1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2018, Joyent, Inc.
24 * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
26 * Copyright 2017 Nexenta Systems, Inc.  All rights reserved.
27 */
28
29/*
30 * DVA-based Adjustable Replacement Cache
31 *
32 * While much of the theory of operation used here is
33 * based on the self-tuning, low overhead replacement cache
34 * presented by Megiddo and Modha at FAST 2003, there are some
35 * significant differences:
36 *
37 * 1. The Megiddo and Modha model assumes any page is evictable.
38 * Pages in its cache cannot be "locked" into memory.  This makes
39 * the eviction algorithm simple: evict the last page in the list.
40 * This also make the performance characteristics easy to reason
41 * about.  Our cache is not so simple.  At any given moment, some
42 * subset of the blocks in the cache are un-evictable because we
43 * have handed out a reference to them.  Blocks are only evictable
44 * when there are no external references active.  This makes
45 * eviction far more problematic:  we choose to evict the evictable
46 * blocks that are the "lowest" in the list.
47 *
48 * There are times when it is not possible to evict the requested
49 * space.  In these circumstances we are unable to adjust the cache
50 * size.  To prevent the cache growing unbounded at these times we
51 * implement a "cache throttle" that slows the flow of new data
52 * into the cache until we can make space available.
53 *
54 * 2. The Megiddo and Modha model assumes a fixed cache size.
55 * Pages are evicted when the cache is full and there is a cache
56 * miss.  Our model has a variable sized cache.  It grows with
57 * high use, but also tries to react to memory pressure from the
58 * operating system: decreasing its size when system memory is
59 * tight.
60 *
61 * 3. The Megiddo and Modha model assumes a fixed page size. All
62 * elements of the cache are therefore exactly the same size.  So
63 * when adjusting the cache size following a cache miss, its simply
64 * a matter of choosing a single page to evict.  In our model, we
65 * have variable sized cache blocks (rangeing from 512 bytes to
66 * 128K bytes).  We therefore choose a set of blocks to evict to make
67 * space for a cache miss that approximates as closely as possible
68 * the space used by the new block.
69 *
70 * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
71 * by N. Megiddo & D. Modha, FAST 2003
72 */
73
74/*
75 * The locking model:
76 *
77 * A new reference to a cache buffer can be obtained in two
78 * ways: 1) via a hash table lookup using the DVA as a key,
79 * or 2) via one of the ARC lists.  The arc_read() interface
80 * uses method 1, while the internal ARC algorithms for
81 * adjusting the cache use method 2.  We therefore provide two
82 * types of locks: 1) the hash table lock array, and 2) the
83 * ARC list locks.
84 *
85 * Buffers do not have their own mutexes, rather they rely on the
86 * hash table mutexes for the bulk of their protection (i.e. most
87 * fields in the arc_buf_hdr_t are protected by these mutexes).
88 *
89 * buf_hash_find() returns the appropriate mutex (held) when it
90 * locates the requested buffer in the hash table.  It returns
91 * NULL for the mutex if the buffer was not in the table.
92 *
93 * buf_hash_remove() expects the appropriate hash mutex to be
94 * already held before it is invoked.
95 *
96 * Each ARC state also has a mutex which is used to protect the
97 * buffer list associated with the state.  When attempting to
98 * obtain a hash table lock while holding an ARC list lock you
99 * must use: mutex_tryenter() to avoid deadlock.  Also note that
100 * the active state mutex must be held before the ghost state mutex.
101 *
102 * It as also possible to register a callback which is run when the
103 * arc_meta_limit is reached and no buffers can be safely evicted.  In
104 * this case the arc user should drop a reference on some arc buffers so
105 * they can be reclaimed and the arc_meta_limit honored.  For example,
106 * when using the ZPL each dentry holds a references on a znode.  These
107 * dentries must be pruned before the arc buffer holding the znode can
108 * be safely evicted.
109 *
110 * Note that the majority of the performance stats are manipulated
111 * with atomic operations.
112 *
113 * The L2ARC uses the l2ad_mtx on each vdev for the following:
114 *
115 *	- L2ARC buflist creation
116 *	- L2ARC buflist eviction
117 *	- L2ARC write completion, which walks L2ARC buflists
118 *	- ARC header destruction, as it removes from L2ARC buflists
119 *	- ARC header release, as it removes from L2ARC buflists
120 */
121
122/*
123 * ARC operation:
124 *
125 * Every block that is in the ARC is tracked by an arc_buf_hdr_t structure.
126 * This structure can point either to a block that is still in the cache or to
127 * one that is only accessible in an L2 ARC device, or it can provide
128 * information about a block that was recently evicted. If a block is
129 * only accessible in the L2ARC, then the arc_buf_hdr_t only has enough
130 * information to retrieve it from the L2ARC device. This information is
131 * stored in the l2arc_buf_hdr_t sub-structure of the arc_buf_hdr_t. A block
132 * that is in this state cannot access the data directly.
133 *
134 * Blocks that are actively being referenced or have not been evicted
135 * are cached in the L1ARC. The L1ARC (l1arc_buf_hdr_t) is a structure within
136 * the arc_buf_hdr_t that will point to the data block in memory. A block can
137 * only be read by a consumer if it has an l1arc_buf_hdr_t. The L1ARC
138 * caches data in two ways -- in a list of ARC buffers (arc_buf_t) and
139 * also in the arc_buf_hdr_t's private physical data block pointer (b_pabd).
140 *
141 * The L1ARC's data pointer may or may not be uncompressed. The ARC has the
142 * ability to store the physical data (b_pabd) associated with the DVA of the
143 * arc_buf_hdr_t. Since the b_pabd is a copy of the on-disk physical block,
144 * it will match its on-disk compression characteristics. This behavior can be
145 * disabled by setting 'zfs_compressed_arc_enabled' to B_FALSE. When the
146 * compressed ARC functionality is disabled, the b_pabd will point to an
147 * uncompressed version of the on-disk data.
148 *
149 * Data in the L1ARC is not accessed by consumers of the ARC directly. Each
150 * arc_buf_hdr_t can have multiple ARC buffers (arc_buf_t) which reference it.
151 * Each ARC buffer (arc_buf_t) is being actively accessed by a specific ARC
152 * consumer. The ARC will provide references to this data and will keep it
153 * cached until it is no longer in use. The ARC caches only the L1ARC's physical
154 * data block and will evict any arc_buf_t that is no longer referenced. The
155 * amount of memory consumed by the arc_buf_ts' data buffers can be seen via the
156 * "overhead_size" kstat.
157 *
158 * Depending on the consumer, an arc_buf_t can be requested in uncompressed or
159 * compressed form. The typical case is that consumers will want uncompressed
160 * data, and when that happens a new data buffer is allocated where the data is
161 * decompressed for them to use. Currently the only consumer who wants
162 * compressed arc_buf_t's is "zfs send", when it streams data exactly as it
163 * exists on disk. When this happens, the arc_buf_t's data buffer is shared
164 * with the arc_buf_hdr_t.
165 *
166 * Here is a diagram showing an arc_buf_hdr_t referenced by two arc_buf_t's. The
167 * first one is owned by a compressed send consumer (and therefore references
168 * the same compressed data buffer as the arc_buf_hdr_t) and the second could be
169 * used by any other consumer (and has its own uncompressed copy of the data
170 * buffer).
171 *
172 *   arc_buf_hdr_t
173 *   +-----------+
174 *   | fields    |
175 *   | common to |
176 *   | L1- and   |
177 *   | L2ARC     |
178 *   +-----------+
179 *   | l2arc_buf_hdr_t
180 *   |           |
181 *   +-----------+
182 *   | l1arc_buf_hdr_t
183 *   |           |              arc_buf_t
184 *   | b_buf     +------------>+-----------+      arc_buf_t
185 *   | b_pabd    +-+           |b_next     +---->+-----------+
186 *   +-----------+ |           |-----------|     |b_next     +-->NULL
187 *                 |           |b_comp = T |     +-----------+
188 *                 |           |b_data     +-+   |b_comp = F |
189 *                 |           +-----------+ |   |b_data     +-+
190 *                 +->+------+               |   +-----------+ |
191 *        compressed  |      |               |                 |
192 *           data     |      |<--------------+                 | uncompressed
193 *                    +------+          compressed,            |     data
194 *                                        shared               +-->+------+
195 *                                         data                    |      |
196 *                                                                 |      |
197 *                                                                 +------+
198 *
199 * When a consumer reads a block, the ARC must first look to see if the
200 * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new
201 * arc_buf_t and either copies uncompressed data into a new data buffer from an
202 * existing uncompressed arc_buf_t, decompresses the hdr's b_pabd buffer into a
203 * new data buffer, or shares the hdr's b_pabd buffer, depending on whether the
204 * hdr is compressed and the desired compression characteristics of the
205 * arc_buf_t consumer. If the arc_buf_t ends up sharing data with the
206 * arc_buf_hdr_t and both of them are uncompressed then the arc_buf_t must be
207 * the last buffer in the hdr's b_buf list, however a shared compressed buf can
208 * be anywhere in the hdr's list.
209 *
210 * The diagram below shows an example of an uncompressed ARC hdr that is
211 * sharing its data with an arc_buf_t (note that the shared uncompressed buf is
212 * the last element in the buf list):
213 *
214 *                arc_buf_hdr_t
215 *                +-----------+
216 *                |           |
217 *                |           |
218 *                |           |
219 *                +-----------+
220 * l2arc_buf_hdr_t|           |
221 *                |           |
222 *                +-----------+
223 * l1arc_buf_hdr_t|           |
224 *                |           |                 arc_buf_t    (shared)
225 *                |    b_buf  +------------>+---------+      arc_buf_t
226 *                |           |             |b_next   +---->+---------+
227 *                |  b_pabd   +-+           |---------|     |b_next   +-->NULL
228 *                +-----------+ |           |         |     +---------+
229 *                              |           |b_data   +-+   |         |
230 *                              |           +---------+ |   |b_data   +-+
231 *                              +->+------+             |   +---------+ |
232 *                                 |      |             |               |
233 *                   uncompressed  |      |             |               |
234 *                        data     +------+             |               |
235 *                                    ^                 +->+------+     |
236 *                                    |       uncompressed |      |     |
237 *                                    |           data     |      |     |
238 *                                    |                    +------+     |
239 *                                    +---------------------------------+
240 *
241 * Writing to the ARC requires that the ARC first discard the hdr's b_pabd
242 * since the physical block is about to be rewritten. The new data contents
243 * will be contained in the arc_buf_t. As the I/O pipeline performs the write,
244 * it may compress the data before writing it to disk. The ARC will be called
245 * with the transformed data and will bcopy the transformed on-disk block into
246 * a newly allocated b_pabd. Writes are always done into buffers which have
247 * either been loaned (and hence are new and don't have other readers) or
248 * buffers which have been released (and hence have their own hdr, if there
249 * were originally other readers of the buf's original hdr). This ensures that
250 * the ARC only needs to update a single buf and its hdr after a write occurs.
251 *
252 * When the L2ARC is in use, it will also take advantage of the b_pabd. The
253 * L2ARC will always write the contents of b_pabd to the L2ARC. This means
254 * that when compressed ARC is enabled that the L2ARC blocks are identical
255 * to the on-disk block in the main data pool. This provides a significant
256 * advantage since the ARC can leverage the bp's checksum when reading from the
257 * L2ARC to determine if the contents are valid. However, if the compressed
258 * ARC is disabled, then the L2ARC's block must be transformed to look
259 * like the physical block in the main data pool before comparing the
260 * checksum and determining its validity.
261 */
262
263#include <sys/spa.h>
264#include <sys/zio.h>
265#include <sys/spa_impl.h>
266#include <sys/zio_compress.h>
267#include <sys/zio_checksum.h>
268#include <sys/zfs_context.h>
269#include <sys/arc.h>
270#include <sys/refcount.h>
271#include <sys/vdev.h>
272#include <sys/vdev_impl.h>
273#include <sys/dsl_pool.h>
274#include <sys/zio_checksum.h>
275#include <sys/multilist.h>
276#include <sys/abd.h>
277#ifdef _KERNEL
278#include <sys/dnlc.h>
279#include <sys/racct.h>
280#endif
281#include <sys/callb.h>
282#include <sys/kstat.h>
283#include <sys/trim_map.h>
284#include <zfs_fletcher.h>
285#include <sys/sdt.h>
286#include <sys/aggsum.h>
287#include <sys/cityhash.h>
288
289#include <machine/vmparam.h>
290
291#ifdef illumos
292#ifndef _KERNEL
293/* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
294boolean_t arc_watch = B_FALSE;
295int arc_procfd;
296#endif
297#endif /* illumos */
298
299static kmutex_t		arc_reclaim_lock;
300static kcondvar_t	arc_reclaim_thread_cv;
301static boolean_t	arc_reclaim_thread_exit;
302static kcondvar_t	arc_reclaim_waiters_cv;
303
304static kmutex_t		arc_dnlc_evicts_lock;
305static kcondvar_t	arc_dnlc_evicts_cv;
306static boolean_t	arc_dnlc_evicts_thread_exit;
307
308uint_t arc_reduce_dnlc_percent = 3;
309
310/*
311 * The number of headers to evict in arc_evict_state_impl() before
312 * dropping the sublist lock and evicting from another sublist. A lower
313 * value means we're more likely to evict the "correct" header (i.e. the
314 * oldest header in the arc state), but comes with higher overhead
315 * (i.e. more invocations of arc_evict_state_impl()).
316 */
317int zfs_arc_evict_batch_limit = 10;
318
319/* number of seconds before growing cache again */
320static int		arc_grow_retry = 60;
321
322/* number of milliseconds before attempting a kmem-cache-reap */
323static int		arc_kmem_cache_reap_retry_ms = 0;
324
325/* shift of arc_c for calculating overflow limit in arc_get_data_impl */
326int		zfs_arc_overflow_shift = 8;
327
328/* shift of arc_c for calculating both min and max arc_p */
329static int		arc_p_min_shift = 4;
330
331/* log2(fraction of arc to reclaim) */
332static int		arc_shrink_shift = 7;
333
334/*
335 * log2(fraction of ARC which must be free to allow growing).
336 * I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
337 * when reading a new block into the ARC, we will evict an equal-sized block
338 * from the ARC.
339 *
340 * This must be less than arc_shrink_shift, so that when we shrink the ARC,
341 * we will still not allow it to grow.
342 */
343int			arc_no_grow_shift = 5;
344
345
346/*
347 * minimum lifespan of a prefetch block in clock ticks
348 * (initialized in arc_init())
349 */
350static int		zfs_arc_min_prefetch_ms = 1;
351static int		zfs_arc_min_prescient_prefetch_ms = 6;
352
353/*
354 * If this percent of memory is free, don't throttle.
355 */
356int arc_lotsfree_percent = 10;
357
358static int arc_dead;
359extern boolean_t zfs_prefetch_disable;
360
361/*
362 * The arc has filled available memory and has now warmed up.
363 */
364static boolean_t arc_warm;
365
366/*
367 * log2 fraction of the zio arena to keep free.
368 */
369int arc_zio_arena_free_shift = 2;
370
371/*
372 * These tunables are for performance analysis.
373 */
374uint64_t zfs_arc_max;
375uint64_t zfs_arc_min;
376uint64_t zfs_arc_meta_limit = 0;
377uint64_t zfs_arc_meta_min = 0;
378uint64_t zfs_arc_dnode_limit = 0;
379uint64_t zfs_arc_dnode_reduce_percent = 10;
380int zfs_arc_grow_retry = 0;
381int zfs_arc_shrink_shift = 0;
382int zfs_arc_no_grow_shift = 0;
383int zfs_arc_p_min_shift = 0;
384uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
385u_int zfs_arc_free_target = 0;
386
387/* Absolute min for arc min / max is 16MB. */
388static uint64_t arc_abs_min = 16 << 20;
389
390/*
391 * ARC dirty data constraints for arc_tempreserve_space() throttle
392 */
393uint_t zfs_arc_dirty_limit_percent = 50;	/* total dirty data limit */
394uint_t zfs_arc_anon_limit_percent = 25;		/* anon block dirty limit */
395uint_t zfs_arc_pool_dirty_percent = 20;		/* each pool's anon allowance */
396
397boolean_t zfs_compressed_arc_enabled = B_TRUE;
398
399static int sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS);
400static int sysctl_vfs_zfs_arc_meta_limit(SYSCTL_HANDLER_ARGS);
401static int sysctl_vfs_zfs_arc_max(SYSCTL_HANDLER_ARGS);
402static int sysctl_vfs_zfs_arc_min(SYSCTL_HANDLER_ARGS);
403static int sysctl_vfs_zfs_arc_no_grow_shift(SYSCTL_HANDLER_ARGS);
404
405#if defined(__FreeBSD__) && defined(_KERNEL)
406static void
407arc_free_target_init(void *unused __unused)
408{
409
410	zfs_arc_free_target = vm_cnt.v_free_target;
411}
412SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY,
413    arc_free_target_init, NULL);
414
415TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit);
416TUNABLE_QUAD("vfs.zfs.arc_meta_min", &zfs_arc_meta_min);
417TUNABLE_INT("vfs.zfs.arc_shrink_shift", &zfs_arc_shrink_shift);
418TUNABLE_INT("vfs.zfs.arc_grow_retry", &zfs_arc_grow_retry);
419TUNABLE_INT("vfs.zfs.arc_no_grow_shift", &zfs_arc_no_grow_shift);
420SYSCTL_DECL(_vfs_zfs);
421SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_max, CTLTYPE_U64 | CTLFLAG_RWTUN,
422    0, sizeof(uint64_t), sysctl_vfs_zfs_arc_max, "QU", "Maximum ARC size");
423SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_min, CTLTYPE_U64 | CTLFLAG_RWTUN,
424    0, sizeof(uint64_t), sysctl_vfs_zfs_arc_min, "QU", "Minimum ARC size");
425SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_no_grow_shift, CTLTYPE_U32 | CTLFLAG_RWTUN,
426    0, sizeof(uint32_t), sysctl_vfs_zfs_arc_no_grow_shift, "U",
427    "log2(fraction of ARC which must be free to allow growing)");
428SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_average_blocksize, CTLFLAG_RDTUN,
429    &zfs_arc_average_blocksize, 0,
430    "ARC average blocksize");
431SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_shrink_shift, CTLFLAG_RW,
432    &arc_shrink_shift, 0,
433    "log2(fraction of arc to reclaim)");
434SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_grow_retry, CTLFLAG_RW,
435    &arc_grow_retry, 0,
436    "Wait in seconds before considering growing ARC");
437SYSCTL_INT(_vfs_zfs, OID_AUTO, compressed_arc_enabled, CTLFLAG_RDTUN,
438    &zfs_compressed_arc_enabled, 0,
439    "Enable compressed ARC");
440SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_kmem_cache_reap_retry_ms, CTLFLAG_RWTUN,
441    &arc_kmem_cache_reap_retry_ms, 0,
442    "Interval between ARC kmem_cache reapings");
443
444/*
445 * We don't have a tunable for arc_free_target due to the dependency on
446 * pagedaemon initialisation.
447 */
448SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_free_target,
449    CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof(u_int),
450    sysctl_vfs_zfs_arc_free_target, "IU",
451    "Desired number of free pages below which ARC triggers reclaim");
452
453static int
454sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS)
455{
456	u_int val;
457	int err;
458
459	val = zfs_arc_free_target;
460	err = sysctl_handle_int(oidp, &val, 0, req);
461	if (err != 0 || req->newptr == NULL)
462		return (err);
463
464	if (val < minfree)
465		return (EINVAL);
466	if (val > vm_cnt.v_page_count)
467		return (EINVAL);
468
469	zfs_arc_free_target = val;
470
471	return (0);
472}
473
474/*
475 * Must be declared here, before the definition of corresponding kstat
476 * macro which uses the same names will confuse the compiler.
477 */
478SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_meta_limit,
479    CTLTYPE_U64 | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof(uint64_t),
480    sysctl_vfs_zfs_arc_meta_limit, "QU",
481    "ARC metadata limit");
482#endif
483
484/*
485 * Note that buffers can be in one of 6 states:
486 *	ARC_anon	- anonymous (discussed below)
487 *	ARC_mru		- recently used, currently cached
488 *	ARC_mru_ghost	- recentely used, no longer in cache
489 *	ARC_mfu		- frequently used, currently cached
490 *	ARC_mfu_ghost	- frequently used, no longer in cache
491 *	ARC_l2c_only	- exists in L2ARC but not other states
492 * When there are no active references to the buffer, they are
493 * are linked onto a list in one of these arc states.  These are
494 * the only buffers that can be evicted or deleted.  Within each
495 * state there are multiple lists, one for meta-data and one for
496 * non-meta-data.  Meta-data (indirect blocks, blocks of dnodes,
497 * etc.) is tracked separately so that it can be managed more
498 * explicitly: favored over data, limited explicitly.
499 *
500 * Anonymous buffers are buffers that are not associated with
501 * a DVA.  These are buffers that hold dirty block copies
502 * before they are written to stable storage.  By definition,
503 * they are "ref'd" and are considered part of arc_mru
504 * that cannot be freed.  Generally, they will aquire a DVA
505 * as they are written and migrate onto the arc_mru list.
506 *
507 * The ARC_l2c_only state is for buffers that are in the second
508 * level ARC but no longer in any of the ARC_m* lists.  The second
509 * level ARC itself may also contain buffers that are in any of
510 * the ARC_m* states - meaning that a buffer can exist in two
511 * places.  The reason for the ARC_l2c_only state is to keep the
512 * buffer header in the hash table, so that reads that hit the
513 * second level ARC benefit from these fast lookups.
514 */
515
516typedef struct arc_state {
517	/*
518	 * list of evictable buffers
519	 */
520	multilist_t *arcs_list[ARC_BUFC_NUMTYPES];
521	/*
522	 * total amount of evictable data in this state
523	 */
524	refcount_t arcs_esize[ARC_BUFC_NUMTYPES];
525	/*
526	 * total amount of data in this state; this includes: evictable,
527	 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA.
528	 */
529	refcount_t arcs_size;
530	/*
531	 * supports the "dbufs" kstat
532	 */
533	arc_state_type_t arcs_state;
534} arc_state_t;
535
536/*
537 * Percentage that can be consumed by dnodes of ARC meta buffers.
538 */
539int zfs_arc_meta_prune = 10000;
540unsigned long zfs_arc_dnode_limit_percent = 10;
541int zfs_arc_meta_strategy = ARC_STRATEGY_META_ONLY;
542int zfs_arc_meta_adjust_restarts = 4096;
543
544SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_meta_strategy, CTLFLAG_RWTUN,
545    &zfs_arc_meta_strategy, 0,
546    "ARC metadata reclamation strategy "
547    "(0 = metadata only, 1 = balance data and metadata)");
548
549/* The 6 states: */
550static arc_state_t ARC_anon;
551static arc_state_t ARC_mru;
552static arc_state_t ARC_mru_ghost;
553static arc_state_t ARC_mfu;
554static arc_state_t ARC_mfu_ghost;
555static arc_state_t ARC_l2c_only;
556
557typedef struct arc_stats {
558	kstat_named_t arcstat_hits;
559	kstat_named_t arcstat_misses;
560	kstat_named_t arcstat_demand_data_hits;
561	kstat_named_t arcstat_demand_data_misses;
562	kstat_named_t arcstat_demand_metadata_hits;
563	kstat_named_t arcstat_demand_metadata_misses;
564	kstat_named_t arcstat_prefetch_data_hits;
565	kstat_named_t arcstat_prefetch_data_misses;
566	kstat_named_t arcstat_prefetch_metadata_hits;
567	kstat_named_t arcstat_prefetch_metadata_misses;
568	kstat_named_t arcstat_mru_hits;
569	kstat_named_t arcstat_mru_ghost_hits;
570	kstat_named_t arcstat_mfu_hits;
571	kstat_named_t arcstat_mfu_ghost_hits;
572	kstat_named_t arcstat_allocated;
573	kstat_named_t arcstat_deleted;
574	/*
575	 * Number of buffers that could not be evicted because the hash lock
576	 * was held by another thread.  The lock may not necessarily be held
577	 * by something using the same buffer, since hash locks are shared
578	 * by multiple buffers.
579	 */
580	kstat_named_t arcstat_mutex_miss;
581	/*
582	 * Number of buffers skipped when updating the access state due to the
583	 * header having already been released after acquiring the hash lock.
584	 */
585	kstat_named_t arcstat_access_skip;
586	/*
587	 * Number of buffers skipped because they have I/O in progress, are
588	 * indirect prefetch buffers that have not lived long enough, or are
589	 * not from the spa we're trying to evict from.
590	 */
591	kstat_named_t arcstat_evict_skip;
592	/*
593	 * Number of times arc_evict_state() was unable to evict enough
594	 * buffers to reach it's target amount.
595	 */
596	kstat_named_t arcstat_evict_not_enough;
597	kstat_named_t arcstat_evict_l2_cached;
598	kstat_named_t arcstat_evict_l2_eligible;
599	kstat_named_t arcstat_evict_l2_ineligible;
600	kstat_named_t arcstat_evict_l2_skip;
601	kstat_named_t arcstat_hash_elements;
602	kstat_named_t arcstat_hash_elements_max;
603	kstat_named_t arcstat_hash_collisions;
604	kstat_named_t arcstat_hash_chains;
605	kstat_named_t arcstat_hash_chain_max;
606	kstat_named_t arcstat_p;
607	kstat_named_t arcstat_c;
608	kstat_named_t arcstat_c_min;
609	kstat_named_t arcstat_c_max;
610	/* Not updated directly; only synced in arc_kstat_update. */
611	kstat_named_t arcstat_size;
612	/*
613	 * Number of compressed bytes stored in the arc_buf_hdr_t's b_pabd.
614	 * Note that the compressed bytes may match the uncompressed bytes
615	 * if the block is either not compressed or compressed arc is disabled.
616	 */
617	kstat_named_t arcstat_compressed_size;
618	/*
619	 * Uncompressed size of the data stored in b_pabd. If compressed
620	 * arc is disabled then this value will be identical to the stat
621	 * above.
622	 */
623	kstat_named_t arcstat_uncompressed_size;
624	/*
625	 * Number of bytes stored in all the arc_buf_t's. This is classified
626	 * as "overhead" since this data is typically short-lived and will
627	 * be evicted from the arc when it becomes unreferenced unless the
628	 * zfs_keep_uncompressed_metadata or zfs_keep_uncompressed_level
629	 * values have been set (see comment in dbuf.c for more information).
630	 */
631	kstat_named_t arcstat_overhead_size;
632	/*
633	 * Number of bytes consumed by internal ARC structures necessary
634	 * for tracking purposes; these structures are not actually
635	 * backed by ARC buffers. This includes arc_buf_hdr_t structures
636	 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only
637	 * caches), and arc_buf_t structures (allocated via arc_buf_t
638	 * cache).
639	 * Not updated directly; only synced in arc_kstat_update.
640	 */
641	kstat_named_t arcstat_hdr_size;
642	/*
643	 * Number of bytes consumed by ARC buffers of type equal to
644	 * ARC_BUFC_DATA. This is generally consumed by buffers backing
645	 * on disk user data (e.g. plain file contents).
646	 * Not updated directly; only synced in arc_kstat_update.
647	 */
648	kstat_named_t arcstat_data_size;
649	/*
650	 * Number of bytes consumed by ARC buffers of type equal to
651	 * ARC_BUFC_METADATA. This is generally consumed by buffers
652	 * backing on disk data that is used for internal ZFS
653	 * structures (e.g. ZAP, dnode, indirect blocks, etc).
654	 * Not updated directly; only synced in arc_kstat_update.
655	 */
656	kstat_named_t arcstat_metadata_size;
657	/*
658	 * Number of bytes consumed by dmu_buf_impl_t objects.
659	 */
660	kstat_named_t arcstat_dbuf_size;
661	/*
662	 * Number of bytes consumed by dnode_t objects.
663	 */
664	kstat_named_t arcstat_dnode_size;
665	/*
666	 * Number of bytes consumed by bonus buffers.
667	 */
668	kstat_named_t arcstat_bonus_size;
669#if defined(__FreeBSD__) && defined(COMPAT_FREEBSD11)
670	/*
671	 * Sum of the previous three counters, provided for compatibility.
672	 */
673	kstat_named_t arcstat_other_size;
674#endif
675	/*
676	 * Total number of bytes consumed by ARC buffers residing in the
677	 * arc_anon state. This includes *all* buffers in the arc_anon
678	 * state; e.g. data, metadata, evictable, and unevictable buffers
679	 * are all included in this value.
680	 * Not updated directly; only synced in arc_kstat_update.
681	 */
682	kstat_named_t arcstat_anon_size;
683	/*
684	 * Number of bytes consumed by ARC buffers that meet the
685	 * following criteria: backing buffers of type ARC_BUFC_DATA,
686	 * residing in the arc_anon state, and are eligible for eviction
687	 * (e.g. have no outstanding holds on the buffer).
688	 * Not updated directly; only synced in arc_kstat_update.
689	 */
690	kstat_named_t arcstat_anon_evictable_data;
691	/*
692	 * Number of bytes consumed by ARC buffers that meet the
693	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
694	 * residing in the arc_anon state, and are eligible for eviction
695	 * (e.g. have no outstanding holds on the buffer).
696	 * Not updated directly; only synced in arc_kstat_update.
697	 */
698	kstat_named_t arcstat_anon_evictable_metadata;
699	/*
700	 * Total number of bytes consumed by ARC buffers residing in the
701	 * arc_mru state. This includes *all* buffers in the arc_mru
702	 * state; e.g. data, metadata, evictable, and unevictable buffers
703	 * are all included in this value.
704	 * Not updated directly; only synced in arc_kstat_update.
705	 */
706	kstat_named_t arcstat_mru_size;
707	/*
708	 * Number of bytes consumed by ARC buffers that meet the
709	 * following criteria: backing buffers of type ARC_BUFC_DATA,
710	 * residing in the arc_mru state, and are eligible for eviction
711	 * (e.g. have no outstanding holds on the buffer).
712	 * Not updated directly; only synced in arc_kstat_update.
713	 */
714	kstat_named_t arcstat_mru_evictable_data;
715	/*
716	 * Number of bytes consumed by ARC buffers that meet the
717	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
718	 * residing in the arc_mru state, and are eligible for eviction
719	 * (e.g. have no outstanding holds on the buffer).
720	 * Not updated directly; only synced in arc_kstat_update.
721	 */
722	kstat_named_t arcstat_mru_evictable_metadata;
723	/*
724	 * Total number of bytes that *would have been* consumed by ARC
725	 * buffers in the arc_mru_ghost state. The key thing to note
726	 * here, is the fact that this size doesn't actually indicate
727	 * RAM consumption. The ghost lists only consist of headers and
728	 * don't actually have ARC buffers linked off of these headers.
729	 * Thus, *if* the headers had associated ARC buffers, these
730	 * buffers *would have* consumed this number of bytes.
731	 * Not updated directly; only synced in arc_kstat_update.
732	 */
733	kstat_named_t arcstat_mru_ghost_size;
734	/*
735	 * Number of bytes that *would have been* consumed by ARC
736	 * buffers that are eligible for eviction, of type
737	 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state.
738	 * Not updated directly; only synced in arc_kstat_update.
739	 */
740	kstat_named_t arcstat_mru_ghost_evictable_data;
741	/*
742	 * Number of bytes that *would have been* consumed by ARC
743	 * buffers that are eligible for eviction, of type
744	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
745	 * Not updated directly; only synced in arc_kstat_update.
746	 */
747	kstat_named_t arcstat_mru_ghost_evictable_metadata;
748	/*
749	 * Total number of bytes consumed by ARC buffers residing in the
750	 * arc_mfu state. This includes *all* buffers in the arc_mfu
751	 * state; e.g. data, metadata, evictable, and unevictable buffers
752	 * are all included in this value.
753	 * Not updated directly; only synced in arc_kstat_update.
754	 */
755	kstat_named_t arcstat_mfu_size;
756	/*
757	 * Number of bytes consumed by ARC buffers that are eligible for
758	 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu
759	 * state.
760	 * Not updated directly; only synced in arc_kstat_update.
761	 */
762	kstat_named_t arcstat_mfu_evictable_data;
763	/*
764	 * Number of bytes consumed by ARC buffers that are eligible for
765	 * eviction, of type ARC_BUFC_METADATA, and reside in the
766	 * arc_mfu state.
767	 * Not updated directly; only synced in arc_kstat_update.
768	 */
769	kstat_named_t arcstat_mfu_evictable_metadata;
770	/*
771	 * Total number of bytes that *would have been* consumed by ARC
772	 * buffers in the arc_mfu_ghost state. See the comment above
773	 * arcstat_mru_ghost_size for more details.
774	 * Not updated directly; only synced in arc_kstat_update.
775	 */
776	kstat_named_t arcstat_mfu_ghost_size;
777	/*
778	 * Number of bytes that *would have been* consumed by ARC
779	 * buffers that are eligible for eviction, of type
780	 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state.
781	 * Not updated directly; only synced in arc_kstat_update.
782	 */
783	kstat_named_t arcstat_mfu_ghost_evictable_data;
784	/*
785	 * Number of bytes that *would have been* consumed by ARC
786	 * buffers that are eligible for eviction, of type
787	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
788	 * Not updated directly; only synced in arc_kstat_update.
789	 */
790	kstat_named_t arcstat_mfu_ghost_evictable_metadata;
791	kstat_named_t arcstat_l2_hits;
792	kstat_named_t arcstat_l2_misses;
793	kstat_named_t arcstat_l2_feeds;
794	kstat_named_t arcstat_l2_rw_clash;
795	kstat_named_t arcstat_l2_read_bytes;
796	kstat_named_t arcstat_l2_write_bytes;
797	kstat_named_t arcstat_l2_writes_sent;
798	kstat_named_t arcstat_l2_writes_done;
799	kstat_named_t arcstat_l2_writes_error;
800	kstat_named_t arcstat_l2_writes_lock_retry;
801	kstat_named_t arcstat_l2_evict_lock_retry;
802	kstat_named_t arcstat_l2_evict_reading;
803	kstat_named_t arcstat_l2_evict_l1cached;
804	kstat_named_t arcstat_l2_free_on_write;
805	kstat_named_t arcstat_l2_abort_lowmem;
806	kstat_named_t arcstat_l2_cksum_bad;
807	kstat_named_t arcstat_l2_io_error;
808	kstat_named_t arcstat_l2_lsize;
809	kstat_named_t arcstat_l2_psize;
810	/* Not updated directly; only synced in arc_kstat_update. */
811	kstat_named_t arcstat_l2_hdr_size;
812	kstat_named_t arcstat_l2_write_trylock_fail;
813	kstat_named_t arcstat_l2_write_passed_headroom;
814	kstat_named_t arcstat_l2_write_spa_mismatch;
815	kstat_named_t arcstat_l2_write_in_l2;
816	kstat_named_t arcstat_l2_write_hdr_io_in_progress;
817	kstat_named_t arcstat_l2_write_not_cacheable;
818	kstat_named_t arcstat_l2_write_full;
819	kstat_named_t arcstat_l2_write_buffer_iter;
820	kstat_named_t arcstat_l2_write_pios;
821	kstat_named_t arcstat_l2_write_buffer_bytes_scanned;
822	kstat_named_t arcstat_l2_write_buffer_list_iter;
823	kstat_named_t arcstat_l2_write_buffer_list_null_iter;
824	kstat_named_t arcstat_memory_throttle_count;
825	kstat_named_t arcstat_memory_direct_count;
826	kstat_named_t arcstat_memory_indirect_count;
827	kstat_named_t arcstat_memory_all_bytes;
828	kstat_named_t arcstat_memory_free_bytes;
829	kstat_named_t arcstat_memory_available_bytes;
830	kstat_named_t arcstat_no_grow;
831	kstat_named_t arcstat_tempreserve;
832	kstat_named_t arcstat_loaned_bytes;
833	kstat_named_t arcstat_prune;
834	/* Not updated directly; only synced in arc_kstat_update. */
835	kstat_named_t arcstat_meta_used;
836	kstat_named_t arcstat_meta_limit;
837	kstat_named_t arcstat_dnode_limit;
838	kstat_named_t arcstat_meta_max;
839	kstat_named_t arcstat_meta_min;
840	kstat_named_t arcstat_async_upgrade_sync;
841	kstat_named_t arcstat_demand_hit_predictive_prefetch;
842	kstat_named_t arcstat_demand_hit_prescient_prefetch;
843} arc_stats_t;
844
845static arc_stats_t arc_stats = {
846	{ "hits",			KSTAT_DATA_UINT64 },
847	{ "misses",			KSTAT_DATA_UINT64 },
848	{ "demand_data_hits",		KSTAT_DATA_UINT64 },
849	{ "demand_data_misses",		KSTAT_DATA_UINT64 },
850	{ "demand_metadata_hits",	KSTAT_DATA_UINT64 },
851	{ "demand_metadata_misses",	KSTAT_DATA_UINT64 },
852	{ "prefetch_data_hits",		KSTAT_DATA_UINT64 },
853	{ "prefetch_data_misses",	KSTAT_DATA_UINT64 },
854	{ "prefetch_metadata_hits",	KSTAT_DATA_UINT64 },
855	{ "prefetch_metadata_misses",	KSTAT_DATA_UINT64 },
856	{ "mru_hits",			KSTAT_DATA_UINT64 },
857	{ "mru_ghost_hits",		KSTAT_DATA_UINT64 },
858	{ "mfu_hits",			KSTAT_DATA_UINT64 },
859	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
860	{ "allocated",			KSTAT_DATA_UINT64 },
861	{ "deleted",			KSTAT_DATA_UINT64 },
862	{ "mutex_miss",			KSTAT_DATA_UINT64 },
863	{ "access_skip",		KSTAT_DATA_UINT64 },
864	{ "evict_skip",			KSTAT_DATA_UINT64 },
865	{ "evict_not_enough",		KSTAT_DATA_UINT64 },
866	{ "evict_l2_cached",		KSTAT_DATA_UINT64 },
867	{ "evict_l2_eligible",		KSTAT_DATA_UINT64 },
868	{ "evict_l2_ineligible",	KSTAT_DATA_UINT64 },
869	{ "evict_l2_skip",		KSTAT_DATA_UINT64 },
870	{ "hash_elements",		KSTAT_DATA_UINT64 },
871	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
872	{ "hash_collisions",		KSTAT_DATA_UINT64 },
873	{ "hash_chains",		KSTAT_DATA_UINT64 },
874	{ "hash_chain_max",		KSTAT_DATA_UINT64 },
875	{ "p",				KSTAT_DATA_UINT64 },
876	{ "c",				KSTAT_DATA_UINT64 },
877	{ "c_min",			KSTAT_DATA_UINT64 },
878	{ "c_max",			KSTAT_DATA_UINT64 },
879	{ "size",			KSTAT_DATA_UINT64 },
880	{ "compressed_size",		KSTAT_DATA_UINT64 },
881	{ "uncompressed_size",		KSTAT_DATA_UINT64 },
882	{ "overhead_size",		KSTAT_DATA_UINT64 },
883	{ "hdr_size",			KSTAT_DATA_UINT64 },
884	{ "data_size",			KSTAT_DATA_UINT64 },
885	{ "metadata_size",		KSTAT_DATA_UINT64 },
886	{ "dbuf_size",			KSTAT_DATA_UINT64 },
887	{ "dnode_size",			KSTAT_DATA_UINT64 },
888	{ "bonus_size",			KSTAT_DATA_UINT64 },
889#if defined(__FreeBSD__) && defined(COMPAT_FREEBSD11)
890	{ "other_size",			KSTAT_DATA_UINT64 },
891#endif
892	{ "anon_size",			KSTAT_DATA_UINT64 },
893	{ "anon_evictable_data",	KSTAT_DATA_UINT64 },
894	{ "anon_evictable_metadata",	KSTAT_DATA_UINT64 },
895	{ "mru_size",			KSTAT_DATA_UINT64 },
896	{ "mru_evictable_data",		KSTAT_DATA_UINT64 },
897	{ "mru_evictable_metadata",	KSTAT_DATA_UINT64 },
898	{ "mru_ghost_size",		KSTAT_DATA_UINT64 },
899	{ "mru_ghost_evictable_data",	KSTAT_DATA_UINT64 },
900	{ "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
901	{ "mfu_size",			KSTAT_DATA_UINT64 },
902	{ "mfu_evictable_data",		KSTAT_DATA_UINT64 },
903	{ "mfu_evictable_metadata",	KSTAT_DATA_UINT64 },
904	{ "mfu_ghost_size",		KSTAT_DATA_UINT64 },
905	{ "mfu_ghost_evictable_data",	KSTAT_DATA_UINT64 },
906	{ "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
907	{ "l2_hits",			KSTAT_DATA_UINT64 },
908	{ "l2_misses",			KSTAT_DATA_UINT64 },
909	{ "l2_feeds",			KSTAT_DATA_UINT64 },
910	{ "l2_rw_clash",		KSTAT_DATA_UINT64 },
911	{ "l2_read_bytes",		KSTAT_DATA_UINT64 },
912	{ "l2_write_bytes",		KSTAT_DATA_UINT64 },
913	{ "l2_writes_sent",		KSTAT_DATA_UINT64 },
914	{ "l2_writes_done",		KSTAT_DATA_UINT64 },
915	{ "l2_writes_error",		KSTAT_DATA_UINT64 },
916	{ "l2_writes_lock_retry",	KSTAT_DATA_UINT64 },
917	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
918	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
919	{ "l2_evict_l1cached",		KSTAT_DATA_UINT64 },
920	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
921	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
922	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
923	{ "l2_io_error",		KSTAT_DATA_UINT64 },
924	{ "l2_size",			KSTAT_DATA_UINT64 },
925	{ "l2_asize",			KSTAT_DATA_UINT64 },
926	{ "l2_hdr_size",		KSTAT_DATA_UINT64 },
927	{ "l2_write_trylock_fail",	KSTAT_DATA_UINT64 },
928	{ "l2_write_passed_headroom",	KSTAT_DATA_UINT64 },
929	{ "l2_write_spa_mismatch",	KSTAT_DATA_UINT64 },
930	{ "l2_write_in_l2",		KSTAT_DATA_UINT64 },
931	{ "l2_write_io_in_progress",	KSTAT_DATA_UINT64 },
932	{ "l2_write_not_cacheable",	KSTAT_DATA_UINT64 },
933	{ "l2_write_full",		KSTAT_DATA_UINT64 },
934	{ "l2_write_buffer_iter",	KSTAT_DATA_UINT64 },
935	{ "l2_write_pios",		KSTAT_DATA_UINT64 },
936	{ "l2_write_buffer_bytes_scanned", KSTAT_DATA_UINT64 },
937	{ "l2_write_buffer_list_iter",	KSTAT_DATA_UINT64 },
938	{ "l2_write_buffer_list_null_iter", KSTAT_DATA_UINT64 },
939	{ "memory_throttle_count",	KSTAT_DATA_UINT64 },
940	{ "memory_direct_count",	KSTAT_DATA_UINT64 },
941	{ "memory_indirect_count",	KSTAT_DATA_UINT64 },
942	{ "memory_all_bytes",		KSTAT_DATA_UINT64 },
943	{ "memory_free_bytes",		KSTAT_DATA_UINT64 },
944	{ "memory_available_bytes",	KSTAT_DATA_UINT64 },
945	{ "arc_no_grow",		KSTAT_DATA_UINT64 },
946	{ "arc_tempreserve",		KSTAT_DATA_UINT64 },
947	{ "arc_loaned_bytes",		KSTAT_DATA_UINT64 },
948	{ "arc_prune",			KSTAT_DATA_UINT64 },
949	{ "arc_meta_used",		KSTAT_DATA_UINT64 },
950	{ "arc_meta_limit",		KSTAT_DATA_UINT64 },
951	{ "arc_dnode_limit",		KSTAT_DATA_UINT64 },
952	{ "arc_meta_max",		KSTAT_DATA_UINT64 },
953	{ "arc_meta_min",		KSTAT_DATA_UINT64 },
954	{ "async_upgrade_sync",		KSTAT_DATA_UINT64 },
955	{ "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 },
956	{ "demand_hit_prescient_prefetch", KSTAT_DATA_UINT64 },
957};
958
959#define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
960
961#define	ARCSTAT_INCR(stat, val) \
962	atomic_add_64(&arc_stats.stat.value.ui64, (val))
963
964#define	ARCSTAT_BUMP(stat)	ARCSTAT_INCR(stat, 1)
965#define	ARCSTAT_BUMPDOWN(stat)	ARCSTAT_INCR(stat, -1)
966
967#define	ARCSTAT_MAX(stat, val) {					\
968	uint64_t m;							\
969	while ((val) > (m = arc_stats.stat.value.ui64) &&		\
970	    (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))	\
971		continue;						\
972}
973
974#define	ARCSTAT_MAXSTAT(stat) \
975	ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
976
977/*
978 * We define a macro to allow ARC hits/misses to be easily broken down by
979 * two separate conditions, giving a total of four different subtypes for
980 * each of hits and misses (so eight statistics total).
981 */
982#define	ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
983	if (cond1) {							\
984		if (cond2) {						\
985			ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
986		} else {						\
987			ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
988		}							\
989	} else {							\
990		if (cond2) {						\
991			ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
992		} else {						\
993			ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
994		}							\
995	}
996
997kstat_t			*arc_ksp;
998static arc_state_t	*arc_anon;
999static arc_state_t	*arc_mru;
1000static arc_state_t	*arc_mru_ghost;
1001static arc_state_t	*arc_mfu;
1002static arc_state_t	*arc_mfu_ghost;
1003static arc_state_t	*arc_l2c_only;
1004
1005/*
1006 * There are several ARC variables that are critical to export as kstats --
1007 * but we don't want to have to grovel around in the kstat whenever we wish to
1008 * manipulate them.  For these variables, we therefore define them to be in
1009 * terms of the statistic variable.  This assures that we are not introducing
1010 * the possibility of inconsistency by having shadow copies of the variables,
1011 * while still allowing the code to be readable.
1012 */
1013#define	arc_p		ARCSTAT(arcstat_p)	/* target size of MRU */
1014#define	arc_c		ARCSTAT(arcstat_c)	/* target size of cache */
1015#define	arc_c_min	ARCSTAT(arcstat_c_min)	/* min target cache size */
1016#define	arc_c_max	ARCSTAT(arcstat_c_max)	/* max target cache size */
1017#define	arc_meta_limit	ARCSTAT(arcstat_meta_limit) /* max size for metadata */
1018#define	arc_dnode_limit	ARCSTAT(arcstat_dnode_limit) /* max size for dnodes */
1019#define	arc_meta_min	ARCSTAT(arcstat_meta_min) /* min size for metadata */
1020#define	arc_meta_max	ARCSTAT(arcstat_meta_max) /* max size of metadata */
1021#define	arc_dbuf_size	ARCSTAT(arcstat_dbuf_size) /* dbuf metadata */
1022#define	arc_dnode_size	ARCSTAT(arcstat_dnode_size) /* dnode metadata */
1023#define	arc_bonus_size	ARCSTAT(arcstat_bonus_size) /* bonus buffer metadata */
1024
1025/* compressed size of entire arc */
1026#define	arc_compressed_size	ARCSTAT(arcstat_compressed_size)
1027/* uncompressed size of entire arc */
1028#define	arc_uncompressed_size	ARCSTAT(arcstat_uncompressed_size)
1029/* number of bytes in the arc from arc_buf_t's */
1030#define	arc_overhead_size	ARCSTAT(arcstat_overhead_size)
1031
1032/*
1033 * There are also some ARC variables that we want to export, but that are
1034 * updated so often that having the canonical representation be the statistic
1035 * variable causes a performance bottleneck. We want to use aggsum_t's for these
1036 * instead, but still be able to export the kstat in the same way as before.
1037 * The solution is to always use the aggsum version, except in the kstat update
1038 * callback.
1039 */
1040aggsum_t arc_size;
1041aggsum_t arc_meta_used;
1042aggsum_t astat_data_size;
1043aggsum_t astat_metadata_size;
1044aggsum_t astat_hdr_size;
1045aggsum_t astat_bonus_size;
1046aggsum_t astat_dnode_size;
1047aggsum_t astat_dbuf_size;
1048aggsum_t astat_l2_hdr_size;
1049
1050static list_t arc_prune_list;
1051static kmutex_t arc_prune_mtx;
1052static taskq_t *arc_prune_taskq;
1053
1054static int		arc_no_grow;	/* Don't try to grow cache size */
1055static uint64_t		arc_tempreserve;
1056static uint64_t		arc_loaned_bytes;
1057
1058typedef struct arc_callback arc_callback_t;
1059
1060struct arc_callback {
1061	void			*acb_private;
1062	arc_read_done_func_t	*acb_done;
1063	arc_buf_t		*acb_buf;
1064	boolean_t		acb_compressed;
1065	zio_t			*acb_zio_dummy;
1066	zio_t			*acb_zio_head;
1067	arc_callback_t		*acb_next;
1068};
1069
1070typedef struct arc_write_callback arc_write_callback_t;
1071
1072struct arc_write_callback {
1073	void			*awcb_private;
1074	arc_write_done_func_t	*awcb_ready;
1075	arc_write_done_func_t	*awcb_children_ready;
1076	arc_write_done_func_t	*awcb_physdone;
1077	arc_write_done_func_t	*awcb_done;
1078	arc_buf_t		*awcb_buf;
1079};
1080
1081/*
1082 * ARC buffers are separated into multiple structs as a memory saving measure:
1083 *   - Common fields struct, always defined, and embedded within it:
1084 *       - L2-only fields, always allocated but undefined when not in L2ARC
1085 *       - L1-only fields, only allocated when in L1ARC
1086 *
1087 *           Buffer in L1                     Buffer only in L2
1088 *    +------------------------+          +------------------------+
1089 *    | arc_buf_hdr_t          |          | arc_buf_hdr_t          |
1090 *    |                        |          |                        |
1091 *    |                        |          |                        |
1092 *    |                        |          |                        |
1093 *    +------------------------+          +------------------------+
1094 *    | l2arc_buf_hdr_t        |          | l2arc_buf_hdr_t        |
1095 *    | (undefined if L1-only) |          |                        |
1096 *    +------------------------+          +------------------------+
1097 *    | l1arc_buf_hdr_t        |
1098 *    |                        |
1099 *    |                        |
1100 *    |                        |
1101 *    |                        |
1102 *    +------------------------+
1103 *
1104 * Because it's possible for the L2ARC to become extremely large, we can wind
1105 * up eating a lot of memory in L2ARC buffer headers, so the size of a header
1106 * is minimized by only allocating the fields necessary for an L1-cached buffer
1107 * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and
1108 * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple
1109 * words in pointers. arc_hdr_realloc() is used to switch a header between
1110 * these two allocation states.
1111 */
1112typedef struct l1arc_buf_hdr {
1113	kmutex_t		b_freeze_lock;
1114	zio_cksum_t		*b_freeze_cksum;
1115#ifdef ZFS_DEBUG
1116	/*
1117	 * Used for debugging with kmem_flags - by allocating and freeing
1118	 * b_thawed when the buffer is thawed, we get a record of the stack
1119	 * trace that thawed it.
1120	 */
1121	void			*b_thawed;
1122#endif
1123
1124	arc_buf_t		*b_buf;
1125	uint32_t		b_bufcnt;
1126	/* for waiting on writes to complete */
1127	kcondvar_t		b_cv;
1128	uint8_t			b_byteswap;
1129
1130	/* protected by arc state mutex */
1131	arc_state_t		*b_state;
1132	multilist_node_t	b_arc_node;
1133
1134	/* updated atomically */
1135	clock_t			b_arc_access;
1136	uint32_t		b_mru_hits;
1137	uint32_t		b_mru_ghost_hits;
1138	uint32_t		b_mfu_hits;
1139	uint32_t		b_mfu_ghost_hits;
1140	uint32_t		b_l2_hits;
1141
1142	/* self protecting */
1143	refcount_t		b_refcnt;
1144
1145	arc_callback_t		*b_acb;
1146	abd_t			*b_pabd;
1147} l1arc_buf_hdr_t;
1148
1149typedef struct l2arc_dev l2arc_dev_t;
1150
1151typedef struct l2arc_buf_hdr {
1152	/* protected by arc_buf_hdr mutex */
1153	l2arc_dev_t		*b_dev;		/* L2ARC device */
1154	uint64_t		b_daddr;	/* disk address, offset byte */
1155	uint32_t		b_hits;
1156
1157	list_node_t		b_l2node;
1158} l2arc_buf_hdr_t;
1159
1160struct arc_buf_hdr {
1161	/* protected by hash lock */
1162	dva_t			b_dva;
1163	uint64_t		b_birth;
1164
1165	arc_buf_contents_t	b_type;
1166	arc_buf_hdr_t		*b_hash_next;
1167	arc_flags_t		b_flags;
1168
1169	/*
1170	 * This field stores the size of the data buffer after
1171	 * compression, and is set in the arc's zio completion handlers.
1172	 * It is in units of SPA_MINBLOCKSIZE (e.g. 1 == 512 bytes).
1173	 *
1174	 * While the block pointers can store up to 32MB in their psize
1175	 * field, we can only store up to 32MB minus 512B. This is due
1176	 * to the bp using a bias of 1, whereas we use a bias of 0 (i.e.
1177	 * a field of zeros represents 512B in the bp). We can't use a
1178	 * bias of 1 since we need to reserve a psize of zero, here, to
1179	 * represent holes and embedded blocks.
1180	 *
1181	 * This isn't a problem in practice, since the maximum size of a
1182	 * buffer is limited to 16MB, so we never need to store 32MB in
1183	 * this field. Even in the upstream illumos code base, the
1184	 * maximum size of a buffer is limited to 16MB.
1185	 */
1186	uint16_t		b_psize;
1187
1188	/*
1189	 * This field stores the size of the data buffer before
1190	 * compression, and cannot change once set. It is in units
1191	 * of SPA_MINBLOCKSIZE (e.g. 2 == 1024 bytes)
1192	 */
1193	uint16_t		b_lsize;	/* immutable */
1194	uint64_t		b_spa;		/* immutable */
1195
1196	/* L2ARC fields. Undefined when not in L2ARC. */
1197	l2arc_buf_hdr_t		b_l2hdr;
1198	/* L1ARC fields. Undefined when in l2arc_only state */
1199	l1arc_buf_hdr_t		b_l1hdr;
1200};
1201
1202#if defined(__FreeBSD__) && defined(_KERNEL)
1203static int
1204sysctl_vfs_zfs_arc_meta_limit(SYSCTL_HANDLER_ARGS)
1205{
1206	uint64_t val;
1207	int err;
1208
1209	val = arc_meta_limit;
1210	err = sysctl_handle_64(oidp, &val, 0, req);
1211	if (err != 0 || req->newptr == NULL)
1212		return (err);
1213
1214        if (val <= 0 || val > arc_c_max)
1215		return (EINVAL);
1216
1217	arc_meta_limit = val;
1218	return (0);
1219}
1220
1221static int
1222sysctl_vfs_zfs_arc_no_grow_shift(SYSCTL_HANDLER_ARGS)
1223{
1224	uint32_t val;
1225	int err;
1226
1227	val = arc_no_grow_shift;
1228	err = sysctl_handle_32(oidp, &val, 0, req);
1229	if (err != 0 || req->newptr == NULL)
1230		return (err);
1231
1232        if (val >= arc_shrink_shift)
1233		return (EINVAL);
1234
1235	arc_no_grow_shift = val;
1236	return (0);
1237}
1238
1239static int
1240sysctl_vfs_zfs_arc_max(SYSCTL_HANDLER_ARGS)
1241{
1242	uint64_t val;
1243	int err;
1244
1245	val = zfs_arc_max;
1246	err = sysctl_handle_64(oidp, &val, 0, req);
1247	if (err != 0 || req->newptr == NULL)
1248		return (err);
1249
1250	if (zfs_arc_max == 0) {
1251		/* Loader tunable so blindly set */
1252		zfs_arc_max = val;
1253		return (0);
1254	}
1255
1256	if (val < arc_abs_min || val > kmem_size())
1257		return (EINVAL);
1258	if (val < arc_c_min)
1259		return (EINVAL);
1260	if (zfs_arc_meta_limit > 0 && val < zfs_arc_meta_limit)
1261		return (EINVAL);
1262
1263	arc_c_max = val;
1264
1265	arc_c = arc_c_max;
1266        arc_p = (arc_c >> 1);
1267
1268	if (zfs_arc_meta_limit == 0) {
1269		/* limit meta-data to 1/4 of the arc capacity */
1270		arc_meta_limit = arc_c_max / 4;
1271	}
1272
1273	/* if kmem_flags are set, lets try to use less memory */
1274	if (kmem_debugging())
1275		arc_c = arc_c / 2;
1276
1277	zfs_arc_max = arc_c;
1278
1279	return (0);
1280}
1281
1282static int
1283sysctl_vfs_zfs_arc_min(SYSCTL_HANDLER_ARGS)
1284{
1285	uint64_t val;
1286	int err;
1287
1288	val = zfs_arc_min;
1289	err = sysctl_handle_64(oidp, &val, 0, req);
1290	if (err != 0 || req->newptr == NULL)
1291		return (err);
1292
1293	if (zfs_arc_min == 0) {
1294		/* Loader tunable so blindly set */
1295		zfs_arc_min = val;
1296		return (0);
1297	}
1298
1299	if (val < arc_abs_min || val > arc_c_max)
1300		return (EINVAL);
1301
1302	arc_c_min = val;
1303
1304	if (zfs_arc_meta_min == 0)
1305                arc_meta_min = arc_c_min / 2;
1306
1307	if (arc_c < arc_c_min)
1308                arc_c = arc_c_min;
1309
1310	zfs_arc_min = arc_c_min;
1311
1312	return (0);
1313}
1314#endif
1315
1316#define	GHOST_STATE(state)	\
1317	((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||	\
1318	(state) == arc_l2c_only)
1319
1320#define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
1321#define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
1322#define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_FLAG_IO_ERROR)
1323#define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_FLAG_PREFETCH)
1324#define	HDR_PRESCIENT_PREFETCH(hdr)	\
1325	((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH)
1326#define	HDR_COMPRESSION_ENABLED(hdr)	\
1327	((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
1328
1329#define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_FLAG_L2CACHE)
1330#define	HDR_L2_READING(hdr)	\
1331	(((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) &&	\
1332	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
1333#define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITING)
1334#define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
1335#define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
1336#define	HDR_SHARED_DATA(hdr)	((hdr)->b_flags & ARC_FLAG_SHARED_DATA)
1337
1338#define	HDR_ISTYPE_METADATA(hdr)	\
1339	((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
1340#define	HDR_ISTYPE_DATA(hdr)	(!HDR_ISTYPE_METADATA(hdr))
1341
1342#define	HDR_HAS_L1HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
1343#define	HDR_HAS_L2HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
1344
1345/* For storing compression mode in b_flags */
1346#define	HDR_COMPRESS_OFFSET	(highbit64(ARC_FLAG_COMPRESS_0) - 1)
1347
1348#define	HDR_GET_COMPRESS(hdr)	((enum zio_compress)BF32_GET((hdr)->b_flags, \
1349	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS))
1350#define	HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \
1351	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS, (cmp));
1352
1353#define	ARC_BUF_LAST(buf)	((buf)->b_next == NULL)
1354#define	ARC_BUF_SHARED(buf)	((buf)->b_flags & ARC_BUF_FLAG_SHARED)
1355#define	ARC_BUF_COMPRESSED(buf)	((buf)->b_flags & ARC_BUF_FLAG_COMPRESSED)
1356
1357/*
1358 * Other sizes
1359 */
1360
1361#define	HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
1362#define	HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
1363
1364/*
1365 * Hash table routines
1366 */
1367
1368#define	HT_LOCK_PAD	CACHE_LINE_SIZE
1369
1370struct ht_lock {
1371	kmutex_t	ht_lock;
1372#ifdef _KERNEL
1373	unsigned char	pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
1374#endif
1375};
1376
1377#define	BUF_LOCKS 256
1378typedef struct buf_hash_table {
1379	uint64_t ht_mask;
1380	arc_buf_hdr_t **ht_table;
1381	struct ht_lock ht_locks[BUF_LOCKS] __aligned(CACHE_LINE_SIZE);
1382} buf_hash_table_t;
1383
1384static buf_hash_table_t buf_hash_table;
1385
1386#define	BUF_HASH_INDEX(spa, dva, birth) \
1387	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
1388#define	BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
1389#define	BUF_HASH_LOCK(idx)	(&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
1390#define	HDR_LOCK(hdr) \
1391	(BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
1392
1393uint64_t zfs_crc64_table[256];
1394
1395/*
1396 * Level 2 ARC
1397 */
1398
1399#define	L2ARC_WRITE_SIZE	(8 * 1024 * 1024)	/* initial write max */
1400#define	L2ARC_HEADROOM		2			/* num of writes */
1401/*
1402 * If we discover during ARC scan any buffers to be compressed, we boost
1403 * our headroom for the next scanning cycle by this percentage multiple.
1404 */
1405#define	L2ARC_HEADROOM_BOOST	200
1406#define	L2ARC_FEED_SECS		1		/* caching interval secs */
1407#define	L2ARC_FEED_MIN_MS	200		/* min caching interval ms */
1408
1409#define	l2arc_writes_sent	ARCSTAT(arcstat_l2_writes_sent)
1410#define	l2arc_writes_done	ARCSTAT(arcstat_l2_writes_done)
1411
1412/* L2ARC Performance Tunables */
1413uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;	/* default max write size */
1414uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;	/* extra write during warmup */
1415uint64_t l2arc_headroom = L2ARC_HEADROOM;	/* number of dev writes */
1416uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
1417uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;	/* interval seconds */
1418uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS;	/* min interval milliseconds */
1419boolean_t l2arc_noprefetch = B_TRUE;		/* don't cache prefetch bufs */
1420boolean_t l2arc_feed_again = B_TRUE;		/* turbo warmup */
1421boolean_t l2arc_norw = B_TRUE;			/* no reads during writes */
1422
1423SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_write_max, CTLFLAG_RW,
1424    &l2arc_write_max, 0, "max write size");
1425SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_write_boost, CTLFLAG_RW,
1426    &l2arc_write_boost, 0, "extra write during warmup");
1427SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_headroom, CTLFLAG_RW,
1428    &l2arc_headroom, 0, "number of dev writes");
1429SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_feed_secs, CTLFLAG_RW,
1430    &l2arc_feed_secs, 0, "interval seconds");
1431SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_feed_min_ms, CTLFLAG_RW,
1432    &l2arc_feed_min_ms, 0, "min interval milliseconds");
1433
1434SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_noprefetch, CTLFLAG_RW,
1435    &l2arc_noprefetch, 0, "don't cache prefetch bufs");
1436SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_feed_again, CTLFLAG_RW,
1437    &l2arc_feed_again, 0, "turbo warmup");
1438SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_norw, CTLFLAG_RW,
1439    &l2arc_norw, 0, "no reads during writes");
1440
1441SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_size, CTLFLAG_RD,
1442    &ARC_anon.arcs_size.rc_count, 0, "size of anonymous state");
1443SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_metadata_esize, CTLFLAG_RD,
1444    &ARC_anon.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1445    "size of anonymous state");
1446SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_data_esize, CTLFLAG_RD,
1447    &ARC_anon.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1448    "size of anonymous state");
1449
1450SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_size, CTLFLAG_RD,
1451    &ARC_mru.arcs_size.rc_count, 0, "size of mru state");
1452SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_metadata_esize, CTLFLAG_RD,
1453    &ARC_mru.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1454    "size of metadata in mru state");
1455SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_data_esize, CTLFLAG_RD,
1456    &ARC_mru.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1457    "size of data in mru state");
1458
1459SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_size, CTLFLAG_RD,
1460    &ARC_mru_ghost.arcs_size.rc_count, 0, "size of mru ghost state");
1461SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_metadata_esize, CTLFLAG_RD,
1462    &ARC_mru_ghost.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1463    "size of metadata in mru ghost state");
1464SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_data_esize, CTLFLAG_RD,
1465    &ARC_mru_ghost.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1466    "size of data in mru ghost state");
1467
1468SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_size, CTLFLAG_RD,
1469    &ARC_mfu.arcs_size.rc_count, 0, "size of mfu state");
1470SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_metadata_esize, CTLFLAG_RD,
1471    &ARC_mfu.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1472    "size of metadata in mfu state");
1473SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_data_esize, CTLFLAG_RD,
1474    &ARC_mfu.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1475    "size of data in mfu state");
1476
1477SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_size, CTLFLAG_RD,
1478    &ARC_mfu_ghost.arcs_size.rc_count, 0, "size of mfu ghost state");
1479SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_metadata_esize, CTLFLAG_RD,
1480    &ARC_mfu_ghost.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1481    "size of metadata in mfu ghost state");
1482SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_data_esize, CTLFLAG_RD,
1483    &ARC_mfu_ghost.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1484    "size of data in mfu ghost state");
1485
1486SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2c_only_size, CTLFLAG_RD,
1487    &ARC_l2c_only.arcs_size.rc_count, 0, "size of mru state");
1488
1489SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_min_prefetch_ms, CTLFLAG_RW,
1490    &zfs_arc_min_prefetch_ms, 0, "Min life of prefetch block in ms");
1491SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_min_prescient_prefetch_ms, CTLFLAG_RW,
1492    &zfs_arc_min_prescient_prefetch_ms, 0, "Min life of prescient prefetched block in ms");
1493
1494/*
1495 * L2ARC Internals
1496 */
1497struct l2arc_dev {
1498	vdev_t			*l2ad_vdev;	/* vdev */
1499	spa_t			*l2ad_spa;	/* spa */
1500	uint64_t		l2ad_hand;	/* next write location */
1501	uint64_t		l2ad_start;	/* first addr on device */
1502	uint64_t		l2ad_end;	/* last addr on device */
1503	boolean_t		l2ad_first;	/* first sweep through */
1504	boolean_t		l2ad_writing;	/* currently writing */
1505	kmutex_t		l2ad_mtx;	/* lock for buffer list */
1506	list_t			l2ad_buflist;	/* buffer list */
1507	list_node_t		l2ad_node;	/* device list node */
1508	refcount_t		l2ad_alloc;	/* allocated bytes */
1509};
1510
1511static list_t L2ARC_dev_list;			/* device list */
1512static list_t *l2arc_dev_list;			/* device list pointer */
1513static kmutex_t l2arc_dev_mtx;			/* device list mutex */
1514static l2arc_dev_t *l2arc_dev_last;		/* last device used */
1515static list_t L2ARC_free_on_write;		/* free after write buf list */
1516static list_t *l2arc_free_on_write;		/* free after write list ptr */
1517static kmutex_t l2arc_free_on_write_mtx;	/* mutex for list */
1518static uint64_t l2arc_ndev;			/* number of devices */
1519
1520typedef struct l2arc_read_callback {
1521	arc_buf_hdr_t		*l2rcb_hdr;		/* read header */
1522	blkptr_t		l2rcb_bp;		/* original blkptr */
1523	zbookmark_phys_t	l2rcb_zb;		/* original bookmark */
1524	int			l2rcb_flags;		/* original flags */
1525	abd_t			*l2rcb_abd;		/* temporary buffer */
1526} l2arc_read_callback_t;
1527
1528typedef struct l2arc_write_callback {
1529	l2arc_dev_t	*l2wcb_dev;		/* device info */
1530	arc_buf_hdr_t	*l2wcb_head;		/* head of write buflist */
1531} l2arc_write_callback_t;
1532
1533typedef struct l2arc_data_free {
1534	/* protected by l2arc_free_on_write_mtx */
1535	abd_t		*l2df_abd;
1536	size_t		l2df_size;
1537	arc_buf_contents_t l2df_type;
1538	list_node_t	l2df_list_node;
1539} l2arc_data_free_t;
1540
1541static kmutex_t l2arc_feed_thr_lock;
1542static kcondvar_t l2arc_feed_thr_cv;
1543static uint8_t l2arc_thread_exit;
1544
1545static abd_t *arc_get_data_abd(arc_buf_hdr_t *, uint64_t, void *);
1546static void *arc_get_data_buf(arc_buf_hdr_t *, uint64_t, void *);
1547static void arc_get_data_impl(arc_buf_hdr_t *, uint64_t, void *);
1548static void arc_free_data_abd(arc_buf_hdr_t *, abd_t *, uint64_t, void *);
1549static void arc_free_data_buf(arc_buf_hdr_t *, void *, uint64_t, void *);
1550static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag);
1551static void arc_hdr_free_pabd(arc_buf_hdr_t *);
1552static void arc_hdr_alloc_pabd(arc_buf_hdr_t *);
1553static void arc_access(arc_buf_hdr_t *, kmutex_t *);
1554static boolean_t arc_is_overflowing();
1555static void arc_buf_watch(arc_buf_t *);
1556static void arc_prune_async(int64_t);
1557
1558static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
1559static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
1560static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
1561static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
1562
1563static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
1564static void l2arc_read_done(zio_t *);
1565
1566static void
1567l2arc_trim(const arc_buf_hdr_t *hdr)
1568{
1569	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
1570
1571	ASSERT(HDR_HAS_L2HDR(hdr));
1572	ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
1573
1574	if (HDR_GET_PSIZE(hdr) != 0) {
1575		trim_map_free(dev->l2ad_vdev, hdr->b_l2hdr.b_daddr,
1576		    HDR_GET_PSIZE(hdr), 0);
1577	}
1578}
1579
1580/*
1581 * We use Cityhash for this. It's fast, and has good hash properties without
1582 * requiring any large static buffers.
1583 */
1584static uint64_t
1585buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
1586{
1587	return (cityhash4(spa, dva->dva_word[0], dva->dva_word[1], birth));
1588}
1589
1590#define	HDR_EMPTY(hdr)						\
1591	((hdr)->b_dva.dva_word[0] == 0 &&			\
1592	(hdr)->b_dva.dva_word[1] == 0)
1593
1594#define	HDR_EQUAL(spa, dva, birth, hdr)				\
1595	((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
1596	((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
1597	((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
1598
1599static void
1600buf_discard_identity(arc_buf_hdr_t *hdr)
1601{
1602	hdr->b_dva.dva_word[0] = 0;
1603	hdr->b_dva.dva_word[1] = 0;
1604	hdr->b_birth = 0;
1605}
1606
1607static arc_buf_hdr_t *
1608buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
1609{
1610	const dva_t *dva = BP_IDENTITY(bp);
1611	uint64_t birth = BP_PHYSICAL_BIRTH(bp);
1612	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
1613	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1614	arc_buf_hdr_t *hdr;
1615
1616	mutex_enter(hash_lock);
1617	for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
1618	    hdr = hdr->b_hash_next) {
1619		if (HDR_EQUAL(spa, dva, birth, hdr)) {
1620			*lockp = hash_lock;
1621			return (hdr);
1622		}
1623	}
1624	mutex_exit(hash_lock);
1625	*lockp = NULL;
1626	return (NULL);
1627}
1628
1629/*
1630 * Insert an entry into the hash table.  If there is already an element
1631 * equal to elem in the hash table, then the already existing element
1632 * will be returned and the new element will not be inserted.
1633 * Otherwise returns NULL.
1634 * If lockp == NULL, the caller is assumed to already hold the hash lock.
1635 */
1636static arc_buf_hdr_t *
1637buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
1638{
1639	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1640	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1641	arc_buf_hdr_t *fhdr;
1642	uint32_t i;
1643
1644	ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1645	ASSERT(hdr->b_birth != 0);
1646	ASSERT(!HDR_IN_HASH_TABLE(hdr));
1647
1648	if (lockp != NULL) {
1649		*lockp = hash_lock;
1650		mutex_enter(hash_lock);
1651	} else {
1652		ASSERT(MUTEX_HELD(hash_lock));
1653	}
1654
1655	for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
1656	    fhdr = fhdr->b_hash_next, i++) {
1657		if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1658			return (fhdr);
1659	}
1660
1661	hdr->b_hash_next = buf_hash_table.ht_table[idx];
1662	buf_hash_table.ht_table[idx] = hdr;
1663	arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1664
1665	/* collect some hash table performance data */
1666	if (i > 0) {
1667		ARCSTAT_BUMP(arcstat_hash_collisions);
1668		if (i == 1)
1669			ARCSTAT_BUMP(arcstat_hash_chains);
1670
1671		ARCSTAT_MAX(arcstat_hash_chain_max, i);
1672	}
1673
1674	ARCSTAT_BUMP(arcstat_hash_elements);
1675	ARCSTAT_MAXSTAT(arcstat_hash_elements);
1676
1677	return (NULL);
1678}
1679
1680static void
1681buf_hash_remove(arc_buf_hdr_t *hdr)
1682{
1683	arc_buf_hdr_t *fhdr, **hdrp;
1684	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1685
1686	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
1687	ASSERT(HDR_IN_HASH_TABLE(hdr));
1688
1689	hdrp = &buf_hash_table.ht_table[idx];
1690	while ((fhdr = *hdrp) != hdr) {
1691		ASSERT3P(fhdr, !=, NULL);
1692		hdrp = &fhdr->b_hash_next;
1693	}
1694	*hdrp = hdr->b_hash_next;
1695	hdr->b_hash_next = NULL;
1696	arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1697
1698	/* collect some hash table performance data */
1699	ARCSTAT_BUMPDOWN(arcstat_hash_elements);
1700
1701	if (buf_hash_table.ht_table[idx] &&
1702	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
1703		ARCSTAT_BUMPDOWN(arcstat_hash_chains);
1704}
1705
1706/*
1707 * Global data structures and functions for the buf kmem cache.
1708 */
1709static kmem_cache_t *hdr_full_cache;
1710static kmem_cache_t *hdr_l2only_cache;
1711static kmem_cache_t *buf_cache;
1712
1713static void
1714buf_fini(void)
1715{
1716	int i;
1717
1718	kmem_free(buf_hash_table.ht_table,
1719	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
1720	for (i = 0; i < BUF_LOCKS; i++)
1721		mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
1722	kmem_cache_destroy(hdr_full_cache);
1723	kmem_cache_destroy(hdr_l2only_cache);
1724	kmem_cache_destroy(buf_cache);
1725}
1726
1727/*
1728 * Constructor callback - called when the cache is empty
1729 * and a new buf is requested.
1730 */
1731/* ARGSUSED */
1732static int
1733hdr_full_cons(void *vbuf, void *unused, int kmflag)
1734{
1735	arc_buf_hdr_t *hdr = vbuf;
1736
1737	bzero(hdr, HDR_FULL_SIZE);
1738	cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
1739	refcount_create(&hdr->b_l1hdr.b_refcnt);
1740	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1741	multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1742	arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1743
1744	return (0);
1745}
1746
1747/* ARGSUSED */
1748static int
1749hdr_l2only_cons(void *vbuf, void *unused, int kmflag)
1750{
1751	arc_buf_hdr_t *hdr = vbuf;
1752
1753	bzero(hdr, HDR_L2ONLY_SIZE);
1754	arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1755
1756	return (0);
1757}
1758
1759/* ARGSUSED */
1760static int
1761buf_cons(void *vbuf, void *unused, int kmflag)
1762{
1763	arc_buf_t *buf = vbuf;
1764
1765	bzero(buf, sizeof (arc_buf_t));
1766	mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL);
1767	arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1768
1769	return (0);
1770}
1771
1772/*
1773 * Destructor callback - called when a cached buf is
1774 * no longer required.
1775 */
1776/* ARGSUSED */
1777static void
1778hdr_full_dest(void *vbuf, void *unused)
1779{
1780	arc_buf_hdr_t *hdr = vbuf;
1781
1782	ASSERT(HDR_EMPTY(hdr));
1783	cv_destroy(&hdr->b_l1hdr.b_cv);
1784	refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1785	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1786	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1787	arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1788}
1789
1790/* ARGSUSED */
1791static void
1792hdr_l2only_dest(void *vbuf, void *unused)
1793{
1794	arc_buf_hdr_t *hdr = vbuf;
1795
1796	ASSERT(HDR_EMPTY(hdr));
1797	arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1798}
1799
1800/* ARGSUSED */
1801static void
1802buf_dest(void *vbuf, void *unused)
1803{
1804	arc_buf_t *buf = vbuf;
1805
1806	mutex_destroy(&buf->b_evict_lock);
1807	arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1808}
1809
1810/*
1811 * Reclaim callback -- invoked when memory is low.
1812 */
1813/* ARGSUSED */
1814static void
1815hdr_recl(void *unused)
1816{
1817	dprintf("hdr_recl called\n");
1818	/*
1819	 * umem calls the reclaim func when we destroy the buf cache,
1820	 * which is after we do arc_fini().
1821	 */
1822	if (!arc_dead)
1823		cv_signal(&arc_reclaim_thread_cv);
1824}
1825
1826static void
1827buf_init(void)
1828{
1829	uint64_t *ct;
1830	uint64_t hsize = 1ULL << 12;
1831	int i, j;
1832
1833	/*
1834	 * The hash table is big enough to fill all of physical memory
1835	 * with an average block size of zfs_arc_average_blocksize (default 8K).
1836	 * By default, the table will take up
1837	 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
1838	 */
1839	while (hsize * zfs_arc_average_blocksize < (uint64_t)physmem * PAGESIZE)
1840		hsize <<= 1;
1841retry:
1842	buf_hash_table.ht_mask = hsize - 1;
1843	buf_hash_table.ht_table =
1844	    kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
1845	if (buf_hash_table.ht_table == NULL) {
1846		ASSERT(hsize > (1ULL << 8));
1847		hsize >>= 1;
1848		goto retry;
1849	}
1850
1851	hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
1852	    0, hdr_full_cons, hdr_full_dest, hdr_recl, NULL, NULL, 0);
1853	hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
1854	    HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, hdr_recl,
1855	    NULL, NULL, 0);
1856	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
1857	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
1858
1859	for (i = 0; i < 256; i++)
1860		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
1861			*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
1862
1863	for (i = 0; i < BUF_LOCKS; i++) {
1864		mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
1865		    NULL, MUTEX_DEFAULT, NULL);
1866	}
1867}
1868
1869/*
1870 * This is the size that the buf occupies in memory. If the buf is compressed,
1871 * it will correspond to the compressed size. You should use this method of
1872 * getting the buf size unless you explicitly need the logical size.
1873 */
1874int32_t
1875arc_buf_size(arc_buf_t *buf)
1876{
1877	return (ARC_BUF_COMPRESSED(buf) ?
1878	    HDR_GET_PSIZE(buf->b_hdr) : HDR_GET_LSIZE(buf->b_hdr));
1879}
1880
1881int32_t
1882arc_buf_lsize(arc_buf_t *buf)
1883{
1884	return (HDR_GET_LSIZE(buf->b_hdr));
1885}
1886
1887enum zio_compress
1888arc_get_compression(arc_buf_t *buf)
1889{
1890	return (ARC_BUF_COMPRESSED(buf) ?
1891	    HDR_GET_COMPRESS(buf->b_hdr) : ZIO_COMPRESS_OFF);
1892}
1893
1894#define	ARC_MINTIME	(hz>>4) /* 62 ms */
1895
1896static inline boolean_t
1897arc_buf_is_shared(arc_buf_t *buf)
1898{
1899	boolean_t shared = (buf->b_data != NULL &&
1900	    buf->b_hdr->b_l1hdr.b_pabd != NULL &&
1901	    abd_is_linear(buf->b_hdr->b_l1hdr.b_pabd) &&
1902	    buf->b_data == abd_to_buf(buf->b_hdr->b_l1hdr.b_pabd));
1903	IMPLY(shared, HDR_SHARED_DATA(buf->b_hdr));
1904	IMPLY(shared, ARC_BUF_SHARED(buf));
1905	IMPLY(shared, ARC_BUF_COMPRESSED(buf) || ARC_BUF_LAST(buf));
1906
1907	/*
1908	 * It would be nice to assert arc_can_share() too, but the "hdr isn't
1909	 * already being shared" requirement prevents us from doing that.
1910	 */
1911
1912	return (shared);
1913}
1914
1915/*
1916 * Free the checksum associated with this header. If there is no checksum, this
1917 * is a no-op.
1918 */
1919static inline void
1920arc_cksum_free(arc_buf_hdr_t *hdr)
1921{
1922	ASSERT(HDR_HAS_L1HDR(hdr));
1923	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1924	if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1925		kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));
1926		hdr->b_l1hdr.b_freeze_cksum = NULL;
1927	}
1928	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1929}
1930
1931/*
1932 * Return true iff at least one of the bufs on hdr is not compressed.
1933 */
1934static boolean_t
1935arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr)
1936{
1937	for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) {
1938		if (!ARC_BUF_COMPRESSED(b)) {
1939			return (B_TRUE);
1940		}
1941	}
1942	return (B_FALSE);
1943}
1944
1945/*
1946 * If we've turned on the ZFS_DEBUG_MODIFY flag, verify that the buf's data
1947 * matches the checksum that is stored in the hdr. If there is no checksum,
1948 * or if the buf is compressed, this is a no-op.
1949 */
1950static void
1951arc_cksum_verify(arc_buf_t *buf)
1952{
1953	arc_buf_hdr_t *hdr = buf->b_hdr;
1954	zio_cksum_t zc;
1955
1956	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1957		return;
1958
1959	if (ARC_BUF_COMPRESSED(buf)) {
1960		ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
1961		    arc_hdr_has_uncompressed_buf(hdr));
1962		return;
1963	}
1964
1965	ASSERT(HDR_HAS_L1HDR(hdr));
1966
1967	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1968	if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) {
1969		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1970		return;
1971	}
1972
1973	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, &zc);
1974	if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))
1975		panic("buffer modified while frozen!");
1976	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1977}
1978
1979static boolean_t
1980arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio)
1981{
1982	enum zio_compress compress = BP_GET_COMPRESS(zio->io_bp);
1983	boolean_t valid_cksum;
1984
1985	ASSERT(!BP_IS_EMBEDDED(zio->io_bp));
1986	VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));
1987
1988	/*
1989	 * We rely on the blkptr's checksum to determine if the block
1990	 * is valid or not. When compressed arc is enabled, the l2arc
1991	 * writes the block to the l2arc just as it appears in the pool.
1992	 * This allows us to use the blkptr's checksum to validate the
1993	 * data that we just read off of the l2arc without having to store
1994	 * a separate checksum in the arc_buf_hdr_t. However, if compressed
1995	 * arc is disabled, then the data written to the l2arc is always
1996	 * uncompressed and won't match the block as it exists in the main
1997	 * pool. When this is the case, we must first compress it if it is
1998	 * compressed on the main pool before we can validate the checksum.
1999	 */
2000	if (!HDR_COMPRESSION_ENABLED(hdr) && compress != ZIO_COMPRESS_OFF) {
2001		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
2002		uint64_t lsize = HDR_GET_LSIZE(hdr);
2003		uint64_t csize;
2004
2005		abd_t *cdata = abd_alloc_linear(HDR_GET_PSIZE(hdr), B_TRUE);
2006		csize = zio_compress_data(compress, zio->io_abd,
2007		    abd_to_buf(cdata), lsize);
2008
2009		ASSERT3U(csize, <=, HDR_GET_PSIZE(hdr));
2010		if (csize < HDR_GET_PSIZE(hdr)) {
2011			/*
2012			 * Compressed blocks are always a multiple of the
2013			 * smallest ashift in the pool. Ideally, we would
2014			 * like to round up the csize to the next
2015			 * spa_min_ashift but that value may have changed
2016			 * since the block was last written. Instead,
2017			 * we rely on the fact that the hdr's psize
2018			 * was set to the psize of the block when it was
2019			 * last written. We set the csize to that value
2020			 * and zero out any part that should not contain
2021			 * data.
2022			 */
2023			abd_zero_off(cdata, csize, HDR_GET_PSIZE(hdr) - csize);
2024			csize = HDR_GET_PSIZE(hdr);
2025		}
2026		zio_push_transform(zio, cdata, csize, HDR_GET_PSIZE(hdr), NULL);
2027	}
2028
2029	/*
2030	 * Block pointers always store the checksum for the logical data.
2031	 * If the block pointer has the gang bit set, then the checksum
2032	 * it represents is for the reconstituted data and not for an
2033	 * individual gang member. The zio pipeline, however, must be able to
2034	 * determine the checksum of each of the gang constituents so it
2035	 * treats the checksum comparison differently than what we need
2036	 * for l2arc blocks. This prevents us from using the
2037	 * zio_checksum_error() interface directly. Instead we must call the
2038	 * zio_checksum_error_impl() so that we can ensure the checksum is
2039	 * generated using the correct checksum algorithm and accounts for the
2040	 * logical I/O size and not just a gang fragment.
2041	 */
2042	valid_cksum = (zio_checksum_error_impl(zio->io_spa, zio->io_bp,
2043	    BP_GET_CHECKSUM(zio->io_bp), zio->io_abd, zio->io_size,
2044	    zio->io_offset, NULL) == 0);
2045	zio_pop_transforms(zio);
2046	return (valid_cksum);
2047}
2048
2049/*
2050 * Given a buf full of data, if ZFS_DEBUG_MODIFY is enabled this computes a
2051 * checksum and attaches it to the buf's hdr so that we can ensure that the buf
2052 * isn't modified later on. If buf is compressed or there is already a checksum
2053 * on the hdr, this is a no-op (we only checksum uncompressed bufs).
2054 */
2055static void
2056arc_cksum_compute(arc_buf_t *buf)
2057{
2058	arc_buf_hdr_t *hdr = buf->b_hdr;
2059
2060	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
2061		return;
2062
2063	ASSERT(HDR_HAS_L1HDR(hdr));
2064
2065	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
2066	if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
2067		ASSERT(arc_hdr_has_uncompressed_buf(hdr));
2068		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2069		return;
2070	} else if (ARC_BUF_COMPRESSED(buf)) {
2071		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2072		return;
2073	}
2074
2075	ASSERT(!ARC_BUF_COMPRESSED(buf));
2076	hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),
2077	    KM_SLEEP);
2078	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL,
2079	    hdr->b_l1hdr.b_freeze_cksum);
2080	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2081#ifdef illumos
2082	arc_buf_watch(buf);
2083#endif
2084}
2085
2086#ifdef illumos
2087#ifndef _KERNEL
2088typedef struct procctl {
2089	long cmd;
2090	prwatch_t prwatch;
2091} procctl_t;
2092#endif
2093
2094/* ARGSUSED */
2095static void
2096arc_buf_unwatch(arc_buf_t *buf)
2097{
2098#ifndef _KERNEL
2099	if (arc_watch) {
2100		int result;
2101		procctl_t ctl;
2102		ctl.cmd = PCWATCH;
2103		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
2104		ctl.prwatch.pr_size = 0;
2105		ctl.prwatch.pr_wflags = 0;
2106		result = write(arc_procfd, &ctl, sizeof (ctl));
2107		ASSERT3U(result, ==, sizeof (ctl));
2108	}
2109#endif
2110}
2111
2112/* ARGSUSED */
2113static void
2114arc_buf_watch(arc_buf_t *buf)
2115{
2116#ifndef _KERNEL
2117	if (arc_watch) {
2118		int result;
2119		procctl_t ctl;
2120		ctl.cmd = PCWATCH;
2121		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
2122		ctl.prwatch.pr_size = arc_buf_size(buf);
2123		ctl.prwatch.pr_wflags = WA_WRITE;
2124		result = write(arc_procfd, &ctl, sizeof (ctl));
2125		ASSERT3U(result, ==, sizeof (ctl));
2126	}
2127#endif
2128}
2129#endif /* illumos */
2130
2131static arc_buf_contents_t
2132arc_buf_type(arc_buf_hdr_t *hdr)
2133{
2134	arc_buf_contents_t type;
2135	if (HDR_ISTYPE_METADATA(hdr)) {
2136		type = ARC_BUFC_METADATA;
2137	} else {
2138		type = ARC_BUFC_DATA;
2139	}
2140	VERIFY3U(hdr->b_type, ==, type);
2141	return (type);
2142}
2143
2144boolean_t
2145arc_is_metadata(arc_buf_t *buf)
2146{
2147	return (HDR_ISTYPE_METADATA(buf->b_hdr) != 0);
2148}
2149
2150static uint32_t
2151arc_bufc_to_flags(arc_buf_contents_t type)
2152{
2153	switch (type) {
2154	case ARC_BUFC_DATA:
2155		/* metadata field is 0 if buffer contains normal data */
2156		return (0);
2157	case ARC_BUFC_METADATA:
2158		return (ARC_FLAG_BUFC_METADATA);
2159	default:
2160		break;
2161	}
2162	panic("undefined ARC buffer type!");
2163	return ((uint32_t)-1);
2164}
2165
2166void
2167arc_buf_thaw(arc_buf_t *buf)
2168{
2169	arc_buf_hdr_t *hdr = buf->b_hdr;
2170
2171	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
2172	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2173
2174	arc_cksum_verify(buf);
2175
2176	/*
2177	 * Compressed buffers do not manipulate the b_freeze_cksum or
2178	 * allocate b_thawed.
2179	 */
2180	if (ARC_BUF_COMPRESSED(buf)) {
2181		ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
2182		    arc_hdr_has_uncompressed_buf(hdr));
2183		return;
2184	}
2185
2186	ASSERT(HDR_HAS_L1HDR(hdr));
2187	arc_cksum_free(hdr);
2188
2189	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
2190#ifdef ZFS_DEBUG
2191	if (zfs_flags & ZFS_DEBUG_MODIFY) {
2192		if (hdr->b_l1hdr.b_thawed != NULL)
2193			kmem_free(hdr->b_l1hdr.b_thawed, 1);
2194		hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP);
2195	}
2196#endif
2197
2198	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2199
2200#ifdef illumos
2201	arc_buf_unwatch(buf);
2202#endif
2203}
2204
2205void
2206arc_buf_freeze(arc_buf_t *buf)
2207{
2208	arc_buf_hdr_t *hdr = buf->b_hdr;
2209	kmutex_t *hash_lock;
2210
2211	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
2212		return;
2213
2214	if (ARC_BUF_COMPRESSED(buf)) {
2215		ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
2216		    arc_hdr_has_uncompressed_buf(hdr));
2217		return;
2218	}
2219
2220	hash_lock = HDR_LOCK(hdr);
2221	mutex_enter(hash_lock);
2222
2223	ASSERT(HDR_HAS_L1HDR(hdr));
2224	ASSERT(hdr->b_l1hdr.b_freeze_cksum != NULL ||
2225	    hdr->b_l1hdr.b_state == arc_anon);
2226	arc_cksum_compute(buf);
2227	mutex_exit(hash_lock);
2228}
2229
2230/*
2231 * The arc_buf_hdr_t's b_flags should never be modified directly. Instead,
2232 * the following functions should be used to ensure that the flags are
2233 * updated in a thread-safe way. When manipulating the flags either
2234 * the hash_lock must be held or the hdr must be undiscoverable. This
2235 * ensures that we're not racing with any other threads when updating
2236 * the flags.
2237 */
2238static inline void
2239arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
2240{
2241	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2242	hdr->b_flags |= flags;
2243}
2244
2245static inline void
2246arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
2247{
2248	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2249	hdr->b_flags &= ~flags;
2250}
2251
2252/*
2253 * Setting the compression bits in the arc_buf_hdr_t's b_flags is
2254 * done in a special way since we have to clear and set bits
2255 * at the same time. Consumers that wish to set the compression bits
2256 * must use this function to ensure that the flags are updated in
2257 * thread-safe manner.
2258 */
2259static void
2260arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp)
2261{
2262	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2263
2264	/*
2265	 * Holes and embedded blocks will always have a psize = 0 so
2266	 * we ignore the compression of the blkptr and set the
2267	 * arc_buf_hdr_t's compression to ZIO_COMPRESS_OFF.
2268	 * Holes and embedded blocks remain anonymous so we don't
2269	 * want to uncompress them. Mark them as uncompressed.
2270	 */
2271	if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) {
2272		arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
2273		HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF);
2274		ASSERT(!HDR_COMPRESSION_ENABLED(hdr));
2275		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
2276	} else {
2277		arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
2278		HDR_SET_COMPRESS(hdr, cmp);
2279		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp);
2280		ASSERT(HDR_COMPRESSION_ENABLED(hdr));
2281	}
2282}
2283
2284/*
2285 * Looks for another buf on the same hdr which has the data decompressed, copies
2286 * from it, and returns true. If no such buf exists, returns false.
2287 */
2288static boolean_t
2289arc_buf_try_copy_decompressed_data(arc_buf_t *buf)
2290{
2291	arc_buf_hdr_t *hdr = buf->b_hdr;
2292	boolean_t copied = B_FALSE;
2293
2294	ASSERT(HDR_HAS_L1HDR(hdr));
2295	ASSERT3P(buf->b_data, !=, NULL);
2296	ASSERT(!ARC_BUF_COMPRESSED(buf));
2297
2298	for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL;
2299	    from = from->b_next) {
2300		/* can't use our own data buffer */
2301		if (from == buf) {
2302			continue;
2303		}
2304
2305		if (!ARC_BUF_COMPRESSED(from)) {
2306			bcopy(from->b_data, buf->b_data, arc_buf_size(buf));
2307			copied = B_TRUE;
2308			break;
2309		}
2310	}
2311
2312	/*
2313	 * There were no decompressed bufs, so there should not be a
2314	 * checksum on the hdr either.
2315	 */
2316	EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL);
2317
2318	return (copied);
2319}
2320
2321/*
2322 * Given a buf that has a data buffer attached to it, this function will
2323 * efficiently fill the buf with data of the specified compression setting from
2324 * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr
2325 * are already sharing a data buf, no copy is performed.
2326 *
2327 * If the buf is marked as compressed but uncompressed data was requested, this
2328 * will allocate a new data buffer for the buf, remove that flag, and fill the
2329 * buf with uncompressed data. You can't request a compressed buf on a hdr with
2330 * uncompressed data, and (since we haven't added support for it yet) if you
2331 * want compressed data your buf must already be marked as compressed and have
2332 * the correct-sized data buffer.
2333 */
2334static int
2335arc_buf_fill(arc_buf_t *buf, boolean_t compressed)
2336{
2337	arc_buf_hdr_t *hdr = buf->b_hdr;
2338	boolean_t hdr_compressed = (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
2339	dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;
2340
2341	ASSERT3P(buf->b_data, !=, NULL);
2342	IMPLY(compressed, hdr_compressed);
2343	IMPLY(compressed, ARC_BUF_COMPRESSED(buf));
2344
2345	if (hdr_compressed == compressed) {
2346		if (!arc_buf_is_shared(buf)) {
2347			abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd,
2348			    arc_buf_size(buf));
2349		}
2350	} else {
2351		ASSERT(hdr_compressed);
2352		ASSERT(!compressed);
2353		ASSERT3U(HDR_GET_LSIZE(hdr), !=, HDR_GET_PSIZE(hdr));
2354
2355		/*
2356		 * If the buf is sharing its data with the hdr, unlink it and
2357		 * allocate a new data buffer for the buf.
2358		 */
2359		if (arc_buf_is_shared(buf)) {
2360			ASSERT(ARC_BUF_COMPRESSED(buf));
2361
2362			/* We need to give the buf it's own b_data */
2363			buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
2364			buf->b_data =
2365			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2366			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2367
2368			/* Previously overhead was 0; just add new overhead */
2369			ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));
2370		} else if (ARC_BUF_COMPRESSED(buf)) {
2371			/* We need to reallocate the buf's b_data */
2372			arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr),
2373			    buf);
2374			buf->b_data =
2375			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2376
2377			/* We increased the size of b_data; update overhead */
2378			ARCSTAT_INCR(arcstat_overhead_size,
2379			    HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr));
2380		}
2381
2382		/*
2383		 * Regardless of the buf's previous compression settings, it
2384		 * should not be compressed at the end of this function.
2385		 */
2386		buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
2387
2388		/*
2389		 * Try copying the data from another buf which already has a
2390		 * decompressed version. If that's not possible, it's time to
2391		 * bite the bullet and decompress the data from the hdr.
2392		 */
2393		if (arc_buf_try_copy_decompressed_data(buf)) {
2394			/* Skip byteswapping and checksumming (already done) */
2395			ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, !=, NULL);
2396			return (0);
2397		} else {
2398			int error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
2399			    hdr->b_l1hdr.b_pabd, buf->b_data,
2400			    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
2401
2402			/*
2403			 * Absent hardware errors or software bugs, this should
2404			 * be impossible, but log it anyway so we can debug it.
2405			 */
2406			if (error != 0) {
2407				zfs_dbgmsg(
2408				    "hdr %p, compress %d, psize %d, lsize %d",
2409				    hdr, HDR_GET_COMPRESS(hdr),
2410				    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
2411				return (SET_ERROR(EIO));
2412			}
2413		}
2414	}
2415
2416	/* Byteswap the buf's data if necessary */
2417	if (bswap != DMU_BSWAP_NUMFUNCS) {
2418		ASSERT(!HDR_SHARED_DATA(hdr));
2419		ASSERT3U(bswap, <, DMU_BSWAP_NUMFUNCS);
2420		dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr));
2421	}
2422
2423	/* Compute the hdr's checksum if necessary */
2424	arc_cksum_compute(buf);
2425
2426	return (0);
2427}
2428
2429int
2430arc_decompress(arc_buf_t *buf)
2431{
2432	return (arc_buf_fill(buf, B_FALSE));
2433}
2434
2435/*
2436 * Return the size of the block, b_pabd, that is stored in the arc_buf_hdr_t.
2437 */
2438static uint64_t
2439arc_hdr_size(arc_buf_hdr_t *hdr)
2440{
2441	uint64_t size;
2442
2443	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
2444	    HDR_GET_PSIZE(hdr) > 0) {
2445		size = HDR_GET_PSIZE(hdr);
2446	} else {
2447		ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0);
2448		size = HDR_GET_LSIZE(hdr);
2449	}
2450	return (size);
2451}
2452
2453/*
2454 * Increment the amount of evictable space in the arc_state_t's refcount.
2455 * We account for the space used by the hdr and the arc buf individually
2456 * so that we can add and remove them from the refcount individually.
2457 */
2458static void
2459arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state)
2460{
2461	arc_buf_contents_t type = arc_buf_type(hdr);
2462
2463	ASSERT(HDR_HAS_L1HDR(hdr));
2464
2465	if (GHOST_STATE(state)) {
2466		ASSERT0(hdr->b_l1hdr.b_bufcnt);
2467		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2468		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2469		(void) refcount_add_many(&state->arcs_esize[type],
2470		    HDR_GET_LSIZE(hdr), hdr);
2471		return;
2472	}
2473
2474	ASSERT(!GHOST_STATE(state));
2475	if (hdr->b_l1hdr.b_pabd != NULL) {
2476		(void) refcount_add_many(&state->arcs_esize[type],
2477		    arc_hdr_size(hdr), hdr);
2478	}
2479	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2480	    buf = buf->b_next) {
2481		if (arc_buf_is_shared(buf))
2482			continue;
2483		(void) refcount_add_many(&state->arcs_esize[type],
2484		    arc_buf_size(buf), buf);
2485	}
2486}
2487
2488/*
2489 * Decrement the amount of evictable space in the arc_state_t's refcount.
2490 * We account for the space used by the hdr and the arc buf individually
2491 * so that we can add and remove them from the refcount individually.
2492 */
2493static void
2494arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state)
2495{
2496	arc_buf_contents_t type = arc_buf_type(hdr);
2497
2498	ASSERT(HDR_HAS_L1HDR(hdr));
2499
2500	if (GHOST_STATE(state)) {
2501		ASSERT0(hdr->b_l1hdr.b_bufcnt);
2502		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2503		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2504		(void) refcount_remove_many(&state->arcs_esize[type],
2505		    HDR_GET_LSIZE(hdr), hdr);
2506		return;
2507	}
2508
2509	ASSERT(!GHOST_STATE(state));
2510	if (hdr->b_l1hdr.b_pabd != NULL) {
2511		(void) refcount_remove_many(&state->arcs_esize[type],
2512		    arc_hdr_size(hdr), hdr);
2513	}
2514	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2515	    buf = buf->b_next) {
2516		if (arc_buf_is_shared(buf))
2517			continue;
2518		(void) refcount_remove_many(&state->arcs_esize[type],
2519		    arc_buf_size(buf), buf);
2520	}
2521}
2522
2523/*
2524 * Add a reference to this hdr indicating that someone is actively
2525 * referencing that memory. When the refcount transitions from 0 to 1,
2526 * we remove it from the respective arc_state_t list to indicate that
2527 * it is not evictable.
2528 */
2529static void
2530add_reference(arc_buf_hdr_t *hdr, void *tag)
2531{
2532	ASSERT(HDR_HAS_L1HDR(hdr));
2533	if (!MUTEX_HELD(HDR_LOCK(hdr))) {
2534		ASSERT(hdr->b_l1hdr.b_state == arc_anon);
2535		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2536		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2537	}
2538
2539	arc_state_t *state = hdr->b_l1hdr.b_state;
2540
2541	if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
2542	    (state != arc_anon)) {
2543		/* We don't use the L2-only state list. */
2544		if (state != arc_l2c_only) {
2545			multilist_remove(state->arcs_list[arc_buf_type(hdr)],
2546			    hdr);
2547			arc_evictable_space_decrement(hdr, state);
2548		}
2549		/* remove the prefetch flag if we get a reference */
2550		arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
2551	}
2552}
2553
2554/*
2555 * Remove a reference from this hdr. When the reference transitions from
2556 * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's
2557 * list making it eligible for eviction.
2558 */
2559static int
2560remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
2561{
2562	int cnt;
2563	arc_state_t *state = hdr->b_l1hdr.b_state;
2564
2565	ASSERT(HDR_HAS_L1HDR(hdr));
2566	ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
2567	ASSERT(!GHOST_STATE(state));
2568
2569	/*
2570	 * arc_l2c_only counts as a ghost state so we don't need to explicitly
2571	 * check to prevent usage of the arc_l2c_only list.
2572	 */
2573	if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
2574	    (state != arc_anon)) {
2575		multilist_insert(state->arcs_list[arc_buf_type(hdr)], hdr);
2576		ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
2577		arc_evictable_space_increment(hdr, state);
2578	}
2579	return (cnt);
2580}
2581
2582/*
2583 * Returns detailed information about a specific arc buffer.  When the
2584 * state_index argument is set the function will calculate the arc header
2585 * list position for its arc state.  Since this requires a linear traversal
2586 * callers are strongly encourage not to do this.  However, it can be helpful
2587 * for targeted analysis so the functionality is provided.
2588 */
2589void
2590arc_buf_info(arc_buf_t *ab, arc_buf_info_t *abi, int state_index)
2591{
2592	arc_buf_hdr_t *hdr = ab->b_hdr;
2593	l1arc_buf_hdr_t *l1hdr = NULL;
2594	l2arc_buf_hdr_t *l2hdr = NULL;
2595	arc_state_t *state = NULL;
2596
2597	memset(abi, 0, sizeof (arc_buf_info_t));
2598
2599	if (hdr == NULL)
2600		return;
2601
2602	abi->abi_flags = hdr->b_flags;
2603
2604	if (HDR_HAS_L1HDR(hdr)) {
2605		l1hdr = &hdr->b_l1hdr;
2606		state = l1hdr->b_state;
2607	}
2608	if (HDR_HAS_L2HDR(hdr))
2609		l2hdr = &hdr->b_l2hdr;
2610
2611	if (l1hdr) {
2612		abi->abi_bufcnt = l1hdr->b_bufcnt;
2613		abi->abi_access = l1hdr->b_arc_access;
2614		abi->abi_mru_hits = l1hdr->b_mru_hits;
2615		abi->abi_mru_ghost_hits = l1hdr->b_mru_ghost_hits;
2616		abi->abi_mfu_hits = l1hdr->b_mfu_hits;
2617		abi->abi_mfu_ghost_hits = l1hdr->b_mfu_ghost_hits;
2618		abi->abi_holds = refcount_count(&l1hdr->b_refcnt);
2619	}
2620
2621	if (l2hdr) {
2622		abi->abi_l2arc_dattr = l2hdr->b_daddr;
2623		abi->abi_l2arc_hits = l2hdr->b_hits;
2624	}
2625
2626	abi->abi_state_type = state ? state->arcs_state : ARC_STATE_ANON;
2627	abi->abi_state_contents = arc_buf_type(hdr);
2628	abi->abi_size = arc_hdr_size(hdr);
2629}
2630
2631/*
2632 * Move the supplied buffer to the indicated state. The hash lock
2633 * for the buffer must be held by the caller.
2634 */
2635static void
2636arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
2637    kmutex_t *hash_lock)
2638{
2639	arc_state_t *old_state;
2640	int64_t refcnt;
2641	uint32_t bufcnt;
2642	boolean_t update_old, update_new;
2643	arc_buf_contents_t buftype = arc_buf_type(hdr);
2644
2645	/*
2646	 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
2647	 * in arc_read() when bringing a buffer out of the L2ARC.  However, the
2648	 * L1 hdr doesn't always exist when we change state to arc_anon before
2649	 * destroying a header, in which case reallocating to add the L1 hdr is
2650	 * pointless.
2651	 */
2652	if (HDR_HAS_L1HDR(hdr)) {
2653		old_state = hdr->b_l1hdr.b_state;
2654		refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt);
2655		bufcnt = hdr->b_l1hdr.b_bufcnt;
2656		update_old = (bufcnt > 0 || hdr->b_l1hdr.b_pabd != NULL);
2657	} else {
2658		old_state = arc_l2c_only;
2659		refcnt = 0;
2660		bufcnt = 0;
2661		update_old = B_FALSE;
2662	}
2663	update_new = update_old;
2664
2665	ASSERT(MUTEX_HELD(hash_lock));
2666	ASSERT3P(new_state, !=, old_state);
2667	ASSERT(!GHOST_STATE(new_state) || bufcnt == 0);
2668	ASSERT(old_state != arc_anon || bufcnt <= 1);
2669
2670	/*
2671	 * If this buffer is evictable, transfer it from the
2672	 * old state list to the new state list.
2673	 */
2674	if (refcnt == 0) {
2675		if (old_state != arc_anon && old_state != arc_l2c_only) {
2676			ASSERT(HDR_HAS_L1HDR(hdr));
2677			multilist_remove(old_state->arcs_list[buftype], hdr);
2678
2679			if (GHOST_STATE(old_state)) {
2680				ASSERT0(bufcnt);
2681				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2682				update_old = B_TRUE;
2683			}
2684			arc_evictable_space_decrement(hdr, old_state);
2685		}
2686		if (new_state != arc_anon && new_state != arc_l2c_only) {
2687
2688			/*
2689			 * An L1 header always exists here, since if we're
2690			 * moving to some L1-cached state (i.e. not l2c_only or
2691			 * anonymous), we realloc the header to add an L1hdr
2692			 * beforehand.
2693			 */
2694			ASSERT(HDR_HAS_L1HDR(hdr));
2695			multilist_insert(new_state->arcs_list[buftype], hdr);
2696
2697			if (GHOST_STATE(new_state)) {
2698				ASSERT0(bufcnt);
2699				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2700				update_new = B_TRUE;
2701			}
2702			arc_evictable_space_increment(hdr, new_state);
2703		}
2704	}
2705
2706	ASSERT(!HDR_EMPTY(hdr));
2707	if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
2708		buf_hash_remove(hdr);
2709
2710	/* adjust state sizes (ignore arc_l2c_only) */
2711
2712	if (update_new && new_state != arc_l2c_only) {
2713		ASSERT(HDR_HAS_L1HDR(hdr));
2714		if (GHOST_STATE(new_state)) {
2715			ASSERT0(bufcnt);
2716
2717			/*
2718			 * When moving a header to a ghost state, we first
2719			 * remove all arc buffers. Thus, we'll have a
2720			 * bufcnt of zero, and no arc buffer to use for
2721			 * the reference. As a result, we use the arc
2722			 * header pointer for the reference.
2723			 */
2724			(void) refcount_add_many(&new_state->arcs_size,
2725			    HDR_GET_LSIZE(hdr), hdr);
2726			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2727		} else {
2728			uint32_t buffers = 0;
2729
2730			/*
2731			 * Each individual buffer holds a unique reference,
2732			 * thus we must remove each of these references one
2733			 * at a time.
2734			 */
2735			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2736			    buf = buf->b_next) {
2737				ASSERT3U(bufcnt, !=, 0);
2738				buffers++;
2739
2740				/*
2741				 * When the arc_buf_t is sharing the data
2742				 * block with the hdr, the owner of the
2743				 * reference belongs to the hdr. Only
2744				 * add to the refcount if the arc_buf_t is
2745				 * not shared.
2746				 */
2747				if (arc_buf_is_shared(buf))
2748					continue;
2749
2750				(void) refcount_add_many(&new_state->arcs_size,
2751				    arc_buf_size(buf), buf);
2752			}
2753			ASSERT3U(bufcnt, ==, buffers);
2754
2755			if (hdr->b_l1hdr.b_pabd != NULL) {
2756				(void) refcount_add_many(&new_state->arcs_size,
2757				    arc_hdr_size(hdr), hdr);
2758			} else {
2759				ASSERT(GHOST_STATE(old_state));
2760			}
2761		}
2762	}
2763
2764	if (update_old && old_state != arc_l2c_only) {
2765		ASSERT(HDR_HAS_L1HDR(hdr));
2766		if (GHOST_STATE(old_state)) {
2767			ASSERT0(bufcnt);
2768			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2769
2770			/*
2771			 * When moving a header off of a ghost state,
2772			 * the header will not contain any arc buffers.
2773			 * We use the arc header pointer for the reference
2774			 * which is exactly what we did when we put the
2775			 * header on the ghost state.
2776			 */
2777
2778			(void) refcount_remove_many(&old_state->arcs_size,
2779			    HDR_GET_LSIZE(hdr), hdr);
2780		} else {
2781			uint32_t buffers = 0;
2782
2783			/*
2784			 * Each individual buffer holds a unique reference,
2785			 * thus we must remove each of these references one
2786			 * at a time.
2787			 */
2788			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2789			    buf = buf->b_next) {
2790				ASSERT3U(bufcnt, !=, 0);
2791				buffers++;
2792
2793				/*
2794				 * When the arc_buf_t is sharing the data
2795				 * block with the hdr, the owner of the
2796				 * reference belongs to the hdr. Only
2797				 * add to the refcount if the arc_buf_t is
2798				 * not shared.
2799				 */
2800				if (arc_buf_is_shared(buf))
2801					continue;
2802
2803				(void) refcount_remove_many(
2804				    &old_state->arcs_size, arc_buf_size(buf),
2805				    buf);
2806			}
2807			ASSERT3U(bufcnt, ==, buffers);
2808			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
2809			(void) refcount_remove_many(
2810			    &old_state->arcs_size, arc_hdr_size(hdr), hdr);
2811		}
2812	}
2813
2814	if (HDR_HAS_L1HDR(hdr))
2815		hdr->b_l1hdr.b_state = new_state;
2816
2817	/*
2818	 * L2 headers should never be on the L2 state list since they don't
2819	 * have L1 headers allocated.
2820	 */
2821	ASSERT(multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
2822	    multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
2823}
2824
2825void
2826arc_space_consume(uint64_t space, arc_space_type_t type)
2827{
2828	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2829
2830	switch (type) {
2831	case ARC_SPACE_DATA:
2832		aggsum_add(&astat_data_size, space);
2833		break;
2834	case ARC_SPACE_META:
2835		aggsum_add(&astat_metadata_size, space);
2836		break;
2837	case ARC_SPACE_BONUS:
2838		aggsum_add(&astat_bonus_size, space);
2839		break;
2840	case ARC_SPACE_DNODE:
2841		aggsum_add(&astat_dnode_size, space);
2842		break;
2843	case ARC_SPACE_DBUF:
2844		aggsum_add(&astat_dbuf_size, space);
2845		break;
2846	case ARC_SPACE_HDRS:
2847		aggsum_add(&astat_hdr_size, space);
2848		break;
2849	case ARC_SPACE_L2HDRS:
2850		aggsum_add(&astat_l2_hdr_size, space);
2851		break;
2852	}
2853
2854	if (type != ARC_SPACE_DATA)
2855		aggsum_add(&arc_meta_used, space);
2856
2857	aggsum_add(&arc_size, space);
2858}
2859
2860void
2861arc_space_return(uint64_t space, arc_space_type_t type)
2862{
2863	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2864
2865	switch (type) {
2866	case ARC_SPACE_DATA:
2867		aggsum_add(&astat_data_size, -space);
2868		break;
2869	case ARC_SPACE_META:
2870		aggsum_add(&astat_metadata_size, -space);
2871		break;
2872	case ARC_SPACE_BONUS:
2873		aggsum_add(&astat_bonus_size, -space);
2874		break;
2875	case ARC_SPACE_DNODE:
2876		aggsum_add(&astat_dnode_size, -space);
2877		break;
2878	case ARC_SPACE_DBUF:
2879		aggsum_add(&astat_dbuf_size, -space);
2880		break;
2881	case ARC_SPACE_HDRS:
2882		aggsum_add(&astat_hdr_size, -space);
2883		break;
2884	case ARC_SPACE_L2HDRS:
2885		aggsum_add(&astat_l2_hdr_size, -space);
2886		break;
2887	}
2888
2889	if (type != ARC_SPACE_DATA) {
2890		ASSERT(aggsum_compare(&arc_meta_used, space) >= 0);
2891		/*
2892		 * We use the upper bound here rather than the precise value
2893		 * because the arc_meta_max value doesn't need to be
2894		 * precise. It's only consumed by humans via arcstats.
2895		 */
2896		if (arc_meta_max < aggsum_upper_bound(&arc_meta_used))
2897			arc_meta_max = aggsum_upper_bound(&arc_meta_used);
2898		aggsum_add(&arc_meta_used, -space);
2899	}
2900
2901	ASSERT(aggsum_compare(&arc_size, space) >= 0);
2902	aggsum_add(&arc_size, -space);
2903}
2904
2905/*
2906 * Given a hdr and a buf, returns whether that buf can share its b_data buffer
2907 * with the hdr's b_pabd.
2908 */
2909static boolean_t
2910arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2911{
2912	/*
2913	 * The criteria for sharing a hdr's data are:
2914	 * 1. the hdr's compression matches the buf's compression
2915	 * 2. the hdr doesn't need to be byteswapped
2916	 * 3. the hdr isn't already being shared
2917	 * 4. the buf is either compressed or it is the last buf in the hdr list
2918	 *
2919	 * Criterion #4 maintains the invariant that shared uncompressed
2920	 * bufs must be the final buf in the hdr's b_buf list. Reading this, you
2921	 * might ask, "if a compressed buf is allocated first, won't that be the
2922	 * last thing in the list?", but in that case it's impossible to create
2923	 * a shared uncompressed buf anyway (because the hdr must be compressed
2924	 * to have the compressed buf). You might also think that #3 is
2925	 * sufficient to make this guarantee, however it's possible
2926	 * (specifically in the rare L2ARC write race mentioned in
2927	 * arc_buf_alloc_impl()) there will be an existing uncompressed buf that
2928	 * is sharable, but wasn't at the time of its allocation. Rather than
2929	 * allow a new shared uncompressed buf to be created and then shuffle
2930	 * the list around to make it the last element, this simply disallows
2931	 * sharing if the new buf isn't the first to be added.
2932	 */
2933	ASSERT3P(buf->b_hdr, ==, hdr);
2934	boolean_t hdr_compressed = HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF;
2935	boolean_t buf_compressed = ARC_BUF_COMPRESSED(buf) != 0;
2936	return (buf_compressed == hdr_compressed &&
2937	    hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&
2938	    !HDR_SHARED_DATA(hdr) &&
2939	    (ARC_BUF_LAST(buf) || ARC_BUF_COMPRESSED(buf)));
2940}
2941
2942/*
2943 * Allocate a buf for this hdr. If you care about the data that's in the hdr,
2944 * or if you want a compressed buffer, pass those flags in. Returns 0 if the
2945 * copy was made successfully, or an error code otherwise.
2946 */
2947static int
2948arc_buf_alloc_impl(arc_buf_hdr_t *hdr, void *tag, boolean_t compressed,
2949    boolean_t fill, arc_buf_t **ret)
2950{
2951	arc_buf_t *buf;
2952
2953	ASSERT(HDR_HAS_L1HDR(hdr));
2954	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
2955	VERIFY(hdr->b_type == ARC_BUFC_DATA ||
2956	    hdr->b_type == ARC_BUFC_METADATA);
2957	ASSERT3P(ret, !=, NULL);
2958	ASSERT3P(*ret, ==, NULL);
2959
2960	buf = *ret = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
2961	buf->b_hdr = hdr;
2962	buf->b_data = NULL;
2963	buf->b_next = hdr->b_l1hdr.b_buf;
2964	buf->b_flags = 0;
2965
2966	add_reference(hdr, tag);
2967
2968	/*
2969	 * We're about to change the hdr's b_flags. We must either
2970	 * hold the hash_lock or be undiscoverable.
2971	 */
2972	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2973
2974	/*
2975	 * Only honor requests for compressed bufs if the hdr is actually
2976	 * compressed.
2977	 */
2978	if (compressed && HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF)
2979		buf->b_flags |= ARC_BUF_FLAG_COMPRESSED;
2980
2981	/*
2982	 * If the hdr's data can be shared then we share the data buffer and
2983	 * set the appropriate bit in the hdr's b_flags to indicate the hdr is
2984	 * sharing it's b_pabd with the arc_buf_t. Otherwise, we allocate a new
2985	 * buffer to store the buf's data.
2986	 *
2987	 * There are two additional restrictions here because we're sharing
2988	 * hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be
2989	 * actively involved in an L2ARC write, because if this buf is used by
2990	 * an arc_write() then the hdr's data buffer will be released when the
2991	 * write completes, even though the L2ARC write might still be using it.
2992	 * Second, the hdr's ABD must be linear so that the buf's user doesn't
2993	 * need to be ABD-aware.
2994	 */
2995	boolean_t can_share = arc_can_share(hdr, buf) && !HDR_L2_WRITING(hdr) &&
2996	    abd_is_linear(hdr->b_l1hdr.b_pabd);
2997
2998	/* Set up b_data and sharing */
2999	if (can_share) {
3000		buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd);
3001		buf->b_flags |= ARC_BUF_FLAG_SHARED;
3002		arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
3003	} else {
3004		buf->b_data =
3005		    arc_get_data_buf(hdr, arc_buf_size(buf), buf);
3006		ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
3007	}
3008	VERIFY3P(buf->b_data, !=, NULL);
3009
3010	hdr->b_l1hdr.b_buf = buf;
3011	hdr->b_l1hdr.b_bufcnt += 1;
3012
3013	/*
3014	 * If the user wants the data from the hdr, we need to either copy or
3015	 * decompress the data.
3016	 */
3017	if (fill) {
3018		return (arc_buf_fill(buf, ARC_BUF_COMPRESSED(buf) != 0));
3019	}
3020
3021	return (0);
3022}
3023
3024static char *arc_onloan_tag = "onloan";
3025
3026static inline void
3027arc_loaned_bytes_update(int64_t delta)
3028{
3029	atomic_add_64(&arc_loaned_bytes, delta);
3030
3031	/* assert that it did not wrap around */
3032	ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
3033}
3034
3035/*
3036 * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
3037 * flight data by arc_tempreserve_space() until they are "returned". Loaned
3038 * buffers must be returned to the arc before they can be used by the DMU or
3039 * freed.
3040 */
3041arc_buf_t *
3042arc_loan_buf(spa_t *spa, boolean_t is_metadata, int size)
3043{
3044	arc_buf_t *buf = arc_alloc_buf(spa, arc_onloan_tag,
3045	    is_metadata ? ARC_BUFC_METADATA : ARC_BUFC_DATA, size);
3046
3047	arc_loaned_bytes_update(arc_buf_size(buf));
3048
3049	return (buf);
3050}
3051
3052arc_buf_t *
3053arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize,
3054    enum zio_compress compression_type)
3055{
3056	arc_buf_t *buf = arc_alloc_compressed_buf(spa, arc_onloan_tag,
3057	    psize, lsize, compression_type);
3058
3059	arc_loaned_bytes_update(arc_buf_size(buf));
3060
3061	return (buf);
3062}
3063
3064
3065/*
3066 * Return a loaned arc buffer to the arc.
3067 */
3068void
3069arc_return_buf(arc_buf_t *buf, void *tag)
3070{
3071	arc_buf_hdr_t *hdr = buf->b_hdr;
3072
3073	ASSERT3P(buf->b_data, !=, NULL);
3074	ASSERT(HDR_HAS_L1HDR(hdr));
3075	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
3076	(void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
3077
3078	arc_loaned_bytes_update(-arc_buf_size(buf));
3079}
3080
3081/* Detach an arc_buf from a dbuf (tag) */
3082void
3083arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
3084{
3085	arc_buf_hdr_t *hdr = buf->b_hdr;
3086
3087	ASSERT3P(buf->b_data, !=, NULL);
3088	ASSERT(HDR_HAS_L1HDR(hdr));
3089	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
3090	(void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
3091
3092	arc_loaned_bytes_update(arc_buf_size(buf));
3093}
3094
3095static void
3096l2arc_free_abd_on_write(abd_t *abd, size_t size, arc_buf_contents_t type)
3097{
3098	l2arc_data_free_t *df = kmem_alloc(sizeof (*df), KM_SLEEP);
3099
3100	df->l2df_abd = abd;
3101	df->l2df_size = size;
3102	df->l2df_type = type;
3103	mutex_enter(&l2arc_free_on_write_mtx);
3104	list_insert_head(l2arc_free_on_write, df);
3105	mutex_exit(&l2arc_free_on_write_mtx);
3106}
3107
3108static void
3109arc_hdr_free_on_write(arc_buf_hdr_t *hdr)
3110{
3111	arc_state_t *state = hdr->b_l1hdr.b_state;
3112	arc_buf_contents_t type = arc_buf_type(hdr);
3113	uint64_t size = arc_hdr_size(hdr);
3114
3115	/* protected by hash lock, if in the hash table */
3116	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
3117		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3118		ASSERT(state != arc_anon && state != arc_l2c_only);
3119
3120		(void) refcount_remove_many(&state->arcs_esize[type],
3121		    size, hdr);
3122	}
3123	(void) refcount_remove_many(&state->arcs_size, size, hdr);
3124	if (type == ARC_BUFC_METADATA) {
3125		arc_space_return(size, ARC_SPACE_META);
3126	} else {
3127		ASSERT(type == ARC_BUFC_DATA);
3128		arc_space_return(size, ARC_SPACE_DATA);
3129	}
3130
3131	l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type);
3132}
3133
3134/*
3135 * Share the arc_buf_t's data with the hdr. Whenever we are sharing the
3136 * data buffer, we transfer the refcount ownership to the hdr and update
3137 * the appropriate kstats.
3138 */
3139static void
3140arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
3141{
3142	arc_state_t *state = hdr->b_l1hdr.b_state;
3143
3144	ASSERT(arc_can_share(hdr, buf));
3145	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3146	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
3147
3148	/*
3149	 * Start sharing the data buffer. We transfer the
3150	 * refcount ownership to the hdr since it always owns
3151	 * the refcount whenever an arc_buf_t is shared.
3152	 */
3153	refcount_transfer_ownership(&state->arcs_size, buf, hdr);
3154	hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf));
3155	abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd,
3156	    HDR_ISTYPE_METADATA(hdr));
3157	arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
3158	buf->b_flags |= ARC_BUF_FLAG_SHARED;
3159
3160	/*
3161	 * Since we've transferred ownership to the hdr we need
3162	 * to increment its compressed and uncompressed kstats and
3163	 * decrement the overhead size.
3164	 */
3165	ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
3166	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
3167	ARCSTAT_INCR(arcstat_overhead_size, -arc_buf_size(buf));
3168}
3169
3170static void
3171arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
3172{
3173	arc_state_t *state = hdr->b_l1hdr.b_state;
3174
3175	ASSERT(arc_buf_is_shared(buf));
3176	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3177	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
3178
3179	/*
3180	 * We are no longer sharing this buffer so we need
3181	 * to transfer its ownership to the rightful owner.
3182	 */
3183	refcount_transfer_ownership(&state->arcs_size, hdr, buf);
3184	arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
3185	abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd);
3186	abd_put(hdr->b_l1hdr.b_pabd);
3187	hdr->b_l1hdr.b_pabd = NULL;
3188	buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
3189
3190	/*
3191	 * Since the buffer is no longer shared between
3192	 * the arc buf and the hdr, count it as overhead.
3193	 */
3194	ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
3195	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3196	ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
3197}
3198
3199/*
3200 * Remove an arc_buf_t from the hdr's buf list and return the last
3201 * arc_buf_t on the list. If no buffers remain on the list then return
3202 * NULL.
3203 */
3204static arc_buf_t *
3205arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf)
3206{
3207	ASSERT(HDR_HAS_L1HDR(hdr));
3208	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
3209
3210	arc_buf_t **bufp = &hdr->b_l1hdr.b_buf;
3211	arc_buf_t *lastbuf = NULL;
3212
3213	/*
3214	 * Remove the buf from the hdr list and locate the last
3215	 * remaining buffer on the list.
3216	 */
3217	while (*bufp != NULL) {
3218		if (*bufp == buf)
3219			*bufp = buf->b_next;
3220
3221		/*
3222		 * If we've removed a buffer in the middle of
3223		 * the list then update the lastbuf and update
3224		 * bufp.
3225		 */
3226		if (*bufp != NULL) {
3227			lastbuf = *bufp;
3228			bufp = &(*bufp)->b_next;
3229		}
3230	}
3231	buf->b_next = NULL;
3232	ASSERT3P(lastbuf, !=, buf);
3233	IMPLY(hdr->b_l1hdr.b_bufcnt > 0, lastbuf != NULL);
3234	IMPLY(hdr->b_l1hdr.b_bufcnt > 0, hdr->b_l1hdr.b_buf != NULL);
3235	IMPLY(lastbuf != NULL, ARC_BUF_LAST(lastbuf));
3236
3237	return (lastbuf);
3238}
3239
3240/*
3241 * Free up buf->b_data and pull the arc_buf_t off of the the arc_buf_hdr_t's
3242 * list and free it.
3243 */
3244static void
3245arc_buf_destroy_impl(arc_buf_t *buf)
3246{
3247	arc_buf_hdr_t *hdr = buf->b_hdr;
3248
3249	/*
3250	 * Free up the data associated with the buf but only if we're not
3251	 * sharing this with the hdr. If we are sharing it with the hdr, the
3252	 * hdr is responsible for doing the free.
3253	 */
3254	if (buf->b_data != NULL) {
3255		/*
3256		 * We're about to change the hdr's b_flags. We must either
3257		 * hold the hash_lock or be undiscoverable.
3258		 */
3259		ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
3260
3261		arc_cksum_verify(buf);
3262#ifdef illumos
3263		arc_buf_unwatch(buf);
3264#endif
3265
3266		if (arc_buf_is_shared(buf)) {
3267			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
3268		} else {
3269			uint64_t size = arc_buf_size(buf);
3270			arc_free_data_buf(hdr, buf->b_data, size, buf);
3271			ARCSTAT_INCR(arcstat_overhead_size, -size);
3272		}
3273		buf->b_data = NULL;
3274
3275		ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
3276		hdr->b_l1hdr.b_bufcnt -= 1;
3277	}
3278
3279	arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
3280
3281	if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) {
3282		/*
3283		 * If the current arc_buf_t is sharing its data buffer with the
3284		 * hdr, then reassign the hdr's b_pabd to share it with the new
3285		 * buffer at the end of the list. The shared buffer is always
3286		 * the last one on the hdr's buffer list.
3287		 *
3288		 * There is an equivalent case for compressed bufs, but since
3289		 * they aren't guaranteed to be the last buf in the list and
3290		 * that is an exceedingly rare case, we just allow that space be
3291		 * wasted temporarily.
3292		 */
3293		if (lastbuf != NULL) {
3294			/* Only one buf can be shared at once */
3295			VERIFY(!arc_buf_is_shared(lastbuf));
3296			/* hdr is uncompressed so can't have compressed buf */
3297			VERIFY(!ARC_BUF_COMPRESSED(lastbuf));
3298
3299			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3300			arc_hdr_free_pabd(hdr);
3301
3302			/*
3303			 * We must setup a new shared block between the
3304			 * last buffer and the hdr. The data would have
3305			 * been allocated by the arc buf so we need to transfer
3306			 * ownership to the hdr since it's now being shared.
3307			 */
3308			arc_share_buf(hdr, lastbuf);
3309		}
3310	} else if (HDR_SHARED_DATA(hdr)) {
3311		/*
3312		 * Uncompressed shared buffers are always at the end
3313		 * of the list. Compressed buffers don't have the
3314		 * same requirements. This makes it hard to
3315		 * simply assert that the lastbuf is shared so
3316		 * we rely on the hdr's compression flags to determine
3317		 * if we have a compressed, shared buffer.
3318		 */
3319		ASSERT3P(lastbuf, !=, NULL);
3320		ASSERT(arc_buf_is_shared(lastbuf) ||
3321		    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
3322	}
3323
3324	/*
3325	 * Free the checksum if we're removing the last uncompressed buf from
3326	 * this hdr.
3327	 */
3328	if (!arc_hdr_has_uncompressed_buf(hdr)) {
3329		arc_cksum_free(hdr);
3330	}
3331
3332	/* clean up the buf */
3333	buf->b_hdr = NULL;
3334	kmem_cache_free(buf_cache, buf);
3335}
3336
3337static void
3338arc_hdr_alloc_pabd(arc_buf_hdr_t *hdr)
3339{
3340	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
3341	ASSERT(HDR_HAS_L1HDR(hdr));
3342	ASSERT(!HDR_SHARED_DATA(hdr));
3343
3344	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3345	hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr);
3346	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3347	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3348
3349	ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
3350	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
3351}
3352
3353static void
3354arc_hdr_free_pabd(arc_buf_hdr_t *hdr)
3355{
3356	ASSERT(HDR_HAS_L1HDR(hdr));
3357	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3358
3359	/*
3360	 * If the hdr is currently being written to the l2arc then
3361	 * we defer freeing the data by adding it to the l2arc_free_on_write
3362	 * list. The l2arc will free the data once it's finished
3363	 * writing it to the l2arc device.
3364	 */
3365	if (HDR_L2_WRITING(hdr)) {
3366		arc_hdr_free_on_write(hdr);
3367		ARCSTAT_BUMP(arcstat_l2_free_on_write);
3368	} else {
3369		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
3370		    arc_hdr_size(hdr), hdr);
3371	}
3372	hdr->b_l1hdr.b_pabd = NULL;
3373	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3374
3375	ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
3376	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3377}
3378
3379static arc_buf_hdr_t *
3380arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize,
3381    enum zio_compress compression_type, arc_buf_contents_t type)
3382{
3383	arc_buf_hdr_t *hdr;
3384
3385	VERIFY(type == ARC_BUFC_DATA || type == ARC_BUFC_METADATA);
3386
3387	hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
3388	ASSERT(HDR_EMPTY(hdr));
3389	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3390	ASSERT3P(hdr->b_l1hdr.b_thawed, ==, NULL);
3391	HDR_SET_PSIZE(hdr, psize);
3392	HDR_SET_LSIZE(hdr, lsize);
3393	hdr->b_spa = spa;
3394	hdr->b_type = type;
3395	hdr->b_flags = 0;
3396	arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR);
3397	arc_hdr_set_compress(hdr, compression_type);
3398
3399	hdr->b_l1hdr.b_state = arc_anon;
3400	hdr->b_l1hdr.b_arc_access = 0;
3401	hdr->b_l1hdr.b_bufcnt = 0;
3402	hdr->b_l1hdr.b_buf = NULL;
3403
3404	/*
3405	 * Allocate the hdr's buffer. This will contain either
3406	 * the compressed or uncompressed data depending on the block
3407	 * it references and compressed arc enablement.
3408	 */
3409	arc_hdr_alloc_pabd(hdr);
3410	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3411
3412	return (hdr);
3413}
3414
3415/*
3416 * Transition between the two allocation states for the arc_buf_hdr struct.
3417 * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
3418 * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
3419 * version is used when a cache buffer is only in the L2ARC in order to reduce
3420 * memory usage.
3421 */
3422static arc_buf_hdr_t *
3423arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
3424{
3425	ASSERT(HDR_HAS_L2HDR(hdr));
3426
3427	arc_buf_hdr_t *nhdr;
3428	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3429
3430	ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) ||
3431	    (old == hdr_l2only_cache && new == hdr_full_cache));
3432
3433	nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);
3434
3435	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
3436	buf_hash_remove(hdr);
3437
3438	bcopy(hdr, nhdr, HDR_L2ONLY_SIZE);
3439
3440	if (new == hdr_full_cache) {
3441		arc_hdr_set_flags(nhdr, ARC_FLAG_HAS_L1HDR);
3442		/*
3443		 * arc_access and arc_change_state need to be aware that a
3444		 * header has just come out of L2ARC, so we set its state to
3445		 * l2c_only even though it's about to change.
3446		 */
3447		nhdr->b_l1hdr.b_state = arc_l2c_only;
3448
3449		/* Verify previous threads set to NULL before freeing */
3450		ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL);
3451	} else {
3452		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3453		ASSERT0(hdr->b_l1hdr.b_bufcnt);
3454		ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3455
3456		/*
3457		 * If we've reached here, We must have been called from
3458		 * arc_evict_hdr(), as such we should have already been
3459		 * removed from any ghost list we were previously on
3460		 * (which protects us from racing with arc_evict_state),
3461		 * thus no locking is needed during this check.
3462		 */
3463		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3464
3465		/*
3466		 * A buffer must not be moved into the arc_l2c_only
3467		 * state if it's not finished being written out to the
3468		 * l2arc device. Otherwise, the b_l1hdr.b_pabd field
3469		 * might try to be accessed, even though it was removed.
3470		 */
3471		VERIFY(!HDR_L2_WRITING(hdr));
3472		VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3473
3474#ifdef ZFS_DEBUG
3475		if (hdr->b_l1hdr.b_thawed != NULL) {
3476			kmem_free(hdr->b_l1hdr.b_thawed, 1);
3477			hdr->b_l1hdr.b_thawed = NULL;
3478		}
3479#endif
3480
3481		arc_hdr_clear_flags(nhdr, ARC_FLAG_HAS_L1HDR);
3482	}
3483	/*
3484	 * The header has been reallocated so we need to re-insert it into any
3485	 * lists it was on.
3486	 */
3487	(void) buf_hash_insert(nhdr, NULL);
3488
3489	ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
3490
3491	mutex_enter(&dev->l2ad_mtx);
3492
3493	/*
3494	 * We must place the realloc'ed header back into the list at
3495	 * the same spot. Otherwise, if it's placed earlier in the list,
3496	 * l2arc_write_buffers() could find it during the function's
3497	 * write phase, and try to write it out to the l2arc.
3498	 */
3499	list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
3500	list_remove(&dev->l2ad_buflist, hdr);
3501
3502	mutex_exit(&dev->l2ad_mtx);
3503
3504	/*
3505	 * Since we're using the pointer address as the tag when
3506	 * incrementing and decrementing the l2ad_alloc refcount, we
3507	 * must remove the old pointer (that we're about to destroy) and
3508	 * add the new pointer to the refcount. Otherwise we'd remove
3509	 * the wrong pointer address when calling arc_hdr_destroy() later.
3510	 */
3511
3512	(void) refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr);
3513	(void) refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(nhdr), nhdr);
3514
3515	buf_discard_identity(hdr);
3516	kmem_cache_free(old, hdr);
3517
3518	return (nhdr);
3519}
3520
3521/*
3522 * Allocate a new arc_buf_hdr_t and arc_buf_t and return the buf to the caller.
3523 * The buf is returned thawed since we expect the consumer to modify it.
3524 */
3525arc_buf_t *
3526arc_alloc_buf(spa_t *spa, void *tag, arc_buf_contents_t type, int32_t size)
3527{
3528	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size,
3529	    ZIO_COMPRESS_OFF, type);
3530	ASSERT(!MUTEX_HELD(HDR_LOCK(hdr)));
3531
3532	arc_buf_t *buf = NULL;
3533	VERIFY0(arc_buf_alloc_impl(hdr, tag, B_FALSE, B_FALSE, &buf));
3534	arc_buf_thaw(buf);
3535
3536	return (buf);
3537}
3538
3539/*
3540 * Allocate a compressed buf in the same manner as arc_alloc_buf. Don't use this
3541 * for bufs containing metadata.
3542 */
3543arc_buf_t *
3544arc_alloc_compressed_buf(spa_t *spa, void *tag, uint64_t psize, uint64_t lsize,
3545    enum zio_compress compression_type)
3546{
3547	ASSERT3U(lsize, >, 0);
3548	ASSERT3U(lsize, >=, psize);
3549	ASSERT(compression_type > ZIO_COMPRESS_OFF);
3550	ASSERT(compression_type < ZIO_COMPRESS_FUNCTIONS);
3551
3552	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
3553	    compression_type, ARC_BUFC_DATA);
3554	ASSERT(!MUTEX_HELD(HDR_LOCK(hdr)));
3555
3556	arc_buf_t *buf = NULL;
3557	VERIFY0(arc_buf_alloc_impl(hdr, tag, B_TRUE, B_FALSE, &buf));
3558	arc_buf_thaw(buf);
3559	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3560
3561	if (!arc_buf_is_shared(buf)) {
3562		/*
3563		 * To ensure that the hdr has the correct data in it if we call
3564		 * arc_decompress() on this buf before it's been written to
3565		 * disk, it's easiest if we just set up sharing between the
3566		 * buf and the hdr.
3567		 */
3568		ASSERT(!abd_is_linear(hdr->b_l1hdr.b_pabd));
3569		arc_hdr_free_pabd(hdr);
3570		arc_share_buf(hdr, buf);
3571	}
3572
3573	return (buf);
3574}
3575
3576static void
3577arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
3578{
3579	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
3580	l2arc_dev_t *dev = l2hdr->b_dev;
3581	uint64_t psize = arc_hdr_size(hdr);
3582
3583	ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
3584	ASSERT(HDR_HAS_L2HDR(hdr));
3585
3586	list_remove(&dev->l2ad_buflist, hdr);
3587
3588	ARCSTAT_INCR(arcstat_l2_psize, -psize);
3589	ARCSTAT_INCR(arcstat_l2_lsize, -HDR_GET_LSIZE(hdr));
3590
3591	vdev_space_update(dev->l2ad_vdev, -psize, 0, 0);
3592
3593	(void) refcount_remove_many(&dev->l2ad_alloc, psize, hdr);
3594	arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
3595}
3596
3597static void
3598arc_hdr_destroy(arc_buf_hdr_t *hdr)
3599{
3600	if (HDR_HAS_L1HDR(hdr)) {
3601		ASSERT(hdr->b_l1hdr.b_buf == NULL ||
3602		    hdr->b_l1hdr.b_bufcnt > 0);
3603		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3604		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3605	}
3606	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3607	ASSERT(!HDR_IN_HASH_TABLE(hdr));
3608
3609	if (!HDR_EMPTY(hdr))
3610		buf_discard_identity(hdr);
3611
3612	if (HDR_HAS_L2HDR(hdr)) {
3613		l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3614		boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx);
3615
3616		if (!buflist_held)
3617			mutex_enter(&dev->l2ad_mtx);
3618
3619		/*
3620		 * Even though we checked this conditional above, we
3621		 * need to check this again now that we have the
3622		 * l2ad_mtx. This is because we could be racing with
3623		 * another thread calling l2arc_evict() which might have
3624		 * destroyed this header's L2 portion as we were waiting
3625		 * to acquire the l2ad_mtx. If that happens, we don't
3626		 * want to re-destroy the header's L2 portion.
3627		 */
3628		if (HDR_HAS_L2HDR(hdr)) {
3629			l2arc_trim(hdr);
3630			arc_hdr_l2hdr_destroy(hdr);
3631		}
3632
3633		if (!buflist_held)
3634			mutex_exit(&dev->l2ad_mtx);
3635	}
3636
3637	if (HDR_HAS_L1HDR(hdr)) {
3638		arc_cksum_free(hdr);
3639
3640		while (hdr->b_l1hdr.b_buf != NULL)
3641			arc_buf_destroy_impl(hdr->b_l1hdr.b_buf);
3642
3643#ifdef ZFS_DEBUG
3644		if (hdr->b_l1hdr.b_thawed != NULL) {
3645			kmem_free(hdr->b_l1hdr.b_thawed, 1);
3646			hdr->b_l1hdr.b_thawed = NULL;
3647		}
3648#endif
3649
3650		if (hdr->b_l1hdr.b_pabd != NULL) {
3651			arc_hdr_free_pabd(hdr);
3652		}
3653	}
3654
3655	ASSERT3P(hdr->b_hash_next, ==, NULL);
3656	if (HDR_HAS_L1HDR(hdr)) {
3657		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3658		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
3659		kmem_cache_free(hdr_full_cache, hdr);
3660	} else {
3661		kmem_cache_free(hdr_l2only_cache, hdr);
3662	}
3663}
3664
3665void
3666arc_buf_destroy(arc_buf_t *buf, void* tag)
3667{
3668	arc_buf_hdr_t *hdr = buf->b_hdr;
3669	kmutex_t *hash_lock = HDR_LOCK(hdr);
3670
3671	if (hdr->b_l1hdr.b_state == arc_anon) {
3672		ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
3673		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3674		VERIFY0(remove_reference(hdr, NULL, tag));
3675		arc_hdr_destroy(hdr);
3676		return;
3677	}
3678
3679	mutex_enter(hash_lock);
3680	ASSERT3P(hdr, ==, buf->b_hdr);
3681	ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
3682	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3683	ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
3684	ASSERT3P(buf->b_data, !=, NULL);
3685
3686	(void) remove_reference(hdr, hash_lock, tag);
3687	arc_buf_destroy_impl(buf);
3688	mutex_exit(hash_lock);
3689}
3690
3691/*
3692 * Evict the arc_buf_hdr that is provided as a parameter. The resultant
3693 * state of the header is dependent on its state prior to entering this
3694 * function. The following transitions are possible:
3695 *
3696 *    - arc_mru -> arc_mru_ghost
3697 *    - arc_mfu -> arc_mfu_ghost
3698 *    - arc_mru_ghost -> arc_l2c_only
3699 *    - arc_mru_ghost -> deleted
3700 *    - arc_mfu_ghost -> arc_l2c_only
3701 *    - arc_mfu_ghost -> deleted
3702 */
3703static int64_t
3704arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
3705{
3706	arc_state_t *evicted_state, *state;
3707	int64_t bytes_evicted = 0;
3708	int min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ?
3709	    zfs_arc_min_prescient_prefetch_ms : zfs_arc_min_prefetch_ms;
3710
3711	ASSERT(MUTEX_HELD(hash_lock));
3712	ASSERT(HDR_HAS_L1HDR(hdr));
3713
3714	state = hdr->b_l1hdr.b_state;
3715	if (GHOST_STATE(state)) {
3716		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3717		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3718
3719		/*
3720		 * l2arc_write_buffers() relies on a header's L1 portion
3721		 * (i.e. its b_pabd field) during it's write phase.
3722		 * Thus, we cannot push a header onto the arc_l2c_only
3723		 * state (removing it's L1 piece) until the header is
3724		 * done being written to the l2arc.
3725		 */
3726		if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
3727			ARCSTAT_BUMP(arcstat_evict_l2_skip);
3728			return (bytes_evicted);
3729		}
3730
3731		ARCSTAT_BUMP(arcstat_deleted);
3732		bytes_evicted += HDR_GET_LSIZE(hdr);
3733
3734		DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
3735
3736		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3737		if (HDR_HAS_L2HDR(hdr)) {
3738			/*
3739			 * This buffer is cached on the 2nd Level ARC;
3740			 * don't destroy the header.
3741			 */
3742			arc_change_state(arc_l2c_only, hdr, hash_lock);
3743			/*
3744			 * dropping from L1+L2 cached to L2-only,
3745			 * realloc to remove the L1 header.
3746			 */
3747			hdr = arc_hdr_realloc(hdr, hdr_full_cache,
3748			    hdr_l2only_cache);
3749		} else {
3750			arc_change_state(arc_anon, hdr, hash_lock);
3751			arc_hdr_destroy(hdr);
3752		}
3753		return (bytes_evicted);
3754	}
3755
3756	ASSERT(state == arc_mru || state == arc_mfu);
3757	evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
3758
3759	/* prefetch buffers have a minimum lifespan */
3760	if (HDR_IO_IN_PROGRESS(hdr) ||
3761	    ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
3762	    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < min_lifetime * hz)) {
3763		ARCSTAT_BUMP(arcstat_evict_skip);
3764		return (bytes_evicted);
3765	}
3766
3767	ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
3768	while (hdr->b_l1hdr.b_buf) {
3769		arc_buf_t *buf = hdr->b_l1hdr.b_buf;
3770		if (!mutex_tryenter(&buf->b_evict_lock)) {
3771			ARCSTAT_BUMP(arcstat_mutex_miss);
3772			break;
3773		}
3774		if (buf->b_data != NULL)
3775			bytes_evicted += HDR_GET_LSIZE(hdr);
3776		mutex_exit(&buf->b_evict_lock);
3777		arc_buf_destroy_impl(buf);
3778	}
3779
3780	if (HDR_HAS_L2HDR(hdr)) {
3781		ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr));
3782	} else {
3783		if (l2arc_write_eligible(hdr->b_spa, hdr)) {
3784			ARCSTAT_INCR(arcstat_evict_l2_eligible,
3785			    HDR_GET_LSIZE(hdr));
3786		} else {
3787			ARCSTAT_INCR(arcstat_evict_l2_ineligible,
3788			    HDR_GET_LSIZE(hdr));
3789		}
3790	}
3791
3792	if (hdr->b_l1hdr.b_bufcnt == 0) {
3793		arc_cksum_free(hdr);
3794
3795		bytes_evicted += arc_hdr_size(hdr);
3796
3797		/*
3798		 * If this hdr is being evicted and has a compressed
3799		 * buffer then we discard it here before we change states.
3800		 * This ensures that the accounting is updated correctly
3801		 * in arc_free_data_impl().
3802		 */
3803		arc_hdr_free_pabd(hdr);
3804
3805		arc_change_state(evicted_state, hdr, hash_lock);
3806		ASSERT(HDR_IN_HASH_TABLE(hdr));
3807		arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
3808		DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
3809	}
3810
3811	return (bytes_evicted);
3812}
3813
3814static uint64_t
3815arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker,
3816    uint64_t spa, int64_t bytes)
3817{
3818	multilist_sublist_t *mls;
3819	uint64_t bytes_evicted = 0;
3820	arc_buf_hdr_t *hdr;
3821	kmutex_t *hash_lock;
3822	int evict_count = 0;
3823
3824	ASSERT3P(marker, !=, NULL);
3825	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
3826
3827	mls = multilist_sublist_lock(ml, idx);
3828
3829	for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL;
3830	    hdr = multilist_sublist_prev(mls, marker)) {
3831		if ((bytes != ARC_EVICT_ALL && bytes_evicted >= bytes) ||
3832		    (evict_count >= zfs_arc_evict_batch_limit))
3833			break;
3834
3835		/*
3836		 * To keep our iteration location, move the marker
3837		 * forward. Since we're not holding hdr's hash lock, we
3838		 * must be very careful and not remove 'hdr' from the
3839		 * sublist. Otherwise, other consumers might mistake the
3840		 * 'hdr' as not being on a sublist when they call the
3841		 * multilist_link_active() function (they all rely on
3842		 * the hash lock protecting concurrent insertions and
3843		 * removals). multilist_sublist_move_forward() was
3844		 * specifically implemented to ensure this is the case
3845		 * (only 'marker' will be removed and re-inserted).
3846		 */
3847		multilist_sublist_move_forward(mls, marker);
3848
3849		/*
3850		 * The only case where the b_spa field should ever be
3851		 * zero, is the marker headers inserted by
3852		 * arc_evict_state(). It's possible for multiple threads
3853		 * to be calling arc_evict_state() concurrently (e.g.
3854		 * dsl_pool_close() and zio_inject_fault()), so we must
3855		 * skip any markers we see from these other threads.
3856		 */
3857		if (hdr->b_spa == 0)
3858			continue;
3859
3860		/* we're only interested in evicting buffers of a certain spa */
3861		if (spa != 0 && hdr->b_spa != spa) {
3862			ARCSTAT_BUMP(arcstat_evict_skip);
3863			continue;
3864		}
3865
3866		hash_lock = HDR_LOCK(hdr);
3867
3868		/*
3869		 * We aren't calling this function from any code path
3870		 * that would already be holding a hash lock, so we're
3871		 * asserting on this assumption to be defensive in case
3872		 * this ever changes. Without this check, it would be
3873		 * possible to incorrectly increment arcstat_mutex_miss
3874		 * below (e.g. if the code changed such that we called
3875		 * this function with a hash lock held).
3876		 */
3877		ASSERT(!MUTEX_HELD(hash_lock));
3878
3879		if (mutex_tryenter(hash_lock)) {
3880			uint64_t evicted = arc_evict_hdr(hdr, hash_lock);
3881			mutex_exit(hash_lock);
3882
3883			bytes_evicted += evicted;
3884
3885			/*
3886			 * If evicted is zero, arc_evict_hdr() must have
3887			 * decided to skip this header, don't increment
3888			 * evict_count in this case.
3889			 */
3890			if (evicted != 0)
3891				evict_count++;
3892
3893			/*
3894			 * If arc_size isn't overflowing, signal any
3895			 * threads that might happen to be waiting.
3896			 *
3897			 * For each header evicted, we wake up a single
3898			 * thread. If we used cv_broadcast, we could
3899			 * wake up "too many" threads causing arc_size
3900			 * to significantly overflow arc_c; since
3901			 * arc_get_data_impl() doesn't check for overflow
3902			 * when it's woken up (it doesn't because it's
3903			 * possible for the ARC to be overflowing while
3904			 * full of un-evictable buffers, and the
3905			 * function should proceed in this case).
3906			 *
3907			 * If threads are left sleeping, due to not
3908			 * using cv_broadcast, they will be woken up
3909			 * just before arc_reclaim_thread() sleeps.
3910			 */
3911			mutex_enter(&arc_reclaim_lock);
3912			if (!arc_is_overflowing())
3913				cv_signal(&arc_reclaim_waiters_cv);
3914			mutex_exit(&arc_reclaim_lock);
3915		} else {
3916			ARCSTAT_BUMP(arcstat_mutex_miss);
3917		}
3918	}
3919
3920	multilist_sublist_unlock(mls);
3921
3922	return (bytes_evicted);
3923}
3924
3925/*
3926 * Evict buffers from the given arc state, until we've removed the
3927 * specified number of bytes. Move the removed buffers to the
3928 * appropriate evict state.
3929 *
3930 * This function makes a "best effort". It skips over any buffers
3931 * it can't get a hash_lock on, and so, may not catch all candidates.
3932 * It may also return without evicting as much space as requested.
3933 *
3934 * If bytes is specified using the special value ARC_EVICT_ALL, this
3935 * will evict all available (i.e. unlocked and evictable) buffers from
3936 * the given arc state; which is used by arc_flush().
3937 */
3938static uint64_t
3939arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
3940    arc_buf_contents_t type)
3941{
3942	uint64_t total_evicted = 0;
3943	multilist_t *ml = state->arcs_list[type];
3944	int num_sublists;
3945	arc_buf_hdr_t **markers;
3946
3947	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
3948
3949	num_sublists = multilist_get_num_sublists(ml);
3950
3951	/*
3952	 * If we've tried to evict from each sublist, made some
3953	 * progress, but still have not hit the target number of bytes
3954	 * to evict, we want to keep trying. The markers allow us to
3955	 * pick up where we left off for each individual sublist, rather
3956	 * than starting from the tail each time.
3957	 */
3958	markers = kmem_zalloc(sizeof (*markers) * num_sublists, KM_SLEEP);
3959	for (int i = 0; i < num_sublists; i++) {
3960		markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
3961
3962		/*
3963		 * A b_spa of 0 is used to indicate that this header is
3964		 * a marker. This fact is used in arc_adjust_type() and
3965		 * arc_evict_state_impl().
3966		 */
3967		markers[i]->b_spa = 0;
3968
3969		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
3970		multilist_sublist_insert_tail(mls, markers[i]);
3971		multilist_sublist_unlock(mls);
3972	}
3973
3974	/*
3975	 * While we haven't hit our target number of bytes to evict, or
3976	 * we're evicting all available buffers.
3977	 */
3978	while (total_evicted < bytes || bytes == ARC_EVICT_ALL) {
3979		int sublist_idx = multilist_get_random_index(ml);
3980		uint64_t scan_evicted = 0;
3981
3982		/*
3983		 * Try to reduce pinned dnodes with a floor of arc_dnode_limit.
3984		 * Request that 10% of the LRUs be scanned by the superblock
3985		 * shrinker.
3986		 */
3987		if (type == ARC_BUFC_DATA && aggsum_compare(&astat_dnode_size,
3988		    arc_dnode_limit) > 0) {
3989			arc_prune_async((aggsum_upper_bound(&astat_dnode_size) -
3990			    arc_dnode_limit) / sizeof (dnode_t) /
3991			    zfs_arc_dnode_reduce_percent);
3992		}
3993
3994		/*
3995		 * Start eviction using a randomly selected sublist,
3996		 * this is to try and evenly balance eviction across all
3997		 * sublists. Always starting at the same sublist
3998		 * (e.g. index 0) would cause evictions to favor certain
3999		 * sublists over others.
4000		 */
4001		for (int i = 0; i < num_sublists; i++) {
4002			uint64_t bytes_remaining;
4003			uint64_t bytes_evicted;
4004
4005			if (bytes == ARC_EVICT_ALL)
4006				bytes_remaining = ARC_EVICT_ALL;
4007			else if (total_evicted < bytes)
4008				bytes_remaining = bytes - total_evicted;
4009			else
4010				break;
4011
4012			bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
4013			    markers[sublist_idx], spa, bytes_remaining);
4014
4015			scan_evicted += bytes_evicted;
4016			total_evicted += bytes_evicted;
4017
4018			/* we've reached the end, wrap to the beginning */
4019			if (++sublist_idx >= num_sublists)
4020				sublist_idx = 0;
4021		}
4022
4023		/*
4024		 * If we didn't evict anything during this scan, we have
4025		 * no reason to believe we'll evict more during another
4026		 * scan, so break the loop.
4027		 */
4028		if (scan_evicted == 0) {
4029			/* This isn't possible, let's make that obvious */
4030			ASSERT3S(bytes, !=, 0);
4031
4032			/*
4033			 * When bytes is ARC_EVICT_ALL, the only way to
4034			 * break the loop is when scan_evicted is zero.
4035			 * In that case, we actually have evicted enough,
4036			 * so we don't want to increment the kstat.
4037			 */
4038			if (bytes != ARC_EVICT_ALL) {
4039				ASSERT3S(total_evicted, <, bytes);
4040				ARCSTAT_BUMP(arcstat_evict_not_enough);
4041			}
4042
4043			break;
4044		}
4045	}
4046
4047	for (int i = 0; i < num_sublists; i++) {
4048		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
4049		multilist_sublist_remove(mls, markers[i]);
4050		multilist_sublist_unlock(mls);
4051
4052		kmem_cache_free(hdr_full_cache, markers[i]);
4053	}
4054	kmem_free(markers, sizeof (*markers) * num_sublists);
4055
4056	return (total_evicted);
4057}
4058
4059/*
4060 * Flush all "evictable" data of the given type from the arc state
4061 * specified. This will not evict any "active" buffers (i.e. referenced).
4062 *
4063 * When 'retry' is set to B_FALSE, the function will make a single pass
4064 * over the state and evict any buffers that it can. Since it doesn't
4065 * continually retry the eviction, it might end up leaving some buffers
4066 * in the ARC due to lock misses.
4067 *
4068 * When 'retry' is set to B_TRUE, the function will continually retry the
4069 * eviction until *all* evictable buffers have been removed from the
4070 * state. As a result, if concurrent insertions into the state are
4071 * allowed (e.g. if the ARC isn't shutting down), this function might
4072 * wind up in an infinite loop, continually trying to evict buffers.
4073 */
4074static uint64_t
4075arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
4076    boolean_t retry)
4077{
4078	uint64_t evicted = 0;
4079
4080	while (refcount_count(&state->arcs_esize[type]) != 0) {
4081		evicted += arc_evict_state(state, spa, ARC_EVICT_ALL, type);
4082
4083		if (!retry)
4084			break;
4085	}
4086
4087	return (evicted);
4088}
4089
4090/*
4091 * Helper function for arc_prune_async() it is responsible for safely
4092 * handling the execution of a registered arc_prune_func_t.
4093 */
4094static void
4095arc_prune_task(void *ptr)
4096{
4097	arc_prune_t *ap = (arc_prune_t *)ptr;
4098	arc_prune_func_t *func = ap->p_pfunc;
4099
4100	if (func != NULL)
4101		func(ap->p_adjust, ap->p_private);
4102
4103	refcount_remove(&ap->p_refcnt, func);
4104}
4105
4106/*
4107 * Notify registered consumers they must drop holds on a portion of the ARC
4108 * buffered they reference.  This provides a mechanism to ensure the ARC can
4109 * honor the arc_meta_limit and reclaim otherwise pinned ARC buffers.  This
4110 * is analogous to dnlc_reduce_cache() but more generic.
4111 *
4112 * This operation is performed asynchronously so it may be safely called
4113 * in the context of the arc_reclaim_thread().  A reference is taken here
4114 * for each registered arc_prune_t and the arc_prune_task() is responsible
4115 * for releasing it once the registered arc_prune_func_t has completed.
4116 */
4117static void
4118arc_prune_async(int64_t adjust)
4119{
4120	arc_prune_t *ap;
4121
4122	mutex_enter(&arc_prune_mtx);
4123	for (ap = list_head(&arc_prune_list); ap != NULL;
4124	    ap = list_next(&arc_prune_list, ap)) {
4125
4126		if (refcount_count(&ap->p_refcnt) >= 2)
4127			continue;
4128
4129		refcount_add(&ap->p_refcnt, ap->p_pfunc);
4130		ap->p_adjust = adjust;
4131		if (taskq_dispatch(arc_prune_taskq, arc_prune_task,
4132		    ap, TQ_SLEEP) == TASKQID_INVALID) {
4133			refcount_remove(&ap->p_refcnt, ap->p_pfunc);
4134			continue;
4135		}
4136		ARCSTAT_BUMP(arcstat_prune);
4137	}
4138	mutex_exit(&arc_prune_mtx);
4139}
4140
4141/*
4142 * Evict the specified number of bytes from the state specified,
4143 * restricting eviction to the spa and type given. This function
4144 * prevents us from trying to evict more from a state's list than
4145 * is "evictable", and to skip evicting altogether when passed a
4146 * negative value for "bytes". In contrast, arc_evict_state() will
4147 * evict everything it can, when passed a negative value for "bytes".
4148 */
4149static uint64_t
4150arc_adjust_impl(arc_state_t *state, uint64_t spa, int64_t bytes,
4151    arc_buf_contents_t type)
4152{
4153	int64_t delta;
4154
4155	if (bytes > 0 && refcount_count(&state->arcs_esize[type]) > 0) {
4156		delta = MIN(refcount_count(&state->arcs_esize[type]), bytes);
4157		return (arc_evict_state(state, spa, delta, type));
4158	}
4159
4160	return (0);
4161}
4162
4163/*
4164 * The goal of this function is to evict enough meta data buffers from the
4165 * ARC in order to enforce the arc_meta_limit.  Achieving this is slightly
4166 * more complicated than it appears because it is common for data buffers
4167 * to have holds on meta data buffers.  In addition, dnode meta data buffers
4168 * will be held by the dnodes in the block preventing them from being freed.
4169 * This means we can't simply traverse the ARC and expect to always find
4170 * enough unheld meta data buffer to release.
4171 *
4172 * Therefore, this function has been updated to make alternating passes
4173 * over the ARC releasing data buffers and then newly unheld meta data
4174 * buffers.  This ensures forward progress is maintained and meta_used
4175 * will decrease.  Normally this is sufficient, but if required the ARC
4176 * will call the registered prune callbacks causing dentry and inodes to
4177 * be dropped from the VFS cache.  This will make dnode meta data buffers
4178 * available for reclaim.
4179 */
4180static uint64_t
4181arc_adjust_meta_balanced(uint64_t meta_used)
4182{
4183	int64_t delta, prune = 0, adjustmnt;
4184	uint64_t total_evicted = 0;
4185	arc_buf_contents_t type = ARC_BUFC_DATA;
4186	int restarts = MAX(zfs_arc_meta_adjust_restarts, 0);
4187
4188restart:
4189	/*
4190	 * This slightly differs than the way we evict from the mru in
4191	 * arc_adjust because we don't have a "target" value (i.e. no
4192	 * "meta" arc_p). As a result, I think we can completely
4193	 * cannibalize the metadata in the MRU before we evict the
4194	 * metadata from the MFU. I think we probably need to implement a
4195	 * "metadata arc_p" value to do this properly.
4196	 */
4197	adjustmnt = meta_used - arc_meta_limit;
4198
4199	if (adjustmnt > 0 && refcount_count(&arc_mru->arcs_esize[type]) > 0) {
4200		delta = MIN(refcount_count(&arc_mru->arcs_esize[type]),
4201		    adjustmnt);
4202		total_evicted += arc_adjust_impl(arc_mru, 0, delta, type);
4203		adjustmnt -= delta;
4204	}
4205
4206	/*
4207	 * We can't afford to recalculate adjustmnt here. If we do,
4208	 * new metadata buffers can sneak into the MRU or ANON lists,
4209	 * thus penalize the MFU metadata. Although the fudge factor is
4210	 * small, it has been empirically shown to be significant for
4211	 * certain workloads (e.g. creating many empty directories). As
4212	 * such, we use the original calculation for adjustmnt, and
4213	 * simply decrement the amount of data evicted from the MRU.
4214	 */
4215
4216	if (adjustmnt > 0 && refcount_count(&arc_mfu->arcs_esize[type]) > 0) {
4217		delta = MIN(refcount_count(&arc_mfu->arcs_esize[type]),
4218		    adjustmnt);
4219		total_evicted += arc_adjust_impl(arc_mfu, 0, delta, type);
4220	}
4221
4222	adjustmnt = meta_used - arc_meta_limit;
4223
4224	if (adjustmnt > 0 &&
4225	    refcount_count(&arc_mru_ghost->arcs_esize[type]) > 0) {
4226		delta = MIN(adjustmnt,
4227		    refcount_count(&arc_mru_ghost->arcs_esize[type]));
4228		total_evicted += arc_adjust_impl(arc_mru_ghost, 0, delta, type);
4229		adjustmnt -= delta;
4230	}
4231
4232	if (adjustmnt > 0 &&
4233	    refcount_count(&arc_mfu_ghost->arcs_esize[type]) > 0) {
4234		delta = MIN(adjustmnt,
4235		    refcount_count(&arc_mfu_ghost->arcs_esize[type]));
4236		total_evicted += arc_adjust_impl(arc_mfu_ghost, 0, delta, type);
4237	}
4238
4239	/*
4240	 * If after attempting to make the requested adjustment to the ARC
4241	 * the meta limit is still being exceeded then request that the
4242	 * higher layers drop some cached objects which have holds on ARC
4243	 * meta buffers.  Requests to the upper layers will be made with
4244	 * increasingly large scan sizes until the ARC is below the limit.
4245	 */
4246	if (meta_used > arc_meta_limit) {
4247		if (type == ARC_BUFC_DATA) {
4248			type = ARC_BUFC_METADATA;
4249		} else {
4250			type = ARC_BUFC_DATA;
4251
4252			if (zfs_arc_meta_prune) {
4253				prune += zfs_arc_meta_prune;
4254				arc_prune_async(prune);
4255			}
4256		}
4257
4258		if (restarts > 0) {
4259			restarts--;
4260			goto restart;
4261		}
4262	}
4263	return (total_evicted);
4264}
4265
4266/*
4267 * Evict metadata buffers from the cache, such that arc_meta_used is
4268 * capped by the arc_meta_limit tunable.
4269 */
4270static uint64_t
4271arc_adjust_meta_only(uint64_t meta_used)
4272{
4273	uint64_t total_evicted = 0;
4274	int64_t target;
4275
4276	/*
4277	 * If we're over the meta limit, we want to evict enough
4278	 * metadata to get back under the meta limit. We don't want to
4279	 * evict so much that we drop the MRU below arc_p, though. If
4280	 * we're over the meta limit more than we're over arc_p, we
4281	 * evict some from the MRU here, and some from the MFU below.
4282	 */
4283	target = MIN((int64_t)(meta_used - arc_meta_limit),
4284	    (int64_t)(refcount_count(&arc_anon->arcs_size) +
4285	    refcount_count(&arc_mru->arcs_size) - arc_p));
4286
4287	total_evicted += arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
4288
4289	/*
4290	 * Similar to the above, we want to evict enough bytes to get us
4291	 * below the meta limit, but not so much as to drop us below the
4292	 * space allotted to the MFU (which is defined as arc_c - arc_p).
4293	 */
4294	target = MIN((int64_t)(meta_used - arc_meta_limit),
4295	    (int64_t)(refcount_count(&arc_mfu->arcs_size) -
4296	    (arc_c - arc_p)));
4297
4298	total_evicted += arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
4299
4300	return (total_evicted);
4301}
4302
4303static uint64_t
4304arc_adjust_meta(uint64_t meta_used)
4305{
4306	if (zfs_arc_meta_strategy == ARC_STRATEGY_META_ONLY)
4307		return (arc_adjust_meta_only(meta_used));
4308	else
4309		return (arc_adjust_meta_balanced(meta_used));
4310}
4311
4312/*
4313 * Return the type of the oldest buffer in the given arc state
4314 *
4315 * This function will select a random sublist of type ARC_BUFC_DATA and
4316 * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist
4317 * is compared, and the type which contains the "older" buffer will be
4318 * returned.
4319 */
4320static arc_buf_contents_t
4321arc_adjust_type(arc_state_t *state)
4322{
4323	multilist_t *data_ml = state->arcs_list[ARC_BUFC_DATA];
4324	multilist_t *meta_ml = state->arcs_list[ARC_BUFC_METADATA];
4325	int data_idx = multilist_get_random_index(data_ml);
4326	int meta_idx = multilist_get_random_index(meta_ml);
4327	multilist_sublist_t *data_mls;
4328	multilist_sublist_t *meta_mls;
4329	arc_buf_contents_t type;
4330	arc_buf_hdr_t *data_hdr;
4331	arc_buf_hdr_t *meta_hdr;
4332
4333	/*
4334	 * We keep the sublist lock until we're finished, to prevent
4335	 * the headers from being destroyed via arc_evict_state().
4336	 */
4337	data_mls = multilist_sublist_lock(data_ml, data_idx);
4338	meta_mls = multilist_sublist_lock(meta_ml, meta_idx);
4339
4340	/*
4341	 * These two loops are to ensure we skip any markers that
4342	 * might be at the tail of the lists due to arc_evict_state().
4343	 */
4344
4345	for (data_hdr = multilist_sublist_tail(data_mls); data_hdr != NULL;
4346	    data_hdr = multilist_sublist_prev(data_mls, data_hdr)) {
4347		if (data_hdr->b_spa != 0)
4348			break;
4349	}
4350
4351	for (meta_hdr = multilist_sublist_tail(meta_mls); meta_hdr != NULL;
4352	    meta_hdr = multilist_sublist_prev(meta_mls, meta_hdr)) {
4353		if (meta_hdr->b_spa != 0)
4354			break;
4355	}
4356
4357	if (data_hdr == NULL && meta_hdr == NULL) {
4358		type = ARC_BUFC_DATA;
4359	} else if (data_hdr == NULL) {
4360		ASSERT3P(meta_hdr, !=, NULL);
4361		type = ARC_BUFC_METADATA;
4362	} else if (meta_hdr == NULL) {
4363		ASSERT3P(data_hdr, !=, NULL);
4364		type = ARC_BUFC_DATA;
4365	} else {
4366		ASSERT3P(data_hdr, !=, NULL);
4367		ASSERT3P(meta_hdr, !=, NULL);
4368
4369		/* The headers can't be on the sublist without an L1 header */
4370		ASSERT(HDR_HAS_L1HDR(data_hdr));
4371		ASSERT(HDR_HAS_L1HDR(meta_hdr));
4372
4373		if (data_hdr->b_l1hdr.b_arc_access <
4374		    meta_hdr->b_l1hdr.b_arc_access) {
4375			type = ARC_BUFC_DATA;
4376		} else {
4377			type = ARC_BUFC_METADATA;
4378		}
4379	}
4380
4381	multilist_sublist_unlock(meta_mls);
4382	multilist_sublist_unlock(data_mls);
4383
4384	return (type);
4385}
4386
4387/*
4388 * Evict buffers from the cache, such that arc_size is capped by arc_c.
4389 */
4390static uint64_t
4391arc_adjust(void)
4392{
4393	uint64_t total_evicted = 0;
4394	uint64_t bytes;
4395	int64_t target;
4396	uint64_t asize = aggsum_value(&arc_size);
4397	uint64_t ameta = aggsum_value(&arc_meta_used);
4398
4399	/*
4400	 * If we're over arc_meta_limit, we want to correct that before
4401	 * potentially evicting data buffers below.
4402	 */
4403	total_evicted += arc_adjust_meta(ameta);
4404
4405	/*
4406	 * Adjust MRU size
4407	 *
4408	 * If we're over the target cache size, we want to evict enough
4409	 * from the list to get back to our target size. We don't want
4410	 * to evict too much from the MRU, such that it drops below
4411	 * arc_p. So, if we're over our target cache size more than
4412	 * the MRU is over arc_p, we'll evict enough to get back to
4413	 * arc_p here, and then evict more from the MFU below.
4414	 */
4415	target = MIN((int64_t)(asize - arc_c),
4416	    (int64_t)(refcount_count(&arc_anon->arcs_size) +
4417	    refcount_count(&arc_mru->arcs_size) + ameta - arc_p));
4418
4419	/*
4420	 * If we're below arc_meta_min, always prefer to evict data.
4421	 * Otherwise, try to satisfy the requested number of bytes to
4422	 * evict from the type which contains older buffers; in an
4423	 * effort to keep newer buffers in the cache regardless of their
4424	 * type. If we cannot satisfy the number of bytes from this
4425	 * type, spill over into the next type.
4426	 */
4427	if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
4428	    ameta > arc_meta_min) {
4429		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
4430		total_evicted += bytes;
4431
4432		/*
4433		 * If we couldn't evict our target number of bytes from
4434		 * metadata, we try to get the rest from data.
4435		 */
4436		target -= bytes;
4437
4438		total_evicted +=
4439		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
4440	} else {
4441		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
4442		total_evicted += bytes;
4443
4444		/*
4445		 * If we couldn't evict our target number of bytes from
4446		 * data, we try to get the rest from metadata.
4447		 */
4448		target -= bytes;
4449
4450		total_evicted +=
4451		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
4452	}
4453
4454	/*
4455	 * Re-sum ARC stats after the first round of evictions.
4456	 */
4457	asize = aggsum_value(&arc_size);
4458	ameta = aggsum_value(&arc_meta_used);
4459
4460	/*
4461	 * Adjust MFU size
4462	 *
4463	 * Now that we've tried to evict enough from the MRU to get its
4464	 * size back to arc_p, if we're still above the target cache
4465	 * size, we evict the rest from the MFU.
4466	 */
4467	target = asize - arc_c;
4468
4469	if (arc_adjust_type(arc_mfu) == ARC_BUFC_METADATA &&
4470	    ameta > arc_meta_min) {
4471		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
4472		total_evicted += bytes;
4473
4474		/*
4475		 * If we couldn't evict our target number of bytes from
4476		 * metadata, we try to get the rest from data.
4477		 */
4478		target -= bytes;
4479
4480		total_evicted +=
4481		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
4482	} else {
4483		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
4484		total_evicted += bytes;
4485
4486		/*
4487		 * If we couldn't evict our target number of bytes from
4488		 * data, we try to get the rest from data.
4489		 */
4490		target -= bytes;
4491
4492		total_evicted +=
4493		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
4494	}
4495
4496	/*
4497	 * Adjust ghost lists
4498	 *
4499	 * In addition to the above, the ARC also defines target values
4500	 * for the ghost lists. The sum of the mru list and mru ghost
4501	 * list should never exceed the target size of the cache, and
4502	 * the sum of the mru list, mfu list, mru ghost list, and mfu
4503	 * ghost list should never exceed twice the target size of the
4504	 * cache. The following logic enforces these limits on the ghost
4505	 * caches, and evicts from them as needed.
4506	 */
4507	target = refcount_count(&arc_mru->arcs_size) +
4508	    refcount_count(&arc_mru_ghost->arcs_size) - arc_c;
4509
4510	bytes = arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_DATA);
4511	total_evicted += bytes;
4512
4513	target -= bytes;
4514
4515	total_evicted +=
4516	    arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_METADATA);
4517
4518	/*
4519	 * We assume the sum of the mru list and mfu list is less than
4520	 * or equal to arc_c (we enforced this above), which means we
4521	 * can use the simpler of the two equations below:
4522	 *
4523	 *	mru + mfu + mru ghost + mfu ghost <= 2 * arc_c
4524	 *		    mru ghost + mfu ghost <= arc_c
4525	 */
4526	target = refcount_count(&arc_mru_ghost->arcs_size) +
4527	    refcount_count(&arc_mfu_ghost->arcs_size) - arc_c;
4528
4529	bytes = arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_DATA);
4530	total_evicted += bytes;
4531
4532	target -= bytes;
4533
4534	total_evicted +=
4535	    arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_METADATA);
4536
4537	return (total_evicted);
4538}
4539
4540void
4541arc_flush(spa_t *spa, boolean_t retry)
4542{
4543	uint64_t guid = 0;
4544
4545	/*
4546	 * If retry is B_TRUE, a spa must not be specified since we have
4547	 * no good way to determine if all of a spa's buffers have been
4548	 * evicted from an arc state.
4549	 */
4550	ASSERT(!retry || spa == 0);
4551
4552	if (spa != NULL)
4553		guid = spa_load_guid(spa);
4554
4555	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
4556	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);
4557
4558	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
4559	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);
4560
4561	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
4562	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);
4563
4564	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
4565	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);
4566}
4567
4568void
4569arc_shrink(int64_t to_free)
4570{
4571	uint64_t asize = aggsum_value(&arc_size);
4572	if (arc_c > arc_c_min) {
4573		DTRACE_PROBE4(arc__shrink, uint64_t, arc_c, uint64_t,
4574			arc_c_min, uint64_t, arc_p, uint64_t, to_free);
4575		if (arc_c > arc_c_min + to_free)
4576			atomic_add_64(&arc_c, -to_free);
4577		else
4578			arc_c = arc_c_min;
4579
4580		atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
4581		if (asize < arc_c)
4582			arc_c = MAX(asize, arc_c_min);
4583		if (arc_p > arc_c)
4584			arc_p = (arc_c >> 1);
4585
4586		DTRACE_PROBE2(arc__shrunk, uint64_t, arc_c, uint64_t,
4587			arc_p);
4588
4589		ASSERT(arc_c >= arc_c_min);
4590		ASSERT((int64_t)arc_p >= 0);
4591	}
4592
4593	if (asize > arc_c) {
4594		DTRACE_PROBE2(arc__shrink_adjust, uint64_t, asize,
4595			uint64_t, arc_c);
4596		(void) arc_adjust();
4597	}
4598}
4599
4600typedef enum free_memory_reason_t {
4601	FMR_UNKNOWN,
4602	FMR_NEEDFREE,
4603	FMR_LOTSFREE,
4604	FMR_SWAPFS_MINFREE,
4605	FMR_PAGES_PP_MAXIMUM,
4606	FMR_HEAP_ARENA,
4607	FMR_ZIO_ARENA,
4608} free_memory_reason_t;
4609
4610int64_t last_free_memory;
4611free_memory_reason_t last_free_reason;
4612
4613/*
4614 * Additional reserve of pages for pp_reserve.
4615 */
4616int64_t arc_pages_pp_reserve = 64;
4617
4618/*
4619 * Additional reserve of pages for swapfs.
4620 */
4621int64_t arc_swapfs_reserve = 64;
4622
4623/*
4624 * Return the amount of memory that can be consumed before reclaim will be
4625 * needed.  Positive if there is sufficient free memory, negative indicates
4626 * the amount of memory that needs to be freed up.
4627 */
4628static int64_t
4629arc_available_memory(void)
4630{
4631	int64_t lowest = INT64_MAX;
4632	int64_t n;
4633	free_memory_reason_t r = FMR_UNKNOWN;
4634
4635#ifdef _KERNEL
4636#ifdef __FreeBSD__
4637	/*
4638	 * Cooperate with pagedaemon when it's time for it to scan
4639	 * and reclaim some pages.
4640	 */
4641	n = PAGESIZE * ((int64_t)freemem - zfs_arc_free_target);
4642	if (n < lowest) {
4643		lowest = n;
4644		r = FMR_LOTSFREE;
4645	}
4646
4647#else
4648	if (needfree > 0) {
4649		n = PAGESIZE * (-needfree);
4650		if (n < lowest) {
4651			lowest = n;
4652			r = FMR_NEEDFREE;
4653		}
4654	}
4655
4656	/*
4657	 * check that we're out of range of the pageout scanner.  It starts to
4658	 * schedule paging if freemem is less than lotsfree and needfree.
4659	 * lotsfree is the high-water mark for pageout, and needfree is the
4660	 * number of needed free pages.  We add extra pages here to make sure
4661	 * the scanner doesn't start up while we're freeing memory.
4662	 */
4663	n = PAGESIZE * (freemem - lotsfree - needfree - desfree);
4664	if (n < lowest) {
4665		lowest = n;
4666		r = FMR_LOTSFREE;
4667	}
4668
4669	/*
4670	 * check to make sure that swapfs has enough space so that anon
4671	 * reservations can still succeed. anon_resvmem() checks that the
4672	 * availrmem is greater than swapfs_minfree, and the number of reserved
4673	 * swap pages.  We also add a bit of extra here just to prevent
4674	 * circumstances from getting really dire.
4675	 */
4676	n = PAGESIZE * (availrmem - swapfs_minfree - swapfs_reserve -
4677	    desfree - arc_swapfs_reserve);
4678	if (n < lowest) {
4679		lowest = n;
4680		r = FMR_SWAPFS_MINFREE;
4681	}
4682
4683
4684	/*
4685	 * Check that we have enough availrmem that memory locking (e.g., via
4686	 * mlock(3C) or memcntl(2)) can still succeed.  (pages_pp_maximum
4687	 * stores the number of pages that cannot be locked; when availrmem
4688	 * drops below pages_pp_maximum, page locking mechanisms such as
4689	 * page_pp_lock() will fail.)
4690	 */
4691	n = PAGESIZE * (availrmem - pages_pp_maximum -
4692	    arc_pages_pp_reserve);
4693	if (n < lowest) {
4694		lowest = n;
4695		r = FMR_PAGES_PP_MAXIMUM;
4696	}
4697
4698#endif	/* __FreeBSD__ */
4699#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC)
4700	/*
4701	 * If we're on an i386 platform, it's possible that we'll exhaust the
4702	 * kernel heap space before we ever run out of available physical
4703	 * memory.  Most checks of the size of the heap_area compare against
4704	 * tune.t_minarmem, which is the minimum available real memory that we
4705	 * can have in the system.  However, this is generally fixed at 25 pages
4706	 * which is so low that it's useless.  In this comparison, we seek to
4707	 * calculate the total heap-size, and reclaim if more than 3/4ths of the
4708	 * heap is allocated.  (Or, in the calculation, if less than 1/4th is
4709	 * free)
4710	 */
4711	n = uma_avail() - (long)(uma_limit() / 4);
4712	if (n < lowest) {
4713		lowest = n;
4714		r = FMR_HEAP_ARENA;
4715	}
4716#endif
4717
4718	/*
4719	 * If zio data pages are being allocated out of a separate heap segment,
4720	 * then enforce that the size of available vmem for this arena remains
4721	 * above about 1/4th (1/(2^arc_zio_arena_free_shift)) free.
4722	 *
4723	 * Note that reducing the arc_zio_arena_free_shift keeps more virtual
4724	 * memory (in the zio_arena) free, which can avoid memory
4725	 * fragmentation issues.
4726	 */
4727	if (zio_arena != NULL) {
4728		n = (int64_t)vmem_size(zio_arena, VMEM_FREE) -
4729		    (vmem_size(zio_arena, VMEM_ALLOC) >>
4730		    arc_zio_arena_free_shift);
4731		if (n < lowest) {
4732			lowest = n;
4733			r = FMR_ZIO_ARENA;
4734		}
4735	}
4736
4737#else	/* _KERNEL */
4738	/* Every 100 calls, free a small amount */
4739	if (spa_get_random(100) == 0)
4740		lowest = -1024;
4741#endif	/* _KERNEL */
4742
4743	last_free_memory = lowest;
4744	last_free_reason = r;
4745	DTRACE_PROBE2(arc__available_memory, int64_t, lowest, int, r);
4746	return (lowest);
4747}
4748
4749
4750/*
4751 * Determine if the system is under memory pressure and is asking
4752 * to reclaim memory. A return value of B_TRUE indicates that the system
4753 * is under memory pressure and that the arc should adjust accordingly.
4754 */
4755static boolean_t
4756arc_reclaim_needed(void)
4757{
4758	return (arc_available_memory() < 0);
4759}
4760
4761extern kmem_cache_t	*zio_buf_cache[];
4762extern kmem_cache_t	*zio_data_buf_cache[];
4763extern kmem_cache_t	*range_seg_cache;
4764extern kmem_cache_t	*abd_chunk_cache;
4765
4766static __noinline void
4767arc_kmem_reap_now(void)
4768{
4769	size_t			i;
4770	kmem_cache_t		*prev_cache = NULL;
4771	kmem_cache_t		*prev_data_cache = NULL;
4772
4773	DTRACE_PROBE(arc__kmem_reap_start);
4774#ifdef _KERNEL
4775	if (aggsum_compare(&arc_meta_used, arc_meta_limit) >= 0) {
4776		/*
4777		 * We are exceeding our meta-data cache limit.
4778		 * Purge some DNLC entries to release holds on meta-data.
4779		 */
4780		dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
4781	}
4782#if defined(__i386)
4783	/*
4784	 * Reclaim unused memory from all kmem caches.
4785	 */
4786	kmem_reap();
4787#endif
4788#endif
4789
4790	/*
4791	 * If a kmem reap is already active, don't schedule more.  We must
4792	 * check for this because kmem_cache_reap_soon() won't actually
4793	 * block on the cache being reaped (this is to prevent callers from
4794	 * becoming implicitly blocked by a system-wide kmem reap -- which,
4795	 * on a system with many, many full magazines, can take minutes).
4796	 */
4797	if (kmem_cache_reap_active())
4798		return;
4799
4800	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
4801		if (zio_buf_cache[i] != prev_cache) {
4802			prev_cache = zio_buf_cache[i];
4803			kmem_cache_reap_soon(zio_buf_cache[i]);
4804		}
4805		if (zio_data_buf_cache[i] != prev_data_cache) {
4806			prev_data_cache = zio_data_buf_cache[i];
4807			kmem_cache_reap_soon(zio_data_buf_cache[i]);
4808		}
4809	}
4810	kmem_cache_reap_soon(abd_chunk_cache);
4811	kmem_cache_reap_soon(buf_cache);
4812	kmem_cache_reap_soon(hdr_full_cache);
4813	kmem_cache_reap_soon(hdr_l2only_cache);
4814	kmem_cache_reap_soon(range_seg_cache);
4815
4816#ifdef illumos
4817	if (zio_arena != NULL) {
4818		/*
4819		 * Ask the vmem arena to reclaim unused memory from its
4820		 * quantum caches.
4821		 */
4822		vmem_qcache_reap(zio_arena);
4823	}
4824#endif
4825	DTRACE_PROBE(arc__kmem_reap_end);
4826}
4827
4828/*
4829 * Threads can block in arc_get_data_impl() waiting for this thread to evict
4830 * enough data and signal them to proceed. When this happens, the threads in
4831 * arc_get_data_impl() are sleeping while holding the hash lock for their
4832 * particular arc header. Thus, we must be careful to never sleep on a
4833 * hash lock in this thread. This is to prevent the following deadlock:
4834 *
4835 *  - Thread A sleeps on CV in arc_get_data_impl() holding hash lock "L",
4836 *    waiting for the reclaim thread to signal it.
4837 *
4838 *  - arc_reclaim_thread() tries to acquire hash lock "L" using mutex_enter,
4839 *    fails, and goes to sleep forever.
4840 *
4841 * This possible deadlock is avoided by always acquiring a hash lock
4842 * using mutex_tryenter() from arc_reclaim_thread().
4843 */
4844/* ARGSUSED */
4845static void
4846arc_reclaim_thread(void *unused __unused)
4847{
4848	hrtime_t		growtime = 0;
4849	hrtime_t		kmem_reap_time = 0;
4850	callb_cpr_t		cpr;
4851
4852	CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG);
4853
4854	mutex_enter(&arc_reclaim_lock);
4855	while (!arc_reclaim_thread_exit) {
4856		uint64_t evicted = 0;
4857
4858		/*
4859		 * This is necessary in order for the mdb ::arc dcmd to
4860		 * show up to date information. Since the ::arc command
4861		 * does not call the kstat's update function, without
4862		 * this call, the command may show stale stats for the
4863		 * anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even
4864		 * with this change, the data might be up to 1 second
4865		 * out of date; but that should suffice. The arc_state_t
4866		 * structures can be queried directly if more accurate
4867		 * information is needed.
4868		 */
4869		if (arc_ksp != NULL)
4870			arc_ksp->ks_update(arc_ksp, KSTAT_READ);
4871
4872		mutex_exit(&arc_reclaim_lock);
4873
4874		/*
4875		 * We call arc_adjust() before (possibly) calling
4876		 * arc_kmem_reap_now(), so that we can wake up
4877		 * arc_get_data_impl() sooner.
4878		 */
4879		evicted = arc_adjust();
4880
4881		int64_t free_memory = arc_available_memory();
4882		if (free_memory < 0) {
4883			hrtime_t curtime = gethrtime();
4884			arc_no_grow = B_TRUE;
4885			arc_warm = B_TRUE;
4886
4887			/*
4888			 * Wait at least zfs_grow_retry (default 60) seconds
4889			 * before considering growing.
4890			 */
4891			growtime = curtime + SEC2NSEC(arc_grow_retry);
4892
4893			/*
4894			 * Wait at least arc_kmem_cache_reap_retry_ms
4895			 * between arc_kmem_reap_now() calls. Without
4896			 * this check it is possible to end up in a
4897			 * situation where we spend lots of time
4898			 * reaping caches, while we're near arc_c_min.
4899			 */
4900			if (curtime >= kmem_reap_time) {
4901				arc_kmem_reap_now();
4902				kmem_reap_time = gethrtime() +
4903				    MSEC2NSEC(arc_kmem_cache_reap_retry_ms);
4904			}
4905
4906			/*
4907			 * If we are still low on memory, shrink the ARC
4908			 * so that we have arc_shrink_min free space.
4909			 */
4910			free_memory = arc_available_memory();
4911
4912			int64_t to_free =
4913			    (arc_c >> arc_shrink_shift) - free_memory;
4914			if (to_free > 0) {
4915#ifdef _KERNEL
4916#ifdef illumos
4917				to_free = MAX(to_free, ptob(needfree));
4918#endif
4919#endif
4920				arc_shrink(to_free);
4921			}
4922		} else if (free_memory < arc_c >> arc_no_grow_shift) {
4923			arc_no_grow = B_TRUE;
4924		} else if (gethrtime() >= growtime) {
4925			arc_no_grow = B_FALSE;
4926		}
4927
4928		mutex_enter(&arc_reclaim_lock);
4929
4930		/*
4931		 * If evicted is zero, we couldn't evict anything via
4932		 * arc_adjust(). This could be due to hash lock
4933		 * collisions, but more likely due to the majority of
4934		 * arc buffers being unevictable. Therefore, even if
4935		 * arc_size is above arc_c, another pass is unlikely to
4936		 * be helpful and could potentially cause us to enter an
4937		 * infinite loop.
4938		 */
4939		if (aggsum_compare(&arc_size, arc_c) <= 0|| evicted == 0) {
4940			/*
4941			 * We're either no longer overflowing, or we
4942			 * can't evict anything more, so we should wake
4943			 * up any threads before we go to sleep.
4944			 */
4945			cv_broadcast(&arc_reclaim_waiters_cv);
4946
4947			/*
4948			 * Block until signaled, or after one second (we
4949			 * might need to perform arc_kmem_reap_now()
4950			 * even if we aren't being signalled)
4951			 */
4952			CALLB_CPR_SAFE_BEGIN(&cpr);
4953			(void) cv_timedwait_hires(&arc_reclaim_thread_cv,
4954			    &arc_reclaim_lock, SEC2NSEC(1), MSEC2NSEC(1), 0);
4955			CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_lock);
4956		}
4957	}
4958
4959	arc_reclaim_thread_exit = B_FALSE;
4960	cv_broadcast(&arc_reclaim_thread_cv);
4961	CALLB_CPR_EXIT(&cpr);		/* drops arc_reclaim_lock */
4962	thread_exit();
4963}
4964
4965static u_int arc_dnlc_evicts_arg;
4966extern struct vfsops zfs_vfsops;
4967
4968static void
4969arc_dnlc_evicts_thread(void *dummy __unused)
4970{
4971	callb_cpr_t cpr;
4972	u_int percent;
4973
4974	CALLB_CPR_INIT(&cpr, &arc_dnlc_evicts_lock, callb_generic_cpr, FTAG);
4975
4976	mutex_enter(&arc_dnlc_evicts_lock);
4977	while (!arc_dnlc_evicts_thread_exit) {
4978		CALLB_CPR_SAFE_BEGIN(&cpr);
4979		(void) cv_wait(&arc_dnlc_evicts_cv, &arc_dnlc_evicts_lock);
4980		CALLB_CPR_SAFE_END(&cpr, &arc_dnlc_evicts_lock);
4981		if (arc_dnlc_evicts_arg != 0) {
4982			percent = arc_dnlc_evicts_arg;
4983			mutex_exit(&arc_dnlc_evicts_lock);
4984#ifdef _KERNEL
4985			vnlru_free(desiredvnodes * percent / 100, &zfs_vfsops);
4986#endif
4987			mutex_enter(&arc_dnlc_evicts_lock);
4988			/*
4989			 * Clear our token only after vnlru_free()
4990			 * pass is done, to avoid false queueing of
4991			 * the requests.
4992			 */
4993			arc_dnlc_evicts_arg = 0;
4994		}
4995	}
4996	arc_dnlc_evicts_thread_exit = FALSE;
4997	cv_broadcast(&arc_dnlc_evicts_cv);
4998	CALLB_CPR_EXIT(&cpr);
4999	thread_exit();
5000}
5001
5002void
5003dnlc_reduce_cache(void *arg)
5004{
5005	u_int percent;
5006
5007	percent = (u_int)(uintptr_t)arg;
5008	mutex_enter(&arc_dnlc_evicts_lock);
5009	if (arc_dnlc_evicts_arg == 0) {
5010		arc_dnlc_evicts_arg = percent;
5011		cv_broadcast(&arc_dnlc_evicts_cv);
5012	}
5013	mutex_exit(&arc_dnlc_evicts_lock);
5014}
5015
5016/*
5017 * Adapt arc info given the number of bytes we are trying to add and
5018 * the state that we are comming from.  This function is only called
5019 * when we are adding new content to the cache.
5020 */
5021static void
5022arc_adapt(int bytes, arc_state_t *state)
5023{
5024	int mult;
5025	uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
5026	int64_t mrug_size = refcount_count(&arc_mru_ghost->arcs_size);
5027	int64_t mfug_size = refcount_count(&arc_mfu_ghost->arcs_size);
5028
5029	if (state == arc_l2c_only)
5030		return;
5031
5032	ASSERT(bytes > 0);
5033	/*
5034	 * Adapt the target size of the MRU list:
5035	 *	- if we just hit in the MRU ghost list, then increase
5036	 *	  the target size of the MRU list.
5037	 *	- if we just hit in the MFU ghost list, then increase
5038	 *	  the target size of the MFU list by decreasing the
5039	 *	  target size of the MRU list.
5040	 */
5041	if (state == arc_mru_ghost) {
5042		mult = (mrug_size >= mfug_size) ? 1 : (mfug_size / mrug_size);
5043		mult = MIN(mult, 10); /* avoid wild arc_p adjustment */
5044
5045		arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
5046	} else if (state == arc_mfu_ghost) {
5047		uint64_t delta;
5048
5049		mult = (mfug_size >= mrug_size) ? 1 : (mrug_size / mfug_size);
5050		mult = MIN(mult, 10);
5051
5052		delta = MIN(bytes * mult, arc_p);
5053		arc_p = MAX(arc_p_min, arc_p - delta);
5054	}
5055	ASSERT((int64_t)arc_p >= 0);
5056
5057	if (arc_reclaim_needed()) {
5058		cv_signal(&arc_reclaim_thread_cv);
5059		return;
5060	}
5061
5062	if (arc_no_grow)
5063		return;
5064
5065	if (arc_c >= arc_c_max)
5066		return;
5067
5068	/*
5069	 * If we're within (2 * maxblocksize) bytes of the target
5070	 * cache size, increment the target cache size
5071	 */
5072	if (aggsum_compare(&arc_size, arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) >
5073	    0) {
5074		DTRACE_PROBE1(arc__inc_adapt, int, bytes);
5075		atomic_add_64(&arc_c, (int64_t)bytes);
5076		if (arc_c > arc_c_max)
5077			arc_c = arc_c_max;
5078		else if (state == arc_anon)
5079			atomic_add_64(&arc_p, (int64_t)bytes);
5080		if (arc_p > arc_c)
5081			arc_p = arc_c;
5082	}
5083	ASSERT((int64_t)arc_p >= 0);
5084}
5085
5086/*
5087 * Check if arc_size has grown past our upper threshold, determined by
5088 * zfs_arc_overflow_shift.
5089 */
5090static boolean_t
5091arc_is_overflowing(void)
5092{
5093	/* Always allow at least one block of overflow */
5094	uint64_t overflow = MAX(SPA_MAXBLOCKSIZE,
5095	    arc_c >> zfs_arc_overflow_shift);
5096
5097	/*
5098	 * We just compare the lower bound here for performance reasons. Our
5099	 * primary goals are to make sure that the arc never grows without
5100	 * bound, and that it can reach its maximum size. This check
5101	 * accomplishes both goals. The maximum amount we could run over by is
5102	 * 2 * aggsum_borrow_multiplier * NUM_CPUS * the average size of a block
5103	 * in the ARC. In practice, that's in the tens of MB, which is low
5104	 * enough to be safe.
5105	 */
5106	return (aggsum_lower_bound(&arc_size) >= arc_c + overflow);
5107}
5108
5109static abd_t *
5110arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
5111{
5112	arc_buf_contents_t type = arc_buf_type(hdr);
5113
5114	arc_get_data_impl(hdr, size, tag);
5115	if (type == ARC_BUFC_METADATA) {
5116		return (abd_alloc(size, B_TRUE));
5117	} else {
5118		ASSERT(type == ARC_BUFC_DATA);
5119		return (abd_alloc(size, B_FALSE));
5120	}
5121}
5122
5123static void *
5124arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
5125{
5126	arc_buf_contents_t type = arc_buf_type(hdr);
5127
5128	arc_get_data_impl(hdr, size, tag);
5129	if (type == ARC_BUFC_METADATA) {
5130		return (zio_buf_alloc(size));
5131	} else {
5132		ASSERT(type == ARC_BUFC_DATA);
5133		return (zio_data_buf_alloc(size));
5134	}
5135}
5136
5137/*
5138 * Allocate a block and return it to the caller. If we are hitting the
5139 * hard limit for the cache size, we must sleep, waiting for the eviction
5140 * thread to catch up. If we're past the target size but below the hard
5141 * limit, we'll only signal the reclaim thread and continue on.
5142 */
5143static void
5144arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
5145{
5146	arc_state_t *state = hdr->b_l1hdr.b_state;
5147	arc_buf_contents_t type = arc_buf_type(hdr);
5148
5149	arc_adapt(size, state);
5150
5151	/*
5152	 * If arc_size is currently overflowing, and has grown past our
5153	 * upper limit, we must be adding data faster than the evict
5154	 * thread can evict. Thus, to ensure we don't compound the
5155	 * problem by adding more data and forcing arc_size to grow even
5156	 * further past it's target size, we halt and wait for the
5157	 * eviction thread to catch up.
5158	 *
5159	 * It's also possible that the reclaim thread is unable to evict
5160	 * enough buffers to get arc_size below the overflow limit (e.g.
5161	 * due to buffers being un-evictable, or hash lock collisions).
5162	 * In this case, we want to proceed regardless if we're
5163	 * overflowing; thus we don't use a while loop here.
5164	 */
5165	if (arc_is_overflowing()) {
5166		mutex_enter(&arc_reclaim_lock);
5167
5168		/*
5169		 * Now that we've acquired the lock, we may no longer be
5170		 * over the overflow limit, lets check.
5171		 *
5172		 * We're ignoring the case of spurious wake ups. If that
5173		 * were to happen, it'd let this thread consume an ARC
5174		 * buffer before it should have (i.e. before we're under
5175		 * the overflow limit and were signalled by the reclaim
5176		 * thread). As long as that is a rare occurrence, it
5177		 * shouldn't cause any harm.
5178		 */
5179		if (arc_is_overflowing()) {
5180			cv_signal(&arc_reclaim_thread_cv);
5181			cv_wait(&arc_reclaim_waiters_cv, &arc_reclaim_lock);
5182		}
5183
5184		mutex_exit(&arc_reclaim_lock);
5185	}
5186
5187	VERIFY3U(hdr->b_type, ==, type);
5188	if (type == ARC_BUFC_METADATA) {
5189		arc_space_consume(size, ARC_SPACE_META);
5190	} else {
5191		arc_space_consume(size, ARC_SPACE_DATA);
5192	}
5193
5194	/*
5195	 * Update the state size.  Note that ghost states have a
5196	 * "ghost size" and so don't need to be updated.
5197	 */
5198	if (!GHOST_STATE(state)) {
5199
5200		(void) refcount_add_many(&state->arcs_size, size, tag);
5201
5202		/*
5203		 * If this is reached via arc_read, the link is
5204		 * protected by the hash lock. If reached via
5205		 * arc_buf_alloc, the header should not be accessed by
5206		 * any other thread. And, if reached via arc_read_done,
5207		 * the hash lock will protect it if it's found in the
5208		 * hash table; otherwise no other thread should be
5209		 * trying to [add|remove]_reference it.
5210		 */
5211		if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
5212			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5213			(void) refcount_add_many(&state->arcs_esize[type],
5214			    size, tag);
5215		}
5216
5217		/*
5218		 * If we are growing the cache, and we are adding anonymous
5219		 * data, and we have outgrown arc_p, update arc_p
5220		 */
5221		if (aggsum_compare(&arc_size, arc_c) < 0 &&
5222		    hdr->b_l1hdr.b_state == arc_anon &&
5223		    (refcount_count(&arc_anon->arcs_size) +
5224		    refcount_count(&arc_mru->arcs_size) > arc_p))
5225			arc_p = MIN(arc_c, arc_p + size);
5226	}
5227	ARCSTAT_BUMP(arcstat_allocated);
5228}
5229
5230static void
5231arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size, void *tag)
5232{
5233	arc_free_data_impl(hdr, size, tag);
5234	abd_free(abd);
5235}
5236
5237static void
5238arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, void *tag)
5239{
5240	arc_buf_contents_t type = arc_buf_type(hdr);
5241
5242	arc_free_data_impl(hdr, size, tag);
5243	if (type == ARC_BUFC_METADATA) {
5244		zio_buf_free(buf, size);
5245	} else {
5246		ASSERT(type == ARC_BUFC_DATA);
5247		zio_data_buf_free(buf, size);
5248	}
5249}
5250
5251/*
5252 * Free the arc data buffer.
5253 */
5254static void
5255arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
5256{
5257	arc_state_t *state = hdr->b_l1hdr.b_state;
5258	arc_buf_contents_t type = arc_buf_type(hdr);
5259
5260	/* protected by hash lock, if in the hash table */
5261	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
5262		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5263		ASSERT(state != arc_anon && state != arc_l2c_only);
5264
5265		(void) refcount_remove_many(&state->arcs_esize[type],
5266		    size, tag);
5267	}
5268	(void) refcount_remove_many(&state->arcs_size, size, tag);
5269
5270	VERIFY3U(hdr->b_type, ==, type);
5271	if (type == ARC_BUFC_METADATA) {
5272		arc_space_return(size, ARC_SPACE_META);
5273	} else {
5274		ASSERT(type == ARC_BUFC_DATA);
5275		arc_space_return(size, ARC_SPACE_DATA);
5276	}
5277}
5278
5279/*
5280 * This routine is called whenever a buffer is accessed.
5281 * NOTE: the hash lock is dropped in this function.
5282 */
5283static void
5284arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
5285{
5286	clock_t now;
5287
5288	ASSERT(MUTEX_HELD(hash_lock));
5289	ASSERT(HDR_HAS_L1HDR(hdr));
5290
5291	if (hdr->b_l1hdr.b_state == arc_anon) {
5292		/*
5293		 * This buffer is not in the cache, and does not
5294		 * appear in our "ghost" list.  Add the new buffer
5295		 * to the MRU state.
5296		 */
5297
5298		ASSERT0(hdr->b_l1hdr.b_arc_access);
5299		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5300		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5301		arc_change_state(arc_mru, hdr, hash_lock);
5302
5303	} else if (hdr->b_l1hdr.b_state == arc_mru) {
5304		now = ddi_get_lbolt();
5305
5306		/*
5307		 * If this buffer is here because of a prefetch, then either:
5308		 * - clear the flag if this is a "referencing" read
5309		 *   (any subsequent access will bump this into the MFU state).
5310		 * or
5311		 * - move the buffer to the head of the list if this is
5312		 *   another prefetch (to make it less likely to be evicted).
5313		 */
5314		if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) {
5315			if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
5316				/* link protected by hash lock */
5317				ASSERT(multilist_link_active(
5318				    &hdr->b_l1hdr.b_arc_node));
5319			} else {
5320				arc_hdr_clear_flags(hdr,
5321				    ARC_FLAG_PREFETCH |
5322				    ARC_FLAG_PRESCIENT_PREFETCH);
5323				ARCSTAT_BUMP(arcstat_mru_hits);
5324			}
5325			hdr->b_l1hdr.b_arc_access = now;
5326			return;
5327		}
5328
5329		/*
5330		 * This buffer has been "accessed" only once so far,
5331		 * but it is still in the cache. Move it to the MFU
5332		 * state.
5333		 */
5334		if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) {
5335			/*
5336			 * More than 125ms have passed since we
5337			 * instantiated this buffer.  Move it to the
5338			 * most frequently used state.
5339			 */
5340			hdr->b_l1hdr.b_arc_access = now;
5341			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5342			arc_change_state(arc_mfu, hdr, hash_lock);
5343		}
5344		atomic_inc_32(&hdr->b_l1hdr.b_mru_hits);
5345		ARCSTAT_BUMP(arcstat_mru_hits);
5346	} else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
5347		arc_state_t	*new_state;
5348		/*
5349		 * This buffer has been "accessed" recently, but
5350		 * was evicted from the cache.  Move it to the
5351		 * MFU state.
5352		 */
5353
5354		if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) {
5355			new_state = arc_mru;
5356			if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0) {
5357				arc_hdr_clear_flags(hdr,
5358				    ARC_FLAG_PREFETCH |
5359				    ARC_FLAG_PRESCIENT_PREFETCH);
5360			}
5361			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5362		} else {
5363			new_state = arc_mfu;
5364			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5365		}
5366
5367		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5368		arc_change_state(new_state, hdr, hash_lock);
5369
5370		atomic_inc_32(&hdr->b_l1hdr.b_mru_ghost_hits);
5371		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
5372	} else if (hdr->b_l1hdr.b_state == arc_mfu) {
5373		/*
5374		 * This buffer has been accessed more than once and is
5375		 * still in the cache.  Keep it in the MFU state.
5376		 *
5377		 * NOTE: an add_reference() that occurred when we did
5378		 * the arc_read() will have kicked this off the list.
5379		 * If it was a prefetch, we will explicitly move it to
5380		 * the head of the list now.
5381		 */
5382
5383		atomic_inc_32(&hdr->b_l1hdr.b_mfu_hits);
5384		ARCSTAT_BUMP(arcstat_mfu_hits);
5385		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5386	} else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
5387		arc_state_t	*new_state = arc_mfu;
5388		/*
5389		 * This buffer has been accessed more than once but has
5390		 * been evicted from the cache.  Move it back to the
5391		 * MFU state.
5392		 */
5393
5394		if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) {
5395			/*
5396			 * This is a prefetch access...
5397			 * move this block back to the MRU state.
5398			 */
5399			new_state = arc_mru;
5400		}
5401
5402		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5403		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5404		arc_change_state(new_state, hdr, hash_lock);
5405
5406		atomic_inc_32(&hdr->b_l1hdr.b_mfu_ghost_hits);
5407		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
5408	} else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
5409		/*
5410		 * This buffer is on the 2nd Level ARC.
5411		 */
5412
5413		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5414		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5415		arc_change_state(arc_mfu, hdr, hash_lock);
5416	} else {
5417		ASSERT(!"invalid arc state");
5418	}
5419}
5420
5421/*
5422 * This routine is called by dbuf_hold() to update the arc_access() state
5423 * which otherwise would be skipped for entries in the dbuf cache.
5424 */
5425void
5426arc_buf_access(arc_buf_t *buf)
5427{
5428	mutex_enter(&buf->b_evict_lock);
5429	arc_buf_hdr_t *hdr = buf->b_hdr;
5430
5431	/*
5432	 * Avoid taking the hash_lock when possible as an optimization.
5433	 * The header must be checked again under the hash_lock in order
5434	 * to handle the case where it is concurrently being released.
5435	 */
5436	if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) {
5437		mutex_exit(&buf->b_evict_lock);
5438		ARCSTAT_BUMP(arcstat_access_skip);
5439		return;
5440	}
5441
5442	kmutex_t *hash_lock = HDR_LOCK(hdr);
5443	mutex_enter(hash_lock);
5444
5445	if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) {
5446		mutex_exit(hash_lock);
5447		mutex_exit(&buf->b_evict_lock);
5448		ARCSTAT_BUMP(arcstat_access_skip);
5449		return;
5450	}
5451
5452	mutex_exit(&buf->b_evict_lock);
5453
5454	ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5455	    hdr->b_l1hdr.b_state == arc_mfu);
5456
5457	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
5458	arc_access(hdr, hash_lock);
5459	mutex_exit(hash_lock);
5460
5461	ARCSTAT_BUMP(arcstat_hits);
5462	ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
5463	    demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data, metadata, hits);
5464}
5465
5466/* a generic arc_read_done_func_t which you can use */
5467/* ARGSUSED */
5468void
5469arc_bcopy_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
5470    arc_buf_t *buf, void *arg)
5471{
5472	if (buf == NULL)
5473		return;
5474
5475	bcopy(buf->b_data, arg, arc_buf_size(buf));
5476	arc_buf_destroy(buf, arg);
5477}
5478
5479/* a generic arc_read_done_func_t */
5480/* ARGSUSED */
5481void
5482arc_getbuf_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
5483    arc_buf_t *buf, void *arg)
5484{
5485	arc_buf_t **bufp = arg;
5486	if (buf == NULL) {
5487		ASSERT(zio == NULL || zio->io_error != 0);
5488		*bufp = NULL;
5489	} else {
5490		ASSERT(zio == NULL || zio->io_error == 0);
5491		*bufp = buf;
5492		ASSERT(buf->b_data != NULL);
5493	}
5494}
5495
5496static void
5497arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)
5498{
5499	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
5500		ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);
5501		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
5502	} else {
5503		if (HDR_COMPRESSION_ENABLED(hdr)) {
5504			ASSERT3U(HDR_GET_COMPRESS(hdr), ==,
5505			    BP_GET_COMPRESS(bp));
5506		}
5507		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
5508		ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp));
5509	}
5510}
5511
5512static void
5513arc_read_done(zio_t *zio)
5514{
5515	arc_buf_hdr_t	*hdr = zio->io_private;
5516	kmutex_t	*hash_lock = NULL;
5517	arc_callback_t	*callback_list;
5518	arc_callback_t	*acb;
5519	boolean_t	freeable = B_FALSE;
5520	boolean_t	no_zio_error = (zio->io_error == 0);
5521
5522	/*
5523	 * The hdr was inserted into hash-table and removed from lists
5524	 * prior to starting I/O.  We should find this header, since
5525	 * it's in the hash table, and it should be legit since it's
5526	 * not possible to evict it during the I/O.  The only possible
5527	 * reason for it not to be found is if we were freed during the
5528	 * read.
5529	 */
5530	if (HDR_IN_HASH_TABLE(hdr)) {
5531		ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp));
5532		ASSERT3U(hdr->b_dva.dva_word[0], ==,
5533		    BP_IDENTITY(zio->io_bp)->dva_word[0]);
5534		ASSERT3U(hdr->b_dva.dva_word[1], ==,
5535		    BP_IDENTITY(zio->io_bp)->dva_word[1]);
5536
5537		arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp,
5538		    &hash_lock);
5539
5540		ASSERT((found == hdr &&
5541		    DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
5542		    (found == hdr && HDR_L2_READING(hdr)));
5543		ASSERT3P(hash_lock, !=, NULL);
5544	}
5545
5546	if (no_zio_error) {
5547		/* byteswap if necessary */
5548		if (BP_SHOULD_BYTESWAP(zio->io_bp)) {
5549			if (BP_GET_LEVEL(zio->io_bp) > 0) {
5550				hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
5551			} else {
5552				hdr->b_l1hdr.b_byteswap =
5553				    DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
5554			}
5555		} else {
5556			hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
5557		}
5558	}
5559
5560	arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
5561	if (l2arc_noprefetch && HDR_PREFETCH(hdr))
5562		arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);
5563
5564	callback_list = hdr->b_l1hdr.b_acb;
5565	ASSERT3P(callback_list, !=, NULL);
5566
5567	if (hash_lock && no_zio_error && hdr->b_l1hdr.b_state == arc_anon) {
5568		/*
5569		 * Only call arc_access on anonymous buffers.  This is because
5570		 * if we've issued an I/O for an evicted buffer, we've already
5571		 * called arc_access (to prevent any simultaneous readers from
5572		 * getting confused).
5573		 */
5574		arc_access(hdr, hash_lock);
5575	}
5576
5577	/*
5578	 * If a read request has a callback (i.e. acb_done is not NULL), then we
5579	 * make a buf containing the data according to the parameters which were
5580	 * passed in. The implementation of arc_buf_alloc_impl() ensures that we
5581	 * aren't needlessly decompressing the data multiple times.
5582	 */
5583	int callback_cnt = 0;
5584	for (acb = callback_list; acb != NULL; acb = acb->acb_next) {
5585		if (!acb->acb_done)
5586			continue;
5587
5588		callback_cnt++;
5589
5590		if (no_zio_error) {
5591			int error = arc_buf_alloc_impl(hdr, acb->acb_private,
5592			    acb->acb_compressed, zio->io_error == 0,
5593			    &acb->acb_buf);
5594			if (error != 0) {
5595				/*
5596				 * Decompression failed.  Set io_error
5597				 * so that when we call acb_done (below),
5598				 * we will indicate that the read failed.
5599				 * Note that in the unusual case where one
5600				 * callback is compressed and another
5601				 * uncompressed, we will mark all of them
5602				 * as failed, even though the uncompressed
5603				 * one can't actually fail.  In this case,
5604				 * the hdr will not be anonymous, because
5605				 * if there are multiple callbacks, it's
5606				 * because multiple threads found the same
5607				 * arc buf in the hash table.
5608				 */
5609				zio->io_error = error;
5610			}
5611		}
5612	}
5613	/*
5614	 * If there are multiple callbacks, we must have the hash lock,
5615	 * because the only way for multiple threads to find this hdr is
5616	 * in the hash table.  This ensures that if there are multiple
5617	 * callbacks, the hdr is not anonymous.  If it were anonymous,
5618	 * we couldn't use arc_buf_destroy() in the error case below.
5619	 */
5620	ASSERT(callback_cnt < 2 || hash_lock != NULL);
5621
5622	hdr->b_l1hdr.b_acb = NULL;
5623	arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5624	if (callback_cnt == 0) {
5625		ASSERT(HDR_PREFETCH(hdr));
5626		ASSERT0(hdr->b_l1hdr.b_bufcnt);
5627		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5628	}
5629
5630	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
5631	    callback_list != NULL);
5632
5633	if (no_zio_error) {
5634		arc_hdr_verify(hdr, zio->io_bp);
5635	} else {
5636		arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
5637		if (hdr->b_l1hdr.b_state != arc_anon)
5638			arc_change_state(arc_anon, hdr, hash_lock);
5639		if (HDR_IN_HASH_TABLE(hdr))
5640			buf_hash_remove(hdr);
5641		freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
5642	}
5643
5644	/*
5645	 * Broadcast before we drop the hash_lock to avoid the possibility
5646	 * that the hdr (and hence the cv) might be freed before we get to
5647	 * the cv_broadcast().
5648	 */
5649	cv_broadcast(&hdr->b_l1hdr.b_cv);
5650
5651	if (hash_lock != NULL) {
5652		mutex_exit(hash_lock);
5653	} else {
5654		/*
5655		 * This block was freed while we waited for the read to
5656		 * complete.  It has been removed from the hash table and
5657		 * moved to the anonymous state (so that it won't show up
5658		 * in the cache).
5659		 */
5660		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
5661		freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
5662	}
5663
5664	/* execute each callback and free its structure */
5665	while ((acb = callback_list) != NULL) {
5666		if (acb->acb_done != NULL) {
5667			if (zio->io_error != 0 && acb->acb_buf != NULL) {
5668				/*
5669				 * If arc_buf_alloc_impl() fails during
5670				 * decompression, the buf will still be
5671				 * allocated, and needs to be freed here.
5672				 */
5673				arc_buf_destroy(acb->acb_buf, acb->acb_private);
5674				acb->acb_buf = NULL;
5675			}
5676			acb->acb_done(zio, &zio->io_bookmark, zio->io_bp,
5677			    acb->acb_buf, acb->acb_private);
5678		}
5679
5680		if (acb->acb_zio_dummy != NULL) {
5681			acb->acb_zio_dummy->io_error = zio->io_error;
5682			zio_nowait(acb->acb_zio_dummy);
5683		}
5684
5685		callback_list = acb->acb_next;
5686		kmem_free(acb, sizeof (arc_callback_t));
5687	}
5688
5689	if (freeable)
5690		arc_hdr_destroy(hdr);
5691}
5692
5693/*
5694 * "Read" the block at the specified DVA (in bp) via the
5695 * cache.  If the block is found in the cache, invoke the provided
5696 * callback immediately and return.  Note that the `zio' parameter
5697 * in the callback will be NULL in this case, since no IO was
5698 * required.  If the block is not in the cache pass the read request
5699 * on to the spa with a substitute callback function, so that the
5700 * requested block will be added to the cache.
5701 *
5702 * If a read request arrives for a block that has a read in-progress,
5703 * either wait for the in-progress read to complete (and return the
5704 * results); or, if this is a read with a "done" func, add a record
5705 * to the read to invoke the "done" func when the read completes,
5706 * and return; or just return.
5707 *
5708 * arc_read_done() will invoke all the requested "done" functions
5709 * for readers of this block.
5710 */
5711int
5712arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_read_done_func_t *done,
5713    void *private, zio_priority_t priority, int zio_flags,
5714    arc_flags_t *arc_flags, const zbookmark_phys_t *zb)
5715{
5716	arc_buf_hdr_t *hdr = NULL;
5717	kmutex_t *hash_lock = NULL;
5718	zio_t *rzio;
5719	uint64_t guid = spa_load_guid(spa);
5720	boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW) != 0;
5721	int rc = 0;
5722
5723	ASSERT(!BP_IS_EMBEDDED(bp) ||
5724	    BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
5725
5726top:
5727	if (!BP_IS_EMBEDDED(bp)) {
5728		/*
5729		 * Embedded BP's have no DVA and require no I/O to "read".
5730		 * Create an anonymous arc buf to back it.
5731		 */
5732		hdr = buf_hash_find(guid, bp, &hash_lock);
5733	}
5734
5735	if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_pabd != NULL) {
5736		arc_buf_t *buf = NULL;
5737		*arc_flags |= ARC_FLAG_CACHED;
5738
5739		if (HDR_IO_IN_PROGRESS(hdr)) {
5740			zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head;
5741
5742			ASSERT3P(head_zio, !=, NULL);
5743			if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&
5744			    priority == ZIO_PRIORITY_SYNC_READ) {
5745				/*
5746				 * This is a sync read that needs to wait for
5747				 * an in-flight async read. Request that the
5748				 * zio have its priority upgraded.
5749				 */
5750				zio_change_priority(head_zio, priority);
5751				DTRACE_PROBE1(arc__async__upgrade__sync,
5752				    arc_buf_hdr_t *, hdr);
5753				ARCSTAT_BUMP(arcstat_async_upgrade_sync);
5754			}
5755			if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) {
5756				arc_hdr_clear_flags(hdr,
5757				    ARC_FLAG_PREDICTIVE_PREFETCH);
5758			}
5759
5760			if (*arc_flags & ARC_FLAG_WAIT) {
5761				cv_wait(&hdr->b_l1hdr.b_cv, hash_lock);
5762				mutex_exit(hash_lock);
5763				goto top;
5764			}
5765			ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
5766
5767			if (done) {
5768				arc_callback_t *acb = NULL;
5769
5770				acb = kmem_zalloc(sizeof (arc_callback_t),
5771				    KM_SLEEP);
5772				acb->acb_done = done;
5773				acb->acb_private = private;
5774				acb->acb_compressed = compressed_read;
5775				if (pio != NULL)
5776					acb->acb_zio_dummy = zio_null(pio,
5777					    spa, NULL, NULL, NULL, zio_flags);
5778
5779				ASSERT3P(acb->acb_done, !=, NULL);
5780				acb->acb_zio_head = head_zio;
5781				acb->acb_next = hdr->b_l1hdr.b_acb;
5782				hdr->b_l1hdr.b_acb = acb;
5783				mutex_exit(hash_lock);
5784				return (0);
5785			}
5786			mutex_exit(hash_lock);
5787			return (0);
5788		}
5789
5790		ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5791		    hdr->b_l1hdr.b_state == arc_mfu);
5792
5793		if (done) {
5794			if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) {
5795				/*
5796				 * This is a demand read which does not have to
5797				 * wait for i/o because we did a predictive
5798				 * prefetch i/o for it, which has completed.
5799				 */
5800				DTRACE_PROBE1(
5801				    arc__demand__hit__predictive__prefetch,
5802				    arc_buf_hdr_t *, hdr);
5803				ARCSTAT_BUMP(
5804				    arcstat_demand_hit_predictive_prefetch);
5805				arc_hdr_clear_flags(hdr,
5806				    ARC_FLAG_PREDICTIVE_PREFETCH);
5807			}
5808
5809			if (hdr->b_flags & ARC_FLAG_PRESCIENT_PREFETCH) {
5810				ARCSTAT_BUMP(
5811                                    arcstat_demand_hit_prescient_prefetch);
5812				arc_hdr_clear_flags(hdr,
5813                                    ARC_FLAG_PRESCIENT_PREFETCH);
5814			}
5815
5816			ASSERT(!BP_IS_EMBEDDED(bp) || !BP_IS_HOLE(bp));
5817			/* Get a buf with the desired data in it. */
5818			rc = arc_buf_alloc_impl(hdr, private,
5819			   compressed_read, B_TRUE, &buf);
5820			if (rc != 0) {
5821				arc_buf_destroy(buf, private);
5822				buf = NULL;
5823			}
5824			ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) ||
5825                            rc == 0 || rc != ENOENT);
5826		} else if (*arc_flags & ARC_FLAG_PREFETCH &&
5827		    refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
5828			arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
5829		}
5830		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
5831		arc_access(hdr, hash_lock);
5832		if (*arc_flags & ARC_FLAG_PRESCIENT_PREFETCH)
5833                        arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH);
5834		if (*arc_flags & ARC_FLAG_L2CACHE)
5835			arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5836		mutex_exit(hash_lock);
5837		ARCSTAT_BUMP(arcstat_hits);
5838		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
5839		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
5840		    data, metadata, hits);
5841
5842		if (done)
5843			done(NULL, zb, bp, buf, private);
5844	} else {
5845		uint64_t lsize = BP_GET_LSIZE(bp);
5846		uint64_t psize = BP_GET_PSIZE(bp);
5847		arc_callback_t *acb;
5848		vdev_t *vd = NULL;
5849		uint64_t addr = 0;
5850		boolean_t devw = B_FALSE;
5851		uint64_t size;
5852
5853		if (hdr == NULL) {
5854			/* this block is not in the cache */
5855			arc_buf_hdr_t *exists = NULL;
5856			arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
5857			hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
5858			    BP_GET_COMPRESS(bp), type);
5859
5860			if (!BP_IS_EMBEDDED(bp)) {
5861				hdr->b_dva = *BP_IDENTITY(bp);
5862				hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
5863				exists = buf_hash_insert(hdr, &hash_lock);
5864			}
5865			if (exists != NULL) {
5866				/* somebody beat us to the hash insert */
5867				mutex_exit(hash_lock);
5868				buf_discard_identity(hdr);
5869				arc_hdr_destroy(hdr);
5870				goto top; /* restart the IO request */
5871			}
5872		} else {
5873			/*
5874			 * This block is in the ghost cache. If it was L2-only
5875			 * (and thus didn't have an L1 hdr), we realloc the
5876			 * header to add an L1 hdr.
5877			 */
5878			if (!HDR_HAS_L1HDR(hdr)) {
5879				hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
5880				    hdr_full_cache);
5881			}
5882			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
5883			ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
5884			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5885			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5886			ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
5887			ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
5888
5889			/*
5890			 * This is a delicate dance that we play here.
5891			 * This hdr is in the ghost list so we access it
5892			 * to move it out of the ghost list before we
5893			 * initiate the read. If it's a prefetch then
5894			 * it won't have a callback so we'll remove the
5895			 * reference that arc_buf_alloc_impl() created. We
5896			 * do this after we've called arc_access() to
5897			 * avoid hitting an assert in remove_reference().
5898			 */
5899			arc_access(hdr, hash_lock);
5900			arc_hdr_alloc_pabd(hdr);
5901		}
5902		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5903		size = arc_hdr_size(hdr);
5904
5905		/*
5906		 * If compression is enabled on the hdr, then will do
5907		 * RAW I/O and will store the compressed data in the hdr's
5908		 * data block. Otherwise, the hdr's data block will contain
5909		 * the uncompressed data.
5910		 */
5911		if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) {
5912			zio_flags |= ZIO_FLAG_RAW;
5913		}
5914
5915		if (*arc_flags & ARC_FLAG_PREFETCH)
5916			arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
5917		if (*arc_flags & ARC_FLAG_PRESCIENT_PREFETCH)
5918			arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH);
5919
5920		if (*arc_flags & ARC_FLAG_L2CACHE)
5921			arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5922		if (BP_GET_LEVEL(bp) > 0)
5923			arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT);
5924		if (*arc_flags & ARC_FLAG_PREDICTIVE_PREFETCH)
5925			arc_hdr_set_flags(hdr, ARC_FLAG_PREDICTIVE_PREFETCH);
5926		ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
5927
5928		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
5929		acb->acb_done = done;
5930		acb->acb_private = private;
5931		acb->acb_compressed = compressed_read;
5932
5933		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
5934		hdr->b_l1hdr.b_acb = acb;
5935		arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5936
5937		if (HDR_HAS_L2HDR(hdr) &&
5938		    (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
5939			devw = hdr->b_l2hdr.b_dev->l2ad_writing;
5940			addr = hdr->b_l2hdr.b_daddr;
5941			/*
5942			 * Lock out L2ARC device removal.
5943			 */
5944			if (vdev_is_dead(vd) ||
5945			    !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
5946				vd = NULL;
5947		}
5948
5949		/*
5950		 * We count both async reads and scrub IOs as asynchronous so
5951		 * that both can be upgraded in the event of a cache hit while
5952		 * the read IO is still in-flight.
5953		 */
5954		if (priority == ZIO_PRIORITY_ASYNC_READ ||
5955		    priority == ZIO_PRIORITY_SCRUB)
5956			arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
5957		else
5958			arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
5959
5960		/*
5961		 * At this point, we have a level 1 cache miss.  Try again in
5962		 * L2ARC if possible.
5963		 */
5964		ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize);
5965
5966		DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
5967		    uint64_t, lsize, zbookmark_phys_t *, zb);
5968		ARCSTAT_BUMP(arcstat_misses);
5969		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
5970		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
5971		    data, metadata, misses);
5972#ifdef _KERNEL
5973#ifdef RACCT
5974		if (racct_enable) {
5975			PROC_LOCK(curproc);
5976			racct_add_force(curproc, RACCT_READBPS, size);
5977			racct_add_force(curproc, RACCT_READIOPS, 1);
5978			PROC_UNLOCK(curproc);
5979		}
5980#endif /* RACCT */
5981		curthread->td_ru.ru_inblock++;
5982#endif
5983
5984		if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
5985			/*
5986			 * Read from the L2ARC if the following are true:
5987			 * 1. The L2ARC vdev was previously cached.
5988			 * 2. This buffer still has L2ARC metadata.
5989			 * 3. This buffer isn't currently writing to the L2ARC.
5990			 * 4. The L2ARC entry wasn't evicted, which may
5991			 *    also have invalidated the vdev.
5992			 * 5. This isn't prefetch and l2arc_noprefetch is set.
5993			 */
5994			if (HDR_HAS_L2HDR(hdr) &&
5995			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
5996			    !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
5997				l2arc_read_callback_t *cb;
5998				abd_t *abd;
5999				uint64_t asize;
6000
6001				DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
6002				ARCSTAT_BUMP(arcstat_l2_hits);
6003				atomic_inc_32(&hdr->b_l2hdr.b_hits);
6004
6005				cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
6006				    KM_SLEEP);
6007				cb->l2rcb_hdr = hdr;
6008				cb->l2rcb_bp = *bp;
6009				cb->l2rcb_zb = *zb;
6010				cb->l2rcb_flags = zio_flags;
6011
6012				asize = vdev_psize_to_asize(vd, size);
6013				if (asize != size) {
6014					abd = abd_alloc_for_io(asize,
6015					    HDR_ISTYPE_METADATA(hdr));
6016					cb->l2rcb_abd = abd;
6017				} else {
6018					abd = hdr->b_l1hdr.b_pabd;
6019				}
6020
6021				ASSERT(addr >= VDEV_LABEL_START_SIZE &&
6022				    addr + asize <= vd->vdev_psize -
6023				    VDEV_LABEL_END_SIZE);
6024
6025				/*
6026				 * l2arc read.  The SCL_L2ARC lock will be
6027				 * released by l2arc_read_done().
6028				 * Issue a null zio if the underlying buffer
6029				 * was squashed to zero size by compression.
6030				 */
6031				ASSERT3U(HDR_GET_COMPRESS(hdr), !=,
6032				    ZIO_COMPRESS_EMPTY);
6033				rzio = zio_read_phys(pio, vd, addr,
6034				    asize, abd,
6035				    ZIO_CHECKSUM_OFF,
6036				    l2arc_read_done, cb, priority,
6037				    zio_flags | ZIO_FLAG_DONT_CACHE |
6038				    ZIO_FLAG_CANFAIL |
6039				    ZIO_FLAG_DONT_PROPAGATE |
6040				    ZIO_FLAG_DONT_RETRY, B_FALSE);
6041				acb->acb_zio_head = rzio;
6042
6043				if (hash_lock != NULL)
6044					mutex_exit(hash_lock);
6045
6046				DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
6047				    zio_t *, rzio);
6048				ARCSTAT_INCR(arcstat_l2_read_bytes, size);
6049
6050				if (*arc_flags & ARC_FLAG_NOWAIT) {
6051					zio_nowait(rzio);
6052					return (0);
6053				}
6054
6055				ASSERT(*arc_flags & ARC_FLAG_WAIT);
6056				if (zio_wait(rzio) == 0)
6057					return (0);
6058
6059				/* l2arc read error; goto zio_read() */
6060				if (hash_lock != NULL)
6061					mutex_enter(hash_lock);
6062			} else {
6063				DTRACE_PROBE1(l2arc__miss,
6064				    arc_buf_hdr_t *, hdr);
6065				ARCSTAT_BUMP(arcstat_l2_misses);
6066				if (HDR_L2_WRITING(hdr))
6067					ARCSTAT_BUMP(arcstat_l2_rw_clash);
6068				spa_config_exit(spa, SCL_L2ARC, vd);
6069			}
6070		} else {
6071			if (vd != NULL)
6072				spa_config_exit(spa, SCL_L2ARC, vd);
6073			if (l2arc_ndev != 0) {
6074				DTRACE_PROBE1(l2arc__miss,
6075				    arc_buf_hdr_t *, hdr);
6076				ARCSTAT_BUMP(arcstat_l2_misses);
6077			}
6078		}
6079
6080		rzio = zio_read(pio, spa, bp, hdr->b_l1hdr.b_pabd, size,
6081		    arc_read_done, hdr, priority, zio_flags, zb);
6082		acb->acb_zio_head = rzio;
6083
6084		if (hash_lock != NULL)
6085			mutex_exit(hash_lock);
6086
6087		if (*arc_flags & ARC_FLAG_WAIT)
6088			return (zio_wait(rzio));
6089
6090		ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
6091		zio_nowait(rzio);
6092	}
6093	return (0);
6094}
6095
6096arc_prune_t *
6097arc_add_prune_callback(arc_prune_func_t *func, void *private)
6098{
6099	arc_prune_t *p;
6100
6101	p = kmem_alloc(sizeof (*p), KM_SLEEP);
6102	p->p_pfunc = func;
6103	p->p_private = private;
6104	list_link_init(&p->p_node);
6105	refcount_create(&p->p_refcnt);
6106
6107	mutex_enter(&arc_prune_mtx);
6108	refcount_add(&p->p_refcnt, &arc_prune_list);
6109	list_insert_head(&arc_prune_list, p);
6110	mutex_exit(&arc_prune_mtx);
6111
6112	return (p);
6113}
6114
6115void
6116arc_remove_prune_callback(arc_prune_t *p)
6117{
6118	boolean_t wait = B_FALSE;
6119	mutex_enter(&arc_prune_mtx);
6120	list_remove(&arc_prune_list, p);
6121	if (refcount_remove(&p->p_refcnt, &arc_prune_list) > 0)
6122		wait = B_TRUE;
6123	mutex_exit(&arc_prune_mtx);
6124
6125	/* wait for arc_prune_task to finish */
6126	if (wait)
6127		taskq_wait(arc_prune_taskq);
6128	ASSERT0(refcount_count(&p->p_refcnt));
6129	refcount_destroy(&p->p_refcnt);
6130	kmem_free(p, sizeof (*p));
6131}
6132
6133/*
6134 * Notify the arc that a block was freed, and thus will never be used again.
6135 */
6136void
6137arc_freed(spa_t *spa, const blkptr_t *bp)
6138{
6139	arc_buf_hdr_t *hdr;
6140	kmutex_t *hash_lock;
6141	uint64_t guid = spa_load_guid(spa);
6142
6143	ASSERT(!BP_IS_EMBEDDED(bp));
6144
6145	hdr = buf_hash_find(guid, bp, &hash_lock);
6146	if (hdr == NULL)
6147		return;
6148
6149	/*
6150	 * We might be trying to free a block that is still doing I/O
6151	 * (i.e. prefetch) or has a reference (i.e. a dedup-ed,
6152	 * dmu_sync-ed block). If this block is being prefetched, then it
6153	 * would still have the ARC_FLAG_IO_IN_PROGRESS flag set on the hdr
6154	 * until the I/O completes. A block may also have a reference if it is
6155	 * part of a dedup-ed, dmu_synced write. The dmu_sync() function would
6156	 * have written the new block to its final resting place on disk but
6157	 * without the dedup flag set. This would have left the hdr in the MRU
6158	 * state and discoverable. When the txg finally syncs it detects that
6159	 * the block was overridden in open context and issues an override I/O.
6160	 * Since this is a dedup block, the override I/O will determine if the
6161	 * block is already in the DDT. If so, then it will replace the io_bp
6162	 * with the bp from the DDT and allow the I/O to finish. When the I/O
6163	 * reaches the done callback, dbuf_write_override_done, it will
6164	 * check to see if the io_bp and io_bp_override are identical.
6165	 * If they are not, then it indicates that the bp was replaced with
6166	 * the bp in the DDT and the override bp is freed. This allows
6167	 * us to arrive here with a reference on a block that is being
6168	 * freed. So if we have an I/O in progress, or a reference to
6169	 * this hdr, then we don't destroy the hdr.
6170	 */
6171	if (!HDR_HAS_L1HDR(hdr) || (!HDR_IO_IN_PROGRESS(hdr) &&
6172	    refcount_is_zero(&hdr->b_l1hdr.b_refcnt))) {
6173		arc_change_state(arc_anon, hdr, hash_lock);
6174		arc_hdr_destroy(hdr);
6175		mutex_exit(hash_lock);
6176	} else {
6177		mutex_exit(hash_lock);
6178	}
6179
6180}
6181
6182/*
6183 * Release this buffer from the cache, making it an anonymous buffer.  This
6184 * must be done after a read and prior to modifying the buffer contents.
6185 * If the buffer has more than one reference, we must make
6186 * a new hdr for the buffer.
6187 */
6188void
6189arc_release(arc_buf_t *buf, void *tag)
6190{
6191	arc_buf_hdr_t *hdr = buf->b_hdr;
6192
6193	/*
6194	 * It would be nice to assert that if it's DMU metadata (level >
6195	 * 0 || it's the dnode file), then it must be syncing context.
6196	 * But we don't know that information at this level.
6197	 */
6198
6199	mutex_enter(&buf->b_evict_lock);
6200
6201	ASSERT(HDR_HAS_L1HDR(hdr));
6202
6203	/*
6204	 * We don't grab the hash lock prior to this check, because if
6205	 * the buffer's header is in the arc_anon state, it won't be
6206	 * linked into the hash table.
6207	 */
6208	if (hdr->b_l1hdr.b_state == arc_anon) {
6209		mutex_exit(&buf->b_evict_lock);
6210		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6211		ASSERT(!HDR_IN_HASH_TABLE(hdr));
6212		ASSERT(!HDR_HAS_L2HDR(hdr));
6213		ASSERT(HDR_EMPTY(hdr));
6214		ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
6215		ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
6216		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
6217
6218		hdr->b_l1hdr.b_arc_access = 0;
6219
6220		/*
6221		 * If the buf is being overridden then it may already
6222		 * have a hdr that is not empty.
6223		 */
6224		buf_discard_identity(hdr);
6225		arc_buf_thaw(buf);
6226
6227		return;
6228	}
6229
6230	kmutex_t *hash_lock = HDR_LOCK(hdr);
6231	mutex_enter(hash_lock);
6232
6233	/*
6234	 * This assignment is only valid as long as the hash_lock is
6235	 * held, we must be careful not to reference state or the
6236	 * b_state field after dropping the lock.
6237	 */
6238	arc_state_t *state = hdr->b_l1hdr.b_state;
6239	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
6240	ASSERT3P(state, !=, arc_anon);
6241
6242	/* this buffer is not on any list */
6243	ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0);
6244
6245	if (HDR_HAS_L2HDR(hdr)) {
6246		mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
6247
6248		/*
6249		 * We have to recheck this conditional again now that
6250		 * we're holding the l2ad_mtx to prevent a race with
6251		 * another thread which might be concurrently calling
6252		 * l2arc_evict(). In that case, l2arc_evict() might have
6253		 * destroyed the header's L2 portion as we were waiting
6254		 * to acquire the l2ad_mtx.
6255		 */
6256		if (HDR_HAS_L2HDR(hdr)) {
6257			l2arc_trim(hdr);
6258			arc_hdr_l2hdr_destroy(hdr);
6259		}
6260
6261		mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
6262	}
6263
6264	/*
6265	 * Do we have more than one buf?
6266	 */
6267	if (hdr->b_l1hdr.b_bufcnt > 1) {
6268		arc_buf_hdr_t *nhdr;
6269		uint64_t spa = hdr->b_spa;
6270		uint64_t psize = HDR_GET_PSIZE(hdr);
6271		uint64_t lsize = HDR_GET_LSIZE(hdr);
6272		enum zio_compress compress = HDR_GET_COMPRESS(hdr);
6273		arc_buf_contents_t type = arc_buf_type(hdr);
6274		VERIFY3U(hdr->b_type, ==, type);
6275
6276		ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
6277		(void) remove_reference(hdr, hash_lock, tag);
6278
6279		if (arc_buf_is_shared(buf) && !ARC_BUF_COMPRESSED(buf)) {
6280			ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
6281			ASSERT(ARC_BUF_LAST(buf));
6282		}
6283
6284		/*
6285		 * Pull the data off of this hdr and attach it to
6286		 * a new anonymous hdr. Also find the last buffer
6287		 * in the hdr's buffer list.
6288		 */
6289		arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
6290		ASSERT3P(lastbuf, !=, NULL);
6291
6292		/*
6293		 * If the current arc_buf_t and the hdr are sharing their data
6294		 * buffer, then we must stop sharing that block.
6295		 */
6296		if (arc_buf_is_shared(buf)) {
6297			VERIFY(!arc_buf_is_shared(lastbuf));
6298
6299			/*
6300			 * First, sever the block sharing relationship between
6301			 * buf and the arc_buf_hdr_t.
6302			 */
6303			arc_unshare_buf(hdr, buf);
6304
6305			/*
6306			 * Now we need to recreate the hdr's b_pabd. Since we
6307			 * have lastbuf handy, we try to share with it, but if
6308			 * we can't then we allocate a new b_pabd and copy the
6309			 * data from buf into it.
6310			 */
6311			if (arc_can_share(hdr, lastbuf)) {
6312				arc_share_buf(hdr, lastbuf);
6313			} else {
6314				arc_hdr_alloc_pabd(hdr);
6315				abd_copy_from_buf(hdr->b_l1hdr.b_pabd,
6316				    buf->b_data, psize);
6317			}
6318			VERIFY3P(lastbuf->b_data, !=, NULL);
6319		} else if (HDR_SHARED_DATA(hdr)) {
6320			/*
6321			 * Uncompressed shared buffers are always at the end
6322			 * of the list. Compressed buffers don't have the
6323			 * same requirements. This makes it hard to
6324			 * simply assert that the lastbuf is shared so
6325			 * we rely on the hdr's compression flags to determine
6326			 * if we have a compressed, shared buffer.
6327			 */
6328			ASSERT(arc_buf_is_shared(lastbuf) ||
6329			    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
6330			ASSERT(!ARC_BUF_SHARED(buf));
6331		}
6332		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
6333		ASSERT3P(state, !=, arc_l2c_only);
6334
6335		(void) refcount_remove_many(&state->arcs_size,
6336		    arc_buf_size(buf), buf);
6337
6338		if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
6339			ASSERT3P(state, !=, arc_l2c_only);
6340			(void) refcount_remove_many(&state->arcs_esize[type],
6341			    arc_buf_size(buf), buf);
6342		}
6343
6344		hdr->b_l1hdr.b_bufcnt -= 1;
6345		arc_cksum_verify(buf);
6346#ifdef illumos
6347		arc_buf_unwatch(buf);
6348#endif
6349
6350		mutex_exit(hash_lock);
6351
6352		/*
6353		 * Allocate a new hdr. The new hdr will contain a b_pabd
6354		 * buffer which will be freed in arc_write().
6355		 */
6356		nhdr = arc_hdr_alloc(spa, psize, lsize, compress, type);
6357		ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL);
6358		ASSERT0(nhdr->b_l1hdr.b_bufcnt);
6359		ASSERT0(refcount_count(&nhdr->b_l1hdr.b_refcnt));
6360		VERIFY3U(nhdr->b_type, ==, type);
6361		ASSERT(!HDR_SHARED_DATA(nhdr));
6362
6363		nhdr->b_l1hdr.b_buf = buf;
6364		nhdr->b_l1hdr.b_bufcnt = 1;
6365		(void) refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
6366		buf->b_hdr = nhdr;
6367
6368		mutex_exit(&buf->b_evict_lock);
6369		(void) refcount_add_many(&arc_anon->arcs_size,
6370		    arc_buf_size(buf), buf);
6371	} else {
6372		mutex_exit(&buf->b_evict_lock);
6373		ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
6374		/* protected by hash lock, or hdr is on arc_anon */
6375		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
6376		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6377		arc_change_state(arc_anon, hdr, hash_lock);
6378		hdr->b_l1hdr.b_arc_access = 0;
6379		mutex_exit(hash_lock);
6380
6381		buf_discard_identity(hdr);
6382		arc_buf_thaw(buf);
6383	}
6384}
6385
6386int
6387arc_released(arc_buf_t *buf)
6388{
6389	int released;
6390
6391	mutex_enter(&buf->b_evict_lock);
6392	released = (buf->b_data != NULL &&
6393	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
6394	mutex_exit(&buf->b_evict_lock);
6395	return (released);
6396}
6397
6398#ifdef ZFS_DEBUG
6399int
6400arc_referenced(arc_buf_t *buf)
6401{
6402	int referenced;
6403
6404	mutex_enter(&buf->b_evict_lock);
6405	referenced = (refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
6406	mutex_exit(&buf->b_evict_lock);
6407	return (referenced);
6408}
6409#endif
6410
6411static void
6412arc_write_ready(zio_t *zio)
6413{
6414	arc_write_callback_t *callback = zio->io_private;
6415	arc_buf_t *buf = callback->awcb_buf;
6416	arc_buf_hdr_t *hdr = buf->b_hdr;
6417	uint64_t psize = BP_IS_HOLE(zio->io_bp) ? 0 : BP_GET_PSIZE(zio->io_bp);
6418
6419	ASSERT(HDR_HAS_L1HDR(hdr));
6420	ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
6421	ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
6422
6423	/*
6424	 * If we're reexecuting this zio because the pool suspended, then
6425	 * cleanup any state that was previously set the first time the
6426	 * callback was invoked.
6427	 */
6428	if (zio->io_flags & ZIO_FLAG_REEXECUTED) {
6429		arc_cksum_free(hdr);
6430#ifdef illumos
6431		arc_buf_unwatch(buf);
6432#endif
6433		if (hdr->b_l1hdr.b_pabd != NULL) {
6434			if (arc_buf_is_shared(buf)) {
6435				arc_unshare_buf(hdr, buf);
6436			} else {
6437				arc_hdr_free_pabd(hdr);
6438			}
6439		}
6440	}
6441	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6442	ASSERT(!HDR_SHARED_DATA(hdr));
6443	ASSERT(!arc_buf_is_shared(buf));
6444
6445	callback->awcb_ready(zio, buf, callback->awcb_private);
6446
6447	if (HDR_IO_IN_PROGRESS(hdr))
6448		ASSERT(zio->io_flags & ZIO_FLAG_REEXECUTED);
6449
6450	arc_cksum_compute(buf);
6451	arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6452
6453	enum zio_compress compress;
6454	if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
6455		compress = ZIO_COMPRESS_OFF;
6456	} else {
6457		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(zio->io_bp));
6458		compress = BP_GET_COMPRESS(zio->io_bp);
6459	}
6460	HDR_SET_PSIZE(hdr, psize);
6461	arc_hdr_set_compress(hdr, compress);
6462
6463
6464	/*
6465	 * Fill the hdr with data. If the hdr is compressed, the data we want
6466	 * is available from the zio, otherwise we can take it from the buf.
6467	 *
6468	 * We might be able to share the buf's data with the hdr here. However,
6469	 * doing so would cause the ARC to be full of linear ABDs if we write a
6470	 * lot of shareable data. As a compromise, we check whether scattered
6471	 * ABDs are allowed, and assume that if they are then the user wants
6472	 * the ARC to be primarily filled with them regardless of the data being
6473	 * written. Therefore, if they're allowed then we allocate one and copy
6474	 * the data into it; otherwise, we share the data directly if we can.
6475	 */
6476	if (zfs_abd_scatter_enabled || !arc_can_share(hdr, buf)) {
6477		arc_hdr_alloc_pabd(hdr);
6478
6479		/*
6480		 * Ideally, we would always copy the io_abd into b_pabd, but the
6481		 * user may have disabled compressed ARC, thus we must check the
6482		 * hdr's compression setting rather than the io_bp's.
6483		 */
6484		if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) {
6485			ASSERT3U(BP_GET_COMPRESS(zio->io_bp), !=,
6486			    ZIO_COMPRESS_OFF);
6487			ASSERT3U(psize, >, 0);
6488
6489			abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize);
6490		} else {
6491			ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr));
6492
6493			abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data,
6494			    arc_buf_size(buf));
6495		}
6496	} else {
6497		ASSERT3P(buf->b_data, ==, abd_to_buf(zio->io_orig_abd));
6498		ASSERT3U(zio->io_orig_size, ==, arc_buf_size(buf));
6499		ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
6500
6501		arc_share_buf(hdr, buf);
6502	}
6503
6504	arc_hdr_verify(hdr, zio->io_bp);
6505}
6506
6507static void
6508arc_write_children_ready(zio_t *zio)
6509{
6510