1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
24 * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
26 * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
27 */
28
29/*
30 * DVA-based Adjustable Replacement Cache
31 *
32 * While much of the theory of operation used here is
33 * based on the self-tuning, low overhead replacement cache
34 * presented by Megiddo and Modha at FAST 2003, there are some
35 * significant differences:
36 *
37 * 1. The Megiddo and Modha model assumes any page is evictable.
38 * Pages in its cache cannot be "locked" into memory.  This makes
39 * the eviction algorithm simple: evict the last page in the list.
40 * This also make the performance characteristics easy to reason
41 * about.  Our cache is not so simple.  At any given moment, some
42 * subset of the blocks in the cache are un-evictable because we
43 * have handed out a reference to them.  Blocks are only evictable
44 * when there are no external references active.  This makes
45 * eviction far more problematic:  we choose to evict the evictable
46 * blocks that are the "lowest" in the list.
47 *
48 * There are times when it is not possible to evict the requested
49 * space.  In these circumstances we are unable to adjust the cache
50 * size.  To prevent the cache growing unbounded at these times we
51 * implement a "cache throttle" that slows the flow of new data
52 * into the cache until we can make space available.
53 *
54 * 2. The Megiddo and Modha model assumes a fixed cache size.
55 * Pages are evicted when the cache is full and there is a cache
56 * miss.  Our model has a variable sized cache.  It grows with
57 * high use, but also tries to react to memory pressure from the
58 * operating system: decreasing its size when system memory is
59 * tight.
60 *
61 * 3. The Megiddo and Modha model assumes a fixed page size. All
62 * elements of the cache are therefore exactly the same size.  So
63 * when adjusting the cache size following a cache miss, its simply
64 * a matter of choosing a single page to evict.  In our model, we
65 * have variable sized cache blocks (rangeing from 512 bytes to
66 * 128K bytes).  We therefore choose a set of blocks to evict to make
67 * space for a cache miss that approximates as closely as possible
68 * the space used by the new block.
69 *
70 * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
71 * by N. Megiddo & D. Modha, FAST 2003
72 */
73
74/*
75 * The locking model:
76 *
77 * A new reference to a cache buffer can be obtained in two
78 * ways: 1) via a hash table lookup using the DVA as a key,
79 * or 2) via one of the ARC lists.  The arc_read() interface
80 * uses method 1, while the internal ARC algorithms for
81 * adjusting the cache use method 2.  We therefore provide two
82 * types of locks: 1) the hash table lock array, and 2) the
83 * ARC list locks.
84 *
85 * Buffers do not have their own mutexes, rather they rely on the
86 * hash table mutexes for the bulk of their protection (i.e. most
87 * fields in the arc_buf_hdr_t are protected by these mutexes).
88 *
89 * buf_hash_find() returns the appropriate mutex (held) when it
90 * locates the requested buffer in the hash table.  It returns
91 * NULL for the mutex if the buffer was not in the table.
92 *
93 * buf_hash_remove() expects the appropriate hash mutex to be
94 * already held before it is invoked.
95 *
96 * Each ARC state also has a mutex which is used to protect the
97 * buffer list associated with the state.  When attempting to
98 * obtain a hash table lock while holding an ARC list lock you
99 * must use: mutex_tryenter() to avoid deadlock.  Also note that
100 * the active state mutex must be held before the ghost state mutex.
101 *
102 * Note that the majority of the performance stats are manipulated
103 * with atomic operations.
104 *
105 * The L2ARC uses the l2ad_mtx on each vdev for the following:
106 *
107 *	- L2ARC buflist creation
108 *	- L2ARC buflist eviction
109 *	- L2ARC write completion, which walks L2ARC buflists
110 *	- ARC header destruction, as it removes from L2ARC buflists
111 *	- ARC header release, as it removes from L2ARC buflists
112 */
113
114/*
115 * ARC operation:
116 *
117 * Every block that is in the ARC is tracked by an arc_buf_hdr_t structure.
118 * This structure can point either to a block that is still in the cache or to
119 * one that is only accessible in an L2 ARC device, or it can provide
120 * information about a block that was recently evicted. If a block is
121 * only accessible in the L2ARC, then the arc_buf_hdr_t only has enough
122 * information to retrieve it from the L2ARC device. This information is
123 * stored in the l2arc_buf_hdr_t sub-structure of the arc_buf_hdr_t. A block
124 * that is in this state cannot access the data directly.
125 *
126 * Blocks that are actively being referenced or have not been evicted
127 * are cached in the L1ARC. The L1ARC (l1arc_buf_hdr_t) is a structure within
128 * the arc_buf_hdr_t that will point to the data block in memory. A block can
129 * only be read by a consumer if it has an l1arc_buf_hdr_t. The L1ARC
130 * caches data in two ways -- in a list of ARC buffers (arc_buf_t) and
131 * also in the arc_buf_hdr_t's private physical data block pointer (b_pabd).
132 *
133 * The L1ARC's data pointer may or may not be uncompressed. The ARC has the
134 * ability to store the physical data (b_pabd) associated with the DVA of the
135 * arc_buf_hdr_t. Since the b_pabd is a copy of the on-disk physical block,
136 * it will match its on-disk compression characteristics. This behavior can be
137 * disabled by setting 'zfs_compressed_arc_enabled' to B_FALSE. When the
138 * compressed ARC functionality is disabled, the b_pabd will point to an
139 * uncompressed version of the on-disk data.
140 *
141 * Data in the L1ARC is not accessed by consumers of the ARC directly. Each
142 * arc_buf_hdr_t can have multiple ARC buffers (arc_buf_t) which reference it.
143 * Each ARC buffer (arc_buf_t) is being actively accessed by a specific ARC
144 * consumer. The ARC will provide references to this data and will keep it
145 * cached until it is no longer in use. The ARC caches only the L1ARC's physical
146 * data block and will evict any arc_buf_t that is no longer referenced. The
147 * amount of memory consumed by the arc_buf_ts' data buffers can be seen via the
148 * "overhead_size" kstat.
149 *
150 * Depending on the consumer, an arc_buf_t can be requested in uncompressed or
151 * compressed form. The typical case is that consumers will want uncompressed
152 * data, and when that happens a new data buffer is allocated where the data is
153 * decompressed for them to use. Currently the only consumer who wants
154 * compressed arc_buf_t's is "zfs send", when it streams data exactly as it
155 * exists on disk. When this happens, the arc_buf_t's data buffer is shared
156 * with the arc_buf_hdr_t.
157 *
158 * Here is a diagram showing an arc_buf_hdr_t referenced by two arc_buf_t's. The
159 * first one is owned by a compressed send consumer (and therefore references
160 * the same compressed data buffer as the arc_buf_hdr_t) and the second could be
161 * used by any other consumer (and has its own uncompressed copy of the data
162 * buffer).
163 *
164 *   arc_buf_hdr_t
165 *   +-----------+
166 *   | fields    |
167 *   | common to |
168 *   | L1- and   |
169 *   | L2ARC     |
170 *   +-----------+
171 *   | l2arc_buf_hdr_t
172 *   |           |
173 *   +-----------+
174 *   | l1arc_buf_hdr_t
175 *   |           |              arc_buf_t
176 *   | b_buf     +------------>+-----------+      arc_buf_t
177 *   | b_pabd    +-+           |b_next     +---->+-----------+
178 *   +-----------+ |           |-----------|     |b_next     +-->NULL
179 *                 |           |b_comp = T |     +-----------+
180 *                 |           |b_data     +-+   |b_comp = F |
181 *                 |           +-----------+ |   |b_data     +-+
182 *                 +->+------+               |   +-----------+ |
183 *        compressed  |      |               |                 |
184 *           data     |      |<--------------+                 | uncompressed
185 *                    +------+          compressed,            |     data
186 *                                        shared               +-->+------+
187 *                                         data                    |      |
188 *                                                                 |      |
189 *                                                                 +------+
190 *
191 * When a consumer reads a block, the ARC must first look to see if the
192 * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new
193 * arc_buf_t and either copies uncompressed data into a new data buffer from an
194 * existing uncompressed arc_buf_t, decompresses the hdr's b_pabd buffer into a
195 * new data buffer, or shares the hdr's b_pabd buffer, depending on whether the
196 * hdr is compressed and the desired compression characteristics of the
197 * arc_buf_t consumer. If the arc_buf_t ends up sharing data with the
198 * arc_buf_hdr_t and both of them are uncompressed then the arc_buf_t must be
199 * the last buffer in the hdr's b_buf list, however a shared compressed buf can
200 * be anywhere in the hdr's list.
201 *
202 * The diagram below shows an example of an uncompressed ARC hdr that is
203 * sharing its data with an arc_buf_t (note that the shared uncompressed buf is
204 * the last element in the buf list):
205 *
206 *                arc_buf_hdr_t
207 *                +-----------+
208 *                |           |
209 *                |           |
210 *                |           |
211 *                +-----------+
212 * l2arc_buf_hdr_t|           |
213 *                |           |
214 *                +-----------+
215 * l1arc_buf_hdr_t|           |
216 *                |           |                 arc_buf_t    (shared)
217 *                |    b_buf  +------------>+---------+      arc_buf_t
218 *                |           |             |b_next   +---->+---------+
219 *                |  b_pabd   +-+           |---------|     |b_next   +-->NULL
220 *                +-----------+ |           |         |     +---------+
221 *                              |           |b_data   +-+   |         |
222 *                              |           +---------+ |   |b_data   +-+
223 *                              +->+------+             |   +---------+ |
224 *                                 |      |             |               |
225 *                   uncompressed  |      |             |               |
226 *                        data     +------+             |               |
227 *                                    ^                 +->+------+     |
228 *                                    |       uncompressed |      |     |
229 *                                    |           data     |      |     |
230 *                                    |                    +------+     |
231 *                                    +---------------------------------+
232 *
233 * Writing to the ARC requires that the ARC first discard the hdr's b_pabd
234 * since the physical block is about to be rewritten. The new data contents
235 * will be contained in the arc_buf_t. As the I/O pipeline performs the write,
236 * it may compress the data before writing it to disk. The ARC will be called
237 * with the transformed data and will bcopy the transformed on-disk block into
238 * a newly allocated b_pabd. Writes are always done into buffers which have
239 * either been loaned (and hence are new and don't have other readers) or
240 * buffers which have been released (and hence have their own hdr, if there
241 * were originally other readers of the buf's original hdr). This ensures that
242 * the ARC only needs to update a single buf and its hdr after a write occurs.
243 *
244 * When the L2ARC is in use, it will also take advantage of the b_pabd. The
245 * L2ARC will always write the contents of b_pabd to the L2ARC. This means
246 * that when compressed ARC is enabled that the L2ARC blocks are identical
247 * to the on-disk block in the main data pool. This provides a significant
248 * advantage since the ARC can leverage the bp's checksum when reading from the
249 * L2ARC to determine if the contents are valid. However, if the compressed
250 * ARC is disabled, then the L2ARC's block must be transformed to look
251 * like the physical block in the main data pool before comparing the
252 * checksum and determining its validity.
253 */
254
255#include <sys/spa.h>
256#include <sys/zio.h>
257#include <sys/spa_impl.h>
258#include <sys/zio_compress.h>
259#include <sys/zio_checksum.h>
260#include <sys/zfs_context.h>
261#include <sys/arc.h>
262#include <sys/refcount.h>
263#include <sys/vdev.h>
264#include <sys/vdev_impl.h>
265#include <sys/dsl_pool.h>
266#include <sys/zio_checksum.h>
267#include <sys/multilist.h>
268#include <sys/abd.h>
269#ifdef _KERNEL
270#include <sys/dnlc.h>
271#include <sys/racct.h>
272#endif
273#include <sys/callb.h>
274#include <sys/kstat.h>
275#include <sys/trim_map.h>
276#include <zfs_fletcher.h>
277#include <sys/sdt.h>
278
279#include <machine/vmparam.h>
280
281#ifdef illumos
282#ifndef _KERNEL
283/* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
284boolean_t arc_watch = B_FALSE;
285int arc_procfd;
286#endif
287#endif /* illumos */
288
289static kmutex_t		arc_reclaim_lock;
290static kcondvar_t	arc_reclaim_thread_cv;
291static boolean_t	arc_reclaim_thread_exit;
292static kcondvar_t	arc_reclaim_waiters_cv;
293
294static kmutex_t		arc_dnlc_evicts_lock;
295static kcondvar_t	arc_dnlc_evicts_cv;
296static boolean_t	arc_dnlc_evicts_thread_exit;
297
298uint_t arc_reduce_dnlc_percent = 3;
299
300/*
301 * The number of headers to evict in arc_evict_state_impl() before
302 * dropping the sublist lock and evicting from another sublist. A lower
303 * value means we're more likely to evict the "correct" header (i.e. the
304 * oldest header in the arc state), but comes with higher overhead
305 * (i.e. more invocations of arc_evict_state_impl()).
306 */
307int zfs_arc_evict_batch_limit = 10;
308
309/* number of seconds before growing cache again */
310static int		arc_grow_retry = 60;
311
312/* shift of arc_c for calculating overflow limit in arc_get_data_impl */
313int		zfs_arc_overflow_shift = 8;
314
315/* shift of arc_c for calculating both min and max arc_p */
316static int		arc_p_min_shift = 4;
317
318/* log2(fraction of arc to reclaim) */
319static int		arc_shrink_shift = 7;
320
321/*
322 * log2(fraction of ARC which must be free to allow growing).
323 * I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
324 * when reading a new block into the ARC, we will evict an equal-sized block
325 * from the ARC.
326 *
327 * This must be less than arc_shrink_shift, so that when we shrink the ARC,
328 * we will still not allow it to grow.
329 */
330int			arc_no_grow_shift = 5;
331
332
333/*
334 * minimum lifespan of a prefetch block in clock ticks
335 * (initialized in arc_init())
336 */
337static int		arc_min_prefetch_lifespan;
338
339/*
340 * If this percent of memory is free, don't throttle.
341 */
342int arc_lotsfree_percent = 10;
343
344static int arc_dead;
345extern boolean_t zfs_prefetch_disable;
346
347/*
348 * The arc has filled available memory and has now warmed up.
349 */
350static boolean_t arc_warm;
351
352/*
353 * These tunables are for performance analysis.
354 */
355uint64_t zfs_arc_max;
356uint64_t zfs_arc_min;
357uint64_t zfs_arc_meta_limit = 0;
358uint64_t zfs_arc_meta_min = 0;
359int zfs_arc_grow_retry = 0;
360int zfs_arc_shrink_shift = 0;
361int zfs_arc_p_min_shift = 0;
362uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
363u_int zfs_arc_free_target = 0;
364
365/* Absolute min for arc min / max is 16MB. */
366static uint64_t arc_abs_min = 16 << 20;
367
368boolean_t zfs_compressed_arc_enabled = B_TRUE;
369
370static int sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS);
371static int sysctl_vfs_zfs_arc_meta_limit(SYSCTL_HANDLER_ARGS);
372static int sysctl_vfs_zfs_arc_max(SYSCTL_HANDLER_ARGS);
373static int sysctl_vfs_zfs_arc_min(SYSCTL_HANDLER_ARGS);
374
375#if defined(__FreeBSD__) && defined(_KERNEL)
376static void
377arc_free_target_init(void *unused __unused)
378{
379
380	zfs_arc_free_target = vm_pageout_wakeup_thresh;
381}
382SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY,
383    arc_free_target_init, NULL);
384
385TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit);
386TUNABLE_QUAD("vfs.zfs.arc_meta_min", &zfs_arc_meta_min);
387TUNABLE_INT("vfs.zfs.arc_shrink_shift", &zfs_arc_shrink_shift);
388SYSCTL_DECL(_vfs_zfs);
389SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_max, CTLTYPE_U64 | CTLFLAG_RWTUN,
390    0, sizeof(uint64_t), sysctl_vfs_zfs_arc_max, "QU", "Maximum ARC size");
391SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_min, CTLTYPE_U64 | CTLFLAG_RWTUN,
392    0, sizeof(uint64_t), sysctl_vfs_zfs_arc_min, "QU", "Minimum ARC size");
393SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_average_blocksize, CTLFLAG_RDTUN,
394    &zfs_arc_average_blocksize, 0,
395    "ARC average blocksize");
396SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_shrink_shift, CTLFLAG_RW,
397    &arc_shrink_shift, 0,
398    "log2(fraction of arc to reclaim)");
399SYSCTL_INT(_vfs_zfs, OID_AUTO, compressed_arc_enabled, CTLFLAG_RDTUN,
400    &zfs_compressed_arc_enabled, 0, "Enable compressed ARC");
401
402/*
403 * We don't have a tunable for arc_free_target due to the dependency on
404 * pagedaemon initialisation.
405 */
406SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_free_target,
407    CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof(u_int),
408    sysctl_vfs_zfs_arc_free_target, "IU",
409    "Desired number of free pages below which ARC triggers reclaim");
410
411static int
412sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS)
413{
414	u_int val;
415	int err;
416
417	val = zfs_arc_free_target;
418	err = sysctl_handle_int(oidp, &val, 0, req);
419	if (err != 0 || req->newptr == NULL)
420		return (err);
421
422	if (val < minfree)
423		return (EINVAL);
424	if (val > vm_cnt.v_page_count)
425		return (EINVAL);
426
427	zfs_arc_free_target = val;
428
429	return (0);
430}
431
432/*
433 * Must be declared here, before the definition of corresponding kstat
434 * macro which uses the same names will confuse the compiler.
435 */
436SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_meta_limit,
437    CTLTYPE_U64 | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof(uint64_t),
438    sysctl_vfs_zfs_arc_meta_limit, "QU",
439    "ARC metadata limit");
440#endif
441
442/*
443 * Note that buffers can be in one of 6 states:
444 *	ARC_anon	- anonymous (discussed below)
445 *	ARC_mru		- recently used, currently cached
446 *	ARC_mru_ghost	- recentely used, no longer in cache
447 *	ARC_mfu		- frequently used, currently cached
448 *	ARC_mfu_ghost	- frequently used, no longer in cache
449 *	ARC_l2c_only	- exists in L2ARC but not other states
450 * When there are no active references to the buffer, they are
451 * are linked onto a list in one of these arc states.  These are
452 * the only buffers that can be evicted or deleted.  Within each
453 * state there are multiple lists, one for meta-data and one for
454 * non-meta-data.  Meta-data (indirect blocks, blocks of dnodes,
455 * etc.) is tracked separately so that it can be managed more
456 * explicitly: favored over data, limited explicitly.
457 *
458 * Anonymous buffers are buffers that are not associated with
459 * a DVA.  These are buffers that hold dirty block copies
460 * before they are written to stable storage.  By definition,
461 * they are "ref'd" and are considered part of arc_mru
462 * that cannot be freed.  Generally, they will aquire a DVA
463 * as they are written and migrate onto the arc_mru list.
464 *
465 * The ARC_l2c_only state is for buffers that are in the second
466 * level ARC but no longer in any of the ARC_m* lists.  The second
467 * level ARC itself may also contain buffers that are in any of
468 * the ARC_m* states - meaning that a buffer can exist in two
469 * places.  The reason for the ARC_l2c_only state is to keep the
470 * buffer header in the hash table, so that reads that hit the
471 * second level ARC benefit from these fast lookups.
472 */
473
474typedef struct arc_state {
475	/*
476	 * list of evictable buffers
477	 */
478	multilist_t *arcs_list[ARC_BUFC_NUMTYPES];
479	/*
480	 * total amount of evictable data in this state
481	 */
482	refcount_t arcs_esize[ARC_BUFC_NUMTYPES];
483	/*
484	 * total amount of data in this state; this includes: evictable,
485	 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA.
486	 */
487	refcount_t arcs_size;
488} arc_state_t;
489
490/* The 6 states: */
491static arc_state_t ARC_anon;
492static arc_state_t ARC_mru;
493static arc_state_t ARC_mru_ghost;
494static arc_state_t ARC_mfu;
495static arc_state_t ARC_mfu_ghost;
496static arc_state_t ARC_l2c_only;
497
498typedef struct arc_stats {
499	kstat_named_t arcstat_hits;
500	kstat_named_t arcstat_misses;
501	kstat_named_t arcstat_demand_data_hits;
502	kstat_named_t arcstat_demand_data_misses;
503	kstat_named_t arcstat_demand_metadata_hits;
504	kstat_named_t arcstat_demand_metadata_misses;
505	kstat_named_t arcstat_prefetch_data_hits;
506	kstat_named_t arcstat_prefetch_data_misses;
507	kstat_named_t arcstat_prefetch_metadata_hits;
508	kstat_named_t arcstat_prefetch_metadata_misses;
509	kstat_named_t arcstat_mru_hits;
510	kstat_named_t arcstat_mru_ghost_hits;
511	kstat_named_t arcstat_mfu_hits;
512	kstat_named_t arcstat_mfu_ghost_hits;
513	kstat_named_t arcstat_allocated;
514	kstat_named_t arcstat_deleted;
515	/*
516	 * Number of buffers that could not be evicted because the hash lock
517	 * was held by another thread.  The lock may not necessarily be held
518	 * by something using the same buffer, since hash locks are shared
519	 * by multiple buffers.
520	 */
521	kstat_named_t arcstat_mutex_miss;
522	/*
523	 * Number of buffers skipped because they have I/O in progress, are
524	 * indrect prefetch buffers that have not lived long enough, or are
525	 * not from the spa we're trying to evict from.
526	 */
527	kstat_named_t arcstat_evict_skip;
528	/*
529	 * Number of times arc_evict_state() was unable to evict enough
530	 * buffers to reach it's target amount.
531	 */
532	kstat_named_t arcstat_evict_not_enough;
533	kstat_named_t arcstat_evict_l2_cached;
534	kstat_named_t arcstat_evict_l2_eligible;
535	kstat_named_t arcstat_evict_l2_ineligible;
536	kstat_named_t arcstat_evict_l2_skip;
537	kstat_named_t arcstat_hash_elements;
538	kstat_named_t arcstat_hash_elements_max;
539	kstat_named_t arcstat_hash_collisions;
540	kstat_named_t arcstat_hash_chains;
541	kstat_named_t arcstat_hash_chain_max;
542	kstat_named_t arcstat_p;
543	kstat_named_t arcstat_c;
544	kstat_named_t arcstat_c_min;
545	kstat_named_t arcstat_c_max;
546	kstat_named_t arcstat_size;
547	/*
548	 * Number of compressed bytes stored in the arc_buf_hdr_t's b_pabd.
549	 * Note that the compressed bytes may match the uncompressed bytes
550	 * if the block is either not compressed or compressed arc is disabled.
551	 */
552	kstat_named_t arcstat_compressed_size;
553	/*
554	 * Uncompressed size of the data stored in b_pabd. If compressed
555	 * arc is disabled then this value will be identical to the stat
556	 * above.
557	 */
558	kstat_named_t arcstat_uncompressed_size;
559	/*
560	 * Number of bytes stored in all the arc_buf_t's. This is classified
561	 * as "overhead" since this data is typically short-lived and will
562	 * be evicted from the arc when it becomes unreferenced unless the
563	 * zfs_keep_uncompressed_metadata or zfs_keep_uncompressed_level
564	 * values have been set (see comment in dbuf.c for more information).
565	 */
566	kstat_named_t arcstat_overhead_size;
567	/*
568	 * Number of bytes consumed by internal ARC structures necessary
569	 * for tracking purposes; these structures are not actually
570	 * backed by ARC buffers. This includes arc_buf_hdr_t structures
571	 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only
572	 * caches), and arc_buf_t structures (allocated via arc_buf_t
573	 * cache).
574	 */
575	kstat_named_t arcstat_hdr_size;
576	/*
577	 * Number of bytes consumed by ARC buffers of type equal to
578	 * ARC_BUFC_DATA. This is generally consumed by buffers backing
579	 * on disk user data (e.g. plain file contents).
580	 */
581	kstat_named_t arcstat_data_size;
582	/*
583	 * Number of bytes consumed by ARC buffers of type equal to
584	 * ARC_BUFC_METADATA. This is generally consumed by buffers
585	 * backing on disk data that is used for internal ZFS
586	 * structures (e.g. ZAP, dnode, indirect blocks, etc).
587	 */
588	kstat_named_t arcstat_metadata_size;
589	/*
590	 * Number of bytes consumed by various buffers and structures
591	 * not actually backed with ARC buffers. This includes bonus
592	 * buffers (allocated directly via zio_buf_* functions),
593	 * dmu_buf_impl_t structures (allocated via dmu_buf_impl_t
594	 * cache), and dnode_t structures (allocated via dnode_t cache).
595	 */
596	kstat_named_t arcstat_other_size;
597	/*
598	 * Total number of bytes consumed by ARC buffers residing in the
599	 * arc_anon state. This includes *all* buffers in the arc_anon
600	 * state; e.g. data, metadata, evictable, and unevictable buffers
601	 * are all included in this value.
602	 */
603	kstat_named_t arcstat_anon_size;
604	/*
605	 * Number of bytes consumed by ARC buffers that meet the
606	 * following criteria: backing buffers of type ARC_BUFC_DATA,
607	 * residing in the arc_anon state, and are eligible for eviction
608	 * (e.g. have no outstanding holds on the buffer).
609	 */
610	kstat_named_t arcstat_anon_evictable_data;
611	/*
612	 * Number of bytes consumed by ARC buffers that meet the
613	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
614	 * residing in the arc_anon state, and are eligible for eviction
615	 * (e.g. have no outstanding holds on the buffer).
616	 */
617	kstat_named_t arcstat_anon_evictable_metadata;
618	/*
619	 * Total number of bytes consumed by ARC buffers residing in the
620	 * arc_mru state. This includes *all* buffers in the arc_mru
621	 * state; e.g. data, metadata, evictable, and unevictable buffers
622	 * are all included in this value.
623	 */
624	kstat_named_t arcstat_mru_size;
625	/*
626	 * Number of bytes consumed by ARC buffers that meet the
627	 * following criteria: backing buffers of type ARC_BUFC_DATA,
628	 * residing in the arc_mru state, and are eligible for eviction
629	 * (e.g. have no outstanding holds on the buffer).
630	 */
631	kstat_named_t arcstat_mru_evictable_data;
632	/*
633	 * Number of bytes consumed by ARC buffers that meet the
634	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
635	 * residing in the arc_mru state, and are eligible for eviction
636	 * (e.g. have no outstanding holds on the buffer).
637	 */
638	kstat_named_t arcstat_mru_evictable_metadata;
639	/*
640	 * Total number of bytes that *would have been* consumed by ARC
641	 * buffers in the arc_mru_ghost state. The key thing to note
642	 * here, is the fact that this size doesn't actually indicate
643	 * RAM consumption. The ghost lists only consist of headers and
644	 * don't actually have ARC buffers linked off of these headers.
645	 * Thus, *if* the headers had associated ARC buffers, these
646	 * buffers *would have* consumed this number of bytes.
647	 */
648	kstat_named_t arcstat_mru_ghost_size;
649	/*
650	 * Number of bytes that *would have been* consumed by ARC
651	 * buffers that are eligible for eviction, of type
652	 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state.
653	 */
654	kstat_named_t arcstat_mru_ghost_evictable_data;
655	/*
656	 * Number of bytes that *would have been* consumed by ARC
657	 * buffers that are eligible for eviction, of type
658	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
659	 */
660	kstat_named_t arcstat_mru_ghost_evictable_metadata;
661	/*
662	 * Total number of bytes consumed by ARC buffers residing in the
663	 * arc_mfu state. This includes *all* buffers in the arc_mfu
664	 * state; e.g. data, metadata, evictable, and unevictable buffers
665	 * are all included in this value.
666	 */
667	kstat_named_t arcstat_mfu_size;
668	/*
669	 * Number of bytes consumed by ARC buffers that are eligible for
670	 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu
671	 * state.
672	 */
673	kstat_named_t arcstat_mfu_evictable_data;
674	/*
675	 * Number of bytes consumed by ARC buffers that are eligible for
676	 * eviction, of type ARC_BUFC_METADATA, and reside in the
677	 * arc_mfu state.
678	 */
679	kstat_named_t arcstat_mfu_evictable_metadata;
680	/*
681	 * Total number of bytes that *would have been* consumed by ARC
682	 * buffers in the arc_mfu_ghost state. See the comment above
683	 * arcstat_mru_ghost_size for more details.
684	 */
685	kstat_named_t arcstat_mfu_ghost_size;
686	/*
687	 * Number of bytes that *would have been* consumed by ARC
688	 * buffers that are eligible for eviction, of type
689	 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state.
690	 */
691	kstat_named_t arcstat_mfu_ghost_evictable_data;
692	/*
693	 * Number of bytes that *would have been* consumed by ARC
694	 * buffers that are eligible for eviction, of type
695	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
696	 */
697	kstat_named_t arcstat_mfu_ghost_evictable_metadata;
698	kstat_named_t arcstat_l2_hits;
699	kstat_named_t arcstat_l2_misses;
700	kstat_named_t arcstat_l2_feeds;
701	kstat_named_t arcstat_l2_rw_clash;
702	kstat_named_t arcstat_l2_read_bytes;
703	kstat_named_t arcstat_l2_write_bytes;
704	kstat_named_t arcstat_l2_writes_sent;
705	kstat_named_t arcstat_l2_writes_done;
706	kstat_named_t arcstat_l2_writes_error;
707	kstat_named_t arcstat_l2_writes_lock_retry;
708	kstat_named_t arcstat_l2_evict_lock_retry;
709	kstat_named_t arcstat_l2_evict_reading;
710	kstat_named_t arcstat_l2_evict_l1cached;
711	kstat_named_t arcstat_l2_free_on_write;
712	kstat_named_t arcstat_l2_abort_lowmem;
713	kstat_named_t arcstat_l2_cksum_bad;
714	kstat_named_t arcstat_l2_io_error;
715	kstat_named_t arcstat_l2_size;
716	kstat_named_t arcstat_l2_asize;
717	kstat_named_t arcstat_l2_hdr_size;
718	kstat_named_t arcstat_l2_write_trylock_fail;
719	kstat_named_t arcstat_l2_write_passed_headroom;
720	kstat_named_t arcstat_l2_write_spa_mismatch;
721	kstat_named_t arcstat_l2_write_in_l2;
722	kstat_named_t arcstat_l2_write_hdr_io_in_progress;
723	kstat_named_t arcstat_l2_write_not_cacheable;
724	kstat_named_t arcstat_l2_write_full;
725	kstat_named_t arcstat_l2_write_buffer_iter;
726	kstat_named_t arcstat_l2_write_pios;
727	kstat_named_t arcstat_l2_write_buffer_bytes_scanned;
728	kstat_named_t arcstat_l2_write_buffer_list_iter;
729	kstat_named_t arcstat_l2_write_buffer_list_null_iter;
730	kstat_named_t arcstat_memory_throttle_count;
731	kstat_named_t arcstat_meta_used;
732	kstat_named_t arcstat_meta_limit;
733	kstat_named_t arcstat_meta_max;
734	kstat_named_t arcstat_meta_min;
735	kstat_named_t arcstat_sync_wait_for_async;
736	kstat_named_t arcstat_demand_hit_predictive_prefetch;
737} arc_stats_t;
738
739static arc_stats_t arc_stats = {
740	{ "hits",			KSTAT_DATA_UINT64 },
741	{ "misses",			KSTAT_DATA_UINT64 },
742	{ "demand_data_hits",		KSTAT_DATA_UINT64 },
743	{ "demand_data_misses",		KSTAT_DATA_UINT64 },
744	{ "demand_metadata_hits",	KSTAT_DATA_UINT64 },
745	{ "demand_metadata_misses",	KSTAT_DATA_UINT64 },
746	{ "prefetch_data_hits",		KSTAT_DATA_UINT64 },
747	{ "prefetch_data_misses",	KSTAT_DATA_UINT64 },
748	{ "prefetch_metadata_hits",	KSTAT_DATA_UINT64 },
749	{ "prefetch_metadata_misses",	KSTAT_DATA_UINT64 },
750	{ "mru_hits",			KSTAT_DATA_UINT64 },
751	{ "mru_ghost_hits",		KSTAT_DATA_UINT64 },
752	{ "mfu_hits",			KSTAT_DATA_UINT64 },
753	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
754	{ "allocated",			KSTAT_DATA_UINT64 },
755	{ "deleted",			KSTAT_DATA_UINT64 },
756	{ "mutex_miss",			KSTAT_DATA_UINT64 },
757	{ "evict_skip",			KSTAT_DATA_UINT64 },
758	{ "evict_not_enough",		KSTAT_DATA_UINT64 },
759	{ "evict_l2_cached",		KSTAT_DATA_UINT64 },
760	{ "evict_l2_eligible",		KSTAT_DATA_UINT64 },
761	{ "evict_l2_ineligible",	KSTAT_DATA_UINT64 },
762	{ "evict_l2_skip",		KSTAT_DATA_UINT64 },
763	{ "hash_elements",		KSTAT_DATA_UINT64 },
764	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
765	{ "hash_collisions",		KSTAT_DATA_UINT64 },
766	{ "hash_chains",		KSTAT_DATA_UINT64 },
767	{ "hash_chain_max",		KSTAT_DATA_UINT64 },
768	{ "p",				KSTAT_DATA_UINT64 },
769	{ "c",				KSTAT_DATA_UINT64 },
770	{ "c_min",			KSTAT_DATA_UINT64 },
771	{ "c_max",			KSTAT_DATA_UINT64 },
772	{ "size",			KSTAT_DATA_UINT64 },
773	{ "compressed_size",		KSTAT_DATA_UINT64 },
774	{ "uncompressed_size",		KSTAT_DATA_UINT64 },
775	{ "overhead_size",		KSTAT_DATA_UINT64 },
776	{ "hdr_size",			KSTAT_DATA_UINT64 },
777	{ "data_size",			KSTAT_DATA_UINT64 },
778	{ "metadata_size",		KSTAT_DATA_UINT64 },
779	{ "other_size",			KSTAT_DATA_UINT64 },
780	{ "anon_size",			KSTAT_DATA_UINT64 },
781	{ "anon_evictable_data",	KSTAT_DATA_UINT64 },
782	{ "anon_evictable_metadata",	KSTAT_DATA_UINT64 },
783	{ "mru_size",			KSTAT_DATA_UINT64 },
784	{ "mru_evictable_data",		KSTAT_DATA_UINT64 },
785	{ "mru_evictable_metadata",	KSTAT_DATA_UINT64 },
786	{ "mru_ghost_size",		KSTAT_DATA_UINT64 },
787	{ "mru_ghost_evictable_data",	KSTAT_DATA_UINT64 },
788	{ "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
789	{ "mfu_size",			KSTAT_DATA_UINT64 },
790	{ "mfu_evictable_data",		KSTAT_DATA_UINT64 },
791	{ "mfu_evictable_metadata",	KSTAT_DATA_UINT64 },
792	{ "mfu_ghost_size",		KSTAT_DATA_UINT64 },
793	{ "mfu_ghost_evictable_data",	KSTAT_DATA_UINT64 },
794	{ "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
795	{ "l2_hits",			KSTAT_DATA_UINT64 },
796	{ "l2_misses",			KSTAT_DATA_UINT64 },
797	{ "l2_feeds",			KSTAT_DATA_UINT64 },
798	{ "l2_rw_clash",		KSTAT_DATA_UINT64 },
799	{ "l2_read_bytes",		KSTAT_DATA_UINT64 },
800	{ "l2_write_bytes",		KSTAT_DATA_UINT64 },
801	{ "l2_writes_sent",		KSTAT_DATA_UINT64 },
802	{ "l2_writes_done",		KSTAT_DATA_UINT64 },
803	{ "l2_writes_error",		KSTAT_DATA_UINT64 },
804	{ "l2_writes_lock_retry",	KSTAT_DATA_UINT64 },
805	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
806	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
807	{ "l2_evict_l1cached",		KSTAT_DATA_UINT64 },
808	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
809	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
810	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
811	{ "l2_io_error",		KSTAT_DATA_UINT64 },
812	{ "l2_size",			KSTAT_DATA_UINT64 },
813	{ "l2_asize",			KSTAT_DATA_UINT64 },
814	{ "l2_hdr_size",		KSTAT_DATA_UINT64 },
815	{ "l2_write_trylock_fail",	KSTAT_DATA_UINT64 },
816	{ "l2_write_passed_headroom",	KSTAT_DATA_UINT64 },
817	{ "l2_write_spa_mismatch",	KSTAT_DATA_UINT64 },
818	{ "l2_write_in_l2",		KSTAT_DATA_UINT64 },
819	{ "l2_write_io_in_progress",	KSTAT_DATA_UINT64 },
820	{ "l2_write_not_cacheable",	KSTAT_DATA_UINT64 },
821	{ "l2_write_full",		KSTAT_DATA_UINT64 },
822	{ "l2_write_buffer_iter",	KSTAT_DATA_UINT64 },
823	{ "l2_write_pios",		KSTAT_DATA_UINT64 },
824	{ "l2_write_buffer_bytes_scanned", KSTAT_DATA_UINT64 },
825	{ "l2_write_buffer_list_iter",	KSTAT_DATA_UINT64 },
826	{ "l2_write_buffer_list_null_iter", KSTAT_DATA_UINT64 },
827	{ "memory_throttle_count",	KSTAT_DATA_UINT64 },
828	{ "arc_meta_used",		KSTAT_DATA_UINT64 },
829	{ "arc_meta_limit",		KSTAT_DATA_UINT64 },
830	{ "arc_meta_max",		KSTAT_DATA_UINT64 },
831	{ "arc_meta_min",		KSTAT_DATA_UINT64 },
832	{ "sync_wait_for_async",	KSTAT_DATA_UINT64 },
833	{ "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 },
834};
835
836#define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
837
838#define	ARCSTAT_INCR(stat, val) \
839	atomic_add_64(&arc_stats.stat.value.ui64, (val))
840
841#define	ARCSTAT_BUMP(stat)	ARCSTAT_INCR(stat, 1)
842#define	ARCSTAT_BUMPDOWN(stat)	ARCSTAT_INCR(stat, -1)
843
844#define	ARCSTAT_MAX(stat, val) {					\
845	uint64_t m;							\
846	while ((val) > (m = arc_stats.stat.value.ui64) &&		\
847	    (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))	\
848		continue;						\
849}
850
851#define	ARCSTAT_MAXSTAT(stat) \
852	ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
853
854/*
855 * We define a macro to allow ARC hits/misses to be easily broken down by
856 * two separate conditions, giving a total of four different subtypes for
857 * each of hits and misses (so eight statistics total).
858 */
859#define	ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
860	if (cond1) {							\
861		if (cond2) {						\
862			ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
863		} else {						\
864			ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
865		}							\
866	} else {							\
867		if (cond2) {						\
868			ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
869		} else {						\
870			ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
871		}							\
872	}
873
874kstat_t			*arc_ksp;
875static arc_state_t	*arc_anon;
876static arc_state_t	*arc_mru;
877static arc_state_t	*arc_mru_ghost;
878static arc_state_t	*arc_mfu;
879static arc_state_t	*arc_mfu_ghost;
880static arc_state_t	*arc_l2c_only;
881
882/*
883 * There are several ARC variables that are critical to export as kstats --
884 * but we don't want to have to grovel around in the kstat whenever we wish to
885 * manipulate them.  For these variables, we therefore define them to be in
886 * terms of the statistic variable.  This assures that we are not introducing
887 * the possibility of inconsistency by having shadow copies of the variables,
888 * while still allowing the code to be readable.
889 */
890#define	arc_size	ARCSTAT(arcstat_size)	/* actual total arc size */
891#define	arc_p		ARCSTAT(arcstat_p)	/* target size of MRU */
892#define	arc_c		ARCSTAT(arcstat_c)	/* target size of cache */
893#define	arc_c_min	ARCSTAT(arcstat_c_min)	/* min target cache size */
894#define	arc_c_max	ARCSTAT(arcstat_c_max)	/* max target cache size */
895#define	arc_meta_limit	ARCSTAT(arcstat_meta_limit) /* max size for metadata */
896#define	arc_meta_min	ARCSTAT(arcstat_meta_min) /* min size for metadata */
897#define	arc_meta_used	ARCSTAT(arcstat_meta_used) /* size of metadata */
898#define	arc_meta_max	ARCSTAT(arcstat_meta_max) /* max size of metadata */
899
900/* compressed size of entire arc */
901#define	arc_compressed_size	ARCSTAT(arcstat_compressed_size)
902/* uncompressed size of entire arc */
903#define	arc_uncompressed_size	ARCSTAT(arcstat_uncompressed_size)
904/* number of bytes in the arc from arc_buf_t's */
905#define	arc_overhead_size	ARCSTAT(arcstat_overhead_size)
906
907static int		arc_no_grow;	/* Don't try to grow cache size */
908static uint64_t		arc_tempreserve;
909static uint64_t		arc_loaned_bytes;
910
911typedef struct arc_callback arc_callback_t;
912
913struct arc_callback {
914	void			*acb_private;
915	arc_done_func_t		*acb_done;
916	arc_buf_t		*acb_buf;
917	boolean_t		acb_compressed;
918	zio_t			*acb_zio_dummy;
919	arc_callback_t		*acb_next;
920};
921
922typedef struct arc_write_callback arc_write_callback_t;
923
924struct arc_write_callback {
925	void		*awcb_private;
926	arc_done_func_t	*awcb_ready;
927	arc_done_func_t	*awcb_children_ready;
928	arc_done_func_t	*awcb_physdone;
929	arc_done_func_t	*awcb_done;
930	arc_buf_t	*awcb_buf;
931};
932
933/*
934 * ARC buffers are separated into multiple structs as a memory saving measure:
935 *   - Common fields struct, always defined, and embedded within it:
936 *       - L2-only fields, always allocated but undefined when not in L2ARC
937 *       - L1-only fields, only allocated when in L1ARC
938 *
939 *           Buffer in L1                     Buffer only in L2
940 *    +------------------------+          +------------------------+
941 *    | arc_buf_hdr_t          |          | arc_buf_hdr_t          |
942 *    |                        |          |                        |
943 *    |                        |          |                        |
944 *    |                        |          |                        |
945 *    +------------------------+          +------------------------+
946 *    | l2arc_buf_hdr_t        |          | l2arc_buf_hdr_t        |
947 *    | (undefined if L1-only) |          |                        |
948 *    +------------------------+          +------------------------+
949 *    | l1arc_buf_hdr_t        |
950 *    |                        |
951 *    |                        |
952 *    |                        |
953 *    |                        |
954 *    +------------------------+
955 *
956 * Because it's possible for the L2ARC to become extremely large, we can wind
957 * up eating a lot of memory in L2ARC buffer headers, so the size of a header
958 * is minimized by only allocating the fields necessary for an L1-cached buffer
959 * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and
960 * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple
961 * words in pointers. arc_hdr_realloc() is used to switch a header between
962 * these two allocation states.
963 */
964typedef struct l1arc_buf_hdr {
965	kmutex_t		b_freeze_lock;
966	zio_cksum_t		*b_freeze_cksum;
967#ifdef ZFS_DEBUG
968	/*
969	 * Used for debugging with kmem_flags - by allocating and freeing
970	 * b_thawed when the buffer is thawed, we get a record of the stack
971	 * trace that thawed it.
972	 */
973	void			*b_thawed;
974#endif
975
976	arc_buf_t		*b_buf;
977	uint32_t		b_bufcnt;
978	/* for waiting on writes to complete */
979	kcondvar_t		b_cv;
980	uint8_t			b_byteswap;
981
982	/* protected by arc state mutex */
983	arc_state_t		*b_state;
984	multilist_node_t	b_arc_node;
985
986	/* updated atomically */
987	clock_t			b_arc_access;
988
989	/* self protecting */
990	refcount_t		b_refcnt;
991
992	arc_callback_t		*b_acb;
993	abd_t			*b_pabd;
994} l1arc_buf_hdr_t;
995
996typedef struct l2arc_dev l2arc_dev_t;
997
998typedef struct l2arc_buf_hdr {
999	/* protected by arc_buf_hdr mutex */
1000	l2arc_dev_t		*b_dev;		/* L2ARC device */
1001	uint64_t		b_daddr;	/* disk address, offset byte */
1002
1003	list_node_t		b_l2node;
1004} l2arc_buf_hdr_t;
1005
1006struct arc_buf_hdr {
1007	/* protected by hash lock */
1008	dva_t			b_dva;
1009	uint64_t		b_birth;
1010
1011	arc_buf_contents_t	b_type;
1012	arc_buf_hdr_t		*b_hash_next;
1013	arc_flags_t		b_flags;
1014
1015	/*
1016	 * This field stores the size of the data buffer after
1017	 * compression, and is set in the arc's zio completion handlers.
1018	 * It is in units of SPA_MINBLOCKSIZE (e.g. 1 == 512 bytes).
1019	 *
1020	 * While the block pointers can store up to 32MB in their psize
1021	 * field, we can only store up to 32MB minus 512B. This is due
1022	 * to the bp using a bias of 1, whereas we use a bias of 0 (i.e.
1023	 * a field of zeros represents 512B in the bp). We can't use a
1024	 * bias of 1 since we need to reserve a psize of zero, here, to
1025	 * represent holes and embedded blocks.
1026	 *
1027	 * This isn't a problem in practice, since the maximum size of a
1028	 * buffer is limited to 16MB, so we never need to store 32MB in
1029	 * this field. Even in the upstream illumos code base, the
1030	 * maximum size of a buffer is limited to 16MB.
1031	 */
1032	uint16_t		b_psize;
1033
1034	/*
1035	 * This field stores the size of the data buffer before
1036	 * compression, and cannot change once set. It is in units
1037	 * of SPA_MINBLOCKSIZE (e.g. 2 == 1024 bytes)
1038	 */
1039	uint16_t		b_lsize;	/* immutable */
1040	uint64_t		b_spa;		/* immutable */
1041
1042	/* L2ARC fields. Undefined when not in L2ARC. */
1043	l2arc_buf_hdr_t		b_l2hdr;
1044	/* L1ARC fields. Undefined when in l2arc_only state */
1045	l1arc_buf_hdr_t		b_l1hdr;
1046};
1047
1048#if defined(__FreeBSD__) && defined(_KERNEL)
1049static int
1050sysctl_vfs_zfs_arc_meta_limit(SYSCTL_HANDLER_ARGS)
1051{
1052	uint64_t val;
1053	int err;
1054
1055	val = arc_meta_limit;
1056	err = sysctl_handle_64(oidp, &val, 0, req);
1057	if (err != 0 || req->newptr == NULL)
1058		return (err);
1059
1060        if (val <= 0 || val > arc_c_max)
1061		return (EINVAL);
1062
1063	arc_meta_limit = val;
1064	return (0);
1065}
1066
1067static int
1068sysctl_vfs_zfs_arc_max(SYSCTL_HANDLER_ARGS)
1069{
1070	uint64_t val;
1071	int err;
1072
1073	val = zfs_arc_max;
1074	err = sysctl_handle_64(oidp, &val, 0, req);
1075	if (err != 0 || req->newptr == NULL)
1076		return (err);
1077
1078	if (zfs_arc_max == 0) {
1079		/* Loader tunable so blindly set */
1080		zfs_arc_max = val;
1081		return (0);
1082	}
1083
1084	if (val < arc_abs_min || val > kmem_size())
1085		return (EINVAL);
1086	if (val < arc_c_min)
1087		return (EINVAL);
1088	if (zfs_arc_meta_limit > 0 && val < zfs_arc_meta_limit)
1089		return (EINVAL);
1090
1091	arc_c_max = val;
1092
1093	arc_c = arc_c_max;
1094        arc_p = (arc_c >> 1);
1095
1096	if (zfs_arc_meta_limit == 0) {
1097		/* limit meta-data to 1/4 of the arc capacity */
1098		arc_meta_limit = arc_c_max / 4;
1099	}
1100
1101	/* if kmem_flags are set, lets try to use less memory */
1102	if (kmem_debugging())
1103		arc_c = arc_c / 2;
1104
1105	zfs_arc_max = arc_c;
1106
1107	return (0);
1108}
1109
1110static int
1111sysctl_vfs_zfs_arc_min(SYSCTL_HANDLER_ARGS)
1112{
1113	uint64_t val;
1114	int err;
1115
1116	val = zfs_arc_min;
1117	err = sysctl_handle_64(oidp, &val, 0, req);
1118	if (err != 0 || req->newptr == NULL)
1119		return (err);
1120
1121	if (zfs_arc_min == 0) {
1122		/* Loader tunable so blindly set */
1123		zfs_arc_min = val;
1124		return (0);
1125	}
1126
1127	if (val < arc_abs_min || val > arc_c_max)
1128		return (EINVAL);
1129
1130	arc_c_min = val;
1131
1132	if (zfs_arc_meta_min == 0)
1133                arc_meta_min = arc_c_min / 2;
1134
1135	if (arc_c < arc_c_min)
1136                arc_c = arc_c_min;
1137
1138	zfs_arc_min = arc_c_min;
1139
1140	return (0);
1141}
1142#endif
1143
1144#define	GHOST_STATE(state)	\
1145	((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||	\
1146	(state) == arc_l2c_only)
1147
1148#define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
1149#define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
1150#define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_FLAG_IO_ERROR)
1151#define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_FLAG_PREFETCH)
1152#define	HDR_COMPRESSION_ENABLED(hdr)	\
1153	((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
1154
1155#define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_FLAG_L2CACHE)
1156#define	HDR_L2_READING(hdr)	\
1157	(((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) &&	\
1158	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
1159#define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITING)
1160#define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
1161#define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
1162#define	HDR_SHARED_DATA(hdr)	((hdr)->b_flags & ARC_FLAG_SHARED_DATA)
1163
1164#define	HDR_ISTYPE_METADATA(hdr)	\
1165	((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
1166#define	HDR_ISTYPE_DATA(hdr)	(!HDR_ISTYPE_METADATA(hdr))
1167
1168#define	HDR_HAS_L1HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
1169#define	HDR_HAS_L2HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
1170
1171/* For storing compression mode in b_flags */
1172#define	HDR_COMPRESS_OFFSET	(highbit64(ARC_FLAG_COMPRESS_0) - 1)
1173
1174#define	HDR_GET_COMPRESS(hdr)	((enum zio_compress)BF32_GET((hdr)->b_flags, \
1175	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS))
1176#define	HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \
1177	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS, (cmp));
1178
1179#define	ARC_BUF_LAST(buf)	((buf)->b_next == NULL)
1180#define	ARC_BUF_SHARED(buf)	((buf)->b_flags & ARC_BUF_FLAG_SHARED)
1181#define	ARC_BUF_COMPRESSED(buf)	((buf)->b_flags & ARC_BUF_FLAG_COMPRESSED)
1182
1183/*
1184 * Other sizes
1185 */
1186
1187#define	HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
1188#define	HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
1189
1190/*
1191 * Hash table routines
1192 */
1193
1194#define	HT_LOCK_PAD	CACHE_LINE_SIZE
1195
1196struct ht_lock {
1197	kmutex_t	ht_lock;
1198#ifdef _KERNEL
1199	unsigned char	pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
1200#endif
1201};
1202
1203#define	BUF_LOCKS 256
1204typedef struct buf_hash_table {
1205	uint64_t ht_mask;
1206	arc_buf_hdr_t **ht_table;
1207	struct ht_lock ht_locks[BUF_LOCKS] __aligned(CACHE_LINE_SIZE);
1208} buf_hash_table_t;
1209
1210static buf_hash_table_t buf_hash_table;
1211
1212#define	BUF_HASH_INDEX(spa, dva, birth) \
1213	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
1214#define	BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
1215#define	BUF_HASH_LOCK(idx)	(&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
1216#define	HDR_LOCK(hdr) \
1217	(BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
1218
1219uint64_t zfs_crc64_table[256];
1220
1221/*
1222 * Level 2 ARC
1223 */
1224
1225#define	L2ARC_WRITE_SIZE	(8 * 1024 * 1024)	/* initial write max */
1226#define	L2ARC_HEADROOM		2			/* num of writes */
1227/*
1228 * If we discover during ARC scan any buffers to be compressed, we boost
1229 * our headroom for the next scanning cycle by this percentage multiple.
1230 */
1231#define	L2ARC_HEADROOM_BOOST	200
1232#define	L2ARC_FEED_SECS		1		/* caching interval secs */
1233#define	L2ARC_FEED_MIN_MS	200		/* min caching interval ms */
1234
1235#define	l2arc_writes_sent	ARCSTAT(arcstat_l2_writes_sent)
1236#define	l2arc_writes_done	ARCSTAT(arcstat_l2_writes_done)
1237
1238/* L2ARC Performance Tunables */
1239uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;	/* default max write size */
1240uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;	/* extra write during warmup */
1241uint64_t l2arc_headroom = L2ARC_HEADROOM;	/* number of dev writes */
1242uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
1243uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;	/* interval seconds */
1244uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS;	/* min interval milliseconds */
1245boolean_t l2arc_noprefetch = B_TRUE;		/* don't cache prefetch bufs */
1246boolean_t l2arc_feed_again = B_TRUE;		/* turbo warmup */
1247boolean_t l2arc_norw = B_TRUE;			/* no reads during writes */
1248
1249SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_write_max, CTLFLAG_RW,
1250    &l2arc_write_max, 0, "max write size");
1251SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_write_boost, CTLFLAG_RW,
1252    &l2arc_write_boost, 0, "extra write during warmup");
1253SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_headroom, CTLFLAG_RW,
1254    &l2arc_headroom, 0, "number of dev writes");
1255SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_feed_secs, CTLFLAG_RW,
1256    &l2arc_feed_secs, 0, "interval seconds");
1257SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_feed_min_ms, CTLFLAG_RW,
1258    &l2arc_feed_min_ms, 0, "min interval milliseconds");
1259
1260SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_noprefetch, CTLFLAG_RW,
1261    &l2arc_noprefetch, 0, "don't cache prefetch bufs");
1262SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_feed_again, CTLFLAG_RW,
1263    &l2arc_feed_again, 0, "turbo warmup");
1264SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_norw, CTLFLAG_RW,
1265    &l2arc_norw, 0, "no reads during writes");
1266
1267SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_size, CTLFLAG_RD,
1268    &ARC_anon.arcs_size.rc_count, 0, "size of anonymous state");
1269SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_metadata_esize, CTLFLAG_RD,
1270    &ARC_anon.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1271    "size of anonymous state");
1272SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_data_esize, CTLFLAG_RD,
1273    &ARC_anon.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1274    "size of anonymous state");
1275
1276SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_size, CTLFLAG_RD,
1277    &ARC_mru.arcs_size.rc_count, 0, "size of mru state");
1278SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_metadata_esize, CTLFLAG_RD,
1279    &ARC_mru.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1280    "size of metadata in mru state");
1281SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_data_esize, CTLFLAG_RD,
1282    &ARC_mru.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1283    "size of data in mru state");
1284
1285SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_size, CTLFLAG_RD,
1286    &ARC_mru_ghost.arcs_size.rc_count, 0, "size of mru ghost state");
1287SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_metadata_esize, CTLFLAG_RD,
1288    &ARC_mru_ghost.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1289    "size of metadata in mru ghost state");
1290SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_data_esize, CTLFLAG_RD,
1291    &ARC_mru_ghost.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1292    "size of data in mru ghost state");
1293
1294SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_size, CTLFLAG_RD,
1295    &ARC_mfu.arcs_size.rc_count, 0, "size of mfu state");
1296SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_metadata_esize, CTLFLAG_RD,
1297    &ARC_mfu.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1298    "size of metadata in mfu state");
1299SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_data_esize, CTLFLAG_RD,
1300    &ARC_mfu.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1301    "size of data in mfu state");
1302
1303SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_size, CTLFLAG_RD,
1304    &ARC_mfu_ghost.arcs_size.rc_count, 0, "size of mfu ghost state");
1305SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_metadata_esize, CTLFLAG_RD,
1306    &ARC_mfu_ghost.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1307    "size of metadata in mfu ghost state");
1308SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_data_esize, CTLFLAG_RD,
1309    &ARC_mfu_ghost.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1310    "size of data in mfu ghost state");
1311
1312SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2c_only_size, CTLFLAG_RD,
1313    &ARC_l2c_only.arcs_size.rc_count, 0, "size of mru state");
1314
1315/*
1316 * L2ARC Internals
1317 */
1318struct l2arc_dev {
1319	vdev_t			*l2ad_vdev;	/* vdev */
1320	spa_t			*l2ad_spa;	/* spa */
1321	uint64_t		l2ad_hand;	/* next write location */
1322	uint64_t		l2ad_start;	/* first addr on device */
1323	uint64_t		l2ad_end;	/* last addr on device */
1324	boolean_t		l2ad_first;	/* first sweep through */
1325	boolean_t		l2ad_writing;	/* currently writing */
1326	kmutex_t		l2ad_mtx;	/* lock for buffer list */
1327	list_t			l2ad_buflist;	/* buffer list */
1328	list_node_t		l2ad_node;	/* device list node */
1329	refcount_t		l2ad_alloc;	/* allocated bytes */
1330};
1331
1332static list_t L2ARC_dev_list;			/* device list */
1333static list_t *l2arc_dev_list;			/* device list pointer */
1334static kmutex_t l2arc_dev_mtx;			/* device list mutex */
1335static l2arc_dev_t *l2arc_dev_last;		/* last device used */
1336static list_t L2ARC_free_on_write;		/* free after write buf list */
1337static list_t *l2arc_free_on_write;		/* free after write list ptr */
1338static kmutex_t l2arc_free_on_write_mtx;	/* mutex for list */
1339static uint64_t l2arc_ndev;			/* number of devices */
1340
1341typedef struct l2arc_read_callback {
1342	arc_buf_hdr_t		*l2rcb_hdr;		/* read header */
1343	blkptr_t		l2rcb_bp;		/* original blkptr */
1344	zbookmark_phys_t	l2rcb_zb;		/* original bookmark */
1345	int			l2rcb_flags;		/* original flags */
1346	abd_t			*l2rcb_abd;		/* temporary buffer */
1347} l2arc_read_callback_t;
1348
1349typedef struct l2arc_write_callback {
1350	l2arc_dev_t	*l2wcb_dev;		/* device info */
1351	arc_buf_hdr_t	*l2wcb_head;		/* head of write buflist */
1352} l2arc_write_callback_t;
1353
1354typedef struct l2arc_data_free {
1355	/* protected by l2arc_free_on_write_mtx */
1356	abd_t		*l2df_abd;
1357	size_t		l2df_size;
1358	arc_buf_contents_t l2df_type;
1359	list_node_t	l2df_list_node;
1360} l2arc_data_free_t;
1361
1362static kmutex_t l2arc_feed_thr_lock;
1363static kcondvar_t l2arc_feed_thr_cv;
1364static uint8_t l2arc_thread_exit;
1365
1366static abd_t *arc_get_data_abd(arc_buf_hdr_t *, uint64_t, void *);
1367static void *arc_get_data_buf(arc_buf_hdr_t *, uint64_t, void *);
1368static void arc_get_data_impl(arc_buf_hdr_t *, uint64_t, void *);
1369static void arc_free_data_abd(arc_buf_hdr_t *, abd_t *, uint64_t, void *);
1370static void arc_free_data_buf(arc_buf_hdr_t *, void *, uint64_t, void *);
1371static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag);
1372static void arc_hdr_free_pabd(arc_buf_hdr_t *);
1373static void arc_hdr_alloc_pabd(arc_buf_hdr_t *);
1374static void arc_access(arc_buf_hdr_t *, kmutex_t *);
1375static boolean_t arc_is_overflowing();
1376static void arc_buf_watch(arc_buf_t *);
1377
1378static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
1379static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
1380static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
1381static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
1382
1383static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
1384static void l2arc_read_done(zio_t *);
1385
1386static void
1387l2arc_trim(const arc_buf_hdr_t *hdr)
1388{
1389	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
1390
1391	ASSERT(HDR_HAS_L2HDR(hdr));
1392	ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
1393
1394	if (HDR_GET_PSIZE(hdr) != 0) {
1395		trim_map_free(dev->l2ad_vdev, hdr->b_l2hdr.b_daddr,
1396		    HDR_GET_PSIZE(hdr), 0);
1397	}
1398}
1399
1400static uint64_t
1401buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
1402{
1403	uint8_t *vdva = (uint8_t *)dva;
1404	uint64_t crc = -1ULL;
1405	int i;
1406
1407	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
1408
1409	for (i = 0; i < sizeof (dva_t); i++)
1410		crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
1411
1412	crc ^= (spa>>8) ^ birth;
1413
1414	return (crc);
1415}
1416
1417#define	HDR_EMPTY(hdr)						\
1418	((hdr)->b_dva.dva_word[0] == 0 &&			\
1419	(hdr)->b_dva.dva_word[1] == 0)
1420
1421#define	HDR_EQUAL(spa, dva, birth, hdr)				\
1422	((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
1423	((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
1424	((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
1425
1426static void
1427buf_discard_identity(arc_buf_hdr_t *hdr)
1428{
1429	hdr->b_dva.dva_word[0] = 0;
1430	hdr->b_dva.dva_word[1] = 0;
1431	hdr->b_birth = 0;
1432}
1433
1434static arc_buf_hdr_t *
1435buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
1436{
1437	const dva_t *dva = BP_IDENTITY(bp);
1438	uint64_t birth = BP_PHYSICAL_BIRTH(bp);
1439	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
1440	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1441	arc_buf_hdr_t *hdr;
1442
1443	mutex_enter(hash_lock);
1444	for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
1445	    hdr = hdr->b_hash_next) {
1446		if (HDR_EQUAL(spa, dva, birth, hdr)) {
1447			*lockp = hash_lock;
1448			return (hdr);
1449		}
1450	}
1451	mutex_exit(hash_lock);
1452	*lockp = NULL;
1453	return (NULL);
1454}
1455
1456/*
1457 * Insert an entry into the hash table.  If there is already an element
1458 * equal to elem in the hash table, then the already existing element
1459 * will be returned and the new element will not be inserted.
1460 * Otherwise returns NULL.
1461 * If lockp == NULL, the caller is assumed to already hold the hash lock.
1462 */
1463static arc_buf_hdr_t *
1464buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
1465{
1466	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1467	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1468	arc_buf_hdr_t *fhdr;
1469	uint32_t i;
1470
1471	ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1472	ASSERT(hdr->b_birth != 0);
1473	ASSERT(!HDR_IN_HASH_TABLE(hdr));
1474
1475	if (lockp != NULL) {
1476		*lockp = hash_lock;
1477		mutex_enter(hash_lock);
1478	} else {
1479		ASSERT(MUTEX_HELD(hash_lock));
1480	}
1481
1482	for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
1483	    fhdr = fhdr->b_hash_next, i++) {
1484		if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1485			return (fhdr);
1486	}
1487
1488	hdr->b_hash_next = buf_hash_table.ht_table[idx];
1489	buf_hash_table.ht_table[idx] = hdr;
1490	arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1491
1492	/* collect some hash table performance data */
1493	if (i > 0) {
1494		ARCSTAT_BUMP(arcstat_hash_collisions);
1495		if (i == 1)
1496			ARCSTAT_BUMP(arcstat_hash_chains);
1497
1498		ARCSTAT_MAX(arcstat_hash_chain_max, i);
1499	}
1500
1501	ARCSTAT_BUMP(arcstat_hash_elements);
1502	ARCSTAT_MAXSTAT(arcstat_hash_elements);
1503
1504	return (NULL);
1505}
1506
1507static void
1508buf_hash_remove(arc_buf_hdr_t *hdr)
1509{
1510	arc_buf_hdr_t *fhdr, **hdrp;
1511	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1512
1513	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
1514	ASSERT(HDR_IN_HASH_TABLE(hdr));
1515
1516	hdrp = &buf_hash_table.ht_table[idx];
1517	while ((fhdr = *hdrp) != hdr) {
1518		ASSERT3P(fhdr, !=, NULL);
1519		hdrp = &fhdr->b_hash_next;
1520	}
1521	*hdrp = hdr->b_hash_next;
1522	hdr->b_hash_next = NULL;
1523	arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1524
1525	/* collect some hash table performance data */
1526	ARCSTAT_BUMPDOWN(arcstat_hash_elements);
1527
1528	if (buf_hash_table.ht_table[idx] &&
1529	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
1530		ARCSTAT_BUMPDOWN(arcstat_hash_chains);
1531}
1532
1533/*
1534 * Global data structures and functions for the buf kmem cache.
1535 */
1536static kmem_cache_t *hdr_full_cache;
1537static kmem_cache_t *hdr_l2only_cache;
1538static kmem_cache_t *buf_cache;
1539
1540static void
1541buf_fini(void)
1542{
1543	int i;
1544
1545	kmem_free(buf_hash_table.ht_table,
1546	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
1547	for (i = 0; i < BUF_LOCKS; i++)
1548		mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
1549	kmem_cache_destroy(hdr_full_cache);
1550	kmem_cache_destroy(hdr_l2only_cache);
1551	kmem_cache_destroy(buf_cache);
1552}
1553
1554/*
1555 * Constructor callback - called when the cache is empty
1556 * and a new buf is requested.
1557 */
1558/* ARGSUSED */
1559static int
1560hdr_full_cons(void *vbuf, void *unused, int kmflag)
1561{
1562	arc_buf_hdr_t *hdr = vbuf;
1563
1564	bzero(hdr, HDR_FULL_SIZE);
1565	cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
1566	refcount_create(&hdr->b_l1hdr.b_refcnt);
1567	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1568	multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1569	arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1570
1571	return (0);
1572}
1573
1574/* ARGSUSED */
1575static int
1576hdr_l2only_cons(void *vbuf, void *unused, int kmflag)
1577{
1578	arc_buf_hdr_t *hdr = vbuf;
1579
1580	bzero(hdr, HDR_L2ONLY_SIZE);
1581	arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1582
1583	return (0);
1584}
1585
1586/* ARGSUSED */
1587static int
1588buf_cons(void *vbuf, void *unused, int kmflag)
1589{
1590	arc_buf_t *buf = vbuf;
1591
1592	bzero(buf, sizeof (arc_buf_t));
1593	mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL);
1594	arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1595
1596	return (0);
1597}
1598
1599/*
1600 * Destructor callback - called when a cached buf is
1601 * no longer required.
1602 */
1603/* ARGSUSED */
1604static void
1605hdr_full_dest(void *vbuf, void *unused)
1606{
1607	arc_buf_hdr_t *hdr = vbuf;
1608
1609	ASSERT(HDR_EMPTY(hdr));
1610	cv_destroy(&hdr->b_l1hdr.b_cv);
1611	refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1612	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1613	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1614	arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1615}
1616
1617/* ARGSUSED */
1618static void
1619hdr_l2only_dest(void *vbuf, void *unused)
1620{
1621	arc_buf_hdr_t *hdr = vbuf;
1622
1623	ASSERT(HDR_EMPTY(hdr));
1624	arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1625}
1626
1627/* ARGSUSED */
1628static void
1629buf_dest(void *vbuf, void *unused)
1630{
1631	arc_buf_t *buf = vbuf;
1632
1633	mutex_destroy(&buf->b_evict_lock);
1634	arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1635}
1636
1637/*
1638 * Reclaim callback -- invoked when memory is low.
1639 */
1640/* ARGSUSED */
1641static void
1642hdr_recl(void *unused)
1643{
1644	dprintf("hdr_recl called\n");
1645	/*
1646	 * umem calls the reclaim func when we destroy the buf cache,
1647	 * which is after we do arc_fini().
1648	 */
1649	if (!arc_dead)
1650		cv_signal(&arc_reclaim_thread_cv);
1651}
1652
1653static void
1654buf_init(void)
1655{
1656	uint64_t *ct;
1657	uint64_t hsize = 1ULL << 12;
1658	int i, j;
1659
1660	/*
1661	 * The hash table is big enough to fill all of physical memory
1662	 * with an average block size of zfs_arc_average_blocksize (default 8K).
1663	 * By default, the table will take up
1664	 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
1665	 */
1666	while (hsize * zfs_arc_average_blocksize < (uint64_t)physmem * PAGESIZE)
1667		hsize <<= 1;
1668retry:
1669	buf_hash_table.ht_mask = hsize - 1;
1670	buf_hash_table.ht_table =
1671	    kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
1672	if (buf_hash_table.ht_table == NULL) {
1673		ASSERT(hsize > (1ULL << 8));
1674		hsize >>= 1;
1675		goto retry;
1676	}
1677
1678	hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
1679	    0, hdr_full_cons, hdr_full_dest, hdr_recl, NULL, NULL, 0);
1680	hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
1681	    HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, hdr_recl,
1682	    NULL, NULL, 0);
1683	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
1684	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
1685
1686	for (i = 0; i < 256; i++)
1687		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
1688			*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
1689
1690	for (i = 0; i < BUF_LOCKS; i++) {
1691		mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
1692		    NULL, MUTEX_DEFAULT, NULL);
1693	}
1694}
1695
1696/*
1697 * This is the size that the buf occupies in memory. If the buf is compressed,
1698 * it will correspond to the compressed size. You should use this method of
1699 * getting the buf size unless you explicitly need the logical size.
1700 */
1701int32_t
1702arc_buf_size(arc_buf_t *buf)
1703{
1704	return (ARC_BUF_COMPRESSED(buf) ?
1705	    HDR_GET_PSIZE(buf->b_hdr) : HDR_GET_LSIZE(buf->b_hdr));
1706}
1707
1708int32_t
1709arc_buf_lsize(arc_buf_t *buf)
1710{
1711	return (HDR_GET_LSIZE(buf->b_hdr));
1712}
1713
1714enum zio_compress
1715arc_get_compression(arc_buf_t *buf)
1716{
1717	return (ARC_BUF_COMPRESSED(buf) ?
1718	    HDR_GET_COMPRESS(buf->b_hdr) : ZIO_COMPRESS_OFF);
1719}
1720
1721#define	ARC_MINTIME	(hz>>4) /* 62 ms */
1722
1723static inline boolean_t
1724arc_buf_is_shared(arc_buf_t *buf)
1725{
1726	boolean_t shared = (buf->b_data != NULL &&
1727	    buf->b_hdr->b_l1hdr.b_pabd != NULL &&
1728	    abd_is_linear(buf->b_hdr->b_l1hdr.b_pabd) &&
1729	    buf->b_data == abd_to_buf(buf->b_hdr->b_l1hdr.b_pabd));
1730	IMPLY(shared, HDR_SHARED_DATA(buf->b_hdr));
1731	IMPLY(shared, ARC_BUF_SHARED(buf));
1732	IMPLY(shared, ARC_BUF_COMPRESSED(buf) || ARC_BUF_LAST(buf));
1733
1734	/*
1735	 * It would be nice to assert arc_can_share() too, but the "hdr isn't
1736	 * already being shared" requirement prevents us from doing that.
1737	 */
1738
1739	return (shared);
1740}
1741
1742/*
1743 * Free the checksum associated with this header. If there is no checksum, this
1744 * is a no-op.
1745 */
1746static inline void
1747arc_cksum_free(arc_buf_hdr_t *hdr)
1748{
1749	ASSERT(HDR_HAS_L1HDR(hdr));
1750	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1751	if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1752		kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));
1753		hdr->b_l1hdr.b_freeze_cksum = NULL;
1754	}
1755	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1756}
1757
1758/*
1759 * Return true iff at least one of the bufs on hdr is not compressed.
1760 */
1761static boolean_t
1762arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr)
1763{
1764	for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) {
1765		if (!ARC_BUF_COMPRESSED(b)) {
1766			return (B_TRUE);
1767		}
1768	}
1769	return (B_FALSE);
1770}
1771
1772/*
1773 * If we've turned on the ZFS_DEBUG_MODIFY flag, verify that the buf's data
1774 * matches the checksum that is stored in the hdr. If there is no checksum,
1775 * or if the buf is compressed, this is a no-op.
1776 */
1777static void
1778arc_cksum_verify(arc_buf_t *buf)
1779{
1780	arc_buf_hdr_t *hdr = buf->b_hdr;
1781	zio_cksum_t zc;
1782
1783	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1784		return;
1785
1786	if (ARC_BUF_COMPRESSED(buf)) {
1787		ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
1788		    arc_hdr_has_uncompressed_buf(hdr));
1789		return;
1790	}
1791
1792	ASSERT(HDR_HAS_L1HDR(hdr));
1793
1794	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1795	if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) {
1796		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1797		return;
1798	}
1799
1800	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, &zc);
1801	if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))
1802		panic("buffer modified while frozen!");
1803	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1804}
1805
1806static boolean_t
1807arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio)
1808{
1809	enum zio_compress compress = BP_GET_COMPRESS(zio->io_bp);
1810	boolean_t valid_cksum;
1811
1812	ASSERT(!BP_IS_EMBEDDED(zio->io_bp));
1813	VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));
1814
1815	/*
1816	 * We rely on the blkptr's checksum to determine if the block
1817	 * is valid or not. When compressed arc is enabled, the l2arc
1818	 * writes the block to the l2arc just as it appears in the pool.
1819	 * This allows us to use the blkptr's checksum to validate the
1820	 * data that we just read off of the l2arc without having to store
1821	 * a separate checksum in the arc_buf_hdr_t. However, if compressed
1822	 * arc is disabled, then the data written to the l2arc is always
1823	 * uncompressed and won't match the block as it exists in the main
1824	 * pool. When this is the case, we must first compress it if it is
1825	 * compressed on the main pool before we can validate the checksum.
1826	 */
1827	if (!HDR_COMPRESSION_ENABLED(hdr) && compress != ZIO_COMPRESS_OFF) {
1828		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
1829		uint64_t lsize = HDR_GET_LSIZE(hdr);
1830		uint64_t csize;
1831
1832		void *cbuf = zio_buf_alloc(HDR_GET_PSIZE(hdr));
1833		csize = zio_compress_data(compress, zio->io_abd, cbuf, lsize);
1834
1835		ASSERT3U(csize, <=, HDR_GET_PSIZE(hdr));
1836		if (csize < HDR_GET_PSIZE(hdr)) {
1837			/*
1838			 * Compressed blocks are always a multiple of the
1839			 * smallest ashift in the pool. Ideally, we would
1840			 * like to round up the csize to the next
1841			 * spa_min_ashift but that value may have changed
1842			 * since the block was last written. Instead,
1843			 * we rely on the fact that the hdr's psize
1844			 * was set to the psize of the block when it was
1845			 * last written. We set the csize to that value
1846			 * and zero out any part that should not contain
1847			 * data.
1848			 */
1849			bzero((char *)cbuf + csize, HDR_GET_PSIZE(hdr) - csize);
1850			csize = HDR_GET_PSIZE(hdr);
1851		}
1852		zio_push_transform(zio, cbuf, csize, HDR_GET_PSIZE(hdr), NULL);
1853	}
1854
1855	/*
1856	 * Block pointers always store the checksum for the logical data.
1857	 * If the block pointer has the gang bit set, then the checksum
1858	 * it represents is for the reconstituted data and not for an
1859	 * individual gang member. The zio pipeline, however, must be able to
1860	 * determine the checksum of each of the gang constituents so it
1861	 * treats the checksum comparison differently than what we need
1862	 * for l2arc blocks. This prevents us from using the
1863	 * zio_checksum_error() interface directly. Instead we must call the
1864	 * zio_checksum_error_impl() so that we can ensure the checksum is
1865	 * generated using the correct checksum algorithm and accounts for the
1866	 * logical I/O size and not just a gang fragment.
1867	 */
1868	valid_cksum = (zio_checksum_error_impl(zio->io_spa, zio->io_bp,
1869	    BP_GET_CHECKSUM(zio->io_bp), zio->io_abd, zio->io_size,
1870	    zio->io_offset, NULL) == 0);
1871	zio_pop_transforms(zio);
1872	return (valid_cksum);
1873}
1874
1875/*
1876 * Given a buf full of data, if ZFS_DEBUG_MODIFY is enabled this computes a
1877 * checksum and attaches it to the buf's hdr so that we can ensure that the buf
1878 * isn't modified later on. If buf is compressed or there is already a checksum
1879 * on the hdr, this is a no-op (we only checksum uncompressed bufs).
1880 */
1881static void
1882arc_cksum_compute(arc_buf_t *buf)
1883{
1884	arc_buf_hdr_t *hdr = buf->b_hdr;
1885
1886	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1887		return;
1888
1889	ASSERT(HDR_HAS_L1HDR(hdr));
1890
1891	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1892	if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1893		ASSERT(arc_hdr_has_uncompressed_buf(hdr));
1894		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1895		return;
1896	} else if (ARC_BUF_COMPRESSED(buf)) {
1897		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1898		return;
1899	}
1900
1901	ASSERT(!ARC_BUF_COMPRESSED(buf));
1902	hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),
1903	    KM_SLEEP);
1904	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL,
1905	    hdr->b_l1hdr.b_freeze_cksum);
1906	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1907#ifdef illumos
1908	arc_buf_watch(buf);
1909#endif
1910}
1911
1912#ifdef illumos
1913#ifndef _KERNEL
1914typedef struct procctl {
1915	long cmd;
1916	prwatch_t prwatch;
1917} procctl_t;
1918#endif
1919
1920/* ARGSUSED */
1921static void
1922arc_buf_unwatch(arc_buf_t *buf)
1923{
1924#ifndef _KERNEL
1925	if (arc_watch) {
1926		int result;
1927		procctl_t ctl;
1928		ctl.cmd = PCWATCH;
1929		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1930		ctl.prwatch.pr_size = 0;
1931		ctl.prwatch.pr_wflags = 0;
1932		result = write(arc_procfd, &ctl, sizeof (ctl));
1933		ASSERT3U(result, ==, sizeof (ctl));
1934	}
1935#endif
1936}
1937
1938/* ARGSUSED */
1939static void
1940arc_buf_watch(arc_buf_t *buf)
1941{
1942#ifndef _KERNEL
1943	if (arc_watch) {
1944		int result;
1945		procctl_t ctl;
1946		ctl.cmd = PCWATCH;
1947		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1948		ctl.prwatch.pr_size = arc_buf_size(buf);
1949		ctl.prwatch.pr_wflags = WA_WRITE;
1950		result = write(arc_procfd, &ctl, sizeof (ctl));
1951		ASSERT3U(result, ==, sizeof (ctl));
1952	}
1953#endif
1954}
1955#endif /* illumos */
1956
1957static arc_buf_contents_t
1958arc_buf_type(arc_buf_hdr_t *hdr)
1959{
1960	arc_buf_contents_t type;
1961	if (HDR_ISTYPE_METADATA(hdr)) {
1962		type = ARC_BUFC_METADATA;
1963	} else {
1964		type = ARC_BUFC_DATA;
1965	}
1966	VERIFY3U(hdr->b_type, ==, type);
1967	return (type);
1968}
1969
1970boolean_t
1971arc_is_metadata(arc_buf_t *buf)
1972{
1973	return (HDR_ISTYPE_METADATA(buf->b_hdr) != 0);
1974}
1975
1976static uint32_t
1977arc_bufc_to_flags(arc_buf_contents_t type)
1978{
1979	switch (type) {
1980	case ARC_BUFC_DATA:
1981		/* metadata field is 0 if buffer contains normal data */
1982		return (0);
1983	case ARC_BUFC_METADATA:
1984		return (ARC_FLAG_BUFC_METADATA);
1985	default:
1986		break;
1987	}
1988	panic("undefined ARC buffer type!");
1989	return ((uint32_t)-1);
1990}
1991
1992void
1993arc_buf_thaw(arc_buf_t *buf)
1994{
1995	arc_buf_hdr_t *hdr = buf->b_hdr;
1996
1997	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
1998	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
1999
2000	arc_cksum_verify(buf);
2001
2002	/*
2003	 * Compressed buffers do not manipulate the b_freeze_cksum or
2004	 * allocate b_thawed.
2005	 */
2006	if (ARC_BUF_COMPRESSED(buf)) {
2007		ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
2008		    arc_hdr_has_uncompressed_buf(hdr));
2009		return;
2010	}
2011
2012	ASSERT(HDR_HAS_L1HDR(hdr));
2013	arc_cksum_free(hdr);
2014
2015	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
2016#ifdef ZFS_DEBUG
2017	if (zfs_flags & ZFS_DEBUG_MODIFY) {
2018		if (hdr->b_l1hdr.b_thawed != NULL)
2019			kmem_free(hdr->b_l1hdr.b_thawed, 1);
2020		hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP);
2021	}
2022#endif
2023
2024	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2025
2026#ifdef illumos
2027	arc_buf_unwatch(buf);
2028#endif
2029}
2030
2031void
2032arc_buf_freeze(arc_buf_t *buf)
2033{
2034	arc_buf_hdr_t *hdr = buf->b_hdr;
2035	kmutex_t *hash_lock;
2036
2037	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
2038		return;
2039
2040	if (ARC_BUF_COMPRESSED(buf)) {
2041		ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
2042		    arc_hdr_has_uncompressed_buf(hdr));
2043		return;
2044	}
2045
2046	hash_lock = HDR_LOCK(hdr);
2047	mutex_enter(hash_lock);
2048
2049	ASSERT(HDR_HAS_L1HDR(hdr));
2050	ASSERT(hdr->b_l1hdr.b_freeze_cksum != NULL ||
2051	    hdr->b_l1hdr.b_state == arc_anon);
2052	arc_cksum_compute(buf);
2053	mutex_exit(hash_lock);
2054}
2055
2056/*
2057 * The arc_buf_hdr_t's b_flags should never be modified directly. Instead,
2058 * the following functions should be used to ensure that the flags are
2059 * updated in a thread-safe way. When manipulating the flags either
2060 * the hash_lock must be held or the hdr must be undiscoverable. This
2061 * ensures that we're not racing with any other threads when updating
2062 * the flags.
2063 */
2064static inline void
2065arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
2066{
2067	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2068	hdr->b_flags |= flags;
2069}
2070
2071static inline void
2072arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
2073{
2074	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2075	hdr->b_flags &= ~flags;
2076}
2077
2078/*
2079 * Setting the compression bits in the arc_buf_hdr_t's b_flags is
2080 * done in a special way since we have to clear and set bits
2081 * at the same time. Consumers that wish to set the compression bits
2082 * must use this function to ensure that the flags are updated in
2083 * thread-safe manner.
2084 */
2085static void
2086arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp)
2087{
2088	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2089
2090	/*
2091	 * Holes and embedded blocks will always have a psize = 0 so
2092	 * we ignore the compression of the blkptr and set the
2093	 * arc_buf_hdr_t's compression to ZIO_COMPRESS_OFF.
2094	 * Holes and embedded blocks remain anonymous so we don't
2095	 * want to uncompress them. Mark them as uncompressed.
2096	 */
2097	if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) {
2098		arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
2099		HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF);
2100		ASSERT(!HDR_COMPRESSION_ENABLED(hdr));
2101		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
2102	} else {
2103		arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
2104		HDR_SET_COMPRESS(hdr, cmp);
2105		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp);
2106		ASSERT(HDR_COMPRESSION_ENABLED(hdr));
2107	}
2108}
2109
2110/*
2111 * Looks for another buf on the same hdr which has the data decompressed, copies
2112 * from it, and returns true. If no such buf exists, returns false.
2113 */
2114static boolean_t
2115arc_buf_try_copy_decompressed_data(arc_buf_t *buf)
2116{
2117	arc_buf_hdr_t *hdr = buf->b_hdr;
2118	boolean_t copied = B_FALSE;
2119
2120	ASSERT(HDR_HAS_L1HDR(hdr));
2121	ASSERT3P(buf->b_data, !=, NULL);
2122	ASSERT(!ARC_BUF_COMPRESSED(buf));
2123
2124	for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL;
2125	    from = from->b_next) {
2126		/* can't use our own data buffer */
2127		if (from == buf) {
2128			continue;
2129		}
2130
2131		if (!ARC_BUF_COMPRESSED(from)) {
2132			bcopy(from->b_data, buf->b_data, arc_buf_size(buf));
2133			copied = B_TRUE;
2134			break;
2135		}
2136	}
2137
2138	/*
2139	 * There were no decompressed bufs, so there should not be a
2140	 * checksum on the hdr either.
2141	 */
2142	EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL);
2143
2144	return (copied);
2145}
2146
2147/*
2148 * Given a buf that has a data buffer attached to it, this function will
2149 * efficiently fill the buf with data of the specified compression setting from
2150 * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr
2151 * are already sharing a data buf, no copy is performed.
2152 *
2153 * If the buf is marked as compressed but uncompressed data was requested, this
2154 * will allocate a new data buffer for the buf, remove that flag, and fill the
2155 * buf with uncompressed data. You can't request a compressed buf on a hdr with
2156 * uncompressed data, and (since we haven't added support for it yet) if you
2157 * want compressed data your buf must already be marked as compressed and have
2158 * the correct-sized data buffer.
2159 */
2160static int
2161arc_buf_fill(arc_buf_t *buf, boolean_t compressed)
2162{
2163	arc_buf_hdr_t *hdr = buf->b_hdr;
2164	boolean_t hdr_compressed = (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
2165	dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;
2166
2167	ASSERT3P(buf->b_data, !=, NULL);
2168	IMPLY(compressed, hdr_compressed);
2169	IMPLY(compressed, ARC_BUF_COMPRESSED(buf));
2170
2171	if (hdr_compressed == compressed) {
2172		if (!arc_buf_is_shared(buf)) {
2173			abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd,
2174			    arc_buf_size(buf));
2175		}
2176	} else {
2177		ASSERT(hdr_compressed);
2178		ASSERT(!compressed);
2179		ASSERT3U(HDR_GET_LSIZE(hdr), !=, HDR_GET_PSIZE(hdr));
2180
2181		/*
2182		 * If the buf is sharing its data with the hdr, unlink it and
2183		 * allocate a new data buffer for the buf.
2184		 */
2185		if (arc_buf_is_shared(buf)) {
2186			ASSERT(ARC_BUF_COMPRESSED(buf));
2187
2188			/* We need to give the buf it's own b_data */
2189			buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
2190			buf->b_data =
2191			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2192			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2193
2194			/* Previously overhead was 0; just add new overhead */
2195			ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));
2196		} else if (ARC_BUF_COMPRESSED(buf)) {
2197			/* We need to reallocate the buf's b_data */
2198			arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr),
2199			    buf);
2200			buf->b_data =
2201			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2202
2203			/* We increased the size of b_data; update overhead */
2204			ARCSTAT_INCR(arcstat_overhead_size,
2205			    HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr));
2206		}
2207
2208		/*
2209		 * Regardless of the buf's previous compression settings, it
2210		 * should not be compressed at the end of this function.
2211		 */
2212		buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
2213
2214		/*
2215		 * Try copying the data from another buf which already has a
2216		 * decompressed version. If that's not possible, it's time to
2217		 * bite the bullet and decompress the data from the hdr.
2218		 */
2219		if (arc_buf_try_copy_decompressed_data(buf)) {
2220			/* Skip byteswapping and checksumming (already done) */
2221			ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, !=, NULL);
2222			return (0);
2223		} else {
2224			int error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
2225			    hdr->b_l1hdr.b_pabd, buf->b_data,
2226			    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
2227
2228			/*
2229			 * Absent hardware errors or software bugs, this should
2230			 * be impossible, but log it anyway so we can debug it.
2231			 */
2232			if (error != 0) {
2233				zfs_dbgmsg(
2234				    "hdr %p, compress %d, psize %d, lsize %d",
2235				    hdr, HDR_GET_COMPRESS(hdr),
2236				    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
2237				return (SET_ERROR(EIO));
2238			}
2239		}
2240	}
2241
2242	/* Byteswap the buf's data if necessary */
2243	if (bswap != DMU_BSWAP_NUMFUNCS) {
2244		ASSERT(!HDR_SHARED_DATA(hdr));
2245		ASSERT3U(bswap, <, DMU_BSWAP_NUMFUNCS);
2246		dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr));
2247	}
2248
2249	/* Compute the hdr's checksum if necessary */
2250	arc_cksum_compute(buf);
2251
2252	return (0);
2253}
2254
2255int
2256arc_decompress(arc_buf_t *buf)
2257{
2258	return (arc_buf_fill(buf, B_FALSE));
2259}
2260
2261/*
2262 * Return the size of the block, b_pabd, that is stored in the arc_buf_hdr_t.
2263 */
2264static uint64_t
2265arc_hdr_size(arc_buf_hdr_t *hdr)
2266{
2267	uint64_t size;
2268
2269	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
2270	    HDR_GET_PSIZE(hdr) > 0) {
2271		size = HDR_GET_PSIZE(hdr);
2272	} else {
2273		ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0);
2274		size = HDR_GET_LSIZE(hdr);
2275	}
2276	return (size);
2277}
2278
2279/*
2280 * Increment the amount of evictable space in the arc_state_t's refcount.
2281 * We account for the space used by the hdr and the arc buf individually
2282 * so that we can add and remove them from the refcount individually.
2283 */
2284static void
2285arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state)
2286{
2287	arc_buf_contents_t type = arc_buf_type(hdr);
2288
2289	ASSERT(HDR_HAS_L1HDR(hdr));
2290
2291	if (GHOST_STATE(state)) {
2292		ASSERT0(hdr->b_l1hdr.b_bufcnt);
2293		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2294		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2295		(void) refcount_add_many(&state->arcs_esize[type],
2296		    HDR_GET_LSIZE(hdr), hdr);
2297		return;
2298	}
2299
2300	ASSERT(!GHOST_STATE(state));
2301	if (hdr->b_l1hdr.b_pabd != NULL) {
2302		(void) refcount_add_many(&state->arcs_esize[type],
2303		    arc_hdr_size(hdr), hdr);
2304	}
2305	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2306	    buf = buf->b_next) {
2307		if (arc_buf_is_shared(buf))
2308			continue;
2309		(void) refcount_add_many(&state->arcs_esize[type],
2310		    arc_buf_size(buf), buf);
2311	}
2312}
2313
2314/*
2315 * Decrement the amount of evictable space in the arc_state_t's refcount.
2316 * We account for the space used by the hdr and the arc buf individually
2317 * so that we can add and remove them from the refcount individually.
2318 */
2319static void
2320arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state)
2321{
2322	arc_buf_contents_t type = arc_buf_type(hdr);
2323
2324	ASSERT(HDR_HAS_L1HDR(hdr));
2325
2326	if (GHOST_STATE(state)) {
2327		ASSERT0(hdr->b_l1hdr.b_bufcnt);
2328		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2329		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2330		(void) refcount_remove_many(&state->arcs_esize[type],
2331		    HDR_GET_LSIZE(hdr), hdr);
2332		return;
2333	}
2334
2335	ASSERT(!GHOST_STATE(state));
2336	if (hdr->b_l1hdr.b_pabd != NULL) {
2337		(void) refcount_remove_many(&state->arcs_esize[type],
2338		    arc_hdr_size(hdr), hdr);
2339	}
2340	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2341	    buf = buf->b_next) {
2342		if (arc_buf_is_shared(buf))
2343			continue;
2344		(void) refcount_remove_many(&state->arcs_esize[type],
2345		    arc_buf_size(buf), buf);
2346	}
2347}
2348
2349/*
2350 * Add a reference to this hdr indicating that someone is actively
2351 * referencing that memory. When the refcount transitions from 0 to 1,
2352 * we remove it from the respective arc_state_t list to indicate that
2353 * it is not evictable.
2354 */
2355static void
2356add_reference(arc_buf_hdr_t *hdr, void *tag)
2357{
2358	ASSERT(HDR_HAS_L1HDR(hdr));
2359	if (!MUTEX_HELD(HDR_LOCK(hdr))) {
2360		ASSERT(hdr->b_l1hdr.b_state == arc_anon);
2361		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2362		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2363	}
2364
2365	arc_state_t *state = hdr->b_l1hdr.b_state;
2366
2367	if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
2368	    (state != arc_anon)) {
2369		/* We don't use the L2-only state list. */
2370		if (state != arc_l2c_only) {
2371			multilist_remove(state->arcs_list[arc_buf_type(hdr)],
2372			    hdr);
2373			arc_evictable_space_decrement(hdr, state);
2374		}
2375		/* remove the prefetch flag if we get a reference */
2376		arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
2377	}
2378}
2379
2380/*
2381 * Remove a reference from this hdr. When the reference transitions from
2382 * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's
2383 * list making it eligible for eviction.
2384 */
2385static int
2386remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
2387{
2388	int cnt;
2389	arc_state_t *state = hdr->b_l1hdr.b_state;
2390
2391	ASSERT(HDR_HAS_L1HDR(hdr));
2392	ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
2393	ASSERT(!GHOST_STATE(state));
2394
2395	/*
2396	 * arc_l2c_only counts as a ghost state so we don't need to explicitly
2397	 * check to prevent usage of the arc_l2c_only list.
2398	 */
2399	if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
2400	    (state != arc_anon)) {
2401		multilist_insert(state->arcs_list[arc_buf_type(hdr)], hdr);
2402		ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
2403		arc_evictable_space_increment(hdr, state);
2404	}
2405	return (cnt);
2406}
2407
2408/*
2409 * Move the supplied buffer to the indicated state. The hash lock
2410 * for the buffer must be held by the caller.
2411 */
2412static void
2413arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
2414    kmutex_t *hash_lock)
2415{
2416	arc_state_t *old_state;
2417	int64_t refcnt;
2418	uint32_t bufcnt;
2419	boolean_t update_old, update_new;
2420	arc_buf_contents_t buftype = arc_buf_type(hdr);
2421
2422	/*
2423	 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
2424	 * in arc_read() when bringing a buffer out of the L2ARC.  However, the
2425	 * L1 hdr doesn't always exist when we change state to arc_anon before
2426	 * destroying a header, in which case reallocating to add the L1 hdr is
2427	 * pointless.
2428	 */
2429	if (HDR_HAS_L1HDR(hdr)) {
2430		old_state = hdr->b_l1hdr.b_state;
2431		refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt);
2432		bufcnt = hdr->b_l1hdr.b_bufcnt;
2433		update_old = (bufcnt > 0 || hdr->b_l1hdr.b_pabd != NULL);
2434	} else {
2435		old_state = arc_l2c_only;
2436		refcnt = 0;
2437		bufcnt = 0;
2438		update_old = B_FALSE;
2439	}
2440	update_new = update_old;
2441
2442	ASSERT(MUTEX_HELD(hash_lock));
2443	ASSERT3P(new_state, !=, old_state);
2444	ASSERT(!GHOST_STATE(new_state) || bufcnt == 0);
2445	ASSERT(old_state != arc_anon || bufcnt <= 1);
2446
2447	/*
2448	 * If this buffer is evictable, transfer it from the
2449	 * old state list to the new state list.
2450	 */
2451	if (refcnt == 0) {
2452		if (old_state != arc_anon && old_state != arc_l2c_only) {
2453			ASSERT(HDR_HAS_L1HDR(hdr));
2454			multilist_remove(old_state->arcs_list[buftype], hdr);
2455
2456			if (GHOST_STATE(old_state)) {
2457				ASSERT0(bufcnt);
2458				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2459				update_old = B_TRUE;
2460			}
2461			arc_evictable_space_decrement(hdr, old_state);
2462		}
2463		if (new_state != arc_anon && new_state != arc_l2c_only) {
2464
2465			/*
2466			 * An L1 header always exists here, since if we're
2467			 * moving to some L1-cached state (i.e. not l2c_only or
2468			 * anonymous), we realloc the header to add an L1hdr
2469			 * beforehand.
2470			 */
2471			ASSERT(HDR_HAS_L1HDR(hdr));
2472			multilist_insert(new_state->arcs_list[buftype], hdr);
2473
2474			if (GHOST_STATE(new_state)) {
2475				ASSERT0(bufcnt);
2476				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2477				update_new = B_TRUE;
2478			}
2479			arc_evictable_space_increment(hdr, new_state);
2480		}
2481	}
2482
2483	ASSERT(!HDR_EMPTY(hdr));
2484	if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
2485		buf_hash_remove(hdr);
2486
2487	/* adjust state sizes (ignore arc_l2c_only) */
2488
2489	if (update_new && new_state != arc_l2c_only) {
2490		ASSERT(HDR_HAS_L1HDR(hdr));
2491		if (GHOST_STATE(new_state)) {
2492			ASSERT0(bufcnt);
2493
2494			/*
2495			 * When moving a header to a ghost state, we first
2496			 * remove all arc buffers. Thus, we'll have a
2497			 * bufcnt of zero, and no arc buffer to use for
2498			 * the reference. As a result, we use the arc
2499			 * header pointer for the reference.
2500			 */
2501			(void) refcount_add_many(&new_state->arcs_size,
2502			    HDR_GET_LSIZE(hdr), hdr);
2503			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2504		} else {
2505			uint32_t buffers = 0;
2506
2507			/*
2508			 * Each individual buffer holds a unique reference,
2509			 * thus we must remove each of these references one
2510			 * at a time.
2511			 */
2512			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2513			    buf = buf->b_next) {
2514				ASSERT3U(bufcnt, !=, 0);
2515				buffers++;
2516
2517				/*
2518				 * When the arc_buf_t is sharing the data
2519				 * block with the hdr, the owner of the
2520				 * reference belongs to the hdr. Only
2521				 * add to the refcount if the arc_buf_t is
2522				 * not shared.
2523				 */
2524				if (arc_buf_is_shared(buf))
2525					continue;
2526
2527				(void) refcount_add_many(&new_state->arcs_size,
2528				    arc_buf_size(buf), buf);
2529			}
2530			ASSERT3U(bufcnt, ==, buffers);
2531
2532			if (hdr->b_l1hdr.b_pabd != NULL) {
2533				(void) refcount_add_many(&new_state->arcs_size,
2534				    arc_hdr_size(hdr), hdr);
2535			} else {
2536				ASSERT(GHOST_STATE(old_state));
2537			}
2538		}
2539	}
2540
2541	if (update_old && old_state != arc_l2c_only) {
2542		ASSERT(HDR_HAS_L1HDR(hdr));
2543		if (GHOST_STATE(old_state)) {
2544			ASSERT0(bufcnt);
2545			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2546
2547			/*
2548			 * When moving a header off of a ghost state,
2549			 * the header will not contain any arc buffers.
2550			 * We use the arc header pointer for the reference
2551			 * which is exactly what we did when we put the
2552			 * header on the ghost state.
2553			 */
2554
2555			(void) refcount_remove_many(&old_state->arcs_size,
2556			    HDR_GET_LSIZE(hdr), hdr);
2557		} else {
2558			uint32_t buffers = 0;
2559
2560			/*
2561			 * Each individual buffer holds a unique reference,
2562			 * thus we must remove each of these references one
2563			 * at a time.
2564			 */
2565			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2566			    buf = buf->b_next) {
2567				ASSERT3U(bufcnt, !=, 0);
2568				buffers++;
2569
2570				/*
2571				 * When the arc_buf_t is sharing the data
2572				 * block with the hdr, the owner of the
2573				 * reference belongs to the hdr. Only
2574				 * add to the refcount if the arc_buf_t is
2575				 * not shared.
2576				 */
2577				if (arc_buf_is_shared(buf))
2578					continue;
2579
2580				(void) refcount_remove_many(
2581				    &old_state->arcs_size, arc_buf_size(buf),
2582				    buf);
2583			}
2584			ASSERT3U(bufcnt, ==, buffers);
2585			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
2586			(void) refcount_remove_many(
2587			    &old_state->arcs_size, arc_hdr_size(hdr), hdr);
2588		}
2589	}
2590
2591	if (HDR_HAS_L1HDR(hdr))
2592		hdr->b_l1hdr.b_state = new_state;
2593
2594	/*
2595	 * L2 headers should never be on the L2 state list since they don't
2596	 * have L1 headers allocated.
2597	 */
2598	ASSERT(multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
2599	    multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
2600}
2601
2602void
2603arc_space_consume(uint64_t space, arc_space_type_t type)
2604{
2605	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2606
2607	switch (type) {
2608	case ARC_SPACE_DATA:
2609		ARCSTAT_INCR(arcstat_data_size, space);
2610		break;
2611	case ARC_SPACE_META:
2612		ARCSTAT_INCR(arcstat_metadata_size, space);
2613		break;
2614	case ARC_SPACE_OTHER:
2615		ARCSTAT_INCR(arcstat_other_size, space);
2616		break;
2617	case ARC_SPACE_HDRS:
2618		ARCSTAT_INCR(arcstat_hdr_size, space);
2619		break;
2620	case ARC_SPACE_L2HDRS:
2621		ARCSTAT_INCR(arcstat_l2_hdr_size, space);
2622		break;
2623	}
2624
2625	if (type != ARC_SPACE_DATA)
2626		ARCSTAT_INCR(arcstat_meta_used, space);
2627
2628	atomic_add_64(&arc_size, space);
2629}
2630
2631void
2632arc_space_return(uint64_t space, arc_space_type_t type)
2633{
2634	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2635
2636	switch (type) {
2637	case ARC_SPACE_DATA:
2638		ARCSTAT_INCR(arcstat_data_size, -space);
2639		break;
2640	case ARC_SPACE_META:
2641		ARCSTAT_INCR(arcstat_metadata_size, -space);
2642		break;
2643	case ARC_SPACE_OTHER:
2644		ARCSTAT_INCR(arcstat_other_size, -space);
2645		break;
2646	case ARC_SPACE_HDRS:
2647		ARCSTAT_INCR(arcstat_hdr_size, -space);
2648		break;
2649	case ARC_SPACE_L2HDRS:
2650		ARCSTAT_INCR(arcstat_l2_hdr_size, -space);
2651		break;
2652	}
2653
2654	if (type != ARC_SPACE_DATA) {
2655		ASSERT(arc_meta_used >= space);
2656		if (arc_meta_max < arc_meta_used)
2657			arc_meta_max = arc_meta_used;
2658		ARCSTAT_INCR(arcstat_meta_used, -space);
2659	}
2660
2661	ASSERT(arc_size >= space);
2662	atomic_add_64(&arc_size, -space);
2663}
2664
2665/*
2666 * Given a hdr and a buf, returns whether that buf can share its b_data buffer
2667 * with the hdr's b_pabd.
2668 */
2669static boolean_t
2670arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2671{
2672	/*
2673	 * The criteria for sharing a hdr's data are:
2674	 * 1. the hdr's compression matches the buf's compression
2675	 * 2. the hdr doesn't need to be byteswapped
2676	 * 3. the hdr isn't already being shared
2677	 * 4. the buf is either compressed or it is the last buf in the hdr list
2678	 *
2679	 * Criterion #4 maintains the invariant that shared uncompressed
2680	 * bufs must be the final buf in the hdr's b_buf list. Reading this, you
2681	 * might ask, "if a compressed buf is allocated first, won't that be the
2682	 * last thing in the list?", but in that case it's impossible to create
2683	 * a shared uncompressed buf anyway (because the hdr must be compressed
2684	 * to have the compressed buf). You might also think that #3 is
2685	 * sufficient to make this guarantee, however it's possible
2686	 * (specifically in the rare L2ARC write race mentioned in
2687	 * arc_buf_alloc_impl()) there will be an existing uncompressed buf that
2688	 * is sharable, but wasn't at the time of its allocation. Rather than
2689	 * allow a new shared uncompressed buf to be created and then shuffle
2690	 * the list around to make it the last element, this simply disallows
2691	 * sharing if the new buf isn't the first to be added.
2692	 */
2693	ASSERT3P(buf->b_hdr, ==, hdr);
2694	boolean_t hdr_compressed = HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF;
2695	boolean_t buf_compressed = ARC_BUF_COMPRESSED(buf) != 0;
2696	return (buf_compressed == hdr_compressed &&
2697	    hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&
2698	    !HDR_SHARED_DATA(hdr) &&
2699	    (ARC_BUF_LAST(buf) || ARC_BUF_COMPRESSED(buf)));
2700}
2701
2702/*
2703 * Allocate a buf for this hdr. If you care about the data that's in the hdr,
2704 * or if you want a compressed buffer, pass those flags in. Returns 0 if the
2705 * copy was made successfully, or an error code otherwise.
2706 */
2707static int
2708arc_buf_alloc_impl(arc_buf_hdr_t *hdr, void *tag, boolean_t compressed,
2709    boolean_t fill, arc_buf_t **ret)
2710{
2711	arc_buf_t *buf;
2712
2713	ASSERT(HDR_HAS_L1HDR(hdr));
2714	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
2715	VERIFY(hdr->b_type == ARC_BUFC_DATA ||
2716	    hdr->b_type == ARC_BUFC_METADATA);
2717	ASSERT3P(ret, !=, NULL);
2718	ASSERT3P(*ret, ==, NULL);
2719
2720	buf = *ret = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
2721	buf->b_hdr = hdr;
2722	buf->b_data = NULL;
2723	buf->b_next = hdr->b_l1hdr.b_buf;
2724	buf->b_flags = 0;
2725
2726	add_reference(hdr, tag);
2727
2728	/*
2729	 * We're about to change the hdr's b_flags. We must either
2730	 * hold the hash_lock or be undiscoverable.
2731	 */
2732	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2733
2734	/*
2735	 * Only honor requests for compressed bufs if the hdr is actually
2736	 * compressed.
2737	 */
2738	if (compressed && HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF)
2739		buf->b_flags |= ARC_BUF_FLAG_COMPRESSED;
2740
2741	/*
2742	 * If the hdr's data can be shared then we share the data buffer and
2743	 * set the appropriate bit in the hdr's b_flags to indicate the hdr is
2744	 * sharing it's b_pabd with the arc_buf_t. Otherwise, we allocate a new
2745	 * buffer to store the buf's data.
2746	 *
2747	 * There are two additional restrictions here because we're sharing
2748	 * hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be
2749	 * actively involved in an L2ARC write, because if this buf is used by
2750	 * an arc_write() then the hdr's data buffer will be released when the
2751	 * write completes, even though the L2ARC write might still be using it.
2752	 * Second, the hdr's ABD must be linear so that the buf's user doesn't
2753	 * need to be ABD-aware.
2754	 */
2755	boolean_t can_share = arc_can_share(hdr, buf) && !HDR_L2_WRITING(hdr) &&
2756	    abd_is_linear(hdr->b_l1hdr.b_pabd);
2757
2758	/* Set up b_data and sharing */
2759	if (can_share) {
2760		buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd);
2761		buf->b_flags |= ARC_BUF_FLAG_SHARED;
2762		arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2763	} else {
2764		buf->b_data =
2765		    arc_get_data_buf(hdr, arc_buf_size(buf), buf);
2766		ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
2767	}
2768	VERIFY3P(buf->b_data, !=, NULL);
2769
2770	hdr->b_l1hdr.b_buf = buf;
2771	hdr->b_l1hdr.b_bufcnt += 1;
2772
2773	/*
2774	 * If the user wants the data from the hdr, we need to either copy or
2775	 * decompress the data.
2776	 */
2777	if (fill) {
2778		return (arc_buf_fill(buf, ARC_BUF_COMPRESSED(buf) != 0));
2779	}
2780
2781	return (0);
2782}
2783
2784static char *arc_onloan_tag = "onloan";
2785
2786static inline void
2787arc_loaned_bytes_update(int64_t delta)
2788{
2789	atomic_add_64(&arc_loaned_bytes, delta);
2790
2791	/* assert that it did not wrap around */
2792	ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
2793}
2794
2795/*
2796 * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
2797 * flight data by arc_tempreserve_space() until they are "returned". Loaned
2798 * buffers must be returned to the arc before they can be used by the DMU or
2799 * freed.
2800 */
2801arc_buf_t *
2802arc_loan_buf(spa_t *spa, boolean_t is_metadata, int size)
2803{
2804	arc_buf_t *buf = arc_alloc_buf(spa, arc_onloan_tag,
2805	    is_metadata ? ARC_BUFC_METADATA : ARC_BUFC_DATA, size);
2806
2807	arc_loaned_bytes_update(size);
2808
2809	return (buf);
2810}
2811
2812arc_buf_t *
2813arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize,
2814    enum zio_compress compression_type)
2815{
2816	arc_buf_t *buf = arc_alloc_compressed_buf(spa, arc_onloan_tag,
2817	    psize, lsize, compression_type);
2818
2819	arc_loaned_bytes_update(psize);
2820
2821	return (buf);
2822}
2823
2824
2825/*
2826 * Return a loaned arc buffer to the arc.
2827 */
2828void
2829arc_return_buf(arc_buf_t *buf, void *tag)
2830{
2831	arc_buf_hdr_t *hdr = buf->b_hdr;
2832
2833	ASSERT3P(buf->b_data, !=, NULL);
2834	ASSERT(HDR_HAS_L1HDR(hdr));
2835	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
2836	(void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2837
2838	arc_loaned_bytes_update(-arc_buf_size(buf));
2839}
2840
2841/* Detach an arc_buf from a dbuf (tag) */
2842void
2843arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
2844{
2845	arc_buf_hdr_t *hdr = buf->b_hdr;
2846
2847	ASSERT3P(buf->b_data, !=, NULL);
2848	ASSERT(HDR_HAS_L1HDR(hdr));
2849	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2850	(void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
2851
2852	arc_loaned_bytes_update(arc_buf_size(buf));
2853}
2854
2855static void
2856l2arc_free_abd_on_write(abd_t *abd, size_t size, arc_buf_contents_t type)
2857{
2858	l2arc_data_free_t *df = kmem_alloc(sizeof (*df), KM_SLEEP);
2859
2860	df->l2df_abd = abd;
2861	df->l2df_size = size;
2862	df->l2df_type = type;
2863	mutex_enter(&l2arc_free_on_write_mtx);
2864	list_insert_head(l2arc_free_on_write, df);
2865	mutex_exit(&l2arc_free_on_write_mtx);
2866}
2867
2868static void
2869arc_hdr_free_on_write(arc_buf_hdr_t *hdr)
2870{
2871	arc_state_t *state = hdr->b_l1hdr.b_state;
2872	arc_buf_contents_t type = arc_buf_type(hdr);
2873	uint64_t size = arc_hdr_size(hdr);
2874
2875	/* protected by hash lock, if in the hash table */
2876	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2877		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2878		ASSERT(state != arc_anon && state != arc_l2c_only);
2879
2880		(void) refcount_remove_many(&state->arcs_esize[type],
2881		    size, hdr);
2882	}
2883	(void) refcount_remove_many(&state->arcs_size, size, hdr);
2884	if (type == ARC_BUFC_METADATA) {
2885		arc_space_return(size, ARC_SPACE_META);
2886	} else {
2887		ASSERT(type == ARC_BUFC_DATA);
2888		arc_space_return(size, ARC_SPACE_DATA);
2889	}
2890
2891	l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type);
2892}
2893
2894/*
2895 * Share the arc_buf_t's data with the hdr. Whenever we are sharing the
2896 * data buffer, we transfer the refcount ownership to the hdr and update
2897 * the appropriate kstats.
2898 */
2899static void
2900arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2901{
2902	arc_state_t *state = hdr->b_l1hdr.b_state;
2903
2904	ASSERT(arc_can_share(hdr, buf));
2905	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2906	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2907
2908	/*
2909	 * Start sharing the data buffer. We transfer the
2910	 * refcount ownership to the hdr since it always owns
2911	 * the refcount whenever an arc_buf_t is shared.
2912	 */
2913	refcount_transfer_ownership(&state->arcs_size, buf, hdr);
2914	hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf));
2915	abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd,
2916	    HDR_ISTYPE_METADATA(hdr));
2917	arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2918	buf->b_flags |= ARC_BUF_FLAG_SHARED;
2919
2920	/*
2921	 * Since we've transferred ownership to the hdr we need
2922	 * to increment its compressed and uncompressed kstats and
2923	 * decrement the overhead size.
2924	 */
2925	ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
2926	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
2927	ARCSTAT_INCR(arcstat_overhead_size, -arc_buf_size(buf));
2928}
2929
2930static void
2931arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2932{
2933	arc_state_t *state = hdr->b_l1hdr.b_state;
2934
2935	ASSERT(arc_buf_is_shared(buf));
2936	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
2937	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2938
2939	/*
2940	 * We are no longer sharing this buffer so we need
2941	 * to transfer its ownership to the rightful owner.
2942	 */
2943	refcount_transfer_ownership(&state->arcs_size, hdr, buf);
2944	arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2945	abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd);
2946	abd_put(hdr->b_l1hdr.b_pabd);
2947	hdr->b_l1hdr.b_pabd = NULL;
2948	buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
2949
2950	/*
2951	 * Since the buffer is no longer shared between
2952	 * the arc buf and the hdr, count it as overhead.
2953	 */
2954	ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
2955	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
2956	ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
2957}
2958
2959/*
2960 * Remove an arc_buf_t from the hdr's buf list and return the last
2961 * arc_buf_t on the list. If no buffers remain on the list then return
2962 * NULL.
2963 */
2964static arc_buf_t *
2965arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2966{
2967	ASSERT(HDR_HAS_L1HDR(hdr));
2968	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2969
2970	arc_buf_t **bufp = &hdr->b_l1hdr.b_buf;
2971	arc_buf_t *lastbuf = NULL;
2972
2973	/*
2974	 * Remove the buf from the hdr list and locate the last
2975	 * remaining buffer on the list.
2976	 */
2977	while (*bufp != NULL) {
2978		if (*bufp == buf)
2979			*bufp = buf->b_next;
2980
2981		/*
2982		 * If we've removed a buffer in the middle of
2983		 * the list then update the lastbuf and update
2984		 * bufp.
2985		 */
2986		if (*bufp != NULL) {
2987			lastbuf = *bufp;
2988			bufp = &(*bufp)->b_next;
2989		}
2990	}
2991	buf->b_next = NULL;
2992	ASSERT3P(lastbuf, !=, buf);
2993	IMPLY(hdr->b_l1hdr.b_bufcnt > 0, lastbuf != NULL);
2994	IMPLY(hdr->b_l1hdr.b_bufcnt > 0, hdr->b_l1hdr.b_buf != NULL);
2995	IMPLY(lastbuf != NULL, ARC_BUF_LAST(lastbuf));
2996
2997	return (lastbuf);
2998}
2999
3000/*
3001 * Free up buf->b_data and pull the arc_buf_t off of the the arc_buf_hdr_t's
3002 * list and free it.
3003 */
3004static void
3005arc_buf_destroy_impl(arc_buf_t *buf)
3006{
3007	arc_buf_hdr_t *hdr = buf->b_hdr;
3008
3009	/*
3010	 * Free up the data associated with the buf but only if we're not
3011	 * sharing this with the hdr. If we are sharing it with the hdr, the
3012	 * hdr is responsible for doing the free.
3013	 */
3014	if (buf->b_data != NULL) {
3015		/*
3016		 * We're about to change the hdr's b_flags. We must either
3017		 * hold the hash_lock or be undiscoverable.
3018		 */
3019		ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
3020
3021		arc_cksum_verify(buf);
3022#ifdef illumos
3023		arc_buf_unwatch(buf);
3024#endif
3025
3026		if (arc_buf_is_shared(buf)) {
3027			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
3028		} else {
3029			uint64_t size = arc_buf_size(buf);
3030			arc_free_data_buf(hdr, buf->b_data, size, buf);
3031			ARCSTAT_INCR(arcstat_overhead_size, -size);
3032		}
3033		buf->b_data = NULL;
3034
3035		ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
3036		hdr->b_l1hdr.b_bufcnt -= 1;
3037	}
3038
3039	arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
3040
3041	if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) {
3042		/*
3043		 * If the current arc_buf_t is sharing its data buffer with the
3044		 * hdr, then reassign the hdr's b_pabd to share it with the new
3045		 * buffer at the end of the list. The shared buffer is always
3046		 * the last one on the hdr's buffer list.
3047		 *
3048		 * There is an equivalent case for compressed bufs, but since
3049		 * they aren't guaranteed to be the last buf in the list and
3050		 * that is an exceedingly rare case, we just allow that space be
3051		 * wasted temporarily.
3052		 */
3053		if (lastbuf != NULL) {
3054			/* Only one buf can be shared at once */
3055			VERIFY(!arc_buf_is_shared(lastbuf));
3056			/* hdr is uncompressed so can't have compressed buf */
3057			VERIFY(!ARC_BUF_COMPRESSED(lastbuf));
3058
3059			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3060			arc_hdr_free_pabd(hdr);
3061
3062			/*
3063			 * We must setup a new shared block between the
3064			 * last buffer and the hdr. The data would have
3065			 * been allocated by the arc buf so we need to transfer
3066			 * ownership to the hdr since it's now being shared.
3067			 */
3068			arc_share_buf(hdr, lastbuf);
3069		}
3070	} else if (HDR_SHARED_DATA(hdr)) {
3071		/*
3072		 * Uncompressed shared buffers are always at the end
3073		 * of the list. Compressed buffers don't have the
3074		 * same requirements. This makes it hard to
3075		 * simply assert that the lastbuf is shared so
3076		 * we rely on the hdr's compression flags to determine
3077		 * if we have a compressed, shared buffer.
3078		 */
3079		ASSERT3P(lastbuf, !=, NULL);
3080		ASSERT(arc_buf_is_shared(lastbuf) ||
3081		    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
3082	}
3083
3084	/*
3085	 * Free the checksum if we're removing the last uncompressed buf from
3086	 * this hdr.
3087	 */
3088	if (!arc_hdr_has_uncompressed_buf(hdr)) {
3089		arc_cksum_free(hdr);
3090	}
3091
3092	/* clean up the buf */
3093	buf->b_hdr = NULL;
3094	kmem_cache_free(buf_cache, buf);
3095}
3096
3097static void
3098arc_hdr_alloc_pabd(arc_buf_hdr_t *hdr)
3099{
3100	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
3101	ASSERT(HDR_HAS_L1HDR(hdr));
3102	ASSERT(!HDR_SHARED_DATA(hdr));
3103
3104	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3105	hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr);
3106	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3107	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3108
3109	ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
3110	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
3111}
3112
3113static void
3114arc_hdr_free_pabd(arc_buf_hdr_t *hdr)
3115{
3116	ASSERT(HDR_HAS_L1HDR(hdr));
3117	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3118
3119	/*
3120	 * If the hdr is currently being written to the l2arc then
3121	 * we defer freeing the data by adding it to the l2arc_free_on_write
3122	 * list. The l2arc will free the data once it's finished
3123	 * writing it to the l2arc device.
3124	 */
3125	if (HDR_L2_WRITING(hdr)) {
3126		arc_hdr_free_on_write(hdr);
3127		ARCSTAT_BUMP(arcstat_l2_free_on_write);
3128	} else {
3129		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
3130		    arc_hdr_size(hdr), hdr);
3131	}
3132	hdr->b_l1hdr.b_pabd = NULL;
3133	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3134
3135	ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
3136	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3137}
3138
3139static arc_buf_hdr_t *
3140arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize,
3141    enum zio_compress compression_type, arc_buf_contents_t type)
3142{
3143	arc_buf_hdr_t *hdr;
3144
3145	VERIFY(type == ARC_BUFC_DATA || type == ARC_BUFC_METADATA);
3146
3147	hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
3148	ASSERT(HDR_EMPTY(hdr));
3149	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3150	ASSERT3P(hdr->b_l1hdr.b_thawed, ==, NULL);
3151	HDR_SET_PSIZE(hdr, psize);
3152	HDR_SET_LSIZE(hdr, lsize);
3153	hdr->b_spa = spa;
3154	hdr->b_type = type;
3155	hdr->b_flags = 0;
3156	arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR);
3157	arc_hdr_set_compress(hdr, compression_type);
3158
3159	hdr->b_l1hdr.b_state = arc_anon;
3160	hdr->b_l1hdr.b_arc_access = 0;
3161	hdr->b_l1hdr.b_bufcnt = 0;
3162	hdr->b_l1hdr.b_buf = NULL;
3163
3164	/*
3165	 * Allocate the hdr's buffer. This will contain either
3166	 * the compressed or uncompressed data depending on the block
3167	 * it references and compressed arc enablement.
3168	 */
3169	arc_hdr_alloc_pabd(hdr);
3170	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3171
3172	return (hdr);
3173}
3174
3175/*
3176 * Transition between the two allocation states for the arc_buf_hdr struct.
3177 * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
3178 * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
3179 * version is used when a cache buffer is only in the L2ARC in order to reduce
3180 * memory usage.
3181 */
3182static arc_buf_hdr_t *
3183arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
3184{
3185	ASSERT(HDR_HAS_L2HDR(hdr));
3186
3187	arc_buf_hdr_t *nhdr;
3188	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3189
3190	ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) ||
3191	    (old == hdr_l2only_cache && new == hdr_full_cache));
3192
3193	nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);
3194
3195	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
3196	buf_hash_remove(hdr);
3197
3198	bcopy(hdr, nhdr, HDR_L2ONLY_SIZE);
3199
3200	if (new == hdr_full_cache) {
3201		arc_hdr_set_flags(nhdr, ARC_FLAG_HAS_L1HDR);
3202		/*
3203		 * arc_access and arc_change_state need to be aware that a
3204		 * header has just come out of L2ARC, so we set its state to
3205		 * l2c_only even though it's about to change.
3206		 */
3207		nhdr->b_l1hdr.b_state = arc_l2c_only;
3208
3209		/* Verify previous threads set to NULL before freeing */
3210		ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL);
3211	} else {
3212		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3213		ASSERT0(hdr->b_l1hdr.b_bufcnt);
3214		ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3215
3216		/*
3217		 * If we've reached here, We must have been called from
3218		 * arc_evict_hdr(), as such we should have already been
3219		 * removed from any ghost list we were previously on
3220		 * (which protects us from racing with arc_evict_state),
3221		 * thus no locking is needed during this check.
3222		 */
3223		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3224
3225		/*
3226		 * A buffer must not be moved into the arc_l2c_only
3227		 * state if it's not finished being written out to the
3228		 * l2arc device. Otherwise, the b_l1hdr.b_pabd field
3229		 * might try to be accessed, even though it was removed.
3230		 */
3231		VERIFY(!HDR_L2_WRITING(hdr));
3232		VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3233
3234#ifdef ZFS_DEBUG
3235		if (hdr->b_l1hdr.b_thawed != NULL) {
3236			kmem_free(hdr->b_l1hdr.b_thawed, 1);
3237			hdr->b_l1hdr.b_thawed = NULL;
3238		}
3239#endif
3240
3241		arc_hdr_clear_flags(nhdr, ARC_FLAG_HAS_L1HDR);
3242	}
3243	/*
3244	 * The header has been reallocated so we need to re-insert it into any
3245	 * lists it was on.
3246	 */
3247	(void) buf_hash_insert(nhdr, NULL);
3248
3249	ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
3250
3251	mutex_enter(&dev->l2ad_mtx);
3252
3253	/*
3254	 * We must place the realloc'ed header back into the list at
3255	 * the same spot. Otherwise, if it's placed earlier in the list,
3256	 * l2arc_write_buffers() could find it during the function's
3257	 * write phase, and try to write it out to the l2arc.
3258	 */
3259	list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
3260	list_remove(&dev->l2ad_buflist, hdr);
3261
3262	mutex_exit(&dev->l2ad_mtx);
3263
3264	/*
3265	 * Since we're using the pointer address as the tag when
3266	 * incrementing and decrementing the l2ad_alloc refcount, we
3267	 * must remove the old pointer (that we're about to destroy) and
3268	 * add the new pointer to the refcount. Otherwise we'd remove
3269	 * the wrong pointer address when calling arc_hdr_destroy() later.
3270	 */
3271
3272	(void) refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr);
3273	(void) refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(nhdr), nhdr);
3274
3275	buf_discard_identity(hdr);
3276	kmem_cache_free(old, hdr);
3277
3278	return (nhdr);
3279}
3280
3281/*
3282 * Allocate a new arc_buf_hdr_t and arc_buf_t and return the buf to the caller.
3283 * The buf is returned thawed since we expect the consumer to modify it.
3284 */
3285arc_buf_t *
3286arc_alloc_buf(spa_t *spa, void *tag, arc_buf_contents_t type, int32_t size)
3287{
3288	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size,
3289	    ZIO_COMPRESS_OFF, type);
3290	ASSERT(!MUTEX_HELD(HDR_LOCK(hdr)));
3291
3292	arc_buf_t *buf = NULL;
3293	VERIFY0(arc_buf_alloc_impl(hdr, tag, B_FALSE, B_FALSE, &buf));
3294	arc_buf_thaw(buf);
3295
3296	return (buf);
3297}
3298
3299/*
3300 * Allocate a compressed buf in the same manner as arc_alloc_buf. Don't use this
3301 * for bufs containing metadata.
3302 */
3303arc_buf_t *
3304arc_alloc_compressed_buf(spa_t *spa, void *tag, uint64_t psize, uint64_t lsize,
3305    enum zio_compress compression_type)
3306{
3307	ASSERT3U(lsize, >, 0);
3308	ASSERT3U(lsize, >=, psize);
3309	ASSERT(compression_type > ZIO_COMPRESS_OFF);
3310	ASSERT(compression_type < ZIO_COMPRESS_FUNCTIONS);
3311
3312	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
3313	    compression_type, ARC_BUFC_DATA);
3314	ASSERT(!MUTEX_HELD(HDR_LOCK(hdr)));
3315
3316	arc_buf_t *buf = NULL;
3317	VERIFY0(arc_buf_alloc_impl(hdr, tag, B_TRUE, B_FALSE, &buf));
3318	arc_buf_thaw(buf);
3319	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3320
3321	if (!arc_buf_is_shared(buf)) {
3322		/*
3323		 * To ensure that the hdr has the correct data in it if we call
3324		 * arc_decompress() on this buf before it's been written to
3325		 * disk, it's easiest if we just set up sharing between the
3326		 * buf and the hdr.
3327		 */
3328		ASSERT(!abd_is_linear(hdr->b_l1hdr.b_pabd));
3329		arc_hdr_free_pabd(hdr);
3330		arc_share_buf(hdr, buf);
3331	}
3332
3333	return (buf);
3334}
3335
3336static void
3337arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
3338{
3339	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
3340	l2arc_dev_t *dev = l2hdr->b_dev;
3341	uint64_t asize = arc_hdr_size(hdr);
3342
3343	ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
3344	ASSERT(HDR_HAS_L2HDR(hdr));
3345
3346	list_remove(&dev->l2ad_buflist, hdr);
3347
3348	ARCSTAT_INCR(arcstat_l2_asize, -asize);
3349	ARCSTAT_INCR(arcstat_l2_size, -HDR_GET_LSIZE(hdr));
3350
3351	vdev_space_update(dev->l2ad_vdev, -asize, 0, 0);
3352
3353	(void) refcount_remove_many(&dev->l2ad_alloc, asize, hdr);
3354	arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
3355}
3356
3357static void
3358arc_hdr_destroy(arc_buf_hdr_t *hdr)
3359{
3360	if (HDR_HAS_L1HDR(hdr)) {
3361		ASSERT(hdr->b_l1hdr.b_buf == NULL ||
3362		    hdr->b_l1hdr.b_bufcnt > 0);
3363		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3364		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3365	}
3366	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3367	ASSERT(!HDR_IN_HASH_TABLE(hdr));
3368
3369	if (!HDR_EMPTY(hdr))
3370		buf_discard_identity(hdr);
3371
3372	if (HDR_HAS_L2HDR(hdr)) {
3373		l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3374		boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx);
3375
3376		if (!buflist_held)
3377			mutex_enter(&dev->l2ad_mtx);
3378
3379		/*
3380		 * Even though we checked this conditional above, we
3381		 * need to check this again now that we have the
3382		 * l2ad_mtx. This is because we could be racing with
3383		 * another thread calling l2arc_evict() which might have
3384		 * destroyed this header's L2 portion as we were waiting
3385		 * to acquire the l2ad_mtx. If that happens, we don't
3386		 * want to re-destroy the header's L2 portion.
3387		 */
3388		if (HDR_HAS_L2HDR(hdr)) {
3389			l2arc_trim(hdr);
3390			arc_hdr_l2hdr_destroy(hdr);
3391		}
3392
3393		if (!buflist_held)
3394			mutex_exit(&dev->l2ad_mtx);
3395	}
3396
3397	if (HDR_HAS_L1HDR(hdr)) {
3398		arc_cksum_free(hdr);
3399
3400		while (hdr->b_l1hdr.b_buf != NULL)
3401			arc_buf_destroy_impl(hdr->b_l1hdr.b_buf);
3402
3403#ifdef ZFS_DEBUG
3404		if (hdr->b_l1hdr.b_thawed != NULL) {
3405			kmem_free(hdr->b_l1hdr.b_thawed, 1);
3406			hdr->b_l1hdr.b_thawed = NULL;
3407		}
3408#endif
3409
3410		if (hdr->b_l1hdr.b_pabd != NULL) {
3411			arc_hdr_free_pabd(hdr);
3412		}
3413	}
3414
3415	ASSERT3P(hdr->b_hash_next, ==, NULL);
3416	if (HDR_HAS_L1HDR(hdr)) {
3417		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3418		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
3419		kmem_cache_free(hdr_full_cache, hdr);
3420	} else {
3421		kmem_cache_free(hdr_l2only_cache, hdr);
3422	}
3423}
3424
3425void
3426arc_buf_destroy(arc_buf_t *buf, void* tag)
3427{
3428	arc_buf_hdr_t *hdr = buf->b_hdr;
3429	kmutex_t *hash_lock = HDR_LOCK(hdr);
3430
3431	if (hdr->b_l1hdr.b_state == arc_anon) {
3432		ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
3433		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3434		VERIFY0(remove_reference(hdr, NULL, tag));
3435		arc_hdr_destroy(hdr);
3436		return;
3437	}
3438
3439	mutex_enter(hash_lock);
3440	ASSERT3P(hdr, ==, buf->b_hdr);
3441	ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
3442	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3443	ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
3444	ASSERT3P(buf->b_data, !=, NULL);
3445
3446	(void) remove_reference(hdr, hash_lock, tag);
3447	arc_buf_destroy_impl(buf);
3448	mutex_exit(hash_lock);
3449}
3450
3451/*
3452 * Evict the arc_buf_hdr that is provided as a parameter. The resultant
3453 * state of the header is dependent on its state prior to entering this
3454 * function. The following transitions are possible:
3455 *
3456 *    - arc_mru -> arc_mru_ghost
3457 *    - arc_mfu -> arc_mfu_ghost
3458 *    - arc_mru_ghost -> arc_l2c_only
3459 *    - arc_mru_ghost -> deleted
3460 *    - arc_mfu_ghost -> arc_l2c_only
3461 *    - arc_mfu_ghost -> deleted
3462 */
3463static int64_t
3464arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
3465{
3466	arc_state_t *evicted_state, *state;
3467	int64_t bytes_evicted = 0;
3468
3469	ASSERT(MUTEX_HELD(hash_lock));
3470	ASSERT(HDR_HAS_L1HDR(hdr));
3471
3472	state = hdr->b_l1hdr.b_state;
3473	if (GHOST_STATE(state)) {
3474		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3475		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3476
3477		/*
3478		 * l2arc_write_buffers() relies on a header's L1 portion
3479		 * (i.e. its b_pabd field) during it's write phase.
3480		 * Thus, we cannot push a header onto the arc_l2c_only
3481		 * state (removing it's L1 piece) until the header is
3482		 * done being written to the l2arc.
3483		 */
3484		if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
3485			ARCSTAT_BUMP(arcstat_evict_l2_skip);
3486			return (bytes_evicted);
3487		}
3488
3489		ARCSTAT_BUMP(arcstat_deleted);
3490		bytes_evicted += HDR_GET_LSIZE(hdr);
3491
3492		DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
3493
3494		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3495		if (HDR_HAS_L2HDR(hdr)) {
3496			/*
3497			 * This buffer is cached on the 2nd Level ARC;
3498			 * don't destroy the header.
3499			 */
3500			arc_change_state(arc_l2c_only, hdr, hash_lock);
3501			/*
3502			 * dropping from L1+L2 cached to L2-only,
3503			 * realloc to remove the L1 header.
3504			 */
3505			hdr = arc_hdr_realloc(hdr, hdr_full_cache,
3506			    hdr_l2only_cache);
3507		} else {
3508			arc_change_state(arc_anon, hdr, hash_lock);
3509			arc_hdr_destroy(hdr);
3510		}
3511		return (bytes_evicted);
3512	}
3513
3514	ASSERT(state == arc_mru || state == arc_mfu);
3515	evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
3516
3517	/* prefetch buffers have a minimum lifespan */
3518	if (HDR_IO_IN_PROGRESS(hdr) ||
3519	    ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
3520	    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
3521	    arc_min_prefetch_lifespan)) {
3522		ARCSTAT_BUMP(arcstat_evict_skip);
3523		return (bytes_evicted);
3524	}
3525
3526	ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
3527	while (hdr->b_l1hdr.b_buf) {
3528		arc_buf_t *buf = hdr->b_l1hdr.b_buf;
3529		if (!mutex_tryenter(&buf->b_evict_lock)) {
3530			ARCSTAT_BUMP(arcstat_mutex_miss);
3531			break;
3532		}
3533		if (buf->b_data != NULL)
3534			bytes_evicted += HDR_GET_LSIZE(hdr);
3535		mutex_exit(&buf->b_evict_lock);
3536		arc_buf_destroy_impl(buf);
3537	}
3538
3539	if (HDR_HAS_L2HDR(hdr)) {
3540		ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr));
3541	} else {
3542		if (l2arc_write_eligible(hdr->b_spa, hdr)) {
3543			ARCSTAT_INCR(arcstat_evict_l2_eligible,
3544			    HDR_GET_LSIZE(hdr));
3545		} else {
3546			ARCSTAT_INCR(arcstat_evict_l2_ineligible,
3547			    HDR_GET_LSIZE(hdr));
3548		}
3549	}
3550
3551	if (hdr->b_l1hdr.b_bufcnt == 0) {
3552		arc_cksum_free(hdr);
3553
3554		bytes_evicted += arc_hdr_size(hdr);
3555
3556		/*
3557		 * If this hdr is being evicted and has a compressed
3558		 * buffer then we discard it here before we change states.
3559		 * This ensures that the accounting is updated correctly
3560		 * in arc_free_data_impl().
3561		 */
3562		arc_hdr_free_pabd(hdr);
3563
3564		arc_change_state(evicted_state, hdr, hash_lock);
3565		ASSERT(HDR_IN_HASH_TABLE(hdr));
3566		arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
3567		DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
3568	}
3569
3570	return (bytes_evicted);
3571}
3572
3573static uint64_t
3574arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker,
3575    uint64_t spa, int64_t bytes)
3576{
3577	multilist_sublist_t *mls;
3578	uint64_t bytes_evicted = 0;
3579	arc_buf_hdr_t *hdr;
3580	kmutex_t *hash_lock;
3581	int evict_count = 0;
3582
3583	ASSERT3P(marker, !=, NULL);
3584	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
3585
3586	mls = multilist_sublist_lock(ml, idx);
3587
3588	for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL;
3589	    hdr = multilist_sublist_prev(mls, marker)) {
3590		if ((bytes != ARC_EVICT_ALL && bytes_evicted >= bytes) ||
3591		    (evict_count >= zfs_arc_evict_batch_limit))
3592			break;
3593
3594		/*
3595		 * To keep our iteration location, move the marker
3596		 * forward. Since we're not holding hdr's hash lock, we
3597		 * must be very careful and not remove 'hdr' from the
3598		 * sublist. Otherwise, other consumers might mistake the
3599		 * 'hdr' as not being on a sublist when they call the
3600		 * multilist_link_active() function (they all rely on
3601		 * the hash lock protecting concurrent insertions and
3602		 * removals). multilist_sublist_move_forward() was
3603		 * specifically implemented to ensure this is the case
3604		 * (only 'marker' will be removed and re-inserted).
3605		 */
3606		multilist_sublist_move_forward(mls, marker);
3607
3608		/*
3609		 * The only case where the b_spa field should ever be
3610		 * zero, is the marker headers inserted by
3611		 * arc_evict_state(). It's possible for multiple threads
3612		 * to be calling arc_evict_state() concurrently (e.g.
3613		 * dsl_pool_close() and zio_inject_fault()), so we must
3614		 * skip any markers we see from these other threads.
3615		 */
3616		if (hdr->b_spa == 0)
3617			continue;
3618
3619		/* we're only interested in evicting buffers of a certain spa */
3620		if (spa != 0 && hdr->b_spa != spa) {
3621			ARCSTAT_BUMP(arcstat_evict_skip);
3622			continue;
3623		}
3624
3625		hash_lock = HDR_LOCK(hdr);
3626
3627		/*
3628		 * We aren't calling this function from any code path
3629		 * that would already be holding a hash lock, so we're
3630		 * asserting on this assumption to be defensive in case
3631		 * this ever changes. Without this check, it would be
3632		 * possible to incorrectly increment arcstat_mutex_miss
3633		 * below (e.g. if the code changed such that we called
3634		 * this function with a hash lock held).
3635		 */
3636		ASSERT(!MUTEX_HELD(hash_lock));
3637
3638		if (mutex_tryenter(hash_lock)) {
3639			uint64_t evicted = arc_evict_hdr(hdr, hash_lock);
3640			mutex_exit(hash_lock);
3641
3642			bytes_evicted += evicted;
3643
3644			/*
3645			 * If evicted is zero, arc_evict_hdr() must have
3646			 * decided to skip this header, don't increment
3647			 * evict_count in this case.
3648			 */
3649			if (evicted != 0)
3650				evict_count++;
3651
3652			/*
3653			 * If arc_size isn't overflowing, signal any
3654			 * threads that might happen to be waiting.
3655			 *
3656			 * For each header evicted, we wake up a single
3657			 * thread. If we used cv_broadcast, we could
3658			 * wake up "too many" threads causing arc_size
3659			 * to significantly overflow arc_c; since
3660			 * arc_get_data_impl() doesn't check for overflow
3661			 * when it's woken up (it doesn't because it's
3662			 * possible for the ARC to be overflowing while
3663			 * full of un-evictable buffers, and the
3664			 * function should proceed in this case).
3665			 *
3666			 * If threads are left sleeping, due to not
3667			 * using cv_broadcast, they will be woken up
3668			 * just before arc_reclaim_thread() sleeps.
3669			 */
3670			mutex_enter(&arc_reclaim_lock);
3671			if (!arc_is_overflowing())
3672				cv_signal(&arc_reclaim_waiters_cv);
3673			mutex_exit(&arc_reclaim_lock);
3674		} else {
3675			ARCSTAT_BUMP(arcstat_mutex_miss);
3676		}
3677	}
3678
3679	multilist_sublist_unlock(mls);
3680
3681	return (bytes_evicted);
3682}
3683
3684/*
3685 * Evict buffers from the given arc state, until we've removed the
3686 * specified number of bytes. Move the removed buffers to the
3687 * appropriate evict state.
3688 *
3689 * This function makes a "best effort". It skips over any buffers
3690 * it can't get a hash_lock on, and so, may not catch all candidates.
3691 * It may also return without evicting as much space as requested.
3692 *
3693 * If bytes is specified using the special value ARC_EVICT_ALL, this
3694 * will evict all available (i.e. unlocked and evictable) buffers from
3695 * the given arc state; which is used by arc_flush().
3696 */
3697static uint64_t
3698arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
3699    arc_buf_contents_t type)
3700{
3701	uint64_t total_evicted = 0;
3702	multilist_t *ml = state->arcs_list[type];
3703	int num_sublists;
3704	arc_buf_hdr_t **markers;
3705
3706	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
3707
3708	num_sublists = multilist_get_num_sublists(ml);
3709
3710	/*
3711	 * If we've tried to evict from each sublist, made some
3712	 * progress, but still have not hit the target number of bytes
3713	 * to evict, we want to keep trying. The markers allow us to
3714	 * pick up where we left off for each individual sublist, rather
3715	 * than starting from the tail each time.
3716	 */
3717	markers = kmem_zalloc(sizeof (*markers) * num_sublists, KM_SLEEP);
3718	for (int i = 0; i < num_sublists; i++) {
3719		markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
3720
3721		/*
3722		 * A b_spa of 0 is used to indicate that this header is
3723		 * a marker. This fact is used in arc_adjust_type() and
3724		 * arc_evict_state_impl().
3725		 */
3726		markers[i]->b_spa = 0;
3727
3728		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
3729		multilist_sublist_insert_tail(mls, markers[i]);
3730		multilist_sublist_unlock(mls);
3731	}
3732
3733	/*
3734	 * While we haven't hit our target number of bytes to evict, or
3735	 * we're evicting all available buffers.
3736	 */
3737	while (total_evicted < bytes || bytes == ARC_EVICT_ALL) {
3738		/*
3739		 * Start eviction using a randomly selected sublist,
3740		 * this is to try and evenly balance eviction across all
3741		 * sublists. Always starting at the same sublist
3742		 * (e.g. index 0) would cause evictions to favor certain
3743		 * sublists over others.
3744		 */
3745		int sublist_idx = multilist_get_random_index(ml);
3746		uint64_t scan_evicted = 0;
3747
3748		for (int i = 0; i < num_sublists; i++) {
3749			uint64_t bytes_remaining;
3750			uint64_t bytes_evicted;
3751
3752			if (bytes == ARC_EVICT_ALL)
3753				bytes_remaining = ARC_EVICT_ALL;
3754			else if (total_evicted < bytes)
3755				bytes_remaining = bytes - total_evicted;
3756			else
3757				break;
3758
3759			bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
3760			    markers[sublist_idx], spa, bytes_remaining);
3761
3762			scan_evicted += bytes_evicted;
3763			total_evicted += bytes_evicted;
3764
3765			/* we've reached the end, wrap to the beginning */
3766			if (++sublist_idx >= num_sublists)
3767				sublist_idx = 0;
3768		}
3769
3770		/*
3771		 * If we didn't evict anything during this scan, we have
3772		 * no reason to believe we'll evict more during another
3773		 * scan, so break the loop.
3774		 */
3775		if (scan_evicted == 0) {
3776			/* This isn't possible, let's make that obvious */
3777			ASSERT3S(bytes, !=, 0);
3778
3779			/*
3780			 * When bytes is ARC_EVICT_ALL, the only way to
3781			 * break the loop is when scan_evicted is zero.
3782			 * In that case, we actually have evicted enough,
3783			 * so we don't want to increment the kstat.
3784			 */
3785			if (bytes != ARC_EVICT_ALL) {
3786				ASSERT3S(total_evicted, <, bytes);
3787				ARCSTAT_BUMP(arcstat_evict_not_enough);
3788			}
3789
3790			break;
3791		}
3792	}
3793
3794	for (int i = 0; i < num_sublists; i++) {
3795		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
3796		multilist_sublist_remove(mls, markers[i]);
3797		multilist_sublist_unlock(mls);
3798
3799		kmem_cache_free(hdr_full_cache, markers[i]);
3800	}
3801	kmem_free(markers, sizeof (*markers) * num_sublists);
3802
3803	return (total_evicted);
3804}
3805
3806/*
3807 * Flush all "evictable" data of the given type from the arc state
3808 * specified. This will not evict any "active" buffers (i.e. referenced).
3809 *
3810 * When 'retry' is set to B_FALSE, the function will make a single pass
3811 * over the state and evict any buffers that it can. Since it doesn't
3812 * continually retry the eviction, it might end up leaving some buffers
3813 * in the ARC due to lock misses.
3814 *
3815 * When 'retry' is set to B_TRUE, the function will continually retry the
3816 * eviction until *all* evictable buffers have been removed from the
3817 * state. As a result, if concurrent insertions into the state are
3818 * allowed (e.g. if the ARC isn't shutting down), this function might
3819 * wind up in an infinite loop, continually trying to evict buffers.
3820 */
3821static uint64_t
3822arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
3823    boolean_t retry)
3824{
3825	uint64_t evicted = 0;
3826
3827	while (refcount_count(&state->arcs_esize[type]) != 0) {
3828		evicted += arc_evict_state(state, spa, ARC_EVICT_ALL, type);
3829
3830		if (!retry)
3831			break;
3832	}
3833
3834	return (evicted);
3835}
3836
3837/*
3838 * Evict the specified number of bytes from the state specified,
3839 * restricting eviction to the spa and type given. This function
3840 * prevents us from trying to evict more from a state's list than
3841 * is "evictable", and to skip evicting altogether when passed a
3842 * negative value for "bytes". In contrast, arc_evict_state() will
3843 * evict everything it can, when passed a negative value for "bytes".
3844 */
3845static uint64_t
3846arc_adjust_impl(arc_state_t *state, uint64_t spa, int64_t bytes,
3847    arc_buf_contents_t type)
3848{
3849	int64_t delta;
3850
3851	if (bytes > 0 && refcount_count(&state->arcs_esize[type]) > 0) {
3852		delta = MIN(refcount_count(&state->arcs_esize[type]), bytes);
3853		return (arc_evict_state(state, spa, delta, type));
3854	}
3855
3856	return (0);
3857}
3858
3859/*
3860 * Evict metadata buffers from the cache, such that arc_meta_used is
3861 * capped by the arc_meta_limit tunable.
3862 */
3863static uint64_t
3864arc_adjust_meta(void)
3865{
3866	uint64_t total_evicted = 0;
3867	int64_t target;
3868
3869	/*
3870	 * If we're over the meta limit, we want to evict enough
3871	 * metadata to get back under the meta limit. We don't want to
3872	 * evict so much that we drop the MRU below arc_p, though. If
3873	 * we're over the meta limit more than we're over arc_p, we
3874	 * evict some from the MRU here, and some from the MFU below.
3875	 */
3876	target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
3877	    (int64_t)(refcount_count(&arc_anon->arcs_size) +
3878	    refcount_count(&arc_mru->arcs_size) - arc_p));
3879
3880	total_evicted += arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
3881
3882	/*
3883	 * Similar to the above, we want to evict enough bytes to get us
3884	 * below the meta limit, but not so much as to drop us below the
3885	 * space allotted to the MFU (which is defined as arc_c - arc_p).
3886	 */
3887	target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
3888	    (int64_t)(refcount_count(&arc_mfu->arcs_size) - (arc_c - arc_p)));
3889
3890	total_evicted += arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
3891
3892	return (total_evicted);
3893}
3894
3895/*
3896 * Return the type of the oldest buffer in the given arc state
3897 *
3898 * This function will select a random sublist of type ARC_BUFC_DATA and
3899 * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist
3900 * is compared, and the type which contains the "older" buffer will be
3901 * returned.
3902 */
3903static arc_buf_contents_t
3904arc_adjust_type(arc_state_t *state)
3905{
3906	multilist_t *data_ml = state->arcs_list[ARC_BUFC_DATA];
3907	multilist_t *meta_ml = state->arcs_list[ARC_BUFC_METADATA];
3908	int data_idx = multilist_get_random_index(data_ml);
3909	int meta_idx = multilist_get_random_index(meta_ml);
3910	multilist_sublist_t *data_mls;
3911	multilist_sublist_t *meta_mls;
3912	arc_buf_contents_t type;
3913	arc_buf_hdr_t *data_hdr;
3914	arc_buf_hdr_t *meta_hdr;
3915
3916	/*
3917	 * We keep the sublist lock until we're finished, to prevent
3918	 * the headers from being destroyed via arc_evict_state().
3919	 */
3920	data_mls = multilist_sublist_lock(data_ml, data_idx);
3921	meta_mls = multilist_sublist_lock(meta_ml, meta_idx);
3922
3923	/*
3924	 * These two loops are to ensure we skip any markers that
3925	 * might be at the tail of the lists due to arc_evict_state().
3926	 */
3927
3928	for (data_hdr = multilist_sublist_tail(data_mls); data_hdr != NULL;
3929	    data_hdr = multilist_sublist_prev(data_mls, data_hdr)) {
3930		if (data_hdr->b_spa != 0)
3931			break;
3932	}
3933
3934	for (meta_hdr = multilist_sublist_tail(meta_mls); meta_hdr != NULL;
3935	    meta_hdr = multilist_sublist_prev(meta_mls, meta_hdr)) {
3936		if (meta_hdr->b_spa != 0)
3937			break;
3938	}
3939
3940	if (data_hdr == NULL && meta_hdr == NULL) {
3941		type = ARC_BUFC_DATA;
3942	} else if (data_hdr == NULL) {
3943		ASSERT3P(meta_hdr, !=, NULL);
3944		type = ARC_BUFC_METADATA;
3945	} else if (meta_hdr == NULL) {
3946		ASSERT3P(data_hdr, !=, NULL);
3947		type = ARC_BUFC_DATA;
3948	} else {
3949		ASSERT3P(data_hdr, !=, NULL);
3950		ASSERT3P(meta_hdr, !=, NULL);
3951
3952		/* The headers can't be on the sublist without an L1 header */
3953		ASSERT(HDR_HAS_L1HDR(data_hdr));
3954		ASSERT(HDR_HAS_L1HDR(meta_hdr));
3955
3956		if (data_hdr->b_l1hdr.b_arc_access <
3957		    meta_hdr->b_l1hdr.b_arc_access) {
3958			type = ARC_BUFC_DATA;
3959		} else {
3960			type = ARC_BUFC_METADATA;
3961		}
3962	}
3963
3964	multilist_sublist_unlock(meta_mls);
3965	multilist_sublist_unlock(data_mls);
3966
3967	return (type);
3968}
3969
3970/*
3971 * Evict buffers from the cache, such that arc_size is capped by arc_c.
3972 */
3973static uint64_t
3974arc_adjust(void)
3975{
3976	uint64_t total_evicted = 0;
3977	uint64_t bytes;
3978	int64_t target;
3979
3980	/*
3981	 * If we're over arc_meta_limit, we want to correct that before
3982	 * potentially evicting data buffers below.
3983	 */
3984	total_evicted += arc_adjust_meta();
3985
3986	/*
3987	 * Adjust MRU size
3988	 *
3989	 * If we're over the target cache size, we want to evict enough
3990	 * from the list to get back to our target size. We don't want
3991	 * to evict too much from the MRU, such that it drops below
3992	 * arc_p. So, if we're over our target cache size more than
3993	 * the MRU is over arc_p, we'll evict enough to get back to
3994	 * arc_p here, and then evict more from the MFU below.
3995	 */
3996	target = MIN((int64_t)(arc_size - arc_c),
3997	    (int64_t)(refcount_count(&arc_anon->arcs_size) +
3998	    refcount_count(&arc_mru->arcs_size) + arc_meta_used - arc_p));
3999
4000	/*
4001	 * If we're below arc_meta_min, always prefer to evict data.
4002	 * Otherwise, try to satisfy the requested number of bytes to
4003	 * evict from the type which contains older buffers; in an
4004	 * effort to keep newer buffers in the cache regardless of their
4005	 * type. If we cannot satisfy the number of bytes from this
4006	 * type, spill over into the next type.
4007	 */
4008	if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
4009	    arc_meta_used > arc_meta_min) {
4010		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
4011		total_evicted += bytes;
4012
4013		/*
4014		 * If we couldn't evict our target number of bytes from
4015		 * metadata, we try to get the rest from data.
4016		 */
4017		target -= bytes;
4018
4019		total_evicted +=
4020		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
4021	} else {
4022		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
4023		total_evicted += bytes;
4024
4025		/*
4026		 * If we couldn't evict our target number of bytes from
4027		 * data, we try to get the rest from metadata.
4028		 */
4029		target -= bytes;
4030
4031		total_evicted +=
4032		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
4033	}
4034
4035	/*
4036	 * Adjust MFU size
4037	 *
4038	 * Now that we've tried to evict enough from the MRU to get its
4039	 * size back to arc_p, if we're still above the target cache
4040	 * size, we evict the rest from the MFU.
4041	 */
4042	target = arc_size - arc_c;
4043
4044	if (arc_adjust_type(arc_mfu) == ARC_BUFC_METADATA &&
4045	    arc_meta_used > arc_meta_min) {
4046		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
4047		total_evicted += bytes;
4048
4049		/*
4050		 * If we couldn't evict our target number of bytes from
4051		 * metadata, we try to get the rest from data.
4052		 */
4053		target -= bytes;
4054
4055		total_evicted +=
4056		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
4057	} else {
4058		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
4059		total_evicted += bytes;
4060
4061		/*
4062		 * If we couldn't evict our target number of bytes from
4063		 * data, we try to get the rest from data.
4064		 */
4065		target -= bytes;
4066
4067		total_evicted +=
4068		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
4069	}
4070
4071	/*
4072	 * Adjust ghost lists
4073	 *
4074	 * In addition to the above, the ARC also defines target values
4075	 * for the ghost lists. The sum of the mru list and mru ghost
4076	 * list should never exceed the target size of the cache, and
4077	 * the sum of the mru list, mfu list, mru ghost list, and mfu
4078	 * ghost list should never exceed twice the target size of the
4079	 * cache. The following logic enforces these limits on the ghost
4080	 * caches, and evicts from them as needed.
4081	 */
4082	target = refcount_count(&arc_mru->arcs_size) +
4083	    refcount_count(&arc_mru_ghost->arcs_size) - arc_c;
4084
4085	bytes = arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_DATA);
4086	total_evicted += bytes;
4087
4088	target -= bytes;
4089
4090	total_evicted +=
4091	    arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_METADATA);
4092
4093	/*
4094	 * We assume the sum of the mru list and mfu list is less than
4095	 * or equal to arc_c (we enforced this above), which means we
4096	 * can use the simpler of the two equations below:
4097	 *
4098	 *	mru + mfu + mru ghost + mfu ghost <= 2 * arc_c
4099	 *		    mru ghost + mfu ghost <= arc_c
4100	 */
4101	target = refcount_count(&arc_mru_ghost->arcs_size) +
4102	    refcount_count(&arc_mfu_ghost->arcs_size) - arc_c;
4103
4104	bytes = arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_DATA);
4105	total_evicted += bytes;
4106
4107	target -= bytes;
4108
4109	total_evicted +=
4110	    arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_METADATA);
4111
4112	return (total_evicted);
4113}
4114
4115void
4116arc_flush(spa_t *spa, boolean_t retry)
4117{
4118	uint64_t guid = 0;
4119
4120	/*
4121	 * If retry is B_TRUE, a spa must not be specified since we have
4122	 * no good way to determine if all of a spa's buffers have been
4123	 * evicted from an arc state.
4124	 */
4125	ASSERT(!retry || spa == 0);
4126
4127	if (spa != NULL)
4128		guid = spa_load_guid(spa);
4129
4130	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
4131	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);
4132
4133	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
4134	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);
4135
4136	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
4137	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);
4138
4139	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
4140	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);
4141}
4142
4143void
4144arc_shrink(int64_t to_free)
4145{
4146	if (arc_c > arc_c_min) {
4147		DTRACE_PROBE4(arc__shrink, uint64_t, arc_c, uint64_t,
4148			arc_c_min, uint64_t, arc_p, uint64_t, to_free);
4149		if (arc_c > arc_c_min + to_free)
4150			atomic_add_64(&arc_c, -to_free);
4151		else
4152			arc_c = arc_c_min;
4153
4154		atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
4155		if (arc_c > arc_size)
4156			arc_c = MAX(arc_size, arc_c_min);
4157		if (arc_p > arc_c)
4158			arc_p = (arc_c >> 1);
4159
4160		DTRACE_PROBE2(arc__shrunk, uint64_t, arc_c, uint64_t,
4161			arc_p);
4162
4163		ASSERT(arc_c >= arc_c_min);
4164		ASSERT((int64_t)arc_p >= 0);
4165	}
4166
4167	if (arc_size > arc_c) {
4168		DTRACE_PROBE2(arc__shrink_adjust, uint64_t, arc_size,
4169			uint64_t, arc_c);
4170		(void) arc_adjust();
4171	}
4172}
4173
4174static long needfree = 0;
4175
4176typedef enum free_memory_reason_t {
4177	FMR_UNKNOWN,
4178	FMR_NEEDFREE,
4179	FMR_LOTSFREE,
4180	FMR_SWAPFS_MINFREE,
4181	FMR_PAGES_PP_MAXIMUM,
4182	FMR_HEAP_ARENA,
4183	FMR_ZIO_ARENA,
4184	FMR_ZIO_FRAG,
4185} free_memory_reason_t;
4186
4187int64_t last_free_memory;
4188free_memory_reason_t last_free_reason;
4189
4190/*
4191 * Additional reserve of pages for pp_reserve.
4192 */
4193int64_t arc_pages_pp_reserve = 64;
4194
4195/*
4196 * Additional reserve of pages for swapfs.
4197 */
4198int64_t arc_swapfs_reserve = 64;
4199
4200/*
4201 * Return the amount of memory that can be consumed before reclaim will be
4202 * needed.  Positive if there is sufficient free memory, negative indicates
4203 * the amount of memory that needs to be freed up.
4204 */
4205static int64_t
4206arc_available_memory(void)
4207{
4208	int64_t lowest = INT64_MAX;
4209	int64_t n;
4210	free_memory_reason_t r = FMR_UNKNOWN;
4211
4212#ifdef _KERNEL
4213	if (needfree > 0) {
4214		n = PAGESIZE * (-needfree);
4215		if (n < lowest) {
4216			lowest = n;
4217			r = FMR_NEEDFREE;
4218		}
4219	}
4220
4221	/*
4222	 * Cooperate with pagedaemon when it's time for it to scan
4223	 * and reclaim some pages.
4224	 */
4225	n = PAGESIZE * ((int64_t)freemem - zfs_arc_free_target);
4226	if (n < lowest) {
4227		lowest = n;
4228		r = FMR_LOTSFREE;
4229	}
4230
4231#ifdef illumos
4232	/*
4233	 * check that we're out of range of the pageout scanner.  It starts to
4234	 * schedule paging if freemem is less than lotsfree and needfree.
4235	 * lotsfree is the high-water mark for pageout, and needfree is the
4236	 * number of needed free pages.  We add extra pages here to make sure
4237	 * the scanner doesn't start up while we're freeing memory.
4238	 */
4239	n = PAGESIZE * (freemem - lotsfree - needfree - desfree);
4240	if (n < lowest) {
4241		lowest = n;
4242		r = FMR_LOTSFREE;
4243	}
4244
4245	/*
4246	 * check to make sure that swapfs has enough space so that anon
4247	 * reservations can still succeed. anon_resvmem() checks that the
4248	 * availrmem is greater than swapfs_minfree, and the number of reserved
4249	 * swap pages.  We also add a bit of extra here just to prevent
4250	 * circumstances from getting really dire.
4251	 */
4252	n = PAGESIZE * (availrmem - swapfs_minfree - swapfs_reserve -
4253	    desfree - arc_swapfs_reserve);
4254	if (n < lowest) {
4255		lowest = n;
4256		r = FMR_SWAPFS_MINFREE;
4257	}
4258
4259
4260	/*
4261	 * Check that we have enough availrmem that memory locking (e.g., via
4262	 * mlock(3C) or memcntl(2)) can still succeed.  (pages_pp_maximum
4263	 * stores the number of pages that cannot be locked; when availrmem
4264	 * drops below pages_pp_maximum, page locking mechanisms such as
4265	 * page_pp_lock() will fail.)
4266	 */
4267	n = PAGESIZE * (availrmem - pages_pp_maximum -
4268	    arc_pages_pp_reserve);
4269	if (n < lowest) {
4270		lowest = n;
4271		r = FMR_PAGES_PP_MAXIMUM;
4272	}
4273
4274#endif	/* illumos */
4275#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC)
4276	/*
4277	 * If we're on an i386 platform, it's possible that we'll exhaust the
4278	 * kernel heap space before we ever run out of available physical
4279	 * memory.  Most checks of the size of the heap_area compare against
4280	 * tune.t_minarmem, which is the minimum available real memory that we
4281	 * can have in the system.  However, this is generally fixed at 25 pages
4282	 * which is so low that it's useless.  In this comparison, we seek to
4283	 * calculate the total heap-size, and reclaim if more than 3/4ths of the
4284	 * heap is allocated.  (Or, in the calculation, if less than 1/4th is
4285	 * free)
4286	 */
4287	n = (int64_t)vmem_size(heap_arena, VMEM_FREE) -
4288	    (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2);
4289	if (n < lowest) {
4290		lowest = n;
4291		r = FMR_HEAP_ARENA;
4292	}
4293#define	zio_arena	NULL
4294#else
4295#define	zio_arena	heap_arena
4296#endif
4297
4298	/*
4299	 * If zio data pages are being allocated out of a separate heap segment,
4300	 * then enforce that the size of available vmem for this arena remains
4301	 * above about 1/16th free.
4302	 *
4303	 * Note: The 1/16th arena free requirement was put in place
4304	 * to aggressively evict memory from the arc in order to avoid
4305	 * memory fragmentation issues.
4306	 */
4307	if (zio_arena != NULL) {
4308		n = (int64_t)vmem_size(zio_arena, VMEM_FREE) -
4309		    (vmem_size(zio_arena, VMEM_ALLOC) >> 4);
4310		if (n < lowest) {
4311			lowest = n;
4312			r = FMR_ZIO_ARENA;
4313		}
4314	}
4315
4316	/*
4317	 * Above limits know nothing about real level of KVA fragmentation.
4318	 * Start aggressive reclamation if too little sequential KVA left.
4319	 */
4320	if (lowest > 0) {
4321		n = (vmem_size(heap_arena, VMEM_MAXFREE) < SPA_MAXBLOCKSIZE) ?
4322		    -((int64_t)vmem_size(heap_arena, VMEM_ALLOC) >> 4) :
4323		    INT64_MAX;
4324		if (n < lowest) {
4325			lowest = n;
4326			r = FMR_ZIO_FRAG;
4327		}
4328	}
4329
4330#else	/* _KERNEL */
4331	/* Every 100 calls, free a small amount */
4332	if (spa_get_random(100) == 0)
4333		lowest = -1024;
4334#endif	/* _KERNEL */
4335
4336	last_free_memory = lowest;
4337	last_free_reason = r;
4338	DTRACE_PROBE2(arc__available_memory, int64_t, lowest, int, r);
4339	return (lowest);
4340}
4341
4342
4343/*
4344 * Determine if the system is under memory pressure and is asking
4345 * to reclaim memory. A return value of B_TRUE indicates that the system
4346 * is under memory pressure and that the arc should adjust accordingly.
4347 */
4348static boolean_t
4349arc_reclaim_needed(void)
4350{
4351	return (arc_available_memory() < 0);
4352}
4353
4354extern kmem_cache_t	*zio_buf_cache[];
4355extern kmem_cache_t	*zio_data_buf_cache[];
4356extern kmem_cache_t	*range_seg_cache;
4357extern kmem_cache_t	*abd_chunk_cache;
4358
4359static __noinline void
4360arc_kmem_reap_now(void)
4361{
4362	size_t			i;
4363	kmem_cache_t		*prev_cache = NULL;
4364	kmem_cache_t		*prev_data_cache = NULL;
4365
4366	DTRACE_PROBE(arc__kmem_reap_start);
4367#ifdef _KERNEL
4368	if (arc_meta_used >= arc_meta_limit) {
4369		/*
4370		 * We are exceeding our meta-data cache limit.
4371		 * Purge some DNLC entries to release holds on meta-data.
4372		 */
4373		dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
4374	}
4375#if defined(__i386)
4376	/*
4377	 * Reclaim unused memory from all kmem caches.
4378	 */
4379	kmem_reap();
4380#endif
4381#endif
4382
4383	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
4384		if (zio_buf_cache[i] != prev_cache) {
4385			prev_cache = zio_buf_cache[i];
4386			kmem_cache_reap_now(zio_buf_cache[i]);
4387		}
4388		if (zio_data_buf_cache[i] != prev_data_cache) {
4389			prev_data_cache = zio_data_buf_cache[i];
4390			kmem_cache_reap_now(zio_data_buf_cache[i]);
4391		}
4392	}
4393	kmem_cache_reap_now(abd_chunk_cache);
4394	kmem_cache_reap_now(buf_cache);
4395	kmem_cache_reap_now(hdr_full_cache);
4396	kmem_cache_reap_now(hdr_l2only_cache);
4397	kmem_cache_reap_now(range_seg_cache);
4398
4399#ifdef illumos
4400	if (zio_arena != NULL) {
4401		/*
4402		 * Ask the vmem arena to reclaim unused memory from its
4403		 * quantum caches.
4404		 */
4405		vmem_qcache_reap(zio_arena);
4406	}
4407#endif
4408	DTRACE_PROBE(arc__kmem_reap_end);
4409}
4410
4411/*
4412 * Threads can block in arc_get_data_impl() waiting for this thread to evict
4413 * enough data and signal them to proceed. When this happens, the threads in
4414 * arc_get_data_impl() are sleeping while holding the hash lock for their
4415 * particular arc header. Thus, we must be careful to never sleep on a
4416 * hash lock in this thread. This is to prevent the following deadlock:
4417 *
4418 *  - Thread A sleeps on CV in arc_get_data_impl() holding hash lock "L",
4419 *    waiting for the reclaim thread to signal it.
4420 *
4421 *  - arc_reclaim_thread() tries to acquire hash lock "L" using mutex_enter,
4422 *    fails, and goes to sleep forever.
4423 *
4424 * This possible deadlock is avoided by always acquiring a hash lock
4425 * using mutex_tryenter() from arc_reclaim_thread().
4426 */
4427static void
4428arc_reclaim_thread(void *dummy __unused)
4429{
4430	hrtime_t		growtime = 0;
4431	callb_cpr_t		cpr;
4432
4433	CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG);
4434
4435	mutex_enter(&arc_reclaim_lock);
4436	while (!arc_reclaim_thread_exit) {
4437		uint64_t evicted = 0;
4438
4439		/*
4440		 * This is necessary in order for the mdb ::arc dcmd to
4441		 * show up to date information. Since the ::arc command
4442		 * does not call the kstat's update function, without
4443		 * this call, the command may show stale stats for the
4444		 * anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even
4445		 * with this change, the data might be up to 1 second
4446		 * out of date; but that should suffice. The arc_state_t
4447		 * structures can be queried directly if more accurate
4448		 * information is needed.
4449		 */
4450		if (arc_ksp != NULL)
4451			arc_ksp->ks_update(arc_ksp, KSTAT_READ);
4452
4453		mutex_exit(&arc_reclaim_lock);
4454
4455		/*
4456		 * We call arc_adjust() before (possibly) calling
4457		 * arc_kmem_reap_now(), so that we can wake up
4458		 * arc_get_data_impl() sooner.
4459		 */
4460		evicted = arc_adjust();
4461
4462		int64_t free_memory = arc_available_memory();
4463		if (free_memory < 0) {
4464
4465			arc_no_grow = B_TRUE;
4466			arc_warm = B_TRUE;
4467
4468			/*
4469			 * Wait at least zfs_grow_retry (default 60) seconds
4470			 * before considering growing.
4471			 */
4472			growtime = gethrtime() + SEC2NSEC(arc_grow_retry);
4473
4474			arc_kmem_reap_now();
4475
4476			/*
4477			 * If we are still low on memory, shrink the ARC
4478			 * so that we have arc_shrink_min free space.
4479			 */
4480			free_memory = arc_available_memory();
4481
4482			int64_t to_free =
4483			    (arc_c >> arc_shrink_shift) - free_memory;
4484			if (to_free > 0) {
4485#ifdef _KERNEL
4486				to_free = MAX(to_free, ptob(needfree));
4487#endif
4488				arc_shrink(to_free);
4489			}
4490		} else if (free_memory < arc_c >> arc_no_grow_shift) {
4491			arc_no_grow = B_TRUE;
4492		} else if (gethrtime() >= growtime) {
4493			arc_no_grow = B_FALSE;
4494		}
4495
4496		mutex_enter(&arc_reclaim_lock);
4497
4498		/*
4499		 * If evicted is zero, we couldn't evict anything via
4500		 * arc_adjust(). This could be due to hash lock
4501		 * collisions, but more likely due to the majority of
4502		 * arc buffers being unevictable. Therefore, even if
4503		 * arc_size is above arc_c, another pass is unlikely to
4504		 * be helpful and could potentially cause us to enter an
4505		 * infinite loop.
4506		 */
4507		if (arc_size <= arc_c || evicted == 0) {
4508#ifdef _KERNEL
4509			needfree = 0;
4510#endif
4511			/*
4512			 * We're either no longer overflowing, or we
4513			 * can't evict anything more, so we should wake
4514			 * up any threads before we go to sleep.
4515			 */
4516			cv_broadcast(&arc_reclaim_waiters_cv);
4517
4518			/*
4519			 * Block until signaled, or after one second (we
4520			 * might need to perform arc_kmem_reap_now()
4521			 * even if we aren't being signalled)
4522			 */
4523			CALLB_CPR_SAFE_BEGIN(&cpr);
4524			(void) cv_timedwait_hires(&arc_reclaim_thread_cv,
4525			    &arc_reclaim_lock, SEC2NSEC(1), MSEC2NSEC(1), 0);
4526			CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_lock);
4527		}
4528	}
4529
4530	arc_reclaim_thread_exit = B_FALSE;
4531	cv_broadcast(&arc_reclaim_thread_cv);
4532	CALLB_CPR_EXIT(&cpr);		/* drops arc_reclaim_lock */
4533	thread_exit();
4534}
4535
4536static u_int arc_dnlc_evicts_arg;
4537extern struct vfsops zfs_vfsops;
4538
4539static void
4540arc_dnlc_evicts_thread(void *dummy __unused)
4541{
4542	callb_cpr_t cpr;
4543	u_int percent;
4544
4545	CALLB_CPR_INIT(&cpr, &arc_dnlc_evicts_lock, callb_generic_cpr, FTAG);
4546
4547	mutex_enter(&arc_dnlc_evicts_lock);
4548	while (!arc_dnlc_evicts_thread_exit) {
4549		CALLB_CPR_SAFE_BEGIN(&cpr);
4550		(void) cv_wait(&arc_dnlc_evicts_cv, &arc_dnlc_evicts_lock);
4551		CALLB_CPR_SAFE_END(&cpr, &arc_dnlc_evicts_lock);
4552		if (arc_dnlc_evicts_arg != 0) {
4553			percent = arc_dnlc_evicts_arg;
4554			mutex_exit(&arc_dnlc_evicts_lock);
4555#ifdef _KERNEL
4556			vnlru_free(desiredvnodes * percent / 100, &zfs_vfsops);
4557#endif
4558			mutex_enter(&arc_dnlc_evicts_lock);
4559			/*
4560			 * Clear our token only after vnlru_free()
4561			 * pass is done, to avoid false queueing of
4562			 * the requests.
4563			 */
4564			arc_dnlc_evicts_arg = 0;
4565		}
4566	}
4567	arc_dnlc_evicts_thread_exit = FALSE;
4568	cv_broadcast(&arc_dnlc_evicts_cv);
4569	CALLB_CPR_EXIT(&cpr);
4570	thread_exit();
4571}
4572
4573void
4574dnlc_reduce_cache(void *arg)
4575{
4576	u_int percent;
4577
4578	percent = (u_int)(uintptr_t)arg;
4579	mutex_enter(&arc_dnlc_evicts_lock);
4580	if (arc_dnlc_evicts_arg == 0) {
4581		arc_dnlc_evicts_arg = percent;
4582		cv_broadcast(&arc_dnlc_evicts_cv);
4583	}
4584	mutex_exit(&arc_dnlc_evicts_lock);
4585}
4586
4587/*
4588 * Adapt arc info given the number of bytes we are trying to add and
4589 * the state that we are comming from.  This function is only called
4590 * when we are adding new content to the cache.
4591 */
4592static void
4593arc_adapt(int bytes, arc_state_t *state)
4594{
4595	int mult;
4596	uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
4597	int64_t mrug_size = refcount_count(&arc_mru_ghost->arcs_size);
4598	int64_t mfug_size = refcount_count(&arc_mfu_ghost->arcs_size);
4599
4600	if (state == arc_l2c_only)
4601		return;
4602
4603	ASSERT(bytes > 0);
4604	/*
4605	 * Adapt the target size of the MRU list:
4606	 *	- if we just hit in the MRU ghost list, then increase
4607	 *	  the target size of the MRU list.
4608	 *	- if we just hit in the MFU ghost list, then increase
4609	 *	  the target size of the MFU list by decreasing the
4610	 *	  target size of the MRU list.
4611	 */
4612	if (state == arc_mru_ghost) {
4613		mult = (mrug_size >= mfug_size) ? 1 : (mfug_size / mrug_size);
4614		mult = MIN(mult, 10); /* avoid wild arc_p adjustment */
4615
4616		arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
4617	} else if (state == arc_mfu_ghost) {
4618		uint64_t delta;
4619
4620		mult = (mfug_size >= mrug_size) ? 1 : (mrug_size / mfug_size);
4621		mult = MIN(mult, 10);
4622
4623		delta = MIN(bytes * mult, arc_p);
4624		arc_p = MAX(arc_p_min, arc_p - delta);
4625	}
4626	ASSERT((int64_t)arc_p >= 0);
4627
4628	if (arc_reclaim_needed()) {
4629		cv_signal(&arc_reclaim_thread_cv);
4630		return;
4631	}
4632
4633	if (arc_no_grow)
4634		return;
4635
4636	if (arc_c >= arc_c_max)
4637		return;
4638
4639	/*
4640	 * If we're within (2 * maxblocksize) bytes of the target
4641	 * cache size, increment the target cache size
4642	 */
4643	if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
4644		DTRACE_PROBE1(arc__inc_adapt, int, bytes);
4645		atomic_add_64(&arc_c, (int64_t)bytes);
4646		if (arc_c > arc_c_max)
4647			arc_c = arc_c_max;
4648		else if (state == arc_anon)
4649			atomic_add_64(&arc_p, (int64_t)bytes);
4650		if (arc_p > arc_c)
4651			arc_p = arc_c;
4652	}
4653	ASSERT((int64_t)arc_p >= 0);
4654}
4655
4656/*
4657 * Check if arc_size has grown past our upper threshold, determined by
4658 * zfs_arc_overflow_shift.
4659 */
4660static boolean_t
4661arc_is_overflowing(void)
4662{
4663	/* Always allow at least one block of overflow */
4664	uint64_t overflow = MAX(SPA_MAXBLOCKSIZE,
4665	    arc_c >> zfs_arc_overflow_shift);
4666
4667	return (arc_size >= arc_c + overflow);
4668}
4669
4670static abd_t *
4671arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
4672{
4673	arc_buf_contents_t type = arc_buf_type(hdr);
4674
4675	arc_get_data_impl(hdr, size, tag);
4676	if (type == ARC_BUFC_METADATA) {
4677		return (abd_alloc(size, B_TRUE));
4678	} else {
4679		ASSERT(type == ARC_BUFC_DATA);
4680		return (abd_alloc(size, B_FALSE));
4681	}
4682}
4683
4684static void *
4685arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
4686{
4687	arc_buf_contents_t type = arc_buf_type(hdr);
4688
4689	arc_get_data_impl(hdr, size, tag);
4690	if (type == ARC_BUFC_METADATA) {
4691		return (zio_buf_alloc(size));
4692	} else {
4693		ASSERT(type == ARC_BUFC_DATA);
4694		return (zio_data_buf_alloc(size));
4695	}
4696}
4697
4698/*
4699 * Allocate a block and return it to the caller. If we are hitting the
4700 * hard limit for the cache size, we must sleep, waiting for the eviction
4701 * thread to catch up. If we're past the target size but below the hard
4702 * limit, we'll only signal the reclaim thread and continue on.
4703 */
4704static void
4705arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
4706{
4707	arc_state_t *state = hdr->b_l1hdr.b_state;
4708	arc_buf_contents_t type = arc_buf_type(hdr);
4709
4710	arc_adapt(size, state);
4711
4712	/*
4713	 * If arc_size is currently overflowing, and has grown past our
4714	 * upper limit, we must be adding data faster than the evict
4715	 * thread can evict. Thus, to ensure we don't compound the
4716	 * problem by adding more data and forcing arc_size to grow even
4717	 * further past it's target size, we halt and wait for the
4718	 * eviction thread to catch up.
4719	 *
4720	 * It's also possible that the reclaim thread is unable to evict
4721	 * enough buffers to get arc_size below the overflow limit (e.g.
4722	 * due to buffers being un-evictable, or hash lock collisions).
4723	 * In this case, we want to proceed regardless if we're
4724	 * overflowing; thus we don't use a while loop here.
4725	 */
4726	if (arc_is_overflowing()) {
4727		mutex_enter(&arc_reclaim_lock);
4728
4729		/*
4730		 * Now that we've acquired the lock, we may no longer be
4731		 * over the overflow limit, lets check.
4732		 *
4733		 * We're ignoring the case of spurious wake ups. If that
4734		 * were to happen, it'd let this thread consume an ARC
4735		 * buffer before it should have (i.e. before we're under
4736		 * the overflow limit and were signalled by the reclaim
4737		 * thread). As long as that is a rare occurrence, it
4738		 * shouldn't cause any harm.
4739		 */
4740		if (arc_is_overflowing()) {
4741			cv_signal(&arc_reclaim_thread_cv);
4742			cv_wait(&arc_reclaim_waiters_cv, &arc_reclaim_lock);
4743		}
4744
4745		mutex_exit(&arc_reclaim_lock);
4746	}
4747
4748	VERIFY3U(hdr->b_type, ==, type);
4749	if (type == ARC_BUFC_METADATA) {
4750		arc_space_consume(size, ARC_SPACE_META);
4751	} else {
4752		arc_space_consume(size, ARC_SPACE_DATA);
4753	}
4754
4755	/*
4756	 * Update the state size.  Note that ghost states have a
4757	 * "ghost size" and so don't need to be updated.
4758	 */
4759	if (!GHOST_STATE(state)) {
4760
4761		(void) refcount_add_many(&state->arcs_size, size, tag);
4762
4763		/*
4764		 * If this is reached via arc_read, the link is
4765		 * protected by the hash lock. If reached via
4766		 * arc_buf_alloc, the header should not be accessed by
4767		 * any other thread. And, if reached via arc_read_done,
4768		 * the hash lock will protect it if it's found in the
4769		 * hash table; otherwise no other thread should be
4770		 * trying to [add|remove]_reference it.
4771		 */
4772		if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
4773			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4774			(void) refcount_add_many(&state->arcs_esize[type],
4775			    size, tag);
4776		}
4777
4778		/*
4779		 * If we are growing the cache, and we are adding anonymous
4780		 * data, and we have outgrown arc_p, update arc_p
4781		 */
4782		if (arc_size < arc_c && hdr->b_l1hdr.b_state == arc_anon &&
4783		    (refcount_count(&arc_anon->arcs_size) +
4784		    refcount_count(&arc_mru->arcs_size) > arc_p))
4785			arc_p = MIN(arc_c, arc_p + size);
4786	}
4787	ARCSTAT_BUMP(arcstat_allocated);
4788}
4789
4790static void
4791arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size, void *tag)
4792{
4793	arc_free_data_impl(hdr, size, tag);
4794	abd_free(abd);
4795}
4796
4797static void
4798arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, void *tag)
4799{
4800	arc_buf_contents_t type = arc_buf_type(hdr);
4801
4802	arc_free_data_impl(hdr, size, tag);
4803	if (type == ARC_BUFC_METADATA) {
4804		zio_buf_free(buf, size);
4805	} else {
4806		ASSERT(type == ARC_BUFC_DATA);
4807		zio_data_buf_free(buf, size);
4808	}
4809}
4810
4811/*
4812 * Free the arc data buffer.
4813 */
4814static void
4815arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
4816{
4817	arc_state_t *state = hdr->b_l1hdr.b_state;
4818	arc_buf_contents_t type = arc_buf_type(hdr);
4819
4820	/* protected by hash lock, if in the hash table */
4821	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
4822		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4823		ASSERT(state != arc_anon && state != arc_l2c_only);
4824
4825		(void) refcount_remove_many(&state->arcs_esize[type],
4826		    size, tag);
4827	}
4828	(void) refcount_remove_many(&state->arcs_size, size, tag);
4829
4830	VERIFY3U(hdr->b_type, ==, type);
4831	if (type == ARC_BUFC_METADATA) {
4832		arc_space_return(size, ARC_SPACE_META);
4833	} else {
4834		ASSERT(type == ARC_BUFC_DATA);
4835		arc_space_return(size, ARC_SPACE_DATA);
4836	}
4837}
4838
4839/*
4840 * This routine is called whenever a buffer is accessed.
4841 * NOTE: the hash lock is dropped in this function.
4842 */
4843static void
4844arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
4845{
4846	clock_t now;
4847
4848	ASSERT(MUTEX_HELD(hash_lock));
4849	ASSERT(HDR_HAS_L1HDR(hdr));
4850
4851	if (hdr->b_l1hdr.b_state == arc_anon) {
4852		/*
4853		 * This buffer is not in the cache, and does not
4854		 * appear in our "ghost" list.  Add the new buffer
4855		 * to the MRU state.
4856		 */
4857
4858		ASSERT0(hdr->b_l1hdr.b_arc_access);
4859		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4860		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
4861		arc_change_state(arc_mru, hdr, hash_lock);
4862
4863	} else if (hdr->b_l1hdr.b_state == arc_mru) {
4864		now = ddi_get_lbolt();
4865
4866		/*
4867		 * If this buffer is here because of a prefetch, then either:
4868		 * - clear the flag if this is a "referencing" read
4869		 *   (any subsequent access will bump this into the MFU state).
4870		 * or
4871		 * - move the buffer to the head of the list if this is
4872		 *   another prefetch (to make it less likely to be evicted).
4873		 */
4874		if (HDR_PREFETCH(hdr)) {
4875			if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
4876				/* link protected by hash lock */
4877				ASSERT(multilist_link_active(
4878				    &hdr->b_l1hdr.b_arc_node));
4879			} else {
4880				arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
4881				ARCSTAT_BUMP(arcstat_mru_hits);
4882			}
4883			hdr->b_l1hdr.b_arc_access = now;
4884			return;
4885		}
4886
4887		/*
4888		 * This buffer has been "accessed" only once so far,
4889		 * but it is still in the cache. Move it to the MFU
4890		 * state.
4891		 */
4892		if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) {
4893			/*
4894			 * More than 125ms have passed since we
4895			 * instantiated this buffer.  Move it to the
4896			 * most frequently used state.
4897			 */
4898			hdr->b_l1hdr.b_arc_access = now;
4899			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4900			arc_change_state(arc_mfu, hdr, hash_lock);
4901		}
4902		ARCSTAT_BUMP(arcstat_mru_hits);
4903	} else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
4904		arc_state_t	*new_state;
4905		/*
4906		 * This buffer has been "accessed" recently, but
4907		 * was evicted from the cache.  Move it to the
4908		 * MFU state.
4909		 */
4910
4911		if (HDR_PREFETCH(hdr)) {
4912			new_state = arc_mru;
4913			if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0)
4914				arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
4915			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
4916		} else {
4917			new_state = arc_mfu;
4918			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4919		}
4920
4921		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4922		arc_change_state(new_state, hdr, hash_lock);
4923
4924		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
4925	} else if (hdr->b_l1hdr.b_state == arc_mfu) {
4926		/*
4927		 * This buffer has been accessed more than once and is
4928		 * still in the cache.  Keep it in the MFU state.
4929		 *
4930		 * NOTE: an add_reference() that occurred when we did
4931		 * the arc_read() will have kicked this off the list.
4932		 * If it was a prefetch, we will explicitly move it to
4933		 * the head of the list now.
4934		 */
4935		if ((HDR_PREFETCH(hdr)) != 0) {
4936			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4937			/* link protected by hash_lock */
4938			ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node));
4939		}
4940		ARCSTAT_BUMP(arcstat_mfu_hits);
4941		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4942	} else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
4943		arc_state_t	*new_state = arc_mfu;
4944		/*
4945		 * This buffer has been accessed more than once but has
4946		 * been evicted from the cache.  Move it back to the
4947		 * MFU state.
4948		 */
4949
4950		if (HDR_PREFETCH(hdr)) {
4951			/*
4952			 * This is a prefetch access...
4953			 * move this block back to the MRU state.
4954			 */
4955			ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
4956			new_state = arc_mru;
4957		}
4958
4959		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4960		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4961		arc_change_state(new_state, hdr, hash_lock);
4962
4963		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
4964	} else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
4965		/*
4966		 * This buffer is on the 2nd Level ARC.
4967		 */
4968
4969		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4970		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4971		arc_change_state(arc_mfu, hdr, hash_lock);
4972	} else {
4973		ASSERT(!"invalid arc state");
4974	}
4975}
4976
4977/* a generic arc_done_func_t which you can use */
4978/* ARGSUSED */
4979void
4980arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
4981{
4982	if (zio == NULL || zio->io_error == 0)
4983		bcopy(buf->b_data, arg, arc_buf_size(buf));
4984	arc_buf_destroy(buf, arg);
4985}
4986
4987/* a generic arc_done_func_t */
4988void
4989arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
4990{
4991	arc_buf_t **bufp = arg;
4992	if (zio && zio->io_error) {
4993		arc_buf_destroy(buf, arg);
4994		*bufp = NULL;
4995	} else {
4996		*bufp = buf;
4997		ASSERT(buf->b_data);
4998	}
4999}
5000
5001static void
5002arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)
5003{
5004	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
5005		ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);
5006		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
5007	} else {
5008		if (HDR_COMPRESSION_ENABLED(hdr)) {
5009			ASSERT3U(HDR_GET_COMPRESS(hdr), ==,
5010			    BP_GET_COMPRESS(bp));
5011		}
5012		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
5013		ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp));
5014	}
5015}
5016
5017static void
5018arc_read_done(zio_t *zio)
5019{
5020	arc_buf_hdr_t	*hdr = zio->io_private;
5021	kmutex_t	*hash_lock = NULL;
5022	arc_callback_t	*callback_list;
5023	arc_callback_t	*acb;
5024	boolean_t	freeable = B_FALSE;
5025	boolean_t	no_zio_error = (zio->io_error == 0);
5026
5027	/*
5028	 * The hdr was inserted into hash-table and removed from lists
5029	 * prior to starting I/O.  We should find this header, since
5030	 * it's in the hash table, and it should be legit since it's
5031	 * not possible to evict it during the I/O.  The only possible
5032	 * reason for it not to be found is if we were freed during the
5033	 * read.
5034	 */
5035	if (HDR_IN_HASH_TABLE(hdr)) {
5036		ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp));
5037		ASSERT3U(hdr->b_dva.dva_word[0], ==,
5038		    BP_IDENTITY(zio->io_bp)->dva_word[0]);
5039		ASSERT3U(hdr->b_dva.dva_word[1], ==,
5040		    BP_IDENTITY(zio->io_bp)->dva_word[1]);
5041
5042		arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp,
5043		    &hash_lock);
5044
5045		ASSERT((found == hdr &&
5046		    DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
5047		    (found == hdr && HDR_L2_READING(hdr)));
5048		ASSERT3P(hash_lock, !=, NULL);
5049	}
5050
5051	if (no_zio_error) {
5052		/* byteswap if necessary */
5053		if (BP_SHOULD_BYTESWAP(zio->io_bp)) {
5054			if (BP_GET_LEVEL(zio->io_bp) > 0) {
5055				hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
5056			} else {
5057				hdr->b_l1hdr.b_byteswap =
5058				    DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
5059			}
5060		} else {
5061			hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
5062		}
5063	}
5064
5065	arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
5066	if (l2arc_noprefetch && HDR_PREFETCH(hdr))
5067		arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);
5068
5069	callback_list = hdr->b_l1hdr.b_acb;
5070	ASSERT3P(callback_list, !=, NULL);
5071
5072	if (hash_lock && no_zio_error && hdr->b_l1hdr.b_state == arc_anon) {
5073		/*
5074		 * Only call arc_access on anonymous buffers.  This is because
5075		 * if we've issued an I/O for an evicted buffer, we've already
5076		 * called arc_access (to prevent any simultaneous readers from
5077		 * getting confused).
5078		 */
5079		arc_access(hdr, hash_lock);
5080	}
5081
5082	/*
5083	 * If a read request has a callback (i.e. acb_done is not NULL), then we
5084	 * make a buf containing the data according to the parameters which were
5085	 * passed in. The implementation of arc_buf_alloc_impl() ensures that we
5086	 * aren't needlessly decompressing the data multiple times.
5087	 */
5088	int callback_cnt = 0;
5089	for (acb = callback_list; acb != NULL; acb = acb->acb_next) {
5090		if (!acb->acb_done)
5091			continue;
5092
5093		/* This is a demand read since prefetches don't use callbacks */
5094		callback_cnt++;
5095
5096		int error = arc_buf_alloc_impl(hdr, acb->acb_private,
5097		    acb->acb_compressed, no_zio_error, &acb->acb_buf);
5098		if (no_zio_error) {
5099			zio->io_error = error;
5100		}
5101	}
5102	hdr->b_l1hdr.b_acb = NULL;
5103	arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5104	if (callback_cnt == 0) {
5105		ASSERT(HDR_PREFETCH(hdr));
5106		ASSERT0(hdr->b_l1hdr.b_bufcnt);
5107		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5108	}
5109
5110	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
5111	    callback_list != NULL);
5112
5113	if (no_zio_error) {
5114		arc_hdr_verify(hdr, zio->io_bp);
5115	} else {
5116		arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
5117		if (hdr->b_l1hdr.b_state != arc_anon)
5118			arc_change_state(arc_anon, hdr, hash_lock);
5119		if (HDR_IN_HASH_TABLE(hdr))
5120			buf_hash_remove(hdr);
5121		freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
5122	}
5123
5124	/*
5125	 * Broadcast before we drop the hash_lock to avoid the possibility
5126	 * that the hdr (and hence the cv) might be freed before we get to
5127	 * the cv_broadcast().
5128	 */
5129	cv_broadcast(&hdr->b_l1hdr.b_cv);
5130
5131	if (hash_lock != NULL) {
5132		mutex_exit(hash_lock);
5133	} else {
5134		/*
5135		 * This block was freed while we waited for the read to
5136		 * complete.  It has been removed from the hash table and
5137		 * moved to the anonymous state (so that it won't show up
5138		 * in the cache).
5139		 */
5140		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
5141		freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
5142	}
5143
5144	/* execute each callback and free its structure */
5145	while ((acb = callback_list) != NULL) {
5146		if (acb->acb_done)
5147			acb->acb_done(zio, acb->acb_buf, acb->acb_private);
5148
5149		if (acb->acb_zio_dummy != NULL) {
5150			acb->acb_zio_dummy->io_error = zio->io_error;
5151			zio_nowait(acb->acb_zio_dummy);
5152		}
5153
5154		callback_list = acb->acb_next;
5155		kmem_free(acb, sizeof (arc_callback_t));
5156	}
5157
5158	if (freeable)
5159		arc_hdr_destroy(hdr);
5160}
5161
5162/*
5163 * "Read" the block at the specified DVA (in bp) via the
5164 * cache.  If the block is found in the cache, invoke the provided
5165 * callback immediately and return.  Note that the `zio' parameter
5166 * in the callback will be NULL in this case, since no IO was
5167 * required.  If the block is not in the cache pass the read request
5168 * on to the spa with a substitute callback function, so that the
5169 * requested block will be added to the cache.
5170 *
5171 * If a read request arrives for a block that has a read in-progress,
5172 * either wait for the in-progress read to complete (and return the
5173 * results); or, if this is a read with a "done" func, add a record
5174 * to the read to invoke the "done" func when the read completes,
5175 * and return; or just return.
5176 *
5177 * arc_read_done() will invoke all the requested "done" functions
5178 * for readers of this block.
5179 */
5180int
5181arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done,
5182    void *private, zio_priority_t priority, int zio_flags,
5183    arc_flags_t *arc_flags, const zbookmark_phys_t *zb)
5184{
5185	arc_buf_hdr_t *hdr = NULL;
5186	kmutex_t *hash_lock = NULL;
5187	zio_t *rzio;
5188	uint64_t guid = spa_load_guid(spa);
5189	boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW) != 0;
5190
5191	ASSERT(!BP_IS_EMBEDDED(bp) ||
5192	    BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
5193
5194top:
5195	if (!BP_IS_EMBEDDED(bp)) {
5196		/*
5197		 * Embedded BP's have no DVA and require no I/O to "read".
5198		 * Create an anonymous arc buf to back it.
5199		 */
5200		hdr = buf_hash_find(guid, bp, &hash_lock);
5201	}
5202
5203	if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_pabd != NULL) {
5204		arc_buf_t *buf = NULL;
5205		*arc_flags |= ARC_FLAG_CACHED;
5206
5207		if (HDR_IO_IN_PROGRESS(hdr)) {
5208
5209			if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&
5210			    priority == ZIO_PRIORITY_SYNC_READ) {
5211				/*
5212				 * This sync read must wait for an
5213				 * in-progress async read (e.g. a predictive
5214				 * prefetch).  Async reads are queued
5215				 * separately at the vdev_queue layer, so
5216				 * this is a form of priority inversion.
5217				 * Ideally, we would "inherit" the demand
5218				 * i/o's priority by moving the i/o from
5219				 * the async queue to the synchronous queue,
5220				 * but there is currently no mechanism to do
5221				 * so.  Track this so that we can evaluate
5222				 * the magnitude of this potential performance
5223				 * problem.
5224				 *
5225				 * Note that if the prefetch i/o is already
5226				 * active (has been issued to the device),
5227				 * the prefetch improved performance, because
5228				 * we issued it sooner than we would have
5229				 * without the prefetch.
5230				 */
5231				DTRACE_PROBE1(arc__sync__wait__for__async,
5232				    arc_buf_hdr_t *, hdr);
5233				ARCSTAT_BUMP(arcstat_sync_wait_for_async);
5234			}
5235			if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) {
5236				arc_hdr_clear_flags(hdr,
5237				    ARC_FLAG_PREDICTIVE_PREFETCH);
5238			}
5239
5240			if (*arc_flags & ARC_FLAG_WAIT) {
5241				cv_wait(&hdr->b_l1hdr.b_cv, hash_lock);
5242				mutex_exit(hash_lock);
5243				goto top;
5244			}
5245			ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
5246
5247			if (done) {
5248				arc_callback_t *acb = NULL;
5249
5250				acb = kmem_zalloc(sizeof (arc_callback_t),
5251				    KM_SLEEP);
5252				acb->acb_done = done;
5253				acb->acb_private = private;
5254				acb->acb_compressed = compressed_read;
5255				if (pio != NULL)
5256					acb->acb_zio_dummy = zio_null(pio,
5257					    spa, NULL, NULL, NULL, zio_flags);
5258
5259				ASSERT3P(acb->acb_done, !=, NULL);
5260				acb->acb_next = hdr->b_l1hdr.b_acb;
5261				hdr->b_l1hdr.b_acb = acb;
5262				mutex_exit(hash_lock);
5263				return (0);
5264			}
5265			mutex_exit(hash_lock);
5266			return (0);
5267		}
5268
5269		ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5270		    hdr->b_l1hdr.b_state == arc_mfu);
5271
5272		if (done) {
5273			if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) {
5274				/*
5275				 * This is a demand read which does not have to
5276				 * wait for i/o because we did a predictive
5277				 * prefetch i/o for it, which has completed.
5278				 */
5279				DTRACE_PROBE1(
5280				    arc__demand__hit__predictive__prefetch,
5281				    arc_buf_hdr_t *, hdr);
5282				ARCSTAT_BUMP(
5283				    arcstat_demand_hit_predictive_prefetch);
5284				arc_hdr_clear_flags(hdr,
5285				    ARC_FLAG_PREDICTIVE_PREFETCH);
5286			}
5287			ASSERT(!BP_IS_EMBEDDED(bp) || !BP_IS_HOLE(bp));
5288
5289			/* Get a buf with the desired data in it. */
5290			VERIFY0(arc_buf_alloc_impl(hdr, private,
5291			    compressed_read, B_TRUE, &buf));
5292		} else if (*arc_flags & ARC_FLAG_PREFETCH &&
5293		    refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
5294			arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
5295		}
5296		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
5297		arc_access(hdr, hash_lock);
5298		if (*arc_flags & ARC_FLAG_L2CACHE)
5299			arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5300		mutex_exit(hash_lock);
5301		ARCSTAT_BUMP(arcstat_hits);
5302		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
5303		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
5304		    data, metadata, hits);
5305
5306		if (done)
5307			done(NULL, buf, private);
5308	} else {
5309		uint64_t lsize = BP_GET_LSIZE(bp);
5310		uint64_t psize = BP_GET_PSIZE(bp);
5311		arc_callback_t *acb;
5312		vdev_t *vd = NULL;
5313		uint64_t addr = 0;
5314		boolean_t devw = B_FALSE;
5315		uint64_t size;
5316
5317		if (hdr == NULL) {
5318			/* this block is not in the cache */
5319			arc_buf_hdr_t *exists = NULL;
5320			arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
5321			hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
5322			    BP_GET_COMPRESS(bp), type);
5323
5324			if (!BP_IS_EMBEDDED(bp)) {
5325				hdr->b_dva = *BP_IDENTITY(bp);
5326				hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
5327				exists = buf_hash_insert(hdr, &hash_lock);
5328			}
5329			if (exists != NULL) {
5330				/* somebody beat us to the hash insert */
5331				mutex_exit(hash_lock);
5332				buf_discard_identity(hdr);
5333				arc_hdr_destroy(hdr);
5334				goto top; /* restart the IO request */
5335			}
5336		} else {
5337			/*
5338			 * This block is in the ghost cache. If it was L2-only
5339			 * (and thus didn't have an L1 hdr), we realloc the
5340			 * header to add an L1 hdr.
5341			 */
5342			if (!HDR_HAS_L1HDR(hdr)) {
5343				hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
5344				    hdr_full_cache);
5345			}
5346			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
5347			ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
5348			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5349			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5350			ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
5351			ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
5352
5353			/*
5354			 * This is a delicate dance that we play here.
5355			 * This hdr is in the ghost list so we access it
5356			 * to move it out of the ghost list before we
5357			 * initiate the read. If it's a prefetch then
5358			 * it won't have a callback so we'll remove the
5359			 * reference that arc_buf_alloc_impl() created. We
5360			 * do this after we've called arc_access() to
5361			 * avoid hitting an assert in remove_reference().
5362			 */
5363			arc_access(hdr, hash_lock);
5364			arc_hdr_alloc_pabd(hdr);
5365		}
5366		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5367		size = arc_hdr_size(hdr);
5368
5369		/*
5370		 * If compression is enabled on the hdr, then will do
5371		 * RAW I/O and will store the compressed data in the hdr's
5372		 * data block. Otherwise, the hdr's data block will contain
5373		 * the uncompressed data.
5374		 */
5375		if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) {
5376			zio_flags |= ZIO_FLAG_RAW;
5377		}
5378
5379		if (*arc_flags & ARC_FLAG_PREFETCH)
5380			arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
5381		if (*arc_flags & ARC_FLAG_L2CACHE)
5382			arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5383		if (BP_GET_LEVEL(bp) > 0)
5384			arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT);
5385		if (*arc_flags & ARC_FLAG_PREDICTIVE_PREFETCH)
5386			arc_hdr_set_flags(hdr, ARC_FLAG_PREDICTIVE_PREFETCH);
5387		ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
5388
5389		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
5390		acb->acb_done = done;
5391		acb->acb_private = private;
5392		acb->acb_compressed = compressed_read;
5393
5394		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
5395		hdr->b_l1hdr.b_acb = acb;
5396		arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5397
5398		if (HDR_HAS_L2HDR(hdr) &&
5399		    (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
5400			devw = hdr->b_l2hdr.b_dev->l2ad_writing;
5401			addr = hdr->b_l2hdr.b_daddr;
5402			/*
5403			 * Lock out device removal.
5404			 */
5405			if (vdev_is_dead(vd) ||
5406			    !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
5407				vd = NULL;
5408		}
5409
5410		if (priority == ZIO_PRIORITY_ASYNC_READ)
5411			arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
5412		else
5413			arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
5414
5415		if (hash_lock != NULL)
5416			mutex_exit(hash_lock);
5417
5418		/*
5419		 * At this point, we have a level 1 cache miss.  Try again in
5420		 * L2ARC if possible.
5421		 */
5422		ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize);
5423
5424		DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
5425		    uint64_t, lsize, zbookmark_phys_t *, zb);
5426		ARCSTAT_BUMP(arcstat_misses);
5427		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
5428		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
5429		    data, metadata, misses);
5430#ifdef _KERNEL
5431#ifdef RACCT
5432		if (racct_enable) {
5433			PROC_LOCK(curproc);
5434			racct_add_force(curproc, RACCT_READBPS, size);
5435			racct_add_force(curproc, RACCT_READIOPS, 1);
5436			PROC_UNLOCK(curproc);
5437		}
5438#endif /* RACCT */
5439		curthread->td_ru.ru_inblock++;
5440#endif
5441
5442		if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
5443			/*
5444			 * Read from the L2ARC if the following are true:
5445			 * 1. The L2ARC vdev was previously cached.
5446			 * 2. This buffer still has L2ARC metadata.
5447			 * 3. This buffer isn't currently writing to the L2ARC.
5448			 * 4. The L2ARC entry wasn't evicted, which may
5449			 *    also have invalidated the vdev.
5450			 * 5. This isn't prefetch and l2arc_noprefetch is set.
5451			 */
5452			if (HDR_HAS_L2HDR(hdr) &&
5453			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
5454			    !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
5455				l2arc_read_callback_t *cb;
5456				abd_t *abd;
5457				uint64_t asize;
5458
5459				DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
5460				ARCSTAT_BUMP(arcstat_l2_hits);
5461
5462				cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
5463				    KM_SLEEP);
5464				cb->l2rcb_hdr = hdr;
5465				cb->l2rcb_bp = *bp;
5466				cb->l2rcb_zb = *zb;
5467				cb->l2rcb_flags = zio_flags;
5468
5469				asize = vdev_psize_to_asize(vd, size);
5470				if (asize != size) {
5471					abd = abd_alloc_for_io(asize,
5472					    HDR_ISTYPE_METADATA(hdr));
5473					cb->l2rcb_abd = abd;
5474				} else {
5475					abd = hdr->b_l1hdr.b_pabd;
5476				}
5477
5478				ASSERT(addr >= VDEV_LABEL_START_SIZE &&
5479				    addr + asize <= vd->vdev_psize -
5480				    VDEV_LABEL_END_SIZE);
5481
5482				/*
5483				 * l2arc read.  The SCL_L2ARC lock will be
5484				 * released by l2arc_read_done().
5485				 * Issue a null zio if the underlying buffer
5486				 * was squashed to zero size by compression.
5487				 */
5488				ASSERT3U(HDR_GET_COMPRESS(hdr), !=,
5489				    ZIO_COMPRESS_EMPTY);
5490				rzio = zio_read_phys(pio, vd, addr,
5491				    asize, abd,
5492				    ZIO_CHECKSUM_OFF,
5493				    l2arc_read_done, cb, priority,
5494				    zio_flags | ZIO_FLAG_DONT_CACHE |
5495				    ZIO_FLAG_CANFAIL |
5496				    ZIO_FLAG_DONT_PROPAGATE |
5497				    ZIO_FLAG_DONT_RETRY, B_FALSE);
5498				DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
5499				    zio_t *, rzio);
5500				ARCSTAT_INCR(arcstat_l2_read_bytes, size);
5501
5502				if (*arc_flags & ARC_FLAG_NOWAIT) {
5503					zio_nowait(rzio);
5504					return (0);
5505				}
5506
5507				ASSERT(*arc_flags & ARC_FLAG_WAIT);
5508				if (zio_wait(rzio) == 0)
5509					return (0);
5510
5511				/* l2arc read error; goto zio_read() */
5512			} else {
5513				DTRACE_PROBE1(l2arc__miss,
5514				    arc_buf_hdr_t *, hdr);
5515				ARCSTAT_BUMP(arcstat_l2_misses);
5516				if (HDR_L2_WRITING(hdr))
5517					ARCSTAT_BUMP(arcstat_l2_rw_clash);
5518				spa_config_exit(spa, SCL_L2ARC, vd);
5519			}
5520		} else {
5521			if (vd != NULL)
5522				spa_config_exit(spa, SCL_L2ARC, vd);
5523			if (l2arc_ndev != 0) {
5524				DTRACE_PROBE1(l2arc__miss,
5525				    arc_buf_hdr_t *, hdr);
5526				ARCSTAT_BUMP(arcstat_l2_misses);
5527			}
5528		}
5529
5530		rzio = zio_read(pio, spa, bp, hdr->b_l1hdr.b_pabd, size,
5531		    arc_read_done, hdr, priority, zio_flags, zb);
5532
5533		if (*arc_flags & ARC_FLAG_WAIT)
5534			return (zio_wait(rzio));
5535
5536		ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
5537		zio_nowait(rzio);
5538	}
5539	return (0);
5540}
5541
5542/*
5543 * Notify the arc that a block was freed, and thus will never be used again.
5544 */
5545void
5546arc_freed(spa_t *spa, const blkptr_t *bp)
5547{
5548	arc_buf_hdr_t *hdr;
5549	kmutex_t *hash_lock;
5550	uint64_t guid = spa_load_guid(spa);
5551
5552	ASSERT(!BP_IS_EMBEDDED(bp));
5553
5554	hdr = buf_hash_find(guid, bp, &hash_lock);
5555	if (hdr == NULL)
5556		return;
5557
5558	/*
5559	 * We might be trying to free a block that is still doing I/O
5560	 * (i.e. prefetch) or has a reference (i.e. a dedup-ed,
5561	 * dmu_sync-ed block). If this block is being prefetched, then it
5562	 * would still have the ARC_FLAG_IO_IN_PROGRESS flag set on the hdr
5563	 * until the I/O completes. A block may also have a reference if it is
5564	 * part of a dedup-ed, dmu_synced write. The dmu_sync() function would
5565	 * have written the new block to its final resting place on disk but
5566	 * without the dedup flag set. This would have left the hdr in the MRU
5567	 * state and discoverable. When the txg finally syncs it detects that
5568	 * the block was overridden in open context and issues an override I/O.
5569	 * Since this is a dedup block, the override I/O will determine if the
5570	 * block is already in the DDT. If so, then it will replace the io_bp
5571	 * with the bp from the DDT and allow the I/O to finish. When the I/O
5572	 * reaches the done callback, dbuf_write_override_done, it will
5573	 * check to see if the io_bp and io_bp_override are identical.
5574	 * If they are not, then it indicates that the bp was replaced with
5575	 * the bp in the DDT and the override bp is freed. This allows
5576	 * us to arrive here with a reference on a block that is being
5577	 * freed. So if we have an I/O in progress, or a reference to
5578	 * this hdr, then we don't destroy the hdr.
5579	 */
5580	if (!HDR_HAS_L1HDR(hdr) || (!HDR_IO_IN_PROGRESS(hdr) &&
5581	    refcount_is_zero(&hdr->b_l1hdr.b_refcnt))) {
5582		arc_change_state(arc_anon, hdr, hash_lock);
5583		arc_hdr_destroy(hdr);
5584		mutex_exit(hash_lock);
5585	} else {
5586		mutex_exit(hash_lock);
5587	}
5588
5589}
5590
5591/*
5592 * Release this buffer from the cache, making it an anonymous buffer.  This
5593 * must be done after a read and prior to modifying the buffer contents.
5594 * If the buffer has more than one reference, we must make
5595 * a new hdr for the buffer.
5596 */
5597void
5598arc_release(arc_buf_t *buf, void *tag)
5599{
5600	arc_buf_hdr_t *hdr = buf->b_hdr;
5601
5602	/*
5603	 * It would be nice to assert that if it's DMU metadata (level >
5604	 * 0 || it's the dnode file), then it must be syncing context.
5605	 * But we don't know that information at this level.
5606	 */
5607
5608	mutex_enter(&buf->b_evict_lock);
5609
5610	ASSERT(HDR_HAS_L1HDR(hdr));
5611
5612	/*
5613	 * We don't grab the hash lock prior to this check, because if
5614	 * the buffer's header is in the arc_anon state, it won't be
5615	 * linked into the hash table.
5616	 */
5617	if (hdr->b_l1hdr.b_state == arc_anon) {
5618		mutex_exit(&buf->b_evict_lock);
5619		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5620		ASSERT(!HDR_IN_HASH_TABLE(hdr));
5621		ASSERT(!HDR_HAS_L2HDR(hdr));
5622		ASSERT(HDR_EMPTY(hdr));
5623		ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
5624		ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
5625		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
5626
5627		hdr->b_l1hdr.b_arc_access = 0;
5628
5629		/*
5630		 * If the buf is being overridden then it may already
5631		 * have a hdr that is not empty.
5632		 */
5633		buf_discard_identity(hdr);
5634		arc_buf_thaw(buf);
5635
5636		return;
5637	}
5638
5639	kmutex_t *hash_lock = HDR_LOCK(hdr);
5640	mutex_enter(hash_lock);
5641
5642	/*
5643	 * This assignment is only valid as long as the hash_lock is
5644	 * held, we must be careful not to reference state or the
5645	 * b_state field after dropping the lock.
5646	 */
5647	arc_state_t *state = hdr->b_l1hdr.b_state;
5648	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
5649	ASSERT3P(state, !=, arc_anon);
5650
5651	/* this buffer is not on any list */
5652	ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0);
5653
5654	if (HDR_HAS_L2HDR(hdr)) {
5655		mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
5656
5657		/*
5658		 * We have to recheck this conditional again now that
5659		 * we're holding the l2ad_mtx to prevent a race with
5660		 * another thread which might be concurrently calling
5661		 * l2arc_evict(). In that case, l2arc_evict() might have
5662		 * destroyed the header's L2 portion as we were waiting
5663		 * to acquire the l2ad_mtx.
5664		 */
5665		if (HDR_HAS_L2HDR(hdr)) {
5666			l2arc_trim(hdr);
5667			arc_hdr_l2hdr_destroy(hdr);
5668		}
5669
5670		mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
5671	}
5672
5673	/*
5674	 * Do we have more than one buf?
5675	 */
5676	if (hdr->b_l1hdr.b_bufcnt > 1) {
5677		arc_buf_hdr_t *nhdr;
5678		uint64_t spa = hdr->b_spa;
5679		uint64_t psize = HDR_GET_PSIZE(hdr);
5680		uint64_t lsize = HDR_GET_LSIZE(hdr);
5681		enum zio_compress compress = HDR_GET_COMPRESS(hdr);
5682		arc_buf_contents_t type = arc_buf_type(hdr);
5683		VERIFY3U(hdr->b_type, ==, type);
5684
5685		ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
5686		(void) remove_reference(hdr, hash_lock, tag);
5687
5688		if (arc_buf_is_shared(buf) && !ARC_BUF_COMPRESSED(buf)) {
5689			ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
5690			ASSERT(ARC_BUF_LAST(buf));
5691		}
5692
5693		/*
5694		 * Pull the data off of this hdr and attach it to
5695		 * a new anonymous hdr. Also find the last buffer
5696		 * in the hdr's buffer list.
5697		 */
5698		arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
5699		ASSERT3P(lastbuf, !=, NULL);
5700
5701		/*
5702		 * If the current arc_buf_t and the hdr are sharing their data
5703		 * buffer, then we must stop sharing that block.
5704		 */
5705		if (arc_buf_is_shared(buf)) {
5706			VERIFY(!arc_buf_is_shared(lastbuf));
5707
5708			/*
5709			 * First, sever the block sharing relationship between
5710			 * buf and the arc_buf_hdr_t.
5711			 */
5712			arc_unshare_buf(hdr, buf);
5713
5714			/*
5715			 * Now we need to recreate the hdr's b_pabd. Since we
5716			 * have lastbuf handy, we try to share with it, but if
5717			 * we can't then we allocate a new b_pabd and copy the
5718			 * data from buf into it.
5719			 */
5720			if (arc_can_share(hdr, lastbuf)) {
5721				arc_share_buf(hdr, lastbuf);
5722			} else {
5723				arc_hdr_alloc_pabd(hdr);
5724				abd_copy_from_buf(hdr->b_l1hdr.b_pabd,
5725				    buf->b_data, psize);
5726			}
5727			VERIFY3P(lastbuf->b_data, !=, NULL);
5728		} else if (HDR_SHARED_DATA(hdr)) {
5729			/*
5730			 * Uncompressed shared buffers are always at the end
5731			 * of the list. Compressed buffers don't have the
5732			 * same requirements. This makes it hard to
5733			 * simply assert that the lastbuf is shared so
5734			 * we rely on the hdr's compression flags to determine
5735			 * if we have a compressed, shared buffer.
5736			 */
5737			ASSERT(arc_buf_is_shared(lastbuf) ||
5738			    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
5739			ASSERT(!ARC_BUF_SHARED(buf));
5740		}
5741		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5742		ASSERT3P(state, !=, arc_l2c_only);
5743
5744		(void) refcount_remove_many(&state->arcs_size,
5745		    arc_buf_size(buf), buf);
5746
5747		if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
5748			ASSERT3P(state, !=, arc_l2c_only);
5749			(void) refcount_remove_many(&state->arcs_esize[type],
5750			    arc_buf_size(buf), buf);
5751		}
5752
5753		hdr->b_l1hdr.b_bufcnt -= 1;
5754		arc_cksum_verify(buf);
5755#ifdef illumos
5756		arc_buf_unwatch(buf);
5757#endif
5758
5759		mutex_exit(hash_lock);
5760
5761		/*
5762		 * Allocate a new hdr. The new hdr will contain a b_pabd
5763		 * buffer which will be freed in arc_write().
5764		 */
5765		nhdr = arc_hdr_alloc(spa, psize, lsize, compress, type);
5766		ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL);
5767		ASSERT0(nhdr->b_l1hdr.b_bufcnt);
5768		ASSERT0(refcount_count(&nhdr->b_l1hdr.b_refcnt));
5769		VERIFY3U(nhdr->b_type, ==, type);
5770		ASSERT(!HDR_SHARED_DATA(nhdr));
5771
5772		nhdr->b_l1hdr.b_buf = buf;
5773		nhdr->b_l1hdr.b_bufcnt = 1;
5774		(void) refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
5775		buf->b_hdr = nhdr;
5776
5777		mutex_exit(&buf->b_evict_lock);
5778		(void) refcount_add_many(&arc_anon->arcs_size,
5779		    arc_buf_size(buf), buf);
5780	} else {
5781		mutex_exit(&buf->b_evict_lock);
5782		ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
5783		/* protected by hash lock, or hdr is on arc_anon */
5784		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
5785		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5786		arc_change_state(arc_anon, hdr, hash_lock);
5787		hdr->b_l1hdr.b_arc_access = 0;
5788		mutex_exit(hash_lock);
5789
5790		buf_discard_identity(hdr);
5791		arc_buf_thaw(buf);
5792	}
5793}
5794
5795int
5796arc_released(arc_buf_t *buf)
5797{
5798	int released;
5799
5800	mutex_enter(&buf->b_evict_lock);
5801	released = (buf->b_data != NULL &&
5802	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
5803	mutex_exit(&buf->b_evict_lock);
5804	return (released);
5805}
5806
5807#ifdef ZFS_DEBUG
5808int
5809arc_referenced(arc_buf_t *buf)
5810{
5811	int referenced;
5812
5813	mutex_enter(&buf->b_evict_lock);
5814	referenced = (refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
5815	mutex_exit(&buf->b_evict_lock);
5816	return (referenced);
5817}
5818#endif
5819
5820static void
5821arc_write_ready(zio_t *zio)
5822{
5823	arc_write_callback_t *callback = zio->io_private;
5824	arc_buf_t *buf = callback->awcb_buf;
5825	arc_buf_hdr_t *hdr = buf->b_hdr;
5826	uint64_t psize = BP_IS_HOLE(zio->io_bp) ? 0 : BP_GET_PSIZE(zio->io_bp);
5827
5828	ASSERT(HDR_HAS_L1HDR(hdr));
5829	ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
5830	ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
5831
5832	/*
5833	 * If we're reexecuting this zio because the pool suspended, then
5834	 * cleanup any state that was previously set the first time the
5835	 * callback was invoked.
5836	 */
5837	if (zio->io_flags & ZIO_FLAG_REEXECUTED) {
5838		arc_cksum_free(hdr);
5839#ifdef illumos
5840		arc_buf_unwatch(buf);
5841#endif
5842		if (hdr->b_l1hdr.b_pabd != NULL) {
5843			if (arc_buf_is_shared(buf)) {
5844				arc_unshare_buf(hdr, buf);
5845			} else {
5846				arc_hdr_free_pabd(hdr);
5847			}
5848		}
5849	}
5850	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
5851	ASSERT(!HDR_SHARED_DATA(hdr));
5852	ASSERT(!arc_buf_is_shared(buf));
5853
5854	callback->awcb_ready(zio, buf, callback->awcb_private);
5855
5856	if (HDR_IO_IN_PROGRESS(hdr))
5857		ASSERT(zio->io_flags & ZIO_FLAG_REEXECUTED);
5858
5859	arc_cksum_compute(buf);
5860	arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5861
5862	enum zio_compress compress;
5863	if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
5864		compress = ZIO_COMPRESS_OFF;
5865	} else {
5866		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(zio->io_bp));
5867		compress = BP_GET_COMPRESS(zio->io_bp);
5868	}
5869	HDR_SET_PSIZE(hdr, psize);
5870	arc_hdr_set_compress(hdr, compress);
5871
5872
5873	/*
5874	 * Fill the hdr with data. If the hdr is compressed, the data we want
5875	 * is available from the zio, otherwise we can take it from the buf.
5876	 *
5877	 * We might be able to share the buf's data with the hdr here. However,
5878	 * doing so would cause the ARC to be full of linear ABDs if we write a
5879	 * lot of shareable data. As a compromise, we check whether scattered
5880	 * ABDs are allowed, and assume that if they are then the user wants
5881	 * the ARC to be primarily filled with them regardless of the data being
5882	 * written. Therefore, if they're allowed then we allocate one and copy
5883	 * the data into it; otherwise, we share the data directly if we can.
5884	 */
5885	if (zfs_abd_scatter_enabled || !arc_can_share(hdr, buf)) {
5886		arc_hdr_alloc_pabd(hdr);
5887
5888		/*
5889		 * Ideally, we would always copy the io_abd into b_pabd, but the
5890		 * user may have disabled compressed ARC, thus we must check the
5891		 * hdr's compression setting rather than the io_bp's.
5892		 */
5893		if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) {
5894			ASSERT3U(BP_GET_COMPRESS(zio->io_bp), !=,
5895			    ZIO_COMPRESS_OFF);
5896			ASSERT3U(psize, >, 0);
5897
5898			abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize);
5899		} else {
5900			ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr));
5901
5902			abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data,
5903			    arc_buf_size(buf));
5904		}
5905	} else {
5906		ASSERT3P(buf->b_data, ==, abd_to_buf(zio->io_orig_abd));
5907		ASSERT3U(zio->io_orig_size, ==, arc_buf_size(buf));
5908		ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
5909
5910		arc_share_buf(hdr, buf);
5911	}
5912
5913	arc_hdr_verify(hdr, zio->io_bp);
5914}
5915
5916static void
5917arc_write_children_ready(zio_t *zio)
5918{
5919	arc_write_callback_t *callback = zio->io_private;
5920	arc_buf_t *buf = callback->awcb_buf;
5921
5922	callback->awcb_children_ready(zio, buf, callback->awcb_private);
5923}
5924
5925/*
5926 * The SPA calls this callback for each physical write that happens on behalf
5927 * of a logical write.  See the comment in dbuf_write_physdone() for details.
5928 */
5929static void
5930arc_write_physdone(zio_t *zio)
5931{
5932	arc_write_callback_t *cb = zio->io_private;
5933	if (cb->awcb_physdone != NULL)
5934		cb->awcb_physdone(zio, cb->awcb_buf, cb->awcb_private);
5935}
5936
5937static void
5938arc_write_done(zio_t *zio)
5939{
5940	arc_write_callback_t *callback = zio->io_private;
5941	arc_buf_t *buf = callback->awcb_buf;
5942	arc_buf_hdr_t *hdr = buf->b_hdr;
5943
5944	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
5945
5946	if (zio->io_error == 0) {
5947		arc_hdr_verify(hdr, zio->io_bp);
5948
5949		if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
5950			buf_discard_identity(hdr);
5951		} else {
5952			hdr->b_dva = *BP_IDENTITY(zio->io_bp);
5953			hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
5954		}
5955	} else {
5956		ASSERT(HDR_EMPTY(hdr));
5957	}
5958
5959	/*
5960	 * If the block to be written was all-zero or compressed enough to be
5961	 * embedded in the BP, no write was performed so there will be no
5962	 * dva/birth/checksum.  The buffer must therefore remain anonymous
5963	 * (and uncached).
5964	 */
5965	if (!HDR_EMPTY(hdr)) {
5966		arc_buf_hdr_t *exists;
5967		kmutex_t *hash_lock;
5968
5969		ASSERT3U(zio->io_error, ==, 0);
5970
5971		arc_cksum_verify(buf);
5972
5973		exists = buf_hash_insert(hdr, &hash_lock);
5974		if (exists != NULL) {
5975			/*
5976			 * This can only happen if we overwrite for
5977			 * sync-to-convergence, because we remove
5978			 * buffers from the hash table when we arc_free().
5979			 */
5980			if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
5981				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
5982					panic("bad overwrite, hdr=%p exists=%p",
5983					    (void *)hdr, (void *)exists);
5984				ASSERT(refcount_is_zero(
5985				    &exists->b_l1hdr.b_refcnt));
5986				arc_change_state(arc_anon, exists, hash_lock);
5987				mutex_exit(hash_lock);
5988				arc_hdr_destroy(exists);
5989				exists = buf_hash_insert(hdr, &hash_lock);
5990				ASSERT3P(exists, ==, NULL);
5991			} else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
5992				/* nopwrite */
5993				ASSERT(zio->io_prop.zp_nopwrite);
5994				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
5995					panic("bad nopwrite, hdr=%p exists=%p",
5996					    (void *)hdr, (void *)exists);
5997			} else {
5998				/* Dedup */
5999				ASSERT(hdr->b_l1hdr.b_bufcnt == 1);
6000				ASSERT(hdr->b_l1hdr.b_state == arc_anon);
6001				ASSERT(BP_GET_DEDUP(zio->io_bp));
6002				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
6003			}
6004		}
6005		arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6006		/* if it's not anon, we are doing a scrub */
6007		if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
6008			arc_access(hdr, hash_lock);
6009		mutex_exit(hash_lock);
6010	} else {
6011		arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6012	}
6013
6014	ASSERT(!refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
6015	callback->awcb_done(zio, buf, callback->awcb_private);
6016
6017	abd_put(zio->io_abd);
6018	kmem_free(callback, sizeof (arc_write_callback_t));
6019}
6020
6021zio_t *
6022arc_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, arc_buf_t *buf,
6023    boolean_t l2arc, const zio_prop_t *zp, arc_done_func_t *ready,
6024    arc_done_func_t *children_ready, arc_done_func_t *physdone,
6025    arc_done_func_t *done, void *private, zio_priority_t priority,
6026    int zio_flags, const zbookmark_phys_t *zb)
6027{
6028	arc_buf_hdr_t *hdr = buf->b_hdr;
6029	arc_write_callback_t *callback;
6030	zio_t *zio;
6031	zio_prop_t localprop = *zp;
6032
6033	ASSERT3P(ready, !=, NULL);
6034	ASSERT3P(done, !=, NULL);
6035	ASSERT(!HDR_IO_ERROR(hdr));
6036	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6037	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6038	ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
6039	if (l2arc)
6040		arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
6041	if (ARC_BUF_COMPRESSED(buf)) {
6042		/*
6043		 * We're writing a pre-compressed buffer.  Make the
6044		 * compression algorithm requested by the zio_prop_t match
6045		 * the pre-compressed buffer's compression algorithm.
6046		 */
6047		localprop.zp_compress = HDR_GET_COMPRESS(hdr);
6048
6049		ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf));
6050		zio_flags |= ZIO_FLAG_RAW;
6051	}
6052	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
6053	callback->awcb_ready = ready;
6054	callback->awcb_children_ready = children_ready;
6055	callback->awcb_physdone = physdone;
6056	callback->awcb_done = done;
6057	callback->awcb_private = private;
6058	callback->awcb_buf = buf;
6059
6060	/*
6061	 * The hdr's b_pabd is now stale, free it now. A new data block
6062	 * will be allocated when the zio pipeline calls arc_write_ready().
6063	 */
6064	if (hdr->b_l1hdr.b_pabd != NULL) {
6065		/*
6066		 * If the buf is currently sharing the data block with
6067		 * the hdr then we need to break that relationship here.
6068		 * The hdr will remain with a NULL data pointer and the
6069		 * buf will take sole ownership of the block.
6070		 */
6071		if (arc_buf_is_shared(buf)) {
6072			arc_unshare_buf(hdr, buf);
6073		} else {
6074			arc_hdr_free_pabd(hdr);
6075		}
6076		VERIFY3P(buf->b_data, !=, NULL);
6077		arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF);
6078	}
6079	ASSERT(!arc_buf_is_shared(buf));
6080	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6081
6082	zio = zio_write(pio, spa, txg, bp,
6083	    abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)),
6084	    HDR_GET_LSIZE(hdr), arc_buf_size(buf), &localprop, arc_write_ready,
6085	    (children_ready != NULL) ? arc_write_children_ready : NULL,
6086	    arc_write_physdone, arc_write_done, callback,
6087	    priority, zio_flags, zb);
6088
6089	return (zio);
6090}
6091
6092static int
6093arc_memory_throttle(uint64_t reserve, uint64_t txg)
6094{
6095#ifdef _KERNEL
6096	uint64_t available_memory = ptob(freemem);
6097	static uint64_t page_load = 0;
6098	static uint64_t last_txg = 0;
6099
6100#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC)
6101	available_memory =
6102	    MIN(available_memory, ptob(vmem_size(heap_arena, VMEM_FREE)));
6103#endif
6104
6105	if (freemem > (uint64_t)physmem * arc_lotsfree_percent / 100)
6106		return (0);
6107
6108	if (txg > last_txg) {
6109		last_txg = txg;
6110		page_load = 0;
6111	}
6112	/*
6113	 * If we are in pageout, we know that memory is already tight,
6114	 * the arc is already going to be evicting, so we just want to
6115	 * continue to let page writes occur as quickly as possible.
6116	 */
6117	if (curproc == pageproc) {
6118		if (page_load > MAX(ptob(minfree), available_memory) / 4)
6119			return (SET_ERROR(ERESTART));
6120		/* Note: reserve is inflated, so we deflate */
6121		page_load += reserve / 8;
6122		return (0);
6123	} else if (page_load > 0 && arc_reclaim_needed()) {
6124		/* memory is low, delay before restarting */
6125		ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
6126		return (SET_ERROR(EAGAIN));
6127	}
6128	page_load = 0;
6129#endif
6130	return (0);
6131}
6132
6133void
6134arc_tempreserve_clear(uint64_t reserve)
6135{
6136	atomic_add_64(&arc_tempreserve, -reserve);
6137	ASSERT((int64_t)arc_tempreserve >= 0);
6138}
6139
6140int
6141arc_tempreserve_space(uint64_t reserve, uint64_t txg)
6142{
6143	int error;
6144	uint64_t anon_size;
6145
6146	if (reserve > arc_c/4 && !arc_no_grow) {
6147		arc_c = MIN(arc_c_max, reserve * 4);
6148		DTRACE_PROBE1(arc__set_reserve, uint64_t, arc_c);
6149	}
6150	if (reserve > arc_c)
6151		return (SET_ERROR(ENOMEM));
6152
6153	/*
6154	 * Don't count loaned bufs as in flight dirty data to prevent long
6155	 * network delays from blocking transactions that are ready to be
6156	 * assigned to a txg.
6157	 */
6158
6159	/* assert that it has not wrapped around */
6160	ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
6161
6162	anon_size = MAX((int64_t)(refcount_count(&arc_anon->arcs_size) -
6163	    arc_loaned_bytes), 0);
6164
6165	/*
6166	 * Writes will, almost always, require additional memory allocations
6167	 * in order to compress/encrypt/etc the data.  We therefore need to
6168	 * make sure that there is sufficient available memory for this.
6169	 */
6170	error = arc_memory_throttle(reserve, txg);
6171	if (error != 0)
6172		return (error);
6173
6174	/*
6175	 * Throttle writes when the amount of dirty data in the cache
6176	 * gets too large.  We try to keep the cache less than half full
6177	 * of dirty blocks so that our sync times don't grow too large.
6178	 * Note: if two requests come in concurrently, we might let them
6179	 * both succeed, when one of them should fail.  Not a huge deal.
6180	 */
6181
6182	if (reserve + arc_tempreserve + anon_size > arc_c / 2 &&
6183	    anon_size > arc_c / 4) {
6184		uint64_t meta_esize =
6185		    refcount_count(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
6186		uint64_t data_esize =
6187		    refcount_count(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
6188		dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
6189		    "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
6190		    arc_tempreserve >> 10, meta_esize >> 10,
6191		    data_esize >> 10, reserve >> 10, arc_c >> 10);
6192		return (SET_ERROR(ERESTART));
6193	}
6194	atomic_add_64(&arc_tempreserve, reserve);
6195	return (0);
6196}
6197
6198static void
6199arc_kstat_update_state(arc_state_t *state, kstat_named_t *size,
6200    kstat_named_t *evict_data, kstat_named_t *evict_metadata)
6201{
6202	size->value.ui64 = refcount_count(&state->arcs_size);
6203	evict_data->value.ui64 =
6204	    refcount_count(&state->arcs_esize[ARC_BUFC_DATA]);
6205	evict_metadata->value.ui64 =
6206	    refcount_count(&state->arcs_esize[ARC_BUFC_METADATA]);
6207}
6208
6209static int
6210arc_kstat_update(kstat_t *ksp, int rw)
6211{
6212	arc_stats_t *as = ksp->ks_data;
6213
6214	if (rw == KSTAT_WRITE) {
6215		return (EACCES);
6216	} else {
6217		arc_kstat_update_state(arc_anon,
6218		    &as->arcstat_anon_size,
6219		    &as->arcstat_anon_evictable_data,
6220		    &as->arcstat_anon_evictable_metadata);
6221		arc_kstat_update_state(arc_mru,
6222		    &as->arcstat_mru_size,
6223		    &as->arcstat_mru_evictable_data,
6224		    &as->arcstat_mru_evictable_metadata);
6225		arc_kstat_update_state(arc_mru_ghost,
6226		    &as->arcstat_mru_ghost_size,
6227		    &as->arcstat_mru_ghost_evictable_data,
6228		    &as->arcstat_mru_ghost_evictable_metadata);
6229		arc_kstat_update_state(arc_mfu,
6230		    &as->arcstat_mfu_size,
6231		    &as->arcstat_mfu_evictable_data,
6232		    &as->arcstat_mfu_evictable_metadata);
6233		arc_kstat_update_state(arc_mfu_ghost,
6234		    &as->arcstat_mfu_ghost_size,
6235		    &as->arcstat_mfu_ghost_evictable_data,
6236		    &as->arcstat_mfu_ghost_evictable_metadata);
6237	}
6238
6239	return (0);
6240}
6241
6242/*
6243 * This function *must* return indices evenly distributed between all
6244 * sublists of the multilist. This is needed due to how the ARC eviction
6245 * code is laid out; arc_evict_state() assumes ARC buffers are evenly
6246 * distributed between all sublists and uses this assumption when
6247 * deciding which sublist to evict from and how much to evict from it.
6248 */
6249unsigned int
6250arc_state_multilist_index_func(multilist_t *ml, void *obj)
6251{
6252	arc_buf_hdr_t *hdr = obj;
6253
6254	/*
6255	 * We rely on b_dva to generate evenly distributed index
6256	 * numbers using buf_hash below. So, as an added precaution,
6257	 * let's make sure we never add empty buffers to the arc lists.
6258	 */
6259	ASSERT(!HDR_EMPTY(hdr));
6260
6261	/*
6262	 * The assumption here, is the hash value for a given
6263	 * arc_buf_hdr_t will remain constant throughout it's lifetime
6264	 * (i.e. it's b_spa, b_dva, and b_birth fields don't change).
6265	 * Thus, we don't need to store the header's sublist index
6266	 * on insertion, as this index can be recalculated on removal.
6267	 *
6268	 * Also, the low order bits of the hash value are thought to be
6269	 * distributed evenly. Otherwise, in the case that the multilist
6270	 * has a power of two number of sublists, each sublists' usage
6271	 * would not be evenly distributed.
6272	 */
6273	return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
6274	    multilist_get_num_sublists(ml));
6275}
6276
6277#ifdef _KERNEL
6278static eventhandler_tag arc_event_lowmem = NULL;
6279
6280static void
6281arc_lowmem(void *arg __unused, int howto __unused)
6282{
6283
6284	mutex_enter(&arc_reclaim_lock);
6285	/* XXX: Memory deficit should be passed as argument. */
6286	needfree = btoc(arc_c >> arc_shrink_shift);
6287	DTRACE_PROBE(arc__needfree);
6288	cv_signal(&arc_reclaim_thread_cv);
6289
6290	/*
6291	 * It is unsafe to block here in arbitrary threads, because we can come
6292	 * here from ARC itself and may hold ARC locks and thus risk a deadlock
6293	 * with ARC reclaim thread.
6294	 */
6295	if (curproc == pageproc)
6296		(void) cv_wait(&arc_reclaim_waiters_cv, &arc_reclaim_lock);
6297	mutex_exit(&arc_reclaim_lock);
6298}
6299#endif
6300
6301static void
6302arc_state_init(void)
6303{
6304	arc_anon = &ARC_anon;
6305	arc_mru = &ARC_mru;
6306	arc_mru_ghost = &ARC_mru_ghost;
6307	arc_mfu = &ARC_mfu;
6308	arc_mfu_ghost = &ARC_mfu_ghost;
6309	arc_l2c_only = &ARC_l2c_only;
6310
6311	arc_mru->arcs_list[ARC_BUFC_METADATA] =
6312	    multilist_create(sizeof (arc_buf_hdr_t),
6313	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6314	    arc_state_multilist_index_func);
6315	arc_mru->arcs_list[ARC_BUFC_DATA] =
6316	    multilist_create(sizeof (arc_buf_hdr_t),
6317	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6318	    arc_state_multilist_index_func);
6319	arc_mru_ghost->arcs_list[ARC_BUFC_METADATA] =
6320	    multilist_create(sizeof (arc_buf_hdr_t),
6321	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6322	    arc_state_multilist_index_func);
6323	arc_mru_ghost->arcs_list[ARC_BUFC_DATA] =
6324	    multilist_create(sizeof (arc_buf_hdr_t),
6325	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6326	    arc_state_multilist_index_func);
6327	arc_mfu->arcs_list[ARC_BUFC_METADATA] =
6328	    multilist_create(sizeof (arc_buf_hdr_t),
6329	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6330	    arc_state_multilist_index_func);
6331	arc_mfu->arcs_list[ARC_BUFC_DATA] =
6332	    multilist_create(sizeof (arc_buf_hdr_t),
6333	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6334	    arc_state_multilist_index_func);
6335	arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA] =
6336	    multilist_create(sizeof (arc_buf_hdr_t),
6337	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6338	    arc_state_multilist_index_func);
6339	arc_mfu_ghost->arcs_list[ARC_BUFC_DATA] =
6340	    multilist_create(sizeof (arc_buf_hdr_t),
6341	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6342	    arc_state_multilist_index_func);
6343	arc_l2c_only->arcs_list[ARC_BUFC_METADATA] =
6344	    multilist_create(sizeof (arc_buf_hdr_t),
6345	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6346	    arc_state_multilist_index_func);
6347	arc_l2c_only->arcs_list[ARC_BUFC_DATA] =
6348	    multilist_create(sizeof (arc_buf_hdr_t),
6349	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6350	    arc_state_multilist_index_func);
6351
6352	refcount_create(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
6353	refcount_create(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
6354	refcount_create(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
6355	refcount_create(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
6356	refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
6357	refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
6358	refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
6359	refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
6360	refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
6361	refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
6362	refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
6363	refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
6364
6365	refcount_create(&arc_anon->arcs_size);
6366	refcount_create(&arc_mru->arcs_size);
6367	refcount_create(&arc_mru_ghost->arcs_size);
6368	refcount_create(&arc_mfu->arcs_size);
6369	refcount_create(&arc_mfu_ghost->arcs_size);
6370	refcount_create(&arc_l2c_only->arcs_size);
6371}
6372
6373static void
6374arc_state_fini(void)
6375{
6376	refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
6377	refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
6378	refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
6379	refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
6380	refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
6381	refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
6382	refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
6383	refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
6384	refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
6385	refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
6386	refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
6387	refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
6388
6389	refcount_destroy(&arc_anon->arcs_size);
6390	refcount_destroy(&arc_mru->arcs_size);
6391	refcount_destroy(&arc_mru_ghost->arcs_size);
6392	refcount_destroy(&arc_mfu->arcs_size);
6393	refcount_destroy(&arc_mfu_ghost->arcs_size);
6394	refcount_destroy(&arc_l2c_only->arcs_size);
6395
6396	multilist_destroy(arc_mru->arcs_list[ARC_BUFC_METADATA]);
6397	multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
6398	multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_METADATA]);
6399	multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
6400	multilist_destroy(arc_mru->arcs_list[ARC_BUFC_DATA]);
6401	multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
6402	multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_DATA]);
6403	multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
6404}
6405
6406uint64_t
6407arc_max_bytes(void)
6408{
6409	return (arc_c_max);
6410}
6411
6412void
6413arc_init(void)
6414{
6415	int i, prefetch_tunable_set = 0;
6416
6417	/*
6418	 * allmem is "all memory that we could possibly use".
6419	 */
6420#ifdef illumos
6421#ifdef _KERNEL
6422	uint64_t allmem = ptob(physmem - swapfs_minfree);
6423#else
6424	uint64_t allmem = (physmem * PAGESIZE) / 2;
6425#endif
6426#else
6427	uint64_t allmem = kmem_size();
6428#endif
6429
6430
6431	mutex_init(&arc_reclaim_lock, NULL, MUTEX_DEFAULT, NULL);
6432	cv_init(&arc_reclaim_thread_cv, NULL, CV_DEFAULT, NULL);
6433	cv_init(&arc_reclaim_waiters_cv, NULL, CV_DEFAULT, NULL);
6434
6435	mutex_init(&arc_dnlc_evicts_lock, NULL, MUTEX_DEFAULT, NULL);
6436	cv_init(&arc_dnlc_evicts_cv, NULL, CV_DEFAULT, NULL);
6437
6438	/* Convert seconds to clock ticks */
6439	arc_min_prefetch_lifespan = 1 * hz;
6440
6441	/* set min cache to 1/32 of all memory, or arc_abs_min, whichever is more */
6442	arc_c_min = MAX(allmem / 32, arc_abs_min);
6443	/* set max to 5/8 of all memory, or all but 1GB, whichever is more */
6444	if (allmem >= 1 <