1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
24 * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
26 * Copyright 2017 Nexenta Systems, Inc.  All rights reserved.
27 */
28
29/*
30 * DVA-based Adjustable Replacement Cache
31 *
32 * While much of the theory of operation used here is
33 * based on the self-tuning, low overhead replacement cache
34 * presented by Megiddo and Modha at FAST 2003, there are some
35 * significant differences:
36 *
37 * 1. The Megiddo and Modha model assumes any page is evictable.
38 * Pages in its cache cannot be "locked" into memory.  This makes
39 * the eviction algorithm simple: evict the last page in the list.
40 * This also make the performance characteristics easy to reason
41 * about.  Our cache is not so simple.  At any given moment, some
42 * subset of the blocks in the cache are un-evictable because we
43 * have handed out a reference to them.  Blocks are only evictable
44 * when there are no external references active.  This makes
45 * eviction far more problematic:  we choose to evict the evictable
46 * blocks that are the "lowest" in the list.
47 *
48 * There are times when it is not possible to evict the requested
49 * space.  In these circumstances we are unable to adjust the cache
50 * size.  To prevent the cache growing unbounded at these times we
51 * implement a "cache throttle" that slows the flow of new data
52 * into the cache until we can make space available.
53 *
54 * 2. The Megiddo and Modha model assumes a fixed cache size.
55 * Pages are evicted when the cache is full and there is a cache
56 * miss.  Our model has a variable sized cache.  It grows with
57 * high use, but also tries to react to memory pressure from the
58 * operating system: decreasing its size when system memory is
59 * tight.
60 *
61 * 3. The Megiddo and Modha model assumes a fixed page size. All
62 * elements of the cache are therefore exactly the same size.  So
63 * when adjusting the cache size following a cache miss, its simply
64 * a matter of choosing a single page to evict.  In our model, we
65 * have variable sized cache blocks (rangeing from 512 bytes to
66 * 128K bytes).  We therefore choose a set of blocks to evict to make
67 * space for a cache miss that approximates as closely as possible
68 * the space used by the new block.
69 *
70 * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
71 * by N. Megiddo & D. Modha, FAST 2003
72 */
73
74/*
75 * The locking model:
76 *
77 * A new reference to a cache buffer can be obtained in two
78 * ways: 1) via a hash table lookup using the DVA as a key,
79 * or 2) via one of the ARC lists.  The arc_read() interface
80 * uses method 1, while the internal ARC algorithms for
81 * adjusting the cache use method 2.  We therefore provide two
82 * types of locks: 1) the hash table lock array, and 2) the
83 * ARC list locks.
84 *
85 * Buffers do not have their own mutexes, rather they rely on the
86 * hash table mutexes for the bulk of their protection (i.e. most
87 * fields in the arc_buf_hdr_t are protected by these mutexes).
88 *
89 * buf_hash_find() returns the appropriate mutex (held) when it
90 * locates the requested buffer in the hash table.  It returns
91 * NULL for the mutex if the buffer was not in the table.
92 *
93 * buf_hash_remove() expects the appropriate hash mutex to be
94 * already held before it is invoked.
95 *
96 * Each ARC state also has a mutex which is used to protect the
97 * buffer list associated with the state.  When attempting to
98 * obtain a hash table lock while holding an ARC list lock you
99 * must use: mutex_tryenter() to avoid deadlock.  Also note that
100 * the active state mutex must be held before the ghost state mutex.
101 *
102 * Note that the majority of the performance stats are manipulated
103 * with atomic operations.
104 *
105 * The L2ARC uses the l2ad_mtx on each vdev for the following:
106 *
107 *	- L2ARC buflist creation
108 *	- L2ARC buflist eviction
109 *	- L2ARC write completion, which walks L2ARC buflists
110 *	- ARC header destruction, as it removes from L2ARC buflists
111 *	- ARC header release, as it removes from L2ARC buflists
112 */
113
114/*
115 * ARC operation:
116 *
117 * Every block that is in the ARC is tracked by an arc_buf_hdr_t structure.
118 * This structure can point either to a block that is still in the cache or to
119 * one that is only accessible in an L2 ARC device, or it can provide
120 * information about a block that was recently evicted. If a block is
121 * only accessible in the L2ARC, then the arc_buf_hdr_t only has enough
122 * information to retrieve it from the L2ARC device. This information is
123 * stored in the l2arc_buf_hdr_t sub-structure of the arc_buf_hdr_t. A block
124 * that is in this state cannot access the data directly.
125 *
126 * Blocks that are actively being referenced or have not been evicted
127 * are cached in the L1ARC. The L1ARC (l1arc_buf_hdr_t) is a structure within
128 * the arc_buf_hdr_t that will point to the data block in memory. A block can
129 * only be read by a consumer if it has an l1arc_buf_hdr_t. The L1ARC
130 * caches data in two ways -- in a list of ARC buffers (arc_buf_t) and
131 * also in the arc_buf_hdr_t's private physical data block pointer (b_pabd).
132 *
133 * The L1ARC's data pointer may or may not be uncompressed. The ARC has the
134 * ability to store the physical data (b_pabd) associated with the DVA of the
135 * arc_buf_hdr_t. Since the b_pabd is a copy of the on-disk physical block,
136 * it will match its on-disk compression characteristics. This behavior can be
137 * disabled by setting 'zfs_compressed_arc_enabled' to B_FALSE. When the
138 * compressed ARC functionality is disabled, the b_pabd will point to an
139 * uncompressed version of the on-disk data.
140 *
141 * Data in the L1ARC is not accessed by consumers of the ARC directly. Each
142 * arc_buf_hdr_t can have multiple ARC buffers (arc_buf_t) which reference it.
143 * Each ARC buffer (arc_buf_t) is being actively accessed by a specific ARC
144 * consumer. The ARC will provide references to this data and will keep it
145 * cached until it is no longer in use. The ARC caches only the L1ARC's physical
146 * data block and will evict any arc_buf_t that is no longer referenced. The
147 * amount of memory consumed by the arc_buf_ts' data buffers can be seen via the
148 * "overhead_size" kstat.
149 *
150 * Depending on the consumer, an arc_buf_t can be requested in uncompressed or
151 * compressed form. The typical case is that consumers will want uncompressed
152 * data, and when that happens a new data buffer is allocated where the data is
153 * decompressed for them to use. Currently the only consumer who wants
154 * compressed arc_buf_t's is "zfs send", when it streams data exactly as it
155 * exists on disk. When this happens, the arc_buf_t's data buffer is shared
156 * with the arc_buf_hdr_t.
157 *
158 * Here is a diagram showing an arc_buf_hdr_t referenced by two arc_buf_t's. The
159 * first one is owned by a compressed send consumer (and therefore references
160 * the same compressed data buffer as the arc_buf_hdr_t) and the second could be
161 * used by any other consumer (and has its own uncompressed copy of the data
162 * buffer).
163 *
164 *   arc_buf_hdr_t
165 *   +-----------+
166 *   | fields    |
167 *   | common to |
168 *   | L1- and   |
169 *   | L2ARC     |
170 *   +-----------+
171 *   | l2arc_buf_hdr_t
172 *   |           |
173 *   +-----------+
174 *   | l1arc_buf_hdr_t
175 *   |           |              arc_buf_t
176 *   | b_buf     +------------>+-----------+      arc_buf_t
177 *   | b_pabd    +-+           |b_next     +---->+-----------+
178 *   +-----------+ |           |-----------|     |b_next     +-->NULL
179 *                 |           |b_comp = T |     +-----------+
180 *                 |           |b_data     +-+   |b_comp = F |
181 *                 |           +-----------+ |   |b_data     +-+
182 *                 +->+------+               |   +-----------+ |
183 *        compressed  |      |               |                 |
184 *           data     |      |<--------------+                 | uncompressed
185 *                    +------+          compressed,            |     data
186 *                                        shared               +-->+------+
187 *                                         data                    |      |
188 *                                                                 |      |
189 *                                                                 +------+
190 *
191 * When a consumer reads a block, the ARC must first look to see if the
192 * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new
193 * arc_buf_t and either copies uncompressed data into a new data buffer from an
194 * existing uncompressed arc_buf_t, decompresses the hdr's b_pabd buffer into a
195 * new data buffer, or shares the hdr's b_pabd buffer, depending on whether the
196 * hdr is compressed and the desired compression characteristics of the
197 * arc_buf_t consumer. If the arc_buf_t ends up sharing data with the
198 * arc_buf_hdr_t and both of them are uncompressed then the arc_buf_t must be
199 * the last buffer in the hdr's b_buf list, however a shared compressed buf can
200 * be anywhere in the hdr's list.
201 *
202 * The diagram below shows an example of an uncompressed ARC hdr that is
203 * sharing its data with an arc_buf_t (note that the shared uncompressed buf is
204 * the last element in the buf list):
205 *
206 *                arc_buf_hdr_t
207 *                +-----------+
208 *                |           |
209 *                |           |
210 *                |           |
211 *                +-----------+
212 * l2arc_buf_hdr_t|           |
213 *                |           |
214 *                +-----------+
215 * l1arc_buf_hdr_t|           |
216 *                |           |                 arc_buf_t    (shared)
217 *                |    b_buf  +------------>+---------+      arc_buf_t
218 *                |           |             |b_next   +---->+---------+
219 *                |  b_pabd   +-+           |---------|     |b_next   +-->NULL
220 *                +-----------+ |           |         |     +---------+
221 *                              |           |b_data   +-+   |         |
222 *                              |           +---------+ |   |b_data   +-+
223 *                              +->+------+             |   +---------+ |
224 *                                 |      |             |               |
225 *                   uncompressed  |      |             |               |
226 *                        data     +------+             |               |
227 *                                    ^                 +->+------+     |
228 *                                    |       uncompressed |      |     |
229 *                                    |           data     |      |     |
230 *                                    |                    +------+     |
231 *                                    +---------------------------------+
232 *
233 * Writing to the ARC requires that the ARC first discard the hdr's b_pabd
234 * since the physical block is about to be rewritten. The new data contents
235 * will be contained in the arc_buf_t. As the I/O pipeline performs the write,
236 * it may compress the data before writing it to disk. The ARC will be called
237 * with the transformed data and will bcopy the transformed on-disk block into
238 * a newly allocated b_pabd. Writes are always done into buffers which have
239 * either been loaned (and hence are new and don't have other readers) or
240 * buffers which have been released (and hence have their own hdr, if there
241 * were originally other readers of the buf's original hdr). This ensures that
242 * the ARC only needs to update a single buf and its hdr after a write occurs.
243 *
244 * When the L2ARC is in use, it will also take advantage of the b_pabd. The
245 * L2ARC will always write the contents of b_pabd to the L2ARC. This means
246 * that when compressed ARC is enabled that the L2ARC blocks are identical
247 * to the on-disk block in the main data pool. This provides a significant
248 * advantage since the ARC can leverage the bp's checksum when reading from the
249 * L2ARC to determine if the contents are valid. However, if the compressed
250 * ARC is disabled, then the L2ARC's block must be transformed to look
251 * like the physical block in the main data pool before comparing the
252 * checksum and determining its validity.
253 */
254
255#include <sys/spa.h>
256#include <sys/zio.h>
257#include <sys/spa_impl.h>
258#include <sys/zio_compress.h>
259#include <sys/zio_checksum.h>
260#include <sys/zfs_context.h>
261#include <sys/arc.h>
262#include <sys/refcount.h>
263#include <sys/vdev.h>
264#include <sys/vdev_impl.h>
265#include <sys/dsl_pool.h>
266#include <sys/zio_checksum.h>
267#include <sys/multilist.h>
268#include <sys/abd.h>
269#ifdef _KERNEL
270#include <sys/dnlc.h>
271#include <sys/racct.h>
272#endif
273#include <sys/callb.h>
274#include <sys/kstat.h>
275#include <sys/trim_map.h>
276#include <zfs_fletcher.h>
277#include <sys/sdt.h>
278
279#include <machine/vmparam.h>
280
281#ifdef illumos
282#ifndef _KERNEL
283/* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
284boolean_t arc_watch = B_FALSE;
285int arc_procfd;
286#endif
287#endif /* illumos */
288
289static kmutex_t		arc_reclaim_lock;
290static kcondvar_t	arc_reclaim_thread_cv;
291static boolean_t	arc_reclaim_thread_exit;
292static kcondvar_t	arc_reclaim_waiters_cv;
293
294static kmutex_t		arc_dnlc_evicts_lock;
295static kcondvar_t	arc_dnlc_evicts_cv;
296static boolean_t	arc_dnlc_evicts_thread_exit;
297
298uint_t arc_reduce_dnlc_percent = 3;
299
300/*
301 * The number of headers to evict in arc_evict_state_impl() before
302 * dropping the sublist lock and evicting from another sublist. A lower
303 * value means we're more likely to evict the "correct" header (i.e. the
304 * oldest header in the arc state), but comes with higher overhead
305 * (i.e. more invocations of arc_evict_state_impl()).
306 */
307int zfs_arc_evict_batch_limit = 10;
308
309/* number of seconds before growing cache again */
310static int		arc_grow_retry = 60;
311
312/* shift of arc_c for calculating overflow limit in arc_get_data_impl */
313int		zfs_arc_overflow_shift = 8;
314
315/* shift of arc_c for calculating both min and max arc_p */
316static int		arc_p_min_shift = 4;
317
318/* log2(fraction of arc to reclaim) */
319static int		arc_shrink_shift = 7;
320
321/*
322 * log2(fraction of ARC which must be free to allow growing).
323 * I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
324 * when reading a new block into the ARC, we will evict an equal-sized block
325 * from the ARC.
326 *
327 * This must be less than arc_shrink_shift, so that when we shrink the ARC,
328 * we will still not allow it to grow.
329 */
330int			arc_no_grow_shift = 5;
331
332
333/*
334 * minimum lifespan of a prefetch block in clock ticks
335 * (initialized in arc_init())
336 */
337static int		arc_min_prefetch_lifespan;
338
339/*
340 * If this percent of memory is free, don't throttle.
341 */
342int arc_lotsfree_percent = 10;
343
344static int arc_dead;
345extern boolean_t zfs_prefetch_disable;
346
347/*
348 * The arc has filled available memory and has now warmed up.
349 */
350static boolean_t arc_warm;
351
352/*
353 * These tunables are for performance analysis.
354 */
355uint64_t zfs_arc_max;
356uint64_t zfs_arc_min;
357uint64_t zfs_arc_meta_limit = 0;
358uint64_t zfs_arc_meta_min = 0;
359int zfs_arc_grow_retry = 0;
360int zfs_arc_shrink_shift = 0;
361int zfs_arc_no_grow_shift = 0;
362int zfs_arc_p_min_shift = 0;
363uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
364u_int zfs_arc_free_target = 0;
365
366/* Absolute min for arc min / max is 16MB. */
367static uint64_t arc_abs_min = 16 << 20;
368
369boolean_t zfs_compressed_arc_enabled = B_TRUE;
370
371static int sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS);
372static int sysctl_vfs_zfs_arc_meta_limit(SYSCTL_HANDLER_ARGS);
373static int sysctl_vfs_zfs_arc_max(SYSCTL_HANDLER_ARGS);
374static int sysctl_vfs_zfs_arc_min(SYSCTL_HANDLER_ARGS);
375static int sysctl_vfs_zfs_arc_no_grow_shift(SYSCTL_HANDLER_ARGS);
376
377#if defined(__FreeBSD__) && defined(_KERNEL)
378static void
379arc_free_target_init(void *unused __unused)
380{
381
382	zfs_arc_free_target = (vm_cnt.v_free_min / 10) * 11;
383}
384SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY,
385    arc_free_target_init, NULL);
386
387TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit);
388TUNABLE_QUAD("vfs.zfs.arc_meta_min", &zfs_arc_meta_min);
389TUNABLE_INT("vfs.zfs.arc_shrink_shift", &zfs_arc_shrink_shift);
390TUNABLE_INT("vfs.zfs.arc_grow_retry", &zfs_arc_grow_retry);
391TUNABLE_INT("vfs.zfs.arc_no_grow_shift", &zfs_arc_no_grow_shift);
392SYSCTL_DECL(_vfs_zfs);
393SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_max, CTLTYPE_U64 | CTLFLAG_RWTUN,
394    0, sizeof(uint64_t), sysctl_vfs_zfs_arc_max, "QU", "Maximum ARC size");
395SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_min, CTLTYPE_U64 | CTLFLAG_RWTUN,
396    0, sizeof(uint64_t), sysctl_vfs_zfs_arc_min, "QU", "Minimum ARC size");
397SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_no_grow_shift, CTLTYPE_U32 | CTLFLAG_RWTUN,
398    0, sizeof(uint32_t), sysctl_vfs_zfs_arc_no_grow_shift, "U",
399    "log2(fraction of ARC which must be free to allow growing)");
400SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_average_blocksize, CTLFLAG_RDTUN,
401    &zfs_arc_average_blocksize, 0,
402    "ARC average blocksize");
403SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_shrink_shift, CTLFLAG_RW,
404    &arc_shrink_shift, 0,
405    "log2(fraction of arc to reclaim)");
406SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_grow_retry, CTLFLAG_RW,
407    &arc_grow_retry, 0,
408    "Wait in seconds before considering growing ARC");
409SYSCTL_INT(_vfs_zfs, OID_AUTO, compressed_arc_enabled, CTLFLAG_RDTUN,
410    &zfs_compressed_arc_enabled, 0, "Enable compressed ARC");
411
412/*
413 * We don't have a tunable for arc_free_target due to the dependency on
414 * pagedaemon initialisation.
415 */
416SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_free_target,
417    CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof(u_int),
418    sysctl_vfs_zfs_arc_free_target, "IU",
419    "Desired number of free pages below which ARC triggers reclaim");
420
421static int
422sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS)
423{
424	u_int val;
425	int err;
426
427	val = zfs_arc_free_target;
428	err = sysctl_handle_int(oidp, &val, 0, req);
429	if (err != 0 || req->newptr == NULL)
430		return (err);
431
432	if (val < minfree)
433		return (EINVAL);
434	if (val > vm_cnt.v_page_count)
435		return (EINVAL);
436
437	zfs_arc_free_target = val;
438
439	return (0);
440}
441
442/*
443 * Must be declared here, before the definition of corresponding kstat
444 * macro which uses the same names will confuse the compiler.
445 */
446SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_meta_limit,
447    CTLTYPE_U64 | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof(uint64_t),
448    sysctl_vfs_zfs_arc_meta_limit, "QU",
449    "ARC metadata limit");
450#endif
451
452/*
453 * Note that buffers can be in one of 6 states:
454 *	ARC_anon	- anonymous (discussed below)
455 *	ARC_mru		- recently used, currently cached
456 *	ARC_mru_ghost	- recentely used, no longer in cache
457 *	ARC_mfu		- frequently used, currently cached
458 *	ARC_mfu_ghost	- frequently used, no longer in cache
459 *	ARC_l2c_only	- exists in L2ARC but not other states
460 * When there are no active references to the buffer, they are
461 * are linked onto a list in one of these arc states.  These are
462 * the only buffers that can be evicted or deleted.  Within each
463 * state there are multiple lists, one for meta-data and one for
464 * non-meta-data.  Meta-data (indirect blocks, blocks of dnodes,
465 * etc.) is tracked separately so that it can be managed more
466 * explicitly: favored over data, limited explicitly.
467 *
468 * Anonymous buffers are buffers that are not associated with
469 * a DVA.  These are buffers that hold dirty block copies
470 * before they are written to stable storage.  By definition,
471 * they are "ref'd" and are considered part of arc_mru
472 * that cannot be freed.  Generally, they will aquire a DVA
473 * as they are written and migrate onto the arc_mru list.
474 *
475 * The ARC_l2c_only state is for buffers that are in the second
476 * level ARC but no longer in any of the ARC_m* lists.  The second
477 * level ARC itself may also contain buffers that are in any of
478 * the ARC_m* states - meaning that a buffer can exist in two
479 * places.  The reason for the ARC_l2c_only state is to keep the
480 * buffer header in the hash table, so that reads that hit the
481 * second level ARC benefit from these fast lookups.
482 */
483
484typedef struct arc_state {
485	/*
486	 * list of evictable buffers
487	 */
488	multilist_t *arcs_list[ARC_BUFC_NUMTYPES];
489	/*
490	 * total amount of evictable data in this state
491	 */
492	refcount_t arcs_esize[ARC_BUFC_NUMTYPES];
493	/*
494	 * total amount of data in this state; this includes: evictable,
495	 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA.
496	 */
497	refcount_t arcs_size;
498} arc_state_t;
499
500/* The 6 states: */
501static arc_state_t ARC_anon;
502static arc_state_t ARC_mru;
503static arc_state_t ARC_mru_ghost;
504static arc_state_t ARC_mfu;
505static arc_state_t ARC_mfu_ghost;
506static arc_state_t ARC_l2c_only;
507
508typedef struct arc_stats {
509	kstat_named_t arcstat_hits;
510	kstat_named_t arcstat_misses;
511	kstat_named_t arcstat_demand_data_hits;
512	kstat_named_t arcstat_demand_data_misses;
513	kstat_named_t arcstat_demand_metadata_hits;
514	kstat_named_t arcstat_demand_metadata_misses;
515	kstat_named_t arcstat_prefetch_data_hits;
516	kstat_named_t arcstat_prefetch_data_misses;
517	kstat_named_t arcstat_prefetch_metadata_hits;
518	kstat_named_t arcstat_prefetch_metadata_misses;
519	kstat_named_t arcstat_mru_hits;
520	kstat_named_t arcstat_mru_ghost_hits;
521	kstat_named_t arcstat_mfu_hits;
522	kstat_named_t arcstat_mfu_ghost_hits;
523	kstat_named_t arcstat_allocated;
524	kstat_named_t arcstat_deleted;
525	/*
526	 * Number of buffers that could not be evicted because the hash lock
527	 * was held by another thread.  The lock may not necessarily be held
528	 * by something using the same buffer, since hash locks are shared
529	 * by multiple buffers.
530	 */
531	kstat_named_t arcstat_mutex_miss;
532	/*
533	 * Number of buffers skipped because they have I/O in progress, are
534	 * indrect prefetch buffers that have not lived long enough, or are
535	 * not from the spa we're trying to evict from.
536	 */
537	kstat_named_t arcstat_evict_skip;
538	/*
539	 * Number of times arc_evict_state() was unable to evict enough
540	 * buffers to reach it's target amount.
541	 */
542	kstat_named_t arcstat_evict_not_enough;
543	kstat_named_t arcstat_evict_l2_cached;
544	kstat_named_t arcstat_evict_l2_eligible;
545	kstat_named_t arcstat_evict_l2_ineligible;
546	kstat_named_t arcstat_evict_l2_skip;
547	kstat_named_t arcstat_hash_elements;
548	kstat_named_t arcstat_hash_elements_max;
549	kstat_named_t arcstat_hash_collisions;
550	kstat_named_t arcstat_hash_chains;
551	kstat_named_t arcstat_hash_chain_max;
552	kstat_named_t arcstat_p;
553	kstat_named_t arcstat_c;
554	kstat_named_t arcstat_c_min;
555	kstat_named_t arcstat_c_max;
556	kstat_named_t arcstat_size;
557	/*
558	 * Number of compressed bytes stored in the arc_buf_hdr_t's b_pabd.
559	 * Note that the compressed bytes may match the uncompressed bytes
560	 * if the block is either not compressed or compressed arc is disabled.
561	 */
562	kstat_named_t arcstat_compressed_size;
563	/*
564	 * Uncompressed size of the data stored in b_pabd. If compressed
565	 * arc is disabled then this value will be identical to the stat
566	 * above.
567	 */
568	kstat_named_t arcstat_uncompressed_size;
569	/*
570	 * Number of bytes stored in all the arc_buf_t's. This is classified
571	 * as "overhead" since this data is typically short-lived and will
572	 * be evicted from the arc when it becomes unreferenced unless the
573	 * zfs_keep_uncompressed_metadata or zfs_keep_uncompressed_level
574	 * values have been set (see comment in dbuf.c for more information).
575	 */
576	kstat_named_t arcstat_overhead_size;
577	/*
578	 * Number of bytes consumed by internal ARC structures necessary
579	 * for tracking purposes; these structures are not actually
580	 * backed by ARC buffers. This includes arc_buf_hdr_t structures
581	 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only
582	 * caches), and arc_buf_t structures (allocated via arc_buf_t
583	 * cache).
584	 */
585	kstat_named_t arcstat_hdr_size;
586	/*
587	 * Number of bytes consumed by ARC buffers of type equal to
588	 * ARC_BUFC_DATA. This is generally consumed by buffers backing
589	 * on disk user data (e.g. plain file contents).
590	 */
591	kstat_named_t arcstat_data_size;
592	/*
593	 * Number of bytes consumed by ARC buffers of type equal to
594	 * ARC_BUFC_METADATA. This is generally consumed by buffers
595	 * backing on disk data that is used for internal ZFS
596	 * structures (e.g. ZAP, dnode, indirect blocks, etc).
597	 */
598	kstat_named_t arcstat_metadata_size;
599	/*
600	 * Number of bytes consumed by various buffers and structures
601	 * not actually backed with ARC buffers. This includes bonus
602	 * buffers (allocated directly via zio_buf_* functions),
603	 * dmu_buf_impl_t structures (allocated via dmu_buf_impl_t
604	 * cache), and dnode_t structures (allocated via dnode_t cache).
605	 */
606	kstat_named_t arcstat_other_size;
607	/*
608	 * Total number of bytes consumed by ARC buffers residing in the
609	 * arc_anon state. This includes *all* buffers in the arc_anon
610	 * state; e.g. data, metadata, evictable, and unevictable buffers
611	 * are all included in this value.
612	 */
613	kstat_named_t arcstat_anon_size;
614	/*
615	 * Number of bytes consumed by ARC buffers that meet the
616	 * following criteria: backing buffers of type ARC_BUFC_DATA,
617	 * residing in the arc_anon state, and are eligible for eviction
618	 * (e.g. have no outstanding holds on the buffer).
619	 */
620	kstat_named_t arcstat_anon_evictable_data;
621	/*
622	 * Number of bytes consumed by ARC buffers that meet the
623	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
624	 * residing in the arc_anon state, and are eligible for eviction
625	 * (e.g. have no outstanding holds on the buffer).
626	 */
627	kstat_named_t arcstat_anon_evictable_metadata;
628	/*
629	 * Total number of bytes consumed by ARC buffers residing in the
630	 * arc_mru state. This includes *all* buffers in the arc_mru
631	 * state; e.g. data, metadata, evictable, and unevictable buffers
632	 * are all included in this value.
633	 */
634	kstat_named_t arcstat_mru_size;
635	/*
636	 * Number of bytes consumed by ARC buffers that meet the
637	 * following criteria: backing buffers of type ARC_BUFC_DATA,
638	 * residing in the arc_mru state, and are eligible for eviction
639	 * (e.g. have no outstanding holds on the buffer).
640	 */
641	kstat_named_t arcstat_mru_evictable_data;
642	/*
643	 * Number of bytes consumed by ARC buffers that meet the
644	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
645	 * residing in the arc_mru state, and are eligible for eviction
646	 * (e.g. have no outstanding holds on the buffer).
647	 */
648	kstat_named_t arcstat_mru_evictable_metadata;
649	/*
650	 * Total number of bytes that *would have been* consumed by ARC
651	 * buffers in the arc_mru_ghost state. The key thing to note
652	 * here, is the fact that this size doesn't actually indicate
653	 * RAM consumption. The ghost lists only consist of headers and
654	 * don't actually have ARC buffers linked off of these headers.
655	 * Thus, *if* the headers had associated ARC buffers, these
656	 * buffers *would have* consumed this number of bytes.
657	 */
658	kstat_named_t arcstat_mru_ghost_size;
659	/*
660	 * Number of bytes that *would have been* consumed by ARC
661	 * buffers that are eligible for eviction, of type
662	 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state.
663	 */
664	kstat_named_t arcstat_mru_ghost_evictable_data;
665	/*
666	 * Number of bytes that *would have been* consumed by ARC
667	 * buffers that are eligible for eviction, of type
668	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
669	 */
670	kstat_named_t arcstat_mru_ghost_evictable_metadata;
671	/*
672	 * Total number of bytes consumed by ARC buffers residing in the
673	 * arc_mfu state. This includes *all* buffers in the arc_mfu
674	 * state; e.g. data, metadata, evictable, and unevictable buffers
675	 * are all included in this value.
676	 */
677	kstat_named_t arcstat_mfu_size;
678	/*
679	 * Number of bytes consumed by ARC buffers that are eligible for
680	 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu
681	 * state.
682	 */
683	kstat_named_t arcstat_mfu_evictable_data;
684	/*
685	 * Number of bytes consumed by ARC buffers that are eligible for
686	 * eviction, of type ARC_BUFC_METADATA, and reside in the
687	 * arc_mfu state.
688	 */
689	kstat_named_t arcstat_mfu_evictable_metadata;
690	/*
691	 * Total number of bytes that *would have been* consumed by ARC
692	 * buffers in the arc_mfu_ghost state. See the comment above
693	 * arcstat_mru_ghost_size for more details.
694	 */
695	kstat_named_t arcstat_mfu_ghost_size;
696	/*
697	 * Number of bytes that *would have been* consumed by ARC
698	 * buffers that are eligible for eviction, of type
699	 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state.
700	 */
701	kstat_named_t arcstat_mfu_ghost_evictable_data;
702	/*
703	 * Number of bytes that *would have been* consumed by ARC
704	 * buffers that are eligible for eviction, of type
705	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
706	 */
707	kstat_named_t arcstat_mfu_ghost_evictable_metadata;
708	kstat_named_t arcstat_l2_hits;
709	kstat_named_t arcstat_l2_misses;
710	kstat_named_t arcstat_l2_feeds;
711	kstat_named_t arcstat_l2_rw_clash;
712	kstat_named_t arcstat_l2_read_bytes;
713	kstat_named_t arcstat_l2_write_bytes;
714	kstat_named_t arcstat_l2_writes_sent;
715	kstat_named_t arcstat_l2_writes_done;
716	kstat_named_t arcstat_l2_writes_error;
717	kstat_named_t arcstat_l2_writes_lock_retry;
718	kstat_named_t arcstat_l2_evict_lock_retry;
719	kstat_named_t arcstat_l2_evict_reading;
720	kstat_named_t arcstat_l2_evict_l1cached;
721	kstat_named_t arcstat_l2_free_on_write;
722	kstat_named_t arcstat_l2_abort_lowmem;
723	kstat_named_t arcstat_l2_cksum_bad;
724	kstat_named_t arcstat_l2_io_error;
725	kstat_named_t arcstat_l2_lsize;
726	kstat_named_t arcstat_l2_psize;
727	kstat_named_t arcstat_l2_hdr_size;
728	kstat_named_t arcstat_l2_write_trylock_fail;
729	kstat_named_t arcstat_l2_write_passed_headroom;
730	kstat_named_t arcstat_l2_write_spa_mismatch;
731	kstat_named_t arcstat_l2_write_in_l2;
732	kstat_named_t arcstat_l2_write_hdr_io_in_progress;
733	kstat_named_t arcstat_l2_write_not_cacheable;
734	kstat_named_t arcstat_l2_write_full;
735	kstat_named_t arcstat_l2_write_buffer_iter;
736	kstat_named_t arcstat_l2_write_pios;
737	kstat_named_t arcstat_l2_write_buffer_bytes_scanned;
738	kstat_named_t arcstat_l2_write_buffer_list_iter;
739	kstat_named_t arcstat_l2_write_buffer_list_null_iter;
740	kstat_named_t arcstat_memory_throttle_count;
741	kstat_named_t arcstat_meta_used;
742	kstat_named_t arcstat_meta_limit;
743	kstat_named_t arcstat_meta_max;
744	kstat_named_t arcstat_meta_min;
745	kstat_named_t arcstat_sync_wait_for_async;
746	kstat_named_t arcstat_demand_hit_predictive_prefetch;
747} arc_stats_t;
748
749static arc_stats_t arc_stats = {
750	{ "hits",			KSTAT_DATA_UINT64 },
751	{ "misses",			KSTAT_DATA_UINT64 },
752	{ "demand_data_hits",		KSTAT_DATA_UINT64 },
753	{ "demand_data_misses",		KSTAT_DATA_UINT64 },
754	{ "demand_metadata_hits",	KSTAT_DATA_UINT64 },
755	{ "demand_metadata_misses",	KSTAT_DATA_UINT64 },
756	{ "prefetch_data_hits",		KSTAT_DATA_UINT64 },
757	{ "prefetch_data_misses",	KSTAT_DATA_UINT64 },
758	{ "prefetch_metadata_hits",	KSTAT_DATA_UINT64 },
759	{ "prefetch_metadata_misses",	KSTAT_DATA_UINT64 },
760	{ "mru_hits",			KSTAT_DATA_UINT64 },
761	{ "mru_ghost_hits",		KSTAT_DATA_UINT64 },
762	{ "mfu_hits",			KSTAT_DATA_UINT64 },
763	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
764	{ "allocated",			KSTAT_DATA_UINT64 },
765	{ "deleted",			KSTAT_DATA_UINT64 },
766	{ "mutex_miss",			KSTAT_DATA_UINT64 },
767	{ "evict_skip",			KSTAT_DATA_UINT64 },
768	{ "evict_not_enough",		KSTAT_DATA_UINT64 },
769	{ "evict_l2_cached",		KSTAT_DATA_UINT64 },
770	{ "evict_l2_eligible",		KSTAT_DATA_UINT64 },
771	{ "evict_l2_ineligible",	KSTAT_DATA_UINT64 },
772	{ "evict_l2_skip",		KSTAT_DATA_UINT64 },
773	{ "hash_elements",		KSTAT_DATA_UINT64 },
774	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
775	{ "hash_collisions",		KSTAT_DATA_UINT64 },
776	{ "hash_chains",		KSTAT_DATA_UINT64 },
777	{ "hash_chain_max",		KSTAT_DATA_UINT64 },
778	{ "p",				KSTAT_DATA_UINT64 },
779	{ "c",				KSTAT_DATA_UINT64 },
780	{ "c_min",			KSTAT_DATA_UINT64 },
781	{ "c_max",			KSTAT_DATA_UINT64 },
782	{ "size",			KSTAT_DATA_UINT64 },
783	{ "compressed_size",		KSTAT_DATA_UINT64 },
784	{ "uncompressed_size",		KSTAT_DATA_UINT64 },
785	{ "overhead_size",		KSTAT_DATA_UINT64 },
786	{ "hdr_size",			KSTAT_DATA_UINT64 },
787	{ "data_size",			KSTAT_DATA_UINT64 },
788	{ "metadata_size",		KSTAT_DATA_UINT64 },
789	{ "other_size",			KSTAT_DATA_UINT64 },
790	{ "anon_size",			KSTAT_DATA_UINT64 },
791	{ "anon_evictable_data",	KSTAT_DATA_UINT64 },
792	{ "anon_evictable_metadata",	KSTAT_DATA_UINT64 },
793	{ "mru_size",			KSTAT_DATA_UINT64 },
794	{ "mru_evictable_data",		KSTAT_DATA_UINT64 },
795	{ "mru_evictable_metadata",	KSTAT_DATA_UINT64 },
796	{ "mru_ghost_size",		KSTAT_DATA_UINT64 },
797	{ "mru_ghost_evictable_data",	KSTAT_DATA_UINT64 },
798	{ "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
799	{ "mfu_size",			KSTAT_DATA_UINT64 },
800	{ "mfu_evictable_data",		KSTAT_DATA_UINT64 },
801	{ "mfu_evictable_metadata",	KSTAT_DATA_UINT64 },
802	{ "mfu_ghost_size",		KSTAT_DATA_UINT64 },
803	{ "mfu_ghost_evictable_data",	KSTAT_DATA_UINT64 },
804	{ "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
805	{ "l2_hits",			KSTAT_DATA_UINT64 },
806	{ "l2_misses",			KSTAT_DATA_UINT64 },
807	{ "l2_feeds",			KSTAT_DATA_UINT64 },
808	{ "l2_rw_clash",		KSTAT_DATA_UINT64 },
809	{ "l2_read_bytes",		KSTAT_DATA_UINT64 },
810	{ "l2_write_bytes",		KSTAT_DATA_UINT64 },
811	{ "l2_writes_sent",		KSTAT_DATA_UINT64 },
812	{ "l2_writes_done",		KSTAT_DATA_UINT64 },
813	{ "l2_writes_error",		KSTAT_DATA_UINT64 },
814	{ "l2_writes_lock_retry",	KSTAT_DATA_UINT64 },
815	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
816	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
817	{ "l2_evict_l1cached",		KSTAT_DATA_UINT64 },
818	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
819	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
820	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
821	{ "l2_io_error",		KSTAT_DATA_UINT64 },
822	{ "l2_size",			KSTAT_DATA_UINT64 },
823	{ "l2_asize",			KSTAT_DATA_UINT64 },
824	{ "l2_hdr_size",		KSTAT_DATA_UINT64 },
825	{ "l2_write_trylock_fail",	KSTAT_DATA_UINT64 },
826	{ "l2_write_passed_headroom",	KSTAT_DATA_UINT64 },
827	{ "l2_write_spa_mismatch",	KSTAT_DATA_UINT64 },
828	{ "l2_write_in_l2",		KSTAT_DATA_UINT64 },
829	{ "l2_write_io_in_progress",	KSTAT_DATA_UINT64 },
830	{ "l2_write_not_cacheable",	KSTAT_DATA_UINT64 },
831	{ "l2_write_full",		KSTAT_DATA_UINT64 },
832	{ "l2_write_buffer_iter",	KSTAT_DATA_UINT64 },
833	{ "l2_write_pios",		KSTAT_DATA_UINT64 },
834	{ "l2_write_buffer_bytes_scanned", KSTAT_DATA_UINT64 },
835	{ "l2_write_buffer_list_iter",	KSTAT_DATA_UINT64 },
836	{ "l2_write_buffer_list_null_iter", KSTAT_DATA_UINT64 },
837	{ "memory_throttle_count",	KSTAT_DATA_UINT64 },
838	{ "arc_meta_used",		KSTAT_DATA_UINT64 },
839	{ "arc_meta_limit",		KSTAT_DATA_UINT64 },
840	{ "arc_meta_max",		KSTAT_DATA_UINT64 },
841	{ "arc_meta_min",		KSTAT_DATA_UINT64 },
842	{ "sync_wait_for_async",	KSTAT_DATA_UINT64 },
843	{ "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 },
844};
845
846#define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
847
848#define	ARCSTAT_INCR(stat, val) \
849	atomic_add_64(&arc_stats.stat.value.ui64, (val))
850
851#define	ARCSTAT_BUMP(stat)	ARCSTAT_INCR(stat, 1)
852#define	ARCSTAT_BUMPDOWN(stat)	ARCSTAT_INCR(stat, -1)
853
854#define	ARCSTAT_MAX(stat, val) {					\
855	uint64_t m;							\
856	while ((val) > (m = arc_stats.stat.value.ui64) &&		\
857	    (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))	\
858		continue;						\
859}
860
861#define	ARCSTAT_MAXSTAT(stat) \
862	ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
863
864/*
865 * We define a macro to allow ARC hits/misses to be easily broken down by
866 * two separate conditions, giving a total of four different subtypes for
867 * each of hits and misses (so eight statistics total).
868 */
869#define	ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
870	if (cond1) {							\
871		if (cond2) {						\
872			ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
873		} else {						\
874			ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
875		}							\
876	} else {							\
877		if (cond2) {						\
878			ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
879		} else {						\
880			ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
881		}							\
882	}
883
884kstat_t			*arc_ksp;
885static arc_state_t	*arc_anon;
886static arc_state_t	*arc_mru;
887static arc_state_t	*arc_mru_ghost;
888static arc_state_t	*arc_mfu;
889static arc_state_t	*arc_mfu_ghost;
890static arc_state_t	*arc_l2c_only;
891
892/*
893 * There are several ARC variables that are critical to export as kstats --
894 * but we don't want to have to grovel around in the kstat whenever we wish to
895 * manipulate them.  For these variables, we therefore define them to be in
896 * terms of the statistic variable.  This assures that we are not introducing
897 * the possibility of inconsistency by having shadow copies of the variables,
898 * while still allowing the code to be readable.
899 */
900#define	arc_size	ARCSTAT(arcstat_size)	/* actual total arc size */
901#define	arc_p		ARCSTAT(arcstat_p)	/* target size of MRU */
902#define	arc_c		ARCSTAT(arcstat_c)	/* target size of cache */
903#define	arc_c_min	ARCSTAT(arcstat_c_min)	/* min target cache size */
904#define	arc_c_max	ARCSTAT(arcstat_c_max)	/* max target cache size */
905#define	arc_meta_limit	ARCSTAT(arcstat_meta_limit) /* max size for metadata */
906#define	arc_meta_min	ARCSTAT(arcstat_meta_min) /* min size for metadata */
907#define	arc_meta_used	ARCSTAT(arcstat_meta_used) /* size of metadata */
908#define	arc_meta_max	ARCSTAT(arcstat_meta_max) /* max size of metadata */
909
910/* compressed size of entire arc */
911#define	arc_compressed_size	ARCSTAT(arcstat_compressed_size)
912/* uncompressed size of entire arc */
913#define	arc_uncompressed_size	ARCSTAT(arcstat_uncompressed_size)
914/* number of bytes in the arc from arc_buf_t's */
915#define	arc_overhead_size	ARCSTAT(arcstat_overhead_size)
916
917static int		arc_no_grow;	/* Don't try to grow cache size */
918static uint64_t		arc_tempreserve;
919static uint64_t		arc_loaned_bytes;
920
921typedef struct arc_callback arc_callback_t;
922
923struct arc_callback {
924	void			*acb_private;
925	arc_done_func_t		*acb_done;
926	arc_buf_t		*acb_buf;
927	boolean_t		acb_compressed;
928	zio_t			*acb_zio_dummy;
929	arc_callback_t		*acb_next;
930};
931
932typedef struct arc_write_callback arc_write_callback_t;
933
934struct arc_write_callback {
935	void		*awcb_private;
936	arc_done_func_t	*awcb_ready;
937	arc_done_func_t	*awcb_children_ready;
938	arc_done_func_t	*awcb_physdone;
939	arc_done_func_t	*awcb_done;
940	arc_buf_t	*awcb_buf;
941};
942
943/*
944 * ARC buffers are separated into multiple structs as a memory saving measure:
945 *   - Common fields struct, always defined, and embedded within it:
946 *       - L2-only fields, always allocated but undefined when not in L2ARC
947 *       - L1-only fields, only allocated when in L1ARC
948 *
949 *           Buffer in L1                     Buffer only in L2
950 *    +------------------------+          +------------------------+
951 *    | arc_buf_hdr_t          |          | arc_buf_hdr_t          |
952 *    |                        |          |                        |
953 *    |                        |          |                        |
954 *    |                        |          |                        |
955 *    +------------------------+          +------------------------+
956 *    | l2arc_buf_hdr_t        |          | l2arc_buf_hdr_t        |
957 *    | (undefined if L1-only) |          |                        |
958 *    +------------------------+          +------------------------+
959 *    | l1arc_buf_hdr_t        |
960 *    |                        |
961 *    |                        |
962 *    |                        |
963 *    |                        |
964 *    +------------------------+
965 *
966 * Because it's possible for the L2ARC to become extremely large, we can wind
967 * up eating a lot of memory in L2ARC buffer headers, so the size of a header
968 * is minimized by only allocating the fields necessary for an L1-cached buffer
969 * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and
970 * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple
971 * words in pointers. arc_hdr_realloc() is used to switch a header between
972 * these two allocation states.
973 */
974typedef struct l1arc_buf_hdr {
975	kmutex_t		b_freeze_lock;
976	zio_cksum_t		*b_freeze_cksum;
977#ifdef ZFS_DEBUG
978	/*
979	 * Used for debugging with kmem_flags - by allocating and freeing
980	 * b_thawed when the buffer is thawed, we get a record of the stack
981	 * trace that thawed it.
982	 */
983	void			*b_thawed;
984#endif
985
986	arc_buf_t		*b_buf;
987	uint32_t		b_bufcnt;
988	/* for waiting on writes to complete */
989	kcondvar_t		b_cv;
990	uint8_t			b_byteswap;
991
992	/* protected by arc state mutex */
993	arc_state_t		*b_state;
994	multilist_node_t	b_arc_node;
995
996	/* updated atomically */
997	clock_t			b_arc_access;
998
999	/* self protecting */
1000	refcount_t		b_refcnt;
1001
1002	arc_callback_t		*b_acb;
1003	abd_t			*b_pabd;
1004} l1arc_buf_hdr_t;
1005
1006typedef struct l2arc_dev l2arc_dev_t;
1007
1008typedef struct l2arc_buf_hdr {
1009	/* protected by arc_buf_hdr mutex */
1010	l2arc_dev_t		*b_dev;		/* L2ARC device */
1011	uint64_t		b_daddr;	/* disk address, offset byte */
1012
1013	list_node_t		b_l2node;
1014} l2arc_buf_hdr_t;
1015
1016struct arc_buf_hdr {
1017	/* protected by hash lock */
1018	dva_t			b_dva;
1019	uint64_t		b_birth;
1020
1021	arc_buf_contents_t	b_type;
1022	arc_buf_hdr_t		*b_hash_next;
1023	arc_flags_t		b_flags;
1024
1025	/*
1026	 * This field stores the size of the data buffer after
1027	 * compression, and is set in the arc's zio completion handlers.
1028	 * It is in units of SPA_MINBLOCKSIZE (e.g. 1 == 512 bytes).
1029	 *
1030	 * While the block pointers can store up to 32MB in their psize
1031	 * field, we can only store up to 32MB minus 512B. This is due
1032	 * to the bp using a bias of 1, whereas we use a bias of 0 (i.e.
1033	 * a field of zeros represents 512B in the bp). We can't use a
1034	 * bias of 1 since we need to reserve a psize of zero, here, to
1035	 * represent holes and embedded blocks.
1036	 *
1037	 * This isn't a problem in practice, since the maximum size of a
1038	 * buffer is limited to 16MB, so we never need to store 32MB in
1039	 * this field. Even in the upstream illumos code base, the
1040	 * maximum size of a buffer is limited to 16MB.
1041	 */
1042	uint16_t		b_psize;
1043
1044	/*
1045	 * This field stores the size of the data buffer before
1046	 * compression, and cannot change once set. It is in units
1047	 * of SPA_MINBLOCKSIZE (e.g. 2 == 1024 bytes)
1048	 */
1049	uint16_t		b_lsize;	/* immutable */
1050	uint64_t		b_spa;		/* immutable */
1051
1052	/* L2ARC fields. Undefined when not in L2ARC. */
1053	l2arc_buf_hdr_t		b_l2hdr;
1054	/* L1ARC fields. Undefined when in l2arc_only state */
1055	l1arc_buf_hdr_t		b_l1hdr;
1056};
1057
1058#if defined(__FreeBSD__) && defined(_KERNEL)
1059static int
1060sysctl_vfs_zfs_arc_meta_limit(SYSCTL_HANDLER_ARGS)
1061{
1062	uint64_t val;
1063	int err;
1064
1065	val = arc_meta_limit;
1066	err = sysctl_handle_64(oidp, &val, 0, req);
1067	if (err != 0 || req->newptr == NULL)
1068		return (err);
1069
1070        if (val <= 0 || val > arc_c_max)
1071		return (EINVAL);
1072
1073	arc_meta_limit = val;
1074	return (0);
1075}
1076
1077static int
1078sysctl_vfs_zfs_arc_no_grow_shift(SYSCTL_HANDLER_ARGS)
1079{
1080	uint32_t val;
1081	int err;
1082
1083	val = arc_no_grow_shift;
1084	err = sysctl_handle_32(oidp, &val, 0, req);
1085	if (err != 0 || req->newptr == NULL)
1086		return (err);
1087
1088        if (val >= arc_shrink_shift)
1089		return (EINVAL);
1090
1091	arc_no_grow_shift = val;
1092	return (0);
1093}
1094
1095static int
1096sysctl_vfs_zfs_arc_max(SYSCTL_HANDLER_ARGS)
1097{
1098	uint64_t val;
1099	int err;
1100
1101	val = zfs_arc_max;
1102	err = sysctl_handle_64(oidp, &val, 0, req);
1103	if (err != 0 || req->newptr == NULL)
1104		return (err);
1105
1106	if (zfs_arc_max == 0) {
1107		/* Loader tunable so blindly set */
1108		zfs_arc_max = val;
1109		return (0);
1110	}
1111
1112	if (val < arc_abs_min || val > kmem_size())
1113		return (EINVAL);
1114	if (val < arc_c_min)
1115		return (EINVAL);
1116	if (zfs_arc_meta_limit > 0 && val < zfs_arc_meta_limit)
1117		return (EINVAL);
1118
1119	arc_c_max = val;
1120
1121	arc_c = arc_c_max;
1122        arc_p = (arc_c >> 1);
1123
1124	if (zfs_arc_meta_limit == 0) {
1125		/* limit meta-data to 1/4 of the arc capacity */
1126		arc_meta_limit = arc_c_max / 4;
1127	}
1128
1129	/* if kmem_flags are set, lets try to use less memory */
1130	if (kmem_debugging())
1131		arc_c = arc_c / 2;
1132
1133	zfs_arc_max = arc_c;
1134
1135	return (0);
1136}
1137
1138static int
1139sysctl_vfs_zfs_arc_min(SYSCTL_HANDLER_ARGS)
1140{
1141	uint64_t val;
1142	int err;
1143
1144	val = zfs_arc_min;
1145	err = sysctl_handle_64(oidp, &val, 0, req);
1146	if (err != 0 || req->newptr == NULL)
1147		return (err);
1148
1149	if (zfs_arc_min == 0) {
1150		/* Loader tunable so blindly set */
1151		zfs_arc_min = val;
1152		return (0);
1153	}
1154
1155	if (val < arc_abs_min || val > arc_c_max)
1156		return (EINVAL);
1157
1158	arc_c_min = val;
1159
1160	if (zfs_arc_meta_min == 0)
1161                arc_meta_min = arc_c_min / 2;
1162
1163	if (arc_c < arc_c_min)
1164                arc_c = arc_c_min;
1165
1166	zfs_arc_min = arc_c_min;
1167
1168	return (0);
1169}
1170#endif
1171
1172#define	GHOST_STATE(state)	\
1173	((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||	\
1174	(state) == arc_l2c_only)
1175
1176#define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
1177#define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
1178#define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_FLAG_IO_ERROR)
1179#define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_FLAG_PREFETCH)
1180#define	HDR_COMPRESSION_ENABLED(hdr)	\
1181	((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
1182
1183#define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_FLAG_L2CACHE)
1184#define	HDR_L2_READING(hdr)	\
1185	(((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) &&	\
1186	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
1187#define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITING)
1188#define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
1189#define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
1190#define	HDR_SHARED_DATA(hdr)	((hdr)->b_flags & ARC_FLAG_SHARED_DATA)
1191
1192#define	HDR_ISTYPE_METADATA(hdr)	\
1193	((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
1194#define	HDR_ISTYPE_DATA(hdr)	(!HDR_ISTYPE_METADATA(hdr))
1195
1196#define	HDR_HAS_L1HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
1197#define	HDR_HAS_L2HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
1198
1199/* For storing compression mode in b_flags */
1200#define	HDR_COMPRESS_OFFSET	(highbit64(ARC_FLAG_COMPRESS_0) - 1)
1201
1202#define	HDR_GET_COMPRESS(hdr)	((enum zio_compress)BF32_GET((hdr)->b_flags, \
1203	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS))
1204#define	HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \
1205	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS, (cmp));
1206
1207#define	ARC_BUF_LAST(buf)	((buf)->b_next == NULL)
1208#define	ARC_BUF_SHARED(buf)	((buf)->b_flags & ARC_BUF_FLAG_SHARED)
1209#define	ARC_BUF_COMPRESSED(buf)	((buf)->b_flags & ARC_BUF_FLAG_COMPRESSED)
1210
1211/*
1212 * Other sizes
1213 */
1214
1215#define	HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
1216#define	HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
1217
1218/*
1219 * Hash table routines
1220 */
1221
1222#define	HT_LOCK_PAD	CACHE_LINE_SIZE
1223
1224struct ht_lock {
1225	kmutex_t	ht_lock;
1226#ifdef _KERNEL
1227	unsigned char	pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
1228#endif
1229};
1230
1231#define	BUF_LOCKS 256
1232typedef struct buf_hash_table {
1233	uint64_t ht_mask;
1234	arc_buf_hdr_t **ht_table;
1235	struct ht_lock ht_locks[BUF_LOCKS] __aligned(CACHE_LINE_SIZE);
1236} buf_hash_table_t;
1237
1238static buf_hash_table_t buf_hash_table;
1239
1240#define	BUF_HASH_INDEX(spa, dva, birth) \
1241	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
1242#define	BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
1243#define	BUF_HASH_LOCK(idx)	(&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
1244#define	HDR_LOCK(hdr) \
1245	(BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
1246
1247uint64_t zfs_crc64_table[256];
1248
1249/*
1250 * Level 2 ARC
1251 */
1252
1253#define	L2ARC_WRITE_SIZE	(8 * 1024 * 1024)	/* initial write max */
1254#define	L2ARC_HEADROOM		2			/* num of writes */
1255/*
1256 * If we discover during ARC scan any buffers to be compressed, we boost
1257 * our headroom for the next scanning cycle by this percentage multiple.
1258 */
1259#define	L2ARC_HEADROOM_BOOST	200
1260#define	L2ARC_FEED_SECS		1		/* caching interval secs */
1261#define	L2ARC_FEED_MIN_MS	200		/* min caching interval ms */
1262
1263#define	l2arc_writes_sent	ARCSTAT(arcstat_l2_writes_sent)
1264#define	l2arc_writes_done	ARCSTAT(arcstat_l2_writes_done)
1265
1266/* L2ARC Performance Tunables */
1267uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;	/* default max write size */
1268uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;	/* extra write during warmup */
1269uint64_t l2arc_headroom = L2ARC_HEADROOM;	/* number of dev writes */
1270uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
1271uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;	/* interval seconds */
1272uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS;	/* min interval milliseconds */
1273boolean_t l2arc_noprefetch = B_TRUE;		/* don't cache prefetch bufs */
1274boolean_t l2arc_feed_again = B_TRUE;		/* turbo warmup */
1275boolean_t l2arc_norw = B_TRUE;			/* no reads during writes */
1276
1277SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_write_max, CTLFLAG_RW,
1278    &l2arc_write_max, 0, "max write size");
1279SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_write_boost, CTLFLAG_RW,
1280    &l2arc_write_boost, 0, "extra write during warmup");
1281SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_headroom, CTLFLAG_RW,
1282    &l2arc_headroom, 0, "number of dev writes");
1283SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_feed_secs, CTLFLAG_RW,
1284    &l2arc_feed_secs, 0, "interval seconds");
1285SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_feed_min_ms, CTLFLAG_RW,
1286    &l2arc_feed_min_ms, 0, "min interval milliseconds");
1287
1288SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_noprefetch, CTLFLAG_RW,
1289    &l2arc_noprefetch, 0, "don't cache prefetch bufs");
1290SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_feed_again, CTLFLAG_RW,
1291    &l2arc_feed_again, 0, "turbo warmup");
1292SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_norw, CTLFLAG_RW,
1293    &l2arc_norw, 0, "no reads during writes");
1294
1295SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_size, CTLFLAG_RD,
1296    &ARC_anon.arcs_size.rc_count, 0, "size of anonymous state");
1297SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_metadata_esize, CTLFLAG_RD,
1298    &ARC_anon.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1299    "size of anonymous state");
1300SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_data_esize, CTLFLAG_RD,
1301    &ARC_anon.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1302    "size of anonymous state");
1303
1304SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_size, CTLFLAG_RD,
1305    &ARC_mru.arcs_size.rc_count, 0, "size of mru state");
1306SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_metadata_esize, CTLFLAG_RD,
1307    &ARC_mru.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1308    "size of metadata in mru state");
1309SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_data_esize, CTLFLAG_RD,
1310    &ARC_mru.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1311    "size of data in mru state");
1312
1313SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_size, CTLFLAG_RD,
1314    &ARC_mru_ghost.arcs_size.rc_count, 0, "size of mru ghost state");
1315SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_metadata_esize, CTLFLAG_RD,
1316    &ARC_mru_ghost.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1317    "size of metadata in mru ghost state");
1318SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_data_esize, CTLFLAG_RD,
1319    &ARC_mru_ghost.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1320    "size of data in mru ghost state");
1321
1322SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_size, CTLFLAG_RD,
1323    &ARC_mfu.arcs_size.rc_count, 0, "size of mfu state");
1324SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_metadata_esize, CTLFLAG_RD,
1325    &ARC_mfu.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1326    "size of metadata in mfu state");
1327SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_data_esize, CTLFLAG_RD,
1328    &ARC_mfu.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1329    "size of data in mfu state");
1330
1331SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_size, CTLFLAG_RD,
1332    &ARC_mfu_ghost.arcs_size.rc_count, 0, "size of mfu ghost state");
1333SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_metadata_esize, CTLFLAG_RD,
1334    &ARC_mfu_ghost.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1335    "size of metadata in mfu ghost state");
1336SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_data_esize, CTLFLAG_RD,
1337    &ARC_mfu_ghost.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1338    "size of data in mfu ghost state");
1339
1340SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2c_only_size, CTLFLAG_RD,
1341    &ARC_l2c_only.arcs_size.rc_count, 0, "size of mru state");
1342
1343/*
1344 * L2ARC Internals
1345 */
1346struct l2arc_dev {
1347	vdev_t			*l2ad_vdev;	/* vdev */
1348	spa_t			*l2ad_spa;	/* spa */
1349	uint64_t		l2ad_hand;	/* next write location */
1350	uint64_t		l2ad_start;	/* first addr on device */
1351	uint64_t		l2ad_end;	/* last addr on device */
1352	boolean_t		l2ad_first;	/* first sweep through */
1353	boolean_t		l2ad_writing;	/* currently writing */
1354	kmutex_t		l2ad_mtx;	/* lock for buffer list */
1355	list_t			l2ad_buflist;	/* buffer list */
1356	list_node_t		l2ad_node;	/* device list node */
1357	refcount_t		l2ad_alloc;	/* allocated bytes */
1358};
1359
1360static list_t L2ARC_dev_list;			/* device list */
1361static list_t *l2arc_dev_list;			/* device list pointer */
1362static kmutex_t l2arc_dev_mtx;			/* device list mutex */
1363static l2arc_dev_t *l2arc_dev_last;		/* last device used */
1364static list_t L2ARC_free_on_write;		/* free after write buf list */
1365static list_t *l2arc_free_on_write;		/* free after write list ptr */
1366static kmutex_t l2arc_free_on_write_mtx;	/* mutex for list */
1367static uint64_t l2arc_ndev;			/* number of devices */
1368
1369typedef struct l2arc_read_callback {
1370	arc_buf_hdr_t		*l2rcb_hdr;		/* read header */
1371	blkptr_t		l2rcb_bp;		/* original blkptr */
1372	zbookmark_phys_t	l2rcb_zb;		/* original bookmark */
1373	int			l2rcb_flags;		/* original flags */
1374	abd_t			*l2rcb_abd;		/* temporary buffer */
1375} l2arc_read_callback_t;
1376
1377typedef struct l2arc_write_callback {
1378	l2arc_dev_t	*l2wcb_dev;		/* device info */
1379	arc_buf_hdr_t	*l2wcb_head;		/* head of write buflist */
1380} l2arc_write_callback_t;
1381
1382typedef struct l2arc_data_free {
1383	/* protected by l2arc_free_on_write_mtx */
1384	abd_t		*l2df_abd;
1385	size_t		l2df_size;
1386	arc_buf_contents_t l2df_type;
1387	list_node_t	l2df_list_node;
1388} l2arc_data_free_t;
1389
1390static kmutex_t l2arc_feed_thr_lock;
1391static kcondvar_t l2arc_feed_thr_cv;
1392static uint8_t l2arc_thread_exit;
1393
1394static abd_t *arc_get_data_abd(arc_buf_hdr_t *, uint64_t, void *);
1395static void *arc_get_data_buf(arc_buf_hdr_t *, uint64_t, void *);
1396static void arc_get_data_impl(arc_buf_hdr_t *, uint64_t, void *);
1397static void arc_free_data_abd(arc_buf_hdr_t *, abd_t *, uint64_t, void *);
1398static void arc_free_data_buf(arc_buf_hdr_t *, void *, uint64_t, void *);
1399static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag);
1400static void arc_hdr_free_pabd(arc_buf_hdr_t *);
1401static void arc_hdr_alloc_pabd(arc_buf_hdr_t *);
1402static void arc_access(arc_buf_hdr_t *, kmutex_t *);
1403static boolean_t arc_is_overflowing();
1404static void arc_buf_watch(arc_buf_t *);
1405
1406static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
1407static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
1408static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
1409static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
1410
1411static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
1412static void l2arc_read_done(zio_t *);
1413
1414static void
1415l2arc_trim(const arc_buf_hdr_t *hdr)
1416{
1417	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
1418
1419	ASSERT(HDR_HAS_L2HDR(hdr));
1420	ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
1421
1422	if (HDR_GET_PSIZE(hdr) != 0) {
1423		trim_map_free(dev->l2ad_vdev, hdr->b_l2hdr.b_daddr,
1424		    HDR_GET_PSIZE(hdr), 0);
1425	}
1426}
1427
1428static uint64_t
1429buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
1430{
1431	uint8_t *vdva = (uint8_t *)dva;
1432	uint64_t crc = -1ULL;
1433	int i;
1434
1435	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
1436
1437	for (i = 0; i < sizeof (dva_t); i++)
1438		crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
1439
1440	crc ^= (spa>>8) ^ birth;
1441
1442	return (crc);
1443}
1444
1445#define	HDR_EMPTY(hdr)						\
1446	((hdr)->b_dva.dva_word[0] == 0 &&			\
1447	(hdr)->b_dva.dva_word[1] == 0)
1448
1449#define	HDR_EQUAL(spa, dva, birth, hdr)				\
1450	((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
1451	((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
1452	((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
1453
1454static void
1455buf_discard_identity(arc_buf_hdr_t *hdr)
1456{
1457	hdr->b_dva.dva_word[0] = 0;
1458	hdr->b_dva.dva_word[1] = 0;
1459	hdr->b_birth = 0;
1460}
1461
1462static arc_buf_hdr_t *
1463buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
1464{
1465	const dva_t *dva = BP_IDENTITY(bp);
1466	uint64_t birth = BP_PHYSICAL_BIRTH(bp);
1467	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
1468	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1469	arc_buf_hdr_t *hdr;
1470
1471	mutex_enter(hash_lock);
1472	for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
1473	    hdr = hdr->b_hash_next) {
1474		if (HDR_EQUAL(spa, dva, birth, hdr)) {
1475			*lockp = hash_lock;
1476			return (hdr);
1477		}
1478	}
1479	mutex_exit(hash_lock);
1480	*lockp = NULL;
1481	return (NULL);
1482}
1483
1484/*
1485 * Insert an entry into the hash table.  If there is already an element
1486 * equal to elem in the hash table, then the already existing element
1487 * will be returned and the new element will not be inserted.
1488 * Otherwise returns NULL.
1489 * If lockp == NULL, the caller is assumed to already hold the hash lock.
1490 */
1491static arc_buf_hdr_t *
1492buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
1493{
1494	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1495	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1496	arc_buf_hdr_t *fhdr;
1497	uint32_t i;
1498
1499	ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1500	ASSERT(hdr->b_birth != 0);
1501	ASSERT(!HDR_IN_HASH_TABLE(hdr));
1502
1503	if (lockp != NULL) {
1504		*lockp = hash_lock;
1505		mutex_enter(hash_lock);
1506	} else {
1507		ASSERT(MUTEX_HELD(hash_lock));
1508	}
1509
1510	for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
1511	    fhdr = fhdr->b_hash_next, i++) {
1512		if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1513			return (fhdr);
1514	}
1515
1516	hdr->b_hash_next = buf_hash_table.ht_table[idx];
1517	buf_hash_table.ht_table[idx] = hdr;
1518	arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1519
1520	/* collect some hash table performance data */
1521	if (i > 0) {
1522		ARCSTAT_BUMP(arcstat_hash_collisions);
1523		if (i == 1)
1524			ARCSTAT_BUMP(arcstat_hash_chains);
1525
1526		ARCSTAT_MAX(arcstat_hash_chain_max, i);
1527	}
1528
1529	ARCSTAT_BUMP(arcstat_hash_elements);
1530	ARCSTAT_MAXSTAT(arcstat_hash_elements);
1531
1532	return (NULL);
1533}
1534
1535static void
1536buf_hash_remove(arc_buf_hdr_t *hdr)
1537{
1538	arc_buf_hdr_t *fhdr, **hdrp;
1539	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1540
1541	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
1542	ASSERT(HDR_IN_HASH_TABLE(hdr));
1543
1544	hdrp = &buf_hash_table.ht_table[idx];
1545	while ((fhdr = *hdrp) != hdr) {
1546		ASSERT3P(fhdr, !=, NULL);
1547		hdrp = &fhdr->b_hash_next;
1548	}
1549	*hdrp = hdr->b_hash_next;
1550	hdr->b_hash_next = NULL;
1551	arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1552
1553	/* collect some hash table performance data */
1554	ARCSTAT_BUMPDOWN(arcstat_hash_elements);
1555
1556	if (buf_hash_table.ht_table[idx] &&
1557	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
1558		ARCSTAT_BUMPDOWN(arcstat_hash_chains);
1559}
1560
1561/*
1562 * Global data structures and functions for the buf kmem cache.
1563 */
1564static kmem_cache_t *hdr_full_cache;
1565static kmem_cache_t *hdr_l2only_cache;
1566static kmem_cache_t *buf_cache;
1567
1568static void
1569buf_fini(void)
1570{
1571	int i;
1572
1573	kmem_free(buf_hash_table.ht_table,
1574	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
1575	for (i = 0; i < BUF_LOCKS; i++)
1576		mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
1577	kmem_cache_destroy(hdr_full_cache);
1578	kmem_cache_destroy(hdr_l2only_cache);
1579	kmem_cache_destroy(buf_cache);
1580}
1581
1582/*
1583 * Constructor callback - called when the cache is empty
1584 * and a new buf is requested.
1585 */
1586/* ARGSUSED */
1587static int
1588hdr_full_cons(void *vbuf, void *unused, int kmflag)
1589{
1590	arc_buf_hdr_t *hdr = vbuf;
1591
1592	bzero(hdr, HDR_FULL_SIZE);
1593	cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
1594	refcount_create(&hdr->b_l1hdr.b_refcnt);
1595	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1596	multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1597	arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1598
1599	return (0);
1600}
1601
1602/* ARGSUSED */
1603static int
1604hdr_l2only_cons(void *vbuf, void *unused, int kmflag)
1605{
1606	arc_buf_hdr_t *hdr = vbuf;
1607
1608	bzero(hdr, HDR_L2ONLY_SIZE);
1609	arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1610
1611	return (0);
1612}
1613
1614/* ARGSUSED */
1615static int
1616buf_cons(void *vbuf, void *unused, int kmflag)
1617{
1618	arc_buf_t *buf = vbuf;
1619
1620	bzero(buf, sizeof (arc_buf_t));
1621	mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL);
1622	arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1623
1624	return (0);
1625}
1626
1627/*
1628 * Destructor callback - called when a cached buf is
1629 * no longer required.
1630 */
1631/* ARGSUSED */
1632static void
1633hdr_full_dest(void *vbuf, void *unused)
1634{
1635	arc_buf_hdr_t *hdr = vbuf;
1636
1637	ASSERT(HDR_EMPTY(hdr));
1638	cv_destroy(&hdr->b_l1hdr.b_cv);
1639	refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1640	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1641	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1642	arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1643}
1644
1645/* ARGSUSED */
1646static void
1647hdr_l2only_dest(void *vbuf, void *unused)
1648{
1649	arc_buf_hdr_t *hdr = vbuf;
1650
1651	ASSERT(HDR_EMPTY(hdr));
1652	arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1653}
1654
1655/* ARGSUSED */
1656static void
1657buf_dest(void *vbuf, void *unused)
1658{
1659	arc_buf_t *buf = vbuf;
1660
1661	mutex_destroy(&buf->b_evict_lock);
1662	arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1663}
1664
1665/*
1666 * Reclaim callback -- invoked when memory is low.
1667 */
1668/* ARGSUSED */
1669static void
1670hdr_recl(void *unused)
1671{
1672	dprintf("hdr_recl called\n");
1673	/*
1674	 * umem calls the reclaim func when we destroy the buf cache,
1675	 * which is after we do arc_fini().
1676	 */
1677	if (!arc_dead)
1678		cv_signal(&arc_reclaim_thread_cv);
1679}
1680
1681static void
1682buf_init(void)
1683{
1684	uint64_t *ct;
1685	uint64_t hsize = 1ULL << 12;
1686	int i, j;
1687
1688	/*
1689	 * The hash table is big enough to fill all of physical memory
1690	 * with an average block size of zfs_arc_average_blocksize (default 8K).
1691	 * By default, the table will take up
1692	 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
1693	 */
1694	while (hsize * zfs_arc_average_blocksize < (uint64_t)physmem * PAGESIZE)
1695		hsize <<= 1;
1696retry:
1697	buf_hash_table.ht_mask = hsize - 1;
1698	buf_hash_table.ht_table =
1699	    kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
1700	if (buf_hash_table.ht_table == NULL) {
1701		ASSERT(hsize > (1ULL << 8));
1702		hsize >>= 1;
1703		goto retry;
1704	}
1705
1706	hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
1707	    0, hdr_full_cons, hdr_full_dest, hdr_recl, NULL, NULL, 0);
1708	hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
1709	    HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, hdr_recl,
1710	    NULL, NULL, 0);
1711	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
1712	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
1713
1714	for (i = 0; i < 256; i++)
1715		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
1716			*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
1717
1718	for (i = 0; i < BUF_LOCKS; i++) {
1719		mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
1720		    NULL, MUTEX_DEFAULT, NULL);
1721	}
1722}
1723
1724/*
1725 * This is the size that the buf occupies in memory. If the buf is compressed,
1726 * it will correspond to the compressed size. You should use this method of
1727 * getting the buf size unless you explicitly need the logical size.
1728 */
1729int32_t
1730arc_buf_size(arc_buf_t *buf)
1731{
1732	return (ARC_BUF_COMPRESSED(buf) ?
1733	    HDR_GET_PSIZE(buf->b_hdr) : HDR_GET_LSIZE(buf->b_hdr));
1734}
1735
1736int32_t
1737arc_buf_lsize(arc_buf_t *buf)
1738{
1739	return (HDR_GET_LSIZE(buf->b_hdr));
1740}
1741
1742enum zio_compress
1743arc_get_compression(arc_buf_t *buf)
1744{
1745	return (ARC_BUF_COMPRESSED(buf) ?
1746	    HDR_GET_COMPRESS(buf->b_hdr) : ZIO_COMPRESS_OFF);
1747}
1748
1749#define	ARC_MINTIME	(hz>>4) /* 62 ms */
1750
1751static inline boolean_t
1752arc_buf_is_shared(arc_buf_t *buf)
1753{
1754	boolean_t shared = (buf->b_data != NULL &&
1755	    buf->b_hdr->b_l1hdr.b_pabd != NULL &&
1756	    abd_is_linear(buf->b_hdr->b_l1hdr.b_pabd) &&
1757	    buf->b_data == abd_to_buf(buf->b_hdr->b_l1hdr.b_pabd));
1758	IMPLY(shared, HDR_SHARED_DATA(buf->b_hdr));
1759	IMPLY(shared, ARC_BUF_SHARED(buf));
1760	IMPLY(shared, ARC_BUF_COMPRESSED(buf) || ARC_BUF_LAST(buf));
1761
1762	/*
1763	 * It would be nice to assert arc_can_share() too, but the "hdr isn't
1764	 * already being shared" requirement prevents us from doing that.
1765	 */
1766
1767	return (shared);
1768}
1769
1770/*
1771 * Free the checksum associated with this header. If there is no checksum, this
1772 * is a no-op.
1773 */
1774static inline void
1775arc_cksum_free(arc_buf_hdr_t *hdr)
1776{
1777	ASSERT(HDR_HAS_L1HDR(hdr));
1778	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1779	if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1780		kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));
1781		hdr->b_l1hdr.b_freeze_cksum = NULL;
1782	}
1783	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1784}
1785
1786/*
1787 * Return true iff at least one of the bufs on hdr is not compressed.
1788 */
1789static boolean_t
1790arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr)
1791{
1792	for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) {
1793		if (!ARC_BUF_COMPRESSED(b)) {
1794			return (B_TRUE);
1795		}
1796	}
1797	return (B_FALSE);
1798}
1799
1800/*
1801 * If we've turned on the ZFS_DEBUG_MODIFY flag, verify that the buf's data
1802 * matches the checksum that is stored in the hdr. If there is no checksum,
1803 * or if the buf is compressed, this is a no-op.
1804 */
1805static void
1806arc_cksum_verify(arc_buf_t *buf)
1807{
1808	arc_buf_hdr_t *hdr = buf->b_hdr;
1809	zio_cksum_t zc;
1810
1811	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1812		return;
1813
1814	if (ARC_BUF_COMPRESSED(buf)) {
1815		ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
1816		    arc_hdr_has_uncompressed_buf(hdr));
1817		return;
1818	}
1819
1820	ASSERT(HDR_HAS_L1HDR(hdr));
1821
1822	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1823	if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) {
1824		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1825		return;
1826	}
1827
1828	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, &zc);
1829	if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))
1830		panic("buffer modified while frozen!");
1831	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1832}
1833
1834static boolean_t
1835arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio)
1836{
1837	enum zio_compress compress = BP_GET_COMPRESS(zio->io_bp);
1838	boolean_t valid_cksum;
1839
1840	ASSERT(!BP_IS_EMBEDDED(zio->io_bp));
1841	VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));
1842
1843	/*
1844	 * We rely on the blkptr's checksum to determine if the block
1845	 * is valid or not. When compressed arc is enabled, the l2arc
1846	 * writes the block to the l2arc just as it appears in the pool.
1847	 * This allows us to use the blkptr's checksum to validate the
1848	 * data that we just read off of the l2arc without having to store
1849	 * a separate checksum in the arc_buf_hdr_t. However, if compressed
1850	 * arc is disabled, then the data written to the l2arc is always
1851	 * uncompressed and won't match the block as it exists in the main
1852	 * pool. When this is the case, we must first compress it if it is
1853	 * compressed on the main pool before we can validate the checksum.
1854	 */
1855	if (!HDR_COMPRESSION_ENABLED(hdr) && compress != ZIO_COMPRESS_OFF) {
1856		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
1857		uint64_t lsize = HDR_GET_LSIZE(hdr);
1858		uint64_t csize;
1859
1860		abd_t *cdata = abd_alloc_linear(HDR_GET_PSIZE(hdr), B_TRUE);
1861		csize = zio_compress_data(compress, zio->io_abd,
1862		    abd_to_buf(cdata), lsize);
1863
1864		ASSERT3U(csize, <=, HDR_GET_PSIZE(hdr));
1865		if (csize < HDR_GET_PSIZE(hdr)) {
1866			/*
1867			 * Compressed blocks are always a multiple of the
1868			 * smallest ashift in the pool. Ideally, we would
1869			 * like to round up the csize to the next
1870			 * spa_min_ashift but that value may have changed
1871			 * since the block was last written. Instead,
1872			 * we rely on the fact that the hdr's psize
1873			 * was set to the psize of the block when it was
1874			 * last written. We set the csize to that value
1875			 * and zero out any part that should not contain
1876			 * data.
1877			 */
1878			abd_zero_off(cdata, csize, HDR_GET_PSIZE(hdr) - csize);
1879			csize = HDR_GET_PSIZE(hdr);
1880		}
1881		zio_push_transform(zio, cdata, csize, HDR_GET_PSIZE(hdr), NULL);
1882	}
1883
1884	/*
1885	 * Block pointers always store the checksum for the logical data.
1886	 * If the block pointer has the gang bit set, then the checksum
1887	 * it represents is for the reconstituted data and not for an
1888	 * individual gang member. The zio pipeline, however, must be able to
1889	 * determine the checksum of each of the gang constituents so it
1890	 * treats the checksum comparison differently than what we need
1891	 * for l2arc blocks. This prevents us from using the
1892	 * zio_checksum_error() interface directly. Instead we must call the
1893	 * zio_checksum_error_impl() so that we can ensure the checksum is
1894	 * generated using the correct checksum algorithm and accounts for the
1895	 * logical I/O size and not just a gang fragment.
1896	 */
1897	valid_cksum = (zio_checksum_error_impl(zio->io_spa, zio->io_bp,
1898	    BP_GET_CHECKSUM(zio->io_bp), zio->io_abd, zio->io_size,
1899	    zio->io_offset, NULL) == 0);
1900	zio_pop_transforms(zio);
1901	return (valid_cksum);
1902}
1903
1904/*
1905 * Given a buf full of data, if ZFS_DEBUG_MODIFY is enabled this computes a
1906 * checksum and attaches it to the buf's hdr so that we can ensure that the buf
1907 * isn't modified later on. If buf is compressed or there is already a checksum
1908 * on the hdr, this is a no-op (we only checksum uncompressed bufs).
1909 */
1910static void
1911arc_cksum_compute(arc_buf_t *buf)
1912{
1913	arc_buf_hdr_t *hdr = buf->b_hdr;
1914
1915	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1916		return;
1917
1918	ASSERT(HDR_HAS_L1HDR(hdr));
1919
1920	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1921	if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1922		ASSERT(arc_hdr_has_uncompressed_buf(hdr));
1923		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1924		return;
1925	} else if (ARC_BUF_COMPRESSED(buf)) {
1926		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1927		return;
1928	}
1929
1930	ASSERT(!ARC_BUF_COMPRESSED(buf));
1931	hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),
1932	    KM_SLEEP);
1933	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL,
1934	    hdr->b_l1hdr.b_freeze_cksum);
1935	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1936#ifdef illumos
1937	arc_buf_watch(buf);
1938#endif
1939}
1940
1941#ifdef illumos
1942#ifndef _KERNEL
1943typedef struct procctl {
1944	long cmd;
1945	prwatch_t prwatch;
1946} procctl_t;
1947#endif
1948
1949/* ARGSUSED */
1950static void
1951arc_buf_unwatch(arc_buf_t *buf)
1952{
1953#ifndef _KERNEL
1954	if (arc_watch) {
1955		int result;
1956		procctl_t ctl;
1957		ctl.cmd = PCWATCH;
1958		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1959		ctl.prwatch.pr_size = 0;
1960		ctl.prwatch.pr_wflags = 0;
1961		result = write(arc_procfd, &ctl, sizeof (ctl));
1962		ASSERT3U(result, ==, sizeof (ctl));
1963	}
1964#endif
1965}
1966
1967/* ARGSUSED */
1968static void
1969arc_buf_watch(arc_buf_t *buf)
1970{
1971#ifndef _KERNEL
1972	if (arc_watch) {
1973		int result;
1974		procctl_t ctl;
1975		ctl.cmd = PCWATCH;
1976		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1977		ctl.prwatch.pr_size = arc_buf_size(buf);
1978		ctl.prwatch.pr_wflags = WA_WRITE;
1979		result = write(arc_procfd, &ctl, sizeof (ctl));
1980		ASSERT3U(result, ==, sizeof (ctl));
1981	}
1982#endif
1983}
1984#endif /* illumos */
1985
1986static arc_buf_contents_t
1987arc_buf_type(arc_buf_hdr_t *hdr)
1988{
1989	arc_buf_contents_t type;
1990	if (HDR_ISTYPE_METADATA(hdr)) {
1991		type = ARC_BUFC_METADATA;
1992	} else {
1993		type = ARC_BUFC_DATA;
1994	}
1995	VERIFY3U(hdr->b_type, ==, type);
1996	return (type);
1997}
1998
1999boolean_t
2000arc_is_metadata(arc_buf_t *buf)
2001{
2002	return (HDR_ISTYPE_METADATA(buf->b_hdr) != 0);
2003}
2004
2005static uint32_t
2006arc_bufc_to_flags(arc_buf_contents_t type)
2007{
2008	switch (type) {
2009	case ARC_BUFC_DATA:
2010		/* metadata field is 0 if buffer contains normal data */
2011		return (0);
2012	case ARC_BUFC_METADATA:
2013		return (ARC_FLAG_BUFC_METADATA);
2014	default:
2015		break;
2016	}
2017	panic("undefined ARC buffer type!");
2018	return ((uint32_t)-1);
2019}
2020
2021void
2022arc_buf_thaw(arc_buf_t *buf)
2023{
2024	arc_buf_hdr_t *hdr = buf->b_hdr;
2025
2026	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
2027	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2028
2029	arc_cksum_verify(buf);
2030
2031	/*
2032	 * Compressed buffers do not manipulate the b_freeze_cksum or
2033	 * allocate b_thawed.
2034	 */
2035	if (ARC_BUF_COMPRESSED(buf)) {
2036		ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
2037		    arc_hdr_has_uncompressed_buf(hdr));
2038		return;
2039	}
2040
2041	ASSERT(HDR_HAS_L1HDR(hdr));
2042	arc_cksum_free(hdr);
2043
2044	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
2045#ifdef ZFS_DEBUG
2046	if (zfs_flags & ZFS_DEBUG_MODIFY) {
2047		if (hdr->b_l1hdr.b_thawed != NULL)
2048			kmem_free(hdr->b_l1hdr.b_thawed, 1);
2049		hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP);
2050	}
2051#endif
2052
2053	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2054
2055#ifdef illumos
2056	arc_buf_unwatch(buf);
2057#endif
2058}
2059
2060void
2061arc_buf_freeze(arc_buf_t *buf)
2062{
2063	arc_buf_hdr_t *hdr = buf->b_hdr;
2064	kmutex_t *hash_lock;
2065
2066	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
2067		return;
2068
2069	if (ARC_BUF_COMPRESSED(buf)) {
2070		ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
2071		    arc_hdr_has_uncompressed_buf(hdr));
2072		return;
2073	}
2074
2075	hash_lock = HDR_LOCK(hdr);
2076	mutex_enter(hash_lock);
2077
2078	ASSERT(HDR_HAS_L1HDR(hdr));
2079	ASSERT(hdr->b_l1hdr.b_freeze_cksum != NULL ||
2080	    hdr->b_l1hdr.b_state == arc_anon);
2081	arc_cksum_compute(buf);
2082	mutex_exit(hash_lock);
2083}
2084
2085/*
2086 * The arc_buf_hdr_t's b_flags should never be modified directly. Instead,
2087 * the following functions should be used to ensure that the flags are
2088 * updated in a thread-safe way. When manipulating the flags either
2089 * the hash_lock must be held or the hdr must be undiscoverable. This
2090 * ensures that we're not racing with any other threads when updating
2091 * the flags.
2092 */
2093static inline void
2094arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
2095{
2096	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2097	hdr->b_flags |= flags;
2098}
2099
2100static inline void
2101arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
2102{
2103	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2104	hdr->b_flags &= ~flags;
2105}
2106
2107/*
2108 * Setting the compression bits in the arc_buf_hdr_t's b_flags is
2109 * done in a special way since we have to clear and set bits
2110 * at the same time. Consumers that wish to set the compression bits
2111 * must use this function to ensure that the flags are updated in
2112 * thread-safe manner.
2113 */
2114static void
2115arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp)
2116{
2117	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2118
2119	/*
2120	 * Holes and embedded blocks will always have a psize = 0 so
2121	 * we ignore the compression of the blkptr and set the
2122	 * arc_buf_hdr_t's compression to ZIO_COMPRESS_OFF.
2123	 * Holes and embedded blocks remain anonymous so we don't
2124	 * want to uncompress them. Mark them as uncompressed.
2125	 */
2126	if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) {
2127		arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
2128		HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF);
2129		ASSERT(!HDR_COMPRESSION_ENABLED(hdr));
2130		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
2131	} else {
2132		arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
2133		HDR_SET_COMPRESS(hdr, cmp);
2134		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp);
2135		ASSERT(HDR_COMPRESSION_ENABLED(hdr));
2136	}
2137}
2138
2139/*
2140 * Looks for another buf on the same hdr which has the data decompressed, copies
2141 * from it, and returns true. If no such buf exists, returns false.
2142 */
2143static boolean_t
2144arc_buf_try_copy_decompressed_data(arc_buf_t *buf)
2145{
2146	arc_buf_hdr_t *hdr = buf->b_hdr;
2147	boolean_t copied = B_FALSE;
2148
2149	ASSERT(HDR_HAS_L1HDR(hdr));
2150	ASSERT3P(buf->b_data, !=, NULL);
2151	ASSERT(!ARC_BUF_COMPRESSED(buf));
2152
2153	for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL;
2154	    from = from->b_next) {
2155		/* can't use our own data buffer */
2156		if (from == buf) {
2157			continue;
2158		}
2159
2160		if (!ARC_BUF_COMPRESSED(from)) {
2161			bcopy(from->b_data, buf->b_data, arc_buf_size(buf));
2162			copied = B_TRUE;
2163			break;
2164		}
2165	}
2166
2167	/*
2168	 * There were no decompressed bufs, so there should not be a
2169	 * checksum on the hdr either.
2170	 */
2171	EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL);
2172
2173	return (copied);
2174}
2175
2176/*
2177 * Given a buf that has a data buffer attached to it, this function will
2178 * efficiently fill the buf with data of the specified compression setting from
2179 * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr
2180 * are already sharing a data buf, no copy is performed.
2181 *
2182 * If the buf is marked as compressed but uncompressed data was requested, this
2183 * will allocate a new data buffer for the buf, remove that flag, and fill the
2184 * buf with uncompressed data. You can't request a compressed buf on a hdr with
2185 * uncompressed data, and (since we haven't added support for it yet) if you
2186 * want compressed data your buf must already be marked as compressed and have
2187 * the correct-sized data buffer.
2188 */
2189static int
2190arc_buf_fill(arc_buf_t *buf, boolean_t compressed)
2191{
2192	arc_buf_hdr_t *hdr = buf->b_hdr;
2193	boolean_t hdr_compressed = (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
2194	dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;
2195
2196	ASSERT3P(buf->b_data, !=, NULL);
2197	IMPLY(compressed, hdr_compressed);
2198	IMPLY(compressed, ARC_BUF_COMPRESSED(buf));
2199
2200	if (hdr_compressed == compressed) {
2201		if (!arc_buf_is_shared(buf)) {
2202			abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd,
2203			    arc_buf_size(buf));
2204		}
2205	} else {
2206		ASSERT(hdr_compressed);
2207		ASSERT(!compressed);
2208		ASSERT3U(HDR_GET_LSIZE(hdr), !=, HDR_GET_PSIZE(hdr));
2209
2210		/*
2211		 * If the buf is sharing its data with the hdr, unlink it and
2212		 * allocate a new data buffer for the buf.
2213		 */
2214		if (arc_buf_is_shared(buf)) {
2215			ASSERT(ARC_BUF_COMPRESSED(buf));
2216
2217			/* We need to give the buf it's own b_data */
2218			buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
2219			buf->b_data =
2220			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2221			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2222
2223			/* Previously overhead was 0; just add new overhead */
2224			ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));
2225		} else if (ARC_BUF_COMPRESSED(buf)) {
2226			/* We need to reallocate the buf's b_data */
2227			arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr),
2228			    buf);
2229			buf->b_data =
2230			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2231
2232			/* We increased the size of b_data; update overhead */
2233			ARCSTAT_INCR(arcstat_overhead_size,
2234			    HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr));
2235		}
2236
2237		/*
2238		 * Regardless of the buf's previous compression settings, it
2239		 * should not be compressed at the end of this function.
2240		 */
2241		buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
2242
2243		/*
2244		 * Try copying the data from another buf which already has a
2245		 * decompressed version. If that's not possible, it's time to
2246		 * bite the bullet and decompress the data from the hdr.
2247		 */
2248		if (arc_buf_try_copy_decompressed_data(buf)) {
2249			/* Skip byteswapping and checksumming (already done) */
2250			ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, !=, NULL);
2251			return (0);
2252		} else {
2253			int error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
2254			    hdr->b_l1hdr.b_pabd, buf->b_data,
2255			    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
2256
2257			/*
2258			 * Absent hardware errors or software bugs, this should
2259			 * be impossible, but log it anyway so we can debug it.
2260			 */
2261			if (error != 0) {
2262				zfs_dbgmsg(
2263				    "hdr %p, compress %d, psize %d, lsize %d",
2264				    hdr, HDR_GET_COMPRESS(hdr),
2265				    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
2266				return (SET_ERROR(EIO));
2267			}
2268		}
2269	}
2270
2271	/* Byteswap the buf's data if necessary */
2272	if (bswap != DMU_BSWAP_NUMFUNCS) {
2273		ASSERT(!HDR_SHARED_DATA(hdr));
2274		ASSERT3U(bswap, <, DMU_BSWAP_NUMFUNCS);
2275		dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr));
2276	}
2277
2278	/* Compute the hdr's checksum if necessary */
2279	arc_cksum_compute(buf);
2280
2281	return (0);
2282}
2283
2284int
2285arc_decompress(arc_buf_t *buf)
2286{
2287	return (arc_buf_fill(buf, B_FALSE));
2288}
2289
2290/*
2291 * Return the size of the block, b_pabd, that is stored in the arc_buf_hdr_t.
2292 */
2293static uint64_t
2294arc_hdr_size(arc_buf_hdr_t *hdr)
2295{
2296	uint64_t size;
2297
2298	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
2299	    HDR_GET_PSIZE(hdr) > 0) {
2300		size = HDR_GET_PSIZE(hdr);
2301	} else {
2302		ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0);
2303		size = HDR_GET_LSIZE(hdr);
2304	}
2305	return (size);
2306}
2307
2308/*
2309 * Increment the amount of evictable space in the arc_state_t's refcount.
2310 * We account for the space used by the hdr and the arc buf individually
2311 * so that we can add and remove them from the refcount individually.
2312 */
2313static void
2314arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state)
2315{
2316	arc_buf_contents_t type = arc_buf_type(hdr);
2317
2318	ASSERT(HDR_HAS_L1HDR(hdr));
2319
2320	if (GHOST_STATE(state)) {
2321		ASSERT0(hdr->b_l1hdr.b_bufcnt);
2322		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2323		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2324		(void) refcount_add_many(&state->arcs_esize[type],
2325		    HDR_GET_LSIZE(hdr), hdr);
2326		return;
2327	}
2328
2329	ASSERT(!GHOST_STATE(state));
2330	if (hdr->b_l1hdr.b_pabd != NULL) {
2331		(void) refcount_add_many(&state->arcs_esize[type],
2332		    arc_hdr_size(hdr), hdr);
2333	}
2334	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2335	    buf = buf->b_next) {
2336		if (arc_buf_is_shared(buf))
2337			continue;
2338		(void) refcount_add_many(&state->arcs_esize[type],
2339		    arc_buf_size(buf), buf);
2340	}
2341}
2342
2343/*
2344 * Decrement the amount of evictable space in the arc_state_t's refcount.
2345 * We account for the space used by the hdr and the arc buf individually
2346 * so that we can add and remove them from the refcount individually.
2347 */
2348static void
2349arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state)
2350{
2351	arc_buf_contents_t type = arc_buf_type(hdr);
2352
2353	ASSERT(HDR_HAS_L1HDR(hdr));
2354
2355	if (GHOST_STATE(state)) {
2356		ASSERT0(hdr->b_l1hdr.b_bufcnt);
2357		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2358		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2359		(void) refcount_remove_many(&state->arcs_esize[type],
2360		    HDR_GET_LSIZE(hdr), hdr);
2361		return;
2362	}
2363
2364	ASSERT(!GHOST_STATE(state));
2365	if (hdr->b_l1hdr.b_pabd != NULL) {
2366		(void) refcount_remove_many(&state->arcs_esize[type],
2367		    arc_hdr_size(hdr), hdr);
2368	}
2369	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2370	    buf = buf->b_next) {
2371		if (arc_buf_is_shared(buf))
2372			continue;
2373		(void) refcount_remove_many(&state->arcs_esize[type],
2374		    arc_buf_size(buf), buf);
2375	}
2376}
2377
2378/*
2379 * Add a reference to this hdr indicating that someone is actively
2380 * referencing that memory. When the refcount transitions from 0 to 1,
2381 * we remove it from the respective arc_state_t list to indicate that
2382 * it is not evictable.
2383 */
2384static void
2385add_reference(arc_buf_hdr_t *hdr, void *tag)
2386{
2387	ASSERT(HDR_HAS_L1HDR(hdr));
2388	if (!MUTEX_HELD(HDR_LOCK(hdr))) {
2389		ASSERT(hdr->b_l1hdr.b_state == arc_anon);
2390		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2391		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2392	}
2393
2394	arc_state_t *state = hdr->b_l1hdr.b_state;
2395
2396	if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
2397	    (state != arc_anon)) {
2398		/* We don't use the L2-only state list. */
2399		if (state != arc_l2c_only) {
2400			multilist_remove(state->arcs_list[arc_buf_type(hdr)],
2401			    hdr);
2402			arc_evictable_space_decrement(hdr, state);
2403		}
2404		/* remove the prefetch flag if we get a reference */
2405		arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
2406	}
2407}
2408
2409/*
2410 * Remove a reference from this hdr. When the reference transitions from
2411 * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's
2412 * list making it eligible for eviction.
2413 */
2414static int
2415remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
2416{
2417	int cnt;
2418	arc_state_t *state = hdr->b_l1hdr.b_state;
2419
2420	ASSERT(HDR_HAS_L1HDR(hdr));
2421	ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
2422	ASSERT(!GHOST_STATE(state));
2423
2424	/*
2425	 * arc_l2c_only counts as a ghost state so we don't need to explicitly
2426	 * check to prevent usage of the arc_l2c_only list.
2427	 */
2428	if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
2429	    (state != arc_anon)) {
2430		multilist_insert(state->arcs_list[arc_buf_type(hdr)], hdr);
2431		ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
2432		arc_evictable_space_increment(hdr, state);
2433	}
2434	return (cnt);
2435}
2436
2437/*
2438 * Move the supplied buffer to the indicated state. The hash lock
2439 * for the buffer must be held by the caller.
2440 */
2441static void
2442arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
2443    kmutex_t *hash_lock)
2444{
2445	arc_state_t *old_state;
2446	int64_t refcnt;
2447	uint32_t bufcnt;
2448	boolean_t update_old, update_new;
2449	arc_buf_contents_t buftype = arc_buf_type(hdr);
2450
2451	/*
2452	 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
2453	 * in arc_read() when bringing a buffer out of the L2ARC.  However, the
2454	 * L1 hdr doesn't always exist when we change state to arc_anon before
2455	 * destroying a header, in which case reallocating to add the L1 hdr is
2456	 * pointless.
2457	 */
2458	if (HDR_HAS_L1HDR(hdr)) {
2459		old_state = hdr->b_l1hdr.b_state;
2460		refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt);
2461		bufcnt = hdr->b_l1hdr.b_bufcnt;
2462		update_old = (bufcnt > 0 || hdr->b_l1hdr.b_pabd != NULL);
2463	} else {
2464		old_state = arc_l2c_only;
2465		refcnt = 0;
2466		bufcnt = 0;
2467		update_old = B_FALSE;
2468	}
2469	update_new = update_old;
2470
2471	ASSERT(MUTEX_HELD(hash_lock));
2472	ASSERT3P(new_state, !=, old_state);
2473	ASSERT(!GHOST_STATE(new_state) || bufcnt == 0);
2474	ASSERT(old_state != arc_anon || bufcnt <= 1);
2475
2476	/*
2477	 * If this buffer is evictable, transfer it from the
2478	 * old state list to the new state list.
2479	 */
2480	if (refcnt == 0) {
2481		if (old_state != arc_anon && old_state != arc_l2c_only) {
2482			ASSERT(HDR_HAS_L1HDR(hdr));
2483			multilist_remove(old_state->arcs_list[buftype], hdr);
2484
2485			if (GHOST_STATE(old_state)) {
2486				ASSERT0(bufcnt);
2487				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2488				update_old = B_TRUE;
2489			}
2490			arc_evictable_space_decrement(hdr, old_state);
2491		}
2492		if (new_state != arc_anon && new_state != arc_l2c_only) {
2493
2494			/*
2495			 * An L1 header always exists here, since if we're
2496			 * moving to some L1-cached state (i.e. not l2c_only or
2497			 * anonymous), we realloc the header to add an L1hdr
2498			 * beforehand.
2499			 */
2500			ASSERT(HDR_HAS_L1HDR(hdr));
2501			multilist_insert(new_state->arcs_list[buftype], hdr);
2502
2503			if (GHOST_STATE(new_state)) {
2504				ASSERT0(bufcnt);
2505				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2506				update_new = B_TRUE;
2507			}
2508			arc_evictable_space_increment(hdr, new_state);
2509		}
2510	}
2511
2512	ASSERT(!HDR_EMPTY(hdr));
2513	if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
2514		buf_hash_remove(hdr);
2515
2516	/* adjust state sizes (ignore arc_l2c_only) */
2517
2518	if (update_new && new_state != arc_l2c_only) {
2519		ASSERT(HDR_HAS_L1HDR(hdr));
2520		if (GHOST_STATE(new_state)) {
2521			ASSERT0(bufcnt);
2522
2523			/*
2524			 * When moving a header to a ghost state, we first
2525			 * remove all arc buffers. Thus, we'll have a
2526			 * bufcnt of zero, and no arc buffer to use for
2527			 * the reference. As a result, we use the arc
2528			 * header pointer for the reference.
2529			 */
2530			(void) refcount_add_many(&new_state->arcs_size,
2531			    HDR_GET_LSIZE(hdr), hdr);
2532			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2533		} else {
2534			uint32_t buffers = 0;
2535
2536			/*
2537			 * Each individual buffer holds a unique reference,
2538			 * thus we must remove each of these references one
2539			 * at a time.
2540			 */
2541			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2542			    buf = buf->b_next) {
2543				ASSERT3U(bufcnt, !=, 0);
2544				buffers++;
2545
2546				/*
2547				 * When the arc_buf_t is sharing the data
2548				 * block with the hdr, the owner of the
2549				 * reference belongs to the hdr. Only
2550				 * add to the refcount if the arc_buf_t is
2551				 * not shared.
2552				 */
2553				if (arc_buf_is_shared(buf))
2554					continue;
2555
2556				(void) refcount_add_many(&new_state->arcs_size,
2557				    arc_buf_size(buf), buf);
2558			}
2559			ASSERT3U(bufcnt, ==, buffers);
2560
2561			if (hdr->b_l1hdr.b_pabd != NULL) {
2562				(void) refcount_add_many(&new_state->arcs_size,
2563				    arc_hdr_size(hdr), hdr);
2564			} else {
2565				ASSERT(GHOST_STATE(old_state));
2566			}
2567		}
2568	}
2569
2570	if (update_old && old_state != arc_l2c_only) {
2571		ASSERT(HDR_HAS_L1HDR(hdr));
2572		if (GHOST_STATE(old_state)) {
2573			ASSERT0(bufcnt);
2574			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2575
2576			/*
2577			 * When moving a header off of a ghost state,
2578			 * the header will not contain any arc buffers.
2579			 * We use the arc header pointer for the reference
2580			 * which is exactly what we did when we put the
2581			 * header on the ghost state.
2582			 */
2583
2584			(void) refcount_remove_many(&old_state->arcs_size,
2585			    HDR_GET_LSIZE(hdr), hdr);
2586		} else {
2587			uint32_t buffers = 0;
2588
2589			/*
2590			 * Each individual buffer holds a unique reference,
2591			 * thus we must remove each of these references one
2592			 * at a time.
2593			 */
2594			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2595			    buf = buf->b_next) {
2596				ASSERT3U(bufcnt, !=, 0);
2597				buffers++;
2598
2599				/*
2600				 * When the arc_buf_t is sharing the data
2601				 * block with the hdr, the owner of the
2602				 * reference belongs to the hdr. Only
2603				 * add to the refcount if the arc_buf_t is
2604				 * not shared.
2605				 */
2606				if (arc_buf_is_shared(buf))
2607					continue;
2608
2609				(void) refcount_remove_many(
2610				    &old_state->arcs_size, arc_buf_size(buf),
2611				    buf);
2612			}
2613			ASSERT3U(bufcnt, ==, buffers);
2614			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
2615			(void) refcount_remove_many(
2616			    &old_state->arcs_size, arc_hdr_size(hdr), hdr);
2617		}
2618	}
2619
2620	if (HDR_HAS_L1HDR(hdr))
2621		hdr->b_l1hdr.b_state = new_state;
2622
2623	/*
2624	 * L2 headers should never be on the L2 state list since they don't
2625	 * have L1 headers allocated.
2626	 */
2627	ASSERT(multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
2628	    multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
2629}
2630
2631void
2632arc_space_consume(uint64_t space, arc_space_type_t type)
2633{
2634	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2635
2636	switch (type) {
2637	case ARC_SPACE_DATA:
2638		ARCSTAT_INCR(arcstat_data_size, space);
2639		break;
2640	case ARC_SPACE_META:
2641		ARCSTAT_INCR(arcstat_metadata_size, space);
2642		break;
2643	case ARC_SPACE_OTHER:
2644		ARCSTAT_INCR(arcstat_other_size, space);
2645		break;
2646	case ARC_SPACE_HDRS:
2647		ARCSTAT_INCR(arcstat_hdr_size, space);
2648		break;
2649	case ARC_SPACE_L2HDRS:
2650		ARCSTAT_INCR(arcstat_l2_hdr_size, space);
2651		break;
2652	}
2653
2654	if (type != ARC_SPACE_DATA)
2655		ARCSTAT_INCR(arcstat_meta_used, space);
2656
2657	atomic_add_64(&arc_size, space);
2658}
2659
2660void
2661arc_space_return(uint64_t space, arc_space_type_t type)
2662{
2663	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2664
2665	switch (type) {
2666	case ARC_SPACE_DATA:
2667		ARCSTAT_INCR(arcstat_data_size, -space);
2668		break;
2669	case ARC_SPACE_META:
2670		ARCSTAT_INCR(arcstat_metadata_size, -space);
2671		break;
2672	case ARC_SPACE_OTHER:
2673		ARCSTAT_INCR(arcstat_other_size, -space);
2674		break;
2675	case ARC_SPACE_HDRS:
2676		ARCSTAT_INCR(arcstat_hdr_size, -space);
2677		break;
2678	case ARC_SPACE_L2HDRS:
2679		ARCSTAT_INCR(arcstat_l2_hdr_size, -space);
2680		break;
2681	}
2682
2683	if (type != ARC_SPACE_DATA) {
2684		ASSERT(arc_meta_used >= space);
2685		if (arc_meta_max < arc_meta_used)
2686			arc_meta_max = arc_meta_used;
2687		ARCSTAT_INCR(arcstat_meta_used, -space);
2688	}
2689
2690	ASSERT(arc_size >= space);
2691	atomic_add_64(&arc_size, -space);
2692}
2693
2694/*
2695 * Given a hdr and a buf, returns whether that buf can share its b_data buffer
2696 * with the hdr's b_pabd.
2697 */
2698static boolean_t
2699arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2700{
2701	/*
2702	 * The criteria for sharing a hdr's data are:
2703	 * 1. the hdr's compression matches the buf's compression
2704	 * 2. the hdr doesn't need to be byteswapped
2705	 * 3. the hdr isn't already being shared
2706	 * 4. the buf is either compressed or it is the last buf in the hdr list
2707	 *
2708	 * Criterion #4 maintains the invariant that shared uncompressed
2709	 * bufs must be the final buf in the hdr's b_buf list. Reading this, you
2710	 * might ask, "if a compressed buf is allocated first, won't that be the
2711	 * last thing in the list?", but in that case it's impossible to create
2712	 * a shared uncompressed buf anyway (because the hdr must be compressed
2713	 * to have the compressed buf). You might also think that #3 is
2714	 * sufficient to make this guarantee, however it's possible
2715	 * (specifically in the rare L2ARC write race mentioned in
2716	 * arc_buf_alloc_impl()) there will be an existing uncompressed buf that
2717	 * is sharable, but wasn't at the time of its allocation. Rather than
2718	 * allow a new shared uncompressed buf to be created and then shuffle
2719	 * the list around to make it the last element, this simply disallows
2720	 * sharing if the new buf isn't the first to be added.
2721	 */
2722	ASSERT3P(buf->b_hdr, ==, hdr);
2723	boolean_t hdr_compressed = HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF;
2724	boolean_t buf_compressed = ARC_BUF_COMPRESSED(buf) != 0;
2725	return (buf_compressed == hdr_compressed &&
2726	    hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&
2727	    !HDR_SHARED_DATA(hdr) &&
2728	    (ARC_BUF_LAST(buf) || ARC_BUF_COMPRESSED(buf)));
2729}
2730
2731/*
2732 * Allocate a buf for this hdr. If you care about the data that's in the hdr,
2733 * or if you want a compressed buffer, pass those flags in. Returns 0 if the
2734 * copy was made successfully, or an error code otherwise.
2735 */
2736static int
2737arc_buf_alloc_impl(arc_buf_hdr_t *hdr, void *tag, boolean_t compressed,
2738    boolean_t fill, arc_buf_t **ret)
2739{
2740	arc_buf_t *buf;
2741
2742	ASSERT(HDR_HAS_L1HDR(hdr));
2743	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
2744	VERIFY(hdr->b_type == ARC_BUFC_DATA ||
2745	    hdr->b_type == ARC_BUFC_METADATA);
2746	ASSERT3P(ret, !=, NULL);
2747	ASSERT3P(*ret, ==, NULL);
2748
2749	buf = *ret = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
2750	buf->b_hdr = hdr;
2751	buf->b_data = NULL;
2752	buf->b_next = hdr->b_l1hdr.b_buf;
2753	buf->b_flags = 0;
2754
2755	add_reference(hdr, tag);
2756
2757	/*
2758	 * We're about to change the hdr's b_flags. We must either
2759	 * hold the hash_lock or be undiscoverable.
2760	 */
2761	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2762
2763	/*
2764	 * Only honor requests for compressed bufs if the hdr is actually
2765	 * compressed.
2766	 */
2767	if (compressed && HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF)
2768		buf->b_flags |= ARC_BUF_FLAG_COMPRESSED;
2769
2770	/*
2771	 * If the hdr's data can be shared then we share the data buffer and
2772	 * set the appropriate bit in the hdr's b_flags to indicate the hdr is
2773	 * sharing it's b_pabd with the arc_buf_t. Otherwise, we allocate a new
2774	 * buffer to store the buf's data.
2775	 *
2776	 * There are two additional restrictions here because we're sharing
2777	 * hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be
2778	 * actively involved in an L2ARC write, because if this buf is used by
2779	 * an arc_write() then the hdr's data buffer will be released when the
2780	 * write completes, even though the L2ARC write might still be using it.
2781	 * Second, the hdr's ABD must be linear so that the buf's user doesn't
2782	 * need to be ABD-aware.
2783	 */
2784	boolean_t can_share = arc_can_share(hdr, buf) && !HDR_L2_WRITING(hdr) &&
2785	    abd_is_linear(hdr->b_l1hdr.b_pabd);
2786
2787	/* Set up b_data and sharing */
2788	if (can_share) {
2789		buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd);
2790		buf->b_flags |= ARC_BUF_FLAG_SHARED;
2791		arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2792	} else {
2793		buf->b_data =
2794		    arc_get_data_buf(hdr, arc_buf_size(buf), buf);
2795		ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
2796	}
2797	VERIFY3P(buf->b_data, !=, NULL);
2798
2799	hdr->b_l1hdr.b_buf = buf;
2800	hdr->b_l1hdr.b_bufcnt += 1;
2801
2802	/*
2803	 * If the user wants the data from the hdr, we need to either copy or
2804	 * decompress the data.
2805	 */
2806	if (fill) {
2807		return (arc_buf_fill(buf, ARC_BUF_COMPRESSED(buf) != 0));
2808	}
2809
2810	return (0);
2811}
2812
2813static char *arc_onloan_tag = "onloan";
2814
2815static inline void
2816arc_loaned_bytes_update(int64_t delta)
2817{
2818	atomic_add_64(&arc_loaned_bytes, delta);
2819
2820	/* assert that it did not wrap around */
2821	ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
2822}
2823
2824/*
2825 * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
2826 * flight data by arc_tempreserve_space() until they are "returned". Loaned
2827 * buffers must be returned to the arc before they can be used by the DMU or
2828 * freed.
2829 */
2830arc_buf_t *
2831arc_loan_buf(spa_t *spa, boolean_t is_metadata, int size)
2832{
2833	arc_buf_t *buf = arc_alloc_buf(spa, arc_onloan_tag,
2834	    is_metadata ? ARC_BUFC_METADATA : ARC_BUFC_DATA, size);
2835
2836	arc_loaned_bytes_update(size);
2837
2838	return (buf);
2839}
2840
2841arc_buf_t *
2842arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize,
2843    enum zio_compress compression_type)
2844{
2845	arc_buf_t *buf = arc_alloc_compressed_buf(spa, arc_onloan_tag,
2846	    psize, lsize, compression_type);
2847
2848	arc_loaned_bytes_update(psize);
2849
2850	return (buf);
2851}
2852
2853
2854/*
2855 * Return a loaned arc buffer to the arc.
2856 */
2857void
2858arc_return_buf(arc_buf_t *buf, void *tag)
2859{
2860	arc_buf_hdr_t *hdr = buf->b_hdr;
2861
2862	ASSERT3P(buf->b_data, !=, NULL);
2863	ASSERT(HDR_HAS_L1HDR(hdr));
2864	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
2865	(void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2866
2867	arc_loaned_bytes_update(-arc_buf_size(buf));
2868}
2869
2870/* Detach an arc_buf from a dbuf (tag) */
2871void
2872arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
2873{
2874	arc_buf_hdr_t *hdr = buf->b_hdr;
2875
2876	ASSERT3P(buf->b_data, !=, NULL);
2877	ASSERT(HDR_HAS_L1HDR(hdr));
2878	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2879	(void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
2880
2881	arc_loaned_bytes_update(arc_buf_size(buf));
2882}
2883
2884static void
2885l2arc_free_abd_on_write(abd_t *abd, size_t size, arc_buf_contents_t type)
2886{
2887	l2arc_data_free_t *df = kmem_alloc(sizeof (*df), KM_SLEEP);
2888
2889	df->l2df_abd = abd;
2890	df->l2df_size = size;
2891	df->l2df_type = type;
2892	mutex_enter(&l2arc_free_on_write_mtx);
2893	list_insert_head(l2arc_free_on_write, df);
2894	mutex_exit(&l2arc_free_on_write_mtx);
2895}
2896
2897static void
2898arc_hdr_free_on_write(arc_buf_hdr_t *hdr)
2899{
2900	arc_state_t *state = hdr->b_l1hdr.b_state;
2901	arc_buf_contents_t type = arc_buf_type(hdr);
2902	uint64_t size = arc_hdr_size(hdr);
2903
2904	/* protected by hash lock, if in the hash table */
2905	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2906		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2907		ASSERT(state != arc_anon && state != arc_l2c_only);
2908
2909		(void) refcount_remove_many(&state->arcs_esize[type],
2910		    size, hdr);
2911	}
2912	(void) refcount_remove_many(&state->arcs_size, size, hdr);
2913	if (type == ARC_BUFC_METADATA) {
2914		arc_space_return(size, ARC_SPACE_META);
2915	} else {
2916		ASSERT(type == ARC_BUFC_DATA);
2917		arc_space_return(size, ARC_SPACE_DATA);
2918	}
2919
2920	l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type);
2921}
2922
2923/*
2924 * Share the arc_buf_t's data with the hdr. Whenever we are sharing the
2925 * data buffer, we transfer the refcount ownership to the hdr and update
2926 * the appropriate kstats.
2927 */
2928static void
2929arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2930{
2931	arc_state_t *state = hdr->b_l1hdr.b_state;
2932
2933	ASSERT(arc_can_share(hdr, buf));
2934	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2935	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2936
2937	/*
2938	 * Start sharing the data buffer. We transfer the
2939	 * refcount ownership to the hdr since it always owns
2940	 * the refcount whenever an arc_buf_t is shared.
2941	 */
2942	refcount_transfer_ownership(&state->arcs_size, buf, hdr);
2943	hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf));
2944	abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd,
2945	    HDR_ISTYPE_METADATA(hdr));
2946	arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2947	buf->b_flags |= ARC_BUF_FLAG_SHARED;
2948
2949	/*
2950	 * Since we've transferred ownership to the hdr we need
2951	 * to increment its compressed and uncompressed kstats and
2952	 * decrement the overhead size.
2953	 */
2954	ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
2955	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
2956	ARCSTAT_INCR(arcstat_overhead_size, -arc_buf_size(buf));
2957}
2958
2959static void
2960arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2961{
2962	arc_state_t *state = hdr->b_l1hdr.b_state;
2963
2964	ASSERT(arc_buf_is_shared(buf));
2965	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
2966	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2967
2968	/*
2969	 * We are no longer sharing this buffer so we need
2970	 * to transfer its ownership to the rightful owner.
2971	 */
2972	refcount_transfer_ownership(&state->arcs_size, hdr, buf);
2973	arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2974	abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd);
2975	abd_put(hdr->b_l1hdr.b_pabd);
2976	hdr->b_l1hdr.b_pabd = NULL;
2977	buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
2978
2979	/*
2980	 * Since the buffer is no longer shared between
2981	 * the arc buf and the hdr, count it as overhead.
2982	 */
2983	ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
2984	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
2985	ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
2986}
2987
2988/*
2989 * Remove an arc_buf_t from the hdr's buf list and return the last
2990 * arc_buf_t on the list. If no buffers remain on the list then return
2991 * NULL.
2992 */
2993static arc_buf_t *
2994arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2995{
2996	ASSERT(HDR_HAS_L1HDR(hdr));
2997	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2998
2999	arc_buf_t **bufp = &hdr->b_l1hdr.b_buf;
3000	arc_buf_t *lastbuf = NULL;
3001
3002	/*
3003	 * Remove the buf from the hdr list and locate the last
3004	 * remaining buffer on the list.
3005	 */
3006	while (*bufp != NULL) {
3007		if (*bufp == buf)
3008			*bufp = buf->b_next;
3009
3010		/*
3011		 * If we've removed a buffer in the middle of
3012		 * the list then update the lastbuf and update
3013		 * bufp.
3014		 */
3015		if (*bufp != NULL) {
3016			lastbuf = *bufp;
3017			bufp = &(*bufp)->b_next;
3018		}
3019	}
3020	buf->b_next = NULL;
3021	ASSERT3P(lastbuf, !=, buf);
3022	IMPLY(hdr->b_l1hdr.b_bufcnt > 0, lastbuf != NULL);
3023	IMPLY(hdr->b_l1hdr.b_bufcnt > 0, hdr->b_l1hdr.b_buf != NULL);
3024	IMPLY(lastbuf != NULL, ARC_BUF_LAST(lastbuf));
3025
3026	return (lastbuf);
3027}
3028
3029/*
3030 * Free up buf->b_data and pull the arc_buf_t off of the the arc_buf_hdr_t's
3031 * list and free it.
3032 */
3033static void
3034arc_buf_destroy_impl(arc_buf_t *buf)
3035{
3036	arc_buf_hdr_t *hdr = buf->b_hdr;
3037
3038	/*
3039	 * Free up the data associated with the buf but only if we're not
3040	 * sharing this with the hdr. If we are sharing it with the hdr, the
3041	 * hdr is responsible for doing the free.
3042	 */
3043	if (buf->b_data != NULL) {
3044		/*
3045		 * We're about to change the hdr's b_flags. We must either
3046		 * hold the hash_lock or be undiscoverable.
3047		 */
3048		ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
3049
3050		arc_cksum_verify(buf);
3051#ifdef illumos
3052		arc_buf_unwatch(buf);
3053#endif
3054
3055		if (arc_buf_is_shared(buf)) {
3056			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
3057		} else {
3058			uint64_t size = arc_buf_size(buf);
3059			arc_free_data_buf(hdr, buf->b_data, size, buf);
3060			ARCSTAT_INCR(arcstat_overhead_size, -size);
3061		}
3062		buf->b_data = NULL;
3063
3064		ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
3065		hdr->b_l1hdr.b_bufcnt -= 1;
3066	}
3067
3068	arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
3069
3070	if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) {
3071		/*
3072		 * If the current arc_buf_t is sharing its data buffer with the
3073		 * hdr, then reassign the hdr's b_pabd to share it with the new
3074		 * buffer at the end of the list. The shared buffer is always
3075		 * the last one on the hdr's buffer list.
3076		 *
3077		 * There is an equivalent case for compressed bufs, but since
3078		 * they aren't guaranteed to be the last buf in the list and
3079		 * that is an exceedingly rare case, we just allow that space be
3080		 * wasted temporarily.
3081		 */
3082		if (lastbuf != NULL) {
3083			/* Only one buf can be shared at once */
3084			VERIFY(!arc_buf_is_shared(lastbuf));
3085			/* hdr is uncompressed so can't have compressed buf */
3086			VERIFY(!ARC_BUF_COMPRESSED(lastbuf));
3087
3088			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3089			arc_hdr_free_pabd(hdr);
3090
3091			/*
3092			 * We must setup a new shared block between the
3093			 * last buffer and the hdr. The data would have
3094			 * been allocated by the arc buf so we need to transfer
3095			 * ownership to the hdr since it's now being shared.
3096			 */
3097			arc_share_buf(hdr, lastbuf);
3098		}
3099	} else if (HDR_SHARED_DATA(hdr)) {
3100		/*
3101		 * Uncompressed shared buffers are always at the end
3102		 * of the list. Compressed buffers don't have the
3103		 * same requirements. This makes it hard to
3104		 * simply assert that the lastbuf is shared so
3105		 * we rely on the hdr's compression flags to determine
3106		 * if we have a compressed, shared buffer.
3107		 */
3108		ASSERT3P(lastbuf, !=, NULL);
3109		ASSERT(arc_buf_is_shared(lastbuf) ||
3110		    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
3111	}
3112
3113	/*
3114	 * Free the checksum if we're removing the last uncompressed buf from
3115	 * this hdr.
3116	 */
3117	if (!arc_hdr_has_uncompressed_buf(hdr)) {
3118		arc_cksum_free(hdr);
3119	}
3120
3121	/* clean up the buf */
3122	buf->b_hdr = NULL;
3123	kmem_cache_free(buf_cache, buf);
3124}
3125
3126static void
3127arc_hdr_alloc_pabd(arc_buf_hdr_t *hdr)
3128{
3129	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
3130	ASSERT(HDR_HAS_L1HDR(hdr));
3131	ASSERT(!HDR_SHARED_DATA(hdr));
3132
3133	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3134	hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr);
3135	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3136	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3137
3138	ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
3139	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
3140}
3141
3142static void
3143arc_hdr_free_pabd(arc_buf_hdr_t *hdr)
3144{
3145	ASSERT(HDR_HAS_L1HDR(hdr));
3146	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3147
3148	/*
3149	 * If the hdr is currently being written to the l2arc then
3150	 * we defer freeing the data by adding it to the l2arc_free_on_write
3151	 * list. The l2arc will free the data once it's finished
3152	 * writing it to the l2arc device.
3153	 */
3154	if (HDR_L2_WRITING(hdr)) {
3155		arc_hdr_free_on_write(hdr);
3156		ARCSTAT_BUMP(arcstat_l2_free_on_write);
3157	} else {
3158		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
3159		    arc_hdr_size(hdr), hdr);
3160	}
3161	hdr->b_l1hdr.b_pabd = NULL;
3162	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3163
3164	ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
3165	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3166}
3167
3168static arc_buf_hdr_t *
3169arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize,
3170    enum zio_compress compression_type, arc_buf_contents_t type)
3171{
3172	arc_buf_hdr_t *hdr;
3173
3174	VERIFY(type == ARC_BUFC_DATA || type == ARC_BUFC_METADATA);
3175
3176	hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
3177	ASSERT(HDR_EMPTY(hdr));
3178	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3179	ASSERT3P(hdr->b_l1hdr.b_thawed, ==, NULL);
3180	HDR_SET_PSIZE(hdr, psize);
3181	HDR_SET_LSIZE(hdr, lsize);
3182	hdr->b_spa = spa;
3183	hdr->b_type = type;
3184	hdr->b_flags = 0;
3185	arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR);
3186	arc_hdr_set_compress(hdr, compression_type);
3187
3188	hdr->b_l1hdr.b_state = arc_anon;
3189	hdr->b_l1hdr.b_arc_access = 0;
3190	hdr->b_l1hdr.b_bufcnt = 0;
3191	hdr->b_l1hdr.b_buf = NULL;
3192
3193	/*
3194	 * Allocate the hdr's buffer. This will contain either
3195	 * the compressed or uncompressed data depending on the block
3196	 * it references and compressed arc enablement.
3197	 */
3198	arc_hdr_alloc_pabd(hdr);
3199	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3200
3201	return (hdr);
3202}
3203
3204/*
3205 * Transition between the two allocation states for the arc_buf_hdr struct.
3206 * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
3207 * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
3208 * version is used when a cache buffer is only in the L2ARC in order to reduce
3209 * memory usage.
3210 */
3211static arc_buf_hdr_t *
3212arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
3213{
3214	ASSERT(HDR_HAS_L2HDR(hdr));
3215
3216	arc_buf_hdr_t *nhdr;
3217	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3218
3219	ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) ||
3220	    (old == hdr_l2only_cache && new == hdr_full_cache));
3221
3222	nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);
3223
3224	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
3225	buf_hash_remove(hdr);
3226
3227	bcopy(hdr, nhdr, HDR_L2ONLY_SIZE);
3228
3229	if (new == hdr_full_cache) {
3230		arc_hdr_set_flags(nhdr, ARC_FLAG_HAS_L1HDR);
3231		/*
3232		 * arc_access and arc_change_state need to be aware that a
3233		 * header has just come out of L2ARC, so we set its state to
3234		 * l2c_only even though it's about to change.
3235		 */
3236		nhdr->b_l1hdr.b_state = arc_l2c_only;
3237
3238		/* Verify previous threads set to NULL before freeing */
3239		ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL);
3240	} else {
3241		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3242		ASSERT0(hdr->b_l1hdr.b_bufcnt);
3243		ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3244
3245		/*
3246		 * If we've reached here, We must have been called from
3247		 * arc_evict_hdr(), as such we should have already been
3248		 * removed from any ghost list we were previously on
3249		 * (which protects us from racing with arc_evict_state),
3250		 * thus no locking is needed during this check.
3251		 */
3252		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3253
3254		/*
3255		 * A buffer must not be moved into the arc_l2c_only
3256		 * state if it's not finished being written out to the
3257		 * l2arc device. Otherwise, the b_l1hdr.b_pabd field
3258		 * might try to be accessed, even though it was removed.
3259		 */
3260		VERIFY(!HDR_L2_WRITING(hdr));
3261		VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3262
3263#ifdef ZFS_DEBUG
3264		if (hdr->b_l1hdr.b_thawed != NULL) {
3265			kmem_free(hdr->b_l1hdr.b_thawed, 1);
3266			hdr->b_l1hdr.b_thawed = NULL;
3267		}
3268#endif
3269
3270		arc_hdr_clear_flags(nhdr, ARC_FLAG_HAS_L1HDR);
3271	}
3272	/*
3273	 * The header has been reallocated so we need to re-insert it into any
3274	 * lists it was on.
3275	 */
3276	(void) buf_hash_insert(nhdr, NULL);
3277
3278	ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
3279
3280	mutex_enter(&dev->l2ad_mtx);
3281
3282	/*
3283	 * We must place the realloc'ed header back into the list at
3284	 * the same spot. Otherwise, if it's placed earlier in the list,
3285	 * l2arc_write_buffers() could find it during the function's
3286	 * write phase, and try to write it out to the l2arc.
3287	 */
3288	list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
3289	list_remove(&dev->l2ad_buflist, hdr);
3290
3291	mutex_exit(&dev->l2ad_mtx);
3292
3293	/*
3294	 * Since we're using the pointer address as the tag when
3295	 * incrementing and decrementing the l2ad_alloc refcount, we
3296	 * must remove the old pointer (that we're about to destroy) and
3297	 * add the new pointer to the refcount. Otherwise we'd remove
3298	 * the wrong pointer address when calling arc_hdr_destroy() later.
3299	 */
3300
3301	(void) refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr);
3302	(void) refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(nhdr), nhdr);
3303
3304	buf_discard_identity(hdr);
3305	kmem_cache_free(old, hdr);
3306
3307	return (nhdr);
3308}
3309
3310/*
3311 * Allocate a new arc_buf_hdr_t and arc_buf_t and return the buf to the caller.
3312 * The buf is returned thawed since we expect the consumer to modify it.
3313 */
3314arc_buf_t *
3315arc_alloc_buf(spa_t *spa, void *tag, arc_buf_contents_t type, int32_t size)
3316{
3317	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size,
3318	    ZIO_COMPRESS_OFF, type);
3319	ASSERT(!MUTEX_HELD(HDR_LOCK(hdr)));
3320
3321	arc_buf_t *buf = NULL;
3322	VERIFY0(arc_buf_alloc_impl(hdr, tag, B_FALSE, B_FALSE, &buf));
3323	arc_buf_thaw(buf);
3324
3325	return (buf);
3326}
3327
3328/*
3329 * Allocate a compressed buf in the same manner as arc_alloc_buf. Don't use this
3330 * for bufs containing metadata.
3331 */
3332arc_buf_t *
3333arc_alloc_compressed_buf(spa_t *spa, void *tag, uint64_t psize, uint64_t lsize,
3334    enum zio_compress compression_type)
3335{
3336	ASSERT3U(lsize, >, 0);
3337	ASSERT3U(lsize, >=, psize);
3338	ASSERT(compression_type > ZIO_COMPRESS_OFF);
3339	ASSERT(compression_type < ZIO_COMPRESS_FUNCTIONS);
3340
3341	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
3342	    compression_type, ARC_BUFC_DATA);
3343	ASSERT(!MUTEX_HELD(HDR_LOCK(hdr)));
3344
3345	arc_buf_t *buf = NULL;
3346	VERIFY0(arc_buf_alloc_impl(hdr, tag, B_TRUE, B_FALSE, &buf));
3347	arc_buf_thaw(buf);
3348	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3349
3350	if (!arc_buf_is_shared(buf)) {
3351		/*
3352		 * To ensure that the hdr has the correct data in it if we call
3353		 * arc_decompress() on this buf before it's been written to
3354		 * disk, it's easiest if we just set up sharing between the
3355		 * buf and the hdr.
3356		 */
3357		ASSERT(!abd_is_linear(hdr->b_l1hdr.b_pabd));
3358		arc_hdr_free_pabd(hdr);
3359		arc_share_buf(hdr, buf);
3360	}
3361
3362	return (buf);
3363}
3364
3365static void
3366arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
3367{
3368	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
3369	l2arc_dev_t *dev = l2hdr->b_dev;
3370	uint64_t psize = arc_hdr_size(hdr);
3371
3372	ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
3373	ASSERT(HDR_HAS_L2HDR(hdr));
3374
3375	list_remove(&dev->l2ad_buflist, hdr);
3376
3377	ARCSTAT_INCR(arcstat_l2_psize, -psize);
3378	ARCSTAT_INCR(arcstat_l2_lsize, -HDR_GET_LSIZE(hdr));
3379
3380	vdev_space_update(dev->l2ad_vdev, -psize, 0, 0);
3381
3382	(void) refcount_remove_many(&dev->l2ad_alloc, psize, hdr);
3383	arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
3384}
3385
3386static void
3387arc_hdr_destroy(arc_buf_hdr_t *hdr)
3388{
3389	if (HDR_HAS_L1HDR(hdr)) {
3390		ASSERT(hdr->b_l1hdr.b_buf == NULL ||
3391		    hdr->b_l1hdr.b_bufcnt > 0);
3392		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3393		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3394	}
3395	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3396	ASSERT(!HDR_IN_HASH_TABLE(hdr));
3397
3398	if (!HDR_EMPTY(hdr))
3399		buf_discard_identity(hdr);
3400
3401	if (HDR_HAS_L2HDR(hdr)) {
3402		l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3403		boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx);
3404
3405		if (!buflist_held)
3406			mutex_enter(&dev->l2ad_mtx);
3407
3408		/*
3409		 * Even though we checked this conditional above, we
3410		 * need to check this again now that we have the
3411		 * l2ad_mtx. This is because we could be racing with
3412		 * another thread calling l2arc_evict() which might have
3413		 * destroyed this header's L2 portion as we were waiting
3414		 * to acquire the l2ad_mtx. If that happens, we don't
3415		 * want to re-destroy the header's L2 portion.
3416		 */
3417		if (HDR_HAS_L2HDR(hdr)) {
3418			l2arc_trim(hdr);
3419			arc_hdr_l2hdr_destroy(hdr);
3420		}
3421
3422		if (!buflist_held)
3423			mutex_exit(&dev->l2ad_mtx);
3424	}
3425
3426	if (HDR_HAS_L1HDR(hdr)) {
3427		arc_cksum_free(hdr);
3428
3429		while (hdr->b_l1hdr.b_buf != NULL)
3430			arc_buf_destroy_impl(hdr->b_l1hdr.b_buf);
3431
3432#ifdef ZFS_DEBUG
3433		if (hdr->b_l1hdr.b_thawed != NULL) {
3434			kmem_free(hdr->b_l1hdr.b_thawed, 1);
3435			hdr->b_l1hdr.b_thawed = NULL;
3436		}
3437#endif
3438
3439		if (hdr->b_l1hdr.b_pabd != NULL) {
3440			arc_hdr_free_pabd(hdr);
3441		}
3442	}
3443
3444	ASSERT3P(hdr->b_hash_next, ==, NULL);
3445	if (HDR_HAS_L1HDR(hdr)) {
3446		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3447		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
3448		kmem_cache_free(hdr_full_cache, hdr);
3449	} else {
3450		kmem_cache_free(hdr_l2only_cache, hdr);
3451	}
3452}
3453
3454void
3455arc_buf_destroy(arc_buf_t *buf, void* tag)
3456{
3457	arc_buf_hdr_t *hdr = buf->b_hdr;
3458	kmutex_t *hash_lock = HDR_LOCK(hdr);
3459
3460	if (hdr->b_l1hdr.b_state == arc_anon) {
3461		ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
3462		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3463		VERIFY0(remove_reference(hdr, NULL, tag));
3464		arc_hdr_destroy(hdr);
3465		return;
3466	}
3467
3468	mutex_enter(hash_lock);
3469	ASSERT3P(hdr, ==, buf->b_hdr);
3470	ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
3471	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3472	ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
3473	ASSERT3P(buf->b_data, !=, NULL);
3474
3475	(void) remove_reference(hdr, hash_lock, tag);
3476	arc_buf_destroy_impl(buf);
3477	mutex_exit(hash_lock);
3478}
3479
3480/*
3481 * Evict the arc_buf_hdr that is provided as a parameter. The resultant
3482 * state of the header is dependent on its state prior to entering this
3483 * function. The following transitions are possible:
3484 *
3485 *    - arc_mru -> arc_mru_ghost
3486 *    - arc_mfu -> arc_mfu_ghost
3487 *    - arc_mru_ghost -> arc_l2c_only
3488 *    - arc_mru_ghost -> deleted
3489 *    - arc_mfu_ghost -> arc_l2c_only
3490 *    - arc_mfu_ghost -> deleted
3491 */
3492static int64_t
3493arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
3494{
3495	arc_state_t *evicted_state, *state;
3496	int64_t bytes_evicted = 0;
3497
3498	ASSERT(MUTEX_HELD(hash_lock));
3499	ASSERT(HDR_HAS_L1HDR(hdr));
3500
3501	state = hdr->b_l1hdr.b_state;
3502	if (GHOST_STATE(state)) {
3503		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3504		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3505
3506		/*
3507		 * l2arc_write_buffers() relies on a header's L1 portion
3508		 * (i.e. its b_pabd field) during it's write phase.
3509		 * Thus, we cannot push a header onto the arc_l2c_only
3510		 * state (removing it's L1 piece) until the header is
3511		 * done being written to the l2arc.
3512		 */
3513		if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
3514			ARCSTAT_BUMP(arcstat_evict_l2_skip);
3515			return (bytes_evicted);
3516		}
3517
3518		ARCSTAT_BUMP(arcstat_deleted);
3519		bytes_evicted += HDR_GET_LSIZE(hdr);
3520
3521		DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
3522
3523		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3524		if (HDR_HAS_L2HDR(hdr)) {
3525			/*
3526			 * This buffer is cached on the 2nd Level ARC;
3527			 * don't destroy the header.
3528			 */
3529			arc_change_state(arc_l2c_only, hdr, hash_lock);
3530			/*
3531			 * dropping from L1+L2 cached to L2-only,
3532			 * realloc to remove the L1 header.
3533			 */
3534			hdr = arc_hdr_realloc(hdr, hdr_full_cache,
3535			    hdr_l2only_cache);
3536		} else {
3537			arc_change_state(arc_anon, hdr, hash_lock);
3538			arc_hdr_destroy(hdr);
3539		}
3540		return (bytes_evicted);
3541	}
3542
3543	ASSERT(state == arc_mru || state == arc_mfu);
3544	evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
3545
3546	/* prefetch buffers have a minimum lifespan */
3547	if (HDR_IO_IN_PROGRESS(hdr) ||
3548	    ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
3549	    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
3550	    arc_min_prefetch_lifespan)) {
3551		ARCSTAT_BUMP(arcstat_evict_skip);
3552		return (bytes_evicted);
3553	}
3554
3555	ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
3556	while (hdr->b_l1hdr.b_buf) {
3557		arc_buf_t *buf = hdr->b_l1hdr.b_buf;
3558		if (!mutex_tryenter(&buf->b_evict_lock)) {
3559			ARCSTAT_BUMP(arcstat_mutex_miss);
3560			break;
3561		}
3562		if (buf->b_data != NULL)
3563			bytes_evicted += HDR_GET_LSIZE(hdr);
3564		mutex_exit(&buf->b_evict_lock);
3565		arc_buf_destroy_impl(buf);
3566	}
3567
3568	if (HDR_HAS_L2HDR(hdr)) {
3569		ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr));
3570	} else {
3571		if (l2arc_write_eligible(hdr->b_spa, hdr)) {
3572			ARCSTAT_INCR(arcstat_evict_l2_eligible,
3573			    HDR_GET_LSIZE(hdr));
3574		} else {
3575			ARCSTAT_INCR(arcstat_evict_l2_ineligible,
3576			    HDR_GET_LSIZE(hdr));
3577		}
3578	}
3579
3580	if (hdr->b_l1hdr.b_bufcnt == 0) {
3581		arc_cksum_free(hdr);
3582
3583		bytes_evicted += arc_hdr_size(hdr);
3584
3585		/*
3586		 * If this hdr is being evicted and has a compressed
3587		 * buffer then we discard it here before we change states.
3588		 * This ensures that the accounting is updated correctly
3589		 * in arc_free_data_impl().
3590		 */
3591		arc_hdr_free_pabd(hdr);
3592
3593		arc_change_state(evicted_state, hdr, hash_lock);
3594		ASSERT(HDR_IN_HASH_TABLE(hdr));
3595		arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
3596		DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
3597	}
3598
3599	return (bytes_evicted);
3600}
3601
3602static uint64_t
3603arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker,
3604    uint64_t spa, int64_t bytes)
3605{
3606	multilist_sublist_t *mls;
3607	uint64_t bytes_evicted = 0;
3608	arc_buf_hdr_t *hdr;
3609	kmutex_t *hash_lock;
3610	int evict_count = 0;
3611
3612	ASSERT3P(marker, !=, NULL);
3613	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
3614
3615	mls = multilist_sublist_lock(ml, idx);
3616
3617	for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL;
3618	    hdr = multilist_sublist_prev(mls, marker)) {
3619		if ((bytes != ARC_EVICT_ALL && bytes_evicted >= bytes) ||
3620		    (evict_count >= zfs_arc_evict_batch_limit))
3621			break;
3622
3623		/*
3624		 * To keep our iteration location, move the marker
3625		 * forward. Since we're not holding hdr's hash lock, we
3626		 * must be very careful and not remove 'hdr' from the
3627		 * sublist. Otherwise, other consumers might mistake the
3628		 * 'hdr' as not being on a sublist when they call the
3629		 * multilist_link_active() function (they all rely on
3630		 * the hash lock protecting concurrent insertions and
3631		 * removals). multilist_sublist_move_forward() was
3632		 * specifically implemented to ensure this is the case
3633		 * (only 'marker' will be removed and re-inserted).
3634		 */
3635		multilist_sublist_move_forward(mls, marker);
3636
3637		/*
3638		 * The only case where the b_spa field should ever be
3639		 * zero, is the marker headers inserted by
3640		 * arc_evict_state(). It's possible for multiple threads
3641		 * to be calling arc_evict_state() concurrently (e.g.
3642		 * dsl_pool_close() and zio_inject_fault()), so we must
3643		 * skip any markers we see from these other threads.
3644		 */
3645		if (hdr->b_spa == 0)
3646			continue;
3647
3648		/* we're only interested in evicting buffers of a certain spa */
3649		if (spa != 0 && hdr->b_spa != spa) {
3650			ARCSTAT_BUMP(arcstat_evict_skip);
3651			continue;
3652		}
3653
3654		hash_lock = HDR_LOCK(hdr);
3655
3656		/*
3657		 * We aren't calling this function from any code path
3658		 * that would already be holding a hash lock, so we're
3659		 * asserting on this assumption to be defensive in case
3660		 * this ever changes. Without this check, it would be
3661		 * possible to incorrectly increment arcstat_mutex_miss
3662		 * below (e.g. if the code changed such that we called
3663		 * this function with a hash lock held).
3664		 */
3665		ASSERT(!MUTEX_HELD(hash_lock));
3666
3667		if (mutex_tryenter(hash_lock)) {
3668			uint64_t evicted = arc_evict_hdr(hdr, hash_lock);
3669			mutex_exit(hash_lock);
3670
3671			bytes_evicted += evicted;
3672
3673			/*
3674			 * If evicted is zero, arc_evict_hdr() must have
3675			 * decided to skip this header, don't increment
3676			 * evict_count in this case.
3677			 */
3678			if (evicted != 0)
3679				evict_count++;
3680
3681			/*
3682			 * If arc_size isn't overflowing, signal any
3683			 * threads that might happen to be waiting.
3684			 *
3685			 * For each header evicted, we wake up a single
3686			 * thread. If we used cv_broadcast, we could
3687			 * wake up "too many" threads causing arc_size
3688			 * to significantly overflow arc_c; since
3689			 * arc_get_data_impl() doesn't check for overflow
3690			 * when it's woken up (it doesn't because it's
3691			 * possible for the ARC to be overflowing while
3692			 * full of un-evictable buffers, and the
3693			 * function should proceed in this case).
3694			 *
3695			 * If threads are left sleeping, due to not
3696			 * using cv_broadcast, they will be woken up
3697			 * just before arc_reclaim_thread() sleeps.
3698			 */
3699			mutex_enter(&arc_reclaim_lock);
3700			if (!arc_is_overflowing())
3701				cv_signal(&arc_reclaim_waiters_cv);
3702			mutex_exit(&arc_reclaim_lock);
3703		} else {
3704			ARCSTAT_BUMP(arcstat_mutex_miss);
3705		}
3706	}
3707
3708	multilist_sublist_unlock(mls);
3709
3710	return (bytes_evicted);
3711}
3712
3713/*
3714 * Evict buffers from the given arc state, until we've removed the
3715 * specified number of bytes. Move the removed buffers to the
3716 * appropriate evict state.
3717 *
3718 * This function makes a "best effort". It skips over any buffers
3719 * it can't get a hash_lock on, and so, may not catch all candidates.
3720 * It may also return without evicting as much space as requested.
3721 *
3722 * If bytes is specified using the special value ARC_EVICT_ALL, this
3723 * will evict all available (i.e. unlocked and evictable) buffers from
3724 * the given arc state; which is used by arc_flush().
3725 */
3726static uint64_t
3727arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
3728    arc_buf_contents_t type)
3729{
3730	uint64_t total_evicted = 0;
3731	multilist_t *ml = state->arcs_list[type];
3732	int num_sublists;
3733	arc_buf_hdr_t **markers;
3734
3735	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
3736
3737	num_sublists = multilist_get_num_sublists(ml);
3738
3739	/*
3740	 * If we've tried to evict from each sublist, made some
3741	 * progress, but still have not hit the target number of bytes
3742	 * to evict, we want to keep trying. The markers allow us to
3743	 * pick up where we left off for each individual sublist, rather
3744	 * than starting from the tail each time.
3745	 */
3746	markers = kmem_zalloc(sizeof (*markers) * num_sublists, KM_SLEEP);
3747	for (int i = 0; i < num_sublists; i++) {
3748		markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
3749
3750		/*
3751		 * A b_spa of 0 is used to indicate that this header is
3752		 * a marker. This fact is used in arc_adjust_type() and
3753		 * arc_evict_state_impl().
3754		 */
3755		markers[i]->b_spa = 0;
3756
3757		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
3758		multilist_sublist_insert_tail(mls, markers[i]);
3759		multilist_sublist_unlock(mls);
3760	}
3761
3762	/*
3763	 * While we haven't hit our target number of bytes to evict, or
3764	 * we're evicting all available buffers.
3765	 */
3766	while (total_evicted < bytes || bytes == ARC_EVICT_ALL) {
3767		/*
3768		 * Start eviction using a randomly selected sublist,
3769		 * this is to try and evenly balance eviction across all
3770		 * sublists. Always starting at the same sublist
3771		 * (e.g. index 0) would cause evictions to favor certain
3772		 * sublists over others.
3773		 */
3774		int sublist_idx = multilist_get_random_index(ml);
3775		uint64_t scan_evicted = 0;
3776
3777		for (int i = 0; i < num_sublists; i++) {
3778			uint64_t bytes_remaining;
3779			uint64_t bytes_evicted;
3780
3781			if (bytes == ARC_EVICT_ALL)
3782				bytes_remaining = ARC_EVICT_ALL;
3783			else if (total_evicted < bytes)
3784				bytes_remaining = bytes - total_evicted;
3785			else
3786				break;
3787
3788			bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
3789			    markers[sublist_idx], spa, bytes_remaining);
3790
3791			scan_evicted += bytes_evicted;
3792			total_evicted += bytes_evicted;
3793
3794			/* we've reached the end, wrap to the beginning */
3795			if (++sublist_idx >= num_sublists)
3796				sublist_idx = 0;
3797		}
3798
3799		/*
3800		 * If we didn't evict anything during this scan, we have
3801		 * no reason to believe we'll evict more during another
3802		 * scan, so break the loop.
3803		 */
3804		if (scan_evicted == 0) {
3805			/* This isn't possible, let's make that obvious */
3806			ASSERT3S(bytes, !=, 0);
3807
3808			/*
3809			 * When bytes is ARC_EVICT_ALL, the only way to
3810			 * break the loop is when scan_evicted is zero.
3811			 * In that case, we actually have evicted enough,
3812			 * so we don't want to increment the kstat.
3813			 */
3814			if (bytes != ARC_EVICT_ALL) {
3815				ASSERT3S(total_evicted, <, bytes);
3816				ARCSTAT_BUMP(arcstat_evict_not_enough);
3817			}
3818
3819			break;
3820		}
3821	}
3822
3823	for (int i = 0; i < num_sublists; i++) {
3824		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
3825		multilist_sublist_remove(mls, markers[i]);
3826		multilist_sublist_unlock(mls);
3827
3828		kmem_cache_free(hdr_full_cache, markers[i]);
3829	}
3830	kmem_free(markers, sizeof (*markers) * num_sublists);
3831
3832	return (total_evicted);
3833}
3834
3835/*
3836 * Flush all "evictable" data of the given type from the arc state
3837 * specified. This will not evict any "active" buffers (i.e. referenced).
3838 *
3839 * When 'retry' is set to B_FALSE, the function will make a single pass
3840 * over the state and evict any buffers that it can. Since it doesn't
3841 * continually retry the eviction, it might end up leaving some buffers
3842 * in the ARC due to lock misses.
3843 *
3844 * When 'retry' is set to B_TRUE, the function will continually retry the
3845 * eviction until *all* evictable buffers have been removed from the
3846 * state. As a result, if concurrent insertions into the state are
3847 * allowed (e.g. if the ARC isn't shutting down), this function might
3848 * wind up in an infinite loop, continually trying to evict buffers.
3849 */
3850static uint64_t
3851arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
3852    boolean_t retry)
3853{
3854	uint64_t evicted = 0;
3855
3856	while (refcount_count(&state->arcs_esize[type]) != 0) {
3857		evicted += arc_evict_state(state, spa, ARC_EVICT_ALL, type);
3858
3859		if (!retry)
3860			break;
3861	}
3862
3863	return (evicted);
3864}
3865
3866/*
3867 * Evict the specified number of bytes from the state specified,
3868 * restricting eviction to the spa and type given. This function
3869 * prevents us from trying to evict more from a state's list than
3870 * is "evictable", and to skip evicting altogether when passed a
3871 * negative value for "bytes". In contrast, arc_evict_state() will
3872 * evict everything it can, when passed a negative value for "bytes".
3873 */
3874static uint64_t
3875arc_adjust_impl(arc_state_t *state, uint64_t spa, int64_t bytes,
3876    arc_buf_contents_t type)
3877{
3878	int64_t delta;
3879
3880	if (bytes > 0 && refcount_count(&state->arcs_esize[type]) > 0) {
3881		delta = MIN(refcount_count(&state->arcs_esize[type]), bytes);
3882		return (arc_evict_state(state, spa, delta, type));
3883	}
3884
3885	return (0);
3886}
3887
3888/*
3889 * Evict metadata buffers from the cache, such that arc_meta_used is
3890 * capped by the arc_meta_limit tunable.
3891 */
3892static uint64_t
3893arc_adjust_meta(void)
3894{
3895	uint64_t total_evicted = 0;
3896	int64_t target;
3897
3898	/*
3899	 * If we're over the meta limit, we want to evict enough
3900	 * metadata to get back under the meta limit. We don't want to
3901	 * evict so much that we drop the MRU below arc_p, though. If
3902	 * we're over the meta limit more than we're over arc_p, we
3903	 * evict some from the MRU here, and some from the MFU below.
3904	 */
3905	target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
3906	    (int64_t)(refcount_count(&arc_anon->arcs_size) +
3907	    refcount_count(&arc_mru->arcs_size) - arc_p));
3908
3909	total_evicted += arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
3910
3911	/*
3912	 * Similar to the above, we want to evict enough bytes to get us
3913	 * below the meta limit, but not so much as to drop us below the
3914	 * space allotted to the MFU (which is defined as arc_c - arc_p).
3915	 */
3916	target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
3917	    (int64_t)(refcount_count(&arc_mfu->arcs_size) - (arc_c - arc_p)));
3918
3919	total_evicted += arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
3920
3921	return (total_evicted);
3922}
3923
3924/*
3925 * Return the type of the oldest buffer in the given arc state
3926 *
3927 * This function will select a random sublist of type ARC_BUFC_DATA and
3928 * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist
3929 * is compared, and the type which contains the "older" buffer will be
3930 * returned.
3931 */
3932static arc_buf_contents_t
3933arc_adjust_type(arc_state_t *state)
3934{
3935	multilist_t *data_ml = state->arcs_list[ARC_BUFC_DATA];
3936	multilist_t *meta_ml = state->arcs_list[ARC_BUFC_METADATA];
3937	int data_idx = multilist_get_random_index(data_ml);
3938	int meta_idx = multilist_get_random_index(meta_ml);
3939	multilist_sublist_t *data_mls;
3940	multilist_sublist_t *meta_mls;
3941	arc_buf_contents_t type;
3942	arc_buf_hdr_t *data_hdr;
3943	arc_buf_hdr_t *meta_hdr;
3944
3945	/*
3946	 * We keep the sublist lock until we're finished, to prevent
3947	 * the headers from being destroyed via arc_evict_state().
3948	 */
3949	data_mls = multilist_sublist_lock(data_ml, data_idx);
3950	meta_mls = multilist_sublist_lock(meta_ml, meta_idx);
3951
3952	/*
3953	 * These two loops are to ensure we skip any markers that
3954	 * might be at the tail of the lists due to arc_evict_state().
3955	 */
3956
3957	for (data_hdr = multilist_sublist_tail(data_mls); data_hdr != NULL;
3958	    data_hdr = multilist_sublist_prev(data_mls, data_hdr)) {
3959		if (data_hdr->b_spa != 0)
3960			break;
3961	}
3962
3963	for (meta_hdr = multilist_sublist_tail(meta_mls); meta_hdr != NULL;
3964	    meta_hdr = multilist_sublist_prev(meta_mls, meta_hdr)) {
3965		if (meta_hdr->b_spa != 0)
3966			break;
3967	}
3968
3969	if (data_hdr == NULL && meta_hdr == NULL) {
3970		type = ARC_BUFC_DATA;
3971	} else if (data_hdr == NULL) {
3972		ASSERT3P(meta_hdr, !=, NULL);
3973		type = ARC_BUFC_METADATA;
3974	} else if (meta_hdr == NULL) {
3975		ASSERT3P(data_hdr, !=, NULL);
3976		type = ARC_BUFC_DATA;
3977	} else {
3978		ASSERT3P(data_hdr, !=, NULL);
3979		ASSERT3P(meta_hdr, !=, NULL);
3980
3981		/* The headers can't be on the sublist without an L1 header */
3982		ASSERT(HDR_HAS_L1HDR(data_hdr));
3983		ASSERT(HDR_HAS_L1HDR(meta_hdr));
3984
3985		if (data_hdr->b_l1hdr.b_arc_access <
3986		    meta_hdr->b_l1hdr.b_arc_access) {
3987			type = ARC_BUFC_DATA;
3988		} else {
3989			type = ARC_BUFC_METADATA;
3990		}
3991	}
3992
3993	multilist_sublist_unlock(meta_mls);
3994	multilist_sublist_unlock(data_mls);
3995
3996	return (type);
3997}
3998
3999/*
4000 * Evict buffers from the cache, such that arc_size is capped by arc_c.
4001 */
4002static uint64_t
4003arc_adjust(void)
4004{
4005	uint64_t total_evicted = 0;
4006	uint64_t bytes;
4007	int64_t target;
4008
4009	/*
4010	 * If we're over arc_meta_limit, we want to correct that before
4011	 * potentially evicting data buffers below.
4012	 */
4013	total_evicted += arc_adjust_meta();
4014
4015	/*
4016	 * Adjust MRU size
4017	 *
4018	 * If we're over the target cache size, we want to evict enough
4019	 * from the list to get back to our target size. We don't want
4020	 * to evict too much from the MRU, such that it drops below
4021	 * arc_p. So, if we're over our target cache size more than
4022	 * the MRU is over arc_p, we'll evict enough to get back to
4023	 * arc_p here, and then evict more from the MFU below.
4024	 */
4025	target = MIN((int64_t)(arc_size - arc_c),
4026	    (int64_t)(refcount_count(&arc_anon->arcs_size) +
4027	    refcount_count(&arc_mru->arcs_size) + arc_meta_used - arc_p));
4028
4029	/*
4030	 * If we're below arc_meta_min, always prefer to evict data.
4031	 * Otherwise, try to satisfy the requested number of bytes to
4032	 * evict from the type which contains older buffers; in an
4033	 * effort to keep newer buffers in the cache regardless of their
4034	 * type. If we cannot satisfy the number of bytes from this
4035	 * type, spill over into the next type.
4036	 */
4037	if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
4038	    arc_meta_used > arc_meta_min) {
4039		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
4040		total_evicted += bytes;
4041
4042		/*
4043		 * If we couldn't evict our target number of bytes from
4044		 * metadata, we try to get the rest from data.
4045		 */
4046		target -= bytes;
4047
4048		total_evicted +=
4049		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
4050	} else {
4051		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
4052		total_evicted += bytes;
4053
4054		/*
4055		 * If we couldn't evict our target number of bytes from
4056		 * data, we try to get the rest from metadata.
4057		 */
4058		target -= bytes;
4059
4060		total_evicted +=
4061		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
4062	}
4063
4064	/*
4065	 * Adjust MFU size
4066	 *
4067	 * Now that we've tried to evict enough from the MRU to get its
4068	 * size back to arc_p, if we're still above the target cache
4069	 * size, we evict the rest from the MFU.
4070	 */
4071	target = arc_size - arc_c;
4072
4073	if (arc_adjust_type(arc_mfu) == ARC_BUFC_METADATA &&
4074	    arc_meta_used > arc_meta_min) {
4075		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
4076		total_evicted += bytes;
4077
4078		/*
4079		 * If we couldn't evict our target number of bytes from
4080		 * metadata, we try to get the rest from data.
4081		 */
4082		target -= bytes;
4083
4084		total_evicted +=
4085		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
4086	} else {
4087		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
4088		total_evicted += bytes;
4089
4090		/*
4091		 * If we couldn't evict our target number of bytes from
4092		 * data, we try to get the rest from data.
4093		 */
4094		target -= bytes;
4095
4096		total_evicted +=
4097		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
4098	}
4099
4100	/*
4101	 * Adjust ghost lists
4102	 *
4103	 * In addition to the above, the ARC also defines target values
4104	 * for the ghost lists. The sum of the mru list and mru ghost
4105	 * list should never exceed the target size of the cache, and
4106	 * the sum of the mru list, mfu list, mru ghost list, and mfu
4107	 * ghost list should never exceed twice the target size of the
4108	 * cache. The following logic enforces these limits on the ghost
4109	 * caches, and evicts from them as needed.
4110	 */
4111	target = refcount_count(&arc_mru->arcs_size) +
4112	    refcount_count(&arc_mru_ghost->arcs_size) - arc_c;
4113
4114	bytes = arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_DATA);
4115	total_evicted += bytes;
4116
4117	target -= bytes;
4118
4119	total_evicted +=
4120	    arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_METADATA);
4121
4122	/*
4123	 * We assume the sum of the mru list and mfu list is less than
4124	 * or equal to arc_c (we enforced this above), which means we
4125	 * can use the simpler of the two equations below:
4126	 *
4127	 *	mru + mfu + mru ghost + mfu ghost <= 2 * arc_c
4128	 *		    mru ghost + mfu ghost <= arc_c
4129	 */
4130	target = refcount_count(&arc_mru_ghost->arcs_size) +
4131	    refcount_count(&arc_mfu_ghost->arcs_size) - arc_c;
4132
4133	bytes = arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_DATA);
4134	total_evicted += bytes;
4135
4136	target -= bytes;
4137
4138	total_evicted +=
4139	    arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_METADATA);
4140
4141	return (total_evicted);
4142}
4143
4144void
4145arc_flush(spa_t *spa, boolean_t retry)
4146{
4147	uint64_t guid = 0;
4148
4149	/*
4150	 * If retry is B_TRUE, a spa must not be specified since we have
4151	 * no good way to determine if all of a spa's buffers have been
4152	 * evicted from an arc state.
4153	 */
4154	ASSERT(!retry || spa == 0);
4155
4156	if (spa != NULL)
4157		guid = spa_load_guid(spa);
4158
4159	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
4160	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);
4161
4162	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
4163	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);
4164
4165	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
4166	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);
4167
4168	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
4169	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);
4170}
4171
4172void
4173arc_shrink(int64_t to_free)
4174{
4175	if (arc_c > arc_c_min) {
4176		DTRACE_PROBE4(arc__shrink, uint64_t, arc_c, uint64_t,
4177			arc_c_min, uint64_t, arc_p, uint64_t, to_free);
4178		if (arc_c > arc_c_min + to_free)
4179			atomic_add_64(&arc_c, -to_free);
4180		else
4181			arc_c = arc_c_min;
4182
4183		atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
4184		if (arc_c > arc_size)
4185			arc_c = MAX(arc_size, arc_c_min);
4186		if (arc_p > arc_c)
4187			arc_p = (arc_c >> 1);
4188
4189		DTRACE_PROBE2(arc__shrunk, uint64_t, arc_c, uint64_t,
4190			arc_p);
4191
4192		ASSERT(arc_c >= arc_c_min);
4193		ASSERT((int64_t)arc_p >= 0);
4194	}
4195
4196	if (arc_size > arc_c) {
4197		DTRACE_PROBE2(arc__shrink_adjust, uint64_t, arc_size,
4198			uint64_t, arc_c);
4199		(void) arc_adjust();
4200	}
4201}
4202
4203typedef enum free_memory_reason_t {
4204	FMR_UNKNOWN,
4205	FMR_NEEDFREE,
4206	FMR_LOTSFREE,
4207	FMR_SWAPFS_MINFREE,
4208	FMR_PAGES_PP_MAXIMUM,
4209	FMR_HEAP_ARENA,
4210	FMR_ZIO_ARENA,
4211} free_memory_reason_t;
4212
4213int64_t last_free_memory;
4214free_memory_reason_t last_free_reason;
4215
4216/*
4217 * Additional reserve of pages for pp_reserve.
4218 */
4219int64_t arc_pages_pp_reserve = 64;
4220
4221/*
4222 * Additional reserve of pages for swapfs.
4223 */
4224int64_t arc_swapfs_reserve = 64;
4225
4226/*
4227 * Return the amount of memory that can be consumed before reclaim will be
4228 * needed.  Positive if there is sufficient free memory, negative indicates
4229 * the amount of memory that needs to be freed up.
4230 */
4231static int64_t
4232arc_available_memory(void)
4233{
4234	int64_t lowest = INT64_MAX;
4235	int64_t n;
4236	free_memory_reason_t r = FMR_UNKNOWN;
4237
4238#ifdef _KERNEL
4239#ifdef __FreeBSD__
4240	/*
4241	 * Cooperate with pagedaemon when it's time for it to scan
4242	 * and reclaim some pages.
4243	 */
4244	n = PAGESIZE * ((int64_t)freemem - zfs_arc_free_target);
4245	if (n < lowest) {
4246		lowest = n;
4247		r = FMR_LOTSFREE;
4248	}
4249
4250#else
4251	if (needfree > 0) {
4252		n = PAGESIZE * (-needfree);
4253		if (n < lowest) {
4254			lowest = n;
4255			r = FMR_NEEDFREE;
4256		}
4257	}
4258
4259	/*
4260	 * check that we're out of range of the pageout scanner.  It starts to
4261	 * schedule paging if freemem is less than lotsfree and needfree.
4262	 * lotsfree is the high-water mark for pageout, and needfree is the
4263	 * number of needed free pages.  We add extra pages here to make sure
4264	 * the scanner doesn't start up while we're freeing memory.
4265	 */
4266	n = PAGESIZE * (freemem - lotsfree - needfree - desfree);
4267	if (n < lowest) {
4268		lowest = n;
4269		r = FMR_LOTSFREE;
4270	}
4271
4272	/*
4273	 * check to make sure that swapfs has enough space so that anon
4274	 * reservations can still succeed. anon_resvmem() checks that the
4275	 * availrmem is greater than swapfs_minfree, and the number of reserved
4276	 * swap pages.  We also add a bit of extra here just to prevent
4277	 * circumstances from getting really dire.
4278	 */
4279	n = PAGESIZE * (availrmem - swapfs_minfree - swapfs_reserve -
4280	    desfree - arc_swapfs_reserve);
4281	if (n < lowest) {
4282		lowest = n;
4283		r = FMR_SWAPFS_MINFREE;
4284	}
4285
4286
4287	/*
4288	 * Check that we have enough availrmem that memory locking (e.g., via
4289	 * mlock(3C) or memcntl(2)) can still succeed.  (pages_pp_maximum
4290	 * stores the number of pages that cannot be locked; when availrmem
4291	 * drops below pages_pp_maximum, page locking mechanisms such as
4292	 * page_pp_lock() will fail.)
4293	 */
4294	n = PAGESIZE * (availrmem - pages_pp_maximum -
4295	    arc_pages_pp_reserve);
4296	if (n < lowest) {
4297		lowest = n;
4298		r = FMR_PAGES_PP_MAXIMUM;
4299	}
4300
4301#endif	/* __FreeBSD__ */
4302#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC)
4303	/*
4304	 * If we're on an i386 platform, it's possible that we'll exhaust the
4305	 * kernel heap space before we ever run out of available physical
4306	 * memory.  Most checks of the size of the heap_area compare against
4307	 * tune.t_minarmem, which is the minimum available real memory that we
4308	 * can have in the system.  However, this is generally fixed at 25 pages
4309	 * which is so low that it's useless.  In this comparison, we seek to
4310	 * calculate the total heap-size, and reclaim if more than 3/4ths of the
4311	 * heap is allocated.  (Or, in the calculation, if less than 1/4th is
4312	 * free)
4313	 */
4314	n = uma_avail() - (long)(uma_limit() / 4);
4315	if (n < lowest) {
4316		lowest = n;
4317		r = FMR_HEAP_ARENA;
4318	}
4319#endif
4320
4321	/*
4322	 * If zio data pages are being allocated out of a separate heap segment,
4323	 * then enforce that the size of available vmem for this arena remains
4324	 * above about 1/16th free.
4325	 *
4326	 * Note: The 1/16th arena free requirement was put in place
4327	 * to aggressively evict memory from the arc in order to avoid
4328	 * memory fragmentation issues.
4329	 */
4330	if (zio_arena != NULL) {
4331		n = (int64_t)vmem_size(zio_arena, VMEM_FREE) -
4332		    (vmem_size(zio_arena, VMEM_ALLOC) >> 4);
4333		if (n < lowest) {
4334			lowest = n;
4335			r = FMR_ZIO_ARENA;
4336		}
4337	}
4338
4339#else	/* _KERNEL */
4340	/* Every 100 calls, free a small amount */
4341	if (spa_get_random(100) == 0)
4342		lowest = -1024;
4343#endif	/* _KERNEL */
4344
4345	last_free_memory = lowest;
4346	last_free_reason = r;
4347	DTRACE_PROBE2(arc__available_memory, int64_t, lowest, int, r);
4348	return (lowest);
4349}
4350
4351
4352/*
4353 * Determine if the system is under memory pressure and is asking
4354 * to reclaim memory. A return value of B_TRUE indicates that the system
4355 * is under memory pressure and that the arc should adjust accordingly.
4356 */
4357static boolean_t
4358arc_reclaim_needed(void)
4359{
4360	return (arc_available_memory() < 0);
4361}
4362
4363extern kmem_cache_t	*zio_buf_cache[];
4364extern kmem_cache_t	*zio_data_buf_cache[];
4365extern kmem_cache_t	*range_seg_cache;
4366extern kmem_cache_t	*abd_chunk_cache;
4367
4368static __noinline void
4369arc_kmem_reap_now(void)
4370{
4371	size_t			i;
4372	kmem_cache_t		*prev_cache = NULL;
4373	kmem_cache_t		*prev_data_cache = NULL;
4374
4375	DTRACE_PROBE(arc__kmem_reap_start);
4376#ifdef _KERNEL
4377	if (arc_meta_used >= arc_meta_limit) {
4378		/*
4379		 * We are exceeding our meta-data cache limit.
4380		 * Purge some DNLC entries to release holds on meta-data.
4381		 */
4382		dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
4383	}
4384#if defined(__i386)
4385	/*
4386	 * Reclaim unused memory from all kmem caches.
4387	 */
4388	kmem_reap();
4389#endif
4390#endif
4391
4392	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
4393		if (zio_buf_cache[i] != prev_cache) {
4394			prev_cache = zio_buf_cache[i];
4395			kmem_cache_reap_now(zio_buf_cache[i]);
4396		}
4397		if (zio_data_buf_cache[i] != prev_data_cache) {
4398			prev_data_cache = zio_data_buf_cache[i];
4399			kmem_cache_reap_now(zio_data_buf_cache[i]);
4400		}
4401	}
4402	kmem_cache_reap_now(abd_chunk_cache);
4403	kmem_cache_reap_now(buf_cache);
4404	kmem_cache_reap_now(hdr_full_cache);
4405	kmem_cache_reap_now(hdr_l2only_cache);
4406	kmem_cache_reap_now(range_seg_cache);
4407
4408#ifdef illumos
4409	if (zio_arena != NULL) {
4410		/*
4411		 * Ask the vmem arena to reclaim unused memory from its
4412		 * quantum caches.
4413		 */
4414		vmem_qcache_reap(zio_arena);
4415	}
4416#endif
4417	DTRACE_PROBE(arc__kmem_reap_end);
4418}
4419
4420/*
4421 * Threads can block in arc_get_data_impl() waiting for this thread to evict
4422 * enough data and signal them to proceed. When this happens, the threads in
4423 * arc_get_data_impl() are sleeping while holding the hash lock for their
4424 * particular arc header. Thus, we must be careful to never sleep on a
4425 * hash lock in this thread. This is to prevent the following deadlock:
4426 *
4427 *  - Thread A sleeps on CV in arc_get_data_impl() holding hash lock "L",
4428 *    waiting for the reclaim thread to signal it.
4429 *
4430 *  - arc_reclaim_thread() tries to acquire hash lock "L" using mutex_enter,
4431 *    fails, and goes to sleep forever.
4432 *
4433 * This possible deadlock is avoided by always acquiring a hash lock
4434 * using mutex_tryenter() from arc_reclaim_thread().
4435 */
4436static void
4437arc_reclaim_thread(void *dummy __unused)
4438{
4439	hrtime_t		growtime = 0;
4440	callb_cpr_t		cpr;
4441
4442	CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG);
4443
4444	mutex_enter(&arc_reclaim_lock);
4445	while (!arc_reclaim_thread_exit) {
4446		uint64_t evicted = 0;
4447
4448		/*
4449		 * This is necessary in order for the mdb ::arc dcmd to
4450		 * show up to date information. Since the ::arc command
4451		 * does not call the kstat's update function, without
4452		 * this call, the command may show stale stats for the
4453		 * anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even
4454		 * with this change, the data might be up to 1 second
4455		 * out of date; but that should suffice. The arc_state_t
4456		 * structures can be queried directly if more accurate
4457		 * information is needed.
4458		 */
4459		if (arc_ksp != NULL)
4460			arc_ksp->ks_update(arc_ksp, KSTAT_READ);
4461
4462		mutex_exit(&arc_reclaim_lock);
4463
4464		/*
4465		 * We call arc_adjust() before (possibly) calling
4466		 * arc_kmem_reap_now(), so that we can wake up
4467		 * arc_get_data_impl() sooner.
4468		 */
4469		evicted = arc_adjust();
4470
4471		int64_t free_memory = arc_available_memory();
4472		if (free_memory < 0) {
4473
4474			arc_no_grow = B_TRUE;
4475			arc_warm = B_TRUE;
4476
4477			/*
4478			 * Wait at least zfs_grow_retry (default 60) seconds
4479			 * before considering growing.
4480			 */
4481			growtime = gethrtime() + SEC2NSEC(arc_grow_retry);
4482
4483			arc_kmem_reap_now();
4484
4485			/*
4486			 * If we are still low on memory, shrink the ARC
4487			 * so that we have arc_shrink_min free space.
4488			 */
4489			free_memory = arc_available_memory();
4490
4491			int64_t to_free =
4492			    (arc_c >> arc_shrink_shift) - free_memory;
4493			if (to_free > 0) {
4494#ifdef _KERNEL
4495#ifdef illumos
4496				to_free = MAX(to_free, ptob(needfree));
4497#endif
4498#endif
4499				arc_shrink(to_free);
4500			}
4501		} else if (free_memory < arc_c >> arc_no_grow_shift) {
4502			arc_no_grow = B_TRUE;
4503		} else if (gethrtime() >= growtime) {
4504			arc_no_grow = B_FALSE;
4505		}
4506
4507		mutex_enter(&arc_reclaim_lock);
4508
4509		/*
4510		 * If evicted is zero, we couldn't evict anything via
4511		 * arc_adjust(). This could be due to hash lock
4512		 * collisions, but more likely due to the majority of
4513		 * arc buffers being unevictable. Therefore, even if
4514		 * arc_size is above arc_c, another pass is unlikely to
4515		 * be helpful and could potentially cause us to enter an
4516		 * infinite loop.
4517		 */
4518		if (arc_size <= arc_c || evicted == 0) {
4519			/*
4520			 * We're either no longer overflowing, or we
4521			 * can't evict anything more, so we should wake
4522			 * up any threads before we go to sleep.
4523			 */
4524			cv_broadcast(&arc_reclaim_waiters_cv);
4525
4526			/*
4527			 * Block until signaled, or after one second (we
4528			 * might need to perform arc_kmem_reap_now()
4529			 * even if we aren't being signalled)
4530			 */
4531			CALLB_CPR_SAFE_BEGIN(&cpr);
4532			(void) cv_timedwait_hires(&arc_reclaim_thread_cv,
4533			    &arc_reclaim_lock, SEC2NSEC(1), MSEC2NSEC(1), 0);
4534			CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_lock);
4535		}
4536	}
4537
4538	arc_reclaim_thread_exit = B_FALSE;
4539	cv_broadcast(&arc_reclaim_thread_cv);
4540	CALLB_CPR_EXIT(&cpr);		/* drops arc_reclaim_lock */
4541	thread_exit();
4542}
4543
4544static u_int arc_dnlc_evicts_arg;
4545extern struct vfsops zfs_vfsops;
4546
4547static void
4548arc_dnlc_evicts_thread(void *dummy __unused)
4549{
4550	callb_cpr_t cpr;
4551	u_int percent;
4552
4553	CALLB_CPR_INIT(&cpr, &arc_dnlc_evicts_lock, callb_generic_cpr, FTAG);
4554
4555	mutex_enter(&arc_dnlc_evicts_lock);
4556	while (!arc_dnlc_evicts_thread_exit) {
4557		CALLB_CPR_SAFE_BEGIN(&cpr);
4558		(void) cv_wait(&arc_dnlc_evicts_cv, &arc_dnlc_evicts_lock);
4559		CALLB_CPR_SAFE_END(&cpr, &arc_dnlc_evicts_lock);
4560		if (arc_dnlc_evicts_arg != 0) {
4561			percent = arc_dnlc_evicts_arg;
4562			mutex_exit(&arc_dnlc_evicts_lock);
4563#ifdef _KERNEL
4564			vnlru_free(desiredvnodes * percent / 100, &zfs_vfsops);
4565#endif
4566			mutex_enter(&arc_dnlc_evicts_lock);
4567			/*
4568			 * Clear our token only after vnlru_free()
4569			 * pass is done, to avoid false queueing of
4570			 * the requests.
4571			 */
4572			arc_dnlc_evicts_arg = 0;
4573		}
4574	}
4575	arc_dnlc_evicts_thread_exit = FALSE;
4576	cv_broadcast(&arc_dnlc_evicts_cv);
4577	CALLB_CPR_EXIT(&cpr);
4578	thread_exit();
4579}
4580
4581void
4582dnlc_reduce_cache(void *arg)
4583{
4584	u_int percent;
4585
4586	percent = (u_int)(uintptr_t)arg;
4587	mutex_enter(&arc_dnlc_evicts_lock);
4588	if (arc_dnlc_evicts_arg == 0) {
4589		arc_dnlc_evicts_arg = percent;
4590		cv_broadcast(&arc_dnlc_evicts_cv);
4591	}
4592	mutex_exit(&arc_dnlc_evicts_lock);
4593}
4594
4595/*
4596 * Adapt arc info given the number of bytes we are trying to add and
4597 * the state that we are comming from.  This function is only called
4598 * when we are adding new content to the cache.
4599 */
4600static void
4601arc_adapt(int bytes, arc_state_t *state)
4602{
4603	int mult;
4604	uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
4605	int64_t mrug_size = refcount_count(&arc_mru_ghost->arcs_size);
4606	int64_t mfug_size = refcount_count(&arc_mfu_ghost->arcs_size);
4607
4608	if (state == arc_l2c_only)
4609		return;
4610
4611	ASSERT(bytes > 0);
4612	/*
4613	 * Adapt the target size of the MRU list:
4614	 *	- if we just hit in the MRU ghost list, then increase
4615	 *	  the target size of the MRU list.
4616	 *	- if we just hit in the MFU ghost list, then increase
4617	 *	  the target size of the MFU list by decreasing the
4618	 *	  target size of the MRU list.
4619	 */
4620	if (state == arc_mru_ghost) {
4621		mult = (mrug_size >= mfug_size) ? 1 : (mfug_size / mrug_size);
4622		mult = MIN(mult, 10); /* avoid wild arc_p adjustment */
4623
4624		arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
4625	} else if (state == arc_mfu_ghost) {
4626		uint64_t delta;
4627
4628		mult = (mfug_size >= mrug_size) ? 1 : (mrug_size / mfug_size);
4629		mult = MIN(mult, 10);
4630
4631		delta = MIN(bytes * mult, arc_p);
4632		arc_p = MAX(arc_p_min, arc_p - delta);
4633	}
4634	ASSERT((int64_t)arc_p >= 0);
4635
4636	if (arc_reclaim_needed()) {
4637		cv_signal(&arc_reclaim_thread_cv);
4638		return;
4639	}
4640
4641	if (arc_no_grow)
4642		return;
4643
4644	if (arc_c >= arc_c_max)
4645		return;
4646
4647	/*
4648	 * If we're within (2 * maxblocksize) bytes of the target
4649	 * cache size, increment the target cache size
4650	 */
4651	if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
4652		DTRACE_PROBE1(arc__inc_adapt, int, bytes);
4653		atomic_add_64(&arc_c, (int64_t)bytes);
4654		if (arc_c > arc_c_max)
4655			arc_c = arc_c_max;
4656		else if (state == arc_anon)
4657			atomic_add_64(&arc_p, (int64_t)bytes);
4658		if (arc_p > arc_c)
4659			arc_p = arc_c;
4660	}
4661	ASSERT((int64_t)arc_p >= 0);
4662}
4663
4664/*
4665 * Check if arc_size has grown past our upper threshold, determined by
4666 * zfs_arc_overflow_shift.
4667 */
4668static boolean_t
4669arc_is_overflowing(void)
4670{
4671	/* Always allow at least one block of overflow */
4672	uint64_t overflow = MAX(SPA_MAXBLOCKSIZE,
4673	    arc_c >> zfs_arc_overflow_shift);
4674
4675	return (arc_size >= arc_c + overflow);
4676}
4677
4678static abd_t *
4679arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
4680{
4681	arc_buf_contents_t type = arc_buf_type(hdr);
4682
4683	arc_get_data_impl(hdr, size, tag);
4684	if (type == ARC_BUFC_METADATA) {
4685		return (abd_alloc(size, B_TRUE));
4686	} else {
4687		ASSERT(type == ARC_BUFC_DATA);
4688		return (abd_alloc(size, B_FALSE));
4689	}
4690}
4691
4692static void *
4693arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
4694{
4695	arc_buf_contents_t type = arc_buf_type(hdr);
4696
4697	arc_get_data_impl(hdr, size, tag);
4698	if (type == ARC_BUFC_METADATA) {
4699		return (zio_buf_alloc(size));
4700	} else {
4701		ASSERT(type == ARC_BUFC_DATA);
4702		return (zio_data_buf_alloc(size));
4703	}
4704}
4705
4706/*
4707 * Allocate a block and return it to the caller. If we are hitting the
4708 * hard limit for the cache size, we must sleep, waiting for the eviction
4709 * thread to catch up. If we're past the target size but below the hard
4710 * limit, we'll only signal the reclaim thread and continue on.
4711 */
4712static void
4713arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
4714{
4715	arc_state_t *state = hdr->b_l1hdr.b_state;
4716	arc_buf_contents_t type = arc_buf_type(hdr);
4717
4718	arc_adapt(size, state);
4719
4720	/*
4721	 * If arc_size is currently overflowing, and has grown past our
4722	 * upper limit, we must be adding data faster than the evict
4723	 * thread can evict. Thus, to ensure we don't compound the
4724	 * problem by adding more data and forcing arc_size to grow even
4725	 * further past it's target size, we halt and wait for the
4726	 * eviction thread to catch up.
4727	 *
4728	 * It's also possible that the reclaim thread is unable to evict
4729	 * enough buffers to get arc_size below the overflow limit (e.g.
4730	 * due to buffers being un-evictable, or hash lock collisions).
4731	 * In this case, we want to proceed regardless if we're
4732	 * overflowing; thus we don't use a while loop here.
4733	 */
4734	if (arc_is_overflowing()) {
4735		mutex_enter(&arc_reclaim_lock);
4736
4737		/*
4738		 * Now that we've acquired the lock, we may no longer be
4739		 * over the overflow limit, lets check.
4740		 *
4741		 * We're ignoring the case of spurious wake ups. If that
4742		 * were to happen, it'd let this thread consume an ARC
4743		 * buffer before it should have (i.e. before we're under
4744		 * the overflow limit and were signalled by the reclaim
4745		 * thread). As long as that is a rare occurrence, it
4746		 * shouldn't cause any harm.
4747		 */
4748		if (arc_is_overflowing()) {
4749			cv_signal(&arc_reclaim_thread_cv);
4750			cv_wait(&arc_reclaim_waiters_cv, &arc_reclaim_lock);
4751		}
4752
4753		mutex_exit(&arc_reclaim_lock);
4754	}
4755
4756	VERIFY3U(hdr->b_type, ==, type);
4757	if (type == ARC_BUFC_METADATA) {
4758		arc_space_consume(size, ARC_SPACE_META);
4759	} else {
4760		arc_space_consume(size, ARC_SPACE_DATA);
4761	}
4762
4763	/*
4764	 * Update the state size.  Note that ghost states have a
4765	 * "ghost size" and so don't need to be updated.
4766	 */
4767	if (!GHOST_STATE(state)) {
4768
4769		(void) refcount_add_many(&state->arcs_size, size, tag);
4770
4771		/*
4772		 * If this is reached via arc_read, the link is
4773		 * protected by the hash lock. If reached via
4774		 * arc_buf_alloc, the header should not be accessed by
4775		 * any other thread. And, if reached via arc_read_done,
4776		 * the hash lock will protect it if it's found in the
4777		 * hash table; otherwise no other thread should be
4778		 * trying to [add|remove]_reference it.
4779		 */
4780		if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
4781			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4782			(void) refcount_add_many(&state->arcs_esize[type],
4783			    size, tag);
4784		}
4785
4786		/*
4787		 * If we are growing the cache, and we are adding anonymous
4788		 * data, and we have outgrown arc_p, update arc_p
4789		 */
4790		if (arc_size < arc_c && hdr->b_l1hdr.b_state == arc_anon &&
4791		    (refcount_count(&arc_anon->arcs_size) +
4792		    refcount_count(&arc_mru->arcs_size) > arc_p))
4793			arc_p = MIN(arc_c, arc_p + size);
4794	}
4795	ARCSTAT_BUMP(arcstat_allocated);
4796}
4797
4798static void
4799arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size, void *tag)
4800{
4801	arc_free_data_impl(hdr, size, tag);
4802	abd_free(abd);
4803}
4804
4805static void
4806arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, void *tag)
4807{
4808	arc_buf_contents_t type = arc_buf_type(hdr);
4809
4810	arc_free_data_impl(hdr, size, tag);
4811	if (type == ARC_BUFC_METADATA) {
4812		zio_buf_free(buf, size);
4813	} else {
4814		ASSERT(type == ARC_BUFC_DATA);
4815		zio_data_buf_free(buf, size);
4816	}
4817}
4818
4819/*
4820 * Free the arc data buffer.
4821 */
4822static void
4823arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
4824{
4825	arc_state_t *state = hdr->b_l1hdr.b_state;
4826	arc_buf_contents_t type = arc_buf_type(hdr);
4827
4828	/* protected by hash lock, if in the hash table */
4829	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
4830		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4831		ASSERT(state != arc_anon && state != arc_l2c_only);
4832
4833		(void) refcount_remove_many(&state->arcs_esize[type],
4834		    size, tag);
4835	}
4836	(void) refcount_remove_many(&state->arcs_size, size, tag);
4837
4838	VERIFY3U(hdr->b_type, ==, type);
4839	if (type == ARC_BUFC_METADATA) {
4840		arc_space_return(size, ARC_SPACE_META);
4841	} else {
4842		ASSERT(type == ARC_BUFC_DATA);
4843		arc_space_return(size, ARC_SPACE_DATA);
4844	}
4845}
4846
4847/*
4848 * This routine is called whenever a buffer is accessed.
4849 * NOTE: the hash lock is dropped in this function.
4850 */
4851static void
4852arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
4853{
4854	clock_t now;
4855
4856	ASSERT(MUTEX_HELD(hash_lock));
4857	ASSERT(HDR_HAS_L1HDR(hdr));
4858
4859	if (hdr->b_l1hdr.b_state == arc_anon) {
4860		/*
4861		 * This buffer is not in the cache, and does not
4862		 * appear in our "ghost" list.  Add the new buffer
4863		 * to the MRU state.
4864		 */
4865
4866		ASSERT0(hdr->b_l1hdr.b_arc_access);
4867		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4868		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
4869		arc_change_state(arc_mru, hdr, hash_lock);
4870
4871	} else if (hdr->b_l1hdr.b_state == arc_mru) {
4872		now = ddi_get_lbolt();
4873
4874		/*
4875		 * If this buffer is here because of a prefetch, then either:
4876		 * - clear the flag if this is a "referencing" read
4877		 *   (any subsequent access will bump this into the MFU state).
4878		 * or
4879		 * - move the buffer to the head of the list if this is
4880		 *   another prefetch (to make it less likely to be evicted).
4881		 */
4882		if (HDR_PREFETCH(hdr)) {
4883			if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
4884				/* link protected by hash lock */
4885				ASSERT(multilist_link_active(
4886				    &hdr->b_l1hdr.b_arc_node));
4887			} else {
4888				arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
4889				ARCSTAT_BUMP(arcstat_mru_hits);
4890			}
4891			hdr->b_l1hdr.b_arc_access = now;
4892			return;
4893		}
4894
4895		/*
4896		 * This buffer has been "accessed" only once so far,
4897		 * but it is still in the cache. Move it to the MFU
4898		 * state.
4899		 */
4900		if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) {
4901			/*
4902			 * More than 125ms have passed since we
4903			 * instantiated this buffer.  Move it to the
4904			 * most frequently used state.
4905			 */
4906			hdr->b_l1hdr.b_arc_access = now;
4907			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4908			arc_change_state(arc_mfu, hdr, hash_lock);
4909		}
4910		ARCSTAT_BUMP(arcstat_mru_hits);
4911	} else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
4912		arc_state_t	*new_state;
4913		/*
4914		 * This buffer has been "accessed" recently, but
4915		 * was evicted from the cache.  Move it to the
4916		 * MFU state.
4917		 */
4918
4919		if (HDR_PREFETCH(hdr)) {
4920			new_state = arc_mru;
4921			if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0)
4922				arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
4923			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
4924		} else {
4925			new_state = arc_mfu;
4926			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4927		}
4928
4929		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4930		arc_change_state(new_state, hdr, hash_lock);
4931
4932		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
4933	} else if (hdr->b_l1hdr.b_state == arc_mfu) {
4934		/*
4935		 * This buffer has been accessed more than once and is
4936		 * still in the cache.  Keep it in the MFU state.
4937		 *
4938		 * NOTE: an add_reference() that occurred when we did
4939		 * the arc_read() will have kicked this off the list.
4940		 * If it was a prefetch, we will explicitly move it to
4941		 * the head of the list now.
4942		 */
4943		if ((HDR_PREFETCH(hdr)) != 0) {
4944			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4945			/* link protected by hash_lock */
4946			ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node));
4947		}
4948		ARCSTAT_BUMP(arcstat_mfu_hits);
4949		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4950	} else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
4951		arc_state_t	*new_state = arc_mfu;
4952		/*
4953		 * This buffer has been accessed more than once but has
4954		 * been evicted from the cache.  Move it back to the
4955		 * MFU state.
4956		 */
4957
4958		if (HDR_PREFETCH(hdr)) {
4959			/*
4960			 * This is a prefetch access...
4961			 * move this block back to the MRU state.
4962			 */
4963			ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
4964			new_state = arc_mru;
4965		}
4966
4967		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4968		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4969		arc_change_state(new_state, hdr, hash_lock);
4970
4971		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
4972	} else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
4973		/*
4974		 * This buffer is on the 2nd Level ARC.
4975		 */
4976
4977		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4978		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4979		arc_change_state(arc_mfu, hdr, hash_lock);
4980	} else {
4981		ASSERT(!"invalid arc state");
4982	}
4983}
4984
4985/* a generic arc_done_func_t which you can use */
4986/* ARGSUSED */
4987void
4988arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
4989{
4990	if (zio == NULL || zio->io_error == 0)
4991		bcopy(buf->b_data, arg, arc_buf_size(buf));
4992	arc_buf_destroy(buf, arg);
4993}
4994
4995/* a generic arc_done_func_t */
4996void
4997arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
4998{
4999	arc_buf_t **bufp = arg;
5000	if (zio && zio->io_error) {
5001		arc_buf_destroy(buf, arg);
5002		*bufp = NULL;
5003	} else {
5004		*bufp = buf;
5005		ASSERT(buf->b_data);
5006	}
5007}
5008
5009static void
5010arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)
5011{
5012	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
5013		ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);
5014		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
5015	} else {
5016		if (HDR_COMPRESSION_ENABLED(hdr)) {
5017			ASSERT3U(HDR_GET_COMPRESS(hdr), ==,
5018			    BP_GET_COMPRESS(bp));
5019		}
5020		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
5021		ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp));
5022	}
5023}
5024
5025static void
5026arc_read_done(zio_t *zio)
5027{
5028	arc_buf_hdr_t	*hdr = zio->io_private;
5029	kmutex_t	*hash_lock = NULL;
5030	arc_callback_t	*callback_list;
5031	arc_callback_t	*acb;
5032	boolean_t	freeable = B_FALSE;
5033	boolean_t	no_zio_error = (zio->io_error == 0);
5034
5035	/*
5036	 * The hdr was inserted into hash-table and removed from lists
5037	 * prior to starting I/O.  We should find this header, since
5038	 * it's in the hash table, and it should be legit since it's
5039	 * not possible to evict it during the I/O.  The only possible
5040	 * reason for it not to be found is if we were freed during the
5041	 * read.
5042	 */
5043	if (HDR_IN_HASH_TABLE(hdr)) {
5044		ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp));
5045		ASSERT3U(hdr->b_dva.dva_word[0], ==,
5046		    BP_IDENTITY(zio->io_bp)->dva_word[0]);
5047		ASSERT3U(hdr->b_dva.dva_word[1], ==,
5048		    BP_IDENTITY(zio->io_bp)->dva_word[1]);
5049
5050		arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp,
5051		    &hash_lock);
5052
5053		ASSERT((found == hdr &&
5054		    DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
5055		    (found == hdr && HDR_L2_READING(hdr)));
5056		ASSERT3P(hash_lock, !=, NULL);
5057	}
5058
5059	if (no_zio_error) {
5060		/* byteswap if necessary */
5061		if (BP_SHOULD_BYTESWAP(zio->io_bp)) {
5062			if (BP_GET_LEVEL(zio->io_bp) > 0) {
5063				hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
5064			} else {
5065				hdr->b_l1hdr.b_byteswap =
5066				    DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
5067			}
5068		} else {
5069			hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
5070		}
5071	}
5072
5073	arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
5074	if (l2arc_noprefetch && HDR_PREFETCH(hdr))
5075		arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);
5076
5077	callback_list = hdr->b_l1hdr.b_acb;
5078	ASSERT3P(callback_list, !=, NULL);
5079
5080	if (hash_lock && no_zio_error && hdr->b_l1hdr.b_state == arc_anon) {
5081		/*
5082		 * Only call arc_access on anonymous buffers.  This is because
5083		 * if we've issued an I/O for an evicted buffer, we've already
5084		 * called arc_access (to prevent any simultaneous readers from
5085		 * getting confused).
5086		 */
5087		arc_access(hdr, hash_lock);
5088	}
5089
5090	/*
5091	 * If a read request has a callback (i.e. acb_done is not NULL), then we
5092	 * make a buf containing the data according to the parameters which were
5093	 * passed in. The implementation of arc_buf_alloc_impl() ensures that we
5094	 * aren't needlessly decompressing the data multiple times.
5095	 */
5096	int callback_cnt = 0;
5097	for (acb = callback_list; acb != NULL; acb = acb->acb_next) {
5098		if (!acb->acb_done)
5099			continue;
5100
5101		/* This is a demand read since prefetches don't use callbacks */
5102		callback_cnt++;
5103
5104		int error = arc_buf_alloc_impl(hdr, acb->acb_private,
5105		    acb->acb_compressed, no_zio_error, &acb->acb_buf);
5106		if (no_zio_error) {
5107			zio->io_error = error;
5108		}
5109	}
5110	hdr->b_l1hdr.b_acb = NULL;
5111	arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5112	if (callback_cnt == 0) {
5113		ASSERT(HDR_PREFETCH(hdr));
5114		ASSERT0(hdr->b_l1hdr.b_bufcnt);
5115		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5116	}
5117
5118	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
5119	    callback_list != NULL);
5120
5121	if (no_zio_error) {
5122		arc_hdr_verify(hdr, zio->io_bp);
5123	} else {
5124		arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
5125		if (hdr->b_l1hdr.b_state != arc_anon)
5126			arc_change_state(arc_anon, hdr, hash_lock);
5127		if (HDR_IN_HASH_TABLE(hdr))
5128			buf_hash_remove(hdr);
5129		freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
5130	}
5131
5132	/*
5133	 * Broadcast before we drop the hash_lock to avoid the possibility
5134	 * that the hdr (and hence the cv) might be freed before we get to
5135	 * the cv_broadcast().
5136	 */
5137	cv_broadcast(&hdr->b_l1hdr.b_cv);
5138
5139	if (hash_lock != NULL) {
5140		mutex_exit(hash_lock);
5141	} else {
5142		/*
5143		 * This block was freed while we waited for the read to
5144		 * complete.  It has been removed from the hash table and
5145		 * moved to the anonymous state (so that it won't show up
5146		 * in the cache).
5147		 */
5148		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
5149		freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
5150	}
5151
5152	/* execute each callback and free its structure */
5153	while ((acb = callback_list) != NULL) {
5154		if (acb->acb_done)
5155			acb->acb_done(zio, acb->acb_buf, acb->acb_private);
5156
5157		if (acb->acb_zio_dummy != NULL) {
5158			acb->acb_zio_dummy->io_error = zio->io_error;
5159			zio_nowait(acb->acb_zio_dummy);
5160		}
5161
5162		callback_list = acb->acb_next;
5163		kmem_free(acb, sizeof (arc_callback_t));
5164	}
5165
5166	if (freeable)
5167		arc_hdr_destroy(hdr);
5168}
5169
5170/*
5171 * "Read" the block at the specified DVA (in bp) via the
5172 * cache.  If the block is found in the cache, invoke the provided
5173 * callback immediately and return.  Note that the `zio' parameter
5174 * in the callback will be NULL in this case, since no IO was
5175 * required.  If the block is not in the cache pass the read request
5176 * on to the spa with a substitute callback function, so that the
5177 * requested block will be added to the cache.
5178 *
5179 * If a read request arrives for a block that has a read in-progress,
5180 * either wait for the in-progress read to complete (and return the
5181 * results); or, if this is a read with a "done" func, add a record
5182 * to the read to invoke the "done" func when the read completes,
5183 * and return; or just return.
5184 *
5185 * arc_read_done() will invoke all the requested "done" functions
5186 * for readers of this block.
5187 */
5188int
5189arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done,
5190    void *private, zio_priority_t priority, int zio_flags,
5191    arc_flags_t *arc_flags, const zbookmark_phys_t *zb)
5192{
5193	arc_buf_hdr_t *hdr = NULL;
5194	kmutex_t *hash_lock = NULL;
5195	zio_t *rzio;
5196	uint64_t guid = spa_load_guid(spa);
5197	boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW) != 0;
5198
5199	ASSERT(!BP_IS_EMBEDDED(bp) ||
5200	    BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
5201
5202top:
5203	if (!BP_IS_EMBEDDED(bp)) {
5204		/*
5205		 * Embedded BP's have no DVA and require no I/O to "read".
5206		 * Create an anonymous arc buf to back it.
5207		 */
5208		hdr = buf_hash_find(guid, bp, &hash_lock);
5209	}
5210
5211	if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_pabd != NULL) {
5212		arc_buf_t *buf = NULL;
5213		*arc_flags |= ARC_FLAG_CACHED;
5214
5215		if (HDR_IO_IN_PROGRESS(hdr)) {
5216
5217			if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&
5218			    priority == ZIO_PRIORITY_SYNC_READ) {
5219				/*
5220				 * This sync read must wait for an
5221				 * in-progress async read (e.g. a predictive
5222				 * prefetch).  Async reads are queued
5223				 * separately at the vdev_queue layer, so
5224				 * this is a form of priority inversion.
5225				 * Ideally, we would "inherit" the demand
5226				 * i/o's priority by moving the i/o from
5227				 * the async queue to the synchronous queue,
5228				 * but there is currently no mechanism to do
5229				 * so.  Track this so that we can evaluate
5230				 * the magnitude of this potential performance
5231				 * problem.
5232				 *
5233				 * Note that if the prefetch i/o is already
5234				 * active (has been issued to the device),
5235				 * the prefetch improved performance, because
5236				 * we issued it sooner than we would have
5237				 * without the prefetch.
5238				 */
5239				DTRACE_PROBE1(arc__sync__wait__for__async,
5240				    arc_buf_hdr_t *, hdr);
5241				ARCSTAT_BUMP(arcstat_sync_wait_for_async);
5242			}
5243			if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) {
5244				arc_hdr_clear_flags(hdr,
5245				    ARC_FLAG_PREDICTIVE_PREFETCH);
5246			}
5247
5248			if (*arc_flags & ARC_FLAG_WAIT) {
5249				cv_wait(&hdr->b_l1hdr.b_cv, hash_lock);
5250				mutex_exit(hash_lock);
5251				goto top;
5252			}
5253			ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
5254
5255			if (done) {
5256				arc_callback_t *acb = NULL;
5257
5258				acb = kmem_zalloc(sizeof (arc_callback_t),
5259				    KM_SLEEP);
5260				acb->acb_done = done;
5261				acb->acb_private = private;
5262				acb->acb_compressed = compressed_read;
5263				if (pio != NULL)
5264					acb->acb_zio_dummy = zio_null(pio,
5265					    spa, NULL, NULL, NULL, zio_flags);
5266
5267				ASSERT3P(acb->acb_done, !=, NULL);
5268				acb->acb_next = hdr->b_l1hdr.b_acb;
5269				hdr->b_l1hdr.b_acb = acb;
5270				mutex_exit(hash_lock);
5271				return (0);
5272			}
5273			mutex_exit(hash_lock);
5274			return (0);
5275		}
5276
5277		ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5278		    hdr->b_l1hdr.b_state == arc_mfu);
5279
5280		if (done) {
5281			if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) {
5282				/*
5283				 * This is a demand read which does not have to
5284				 * wait for i/o because we did a predictive
5285				 * prefetch i/o for it, which has completed.
5286				 */
5287				DTRACE_PROBE1(
5288				    arc__demand__hit__predictive__prefetch,
5289				    arc_buf_hdr_t *, hdr);
5290				ARCSTAT_BUMP(
5291				    arcstat_demand_hit_predictive_prefetch);
5292				arc_hdr_clear_flags(hdr,
5293				    ARC_FLAG_PREDICTIVE_PREFETCH);
5294			}
5295			ASSERT(!BP_IS_EMBEDDED(bp) || !BP_IS_HOLE(bp));
5296
5297			/* Get a buf with the desired data in it. */
5298			VERIFY0(arc_buf_alloc_impl(hdr, private,
5299			    compressed_read, B_TRUE, &buf));
5300		} else if (*arc_flags & ARC_FLAG_PREFETCH &&
5301		    refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
5302			arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
5303		}
5304		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
5305		arc_access(hdr, hash_lock);
5306		if (*arc_flags & ARC_FLAG_L2CACHE)
5307			arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5308		mutex_exit(hash_lock);
5309		ARCSTAT_BUMP(arcstat_hits);
5310		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
5311		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
5312		    data, metadata, hits);
5313
5314		if (done)
5315			done(NULL, buf, private);
5316	} else {
5317		uint64_t lsize = BP_GET_LSIZE(bp);
5318		uint64_t psize = BP_GET_PSIZE(bp);
5319		arc_callback_t *acb;
5320		vdev_t *vd = NULL;
5321		uint64_t addr = 0;
5322		boolean_t devw = B_FALSE;
5323		uint64_t size;
5324
5325		if (hdr == NULL) {
5326			/* this block is not in the cache */
5327			arc_buf_hdr_t *exists = NULL;
5328			arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
5329			hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
5330			    BP_GET_COMPRESS(bp), type);
5331
5332			if (!BP_IS_EMBEDDED(bp)) {
5333				hdr->b_dva = *BP_IDENTITY(bp);
5334				hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
5335				exists = buf_hash_insert(hdr, &hash_lock);
5336			}
5337			if (exists != NULL) {
5338				/* somebody beat us to the hash insert */
5339				mutex_exit(hash_lock);
5340				buf_discard_identity(hdr);
5341				arc_hdr_destroy(hdr);
5342				goto top; /* restart the IO request */
5343			}
5344		} else {
5345			/*
5346			 * This block is in the ghost cache. If it was L2-only
5347			 * (and thus didn't have an L1 hdr), we realloc the
5348			 * header to add an L1 hdr.
5349			 */
5350			if (!HDR_HAS_L1HDR(hdr)) {
5351				hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
5352				    hdr_full_cache);
5353			}
5354			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
5355			ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
5356			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5357			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5358			ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
5359			ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
5360
5361			/*
5362			 * This is a delicate dance that we play here.
5363			 * This hdr is in the ghost list so we access it
5364			 * to move it out of the ghost list before we
5365			 * initiate the read. If it's a prefetch then
5366			 * it won't have a callback so we'll remove the
5367			 * reference that arc_buf_alloc_impl() created. We
5368			 * do this after we've called arc_access() to
5369			 * avoid hitting an assert in remove_reference().
5370			 */
5371			arc_access(hdr, hash_lock);
5372			arc_hdr_alloc_pabd(hdr);
5373		}
5374		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5375		size = arc_hdr_size(hdr);
5376
5377		/*
5378		 * If compression is enabled on the hdr, then will do
5379		 * RAW I/O and will store the compressed data in the hdr's
5380		 * data block. Otherwise, the hdr's data block will contain
5381		 * the uncompressed data.
5382		 */
5383		if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) {
5384			zio_flags |= ZIO_FLAG_RAW;
5385		}
5386
5387		if (*arc_flags & ARC_FLAG_PREFETCH)
5388			arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
5389		if (*arc_flags & ARC_FLAG_L2CACHE)
5390			arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5391		if (BP_GET_LEVEL(bp) > 0)
5392			arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT);
5393		if (*arc_flags & ARC_FLAG_PREDICTIVE_PREFETCH)
5394			arc_hdr_set_flags(hdr, ARC_FLAG_PREDICTIVE_PREFETCH);
5395		ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
5396
5397		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
5398		acb->acb_done = done;
5399		acb->acb_private = private;
5400		acb->acb_compressed = compressed_read;
5401
5402		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
5403		hdr->b_l1hdr.b_acb = acb;
5404		arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5405
5406		if (HDR_HAS_L2HDR(hdr) &&
5407		    (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
5408			devw = hdr->b_l2hdr.b_dev->l2ad_writing;
5409			addr = hdr->b_l2hdr.b_daddr;
5410			/*
5411			 * Lock out device removal.
5412			 */
5413			if (vdev_is_dead(vd) ||
5414			    !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
5415				vd = NULL;
5416		}
5417
5418		if (priority == ZIO_PRIORITY_ASYNC_READ)
5419			arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
5420		else
5421			arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
5422
5423		if (hash_lock != NULL)
5424			mutex_exit(hash_lock);
5425
5426		/*
5427		 * At this point, we have a level 1 cache miss.  Try again in
5428		 * L2ARC if possible.
5429		 */
5430		ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize);
5431
5432		DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
5433		    uint64_t, lsize, zbookmark_phys_t *, zb);
5434		ARCSTAT_BUMP(arcstat_misses);
5435		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
5436		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
5437		    data, metadata, misses);
5438#ifdef _KERNEL
5439#ifdef RACCT
5440		if (racct_enable) {
5441			PROC_LOCK(curproc);
5442			racct_add_force(curproc, RACCT_READBPS, size);
5443			racct_add_force(curproc, RACCT_READIOPS, 1);
5444			PROC_UNLOCK(curproc);
5445		}
5446#endif /* RACCT */
5447		curthread->td_ru.ru_inblock++;
5448#endif
5449
5450		if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
5451			/*
5452			 * Read from the L2ARC if the following are true:
5453			 * 1. The L2ARC vdev was previously cached.
5454			 * 2. This buffer still has L2ARC metadata.
5455			 * 3. This buffer isn't currently writing to the L2ARC.
5456			 * 4. The L2ARC entry wasn't evicted, which may
5457			 *    also have invalidated the vdev.
5458			 * 5. This isn't prefetch and l2arc_noprefetch is set.
5459			 */
5460			if (HDR_HAS_L2HDR(hdr) &&
5461			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
5462			    !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
5463				l2arc_read_callback_t *cb;
5464				abd_t *abd;
5465				uint64_t asize;
5466
5467				DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
5468				ARCSTAT_BUMP(arcstat_l2_hits);
5469
5470				cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
5471				    KM_SLEEP);
5472				cb->l2rcb_hdr = hdr;
5473				cb->l2rcb_bp = *bp;
5474				cb->l2rcb_zb = *zb;
5475				cb->l2rcb_flags = zio_flags;
5476
5477				asize = vdev_psize_to_asize(vd, size);
5478				if (asize != size) {
5479					abd = abd_alloc_for_io(asize,
5480					    HDR_ISTYPE_METADATA(hdr));
5481					cb->l2rcb_abd = abd;
5482				} else {
5483					abd = hdr->b_l1hdr.b_pabd;
5484				}
5485
5486				ASSERT(addr >= VDEV_LABEL_START_SIZE &&
5487				    addr + asize <= vd->vdev_psize -
5488				    VDEV_LABEL_END_SIZE);
5489
5490				/*
5491				 * l2arc read.  The SCL_L2ARC lock will be
5492				 * released by l2arc_read_done().
5493				 * Issue a null zio if the underlying buffer
5494				 * was squashed to zero size by compression.
5495				 */
5496				ASSERT3U(HDR_GET_COMPRESS(hdr), !=,
5497				    ZIO_COMPRESS_EMPTY);
5498				rzio = zio_read_phys(pio, vd, addr,
5499				    asize, abd,
5500				    ZIO_CHECKSUM_OFF,
5501				    l2arc_read_done, cb, priority,
5502				    zio_flags | ZIO_FLAG_DONT_CACHE |
5503				    ZIO_FLAG_CANFAIL |
5504				    ZIO_FLAG_DONT_PROPAGATE |
5505				    ZIO_FLAG_DONT_RETRY, B_FALSE);
5506				DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
5507				    zio_t *, rzio);
5508				ARCSTAT_INCR(arcstat_l2_read_bytes, size);
5509
5510				if (*arc_flags & ARC_FLAG_NOWAIT) {
5511					zio_nowait(rzio);
5512					return (0);
5513				}
5514
5515				ASSERT(*arc_flags & ARC_FLAG_WAIT);
5516				if (zio_wait(rzio) == 0)
5517					return (0);
5518
5519				/* l2arc read error; goto zio_read() */
5520			} else {
5521				DTRACE_PROBE1(l2arc__miss,
5522				    arc_buf_hdr_t *, hdr);
5523				ARCSTAT_BUMP(arcstat_l2_misses);
5524				if (HDR_L2_WRITING(hdr))
5525					ARCSTAT_BUMP(arcstat_l2_rw_clash);
5526				spa_config_exit(spa, SCL_L2ARC, vd);
5527			}
5528		} else {
5529			if (vd != NULL)
5530				spa_config_exit(spa, SCL_L2ARC, vd);
5531			if (l2arc_ndev != 0) {
5532				DTRACE_PROBE1(l2arc__miss,
5533				    arc_buf_hdr_t *, hdr);
5534				ARCSTAT_BUMP(arcstat_l2_misses);
5535			}
5536		}
5537
5538		rzio = zio_read(pio, spa, bp, hdr->b_l1hdr.b_pabd, size,
5539		    arc_read_done, hdr, priority, zio_flags, zb);
5540
5541		if (*arc_flags & ARC_FLAG_WAIT)
5542			return (zio_wait(rzio));
5543
5544		ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
5545		zio_nowait(rzio);
5546	}
5547	return (0);
5548}
5549
5550/*
5551 * Notify the arc that a block was freed, and thus will never be used again.
5552 */
5553void
5554arc_freed(spa_t *spa, const blkptr_t *bp)
5555{
5556	arc_buf_hdr_t *hdr;
5557	kmutex_t *hash_lock;
5558	uint64_t guid = spa_load_guid(spa);
5559
5560	ASSERT(!BP_IS_EMBEDDED(bp));
5561
5562	hdr = buf_hash_find(guid, bp, &hash_lock);
5563	if (hdr == NULL)
5564		return;
5565
5566	/*
5567	 * We might be trying to free a block that is still doing I/O
5568	 * (i.e. prefetch) or has a reference (i.e. a dedup-ed,
5569	 * dmu_sync-ed block). If this block is being prefetched, then it
5570	 * would still have the ARC_FLAG_IO_IN_PROGRESS flag set on the hdr
5571	 * until the I/O completes. A block may also have a reference if it is
5572	 * part of a dedup-ed, dmu_synced write. The dmu_sync() function would
5573	 * have written the new block to its final resting place on disk but
5574	 * without the dedup flag set. This would have left the hdr in the MRU
5575	 * state and discoverable. When the txg finally syncs it detects that
5576	 * the block was overridden in open context and issues an override I/O.
5577	 * Since this is a dedup block, the override I/O will determine if the
5578	 * block is already in the DDT. If so, then it will replace the io_bp
5579	 * with the bp from the DDT and allow the I/O to finish. When the I/O
5580	 * reaches the done callback, dbuf_write_override_done, it will
5581	 * check to see if the io_bp and io_bp_override are identical.
5582	 * If they are not, then it indicates that the bp was replaced with
5583	 * the bp in the DDT and the override bp is freed. This allows
5584	 * us to arrive here with a reference on a block that is being
5585	 * freed. So if we have an I/O in progress, or a reference to
5586	 * this hdr, then we don't destroy the hdr.
5587	 */
5588	if (!HDR_HAS_L1HDR(hdr) || (!HDR_IO_IN_PROGRESS(hdr) &&
5589	    refcount_is_zero(&hdr->b_l1hdr.b_refcnt))) {
5590		arc_change_state(arc_anon, hdr, hash_lock);
5591		arc_hdr_destroy(hdr);
5592		mutex_exit(hash_lock);
5593	} else {
5594		mutex_exit(hash_lock);
5595	}
5596
5597}
5598
5599/*
5600 * Release this buffer from the cache, making it an anonymous buffer.  This
5601 * must be done after a read and prior to modifying the buffer contents.
5602 * If the buffer has more than one reference, we must make
5603 * a new hdr for the buffer.
5604 */
5605void
5606arc_release(arc_buf_t *buf, void *tag)
5607{
5608	arc_buf_hdr_t *hdr = buf->b_hdr;
5609
5610	/*
5611	 * It would be nice to assert that if it's DMU metadata (level >
5612	 * 0 || it's the dnode file), then it must be syncing context.
5613	 * But we don't know that information at this level.
5614	 */
5615
5616	mutex_enter(&buf->b_evict_lock);
5617
5618	ASSERT(HDR_HAS_L1HDR(hdr));
5619
5620	/*
5621	 * We don't grab the hash lock prior to this check, because if
5622	 * the buffer's header is in the arc_anon state, it won't be
5623	 * linked into the hash table.
5624	 */
5625	if (hdr->b_l1hdr.b_state == arc_anon) {
5626		mutex_exit(&buf->b_evict_lock);
5627		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5628		ASSERT(!HDR_IN_HASH_TABLE(hdr));
5629		ASSERT(!HDR_HAS_L2HDR(hdr));
5630		ASSERT(HDR_EMPTY(hdr));
5631		ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
5632		ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
5633		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
5634
5635		hdr->b_l1hdr.b_arc_access = 0;
5636
5637		/*
5638		 * If the buf is being overridden then it may already
5639		 * have a hdr that is not empty.
5640		 */
5641		buf_discard_identity(hdr);
5642		arc_buf_thaw(buf);
5643
5644		return;
5645	}
5646
5647	kmutex_t *hash_lock = HDR_LOCK(hdr);
5648	mutex_enter(hash_lock);
5649
5650	/*
5651	 * This assignment is only valid as long as the hash_lock is
5652	 * held, we must be careful not to reference state or the
5653	 * b_state field after dropping the lock.
5654	 */
5655	arc_state_t *state = hdr->b_l1hdr.b_state;
5656	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
5657	ASSERT3P(state, !=, arc_anon);
5658
5659	/* this buffer is not on any list */
5660	ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0);
5661
5662	if (HDR_HAS_L2HDR(hdr)) {
5663		mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
5664
5665		/*
5666		 * We have to recheck this conditional again now that
5667		 * we're holding the l2ad_mtx to prevent a race with
5668		 * another thread which might be concurrently calling
5669		 * l2arc_evict(). In that case, l2arc_evict() might have
5670		 * destroyed the header's L2 portion as we were waiting
5671		 * to acquire the l2ad_mtx.
5672		 */
5673		if (HDR_HAS_L2HDR(hdr)) {
5674			l2arc_trim(hdr);
5675			arc_hdr_l2hdr_destroy(hdr);
5676		}
5677
5678		mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
5679	}
5680
5681	/*
5682	 * Do we have more than one buf?
5683	 */
5684	if (hdr->b_l1hdr.b_bufcnt > 1) {
5685		arc_buf_hdr_t *nhdr;
5686		uint64_t spa = hdr->b_spa;
5687		uint64_t psize = HDR_GET_PSIZE(hdr);
5688		uint64_t lsize = HDR_GET_LSIZE(hdr);
5689		enum zio_compress compress = HDR_GET_COMPRESS(hdr);
5690		arc_buf_contents_t type = arc_buf_type(hdr);
5691		VERIFY3U(hdr->b_type, ==, type);
5692
5693		ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
5694		(void) remove_reference(hdr, hash_lock, tag);
5695
5696		if (arc_buf_is_shared(buf) && !ARC_BUF_COMPRESSED(buf)) {
5697			ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
5698			ASSERT(ARC_BUF_LAST(buf));
5699		}
5700
5701		/*
5702		 * Pull the data off of this hdr and attach it to
5703		 * a new anonymous hdr. Also find the last buffer
5704		 * in the hdr's buffer list.
5705		 */
5706		arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
5707		ASSERT3P(lastbuf, !=, NULL);
5708
5709		/*
5710		 * If the current arc_buf_t and the hdr are sharing their data
5711		 * buffer, then we must stop sharing that block.
5712		 */
5713		if (arc_buf_is_shared(buf)) {
5714			VERIFY(!arc_buf_is_shared(lastbuf));
5715
5716			/*
5717			 * First, sever the block sharing relationship between
5718			 * buf and the arc_buf_hdr_t.
5719			 */
5720			arc_unshare_buf(hdr, buf);
5721
5722			/*
5723			 * Now we need to recreate the hdr's b_pabd. Since we
5724			 * have lastbuf handy, we try to share with it, but if
5725			 * we can't then we allocate a new b_pabd and copy the
5726			 * data from buf into it.
5727			 */
5728			if (arc_can_share(hdr, lastbuf)) {
5729				arc_share_buf(hdr, lastbuf);
5730			} else {
5731				arc_hdr_alloc_pabd(hdr);
5732				abd_copy_from_buf(hdr->b_l1hdr.b_pabd,
5733				    buf->b_data, psize);
5734			}
5735			VERIFY3P(lastbuf->b_data, !=, NULL);
5736		} else if (HDR_SHARED_DATA(hdr)) {
5737			/*
5738			 * Uncompressed shared buffers are always at the end
5739			 * of the list. Compressed buffers don't have the
5740			 * same requirements. This makes it hard to
5741			 * simply assert that the lastbuf is shared so
5742			 * we rely on the hdr's compression flags to determine
5743			 * if we have a compressed, shared buffer.
5744			 */
5745			ASSERT(arc_buf_is_shared(lastbuf) ||
5746			    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
5747			ASSERT(!ARC_BUF_SHARED(buf));
5748		}
5749		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5750		ASSERT3P(state, !=, arc_l2c_only);
5751
5752		(void) refcount_remove_many(&state->arcs_size,
5753		    arc_buf_size(buf), buf);
5754
5755		if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
5756			ASSERT3P(state, !=, arc_l2c_only);
5757			(void) refcount_remove_many(&state->arcs_esize[type],
5758			    arc_buf_size(buf), buf);
5759		}
5760
5761		hdr->b_l1hdr.b_bufcnt -= 1;
5762		arc_cksum_verify(buf);
5763#ifdef illumos
5764		arc_buf_unwatch(buf);
5765#endif
5766
5767		mutex_exit(hash_lock);
5768
5769		/*
5770		 * Allocate a new hdr. The new hdr will contain a b_pabd
5771		 * buffer which will be freed in arc_write().
5772		 */
5773		nhdr = arc_hdr_alloc(spa, psize, lsize, compress, type);
5774		ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL);
5775		ASSERT0(nhdr->b_l1hdr.b_bufcnt);
5776		ASSERT0(refcount_count(&nhdr->b_l1hdr.b_refcnt));
5777		VERIFY3U(nhdr->b_type, ==, type);
5778		ASSERT(!HDR_SHARED_DATA(nhdr));
5779
5780		nhdr->b_l1hdr.b_buf = buf;
5781		nhdr->b_l1hdr.b_bufcnt = 1;
5782		(void) refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
5783		buf->b_hdr = nhdr;
5784
5785		mutex_exit(&buf->b_evict_lock);
5786		(void) refcount_add_many(&arc_anon->arcs_size,
5787		    arc_buf_size(buf), buf);
5788	} else {
5789		mutex_exit(&buf->b_evict_lock);
5790		ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
5791		/* protected by hash lock, or hdr is on arc_anon */
5792		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
5793		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5794		arc_change_state(arc_anon, hdr, hash_lock);
5795		hdr->b_l1hdr.b_arc_access = 0;
5796		mutex_exit(hash_lock);
5797
5798		buf_discard_identity(hdr);
5799		arc_buf_thaw(buf);
5800	}
5801}
5802
5803int
5804arc_released(arc_buf_t *buf)
5805{
5806	int released;
5807
5808	mutex_enter(&buf->b_evict_lock);
5809	released = (buf->b_data != NULL &&
5810	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
5811	mutex_exit(&buf->b_evict_lock);
5812	return (released);
5813}
5814
5815#ifdef ZFS_DEBUG
5816int
5817arc_referenced(arc_buf_t *buf)
5818{
5819	int referenced;
5820
5821	mutex_enter(&buf->b_evict_lock);
5822	referenced = (refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
5823	mutex_exit(&buf->b_evict_lock);
5824	return (referenced);
5825}
5826#endif
5827
5828static void
5829arc_write_ready(zio_t *zio)
5830{
5831	arc_write_callback_t *callback = zio->io_private;
5832	arc_buf_t *buf = callback->awcb_buf;
5833	arc_buf_hdr_t *hdr = buf->b_hdr;
5834	uint64_t psize = BP_IS_HOLE(zio->io_bp) ? 0 : BP_GET_PSIZE(zio->io_bp);
5835
5836	ASSERT(HDR_HAS_L1HDR(hdr));
5837	ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
5838	ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
5839
5840	/*
5841	 * If we're reexecuting this zio because the pool suspended, then
5842	 * cleanup any state that was previously set the first time the
5843	 * callback was invoked.
5844	 */
5845	if (zio->io_flags & ZIO_FLAG_REEXECUTED) {
5846		arc_cksum_free(hdr);
5847#ifdef illumos
5848		arc_buf_unwatch(buf);
5849#endif
5850		if (hdr->b_l1hdr.b_pabd != NULL) {
5851			if (arc_buf_is_shared(buf)) {
5852				arc_unshare_buf(hdr, buf);
5853			} else {
5854				arc_hdr_free_pabd(hdr);
5855			}
5856		}
5857	}
5858	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
5859	ASSERT(!HDR_SHARED_DATA(hdr));
5860	ASSERT(!arc_buf_is_shared(buf));
5861
5862	callback->awcb_ready(zio, buf, callback->awcb_private);
5863
5864	if (HDR_IO_IN_PROGRESS(hdr))
5865		ASSERT(zio->io_flags & ZIO_FLAG_REEXECUTED);
5866
5867	arc_cksum_compute(buf);
5868	arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5869
5870	enum zio_compress compress;
5871	if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
5872		compress = ZIO_COMPRESS_OFF;
5873	} else {
5874		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(zio->io_bp));
5875		compress = BP_GET_COMPRESS(zio->io_bp);
5876	}
5877	HDR_SET_PSIZE(hdr, psize);
5878	arc_hdr_set_compress(hdr, compress);
5879
5880
5881	/*
5882	 * Fill the hdr with data. If the hdr is compressed, the data we want
5883	 * is available from the zio, otherwise we can take it from the buf.
5884	 *
5885	 * We might be able to share the buf's data with the hdr here. However,
5886	 * doing so would cause the ARC to be full of linear ABDs if we write a
5887	 * lot of shareable data. As a compromise, we check whether scattered
5888	 * ABDs are allowed, and assume that if they are then the user wants
5889	 * the ARC to be primarily filled with them regardless of the data being
5890	 * written. Therefore, if they're allowed then we allocate one and copy
5891	 * the data into it; otherwise, we share the data directly if we can.
5892	 */
5893	if (zfs_abd_scatter_enabled || !arc_can_share(hdr, buf)) {
5894		arc_hdr_alloc_pabd(hdr);
5895
5896		/*
5897		 * Ideally, we would always copy the io_abd into b_pabd, but the
5898		 * user may have disabled compressed ARC, thus we must check the
5899		 * hdr's compression setting rather than the io_bp's.
5900		 */
5901		if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) {
5902			ASSERT3U(BP_GET_COMPRESS(zio->io_bp), !=,
5903			    ZIO_COMPRESS_OFF);
5904			ASSERT3U(psize, >, 0);
5905
5906			abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize);
5907		} else {
5908			ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr));
5909
5910			abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data,
5911			    arc_buf_size(buf));
5912		}
5913	} else {
5914		ASSERT3P(buf->b_data, ==, abd_to_buf(zio->io_orig_abd));
5915		ASSERT3U(zio->io_orig_size, ==, arc_buf_size(buf));
5916		ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
5917
5918		arc_share_buf(hdr, buf);
5919	}
5920
5921	arc_hdr_verify(hdr, zio->io_bp);
5922}
5923
5924static void
5925arc_write_children_ready(zio_t *zio)
5926{
5927	arc_write_callback_t *callback = zio->io_private;
5928	arc_buf_t *buf = callback->awcb_buf;
5929
5930	callback->awcb_children_ready(zio, buf, callback->awcb_private);
5931}
5932
5933/*
5934 * The SPA calls this callback for each physical write that happens on behalf
5935 * of a logical write.  See the comment in dbuf_write_physdone() for details.
5936 */
5937static void
5938arc_write_physdone(zio_t *zio)
5939{
5940	arc_write_callback_t *cb = zio->io_private;
5941	if (cb->awcb_physdone != NULL)
5942		cb->awcb_physdone(zio, cb->awcb_buf, cb->awcb_private);
5943}
5944
5945static void
5946arc_write_done(zio_t *zio)
5947{
5948	arc_write_callback_t *callback = zio->io_private;
5949	arc_buf_t *buf = callback->awcb_buf;
5950	arc_buf_hdr_t *hdr = buf->b_hdr;
5951
5952	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
5953
5954	if (zio->io_error == 0) {
5955		arc_hdr_verify(hdr, zio->io_bp);
5956
5957		if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
5958			buf_discard_identity(hdr);
5959		} else {
5960			hdr->b_dva = *BP_IDENTITY(zio->io_bp);
5961			hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
5962		}
5963	} else {
5964		ASSERT(HDR_EMPTY(hdr));
5965	}
5966
5967	/*
5968	 * If the block to be written was all-zero or compressed enough to be
5969	 * embedded in the BP, no write was performed so there will be no
5970	 * dva/birth/checksum.  The buffer must therefore remain anonymous
5971	 * (and uncached).
5972	 */
5973	if (!HDR_EMPTY(hdr)) {
5974		arc_buf_hdr_t *exists;
5975		kmutex_t *hash_lock;
5976
5977		ASSERT3U(zio->io_error, ==, 0);
5978
5979		arc_cksum_verify(buf);
5980
5981		exists = buf_hash_insert(hdr, &hash_lock);
5982		if (exists != NULL) {
5983			/*
5984			 * This can only happen if we overwrite for
5985			 * sync-to-convergence, because we remove
5986			 * buffers from the hash table when we arc_free().
5987			 */
5988			if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
5989				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
5990					panic("bad overwrite, hdr=%p exists=%p",
5991					    (void *)hdr, (void *)exists);
5992				ASSERT(refcount_is_zero(
5993				    &exists->b_l1hdr.b_refcnt));
5994				arc_change_state(arc_anon, exists, hash_lock);
5995				mutex_exit(hash_lock);
5996				arc_hdr_destroy(exists);
5997				exists = buf_hash_insert(hdr, &hash_lock);
5998				ASSERT3P(exists, ==, NULL);
5999			} else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
6000				/* nopwrite */
6001				ASSERT(zio->io_prop.zp_nopwrite);
6002				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
6003					panic("bad nopwrite, hdr=%p exists=%p",
6004					    (void *)hdr, (void *)exists);
6005			} else {
6006				/* Dedup */
6007				ASSERT(hdr->b_l1hdr.b_bufcnt == 1);
6008				ASSERT(hdr->b_l1hdr.b_state == arc_anon);
6009				ASSERT(BP_GET_DEDUP(zio->io_bp));
6010				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
6011			}
6012		}
6013		arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6014		/* if it's not anon, we are doing a scrub */
6015		if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
6016			arc_access(hdr, hash_lock);
6017		mutex_exit(hash_lock);
6018	} else {
6019		arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6020	}
6021
6022	ASSERT(!refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
6023	callback->awcb_done(zio, buf, callback->awcb_private);
6024
6025	abd_put(zio->io_abd);
6026	kmem_free(callback, sizeof (arc_write_callback_t));
6027}
6028
6029zio_t *
6030arc_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, arc_buf_t *buf,
6031    boolean_t l2arc, const zio_prop_t *zp, arc_done_func_t *ready,
6032    arc_done_func_t *children_ready, arc_done_func_t *physdone,
6033    arc_done_func_t *done, void *private, zio_priority_t priority,
6034    int zio_flags, const zbookmark_phys_t *zb)
6035{
6036	arc_buf_hdr_t *hdr = buf->b_hdr;
6037	arc_write_callback_t *callback;
6038	zio_t *zio;
6039	zio_prop_t localprop = *zp;
6040
6041	ASSERT3P(ready, !=, NULL);
6042	ASSERT3P(done, !=, NULL);
6043	ASSERT(!HDR_IO_ERROR(hdr));
6044	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6045	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6046	ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
6047	if (l2arc)
6048		arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
6049	if (ARC_BUF_COMPRESSED(buf)) {
6050		/*
6051		 * We're writing a pre-compressed buffer.  Make the
6052		 * compression algorithm requested by the zio_prop_t match
6053		 * the pre-compressed buffer's compression algorithm.
6054		 */
6055		localprop.zp_compress = HDR_GET_COMPRESS(hdr);
6056
6057		ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf));
6058		zio_flags |= ZIO_FLAG_RAW;
6059	}
6060	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
6061	callback->awcb_ready = ready;
6062	callback->awcb_children_ready = children_ready;
6063	callback->awcb_physdone = physdone;
6064	callback->awcb_done = done;
6065	callback->awcb_private = private;
6066	callback->awcb_buf = buf;
6067
6068	/*
6069	 * The hdr's b_pabd is now stale, free it now. A new data block
6070	 * will be allocated when the zio pipeline calls arc_write_ready().
6071	 */
6072	if (hdr->b_l1hdr.b_pabd != NULL) {
6073		/*
6074		 * If the buf is currently sharing the data block with
6075		 * the hdr then we need to break that relationship here.
6076		 * The hdr will remain with a NULL data pointer and the
6077		 * buf will take sole ownership of the block.
6078		 */
6079		if (arc_buf_is_shared(buf)) {
6080			arc_unshare_buf(hdr, buf);
6081		} else {
6082			arc_hdr_free_pabd(hdr);
6083		}
6084		VERIFY3P(buf->b_data, !=, NULL);
6085		arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF);
6086	}
6087	ASSERT(!arc_buf_is_shared(buf));
6088	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6089
6090	zio = zio_write(pio, spa, txg, bp,
6091	    abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)),
6092	    HDR_GET_LSIZE(hdr), arc_buf_size(buf), &localprop, arc_write_ready,
6093	    (children_ready != NULL) ? arc_write_children_ready : NULL,
6094	    arc_write_physdone, arc_write_done, callback,
6095	    priority, zio_flags, zb);
6096
6097	return (zio);
6098}
6099
6100static int
6101arc_memory_throttle(uint64_t reserve, uint64_t txg)
6102{
6103#ifdef _KERNEL
6104	uint64_t available_memory = ptob(freemem);
6105	static uint64_t page_load = 0;
6106	static uint64_t last_txg = 0;
6107
6108#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC)
6109	available_memory = MIN(available_memory, uma_avail());
6110#endif
6111
6112	if (freemem > (uint64_t)physmem * arc_lotsfree_percent / 100)
6113		return (0);
6114
6115	if (txg > last_txg) {
6116		last_txg = txg;
6117		page_load = 0;
6118	}
6119	/*
6120	 * If we are in pageout, we know that memory is already tight,
6121	 * the arc is already going to be evicting, so we just want to
6122	 * continue to let page writes occur as quickly as possible.
6123	 */
6124	if (curproc == pageproc) {
6125		if (page_load > MAX(ptob(minfree), available_memory) / 4)
6126			return (SET_ERROR(ERESTART));
6127		/* Note: reserve is inflated, so we deflate */
6128		page_load += reserve / 8;
6129		return (0);
6130	} else if (page_load > 0 && arc_reclaim_needed()) {
6131		/* memory is low, delay before restarting */
6132		ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
6133		return (SET_ERROR(EAGAIN));
6134	}
6135	page_load = 0;
6136#endif
6137	return (0);
6138}
6139
6140void
6141arc_tempreserve_clear(uint64_t reserve)
6142{
6143	atomic_add_64(&arc_tempreserve, -reserve);
6144	ASSERT((int64_t)arc_tempreserve >= 0);
6145}
6146
6147int
6148arc_tempreserve_space(uint64_t reserve, uint64_t txg)
6149{
6150	int error;
6151	uint64_t anon_size;
6152
6153	if (reserve > arc_c/4 && !arc_no_grow) {
6154		arc_c = MIN(arc_c_max, reserve * 4);
6155		DTRACE_PROBE1(arc__set_reserve, uint64_t, arc_c);
6156	}
6157	if (reserve > arc_c)
6158		return (SET_ERROR(ENOMEM));
6159
6160	/*
6161	 * Don't count loaned bufs as in flight dirty data to prevent long
6162	 * network delays from blocking transactions that are ready to be
6163	 * assigned to a txg.
6164	 */
6165
6166	/* assert that it has not wrapped around */
6167	ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
6168
6169	anon_size = MAX((int64_t)(refcount_count(&arc_anon->arcs_size) -
6170	    arc_loaned_bytes), 0);
6171
6172	/*
6173	 * Writes will, almost always, require additional memory allocations
6174	 * in order to compress/encrypt/etc the data.  We therefore need to
6175	 * make sure that there is sufficient available memory for this.
6176	 */
6177	error = arc_memory_throttle(reserve, txg);
6178	if (error != 0)
6179		return (error);
6180
6181	/*
6182	 * Throttle writes when the amount of dirty data in the cache
6183	 * gets too large.  We try to keep the cache less than half full
6184	 * of dirty blocks so that our sync times don't grow too large.
6185	 * Note: if two requests come in concurrently, we might let them
6186	 * both succeed, when one of them should fail.  Not a huge deal.
6187	 */
6188
6189	if (reserve + arc_tempreserve + anon_size > arc_c / 2 &&
6190	    anon_size > arc_c / 4) {
6191		uint64_t meta_esize =
6192		    refcount_count(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
6193		uint64_t data_esize =
6194		    refcount_count(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
6195		dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
6196		    "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
6197		    arc_tempreserve >> 10, meta_esize >> 10,
6198		    data_esize >> 10, reserve >> 10, arc_c >> 10);
6199		return (SET_ERROR(ERESTART));
6200	}
6201	atomic_add_64(&arc_tempreserve, reserve);
6202	return (0);
6203}
6204
6205static void
6206arc_kstat_update_state(arc_state_t *state, kstat_named_t *size,
6207    kstat_named_t *evict_data, kstat_named_t *evict_metadata)
6208{
6209	size->value.ui64 = refcount_count(&state->arcs_size);
6210	evict_data->value.ui64 =
6211	    refcount_count(&state->arcs_esize[ARC_BUFC_DATA]);
6212	evict_metadata->value.ui64 =
6213	    refcount_count(&state->arcs_esize[ARC_BUFC_METADATA]);
6214}
6215
6216static int
6217arc_kstat_update(kstat_t *ksp, int rw)
6218{
6219	arc_stats_t *as = ksp->ks_data;
6220
6221	if (rw == KSTAT_WRITE) {
6222		return (EACCES);
6223	} else {
6224		arc_kstat_update_state(arc_anon,
6225		    &as->arcstat_anon_size,
6226		    &as->arcstat_anon_evictable_data,
6227		    &as->arcstat_anon_evictable_metadata);
6228		arc_kstat_update_state(arc_mru,
6229		    &as->arcstat_mru_size,
6230		    &as->arcstat_mru_evictable_data,
6231		    &as->arcstat_mru_evictable_metadata);
6232		arc_kstat_update_state(arc_mru_ghost,
6233		    &as->arcstat_mru_ghost_size,
6234		    &as->arcstat_mru_ghost_evictable_data,
6235		    &as->arcstat_mru_ghost_evictable_metadata);
6236		arc_kstat_update_state(arc_mfu,
6237		    &as->arcstat_mfu_size,
6238		    &as->arcstat_mfu_evictable_data,
6239		    &as->arcstat_mfu_evictable_metadata);
6240		arc_kstat_update_state(arc_mfu_ghost,
6241		    &as->arcstat_mfu_ghost_size,
6242		    &as->arcstat_mfu_ghost_evictable_data,
6243		    &as->arcstat_mfu_ghost_evictable_metadata);
6244	}
6245
6246	return (0);
6247}
6248
6249/*
6250 * This function *must* return indices evenly distributed between all
6251 * sublists of the multilist. This is needed due to how the ARC eviction
6252 * code is laid out; arc_evict_state() assumes ARC buffers are evenly
6253 * distributed between all sublists and uses this assumption when
6254 * deciding which sublist to evict from and how much to evict from it.
6255 */
6256unsigned int
6257arc_state_multilist_index_func(multilist_t *ml, void *obj)
6258{
6259	arc_buf_hdr_t *hdr = obj;
6260
6261	/*
6262	 * We rely on b_dva to generate evenly distributed index
6263	 * numbers using buf_hash below. So, as an added precaution,
6264	 * let's make sure we never add empty buffers to the arc lists.
6265	 */
6266	ASSERT(!HDR_EMPTY(hdr));
6267
6268	/*
6269	 * The assumption here, is the hash value for a given
6270	 * arc_buf_hdr_t will remain constant throughout it's lifetime
6271	 * (i.e. it's b_spa, b_dva, and b_birth fields don't change).
6272	 * Thus, we don't need to store the header's sublist index
6273	 * on insertion, as this index can be recalculated on removal.
6274	 *
6275	 * Also, the low order bits of the hash value are thought to be
6276	 * distributed evenly. Otherwise, in the case that the multilist
6277	 * has a power of two number of sublists, each sublists' usage
6278	 * would not be evenly distributed.
6279	 */
6280	return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
6281	    multilist_get_num_sublists(ml));
6282}
6283
6284#ifdef _KERNEL
6285static eventhandler_tag arc_event_lowmem = NULL;
6286
6287static void
6288arc_lowmem(void *arg __unused, int howto __unused)
6289{
6290
6291	mutex_enter(&arc_reclaim_lock);
6292	DTRACE_PROBE1(arc__needfree, int64_t, ((int64_t)freemem - zfs_arc_free_target) * PAGESIZE);
6293	cv_signal(&arc_reclaim_thread_cv);
6294
6295	/*
6296	 * It is unsafe to block here in arbitrary threads, because we can come
6297	 * here from ARC itself and may hold ARC locks and thus risk a deadlock
6298	 * with ARC reclaim thread.
6299	 */
6300	if (curproc == pageproc)
6301		(void) cv_wait(&arc_reclaim_waiters_cv, &arc_reclaim_lock);
6302	mutex_exit(&arc_reclaim_lock);
6303}
6304#endif
6305
6306static void
6307arc_state_init(void)
6308{
6309	arc_anon = &ARC_anon;
6310	arc_mru = &ARC_mru;
6311	arc_mru_ghost = &ARC_mru_ghost;
6312	arc_mfu = &ARC_mfu;
6313	arc_mfu_ghost = &ARC_mfu_ghost;
6314	arc_l2c_only = &ARC_l2c_only;
6315
6316	arc_mru->arcs_list[ARC_BUFC_METADATA] =
6317	    multilist_create(sizeof (arc_buf_hdr_t),
6318	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6319	    arc_state_multilist_index_func);
6320	arc_mru->arcs_list[ARC_BUFC_DATA] =
6321	    multilist_create(sizeof (arc_buf_hdr_t),
6322	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6323	    arc_state_multilist_index_func);
6324	arc_mru_ghost->arcs_list[ARC_BUFC_METADATA] =
6325	    multilist_create(sizeof (arc_buf_hdr_t),
6326	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6327	    arc_state_multilist_index_func);
6328	arc_mru_ghost->arcs_list[ARC_BUFC_DATA] =
6329	    multilist_create(sizeof (arc_buf_hdr_t),
6330	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6331	    arc_state_multilist_index_func);
6332	arc_mfu->arcs_list[ARC_BUFC_METADATA] =
6333	    multilist_create(sizeof (arc_buf_hdr_t),
6334	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6335	    arc_state_multilist_index_func);
6336	arc_mfu->arcs_list[ARC_BUFC_DATA] =
6337	    multilist_create(sizeof (arc_buf_hdr_t),
6338	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6339	    arc_state_multilist_index_func);
6340	arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA] =
6341	    multilist_create(sizeof (arc_buf_hdr_t),
6342	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6343	    arc_state_multilist_index_func);
6344	arc_mfu_ghost->arcs_list[ARC_BUFC_DATA] =
6345	    multilist_create(sizeof (arc_buf_hdr_t),
6346	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6347	    arc_state_multilist_index_func);
6348	arc_l2c_only->arcs_list[ARC_BUFC_METADATA] =
6349	    multilist_create(sizeof (arc_buf_hdr_t),
6350	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6351	    arc_state_multilist_index_func);
6352	arc_l2c_only->arcs_list[ARC_BUFC_DATA] =
6353	    multilist_create(sizeof (arc_buf_hdr_t),
6354	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6355	    arc_state_multilist_index_func);
6356
6357	refcount_create(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
6358	refcount_create(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
6359	refcount_create(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
6360	refcount_create(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
6361	refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
6362	refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
6363	refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
6364	refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
6365	refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
6366	refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
6367	refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
6368	refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
6369
6370	refcount_create(&arc_anon->arcs_size);
6371	refcount_create(&arc_mru->arcs_size);
6372	refcount_create(&arc_mru_ghost->arcs_size);
6373	refcount_create(&arc_mfu->arcs_size);
6374	refcount_create(&arc_mfu_ghost->arcs_size);
6375	refcount_create(&arc_l2c_only->arcs_size);
6376}
6377
6378static void
6379arc_state_fini(void)
6380{
6381	refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
6382	refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
6383	refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
6384	refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
6385	refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
6386	refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
6387	refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
6388	refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
6389	refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
6390	refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
6391	refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
6392	refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
6393
6394	refcount_destroy(&arc_anon->arcs_size);
6395	refcount_destroy(&arc_mru->arcs_size);
6396	refcount_destroy(&arc_mru_ghost->arcs_size);
6397	refcount_destroy(&arc_mfu->arcs_size);
6398	refcount_destroy(&arc_mfu_ghost->arcs_size);
6399	refcount_destroy(&arc_l2c_only->arcs_size);
6400
6401	multilist_destroy(arc_mru->arcs_list[ARC_BUFC_METADATA]);
6402	multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
6403	multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_METADATA]);
6404	multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
6405	multilist_destroy(arc_mru->arcs_list[ARC_BUFC_DATA]);
6406	multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
6407	multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_DATA]);
6408	multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
6409}
6410
6411uint64_t
6412arc_max_bytes(void)
6413{
6414	return (arc_c_max);
6415}
6416
6417void
6418arc_init(void)
6419{
6420	int i, prefetch_tunable_set = 0;
6421
6422	/*
6423	 * allmem is "all memory that we could possibly use".
6424	 */
6425#ifdef illumos
6426#ifdef _KERNEL
6427	uint64_t allmem = ptob(physmem - swapfs_minfree);
6428#else
6429	uint64_t allmem = (physmem * PAGESIZE) / 2;
6430#endif
6431#else
6432	uint64_t allmem = kmem_size();
6433#endif
6434
6435
6436	mutex_init(&arc_reclaim_lock, NULL, MUTEX_DEFAULT, NULL);
6437	cv_init(&arc_reclaim_thread_cv, NULL, CV_DEFAULT, NULL);
6438	cv_init(&arc_reclaim_waiters_cv, NULL, CV_DEFAULT, NULL);
6439
6440	mutex_init(&arc_dnlc_evicts_lock, NULL, MUTEX_DEFAULT, NULL);
6441	cv_init(&arc_dnlc_evicts_cv, NULL, CV_DEFAULT, NULL);
6442
6443	/* Convert seconds to clock ticks */
6444	arc_min_prefetch_lifespan = 1 * hz;
6445
6446	/* set min cache to 1/32 of all memory, or arc_abs_min, whichever is more */
6447	arc_c_min = MAX(allmem / 32, arc_abs_min);
6448	/* set max to 5/8 of all memory, or all but 1GB, whichever is more */
6449	if (allmem >= 1 << 30)
6450		arc_c_max = allmem - (1 << 30);
6451	else
6452		arc_c_max = arc_c_min;
6453	arc_c_max =