xref: /illumos-gate/usr/src/uts/common/fs/zfs/arc.c (revision f7170741)
1fa9e4066Sahrens /*
2fa9e4066Sahrens  * CDDL HEADER START
3fa9e4066Sahrens  *
4fa9e4066Sahrens  * The contents of this file are subject to the terms of the
5033f9833Sek  * Common Development and Distribution License (the "License").
6033f9833Sek  * You may not use this file except in compliance with the License.
7fa9e4066Sahrens  *
8fa9e4066Sahrens  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9fa9e4066Sahrens  * or http://www.opensolaris.org/os/licensing.
10fa9e4066Sahrens  * See the License for the specific language governing permissions
11fa9e4066Sahrens  * and limitations under the License.
12fa9e4066Sahrens  *
13fa9e4066Sahrens  * When distributing Covered Code, include this CDDL HEADER in each
14fa9e4066Sahrens  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15fa9e4066Sahrens  * If applicable, add the following below this CDDL HEADER, with the
16fa9e4066Sahrens  * fields enclosed by brackets "[]" replaced with your own identifying
17fa9e4066Sahrens  * information: Portions Copyright [yyyy] [name of copyright owner]
18fa9e4066Sahrens  *
19fa9e4066Sahrens  * CDDL HEADER END
20fa9e4066Sahrens  */
21fa9e4066Sahrens /*
223f9d6ad7SLin Ling  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23e9103aaeSGarrett D'Amore  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
24be6fd75aSMatthew Ahrens  * Copyright (c) 2013 by Delphix. All rights reserved.
25aad02571SSaso Kiselkov  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
26fa9e4066Sahrens  */
27fa9e4066Sahrens 
28fa9e4066Sahrens /*
2944cb6abcSbmc  * DVA-based Adjustable Replacement Cache
30fa9e4066Sahrens  *
31ea8dc4b6Seschrock  * While much of the theory of operation used here is
32ea8dc4b6Seschrock  * based on the self-tuning, low overhead replacement cache
33fa9e4066Sahrens  * presented by Megiddo and Modha at FAST 2003, there are some
34fa9e4066Sahrens  * significant differences:
35fa9e4066Sahrens  *
36fa9e4066Sahrens  * 1. The Megiddo and Modha model assumes any page is evictable.
37fa9e4066Sahrens  * Pages in its cache cannot be "locked" into memory.  This makes
38fa9e4066Sahrens  * the eviction algorithm simple: evict the last page in the list.
39fa9e4066Sahrens  * This also make the performance characteristics easy to reason
40fa9e4066Sahrens  * about.  Our cache is not so simple.  At any given moment, some
41fa9e4066Sahrens  * subset of the blocks in the cache are un-evictable because we
42fa9e4066Sahrens  * have handed out a reference to them.  Blocks are only evictable
43fa9e4066Sahrens  * when there are no external references active.  This makes
44fa9e4066Sahrens  * eviction far more problematic:  we choose to evict the evictable
45fa9e4066Sahrens  * blocks that are the "lowest" in the list.
46fa9e4066Sahrens  *
47fa9e4066Sahrens  * There are times when it is not possible to evict the requested
48fa9e4066Sahrens  * space.  In these circumstances we are unable to adjust the cache
49fa9e4066Sahrens  * size.  To prevent the cache growing unbounded at these times we
50fa94a07fSbrendan  * implement a "cache throttle" that slows the flow of new data
51fa94a07fSbrendan  * into the cache until we can make space available.
52fa9e4066Sahrens  *
53fa9e4066Sahrens  * 2. The Megiddo and Modha model assumes a fixed cache size.
54fa9e4066Sahrens  * Pages are evicted when the cache is full and there is a cache
55fa9e4066Sahrens  * miss.  Our model has a variable sized cache.  It grows with
56fa94a07fSbrendan  * high use, but also tries to react to memory pressure from the
57fa9e4066Sahrens  * operating system: decreasing its size when system memory is
58fa9e4066Sahrens  * tight.
59fa9e4066Sahrens  *
60fa9e4066Sahrens  * 3. The Megiddo and Modha model assumes a fixed page size. All
61*f7170741SWill Andrews  * elements of the cache are therefore exactly the same size.  So
62fa9e4066Sahrens  * when adjusting the cache size following a cache miss, its simply
63fa9e4066Sahrens  * a matter of choosing a single page to evict.  In our model, we
64fa9e4066Sahrens  * have variable sized cache blocks (rangeing from 512 bytes to
65*f7170741SWill Andrews  * 128K bytes).  We therefore choose a set of blocks to evict to make
66fa9e4066Sahrens  * space for a cache miss that approximates as closely as possible
67fa9e4066Sahrens  * the space used by the new block.
68fa9e4066Sahrens  *
69fa9e4066Sahrens  * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
70fa9e4066Sahrens  * by N. Megiddo & D. Modha, FAST 2003
71fa9e4066Sahrens  */
72fa9e4066Sahrens 
73fa9e4066Sahrens /*
74fa9e4066Sahrens  * The locking model:
75fa9e4066Sahrens  *
76fa9e4066Sahrens  * A new reference to a cache buffer can be obtained in two
77fa9e4066Sahrens  * ways: 1) via a hash table lookup using the DVA as a key,
78fa94a07fSbrendan  * or 2) via one of the ARC lists.  The arc_read() interface
79fa9e4066Sahrens  * uses method 1, while the internal arc algorithms for
80*f7170741SWill Andrews  * adjusting the cache use method 2.  We therefore provide two
81fa9e4066Sahrens  * types of locks: 1) the hash table lock array, and 2) the
82fa9e4066Sahrens  * arc list locks.
83fa9e4066Sahrens  *
84fc98fea5SBart Coddens  * Buffers do not have their own mutexes, rather they rely on the
85fc98fea5SBart Coddens  * hash table mutexes for the bulk of their protection (i.e. most
86fc98fea5SBart Coddens  * fields in the arc_buf_hdr_t are protected by these mutexes).
87fa9e4066Sahrens  *
88fa9e4066Sahrens  * buf_hash_find() returns the appropriate mutex (held) when it
89fa9e4066Sahrens  * locates the requested buffer in the hash table.  It returns
90fa9e4066Sahrens  * NULL for the mutex if the buffer was not in the table.
91fa9e4066Sahrens  *
92fa9e4066Sahrens  * buf_hash_remove() expects the appropriate hash mutex to be
93fa9e4066Sahrens  * already held before it is invoked.
94fa9e4066Sahrens  *
95fa9e4066Sahrens  * Each arc state also has a mutex which is used to protect the
96fa9e4066Sahrens  * buffer list associated with the state.  When attempting to
97fa9e4066Sahrens  * obtain a hash table lock while holding an arc list lock you
98fa9e4066Sahrens  * must use: mutex_tryenter() to avoid deadlock.  Also note that
9944eda4d7Smaybee  * the active state mutex must be held before the ghost state mutex.
100fa9e4066Sahrens  *
101ea8dc4b6Seschrock  * Arc buffers may have an associated eviction callback function.
102ea8dc4b6Seschrock  * This function will be invoked prior to removing the buffer (e.g.
103ea8dc4b6Seschrock  * in arc_do_user_evicts()).  Note however that the data associated
104ea8dc4b6Seschrock  * with the buffer may be evicted prior to the callback.  The callback
105ea8dc4b6Seschrock  * must be made with *no locks held* (to prevent deadlock).  Additionally,
106ea8dc4b6Seschrock  * the users of callbacks must ensure that their private data is
107ea8dc4b6Seschrock  * protected from simultaneous callbacks from arc_buf_evict()
108ea8dc4b6Seschrock  * and arc_do_user_evicts().
109ea8dc4b6Seschrock  *
110fa9e4066Sahrens  * Note that the majority of the performance stats are manipulated
111fa9e4066Sahrens  * with atomic operations.
112fa94a07fSbrendan  *
113fa94a07fSbrendan  * The L2ARC uses the l2arc_buflist_mtx global mutex for the following:
114fa94a07fSbrendan  *
115fa94a07fSbrendan  *	- L2ARC buflist creation
116fa94a07fSbrendan  *	- L2ARC buflist eviction
117fa94a07fSbrendan  *	- L2ARC write completion, which walks L2ARC buflists
118fa94a07fSbrendan  *	- ARC header destruction, as it removes from L2ARC buflists
119fa94a07fSbrendan  *	- ARC header release, as it removes from L2ARC buflists
120fa9e4066Sahrens  */
121fa9e4066Sahrens 
122fa9e4066Sahrens #include <sys/spa.h>
123fa9e4066Sahrens #include <sys/zio.h>
124aad02571SSaso Kiselkov #include <sys/zio_compress.h>
125fa9e4066Sahrens #include <sys/zfs_context.h>
126fa9e4066Sahrens #include <sys/arc.h>
127fa9e4066Sahrens #include <sys/refcount.h>
128c5904d13Seschrock #include <sys/vdev.h>
129573ca77eSGeorge Wilson #include <sys/vdev_impl.h>
130fa9e4066Sahrens #ifdef _KERNEL
131fa9e4066Sahrens #include <sys/vmsystm.h>
132fa9e4066Sahrens #include <vm/anon.h>
133fa9e4066Sahrens #include <sys/fs/swapnode.h>
134033f9833Sek #include <sys/dnlc.h>
135fa9e4066Sahrens #endif
136fa9e4066Sahrens #include <sys/callb.h>
13744cb6abcSbmc #include <sys/kstat.h>
138b24ab676SJeff Bonwick #include <zfs_fletcher.h>
139fa9e4066Sahrens 
140cd1c8b85SMatthew Ahrens #ifndef _KERNEL
141cd1c8b85SMatthew Ahrens /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
142cd1c8b85SMatthew Ahrens boolean_t arc_watch = B_FALSE;
143cd1c8b85SMatthew Ahrens int arc_procfd;
144cd1c8b85SMatthew Ahrens #endif
145cd1c8b85SMatthew Ahrens 
146fa9e4066Sahrens static kmutex_t		arc_reclaim_thr_lock;
147fa9e4066Sahrens static kcondvar_t	arc_reclaim_thr_cv;	/* used to signal reclaim thr */
148fa9e4066Sahrens static uint8_t		arc_thread_exit;
149fa9e4066Sahrens 
1501ab7f2deSmaybee extern int zfs_write_limit_shift;
1511ab7f2deSmaybee extern uint64_t zfs_write_limit_max;
15205715f94SMark Maybee extern kmutex_t zfs_write_limit_lock;
1531ab7f2deSmaybee 
154033f9833Sek #define	ARC_REDUCE_DNLC_PERCENT	3
155033f9833Sek uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT;
156033f9833Sek 
157fa9e4066Sahrens typedef enum arc_reclaim_strategy {
158fa9e4066Sahrens 	ARC_RECLAIM_AGGR,		/* Aggressive reclaim strategy */
159fa9e4066Sahrens 	ARC_RECLAIM_CONS		/* Conservative reclaim strategy */
160fa9e4066Sahrens } arc_reclaim_strategy_t;
161fa9e4066Sahrens 
162fa9e4066Sahrens /* number of seconds before growing cache again */
163fa9e4066Sahrens static int		arc_grow_retry = 60;
164fa9e4066Sahrens 
1655a98e54bSBrendan Gregg - Sun Microsystems /* shift of arc_c for calculating both min and max arc_p */
1665a98e54bSBrendan Gregg - Sun Microsystems static int		arc_p_min_shift = 4;
1675a98e54bSBrendan Gregg - Sun Microsystems 
1685a98e54bSBrendan Gregg - Sun Microsystems /* log2(fraction of arc to reclaim) */
1695a98e54bSBrendan Gregg - Sun Microsystems static int		arc_shrink_shift = 5;
1705a98e54bSBrendan Gregg - Sun Microsystems 
17113506d1eSmaybee /*
172b19a79ecSperrin  * minimum lifespan of a prefetch block in clock ticks
173b19a79ecSperrin  * (initialized in arc_init())
17413506d1eSmaybee  */
175b19a79ecSperrin static int		arc_min_prefetch_lifespan;
17613506d1eSmaybee 
177fa9e4066Sahrens static int arc_dead;
178fa9e4066Sahrens 
1793a737e0dSbrendan /*
1803a737e0dSbrendan  * The arc has filled available memory and has now warmed up.
1813a737e0dSbrendan  */
1823a737e0dSbrendan static boolean_t arc_warm;
1833a737e0dSbrendan 
184a2eea2e1Sahrens /*
185a2eea2e1Sahrens  * These tunables are for performance analysis.
186a2eea2e1Sahrens  */
187a2eea2e1Sahrens uint64_t zfs_arc_max;
188a2eea2e1Sahrens uint64_t zfs_arc_min;
1891116048bSek uint64_t zfs_arc_meta_limit = 0;
1905a98e54bSBrendan Gregg - Sun Microsystems int zfs_arc_grow_retry = 0;
1915a98e54bSBrendan Gregg - Sun Microsystems int zfs_arc_shrink_shift = 0;
1925a98e54bSBrendan Gregg - Sun Microsystems int zfs_arc_p_min_shift = 0;
1939253d63dSGeorge Wilson int zfs_disable_dup_eviction = 0;
194a2eea2e1Sahrens 
195fa9e4066Sahrens /*
196fa94a07fSbrendan  * Note that buffers can be in one of 6 states:
197fa9e4066Sahrens  *	ARC_anon	- anonymous (discussed below)
198ea8dc4b6Seschrock  *	ARC_mru		- recently used, currently cached
199ea8dc4b6Seschrock  *	ARC_mru_ghost	- recentely used, no longer in cache
200ea8dc4b6Seschrock  *	ARC_mfu		- frequently used, currently cached
201ea8dc4b6Seschrock  *	ARC_mfu_ghost	- frequently used, no longer in cache
202fa94a07fSbrendan  *	ARC_l2c_only	- exists in L2ARC but not other states
2030e8c6158Smaybee  * When there are no active references to the buffer, they are
2040e8c6158Smaybee  * are linked onto a list in one of these arc states.  These are
2050e8c6158Smaybee  * the only buffers that can be evicted or deleted.  Within each
2060e8c6158Smaybee  * state there are multiple lists, one for meta-data and one for
2070e8c6158Smaybee  * non-meta-data.  Meta-data (indirect blocks, blocks of dnodes,
2080e8c6158Smaybee  * etc.) is tracked separately so that it can be managed more
209fa94a07fSbrendan  * explicitly: favored over data, limited explicitly.
210fa9e4066Sahrens  *
211fa9e4066Sahrens  * Anonymous buffers are buffers that are not associated with
212fa9e4066Sahrens  * a DVA.  These are buffers that hold dirty block copies
213fa9e4066Sahrens  * before they are written to stable storage.  By definition,
214ea8dc4b6Seschrock  * they are "ref'd" and are considered part of arc_mru
215fa9e4066Sahrens  * that cannot be freed.  Generally, they will aquire a DVA
216ea8dc4b6Seschrock  * as they are written and migrate onto the arc_mru list.
217fa94a07fSbrendan  *
218fa94a07fSbrendan  * The ARC_l2c_only state is for buffers that are in the second
219fa94a07fSbrendan  * level ARC but no longer in any of the ARC_m* lists.  The second
220fa94a07fSbrendan  * level ARC itself may also contain buffers that are in any of
221fa94a07fSbrendan  * the ARC_m* states - meaning that a buffer can exist in two
222fa94a07fSbrendan  * places.  The reason for the ARC_l2c_only state is to keep the
223fa94a07fSbrendan  * buffer header in the hash table, so that reads that hit the
224fa94a07fSbrendan  * second level ARC benefit from these fast lookups.
225fa9e4066Sahrens  */
226fa9e4066Sahrens 
227fa9e4066Sahrens typedef struct arc_state {
2280e8c6158Smaybee 	list_t	arcs_list[ARC_BUFC_NUMTYPES];	/* list of evictable buffers */
2290e8c6158Smaybee 	uint64_t arcs_lsize[ARC_BUFC_NUMTYPES];	/* amount of evictable data */
2300e8c6158Smaybee 	uint64_t arcs_size;	/* total amount of data in this state */
23144cb6abcSbmc 	kmutex_t arcs_mtx;
232fa9e4066Sahrens } arc_state_t;
233fa9e4066Sahrens 
234fa94a07fSbrendan /* The 6 states: */
235fa9e4066Sahrens static arc_state_t ARC_anon;
236ea8dc4b6Seschrock static arc_state_t ARC_mru;
237ea8dc4b6Seschrock static arc_state_t ARC_mru_ghost;
238ea8dc4b6Seschrock static arc_state_t ARC_mfu;
239ea8dc4b6Seschrock static arc_state_t ARC_mfu_ghost;
240fa94a07fSbrendan static arc_state_t ARC_l2c_only;
241fa9e4066Sahrens 
24244cb6abcSbmc typedef struct arc_stats {
24344cb6abcSbmc 	kstat_named_t arcstat_hits;
24444cb6abcSbmc 	kstat_named_t arcstat_misses;
24544cb6abcSbmc 	kstat_named_t arcstat_demand_data_hits;
24644cb6abcSbmc 	kstat_named_t arcstat_demand_data_misses;
24744cb6abcSbmc 	kstat_named_t arcstat_demand_metadata_hits;
24844cb6abcSbmc 	kstat_named_t arcstat_demand_metadata_misses;
24944cb6abcSbmc 	kstat_named_t arcstat_prefetch_data_hits;
25044cb6abcSbmc 	kstat_named_t arcstat_prefetch_data_misses;
25144cb6abcSbmc 	kstat_named_t arcstat_prefetch_metadata_hits;
25244cb6abcSbmc 	kstat_named_t arcstat_prefetch_metadata_misses;
25344cb6abcSbmc 	kstat_named_t arcstat_mru_hits;
25444cb6abcSbmc 	kstat_named_t arcstat_mru_ghost_hits;
25544cb6abcSbmc 	kstat_named_t arcstat_mfu_hits;
25644cb6abcSbmc 	kstat_named_t arcstat_mfu_ghost_hits;
25744cb6abcSbmc 	kstat_named_t arcstat_deleted;
25844cb6abcSbmc 	kstat_named_t arcstat_recycle_miss;
2593e30c24aSWill Andrews 	/*
2603e30c24aSWill Andrews 	 * Number of buffers that could not be evicted because the hash lock
2613e30c24aSWill Andrews 	 * was held by another thread.  The lock may not necessarily be held
2623e30c24aSWill Andrews 	 * by something using the same buffer, since hash locks are shared
2633e30c24aSWill Andrews 	 * by multiple buffers.
2643e30c24aSWill Andrews 	 */
26544cb6abcSbmc 	kstat_named_t arcstat_mutex_miss;
2663e30c24aSWill Andrews 	/*
2673e30c24aSWill Andrews 	 * Number of buffers skipped because they have I/O in progress, are
2683e30c24aSWill Andrews 	 * indrect prefetch buffers that have not lived long enough, or are
2693e30c24aSWill Andrews 	 * not from the spa we're trying to evict from.
2703e30c24aSWill Andrews 	 */
27144cb6abcSbmc 	kstat_named_t arcstat_evict_skip;
2725ea40c06SBrendan Gregg - Sun Microsystems 	kstat_named_t arcstat_evict_l2_cached;
2735ea40c06SBrendan Gregg - Sun Microsystems 	kstat_named_t arcstat_evict_l2_eligible;
2745ea40c06SBrendan Gregg - Sun Microsystems 	kstat_named_t arcstat_evict_l2_ineligible;
27544cb6abcSbmc 	kstat_named_t arcstat_hash_elements;
27644cb6abcSbmc 	kstat_named_t arcstat_hash_elements_max;
27744cb6abcSbmc 	kstat_named_t arcstat_hash_collisions;
27844cb6abcSbmc 	kstat_named_t arcstat_hash_chains;
27944cb6abcSbmc 	kstat_named_t arcstat_hash_chain_max;
28044cb6abcSbmc 	kstat_named_t arcstat_p;
28144cb6abcSbmc 	kstat_named_t arcstat_c;
28244cb6abcSbmc 	kstat_named_t arcstat_c_min;
28344cb6abcSbmc 	kstat_named_t arcstat_c_max;
28444cb6abcSbmc 	kstat_named_t arcstat_size;
285fa94a07fSbrendan 	kstat_named_t arcstat_hdr_size;
2865a98e54bSBrendan Gregg - Sun Microsystems 	kstat_named_t arcstat_data_size;
2875a98e54bSBrendan Gregg - Sun Microsystems 	kstat_named_t arcstat_other_size;
288fa94a07fSbrendan 	kstat_named_t arcstat_l2_hits;
289fa94a07fSbrendan 	kstat_named_t arcstat_l2_misses;
290fa94a07fSbrendan 	kstat_named_t arcstat_l2_feeds;
291fa94a07fSbrendan 	kstat_named_t arcstat_l2_rw_clash;
2925a98e54bSBrendan Gregg - Sun Microsystems 	kstat_named_t arcstat_l2_read_bytes;
2935a98e54bSBrendan Gregg - Sun Microsystems 	kstat_named_t arcstat_l2_write_bytes;
294fa94a07fSbrendan 	kstat_named_t arcstat_l2_writes_sent;
295fa94a07fSbrendan 	kstat_named_t arcstat_l2_writes_done;
296fa94a07fSbrendan 	kstat_named_t arcstat_l2_writes_error;
297fa94a07fSbrendan 	kstat_named_t arcstat_l2_writes_hdr_miss;
298fa94a07fSbrendan 	kstat_named_t arcstat_l2_evict_lock_retry;
299fa94a07fSbrendan 	kstat_named_t arcstat_l2_evict_reading;
300fa94a07fSbrendan 	kstat_named_t arcstat_l2_free_on_write;
301fa94a07fSbrendan 	kstat_named_t arcstat_l2_abort_lowmem;
302fa94a07fSbrendan 	kstat_named_t arcstat_l2_cksum_bad;
303fa94a07fSbrendan 	kstat_named_t arcstat_l2_io_error;
304fa94a07fSbrendan 	kstat_named_t arcstat_l2_size;
305aad02571SSaso Kiselkov 	kstat_named_t arcstat_l2_asize;
306fa94a07fSbrendan 	kstat_named_t arcstat_l2_hdr_size;
307aad02571SSaso Kiselkov 	kstat_named_t arcstat_l2_compress_successes;
308aad02571SSaso Kiselkov 	kstat_named_t arcstat_l2_compress_zeros;
309aad02571SSaso Kiselkov 	kstat_named_t arcstat_l2_compress_failures;
3101ab7f2deSmaybee 	kstat_named_t arcstat_memory_throttle_count;
3119253d63dSGeorge Wilson 	kstat_named_t arcstat_duplicate_buffers;
3129253d63dSGeorge Wilson 	kstat_named_t arcstat_duplicate_buffers_size;
3139253d63dSGeorge Wilson 	kstat_named_t arcstat_duplicate_reads;
31420128a08SGeorge Wilson 	kstat_named_t arcstat_meta_used;
31520128a08SGeorge Wilson 	kstat_named_t arcstat_meta_limit;
31620128a08SGeorge Wilson 	kstat_named_t arcstat_meta_max;
31744cb6abcSbmc } arc_stats_t;
31844cb6abcSbmc 
31944cb6abcSbmc static arc_stats_t arc_stats = {
32044cb6abcSbmc 	{ "hits",			KSTAT_DATA_UINT64 },
32144cb6abcSbmc 	{ "misses",			KSTAT_DATA_UINT64 },
32244cb6abcSbmc 	{ "demand_data_hits",		KSTAT_DATA_UINT64 },
32344cb6abcSbmc 	{ "demand_data_misses",		KSTAT_DATA_UINT64 },
32444cb6abcSbmc 	{ "demand_metadata_hits",	KSTAT_DATA_UINT64 },
32544cb6abcSbmc 	{ "demand_metadata_misses",	KSTAT_DATA_UINT64 },
32644cb6abcSbmc 	{ "prefetch_data_hits",		KSTAT_DATA_UINT64 },
32744cb6abcSbmc 	{ "prefetch_data_misses",	KSTAT_DATA_UINT64 },
32844cb6abcSbmc 	{ "prefetch_metadata_hits",	KSTAT_DATA_UINT64 },
32944cb6abcSbmc 	{ "prefetch_metadata_misses",	KSTAT_DATA_UINT64 },
33044cb6abcSbmc 	{ "mru_hits",			KSTAT_DATA_UINT64 },
33144cb6abcSbmc 	{ "mru_ghost_hits",		KSTAT_DATA_UINT64 },
33244cb6abcSbmc 	{ "mfu_hits",			KSTAT_DATA_UINT64 },
33344cb6abcSbmc 	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
33444cb6abcSbmc 	{ "deleted",			KSTAT_DATA_UINT64 },
33544cb6abcSbmc 	{ "recycle_miss",		KSTAT_DATA_UINT64 },
33644cb6abcSbmc 	{ "mutex_miss",			KSTAT_DATA_UINT64 },
33744cb6abcSbmc 	{ "evict_skip",			KSTAT_DATA_UINT64 },
3385ea40c06SBrendan Gregg - Sun Microsystems 	{ "evict_l2_cached",		KSTAT_DATA_UINT64 },
3395ea40c06SBrendan Gregg - Sun Microsystems 	{ "evict_l2_eligible",		KSTAT_DATA_UINT64 },
3405ea40c06SBrendan Gregg - Sun Microsystems 	{ "evict_l2_ineligible",	KSTAT_DATA_UINT64 },
34144cb6abcSbmc 	{ "hash_elements",		KSTAT_DATA_UINT64 },
34244cb6abcSbmc 	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
34344cb6abcSbmc 	{ "hash_collisions",		KSTAT_DATA_UINT64 },
34444cb6abcSbmc 	{ "hash_chains",		KSTAT_DATA_UINT64 },
34544cb6abcSbmc 	{ "hash_chain_max",		KSTAT_DATA_UINT64 },
34644cb6abcSbmc 	{ "p",				KSTAT_DATA_UINT64 },
34744cb6abcSbmc 	{ "c",				KSTAT_DATA_UINT64 },
34844cb6abcSbmc 	{ "c_min",			KSTAT_DATA_UINT64 },
34944cb6abcSbmc 	{ "c_max",			KSTAT_DATA_UINT64 },
350fa94a07fSbrendan 	{ "size",			KSTAT_DATA_UINT64 },
351fa94a07fSbrendan 	{ "hdr_size",			KSTAT_DATA_UINT64 },
3525a98e54bSBrendan Gregg - Sun Microsystems 	{ "data_size",			KSTAT_DATA_UINT64 },
3535a98e54bSBrendan Gregg - Sun Microsystems 	{ "other_size",			KSTAT_DATA_UINT64 },
354fa94a07fSbrendan 	{ "l2_hits",			KSTAT_DATA_UINT64 },
355fa94a07fSbrendan 	{ "l2_misses",			KSTAT_DATA_UINT64 },
356fa94a07fSbrendan 	{ "l2_feeds",			KSTAT_DATA_UINT64 },
357fa94a07fSbrendan 	{ "l2_rw_clash",		KSTAT_DATA_UINT64 },
3585a98e54bSBrendan Gregg - Sun Microsystems 	{ "l2_read_bytes",		KSTAT_DATA_UINT64 },
3595a98e54bSBrendan Gregg - Sun Microsystems 	{ "l2_write_bytes",		KSTAT_DATA_UINT64 },
360fa94a07fSbrendan 	{ "l2_writes_sent",		KSTAT_DATA_UINT64 },
361fa94a07fSbrendan 	{ "l2_writes_done",		KSTAT_DATA_UINT64 },
362fa94a07fSbrendan 	{ "l2_writes_error",		KSTAT_DATA_UINT64 },
363fa94a07fSbrendan 	{ "l2_writes_hdr_miss",		KSTAT_DATA_UINT64 },
364fa94a07fSbrendan 	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
365fa94a07fSbrendan 	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
366fa94a07fSbrendan 	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
367fa94a07fSbrendan 	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
368fa94a07fSbrendan 	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
369fa94a07fSbrendan 	{ "l2_io_error",		KSTAT_DATA_UINT64 },
370fa94a07fSbrendan 	{ "l2_size",			KSTAT_DATA_UINT64 },
371aad02571SSaso Kiselkov 	{ "l2_asize",			KSTAT_DATA_UINT64 },
3721ab7f2deSmaybee 	{ "l2_hdr_size",		KSTAT_DATA_UINT64 },
373aad02571SSaso Kiselkov 	{ "l2_compress_successes",	KSTAT_DATA_UINT64 },
374aad02571SSaso Kiselkov 	{ "l2_compress_zeros",		KSTAT_DATA_UINT64 },
375aad02571SSaso Kiselkov 	{ "l2_compress_failures",	KSTAT_DATA_UINT64 },
3769253d63dSGeorge Wilson 	{ "memory_throttle_count",	KSTAT_DATA_UINT64 },
3779253d63dSGeorge Wilson 	{ "duplicate_buffers",		KSTAT_DATA_UINT64 },
3789253d63dSGeorge Wilson 	{ "duplicate_buffers_size",	KSTAT_DATA_UINT64 },
37920128a08SGeorge Wilson 	{ "duplicate_reads",		KSTAT_DATA_UINT64 },
38020128a08SGeorge Wilson 	{ "arc_meta_used",		KSTAT_DATA_UINT64 },
38120128a08SGeorge Wilson 	{ "arc_meta_limit",		KSTAT_DATA_UINT64 },
38220128a08SGeorge Wilson 	{ "arc_meta_max",		KSTAT_DATA_UINT64 }
38344cb6abcSbmc };
38444cb6abcSbmc 
38544cb6abcSbmc #define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
38644cb6abcSbmc 
38744cb6abcSbmc #define	ARCSTAT_INCR(stat, val) \
388*f7170741SWill Andrews 	atomic_add_64(&arc_stats.stat.value.ui64, (val))
38944cb6abcSbmc 
390b24ab676SJeff Bonwick #define	ARCSTAT_BUMP(stat)	ARCSTAT_INCR(stat, 1)
39144cb6abcSbmc #define	ARCSTAT_BUMPDOWN(stat)	ARCSTAT_INCR(stat, -1)
39244cb6abcSbmc 
39344cb6abcSbmc #define	ARCSTAT_MAX(stat, val) {					\
39444cb6abcSbmc 	uint64_t m;							\
39544cb6abcSbmc 	while ((val) > (m = arc_stats.stat.value.ui64) &&		\
39644cb6abcSbmc 	    (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))	\
39744cb6abcSbmc 		continue;						\
39844cb6abcSbmc }
39944cb6abcSbmc 
40044cb6abcSbmc #define	ARCSTAT_MAXSTAT(stat) \
40144cb6abcSbmc 	ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
40244cb6abcSbmc 
40344cb6abcSbmc /*
40444cb6abcSbmc  * We define a macro to allow ARC hits/misses to be easily broken down by
40544cb6abcSbmc  * two separate conditions, giving a total of four different subtypes for
40644cb6abcSbmc  * each of hits and misses (so eight statistics total).
40744cb6abcSbmc  */
40844cb6abcSbmc #define	ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
40944cb6abcSbmc 	if (cond1) {							\
41044cb6abcSbmc 		if (cond2) {						\
41144cb6abcSbmc 			ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
41244cb6abcSbmc 		} else {						\
41344cb6abcSbmc 			ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
41444cb6abcSbmc 		}							\
41544cb6abcSbmc 	} else {							\
41644cb6abcSbmc 		if (cond2) {						\
41744cb6abcSbmc 			ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
41844cb6abcSbmc 		} else {						\
41944cb6abcSbmc 			ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
42044cb6abcSbmc 		}							\
42144cb6abcSbmc 	}
42244cb6abcSbmc 
42344cb6abcSbmc kstat_t			*arc_ksp;
424b24ab676SJeff Bonwick static arc_state_t	*arc_anon;
42544cb6abcSbmc static arc_state_t	*arc_mru;
42644cb6abcSbmc static arc_state_t	*arc_mru_ghost;
42744cb6abcSbmc static arc_state_t	*arc_mfu;
42844cb6abcSbmc static arc_state_t	*arc_mfu_ghost;
429fa94a07fSbrendan static arc_state_t	*arc_l2c_only;
43044cb6abcSbmc 
43144cb6abcSbmc /*
43244cb6abcSbmc  * There are several ARC variables that are critical to export as kstats --
43344cb6abcSbmc  * but we don't want to have to grovel around in the kstat whenever we wish to
43444cb6abcSbmc  * manipulate them.  For these variables, we therefore define them to be in
43544cb6abcSbmc  * terms of the statistic variable.  This assures that we are not introducing
43644cb6abcSbmc  * the possibility of inconsistency by having shadow copies of the variables,
43744cb6abcSbmc  * while still allowing the code to be readable.
43844cb6abcSbmc  */
43944cb6abcSbmc #define	arc_size	ARCSTAT(arcstat_size)	/* actual total arc size */
44044cb6abcSbmc #define	arc_p		ARCSTAT(arcstat_p)	/* target size of MRU */
44144cb6abcSbmc #define	arc_c		ARCSTAT(arcstat_c)	/* target size of cache */
44244cb6abcSbmc #define	arc_c_min	ARCSTAT(arcstat_c_min)	/* min target cache size */
44344cb6abcSbmc #define	arc_c_max	ARCSTAT(arcstat_c_max)	/* max target cache size */
44420128a08SGeorge Wilson #define	arc_meta_limit	ARCSTAT(arcstat_meta_limit) /* max size for metadata */
44520128a08SGeorge Wilson #define	arc_meta_used	ARCSTAT(arcstat_meta_used) /* size of metadata */
44620128a08SGeorge Wilson #define	arc_meta_max	ARCSTAT(arcstat_meta_max) /* max size of metadata */
44744cb6abcSbmc 
448aad02571SSaso Kiselkov #define	L2ARC_IS_VALID_COMPRESS(_c_) \
449aad02571SSaso Kiselkov 	((_c_) == ZIO_COMPRESS_LZ4 || (_c_) == ZIO_COMPRESS_EMPTY)
450aad02571SSaso Kiselkov 
45144cb6abcSbmc static int		arc_no_grow;	/* Don't try to grow cache size */
45244cb6abcSbmc static uint64_t		arc_tempreserve;
4532fdbea25SAleksandr Guzovskiy static uint64_t		arc_loaned_bytes;
454fa9e4066Sahrens 
455fa94a07fSbrendan typedef struct l2arc_buf_hdr l2arc_buf_hdr_t;
456fa94a07fSbrendan 
457fa9e4066Sahrens typedef struct arc_callback arc_callback_t;
458fa9e4066Sahrens 
459fa9e4066Sahrens struct arc_callback {
460fa9e4066Sahrens 	void			*acb_private;
461c717a561Smaybee 	arc_done_func_t		*acb_done;
462fa9e4066Sahrens 	arc_buf_t		*acb_buf;
463fa9e4066Sahrens 	zio_t			*acb_zio_dummy;
464fa9e4066Sahrens 	arc_callback_t		*acb_next;
465fa9e4066Sahrens };
466fa9e4066Sahrens 
467c717a561Smaybee typedef struct arc_write_callback arc_write_callback_t;
468c717a561Smaybee 
469c717a561Smaybee struct arc_write_callback {
470c717a561Smaybee 	void		*awcb_private;
471c717a561Smaybee 	arc_done_func_t	*awcb_ready;
472c717a561Smaybee 	arc_done_func_t	*awcb_done;
473c717a561Smaybee 	arc_buf_t	*awcb_buf;
474c717a561Smaybee };
475c717a561Smaybee 
476fa9e4066Sahrens struct arc_buf_hdr {
477fa9e4066Sahrens 	/* protected by hash lock */
478fa9e4066Sahrens 	dva_t			b_dva;
479fa9e4066Sahrens 	uint64_t		b_birth;
480fa9e4066Sahrens 	uint64_t		b_cksum0;
481fa9e4066Sahrens 
4826b4acc8bSahrens 	kmutex_t		b_freeze_lock;
4836b4acc8bSahrens 	zio_cksum_t		*b_freeze_cksum;
4843f9d6ad7SLin Ling 	void			*b_thawed;
4856b4acc8bSahrens 
486fa9e4066Sahrens 	arc_buf_hdr_t		*b_hash_next;
487fa9e4066Sahrens 	arc_buf_t		*b_buf;
488fa9e4066Sahrens 	uint32_t		b_flags;
489ea8dc4b6Seschrock 	uint32_t		b_datacnt;
490fa9e4066Sahrens 
491fa9e4066Sahrens 	arc_callback_t		*b_acb;
492ad23a2dbSjohansen 	kcondvar_t		b_cv;
493ad23a2dbSjohansen 
494ad23a2dbSjohansen 	/* immutable */
495ad23a2dbSjohansen 	arc_buf_contents_t	b_type;
496ad23a2dbSjohansen 	uint64_t		b_size;
497ac05c741SMark Maybee 	uint64_t		b_spa;
498fa9e4066Sahrens 
499fa9e4066Sahrens 	/* protected by arc state mutex */
500fa9e4066Sahrens 	arc_state_t		*b_state;
501fa9e4066Sahrens 	list_node_t		b_arc_node;
502fa9e4066Sahrens 
503fa9e4066Sahrens 	/* updated atomically */
504fa9e4066Sahrens 	clock_t			b_arc_access;
505fa9e4066Sahrens 
506fa9e4066Sahrens 	/* self protecting */
507fa9e4066Sahrens 	refcount_t		b_refcnt;
508fa94a07fSbrendan 
509fa94a07fSbrendan 	l2arc_buf_hdr_t		*b_l2hdr;
510fa94a07fSbrendan 	list_node_t		b_l2node;
511fa9e4066Sahrens };
512fa9e4066Sahrens 
513ea8dc4b6Seschrock static arc_buf_t *arc_eviction_list;
514ea8dc4b6Seschrock static kmutex_t arc_eviction_mtx;
51540d7d650Smaybee static arc_buf_hdr_t arc_eviction_hdr;
51644eda4d7Smaybee static void arc_get_data_buf(arc_buf_t *buf);
51744eda4d7Smaybee static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock);
5180e8c6158Smaybee static int arc_evict_needed(arc_buf_contents_t type);
519ac05c741SMark Maybee static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes);
520cd1c8b85SMatthew Ahrens static void arc_buf_watch(arc_buf_t *buf);
521ea8dc4b6Seschrock 
5225ea40c06SBrendan Gregg - Sun Microsystems static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab);
5235ea40c06SBrendan Gregg - Sun Microsystems 
524ea8dc4b6Seschrock #define	GHOST_STATE(state)	\
525fa94a07fSbrendan 	((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||	\
526fa94a07fSbrendan 	(state) == arc_l2c_only)
527ea8dc4b6Seschrock 
528fa9e4066Sahrens /*
529fa9e4066Sahrens  * Private ARC flags.  These flags are private ARC only flags that will show up
530fa9e4066Sahrens  * in b_flags in the arc_hdr_buf_t.  Some flags are publicly declared, and can
531fa9e4066Sahrens  * be passed in as arc_flags in things like arc_read.  However, these flags
532fa9e4066Sahrens  * should never be passed and should only be set by ARC code.  When adding new
533fa9e4066Sahrens  * public flags, make sure not to smash the private ones.
534fa9e4066Sahrens  */
535fa9e4066Sahrens 
536ea8dc4b6Seschrock #define	ARC_IN_HASH_TABLE	(1 << 9)	/* this buffer is hashed */
537fa9e4066Sahrens #define	ARC_IO_IN_PROGRESS	(1 << 10)	/* I/O in progress for buf */
538fa9e4066Sahrens #define	ARC_IO_ERROR		(1 << 11)	/* I/O failed for buf */
539fa9e4066Sahrens #define	ARC_FREED_IN_READ	(1 << 12)	/* buf freed while in read */
540ea8dc4b6Seschrock #define	ARC_BUF_AVAILABLE	(1 << 13)	/* block not in active use */
54113506d1eSmaybee #define	ARC_INDIRECT		(1 << 14)	/* this is an indirect block */
542fa94a07fSbrendan #define	ARC_FREE_IN_PROGRESS	(1 << 15)	/* hdr about to be freed */
5433baa08fcSek #define	ARC_L2_WRITING		(1 << 16)	/* L2ARC write in progress */
5443baa08fcSek #define	ARC_L2_EVICTED		(1 << 17)	/* evicted during I/O */
5453baa08fcSek #define	ARC_L2_WRITE_HEAD	(1 << 18)	/* head of write list */
546fa9e4066Sahrens 
547ea8dc4b6Seschrock #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_IN_HASH_TABLE)
548fa9e4066Sahrens #define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS)
549fa9e4066Sahrens #define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_IO_ERROR)
5505a98e54bSBrendan Gregg - Sun Microsystems #define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_PREFETCH)
551fa9e4066Sahrens #define	HDR_FREED_IN_READ(hdr)	((hdr)->b_flags & ARC_FREED_IN_READ)
552ea8dc4b6Seschrock #define	HDR_BUF_AVAILABLE(hdr)	((hdr)->b_flags & ARC_BUF_AVAILABLE)
553fa94a07fSbrendan #define	HDR_FREE_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FREE_IN_PROGRESS)
5543baa08fcSek #define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_L2CACHE)
5553a737e0dSbrendan #define	HDR_L2_READING(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS &&	\
5563a737e0dSbrendan 				    (hdr)->b_l2hdr != NULL)
557fa94a07fSbrendan #define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_L2_WRITING)
558fa94a07fSbrendan #define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_L2_EVICTED)
559fa94a07fSbrendan #define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_L2_WRITE_HEAD)
560fa9e4066Sahrens 
561e6c728e1Sbrendan /*
562e6c728e1Sbrendan  * Other sizes
563e6c728e1Sbrendan  */
564e6c728e1Sbrendan 
565e6c728e1Sbrendan #define	HDR_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
566e6c728e1Sbrendan #define	L2HDR_SIZE ((int64_t)sizeof (l2arc_buf_hdr_t))
567e6c728e1Sbrendan 
568fa9e4066Sahrens /*
569fa9e4066Sahrens  * Hash table routines
570fa9e4066Sahrens  */
571fa9e4066Sahrens 
572fa9e4066Sahrens #define	HT_LOCK_PAD	64
573fa9e4066Sahrens 
574fa9e4066Sahrens struct ht_lock {
575fa9e4066Sahrens 	kmutex_t	ht_lock;
576fa9e4066Sahrens #ifdef _KERNEL
577fa9e4066Sahrens 	unsigned char	pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
578fa9e4066Sahrens #endif
579fa9e4066Sahrens };
580fa9e4066Sahrens 
581fa9e4066Sahrens #define	BUF_LOCKS 256
582fa9e4066Sahrens typedef struct buf_hash_table {
583fa9e4066Sahrens 	uint64_t ht_mask;
584fa9e4066Sahrens 	arc_buf_hdr_t **ht_table;
585fa9e4066Sahrens 	struct ht_lock ht_locks[BUF_LOCKS];
586fa9e4066Sahrens } buf_hash_table_t;
587fa9e4066Sahrens 
588fa9e4066Sahrens static buf_hash_table_t buf_hash_table;
589fa9e4066Sahrens 
590fa9e4066Sahrens #define	BUF_HASH_INDEX(spa, dva, birth) \
591fa9e4066Sahrens 	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
592fa9e4066Sahrens #define	BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
593fa9e4066Sahrens #define	BUF_HASH_LOCK(idx)	(&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
5943f9d6ad7SLin Ling #define	HDR_LOCK(hdr) \
5953f9d6ad7SLin Ling 	(BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
596fa9e4066Sahrens 
597fa9e4066Sahrens uint64_t zfs_crc64_table[256];
598fa9e4066Sahrens 
599fa94a07fSbrendan /*
600fa94a07fSbrendan  * Level 2 ARC
601fa94a07fSbrendan  */
602fa94a07fSbrendan 
603fa94a07fSbrendan #define	L2ARC_WRITE_SIZE	(8 * 1024 * 1024)	/* initial write max */
604aad02571SSaso Kiselkov #define	L2ARC_HEADROOM		2			/* num of writes */
605aad02571SSaso Kiselkov /*
606aad02571SSaso Kiselkov  * If we discover during ARC scan any buffers to be compressed, we boost
607aad02571SSaso Kiselkov  * our headroom for the next scanning cycle by this percentage multiple.
608aad02571SSaso Kiselkov  */
609aad02571SSaso Kiselkov #define	L2ARC_HEADROOM_BOOST	200
6105a98e54bSBrendan Gregg - Sun Microsystems #define	L2ARC_FEED_SECS		1		/* caching interval secs */
6115a98e54bSBrendan Gregg - Sun Microsystems #define	L2ARC_FEED_MIN_MS	200		/* min caching interval ms */
612fa94a07fSbrendan 
613fa94a07fSbrendan #define	l2arc_writes_sent	ARCSTAT(arcstat_l2_writes_sent)
614fa94a07fSbrendan #define	l2arc_writes_done	ARCSTAT(arcstat_l2_writes_done)
615fa94a07fSbrendan 
616*f7170741SWill Andrews /* L2ARC Performance Tunables */
617fa94a07fSbrendan uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;	/* default max write size */
6183a737e0dSbrendan uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;	/* extra write during warmup */
619fa94a07fSbrendan uint64_t l2arc_headroom = L2ARC_HEADROOM;	/* number of dev writes */
620aad02571SSaso Kiselkov uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
621fa94a07fSbrendan uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;	/* interval seconds */
6225a98e54bSBrendan Gregg - Sun Microsystems uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS;	/* min interval milliseconds */
623fa94a07fSbrendan boolean_t l2arc_noprefetch = B_TRUE;		/* don't cache prefetch bufs */
6245a98e54bSBrendan Gregg - Sun Microsystems boolean_t l2arc_feed_again = B_TRUE;		/* turbo warmup */
6255a98e54bSBrendan Gregg - Sun Microsystems boolean_t l2arc_norw = B_TRUE;			/* no reads during writes */
626fa94a07fSbrendan 
627fa94a07fSbrendan /*
628fa94a07fSbrendan  * L2ARC Internals
629fa94a07fSbrendan  */
630fa94a07fSbrendan typedef struct l2arc_dev {
631fa94a07fSbrendan 	vdev_t			*l2ad_vdev;	/* vdev */
632fa94a07fSbrendan 	spa_t			*l2ad_spa;	/* spa */
633fa94a07fSbrendan 	uint64_t		l2ad_hand;	/* next write location */
634fa94a07fSbrendan 	uint64_t		l2ad_start;	/* first addr on device */
635fa94a07fSbrendan 	uint64_t		l2ad_end;	/* last addr on device */
636fa94a07fSbrendan 	uint64_t		l2ad_evict;	/* last addr eviction reached */
637fa94a07fSbrendan 	boolean_t		l2ad_first;	/* first sweep through */
6385a98e54bSBrendan Gregg - Sun Microsystems 	boolean_t		l2ad_writing;	/* currently writing */
639fa94a07fSbrendan 	list_t			*l2ad_buflist;	/* buffer list */
640fa94a07fSbrendan 	list_node_t		l2ad_node;	/* device list node */
641fa94a07fSbrendan } l2arc_dev_t;
642fa94a07fSbrendan 
643fa94a07fSbrendan static list_t L2ARC_dev_list;			/* device list */
644fa94a07fSbrendan static list_t *l2arc_dev_list;			/* device list pointer */
645fa94a07fSbrendan static kmutex_t l2arc_dev_mtx;			/* device list mutex */
646fa94a07fSbrendan static l2arc_dev_t *l2arc_dev_last;		/* last device used */
647fa94a07fSbrendan static kmutex_t l2arc_buflist_mtx;		/* mutex for all buflists */
648fa94a07fSbrendan static list_t L2ARC_free_on_write;		/* free after write buf list */
649fa94a07fSbrendan static list_t *l2arc_free_on_write;		/* free after write list ptr */
650fa94a07fSbrendan static kmutex_t l2arc_free_on_write_mtx;	/* mutex for list */
651fa94a07fSbrendan static uint64_t l2arc_ndev;			/* number of devices */
652fa94a07fSbrendan 
653fa94a07fSbrendan typedef struct l2arc_read_callback {
654aad02571SSaso Kiselkov 	arc_buf_t		*l2rcb_buf;		/* read buffer */
655aad02571SSaso Kiselkov 	spa_t			*l2rcb_spa;		/* spa */
656aad02571SSaso Kiselkov 	blkptr_t		l2rcb_bp;		/* original blkptr */
657aad02571SSaso Kiselkov 	zbookmark_t		l2rcb_zb;		/* original bookmark */
658aad02571SSaso Kiselkov 	int			l2rcb_flags;		/* original flags */
659aad02571SSaso Kiselkov 	enum zio_compress	l2rcb_compress;		/* applied compress */
660fa94a07fSbrendan } l2arc_read_callback_t;
661fa94a07fSbrendan 
662fa94a07fSbrendan typedef struct l2arc_write_callback {
663fa94a07fSbrendan 	l2arc_dev_t	*l2wcb_dev;		/* device info */
664fa94a07fSbrendan 	arc_buf_hdr_t	*l2wcb_head;		/* head of write buflist */
665fa94a07fSbrendan } l2arc_write_callback_t;
666fa94a07fSbrendan 
667fa94a07fSbrendan struct l2arc_buf_hdr {
668fa94a07fSbrendan 	/* protected by arc_buf_hdr  mutex */
669aad02571SSaso Kiselkov 	l2arc_dev_t		*b_dev;		/* L2ARC device */
670aad02571SSaso Kiselkov 	uint64_t		b_daddr;	/* disk address, offset byte */
671aad02571SSaso Kiselkov 	/* compression applied to buffer data */
672aad02571SSaso Kiselkov 	enum zio_compress	b_compress;
673aad02571SSaso Kiselkov 	/* real alloc'd buffer size depending on b_compress applied */
674aad02571SSaso Kiselkov 	int			b_asize;
675aad02571SSaso Kiselkov 	/* temporary buffer holder for in-flight compressed data */
676aad02571SSaso Kiselkov 	void			*b_tmp_cdata;
677fa94a07fSbrendan };
678fa94a07fSbrendan 
679fa94a07fSbrendan typedef struct l2arc_data_free {
680fa94a07fSbrendan 	/* protected by l2arc_free_on_write_mtx */
681fa94a07fSbrendan 	void		*l2df_data;
682fa94a07fSbrendan 	size_t		l2df_size;
683fa94a07fSbrendan 	void		(*l2df_func)(void *, size_t);
684fa94a07fSbrendan 	list_node_t	l2df_list_node;
685fa94a07fSbrendan } l2arc_data_free_t;
686fa94a07fSbrendan 
687fa94a07fSbrendan static kmutex_t l2arc_feed_thr_lock;
688fa94a07fSbrendan static kcondvar_t l2arc_feed_thr_cv;
689fa94a07fSbrendan static uint8_t l2arc_thread_exit;
690fa94a07fSbrendan 
691fa94a07fSbrendan static void l2arc_read_done(zio_t *zio);
692fa94a07fSbrendan static void l2arc_hdr_stat_add(void);
693fa94a07fSbrendan static void l2arc_hdr_stat_remove(void);
694fa94a07fSbrendan 
695aad02571SSaso Kiselkov static boolean_t l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr);
696aad02571SSaso Kiselkov static void l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr,
697aad02571SSaso Kiselkov     enum zio_compress c);
698aad02571SSaso Kiselkov static void l2arc_release_cdata_buf(arc_buf_hdr_t *ab);
699aad02571SSaso Kiselkov 
700fa9e4066Sahrens static uint64_t
701ac05c741SMark Maybee buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
702fa9e4066Sahrens {
703fa9e4066Sahrens 	uint8_t *vdva = (uint8_t *)dva;
704fa9e4066Sahrens 	uint64_t crc = -1ULL;
705fa9e4066Sahrens 	int i;
706fa9e4066Sahrens 
707fa9e4066Sahrens 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
708fa9e4066Sahrens 
709fa9e4066Sahrens 	for (i = 0; i < sizeof (dva_t); i++)
710fa9e4066Sahrens 		crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
711fa9e4066Sahrens 
712ac05c741SMark Maybee 	crc ^= (spa>>8) ^ birth;
713fa9e4066Sahrens 
714fa9e4066Sahrens 	return (crc);
715fa9e4066Sahrens }
716fa9e4066Sahrens 
717fa9e4066Sahrens #define	BUF_EMPTY(buf)						\
718fa9e4066Sahrens 	((buf)->b_dva.dva_word[0] == 0 &&			\
719fa9e4066Sahrens 	(buf)->b_dva.dva_word[1] == 0 &&			\
720fa9e4066Sahrens 	(buf)->b_birth == 0)
721fa9e4066Sahrens 
722fa9e4066Sahrens #define	BUF_EQUAL(spa, dva, birth, buf)				\
723fa9e4066Sahrens 	((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
724fa9e4066Sahrens 	((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
725fa9e4066Sahrens 	((buf)->b_birth == birth) && ((buf)->b_spa == spa)
726fa9e4066Sahrens 
7273f9d6ad7SLin Ling static void
7283f9d6ad7SLin Ling buf_discard_identity(arc_buf_hdr_t *hdr)
7293f9d6ad7SLin Ling {
7303f9d6ad7SLin Ling 	hdr->b_dva.dva_word[0] = 0;
7313f9d6ad7SLin Ling 	hdr->b_dva.dva_word[1] = 0;
7323f9d6ad7SLin Ling 	hdr->b_birth = 0;
7333f9d6ad7SLin Ling 	hdr->b_cksum0 = 0;
7343f9d6ad7SLin Ling }
7353f9d6ad7SLin Ling 
736fa9e4066Sahrens static arc_buf_hdr_t *
737ac05c741SMark Maybee buf_hash_find(uint64_t spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp)
738fa9e4066Sahrens {
739fa9e4066Sahrens 	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
740fa9e4066Sahrens 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
741fa9e4066Sahrens 	arc_buf_hdr_t *buf;
742fa9e4066Sahrens 
743fa9e4066Sahrens 	mutex_enter(hash_lock);
744fa9e4066Sahrens 	for (buf = buf_hash_table.ht_table[idx]; buf != NULL;
745fa9e4066Sahrens 	    buf = buf->b_hash_next) {
746fa9e4066Sahrens 		if (BUF_EQUAL(spa, dva, birth, buf)) {
747fa9e4066Sahrens 			*lockp = hash_lock;
748fa9e4066Sahrens 			return (buf);
749fa9e4066Sahrens 		}
750fa9e4066Sahrens 	}
751fa9e4066Sahrens 	mutex_exit(hash_lock);
752fa9e4066Sahrens 	*lockp = NULL;
753fa9e4066Sahrens 	return (NULL);
754fa9e4066Sahrens }
755fa9e4066Sahrens 
756fa9e4066Sahrens /*
757fa9e4066Sahrens  * Insert an entry into the hash table.  If there is already an element
758fa9e4066Sahrens  * equal to elem in the hash table, then the already existing element
759fa9e4066Sahrens  * will be returned and the new element will not be inserted.
760fa9e4066Sahrens  * Otherwise returns NULL.
761fa9e4066Sahrens  */
762fa9e4066Sahrens static arc_buf_hdr_t *
763fa9e4066Sahrens buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp)
764fa9e4066Sahrens {
765fa9e4066Sahrens 	uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
766fa9e4066Sahrens 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
767fa9e4066Sahrens 	arc_buf_hdr_t *fbuf;
76844cb6abcSbmc 	uint32_t i;
769fa9e4066Sahrens 
770ea8dc4b6Seschrock 	ASSERT(!HDR_IN_HASH_TABLE(buf));
771fa9e4066Sahrens 	*lockp = hash_lock;
772fa9e4066Sahrens 	mutex_enter(hash_lock);
773fa9e4066Sahrens 	for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL;
774fa9e4066Sahrens 	    fbuf = fbuf->b_hash_next, i++) {
775fa9e4066Sahrens 		if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf))
776fa9e4066Sahrens 			return (fbuf);
777fa9e4066Sahrens 	}
778fa9e4066Sahrens 
779fa9e4066Sahrens 	buf->b_hash_next = buf_hash_table.ht_table[idx];
780fa9e4066Sahrens 	buf_hash_table.ht_table[idx] = buf;
781ea8dc4b6Seschrock 	buf->b_flags |= ARC_IN_HASH_TABLE;
782fa9e4066Sahrens 
783fa9e4066Sahrens 	/* collect some hash table performance data */
784fa9e4066Sahrens 	if (i > 0) {
78544cb6abcSbmc 		ARCSTAT_BUMP(arcstat_hash_collisions);
786fa9e4066Sahrens 		if (i == 1)
78744cb6abcSbmc 			ARCSTAT_BUMP(arcstat_hash_chains);
78844cb6abcSbmc 
78944cb6abcSbmc 		ARCSTAT_MAX(arcstat_hash_chain_max, i);
790fa9e4066Sahrens 	}
79144cb6abcSbmc 
79244cb6abcSbmc 	ARCSTAT_BUMP(arcstat_hash_elements);
79344cb6abcSbmc 	ARCSTAT_MAXSTAT(arcstat_hash_elements);
794fa9e4066Sahrens 
795fa9e4066Sahrens 	return (NULL);
796fa9e4066Sahrens }
797fa9e4066Sahrens 
798fa9e4066Sahrens static void
799fa9e4066Sahrens buf_hash_remove(arc_buf_hdr_t *buf)
800fa9e4066Sahrens {
801fa9e4066Sahrens 	arc_buf_hdr_t *fbuf, **bufp;
802fa9e4066Sahrens 	uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
803fa9e4066Sahrens 
804fa9e4066Sahrens 	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
805ea8dc4b6Seschrock 	ASSERT(HDR_IN_HASH_TABLE(buf));
806fa9e4066Sahrens 
807fa9e4066Sahrens 	bufp = &buf_hash_table.ht_table[idx];
808fa9e4066Sahrens 	while ((fbuf = *bufp) != buf) {
809fa9e4066Sahrens 		ASSERT(fbuf != NULL);
810fa9e4066Sahrens 		bufp = &fbuf->b_hash_next;
811fa9e4066Sahrens 	}
812fa9e4066Sahrens 	*bufp = buf->b_hash_next;
813fa9e4066Sahrens 	buf->b_hash_next = NULL;
814ea8dc4b6Seschrock 	buf->b_flags &= ~ARC_IN_HASH_TABLE;
815fa9e4066Sahrens 
816fa9e4066Sahrens 	/* collect some hash table performance data */
81744cb6abcSbmc 	ARCSTAT_BUMPDOWN(arcstat_hash_elements);
81844cb6abcSbmc 
819fa9e4066Sahrens 	if (buf_hash_table.ht_table[idx] &&
820fa9e4066Sahrens 	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
82144cb6abcSbmc 		ARCSTAT_BUMPDOWN(arcstat_hash_chains);
822fa9e4066Sahrens }
823fa9e4066Sahrens 
824fa9e4066Sahrens /*
825fa9e4066Sahrens  * Global data structures and functions for the buf kmem cache.
826fa9e4066Sahrens  */
827fa9e4066Sahrens static kmem_cache_t *hdr_cache;
828fa9e4066Sahrens static kmem_cache_t *buf_cache;
829fa9e4066Sahrens 
830fa9e4066Sahrens static void
831fa9e4066Sahrens buf_fini(void)
832fa9e4066Sahrens {
833fa9e4066Sahrens 	int i;
834fa9e4066Sahrens 
835fa9e4066Sahrens 	kmem_free(buf_hash_table.ht_table,
836fa9e4066Sahrens 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
837fa9e4066Sahrens 	for (i = 0; i < BUF_LOCKS; i++)
838fa9e4066Sahrens 		mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
839fa9e4066Sahrens 	kmem_cache_destroy(hdr_cache);
840fa9e4066Sahrens 	kmem_cache_destroy(buf_cache);
841fa9e4066Sahrens }
842fa9e4066Sahrens 
843fa9e4066Sahrens /*
844fa9e4066Sahrens  * Constructor callback - called when the cache is empty
845fa9e4066Sahrens  * and a new buf is requested.
846fa9e4066Sahrens  */
847fa9e4066Sahrens /* ARGSUSED */
848fa9e4066Sahrens static int
849fa9e4066Sahrens hdr_cons(void *vbuf, void *unused, int kmflag)
850fa9e4066Sahrens {
851fa9e4066Sahrens 	arc_buf_hdr_t *buf = vbuf;
852fa9e4066Sahrens 
853fa9e4066Sahrens 	bzero(buf, sizeof (arc_buf_hdr_t));
854fa9e4066Sahrens 	refcount_create(&buf->b_refcnt);
855fa9e4066Sahrens 	cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL);
856c25056deSgw 	mutex_init(&buf->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
8575a98e54bSBrendan Gregg - Sun Microsystems 	arc_space_consume(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
858fa94a07fSbrendan 
859fa9e4066Sahrens 	return (0);
860fa9e4066Sahrens }
861fa9e4066Sahrens 
8626f83844dSMark Maybee /* ARGSUSED */
8636f83844dSMark Maybee static int
8646f83844dSMark Maybee buf_cons(void *vbuf, void *unused, int kmflag)
8656f83844dSMark Maybee {
8666f83844dSMark Maybee 	arc_buf_t *buf = vbuf;
8676f83844dSMark Maybee 
8686f83844dSMark Maybee 	bzero(buf, sizeof (arc_buf_t));
8693f9d6ad7SLin Ling 	mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL);
8705a98e54bSBrendan Gregg - Sun Microsystems 	arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
8715a98e54bSBrendan Gregg - Sun Microsystems 
8726f83844dSMark Maybee 	return (0);
8736f83844dSMark Maybee }
8746f83844dSMark Maybee 
875fa9e4066Sahrens /*
876fa9e4066Sahrens  * Destructor callback - called when a cached buf is
877fa9e4066Sahrens  * no longer required.
878fa9e4066Sahrens  */
879fa9e4066Sahrens /* ARGSUSED */
880fa9e4066Sahrens static void
881fa9e4066Sahrens hdr_dest(void *vbuf, void *unused)
882fa9e4066Sahrens {
883fa9e4066Sahrens 	arc_buf_hdr_t *buf = vbuf;
884fa9e4066Sahrens 
885b24ab676SJeff Bonwick 	ASSERT(BUF_EMPTY(buf));
886fa9e4066Sahrens 	refcount_destroy(&buf->b_refcnt);
887fa9e4066Sahrens 	cv_destroy(&buf->b_cv);
888c25056deSgw 	mutex_destroy(&buf->b_freeze_lock);
8895a98e54bSBrendan Gregg - Sun Microsystems 	arc_space_return(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
890fa9e4066Sahrens }
891fa9e4066Sahrens 
8926f83844dSMark Maybee /* ARGSUSED */
8936f83844dSMark Maybee static void
8946f83844dSMark Maybee buf_dest(void *vbuf, void *unused)
8956f83844dSMark Maybee {
8966f83844dSMark Maybee 	arc_buf_t *buf = vbuf;
8976f83844dSMark Maybee 
8983f9d6ad7SLin Ling 	mutex_destroy(&buf->b_evict_lock);
8995a98e54bSBrendan Gregg - Sun Microsystems 	arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
9006f83844dSMark Maybee }
9016f83844dSMark Maybee 
902fa9e4066Sahrens /*
903fa9e4066Sahrens  * Reclaim callback -- invoked when memory is low.
904fa9e4066Sahrens  */
905fa9e4066Sahrens /* ARGSUSED */
906fa9e4066Sahrens static void
907fa9e4066Sahrens hdr_recl(void *unused)
908fa9e4066Sahrens {
909fa9e4066Sahrens 	dprintf("hdr_recl called\n");
91049e3519aSmaybee 	/*
91149e3519aSmaybee 	 * umem calls the reclaim func when we destroy the buf cache,
91249e3519aSmaybee 	 * which is after we do arc_fini().
91349e3519aSmaybee 	 */
91449e3519aSmaybee 	if (!arc_dead)
91549e3519aSmaybee 		cv_signal(&arc_reclaim_thr_cv);
916fa9e4066Sahrens }
917fa9e4066Sahrens 
918fa9e4066Sahrens static void
919fa9e4066Sahrens buf_init(void)
920fa9e4066Sahrens {
921fa9e4066Sahrens 	uint64_t *ct;
922ea8dc4b6Seschrock 	uint64_t hsize = 1ULL << 12;
923fa9e4066Sahrens 	int i, j;
924fa9e4066Sahrens 
925fa9e4066Sahrens 	/*
926fa9e4066Sahrens 	 * The hash table is big enough to fill all of physical memory
927ea8dc4b6Seschrock 	 * with an average 64K block size.  The table will take up
928ea8dc4b6Seschrock 	 * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers).
929fa9e4066Sahrens 	 */
930ea8dc4b6Seschrock 	while (hsize * 65536 < physmem * PAGESIZE)
931fa9e4066Sahrens 		hsize <<= 1;
932ea8dc4b6Seschrock retry:
933fa9e4066Sahrens 	buf_hash_table.ht_mask = hsize - 1;
934ea8dc4b6Seschrock 	buf_hash_table.ht_table =
935ea8dc4b6Seschrock 	    kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
936ea8dc4b6Seschrock 	if (buf_hash_table.ht_table == NULL) {
937ea8dc4b6Seschrock 		ASSERT(hsize > (1ULL << 8));
938ea8dc4b6Seschrock 		hsize >>= 1;
939ea8dc4b6Seschrock 		goto retry;
940ea8dc4b6Seschrock 	}
941fa9e4066Sahrens 
942fa9e4066Sahrens 	hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t),
943fa9e4066Sahrens 	    0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0);
944fa9e4066Sahrens 	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
9456f83844dSMark Maybee 	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
946fa9e4066Sahrens 
947fa9e4066Sahrens 	for (i = 0; i < 256; i++)
948fa9e4066Sahrens 		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
949fa9e4066Sahrens 			*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
950fa9e4066Sahrens 
951fa9e4066Sahrens 	for (i = 0; i < BUF_LOCKS; i++) {
952fa9e4066Sahrens 		mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
953fa9e4066Sahrens 		    NULL, MUTEX_DEFAULT, NULL);
954fa9e4066Sahrens 	}
955fa9e4066Sahrens }
956fa9e4066Sahrens 
957fa9e4066Sahrens #define	ARC_MINTIME	(hz>>4) /* 62 ms */
958fa9e4066Sahrens 
9596b4acc8bSahrens static void
9606b4acc8bSahrens arc_cksum_verify(arc_buf_t *buf)
9616b4acc8bSahrens {
9626b4acc8bSahrens 	zio_cksum_t zc;
9636b4acc8bSahrens 
964cc60fd72Sahrens 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
9656b4acc8bSahrens 		return;
9666b4acc8bSahrens 
9676b4acc8bSahrens 	mutex_enter(&buf->b_hdr->b_freeze_lock);
9683ccfa83cSahrens 	if (buf->b_hdr->b_freeze_cksum == NULL ||
9693ccfa83cSahrens 	    (buf->b_hdr->b_flags & ARC_IO_ERROR)) {
9706b4acc8bSahrens 		mutex_exit(&buf->b_hdr->b_freeze_lock);
9716b4acc8bSahrens 		return;
9726b4acc8bSahrens 	}
9736b4acc8bSahrens 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
9746b4acc8bSahrens 	if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc))
9756b4acc8bSahrens 		panic("buffer modified while frozen!");
9766b4acc8bSahrens 	mutex_exit(&buf->b_hdr->b_freeze_lock);
9776b4acc8bSahrens }
9786b4acc8bSahrens 
979fa94a07fSbrendan static int
980fa94a07fSbrendan arc_cksum_equal(arc_buf_t *buf)
981fa94a07fSbrendan {
982fa94a07fSbrendan 	zio_cksum_t zc;
983fa94a07fSbrendan 	int equal;
984fa94a07fSbrendan 
985fa94a07fSbrendan 	mutex_enter(&buf->b_hdr->b_freeze_lock);
986fa94a07fSbrendan 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
987fa94a07fSbrendan 	equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc);
988fa94a07fSbrendan 	mutex_exit(&buf->b_hdr->b_freeze_lock);
989fa94a07fSbrendan 
990fa94a07fSbrendan 	return (equal);
991fa94a07fSbrendan }
992fa94a07fSbrendan 
9936b4acc8bSahrens static void
994fa94a07fSbrendan arc_cksum_compute(arc_buf_t *buf, boolean_t force)
9956b4acc8bSahrens {
996fa94a07fSbrendan 	if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY))
9976b4acc8bSahrens 		return;
9986b4acc8bSahrens 
9996b4acc8bSahrens 	mutex_enter(&buf->b_hdr->b_freeze_lock);
10006b4acc8bSahrens 	if (buf->b_hdr->b_freeze_cksum != NULL) {
10016b4acc8bSahrens 		mutex_exit(&buf->b_hdr->b_freeze_lock);
10026b4acc8bSahrens 		return;
10036b4acc8bSahrens 	}
10046b4acc8bSahrens 	buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
10056b4acc8bSahrens 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
10066b4acc8bSahrens 	    buf->b_hdr->b_freeze_cksum);
10076b4acc8bSahrens 	mutex_exit(&buf->b_hdr->b_freeze_lock);
1008cd1c8b85SMatthew Ahrens 	arc_buf_watch(buf);
1009cd1c8b85SMatthew Ahrens }
1010cd1c8b85SMatthew Ahrens 
1011cd1c8b85SMatthew Ahrens #ifndef _KERNEL
1012cd1c8b85SMatthew Ahrens typedef struct procctl {
1013cd1c8b85SMatthew Ahrens 	long cmd;
1014cd1c8b85SMatthew Ahrens 	prwatch_t prwatch;
1015cd1c8b85SMatthew Ahrens } procctl_t;
1016cd1c8b85SMatthew Ahrens #endif
1017cd1c8b85SMatthew Ahrens 
1018cd1c8b85SMatthew Ahrens /* ARGSUSED */
1019cd1c8b85SMatthew Ahrens static void
1020cd1c8b85SMatthew Ahrens arc_buf_unwatch(arc_buf_t *buf)
1021cd1c8b85SMatthew Ahrens {
1022cd1c8b85SMatthew Ahrens #ifndef _KERNEL
1023cd1c8b85SMatthew Ahrens 	if (arc_watch) {
1024cd1c8b85SMatthew Ahrens 		int result;
1025cd1c8b85SMatthew Ahrens 		procctl_t ctl;
1026cd1c8b85SMatthew Ahrens 		ctl.cmd = PCWATCH;
1027cd1c8b85SMatthew Ahrens 		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1028cd1c8b85SMatthew Ahrens 		ctl.prwatch.pr_size = 0;
1029cd1c8b85SMatthew Ahrens 		ctl.prwatch.pr_wflags = 0;
1030cd1c8b85SMatthew Ahrens 		result = write(arc_procfd, &ctl, sizeof (ctl));
1031cd1c8b85SMatthew Ahrens 		ASSERT3U(result, ==, sizeof (ctl));
1032cd1c8b85SMatthew Ahrens 	}
1033cd1c8b85SMatthew Ahrens #endif
1034cd1c8b85SMatthew Ahrens }
1035cd1c8b85SMatthew Ahrens 
1036cd1c8b85SMatthew Ahrens /* ARGSUSED */
1037cd1c8b85SMatthew Ahrens static void
1038cd1c8b85SMatthew Ahrens arc_buf_watch(arc_buf_t *buf)
1039cd1c8b85SMatthew Ahrens {
1040cd1c8b85SMatthew Ahrens #ifndef _KERNEL
1041cd1c8b85SMatthew Ahrens 	if (arc_watch) {
1042cd1c8b85SMatthew Ahrens 		int result;
1043cd1c8b85SMatthew Ahrens 		procctl_t ctl;
1044cd1c8b85SMatthew Ahrens 		ctl.cmd = PCWATCH;
1045cd1c8b85SMatthew Ahrens 		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1046cd1c8b85SMatthew Ahrens 		ctl.prwatch.pr_size = buf->b_hdr->b_size;
1047cd1c8b85SMatthew Ahrens 		ctl.prwatch.pr_wflags = WA_WRITE;
1048cd1c8b85SMatthew Ahrens 		result = write(arc_procfd, &ctl, sizeof (ctl));
1049cd1c8b85SMatthew Ahrens 		ASSERT3U(result, ==, sizeof (ctl));
1050cd1c8b85SMatthew Ahrens 	}
1051cd1c8b85SMatthew Ahrens #endif
10526b4acc8bSahrens }
10536b4acc8bSahrens 
10546b4acc8bSahrens void
10556b4acc8bSahrens arc_buf_thaw(arc_buf_t *buf)
10566b4acc8bSahrens {
1057fa94a07fSbrendan 	if (zfs_flags & ZFS_DEBUG_MODIFY) {
1058fa94a07fSbrendan 		if (buf->b_hdr->b_state != arc_anon)
1059fa94a07fSbrendan 			panic("modifying non-anon buffer!");
1060fa94a07fSbrendan 		if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS)
1061fa94a07fSbrendan 			panic("modifying buffer while i/o in progress!");
1062fa94a07fSbrendan 		arc_cksum_verify(buf);
1063fa94a07fSbrendan 	}
10646b4acc8bSahrens 
10656b4acc8bSahrens 	mutex_enter(&buf->b_hdr->b_freeze_lock);
10666b4acc8bSahrens 	if (buf->b_hdr->b_freeze_cksum != NULL) {
10676b4acc8bSahrens 		kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t));
10686b4acc8bSahrens 		buf->b_hdr->b_freeze_cksum = NULL;
10696b4acc8bSahrens 	}
10703f9d6ad7SLin Ling 
10713f9d6ad7SLin Ling 	if (zfs_flags & ZFS_DEBUG_MODIFY) {
10723f9d6ad7SLin Ling 		if (buf->b_hdr->b_thawed)
10733f9d6ad7SLin Ling 			kmem_free(buf->b_hdr->b_thawed, 1);
10743f9d6ad7SLin Ling 		buf->b_hdr->b_thawed = kmem_alloc(1, KM_SLEEP);
10753f9d6ad7SLin Ling 	}
10763f9d6ad7SLin Ling 
10776b4acc8bSahrens 	mutex_exit(&buf->b_hdr->b_freeze_lock);
1078cd1c8b85SMatthew Ahrens 
1079cd1c8b85SMatthew Ahrens 	arc_buf_unwatch(buf);
10806b4acc8bSahrens }
10816b4acc8bSahrens 
10826b4acc8bSahrens void
10836b4acc8bSahrens arc_buf_freeze(arc_buf_t *buf)
10846b4acc8bSahrens {
10853f9d6ad7SLin Ling 	kmutex_t *hash_lock;
10863f9d6ad7SLin Ling 
1087cc60fd72Sahrens 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1088cc60fd72Sahrens 		return;
1089cc60fd72Sahrens 
10903f9d6ad7SLin Ling 	hash_lock = HDR_LOCK(buf->b_hdr);
10913f9d6ad7SLin Ling 	mutex_enter(hash_lock);
10923f9d6ad7SLin Ling 
10936b4acc8bSahrens 	ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
109444cb6abcSbmc 	    buf->b_hdr->b_state == arc_anon);
1095fa94a07fSbrendan 	arc_cksum_compute(buf, B_FALSE);
10963f9d6ad7SLin Ling 	mutex_exit(hash_lock);
1097cd1c8b85SMatthew Ahrens 
10986b4acc8bSahrens }
10996b4acc8bSahrens 
1100fa9e4066Sahrens static void
1101fa9e4066Sahrens add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
1102fa9e4066Sahrens {
1103fa9e4066Sahrens 	ASSERT(MUTEX_HELD(hash_lock));
1104fa9e4066Sahrens 
1105fa9e4066Sahrens 	if ((refcount_add(&ab->b_refcnt, tag) == 1) &&
110644cb6abcSbmc 	    (ab->b_state != arc_anon)) {
1107c0a81264Sek 		uint64_t delta = ab->b_size * ab->b_datacnt;
11080e8c6158Smaybee 		list_t *list = &ab->b_state->arcs_list[ab->b_type];
11090e8c6158Smaybee 		uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type];
1110fa9e4066Sahrens 
111144cb6abcSbmc 		ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx));
111244cb6abcSbmc 		mutex_enter(&ab->b_state->arcs_mtx);
1113fa9e4066Sahrens 		ASSERT(list_link_active(&ab->b_arc_node));
11140e8c6158Smaybee 		list_remove(list, ab);
1115ea8dc4b6Seschrock 		if (GHOST_STATE(ab->b_state)) {
1116fb09f5aaSMadhav Suresh 			ASSERT0(ab->b_datacnt);
1117ea8dc4b6Seschrock 			ASSERT3P(ab->b_buf, ==, NULL);
1118ea8dc4b6Seschrock 			delta = ab->b_size;
1119ea8dc4b6Seschrock 		}
1120ea8dc4b6Seschrock 		ASSERT(delta > 0);
11210e8c6158Smaybee 		ASSERT3U(*size, >=, delta);
11220e8c6158Smaybee 		atomic_add_64(size, -delta);
112344cb6abcSbmc 		mutex_exit(&ab->b_state->arcs_mtx);
1124088f3894Sahrens 		/* remove the prefetch flag if we get a reference */
112513506d1eSmaybee 		if (ab->b_flags & ARC_PREFETCH)
112613506d1eSmaybee 			ab->b_flags &= ~ARC_PREFETCH;
1127fa9e4066Sahrens 	}
1128fa9e4066Sahrens }
1129fa9e4066Sahrens 
1130fa9e4066Sahrens static int
1131fa9e4066Sahrens remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
1132fa9e4066Sahrens {
1133fa9e4066Sahrens 	int cnt;
113444cb6abcSbmc 	arc_state_t *state = ab->b_state;
1135fa9e4066Sahrens 
113644cb6abcSbmc 	ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
113744cb6abcSbmc 	ASSERT(!GHOST_STATE(state));
1138fa9e4066Sahrens 
1139fa9e4066Sahrens 	if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) &&
114044cb6abcSbmc 	    (state != arc_anon)) {
11410e8c6158Smaybee 		uint64_t *size = &state->arcs_lsize[ab->b_type];
11420e8c6158Smaybee 
114344cb6abcSbmc 		ASSERT(!MUTEX_HELD(&state->arcs_mtx));
114444cb6abcSbmc 		mutex_enter(&state->arcs_mtx);
1145fa9e4066Sahrens 		ASSERT(!list_link_active(&ab->b_arc_node));
11460e8c6158Smaybee 		list_insert_head(&state->arcs_list[ab->b_type], ab);
1147ea8dc4b6Seschrock 		ASSERT(ab->b_datacnt > 0);
11480e8c6158Smaybee 		atomic_add_64(size, ab->b_size * ab->b_datacnt);
114944cb6abcSbmc 		mutex_exit(&state->arcs_mtx);
1150fa9e4066Sahrens 	}
1151fa9e4066Sahrens 	return (cnt);
1152fa9e4066Sahrens }
1153fa9e4066Sahrens 
1154fa9e4066Sahrens /*
1155fa9e4066Sahrens  * Move the supplied buffer to the indicated state.  The mutex
1156fa9e4066Sahrens  * for the buffer must be held by the caller.
1157fa9e4066Sahrens  */
1158fa9e4066Sahrens static void
1159ea8dc4b6Seschrock arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock)
1160fa9e4066Sahrens {
1161ea8dc4b6Seschrock 	arc_state_t *old_state = ab->b_state;
1162c0a81264Sek 	int64_t refcnt = refcount_count(&ab->b_refcnt);
1163c0a81264Sek 	uint64_t from_delta, to_delta;
1164fa9e4066Sahrens 
1165fa9e4066Sahrens 	ASSERT(MUTEX_HELD(hash_lock));
1166ea8dc4b6Seschrock 	ASSERT(new_state != old_state);
1167ea8dc4b6Seschrock 	ASSERT(refcnt == 0 || ab->b_datacnt > 0);
1168ea8dc4b6Seschrock 	ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state));
1169b24ab676SJeff Bonwick 	ASSERT(ab->b_datacnt <= 1 || old_state != arc_anon);
1170ea8dc4b6Seschrock 
1171ea8dc4b6Seschrock 	from_delta = to_delta = ab->b_datacnt * ab->b_size;
1172fa9e4066Sahrens 
1173fa9e4066Sahrens 	/*
1174fa9e4066Sahrens 	 * If this buffer is evictable, transfer it from the
1175fa9e4066Sahrens 	 * old state list to the new state list.
1176fa9e4066Sahrens 	 */
1177ea8dc4b6Seschrock 	if (refcnt == 0) {
117844cb6abcSbmc 		if (old_state != arc_anon) {
117944cb6abcSbmc 			int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx);
11800e8c6158Smaybee 			uint64_t *size = &old_state->arcs_lsize[ab->b_type];
1181ea8dc4b6Seschrock 
1182ea8dc4b6Seschrock 			if (use_mutex)
118344cb6abcSbmc 				mutex_enter(&old_state->arcs_mtx);
1184fa9e4066Sahrens 
1185fa9e4066Sahrens 			ASSERT(list_link_active(&ab->b_arc_node));
11860e8c6158Smaybee 			list_remove(&old_state->arcs_list[ab->b_type], ab);
1187ea8dc4b6Seschrock 
118813506d1eSmaybee 			/*
118913506d1eSmaybee 			 * If prefetching out of the ghost cache,
11903f9d6ad7SLin Ling 			 * we will have a non-zero datacnt.
119113506d1eSmaybee 			 */
119213506d1eSmaybee 			if (GHOST_STATE(old_state) && ab->b_datacnt == 0) {
119313506d1eSmaybee 				/* ghost elements have a ghost size */
1194ea8dc4b6Seschrock 				ASSERT(ab->b_buf == NULL);
1195ea8dc4b6Seschrock 				from_delta = ab->b_size;
1196ea8dc4b6Seschrock 			}
11970e8c6158Smaybee 			ASSERT3U(*size, >=, from_delta);
11980e8c6158Smaybee 			atomic_add_64(size, -from_delta);
1199ea8dc4b6Seschrock 
1200ea8dc4b6Seschrock 			if (use_mutex)
120144cb6abcSbmc 				mutex_exit(&old_state->arcs_mtx);
1202fa9e4066Sahrens 		}
120344cb6abcSbmc 		if (new_state != arc_anon) {
120444cb6abcSbmc 			int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx);
12050e8c6158Smaybee 			uint64_t *size = &new_state->arcs_lsize[ab->b_type];
1206fa9e4066Sahrens 
1207ea8dc4b6Seschrock 			if (use_mutex)
120844cb6abcSbmc 				mutex_enter(&new_state->arcs_mtx);
1209ea8dc4b6Seschrock 
12100e8c6158Smaybee 			list_insert_head(&new_state->arcs_list[ab->b_type], ab);
1211ea8dc4b6Seschrock 
1212ea8dc4b6Seschrock 			/* ghost elements have a ghost size */
1213ea8dc4b6Seschrock 			if (GHOST_STATE(new_state)) {
1214ea8dc4b6Seschrock 				ASSERT(ab->b_datacnt == 0);
1215ea8dc4b6Seschrock 				ASSERT(ab->b_buf == NULL);
1216ea8dc4b6Seschrock 				to_delta = ab->b_size;
1217ea8dc4b6Seschrock 			}
12180e8c6158Smaybee 			atomic_add_64(size, to_delta);
1219ea8dc4b6Seschrock 
1220ea8dc4b6Seschrock 			if (use_mutex)
122144cb6abcSbmc 				mutex_exit(&new_state->arcs_mtx);
1222fa9e4066Sahrens 		}
1223fa9e4066Sahrens 	}
1224fa9e4066Sahrens 
1225fa9e4066Sahrens 	ASSERT(!BUF_EMPTY(ab));
12263f9d6ad7SLin Ling 	if (new_state == arc_anon && HDR_IN_HASH_TABLE(ab))
1227fa9e4066Sahrens 		buf_hash_remove(ab);
1228fa9e4066Sahrens 
1229ea8dc4b6Seschrock 	/* adjust state sizes */
1230ea8dc4b6Seschrock 	if (to_delta)
123144cb6abcSbmc 		atomic_add_64(&new_state->arcs_size, to_delta);
1232ea8dc4b6Seschrock 	if (from_delta) {
123344cb6abcSbmc 		ASSERT3U(old_state->arcs_size, >=, from_delta);
123444cb6abcSbmc 		atomic_add_64(&old_state->arcs_size, -from_delta);
1235fa9e4066Sahrens 	}
1236fa9e4066Sahrens 	ab->b_state = new_state;
1237fa94a07fSbrendan 
1238fa94a07fSbrendan 	/* adjust l2arc hdr stats */
1239fa94a07fSbrendan 	if (new_state == arc_l2c_only)
1240fa94a07fSbrendan 		l2arc_hdr_stat_add();
1241fa94a07fSbrendan 	else if (old_state == arc_l2c_only)
1242fa94a07fSbrendan 		l2arc_hdr_stat_remove();
1243fa9e4066Sahrens }
1244fa9e4066Sahrens 
12450e8c6158Smaybee void
12465a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(uint64_t space, arc_space_type_t type)
12470e8c6158Smaybee {
12485a98e54bSBrendan Gregg - Sun Microsystems 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
12495a98e54bSBrendan Gregg - Sun Microsystems 
12505a98e54bSBrendan Gregg - Sun Microsystems 	switch (type) {
12515a98e54bSBrendan Gregg - Sun Microsystems 	case ARC_SPACE_DATA:
12525a98e54bSBrendan Gregg - Sun Microsystems 		ARCSTAT_INCR(arcstat_data_size, space);
12535a98e54bSBrendan Gregg - Sun Microsystems 		break;
12545a98e54bSBrendan Gregg - Sun Microsystems 	case ARC_SPACE_OTHER:
12555a98e54bSBrendan Gregg - Sun Microsystems 		ARCSTAT_INCR(arcstat_other_size, space);
12565a98e54bSBrendan Gregg - Sun Microsystems 		break;
12575a98e54bSBrendan Gregg - Sun Microsystems 	case ARC_SPACE_HDRS:
12585a98e54bSBrendan Gregg - Sun Microsystems 		ARCSTAT_INCR(arcstat_hdr_size, space);
12595a98e54bSBrendan Gregg - Sun Microsystems 		break;
12605a98e54bSBrendan Gregg - Sun Microsystems 	case ARC_SPACE_L2HDRS:
12615a98e54bSBrendan Gregg - Sun Microsystems 		ARCSTAT_INCR(arcstat_l2_hdr_size, space);
12625a98e54bSBrendan Gregg - Sun Microsystems 		break;
12635a98e54bSBrendan Gregg - Sun Microsystems 	}
12645a98e54bSBrendan Gregg - Sun Microsystems 
126520128a08SGeorge Wilson 	ARCSTAT_INCR(arcstat_meta_used, space);
12660e8c6158Smaybee 	atomic_add_64(&arc_size, space);
12670e8c6158Smaybee }
12680e8c6158Smaybee 
12690e8c6158Smaybee void
12705a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(uint64_t space, arc_space_type_t type)
12710e8c6158Smaybee {
12725a98e54bSBrendan Gregg - Sun Microsystems 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
12735a98e54bSBrendan Gregg - Sun Microsystems 
12745a98e54bSBrendan Gregg - Sun Microsystems 	switch (type) {
12755a98e54bSBrendan Gregg - Sun Microsystems 	case ARC_SPACE_DATA:
12765a98e54bSBrendan Gregg - Sun Microsystems 		ARCSTAT_INCR(arcstat_data_size, -space);
12775a98e54bSBrendan Gregg - Sun Microsystems 		break;
12785a98e54bSBrendan Gregg - Sun Microsystems 	case ARC_SPACE_OTHER:
12795a98e54bSBrendan Gregg - Sun Microsystems 		ARCSTAT_INCR(arcstat_other_size, -space);
12805a98e54bSBrendan Gregg - Sun Microsystems 		break;
12815a98e54bSBrendan Gregg - Sun Microsystems 	case ARC_SPACE_HDRS:
12825a98e54bSBrendan Gregg - Sun Microsystems 		ARCSTAT_INCR(arcstat_hdr_size, -space);
12835a98e54bSBrendan Gregg - Sun Microsystems 		break;
12845a98e54bSBrendan Gregg - Sun Microsystems 	case ARC_SPACE_L2HDRS:
12855a98e54bSBrendan Gregg - Sun Microsystems 		ARCSTAT_INCR(arcstat_l2_hdr_size, -space);
12865a98e54bSBrendan Gregg - Sun Microsystems 		break;
12875a98e54bSBrendan Gregg - Sun Microsystems 	}
12885a98e54bSBrendan Gregg - Sun Microsystems 
12890e8c6158Smaybee 	ASSERT(arc_meta_used >= space);
12900e8c6158Smaybee 	if (arc_meta_max < arc_meta_used)
12910e8c6158Smaybee 		arc_meta_max = arc_meta_used;
129220128a08SGeorge Wilson 	ARCSTAT_INCR(arcstat_meta_used, -space);
12930e8c6158Smaybee 	ASSERT(arc_size >= space);
12940e8c6158Smaybee 	atomic_add_64(&arc_size, -space);
12950e8c6158Smaybee }
12960e8c6158Smaybee 
12970e8c6158Smaybee void *
12980e8c6158Smaybee arc_data_buf_alloc(uint64_t size)
12990e8c6158Smaybee {
13000e8c6158Smaybee 	if (arc_evict_needed(ARC_BUFC_DATA))
13010e8c6158Smaybee 		cv_signal(&arc_reclaim_thr_cv);
13020e8c6158Smaybee 	atomic_add_64(&arc_size, size);
13030e8c6158Smaybee 	return (zio_data_buf_alloc(size));
13040e8c6158Smaybee }
13050e8c6158Smaybee 
13060e8c6158Smaybee void
13070e8c6158Smaybee arc_data_buf_free(void *buf, uint64_t size)
13080e8c6158Smaybee {
13090e8c6158Smaybee 	zio_data_buf_free(buf, size);
13100e8c6158Smaybee 	ASSERT(arc_size >= size);
13110e8c6158Smaybee 	atomic_add_64(&arc_size, -size);
13120e8c6158Smaybee }
13130e8c6158Smaybee 
1314fa9e4066Sahrens arc_buf_t *
1315ad23a2dbSjohansen arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type)
1316fa9e4066Sahrens {
1317fa9e4066Sahrens 	arc_buf_hdr_t *hdr;
1318fa9e4066Sahrens 	arc_buf_t *buf;
1319fa9e4066Sahrens 
1320fa9e4066Sahrens 	ASSERT3U(size, >, 0);
13211ab7f2deSmaybee 	hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
1322fa9e4066Sahrens 	ASSERT(BUF_EMPTY(hdr));
1323fa9e4066Sahrens 	hdr->b_size = size;
1324ad23a2dbSjohansen 	hdr->b_type = type;
1325e9103aaeSGarrett D'Amore 	hdr->b_spa = spa_load_guid(spa);
132644cb6abcSbmc 	hdr->b_state = arc_anon;
1327fa9e4066Sahrens 	hdr->b_arc_access = 0;
13281ab7f2deSmaybee 	buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1329fa9e4066Sahrens 	buf->b_hdr = hdr;
133044eda4d7Smaybee 	buf->b_data = NULL;
1331ea8dc4b6Seschrock 	buf->b_efunc = NULL;
1332ea8dc4b6Seschrock 	buf->b_private = NULL;
1333fa9e4066Sahrens 	buf->b_next = NULL;
1334fa9e4066Sahrens 	hdr->b_buf = buf;
133544eda4d7Smaybee 	arc_get_data_buf(buf);
1336ea8dc4b6Seschrock 	hdr->b_datacnt = 1;
1337fa9e4066Sahrens 	hdr->b_flags = 0;
1338fa9e4066Sahrens 	ASSERT(refcount_is_zero(&hdr->b_refcnt));
1339fa9e4066Sahrens 	(void) refcount_add(&hdr->b_refcnt, tag);
1340fa9e4066Sahrens 
1341fa9e4066Sahrens 	return (buf);
1342fa9e4066Sahrens }
1343fa9e4066Sahrens 
13442fdbea25SAleksandr Guzovskiy static char *arc_onloan_tag = "onloan";
13452fdbea25SAleksandr Guzovskiy 
13462fdbea25SAleksandr Guzovskiy /*
13472fdbea25SAleksandr Guzovskiy  * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
13482fdbea25SAleksandr Guzovskiy  * flight data by arc_tempreserve_space() until they are "returned". Loaned
13492fdbea25SAleksandr Guzovskiy  * buffers must be returned to the arc before they can be used by the DMU or
13502fdbea25SAleksandr Guzovskiy  * freed.
13512fdbea25SAleksandr Guzovskiy  */
13522fdbea25SAleksandr Guzovskiy arc_buf_t *
13532fdbea25SAleksandr Guzovskiy arc_loan_buf(spa_t *spa, int size)
13542fdbea25SAleksandr Guzovskiy {
13552fdbea25SAleksandr Guzovskiy 	arc_buf_t *buf;
13562fdbea25SAleksandr Guzovskiy 
13572fdbea25SAleksandr Guzovskiy 	buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA);
13582fdbea25SAleksandr Guzovskiy 
13592fdbea25SAleksandr Guzovskiy 	atomic_add_64(&arc_loaned_bytes, size);
13602fdbea25SAleksandr Guzovskiy 	return (buf);
13612fdbea25SAleksandr Guzovskiy }
13622fdbea25SAleksandr Guzovskiy 
13632fdbea25SAleksandr Guzovskiy /*
13642fdbea25SAleksandr Guzovskiy  * Return a loaned arc buffer to the arc.
13652fdbea25SAleksandr Guzovskiy  */
13662fdbea25SAleksandr Guzovskiy void
13672fdbea25SAleksandr Guzovskiy arc_return_buf(arc_buf_t *buf, void *tag)
13682fdbea25SAleksandr Guzovskiy {
13692fdbea25SAleksandr Guzovskiy 	arc_buf_hdr_t *hdr = buf->b_hdr;
13702fdbea25SAleksandr Guzovskiy 
13712fdbea25SAleksandr Guzovskiy 	ASSERT(buf->b_data != NULL);
1372c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	(void) refcount_add(&hdr->b_refcnt, tag);
1373c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	(void) refcount_remove(&hdr->b_refcnt, arc_onloan_tag);
13742fdbea25SAleksandr Guzovskiy 
13752fdbea25SAleksandr Guzovskiy 	atomic_add_64(&arc_loaned_bytes, -hdr->b_size);
13762fdbea25SAleksandr Guzovskiy }
13772fdbea25SAleksandr Guzovskiy 
1378c242f9a0Schunli zhang - Sun Microsystems - Irvine United States /* Detach an arc_buf from a dbuf (tag) */
1379c242f9a0Schunli zhang - Sun Microsystems - Irvine United States void
1380c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
1381c242f9a0Schunli zhang - Sun Microsystems - Irvine United States {
1382c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	arc_buf_hdr_t *hdr;
1383c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1384c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	ASSERT(buf->b_data != NULL);
1385c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	hdr = buf->b_hdr;
1386c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	(void) refcount_add(&hdr->b_refcnt, arc_onloan_tag);
1387c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	(void) refcount_remove(&hdr->b_refcnt, tag);
1388c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	buf->b_efunc = NULL;
1389c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	buf->b_private = NULL;
1390c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1391c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	atomic_add_64(&arc_loaned_bytes, hdr->b_size);
1392c242f9a0Schunli zhang - Sun Microsystems - Irvine United States }
1393c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
139444eda4d7Smaybee static arc_buf_t *
139544eda4d7Smaybee arc_buf_clone(arc_buf_t *from)
1396ea8dc4b6Seschrock {
139744eda4d7Smaybee 	arc_buf_t *buf;
139844eda4d7Smaybee 	arc_buf_hdr_t *hdr = from->b_hdr;
139944eda4d7Smaybee 	uint64_t size = hdr->b_size;
1400ea8dc4b6Seschrock 
1401b24ab676SJeff Bonwick 	ASSERT(hdr->b_state != arc_anon);
1402b24ab676SJeff Bonwick 
14031ab7f2deSmaybee 	buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
140444eda4d7Smaybee 	buf->b_hdr = hdr;
140544eda4d7Smaybee 	buf->b_data = NULL;
140644eda4d7Smaybee 	buf->b_efunc = NULL;
140744eda4d7Smaybee 	buf->b_private = NULL;
140844eda4d7Smaybee 	buf->b_next = hdr->b_buf;
140944eda4d7Smaybee 	hdr->b_buf = buf;
141044eda4d7Smaybee 	arc_get_data_buf(buf);
141144eda4d7Smaybee 	bcopy(from->b_data, buf->b_data, size);
14129253d63dSGeorge Wilson 
14139253d63dSGeorge Wilson 	/*
14149253d63dSGeorge Wilson 	 * This buffer already exists in the arc so create a duplicate
14159253d63dSGeorge Wilson 	 * copy for the caller.  If the buffer is associated with user data
14169253d63dSGeorge Wilson 	 * then track the size and number of duplicates.  These stats will be
14179253d63dSGeorge Wilson 	 * updated as duplicate buffers are created and destroyed.
14189253d63dSGeorge Wilson 	 */
14199253d63dSGeorge Wilson 	if (hdr->b_type == ARC_BUFC_DATA) {
14209253d63dSGeorge Wilson 		ARCSTAT_BUMP(arcstat_duplicate_buffers);
14219253d63dSGeorge Wilson 		ARCSTAT_INCR(arcstat_duplicate_buffers_size, size);
14229253d63dSGeorge Wilson 	}
142344eda4d7Smaybee 	hdr->b_datacnt += 1;
142444eda4d7Smaybee 	return (buf);
1425ea8dc4b6Seschrock }
1426ea8dc4b6Seschrock 
1427ea8dc4b6Seschrock void
1428ea8dc4b6Seschrock arc_buf_add_ref(arc_buf_t *buf, void* tag)
1429ea8dc4b6Seschrock {
143040d7d650Smaybee 	arc_buf_hdr_t *hdr;
1431ea8dc4b6Seschrock 	kmutex_t *hash_lock;
1432ea8dc4b6Seschrock 
14339b23f181Smaybee 	/*
14346f83844dSMark Maybee 	 * Check to see if this buffer is evicted.  Callers
14356f83844dSMark Maybee 	 * must verify b_data != NULL to know if the add_ref
14366f83844dSMark Maybee 	 * was successful.
14379b23f181Smaybee 	 */
14383f9d6ad7SLin Ling 	mutex_enter(&buf->b_evict_lock);
14396f83844dSMark Maybee 	if (buf->b_data == NULL) {
14403f9d6ad7SLin Ling 		mutex_exit(&buf->b_evict_lock);
14419b23f181Smaybee 		return;
144240d7d650Smaybee 	}
14433f9d6ad7SLin Ling 	hash_lock = HDR_LOCK(buf->b_hdr);
14449b23f181Smaybee 	mutex_enter(hash_lock);
14453f9d6ad7SLin Ling 	hdr = buf->b_hdr;
14463f9d6ad7SLin Ling 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
14473f9d6ad7SLin Ling 	mutex_exit(&buf->b_evict_lock);
1448ea8dc4b6Seschrock 
144944cb6abcSbmc 	ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
1450ea8dc4b6Seschrock 	add_reference(hdr, hash_lock, tag);
14515a98e54bSBrendan Gregg - Sun Microsystems 	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
145244eda4d7Smaybee 	arc_access(hdr, hash_lock);
145344eda4d7Smaybee 	mutex_exit(hash_lock);
145444cb6abcSbmc 	ARCSTAT_BUMP(arcstat_hits);
145544cb6abcSbmc 	ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
145644cb6abcSbmc 	    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
145744cb6abcSbmc 	    data, metadata, hits);
1458ea8dc4b6Seschrock }
1459ea8dc4b6Seschrock 
1460fa94a07fSbrendan /*
1461fa94a07fSbrendan  * Free the arc data buffer.  If it is an l2arc write in progress,
1462fa94a07fSbrendan  * the buffer is placed on l2arc_free_on_write to be freed later.
1463fa94a07fSbrendan  */
1464fa94a07fSbrendan static void
1465cd1c8b85SMatthew Ahrens arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t))
1466fa94a07fSbrendan {
1467cd1c8b85SMatthew Ahrens 	arc_buf_hdr_t *hdr = buf->b_hdr;
1468cd1c8b85SMatthew Ahrens 
1469fa94a07fSbrendan 	if (HDR_L2_WRITING(hdr)) {
1470fa94a07fSbrendan 		l2arc_data_free_t *df;
1471fa94a07fSbrendan 		df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP);
1472cd1c8b85SMatthew Ahrens 		df->l2df_data = buf->b_data;
1473cd1c8b85SMatthew Ahrens 		df->l2df_size = hdr->b_size;
1474fa94a07fSbrendan 		df->l2df_func = free_func;
1475fa94a07fSbrendan 		mutex_enter(&l2arc_free_on_write_mtx);
1476fa94a07fSbrendan 		list_insert_head(l2arc_free_on_write, df);
1477fa94a07fSbrendan 		mutex_exit(&l2arc_free_on_write_mtx);
1478fa94a07fSbrendan 		ARCSTAT_BUMP(arcstat_l2_free_on_write);
1479fa94a07fSbrendan 	} else {
1480cd1c8b85SMatthew Ahrens 		free_func(buf->b_data, hdr->b_size);
1481fa94a07fSbrendan 	}
1482fa94a07fSbrendan }
1483fa94a07fSbrendan 
1484ea8dc4b6Seschrock static void
148544eda4d7Smaybee arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all)
1486ea8dc4b6Seschrock {
1487ea8dc4b6Seschrock 	arc_buf_t **bufp;
1488ea8dc4b6Seschrock 
1489ea8dc4b6Seschrock 	/* free up data associated with the buf */
1490ea8dc4b6Seschrock 	if (buf->b_data) {
1491ea8dc4b6Seschrock 		arc_state_t *state = buf->b_hdr->b_state;
1492ea8dc4b6Seschrock 		uint64_t size = buf->b_hdr->b_size;
1493ad23a2dbSjohansen 		arc_buf_contents_t type = buf->b_hdr->b_type;
1494ea8dc4b6Seschrock 
14956b4acc8bSahrens 		arc_cksum_verify(buf);
1496cd1c8b85SMatthew Ahrens 		arc_buf_unwatch(buf);
1497b24ab676SJeff Bonwick 
149844eda4d7Smaybee 		if (!recycle) {
1499ad23a2dbSjohansen 			if (type == ARC_BUFC_METADATA) {
1500cd1c8b85SMatthew Ahrens 				arc_buf_data_free(buf, zio_buf_free);
15015a98e54bSBrendan Gregg - Sun Microsystems 				arc_space_return(size, ARC_SPACE_DATA);
1502ad23a2dbSjohansen 			} else {
1503ad23a2dbSjohansen 				ASSERT(type == ARC_BUFC_DATA);
1504cd1c8b85SMatthew Ahrens 				arc_buf_data_free(buf, zio_data_buf_free);
15055a98e54bSBrendan Gregg - Sun Microsystems 				ARCSTAT_INCR(arcstat_data_size, -size);
15060e8c6158Smaybee 				atomic_add_64(&arc_size, -size);
1507ad23a2dbSjohansen 			}
150844eda4d7Smaybee 		}
1509ea8dc4b6Seschrock 		if (list_link_active(&buf->b_hdr->b_arc_node)) {
15100e8c6158Smaybee 			uint64_t *cnt = &state->arcs_lsize[type];
15110e8c6158Smaybee 
1512ea8dc4b6Seschrock 			ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt));
151344cb6abcSbmc 			ASSERT(state != arc_anon);
15140e8c6158Smaybee 
15150e8c6158Smaybee 			ASSERT3U(*cnt, >=, size);
15160e8c6158Smaybee 			atomic_add_64(cnt, -size);
1517ea8dc4b6Seschrock 		}
151844cb6abcSbmc 		ASSERT3U(state->arcs_size, >=, size);
151944cb6abcSbmc 		atomic_add_64(&state->arcs_size, -size);
1520ea8dc4b6Seschrock 		buf->b_data = NULL;
15219253d63dSGeorge Wilson 
15229253d63dSGeorge Wilson 		/*
15239253d63dSGeorge Wilson 		 * If we're destroying a duplicate buffer make sure
15249253d63dSGeorge Wilson 		 * that the appropriate statistics are updated.
15259253d63dSGeorge Wilson 		 */
15269253d63dSGeorge Wilson 		if (buf->b_hdr->b_datacnt > 1 &&
15279253d63dSGeorge Wilson 		    buf->b_hdr->b_type == ARC_BUFC_DATA) {
15289253d63dSGeorge Wilson 			ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
15299253d63dSGeorge Wilson 			ARCSTAT_INCR(arcstat_duplicate_buffers_size, -size);
15309253d63dSGeorge Wilson 		}
1531ea8dc4b6Seschrock 		ASSERT(buf->b_hdr->b_datacnt > 0);
1532ea8dc4b6Seschrock 		buf->b_hdr->b_datacnt -= 1;
1533ea8dc4b6Seschrock 	}
1534ea8dc4b6Seschrock 
1535ea8dc4b6Seschrock 	/* only remove the buf if requested */
1536ea8dc4b6Seschrock 	if (!all)
1537ea8dc4b6Seschrock 		return;
1538ea8dc4b6Seschrock 
1539ea8dc4b6Seschrock 	/* remove the buf from the hdr list */
1540ea8dc4b6Seschrock 	for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next)
1541ea8dc4b6Seschrock 		continue;
1542ea8dc4b6Seschrock 	*bufp = buf->b_next;
15433f9d6ad7SLin Ling 	buf->b_next = NULL;
1544ea8dc4b6Seschrock 
1545ea8dc4b6Seschrock 	ASSERT(buf->b_efunc == NULL);
1546ea8dc4b6Seschrock 
1547ea8dc4b6Seschrock 	/* clean up the buf */
1548ea8dc4b6Seschrock 	buf->b_hdr = NULL;
1549ea8dc4b6Seschrock 	kmem_cache_free(buf_cache, buf);
1550ea8dc4b6Seschrock }
1551ea8dc4b6Seschrock 
1552fa9e4066Sahrens static void
1553ea8dc4b6Seschrock arc_hdr_destroy(arc_buf_hdr_t *hdr)
1554fa9e4066Sahrens {
1555fa9e4066Sahrens 	ASSERT(refcount_is_zero(&hdr->b_refcnt));
155644cb6abcSbmc 	ASSERT3P(hdr->b_state, ==, arc_anon);
1557ea8dc4b6Seschrock 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
1558b24ab676SJeff Bonwick 	l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr;
1559fa9e4066Sahrens 
1560b24ab676SJeff Bonwick 	if (l2hdr != NULL) {
1561b24ab676SJeff Bonwick 		boolean_t buflist_held = MUTEX_HELD(&l2arc_buflist_mtx);
1562b24ab676SJeff Bonwick 		/*
1563b24ab676SJeff Bonwick 		 * To prevent arc_free() and l2arc_evict() from
1564b24ab676SJeff Bonwick 		 * attempting to free the same buffer at the same time,
1565b24ab676SJeff Bonwick 		 * a FREE_IN_PROGRESS flag is given to arc_free() to
1566b24ab676SJeff Bonwick 		 * give it priority.  l2arc_evict() can't destroy this
1567b24ab676SJeff Bonwick 		 * header while we are waiting on l2arc_buflist_mtx.
1568b24ab676SJeff Bonwick 		 *
1569b24ab676SJeff Bonwick 		 * The hdr may be removed from l2ad_buflist before we
1570b24ab676SJeff Bonwick 		 * grab l2arc_buflist_mtx, so b_l2hdr is rechecked.
1571b24ab676SJeff Bonwick 		 */
1572b24ab676SJeff Bonwick 		if (!buflist_held) {
1573fa94a07fSbrendan 			mutex_enter(&l2arc_buflist_mtx);
1574b24ab676SJeff Bonwick 			l2hdr = hdr->b_l2hdr;
1575fa94a07fSbrendan 		}
1576b24ab676SJeff Bonwick 
1577b24ab676SJeff Bonwick 		if (l2hdr != NULL) {
1578b24ab676SJeff Bonwick 			list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
1579b24ab676SJeff Bonwick 			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
1580aad02571SSaso Kiselkov 			ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
1581b24ab676SJeff Bonwick 			kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
1582b24ab676SJeff Bonwick 			if (hdr->b_state == arc_l2c_only)
1583b24ab676SJeff Bonwick 				l2arc_hdr_stat_remove();
1584b24ab676SJeff Bonwick 			hdr->b_l2hdr = NULL;
1585b24ab676SJeff Bonwick 		}
1586b24ab676SJeff Bonwick 
1587b24ab676SJeff Bonwick 		if (!buflist_held)
1588b24ab676SJeff Bonwick 			mutex_exit(&l2arc_buflist_mtx);
1589fa94a07fSbrendan 	}
1590fa94a07fSbrendan 
1591fa9e4066Sahrens 	if (!BUF_EMPTY(hdr)) {
1592ea8dc4b6Seschrock 		ASSERT(!HDR_IN_HASH_TABLE(hdr));
15933f9d6ad7SLin Ling 		buf_discard_identity(hdr);
1594fa9e4066Sahrens 	}
1595ea8dc4b6Seschrock 	while (hdr->b_buf) {
1596fa9e4066Sahrens 		arc_buf_t *buf = hdr->b_buf;
1597fa9e4066Sahrens 
1598ea8dc4b6Seschrock 		if (buf->b_efunc) {
1599ea8dc4b6Seschrock 			mutex_enter(&arc_eviction_mtx);
16003f9d6ad7SLin Ling 			mutex_enter(&buf->b_evict_lock);
1601ea8dc4b6Seschrock 			ASSERT(buf->b_hdr != NULL);
160244eda4d7Smaybee 			arc_buf_destroy(hdr->b_buf, FALSE, FALSE);
1603ea8dc4b6Seschrock 			hdr->b_buf = buf->b_next;
160440d7d650Smaybee 			buf->b_hdr = &arc_eviction_hdr;
1605ea8dc4b6Seschrock 			buf->b_next = arc_eviction_list;
1606ea8dc4b6Seschrock 			arc_eviction_list = buf;
16073f9d6ad7SLin Ling 			mutex_exit(&buf->b_evict_lock);
1608ea8dc4b6Seschrock 			mutex_exit(&arc_eviction_mtx);
1609ea8dc4b6Seschrock 		} else {
161044eda4d7Smaybee 			arc_buf_destroy(hdr->b_buf, FALSE, TRUE);
1611ea8dc4b6Seschrock 		}
1612fa9e4066Sahrens 	}
16136b4acc8bSahrens 	if (hdr->b_freeze_cksum != NULL) {
16146b4acc8bSahrens 		kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
16156b4acc8bSahrens 		hdr->b_freeze_cksum = NULL;
16166b4acc8bSahrens 	}
16173f9d6ad7SLin Ling 	if (hdr->b_thawed) {
16183f9d6ad7SLin Ling 		kmem_free(hdr->b_thawed, 1);
16193f9d6ad7SLin Ling 		hdr->b_thawed = NULL;
16203f9d6ad7SLin Ling 	}
1621ea8dc4b6Seschrock 
1622fa9e4066Sahrens 	ASSERT(!list_link_active(&hdr->b_arc_node));
1623fa9e4066Sahrens 	ASSERT3P(hdr->b_hash_next, ==, NULL);
1624fa9e4066Sahrens 	ASSERT3P(hdr->b_acb, ==, NULL);
1625fa9e4066Sahrens 	kmem_cache_free(hdr_cache, hdr);
1626fa9e4066Sahrens }
1627fa9e4066Sahrens 
1628fa9e4066Sahrens void
1629fa9e4066Sahrens arc_buf_free(arc_buf_t *buf, void *tag)
1630fa9e4066Sahrens {
1631fa9e4066Sahrens 	arc_buf_hdr_t *hdr = buf->b_hdr;
163244cb6abcSbmc 	int hashed = hdr->b_state != arc_anon;
1633fa9e4066Sahrens 
1634ea8dc4b6Seschrock 	ASSERT(buf->b_efunc == NULL);
1635ea8dc4b6Seschrock 	ASSERT(buf->b_data != NULL);
1636ea8dc4b6Seschrock 
1637ea8dc4b6Seschrock 	if (hashed) {
1638ea8dc4b6Seschrock 		kmutex_t *hash_lock = HDR_LOCK(hdr);
1639ea8dc4b6Seschrock 
1640ea8dc4b6Seschrock 		mutex_enter(hash_lock);
16413f9d6ad7SLin Ling 		hdr = buf->b_hdr;
16423f9d6ad7SLin Ling 		ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
16433f9d6ad7SLin Ling 
1644ea8dc4b6Seschrock 		(void) remove_reference(hdr, hash_lock, tag);
1645b24ab676SJeff Bonwick 		if (hdr->b_datacnt > 1) {
164644eda4d7Smaybee 			arc_buf_destroy(buf, FALSE, TRUE);
1647b24ab676SJeff Bonwick 		} else {
1648b24ab676SJeff Bonwick 			ASSERT(buf == hdr->b_buf);
1649b24ab676SJeff Bonwick 			ASSERT(buf->b_efunc == NULL);
1650ea8dc4b6Seschrock 			hdr->b_flags |= ARC_BUF_AVAILABLE;
1651b24ab676SJeff Bonwick 		}
1652fa9e4066Sahrens 		mutex_exit(hash_lock);
1653ea8dc4b6Seschrock 	} else if (HDR_IO_IN_PROGRESS(hdr)) {
1654ea8dc4b6Seschrock 		int destroy_hdr;
1655ea8dc4b6Seschrock 		/*
1656ea8dc4b6Seschrock 		 * We are in the middle of an async write.  Don't destroy
1657ea8dc4b6Seschrock 		 * this buffer unless the write completes before we finish
1658ea8dc4b6Seschrock 		 * decrementing the reference count.
1659ea8dc4b6Seschrock 		 */
1660ea8dc4b6Seschrock 		mutex_enter(&arc_eviction_mtx);
1661ea8dc4b6Seschrock 		(void) remove_reference(hdr, NULL, tag);
1662ea8dc4b6Seschrock 		ASSERT(refcount_is_zero(&hdr->b_refcnt));
1663ea8dc4b6Seschrock 		destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
1664ea8dc4b6Seschrock 		mutex_exit(&arc_eviction_mtx);
1665ea8dc4b6Seschrock 		if (destroy_hdr)
1666ea8dc4b6Seschrock 			arc_hdr_destroy(hdr);
1667ea8dc4b6Seschrock 	} else {
16683f9d6ad7SLin Ling 		if (remove_reference(hdr, NULL, tag) > 0)
166944eda4d7Smaybee 			arc_buf_destroy(buf, FALSE, TRUE);
16703f9d6ad7SLin Ling 		else
1671ea8dc4b6Seschrock 			arc_hdr_destroy(hdr);
1672fa9e4066Sahrens 	}
1673ea8dc4b6Seschrock }
1674fa9e4066Sahrens 
16753b2aab18SMatthew Ahrens boolean_t
1676ea8dc4b6Seschrock arc_buf_remove_ref(arc_buf_t *buf, void* tag)
1677ea8dc4b6Seschrock {
1678ea8dc4b6Seschrock 	arc_buf_hdr_t *hdr = buf->b_hdr;
1679ea8dc4b6Seschrock 	kmutex_t *hash_lock = HDR_LOCK(hdr);
16803b2aab18SMatthew Ahrens 	boolean_t no_callback = (buf->b_efunc == NULL);
1681fa9e4066Sahrens 
168244cb6abcSbmc 	if (hdr->b_state == arc_anon) {
1683b24ab676SJeff Bonwick 		ASSERT(hdr->b_datacnt == 1);
1684ea8dc4b6Seschrock 		arc_buf_free(buf, tag);
1685ea8dc4b6Seschrock 		return (no_callback);
1686ea8dc4b6Seschrock 	}
1687ea8dc4b6Seschrock 
1688ea8dc4b6Seschrock 	mutex_enter(hash_lock);
16893f9d6ad7SLin Ling 	hdr = buf->b_hdr;
16903f9d6ad7SLin Ling 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
169144cb6abcSbmc 	ASSERT(hdr->b_state != arc_anon);
1692ea8dc4b6Seschrock 	ASSERT(buf->b_data != NULL);
1693ea8dc4b6Seschrock 
1694ea8dc4b6Seschrock 	(void) remove_reference(hdr, hash_lock, tag);
1695ea8dc4b6Seschrock 	if (hdr->b_datacnt > 1) {
1696ea8dc4b6Seschrock 		if (no_callback)
169744eda4d7Smaybee 			arc_buf_destroy(buf, FALSE, TRUE);
1698ea8dc4b6Seschrock 	} else if (no_callback) {
1699ea8dc4b6Seschrock 		ASSERT(hdr->b_buf == buf && buf->b_next == NULL);
1700b24ab676SJeff Bonwick 		ASSERT(buf->b_efunc == NULL);
1701ea8dc4b6Seschrock 		hdr->b_flags |= ARC_BUF_AVAILABLE;
1702ea8dc4b6Seschrock 	}
1703ea8dc4b6Seschrock 	ASSERT(no_callback || hdr->b_datacnt > 1 ||
1704ea8dc4b6Seschrock 	    refcount_is_zero(&hdr->b_refcnt));
1705ea8dc4b6Seschrock 	mutex_exit(hash_lock);
1706ea8dc4b6Seschrock 	return (no_callback);
1707fa9e4066Sahrens }
1708fa9e4066Sahrens 
1709fa9e4066Sahrens int
1710fa9e4066Sahrens arc_buf_size(arc_buf_t *buf)
1711fa9e4066Sahrens {
1712fa9e4066Sahrens 	return (buf->b_hdr->b_size);
1713fa9e4066Sahrens }
1714fa9e4066Sahrens 
17159253d63dSGeorge Wilson /*
17169253d63dSGeorge Wilson  * Called from the DMU to determine if the current buffer should be
17179253d63dSGeorge Wilson  * evicted. In order to ensure proper locking, the eviction must be initiated
17189253d63dSGeorge Wilson  * from the DMU. Return true if the buffer is associated with user data and
17199253d63dSGeorge Wilson  * duplicate buffers still exist.
17209253d63dSGeorge Wilson  */
17219253d63dSGeorge Wilson boolean_t
17229253d63dSGeorge Wilson arc_buf_eviction_needed(arc_buf_t *buf)
17239253d63dSGeorge Wilson {
17249253d63dSGeorge Wilson 	arc_buf_hdr_t *hdr;
17259253d63dSGeorge Wilson 	boolean_t evict_needed = B_FALSE;
17269253d63dSGeorge Wilson 
17279253d63dSGeorge Wilson 	if (zfs_disable_dup_eviction)
17289253d63dSGeorge Wilson 		return (B_FALSE);
17299253d63dSGeorge Wilson 
17309253d63dSGeorge Wilson 	mutex_enter(&buf->b_evict_lock);
17319253d63dSGeorge Wilson 	hdr = buf->b_hdr;
17329253d63dSGeorge Wilson 	if (hdr == NULL) {
17339253d63dSGeorge Wilson 		/*
17349253d63dSGeorge Wilson 		 * We are in arc_do_user_evicts(); let that function
17359253d63dSGeorge Wilson 		 * perform the eviction.
17369253d63dSGeorge Wilson 		 */
17379253d63dSGeorge Wilson 		ASSERT(buf->b_data == NULL);
17389253d63dSGeorge Wilson 		mutex_exit(&buf->b_evict_lock);
17399253d63dSGeorge Wilson 		return (B_FALSE);
17409253d63dSGeorge Wilson 	} else if (buf->b_data == NULL) {
17419253d63dSGeorge Wilson 		/*
17429253d63dSGeorge Wilson 		 * We have already been added to the arc eviction list;
17439253d63dSGeorge Wilson 		 * recommend eviction.
17449253d63dSGeorge Wilson 		 */
17459253d63dSGeorge Wilson 		ASSERT3P(hdr, ==, &arc_eviction_hdr);
17469253d63dSGeorge Wilson 		mutex_exit(&buf->b_evict_lock);
17479253d63dSGeorge Wilson 		return (B_TRUE);
17489253d63dSGeorge Wilson 	}
17499253d63dSGeorge Wilson 
17509253d63dSGeorge Wilson 	if (hdr->b_datacnt > 1 && hdr->b_type == ARC_BUFC_DATA)
17519253d63dSGeorge Wilson 		evict_needed = B_TRUE;
17529253d63dSGeorge Wilson 
17539253d63dSGeorge Wilson 	mutex_exit(&buf->b_evict_lock);
17549253d63dSGeorge Wilson 	return (evict_needed);
17559253d63dSGeorge Wilson }
17569253d63dSGeorge Wilson 
1757fa9e4066Sahrens /*
1758fa9e4066Sahrens  * Evict buffers from list until we've removed the specified number of
1759fa9e4066Sahrens  * bytes.  Move the removed buffers to the appropriate evict state.
176044eda4d7Smaybee  * If the recycle flag is set, then attempt to "recycle" a buffer:
176144eda4d7Smaybee  * - look for a buffer to evict that is `bytes' long.
176244eda4d7Smaybee  * - return the data block from this buffer rather than freeing it.
176344eda4d7Smaybee  * This flag is used by callers that are trying to make space for a
176444eda4d7Smaybee  * new buffer in a full arc cache.
1765874395d5Smaybee  *
1766874395d5Smaybee  * This function makes a "best effort".  It skips over any buffers
1767874395d5Smaybee  * it can't get a hash_lock on, and so may not catch all candidates.
1768874395d5Smaybee  * It may also return without evicting as much space as requested.
1769fa9e4066Sahrens  */
177044eda4d7Smaybee static void *
1771ac05c741SMark Maybee arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
1772ad23a2dbSjohansen     arc_buf_contents_t type)
1773fa9e4066Sahrens {
1774fa9e4066Sahrens 	arc_state_t *evicted_state;
177544eda4d7Smaybee 	uint64_t bytes_evicted = 0, skipped = 0, missed = 0;
17763fa51506Smaybee 	arc_buf_hdr_t *ab, *ab_prev = NULL;
17770e8c6158Smaybee 	list_t *list = &state->arcs_list[type];
1778fa9e4066Sahrens 	kmutex_t *hash_lock;
177944eda4d7Smaybee 	boolean_t have_lock;
17803fa51506Smaybee 	void *stolen = NULL;
1781fa9e4066Sahrens 
178244cb6abcSbmc 	ASSERT(state == arc_mru || state == arc_mfu);
1783fa9e4066Sahrens 
178444cb6abcSbmc 	evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
1785fa9e4066Sahrens 
178644cb6abcSbmc 	mutex_enter(&state->arcs_mtx);
178744cb6abcSbmc 	mutex_enter(&evicted_state->arcs_mtx);
1788fa9e4066Sahrens 
17890e8c6158Smaybee 	for (ab = list_tail(list); ab; ab = ab_prev) {
17900e8c6158Smaybee 		ab_prev = list_prev(list, ab);
179113506d1eSmaybee 		/* prefetch buffers have a minimum lifespan */
179244eda4d7Smaybee 		if (HDR_IO_IN_PROGRESS(ab) ||
1793874395d5Smaybee 		    (spa && ab->b_spa != spa) ||
179444eda4d7Smaybee 		    (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) &&
1795d3d50737SRafael Vanoni 		    ddi_get_lbolt() - ab->b_arc_access <
1796d3d50737SRafael Vanoni 		    arc_min_prefetch_lifespan)) {
179713506d1eSmaybee 			skipped++;
179813506d1eSmaybee 			continue;
179913506d1eSmaybee 		}
18003fa51506Smaybee 		/* "lookahead" for better eviction candidate */
18013fa51506Smaybee 		if (recycle && ab->b_size != bytes &&
18023fa51506Smaybee 		    ab_prev && ab_prev->b_size == bytes)
180344eda4d7Smaybee 			continue;
1804fa9e4066Sahrens 		hash_lock = HDR_LOCK(ab);
180544eda4d7Smaybee 		have_lock = MUTEX_HELD(hash_lock);
180644eda4d7Smaybee 		if (have_lock || mutex_tryenter(hash_lock)) {
1807fb09f5aaSMadhav Suresh 			ASSERT0(refcount_count(&ab->b_refcnt));
1808ea8dc4b6Seschrock 			ASSERT(ab->b_datacnt > 0);
1809ea8dc4b6Seschrock 			while (ab->b_buf) {
1810ea8dc4b6Seschrock 				arc_buf_t *buf = ab->b_buf;
18113f9d6ad7SLin Ling 				if (!mutex_tryenter(&buf->b_evict_lock)) {
18126f83844dSMark Maybee 					missed += 1;
18136f83844dSMark Maybee 					break;
18146f83844dSMark Maybee 				}
181544eda4d7Smaybee 				if (buf->b_data) {
1816ea8dc4b6Seschrock 					bytes_evicted += ab->b_size;
1817ad23a2dbSjohansen 					if (recycle && ab->b_type == type &&
1818fa94a07fSbrendan 					    ab->b_size == bytes &&
1819fa94a07fSbrendan 					    !HDR_L2_WRITING(ab)) {
18203fa51506Smaybee 						stolen = buf->b_data;
18213fa51506Smaybee 						recycle = FALSE;
18223fa51506Smaybee 					}
182344eda4d7Smaybee 				}
1824ea8dc4b6Seschrock 				if (buf->b_efunc) {
1825ea8dc4b6Seschrock 					mutex_enter(&arc_eviction_mtx);
18263fa51506Smaybee 					arc_buf_destroy(buf,
18273fa51506Smaybee 					    buf->b_data == stolen, FALSE);
1828ea8dc4b6Seschrock 					ab->b_buf = buf->b_next;
182940d7d650Smaybee 					buf->b_hdr = &arc_eviction_hdr;
1830ea8dc4b6Seschrock 					buf->b_next = arc_eviction_list;
1831ea8dc4b6Seschrock 					arc_eviction_list = buf;
1832ea8dc4b6Seschrock 					mutex_exit(&arc_eviction_mtx);
18333f9d6ad7SLin Ling 					mutex_exit(&buf->b_evict_lock);
1834ea8dc4b6Seschrock 				} else {
18353f9d6ad7SLin Ling 					mutex_exit(&buf->b_evict_lock);
18363fa51506Smaybee 					arc_buf_destroy(buf,
18373fa51506Smaybee 					    buf->b_data == stolen, TRUE);
1838ea8dc4b6Seschrock 				}
1839ea8dc4b6Seschrock 			}
18405ea40c06SBrendan Gregg - Sun Microsystems 
18415ea40c06SBrendan Gregg - Sun Microsystems 			if (ab->b_l2hdr) {
18425ea40c06SBrendan Gregg - Sun Microsystems 				ARCSTAT_INCR(arcstat_evict_l2_cached,
18435ea40c06SBrendan Gregg - Sun Microsystems 				    ab->b_size);
18445ea40c06SBrendan Gregg - Sun Microsystems 			} else {
18455ea40c06SBrendan Gregg - Sun Microsystems 				if (l2arc_write_eligible(ab->b_spa, ab)) {
18465ea40c06SBrendan Gregg - Sun Microsystems 					ARCSTAT_INCR(arcstat_evict_l2_eligible,
18475ea40c06SBrendan Gregg - Sun Microsystems 					    ab->b_size);
18485ea40c06SBrendan Gregg - Sun Microsystems 				} else {
18495ea40c06SBrendan Gregg - Sun Microsystems 					ARCSTAT_INCR(
18505ea40c06SBrendan Gregg - Sun Microsystems 					    arcstat_evict_l2_ineligible,
18515ea40c06SBrendan Gregg - Sun Microsystems 					    ab->b_size);
18525ea40c06SBrendan Gregg - Sun Microsystems 				}
18535ea40c06SBrendan Gregg - Sun Microsystems 			}
18545ea40c06SBrendan Gregg - Sun Microsystems 
18556f83844dSMark Maybee 			if (ab->b_datacnt == 0) {
18566f83844dSMark Maybee 				arc_change_state(evicted_state, ab, hash_lock);
18576f83844dSMark Maybee 				ASSERT(HDR_IN_HASH_TABLE(ab));
18586f83844dSMark Maybee 				ab->b_flags |= ARC_IN_HASH_TABLE;
18596f83844dSMark Maybee 				ab->b_flags &= ~ARC_BUF_AVAILABLE;
18606f83844dSMark Maybee 				DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab);
18616f83844dSMark Maybee 			}
186244eda4d7Smaybee 			if (!have_lock)
186344eda4d7Smaybee 				mutex_exit(hash_lock);
1864ea8dc4b6Seschrock 			if (bytes >= 0 && bytes_evicted >= bytes)
1865fa9e4066Sahrens 				break;
1866fa9e4066Sahrens 		} else {
186744eda4d7Smaybee 			missed += 1;
1868fa9e4066Sahrens 		}
1869fa9e4066Sahrens 	}
187044cb6abcSbmc 
187144cb6abcSbmc 	mutex_exit(&evicted_state->arcs_mtx);
187244cb6abcSbmc 	mutex_exit(&state->arcs_mtx);
1873fa9e4066Sahrens 
1874fa9e4066Sahrens 	if (bytes_evicted < bytes)
1875fa9e4066Sahrens 		dprintf("only evicted %lld bytes from %x",
1876fa9e4066Sahrens 		    (longlong_t)bytes_evicted, state);
1877fa9e4066Sahrens 
187844eda4d7Smaybee 	if (skipped)
187944cb6abcSbmc 		ARCSTAT_INCR(arcstat_evict_skip, skipped);
188044cb6abcSbmc 
188144eda4d7Smaybee 	if (missed)
188244cb6abcSbmc 		ARCSTAT_INCR(arcstat_mutex_miss, missed);
1883f4d2e9e6Smaybee 
1884f4d2e9e6Smaybee 	/*
18853b2aab18SMatthew Ahrens 	 * We have just evicted some data into the ghost state, make
1886f4d2e9e6Smaybee 	 * sure we also adjust the ghost state size if necessary.
1887f4d2e9e6Smaybee 	 */
1888f4d2e9e6Smaybee 	if (arc_no_grow &&
1889f4d2e9e6Smaybee 	    arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size > arc_c) {
1890f4d2e9e6Smaybee 		int64_t mru_over = arc_anon->arcs_size + arc_mru->arcs_size +
1891f4d2e9e6Smaybee 		    arc_mru_ghost->arcs_size - arc_c;
1892f4d2e9e6Smaybee 
1893f4d2e9e6Smaybee 		if (mru_over > 0 && arc_mru_ghost->arcs_lsize[type] > 0) {
1894f4d2e9e6Smaybee 			int64_t todelete =
1895f4d2e9e6Smaybee 			    MIN(arc_mru_ghost->arcs_lsize[type], mru_over);
1896874395d5Smaybee 			arc_evict_ghost(arc_mru_ghost, NULL, todelete);
1897f4d2e9e6Smaybee 		} else if (arc_mfu_ghost->arcs_lsize[type] > 0) {
1898f4d2e9e6Smaybee 			int64_t todelete = MIN(arc_mfu_ghost->arcs_lsize[type],
1899f4d2e9e6Smaybee 			    arc_mru_ghost->arcs_size +
1900f4d2e9e6Smaybee 			    arc_mfu_ghost->arcs_size - arc_c);
1901874395d5Smaybee 			arc_evict_ghost(arc_mfu_ghost, NULL, todelete);
1902f4d2e9e6Smaybee 		}
1903f4d2e9e6Smaybee 	}
190444cb6abcSbmc 
19053fa51506Smaybee 	return (stolen);
1906fa9e4066Sahrens }
1907fa9e4066Sahrens 
1908fa9e4066Sahrens /*
1909fa9e4066Sahrens  * Remove buffers from list until we've removed the specified number of
1910fa9e4066Sahrens  * bytes.  Destroy the buffers that are removed.
1911fa9e4066Sahrens  */
1912fa9e4066Sahrens static void
1913ac05c741SMark Maybee arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes)
1914fa9e4066Sahrens {
1915fa9e4066Sahrens 	arc_buf_hdr_t *ab, *ab_prev;
1916b802aa8cSSanjeev Bagewadi 	arc_buf_hdr_t marker = { 0 };
19170e8c6158Smaybee 	list_t *list = &state->arcs_list[ARC_BUFC_DATA];
1918fa9e4066Sahrens 	kmutex_t *hash_lock;
1919ea8dc4b6Seschrock 	uint64_t bytes_deleted = 0;
1920c0a81264Sek 	uint64_t bufs_skipped = 0;
1921fa9e4066Sahrens 
1922ea8dc4b6Seschrock 	ASSERT(GHOST_STATE(state));
1923fa9e4066Sahrens top:
192444cb6abcSbmc 	mutex_enter(&state->arcs_mtx);
19250e8c6158Smaybee 	for (ab = list_tail(list); ab; ab = ab_prev) {
19260e8c6158Smaybee 		ab_prev = list_prev(list, ab);
1927874395d5Smaybee 		if (spa && ab->b_spa != spa)
1928874395d5Smaybee 			continue;
1929b802aa8cSSanjeev Bagewadi 
1930b802aa8cSSanjeev Bagewadi 		/* ignore markers */
1931b802aa8cSSanjeev Bagewadi 		if (ab->b_spa == 0)
1932b802aa8cSSanjeev Bagewadi 			continue;
1933b802aa8cSSanjeev Bagewadi 
1934fa9e4066Sahrens 		hash_lock = HDR_LOCK(ab);
19357e453561SWilliam Gorrell 		/* caller may be trying to modify this buffer, skip it */
19367e453561SWilliam Gorrell 		if (MUTEX_HELD(hash_lock))
19377e453561SWilliam Gorrell 			continue;
19387e453561SWilliam Gorrell 		if (mutex_tryenter(hash_lock)) {
193913506d1eSmaybee 			ASSERT(!HDR_IO_IN_PROGRESS(ab));
1940ea8dc4b6Seschrock 			ASSERT(ab->b_buf == NULL);
194144cb6abcSbmc 			ARCSTAT_BUMP(arcstat_deleted);
1942fa9e4066Sahrens 			bytes_deleted += ab->b_size;
1943fa94a07fSbrendan 
1944fa94a07fSbrendan 			if (ab->b_l2hdr != NULL) {
1945fa94a07fSbrendan 				/*
1946fa94a07fSbrendan 				 * This buffer is cached on the 2nd Level ARC;
1947fa94a07fSbrendan 				 * don't destroy the header.
1948fa94a07fSbrendan 				 */
1949fa94a07fSbrendan 				arc_change_state(arc_l2c_only, ab, hash_lock);
19507e453561SWilliam Gorrell 				mutex_exit(hash_lock);
1951fa94a07fSbrendan 			} else {
1952fa94a07fSbrendan 				arc_change_state(arc_anon, ab, hash_lock);
19537e453561SWilliam Gorrell 				mutex_exit(hash_lock);
1954fa94a07fSbrendan 				arc_hdr_destroy(ab);
1955fa94a07fSbrendan 			}
1956fa94a07fSbrendan 
1957ea8dc4b6Seschrock 			DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab);
1958fa9e4066Sahrens 			if (bytes >= 0 && bytes_deleted >= bytes)
1959fa9e4066Sahrens 				break;
1960b802aa8cSSanjeev Bagewadi 		} else if (bytes < 0) {
1961b802aa8cSSanjeev Bagewadi 			/*
1962b802aa8cSSanjeev Bagewadi 			 * Insert a list marker and then wait for the
1963b802aa8cSSanjeev Bagewadi 			 * hash lock to become available. Once its
1964b802aa8cSSanjeev Bagewadi 			 * available, restart from where we left off.
1965b802aa8cSSanjeev Bagewadi 			 */
1966b802aa8cSSanjeev Bagewadi 			list_insert_after(list, ab, &marker);
1967b802aa8cSSanjeev Bagewadi 			mutex_exit(&state->arcs_mtx);
1968b802aa8cSSanjeev Bagewadi 			mutex_enter(hash_lock);
1969b802aa8cSSanjeev Bagewadi 			mutex_exit(hash_lock);
1970b802aa8cSSanjeev Bagewadi 			mutex_enter(&state->arcs_mtx);
1971b802aa8cSSanjeev Bagewadi 			ab_prev = list_prev(list, &marker);
1972b802aa8cSSanjeev Bagewadi 			list_remove(list, &marker);
1973b802aa8cSSanjeev Bagewadi 		} else
1974fa9e4066Sahrens 			bufs_skipped += 1;
1975fa9e4066Sahrens 	}
197644cb6abcSbmc 	mutex_exit(&state->arcs_mtx);
1977fa9e4066Sahrens 
19780e8c6158Smaybee 	if (list == &state->arcs_list[ARC_BUFC_DATA] &&
19790e8c6158Smaybee 	    (bytes < 0 || bytes_deleted < bytes)) {
19800e8c6158Smaybee 		list = &state->arcs_list[ARC_BUFC_METADATA];
19810e8c6158Smaybee 		goto top;
19820e8c6158Smaybee 	}
19830e8c6158Smaybee 
1984fa9e4066Sahrens 	if (bufs_skipped) {
198544cb6abcSbmc 		ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped);
1986fa9e4066Sahrens 		ASSERT(bytes >= 0);
1987fa9e4066Sahrens 	}
1988fa9e4066Sahrens 
1989fa9e4066Sahrens 	if (bytes_deleted < bytes)
1990fa9e4066Sahrens 		dprintf("only deleted %lld bytes from %p",
1991fa9e4066Sahrens 		    (longlong_t)bytes_deleted, state);
1992fa9e4066Sahrens }
1993fa9e4066Sahrens 
1994fa9e4066Sahrens static void
1995fa9e4066Sahrens arc_adjust(void)
1996fa9e4066Sahrens {
19975a98e54bSBrendan Gregg - Sun Microsystems 	int64_t adjustment, delta;
1998fa9e4066Sahrens 
19995a98e54bSBrendan Gregg - Sun Microsystems 	/*
20005a98e54bSBrendan Gregg - Sun Microsystems 	 * Adjust MRU size
20015a98e54bSBrendan Gregg - Sun Microsystems 	 */
20025a98e54bSBrendan Gregg - Sun Microsystems 
20033e4e8481STom Erickson 	adjustment = MIN((int64_t)(arc_size - arc_c),
20043e4e8481STom Erickson 	    (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used -
20053e4e8481STom Erickson 	    arc_p));
2006fa9e4066Sahrens 
20075a98e54bSBrendan Gregg - Sun Microsystems 	if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) {
20085a98e54bSBrendan Gregg - Sun Microsystems 		delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment);
20095a98e54bSBrendan Gregg - Sun Microsystems 		(void) arc_evict(arc_mru, NULL, delta, FALSE, ARC_BUFC_DATA);
20105a98e54bSBrendan Gregg - Sun Microsystems 		adjustment -= delta;
20110e8c6158Smaybee 	}
20120e8c6158Smaybee 
20135a98e54bSBrendan Gregg - Sun Microsystems 	if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) {
20145a98e54bSBrendan Gregg - Sun Microsystems 		delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment);
20155a98e54bSBrendan Gregg - Sun Microsystems 		(void) arc_evict(arc_mru, NULL, delta, FALSE,
2016874395d5Smaybee 		    ARC_BUFC_METADATA);
2017fa9e4066Sahrens 	}
2018fa9e4066Sahrens 
20195a98e54bSBrendan Gregg - Sun Microsystems 	/*
20205a98e54bSBrendan Gregg - Sun Microsystems 	 * Adjust MFU size
20215a98e54bSBrendan Gregg - Sun Microsystems 	 */
2022fa9e4066Sahrens 
20235a98e54bSBrendan Gregg - Sun Microsystems 	adjustment = arc_size - arc_c;
20245a98e54bSBrendan Gregg - Sun Microsystems 
20255a98e54bSBrendan Gregg - Sun Microsystems 	if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) {
20265a98e54bSBrendan Gregg - Sun Microsystems 		delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]);
20275a98e54bSBrendan Gregg - Sun Microsystems 		(void) arc_evict(arc_mfu, NULL, delta, FALSE, ARC_BUFC_DATA);
20285a98e54bSBrendan Gregg - Sun Microsystems 		adjustment -= delta;
2029fa9e4066Sahrens 	}
2030fa9e4066Sahrens 
20315a98e54bSBrendan Gregg - Sun Microsystems 	if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) {
20325a98e54bSBrendan Gregg - Sun Microsystems 		int64_t delta = MIN(adjustment,
20335a98e54bSBrendan Gregg - Sun Microsystems 		    arc_mfu->arcs_lsize[ARC_BUFC_METADATA]);
20345a98e54bSBrendan Gregg - Sun Microsystems 		(void) arc_evict(arc_mfu, NULL, delta, FALSE,
20355a98e54bSBrendan Gregg - Sun Microsystems 		    ARC_BUFC_METADATA);
20365a98e54bSBrendan Gregg - Sun Microsystems 	}
2037fa9e4066Sahrens 
20385a98e54bSBrendan Gregg - Sun Microsystems 	/*
20395a98e54bSBrendan Gregg - Sun Microsystems 	 * Adjust ghost lists
20405a98e54bSBrendan Gregg - Sun Microsystems 	 */
2041fa9e4066Sahrens 
20425a98e54bSBrendan Gregg - Sun Microsystems 	adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c;
2043fa9e4066Sahrens 
20445a98e54bSBrendan Gregg - Sun Microsystems 	if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) {
20455a98e54bSBrendan Gregg - Sun Microsystems 		delta = MIN(arc_mru_ghost->arcs_size, adjustment);
20465a98e54bSBrendan Gregg - Sun Microsystems 		arc_evict_ghost(arc_mru_ghost, NULL, delta);
20475a98e54bSBrendan Gregg - Sun Microsystems 	}
20480e8c6158Smaybee 
20495a98e54bSBrendan Gregg - Sun Microsystems 	adjustment =
20505a98e54bSBrendan Gregg - Sun Microsystems 	    arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c;
20515a98e54bSBrendan Gregg - Sun Microsystems 
20525a98e54bSBrendan Gregg - Sun Microsystems 	if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) {
20535a98e54bSBrendan Gregg - Sun Microsystems 		delta = MIN(arc_mfu_ghost->arcs_size, adjustment);
20545a98e54bSBrendan Gregg - Sun Microsystems 		arc_evict_ghost(arc_mfu_ghost, NULL, delta);
2055fa9e4066Sahrens 	}
2056fa9e4066Sahrens }
2057fa9e4066Sahrens 
2058ea8dc4b6Seschrock static void
2059ea8dc4b6Seschrock arc_do_user_evicts(void)
2060ea8dc4b6Seschrock {
2061ea8dc4b6Seschrock 	mutex_enter(&arc_eviction_mtx);
2062ea8dc4b6Seschrock 	while (arc_eviction_list != NULL) {
2063ea8dc4b6Seschrock 		arc_buf_t *buf = arc_eviction_list;
2064ea8dc4b6Seschrock 		arc_eviction_list = buf->b_next;
20653f9d6ad7SLin Ling 		mutex_enter(&buf->b_evict_lock);
2066ea8dc4b6Seschrock 		buf->b_hdr = NULL;
20673f9d6ad7SLin Ling 		mutex_exit(&buf->b_evict_lock);
2068ea8dc4b6Seschrock 		mutex_exit(&arc_eviction_mtx);
2069ea8dc4b6Seschrock 
2070dd6ef538Smaybee 		if (buf->b_efunc != NULL)
2071dd6ef538Smaybee 			VERIFY(buf->b_efunc(buf) == 0);
2072ea8dc4b6Seschrock 
2073ea8dc4b6Seschrock 		buf->b_efunc = NULL;
2074ea8dc4b6Seschrock 		buf->b_private = NULL;
2075ea8dc4b6Seschrock 		kmem_cache_free(buf_cache, buf);
2076ea8dc4b6Seschrock 		mutex_enter(&arc_eviction_mtx);
2077ea8dc4b6Seschrock 	}
2078ea8dc4b6Seschrock 	mutex_exit(&arc_eviction_mtx);
2079ea8dc4b6Seschrock }
2080ea8dc4b6Seschrock 
2081fa9e4066Sahrens /*
2082874395d5Smaybee  * Flush all *evictable* data from the cache for the given spa.
2083fa9e4066Sahrens  * NOTE: this will not touch "active" (i.e. referenced) data.
2084fa9e4066Sahrens  */
2085fa9e4066Sahrens void
2086874395d5Smaybee arc_flush(spa_t *spa)
2087fa9e4066Sahrens {
2088ac05c741SMark Maybee 	uint64_t guid = 0;
2089ac05c741SMark Maybee 
2090ac05c741SMark Maybee 	if (spa)
2091e9103aaeSGarrett D'Amore 		guid = spa_load_guid(spa);
2092ac05c741SMark Maybee 
2093874395d5Smaybee 	while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) {
2094ac05c741SMark Maybee 		(void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA);
2095874395d5Smaybee 		if (spa)
2096874395d5Smaybee 			break;
2097874395d5Smaybee 	}
2098874395d5Smaybee 	while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) {
2099ac05c741SMark Maybee 		(void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA);
2100874395d5Smaybee 		if (spa)
2101874395d5Smaybee 			break;
2102874395d5Smaybee 	}
2103874395d5Smaybee 	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) {
2104ac05c741SMark Maybee 		(void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA);
2105874395d5Smaybee 		if (spa)
2106874395d5Smaybee 			break;
2107874395d5Smaybee 	}
2108874395d5Smaybee 	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) {
2109ac05c741SMark Maybee 		(void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA);
2110874395d5Smaybee 		if (spa)
2111874395d5Smaybee 			break;
2112874395d5Smaybee 	}
2113874395d5Smaybee 
2114ac05c741SMark Maybee 	arc_evict_ghost(arc_mru_ghost, guid, -1);
2115ac05c741SMark Maybee 	arc_evict_ghost(arc_mfu_ghost, guid, -1);
2116ea8dc4b6Seschrock 
2117ea8dc4b6Seschrock 	mutex_enter(&arc_reclaim_thr_lock);
2118ea8dc4b6Seschrock 	arc_do_user_evicts();
2119ea8dc4b6Seschrock 	mutex_exit(&arc_reclaim_thr_lock);
2120874395d5Smaybee 	ASSERT(spa || arc_eviction_list == NULL);
2121fa9e4066Sahrens }
2122fa9e4066Sahrens 
2123fa9e4066Sahrens void
212449e3519aSmaybee arc_shrink(void)
2125fa9e4066Sahrens {
212644cb6abcSbmc 	if (arc_c > arc_c_min) {
212749e3519aSmaybee 		uint64_t to_free;
2128fa9e4066Sahrens 
21293cff2f43Sstans #ifdef _KERNEL
213044cb6abcSbmc 		to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree));
21313cff2f43Sstans #else
213244cb6abcSbmc 		to_free = arc_c >> arc_shrink_shift;
21333cff2f43Sstans #endif
213444cb6abcSbmc 		if (arc_c > arc_c_min + to_free)
213544cb6abcSbmc 			atomic_add_64(&arc_c, -to_free);
213649e3519aSmaybee 		else
213744cb6abcSbmc 			arc_c = arc_c_min;
213844cb6abcSbmc 
213944cb6abcSbmc 		atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
214044cb6abcSbmc 		if (arc_c > arc_size)
214144cb6abcSbmc 			arc_c = MAX(arc_size, arc_c_min);
214244cb6abcSbmc 		if (arc_p > arc_c)
214344cb6abcSbmc 			arc_p = (arc_c >> 1);
214444cb6abcSbmc 		ASSERT(arc_c >= arc_c_min);
214544cb6abcSbmc 		ASSERT((int64_t)arc_p >= 0);
214649e3519aSmaybee 	}
2147fa9e4066Sahrens 
214844cb6abcSbmc 	if (arc_size > arc_c)
214949e3519aSmaybee 		arc_adjust();
2150fa9e4066Sahrens }
2151fa9e4066Sahrens 
215294dd93aeSGeorge Wilson /*
215394dd93aeSGeorge Wilson  * Determine if the system is under memory pressure and is asking
215494dd93aeSGeorge Wilson  * to reclaim memory. A return value of 1 indicates that the system
215594dd93aeSGeorge Wilson  * is under memory pressure and that the arc should adjust accordingly.
215694dd93aeSGeorge Wilson  */
2157fa9e4066Sahrens static int
2158fa9e4066Sahrens arc_reclaim_needed(void)
2159fa9e4066Sahrens {
2160fa9e4066Sahrens 	uint64_t extra;
2161fa9e4066Sahrens 
2162fa9e4066Sahrens #ifdef _KERNEL
21633cff2f43Sstans 
21643cff2f43Sstans 	if (needfree)
21653cff2f43Sstans 		return (1);
21663cff2f43Sstans 
2167fa9e4066Sahrens 	/*
2168fa9e4066Sahrens 	 * take 'desfree' extra pages, so we reclaim sooner, rather than later
2169fa9e4066Sahrens 	 */
2170fa9e4066Sahrens 	extra = desfree;
2171fa9e4066Sahrens 
2172fa9e4066Sahrens 	/*
2173fa9e4066Sahrens 	 * check that we're out of range of the pageout scanner.  It starts to
2174fa9e4066Sahrens 	 * schedule paging if freemem is less than lotsfree and needfree.
2175fa9e4066Sahrens 	 * lotsfree is the high-water mark for pageout, and needfree is the
2176fa9e4066Sahrens 	 * number of needed free pages.  We add extra pages here to make sure
2177fa9e4066Sahrens 	 * the scanner doesn't start up while we're freeing memory.
2178fa9e4066Sahrens 	 */
2179fa9e4066Sahrens 	if (freemem < lotsfree + needfree + extra)
2180fa9e4066Sahrens 		return (1);
2181fa9e4066Sahrens 
2182fa9e4066Sahrens 	/*
2183fa9e4066Sahrens 	 * check to make sure that swapfs has enough space so that anon
2184fa94a07fSbrendan 	 * reservations can still succeed. anon_resvmem() checks that the
2185fa9e4066Sahrens 	 * availrmem is greater than swapfs_minfree, and the number of reserved
2186fa9e4066Sahrens 	 * swap pages.  We also add a bit of extra here just to prevent
2187fa9e4066Sahrens 	 * circumstances from getting really dire.
2188fa9e4066Sahrens 	 */
2189fa9e4066Sahrens 	if (availrmem < swapfs_minfree + swapfs_reserve + extra)
2190fa9e4066Sahrens 		return (1);
2191fa9e4066Sahrens 
21925dc8af33Smaybee #if defined(__i386)
2193fa9e4066Sahrens 	/*
2194fa9e4066Sahrens 	 * If we're on an i386 platform, it's possible that we'll exhaust the
2195fa9e4066Sahrens 	 * kernel heap space before we ever run out of available physical
2196fa9e4066Sahrens 	 * memory.  Most checks of the size of the heap_area compare against
2197fa9e4066Sahrens 	 * tune.t_minarmem, which is the minimum available real memory that we
2198fa9e4066Sahrens 	 * can have in the system.  However, this is generally fixed at 25 pages
2199fa9e4066Sahrens 	 * which is so low that it's useless.  In this comparison, we seek to
2200fa9e4066Sahrens 	 * calculate the total heap-size, and reclaim if more than 3/4ths of the
2201fa94a07fSbrendan 	 * heap is allocated.  (Or, in the calculation, if less than 1/4th is
2202fa9e4066Sahrens 	 * free)
2203fa9e4066Sahrens 	 */
220494dd93aeSGeorge Wilson 	if (vmem_size(heap_arena, VMEM_FREE) <
220594dd93aeSGeorge Wilson 	    (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2))
2206fa9e4066Sahrens 		return (1);
2207fa9e4066Sahrens #endif
2208fa9e4066Sahrens 
220994dd93aeSGeorge Wilson 	/*
221094dd93aeSGeorge Wilson 	 * If zio data pages are being allocated out of a separate heap segment,
221194dd93aeSGeorge Wilson 	 * then enforce that the size of available vmem for this arena remains
221294dd93aeSGeorge Wilson 	 * above about 1/16th free.
221394dd93aeSGeorge Wilson 	 *
221494dd93aeSGeorge Wilson 	 * Note: The 1/16th arena free requirement was put in place
221594dd93aeSGeorge Wilson 	 * to aggressively evict memory from the arc in order to avoid
221694dd93aeSGeorge Wilson 	 * memory fragmentation issues.
221794dd93aeSGeorge Wilson 	 */
221894dd93aeSGeorge Wilson 	if (zio_arena != NULL &&
221994dd93aeSGeorge Wilson 	    vmem_size(zio_arena, VMEM_FREE) <
222094dd93aeSGeorge Wilson 	    (vmem_size(zio_arena, VMEM_ALLOC) >> 4))
222194dd93aeSGeorge Wilson 		return (1);
2222fa9e4066Sahrens #else
2223fa9e4066Sahrens 	if (spa_get_random(100) == 0)
2224fa9e4066Sahrens 		return (1);
2225fa9e4066Sahrens #endif
2226fa9e4066Sahrens 	return (0);
2227fa9e4066Sahrens }
2228fa9e4066Sahrens 
2229fa9e4066Sahrens static void
2230fa9e4066Sahrens arc_kmem_reap_now(arc_reclaim_strategy_t strat)
2231fa9e4066Sahrens {
2232fa9e4066Sahrens 	size_t			i;
2233fa9e4066Sahrens 	kmem_cache_t		*prev_cache = NULL;
2234ad23a2dbSjohansen 	kmem_cache_t		*prev_data_cache = NULL;
2235fa9e4066Sahrens 	extern kmem_cache_t	*zio_buf_cache[];
2236ad23a2dbSjohansen 	extern kmem_cache_t	*zio_data_buf_cache[];
2237fa9e4066Sahrens 
2238033f9833Sek #ifdef _KERNEL
22390e8c6158Smaybee 	if (arc_meta_used >= arc_meta_limit) {
22400e8c6158Smaybee 		/*
22410e8c6158Smaybee 		 * We are exceeding our meta-data cache limit.
22420e8c6158Smaybee 		 * Purge some DNLC entries to release holds on meta-data.
22430e8c6158Smaybee 		 */
22440e8c6158Smaybee 		dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
22450e8c6158Smaybee 	}
22465dc8af33Smaybee #if defined(__i386)
22475dc8af33Smaybee 	/*
22485dc8af33Smaybee 	 * Reclaim unused memory from all kmem caches.
22495dc8af33Smaybee 	 */
22505dc8af33Smaybee 	kmem_reap();
22515dc8af33Smaybee #endif
2252033f9833Sek #endif
2253033f9833Sek 
2254fa9e4066Sahrens 	/*
2255fa94a07fSbrendan 	 * An aggressive reclamation will shrink the cache size as well as
2256ea8dc4b6Seschrock 	 * reap free buffers from the arc kmem caches.
2257fa9e4066Sahrens 	 */
2258fa9e4066Sahrens 	if (strat == ARC_RECLAIM_AGGR)
225949e3519aSmaybee 		arc_shrink();
2260fa9e4066Sahrens 
2261fa9e4066Sahrens 	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
2262fa9e4066Sahrens 		if (zio_buf_cache[i] != prev_cache) {
2263fa9e4066Sahrens 			prev_cache = zio_buf_cache[i];
2264fa9e4066Sahrens 			kmem_cache_reap_now(zio_buf_cache[i]);
2265fa9e4066Sahrens 		}
2266ad23a2dbSjohansen 		if (zio_data_buf_cache[i] != prev_data_cache) {
2267ad23a2dbSjohansen 			prev_data_cache = zio_data_buf_cache[i];
2268ad23a2dbSjohansen 			kmem_cache_reap_now(zio_data_buf_cache[i]);
2269ad23a2dbSjohansen 		}
2270fa9e4066Sahrens 	}
2271ea8dc4b6Seschrock 	kmem_cache_reap_now(buf_cache);
2272ea8dc4b6Seschrock 	kmem_cache_reap_now(hdr_cache);
227394dd93aeSGeorge Wilson 
227494dd93aeSGeorge Wilson 	/*
227594dd93aeSGeorge Wilson 	 * Ask the vmem areana to reclaim unused memory from its
227694dd93aeSGeorge Wilson 	 * quantum caches.
227794dd93aeSGeorge Wilson 	 */
227894dd93aeSGeorge Wilson 	if (zio_arena != NULL && strat == ARC_RECLAIM_AGGR)
227994dd93aeSGeorge Wilson 		vmem_qcache_reap(zio_arena);
2280fa9e4066Sahrens }
2281fa9e4066Sahrens 
2282fa9e4066Sahrens static void
2283fa9e4066Sahrens arc_reclaim_thread(void)
2284fa9e4066Sahrens {
2285fa9e4066Sahrens 	clock_t			growtime = 0;
2286fa9e4066Sahrens 	arc_reclaim_strategy_t	last_reclaim = ARC_RECLAIM_CONS;
2287fa9e4066Sahrens 	callb_cpr_t		cpr;
2288fa9e4066Sahrens 
2289fa9e4066Sahrens 	CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG);
2290fa9e4066Sahrens 
2291fa9e4066Sahrens 	mutex_enter(&arc_reclaim_thr_lock);
2292fa9e4066Sahrens 	while (arc_thread_exit == 0) {
2293fa9e4066Sahrens 		if (arc_reclaim_needed()) {
2294fa9e4066Sahrens 
229544cb6abcSbmc 			if (arc_no_grow) {
2296fa9e4066Sahrens 				if (last_reclaim == ARC_RECLAIM_CONS) {
2297fa9e4066Sahrens 					last_reclaim = ARC_RECLAIM_AGGR;
2298fa9e4066Sahrens 				} else {
2299fa9e4066Sahrens 					last_reclaim = ARC_RECLAIM_CONS;
2300fa9e4066Sahrens 				}
2301fa9e4066Sahrens 			} else {
230244cb6abcSbmc 				arc_no_grow = TRUE;
2303fa9e4066Sahrens 				last_reclaim = ARC_RECLAIM_AGGR;
2304fa9e4066Sahrens 				membar_producer();
2305fa9e4066Sahrens 			}
2306fa9e4066Sahrens 
2307fa9e4066Sahrens 			/* reset the growth delay for every reclaim */
2308d3d50737SRafael Vanoni 			growtime = ddi_get_lbolt() + (arc_grow_retry * hz);
2309fa9e4066Sahrens 
2310fa9e4066Sahrens 			arc_kmem_reap_now(last_reclaim);
23113a737e0dSbrendan 			arc_warm = B_TRUE;
2312fa9e4066Sahrens 
2313d3d50737SRafael Vanoni 		} else if (arc_no_grow && ddi_get_lbolt() >= growtime) {
231444cb6abcSbmc 			arc_no_grow = FALSE;
2315fa9e4066Sahrens 		}
2316fa9e4066Sahrens 
23173e4e8481STom Erickson 		arc_adjust();
2318641fbdaeSmaybee 
2319ea8dc4b6Seschrock 		if (arc_eviction_list != NULL)
2320ea8dc4b6Seschrock 			arc_do_user_evicts();
2321ea8dc4b6Seschrock 
2322fa9e4066Sahrens 		/* block until needed, or one second, whichever is shorter */
2323fa9e4066Sahrens 		CALLB_CPR_SAFE_BEGIN(&cpr);
2324fa9e4066Sahrens 		(void) cv_timedwait(&arc_reclaim_thr_cv,
2325d3d50737SRafael Vanoni 		    &arc_reclaim_thr_lock, (ddi_get_lbolt() + hz));
2326fa9e4066Sahrens 		CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock);
2327fa9e4066Sahrens 	}
2328fa9e4066Sahrens 
2329fa9e4066Sahrens 	arc_thread_exit = 0;
2330fa9e4066Sahrens 	cv_broadcast(&arc_reclaim_thr_cv);
2331fa9e4066Sahrens 	CALLB_CPR_EXIT(&cpr);		/* drops arc_reclaim_thr_lock */
2332fa9e4066Sahrens 	thread_exit();
2333fa9e4066Sahrens }
2334fa9e4066Sahrens 
2335ea8dc4b6Seschrock /*
2336ea8dc4b6Seschrock  * Adapt arc info given the number of bytes we are trying to add and
2337ea8dc4b6Seschrock  * the state that we are comming from.  This function is only called
2338ea8dc4b6Seschrock  * when we are adding new content to the cache.
2339ea8dc4b6Seschrock  */
2340fa9e4066Sahrens static void
2341ea8dc4b6Seschrock arc_adapt(int bytes, arc_state_t *state)
2342fa9e4066Sahrens {
2343ea8dc4b6Seschrock 	int mult;
23445a98e54bSBrendan Gregg - Sun Microsystems 	uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
2345ea8dc4b6Seschrock 
2346fa94a07fSbrendan 	if (state == arc_l2c_only)
2347fa94a07fSbrendan 		return;
2348fa94a07fSbrendan 
2349ea8dc4b6Seschrock 	ASSERT(bytes > 0);
2350fa9e4066Sahrens 	/*
2351ea8dc4b6Seschrock 	 * Adapt the target size of the MRU list:
2352ea8dc4b6Seschrock 	 *	- if we just hit in the MRU ghost list, then increase
2353ea8dc4b6Seschrock 	 *	  the target size of the MRU list.
2354ea8dc4b6Seschrock 	 *	- if we just hit in the MFU ghost list, then increase
2355ea8dc4b6Seschrock 	 *	  the target size of the MFU list by decreasing the
2356ea8dc4b6Seschrock 	 *	  target size of the MRU list.
2357fa9e4066Sahrens 	 */
235844cb6abcSbmc 	if (state == arc_mru_ghost) {
235944cb6abcSbmc 		mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ?
236044cb6abcSbmc 		    1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size));
23613e4e8481STom Erickson 		mult = MIN(mult, 10); /* avoid wild arc_p adjustment */
2362ea8dc4b6Seschrock 
23635a98e54bSBrendan Gregg - Sun Microsystems 		arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
236444cb6abcSbmc 	} else if (state == arc_mfu_ghost) {
23655a98e54bSBrendan Gregg - Sun Microsystems 		uint64_t delta;
23665a98e54bSBrendan Gregg - Sun Microsystems 
236744cb6abcSbmc 		mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ?
236844cb6abcSbmc 		    1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size));
23693e4e8481STom Erickson 		mult = MIN(mult, 10);
2370ea8dc4b6Seschrock 
23715a98e54bSBrendan Gregg - Sun Microsystems 		delta = MIN(bytes * mult, arc_p);
23725a98e54bSBrendan Gregg - Sun Microsystems 		arc_p = MAX(arc_p_min, arc_p - delta);
2373ea8dc4b6Seschrock 	}
237444cb6abcSbmc 	ASSERT((int64_t)arc_p >= 0);
2375fa9e4066Sahrens 
2376fa9e4066Sahrens 	if (arc_reclaim_needed()) {
2377fa9e4066Sahrens 		cv_signal(&arc_reclaim_thr_cv);
2378fa9e4066Sahrens 		return;
2379fa9e4066Sahrens 	}
2380fa9e4066Sahrens 
238144cb6abcSbmc 	if (arc_no_grow)
2382fa9e4066Sahrens 		return;
2383fa9e4066Sahrens 
238444cb6abcSbmc 	if (arc_c >= arc_c_max)
2385ea8dc4b6Seschrock 		return;
2386ea8dc4b6Seschrock 
2387fa9e4066Sahrens 	/*
2388ea8dc4b6Seschrock 	 * If we're within (2 * maxblocksize) bytes of the target
2389ea8dc4b6Seschrock 	 * cache size, increment the target cache size
2390fa9e4066Sahrens 	 */
239144cb6abcSbmc 	if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
239244cb6abcSbmc 		atomic_add_64(&arc_c, (int64_t)bytes);
239344cb6abcSbmc 		if (arc_c > arc_c_max)
239444cb6abcSbmc 			arc_c = arc_c_max;
239544cb6abcSbmc 		else if (state == arc_anon)
239644cb6abcSbmc 			atomic_add_64(&arc_p, (int64_t)bytes);
239744cb6abcSbmc 		if (arc_p > arc_c)
239844cb6abcSbmc 			arc_p = arc_c;
2399fa9e4066Sahrens 	}
240044cb6abcSbmc 	ASSERT((int64_t)arc_p >= 0);
2401fa9e4066Sahrens }
2402fa9e4066Sahrens 
2403fa9e4066Sahrens /*
2404ea8dc4b6Seschrock  * Check if the cache has reached its limits and eviction is required
2405ea8dc4b6Seschrock  * prior to insert.
2406fa9e4066Sahrens  */
2407fa9e4066Sahrens static int
24080e8c6158Smaybee arc_evict_needed(arc_buf_contents_t type)
2409fa9e4066Sahrens {
24100e8c6158Smaybee 	if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)
24110e8c6158Smaybee 		return (1);
24120e8c6158Smaybee 
2413fa9e4066Sahrens 	if (arc_reclaim_needed())
2414fa9e4066Sahrens 		return (1);
2415fa9e4066Sahrens 
241644cb6abcSbmc 	return (arc_size > arc_c);
2417fa9e4066Sahrens }
2418fa9e4066Sahrens 
2419fa9e4066Sahrens /*
242044eda4d7Smaybee  * The buffer, supplied as the first argument, needs a data block.
242144eda4d7Smaybee  * So, if we are at cache max, determine which cache should be victimized.
242244eda4d7Smaybee  * We have the following cases:
2423fa9e4066Sahrens  *
242444cb6abcSbmc  * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) ->
2425fa9e4066Sahrens  * In this situation if we're out of space, but the resident size of the MFU is
2426fa9e4066Sahrens  * under the limit, victimize the MFU cache to satisfy this insertion request.
2427fa9e4066Sahrens  *
242844cb6abcSbmc  * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) ->
2429fa9e4066Sahrens  * Here, we've used up all of the available space for the MRU, so we need to
2430fa9e4066Sahrens  * evict from our own cache instead.  Evict from the set of resident MRU
2431fa9e4066Sahrens  * entries.
2432fa9e4066Sahrens  *
243344cb6abcSbmc  * 3. Insert for MFU (c - p) > sizeof(arc_mfu) ->
2434fa9e4066Sahrens  * c minus p represents the MFU space in the cache, since p is the size of the
2435fa9e4066Sahrens  * cache that is dedicated to the MRU.  In this situation there's still space on
2436fa9e4066Sahrens  * the MFU side, so the MRU side needs to be victimized.
2437fa9e4066Sahrens  *
243844cb6abcSbmc  * 4. Insert for MFU (c - p) < sizeof(arc_mfu) ->
2439fa9e4066Sahrens  * MFU's resident set is consuming more space than it has been allotted.  In
2440fa9e4066Sahrens  * this situation, we must victimize our own cache, the MFU, for this insertion.
2441fa9e4066Sahrens  */
2442fa9e4066Sahrens static void
244344eda4d7Smaybee arc_get_data_buf(arc_buf_t *buf)
2444fa9e4066Sahrens {
2445ad23a2dbSjohansen 	arc_state_t		*state = buf->b_hdr->b_state;
2446ad23a2dbSjohansen 	uint64_t		size = buf->b_hdr->b_size;
2447ad23a2dbSjohansen 	arc_buf_contents_t	type = buf->b_hdr->b_type;
2448fa9e4066Sahrens 
244944eda4d7Smaybee 	arc_adapt(size, state);
2450fa9e4066Sahrens 
245144eda4d7Smaybee 	/*
245244eda4d7Smaybee 	 * We have not yet reached cache maximum size,
245344eda4d7Smaybee 	 * just allocate a new buffer.
245444eda4d7Smaybee 	 */
24550e8c6158Smaybee 	if (!arc_evict_needed(type)) {
2456ad23a2dbSjohansen 		if (type == ARC_BUFC_METADATA) {
2457ad23a2dbSjohansen 			buf->b_data = zio_buf_alloc(size);
24585a98e54bSBrendan Gregg - Sun Microsystems 			arc_space_consume(size, ARC_SPACE_DATA);
2459ad23a2dbSjohansen 		} else {
2460ad23a2dbSjohansen 			ASSERT(type == ARC_BUFC_DATA);
2461ad23a2dbSjohansen 			buf->b_data = zio_data_buf_alloc(size);
24625a98e54bSBrendan Gregg - Sun Microsystems 			ARCSTAT_INCR(arcstat_data_size, size);
24630e8c6158Smaybee 			atomic_add_64(&arc_size, size);
2464ad23a2dbSjohansen 		}
246544eda4d7Smaybee 		goto out;
246644eda4d7Smaybee 	}
246744eda4d7Smaybee 
246844eda4d7Smaybee 	/*
246944eda4d7Smaybee 	 * If we are prefetching from the mfu ghost list, this buffer
247044eda4d7Smaybee 	 * will end up on the mru list; so steal space from there.
247144eda4d7Smaybee 	 */
247244cb6abcSbmc 	if (state == arc_mfu_ghost)
247344cb6abcSbmc 		state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu;
247444cb6abcSbmc 	else if (state == arc_mru_ghost)
247544cb6abcSbmc 		state = arc_mru;
247644cb6abcSbmc 
247744cb6abcSbmc 	if (state == arc_mru || state == arc_anon) {
247844cb6abcSbmc 		uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size;
24795a98e54bSBrendan Gregg - Sun Microsystems 		state = (arc_mfu->arcs_lsize[type] >= size &&
24800e8c6158Smaybee 		    arc_p > mru_used) ? arc_mfu : arc_mru;
2481fa9e4066Sahrens 	} else {
248244eda4d7Smaybee 		/* MFU cases */
248344cb6abcSbmc 		uint64_t mfu_space = arc_c - arc_p;
24845a98e54bSBrendan Gregg - Sun Microsystems 		state =  (arc_mru->arcs_lsize[type] >= size &&
24850e8c6158Smaybee 		    mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu;
248644eda4d7Smaybee 	}
2487874395d5Smaybee 	if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) {
2488ad23a2dbSjohansen 		if (type == ARC_BUFC_METADATA) {
2489ad23a2dbSjohansen 			buf->b_data = zio_buf_alloc(size);
24905a98e54bSBrendan Gregg - Sun Microsystems 			arc_space_consume(size, ARC_SPACE_DATA);
2491ad23a2dbSjohansen 		} else {
2492ad23a2dbSjohansen 			ASSERT(type == ARC_BUFC_DATA);
2493ad23a2dbSjohansen 			buf->b_data = zio_data_buf_alloc(size);
24945a98e54bSBrendan Gregg - Sun Microsystems 			ARCSTAT_INCR(arcstat_data_size, size);
24950e8c6158Smaybee 			atomic_add_64(&arc_size, size);
2496ad23a2dbSjohansen 		}
249744cb6abcSbmc 		ARCSTAT_BUMP(arcstat_recycle_miss);
249844eda4d7Smaybee 	}
249944eda4d7Smaybee 	ASSERT(buf->b_data != NULL);
250044eda4d7Smaybee out:
250144eda4d7Smaybee 	/*
250244eda4d7Smaybee 	 * Update the state size.  Note that ghost states have a
250344eda4d7Smaybee 	 * "ghost size" and so don't need to be updated.
250444eda4d7Smaybee 	 */
250544eda4d7Smaybee 	if (!GHOST_STATE(buf->b_hdr->b_state)) {
250644eda4d7Smaybee 		arc_buf_hdr_t *hdr = buf->b_hdr;
250744eda4d7Smaybee 
250844cb6abcSbmc 		atomic_add_64(&hdr->b_state->arcs_size, size);
250944eda4d7Smaybee 		if (list_link_active(&hdr->b_arc_node)) {
251044eda4d7Smaybee 			ASSERT(refcount_is_zero(&hdr->b_refcnt));
25110e8c6158Smaybee 			atomic_add_64(&hdr->b_state->arcs_lsize[type], size);
2512fa9e4066Sahrens 		}
2513641fbdaeSmaybee 		/*
2514641fbdaeSmaybee 		 * If we are growing the cache, and we are adding anonymous
251544cb6abcSbmc 		 * data, and we have outgrown arc_p, update arc_p
2516641fbdaeSmaybee 		 */
251744cb6abcSbmc 		if (arc_size < arc_c && hdr->b_state == arc_anon &&
251844cb6abcSbmc 		    arc_anon->arcs_size + arc_mru->arcs_size > arc_p)
251944cb6abcSbmc 			arc_p = MIN(arc_c, arc_p + size);
2520fa9e4066Sahrens 	}
2521fa9e4066Sahrens }
2522fa9e4066Sahrens 
2523fa9e4066Sahrens /*
2524fa9e4066Sahrens  * This routine is called whenever a buffer is accessed.
2525ea8dc4b6Seschrock  * NOTE: the hash lock is dropped in this function.
2526fa9e4066Sahrens  */
2527fa9e4066Sahrens static void
252844eda4d7Smaybee arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock)
2529fa9e4066Sahrens {
2530d3d50737SRafael Vanoni 	clock_t now;
2531d3d50737SRafael Vanoni 
2532fa9e4066Sahrens 	ASSERT(MUTEX_HELD(hash_lock));
2533fa9e4066Sahrens 
253444cb6abcSbmc 	if (buf->b_state == arc_anon) {
2535fa9e4066Sahrens 		/*
2536fa9e4066Sahrens 		 * This buffer is not in the cache, and does not
2537fa9e4066Sahrens 		 * appear in our "ghost" list.  Add the new buffer
2538fa9e4066Sahrens 		 * to the MRU state.
2539fa9e4066Sahrens 		 */
2540fa9e4066Sahrens 
2541fa9e4066Sahrens 		ASSERT(buf->b_arc_access == 0);
2542d3d50737SRafael Vanoni 		buf->b_arc_access = ddi_get_lbolt();
2543ea8dc4b6Seschrock 		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
254444cb6abcSbmc 		arc_change_state(arc_mru, buf, hash_lock);
2545fa9e4066Sahrens 
254644cb6abcSbmc 	} else if (buf->b_state == arc_mru) {
2547d3d50737SRafael Vanoni 		now = ddi_get_lbolt();
2548d3d50737SRafael Vanoni 
2549fa9e4066Sahrens 		/*
255013506d1eSmaybee 		 * If this buffer is here because of a prefetch, then either:
255113506d1eSmaybee 		 * - clear the flag if this is a "referencing" read
255213506d1eSmaybee 		 *   (any subsequent access will bump this into the MFU state).
255313506d1eSmaybee 		 * or
255413506d1eSmaybee 		 * - move the buffer to the head of the list if this is
255513506d1eSmaybee 		 *   another prefetch (to make it less likely to be evicted).
2556fa9e4066Sahrens 		 */
2557fa9e4066Sahrens 		if ((buf->b_flags & ARC_PREFETCH) != 0) {
255813506d1eSmaybee 			if (refcount_count(&buf->b_refcnt) == 0) {
255913506d1eSmaybee 				ASSERT(list_link_active(&buf->b_arc_node));
256013506d1eSmaybee 			} else {
256113506d1eSmaybee 				buf->b_flags &= ~ARC_PREFETCH;
256244cb6abcSbmc 				ARCSTAT_BUMP(arcstat_mru_hits);
256313506d1eSmaybee 			}
2564d3d50737SRafael Vanoni 			buf->b_arc_access = now;
2565fa9e4066Sahrens 			return;
2566fa9e4066Sahrens 		}
2567fa9e4066Sahrens 
2568fa9e4066Sahrens 		/*
2569fa9e4066Sahrens 		 * This buffer has been "accessed" only once so far,
2570fa9e4066Sahrens 		 * but it is still in the cache. Move it to the MFU
2571fa9e4066Sahrens 		 * state.
2572fa9e4066Sahrens 		 */
2573d3d50737SRafael Vanoni 		if (now > buf->b_arc_access + ARC_MINTIME) {
2574fa9e4066Sahrens 			/*
2575fa9e4066Sahrens 			 * More than 125ms have passed since we
2576fa9e4066Sahrens 			 * instantiated this buffer.  Move it to the
2577fa9e4066Sahrens 			 * most frequently used state.
2578fa9e4066Sahrens 			 */
2579d3d50737SRafael Vanoni 			buf->b_arc_access = now;
2580ea8dc4b6Seschrock 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
258144cb6abcSbmc 			arc_change_state(arc_mfu, buf, hash_lock);
2582fa9e4066Sahrens 		}
258344cb6abcSbmc 		ARCSTAT_BUMP(arcstat_mru_hits);
258444cb6abcSbmc 	} else if (buf->b_state == arc_mru_ghost) {
2585fa9e4066Sahrens 		arc_state_t	*new_state;
2586fa9e4066Sahrens 		/*
2587fa9e4066Sahrens 		 * This buffer has been "accessed" recently, but
2588fa9e4066Sahrens 		 * was evicted from the cache.  Move it to the
2589fa9e4066Sahrens 		 * MFU state.
2590fa9e4066Sahrens 		 */
2591fa9e4066Sahrens 
2592fa9e4066Sahrens 		if (buf->b_flags & ARC_PREFETCH) {
259344cb6abcSbmc 			new_state = arc_mru;
259413506d1eSmaybee 			if (refcount_count(&buf->b_refcnt) > 0)
259513506d1eSmaybee 				buf->b_flags &= ~ARC_PREFETCH;
2596ea8dc4b6Seschrock 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
2597fa9e4066Sahrens 		} else {
259844cb6abcSbmc 			new_state = arc_mfu;
2599ea8dc4b6Seschrock 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2600fa9e4066Sahrens 		}
2601fa9e4066Sahrens 
2602d3d50737SRafael Vanoni 		buf->b_arc_access = ddi_get_lbolt();
2603fa9e4066Sahrens 		arc_change_state(new_state, buf, hash_lock);
2604fa9e4066Sahrens 
260544cb6abcSbmc 		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
260644cb6abcSbmc 	} else if (buf->b_state == arc_mfu) {
2607fa9e4066Sahrens 		/*
2608fa9e4066Sahrens 		 * This buffer has been accessed more than once and is
2609fa9e4066Sahrens 		 * still in the cache.  Keep it in the MFU state.
2610fa9e4066Sahrens 		 *
261113506d1eSmaybee 		 * NOTE: an add_reference() that occurred when we did
261213506d1eSmaybee 		 * the arc_read() will have kicked this off the list.
261313506d1eSmaybee 		 * If it was a prefetch, we will explicitly move it to
261413506d1eSmaybee 		 * the head of the list now.
2615fa9e4066Sahrens 		 */
261613506d1eSmaybee 		if ((buf->b_flags & ARC_PREFETCH) != 0) {
261713506d1eSmaybee 			ASSERT(refcount_count(&buf->b_refcnt) == 0);
261813506d1eSmaybee 			ASSERT(list_link_active(&buf->b_arc_node));
261913506d1eSmaybee 		}
262044cb6abcSbmc 		ARCSTAT_BUMP(arcstat_mfu_hits);
2621d3d50737SRafael Vanoni 		buf->b_arc_access = ddi_get_lbolt();
262244cb6abcSbmc 	} else if (buf->b_state == arc_mfu_ghost) {
262344cb6abcSbmc 		arc_state_t	*new_state = arc_mfu;
2624fa9e4066Sahrens 		/*
2625fa9e4066Sahrens 		 * This buffer has been accessed more than once but has
2626fa9e4066Sahrens 		 * been evicted from the cache.  Move it back to the
2627fa9e4066Sahrens 		 * MFU state.
2628fa9e4066Sahrens 		 */
2629fa9e4066Sahrens 
263013506d1eSmaybee 		if (buf->b_flags & ARC_PREFETCH) {
263113506d1eSmaybee 			/*
263213506d1eSmaybee 			 * This is a prefetch access...
263313506d1eSmaybee 			 * move this block back to the MRU state.
263413506d1eSmaybee 			 */
2635fb09f5aaSMadhav Suresh 			ASSERT0(refcount_count(&buf->b_refcnt));
263644cb6abcSbmc 			new_state = arc_mru;
263713506d1eSmaybee 		}
263813506d1eSmaybee 
2639d3d50737SRafael Vanoni 		buf->b_arc_access = ddi_get_lbolt();
2640ea8dc4b6Seschrock 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
264113506d1eSmaybee 		arc_change_state(new_state, buf, hash_lock);
2642fa9e4066Sahrens 
264344cb6abcSbmc 		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
2644fa94a07fSbrendan 	} else if (buf->b_state == arc_l2c_only) {
2645fa94a07fSbrendan 		/*
2646fa94a07fSbrendan 		 * This buffer is on the 2nd Level ARC.
2647fa94a07fSbrendan 		 */
2648fa94a07fSbrendan 
2649d3d50737SRafael Vanoni 		buf->b_arc_access = ddi_get_lbolt();
2650fa94a07fSbrendan 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2651fa94a07fSbrendan 		arc_change_state(arc_mfu, buf, hash_lock);
2652fa9e4066Sahrens 	} else {
2653fa9e4066Sahrens 		ASSERT(!"invalid arc state");
2654fa9e4066Sahrens 	}
2655fa9e4066Sahrens }
2656fa9e4066Sahrens 
2657fa9e4066Sahrens /* a generic arc_done_func_t which you can use */
2658fa9e4066Sahrens /* ARGSUSED */
2659fa9e4066Sahrens void
2660fa9e4066Sahrens arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
2661fa9e4066Sahrens {
26623f9d6ad7SLin Ling 	if (zio == NULL || zio->io_error == 0)
26633f9d6ad7SLin Ling 		bcopy(buf->b_data, arg, buf->b_hdr->b_size);
26643b2aab18SMatthew Ahrens 	VERIFY(arc_buf_remove_ref(buf, arg));
2665fa9e4066Sahrens }
2666fa9e4066Sahrens 
26670e8c6158Smaybee /* a generic arc_done_func_t */
2668fa9e4066Sahrens void
2669fa9e4066Sahrens arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
2670fa9e4066Sahrens {
2671fa9e4066Sahrens 	arc_buf_t **bufp = arg;
2672fa9e4066Sahrens 	if (zio && zio->io_error) {
26733b2aab18SMatthew Ahrens 		VERIFY(arc_buf_remove_ref(buf, arg));
2674fa9e4066Sahrens 		*bufp = NULL;
2675fa9e4066Sahrens 	} else {
2676fa9e4066Sahrens 		*bufp = buf;
26773f9d6ad7SLin Ling 		ASSERT(buf->b_data);
2678fa9e4066Sahrens 	}
2679fa9e4066Sahrens }
2680fa9e4066Sahrens 
2681fa9e4066Sahrens static void
2682fa9e4066Sahrens arc_read_done(zio_t *zio)
2683fa9e4066Sahrens {
2684bbf4a8dfSmaybee 	arc_buf_hdr_t	*hdr, *found;
2685fa9e4066Sahrens 	arc_buf_t	*buf;
2686fa9e4066Sahrens 	arc_buf_t	*abuf;	/* buffer we're assigning to callback */
2687fa9e4066Sahrens 	kmutex_t	*hash_lock;
2688fa9e4066Sahrens 	arc_callback_t	*callback_list, *acb;
2689fa9e4066Sahrens 	int		freeable = FALSE;
2690fa9e4066Sahrens 
2691fa9e4066Sahrens 	buf = zio->io_private;
2692fa9e4066Sahrens 	hdr = buf->b_hdr;
2693fa9e4066Sahrens 
2694bbf4a8dfSmaybee 	/*
2695bbf4a8dfSmaybee 	 * The hdr was inserted into hash-table and removed from lists
2696bbf4a8dfSmaybee 	 * prior to starting I/O.  We should find this header, since
2697bbf4a8dfSmaybee 	 * it's in the hash table, and it should be legit since it's
2698bbf4a8dfSmaybee 	 * not possible to evict it during the I/O.  The only possible
2699bbf4a8dfSmaybee 	 * reason for it not to be found is if we were freed during the
2700bbf4a8dfSmaybee 	 * read.
2701bbf4a8dfSmaybee 	 */
2702ac05c741SMark Maybee 	found = buf_hash_find(hdr->b_spa, &hdr->b_dva, hdr->b_birth,
27036b4acc8bSahrens 	    &hash_lock);
2704fa9e4066Sahrens 
2705bbf4a8dfSmaybee 	ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) ||
2706fa94a07fSbrendan 	    (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
2707fa94a07fSbrendan 	    (found == hdr && HDR_L2_READING(hdr)));
2708fa94a07fSbrendan 
27093a737e0dSbrendan 	hdr->b_flags &= ~ARC_L2_EVICTED;
2710fa94a07fSbrendan 	if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))
27113baa08fcSek 		hdr->b_flags &= ~ARC_L2CACHE;
2712fa9e4066Sahrens 
2713fa9e4066Sahrens 	/* byteswap if necessary */
2714fa9e4066Sahrens 	callback_list = hdr->b_acb;
2715fa9e4066Sahrens 	ASSERT(callback_list != NULL);
27168e0f0d3dSWilliam Gorrell 	if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
2717ad135b5dSChristopher Siden 		dmu_object_byteswap_t bswap =
2718ad135b5dSChristopher Siden 		    DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
2719088f3894Sahrens 		arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ?
2720088f3894Sahrens 		    byteswap_uint64_array :
2721ad135b5dSChristopher Siden 		    dmu_ot_byteswap[bswap].ob_func;
2722088f3894Sahrens 		func(buf->b_data, hdr->b_size);
2723088f3894Sahrens 	}
2724fa9e4066Sahrens 
2725fa94a07fSbrendan 	arc_cksum_compute(buf, B_FALSE);
2726cd1c8b85SMatthew Ahrens 	arc_buf_watch(buf);
27276b4acc8bSahrens 
2728b24ab676SJeff Bonwick 	if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) {
2729b24ab676SJeff Bonwick 		/*
2730b24ab676SJeff Bonwick 		 * Only call arc_access on anonymous buffers.  This is because
2731b24ab676SJeff Bonwick 		 * if we've issued an I/O for an evicted buffer, we've already
2732b24ab676SJeff Bonwick 		 * called arc_access (to prevent any simultaneous readers from
2733b24ab676SJeff Bonwick 		 * getting confused).
2734b24ab676SJeff Bonwick 		 */
2735b24ab676SJeff Bonwick 		arc_access(hdr, hash_lock);
2736b24ab676SJeff Bonwick 	}
2737b24ab676SJeff Bonwick 
2738fa9e4066Sahrens 	/* create copies of the data buffer for the callers */
2739fa9e4066Sahrens 	abuf = buf;
2740fa9e4066Sahrens 	for (acb = callback_list; acb; acb = acb->acb_next) {
2741fa9e4066Sahrens 		if (acb->acb_done) {
27429253d63dSGeorge Wilson 			if (abuf == NULL) {
27439253d63dSGeorge Wilson 				ARCSTAT_BUMP(arcstat_duplicate_reads);
274444eda4d7Smaybee 				abuf = arc_buf_clone(buf);
27459253d63dSGeorge Wilson 			}
2746fa9e4066Sahrens 			acb->acb_buf = abuf;
2747fa9e4066Sahrens 			abuf = NULL;
2748fa9e4066Sahrens 		}
2749fa9e4066Sahrens 	}
2750fa9e4066Sahrens 	hdr->b_acb = NULL;
2751fa9e4066Sahrens 	hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
2752ea8dc4b6Seschrock 	ASSERT(!HDR_BUF_AVAILABLE(hdr));
2753b24ab676SJeff Bonwick 	if (abuf == buf) {
2754b24ab676SJeff Bonwick 		ASSERT(buf->b_efunc == NULL);
2755b24ab676SJeff Bonwick 		ASSERT(hdr->b_datacnt == 1);
2756ea8dc4b6Seschrock 		hdr->b_flags |= ARC_BUF_AVAILABLE;
2757b24ab676SJeff Bonwick 	}
2758fa9e4066Sahrens 
2759fa9e4066Sahrens 	ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL);
2760fa9e4066Sahrens 
2761fa9e4066Sahrens 	if (zio->io_error != 0) {
2762fa9e4066Sahrens 		hdr->b_flags |= ARC_IO_ERROR;
276344cb6abcSbmc 		if (hdr->b_state != arc_anon)
276444cb6abcSbmc 			arc_change_state(arc_anon, hdr, hash_lock);
2765ea8dc4b6Seschrock 		if (HDR_IN_HASH_TABLE(hdr))
2766ea8dc4b6Seschrock 			buf_hash_remove(hdr);
2767fa9e4066Sahrens 		freeable = refcount_is_zero(&hdr->b_refcnt);
2768fa9e4066Sahrens 	}
2769fa9e4066Sahrens 
2770ea8dc4b6Seschrock 	/*
277113506d1eSmaybee 	 * Broadcast before we drop the hash_lock to avoid the possibility
277213506d1eSmaybee 	 * that the hdr (and hence the cv) might be freed before we get to
277313506d1eSmaybee 	 * the cv_broadcast().
2774ea8dc4b6Seschrock 	 */
2775ea8dc4b6Seschrock 	cv_broadcast(&hdr->b_cv);
2776ea8dc4b6Seschrock 
2777bbf4a8dfSmaybee 	if (hash_lock) {
277844eda4d7Smaybee 		mutex_exit(hash_lock);
2779fa9e4066Sahrens 	} else {
2780fa9e4066Sahrens 		/*
2781fa9e4066Sahrens 		 * This block was freed while we waited for the read to
2782fa9e4066Sahrens 		 * complete.  It has been removed from the hash table and
2783fa9e4066Sahrens 		 * moved to the anonymous state (so that it won't show up
2784fa9e4066Sahrens 		 * in the cache).
2785fa9e4066Sahrens 		 */
278644cb6abcSbmc 		ASSERT3P(hdr->b_state, ==, arc_anon);
2787fa9e4066Sahrens 		freeable = refcount_is_zero(&hdr->b_refcnt);
2788fa9e4066Sahrens 	}
2789fa9e4066Sahrens 
2790fa9e4066Sahrens 	/* execute each callback and free its structure */
2791fa9e4066Sahrens 	while ((acb = callback_list) != NULL) {
2792fa9e4066Sahrens 		if (acb->acb_done)
2793fa9e4066Sahrens 			acb->acb_done(zio, acb->acb_buf, acb->acb_private);
2794fa9e4066Sahrens 
2795fa9e4066Sahrens 		if (acb->acb_zio_dummy != NULL) {
2796fa9e4066Sahrens 			acb->acb_zio_dummy->io_error = zio->io_error;
2797fa9e4066Sahrens 			zio_nowait(acb->acb_zio_dummy);
2798fa9e4066Sahrens 		}
2799fa9e4066Sahrens 
2800fa9e4066Sahrens 		callback_list = acb->acb_next;
2801fa9e4066Sahrens 		kmem_free(acb, sizeof (arc_callback_t));
2802fa9e4066Sahrens 	}
2803fa9e4066Sahrens 
2804fa9e4066Sahrens 	if (freeable)
2805ea8dc4b6Seschrock 		arc_hdr_destroy(hdr);
2806fa9e4066Sahrens }
2807fa9e4066Sahrens 
2808fa9e4066Sahrens /*
2809fc98fea5SBart Coddens  * "Read" the block at the specified DVA (in bp) via the
2810fa9e4066Sahrens  * cache.  If the block is found in the cache, invoke the provided
2811fa9e4066Sahrens  * callback immediately and return.  Note that the `zio' parameter
2812fa9e4066Sahrens  * in the callback will be NULL in this case, since no IO was
2813fa9e4066Sahrens  * required.  If the block is not in the cache pass the read request
2814fa9e4066Sahrens  * on to the spa with a substitute callback function, so that the
2815fa9e4066Sahrens  * requested block will be added to the cache.
2816fa9e4066Sahrens  *
2817fa9e4066Sahrens  * If a read request arrives for a block that has a read in-progress,
2818fa9e4066Sahrens  * either wait for the in-progress read to complete (and return the
2819fa9e4066Sahrens  * results); or, if this is a read with a "done" func, add a record
2820fa9e4066Sahrens  * to the read to invoke the "done" func when the read completes,
2821fa9e4066Sahrens  * and return; or just return.
2822fa9e4066Sahrens  *
2823fa9e4066Sahrens  * arc_read_done() will invoke all the requested "done" functions
2824fa9e4066Sahrens  * for readers of this block.
2825fa9e4066Sahrens  */
2826fa9e4066Sahrens int
28271b912ec7SGeorge Wilson arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done,
28281b912ec7SGeorge Wilson     void *private, int priority, int zio_flags, uint32_t *arc_flags,
28291b912ec7SGeorge Wilson     const zbookmark_t *zb)
2830fa9e4066Sahrens {
2831fa9e4066Sahrens 	arc_buf_hdr_t *hdr;
2832d5285caeSGeorge Wilson 	arc_buf_t *buf = NULL;
2833fa9e4066Sahrens 	kmutex_t *hash_lock;
2834fa94a07fSbrendan 	zio_t *rzio;
2835e9103aaeSGarrett D'Amore 	uint64_t guid = spa_load_guid(spa);
2836fa9e4066Sahrens 
2837fa9e4066Sahrens top:
2838b24ab676SJeff Bonwick 	hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp),
2839b24ab676SJeff Bonwick 	    &hash_lock);
2840ea8dc4b6Seschrock 	if (hdr && hdr->b_datacnt > 0) {
2841fa9e4066Sahrens 
284213506d1eSmaybee 		*arc_flags |= ARC_CACHED;
284313506d1eSmaybee 
2844fa9e4066Sahrens 		if (HDR_IO_IN_PROGRESS(hdr)) {
284513506d1eSmaybee 
284613506d1eSmaybee 			if (*arc_flags & ARC_WAIT) {
284713506d1eSmaybee 				cv_wait(&hdr->b_cv, hash_lock);
284813506d1eSmaybee 				mutex_exit(hash_lock);
284913506d1eSmaybee 				goto top;
285013506d1eSmaybee 			}
285113506d1eSmaybee 			ASSERT(*arc_flags & ARC_NOWAIT);
285213506d1eSmaybee 
285313506d1eSmaybee 			if (done) {
2854fa9e4066Sahrens 				arc_callback_t	*acb = NULL;
2855fa9e4066Sahrens 
2856fa9e4066Sahrens 				acb = kmem_zalloc(sizeof (arc_callback_t),
2857fa9e4066Sahrens 				    KM_SLEEP);
2858fa9e4066Sahrens 				acb->acb_done = done;
2859fa9e4066Sahrens 				acb->acb_private = private;
2860fa9e4066Sahrens 				if (pio != NULL)
2861fa9e4066Sahrens 					acb->acb_zio_dummy = zio_null(pio,
2862a3f829aeSBill Moore 					    spa, NULL, NULL, NULL, zio_flags);
2863fa9e4066Sahrens 
2864fa9e4066Sahrens 				ASSERT(acb->acb_done != NULL);
2865fa9e4066Sahrens 				acb->acb_next = hdr->b_acb;
2866fa9e4066Sahrens 				hdr->b_acb = acb;
2867fa9e4066Sahrens 				add_reference(hdr, hash_lock, private);
2868fa9e4066Sahrens 				mutex_exit(hash_lock);
2869fa9e4066Sahrens 				return (0);
2870fa9e4066Sahrens 			}
2871fa9e4066Sahrens 			mutex_exit(hash_lock);
2872fa9e4066Sahrens 			return (0);
2873fa9e4066Sahrens 		}
2874fa9e4066Sahrens 
287544cb6abcSbmc 		ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
2876fa9e4066Sahrens 
2877ea8dc4b6Seschrock 		if (done) {
287844eda4d7Smaybee 			add_reference(hdr, hash_lock, private);
2879ea8dc4b6Seschrock 			/*
2880ea8dc4b6Seschrock 			 * If this block is already in use, create a new
2881ea8dc4b6Seschrock 			 * copy of the data so that we will be guaranteed
2882ea8dc4b6Seschrock 			 * that arc_release() will always succeed.
2883ea8dc4b6Seschrock 			 */
2884fa9e4066Sahrens 			buf = hdr->b_buf;
2885ea8dc4b6Seschrock 			ASSERT(buf);
2886ea8dc4b6Seschrock 			ASSERT(buf->b_data);
288744eda4d7Smaybee 			if (HDR_BUF_AVAILABLE(hdr)) {
2888ea8dc4b6Seschrock 				ASSERT(buf->b_efunc == NULL);
2889ea8dc4b6Seschrock 				hdr->b_flags &= ~ARC_BUF_AVAILABLE;
289044eda4d7Smaybee 			} else {
289144eda4d7Smaybee 				buf = arc_buf_clone(buf);
2892ea8dc4b6Seschrock 			}
2893b24ab676SJeff Bonwick 
289413506d1eSmaybee 		} else if (*arc_flags & ARC_PREFETCH &&
289513506d1eSmaybee 		    refcount_count(&hdr->b_refcnt) == 0) {
289613506d1eSmaybee 			hdr->b_flags |= ARC_PREFETCH;
2897fa9e4066Sahrens 		}
2898fa9e4066Sahrens 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
289944eda4d7Smaybee 		arc_access(hdr, hash_lock);
29003baa08fcSek 		if (*arc_flags & ARC_L2CACHE)
29013baa08fcSek 			hdr->b_flags |= ARC_L2CACHE;
2902aad02571SSaso Kiselkov 		if (*arc_flags & ARC_L2COMPRESS)
2903aad02571SSaso Kiselkov 			hdr->b_flags |= ARC_L2COMPRESS;
290444eda4d7Smaybee 		mutex_exit(hash_lock);
290544cb6abcSbmc 		ARCSTAT_BUMP(arcstat_hits);
290644cb6abcSbmc 		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
290744cb6abcSbmc 		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
290844cb6abcSbmc 		    data, metadata, hits);
290944cb6abcSbmc 
2910fa9e4066Sahrens 		if (done)
2911fa9e4066Sahrens 			done(NULL, buf, private);
2912fa9e4066Sahrens 	} else {
2913fa9e4066Sahrens 		uint64_t size = BP_GET_LSIZE(bp);
2914fa9e4066Sahrens 		arc_callback_t	*acb;
29153a737e0dSbrendan 		vdev_t *vd = NULL;
2916d5285caeSGeorge Wilson 		uint64_t addr = 0;
29175a98e54bSBrendan Gregg - Sun Microsystems 		boolean_t devw = B_FALSE;
2918fa9e4066Sahrens 
2919fa9e4066Sahrens 		if (hdr == NULL) {
2920fa9e4066Sahrens 			/* this block is not in the cache */
2921fa9e4066Sahrens 			arc_buf_hdr_t	*exists;
2922ad23a2dbSjohansen 			arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
2923ad23a2dbSjohansen 			buf = arc_buf_alloc(spa, size, private, type);
2924fa9e4066Sahrens 			hdr = buf->b_hdr;
2925fa9e4066Sahrens 			hdr->b_dva = *BP_IDENTITY(bp);
2926b24ab676SJeff Bonwick 			hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
2927fa9e4066Sahrens 			hdr->b_cksum0 = bp->blk_cksum.zc_word[0];
2928fa9e4066Sahrens 			exists = buf_hash_insert(hdr, &hash_lock);
2929fa9e4066Sahrens 			if (exists) {
2930fa9e4066Sahrens 				/* somebody beat us to the hash insert */
2931fa9e4066Sahrens 				mutex_exit(hash_lock);
29323f9d6ad7SLin Ling 				buf_discard_identity(hdr);
2933ea8dc4b6Seschrock 				(void) arc_buf_remove_ref(buf, private);
2934fa9e4066Sahrens 				goto top; /* restart the IO request */
2935fa9e4066Sahrens 			}
293613506d1eSmaybee 			/* if this is a prefetch, we don't have a reference */
293713506d1eSmaybee 			if (*arc_flags & ARC_PREFETCH) {
293813506d1eSmaybee 				(void) remove_reference(hdr, hash_lock,
293913506d1eSmaybee 				    private);
294013506d1eSmaybee 				hdr->b_flags |= ARC_PREFETCH;
294113506d1eSmaybee 			}
29423baa08fcSek 			if (*arc_flags & ARC_L2CACHE)
29433baa08fcSek 				hdr->b_flags |= ARC_L2CACHE;
2944aad02571SSaso Kiselkov 			if (*arc_flags & ARC_L2COMPRESS)
2945aad02571SSaso Kiselkov 				hdr->b_flags |= ARC_L2COMPRESS;
294613506d1eSmaybee 			if (BP_GET_LEVEL(bp) > 0)
294713506d1eSmaybee 				hdr->b_flags |= ARC_INDIRECT;
2948fa9e4066Sahrens 		} else {
2949fa9e4066Sahrens 			/* this block is in the ghost cache */
2950ea8dc4b6Seschrock 			ASSERT(GHOST_STATE(hdr->b_state));
2951ea8dc4b6Seschrock 			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2952fb09f5aaSMadhav Suresh 			ASSERT0(refcount_count(&hdr->b_refcnt));
2953ea8dc4b6Seschrock 			ASSERT(hdr->b_buf == NULL);
295413506d1eSmaybee 
295513506d1eSmaybee 			/* if this is a prefetch, we don't have a reference */
295613506d1eSmaybee 			if (*arc_flags & ARC_PREFETCH)
295713506d1eSmaybee 				hdr->b_flags |= ARC_PREFETCH;
295813506d1eSmaybee 			else
295913506d1eSmaybee 				add_reference(hdr, hash_lock, private);
29603baa08fcSek 			if (*arc_flags & ARC_L2CACHE)
29613baa08fcSek 				hdr->b_flags |= ARC_L2CACHE;
2962aad02571SSaso Kiselkov 			if (*arc_flags & ARC_L2COMPRESS)
2963aad02571SSaso Kiselkov 				hdr->b_flags |= ARC_L2COMPRESS;
29641ab7f2deSmaybee 			buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
2965fa9e4066Sahrens 			buf->b_hdr = hdr;
296644eda4d7Smaybee 			buf->b_data = NULL;
2967ea8dc4b6Seschrock 			buf->b_efunc = NULL;
2968ea8dc4b6Seschrock 			buf->b_private = NULL;
2969fa9e4066Sahrens 			buf->b_next = NULL;
2970fa9e4066Sahrens 			hdr->b_buf = buf;
2971ea8dc4b6Seschrock 			ASSERT(hdr->b_datacnt == 0);
2972ea8dc4b6Seschrock 			hdr->b_datacnt = 1;
29735614b00aSWilliam Gorrell 			arc_get_data_buf(buf);
29747e453561SWilliam Gorrell 			arc_access(hdr, hash_lock);
2975fa9e4066Sahrens 		}
2976fa9e4066Sahrens 
29775614b00aSWilliam Gorrell 		ASSERT(!GHOST_STATE(hdr->b_state));
29785614b00aSWilliam Gorrell 
2979fa9e4066Sahrens 		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
2980fa9e4066Sahrens 		acb->acb_done = done;
2981fa9e4066Sahrens 		acb->acb_private = private;
2982fa9e4066Sahrens 
2983fa9e4066Sahrens 		ASSERT(hdr->b_acb == NULL);
2984fa9e4066Sahrens 		hdr->b_acb = acb;
2985fa9e4066Sahrens 		hdr->b_flags |= ARC_IO_IN_PROGRESS;
2986fa9e4066Sahrens 
2987e14bb325SJeff Bonwick 		if (HDR_L2CACHE(hdr) && hdr->b_l2hdr != NULL &&
2988e14bb325SJeff Bonwick 		    (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) {
29895a98e54bSBrendan Gregg - Sun Microsystems 			devw = hdr->b_l2hdr->b_dev->l2ad_writing;
29903a737e0dSbrendan 			addr = hdr->b_l2hdr->b_daddr;
2991e14bb325SJeff Bonwick 			/*
2992e14bb325SJeff Bonwick 			 * Lock out device removal.
2993e14bb325SJeff Bonwick 			 */
2994e14bb325SJeff Bonwick 			if (vdev_is_dead(vd) ||
2995e14bb325SJeff Bonwick 			    !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
2996e14bb325SJeff Bonwick 				vd = NULL;
29973a737e0dSbrendan 		}
29983a737e0dSbrendan 
29993a737e0dSbrendan 		mutex_exit(hash_lock);
30003a737e0dSbrendan 
30013e30c24aSWill Andrews 		/*
30023e30c24aSWill Andrews 		 * At this point, we have a level 1 cache miss.  Try again in
30033e30c24aSWill Andrews 		 * L2ARC if possible.
30043e30c24aSWill Andrews 		 */
3005fa9e4066Sahrens 		ASSERT3U(hdr->b_size, ==, size);
30065c28183bSBrendan Gregg - Sun Microsystems 		DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
30075c28183bSBrendan Gregg - Sun Microsystems 		    uint64_t, size, zbookmark_t *, zb);
300844cb6abcSbmc 		ARCSTAT_BUMP(arcstat_misses);
300944cb6abcSbmc 		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
301044cb6abcSbmc 		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
301144cb6abcSbmc 		    data, metadata, misses);
3012ea8dc4b6Seschrock 
30135a98e54bSBrendan Gregg - Sun Microsystems 		if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
3014fa94a07fSbrendan 			/*
3015fa94a07fSbrendan 			 * Read from the L2ARC if the following are true:
30163a737e0dSbrendan 			 * 1. The L2ARC vdev was previously cached.
30173a737e0dSbrendan 			 * 2. This buffer still has L2ARC metadata.
30183a737e0dSbrendan 			 * 3. This buffer isn't currently writing to the L2ARC.
30193a737e0dSbrendan 			 * 4. The L2ARC entry wasn't evicted, which may
30203a737e0dSbrendan 			 *    also have invalidated the vdev.
30215a98e54bSBrendan Gregg - Sun Microsystems 			 * 5. This isn't prefetch and l2arc_noprefetch is set.
3022fa94a07fSbrendan 			 */
3023e14bb325SJeff Bonwick 			if (hdr->b_l2hdr != NULL &&
30245a98e54bSBrendan Gregg - Sun Microsystems 			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
30255a98e54bSBrendan Gregg - Sun Microsystems 			    !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
3026fa94a07fSbrendan 				l2arc_read_callback_t *cb;
3027fa94a07fSbrendan 
3028c5904d13Seschrock 				DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
3029c5904d13Seschrock 				ARCSTAT_BUMP(arcstat_l2_hits);
3030c5904d13Seschrock 
3031fa94a07fSbrendan 				cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
3032fa94a07fSbrendan 				    KM_SLEEP);
3033fa94a07fSbrendan 				cb->l2rcb_buf = buf;
3034fa94a07fSbrendan 				cb->l2rcb_spa = spa;
3035fa94a07fSbrendan 				cb->l2rcb_bp = *bp;
3036fa94a07fSbrendan 				cb->l2rcb_zb = *zb;
30373baa08fcSek 				cb->l2rcb_flags = zio_flags;
3038aad02571SSaso Kiselkov 				cb->l2rcb_compress = hdr->b_l2hdr->b_compress;
3039fa94a07fSbrendan 
3040d5285caeSGeorge Wilson 				ASSERT(addr >= VDEV_LABEL_START_SIZE &&
3041d5285caeSGeorge Wilson 				    addr + size < vd->vdev_psize -
3042d5285caeSGeorge Wilson 				    VDEV_LABEL_END_SIZE);
3043d5285caeSGeorge Wilson 
3044fa94a07fSbrendan 				/*
3045e14bb325SJeff Bonwick 				 * l2arc read.  The SCL_L2ARC lock will be
3046e14bb325SJeff Bonwick 				 * released by l2arc_read_done().
3047aad02571SSaso Kiselkov 				 * Issue a null zio if the underlying buffer
3048aad02571SSaso Kiselkov 				 * was squashed to zero size by compression.
3049fa94a07fSbrendan 				 */
3050aad02571SSaso Kiselkov 				if (hdr->b_l2hdr->b_compress ==
3051aad02571SSaso Kiselkov 				    ZIO_COMPRESS_EMPTY) {
3052aad02571SSaso Kiselkov 					rzio = zio_null(pio, spa, vd,
3053aad02571SSaso Kiselkov 					    l2arc_read_done, cb,
3054aad02571SSaso Kiselkov 					    zio_flags | ZIO_FLAG_DONT_CACHE |
3055aad02571SSaso Kiselkov 					    ZIO_FLAG_CANFAIL |
3056aad02571SSaso Kiselkov 					    ZIO_FLAG_DONT_PROPAGATE |
3057aad02571SSaso Kiselkov 					    ZIO_FLAG_DONT_RETRY);
3058aad02571SSaso Kiselkov 				} else {
3059aad02571SSaso Kiselkov 					rzio = zio_read_phys(pio, vd, addr,
3060aad02571SSaso Kiselkov 					    hdr->b_l2hdr->b_asize,
3061aad02571SSaso Kiselkov 					    buf->b_data, ZIO_CHECKSUM_OFF,
3062aad02571SSaso Kiselkov 					    l2arc_read_done, cb, priority,
3063aad02571SSaso Kiselkov 					    zio_flags | ZIO_FLAG_DONT_CACHE |
3064aad02571SSaso Kiselkov 					    ZIO_FLAG_CANFAIL |
3065aad02571SSaso Kiselkov 					    ZIO_FLAG_DONT_PROPAGATE |
3066aad02571SSaso Kiselkov 					    ZIO_FLAG_DONT_RETRY, B_FALSE);
3067aad02571SSaso Kiselkov 				}
3068fa94a07fSbrendan 				DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
3069fa94a07fSbrendan 				    zio_t *, rzio);
3070aad02571SSaso Kiselkov 				ARCSTAT_INCR(arcstat_l2_read_bytes,
3071aad02571SSaso Kiselkov 				    hdr->b_l2hdr->b_asize);
3072fa94a07fSbrendan 
30733a737e0dSbrendan 				if (*arc_flags & ARC_NOWAIT) {
30743a737e0dSbrendan 					zio_nowait(rzio);
30753a737e0dSbrendan 					return (0);
30763a737e0dSbrendan 				}
3077fa94a07fSbrendan 
30783a737e0dSbrendan 				ASSERT(*arc_flags & ARC_WAIT);
30793a737e0dSbrendan 				if (zio_wait(rzio) == 0)
30803a737e0dSbrendan 					return (0);
30813a737e0dSbrendan 
30823a737e0dSbrendan 				/* l2arc read error; goto zio_read() */
3083fa94a07fSbrendan 			} else {
3084fa94a07fSbrendan 				DTRACE_PROBE1(l2arc__miss,
3085fa94a07fSbrendan 				    arc_buf_hdr_t *, hdr);
3086fa94a07fSbrendan 				ARCSTAT_BUMP(arcstat_l2_misses);
3087fa94a07fSbrendan 				if (HDR_L2_WRITING(hdr))
3088fa94a07fSbrendan 					ARCSTAT_BUMP(arcstat_l2_rw_clash);
3089e14bb325SJeff Bonwick 				spa_config_exit(spa, SCL_L2ARC, vd);
3090fa94a07fSbrendan 			}
30915a98e54bSBrendan Gregg - Sun Microsystems 		} else {
309276a25fafSBill Moore 			if (vd != NULL)
309376a25fafSBill Moore 				spa_config_exit(spa, SCL_L2ARC, vd);
30945a98e54bSBrendan Gregg - Sun Microsystems 			if (l2arc_ndev != 0) {
30955a98e54bSBrendan Gregg - Sun Microsystems 				DTRACE_PROBE1(l2arc__miss,
30965a98e54bSBrendan Gregg - Sun Microsystems 				    arc_buf_hdr_t *, hdr);
30975a98e54bSBrendan Gregg - Sun Microsystems 				ARCSTAT_BUMP(arcstat_l2_misses);
30985a98e54bSBrendan Gregg - Sun Microsystems 			}
3099fa94a07fSbrendan 		}
3100c5904d13Seschrock 
3101fa9e4066Sahrens 		rzio = zio_read(pio, spa, bp, buf->b_data, size,
31023baa08fcSek 		    arc_read_done, buf, priority, zio_flags, zb);
3103fa9e4066Sahrens 
310413506d1eSmaybee 		if (*arc_flags & ARC_WAIT)
3105fa9e4066Sahrens 			return (zio_wait(rzio));
3106fa9e4066Sahrens 
310713506d1eSmaybee 		ASSERT(*arc_flags & ARC_NOWAIT);
3108fa9e4066Sahrens 		zio_nowait(rzio);
3109fa9e4066Sahrens 	}
3110fa9e4066Sahrens 	return (0);
3111fa9e4066Sahrens }
3112fa9e4066Sahrens 
3113ea8dc4b6Seschrock void
3114ea8dc4b6Seschrock arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private)
3115ea8dc4b6Seschrock {
3116ea8dc4b6Seschrock 	ASSERT(buf->b_hdr != NULL);
311744cb6abcSbmc 	ASSERT(buf->b_hdr->b_state != arc_anon);
3118ea8dc4b6Seschrock 	ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL);
3119b24ab676SJeff Bonwick 	ASSERT(buf->b_efunc == NULL);
3120b24ab676SJeff Bonwick 	ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr));
3121b24ab676SJeff Bonwick 
3122ea8dc4b6Seschrock 	buf->b_efunc = func;
3123ea8dc4b6Seschrock 	buf->b_private = private;
3124ea8dc4b6Seschrock }
3125ea8dc4b6Seschrock 
31266e6d5868SMatthew Ahrens /*
31276e6d5868SMatthew Ahrens  * Notify the arc that a block was freed, and thus will never be used again.
31286e6d5868SMatthew Ahrens  */
31296e6d5868SMatthew Ahrens void
31306e6d5868SMatthew Ahrens arc_freed(spa_t *spa, const blkptr_t *bp)
31316e6d5868SMatthew Ahrens {
31326e6d5868SMatthew Ahrens 	arc_buf_hdr_t *hdr;
31336e6d5868SMatthew Ahrens 	kmutex_t *hash_lock;
31346e6d5868SMatthew Ahrens 	uint64_t guid = spa_load_guid(spa);
31356e6d5868SMatthew Ahrens 
31366e6d5868SMatthew Ahrens 	hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp),
31376e6d5868SMatthew Ahrens 	    &hash_lock);
31386e6d5868SMatthew Ahrens 	if (hdr == NULL)
31396e6d5868SMatthew Ahrens 		return;
31406e6d5868SMatthew Ahrens 	if (HDR_BUF_AVAILABLE(hdr)) {
31416e6d5868SMatthew Ahrens 		arc_buf_t *buf = hdr->b_buf;
31426e6d5868SMatthew Ahrens 		add_reference(hdr, hash_lock, FTAG);
31436e6d5868SMatthew Ahrens 		hdr->b_flags &= ~ARC_BUF_AVAILABLE;
31446e6d5868SMatthew Ahrens 		mutex_exit(hash_lock);
31456e6d5868SMatthew Ahrens 
31466e6d5868SMatthew Ahrens 		arc_release(buf, FTAG);
31476e6d5868SMatthew Ahrens 		(void) arc_buf_remove_ref(buf, FTAG);
31486e6d5868SMatthew Ahrens 	} else {
31496e6d5868SMatthew Ahrens 		mutex_exit(hash_lock);
31506e6d5868SMatthew Ahrens 	}
31516e6d5868SMatthew Ahrens 
31526e6d5868SMatthew Ahrens }
31536e6d5868SMatthew Ahrens 
3154ea8dc4b6Seschrock /*
3155ea8dc4b6Seschrock  * This is used by the DMU to let the ARC know that a buffer is
3156ea8dc4b6Seschrock  * being evicted, so the ARC should clean up.  If this arc buf
3157ea8dc4b6Seschrock  * is not yet in the evicted state, it will be put there.
3158ea8dc4b6Seschrock  */
3159ea8dc4b6Seschrock int
3160ea8dc4b6Seschrock arc_buf_evict(arc_buf_t *buf)
3161ea8dc4b6Seschrock {
316240d7d650Smaybee 	arc_buf_hdr_t *hdr;
3163ea8dc4b6Seschrock 	kmutex_t *hash_lock;
3164ea8dc4b6Seschrock 	arc_buf_t **bufp;
3165ea8dc4b6Seschrock 
31663f9d6ad7SLin Ling 	mutex_enter(&buf->b_evict_lock);
316740d7d650Smaybee 	hdr = buf->b_hdr;
3168ea8dc4b6Seschrock 	if (hdr == NULL) {
3169ea8dc4b6Seschrock 		/*
3170ea8dc4b6Seschrock 		 * We are in arc_do_user_evicts().
3171ea8dc4b6Seschrock 		 */
3172ea8dc4b6Seschrock 		ASSERT(buf->b_data == NULL);
31733f9d6ad7SLin Ling 		mutex_exit(&buf->b_evict_lock);
3174ea8dc4b6Seschrock 		return (0);
31756f83844dSMark Maybee 	} else if (buf->b_data == NULL) {
31766f83844dSMark Maybee 		arc_buf_t copy = *buf; /* structure assignment */
31779b23f181Smaybee 		/*
31786f83844dSMark Maybee 		 * We are on the eviction list; process this buffer now
31796f83844dSMark Maybee 		 * but let arc_do_user_evicts() do the reaping.
31809b23f181Smaybee 		 */
31816f83844dSMark Maybee 		buf->b_efunc = NULL;
31823f9d6ad7SLin Ling 		mutex_exit(&buf->b_evict_lock);
31836f83844dSMark Maybee 		VERIFY(copy.b_efunc(&copy) == 0);
31846f83844dSMark Maybee 		return (1);
31859b23f181Smaybee 	}
31866f83844dSMark Maybee 	hash_lock = HDR_LOCK(hdr);
31876f83844dSMark Maybee 	mutex_enter(hash_lock);
31883f9d6ad7SLin Ling 	hdr = buf->b_hdr;
31893f9d6ad7SLin Ling 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
31909b23f181Smaybee 
31919b23f181Smaybee 	ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt);
319244cb6abcSbmc 	ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
3193ea8dc4b6Seschrock 
3194ea8dc4b6Seschrock 	/*
3195ea8dc4b6Seschrock 	 * Pull this buffer off of the hdr
3196ea8dc4b6Seschrock 	 */
3197ea8dc4b6Seschrock 	bufp = &hdr->b_buf;
3198ea8dc4b6Seschrock 	while (*bufp != buf)
3199ea8dc4b6Seschrock 		bufp = &(*bufp)->b_next;
3200ea8dc4b6Seschrock 	*bufp = buf->b_next;
3201ea8dc4b6Seschrock 
3202ea8dc4b6Seschrock 	ASSERT(buf->b_data != NULL);
320344eda4d7Smaybee 	arc_buf_destroy(buf, FALSE, FALSE);
3204ea8dc4b6Seschrock 
3205ea8dc4b6Seschrock 	if (hdr->b_datacnt == 0) {
3206ea8dc4b6Seschrock 		arc_state_t *old_state = hdr->b_state;
3207ea8dc4b6Seschrock 		arc_state_t *evicted_state;
3208ea8dc4b6Seschrock 
32093f9d6ad7SLin Ling 		ASSERT(hdr->b_buf == NULL);
3210ea8dc4b6Seschrock 		ASSERT(refcount_is_zero(&hdr->b_refcnt));
3211ea8dc4b6Seschrock 
3212ea8dc4b6Seschrock 		evicted_state =
321344cb6abcSbmc 		    (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
3214ea8dc4b6Seschrock 
321544cb6abcSbmc 		mutex_enter(&old_state->arcs_mtx);
321644cb6abcSbmc 		mutex_enter(&evicted_state->arcs_mtx);
3217ea8dc4b6Seschrock 
3218ea8dc4b6Seschrock 		arc_change_state(evicted_state, hdr, hash_lock);
3219ea8dc4b6Seschrock 		ASSERT(HDR_IN_HASH_TABLE(hdr));
3220fa94a07fSbrendan 		hdr->b_flags |= ARC_IN_HASH_TABLE;
3221fa94a07fSbrendan 		hdr->b_flags &= ~ARC_BUF_AVAILABLE;
3222ea8dc4b6Seschrock 
322344cb6abcSbmc 		mutex_exit(&evicted_state->arcs_mtx);
322444cb6abcSbmc 		mutex_exit(&old_state->arcs_mtx);
3225ea8dc4b6Seschrock 	}
3226ea8dc4b6Seschrock 	mutex_exit(hash_lock);
32273f9d6ad7SLin Ling 	mutex_exit(&buf->b_evict_lock);
3228dd6ef538Smaybee 
3229ea8dc4b6Seschrock 	VERIFY(buf->b_efunc(buf) == 0);
3230ea8dc4b6Seschrock 	buf->b_efunc = NULL;
3231ea8dc4b6Seschrock 	buf->b_private = NULL;
3232ea8dc4b6Seschrock 	buf->b_hdr = NULL;
32333f9d6ad7SLin Ling 	buf->b_next = NULL;
3234ea8dc4b6Seschrock 	kmem_cache_free(buf_cache, buf);
3235ea8dc4b6Seschrock 	return (1);
3236ea8dc4b6Seschrock }
3237ea8dc4b6Seschrock 
3238fa9e4066Sahrens /*
32393e30c24aSWill Andrews  * Release this buffer from the cache, making it an anonymous buffer.  This
32403e30c24aSWill Andrews  * must be done after a read and prior to modifying the buffer contents.
3241fa9e4066Sahrens  * If the buffer has more than one reference, we must make
3242088f3894Sahrens  * a new hdr for the buffer.
3243fa9e4066Sahrens  */
3244fa9e4066Sahrens void
3245fa9e4066Sahrens arc_release(arc_buf_t *buf, void *tag)
3246fa9e4066Sahrens {
32476f83844dSMark Maybee 	arc_buf_hdr_t *hdr;
32483f9d6ad7SLin Ling 	kmutex_t *hash_lock = NULL;
32496f83844dSMark Maybee 	l2arc_buf_hdr_t *l2hdr;
3250fa94a07fSbrendan 	uint64_t buf_size;
3251fa9e4066Sahrens 
32523f9d6ad7SLin Ling 	/*
32533f9d6ad7SLin Ling 	 * It would be nice to assert that if it's DMU metadata (level >
32543f9d6ad7SLin Ling 	 * 0 || it's the dnode file), then it must be syncing context.
32553f9d6ad7SLin Ling 	 * But we don't know that information at this level.
32563f9d6ad7SLin Ling 	 */
32573f9d6ad7SLin Ling 
32583f9d6ad7SLin Ling 	mutex_enter(&buf->b_evict_lock);
32596f83844dSMark Maybee 	hdr = buf->b_hdr;
32606f83844dSMark Maybee 
3261fa9e4066Sahrens 	/* this buffer is not on any list */
3262fa9e4066Sahrens 	ASSERT(refcount_count(&hdr->b_refcnt) > 0);
3263fa9e4066Sahrens 
326444cb6abcSbmc 	if (hdr->b_state == arc_anon) {
3265fa9e4066Sahrens 		/* this buffer is already released */
3266ea8dc4b6Seschrock 		ASSERT(buf->b_efunc == NULL);
32670a95608cSBrendan Gregg - Sun Microsystems 	} else {
32680a95608cSBrendan Gregg - Sun Microsystems 		hash_lock = HDR_LOCK(hdr);
32690a95608cSBrendan Gregg - Sun Microsystems 		mutex_enter(hash_lock);
32703f9d6ad7SLin Ling 		hdr = buf->b_hdr;
32713f9d6ad7SLin Ling 		ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3272fa9e4066Sahrens 	}
3273fa9e4066Sahrens 
32746f83844dSMark Maybee 	l2hdr = hdr->b_l2hdr;
32756f83844dSMark Maybee 	if (l2hdr) {
32766f83844dSMark Maybee 		mutex_enter(&l2arc_buflist_mtx);
32776f83844dSMark Maybee 		hdr->b_l2hdr = NULL;
32786f83844dSMark Maybee 	}
3279d5285caeSGeorge Wilson 	buf_size = hdr->b_size;
32806f83844dSMark Maybee 
3281ea8dc4b6Seschrock 	/*
3282ea8dc4b6Seschrock 	 * Do we have more than one buf?
3283ea8dc4b6Seschrock 	 */
32846f83844dSMark Maybee 	if (hdr->b_datacnt > 1) {
3285fa9e4066Sahrens 		arc_buf_hdr_t *nhdr;
3286fa9e4066Sahrens 		arc_buf_t **bufp;
3287fa9e4066Sahrens 		uint64_t blksz = hdr->b_size;
3288ac05c741SMark Maybee 		uint64_t spa = hdr->b_spa;
3289ad23a2dbSjohansen 		arc_buf_contents_t type = hdr->b_type;
3290fa94a07fSbrendan 		uint32_t flags = hdr->b_flags;
3291fa9e4066Sahrens 
32926f83844dSMark Maybee 		ASSERT(hdr->b_buf != buf || buf->b_next != NULL);
3293fa9e4066Sahrens 		/*
32943f9d6ad7SLin Ling 		 * Pull the data off of this hdr and attach it to
32953f9d6ad7SLin Ling 		 * a new anonymous hdr.
3296fa9e4066Sahrens 		 */
3297ea8dc4b6Seschrock 		(void) remove_reference(hdr, hash_lock, tag);
3298fa9e4066Sahrens 		bufp = &hdr->b_buf;
3299ea8dc4b6Seschrock 		while (*bufp != buf)
3300fa9e4066Sahrens 			bufp = &(*bufp)->b_next;
33013f9d6ad7SLin Ling 		*bufp = buf->b_next;
3302af2c4821Smaybee 		buf->b_next = NULL;
3303ea8dc4b6Seschrock 
330444cb6abcSbmc 		ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size);
330544cb6abcSbmc 		atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
3306ea8dc4b6Seschrock 		if (refcount_is_zero(&hdr->b_refcnt)) {
33070e8c6158Smaybee 			uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
33080e8c6158Smaybee 			ASSERT3U(*size, >=, hdr->b_size);
33090e8c6158Smaybee 			atomic_add_64(size, -hdr->b_size);
3310ea8dc4b6Seschrock 		}
33119253d63dSGeorge Wilson 
33129253d63dSGeorge Wilson 		/*
33139253d63dSGeorge Wilson 		 * We're releasing a duplicate user data buffer, update
33149253d63dSGeorge Wilson 		 * our statistics accordingly.
33159253d63dSGeorge Wilson 		 */
33169253d63dSGeorge Wilson 		if (hdr->b_type == ARC_BUFC_DATA) {
33179253d63dSGeorge Wilson 			ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
33189253d63dSGeorge Wilson 			ARCSTAT_INCR(arcstat_duplicate_buffers_size,
33199253d63dSGeorge Wilson 			    -hdr->b_size);
33209253d63dSGeorge Wilson 		}
3321ea8dc4b6Seschrock 		hdr->b_datacnt -= 1;
3322c717a561Smaybee 		arc_cksum_verify(buf);
3323cd1c8b85SMatthew Ahrens 		arc_buf_unwatch(buf);
3324ea8dc4b6Seschrock 
3325fa9e4066Sahrens 		mutex_exit(hash_lock);
3326fa9e4066Sahrens 
33271ab7f2deSmaybee 		nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
3328fa9e4066Sahrens 		nhdr->b_size = blksz;
3329fa9e4066Sahrens 		nhdr->b_spa = spa;
3330ad23a2dbSjohansen 		nhdr->b_type = type;
3331fa9e4066Sahrens 		nhdr->b_buf = buf;
333244cb6abcSbmc 		nhdr->b_state = arc_anon;
3333fa9e4066Sahrens 		nhdr->b_arc_access = 0;
3334fa94a07fSbrendan 		nhdr->b_flags = flags & ARC_L2_WRITING;
3335fa94a07fSbrendan 		nhdr->b_l2hdr = NULL;
3336ea8dc4b6Seschrock 		nhdr->b_datacnt = 1;
3337c717a561Smaybee 		nhdr->b_freeze_cksum = NULL;
3338fa9e4066Sahrens 		(void) refcount_add(&nhdr->b_refcnt, tag);
3339af2c4821Smaybee 		buf->b_hdr = nhdr;
33403f9d6ad7SLin Ling 		mutex_exit(&buf->b_evict_lock);
334144cb6abcSbmc 		atomic_add_64(&arc_anon->arcs_size, blksz);
3342fa9e4066Sahrens 	} else {
33433f9d6ad7SLin Ling 		mutex_exit(&buf->b_evict_lock);
3344ea8dc4b6Seschrock 		ASSERT(refcount_count(&hdr->b_refcnt) == 1);
3345fa9e4066Sahrens 		ASSERT(!list_link_active(&hdr->b_arc_node));
3346fa9e4066Sahrens 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
33473f9d6ad7SLin Ling 		if (hdr->b_state != arc_anon)
33483f9d6ad7SLin Ling 			arc_change_state(arc_anon, hdr, hash_lock);
3349fa9e4066Sahrens 		hdr->b_arc_access = 0;
33503f9d6ad7SLin Ling 		if (hash_lock)
33513f9d6ad7SLin Ling 			mutex_exit(hash_lock);
3352fa94a07fSbrendan 
33533f9d6ad7SLin Ling 		buf_discard_identity(hdr);
3354c717a561Smaybee 		arc_buf_thaw(buf);
3355fa9e4066Sahrens 	}
3356ea8dc4b6Seschrock 	buf->b_efunc = NULL;
3357ea8dc4b6Seschrock 	buf->b_private = NULL;
3358fa94a07fSbrendan 
3359fa94a07fSbrendan 	if (l2hdr) {
3360aad02571SSaso Kiselkov 		ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
3361fa94a07fSbrendan 		list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
3362fa94a07fSbrendan 		kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
3363fa94a07fSbrendan 		ARCSTAT_INCR(arcstat_l2_size, -buf_size);
3364fa94a07fSbrendan 		mutex_exit(&l2arc_buflist_mtx);
33656f83844dSMark Maybee 	}
3366fa9e4066Sahrens }
3367fa9e4066Sahrens 
3368fa9e4066Sahrens int
3369fa9e4066Sahrens arc_released(arc_buf_t *buf)
3370fa9e4066Sahrens {
33716f83844dSMark Maybee 	int released;
33726f83844dSMark Maybee 
33733f9d6ad7SLin Ling 	mutex_enter(&buf->b_evict_lock);
33746f83844dSMark Maybee 	released = (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon);
33753f9d6ad7SLin Ling 	mutex_exit(&buf->b_evict_lock);
33766f83844dSMark Maybee 	return (released);
3377ea8dc4b6Seschrock }
3378ea8dc4b6Seschrock 
3379ea8dc4b6Seschrock int
3380ea8dc4b6Seschrock arc_has_callback(arc_buf_t *buf)
3381ea8dc4b6Seschrock {
33826f83844dSMark Maybee 	int callback;
33836f83844dSMark Maybee 
33843f9d6ad7SLin Ling 	mutex_enter(&buf->b_evict_lock);
33856f83844dSMark Maybee 	callback = (buf->b_efunc != NULL);
33863f9d6ad7SLin Ling 	mutex_exit(&buf->b_evict_lock);
33876f83844dSMark Maybee 	return (callback);
3388fa9e4066Sahrens }
3389fa9e4066Sahrens 
3390ea8dc4b6Seschrock #ifdef ZFS_DEBUG
3391ea8dc4b6Seschrock int
3392ea8dc4b6Seschrock arc_referenced(arc_buf_t *buf)
3393ea8dc4b6Seschrock {
33946f83844dSMark Maybee 	int referenced;
33956f83844dSMark Maybee 
33963f9d6ad7SLin Ling 	mutex_enter(&buf->b_evict_lock);
33976f83844dSMark Maybee 	referenced = (refcount_count(&buf->b_hdr->b_refcnt));
33983f9d6ad7SLin Ling 	mutex_exit(&buf->b_evict_lock);
33996f83844dSMark Maybee 	return (referenced);
3400ea8dc4b6Seschrock }
3401ea8dc4b6Seschrock #endif
3402ea8dc4b6Seschrock 
3403c717a561Smaybee static void
3404c717a561Smaybee arc_write_ready(zio_t *zio)
3405c717a561Smaybee {
3406c717a561Smaybee 	arc_write_callback_t *callback = zio->io_private;
3407c717a561Smaybee 	arc_buf_t *buf = callback->awcb_buf;
34080a4e9518Sgw 	arc_buf_hdr_t *hdr = buf->b_hdr;
3409c717a561Smaybee 
3410e14bb325SJeff Bonwick 	ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt));
3411e14bb325SJeff Bonwick 	callback->awcb_ready(zio, buf, callback->awcb_private);
3412e14bb325SJeff Bonwick 
34130a4e9518Sgw 	/*
34140a4e9518Sgw 	 * If the IO is already in progress, then this is a re-write
3415e14bb325SJeff Bonwick 	 * attempt, so we need to thaw and re-compute the cksum.
3416e14bb325SJeff Bonwick 	 * It is the responsibility of the callback to handle the
3417e14bb325SJeff Bonwick 	 * accounting for any re-write attempt.
34180a4e9518Sgw 	 */
34190a4e9518Sgw 	if (HDR_IO_IN_PROGRESS(hdr)) {
34200a4e9518Sgw 		mutex_enter(&hdr->b_freeze_lock);
34210a4e9518Sgw 		if (hdr->b_freeze_cksum != NULL) {
34220a4e9518Sgw 			kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
34230a4e9518Sgw 			hdr->b_freeze_cksum = NULL;
34240a4e9518Sgw 		}
34250a4e9518Sgw 		mutex_exit(&hdr->b_freeze_lock);
34260a4e9518Sgw 	}
3427fa94a07fSbrendan 	arc_cksum_compute(buf, B_FALSE);
34280a4e9518Sgw 	hdr->b_flags |= ARC_IO_IN_PROGRESS;
3429c717a561Smaybee }
3430c717a561Smaybee 
3431fa9e4066Sahrens static void
3432fa9e4066Sahrens arc_write_done(zio_t *zio)
3433fa9e4066Sahrens {
3434c717a561Smaybee 	arc_write_callback_t *callback = zio->io_private;
3435c717a561Smaybee 	arc_buf_t *buf = callback->awcb_buf;
3436c717a561Smaybee 	arc_buf_hdr_t *hdr = buf->b_hdr;
3437fa9e4066Sahrens 
3438b24ab676SJeff Bonwick 	ASSERT(hdr->b_acb == NULL);
3439b24ab676SJeff Bonwick 
3440b24ab676SJeff Bonwick 	if (zio->io_error == 0) {
3441b24ab676SJeff Bonwick 		hdr->b_dva = *BP_IDENTITY(zio->io_bp);
3442b24ab676SJeff Bonwick 		hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
3443b24ab676SJeff Bonwick 		hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0];
3444b24ab676SJeff Bonwick 	} else {
3445b24ab676SJeff Bonwick 		ASSERT(BUF_EMPTY(hdr));
3446b24ab676SJeff Bonwick 	}
3447fa9e4066Sahrens 
3448ea8dc4b6Seschrock 	/*
3449ea8dc4b6Seschrock 	 * If the block to be written was all-zero, we may have
3450ea8dc4b6Seschrock 	 * compressed it away.  In this case no write was performed
34513f9d6ad7SLin Ling 	 * so there will be no dva/birth/checksum.  The buffer must
34523f9d6ad7SLin Ling 	 * therefore remain anonymous (and uncached).
3453ea8dc4b6Seschrock 	 */
3454fa9e4066Sahrens 	if (!BUF_EMPTY(hdr)) {
3455fa9e4066Sahrens 		arc_buf_hdr_t *exists;
3456fa9e4066Sahrens 		kmutex_t *hash_lock;
3457fa9e4066Sahrens 
3458b24ab676SJeff Bonwick 		ASSERT(zio->io_error == 0);
3459b24ab676SJeff Bonwick 
34606b4acc8bSahrens 		arc_cksum_verify(buf);
34616b4acc8bSahrens 
3462fa9e4066Sahrens 		exists = buf_hash_insert(hdr, &hash_lock);
3463fa9e4066Sahrens 		if (exists) {
3464fa9e4066Sahrens 			/*
3465fa9e4066Sahrens 			 * This can only happen if we overwrite for
3466fa9e4066Sahrens 			 * sync-to-convergence, because we remove
3467fa9e4066Sahrens 			 * buffers from the hash table when we arc_free().
3468fa9e4066Sahrens 			 */
3469b24ab676SJeff Bonwick 			if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
3470b24ab676SJeff Bonwick 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
3471b24ab676SJeff Bonwick 					panic("bad overwrite, hdr=%p exists=%p",
3472b24ab676SJeff Bonwick 					    (void *)hdr, (void *)exists);
3473b24ab676SJeff Bonwick 				ASSERT(refcount_is_zero(&exists->b_refcnt));
3474b24ab676SJeff Bonwick 				arc_change_state(arc_anon, exists, hash_lock);
3475b24ab676SJeff Bonwick 				mutex_exit(hash_lock);
3476b24ab676SJeff Bonwick 				arc_hdr_destroy(exists);
3477b24ab676SJeff Bonwick 				exists = buf_hash_insert(hdr, &hash_lock);
3478b24ab676SJeff Bonwick 				ASSERT3P(exists, ==, NULL);
347980901aeaSGeorge Wilson 			} else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
348080901aeaSGeorge Wilson 				/* nopwrite */
348180901aeaSGeorge Wilson 				ASSERT(zio->io_prop.zp_nopwrite);
348280901aeaSGeorge Wilson 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
348380901aeaSGeorge Wilson 					panic("bad nopwrite, hdr=%p exists=%p",
348480901aeaSGeorge Wilson 					    (void *)hdr, (void *)exists);
3485b24ab676SJeff Bonwick 			} else {
3486b24ab676SJeff Bonwick 				/* Dedup */
3487b24ab676SJeff Bonwick 				ASSERT(hdr->b_datacnt == 1);
3488b24ab676SJeff Bonwick 				ASSERT(hdr->b_state == arc_anon);
3489b24ab676SJeff Bonwick 				ASSERT(BP_GET_DEDUP(zio->io_bp));
3490b24ab676SJeff Bonwick 				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
3491ae46e4c7SMatthew Ahrens 			}
3492fa9e4066Sahrens 		}
3493ea8dc4b6Seschrock 		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3494088f3894Sahrens 		/* if it's not anon, we are doing a scrub */
3495b24ab676SJeff Bonwick 		if (!exists && hdr->b_state == arc_anon)
3496088f3894Sahrens 			arc_access(hdr, hash_lock);
349744eda4d7Smaybee 		mutex_exit(hash_lock);
3498ea8dc4b6Seschrock 	} else {
3499ea8dc4b6Seschrock 		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3500fa9e4066Sahrens 	}
3501ea8dc4b6Seschrock 
3502b24ab676SJeff Bonwick 	ASSERT(!refcount_is_zero(&hdr->b_refcnt));
3503b24ab676SJeff Bonwick 	callback->awcb_done(zio, buf, callback->awcb_private);
3504fa9e4066Sahrens 
3505c717a561Smaybee 	kmem_free(callback, sizeof (arc_write_callback_t));
3506fa9e4066Sahrens }
3507fa9e4066Sahrens 
3508c717a561Smaybee zio_t *
3509b24ab676SJeff Bonwick arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
3510aad02571SSaso Kiselkov     blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, boolean_t l2arc_compress,
3511aad02571SSaso Kiselkov     const zio_prop_t *zp, arc_done_func_t *ready, arc_done_func_t *done,
3512aad02571SSaso Kiselkov     void *private, int priority, int zio_flags, const zbookmark_t *zb)
3513fa9e4066Sahrens {
3514fa9e4066Sahrens 	arc_buf_hdr_t *hdr = buf->b_hdr;
3515c717a561Smaybee 	arc_write_callback_t *callback;
3516e14bb325SJeff Bonwick 	zio_t *zio;
3517fa9e4066Sahrens 
3518e14bb325SJeff Bonwick 	ASSERT(ready != NULL);
3519b24ab676SJeff Bonwick 	ASSERT(done != NULL);
3520fa9e4066Sahrens 	ASSERT(!HDR_IO_ERROR(hdr));
3521c5c6ffa0Smaybee 	ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0);
3522b24ab676SJeff Bonwick 	ASSERT(hdr->b_acb == NULL);
35233baa08fcSek 	if (l2arc)
35243baa08fcSek 		hdr->b_flags |= ARC_L2CACHE;
3525aad02571SSaso Kiselkov 	if (l2arc_compress)
3526aad02571SSaso Kiselkov 		hdr->b_flags |= ARC_L2COMPRESS;
3527c717a561Smaybee 	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
3528c717a561Smaybee 	callback->awcb_ready = ready;
3529c717a561Smaybee 	callback->awcb_done = done;
3530c717a561Smaybee 	callback->awcb_private = private;
3531c717a561Smaybee 	callback->awcb_buf = buf;
3532088f3894Sahrens 
3533b24ab676SJeff Bonwick 	zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp,
3534e14bb325SJeff Bonwick 	    arc_write_ready, arc_write_done, callback, priority, zio_flags, zb);
3535fa9e4066Sahrens 
3536c717a561Smaybee 	return (zio);
3537fa9e4066Sahrens }
3538fa9e4066Sahrens 
35391ab7f2deSmaybee static int
35402fdbea25SAleksandr Guzovskiy arc_memory_throttle(uint64_t reserve, uint64_t inflight_data, uint64_t txg)
35411ab7f2deSmaybee {
35421ab7f2deSmaybee #ifdef _KERNEL
35431ab7f2deSmaybee 	uint64_t available_memory = ptob(freemem);
35441ab7f2deSmaybee 	static uint64_t page_load = 0;
35451ab7f2deSmaybee 	static uint64_t last_txg = 0;
35461ab7f2deSmaybee 
35471ab7f2deSmaybee #if defined(__i386)
35481ab7f2deSmaybee 	available_memory =
35491ab7f2deSmaybee 	    MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));
35501ab7f2deSmaybee #endif
35511ab7f2deSmaybee 	if (available_memory >= zfs_write_limit_max)
35521ab7f2deSmaybee 		return (0);
35531ab7f2deSmaybee 
35541ab7f2deSmaybee 	if (txg > last_txg) {
35551ab7f2deSmaybee 		last_txg = txg;
35561ab7f2deSmaybee 		page_load = 0;
35571ab7f2deSmaybee 	}
35581ab7f2deSmaybee 	/*
35591ab7f2deSmaybee 	 * If we are in pageout, we know that memory is already tight,
35601ab7f2deSmaybee 	 * the arc is already going to be evicting, so we just want to
35611ab7f2deSmaybee 	 * continue to let page writes occur as quickly as possible.
35621ab7f2deSmaybee 	 */
35631ab7f2deSmaybee 	if (curproc == proc_pageout) {
35641ab7f2deSmaybee 		if (page_load > MAX(ptob(minfree), available_memory) / 4)
3565be6fd75aSMatthew Ahrens 			return (SET_ERROR(ERESTART));
35661ab7f2deSmaybee 		/* Note: reserve is inflated, so we deflate */
35671ab7f2deSmaybee 		page_load += reserve / 8;
35681ab7f2deSmaybee 		return (0);
35691ab7f2deSmaybee 	} else if (page_load > 0 && arc_reclaim_needed()) {
35701ab7f2deSmaybee 		/* memory is low, delay before restarting */
35711ab7f2deSmaybee 		ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
3572be6fd75aSMatthew Ahrens 		return (SET_ERROR(EAGAIN));
35731ab7f2deSmaybee 	}
35741ab7f2deSmaybee 	page_load = 0;
35751ab7f2deSmaybee 
35761ab7f2deSmaybee 	if (arc_size > arc_c_min) {
35771ab7f2deSmaybee 		uint64_t evictable_memory =
35781ab7f2deSmaybee 		    arc_mru->arcs_lsize[ARC_BUFC_DATA] +
35791ab7f2deSmaybee 		    arc_mru->arcs_lsize[ARC_BUFC_METADATA] +
35801ab7f2deSmaybee 		    arc_mfu->arcs_lsize[ARC_BUFC_DATA] +
35811ab7f2deSmaybee 		    arc_mfu->arcs_lsize[ARC_BUFC_METADATA];
35821ab7f2deSmaybee 		available_memory += MIN(evictable_memory, arc_size - arc_c_min);
35831ab7f2deSmaybee 	}
35841ab7f2deSmaybee 
35851ab7f2deSmaybee 	if (inflight_data > available_memory / 4) {
35861ab7f2deSmaybee 		ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
3587be6fd75aSMatthew Ahrens 		return (SET_ERROR(ERESTART));
35881ab7f2deSmaybee 	}
35891ab7f2deSmaybee #endif
35901ab7f2deSmaybee 	return (0);
35911ab7f2deSmaybee }
35921ab7f2deSmaybee 
3593fa9e4066Sahrens void
35941ab7f2deSmaybee arc_tempreserve_clear(uint64_t reserve)
3595fa9e4066Sahrens {
35961ab7f2deSmaybee 	atomic_add_64(&arc_tempreserve, -reserve);
3597fa9e4066Sahrens 	ASSERT((int64_t)arc_tempreserve >= 0);
3598fa9e4066Sahrens }
3599fa9e4066Sahrens 
3600fa9e4066Sahrens int
36011ab7f2deSmaybee arc_tempreserve_space(uint64_t reserve, uint64_t txg)
3602fa9e4066Sahrens {
36031ab7f2deSmaybee 	int error;
36042fdbea25SAleksandr Guzovskiy 	uint64_t anon_size;
36051ab7f2deSmaybee 
3606fa9e4066Sahrens #ifdef ZFS_DEBUG
3607fa9e4066Sahrens 	/*
3608fa9e4066Sahrens 	 * Once in a while, fail for no reason.  Everything should cope.
3609fa9e4066Sahrens 	 */
3610fa9e4066Sahrens 	if (spa_get_random(10000) == 0) {
3611fa9e4066Sahrens 		dprintf("forcing random failure\n");
3612be6fd75aSMatthew Ahrens 		return (SET_ERROR(ERESTART));
3613fa9e4066Sahrens 	}
3614fa9e4066Sahrens #endif
36151ab7f2deSmaybee 	if (reserve > arc_c/4 && !arc_no_grow)
36161ab7f2deSmaybee 		arc_c = MIN(arc_c_max, reserve * 4);
36171ab7f2deSmaybee 	if (reserve > arc_c)
3618be6fd75aSMatthew Ahrens 		return (SET_ERROR(ENOMEM));
3619112fe045Smaybee 
36202fdbea25SAleksandr Guzovskiy 	/*
36212fdbea25SAleksandr Guzovskiy 	 * Don't count loaned bufs as in flight dirty data to prevent long
36222fdbea25SAleksandr Guzovskiy 	 * network delays from blocking transactions that are ready to be
36232fdbea25SAleksandr Guzovskiy 	 * assigned to a txg.
36242fdbea25SAleksandr Guzovskiy 	 */
36252fdbea25SAleksandr Guzovskiy 	anon_size = MAX((int64_t)(arc_anon->arcs_size - arc_loaned_bytes), 0);
36262fdbea25SAleksandr Guzovskiy 
36271ab7f2deSmaybee 	/*
36281ab7f2deSmaybee 	 * Writes will, almost always, require additional memory allocations
3629*f7170741SWill Andrews 	 * in order to compress/encrypt/etc the data.  We therefore need to
36301ab7f2deSmaybee 	 * make sure that there is sufficient available memory for this.
36311ab7f2deSmaybee 	 */
36322fdbea25SAleksandr Guzovskiy 	if (error = arc_memory_throttle(reserve, anon_size, txg))
36331ab7f2deSmaybee 		return (error);
36341ab7f2deSmaybee 
3635fa9e4066Sahrens 	/*
3636112fe045Smaybee 	 * Throttle writes when the amount of dirty data in the cache
3637112fe045Smaybee 	 * gets too large.  We try to keep the cache less than half full
3638112fe045Smaybee 	 * of dirty blocks so that our sync times don't grow too large.
3639112fe045Smaybee 	 * Note: if two requests come in concurrently, we might let them
3640112fe045Smaybee 	 * both succeed, when one of them should fail.  Not a huge deal.
3641fa9e4066Sahrens 	 */
36422fdbea25SAleksandr Guzovskiy 
36432fdbea25SAleksandr Guzovskiy 	if (reserve + arc_tempreserve + anon_size > arc_c / 2 &&
36442fdbea25SAleksandr Guzovskiy 	    anon_size > arc_c / 4) {
36450e8c6158Smaybee 		dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
36460e8c6158Smaybee 		    "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
36470e8c6158Smaybee 		    arc_tempreserve>>10,
36480e8c6158Smaybee 		    arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10,
36490e8c6158Smaybee 		    arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10,
36501ab7f2deSmaybee 		    reserve>>10, arc_c>>10);
3651be6fd75aSMatthew Ahrens 		return (SET_ERROR(ERESTART));
3652fa9e4066Sahrens 	}
36531ab7f2deSmaybee 	atomic_add_64(&arc_tempreserve, reserve);
3654fa9e4066Sahrens 	return (0);
3655fa9e4066Sahrens }
3656fa9e4066Sahrens 
3657fa9e4066Sahrens void
3658fa9e4066Sahrens arc_init(void)
3659fa9e4066Sahrens {
3660fa9e4066Sahrens 	mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL);
3661fa9e4066Sahrens 	cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL);
3662fa9e4066Sahrens 
366313506d1eSmaybee 	/* Convert seconds to clock ticks */
3664b19a79ecSperrin 	arc_min_prefetch_lifespan = 1 * hz;
366513506d1eSmaybee 
3666fa9e4066Sahrens 	/* Start out with 1/8 of all memory */
366744cb6abcSbmc 	arc_c = physmem * PAGESIZE / 8;
3668fa9e4066Sahrens 
3669fa9e4066Sahrens #ifdef _KERNEL
3670fa9e4066Sahrens 	/*
3671fa9e4066Sahrens 	 * On architectures where the physical memory can be larger
3672fa9e4066Sahrens 	 * than the addressable space (intel in 32-bit mode), we may
3673fa9e4066Sahrens 	 * need to limit the cache to 1/8 of VM size.
3674fa9e4066Sahrens 	 */
367544cb6abcSbmc 	arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8);
3676fa9e4066Sahrens #endif
3677fa9e4066Sahrens 
3678112fe045Smaybee 	/* set min cache to 1/32 of all memory, or 64MB, whichever is more */
367944cb6abcSbmc 	arc_c_min = MAX(arc_c / 4, 64<<20);
3680112fe045Smaybee 	/* set max to 3/4 of all memory, or all but 1GB, whichever is more */
368144cb6abcSbmc 	if (arc_c * 8 >= 1<<30)
368244cb6abcSbmc 		arc_c_max = (arc_c * 8) - (1<<30);
3683fa9e4066Sahrens 	else
368444cb6abcSbmc 		arc_c_max = arc_c_min;
368544cb6abcSbmc 	arc_c_max = MAX(arc_c * 6, arc_c_max);
3686a2eea2e1Sahrens 
3687a2eea2e1Sahrens 	/*
3688a2eea2e1Sahrens 	 * Allow the tunables to override our calculations if they are
3689a2eea2e1Sahrens 	 * reasonable (ie. over 64MB)
3690a2eea2e1Sahrens 	 */
3691a2eea2e1Sahrens 	if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE)
369244cb6abcSbmc 		arc_c_max = zfs_arc_max;
369344cb6abcSbmc 	if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max)
369444cb6abcSbmc 		arc_c_min = zfs_arc_min;
3695a2eea2e1Sahrens 
369644cb6abcSbmc 	arc_c = arc_c_max;
369744cb6abcSbmc 	arc_p = (arc_c >> 1);
3698fa9e4066Sahrens 
36990e8c6158Smaybee 	/* limit meta-data to 1/4 of the arc capacity */
37000e8c6158Smaybee 	arc_meta_limit = arc_c_max / 4;
37011116048bSek 
37021116048bSek 	/* Allow the tunable to override if it is reasonable */
37031116048bSek 	if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max)
37041116048bSek 		arc_meta_limit = zfs_arc_meta_limit;
37051116048bSek 
37060e8c6158Smaybee 	if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0)
37070e8c6158Smaybee 		arc_c_min = arc_meta_limit / 2;
37080e8c6158Smaybee 
37095a98e54bSBrendan Gregg - Sun Microsystems 	if (zfs_arc_grow_retry > 0)
37105a98e54bSBrendan Gregg - Sun Microsystems 		arc_grow_retry = zfs_arc_grow_retry;
37115a98e54bSBrendan Gregg - Sun Microsystems 
37125a98e54bSBrendan Gregg - Sun Microsystems 	if (zfs_arc_shrink_shift > 0)
37135a98e54bSBrendan Gregg - Sun Microsystems 		arc_shrink_shift = zfs_arc_shrink_shift;
37145a98e54bSBrendan Gregg - Sun Microsystems 
37155a98e54bSBrendan Gregg - Sun Microsystems 	if (zfs_arc_p_min_shift > 0)
37165a98e54bSBrendan Gregg - Sun Microsystems 		arc_p_min_shift = zfs_arc_p_min_shift;
37175a98e54bSBrendan Gregg - Sun Microsystems 
3718fa9e4066Sahrens 	/* if kmem_flags are set, lets try to use less memory */
3719fa9e4066Sahrens 	if (kmem_debugging())
372044cb6abcSbmc 		arc_c = arc_c / 2;
372144cb6abcSbmc 	if (arc_c < arc_c_min)
372244cb6abcSbmc 		arc_c = arc_c_min;
372344cb6abcSbmc 
372444cb6abcSbmc 	arc_anon = &ARC_anon;
372544cb6abcSbmc 	arc_mru = &ARC_mru;
372644cb6abcSbmc 	arc_mru_ghost = &ARC_mru_ghost;
372744cb6abcSbmc 	arc_mfu = &ARC_mfu;
372844cb6abcSbmc 	arc_mfu_ghost = &ARC_mfu_ghost;
3729fa94a07fSbrendan 	arc_l2c_only = &ARC_l2c_only;
373044cb6abcSbmc 	arc_size = 0;
373144cb6abcSbmc 
373244cb6abcSbmc 	mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
373344cb6abcSbmc 	mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
373444cb6abcSbmc 	mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
373544cb6abcSbmc 	mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
373644cb6abcSbmc 	mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3737fa94a07fSbrendan 	mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
373844cb6abcSbmc 
37390e8c6158Smaybee 	list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
37400e8c6158Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
37410e8c6158Smaybee 	list_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
37420e8c6158Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
37430e8c6158Smaybee 	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
37440e8c6158Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
37450e8c6158Smaybee 	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
37460e8c6158Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
37470e8c6158Smaybee 	list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
37480e8c6158Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
37490e8c6158Smaybee 	list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
37500e8c6158Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
37510e8c6158Smaybee 	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
37520e8c6158Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
37530e8c6158Smaybee 	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
37540e8c6158Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3755fa94a07fSbrendan 	list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
3756fa94a07fSbrendan 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3757fa94a07fSbrendan 	list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
3758fa94a07fSbrendan 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3759fa9e4066Sahrens 
3760fa9e4066Sahrens 	buf_init();
3761fa9e4066Sahrens 
3762fa9e4066Sahrens 	arc_thread_exit = 0;
3763ea8dc4b6Seschrock 	arc_eviction_list = NULL;
3764ea8dc4b6Seschrock 	mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL);
376540d7d650Smaybee 	bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t));
3766fa9e4066Sahrens 
376744cb6abcSbmc 	arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
376844cb6abcSbmc 	    sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
376944cb6abcSbmc 
377044cb6abcSbmc 	if (arc_ksp != NULL) {
377144cb6abcSbmc 		arc_ksp->ks_data = &arc_stats;
377244cb6abcSbmc 		kstat_install(arc_ksp);
377344cb6abcSbmc 	}
377444cb6abcSbmc 
3775fa9e4066Sahrens 	(void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
3776fa9e4066Sahrens 	    TS_RUN, minclsyspri);
377749e3519aSmaybee 
377849e3519aSmaybee 	arc_dead = FALSE;
37793a737e0dSbrendan 	arc_warm = B_FALSE;
37801ab7f2deSmaybee 
37811ab7f2deSmaybee 	if (zfs_write_limit_max == 0)
378205715f94SMark Maybee 		zfs_write_limit_max = ptob(physmem) >> zfs_write_limit_shift;
37831ab7f2deSmaybee 	else
37841ab7f2deSmaybee 		zfs_write_limit_shift = 0;
378505715f94SMark Maybee 	mutex_init(&zfs_write_limit_lock, NULL, MUTEX_DEFAULT, NULL);
3786fa9e4066Sahrens }
3787fa9e4066Sahrens 
3788fa9e4066Sahrens void
3789fa9e4066Sahrens arc_fini(void)
3790fa9e4066Sahrens {
3791fa9e4066Sahrens 	mutex_enter(&arc_reclaim_thr_lock);
3792fa9e4066Sahrens 	arc_thread_exit = 1;
3793fa9e4066Sahrens 	while (arc_thread_exit != 0)
3794fa9e4066Sahrens 		cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock);
3795fa9e4066Sahrens 	mutex_exit(&arc_reclaim_thr_lock);
3796fa9e4066Sahrens 
3797874395d5Smaybee 	arc_flush(NULL);
3798fa9e4066Sahrens 
3799fa9e4066Sahrens 	arc_dead = TRUE;
3800fa9e4066Sahrens 
380144cb6abcSbmc 	if (arc_ksp != NULL) {
380244cb6abcSbmc 		kstat_delete(arc_ksp);
380344cb6abcSbmc 		arc_ksp = NULL;
380444cb6abcSbmc 	}
380544cb6abcSbmc 
3806ea8dc4b6Seschrock 	mutex_destroy(&arc_eviction_mtx);
3807fa9e4066Sahrens 	mutex_destroy(&arc_reclaim_thr_lock);
3808fa9e4066Sahrens 	cv_destroy(&arc_reclaim_thr_cv);
3809fa9e4066Sahrens 
38100e8c6158Smaybee 	list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
38110e8c6158Smaybee 	list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
38120e8c6158Smaybee 	list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
38130e8c6158Smaybee 	list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
38140e8c6158Smaybee 	list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
38150e8c6158Smaybee 	list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
38160e8c6158Smaybee 	list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
38170e8c6158Smaybee 	list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
3818fa9e4066Sahrens 
381944cb6abcSbmc 	mutex_destroy(&arc_anon->arcs_mtx);
382044cb6abcSbmc 	mutex_destroy(&arc_mru->arcs_mtx);
382144cb6abcSbmc 	mutex_destroy(&arc_mru_ghost->arcs_mtx);
382244cb6abcSbmc 	mutex_destroy(&arc_mfu->arcs_mtx);
382344cb6abcSbmc 	mutex_destroy(&arc_mfu_ghost->arcs_mtx);
3824b5e70f97SRicardo M. Correia 	mutex_destroy(&arc_l2c_only->arcs_mtx);
38255ad82045Snd 
382605715f94SMark Maybee 	mutex_destroy(&zfs_write_limit_lock);
382705715f94SMark Maybee 
3828fa9e4066Sahrens 	buf_fini();
38292fdbea25SAleksandr Guzovskiy 
38302fdbea25SAleksandr Guzovskiy 	ASSERT(arc_loaned_bytes == 0);
3831fa9e4066Sahrens }
3832fa94a07fSbrendan 
3833fa94a07fSbrendan /*
3834fa94a07fSbrendan  * Level 2 ARC
3835fa94a07fSbrendan  *
3836fa94a07fSbrendan  * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
3837fa94a07fSbrendan  * It uses dedicated storage devices to hold cached data, which are populated
3838fa94a07fSbrendan  * using large infrequent writes.  The main role of this cache is to boost
3839fa94a07fSbrendan  * the performance of random read workloads.  The intended L2ARC devices
3840fa94a07fSbrendan  * include short-stroked disks, solid state disks, and other media with
3841fa94a07fSbrendan  * substantially faster read latency than disk.
3842fa94a07fSbrendan  *
3843fa94a07fSbrendan  *                 +-----------------------+
3844fa94a07fSbrendan  *                 |         ARC           |
3845fa94a07fSbrendan  *                 +-----------------------+
3846fa94a07fSbrendan  *                    |         ^     ^
3847fa94a07fSbrendan  *                    |         |     |
3848fa94a07fSbrendan  *      l2arc_feed_thread()    arc_read()
3849fa94a07fSbrendan  *                    |         |     |
3850fa94a07fSbrendan  *                    |  l2arc read   |
3851fa94a07fSbrendan  *                    V         |     |
3852fa94a07fSbrendan  *               +---------------+    |
3853fa94a07fSbrendan  *               |     L2ARC     |    |
3854fa94a07fSbrendan  *               +---------------+    |
3855fa94a07fSbrendan  *                   |    ^           |
3856fa94a07fSbrendan  *          l2arc_write() |           |
3857fa94a07fSbrendan  *                   |    |           |
3858fa94a07fSbrendan  *                   V    |           |
3859fa94a07fSbrendan  *                 +-------+      +-------+
3860fa94a07fSbrendan  *                 | vdev  |      | vdev  |
3861fa94a07fSbrendan  *                 | cache |      | cache |
3862fa94a07fSbrendan  *                 +-------+      +-------+
3863fa94a07fSbrendan  *                 +=========+     .-----.
3864fa94a07fSbrendan  *                 :  L2ARC  :    |-_____-|
3865fa94a07fSbrendan  *                 : devices :    | Disks |
3866fa94a07fSbrendan  *                 +=========+    `-_____-'
3867fa94a07fSbrendan  *
3868fa94a07fSbrendan  * Read requests are satisfied from the following sources, in order:
3869fa94a07fSbrendan  *
3870fa94a07fSbrendan  *	1) ARC
3871fa94a07fSbrendan  *	2) vdev cache of L2ARC devices
3872fa94a07fSbrendan  *	3) L2ARC devices
3873fa94a07fSbrendan  *	4) vdev cache of disks
3874fa94a07fSbrendan  *	5) disks
3875fa94a07fSbrendan  *
3876fa94a07fSbrendan  * Some L2ARC device types exhibit extremely slow write performance.
3877fa94a07fSbrendan  * To accommodate for this there are some significant differences between
3878fa94a07fSbrendan  * the L2ARC and traditional cache design:
3879fa94a07fSbrendan  *
3880fa94a07fSbrendan  * 1. There is no eviction path from the ARC to the L2ARC.  Evictions from
3881fa94a07fSbrendan  * the ARC behave as usual, freeing buffers and placing headers on ghost
3882fa94a07fSbrendan  * lists.  The ARC does not send buffers to the L2ARC during eviction as
3883fa94a07fSbrendan  * this would add inflated write latencies for all ARC memory pressure.
3884fa94a07fSbrendan  *
3885fa94a07fSbrendan  * 2. The L2ARC attempts to cache data from the ARC before it is evicted.
3886fa94a07fSbrendan  * It does this by periodically scanning buffers from the eviction-end of
3887fa94a07fSbrendan  * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
3888aad02571SSaso Kiselkov  * not already there. It scans until a headroom of buffers is satisfied,
3889aad02571SSaso Kiselkov  * which itself is a buffer for ARC eviction. If a compressible buffer is
3890aad02571SSaso Kiselkov  * found during scanning and selected for writing to an L2ARC device, we
3891aad02571SSaso Kiselkov  * temporarily boost scanning headroom during the next scan cycle to make
3892aad02571SSaso Kiselkov  * sure we adapt to compression effects (which might significantly reduce
3893aad02571SSaso Kiselkov  * the data volume we write to L2ARC). The thread that does this is
3894fa94a07fSbrendan  * l2arc_feed_thread(), illustrated below; example sizes are included to
3895fa94a07fSbrendan  * provide a better sense of ratio than this diagram:
3896fa94a07fSbrendan  *
3897fa94a07fSbrendan  *	       head -->                        tail
3898fa94a07fSbrendan  *	        +---------------------+----------+
3899fa94a07fSbrendan  *	ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->.   # already on L2ARC
3900fa94a07fSbrendan  *	        +---------------------+----------+   |   o L2ARC eligible
3901fa94a07fSbrendan  *	ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->|   : ARC buffer
3902fa94a07fSbrendan  *	        +---------------------+----------+   |
3903fa94a07fSbrendan  *	             15.9 Gbytes      ^ 32 Mbytes    |
3904fa94a07fSbrendan  *	                           headroom          |
3905fa94a07fSbrendan  *	                                      l2arc_feed_thread()
3906fa94a07fSbrendan  *	                                             |
3907fa94a07fSbrendan  *	                 l2arc write hand <--[oooo]--'
3908fa94a07fSbrendan  *	                         |           8 Mbyte
3909fa94a07fSbrendan  *	                         |          write max
3910fa94a07fSbrendan  *	                         V
3911fa94a07fSbrendan  *		  +==============================+
3912fa94a07fSbrendan  *	L2ARC dev |####|#|###|###|    |####| ... |
3913fa94a07fSbrendan  *	          +==============================+
3914fa94a07fSbrendan  *	                     32 Gbytes
3915fa94a07fSbrendan  *
3916fa94a07fSbrendan  * 3. If an ARC buffer is copied to the L2ARC but then hit instead of
3917fa94a07fSbrendan  * evicted, then the L2ARC has cached a buffer much sooner than it probably
3918fa94a07fSbrendan  * needed to, potentially wasting L2ARC device bandwidth and storage.  It is
3919fa94a07fSbrendan  * safe to say that this is an uncommon case, since buffers at the end of
3920fa94a07fSbrendan  * the ARC lists have moved there due to inactivity.
3921fa94a07fSbrendan  *
3922fa94a07fSbrendan  * 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
3923fa94a07fSbrendan  * then the L2ARC simply misses copying some buffers.  This serves as a
3924fa94a07fSbrendan  * pressure valve to prevent heavy read workloads from both stalling the ARC
3925fa94a07fSbrendan  * with waits and clogging the L2ARC with writes.  This also helps prevent
3926fa94a07fSbrendan  * the potential for the L2ARC to churn if it attempts to cache content too
3927fa94a07fSbrendan  * quickly, such as during backups of the entire pool.
3928fa94a07fSbrendan  *
39293a737e0dSbrendan  * 5. After system boot and before the ARC has filled main memory, there are
39303a737e0dSbrendan  * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
39313a737e0dSbrendan  * lists can remain mostly static.  Instead of searching from tail of these
39323a737e0dSbrendan  * lists as pictured, the l2arc_feed_thread() will search from the list heads
39333a737e0dSbrendan  * for eligible buffers, greatly increasing its chance of finding them.
39343a737e0dSbrendan  *
39353a737e0dSbrendan  * The L2ARC device write speed is also boosted during this time so that
39363a737e0dSbrendan  * the L2ARC warms up faster.  Since there have been no ARC evictions yet,
39373a737e0dSbrendan  * there are no L2ARC reads, and no fear of degrading read performance
39383a737e0dSbrendan  * through increased writes.
39393a737e0dSbrendan  *
39403a737e0dSbrendan  * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
3941fa94a07fSbrendan  * the vdev queue can aggregate them into larger and fewer writes.  Each
3942fa94a07fSbrendan  * device is written to in a rotor fashion, sweeping writes through
3943fa94a07fSbrendan  * available space then repeating.
3944fa94a07fSbrendan  *
39453a737e0dSbrendan  * 7. The L2ARC does not store dirty content.  It never needs to flush
3946fa94a07fSbrendan  * write buffers back to disk based storage.
3947fa94a07fSbrendan  *
39483a737e0dSbrendan  * 8. If an ARC buffer is written (and dirtied) which also exists in the
3949fa94a07fSbrendan  * L2ARC, the now stale L2ARC buffer is immediately dropped.
3950fa94a07fSbrendan  *
3951fa94a07fSbrendan  * The performance of the L2ARC can be tweaked by a number of tunables, which
3952fa94a07fSbrendan  * may be necessary for different workloads:
3953fa94a07fSbrendan  *
3954fa94a07fSbrendan  *	l2arc_write_max		max write bytes per interval
39553a737e0dSbrendan  *	l2arc_write_boost	extra write bytes during device warmup
3956fa94a07fSbrendan  *	l2arc_noprefetch	skip caching prefetched buffers
3957fa94a07fSbrendan  *	l2arc_headroom		number of max device writes to precache
3958aad02571SSaso Kiselkov  *	l2arc_headroom_boost	when we find compressed buffers during ARC
3959aad02571SSaso Kiselkov  *				scanning, we multiply headroom by this
3960aad02571SSaso Kiselkov  *				percentage factor for the next scan cycle,
3961aad02571SSaso Kiselkov  *				since more compressed buffers are likely to
3962aad02571SSaso Kiselkov  *				be present
3963fa94a07fSbrendan  *	l2arc_feed_secs		seconds between L2ARC writing
3964fa94a07fSbrendan  *
3965fa94a07fSbrendan  * Tunables may be removed or added as future performance improvements are
3966fa94a07fSbrendan  * integrated, and also may become zpool properties.
39675a98e54bSBrendan Gregg - Sun Microsystems  *
39685a98e54bSBrendan Gregg - Sun Microsystems  * There are three key functions that control how the L2ARC warms up:
39695a98e54bSBrendan Gregg - Sun Microsystems  *
39705a98e54bSBrendan Gregg - Sun Microsystems  *	l2arc_write_eligible()	check if a buffer is eligible to cache
39715a98e54bSBrendan Gregg - Sun Microsystems  *	l2arc_write_size()	calculate how much to write
39725a98e54bSBrendan Gregg - Sun Microsystems  *	l2arc_write_interval()	calculate sleep delay between writes
39735a98e54bSBrendan Gregg - Sun Microsystems  *
39745a98e54bSBrendan Gregg - Sun Microsystems  * These three functions determine what to write, how much, and how quickly
39755a98e54bSBrendan Gregg - Sun Microsystems  * to send writes.
3976fa94a07fSbrendan  */
3977fa94a07fSbrendan 
39785a98e54bSBrendan Gregg - Sun Microsystems static boolean_t
3979ac05c741SMark Maybee l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab)
39805a98e54bSBrendan Gregg - Sun Microsystems {
39815a98e54bSBrendan Gregg - Sun Microsystems 	/*
39825a98e54bSBrendan Gregg - Sun Microsystems 	 * A buffer is *not* eligible for the L2ARC if it:
39835a98e54bSBrendan Gregg - Sun Microsystems 	 * 1. belongs to a different spa.
39845ea40c06SBrendan Gregg - Sun Microsystems 	 * 2. is already cached on the L2ARC.
39855ea40c06SBrendan Gregg - Sun Microsystems 	 * 3. has an I/O in progress (it may be an incomplete read).
39865ea40c06SBrendan Gregg - Sun Microsystems 	 * 4. is flagged not eligible (zfs property).
39875a98e54bSBrendan Gregg - Sun Microsystems 	 */
39885ea40c06SBrendan Gregg - Sun Microsystems 	if (ab->b_spa != spa_guid || ab->b_l2hdr != NULL ||
39895a98e54bSBrendan Gregg - Sun Microsystems 	    HDR_IO_IN_PROGRESS(ab) || !HDR_L2CACHE(ab))
39905a98e54bSBrendan Gregg - Sun Microsystems 		return (B_FALSE);
39915a98e54bSBrendan Gregg - Sun Microsystems 
39925a98e54bSBrendan Gregg - Sun Microsystems 	return (B_TRUE);
39935a98e54bSBrendan Gregg - Sun Microsystems }
39945a98e54bSBrendan Gregg - Sun Microsystems 
39955a98e54bSBrendan Gregg - Sun Microsystems static uint64_t
3996aad02571SSaso Kiselkov l2arc_write_size(void)
39975a98e54bSBrendan Gregg - Sun Microsystems {
39985a98e54bSBrendan Gregg - Sun Microsystems 	uint64_t size;
39995a98e54bSBrendan Gregg - Sun Microsystems 
4000aad02571SSaso Kiselkov 	/*
4001aad02571SSaso Kiselkov 	 * Make sure our globals have meaningful values in case the user
4002aad02571SSaso Kiselkov 	 * altered them.
4003aad02571SSaso Kiselkov 	 */
4004aad02571SSaso Kiselkov 	size = l2arc_write_max;
4005aad02571SSaso Kiselkov 	if (size == 0) {
4006aad02571SSaso Kiselkov 		cmn_err(CE_NOTE, "Bad value for l2arc_write_max, value must "
4007aad02571SSaso Kiselkov 		    "be greater than zero, resetting it to the default (%d)",
4008aad02571SSaso Kiselkov 		    L2ARC_WRITE_SIZE);
4009aad02571SSaso Kiselkov 		size = l2arc_write_max = L2ARC_WRITE_SIZE;
4010aad02571SSaso Kiselkov 	}
40115a98e54bSBrendan Gregg - Sun Microsystems 
40125a98e54bSBrendan Gregg - Sun Microsystems 	if (arc_warm == B_FALSE)
4013aad02571SSaso Kiselkov 		size += l2arc_write_boost;
40145a98e54bSBrendan Gregg - Sun Microsystems 
40155a98e54bSBrendan Gregg - Sun Microsystems 	return (size);
40165a98e54bSBrendan Gregg - Sun Microsystems 
40175a98e54bSBrendan Gregg - Sun Microsystems }
40185a98e54bSBrendan Gregg - Sun Microsystems 
40195a98e54bSBrendan Gregg - Sun Microsystems static clock_t
40205a98e54bSBrendan Gregg - Sun Microsystems l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
40215a98e54bSBrendan Gregg - Sun Microsystems {
4022d3d50737SRafael Vanoni 	clock_t interval, next, now;
40235a98e54bSBrendan Gregg - Sun Microsystems 
40245a98e54bSBrendan Gregg - Sun Microsystems 	/*
40255a98e54bSBrendan Gregg - Sun Microsystems 	 * If the ARC lists are busy, increase our write rate; if the
40265a98e54bSBrendan Gregg - Sun Microsystems 	 * lists are stale, idle back.  This is achieved by checking
40275a98e54bSBrendan Gregg - Sun Microsystems 	 * how much we previously wrote - if it was more than half of
40285a98e54bSBrendan Gregg - Sun Microsystems 	 * what we wanted, schedule the next write much sooner.
40295a98e54bSBrendan Gregg - Sun Microsystems 	 */
40305a98e54bSBrendan Gregg - Sun Microsystems 	if (l2arc_feed_again && wrote > (wanted / 2))
40315a98e54bSBrendan Gregg - Sun Microsystems 		interval = (hz * l2arc_feed_min_ms) / 1000;
40325a98e54bSBrendan Gregg - Sun Microsystems 	else
40335a98e54bSBrendan Gregg - Sun Microsystems 		interval = hz * l2arc_feed_secs;
40345a98e54bSBrendan Gregg - Sun Microsystems 
4035d3d50737SRafael Vanoni 	now = ddi_get_lbolt();
4036d3d50737SRafael Vanoni 	next = MAX(now, MIN(now + interval, began + interval));
40375a98e54bSBrendan Gregg - Sun Microsystems 
40385a98e54bSBrendan Gregg - Sun Microsystems 	return (next);
40395a98e54bSBrendan Gregg - Sun Microsystems }
40405a98e54bSBrendan Gregg - Sun Microsystems 
4041fa94a07fSbrendan static void
4042fa94a07fSbrendan l2arc_hdr_stat_add(void)
4043fa94a07fSbrendan {
4044e6c728e1Sbrendan 	ARCSTAT_INCR(arcstat_l2_hdr_size, HDR_SIZE + L2HDR_SIZE);
4045e6c728e1Sbrendan 	ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE);
4046fa94a07fSbrendan }
4047fa94a07fSbrendan 
4048fa94a07fSbrendan static void
4049fa94a07fSbrendan l2arc_hdr_stat_remove(void)
4050fa94a07fSbrendan {
4051e6c728e1Sbrendan 	ARCSTAT_INCR(arcstat_l2_hdr_size, -(HDR_SIZE + L2HDR_SIZE));
4052e6c728e1Sbrendan 	ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE);
4053fa94a07fSbrendan }
4054fa94a07fSbrendan 
4055fa94a07fSbrendan /*
4056fa94a07fSbrendan  * Cycle through L2ARC devices.  This is how L2ARC load balances.
40573a737e0dSbrendan  * If a device is returned, this also returns holding the spa config lock.
4058fa94a07fSbrendan  */
4059fa94a07fSbrendan static l2arc_dev_t *
4060fa94a07fSbrendan l2arc_dev_get_next(void)
4061fa94a07fSbrendan {
40623a737e0dSbrendan 	l2arc_dev_t *first, *next = NULL;
40633a737e0dSbrendan 
40643a737e0dSbrendan 	/*
40653a737e0dSbrendan 	 * Lock out the removal of spas (spa_namespace_lock), then removal
40663a737e0dSbrendan 	 * of cache devices (l2arc_dev_mtx).  Once a device has been selected,
40673a737e0dSbrendan 	 * both locks will be dropped and a spa config lock held instead.
40683a737e0dSbrendan 	 */
40693a737e0dSbrendan 	mutex_enter(&spa_namespace_lock);
40703a737e0dSbrendan 	mutex_enter(&l2arc_dev_mtx);
4071fa94a07fSbrendan 
4072c5904d13Seschrock 	/* if there are no vdevs, there is nothing to do */
4073c5904d13Seschrock 	if (l2arc_ndev == 0)
40743a737e0dSbrendan 		goto out;
4075c5904d13Seschrock 
4076c5904d13Seschrock 	first = NULL;
4077c5904d13Seschrock 	next = l2arc_dev_last;
4078c5904d13Seschrock 	do {
4079c5904d13Seschrock 		/* loop around the list looking for a non-faulted vdev */
4080c5904d13Seschrock 		if (next == NULL) {
4081fa94a07fSbrendan 			next = list_head(l2arc_dev_list);
4082c5904d13Seschrock 		} else {
4083c5904d13Seschrock 			next = list_next(l2arc_dev_list, next);
4084c5904d13Seschrock 			if (next == NULL)
4085c5904d13Seschrock 				next = list_head(l2arc_dev_list);
4086c5904d13Seschrock 		}
4087c5904d13Seschrock 
4088c5904d13Seschrock 		/* if we have come back to the start, bail out */
4089c5904d13Seschrock 		if (first == NULL)
4090c5904d13Seschrock 			first = next;
4091c5904d13Seschrock 		else if (next == first)
4092c5904d13Seschrock 			break;
4093c5904d13Seschrock 
4094c5904d13Seschrock 	} while (vdev_is_dead(next->l2ad_vdev));
4095c5904d13Seschrock 
4096c5904d13Seschrock 	/* if we were unable to find any usable vdevs, return NULL */
4097c5904d13Seschrock 	if (vdev_is_dead(next->l2ad_vdev))
40983a737e0dSbrendan 		next = NULL;
4099fa94a07fSbrendan 
4100fa94a07fSbrendan 	l2arc_dev_last = next;
4101fa94a07fSbrendan 
41023a737e0dSbrendan out:
41033a737e0dSbrendan 	mutex_exit(&l2arc_dev_mtx);
41043a737e0dSbrendan 
41053a737e0dSbrendan 	/*
41063a737e0dSbrendan 	 * Grab the config lock to prevent the 'next' device from being
41073a737e0dSbrendan 	 * removed while we are writing to it.
41083a737e0dSbrendan 	 */
41093a737e0dSbrendan 	if (next != NULL)
4110e14bb325SJeff Bonwick 		spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
41113a737e0dSbrendan 	mutex_exit(&spa_namespace_lock);
41123a737e0dSbrendan 
4113fa94a07fSbrendan 	return (next);
4114fa94a07fSbrendan }
4115fa94a07fSbrendan 
41163a737e0dSbrendan /*
41173a737e0dSbrendan  * Free buffers that were tagged for destruction.
41183a737e0dSbrendan  */
41193a737e0dSbrendan static void
41203a737e0dSbrendan l2arc_do_free_on_write()
41213a737e0dSbrendan {
41223a737e0dSbrendan 	list_t *buflist;
41233a737e0dSbrendan 	l2arc_data_free_t *df, *df_prev;
41243a737e0dSbrendan 
41253a737e0dSbrendan 	mutex_enter(&l2arc_free_on_write_mtx);
41263a737e0dSbrendan 	buflist = l2arc_free_on_write;
41273a737e0dSbrendan 
41283a737e0dSbrendan 	for (df = list_tail(buflist); df; df = df_prev) {
41293a737e0dSbrendan 		df_prev = list_prev(buflist, df);
41303a737e0dSbrendan 		ASSERT(df->l2df_data != NULL);
41313a737e0dSbrendan 		ASSERT(df->l2df_func != NULL);
41323a737e0dSbrendan 		df->l2df_func(df->l2df_data, df->l2df_size);
41333a737e0dSbrendan 		list_remove(buflist, df);
41343a737e0dSbrendan 		kmem_free(df, sizeof (l2arc_data_free_t));
41353a737e0dSbrendan 	}
41363a737e0dSbrendan 
41373a737e0dSbrendan 	mutex_exit(&l2arc_free_on_write_mtx);
41383a737e0dSbrendan }
41393a737e0dSbrendan 
4140fa94a07fSbrendan /*
4141fa94a07fSbrendan  * A write to a cache device has completed.  Update all headers to allow
4142fa94a07fSbrendan  * reads from these buffers to begin.
4143fa94a07fSbrendan  */
4144fa94a07fSbrendan static void
4145fa94a07fSbrendan l2arc_write_done(zio_t *zio)
4146fa94a07fSbrendan {
4147fa94a07fSbrendan 	l2arc_write_callback_t *cb;
4148fa94a07fSbrendan 	l2arc_dev_t *dev;
4149fa94a07fSbrendan 	list_t *buflist;
4150fa94a07fSbrendan 	arc_buf_hdr_t *head, *ab, *ab_prev;
41513a737e0dSbrendan 	l2arc_buf_hdr_t *abl2;
4152fa94a07fSbrendan 	kmutex_t *hash_lock;
4153fa94a07fSbrendan 
4154fa94a07fSbrendan 	cb = zio->io_private;
4155fa94a07fSbrendan 	ASSERT(cb != NULL);
4156fa94a07fSbrendan 	dev = cb->l2wcb_dev;
4157fa94a07fSbrendan 	ASSERT(dev != NULL);
4158fa94a07fSbrendan 	head = cb->l2wcb_head;
4159fa94a07fSbrendan 	ASSERT(head != NULL);
4160fa94a07fSbrendan 	buflist = dev->l2ad_buflist;
4161fa94a07fSbrendan 	ASSERT(buflist != NULL);
4162fa94a07fSbrendan 	DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
4163fa94a07fSbrendan 	    l2arc_write_callback_t *, cb);
4164fa94a07fSbrendan 
4165fa94a07fSbrendan 	if (zio->io_error != 0)
4166fa94a07fSbrendan 		ARCSTAT_BUMP(arcstat_l2_writes_error);
4167fa94a07fSbrendan 
4168fa94a07fSbrendan 	mutex_enter(&l2arc_buflist_mtx);
4169fa94a07fSbrendan 
4170fa94a07fSbrendan 	/*
4171fa94a07fSbrendan 	 * All writes completed, or an error was hit.
4172fa94a07fSbrendan 	 */
4173fa94a07fSbrendan 	for (ab = list_prev(buflist, head); ab; ab = ab_prev) {
4174fa94a07fSbrendan 		ab_prev = list_prev(buflist, ab);
4175fa94a07fSbrendan 
4176fa94a07fSbrendan 		hash_lock = HDR_LOCK(ab);
4177fa94a07fSbrendan 		if (!mutex_tryenter(hash_lock)) {
4178fa94a07fSbrendan 			/*
4179fa94a07fSbrendan 			 * This buffer misses out.  It may be in a stage
4180fa94a07fSbrendan 			 * of eviction.  Its ARC_L2_WRITING flag will be
4181fa94a07fSbrendan 			 * left set, denying reads to this buffer.
4182fa94a07fSbrendan 			 */
4183fa94a07fSbrendan 			ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss);
4184fa94a07fSbrendan 			continue;
4185fa94a07fSbrendan 		}
4186fa94a07fSbrendan 
4187aad02571SSaso Kiselkov 		abl2 = ab->b_l2hdr;
4188aad02571SSaso Kiselkov 
4189aad02571SSaso Kiselkov 		/*
4190aad02571SSaso Kiselkov 		 * Release the temporary compressed buffer as soon as possible.
4191aad02571SSaso Kiselkov 		 */
4192aad02571SSaso Kiselkov 		if (abl2->b_compress != ZIO_COMPRESS_OFF)
4193aad02571SSaso Kiselkov 			l2arc_release_cdata_buf(ab);
4194aad02571SSaso Kiselkov 
4195fa94a07fSbrendan 		if (zio->io_error != 0) {
4196fa94a07fSbrendan 			/*
41973a737e0dSbrendan 			 * Error - drop L2ARC entry.
4198fa94a07fSbrendan 			 */
41993a737e0dSbrendan 			list_remove(buflist, ab);
4200aad02571SSaso Kiselkov 			ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
4201fa94a07fSbrendan 			ab->b_l2hdr = NULL;
42023a737e0dSbrendan 			kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
42033a737e0dSbrendan 			ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
4204fa94a07fSbrendan 		}
4205fa94a07fSbrendan 
4206fa94a07fSbrendan 		/*
4207fa94a07fSbrendan 		 * Allow ARC to begin reads to this L2ARC entry.
4208fa94a07fSbrendan 		 */
4209fa94a07fSbrendan 		ab->b_flags &= ~ARC_L2_WRITING;
4210fa94a07fSbrendan 
4211fa94a07fSbrendan 		mutex_exit(hash_lock);
4212fa94a07fSbrendan 	}
4213fa94a07fSbrendan 
4214fa94a07fSbrendan 	atomic_inc_64(&l2arc_writes_done);
4215fa94a07fSbrendan 	list_remove(buflist, head);
4216fa94a07fSbrendan 	kmem_cache_free(hdr_cache, head);
4217fa94a07fSbrendan 	mutex_exit(&l2arc_buflist_mtx);
4218fa94a07fSbrendan 
42193a737e0dSbrendan 	l2arc_do_free_on_write();
4220fa94a07fSbrendan 
4221fa94a07fSbrendan 	kmem_free(cb, sizeof (l2arc_write_callback_t));
4222fa94a07fSbrendan }
4223fa94a07fSbrendan 
4224fa94a07fSbrendan /*
4225fa94a07fSbrendan  * A read to a cache device completed.  Validate buffer contents before
4226fa94a07fSbrendan  * handing over to the regular ARC routines.
4227fa94a07fSbrendan  */
4228fa94a07fSbrendan static void
4229fa94a07fSbrendan l2arc_read_done(zio_t *zio)
4230fa94a07fSbrendan {
4231fa94a07fSbrendan 	l2arc_read_callback_t *cb;
4232fa94a07fSbrendan 	arc_buf_hdr_t *hdr;
4233fa94a07fSbrendan 	arc_buf_t *buf;
4234fa94a07fSbrendan 	kmutex_t *hash_lock;
42353a737e0dSbrendan 	int equal;
4236fa94a07fSbrendan 
4237e14bb325SJeff Bonwick 	ASSERT(zio->io_vd != NULL);
4238e14bb325SJeff Bonwick 	ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
4239e14bb325SJeff Bonwick 
4240e14bb325SJeff Bonwick 	spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
4241e14bb325SJeff Bonwick 
4242fa94a07fSbrendan 	cb = zio->io_private;
4243fa94a07fSbrendan 	ASSERT(cb != NULL);
4244fa94a07fSbrendan 	buf = cb->l2rcb_buf;
4245fa94a07fSbrendan 	ASSERT(buf != NULL);
4246fa94a07fSbrendan 
42473f9d6ad7SLin Ling 	hash_lock = HDR_LOCK(buf->b_hdr);
4248fa94a07fSbrendan 	mutex_enter(hash_lock);
42493f9d6ad7SLin Ling 	hdr = buf->b_hdr;
42503f9d6ad7SLin Ling 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
4251fa94a07fSbrendan 
4252aad02571SSaso Kiselkov 	/*
4253aad02571SSaso Kiselkov 	 * If the buffer was compressed, decompress it first.
4254aad02571SSaso Kiselkov 	 */
4255aad02571SSaso Kiselkov 	if (cb->l2rcb_compress != ZIO_COMPRESS_OFF)
4256aad02571SSaso Kiselkov 		l2arc_decompress_zio(zio, hdr, cb->l2rcb_compress);
4257aad02571SSaso Kiselkov 	ASSERT(zio->io_data != NULL);
4258aad02571SSaso Kiselkov 
4259fa94a07fSbrendan 	/*
4260fa94a07fSbrendan 	 * Check this survived the L2ARC journey.
4261fa94a07fSbrendan 	 */
4262fa94a07fSbrendan 	equal = arc_cksum_equal(buf);
4263fa94a07fSbrendan 	if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) {
4264fa94a07fSbrendan 		mutex_exit(hash_lock);
4265fa94a07fSbrendan 		zio->io_private = buf;
4266e14bb325SJeff Bonwick 		zio->io_bp_copy = cb->l2rcb_bp;	/* XXX fix in L2ARC 2.0	*/
4267e14bb325SJeff Bonwick 		zio->io_bp = &zio->io_bp_copy;	/* XXX fix in L2ARC 2.0	*/
4268fa94a07fSbrendan 		arc_read_done(zio);
4269fa94a07fSbrendan 	} else {
4270fa94a07fSbrendan 		mutex_exit(hash_lock);
4271fa94a07fSbrendan 		/*
4272fa94a07fSbrendan 		 * Buffer didn't survive caching.  Increment stats and
4273fa94a07fSbrendan 		 * reissue to the original storage device.
4274fa94a07fSbrendan 		 */
42753a737e0dSbrendan 		if (zio->io_error != 0) {
4276fa94a07fSbrendan 			ARCSTAT_BUMP(arcstat_l2_io_error);
42773a737e0dSbrendan 		} else {
4278be6fd75aSMatthew Ahrens 			zio->io_error = SET_ERROR(EIO);
42793a737e0dSbrendan 		}
4280fa94a07fSbrendan 		if (!equal)
4281fa94a07fSbrendan 			ARCSTAT_BUMP(arcstat_l2_cksum_bad);
4282fa94a07fSbrendan 
4283e14bb325SJeff Bonwick 		/*
4284e14bb325SJeff Bonwick 		 * If there's no waiter, issue an async i/o to the primary
4285e14bb325SJeff Bonwick 		 * storage now.  If there *is* a waiter, the caller must
4286e14bb325SJeff Bonwick 		 * issue the i/o in a context where it's OK to block.
4287e14bb325SJeff Bonwick 		 */
4288a3f829aeSBill Moore 		if (zio->io_waiter == NULL) {
4289a3f829aeSBill Moore 			zio_t *pio = zio_unique_parent(zio);
4290a3f829aeSBill Moore 
4291a3f829aeSBill Moore 			ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
4292a3f829aeSBill Moore 
4293a3f829aeSBill Moore 			zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp,
4294e14bb325SJeff Bonwick 			    buf->b_data, zio->io_size, arc_read_done, buf,
4295e14bb325SJeff Bonwick 			    zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb));
4296a3f829aeSBill Moore 		}
4297fa94a07fSbrendan 	}
4298fa94a07fSbrendan 
4299fa94a07fSbrendan 	kmem_free(cb, sizeof (l2arc_read_callback_t));
4300fa94a07fSbrendan }
4301fa94a07fSbrendan 
4302fa94a07fSbrendan /*
4303fa94a07fSbrendan  * This is the list priority from which the L2ARC will search for pages to
4304fa94a07fSbrendan  * cache.  This is used within loops (0..3) to cycle through lists in the
4305fa94a07fSbrendan  * desired order.  This order can have a significant effect on cache
4306fa94a07fSbrendan  * performance.
4307fa94a07fSbrendan  *
4308fa94a07fSbrendan  * Currently the metadata lists are hit first, MFU then MRU, followed by
4309fa94a07fSbrendan  * the data lists.  This function returns a locked list, and also returns
4310fa94a07fSbrendan  * the lock pointer.
4311fa94a07fSbrendan  */
4312fa94a07fSbrendan static list_t *
4313fa94a07fSbrendan l2arc_list_locked(int list_num, kmutex_t **lock)
4314fa94a07fSbrendan {
4315d5285caeSGeorge Wilson 	list_t *list = NULL;
4316fa94a07fSbrendan 
4317fa94a07fSbrendan 	ASSERT(list_num >= 0 && list_num <= 3);
4318fa94a07fSbrendan 
4319fa94a07fSbrendan 	switch (list_num) {
4320fa94a07fSbrendan 	case 0:
4321fa94a07fSbrendan 		list = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
4322fa94a07fSbrendan 		*lock = &arc_mfu->arcs_mtx;
4323fa94a07fSbrendan 		break;
4324fa94a07fSbrendan 	case 1:
4325fa94a07fSbrendan 		list = &arc_mru->arcs_list[ARC_BUFC_METADATA];
4326fa94a07fSbrendan 		*lock = &arc_mru->arcs_mtx;
4327fa94a07fSbrendan 		break;
4328fa94a07fSbrendan 	case 2:
4329fa94a07fSbrendan 		list = &arc_mfu->arcs_list[ARC_BUFC_DATA];
4330fa94a07fSbrendan 		*lock = &arc_mfu->arcs_mtx;
4331fa94a07fSbrendan 		break;
4332fa94a07fSbrendan 	case 3:
4333fa94a07fSbrendan 		list = &arc_mru->arcs_list[ARC_BUFC_DATA];
4334fa94a07fSbrendan 		*lock = &arc_mru->arcs_mtx;
4335fa94a07fSbrendan 		break;
4336fa94a07fSbrendan 	}
4337fa94a07fSbrendan 
4338fa94a07fSbrendan 	ASSERT(!(MUTEX_HELD(*lock)));
4339fa94a07fSbrendan 	mutex_enter(*lock);
4340fa94a07fSbrendan 	return (list);
4341fa94a07fSbrendan }
4342fa94a07fSbrendan 
4343fa94a07fSbrendan /*
4344fa94a07fSbrendan  * Evict buffers from the device write hand to the distance specified in
4345fa94a07fSbrendan  * bytes.  This distance may span populated buffers, it may span nothing.
4346fa94a07fSbrendan  * This is clearing a region on the L2ARC device ready for writing.
4347fa94a07fSbrendan  * If the 'all' boolean is set, every buffer is evicted.
4348fa94a07fSbrendan  */
4349fa94a07fSbrendan static void
4350fa94a07fSbrendan l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
4351fa94a07fSbrendan {
4352fa94a07fSbrendan 	list_t *buflist;
4353fa94a07fSbrendan 	l2arc_buf_hdr_t *abl2;
4354fa94a07fSbrendan 	arc_buf_hdr_t *ab, *ab_prev;
4355fa94a07fSbrendan 	kmutex_t *hash_lock;
4356fa94a07fSbrendan 	uint64_t taddr;
4357fa94a07fSbrendan 
4358fa94a07fSbrendan 	buflist = dev->l2ad_buflist;
4359fa94a07fSbrendan 
4360fa94a07fSbrendan 	if (buflist == NULL)
4361fa94a07fSbrendan 		return;
4362fa94a07fSbrendan 
4363fa94a07fSbrendan 	if (!all && dev->l2ad_first) {
4364fa94a07fSbrendan 		/*
4365fa94a07fSbrendan 		 * This is the first sweep through the device.  There is
4366fa94a07fSbrendan 		 * nothing to evict.
4367fa94a07fSbrendan 		 */
4368fa94a07fSbrendan 		return;
4369fa94a07fSbrendan 	}
4370fa94a07fSbrendan 
43713a737e0dSbrendan 	if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) {
4372fa94a07fSbrendan 		/*
4373fa94a07fSbrendan 		 * When nearing the end of the device, evict to the end
4374fa94a07fSbrendan 		 * before the device write hand jumps to the start.
4375fa94a07fSbrendan 		 */
4376fa94a07fSbrendan 		taddr = dev->l2ad_end;
4377fa94a07fSbrendan 	} else {
4378fa94a07fSbrendan 		taddr = dev->l2ad_hand + distance;
4379fa94a07fSbrendan 	}
4380fa94a07fSbrendan 	DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist,
4381fa94a07fSbrendan 	    uint64_t, taddr, boolean_t, all);
4382fa94a07fSbrendan 
4383fa94a07fSbrendan top:
4384fa94a07fSbrendan 	mutex_enter(&l2arc_buflist_mtx);
4385fa94a07fSbrendan 	for (ab = list_tail(buflist); ab; ab = ab_prev) {
4386fa94a07fSbrendan 		ab_prev = list_prev(buflist, ab);
4387fa94a07fSbrendan 
4388fa94a07fSbrendan 		hash_lock = HDR_LOCK(ab);
4389fa94a07fSbrendan 		if (!mutex_tryenter(hash_lock)) {
4390fa94a07fSbrendan 			/*
4391fa94a07fSbrendan 			 * Missed the hash lock.  Retry.
4392fa94a07fSbrendan 			 */
4393fa94a07fSbrendan 			ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
4394fa94a07fSbrendan 			mutex_exit(&l2arc_buflist_mtx);
4395fa94a07fSbrendan 			mutex_enter(hash_lock);
4396fa94a07fSbrendan 			mutex_exit(hash_lock);
4397fa94a07fSbrendan 			goto top;
4398fa94a07fSbrendan 		}
4399fa94a07fSbrendan 
4400fa94a07fSbrendan 		if (HDR_L2_WRITE_HEAD(ab)) {
4401fa94a07fSbrendan 			/*
4402fa94a07fSbrendan 			 * We hit a write head node.  Leave it for
4403fa94a07fSbrendan 			 * l2arc_write_done().
4404fa94a07fSbrendan 			 */
4405fa94a07fSbrendan 			list_remove(buflist, ab);
4406fa94a07fSbrendan 			mutex_exit(hash_lock);
4407fa94a07fSbrendan 			continue;
4408fa94a07fSbrendan 		}
4409fa94a07fSbrendan 
4410fa94a07fSbrendan 		if (!all && ab->b_l2hdr != NULL &&
4411fa94a07fSbrendan 		    (ab->b_l2hdr->b_daddr > taddr ||
4412fa94a07fSbrendan 		    ab->b_l2hdr->b_daddr < dev->l2ad_hand)) {
4413fa94a07fSbrendan 			/*
4414fa94a07fSbrendan 			 * We've evicted to the target address,
4415fa94a07fSbrendan 			 * or the end of the device.
4416fa94a07fSbrendan 			 */
4417fa94a07fSbrendan 			mutex_exit(hash_lock);
4418fa94a07fSbrendan 			break;
4419fa94a07fSbrendan 		}
4420fa94a07fSbrendan 
4421fa94a07fSbrendan 		if (HDR_FREE_IN_PROGRESS(ab)) {
4422fa94a07fSbrendan 			/*
4423fa94a07fSbrendan 			 * Already on the path to destruction.
4424fa94a07fSbrendan 			 */
4425fa94a07fSbrendan 			mutex_exit(hash_lock);
4426fa94a07fSbrendan 			continue;
4427fa94a07fSbrendan 		}
4428fa94a07fSbrendan 
4429fa94a07fSbrendan 		if (ab->b_state == arc_l2c_only) {
4430fa94a07fSbrendan 			ASSERT(!HDR_L2_READING(ab));
4431fa94a07fSbrendan 			/*
4432fa94a07fSbrendan 			 * This doesn't exist in the ARC.  Destroy.
4433fa94a07fSbrendan 			 * arc_hdr_destroy() will call list_remove()
4434fa94a07fSbrendan 			 * and decrement arcstat_l2_size.
4435fa94a07fSbrendan 			 */
4436fa94a07fSbrendan 			arc_change_state(arc_anon, ab, hash_lock);
4437fa94a07fSbrendan 			arc_hdr_destroy(ab);
4438fa94a07fSbrendan 		} else {
44393a737e0dSbrendan 			/*
44403a737e0dSbrendan 			 * Invalidate issued or about to be issued
44413a737e0dSbrendan 			 * reads, since we may be about to write
44423a737e0dSbrendan 			 * over this location.
44433a737e0dSbrendan 			 */
44443a737e0dSbrendan 			if (HDR_L2_READING(ab)) {
44453a737e0dSbrendan 				ARCSTAT_BUMP(arcstat_l2_evict_reading);
44463a737e0dSbrendan 				ab->b_flags |= ARC_L2_EVICTED;
44473a737e0dSbrendan 			}
44483a737e0dSbrendan 
4449fa94a07fSbrendan 			/*
4450fa94a07fSbrendan 			 * Tell ARC this no longer exists in L2ARC.
4451fa94a07fSbrendan 			 */
4452fa94a07fSbrendan 			if (ab->b_l2hdr != NULL) {
4453fa94a07fSbrendan 				abl2 = ab->b_l2hdr;
4454aad02571SSaso Kiselkov 				ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
4455fa94a07fSbrendan 				ab->b_l2hdr = NULL;
4456fa94a07fSbrendan 				kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
4457fa94a07fSbrendan 				ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
4458fa94a07fSbrendan 			}
4459fa94a07fSbrendan 			list_remove(buflist, ab);
4460fa94a07fSbrendan 
4461fa94a07fSbrendan 			/*
4462fa94a07fSbrendan 			 * This may have been leftover after a
4463fa94a07fSbrendan 			 * failed write.
4464fa94a07fSbrendan 			 */
4465fa94a07fSbrendan 			ab->b_flags &= ~ARC_L2_WRITING;
4466fa94a07fSbrendan 		}
4467fa94a07fSbrendan 		mutex_exit(hash_lock);
4468fa94a07fSbrendan 	}
4469fa94a07fSbrendan 	mutex_exit(&l2arc_buflist_mtx);
4470fa94a07fSbrendan 
4471b24ab676SJeff Bonwick 	vdev_space_update(dev->l2ad_vdev, -(taddr - dev->l2ad_evict), 0, 0);
4472fa94a07fSbrendan 	dev->l2ad_evict = taddr;
4473fa94a07fSbrendan }
4474fa94a07fSbrendan 
4475fa94a07fSbrendan /*
4476fa94a07fSbrendan  * Find and write ARC buffers to the L2ARC device.
4477fa94a07fSbrendan  *
4478fa94a07fSbrendan  * An ARC_L2_WRITING flag is set so that the L2ARC buffers are not valid
4479fa94a07fSbrendan  * for reading until they have completed writing.
4480aad02571SSaso Kiselkov  * The headroom_boost is an in-out parameter used to maintain headroom boost
4481aad02571SSaso Kiselkov  * state between calls to this function.
4482aad02571SSaso Kiselkov  *
4483aad02571SSaso Kiselkov  * Returns the number of bytes actually written (which may be smaller than
4484aad02571SSaso Kiselkov  * the delta by which the device hand has changed due to alignment).
4485fa94a07fSbrendan  */
44865a98e54bSBrendan Gregg - Sun Microsystems static uint64_t
4487aad02571SSaso Kiselkov l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
4488aad02571SSaso Kiselkov     boolean_t *headroom_boost)
4489fa94a07fSbrendan {
4490fa94a07fSbrendan 	arc_buf_hdr_t *ab, *ab_prev, *head;
4491fa94a07fSbrendan 	list_t *list;
4492aad02571SSaso Kiselkov 	uint64_t write_asize, write_psize, write_sz, headroom,
4493aad02571SSaso Kiselkov 	    buf_compress_minsz;
4494fa94a07fSbrendan 	void *buf_data;
4495aad02571SSaso Kiselkov 	kmutex_t *list_lock;
4496aad02571SSaso Kiselkov 	boolean_t full;
4497fa94a07fSbrendan 	l2arc_write_callback_t *cb;
4498fa94a07fSbrendan 	zio_t *pio, *wzio;
4499e9103aaeSGarrett D'Amore 	uint64_t guid = spa_load_guid(spa);
4500aad02571SSaso Kiselkov 	const boolean_t do_headroom_boost = *headroom_boost;
4501fa94a07fSbrendan 
4502fa94a07fSbrendan 	ASSERT(dev->l2ad_vdev != NULL);
4503fa94a07fSbrendan 
4504aad02571SSaso Kiselkov 	/* Lower the flag now, we might want to raise it again later. */
4505aad02571SSaso Kiselkov 	*headroom_boost = B_FALSE;
4506aad02571SSaso Kiselkov 
4507fa94a07fSbrendan 	pio = NULL;
4508aad02571SSaso Kiselkov 	write_sz = write_asize = write_psize = 0;
4509fa94a07fSbrendan 	full = B_FALSE;
45101ab7f2deSmaybee 	head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
4511fa94a07fSbrendan 	head->b_flags |= ARC_L2_WRITE_HEAD;
4512fa94a07fSbrendan 
4513aad02571SSaso Kiselkov 	/*
4514aad02571SSaso Kiselkov 	 * We will want to try to compress buffers that are at least 2x the
4515aad02571SSaso Kiselkov 	 * device sector size.
4516aad02571SSaso Kiselkov 	 */
4517aad02571SSaso Kiselkov 	buf_compress_minsz = 2 << dev->l2ad_vdev->vdev_ashift;
4518aad02571SSaso Kiselkov 
4519fa94a07fSbrendan 	/*
4520fa94a07fSbrendan 	 * Copy buffers for L2ARC writing.
4521fa94a07fSbrendan 	 */
4522fa94a07fSbrendan 	mutex_enter(&l2arc_buflist_mtx);
4523fa94a07fSbrendan 	for (int try = 0; try <= 3; try++) {
4524aad02571SSaso Kiselkov 		uint64_t passed_sz = 0;
4525aad02571SSaso Kiselkov 
4526fa94a07fSbrendan 		list = l2arc_list_locked(try, &list_lock);
4527fa94a07fSbrendan 
45283a737e0dSbrendan 		/*
45293a737e0dSbrendan 		 * L2ARC fast warmup.
45303a737e0dSbrendan 		 *
45313a737e0dSbrendan 		 * Until the ARC is warm and starts to evict, read from the
45323a737e0dSbrendan 		 * head of the ARC lists rather than the tail.
45333a737e0dSbrendan 		 */
45343a737e0dSbrendan 		if (arc_warm == B_FALSE)
45353a737e0dSbrendan 			ab = list_head(list);
45363a737e0dSbrendan 		else
45373a737e0dSbrendan 			ab = list_tail(list);
45383a737e0dSbrendan 
4539aad02571SSaso Kiselkov 		headroom = target_sz * l2arc_headroom;
4540aad02571SSaso Kiselkov 		if (do_headroom_boost)
4541aad02571SSaso Kiselkov 			headroom = (headroom * l2arc_headroom_boost) / 100;
4542aad02571SSaso Kiselkov 
45433a737e0dSbrendan 		for (; ab; ab = ab_prev) {
4544aad02571SSaso Kiselkov 			l2arc_buf_hdr_t *l2hdr;
4545aad02571SSaso Kiselkov 			kmutex_t *hash_lock;
4546aad02571SSaso Kiselkov 			uint64_t buf_sz;
4547aad02571SSaso Kiselkov 
45483a737e0dSbrendan 			if (arc_warm == B_FALSE)
45493a737e0dSbrendan 				ab_prev = list_next(list, ab);
45503a737e0dSbrendan 			else
45513a737e0dSbrendan 				ab_prev = list_prev(list, ab);
4552fa94a07fSbrendan 
4553fa94a07fSbrendan 			hash_lock = HDR_LOCK(ab);
4554aad02571SSaso Kiselkov 			if (!mutex_tryenter(hash_lock)) {
4555fa94a07fSbrendan 				/*
4556fa94a07fSbrendan 				 * Skip this buffer rather than waiting.
4557fa94a07fSbrendan 				 */
4558fa94a07fSbrendan 				continue;
4559fa94a07fSbrendan 			}
4560fa94a07fSbrendan 
4561fa94a07fSbrendan 			passed_sz += ab->b_size;
4562fa94a07fSbrendan 			if (passed_sz > headroom) {
4563fa94a07fSbrendan 				/*
4564fa94a07fSbrendan 				 * Searched too far.
4565fa94a07fSbrendan 				 */
4566fa94a07fSbrendan 				mutex_exit(hash_lock);
4567fa94a07fSbrendan 				break;
4568fa94a07fSbrendan 			}
4569fa94a07fSbrendan 
4570ac05c741SMark Maybee 			if (!l2arc_write_eligible(guid, ab)) {
4571fa94a07fSbrendan 				mutex_exit(hash_lock);
4572fa94a07fSbrendan 				continue;
4573fa94a07fSbrendan 			}
4574fa94a07fSbrendan 
4575fa94a07fSbrendan 			if ((write_sz + ab->b_size) > target_sz) {
4576fa94a07fSbrendan 				full = B_TRUE;
4577fa94a07fSbrendan 				mutex_exit(hash_lock);
4578fa94a07fSbrendan 				break;
4579fa94a07fSbrendan 			}
4580fa94a07fSbrendan 
4581fa94a07fSbrendan 			if (pio == NULL) {
4582fa94a07fSbrendan 				/*
4583fa94a07fSbrendan 				 * Insert a dummy header on the buflist so
4584fa94a07fSbrendan 				 * l2arc_write_done() can find where the
4585fa94a07fSbrendan 				 * write buffers begin without searching.
4586fa94a07fSbrendan 				 */
4587fa94a07fSbrendan 				list_insert_head(dev->l2ad_buflist, head);
4588fa94a07fSbrendan 
4589fa94a07fSbrendan 				cb = kmem_alloc(
4590fa94a07fSbrendan 				    sizeof (l2arc_write_callback_t), KM_SLEEP);
4591fa94a07fSbrendan 				cb->l2wcb_dev = dev;
4592fa94a07fSbrendan 				cb->l2wcb_head = head;
4593fa94a07fSbrendan 				pio = zio_root(spa, l2arc_write_done, cb,
4594fa94a07fSbrendan 				    ZIO_FLAG_CANFAIL);
4595fa94a07fSbrendan 			}
4596fa94a07fSbrendan 
4597fa94a07fSbrendan 			/*
4598fa94a07fSbrendan 			 * Create and add a new L2ARC header.
4599fa94a07fSbrendan 			 */
4600aad02571SSaso Kiselkov 			l2hdr = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP);
4601aad02571SSaso Kiselkov 			l2hdr->b_dev = dev;
4602fa94a07fSbrendan 			ab->b_flags |= ARC_L2_WRITING;
4603aad02571SSaso Kiselkov 
4604aad02571SSaso Kiselkov 			/*
4605aad02571SSaso Kiselkov 			 * Temporarily stash the data buffer in b_tmp_cdata.
4606aad02571SSaso Kiselkov 			 * The subsequent write step will pick it up from
4607aad02571SSaso Kiselkov 			 * there. This is because can't access ab->b_buf
4608aad02571SSaso Kiselkov 			 * without holding the hash_lock, which we in turn
4609aad02571SSaso Kiselkov 			 * can't access without holding the ARC list locks
4610aad02571SSaso Kiselkov 			 * (which we want to avoid during compression/writing).
4611aad02571SSaso Kiselkov 			 */
4612aad02571SSaso Kiselkov 			l2hdr->b_compress = ZIO_COMPRESS_OFF;
4613aad02571SSaso Kiselkov 			l2hdr->b_asize = ab->b_size;
4614aad02571SSaso Kiselkov 			l2hdr->b_tmp_cdata = ab->b_buf->b_data;
4615aad02571SSaso Kiselkov 
4616fa94a07fSbrendan 			buf_sz = ab->b_size;
4617aad02571SSaso Kiselkov 			ab->b_l2hdr = l2hdr;
4618aad02571SSaso Kiselkov 
4619aad02571SSaso Kiselkov 			list_insert_head(dev->l2ad_buflist, ab);
4620fa94a07fSbrendan 
4621fa94a07fSbrendan 			/*
4622fa94a07fSbrendan 			 * Compute and store the buffer cksum before
4623fa94a07fSbrendan 			 * writing.  On debug the cksum is verified first.
4624fa94a07fSbrendan 			 */
4625fa94a07fSbrendan 			arc_cksum_verify(ab->b_buf);
4626fa94a07fSbrendan 			arc_cksum_compute(ab->b_buf, B_TRUE);
4627fa94a07fSbrendan 
4628fa94a07fSbrendan 			mutex_exit(hash_lock);
4629fa94a07fSbrendan 
4630aad02571SSaso Kiselkov 			write_sz += buf_sz;
4631aad02571SSaso Kiselkov 		}
4632aad02571SSaso Kiselkov 
4633aad02571SSaso Kiselkov 		mutex_exit(list_lock);
4634aad02571SSaso Kiselkov 
4635aad02571SSaso Kiselkov 		if (full == B_TRUE)
4636aad02571SSaso Kiselkov 			break;
4637aad02571SSaso Kiselkov 	}
4638aad02571SSaso Kiselkov 
4639aad02571SSaso Kiselkov 	/* No buffers selected for writing? */
4640aad02571SSaso Kiselkov 	if (pio == NULL) {
4641aad02571SSaso Kiselkov 		ASSERT0(write_sz);
4642aad02571SSaso Kiselkov 		mutex_exit(&l2arc_buflist_mtx);
4643aad02571SSaso Kiselkov 		kmem_cache_free(hdr_cache, head);
4644aad02571SSaso Kiselkov 		return (0);
4645aad02571SSaso Kiselkov 	}
4646aad02571SSaso Kiselkov 
4647aad02571SSaso Kiselkov 	/*
4648aad02571SSaso Kiselkov 	 * Now start writing the buffers. We're starting at the write head
4649aad02571SSaso Kiselkov 	 * and work backwards, retracing the course of the buffer selector
4650aad02571SSaso Kiselkov 	 * loop above.
4651aad02571SSaso Kiselkov 	 */
4652aad02571SSaso Kiselkov 	for (ab = list_prev(dev->l2ad_buflist, head); ab;
4653aad02571SSaso Kiselkov 	    ab = list_prev(dev->l2ad_buflist, ab)) {
4654aad02571SSaso Kiselkov 		l2arc_buf_hdr_t *l2hdr;
4655aad02571SSaso Kiselkov 		uint64_t buf_sz;
4656aad02571SSaso Kiselkov 
4657aad02571SSaso Kiselkov 		/*
4658aad02571SSaso Kiselkov 		 * We shouldn't need to lock the buffer here, since we flagged
4659aad02571SSaso Kiselkov 		 * it as ARC_L2_WRITING in the previous step, but we must take
4660aad02571SSaso Kiselkov 		 * care to only access its L2 cache parameters. In particular,
4661aad02571SSaso Kiselkov 		 * ab->b_buf may be invalid by now due to ARC eviction.
4662aad02571SSaso Kiselkov 		 */
4663aad02571SSaso Kiselkov 		l2hdr = ab->b_l2hdr;
4664aad02571SSaso Kiselkov 		l2hdr->b_daddr = dev->l2ad_hand;
4665aad02571SSaso Kiselkov 
4666aad02571SSaso Kiselkov 		if ((ab->b_flags & ARC_L2COMPRESS) &&
4667aad02571SSaso Kiselkov 		    l2hdr->b_asize >= buf_compress_minsz) {
4668aad02571SSaso Kiselkov 			if (l2arc_compress_buf(l2hdr)) {
4669aad02571SSaso Kiselkov 				/*
4670aad02571SSaso Kiselkov 				 * If compression succeeded, enable headroom
4671aad02571SSaso Kiselkov 				 * boost on the next scan cycle.
4672aad02571SSaso Kiselkov 				 */
4673aad02571SSaso Kiselkov 				*headroom_boost = B_TRUE;
4674aad02571SSaso Kiselkov 			}
4675aad02571SSaso Kiselkov 		}
4676aad02571SSaso Kiselkov 
4677aad02571SSaso Kiselkov 		/*
4678aad02571SSaso Kiselkov 		 * Pick up the buffer data we had previously stashed away
4679aad02571SSaso Kiselkov 		 * (and now potentially also compressed).
4680aad02571SSaso Kiselkov 		 */
4681aad02571SSaso Kiselkov 		buf_data = l2hdr->b_tmp_cdata;
4682aad02571SSaso Kiselkov 		buf_sz = l2hdr->b_asize;
4683aad02571SSaso Kiselkov 
4684aad02571SSaso Kiselkov 		/* Compression may have squashed the buffer to zero length. */
4685aad02571SSaso Kiselkov 		if (buf_sz != 0) {
4686aad02571SSaso Kiselkov 			uint64_t buf_p_sz;
4687aad02571SSaso Kiselkov 
4688fa94a07fSbrendan 			wzio = zio_write_phys(pio, dev->l2ad_vdev,
4689fa94a07fSbrendan 			    dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF,
4690fa94a07fSbrendan 			    NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE,
4691fa94a07fSbrendan 			    ZIO_FLAG_CANFAIL, B_FALSE);
4692fa94a07fSbrendan 
4693fa94a07fSbrendan 			DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
4694fa94a07fSbrendan 			    zio_t *, wzio);
4695fa94a07fSbrendan 			(void) zio_nowait(wzio);
4696fa94a07fSbrendan 
4697aad02571SSaso Kiselkov 			write_asize += buf_sz;
4698e14bb325SJeff Bonwick 			/*
4699e14bb325SJeff Bonwick 			 * Keep the clock hand suitably device-aligned.
4700e14bb325SJeff Bonwick 			 */
4701aad02571SSaso Kiselkov 			buf_p_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
4702aad02571SSaso Kiselkov 			write_psize += buf_p_sz;
4703aad02571SSaso Kiselkov 			dev->l2ad_hand += buf_p_sz;
4704fa94a07fSbrendan 		}
4705fa94a07fSbrendan 	}
4706fa94a07fSbrendan 
4707aad02571SSaso Kiselkov 	mutex_exit(&l2arc_buflist_mtx);
4708fa94a07fSbrendan 
4709aad02571SSaso Kiselkov 	ASSERT3U(write_asize, <=, target_sz);
4710fa94a07fSbrendan 	ARCSTAT_BUMP(arcstat_l2_writes_sent);
4711aad02571SSaso Kiselkov 	ARCSTAT_INCR(arcstat_l2_write_bytes, write_asize);
4712fa94a07fSbrendan 	ARCSTAT_INCR(arcstat_l2_size, write_sz);
4713aad02571SSaso Kiselkov 	ARCSTAT_INCR(arcstat_l2_asize, write_asize);
4714aad02571SSaso Kiselkov 	vdev_space_update(dev->l2ad_vdev, write_psize, 0, 0);
4715fa94a07fSbrendan 
4716fa94a07fSbrendan 	/*
4717fa94a07fSbrendan 	 * Bump device hand to the device start if it is approaching the end.
4718fa94a07fSbrendan 	 * l2arc_evict() will already have evicted ahead for this case.
4719fa94a07fSbrendan 	 */
47203a737e0dSbrendan 	if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
4721b24ab676SJeff Bonwick 		vdev_space_update(dev->l2ad_vdev,
4722b24ab676SJeff Bonwick 		    dev->l2ad_end - dev->l2ad_hand, 0, 0);
4723fa94a07fSbrendan 		dev->l2ad_hand = dev->l2ad_start;
4724fa94a07fSbrendan 		dev->l2ad_evict = dev->l2ad_start;
4725fa94a07fSbrendan 		dev->l2ad_first = B_FALSE;
4726fa94a07fSbrendan 	}
4727fa94a07fSbrendan 
47285a98e54bSBrendan Gregg - Sun Microsystems 	dev->l2ad_writing = B_TRUE;
4729fa94a07fSbrendan 	(void) zio_wait(pio);
47305a98e54bSBrendan Gregg - Sun Microsystems 	dev->l2ad_writing = B_FALSE;
47315a98e54bSBrendan Gregg - Sun Microsystems 
4732aad02571SSaso Kiselkov 	return (write_asize);
4733aad02571SSaso Kiselkov }
4734aad02571SSaso Kiselkov 
4735aad02571SSaso Kiselkov /*
4736aad02571SSaso Kiselkov  * Compresses an L2ARC buffer.
4737aad02571SSaso Kiselkov  * The data to be compressed must be prefilled in l2hdr->b_tmp_cdata and its
4738aad02571SSaso Kiselkov  * size in l2hdr->b_asize. This routine tries to compress the data and
4739aad02571SSaso Kiselkov  * depending on the compression result there are three possible outcomes:
4740aad02571SSaso Kiselkov  * *) The buffer was incompressible. The original l2hdr contents were left
4741aad02571SSaso Kiselkov  *    untouched and are ready for writing to an L2 device.
4742aad02571SSaso Kiselkov  * *) The buffer was all-zeros, so there is no need to write it to an L2
4743aad02571SSaso Kiselkov  *    device. To indicate this situation b_tmp_cdata is NULL'ed, b_asize is
4744aad02571SSaso Kiselkov  *    set to zero and b_compress is set to ZIO_COMPRESS_EMPTY.
4745aad02571SSaso Kiselkov  * *) Compression succeeded and b_tmp_cdata was replaced with a temporary
4746aad02571SSaso Kiselkov  *    data buffer which holds the compressed data to be written, and b_asize
4747aad02571SSaso Kiselkov  *    tells us how much data there is. b_compress is set to the appropriate
4748aad02571SSaso Kiselkov  *    compression algorithm. Once writing is done, invoke
4749aad02571SSaso Kiselkov  *    l2arc_release_cdata_buf on this l2hdr to free this temporary buffer.
4750aad02571SSaso Kiselkov  *
4751aad02571SSaso Kiselkov  * Returns B_TRUE if compression succeeded, or B_FALSE if it didn't (the
4752aad02571SSaso Kiselkov  * buffer was incompressible).
4753aad02571SSaso Kiselkov  */
4754aad02571SSaso Kiselkov static boolean_t
4755aad02571SSaso Kiselkov l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr)
4756aad02571SSaso Kiselkov {
4757aad02571SSaso Kiselkov 	void *cdata;
4758aad02571SSaso Kiselkov 	size_t csize, len;
4759aad02571SSaso Kiselkov 
4760aad02571SSaso Kiselkov 	ASSERT(l2hdr->b_compress == ZIO_COMPRESS_OFF);
4761aad02571SSaso Kiselkov 	ASSERT(l2hdr->b_tmp_cdata != NULL);
4762aad02571SSaso Kiselkov 
4763aad02571SSaso Kiselkov 	len = l2hdr->b_asize;
4764aad02571SSaso Kiselkov 	cdata = zio_data_buf_alloc(len);
4765aad02571SSaso Kiselkov 	csize = zio_compress_data(ZIO_COMPRESS_LZ4, l2hdr->b_tmp_cdata,
4766aad02571SSaso Kiselkov 	    cdata, l2hdr->b_asize);
4767aad02571SSaso Kiselkov 
4768aad02571SSaso Kiselkov 	if (csize == 0) {
4769aad02571SSaso Kiselkov 		/* zero block, indicate that there's nothing to write */
4770aad02571SSaso Kiselkov 		zio_data_buf_free(cdata, len);
4771aad02571SSaso Kiselkov 		l2hdr->b_compress = ZIO_COMPRESS_EMPTY;
4772aad02571SSaso Kiselkov 		l2hdr->b_asize = 0;
4773aad02571SSaso Kiselkov 		l2hdr->b_tmp_cdata = NULL;
4774aad02571SSaso Kiselkov 		ARCSTAT_BUMP(arcstat_l2_compress_zeros);
4775aad02571SSaso Kiselkov 		return (B_TRUE);
4776aad02571SSaso Kiselkov 	} else if (csize > 0 && csize < len) {
4777aad02571SSaso Kiselkov 		/*
4778aad02571SSaso Kiselkov 		 * Compression succeeded, we'll keep the cdata around for
4779aad02571SSaso Kiselkov 		 * writing and release it afterwards.
4780aad02571SSaso Kiselkov 		 */
4781aad02571SSaso Kiselkov 		l2hdr->b_compress = ZIO_COMPRESS_LZ4;
4782aad02571SSaso Kiselkov 		l2hdr->b_asize = csize;
4783aad02571SSaso Kiselkov 		l2hdr->b_tmp_cdata = cdata;
4784aad02571SSaso Kiselkov 		ARCSTAT_BUMP(arcstat_l2_compress_successes);
4785aad02571SSaso Kiselkov 		return (B_TRUE);
4786aad02571SSaso Kiselkov 	} else {
4787aad02571SSaso Kiselkov 		/*
4788aad02571SSaso Kiselkov 		 * Compression failed, release the compressed buffer.
4789aad02571SSaso Kiselkov 		 * l2hdr will be left unmodified.
4790aad02571SSaso Kiselkov 		 */
4791aad02571SSaso Kiselkov 		zio_data_buf_free(cdata, len);
4792aad02571SSaso Kiselkov 		ARCSTAT_BUMP(arcstat_l2_compress_failures);
4793aad02571SSaso Kiselkov 		return (B_FALSE);
4794aad02571SSaso Kiselkov 	}
4795aad02571SSaso Kiselkov }
4796aad02571SSaso Kiselkov 
4797aad02571SSaso Kiselkov /*
4798aad02571SSaso Kiselkov  * Decompresses a zio read back from an l2arc device. On success, the
4799aad02571SSaso Kiselkov  * underlying zio's io_data buffer is overwritten by the uncompressed
4800aad02571SSaso Kiselkov  * version. On decompression error (corrupt compressed stream), the
4801aad02571SSaso Kiselkov  * zio->io_error value is set to signal an I/O error.
4802aad02571SSaso Kiselkov  *
4803aad02571SSaso Kiselkov  * Please note that the compressed data stream is not checksummed, so
4804aad02571SSaso Kiselkov  * if the underlying device is experiencing data corruption, we may feed
4805aad02571SSaso Kiselkov  * corrupt data to the decompressor, so the decompressor needs to be
4806aad02571SSaso Kiselkov  * able to handle this situation (LZ4 does).
4807aad02571SSaso Kiselkov  */
4808aad02571SSaso Kiselkov static void
4809aad02571SSaso Kiselkov l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
4810aad02571SSaso Kiselkov {
4811aad02571SSaso Kiselkov 	ASSERT(L2ARC_IS_VALID_COMPRESS(c));
4812aad02571SSaso Kiselkov 
4813aad02571SSaso Kiselkov 	if (zio->io_error != 0) {
4814aad02571SSaso Kiselkov 		/*
4815aad02571SSaso Kiselkov 		 * An io error has occured, just restore the original io
4816aad02571SSaso Kiselkov 		 * size in preparation for a main pool read.
4817aad02571SSaso Kiselkov 		 */
4818aad02571SSaso Kiselkov 		zio->io_orig_size = zio->io_size = hdr->b_size;
4819aad02571SSaso Kiselkov 		return;
4820aad02571SSaso Kiselkov 	}
4821aad02571SSaso Kiselkov 
4822aad02571SSaso Kiselkov 	if (c == ZIO_COMPRESS_EMPTY) {
4823aad02571SSaso Kiselkov 		/*
4824aad02571SSaso Kiselkov 		 * An empty buffer results in a null zio, which means we
4825aad02571SSaso Kiselkov 		 * need to fill its io_data after we're done restoring the
4826aad02571SSaso Kiselkov 		 * buffer's contents.
4827aad02571SSaso Kiselkov 		 */
4828aad02571SSaso Kiselkov 		ASSERT(hdr->b_buf != NULL);
4829aad02571SSaso Kiselkov 		bzero(hdr->b_buf->b_data, hdr->b_size);
4830aad02571SSaso Kiselkov 		zio->io_data = zio->io_orig_data = hdr->b_buf->b_data;
4831aad02571SSaso Kiselkov 	} else {
4832aad02571SSaso Kiselkov 		ASSERT(zio->io_data != NULL);
4833aad02571SSaso Kiselkov 		/*
4834aad02571SSaso Kiselkov 		 * We copy the compressed data from the start of the arc buffer
4835aad02571SSaso Kiselkov 		 * (the zio_read will have pulled in only what we need, the
4836aad02571SSaso Kiselkov 		 * rest is garbage which we will overwrite at decompression)
4837aad02571SSaso Kiselkov 		 * and then decompress back to the ARC data buffer. This way we
4838aad02571SSaso Kiselkov 		 * can minimize copying by simply decompressing back over the
4839aad02571SSaso Kiselkov 		 * original compressed data (rather than decompressing to an
4840aad02571SSaso Kiselkov 		 * aux buffer and then copying back the uncompressed buffer,
4841aad02571SSaso Kiselkov 		 * which is likely to be much larger).
4842aad02571SSaso Kiselkov 		 */
4843aad02571SSaso Kiselkov 		uint64_t csize;
4844aad02571SSaso Kiselkov 		void *cdata;
4845aad02571SSaso Kiselkov 
4846aad02571SSaso Kiselkov 		csize = zio->io_size;
4847aad02571SSaso Kiselkov 		cdata = zio_data_buf_alloc(csize);
4848aad02571SSaso Kiselkov 		bcopy(zio->io_data, cdata, csize);
4849aad02571SSaso Kiselkov 		if (zio_decompress_data(c, cdata, zio->io_data, csize,
4850aad02571SSaso Kiselkov 		    hdr->b_size) != 0)
4851aad02571SSaso Kiselkov 			zio->io_error = EIO;
4852aad02571SSaso Kiselkov 		zio_data_buf_free(cdata, csize);
4853aad02571SSaso Kiselkov 	}
4854aad02571SSaso Kiselkov 
4855aad02571SSaso Kiselkov 	/* Restore the expected uncompressed IO size. */
4856aad02571SSaso Kiselkov 	zio->io_orig_size = zio->io_size = hdr->b_size;
4857aad02571SSaso Kiselkov }
4858aad02571SSaso Kiselkov 
4859aad02571SSaso Kiselkov /*
4860aad02571SSaso Kiselkov  * Releases the temporary b_tmp_cdata buffer in an l2arc header structure.
4861aad02571SSaso Kiselkov  * This buffer serves as a temporary holder of compressed data while
4862aad02571SSaso Kiselkov  * the buffer entry is being written to an l2arc device. Once that is
4863aad02571SSaso Kiselkov  * done, we can dispose of it.
4864aad02571SSaso Kiselkov  */
4865aad02571SSaso Kiselkov static void
4866aad02571SSaso Kiselkov l2arc_release_cdata_buf(arc_buf_hdr_t *ab)
4867aad02571SSaso Kiselkov {
4868aad02571SSaso Kiselkov 	l2arc_buf_hdr_t *l2hdr = ab->b_l2hdr;
4869aad02571SSaso Kiselkov 
4870aad02571SSaso Kiselkov 	if (l2hdr->b_compress == ZIO_COMPRESS_LZ4) {
4871aad02571SSaso Kiselkov 		/*
4872aad02571SSaso Kiselkov 		 * If the data was compressed, then we've allocated a
4873aad02571SSaso Kiselkov 		 * temporary buffer for it, so now we need to release it.
4874aad02571SSaso Kiselkov 		 */
4875aad02571SSaso Kiselkov 		ASSERT(l2hdr->b_tmp_cdata != NULL);
4876aad02571SSaso Kiselkov 		zio_data_buf_free(l2hdr->b_tmp_cdata, ab->b_size);
4877aad02571SSaso Kiselkov 	}
4878aad02571SSaso Kiselkov 	l2hdr->b_tmp_cdata = NULL;
4879fa94a07fSbrendan }
4880fa94a07fSbrendan 
4881fa94a07fSbrendan /*
4882fa94a07fSbrendan  * This thread feeds the L2ARC at regular intervals.  This is the beating
4883fa94a07fSbrendan  * heart of the L2ARC.
4884fa94a07fSbrendan  */
4885fa94a07fSbrendan static void
4886fa94a07fSbrendan l2arc_feed_thread(void)
4887fa94a07fSbrendan {
4888fa94a07fSbrendan 	callb_cpr_t cpr;
4889fa94a07fSbrendan 	l2arc_dev_t *dev;
4890fa94a07fSbrendan 	spa_t *spa;
48915a98e54bSBrendan Gregg - Sun Microsystems 	uint64_t size, wrote;
4892d3d50737SRafael Vanoni 	clock_t begin, next = ddi_get_lbolt();
4893aad02571SSaso Kiselkov 	boolean_t headroom_boost = B_FALSE;
4894fa94a07fSbrendan 
4895fa94a07fSbrendan 	CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
4896fa94a07fSbrendan 
4897fa94a07fSbrendan 	mutex_enter(&l2arc_feed_thr_lock);
4898fa94a07fSbrendan 
4899fa94a07fSbrendan 	while (l2arc_thread_exit == 0) {
4900fa94a07fSbrendan 		CALLB_CPR_SAFE_BEGIN(&cpr);
4901fa94a07fSbrendan 		(void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock,
49025a98e54bSBrendan Gregg - Sun Microsystems 		    next);
4903fa94a07fSbrendan 		CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
4904d3d50737SRafael Vanoni 		next = ddi_get_lbolt() + hz;
4905fa94a07fSbrendan 
49063a737e0dSbrendan 		/*
49073a737e0dSbrendan 		 * Quick check for L2ARC devices.
49083a737e0dSbrendan 		 */
4909c5904d13Seschrock 		mutex_enter(&l2arc_dev_mtx);
49103a737e0dSbrendan 		if (l2arc_ndev == 0) {
49113a737e0dSbrendan 			mutex_exit(&l2arc_dev_mtx);
49123a737e0dSbrendan 			continue;
49133a737e0dSbrendan 		}
49143a737e0dSbrendan 		mutex_exit(&l2arc_dev_mtx);
4915d3d50737SRafael Vanoni 		begin = ddi_get_lbolt();
4916c5904d13Seschrock 
4917fa94a07fSbrendan 		/*
4918c5904d13Seschrock 		 * This selects the next l2arc device to write to, and in
4919c5904d13Seschrock 		 * doing so the next spa to feed from: dev->l2ad_spa.   This
49203a737e0dSbrendan 		 * will return NULL if there are now no l2arc devices or if
49213a737e0dSbrendan 		 * they are all faulted.
49223a737e0dSbrendan 		 *
49233a737e0dSbrendan 		 * If a device is returned, its spa's config lock is also
49243a737e0dSbrendan 		 * held to prevent device removal.  l2arc_dev_get_next()
49253a737e0dSbrendan 		 * will grab and release l2arc_dev_mtx.
4926fa94a07fSbrendan 		 */
49273a737e0dSbrendan 		if ((dev = l2arc_dev_get_next()) == NULL)
4928fa94a07fSbrendan 			continue;
49293a737e0dSbrendan 
49303a737e0dSbrendan 		spa = dev->l2ad_spa;
49313a737e0dSbrendan 		ASSERT(spa != NULL);
4932fa94a07fSbrendan 
4933f9af39baSGeorge Wilson 		/*
4934f9af39baSGeorge Wilson 		 * If the pool is read-only then force the feed thread to
4935f9af39baSGeorge Wilson 		 * sleep a little longer.
4936f9af39baSGeorge Wilson 		 */
4937f9af39baSGeorge Wilson 		if (!spa_writeable(spa)) {
4938f9af39baSGeorge Wilson 			next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz;
4939f9af39baSGeorge Wilson 			spa_config_exit(spa, SCL_L2ARC, dev);
4940f9af39baSGeorge Wilson 			continue;
4941f9af39baSGeorge Wilson 		}
4942f9af39baSGeorge Wilson 
4943fa94a07fSbrendan 		/*
4944fa94a07fSbrendan 		 * Avoid contributing to memory pressure.
4945fa94a07fSbrendan 		 */
4946fa94a07fSbrendan 		if (arc_reclaim_needed()) {
4947fa94a07fSbrendan 			ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
4948e14bb325SJeff Bonwick 			spa_config_exit(spa, SCL_L2ARC, dev);
4949fa94a07fSbrendan 			continue;
4950fa94a07fSbrendan 		}
4951fa94a07fSbrendan 
4952fa94a07fSbrendan 		ARCSTAT_BUMP(arcstat_l2_feeds);
4953fa94a07fSbrendan 
4954aad02571SSaso Kiselkov 		size = l2arc_write_size();
49553a737e0dSbrendan 
4956fa94a07fSbrendan 		/*
4957fa94a07fSbrendan 		 * Evict L2ARC buffers that will be overwritten.
4958fa94a07fSbrendan 		 */
49593a737e0dSbrendan 		l2arc_evict(dev, size, B_FALSE);
4960fa94a07fSbrendan 
4961fa94a07fSbrendan 		/*
4962fa94a07fSbrendan 		 * Write ARC buffers.
4963fa94a07fSbrendan 		 */
4964aad02571SSaso Kiselkov 		wrote = l2arc_write_buffers(spa, dev, size, &headroom_boost);
49655a98e54bSBrendan Gregg - Sun Microsystems 
49665a98e54bSBrendan Gregg - Sun Microsystems 		/*
49675a98e54bSBrendan Gregg - Sun Microsystems 		 * Calculate interval between writes.
49685a98e54bSBrendan Gregg - Sun Microsystems 		 */
49695a98e54bSBrendan Gregg - Sun Microsystems 		next = l2arc_write_interval(begin, size, wrote);
4970e14bb325SJeff Bonwick 		spa_config_exit(spa, SCL_L2ARC, dev);
4971fa94a07fSbrendan 	}
4972fa94a07fSbrendan 
4973fa94a07fSbrendan 	l2arc_thread_exit = 0;
4974fa94a07fSbrendan 	cv_broadcast(&l2arc_feed_thr_cv);
4975fa94a07fSbrendan 	CALLB_CPR_EXIT(&cpr);		/* drops l2arc_feed_thr_lock */
4976fa94a07fSbrendan 	thread_exit();
4977fa94a07fSbrendan }
4978fa94a07fSbrendan 
4979c5904d13Seschrock boolean_t
4980c5904d13Seschrock l2arc_vdev_present(vdev_t *vd)
4981c5904d13Seschrock {
4982c5904d13Seschrock 	l2arc_dev_t *dev;
4983c5904d13Seschrock 
4984c5904d13Seschrock 	mutex_enter(&l2arc_dev_mtx);
4985c5904d13Seschrock 	for (dev = list_head(l2arc_dev_list); dev != NULL;
4986c5904d13Seschrock 	    dev = list_next(l2arc_dev_list, dev)) {
4987c5904d13Seschrock 		if (dev->l2ad_vdev == vd)
4988c5904d13Seschrock 			break;
4989c5904d13Seschrock 	}
4990c5904d13Seschrock 	mutex_exit(&l2arc_dev_mtx);
4991c5904d13Seschrock 
4992c5904d13Seschrock 	return (dev != NULL);
4993c5904d13Seschrock }
4994c5904d13Seschrock 
4995fa94a07fSbrendan /*
4996fa94a07fSbrendan  * Add a vdev for use by the L2ARC.  By this point the spa has already
4997fa94a07fSbrendan  * validated the vdev and opened it.
4998fa94a07fSbrendan  */
4999fa94a07fSbrendan void
5000573ca77eSGeorge Wilson l2arc_add_vdev(spa_t *spa, vdev_t *vd)
5001fa94a07fSbrendan {
5002fa94a07fSbrendan 	l2arc_dev_t *adddev;
5003fa94a07fSbrendan 
5004c5904d13Seschrock 	ASSERT(!l2arc_vdev_present(vd));
5005c5904d13Seschrock 
5006fa94a07fSbrendan 	/*
5007fa94a07fSbrendan 	 * Create a new l2arc device entry.
5008fa94a07fSbrendan 	 */
5009fa94a07fSbrendan 	adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
5010fa94a07fSbrendan 	adddev->l2ad_spa = spa;
5011fa94a07fSbrendan 	adddev->l2ad_vdev = vd;
5012573ca77eSGeorge Wilson 	adddev->l2ad_start = VDEV_LABEL_START_SIZE;
5013573ca77eSGeorge Wilson 	adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
5014fa94a07fSbrendan 	adddev->l2ad_hand = adddev->l2ad_start;
5015fa94a07fSbrendan 	adddev->l2ad_evict = adddev->l2ad_start;
5016fa94a07fSbrendan 	adddev->l2ad_first = B_TRUE;
50175a98e54bSBrendan Gregg - Sun Microsystems 	adddev->l2ad_writing = B_FALSE;
5018fa94a07fSbrendan 
5019fa94a07fSbrendan 	/*
5020fa94a07fSbrendan 	 * This is a list of all ARC buffers that are still valid on the
5021fa94a07fSbrendan 	 * device.
5022fa94a07fSbrendan 	 */
5023fa94a07fSbrendan 	adddev->l2ad_buflist = kmem_zalloc(sizeof (list_t), KM_SLEEP);
5024fa94a07fSbrendan 	list_create(adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
5025fa94a07fSbrendan 	    offsetof(arc_buf_hdr_t, b_l2node));
5026fa94a07fSbrendan 
5027b24ab676SJeff Bonwick 	vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
5028fa94a07fSbrendan 
5029fa94a07fSbrendan 	/*
5030fa94a07fSbrendan 	 * Add device to global list
5031fa94a07fSbrendan 	 */
5032fa94a07fSbrendan 	mutex_enter(&l2arc_dev_mtx);
5033fa94a07fSbrendan 	list_insert_head(l2arc_dev_list, adddev);
5034fa94a07fSbrendan 	atomic_inc_64(&l2arc_ndev);
5035fa94a07fSbrendan 	mutex_exit(&l2arc_dev_mtx);
5036fa94a07fSbrendan }
5037fa94a07fSbrendan 
5038fa94a07fSbrendan /*
5039fa94a07fSbrendan  * Remove a vdev from the L2ARC.
5040fa94a07fSbrendan  */
5041fa94a07fSbrendan void
5042fa94a07fSbrendan l2arc_remove_vdev(vdev_t *vd)
5043fa94a07fSbrendan {
5044fa94a07fSbrendan 	l2arc_dev_t *dev, *nextdev, *remdev = NULL;
5045fa94a07fSbrendan 
5046fa94a07fSbrendan 	/*
5047fa94a07fSbrendan 	 * Find the device by vdev
5048fa94a07fSbrendan 	 */
5049fa94a07fSbrendan 	mutex_enter(&l2arc_dev_mtx);
5050fa94a07fSbrendan 	for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) {
5051fa94a07fSbrendan 		nextdev = list_next(l2arc_dev_list, dev);
5052fa94a07fSbrendan 		if (vd == dev->l2ad_vdev) {
5053fa94a07fSbrendan 			remdev = dev;
5054fa94a07fSbrendan 			break;
5055fa94a07fSbrendan 		}
5056fa94a07fSbrendan 	}
5057fa94a07fSbrendan 	ASSERT(remdev != NULL);
5058fa94a07fSbrendan 
5059fa94a07fSbrendan 	/*
5060fa94a07fSbrendan 	 * Remove device from global list
5061fa94a07fSbrendan 	 */
5062fa94a07fSbrendan 	list_remove(l2arc_dev_list, remdev);
5063fa94a07fSbrendan 	l2arc_dev_last = NULL;		/* may have been invalidated */
50643a737e0dSbrendan 	atomic_dec_64(&l2arc_ndev);
50653a737e0dSbrendan 	mutex_exit(&l2arc_dev_mtx);
5066fa94a07fSbrendan 
5067fa94a07fSbrendan 	/*
5068fa94a07fSbrendan 	 * Clear all buflists and ARC references.  L2ARC device flush.
5069fa94a07fSbrendan 	 */
5070fa94a07fSbrendan 	l2arc_evict(remdev, 0, B_TRUE);
5071fa94a07fSbrendan 	list_destroy(remdev->l2ad_buflist);
5072fa94a07fSbrendan 	kmem_free(remdev->l2ad_buflist, sizeof (list_t));
5073fa94a07fSbrendan 	kmem_free(remdev, sizeof (l2arc_dev_t));
5074fa94a07fSbrendan }
5075fa94a07fSbrendan 
5076fa94a07fSbrendan void
5077e14bb325SJeff Bonwick l2arc_init(void)
5078fa94a07fSbrendan {
5079fa94a07fSbrendan 	l2arc_thread_exit = 0;
5080fa94a07fSbrendan 	l2arc_ndev = 0;
5081fa94a07fSbrendan 	l2arc_writes_sent = 0;
5082fa94a07fSbrendan 	l2arc_writes_done = 0;
5083fa94a07fSbrendan 
5084fa94a07fSbrendan 	mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
5085fa94a07fSbrendan 	cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
5086fa94a07fSbrendan 	mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
5087fa94a07fSbrendan 	mutex_init(&l2arc_buflist_mtx, NULL, MUTEX_DEFAULT, NULL);
5088fa94a07fSbrendan 	mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
5089fa94a07fSbrendan 
5090fa94a07fSbrendan 	l2arc_dev_list = &L2ARC_dev_list;
5091fa94a07fSbrendan 	l2arc_free_on_write = &L2ARC_free_on_write;
5092fa94a07fSbrendan 	list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
5093fa94a07fSbrendan 	    offsetof(l2arc_dev_t, l2ad_node));
5094fa94a07fSbrendan 	list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
5095fa94a07fSbrendan 	    offsetof(l2arc_data_free_t, l2df_list_node));
5096fa94a07fSbrendan }
5097fa94a07fSbrendan 
5098fa94a07fSbrendan void
5099e14bb325SJeff Bonwick l2arc_fini(void)
5100fa94a07fSbrendan {
51013a737e0dSbrendan 	/*
51023a737e0dSbrendan 	 * This is called from dmu_fini(), which is called from spa_fini();
51033a737e0dSbrendan 	 * Because of this, we can assume that all l2arc devices have
51043a737e0dSbrendan 	 * already been removed when the pools themselves were removed.
51053a737e0dSbrendan 	 */
51063a737e0dSbrendan 
51073a737e0dSbrendan 	l2arc_do_free_on_write();
51083a737e0dSbrendan 
5109fa94a07fSbrendan 	mutex_destroy(&l2arc_feed_thr_lock);
5110fa94a07fSbrendan 	cv_destroy(&l2arc_feed_thr_cv);
5111fa94a07fSbrendan 	mutex_destroy(&l2arc_dev_mtx);
5112fa94a07fSbrendan 	mutex_destroy(&l2arc_buflist_mtx);
5113fa94a07fSbrendan 	mutex_destroy(&l2arc_free_on_write_mtx);
5114fa94a07fSbrendan 
5115fa94a07fSbrendan 	list_destroy(l2arc_dev_list);
5116fa94a07fSbrendan 	list_destroy(l2arc_free_on_write);
5117fa94a07fSbrendan }
5118e14bb325SJeff Bonwick 
5119e14bb325SJeff Bonwick void
5120e14bb325SJeff Bonwick l2arc_start(void)
5121e14bb325SJeff Bonwick {
51228ad4d6ddSJeff Bonwick 	if (!(spa_mode_global & FWRITE))
5123e14bb325SJeff Bonwick 		return;
5124e14bb325SJeff Bonwick 
5125e14bb325SJeff Bonwick 	(void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
5126e14bb325SJeff Bonwick 	    TS_RUN, minclsyspri);
5127e14bb325SJeff Bonwick }
5128e14bb325SJeff Bonwick 
5129e14bb325SJeff Bonwick void
5130e14bb325SJeff Bonwick l2arc_stop(void)
5131e14bb325SJeff Bonwick {
51328ad4d6ddSJeff Bonwick 	if (!(spa_mode_global & FWRITE))
5133e14bb325SJeff Bonwick 		return;
5134e14bb325SJeff Bonwick 
5135e14bb325SJeff Bonwick 	mutex_enter(&l2arc_feed_thr_lock);
5136e14bb325SJeff Bonwick 	cv_signal(&l2arc_feed_thr_cv);	/* kick thread out of startup */
5137e14bb325SJeff Bonwick 	l2arc_thread_exit = 1;
5138e14bb325SJeff Bonwick 	while (l2arc_thread_exit != 0)
5139e14bb325SJeff Bonwick 		cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
5140e14bb325SJeff Bonwick 	mutex_exit(&l2arc_feed_thr_lock);
5141e14bb325SJeff Bonwick }
5142