1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5033f9833Sek * Common Development and Distribution License (the "License"). 6033f9833Sek * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 22*5a98e54bSBrendan Gregg - Sun Microsystems * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23fa9e4066Sahrens * Use is subject to license terms. 24fa9e4066Sahrens */ 25fa9e4066Sahrens 26fa9e4066Sahrens /* 2744cb6abcSbmc * DVA-based Adjustable Replacement Cache 28fa9e4066Sahrens * 29ea8dc4b6Seschrock * While much of the theory of operation used here is 30ea8dc4b6Seschrock * based on the self-tuning, low overhead replacement cache 31fa9e4066Sahrens * presented by Megiddo and Modha at FAST 2003, there are some 32fa9e4066Sahrens * significant differences: 33fa9e4066Sahrens * 34fa9e4066Sahrens * 1. The Megiddo and Modha model assumes any page is evictable. 35fa9e4066Sahrens * Pages in its cache cannot be "locked" into memory. This makes 36fa9e4066Sahrens * the eviction algorithm simple: evict the last page in the list. 37fa9e4066Sahrens * This also make the performance characteristics easy to reason 38fa9e4066Sahrens * about. Our cache is not so simple. At any given moment, some 39fa9e4066Sahrens * subset of the blocks in the cache are un-evictable because we 40fa9e4066Sahrens * have handed out a reference to them. Blocks are only evictable 41fa9e4066Sahrens * when there are no external references active. This makes 42fa9e4066Sahrens * eviction far more problematic: we choose to evict the evictable 43fa9e4066Sahrens * blocks that are the "lowest" in the list. 44fa9e4066Sahrens * 45fa9e4066Sahrens * There are times when it is not possible to evict the requested 46fa9e4066Sahrens * space. In these circumstances we are unable to adjust the cache 47fa9e4066Sahrens * size. To prevent the cache growing unbounded at these times we 48fa94a07fSbrendan * implement a "cache throttle" that slows the flow of new data 49fa94a07fSbrendan * into the cache until we can make space available. 50fa9e4066Sahrens * 51fa9e4066Sahrens * 2. The Megiddo and Modha model assumes a fixed cache size. 52fa9e4066Sahrens * Pages are evicted when the cache is full and there is a cache 53fa9e4066Sahrens * miss. Our model has a variable sized cache. It grows with 54fa94a07fSbrendan * high use, but also tries to react to memory pressure from the 55fa9e4066Sahrens * operating system: decreasing its size when system memory is 56fa9e4066Sahrens * tight. 57fa9e4066Sahrens * 58fa9e4066Sahrens * 3. The Megiddo and Modha model assumes a fixed page size. All 59fa9e4066Sahrens * elements of the cache are therefor exactly the same size. So 60fa9e4066Sahrens * when adjusting the cache size following a cache miss, its simply 61fa9e4066Sahrens * a matter of choosing a single page to evict. In our model, we 62fa9e4066Sahrens * have variable sized cache blocks (rangeing from 512 bytes to 63fa9e4066Sahrens * 128K bytes). We therefor choose a set of blocks to evict to make 64fa9e4066Sahrens * space for a cache miss that approximates as closely as possible 65fa9e4066Sahrens * the space used by the new block. 66fa9e4066Sahrens * 67fa9e4066Sahrens * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache" 68fa9e4066Sahrens * by N. Megiddo & D. Modha, FAST 2003 69fa9e4066Sahrens */ 70fa9e4066Sahrens 71fa9e4066Sahrens /* 72fa9e4066Sahrens * The locking model: 73fa9e4066Sahrens * 74fa9e4066Sahrens * A new reference to a cache buffer can be obtained in two 75fa9e4066Sahrens * ways: 1) via a hash table lookup using the DVA as a key, 76fa94a07fSbrendan * or 2) via one of the ARC lists. The arc_read() interface 77fa9e4066Sahrens * uses method 1, while the internal arc algorithms for 78fa9e4066Sahrens * adjusting the cache use method 2. We therefor provide two 79fa9e4066Sahrens * types of locks: 1) the hash table lock array, and 2) the 80fa9e4066Sahrens * arc list locks. 81fa9e4066Sahrens * 82fa9e4066Sahrens * Buffers do not have their own mutexs, rather they rely on the 83fa9e4066Sahrens * hash table mutexs for the bulk of their protection (i.e. most 84fa9e4066Sahrens * fields in the arc_buf_hdr_t are protected by these mutexs). 85fa9e4066Sahrens * 86fa9e4066Sahrens * buf_hash_find() returns the appropriate mutex (held) when it 87fa9e4066Sahrens * locates the requested buffer in the hash table. It returns 88fa9e4066Sahrens * NULL for the mutex if the buffer was not in the table. 89fa9e4066Sahrens * 90fa9e4066Sahrens * buf_hash_remove() expects the appropriate hash mutex to be 91fa9e4066Sahrens * already held before it is invoked. 92fa9e4066Sahrens * 93fa9e4066Sahrens * Each arc state also has a mutex which is used to protect the 94fa9e4066Sahrens * buffer list associated with the state. When attempting to 95fa9e4066Sahrens * obtain a hash table lock while holding an arc list lock you 96fa9e4066Sahrens * must use: mutex_tryenter() to avoid deadlock. Also note that 9744eda4d7Smaybee * the active state mutex must be held before the ghost state mutex. 98fa9e4066Sahrens * 99ea8dc4b6Seschrock * Arc buffers may have an associated eviction callback function. 100ea8dc4b6Seschrock * This function will be invoked prior to removing the buffer (e.g. 101ea8dc4b6Seschrock * in arc_do_user_evicts()). Note however that the data associated 102ea8dc4b6Seschrock * with the buffer may be evicted prior to the callback. The callback 103ea8dc4b6Seschrock * must be made with *no locks held* (to prevent deadlock). Additionally, 104ea8dc4b6Seschrock * the users of callbacks must ensure that their private data is 105ea8dc4b6Seschrock * protected from simultaneous callbacks from arc_buf_evict() 106ea8dc4b6Seschrock * and arc_do_user_evicts(). 107ea8dc4b6Seschrock * 108fa9e4066Sahrens * Note that the majority of the performance stats are manipulated 109fa9e4066Sahrens * with atomic operations. 110fa94a07fSbrendan * 111fa94a07fSbrendan * The L2ARC uses the l2arc_buflist_mtx global mutex for the following: 112fa94a07fSbrendan * 113fa94a07fSbrendan * - L2ARC buflist creation 114fa94a07fSbrendan * - L2ARC buflist eviction 115fa94a07fSbrendan * - L2ARC write completion, which walks L2ARC buflists 116fa94a07fSbrendan * - ARC header destruction, as it removes from L2ARC buflists 117fa94a07fSbrendan * - ARC header release, as it removes from L2ARC buflists 118fa9e4066Sahrens */ 119fa9e4066Sahrens 120fa9e4066Sahrens #include <sys/spa.h> 121fa9e4066Sahrens #include <sys/zio.h> 1226b4acc8bSahrens #include <sys/zio_checksum.h> 123fa9e4066Sahrens #include <sys/zfs_context.h> 124fa9e4066Sahrens #include <sys/arc.h> 125fa9e4066Sahrens #include <sys/refcount.h> 126c5904d13Seschrock #include <sys/vdev.h> 127fa9e4066Sahrens #ifdef _KERNEL 128fa9e4066Sahrens #include <sys/vmsystm.h> 129fa9e4066Sahrens #include <vm/anon.h> 130fa9e4066Sahrens #include <sys/fs/swapnode.h> 131033f9833Sek #include <sys/dnlc.h> 132fa9e4066Sahrens #endif 133fa9e4066Sahrens #include <sys/callb.h> 13444cb6abcSbmc #include <sys/kstat.h> 135fa9e4066Sahrens 136fa9e4066Sahrens static kmutex_t arc_reclaim_thr_lock; 137fa9e4066Sahrens static kcondvar_t arc_reclaim_thr_cv; /* used to signal reclaim thr */ 138fa9e4066Sahrens static uint8_t arc_thread_exit; 139fa9e4066Sahrens 1401ab7f2deSmaybee extern int zfs_write_limit_shift; 1411ab7f2deSmaybee extern uint64_t zfs_write_limit_max; 14205715f94SMark Maybee extern kmutex_t zfs_write_limit_lock; 1431ab7f2deSmaybee 144033f9833Sek #define ARC_REDUCE_DNLC_PERCENT 3 145033f9833Sek uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT; 146033f9833Sek 147fa9e4066Sahrens typedef enum arc_reclaim_strategy { 148fa9e4066Sahrens ARC_RECLAIM_AGGR, /* Aggressive reclaim strategy */ 149fa9e4066Sahrens ARC_RECLAIM_CONS /* Conservative reclaim strategy */ 150fa9e4066Sahrens } arc_reclaim_strategy_t; 151fa9e4066Sahrens 152fa9e4066Sahrens /* number of seconds before growing cache again */ 153fa9e4066Sahrens static int arc_grow_retry = 60; 154fa9e4066Sahrens 155*5a98e54bSBrendan Gregg - Sun Microsystems /* shift of arc_c for calculating both min and max arc_p */ 156*5a98e54bSBrendan Gregg - Sun Microsystems static int arc_p_min_shift = 4; 157*5a98e54bSBrendan Gregg - Sun Microsystems 158*5a98e54bSBrendan Gregg - Sun Microsystems /* log2(fraction of arc to reclaim) */ 159*5a98e54bSBrendan Gregg - Sun Microsystems static int arc_shrink_shift = 5; 160*5a98e54bSBrendan Gregg - Sun Microsystems 16113506d1eSmaybee /* 162b19a79ecSperrin * minimum lifespan of a prefetch block in clock ticks 163b19a79ecSperrin * (initialized in arc_init()) 16413506d1eSmaybee */ 165b19a79ecSperrin static int arc_min_prefetch_lifespan; 16613506d1eSmaybee 167fa9e4066Sahrens static int arc_dead; 168fa9e4066Sahrens 1693a737e0dSbrendan /* 1703a737e0dSbrendan * The arc has filled available memory and has now warmed up. 1713a737e0dSbrendan */ 1723a737e0dSbrendan static boolean_t arc_warm; 1733a737e0dSbrendan 174a2eea2e1Sahrens /* 175a2eea2e1Sahrens * These tunables are for performance analysis. 176a2eea2e1Sahrens */ 177a2eea2e1Sahrens uint64_t zfs_arc_max; 178a2eea2e1Sahrens uint64_t zfs_arc_min; 1791116048bSek uint64_t zfs_arc_meta_limit = 0; 180088f3894Sahrens int zfs_mdcomp_disable = 0; 181*5a98e54bSBrendan Gregg - Sun Microsystems int zfs_arc_grow_retry = 0; 182*5a98e54bSBrendan Gregg - Sun Microsystems int zfs_arc_shrink_shift = 0; 183*5a98e54bSBrendan Gregg - Sun Microsystems int zfs_arc_p_min_shift = 0; 184a2eea2e1Sahrens 185fa9e4066Sahrens /* 186fa94a07fSbrendan * Note that buffers can be in one of 6 states: 187fa9e4066Sahrens * ARC_anon - anonymous (discussed below) 188ea8dc4b6Seschrock * ARC_mru - recently used, currently cached 189ea8dc4b6Seschrock * ARC_mru_ghost - recentely used, no longer in cache 190ea8dc4b6Seschrock * ARC_mfu - frequently used, currently cached 191ea8dc4b6Seschrock * ARC_mfu_ghost - frequently used, no longer in cache 192fa94a07fSbrendan * ARC_l2c_only - exists in L2ARC but not other states 1930e8c6158Smaybee * When there are no active references to the buffer, they are 1940e8c6158Smaybee * are linked onto a list in one of these arc states. These are 1950e8c6158Smaybee * the only buffers that can be evicted or deleted. Within each 1960e8c6158Smaybee * state there are multiple lists, one for meta-data and one for 1970e8c6158Smaybee * non-meta-data. Meta-data (indirect blocks, blocks of dnodes, 1980e8c6158Smaybee * etc.) is tracked separately so that it can be managed more 199fa94a07fSbrendan * explicitly: favored over data, limited explicitly. 200fa9e4066Sahrens * 201fa9e4066Sahrens * Anonymous buffers are buffers that are not associated with 202fa9e4066Sahrens * a DVA. These are buffers that hold dirty block copies 203fa9e4066Sahrens * before they are written to stable storage. By definition, 204ea8dc4b6Seschrock * they are "ref'd" and are considered part of arc_mru 205fa9e4066Sahrens * that cannot be freed. Generally, they will aquire a DVA 206ea8dc4b6Seschrock * as they are written and migrate onto the arc_mru list. 207fa94a07fSbrendan * 208fa94a07fSbrendan * The ARC_l2c_only state is for buffers that are in the second 209fa94a07fSbrendan * level ARC but no longer in any of the ARC_m* lists. The second 210fa94a07fSbrendan * level ARC itself may also contain buffers that are in any of 211fa94a07fSbrendan * the ARC_m* states - meaning that a buffer can exist in two 212fa94a07fSbrendan * places. The reason for the ARC_l2c_only state is to keep the 213fa94a07fSbrendan * buffer header in the hash table, so that reads that hit the 214fa94a07fSbrendan * second level ARC benefit from these fast lookups. 215fa9e4066Sahrens */ 216fa9e4066Sahrens 217fa9e4066Sahrens typedef struct arc_state { 2180e8c6158Smaybee list_t arcs_list[ARC_BUFC_NUMTYPES]; /* list of evictable buffers */ 2190e8c6158Smaybee uint64_t arcs_lsize[ARC_BUFC_NUMTYPES]; /* amount of evictable data */ 2200e8c6158Smaybee uint64_t arcs_size; /* total amount of data in this state */ 22144cb6abcSbmc kmutex_t arcs_mtx; 222fa9e4066Sahrens } arc_state_t; 223fa9e4066Sahrens 224fa94a07fSbrendan /* The 6 states: */ 225fa9e4066Sahrens static arc_state_t ARC_anon; 226ea8dc4b6Seschrock static arc_state_t ARC_mru; 227ea8dc4b6Seschrock static arc_state_t ARC_mru_ghost; 228ea8dc4b6Seschrock static arc_state_t ARC_mfu; 229ea8dc4b6Seschrock static arc_state_t ARC_mfu_ghost; 230fa94a07fSbrendan static arc_state_t ARC_l2c_only; 231fa9e4066Sahrens 23244cb6abcSbmc typedef struct arc_stats { 23344cb6abcSbmc kstat_named_t arcstat_hits; 23444cb6abcSbmc kstat_named_t arcstat_misses; 23544cb6abcSbmc kstat_named_t arcstat_demand_data_hits; 23644cb6abcSbmc kstat_named_t arcstat_demand_data_misses; 23744cb6abcSbmc kstat_named_t arcstat_demand_metadata_hits; 23844cb6abcSbmc kstat_named_t arcstat_demand_metadata_misses; 23944cb6abcSbmc kstat_named_t arcstat_prefetch_data_hits; 24044cb6abcSbmc kstat_named_t arcstat_prefetch_data_misses; 24144cb6abcSbmc kstat_named_t arcstat_prefetch_metadata_hits; 24244cb6abcSbmc kstat_named_t arcstat_prefetch_metadata_misses; 24344cb6abcSbmc kstat_named_t arcstat_mru_hits; 24444cb6abcSbmc kstat_named_t arcstat_mru_ghost_hits; 24544cb6abcSbmc kstat_named_t arcstat_mfu_hits; 24644cb6abcSbmc kstat_named_t arcstat_mfu_ghost_hits; 24744cb6abcSbmc kstat_named_t arcstat_deleted; 24844cb6abcSbmc kstat_named_t arcstat_recycle_miss; 24944cb6abcSbmc kstat_named_t arcstat_mutex_miss; 25044cb6abcSbmc kstat_named_t arcstat_evict_skip; 25144cb6abcSbmc kstat_named_t arcstat_hash_elements; 25244cb6abcSbmc kstat_named_t arcstat_hash_elements_max; 25344cb6abcSbmc kstat_named_t arcstat_hash_collisions; 25444cb6abcSbmc kstat_named_t arcstat_hash_chains; 25544cb6abcSbmc kstat_named_t arcstat_hash_chain_max; 25644cb6abcSbmc kstat_named_t arcstat_p; 25744cb6abcSbmc kstat_named_t arcstat_c; 25844cb6abcSbmc kstat_named_t arcstat_c_min; 25944cb6abcSbmc kstat_named_t arcstat_c_max; 26044cb6abcSbmc kstat_named_t arcstat_size; 261fa94a07fSbrendan kstat_named_t arcstat_hdr_size; 262*5a98e54bSBrendan Gregg - Sun Microsystems kstat_named_t arcstat_data_size; 263*5a98e54bSBrendan Gregg - Sun Microsystems kstat_named_t arcstat_other_size; 264fa94a07fSbrendan kstat_named_t arcstat_l2_hits; 265fa94a07fSbrendan kstat_named_t arcstat_l2_misses; 266fa94a07fSbrendan kstat_named_t arcstat_l2_feeds; 267fa94a07fSbrendan kstat_named_t arcstat_l2_rw_clash; 268*5a98e54bSBrendan Gregg - Sun Microsystems kstat_named_t arcstat_l2_read_bytes; 269*5a98e54bSBrendan Gregg - Sun Microsystems kstat_named_t arcstat_l2_write_bytes; 270fa94a07fSbrendan kstat_named_t arcstat_l2_writes_sent; 271fa94a07fSbrendan kstat_named_t arcstat_l2_writes_done; 272fa94a07fSbrendan kstat_named_t arcstat_l2_writes_error; 273fa94a07fSbrendan kstat_named_t arcstat_l2_writes_hdr_miss; 274fa94a07fSbrendan kstat_named_t arcstat_l2_evict_lock_retry; 275fa94a07fSbrendan kstat_named_t arcstat_l2_evict_reading; 276fa94a07fSbrendan kstat_named_t arcstat_l2_free_on_write; 277fa94a07fSbrendan kstat_named_t arcstat_l2_abort_lowmem; 278fa94a07fSbrendan kstat_named_t arcstat_l2_cksum_bad; 279fa94a07fSbrendan kstat_named_t arcstat_l2_io_error; 280fa94a07fSbrendan kstat_named_t arcstat_l2_size; 281fa94a07fSbrendan kstat_named_t arcstat_l2_hdr_size; 2821ab7f2deSmaybee kstat_named_t arcstat_memory_throttle_count; 28344cb6abcSbmc } arc_stats_t; 28444cb6abcSbmc 28544cb6abcSbmc static arc_stats_t arc_stats = { 28644cb6abcSbmc { "hits", KSTAT_DATA_UINT64 }, 28744cb6abcSbmc { "misses", KSTAT_DATA_UINT64 }, 28844cb6abcSbmc { "demand_data_hits", KSTAT_DATA_UINT64 }, 28944cb6abcSbmc { "demand_data_misses", KSTAT_DATA_UINT64 }, 29044cb6abcSbmc { "demand_metadata_hits", KSTAT_DATA_UINT64 }, 29144cb6abcSbmc { "demand_metadata_misses", KSTAT_DATA_UINT64 }, 29244cb6abcSbmc { "prefetch_data_hits", KSTAT_DATA_UINT64 }, 29344cb6abcSbmc { "prefetch_data_misses", KSTAT_DATA_UINT64 }, 29444cb6abcSbmc { "prefetch_metadata_hits", KSTAT_DATA_UINT64 }, 29544cb6abcSbmc { "prefetch_metadata_misses", KSTAT_DATA_UINT64 }, 29644cb6abcSbmc { "mru_hits", KSTAT_DATA_UINT64 }, 29744cb6abcSbmc { "mru_ghost_hits", KSTAT_DATA_UINT64 }, 29844cb6abcSbmc { "mfu_hits", KSTAT_DATA_UINT64 }, 29944cb6abcSbmc { "mfu_ghost_hits", KSTAT_DATA_UINT64 }, 30044cb6abcSbmc { "deleted", KSTAT_DATA_UINT64 }, 30144cb6abcSbmc { "recycle_miss", KSTAT_DATA_UINT64 }, 30244cb6abcSbmc { "mutex_miss", KSTAT_DATA_UINT64 }, 30344cb6abcSbmc { "evict_skip", KSTAT_DATA_UINT64 }, 30444cb6abcSbmc { "hash_elements", KSTAT_DATA_UINT64 }, 30544cb6abcSbmc { "hash_elements_max", KSTAT_DATA_UINT64 }, 30644cb6abcSbmc { "hash_collisions", KSTAT_DATA_UINT64 }, 30744cb6abcSbmc { "hash_chains", KSTAT_DATA_UINT64 }, 30844cb6abcSbmc { "hash_chain_max", KSTAT_DATA_UINT64 }, 30944cb6abcSbmc { "p", KSTAT_DATA_UINT64 }, 31044cb6abcSbmc { "c", KSTAT_DATA_UINT64 }, 31144cb6abcSbmc { "c_min", KSTAT_DATA_UINT64 }, 31244cb6abcSbmc { "c_max", KSTAT_DATA_UINT64 }, 313fa94a07fSbrendan { "size", KSTAT_DATA_UINT64 }, 314fa94a07fSbrendan { "hdr_size", KSTAT_DATA_UINT64 }, 315*5a98e54bSBrendan Gregg - Sun Microsystems { "data_size", KSTAT_DATA_UINT64 }, 316*5a98e54bSBrendan Gregg - Sun Microsystems { "other_size", KSTAT_DATA_UINT64 }, 317fa94a07fSbrendan { "l2_hits", KSTAT_DATA_UINT64 }, 318fa94a07fSbrendan { "l2_misses", KSTAT_DATA_UINT64 }, 319fa94a07fSbrendan { "l2_feeds", KSTAT_DATA_UINT64 }, 320fa94a07fSbrendan { "l2_rw_clash", KSTAT_DATA_UINT64 }, 321*5a98e54bSBrendan Gregg - Sun Microsystems { "l2_read_bytes", KSTAT_DATA_UINT64 }, 322*5a98e54bSBrendan Gregg - Sun Microsystems { "l2_write_bytes", KSTAT_DATA_UINT64 }, 323fa94a07fSbrendan { "l2_writes_sent", KSTAT_DATA_UINT64 }, 324fa94a07fSbrendan { "l2_writes_done", KSTAT_DATA_UINT64 }, 325fa94a07fSbrendan { "l2_writes_error", KSTAT_DATA_UINT64 }, 326fa94a07fSbrendan { "l2_writes_hdr_miss", KSTAT_DATA_UINT64 }, 327fa94a07fSbrendan { "l2_evict_lock_retry", KSTAT_DATA_UINT64 }, 328fa94a07fSbrendan { "l2_evict_reading", KSTAT_DATA_UINT64 }, 329fa94a07fSbrendan { "l2_free_on_write", KSTAT_DATA_UINT64 }, 330fa94a07fSbrendan { "l2_abort_lowmem", KSTAT_DATA_UINT64 }, 331fa94a07fSbrendan { "l2_cksum_bad", KSTAT_DATA_UINT64 }, 332fa94a07fSbrendan { "l2_io_error", KSTAT_DATA_UINT64 }, 333fa94a07fSbrendan { "l2_size", KSTAT_DATA_UINT64 }, 3341ab7f2deSmaybee { "l2_hdr_size", KSTAT_DATA_UINT64 }, 3351ab7f2deSmaybee { "memory_throttle_count", KSTAT_DATA_UINT64 } 33644cb6abcSbmc }; 33744cb6abcSbmc 33844cb6abcSbmc #define ARCSTAT(stat) (arc_stats.stat.value.ui64) 33944cb6abcSbmc 34044cb6abcSbmc #define ARCSTAT_INCR(stat, val) \ 34144cb6abcSbmc atomic_add_64(&arc_stats.stat.value.ui64, (val)); 34244cb6abcSbmc 34344cb6abcSbmc #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) 34444cb6abcSbmc #define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1) 34544cb6abcSbmc 34644cb6abcSbmc #define ARCSTAT_MAX(stat, val) { \ 34744cb6abcSbmc uint64_t m; \ 34844cb6abcSbmc while ((val) > (m = arc_stats.stat.value.ui64) && \ 34944cb6abcSbmc (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \ 35044cb6abcSbmc continue; \ 35144cb6abcSbmc } 35244cb6abcSbmc 35344cb6abcSbmc #define ARCSTAT_MAXSTAT(stat) \ 35444cb6abcSbmc ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64) 35544cb6abcSbmc 35644cb6abcSbmc /* 35744cb6abcSbmc * We define a macro to allow ARC hits/misses to be easily broken down by 35844cb6abcSbmc * two separate conditions, giving a total of four different subtypes for 35944cb6abcSbmc * each of hits and misses (so eight statistics total). 36044cb6abcSbmc */ 36144cb6abcSbmc #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \ 36244cb6abcSbmc if (cond1) { \ 36344cb6abcSbmc if (cond2) { \ 36444cb6abcSbmc ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \ 36544cb6abcSbmc } else { \ 36644cb6abcSbmc ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \ 36744cb6abcSbmc } \ 36844cb6abcSbmc } else { \ 36944cb6abcSbmc if (cond2) { \ 37044cb6abcSbmc ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \ 37144cb6abcSbmc } else { \ 37244cb6abcSbmc ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\ 37344cb6abcSbmc } \ 37444cb6abcSbmc } 37544cb6abcSbmc 37644cb6abcSbmc kstat_t *arc_ksp; 37744cb6abcSbmc static arc_state_t *arc_anon; 37844cb6abcSbmc static arc_state_t *arc_mru; 37944cb6abcSbmc static arc_state_t *arc_mru_ghost; 38044cb6abcSbmc static arc_state_t *arc_mfu; 38144cb6abcSbmc static arc_state_t *arc_mfu_ghost; 382fa94a07fSbrendan static arc_state_t *arc_l2c_only; 38344cb6abcSbmc 38444cb6abcSbmc /* 38544cb6abcSbmc * There are several ARC variables that are critical to export as kstats -- 38644cb6abcSbmc * but we don't want to have to grovel around in the kstat whenever we wish to 38744cb6abcSbmc * manipulate them. For these variables, we therefore define them to be in 38844cb6abcSbmc * terms of the statistic variable. This assures that we are not introducing 38944cb6abcSbmc * the possibility of inconsistency by having shadow copies of the variables, 39044cb6abcSbmc * while still allowing the code to be readable. 39144cb6abcSbmc */ 39244cb6abcSbmc #define arc_size ARCSTAT(arcstat_size) /* actual total arc size */ 39344cb6abcSbmc #define arc_p ARCSTAT(arcstat_p) /* target size of MRU */ 39444cb6abcSbmc #define arc_c ARCSTAT(arcstat_c) /* target size of cache */ 39544cb6abcSbmc #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ 39644cb6abcSbmc #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ 39744cb6abcSbmc 39844cb6abcSbmc static int arc_no_grow; /* Don't try to grow cache size */ 39944cb6abcSbmc static uint64_t arc_tempreserve; 4000e8c6158Smaybee static uint64_t arc_meta_used; 4010e8c6158Smaybee static uint64_t arc_meta_limit; 4020e8c6158Smaybee static uint64_t arc_meta_max = 0; 403fa9e4066Sahrens 404fa94a07fSbrendan typedef struct l2arc_buf_hdr l2arc_buf_hdr_t; 405fa94a07fSbrendan 406fa9e4066Sahrens typedef struct arc_callback arc_callback_t; 407fa9e4066Sahrens 408fa9e4066Sahrens struct arc_callback { 409fa9e4066Sahrens void *acb_private; 410c717a561Smaybee arc_done_func_t *acb_done; 411fa9e4066Sahrens arc_buf_t *acb_buf; 412fa9e4066Sahrens zio_t *acb_zio_dummy; 413fa9e4066Sahrens arc_callback_t *acb_next; 414fa9e4066Sahrens }; 415fa9e4066Sahrens 416c717a561Smaybee typedef struct arc_write_callback arc_write_callback_t; 417c717a561Smaybee 418c717a561Smaybee struct arc_write_callback { 419c717a561Smaybee void *awcb_private; 420c717a561Smaybee arc_done_func_t *awcb_ready; 421c717a561Smaybee arc_done_func_t *awcb_done; 422c717a561Smaybee arc_buf_t *awcb_buf; 423c717a561Smaybee }; 424c717a561Smaybee 425fa9e4066Sahrens struct arc_buf_hdr { 426fa9e4066Sahrens /* protected by hash lock */ 427fa9e4066Sahrens dva_t b_dva; 428fa9e4066Sahrens uint64_t b_birth; 429fa9e4066Sahrens uint64_t b_cksum0; 430fa9e4066Sahrens 4316b4acc8bSahrens kmutex_t b_freeze_lock; 4326b4acc8bSahrens zio_cksum_t *b_freeze_cksum; 4336b4acc8bSahrens 434fa9e4066Sahrens arc_buf_hdr_t *b_hash_next; 435fa9e4066Sahrens arc_buf_t *b_buf; 436fa9e4066Sahrens uint32_t b_flags; 437ea8dc4b6Seschrock uint32_t b_datacnt; 438fa9e4066Sahrens 439fa9e4066Sahrens arc_callback_t *b_acb; 440ad23a2dbSjohansen kcondvar_t b_cv; 441ad23a2dbSjohansen 442ad23a2dbSjohansen /* immutable */ 443ad23a2dbSjohansen arc_buf_contents_t b_type; 444ad23a2dbSjohansen uint64_t b_size; 445ad23a2dbSjohansen spa_t *b_spa; 446fa9e4066Sahrens 447fa9e4066Sahrens /* protected by arc state mutex */ 448fa9e4066Sahrens arc_state_t *b_state; 449fa9e4066Sahrens list_node_t b_arc_node; 450fa9e4066Sahrens 451fa9e4066Sahrens /* updated atomically */ 452fa9e4066Sahrens clock_t b_arc_access; 453fa9e4066Sahrens 454fa9e4066Sahrens /* self protecting */ 455fa9e4066Sahrens refcount_t b_refcnt; 456fa94a07fSbrendan 457fa94a07fSbrendan l2arc_buf_hdr_t *b_l2hdr; 458fa94a07fSbrendan list_node_t b_l2node; 459fa9e4066Sahrens }; 460fa9e4066Sahrens 461ea8dc4b6Seschrock static arc_buf_t *arc_eviction_list; 462ea8dc4b6Seschrock static kmutex_t arc_eviction_mtx; 46340d7d650Smaybee static arc_buf_hdr_t arc_eviction_hdr; 46444eda4d7Smaybee static void arc_get_data_buf(arc_buf_t *buf); 46544eda4d7Smaybee static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock); 4660e8c6158Smaybee static int arc_evict_needed(arc_buf_contents_t type); 467874395d5Smaybee static void arc_evict_ghost(arc_state_t *state, spa_t *spa, int64_t bytes); 468ea8dc4b6Seschrock 469ea8dc4b6Seschrock #define GHOST_STATE(state) \ 470fa94a07fSbrendan ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \ 471fa94a07fSbrendan (state) == arc_l2c_only) 472ea8dc4b6Seschrock 473fa9e4066Sahrens /* 474fa9e4066Sahrens * Private ARC flags. These flags are private ARC only flags that will show up 475fa9e4066Sahrens * in b_flags in the arc_hdr_buf_t. Some flags are publicly declared, and can 476fa9e4066Sahrens * be passed in as arc_flags in things like arc_read. However, these flags 477fa9e4066Sahrens * should never be passed and should only be set by ARC code. When adding new 478fa9e4066Sahrens * public flags, make sure not to smash the private ones. 479fa9e4066Sahrens */ 480fa9e4066Sahrens 481ea8dc4b6Seschrock #define ARC_IN_HASH_TABLE (1 << 9) /* this buffer is hashed */ 482fa9e4066Sahrens #define ARC_IO_IN_PROGRESS (1 << 10) /* I/O in progress for buf */ 483fa9e4066Sahrens #define ARC_IO_ERROR (1 << 11) /* I/O failed for buf */ 484fa9e4066Sahrens #define ARC_FREED_IN_READ (1 << 12) /* buf freed while in read */ 485ea8dc4b6Seschrock #define ARC_BUF_AVAILABLE (1 << 13) /* block not in active use */ 48613506d1eSmaybee #define ARC_INDIRECT (1 << 14) /* this is an indirect block */ 487fa94a07fSbrendan #define ARC_FREE_IN_PROGRESS (1 << 15) /* hdr about to be freed */ 4883baa08fcSek #define ARC_L2_WRITING (1 << 16) /* L2ARC write in progress */ 4893baa08fcSek #define ARC_L2_EVICTED (1 << 17) /* evicted during I/O */ 4903baa08fcSek #define ARC_L2_WRITE_HEAD (1 << 18) /* head of write list */ 4913baa08fcSek #define ARC_STORED (1 << 19) /* has been store()d to */ 492fa9e4066Sahrens 493ea8dc4b6Seschrock #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_IN_HASH_TABLE) 494fa9e4066Sahrens #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS) 495fa9e4066Sahrens #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_IO_ERROR) 496*5a98e54bSBrendan Gregg - Sun Microsystems #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_PREFETCH) 497fa9e4066Sahrens #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FREED_IN_READ) 498ea8dc4b6Seschrock #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_BUF_AVAILABLE) 499fa94a07fSbrendan #define HDR_FREE_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FREE_IN_PROGRESS) 5003baa08fcSek #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_L2CACHE) 5013a737e0dSbrendan #define HDR_L2_READING(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS && \ 5023a737e0dSbrendan (hdr)->b_l2hdr != NULL) 503fa94a07fSbrendan #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_L2_WRITING) 504fa94a07fSbrendan #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_L2_EVICTED) 505fa94a07fSbrendan #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_L2_WRITE_HEAD) 506fa9e4066Sahrens 507e6c728e1Sbrendan /* 508e6c728e1Sbrendan * Other sizes 509e6c728e1Sbrendan */ 510e6c728e1Sbrendan 511e6c728e1Sbrendan #define HDR_SIZE ((int64_t)sizeof (arc_buf_hdr_t)) 512e6c728e1Sbrendan #define L2HDR_SIZE ((int64_t)sizeof (l2arc_buf_hdr_t)) 513e6c728e1Sbrendan 514fa9e4066Sahrens /* 515fa9e4066Sahrens * Hash table routines 516fa9e4066Sahrens */ 517fa9e4066Sahrens 518fa9e4066Sahrens #define HT_LOCK_PAD 64 519fa9e4066Sahrens 520fa9e4066Sahrens struct ht_lock { 521fa9e4066Sahrens kmutex_t ht_lock; 522fa9e4066Sahrens #ifdef _KERNEL 523fa9e4066Sahrens unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))]; 524fa9e4066Sahrens #endif 525fa9e4066Sahrens }; 526fa9e4066Sahrens 527fa9e4066Sahrens #define BUF_LOCKS 256 528fa9e4066Sahrens typedef struct buf_hash_table { 529fa9e4066Sahrens uint64_t ht_mask; 530fa9e4066Sahrens arc_buf_hdr_t **ht_table; 531fa9e4066Sahrens struct ht_lock ht_locks[BUF_LOCKS]; 532fa9e4066Sahrens } buf_hash_table_t; 533fa9e4066Sahrens 534fa9e4066Sahrens static buf_hash_table_t buf_hash_table; 535fa9e4066Sahrens 536fa9e4066Sahrens #define BUF_HASH_INDEX(spa, dva, birth) \ 537fa9e4066Sahrens (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask) 538fa9e4066Sahrens #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)]) 539fa9e4066Sahrens #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock)) 540fa9e4066Sahrens #define HDR_LOCK(buf) \ 541fa9e4066Sahrens (BUF_HASH_LOCK(BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth))) 542fa9e4066Sahrens 543fa9e4066Sahrens uint64_t zfs_crc64_table[256]; 544fa9e4066Sahrens 545fa94a07fSbrendan /* 546fa94a07fSbrendan * Level 2 ARC 547fa94a07fSbrendan */ 548fa94a07fSbrendan 549fa94a07fSbrendan #define L2ARC_WRITE_SIZE (8 * 1024 * 1024) /* initial write max */ 550*5a98e54bSBrendan Gregg - Sun Microsystems #define L2ARC_HEADROOM 2 /* num of writes */ 551*5a98e54bSBrendan Gregg - Sun Microsystems #define L2ARC_FEED_SECS 1 /* caching interval secs */ 552*5a98e54bSBrendan Gregg - Sun Microsystems #define L2ARC_FEED_MIN_MS 200 /* min caching interval ms */ 553fa94a07fSbrendan 554fa94a07fSbrendan #define l2arc_writes_sent ARCSTAT(arcstat_l2_writes_sent) 555fa94a07fSbrendan #define l2arc_writes_done ARCSTAT(arcstat_l2_writes_done) 556fa94a07fSbrendan 557fa94a07fSbrendan /* 558fa94a07fSbrendan * L2ARC Performance Tunables 559fa94a07fSbrendan */ 560fa94a07fSbrendan uint64_t l2arc_write_max = L2ARC_WRITE_SIZE; /* default max write size */ 5613a737e0dSbrendan uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE; /* extra write during warmup */ 562fa94a07fSbrendan uint64_t l2arc_headroom = L2ARC_HEADROOM; /* number of dev writes */ 563fa94a07fSbrendan uint64_t l2arc_feed_secs = L2ARC_FEED_SECS; /* interval seconds */ 564*5a98e54bSBrendan Gregg - Sun Microsystems uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval milliseconds */ 565fa94a07fSbrendan boolean_t l2arc_noprefetch = B_TRUE; /* don't cache prefetch bufs */ 566*5a98e54bSBrendan Gregg - Sun Microsystems boolean_t l2arc_feed_again = B_TRUE; /* turbo warmup */ 567*5a98e54bSBrendan Gregg - Sun Microsystems boolean_t l2arc_norw = B_TRUE; /* no reads during writes */ 568fa94a07fSbrendan 569fa94a07fSbrendan /* 570fa94a07fSbrendan * L2ARC Internals 571fa94a07fSbrendan */ 572fa94a07fSbrendan typedef struct l2arc_dev { 573fa94a07fSbrendan vdev_t *l2ad_vdev; /* vdev */ 574fa94a07fSbrendan spa_t *l2ad_spa; /* spa */ 575fa94a07fSbrendan uint64_t l2ad_hand; /* next write location */ 576fa94a07fSbrendan uint64_t l2ad_write; /* desired write size, bytes */ 5773a737e0dSbrendan uint64_t l2ad_boost; /* warmup write boost, bytes */ 578fa94a07fSbrendan uint64_t l2ad_start; /* first addr on device */ 579fa94a07fSbrendan uint64_t l2ad_end; /* last addr on device */ 580fa94a07fSbrendan uint64_t l2ad_evict; /* last addr eviction reached */ 581fa94a07fSbrendan boolean_t l2ad_first; /* first sweep through */ 582*5a98e54bSBrendan Gregg - Sun Microsystems boolean_t l2ad_writing; /* currently writing */ 583fa94a07fSbrendan list_t *l2ad_buflist; /* buffer list */ 584fa94a07fSbrendan list_node_t l2ad_node; /* device list node */ 585fa94a07fSbrendan } l2arc_dev_t; 586fa94a07fSbrendan 587fa94a07fSbrendan static list_t L2ARC_dev_list; /* device list */ 588fa94a07fSbrendan static list_t *l2arc_dev_list; /* device list pointer */ 589fa94a07fSbrendan static kmutex_t l2arc_dev_mtx; /* device list mutex */ 590fa94a07fSbrendan static l2arc_dev_t *l2arc_dev_last; /* last device used */ 591fa94a07fSbrendan static kmutex_t l2arc_buflist_mtx; /* mutex for all buflists */ 592fa94a07fSbrendan static list_t L2ARC_free_on_write; /* free after write buf list */ 593fa94a07fSbrendan static list_t *l2arc_free_on_write; /* free after write list ptr */ 594fa94a07fSbrendan static kmutex_t l2arc_free_on_write_mtx; /* mutex for list */ 595fa94a07fSbrendan static uint64_t l2arc_ndev; /* number of devices */ 596fa94a07fSbrendan 597fa94a07fSbrendan typedef struct l2arc_read_callback { 598fa94a07fSbrendan arc_buf_t *l2rcb_buf; /* read buffer */ 599fa94a07fSbrendan spa_t *l2rcb_spa; /* spa */ 600fa94a07fSbrendan blkptr_t l2rcb_bp; /* original blkptr */ 601fa94a07fSbrendan zbookmark_t l2rcb_zb; /* original bookmark */ 602fa94a07fSbrendan int l2rcb_flags; /* original flags */ 603fa94a07fSbrendan } l2arc_read_callback_t; 604fa94a07fSbrendan 605fa94a07fSbrendan typedef struct l2arc_write_callback { 606fa94a07fSbrendan l2arc_dev_t *l2wcb_dev; /* device info */ 607fa94a07fSbrendan arc_buf_hdr_t *l2wcb_head; /* head of write buflist */ 608fa94a07fSbrendan } l2arc_write_callback_t; 609fa94a07fSbrendan 610fa94a07fSbrendan struct l2arc_buf_hdr { 611fa94a07fSbrendan /* protected by arc_buf_hdr mutex */ 612fa94a07fSbrendan l2arc_dev_t *b_dev; /* L2ARC device */ 613fa94a07fSbrendan daddr_t b_daddr; /* disk address, offset byte */ 614fa94a07fSbrendan }; 615fa94a07fSbrendan 616fa94a07fSbrendan typedef struct l2arc_data_free { 617fa94a07fSbrendan /* protected by l2arc_free_on_write_mtx */ 618fa94a07fSbrendan void *l2df_data; 619fa94a07fSbrendan size_t l2df_size; 620fa94a07fSbrendan void (*l2df_func)(void *, size_t); 621fa94a07fSbrendan list_node_t l2df_list_node; 622fa94a07fSbrendan } l2arc_data_free_t; 623fa94a07fSbrendan 624fa94a07fSbrendan static kmutex_t l2arc_feed_thr_lock; 625fa94a07fSbrendan static kcondvar_t l2arc_feed_thr_cv; 626fa94a07fSbrendan static uint8_t l2arc_thread_exit; 627fa94a07fSbrendan 628fa94a07fSbrendan static void l2arc_read_done(zio_t *zio); 629fa94a07fSbrendan static void l2arc_hdr_stat_add(void); 630fa94a07fSbrendan static void l2arc_hdr_stat_remove(void); 631fa94a07fSbrendan 632fa9e4066Sahrens static uint64_t 633088f3894Sahrens buf_hash(spa_t *spa, const dva_t *dva, uint64_t birth) 634fa9e4066Sahrens { 635fa9e4066Sahrens uintptr_t spav = (uintptr_t)spa; 636fa9e4066Sahrens uint8_t *vdva = (uint8_t *)dva; 637fa9e4066Sahrens uint64_t crc = -1ULL; 638fa9e4066Sahrens int i; 639fa9e4066Sahrens 640fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 641fa9e4066Sahrens 642fa9e4066Sahrens for (i = 0; i < sizeof (dva_t); i++) 643fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF]; 644fa9e4066Sahrens 645fa9e4066Sahrens crc ^= (spav>>8) ^ birth; 646fa9e4066Sahrens 647fa9e4066Sahrens return (crc); 648fa9e4066Sahrens } 649fa9e4066Sahrens 650fa9e4066Sahrens #define BUF_EMPTY(buf) \ 651fa9e4066Sahrens ((buf)->b_dva.dva_word[0] == 0 && \ 652fa9e4066Sahrens (buf)->b_dva.dva_word[1] == 0 && \ 653fa9e4066Sahrens (buf)->b_birth == 0) 654fa9e4066Sahrens 655fa9e4066Sahrens #define BUF_EQUAL(spa, dva, birth, buf) \ 656fa9e4066Sahrens ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \ 657fa9e4066Sahrens ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \ 658fa9e4066Sahrens ((buf)->b_birth == birth) && ((buf)->b_spa == spa) 659fa9e4066Sahrens 660fa9e4066Sahrens static arc_buf_hdr_t * 661088f3894Sahrens buf_hash_find(spa_t *spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp) 662fa9e4066Sahrens { 663fa9e4066Sahrens uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); 664fa9e4066Sahrens kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 665fa9e4066Sahrens arc_buf_hdr_t *buf; 666fa9e4066Sahrens 667fa9e4066Sahrens mutex_enter(hash_lock); 668fa9e4066Sahrens for (buf = buf_hash_table.ht_table[idx]; buf != NULL; 669fa9e4066Sahrens buf = buf->b_hash_next) { 670fa9e4066Sahrens if (BUF_EQUAL(spa, dva, birth, buf)) { 671fa9e4066Sahrens *lockp = hash_lock; 672fa9e4066Sahrens return (buf); 673fa9e4066Sahrens } 674fa9e4066Sahrens } 675fa9e4066Sahrens mutex_exit(hash_lock); 676fa9e4066Sahrens *lockp = NULL; 677fa9e4066Sahrens return (NULL); 678fa9e4066Sahrens } 679fa9e4066Sahrens 680fa9e4066Sahrens /* 681fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 682fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 683fa9e4066Sahrens * will be returned and the new element will not be inserted. 684fa9e4066Sahrens * Otherwise returns NULL. 685fa9e4066Sahrens */ 686fa9e4066Sahrens static arc_buf_hdr_t * 687fa9e4066Sahrens buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp) 688fa9e4066Sahrens { 689fa9e4066Sahrens uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 690fa9e4066Sahrens kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 691fa9e4066Sahrens arc_buf_hdr_t *fbuf; 69244cb6abcSbmc uint32_t i; 693fa9e4066Sahrens 694ea8dc4b6Seschrock ASSERT(!HDR_IN_HASH_TABLE(buf)); 695fa9e4066Sahrens *lockp = hash_lock; 696fa9e4066Sahrens mutex_enter(hash_lock); 697fa9e4066Sahrens for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL; 698fa9e4066Sahrens fbuf = fbuf->b_hash_next, i++) { 699fa9e4066Sahrens if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf)) 700fa9e4066Sahrens return (fbuf); 701fa9e4066Sahrens } 702fa9e4066Sahrens 703fa9e4066Sahrens buf->b_hash_next = buf_hash_table.ht_table[idx]; 704fa9e4066Sahrens buf_hash_table.ht_table[idx] = buf; 705ea8dc4b6Seschrock buf->b_flags |= ARC_IN_HASH_TABLE; 706fa9e4066Sahrens 707fa9e4066Sahrens /* collect some hash table performance data */ 708fa9e4066Sahrens if (i > 0) { 70944cb6abcSbmc ARCSTAT_BUMP(arcstat_hash_collisions); 710fa9e4066Sahrens if (i == 1) 71144cb6abcSbmc ARCSTAT_BUMP(arcstat_hash_chains); 71244cb6abcSbmc 71344cb6abcSbmc ARCSTAT_MAX(arcstat_hash_chain_max, i); 714fa9e4066Sahrens } 71544cb6abcSbmc 71644cb6abcSbmc ARCSTAT_BUMP(arcstat_hash_elements); 71744cb6abcSbmc ARCSTAT_MAXSTAT(arcstat_hash_elements); 718fa9e4066Sahrens 719fa9e4066Sahrens return (NULL); 720fa9e4066Sahrens } 721fa9e4066Sahrens 722fa9e4066Sahrens static void 723fa9e4066Sahrens buf_hash_remove(arc_buf_hdr_t *buf) 724fa9e4066Sahrens { 725fa9e4066Sahrens arc_buf_hdr_t *fbuf, **bufp; 726fa9e4066Sahrens uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 727fa9e4066Sahrens 728fa9e4066Sahrens ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx))); 729ea8dc4b6Seschrock ASSERT(HDR_IN_HASH_TABLE(buf)); 730fa9e4066Sahrens 731fa9e4066Sahrens bufp = &buf_hash_table.ht_table[idx]; 732fa9e4066Sahrens while ((fbuf = *bufp) != buf) { 733fa9e4066Sahrens ASSERT(fbuf != NULL); 734fa9e4066Sahrens bufp = &fbuf->b_hash_next; 735fa9e4066Sahrens } 736fa9e4066Sahrens *bufp = buf->b_hash_next; 737fa9e4066Sahrens buf->b_hash_next = NULL; 738ea8dc4b6Seschrock buf->b_flags &= ~ARC_IN_HASH_TABLE; 739fa9e4066Sahrens 740fa9e4066Sahrens /* collect some hash table performance data */ 74144cb6abcSbmc ARCSTAT_BUMPDOWN(arcstat_hash_elements); 74244cb6abcSbmc 743fa9e4066Sahrens if (buf_hash_table.ht_table[idx] && 744fa9e4066Sahrens buf_hash_table.ht_table[idx]->b_hash_next == NULL) 74544cb6abcSbmc ARCSTAT_BUMPDOWN(arcstat_hash_chains); 746fa9e4066Sahrens } 747fa9e4066Sahrens 748fa9e4066Sahrens /* 749fa9e4066Sahrens * Global data structures and functions for the buf kmem cache. 750fa9e4066Sahrens */ 751fa9e4066Sahrens static kmem_cache_t *hdr_cache; 752fa9e4066Sahrens static kmem_cache_t *buf_cache; 753fa9e4066Sahrens 754fa9e4066Sahrens static void 755fa9e4066Sahrens buf_fini(void) 756fa9e4066Sahrens { 757fa9e4066Sahrens int i; 758fa9e4066Sahrens 759fa9e4066Sahrens kmem_free(buf_hash_table.ht_table, 760fa9e4066Sahrens (buf_hash_table.ht_mask + 1) * sizeof (void *)); 761fa9e4066Sahrens for (i = 0; i < BUF_LOCKS; i++) 762fa9e4066Sahrens mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock); 763fa9e4066Sahrens kmem_cache_destroy(hdr_cache); 764fa9e4066Sahrens kmem_cache_destroy(buf_cache); 765fa9e4066Sahrens } 766fa9e4066Sahrens 767fa9e4066Sahrens /* 768fa9e4066Sahrens * Constructor callback - called when the cache is empty 769fa9e4066Sahrens * and a new buf is requested. 770fa9e4066Sahrens */ 771fa9e4066Sahrens /* ARGSUSED */ 772fa9e4066Sahrens static int 773fa9e4066Sahrens hdr_cons(void *vbuf, void *unused, int kmflag) 774fa9e4066Sahrens { 775fa9e4066Sahrens arc_buf_hdr_t *buf = vbuf; 776fa9e4066Sahrens 777fa9e4066Sahrens bzero(buf, sizeof (arc_buf_hdr_t)); 778fa9e4066Sahrens refcount_create(&buf->b_refcnt); 779fa9e4066Sahrens cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL); 780c25056deSgw mutex_init(&buf->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL); 781*5a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS); 782fa94a07fSbrendan 783fa9e4066Sahrens return (0); 784fa9e4066Sahrens } 785fa9e4066Sahrens 7866f83844dSMark Maybee /* ARGSUSED */ 7876f83844dSMark Maybee static int 7886f83844dSMark Maybee buf_cons(void *vbuf, void *unused, int kmflag) 7896f83844dSMark Maybee { 7906f83844dSMark Maybee arc_buf_t *buf = vbuf; 7916f83844dSMark Maybee 7926f83844dSMark Maybee bzero(buf, sizeof (arc_buf_t)); 7936f83844dSMark Maybee rw_init(&buf->b_lock, NULL, RW_DEFAULT, NULL); 794*5a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS); 795*5a98e54bSBrendan Gregg - Sun Microsystems 7966f83844dSMark Maybee return (0); 7976f83844dSMark Maybee } 7986f83844dSMark Maybee 799fa9e4066Sahrens /* 800fa9e4066Sahrens * Destructor callback - called when a cached buf is 801fa9e4066Sahrens * no longer required. 802fa9e4066Sahrens */ 803fa9e4066Sahrens /* ARGSUSED */ 804fa9e4066Sahrens static void 805fa9e4066Sahrens hdr_dest(void *vbuf, void *unused) 806fa9e4066Sahrens { 807fa9e4066Sahrens arc_buf_hdr_t *buf = vbuf; 808fa9e4066Sahrens 809fa9e4066Sahrens refcount_destroy(&buf->b_refcnt); 810fa9e4066Sahrens cv_destroy(&buf->b_cv); 811c25056deSgw mutex_destroy(&buf->b_freeze_lock); 812*5a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS); 813fa9e4066Sahrens } 814fa9e4066Sahrens 8156f83844dSMark Maybee /* ARGSUSED */ 8166f83844dSMark Maybee static void 8176f83844dSMark Maybee buf_dest(void *vbuf, void *unused) 8186f83844dSMark Maybee { 8196f83844dSMark Maybee arc_buf_t *buf = vbuf; 8206f83844dSMark Maybee 8216f83844dSMark Maybee rw_destroy(&buf->b_lock); 822*5a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS); 8236f83844dSMark Maybee } 8246f83844dSMark Maybee 825fa9e4066Sahrens /* 826fa9e4066Sahrens * Reclaim callback -- invoked when memory is low. 827fa9e4066Sahrens */ 828fa9e4066Sahrens /* ARGSUSED */ 829fa9e4066Sahrens static void 830fa9e4066Sahrens hdr_recl(void *unused) 831fa9e4066Sahrens { 832fa9e4066Sahrens dprintf("hdr_recl called\n"); 83349e3519aSmaybee /* 83449e3519aSmaybee * umem calls the reclaim func when we destroy the buf cache, 83549e3519aSmaybee * which is after we do arc_fini(). 83649e3519aSmaybee */ 83749e3519aSmaybee if (!arc_dead) 83849e3519aSmaybee cv_signal(&arc_reclaim_thr_cv); 839fa9e4066Sahrens } 840fa9e4066Sahrens 841fa9e4066Sahrens static void 842fa9e4066Sahrens buf_init(void) 843fa9e4066Sahrens { 844fa9e4066Sahrens uint64_t *ct; 845ea8dc4b6Seschrock uint64_t hsize = 1ULL << 12; 846fa9e4066Sahrens int i, j; 847fa9e4066Sahrens 848fa9e4066Sahrens /* 849fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 850ea8dc4b6Seschrock * with an average 64K block size. The table will take up 851ea8dc4b6Seschrock * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers). 852fa9e4066Sahrens */ 853ea8dc4b6Seschrock while (hsize * 65536 < physmem * PAGESIZE) 854fa9e4066Sahrens hsize <<= 1; 855ea8dc4b6Seschrock retry: 856fa9e4066Sahrens buf_hash_table.ht_mask = hsize - 1; 857ea8dc4b6Seschrock buf_hash_table.ht_table = 858ea8dc4b6Seschrock kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP); 859ea8dc4b6Seschrock if (buf_hash_table.ht_table == NULL) { 860ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 8)); 861ea8dc4b6Seschrock hsize >>= 1; 862ea8dc4b6Seschrock goto retry; 863ea8dc4b6Seschrock } 864fa9e4066Sahrens 865fa9e4066Sahrens hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t), 866fa9e4066Sahrens 0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0); 867fa9e4066Sahrens buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t), 8686f83844dSMark Maybee 0, buf_cons, buf_dest, NULL, NULL, NULL, 0); 869fa9e4066Sahrens 870fa9e4066Sahrens for (i = 0; i < 256; i++) 871fa9e4066Sahrens for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--) 872fa9e4066Sahrens *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY); 873fa9e4066Sahrens 874fa9e4066Sahrens for (i = 0; i < BUF_LOCKS; i++) { 875fa9e4066Sahrens mutex_init(&buf_hash_table.ht_locks[i].ht_lock, 876fa9e4066Sahrens NULL, MUTEX_DEFAULT, NULL); 877fa9e4066Sahrens } 878fa9e4066Sahrens } 879fa9e4066Sahrens 880fa9e4066Sahrens #define ARC_MINTIME (hz>>4) /* 62 ms */ 881fa9e4066Sahrens 8826b4acc8bSahrens static void 8836b4acc8bSahrens arc_cksum_verify(arc_buf_t *buf) 8846b4acc8bSahrens { 8856b4acc8bSahrens zio_cksum_t zc; 8866b4acc8bSahrens 887cc60fd72Sahrens if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 8886b4acc8bSahrens return; 8896b4acc8bSahrens 8906b4acc8bSahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 8913ccfa83cSahrens if (buf->b_hdr->b_freeze_cksum == NULL || 8923ccfa83cSahrens (buf->b_hdr->b_flags & ARC_IO_ERROR)) { 8936b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 8946b4acc8bSahrens return; 8956b4acc8bSahrens } 8966b4acc8bSahrens fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 8976b4acc8bSahrens if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc)) 8986b4acc8bSahrens panic("buffer modified while frozen!"); 8996b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 9006b4acc8bSahrens } 9016b4acc8bSahrens 902fa94a07fSbrendan static int 903fa94a07fSbrendan arc_cksum_equal(arc_buf_t *buf) 904fa94a07fSbrendan { 905fa94a07fSbrendan zio_cksum_t zc; 906fa94a07fSbrendan int equal; 907fa94a07fSbrendan 908fa94a07fSbrendan mutex_enter(&buf->b_hdr->b_freeze_lock); 909fa94a07fSbrendan fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 910fa94a07fSbrendan equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc); 911fa94a07fSbrendan mutex_exit(&buf->b_hdr->b_freeze_lock); 912fa94a07fSbrendan 913fa94a07fSbrendan return (equal); 914fa94a07fSbrendan } 915fa94a07fSbrendan 9166b4acc8bSahrens static void 917fa94a07fSbrendan arc_cksum_compute(arc_buf_t *buf, boolean_t force) 9186b4acc8bSahrens { 919fa94a07fSbrendan if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY)) 9206b4acc8bSahrens return; 9216b4acc8bSahrens 9226b4acc8bSahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 9236b4acc8bSahrens if (buf->b_hdr->b_freeze_cksum != NULL) { 9246b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 9256b4acc8bSahrens return; 9266b4acc8bSahrens } 9276b4acc8bSahrens buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP); 9286b4acc8bSahrens fletcher_2_native(buf->b_data, buf->b_hdr->b_size, 9296b4acc8bSahrens buf->b_hdr->b_freeze_cksum); 9306b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 9316b4acc8bSahrens } 9326b4acc8bSahrens 9336b4acc8bSahrens void 9346b4acc8bSahrens arc_buf_thaw(arc_buf_t *buf) 9356b4acc8bSahrens { 936fa94a07fSbrendan if (zfs_flags & ZFS_DEBUG_MODIFY) { 937fa94a07fSbrendan if (buf->b_hdr->b_state != arc_anon) 938fa94a07fSbrendan panic("modifying non-anon buffer!"); 939fa94a07fSbrendan if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS) 940fa94a07fSbrendan panic("modifying buffer while i/o in progress!"); 941fa94a07fSbrendan arc_cksum_verify(buf); 942fa94a07fSbrendan } 9436b4acc8bSahrens 9446b4acc8bSahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 9456b4acc8bSahrens if (buf->b_hdr->b_freeze_cksum != NULL) { 9466b4acc8bSahrens kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 9476b4acc8bSahrens buf->b_hdr->b_freeze_cksum = NULL; 9486b4acc8bSahrens } 9496b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 9506b4acc8bSahrens } 9516b4acc8bSahrens 9526b4acc8bSahrens void 9536b4acc8bSahrens arc_buf_freeze(arc_buf_t *buf) 9546b4acc8bSahrens { 955cc60fd72Sahrens if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 956cc60fd72Sahrens return; 957cc60fd72Sahrens 9586b4acc8bSahrens ASSERT(buf->b_hdr->b_freeze_cksum != NULL || 95944cb6abcSbmc buf->b_hdr->b_state == arc_anon); 960fa94a07fSbrendan arc_cksum_compute(buf, B_FALSE); 9616b4acc8bSahrens } 9626b4acc8bSahrens 963fa9e4066Sahrens static void 964fa9e4066Sahrens add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 965fa9e4066Sahrens { 966fa9e4066Sahrens ASSERT(MUTEX_HELD(hash_lock)); 967fa9e4066Sahrens 968fa9e4066Sahrens if ((refcount_add(&ab->b_refcnt, tag) == 1) && 96944cb6abcSbmc (ab->b_state != arc_anon)) { 970c0a81264Sek uint64_t delta = ab->b_size * ab->b_datacnt; 9710e8c6158Smaybee list_t *list = &ab->b_state->arcs_list[ab->b_type]; 9720e8c6158Smaybee uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type]; 973fa9e4066Sahrens 97444cb6abcSbmc ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx)); 97544cb6abcSbmc mutex_enter(&ab->b_state->arcs_mtx); 976fa9e4066Sahrens ASSERT(list_link_active(&ab->b_arc_node)); 9770e8c6158Smaybee list_remove(list, ab); 978ea8dc4b6Seschrock if (GHOST_STATE(ab->b_state)) { 979ea8dc4b6Seschrock ASSERT3U(ab->b_datacnt, ==, 0); 980ea8dc4b6Seschrock ASSERT3P(ab->b_buf, ==, NULL); 981ea8dc4b6Seschrock delta = ab->b_size; 982ea8dc4b6Seschrock } 983ea8dc4b6Seschrock ASSERT(delta > 0); 9840e8c6158Smaybee ASSERT3U(*size, >=, delta); 9850e8c6158Smaybee atomic_add_64(size, -delta); 98644cb6abcSbmc mutex_exit(&ab->b_state->arcs_mtx); 987088f3894Sahrens /* remove the prefetch flag if we get a reference */ 98813506d1eSmaybee if (ab->b_flags & ARC_PREFETCH) 98913506d1eSmaybee ab->b_flags &= ~ARC_PREFETCH; 990fa9e4066Sahrens } 991fa9e4066Sahrens } 992fa9e4066Sahrens 993fa9e4066Sahrens static int 994fa9e4066Sahrens remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 995fa9e4066Sahrens { 996fa9e4066Sahrens int cnt; 99744cb6abcSbmc arc_state_t *state = ab->b_state; 998fa9e4066Sahrens 99944cb6abcSbmc ASSERT(state == arc_anon || MUTEX_HELD(hash_lock)); 100044cb6abcSbmc ASSERT(!GHOST_STATE(state)); 1001fa9e4066Sahrens 1002fa9e4066Sahrens if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) && 100344cb6abcSbmc (state != arc_anon)) { 10040e8c6158Smaybee uint64_t *size = &state->arcs_lsize[ab->b_type]; 10050e8c6158Smaybee 100644cb6abcSbmc ASSERT(!MUTEX_HELD(&state->arcs_mtx)); 100744cb6abcSbmc mutex_enter(&state->arcs_mtx); 1008fa9e4066Sahrens ASSERT(!list_link_active(&ab->b_arc_node)); 10090e8c6158Smaybee list_insert_head(&state->arcs_list[ab->b_type], ab); 1010ea8dc4b6Seschrock ASSERT(ab->b_datacnt > 0); 10110e8c6158Smaybee atomic_add_64(size, ab->b_size * ab->b_datacnt); 101244cb6abcSbmc mutex_exit(&state->arcs_mtx); 1013fa9e4066Sahrens } 1014fa9e4066Sahrens return (cnt); 1015fa9e4066Sahrens } 1016fa9e4066Sahrens 1017fa9e4066Sahrens /* 1018fa9e4066Sahrens * Move the supplied buffer to the indicated state. The mutex 1019fa9e4066Sahrens * for the buffer must be held by the caller. 1020fa9e4066Sahrens */ 1021fa9e4066Sahrens static void 1022ea8dc4b6Seschrock arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock) 1023fa9e4066Sahrens { 1024ea8dc4b6Seschrock arc_state_t *old_state = ab->b_state; 1025c0a81264Sek int64_t refcnt = refcount_count(&ab->b_refcnt); 1026c0a81264Sek uint64_t from_delta, to_delta; 1027fa9e4066Sahrens 1028fa9e4066Sahrens ASSERT(MUTEX_HELD(hash_lock)); 1029ea8dc4b6Seschrock ASSERT(new_state != old_state); 1030ea8dc4b6Seschrock ASSERT(refcnt == 0 || ab->b_datacnt > 0); 1031ea8dc4b6Seschrock ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state)); 1032ea8dc4b6Seschrock 1033ea8dc4b6Seschrock from_delta = to_delta = ab->b_datacnt * ab->b_size; 1034fa9e4066Sahrens 1035fa9e4066Sahrens /* 1036fa9e4066Sahrens * If this buffer is evictable, transfer it from the 1037fa9e4066Sahrens * old state list to the new state list. 1038fa9e4066Sahrens */ 1039ea8dc4b6Seschrock if (refcnt == 0) { 104044cb6abcSbmc if (old_state != arc_anon) { 104144cb6abcSbmc int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx); 10420e8c6158Smaybee uint64_t *size = &old_state->arcs_lsize[ab->b_type]; 1043ea8dc4b6Seschrock 1044ea8dc4b6Seschrock if (use_mutex) 104544cb6abcSbmc mutex_enter(&old_state->arcs_mtx); 1046fa9e4066Sahrens 1047fa9e4066Sahrens ASSERT(list_link_active(&ab->b_arc_node)); 10480e8c6158Smaybee list_remove(&old_state->arcs_list[ab->b_type], ab); 1049ea8dc4b6Seschrock 105013506d1eSmaybee /* 105113506d1eSmaybee * If prefetching out of the ghost cache, 105213506d1eSmaybee * we will have a non-null datacnt. 105313506d1eSmaybee */ 105413506d1eSmaybee if (GHOST_STATE(old_state) && ab->b_datacnt == 0) { 105513506d1eSmaybee /* ghost elements have a ghost size */ 1056ea8dc4b6Seschrock ASSERT(ab->b_buf == NULL); 1057ea8dc4b6Seschrock from_delta = ab->b_size; 1058ea8dc4b6Seschrock } 10590e8c6158Smaybee ASSERT3U(*size, >=, from_delta); 10600e8c6158Smaybee atomic_add_64(size, -from_delta); 1061ea8dc4b6Seschrock 1062ea8dc4b6Seschrock if (use_mutex) 106344cb6abcSbmc mutex_exit(&old_state->arcs_mtx); 1064fa9e4066Sahrens } 106544cb6abcSbmc if (new_state != arc_anon) { 106644cb6abcSbmc int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx); 10670e8c6158Smaybee uint64_t *size = &new_state->arcs_lsize[ab->b_type]; 1068fa9e4066Sahrens 1069ea8dc4b6Seschrock if (use_mutex) 107044cb6abcSbmc mutex_enter(&new_state->arcs_mtx); 1071ea8dc4b6Seschrock 10720e8c6158Smaybee list_insert_head(&new_state->arcs_list[ab->b_type], ab); 1073ea8dc4b6Seschrock 1074ea8dc4b6Seschrock /* ghost elements have a ghost size */ 1075ea8dc4b6Seschrock if (GHOST_STATE(new_state)) { 1076ea8dc4b6Seschrock ASSERT(ab->b_datacnt == 0); 1077ea8dc4b6Seschrock ASSERT(ab->b_buf == NULL); 1078ea8dc4b6Seschrock to_delta = ab->b_size; 1079ea8dc4b6Seschrock } 10800e8c6158Smaybee atomic_add_64(size, to_delta); 1081ea8dc4b6Seschrock 1082ea8dc4b6Seschrock if (use_mutex) 108344cb6abcSbmc mutex_exit(&new_state->arcs_mtx); 1084fa9e4066Sahrens } 1085fa9e4066Sahrens } 1086fa9e4066Sahrens 1087fa9e4066Sahrens ASSERT(!BUF_EMPTY(ab)); 1088fa94a07fSbrendan if (new_state == arc_anon) { 1089fa9e4066Sahrens buf_hash_remove(ab); 1090fa9e4066Sahrens } 1091fa9e4066Sahrens 1092ea8dc4b6Seschrock /* adjust state sizes */ 1093ea8dc4b6Seschrock if (to_delta) 109444cb6abcSbmc atomic_add_64(&new_state->arcs_size, to_delta); 1095ea8dc4b6Seschrock if (from_delta) { 109644cb6abcSbmc ASSERT3U(old_state->arcs_size, >=, from_delta); 109744cb6abcSbmc atomic_add_64(&old_state->arcs_size, -from_delta); 1098fa9e4066Sahrens } 1099fa9e4066Sahrens ab->b_state = new_state; 1100fa94a07fSbrendan 1101fa94a07fSbrendan /* adjust l2arc hdr stats */ 1102fa94a07fSbrendan if (new_state == arc_l2c_only) 1103fa94a07fSbrendan l2arc_hdr_stat_add(); 1104fa94a07fSbrendan else if (old_state == arc_l2c_only) 1105fa94a07fSbrendan l2arc_hdr_stat_remove(); 1106fa9e4066Sahrens } 1107fa9e4066Sahrens 11080e8c6158Smaybee void 1109*5a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(uint64_t space, arc_space_type_t type) 11100e8c6158Smaybee { 1111*5a98e54bSBrendan Gregg - Sun Microsystems ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 1112*5a98e54bSBrendan Gregg - Sun Microsystems 1113*5a98e54bSBrendan Gregg - Sun Microsystems switch (type) { 1114*5a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_DATA: 1115*5a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, space); 1116*5a98e54bSBrendan Gregg - Sun Microsystems break; 1117*5a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_OTHER: 1118*5a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_other_size, space); 1119*5a98e54bSBrendan Gregg - Sun Microsystems break; 1120*5a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_HDRS: 1121*5a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_hdr_size, space); 1122*5a98e54bSBrendan Gregg - Sun Microsystems break; 1123*5a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_L2HDRS: 1124*5a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_l2_hdr_size, space); 1125*5a98e54bSBrendan Gregg - Sun Microsystems break; 1126*5a98e54bSBrendan Gregg - Sun Microsystems } 1127*5a98e54bSBrendan Gregg - Sun Microsystems 11280e8c6158Smaybee atomic_add_64(&arc_meta_used, space); 11290e8c6158Smaybee atomic_add_64(&arc_size, space); 11300e8c6158Smaybee } 11310e8c6158Smaybee 11320e8c6158Smaybee void 1133*5a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(uint64_t space, arc_space_type_t type) 11340e8c6158Smaybee { 1135*5a98e54bSBrendan Gregg - Sun Microsystems ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 1136*5a98e54bSBrendan Gregg - Sun Microsystems 1137*5a98e54bSBrendan Gregg - Sun Microsystems switch (type) { 1138*5a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_DATA: 1139*5a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, -space); 1140*5a98e54bSBrendan Gregg - Sun Microsystems break; 1141*5a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_OTHER: 1142*5a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_other_size, -space); 1143*5a98e54bSBrendan Gregg - Sun Microsystems break; 1144*5a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_HDRS: 1145*5a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_hdr_size, -space); 1146*5a98e54bSBrendan Gregg - Sun Microsystems break; 1147*5a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_L2HDRS: 1148*5a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_l2_hdr_size, -space); 1149*5a98e54bSBrendan Gregg - Sun Microsystems break; 1150*5a98e54bSBrendan Gregg - Sun Microsystems } 1151*5a98e54bSBrendan Gregg - Sun Microsystems 11520e8c6158Smaybee ASSERT(arc_meta_used >= space); 11530e8c6158Smaybee if (arc_meta_max < arc_meta_used) 11540e8c6158Smaybee arc_meta_max = arc_meta_used; 11550e8c6158Smaybee atomic_add_64(&arc_meta_used, -space); 11560e8c6158Smaybee ASSERT(arc_size >= space); 11570e8c6158Smaybee atomic_add_64(&arc_size, -space); 11580e8c6158Smaybee } 11590e8c6158Smaybee 11600e8c6158Smaybee void * 11610e8c6158Smaybee arc_data_buf_alloc(uint64_t size) 11620e8c6158Smaybee { 11630e8c6158Smaybee if (arc_evict_needed(ARC_BUFC_DATA)) 11640e8c6158Smaybee cv_signal(&arc_reclaim_thr_cv); 11650e8c6158Smaybee atomic_add_64(&arc_size, size); 11660e8c6158Smaybee return (zio_data_buf_alloc(size)); 11670e8c6158Smaybee } 11680e8c6158Smaybee 11690e8c6158Smaybee void 11700e8c6158Smaybee arc_data_buf_free(void *buf, uint64_t size) 11710e8c6158Smaybee { 11720e8c6158Smaybee zio_data_buf_free(buf, size); 11730e8c6158Smaybee ASSERT(arc_size >= size); 11740e8c6158Smaybee atomic_add_64(&arc_size, -size); 11750e8c6158Smaybee } 11760e8c6158Smaybee 1177fa9e4066Sahrens arc_buf_t * 1178ad23a2dbSjohansen arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type) 1179fa9e4066Sahrens { 1180fa9e4066Sahrens arc_buf_hdr_t *hdr; 1181fa9e4066Sahrens arc_buf_t *buf; 1182fa9e4066Sahrens 1183fa9e4066Sahrens ASSERT3U(size, >, 0); 11841ab7f2deSmaybee hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 1185fa9e4066Sahrens ASSERT(BUF_EMPTY(hdr)); 1186fa9e4066Sahrens hdr->b_size = size; 1187ad23a2dbSjohansen hdr->b_type = type; 1188fa9e4066Sahrens hdr->b_spa = spa; 118944cb6abcSbmc hdr->b_state = arc_anon; 1190fa9e4066Sahrens hdr->b_arc_access = 0; 11911ab7f2deSmaybee buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 1192fa9e4066Sahrens buf->b_hdr = hdr; 119344eda4d7Smaybee buf->b_data = NULL; 1194ea8dc4b6Seschrock buf->b_efunc = NULL; 1195ea8dc4b6Seschrock buf->b_private = NULL; 1196fa9e4066Sahrens buf->b_next = NULL; 1197fa9e4066Sahrens hdr->b_buf = buf; 119844eda4d7Smaybee arc_get_data_buf(buf); 1199ea8dc4b6Seschrock hdr->b_datacnt = 1; 1200fa9e4066Sahrens hdr->b_flags = 0; 1201fa9e4066Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1202fa9e4066Sahrens (void) refcount_add(&hdr->b_refcnt, tag); 1203fa9e4066Sahrens 1204fa9e4066Sahrens return (buf); 1205fa9e4066Sahrens } 1206fa9e4066Sahrens 120744eda4d7Smaybee static arc_buf_t * 120844eda4d7Smaybee arc_buf_clone(arc_buf_t *from) 1209ea8dc4b6Seschrock { 121044eda4d7Smaybee arc_buf_t *buf; 121144eda4d7Smaybee arc_buf_hdr_t *hdr = from->b_hdr; 121244eda4d7Smaybee uint64_t size = hdr->b_size; 1213ea8dc4b6Seschrock 12141ab7f2deSmaybee buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 121544eda4d7Smaybee buf->b_hdr = hdr; 121644eda4d7Smaybee buf->b_data = NULL; 121744eda4d7Smaybee buf->b_efunc = NULL; 121844eda4d7Smaybee buf->b_private = NULL; 121944eda4d7Smaybee buf->b_next = hdr->b_buf; 122044eda4d7Smaybee hdr->b_buf = buf; 122144eda4d7Smaybee arc_get_data_buf(buf); 122244eda4d7Smaybee bcopy(from->b_data, buf->b_data, size); 122344eda4d7Smaybee hdr->b_datacnt += 1; 122444eda4d7Smaybee return (buf); 1225ea8dc4b6Seschrock } 1226ea8dc4b6Seschrock 1227ea8dc4b6Seschrock void 1228ea8dc4b6Seschrock arc_buf_add_ref(arc_buf_t *buf, void* tag) 1229ea8dc4b6Seschrock { 123040d7d650Smaybee arc_buf_hdr_t *hdr; 1231ea8dc4b6Seschrock kmutex_t *hash_lock; 1232ea8dc4b6Seschrock 12339b23f181Smaybee /* 12346f83844dSMark Maybee * Check to see if this buffer is evicted. Callers 12356f83844dSMark Maybee * must verify b_data != NULL to know if the add_ref 12366f83844dSMark Maybee * was successful. 12379b23f181Smaybee */ 12386f83844dSMark Maybee rw_enter(&buf->b_lock, RW_READER); 12396f83844dSMark Maybee if (buf->b_data == NULL) { 12406f83844dSMark Maybee rw_exit(&buf->b_lock); 12419b23f181Smaybee return; 124240d7d650Smaybee } 12436f83844dSMark Maybee hdr = buf->b_hdr; 12446f83844dSMark Maybee ASSERT(hdr != NULL); 12459b23f181Smaybee hash_lock = HDR_LOCK(hdr); 12469b23f181Smaybee mutex_enter(hash_lock); 12476f83844dSMark Maybee rw_exit(&buf->b_lock); 1248ea8dc4b6Seschrock 124944cb6abcSbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 1250ea8dc4b6Seschrock add_reference(hdr, hash_lock, tag); 1251*5a98e54bSBrendan Gregg - Sun Microsystems DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 125244eda4d7Smaybee arc_access(hdr, hash_lock); 125344eda4d7Smaybee mutex_exit(hash_lock); 125444cb6abcSbmc ARCSTAT_BUMP(arcstat_hits); 125544cb6abcSbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 125644cb6abcSbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 125744cb6abcSbmc data, metadata, hits); 1258ea8dc4b6Seschrock } 1259ea8dc4b6Seschrock 1260fa94a07fSbrendan /* 1261fa94a07fSbrendan * Free the arc data buffer. If it is an l2arc write in progress, 1262fa94a07fSbrendan * the buffer is placed on l2arc_free_on_write to be freed later. 1263fa94a07fSbrendan */ 1264fa94a07fSbrendan static void 1265fa94a07fSbrendan arc_buf_data_free(arc_buf_hdr_t *hdr, void (*free_func)(void *, size_t), 1266fa94a07fSbrendan void *data, size_t size) 1267fa94a07fSbrendan { 1268fa94a07fSbrendan if (HDR_L2_WRITING(hdr)) { 1269fa94a07fSbrendan l2arc_data_free_t *df; 1270fa94a07fSbrendan df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP); 1271fa94a07fSbrendan df->l2df_data = data; 1272fa94a07fSbrendan df->l2df_size = size; 1273fa94a07fSbrendan df->l2df_func = free_func; 1274fa94a07fSbrendan mutex_enter(&l2arc_free_on_write_mtx); 1275fa94a07fSbrendan list_insert_head(l2arc_free_on_write, df); 1276fa94a07fSbrendan mutex_exit(&l2arc_free_on_write_mtx); 1277fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_free_on_write); 1278fa94a07fSbrendan } else { 1279fa94a07fSbrendan free_func(data, size); 1280fa94a07fSbrendan } 1281fa94a07fSbrendan } 1282fa94a07fSbrendan 1283ea8dc4b6Seschrock static void 128444eda4d7Smaybee arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all) 1285ea8dc4b6Seschrock { 1286ea8dc4b6Seschrock arc_buf_t **bufp; 1287ea8dc4b6Seschrock 1288ea8dc4b6Seschrock /* free up data associated with the buf */ 1289ea8dc4b6Seschrock if (buf->b_data) { 1290ea8dc4b6Seschrock arc_state_t *state = buf->b_hdr->b_state; 1291ea8dc4b6Seschrock uint64_t size = buf->b_hdr->b_size; 1292ad23a2dbSjohansen arc_buf_contents_t type = buf->b_hdr->b_type; 1293ea8dc4b6Seschrock 12946b4acc8bSahrens arc_cksum_verify(buf); 129544eda4d7Smaybee if (!recycle) { 1296ad23a2dbSjohansen if (type == ARC_BUFC_METADATA) { 1297fa94a07fSbrendan arc_buf_data_free(buf->b_hdr, zio_buf_free, 1298fa94a07fSbrendan buf->b_data, size); 1299*5a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(size, ARC_SPACE_DATA); 1300ad23a2dbSjohansen } else { 1301ad23a2dbSjohansen ASSERT(type == ARC_BUFC_DATA); 1302fa94a07fSbrendan arc_buf_data_free(buf->b_hdr, 1303fa94a07fSbrendan zio_data_buf_free, buf->b_data, size); 1304*5a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, -size); 13050e8c6158Smaybee atomic_add_64(&arc_size, -size); 1306ad23a2dbSjohansen } 130744eda4d7Smaybee } 1308ea8dc4b6Seschrock if (list_link_active(&buf->b_hdr->b_arc_node)) { 13090e8c6158Smaybee uint64_t *cnt = &state->arcs_lsize[type]; 13100e8c6158Smaybee 1311ea8dc4b6Seschrock ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt)); 131244cb6abcSbmc ASSERT(state != arc_anon); 13130e8c6158Smaybee 13140e8c6158Smaybee ASSERT3U(*cnt, >=, size); 13150e8c6158Smaybee atomic_add_64(cnt, -size); 1316ea8dc4b6Seschrock } 131744cb6abcSbmc ASSERT3U(state->arcs_size, >=, size); 131844cb6abcSbmc atomic_add_64(&state->arcs_size, -size); 1319ea8dc4b6Seschrock buf->b_data = NULL; 1320ea8dc4b6Seschrock ASSERT(buf->b_hdr->b_datacnt > 0); 1321ea8dc4b6Seschrock buf->b_hdr->b_datacnt -= 1; 1322ea8dc4b6Seschrock } 1323ea8dc4b6Seschrock 1324ea8dc4b6Seschrock /* only remove the buf if requested */ 1325ea8dc4b6Seschrock if (!all) 1326ea8dc4b6Seschrock return; 1327ea8dc4b6Seschrock 1328ea8dc4b6Seschrock /* remove the buf from the hdr list */ 1329ea8dc4b6Seschrock for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next) 1330ea8dc4b6Seschrock continue; 1331ea8dc4b6Seschrock *bufp = buf->b_next; 1332ea8dc4b6Seschrock 1333ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 1334ea8dc4b6Seschrock 1335ea8dc4b6Seschrock /* clean up the buf */ 1336ea8dc4b6Seschrock buf->b_hdr = NULL; 1337ea8dc4b6Seschrock kmem_cache_free(buf_cache, buf); 1338ea8dc4b6Seschrock } 1339ea8dc4b6Seschrock 1340fa9e4066Sahrens static void 1341ea8dc4b6Seschrock arc_hdr_destroy(arc_buf_hdr_t *hdr) 1342fa9e4066Sahrens { 1343fa9e4066Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt)); 134444cb6abcSbmc ASSERT3P(hdr->b_state, ==, arc_anon); 1345ea8dc4b6Seschrock ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 1346088f3894Sahrens ASSERT(!(hdr->b_flags & ARC_STORED)); 1347fa9e4066Sahrens 1348fa94a07fSbrendan if (hdr->b_l2hdr != NULL) { 1349fa94a07fSbrendan if (!MUTEX_HELD(&l2arc_buflist_mtx)) { 1350fa94a07fSbrendan /* 1351fa94a07fSbrendan * To prevent arc_free() and l2arc_evict() from 1352fa94a07fSbrendan * attempting to free the same buffer at the same time, 1353fa94a07fSbrendan * a FREE_IN_PROGRESS flag is given to arc_free() to 1354fa94a07fSbrendan * give it priority. l2arc_evict() can't destroy this 1355fa94a07fSbrendan * header while we are waiting on l2arc_buflist_mtx. 135649cf58c0SBrendan Gregg - Sun Microsystems * 135749cf58c0SBrendan Gregg - Sun Microsystems * The hdr may be removed from l2ad_buflist before we 135849cf58c0SBrendan Gregg - Sun Microsystems * grab l2arc_buflist_mtx, so b_l2hdr is rechecked. 1359fa94a07fSbrendan */ 1360fa94a07fSbrendan mutex_enter(&l2arc_buflist_mtx); 136149cf58c0SBrendan Gregg - Sun Microsystems if (hdr->b_l2hdr != NULL) { 136249cf58c0SBrendan Gregg - Sun Microsystems list_remove(hdr->b_l2hdr->b_dev->l2ad_buflist, 136349cf58c0SBrendan Gregg - Sun Microsystems hdr); 136449cf58c0SBrendan Gregg - Sun Microsystems } 1365fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 1366fa94a07fSbrendan } else { 1367fa94a07fSbrendan list_remove(hdr->b_l2hdr->b_dev->l2ad_buflist, hdr); 1368fa94a07fSbrendan } 1369fa94a07fSbrendan ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size); 1370fa94a07fSbrendan kmem_free(hdr->b_l2hdr, sizeof (l2arc_buf_hdr_t)); 1371fa94a07fSbrendan if (hdr->b_state == arc_l2c_only) 1372fa94a07fSbrendan l2arc_hdr_stat_remove(); 1373fa94a07fSbrendan hdr->b_l2hdr = NULL; 1374fa94a07fSbrendan } 1375fa94a07fSbrendan 1376fa9e4066Sahrens if (!BUF_EMPTY(hdr)) { 1377ea8dc4b6Seschrock ASSERT(!HDR_IN_HASH_TABLE(hdr)); 1378fa9e4066Sahrens bzero(&hdr->b_dva, sizeof (dva_t)); 1379fa9e4066Sahrens hdr->b_birth = 0; 1380fa9e4066Sahrens hdr->b_cksum0 = 0; 1381fa9e4066Sahrens } 1382ea8dc4b6Seschrock while (hdr->b_buf) { 1383fa9e4066Sahrens arc_buf_t *buf = hdr->b_buf; 1384fa9e4066Sahrens 1385ea8dc4b6Seschrock if (buf->b_efunc) { 1386ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 13876f83844dSMark Maybee rw_enter(&buf->b_lock, RW_WRITER); 1388ea8dc4b6Seschrock ASSERT(buf->b_hdr != NULL); 138944eda4d7Smaybee arc_buf_destroy(hdr->b_buf, FALSE, FALSE); 1390ea8dc4b6Seschrock hdr->b_buf = buf->b_next; 139140d7d650Smaybee buf->b_hdr = &arc_eviction_hdr; 1392ea8dc4b6Seschrock buf->b_next = arc_eviction_list; 1393ea8dc4b6Seschrock arc_eviction_list = buf; 13946f83844dSMark Maybee rw_exit(&buf->b_lock); 1395ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1396ea8dc4b6Seschrock } else { 139744eda4d7Smaybee arc_buf_destroy(hdr->b_buf, FALSE, TRUE); 1398ea8dc4b6Seschrock } 1399fa9e4066Sahrens } 14006b4acc8bSahrens if (hdr->b_freeze_cksum != NULL) { 14016b4acc8bSahrens kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 14026b4acc8bSahrens hdr->b_freeze_cksum = NULL; 14036b4acc8bSahrens } 1404ea8dc4b6Seschrock 1405fa9e4066Sahrens ASSERT(!list_link_active(&hdr->b_arc_node)); 1406fa9e4066Sahrens ASSERT3P(hdr->b_hash_next, ==, NULL); 1407fa9e4066Sahrens ASSERT3P(hdr->b_acb, ==, NULL); 1408fa9e4066Sahrens kmem_cache_free(hdr_cache, hdr); 1409fa9e4066Sahrens } 1410fa9e4066Sahrens 1411fa9e4066Sahrens void 1412fa9e4066Sahrens arc_buf_free(arc_buf_t *buf, void *tag) 1413fa9e4066Sahrens { 1414fa9e4066Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 141544cb6abcSbmc int hashed = hdr->b_state != arc_anon; 1416fa9e4066Sahrens 1417ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 1418ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 1419ea8dc4b6Seschrock 1420ea8dc4b6Seschrock if (hashed) { 1421ea8dc4b6Seschrock kmutex_t *hash_lock = HDR_LOCK(hdr); 1422ea8dc4b6Seschrock 1423ea8dc4b6Seschrock mutex_enter(hash_lock); 1424ea8dc4b6Seschrock (void) remove_reference(hdr, hash_lock, tag); 1425ea8dc4b6Seschrock if (hdr->b_datacnt > 1) 142644eda4d7Smaybee arc_buf_destroy(buf, FALSE, TRUE); 1427ea8dc4b6Seschrock else 1428ea8dc4b6Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 1429fa9e4066Sahrens mutex_exit(hash_lock); 1430ea8dc4b6Seschrock } else if (HDR_IO_IN_PROGRESS(hdr)) { 1431ea8dc4b6Seschrock int destroy_hdr; 1432ea8dc4b6Seschrock /* 1433ea8dc4b6Seschrock * We are in the middle of an async write. Don't destroy 1434ea8dc4b6Seschrock * this buffer unless the write completes before we finish 1435ea8dc4b6Seschrock * decrementing the reference count. 1436ea8dc4b6Seschrock */ 1437ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 1438ea8dc4b6Seschrock (void) remove_reference(hdr, NULL, tag); 1439ea8dc4b6Seschrock ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1440ea8dc4b6Seschrock destroy_hdr = !HDR_IO_IN_PROGRESS(hdr); 1441ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1442ea8dc4b6Seschrock if (destroy_hdr) 1443ea8dc4b6Seschrock arc_hdr_destroy(hdr); 1444ea8dc4b6Seschrock } else { 1445ea8dc4b6Seschrock if (remove_reference(hdr, NULL, tag) > 0) { 1446ea8dc4b6Seschrock ASSERT(HDR_IO_ERROR(hdr)); 144744eda4d7Smaybee arc_buf_destroy(buf, FALSE, TRUE); 1448ea8dc4b6Seschrock } else { 1449ea8dc4b6Seschrock arc_hdr_destroy(hdr); 1450ea8dc4b6Seschrock } 1451fa9e4066Sahrens } 1452ea8dc4b6Seschrock } 1453fa9e4066Sahrens 1454ea8dc4b6Seschrock int 1455ea8dc4b6Seschrock arc_buf_remove_ref(arc_buf_t *buf, void* tag) 1456ea8dc4b6Seschrock { 1457ea8dc4b6Seschrock arc_buf_hdr_t *hdr = buf->b_hdr; 1458ea8dc4b6Seschrock kmutex_t *hash_lock = HDR_LOCK(hdr); 1459ea8dc4b6Seschrock int no_callback = (buf->b_efunc == NULL); 1460fa9e4066Sahrens 146144cb6abcSbmc if (hdr->b_state == arc_anon) { 1462ea8dc4b6Seschrock arc_buf_free(buf, tag); 1463ea8dc4b6Seschrock return (no_callback); 1464ea8dc4b6Seschrock } 1465ea8dc4b6Seschrock 1466ea8dc4b6Seschrock mutex_enter(hash_lock); 146744cb6abcSbmc ASSERT(hdr->b_state != arc_anon); 1468ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 1469ea8dc4b6Seschrock 1470ea8dc4b6Seschrock (void) remove_reference(hdr, hash_lock, tag); 1471ea8dc4b6Seschrock if (hdr->b_datacnt > 1) { 1472ea8dc4b6Seschrock if (no_callback) 147344eda4d7Smaybee arc_buf_destroy(buf, FALSE, TRUE); 1474ea8dc4b6Seschrock } else if (no_callback) { 1475ea8dc4b6Seschrock ASSERT(hdr->b_buf == buf && buf->b_next == NULL); 1476ea8dc4b6Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 1477ea8dc4b6Seschrock } 1478ea8dc4b6Seschrock ASSERT(no_callback || hdr->b_datacnt > 1 || 1479ea8dc4b6Seschrock refcount_is_zero(&hdr->b_refcnt)); 1480ea8dc4b6Seschrock mutex_exit(hash_lock); 1481ea8dc4b6Seschrock return (no_callback); 1482fa9e4066Sahrens } 1483fa9e4066Sahrens 1484fa9e4066Sahrens int 1485fa9e4066Sahrens arc_buf_size(arc_buf_t *buf) 1486fa9e4066Sahrens { 1487fa9e4066Sahrens return (buf->b_hdr->b_size); 1488fa9e4066Sahrens } 1489fa9e4066Sahrens 1490fa9e4066Sahrens /* 1491fa9e4066Sahrens * Evict buffers from list until we've removed the specified number of 1492fa9e4066Sahrens * bytes. Move the removed buffers to the appropriate evict state. 149344eda4d7Smaybee * If the recycle flag is set, then attempt to "recycle" a buffer: 149444eda4d7Smaybee * - look for a buffer to evict that is `bytes' long. 149544eda4d7Smaybee * - return the data block from this buffer rather than freeing it. 149644eda4d7Smaybee * This flag is used by callers that are trying to make space for a 149744eda4d7Smaybee * new buffer in a full arc cache. 1498874395d5Smaybee * 1499874395d5Smaybee * This function makes a "best effort". It skips over any buffers 1500874395d5Smaybee * it can't get a hash_lock on, and so may not catch all candidates. 1501874395d5Smaybee * It may also return without evicting as much space as requested. 1502fa9e4066Sahrens */ 150344eda4d7Smaybee static void * 1504874395d5Smaybee arc_evict(arc_state_t *state, spa_t *spa, int64_t bytes, boolean_t recycle, 1505ad23a2dbSjohansen arc_buf_contents_t type) 1506fa9e4066Sahrens { 1507fa9e4066Sahrens arc_state_t *evicted_state; 150844eda4d7Smaybee uint64_t bytes_evicted = 0, skipped = 0, missed = 0; 15093fa51506Smaybee arc_buf_hdr_t *ab, *ab_prev = NULL; 15100e8c6158Smaybee list_t *list = &state->arcs_list[type]; 1511fa9e4066Sahrens kmutex_t *hash_lock; 151244eda4d7Smaybee boolean_t have_lock; 15133fa51506Smaybee void *stolen = NULL; 1514fa9e4066Sahrens 151544cb6abcSbmc ASSERT(state == arc_mru || state == arc_mfu); 1516fa9e4066Sahrens 151744cb6abcSbmc evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 1518fa9e4066Sahrens 151944cb6abcSbmc mutex_enter(&state->arcs_mtx); 152044cb6abcSbmc mutex_enter(&evicted_state->arcs_mtx); 1521fa9e4066Sahrens 15220e8c6158Smaybee for (ab = list_tail(list); ab; ab = ab_prev) { 15230e8c6158Smaybee ab_prev = list_prev(list, ab); 152413506d1eSmaybee /* prefetch buffers have a minimum lifespan */ 152544eda4d7Smaybee if (HDR_IO_IN_PROGRESS(ab) || 1526874395d5Smaybee (spa && ab->b_spa != spa) || 152744eda4d7Smaybee (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) && 152844eda4d7Smaybee lbolt - ab->b_arc_access < arc_min_prefetch_lifespan)) { 152913506d1eSmaybee skipped++; 153013506d1eSmaybee continue; 153113506d1eSmaybee } 15323fa51506Smaybee /* "lookahead" for better eviction candidate */ 15333fa51506Smaybee if (recycle && ab->b_size != bytes && 15343fa51506Smaybee ab_prev && ab_prev->b_size == bytes) 153544eda4d7Smaybee continue; 1536fa9e4066Sahrens hash_lock = HDR_LOCK(ab); 153744eda4d7Smaybee have_lock = MUTEX_HELD(hash_lock); 153844eda4d7Smaybee if (have_lock || mutex_tryenter(hash_lock)) { 1539fa9e4066Sahrens ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0); 1540ea8dc4b6Seschrock ASSERT(ab->b_datacnt > 0); 1541ea8dc4b6Seschrock while (ab->b_buf) { 1542ea8dc4b6Seschrock arc_buf_t *buf = ab->b_buf; 15436f83844dSMark Maybee if (!rw_tryenter(&buf->b_lock, RW_WRITER)) { 15446f83844dSMark Maybee missed += 1; 15456f83844dSMark Maybee break; 15466f83844dSMark Maybee } 154744eda4d7Smaybee if (buf->b_data) { 1548ea8dc4b6Seschrock bytes_evicted += ab->b_size; 1549ad23a2dbSjohansen if (recycle && ab->b_type == type && 1550fa94a07fSbrendan ab->b_size == bytes && 1551fa94a07fSbrendan !HDR_L2_WRITING(ab)) { 15523fa51506Smaybee stolen = buf->b_data; 15533fa51506Smaybee recycle = FALSE; 15543fa51506Smaybee } 155544eda4d7Smaybee } 1556ea8dc4b6Seschrock if (buf->b_efunc) { 1557ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 15583fa51506Smaybee arc_buf_destroy(buf, 15593fa51506Smaybee buf->b_data == stolen, FALSE); 1560ea8dc4b6Seschrock ab->b_buf = buf->b_next; 156140d7d650Smaybee buf->b_hdr = &arc_eviction_hdr; 1562ea8dc4b6Seschrock buf->b_next = arc_eviction_list; 1563ea8dc4b6Seschrock arc_eviction_list = buf; 1564ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 15656f83844dSMark Maybee rw_exit(&buf->b_lock); 1566ea8dc4b6Seschrock } else { 15676f83844dSMark Maybee rw_exit(&buf->b_lock); 15683fa51506Smaybee arc_buf_destroy(buf, 15693fa51506Smaybee buf->b_data == stolen, TRUE); 1570ea8dc4b6Seschrock } 1571ea8dc4b6Seschrock } 15726f83844dSMark Maybee if (ab->b_datacnt == 0) { 15736f83844dSMark Maybee arc_change_state(evicted_state, ab, hash_lock); 15746f83844dSMark Maybee ASSERT(HDR_IN_HASH_TABLE(ab)); 15756f83844dSMark Maybee ab->b_flags |= ARC_IN_HASH_TABLE; 15766f83844dSMark Maybee ab->b_flags &= ~ARC_BUF_AVAILABLE; 15776f83844dSMark Maybee DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab); 15786f83844dSMark Maybee } 157944eda4d7Smaybee if (!have_lock) 158044eda4d7Smaybee mutex_exit(hash_lock); 1581ea8dc4b6Seschrock if (bytes >= 0 && bytes_evicted >= bytes) 1582fa9e4066Sahrens break; 1583fa9e4066Sahrens } else { 158444eda4d7Smaybee missed += 1; 1585fa9e4066Sahrens } 1586fa9e4066Sahrens } 158744cb6abcSbmc 158844cb6abcSbmc mutex_exit(&evicted_state->arcs_mtx); 158944cb6abcSbmc mutex_exit(&state->arcs_mtx); 1590fa9e4066Sahrens 1591fa9e4066Sahrens if (bytes_evicted < bytes) 1592fa9e4066Sahrens dprintf("only evicted %lld bytes from %x", 1593fa9e4066Sahrens (longlong_t)bytes_evicted, state); 1594fa9e4066Sahrens 159544eda4d7Smaybee if (skipped) 159644cb6abcSbmc ARCSTAT_INCR(arcstat_evict_skip, skipped); 159744cb6abcSbmc 159844eda4d7Smaybee if (missed) 159944cb6abcSbmc ARCSTAT_INCR(arcstat_mutex_miss, missed); 1600f4d2e9e6Smaybee 1601f4d2e9e6Smaybee /* 1602f4d2e9e6Smaybee * We have just evicted some date into the ghost state, make 1603f4d2e9e6Smaybee * sure we also adjust the ghost state size if necessary. 1604f4d2e9e6Smaybee */ 1605f4d2e9e6Smaybee if (arc_no_grow && 1606f4d2e9e6Smaybee arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size > arc_c) { 1607f4d2e9e6Smaybee int64_t mru_over = arc_anon->arcs_size + arc_mru->arcs_size + 1608f4d2e9e6Smaybee arc_mru_ghost->arcs_size - arc_c; 1609f4d2e9e6Smaybee 1610f4d2e9e6Smaybee if (mru_over > 0 && arc_mru_ghost->arcs_lsize[type] > 0) { 1611f4d2e9e6Smaybee int64_t todelete = 1612f4d2e9e6Smaybee MIN(arc_mru_ghost->arcs_lsize[type], mru_over); 1613874395d5Smaybee arc_evict_ghost(arc_mru_ghost, NULL, todelete); 1614f4d2e9e6Smaybee } else if (arc_mfu_ghost->arcs_lsize[type] > 0) { 1615f4d2e9e6Smaybee int64_t todelete = MIN(arc_mfu_ghost->arcs_lsize[type], 1616f4d2e9e6Smaybee arc_mru_ghost->arcs_size + 1617f4d2e9e6Smaybee arc_mfu_ghost->arcs_size - arc_c); 1618874395d5Smaybee arc_evict_ghost(arc_mfu_ghost, NULL, todelete); 1619f4d2e9e6Smaybee } 1620f4d2e9e6Smaybee } 162144cb6abcSbmc 16223fa51506Smaybee return (stolen); 1623fa9e4066Sahrens } 1624fa9e4066Sahrens 1625fa9e4066Sahrens /* 1626fa9e4066Sahrens * Remove buffers from list until we've removed the specified number of 1627fa9e4066Sahrens * bytes. Destroy the buffers that are removed. 1628fa9e4066Sahrens */ 1629fa9e4066Sahrens static void 1630874395d5Smaybee arc_evict_ghost(arc_state_t *state, spa_t *spa, int64_t bytes) 1631fa9e4066Sahrens { 1632fa9e4066Sahrens arc_buf_hdr_t *ab, *ab_prev; 16330e8c6158Smaybee list_t *list = &state->arcs_list[ARC_BUFC_DATA]; 1634fa9e4066Sahrens kmutex_t *hash_lock; 1635ea8dc4b6Seschrock uint64_t bytes_deleted = 0; 1636c0a81264Sek uint64_t bufs_skipped = 0; 1637fa9e4066Sahrens 1638ea8dc4b6Seschrock ASSERT(GHOST_STATE(state)); 1639fa9e4066Sahrens top: 164044cb6abcSbmc mutex_enter(&state->arcs_mtx); 16410e8c6158Smaybee for (ab = list_tail(list); ab; ab = ab_prev) { 16420e8c6158Smaybee ab_prev = list_prev(list, ab); 1643874395d5Smaybee if (spa && ab->b_spa != spa) 1644874395d5Smaybee continue; 1645fa9e4066Sahrens hash_lock = HDR_LOCK(ab); 1646fa9e4066Sahrens if (mutex_tryenter(hash_lock)) { 164713506d1eSmaybee ASSERT(!HDR_IO_IN_PROGRESS(ab)); 1648ea8dc4b6Seschrock ASSERT(ab->b_buf == NULL); 164944cb6abcSbmc ARCSTAT_BUMP(arcstat_deleted); 1650fa9e4066Sahrens bytes_deleted += ab->b_size; 1651fa94a07fSbrendan 1652fa94a07fSbrendan if (ab->b_l2hdr != NULL) { 1653fa94a07fSbrendan /* 1654fa94a07fSbrendan * This buffer is cached on the 2nd Level ARC; 1655fa94a07fSbrendan * don't destroy the header. 1656fa94a07fSbrendan */ 1657fa94a07fSbrendan arc_change_state(arc_l2c_only, ab, hash_lock); 1658fa94a07fSbrendan mutex_exit(hash_lock); 1659fa94a07fSbrendan } else { 1660fa94a07fSbrendan arc_change_state(arc_anon, ab, hash_lock); 1661fa94a07fSbrendan mutex_exit(hash_lock); 1662fa94a07fSbrendan arc_hdr_destroy(ab); 1663fa94a07fSbrendan } 1664fa94a07fSbrendan 1665ea8dc4b6Seschrock DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab); 1666fa9e4066Sahrens if (bytes >= 0 && bytes_deleted >= bytes) 1667fa9e4066Sahrens break; 1668fa9e4066Sahrens } else { 1669fa9e4066Sahrens if (bytes < 0) { 167044cb6abcSbmc mutex_exit(&state->arcs_mtx); 1671fa9e4066Sahrens mutex_enter(hash_lock); 1672fa9e4066Sahrens mutex_exit(hash_lock); 1673fa9e4066Sahrens goto top; 1674fa9e4066Sahrens } 1675fa9e4066Sahrens bufs_skipped += 1; 1676fa9e4066Sahrens } 1677fa9e4066Sahrens } 167844cb6abcSbmc mutex_exit(&state->arcs_mtx); 1679fa9e4066Sahrens 16800e8c6158Smaybee if (list == &state->arcs_list[ARC_BUFC_DATA] && 16810e8c6158Smaybee (bytes < 0 || bytes_deleted < bytes)) { 16820e8c6158Smaybee list = &state->arcs_list[ARC_BUFC_METADATA]; 16830e8c6158Smaybee goto top; 16840e8c6158Smaybee } 16850e8c6158Smaybee 1686fa9e4066Sahrens if (bufs_skipped) { 168744cb6abcSbmc ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped); 1688fa9e4066Sahrens ASSERT(bytes >= 0); 1689fa9e4066Sahrens } 1690fa9e4066Sahrens 1691fa9e4066Sahrens if (bytes_deleted < bytes) 1692fa9e4066Sahrens dprintf("only deleted %lld bytes from %p", 1693fa9e4066Sahrens (longlong_t)bytes_deleted, state); 1694fa9e4066Sahrens } 1695fa9e4066Sahrens 1696fa9e4066Sahrens static void 1697fa9e4066Sahrens arc_adjust(void) 1698fa9e4066Sahrens { 1699*5a98e54bSBrendan Gregg - Sun Microsystems int64_t adjustment, delta; 1700fa9e4066Sahrens 1701*5a98e54bSBrendan Gregg - Sun Microsystems /* 1702*5a98e54bSBrendan Gregg - Sun Microsystems * Adjust MRU size 1703*5a98e54bSBrendan Gregg - Sun Microsystems */ 1704*5a98e54bSBrendan Gregg - Sun Microsystems 1705*5a98e54bSBrendan Gregg - Sun Microsystems adjustment = MIN(arc_size - arc_c, 1706*5a98e54bSBrendan Gregg - Sun Microsystems arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used - arc_p); 1707fa9e4066Sahrens 1708*5a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) { 1709*5a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment); 1710*5a98e54bSBrendan Gregg - Sun Microsystems (void) arc_evict(arc_mru, NULL, delta, FALSE, ARC_BUFC_DATA); 1711*5a98e54bSBrendan Gregg - Sun Microsystems adjustment -= delta; 17120e8c6158Smaybee } 17130e8c6158Smaybee 1714*5a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) { 1715*5a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment); 1716*5a98e54bSBrendan Gregg - Sun Microsystems (void) arc_evict(arc_mru, NULL, delta, FALSE, 1717874395d5Smaybee ARC_BUFC_METADATA); 1718fa9e4066Sahrens } 1719fa9e4066Sahrens 1720*5a98e54bSBrendan Gregg - Sun Microsystems /* 1721*5a98e54bSBrendan Gregg - Sun Microsystems * Adjust MFU size 1722*5a98e54bSBrendan Gregg - Sun Microsystems */ 1723fa9e4066Sahrens 1724*5a98e54bSBrendan Gregg - Sun Microsystems adjustment = arc_size - arc_c; 1725*5a98e54bSBrendan Gregg - Sun Microsystems 1726*5a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) { 1727*5a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]); 1728*5a98e54bSBrendan Gregg - Sun Microsystems (void) arc_evict(arc_mfu, NULL, delta, FALSE, ARC_BUFC_DATA); 1729*5a98e54bSBrendan Gregg - Sun Microsystems adjustment -= delta; 1730fa9e4066Sahrens } 1731fa9e4066Sahrens 1732*5a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) { 1733*5a98e54bSBrendan Gregg - Sun Microsystems int64_t delta = MIN(adjustment, 1734*5a98e54bSBrendan Gregg - Sun Microsystems arc_mfu->arcs_lsize[ARC_BUFC_METADATA]); 1735*5a98e54bSBrendan Gregg - Sun Microsystems (void) arc_evict(arc_mfu, NULL, delta, FALSE, 1736*5a98e54bSBrendan Gregg - Sun Microsystems ARC_BUFC_METADATA); 1737*5a98e54bSBrendan Gregg - Sun Microsystems } 1738fa9e4066Sahrens 1739*5a98e54bSBrendan Gregg - Sun Microsystems /* 1740*5a98e54bSBrendan Gregg - Sun Microsystems * Adjust ghost lists 1741*5a98e54bSBrendan Gregg - Sun Microsystems */ 1742fa9e4066Sahrens 1743*5a98e54bSBrendan Gregg - Sun Microsystems adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c; 1744fa9e4066Sahrens 1745*5a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) { 1746*5a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(arc_mru_ghost->arcs_size, adjustment); 1747*5a98e54bSBrendan Gregg - Sun Microsystems arc_evict_ghost(arc_mru_ghost, NULL, delta); 1748*5a98e54bSBrendan Gregg - Sun Microsystems } 17490e8c6158Smaybee 1750*5a98e54bSBrendan Gregg - Sun Microsystems adjustment = 1751*5a98e54bSBrendan Gregg - Sun Microsystems arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c; 1752*5a98e54bSBrendan Gregg - Sun Microsystems 1753*5a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) { 1754*5a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(arc_mfu_ghost->arcs_size, adjustment); 1755*5a98e54bSBrendan Gregg - Sun Microsystems arc_evict_ghost(arc_mfu_ghost, NULL, delta); 1756fa9e4066Sahrens } 1757fa9e4066Sahrens } 1758fa9e4066Sahrens 1759ea8dc4b6Seschrock static void 1760ea8dc4b6Seschrock arc_do_user_evicts(void) 1761ea8dc4b6Seschrock { 1762ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 1763ea8dc4b6Seschrock while (arc_eviction_list != NULL) { 1764ea8dc4b6Seschrock arc_buf_t *buf = arc_eviction_list; 1765ea8dc4b6Seschrock arc_eviction_list = buf->b_next; 17666f83844dSMark Maybee rw_enter(&buf->b_lock, RW_WRITER); 1767ea8dc4b6Seschrock buf->b_hdr = NULL; 17686f83844dSMark Maybee rw_exit(&buf->b_lock); 1769ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1770ea8dc4b6Seschrock 1771dd6ef538Smaybee if (buf->b_efunc != NULL) 1772dd6ef538Smaybee VERIFY(buf->b_efunc(buf) == 0); 1773ea8dc4b6Seschrock 1774ea8dc4b6Seschrock buf->b_efunc = NULL; 1775ea8dc4b6Seschrock buf->b_private = NULL; 1776ea8dc4b6Seschrock kmem_cache_free(buf_cache, buf); 1777ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 1778ea8dc4b6Seschrock } 1779ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1780ea8dc4b6Seschrock } 1781ea8dc4b6Seschrock 1782fa9e4066Sahrens /* 1783874395d5Smaybee * Flush all *evictable* data from the cache for the given spa. 1784fa9e4066Sahrens * NOTE: this will not touch "active" (i.e. referenced) data. 1785fa9e4066Sahrens */ 1786fa9e4066Sahrens void 1787874395d5Smaybee arc_flush(spa_t *spa) 1788fa9e4066Sahrens { 1789874395d5Smaybee while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) { 1790874395d5Smaybee (void) arc_evict(arc_mru, spa, -1, FALSE, ARC_BUFC_DATA); 1791874395d5Smaybee if (spa) 1792874395d5Smaybee break; 1793874395d5Smaybee } 1794874395d5Smaybee while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) { 1795874395d5Smaybee (void) arc_evict(arc_mru, spa, -1, FALSE, ARC_BUFC_METADATA); 1796874395d5Smaybee if (spa) 1797874395d5Smaybee break; 1798874395d5Smaybee } 1799874395d5Smaybee while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) { 1800874395d5Smaybee (void) arc_evict(arc_mfu, spa, -1, FALSE, ARC_BUFC_DATA); 1801874395d5Smaybee if (spa) 1802874395d5Smaybee break; 1803874395d5Smaybee } 1804874395d5Smaybee while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) { 1805874395d5Smaybee (void) arc_evict(arc_mfu, spa, -1, FALSE, ARC_BUFC_METADATA); 1806874395d5Smaybee if (spa) 1807874395d5Smaybee break; 1808874395d5Smaybee } 1809874395d5Smaybee 1810874395d5Smaybee arc_evict_ghost(arc_mru_ghost, spa, -1); 1811874395d5Smaybee arc_evict_ghost(arc_mfu_ghost, spa, -1); 1812ea8dc4b6Seschrock 1813ea8dc4b6Seschrock mutex_enter(&arc_reclaim_thr_lock); 1814ea8dc4b6Seschrock arc_do_user_evicts(); 1815ea8dc4b6Seschrock mutex_exit(&arc_reclaim_thr_lock); 1816874395d5Smaybee ASSERT(spa || arc_eviction_list == NULL); 1817fa9e4066Sahrens } 1818fa9e4066Sahrens 1819fa9e4066Sahrens void 182049e3519aSmaybee arc_shrink(void) 1821fa9e4066Sahrens { 182244cb6abcSbmc if (arc_c > arc_c_min) { 182349e3519aSmaybee uint64_t to_free; 1824fa9e4066Sahrens 18253cff2f43Sstans #ifdef _KERNEL 182644cb6abcSbmc to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree)); 18273cff2f43Sstans #else 182844cb6abcSbmc to_free = arc_c >> arc_shrink_shift; 18293cff2f43Sstans #endif 183044cb6abcSbmc if (arc_c > arc_c_min + to_free) 183144cb6abcSbmc atomic_add_64(&arc_c, -to_free); 183249e3519aSmaybee else 183344cb6abcSbmc arc_c = arc_c_min; 183444cb6abcSbmc 183544cb6abcSbmc atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift)); 183644cb6abcSbmc if (arc_c > arc_size) 183744cb6abcSbmc arc_c = MAX(arc_size, arc_c_min); 183844cb6abcSbmc if (arc_p > arc_c) 183944cb6abcSbmc arc_p = (arc_c >> 1); 184044cb6abcSbmc ASSERT(arc_c >= arc_c_min); 184144cb6abcSbmc ASSERT((int64_t)arc_p >= 0); 184249e3519aSmaybee } 1843fa9e4066Sahrens 184444cb6abcSbmc if (arc_size > arc_c) 184549e3519aSmaybee arc_adjust(); 1846fa9e4066Sahrens } 1847fa9e4066Sahrens 1848fa9e4066Sahrens static int 1849fa9e4066Sahrens arc_reclaim_needed(void) 1850fa9e4066Sahrens { 1851fa9e4066Sahrens uint64_t extra; 1852fa9e4066Sahrens 1853fa9e4066Sahrens #ifdef _KERNEL 18543cff2f43Sstans 18553cff2f43Sstans if (needfree) 18563cff2f43Sstans return (1); 18573cff2f43Sstans 1858fa9e4066Sahrens /* 1859fa9e4066Sahrens * take 'desfree' extra pages, so we reclaim sooner, rather than later 1860fa9e4066Sahrens */ 1861fa9e4066Sahrens extra = desfree; 1862fa9e4066Sahrens 1863fa9e4066Sahrens /* 1864fa9e4066Sahrens * check that we're out of range of the pageout scanner. It starts to 1865fa9e4066Sahrens * schedule paging if freemem is less than lotsfree and needfree. 1866fa9e4066Sahrens * lotsfree is the high-water mark for pageout, and needfree is the 1867fa9e4066Sahrens * number of needed free pages. We add extra pages here to make sure 1868fa9e4066Sahrens * the scanner doesn't start up while we're freeing memory. 1869fa9e4066Sahrens */ 1870fa9e4066Sahrens if (freemem < lotsfree + needfree + extra) 1871fa9e4066Sahrens return (1); 1872fa9e4066Sahrens 1873fa9e4066Sahrens /* 1874fa9e4066Sahrens * check to make sure that swapfs has enough space so that anon 1875fa94a07fSbrendan * reservations can still succeed. anon_resvmem() checks that the 1876fa9e4066Sahrens * availrmem is greater than swapfs_minfree, and the number of reserved 1877fa9e4066Sahrens * swap pages. We also add a bit of extra here just to prevent 1878fa9e4066Sahrens * circumstances from getting really dire. 1879fa9e4066Sahrens */ 1880fa9e4066Sahrens if (availrmem < swapfs_minfree + swapfs_reserve + extra) 1881fa9e4066Sahrens return (1); 1882fa9e4066Sahrens 18835dc8af33Smaybee #if defined(__i386) 1884fa9e4066Sahrens /* 1885fa9e4066Sahrens * If we're on an i386 platform, it's possible that we'll exhaust the 1886fa9e4066Sahrens * kernel heap space before we ever run out of available physical 1887fa9e4066Sahrens * memory. Most checks of the size of the heap_area compare against 1888fa9e4066Sahrens * tune.t_minarmem, which is the minimum available real memory that we 1889fa9e4066Sahrens * can have in the system. However, this is generally fixed at 25 pages 1890fa9e4066Sahrens * which is so low that it's useless. In this comparison, we seek to 1891fa9e4066Sahrens * calculate the total heap-size, and reclaim if more than 3/4ths of the 1892fa94a07fSbrendan * heap is allocated. (Or, in the calculation, if less than 1/4th is 1893fa9e4066Sahrens * free) 1894fa9e4066Sahrens */ 1895fa9e4066Sahrens if (btop(vmem_size(heap_arena, VMEM_FREE)) < 1896fa9e4066Sahrens (btop(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2)) 1897fa9e4066Sahrens return (1); 1898fa9e4066Sahrens #endif 1899fa9e4066Sahrens 1900fa9e4066Sahrens #else 1901fa9e4066Sahrens if (spa_get_random(100) == 0) 1902fa9e4066Sahrens return (1); 1903fa9e4066Sahrens #endif 1904fa9e4066Sahrens return (0); 1905fa9e4066Sahrens } 1906fa9e4066Sahrens 1907fa9e4066Sahrens static void 1908fa9e4066Sahrens arc_kmem_reap_now(arc_reclaim_strategy_t strat) 1909fa9e4066Sahrens { 1910fa9e4066Sahrens size_t i; 1911fa9e4066Sahrens kmem_cache_t *prev_cache = NULL; 1912ad23a2dbSjohansen kmem_cache_t *prev_data_cache = NULL; 1913fa9e4066Sahrens extern kmem_cache_t *zio_buf_cache[]; 1914ad23a2dbSjohansen extern kmem_cache_t *zio_data_buf_cache[]; 1915fa9e4066Sahrens 1916033f9833Sek #ifdef _KERNEL 19170e8c6158Smaybee if (arc_meta_used >= arc_meta_limit) { 19180e8c6158Smaybee /* 19190e8c6158Smaybee * We are exceeding our meta-data cache limit. 19200e8c6158Smaybee * Purge some DNLC entries to release holds on meta-data. 19210e8c6158Smaybee */ 19220e8c6158Smaybee dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); 19230e8c6158Smaybee } 19245dc8af33Smaybee #if defined(__i386) 19255dc8af33Smaybee /* 19265dc8af33Smaybee * Reclaim unused memory from all kmem caches. 19275dc8af33Smaybee */ 19285dc8af33Smaybee kmem_reap(); 19295dc8af33Smaybee #endif 1930033f9833Sek #endif 1931033f9833Sek 1932fa9e4066Sahrens /* 1933fa94a07fSbrendan * An aggressive reclamation will shrink the cache size as well as 1934ea8dc4b6Seschrock * reap free buffers from the arc kmem caches. 1935fa9e4066Sahrens */ 1936fa9e4066Sahrens if (strat == ARC_RECLAIM_AGGR) 193749e3519aSmaybee arc_shrink(); 1938fa9e4066Sahrens 1939fa9e4066Sahrens for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { 1940fa9e4066Sahrens if (zio_buf_cache[i] != prev_cache) { 1941fa9e4066Sahrens prev_cache = zio_buf_cache[i]; 1942fa9e4066Sahrens kmem_cache_reap_now(zio_buf_cache[i]); 1943fa9e4066Sahrens } 1944ad23a2dbSjohansen if (zio_data_buf_cache[i] != prev_data_cache) { 1945ad23a2dbSjohansen prev_data_cache = zio_data_buf_cache[i]; 1946ad23a2dbSjohansen kmem_cache_reap_now(zio_data_buf_cache[i]); 1947ad23a2dbSjohansen } 1948fa9e4066Sahrens } 1949ea8dc4b6Seschrock kmem_cache_reap_now(buf_cache); 1950ea8dc4b6Seschrock kmem_cache_reap_now(hdr_cache); 1951fa9e4066Sahrens } 1952fa9e4066Sahrens 1953fa9e4066Sahrens static void 1954fa9e4066Sahrens arc_reclaim_thread(void) 1955fa9e4066Sahrens { 1956fa9e4066Sahrens clock_t growtime = 0; 1957fa9e4066Sahrens arc_reclaim_strategy_t last_reclaim = ARC_RECLAIM_CONS; 1958fa9e4066Sahrens callb_cpr_t cpr; 1959fa9e4066Sahrens 1960fa9e4066Sahrens CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG); 1961fa9e4066Sahrens 1962fa9e4066Sahrens mutex_enter(&arc_reclaim_thr_lock); 1963fa9e4066Sahrens while (arc_thread_exit == 0) { 1964fa9e4066Sahrens if (arc_reclaim_needed()) { 1965fa9e4066Sahrens 196644cb6abcSbmc if (arc_no_grow) { 1967fa9e4066Sahrens if (last_reclaim == ARC_RECLAIM_CONS) { 1968fa9e4066Sahrens last_reclaim = ARC_RECLAIM_AGGR; 1969fa9e4066Sahrens } else { 1970fa9e4066Sahrens last_reclaim = ARC_RECLAIM_CONS; 1971fa9e4066Sahrens } 1972fa9e4066Sahrens } else { 197344cb6abcSbmc arc_no_grow = TRUE; 1974fa9e4066Sahrens last_reclaim = ARC_RECLAIM_AGGR; 1975fa9e4066Sahrens membar_producer(); 1976fa9e4066Sahrens } 1977fa9e4066Sahrens 1978fa9e4066Sahrens /* reset the growth delay for every reclaim */ 1979fa9e4066Sahrens growtime = lbolt + (arc_grow_retry * hz); 1980fa9e4066Sahrens 1981fa9e4066Sahrens arc_kmem_reap_now(last_reclaim); 19823a737e0dSbrendan arc_warm = B_TRUE; 1983fa9e4066Sahrens 19840e8c6158Smaybee } else if (arc_no_grow && lbolt >= growtime) { 198544cb6abcSbmc arc_no_grow = FALSE; 1986fa9e4066Sahrens } 1987fa9e4066Sahrens 198844cb6abcSbmc if (2 * arc_c < arc_size + 198944cb6abcSbmc arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size) 1990641fbdaeSmaybee arc_adjust(); 1991641fbdaeSmaybee 1992ea8dc4b6Seschrock if (arc_eviction_list != NULL) 1993ea8dc4b6Seschrock arc_do_user_evicts(); 1994ea8dc4b6Seschrock 1995fa9e4066Sahrens /* block until needed, or one second, whichever is shorter */ 1996fa9e4066Sahrens CALLB_CPR_SAFE_BEGIN(&cpr); 1997fa9e4066Sahrens (void) cv_timedwait(&arc_reclaim_thr_cv, 1998fa9e4066Sahrens &arc_reclaim_thr_lock, (lbolt + hz)); 1999fa9e4066Sahrens CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock); 2000fa9e4066Sahrens } 2001fa9e4066Sahrens 2002fa9e4066Sahrens arc_thread_exit = 0; 2003fa9e4066Sahrens cv_broadcast(&arc_reclaim_thr_cv); 2004fa9e4066Sahrens CALLB_CPR_EXIT(&cpr); /* drops arc_reclaim_thr_lock */ 2005fa9e4066Sahrens thread_exit(); 2006fa9e4066Sahrens } 2007fa9e4066Sahrens 2008ea8dc4b6Seschrock /* 2009ea8dc4b6Seschrock * Adapt arc info given the number of bytes we are trying to add and 2010ea8dc4b6Seschrock * the state that we are comming from. This function is only called 2011ea8dc4b6Seschrock * when we are adding new content to the cache. 2012ea8dc4b6Seschrock */ 2013fa9e4066Sahrens static void 2014ea8dc4b6Seschrock arc_adapt(int bytes, arc_state_t *state) 2015fa9e4066Sahrens { 2016ea8dc4b6Seschrock int mult; 2017*5a98e54bSBrendan Gregg - Sun Microsystems uint64_t arc_p_min = (arc_c >> arc_p_min_shift); 2018ea8dc4b6Seschrock 2019fa94a07fSbrendan if (state == arc_l2c_only) 2020fa94a07fSbrendan return; 2021fa94a07fSbrendan 2022ea8dc4b6Seschrock ASSERT(bytes > 0); 2023fa9e4066Sahrens /* 2024ea8dc4b6Seschrock * Adapt the target size of the MRU list: 2025ea8dc4b6Seschrock * - if we just hit in the MRU ghost list, then increase 2026ea8dc4b6Seschrock * the target size of the MRU list. 2027ea8dc4b6Seschrock * - if we just hit in the MFU ghost list, then increase 2028ea8dc4b6Seschrock * the target size of the MFU list by decreasing the 2029ea8dc4b6Seschrock * target size of the MRU list. 2030fa9e4066Sahrens */ 203144cb6abcSbmc if (state == arc_mru_ghost) { 203244cb6abcSbmc mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ? 203344cb6abcSbmc 1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size)); 2034ea8dc4b6Seschrock 2035*5a98e54bSBrendan Gregg - Sun Microsystems arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult); 203644cb6abcSbmc } else if (state == arc_mfu_ghost) { 2037*5a98e54bSBrendan Gregg - Sun Microsystems uint64_t delta; 2038*5a98e54bSBrendan Gregg - Sun Microsystems 203944cb6abcSbmc mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ? 204044cb6abcSbmc 1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size)); 2041ea8dc4b6Seschrock 2042*5a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(bytes * mult, arc_p); 2043*5a98e54bSBrendan Gregg - Sun Microsystems arc_p = MAX(arc_p_min, arc_p - delta); 2044ea8dc4b6Seschrock } 204544cb6abcSbmc ASSERT((int64_t)arc_p >= 0); 2046fa9e4066Sahrens 2047fa9e4066Sahrens if (arc_reclaim_needed()) { 2048fa9e4066Sahrens cv_signal(&arc_reclaim_thr_cv); 2049fa9e4066Sahrens return; 2050fa9e4066Sahrens } 2051fa9e4066Sahrens 205244cb6abcSbmc if (arc_no_grow) 2053fa9e4066Sahrens return; 2054fa9e4066Sahrens 205544cb6abcSbmc if (arc_c >= arc_c_max) 2056ea8dc4b6Seschrock return; 2057ea8dc4b6Seschrock 2058fa9e4066Sahrens /* 2059ea8dc4b6Seschrock * If we're within (2 * maxblocksize) bytes of the target 2060ea8dc4b6Seschrock * cache size, increment the target cache size 2061fa9e4066Sahrens */ 206244cb6abcSbmc if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) { 206344cb6abcSbmc atomic_add_64(&arc_c, (int64_t)bytes); 206444cb6abcSbmc if (arc_c > arc_c_max) 206544cb6abcSbmc arc_c = arc_c_max; 206644cb6abcSbmc else if (state == arc_anon) 206744cb6abcSbmc atomic_add_64(&arc_p, (int64_t)bytes); 206844cb6abcSbmc if (arc_p > arc_c) 206944cb6abcSbmc arc_p = arc_c; 2070fa9e4066Sahrens } 207144cb6abcSbmc ASSERT((int64_t)arc_p >= 0); 2072fa9e4066Sahrens } 2073fa9e4066Sahrens 2074fa9e4066Sahrens /* 2075ea8dc4b6Seschrock * Check if the cache has reached its limits and eviction is required 2076ea8dc4b6Seschrock * prior to insert. 2077fa9e4066Sahrens */ 2078fa9e4066Sahrens static int 20790e8c6158Smaybee arc_evict_needed(arc_buf_contents_t type) 2080fa9e4066Sahrens { 20810e8c6158Smaybee if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit) 20820e8c6158Smaybee return (1); 20830e8c6158Smaybee 20840e8c6158Smaybee #ifdef _KERNEL 20850e8c6158Smaybee /* 20860e8c6158Smaybee * If zio data pages are being allocated out of a separate heap segment, 20870e8c6158Smaybee * then enforce that the size of available vmem for this area remains 20880e8c6158Smaybee * above about 1/32nd free. 20890e8c6158Smaybee */ 20900e8c6158Smaybee if (type == ARC_BUFC_DATA && zio_arena != NULL && 20910e8c6158Smaybee vmem_size(zio_arena, VMEM_FREE) < 20920e8c6158Smaybee (vmem_size(zio_arena, VMEM_ALLOC) >> 5)) 20930e8c6158Smaybee return (1); 20940e8c6158Smaybee #endif 20950e8c6158Smaybee 2096fa9e4066Sahrens if (arc_reclaim_needed()) 2097fa9e4066Sahrens return (1); 2098fa9e4066Sahrens 209944cb6abcSbmc return (arc_size > arc_c); 2100fa9e4066Sahrens } 2101fa9e4066Sahrens 2102fa9e4066Sahrens /* 210344eda4d7Smaybee * The buffer, supplied as the first argument, needs a data block. 210444eda4d7Smaybee * So, if we are at cache max, determine which cache should be victimized. 210544eda4d7Smaybee * We have the following cases: 2106fa9e4066Sahrens * 210744cb6abcSbmc * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) -> 2108fa9e4066Sahrens * In this situation if we're out of space, but the resident size of the MFU is 2109fa9e4066Sahrens * under the limit, victimize the MFU cache to satisfy this insertion request. 2110fa9e4066Sahrens * 211144cb6abcSbmc * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) -> 2112fa9e4066Sahrens * Here, we've used up all of the available space for the MRU, so we need to 2113fa9e4066Sahrens * evict from our own cache instead. Evict from the set of resident MRU 2114fa9e4066Sahrens * entries. 2115fa9e4066Sahrens * 211644cb6abcSbmc * 3. Insert for MFU (c - p) > sizeof(arc_mfu) -> 2117fa9e4066Sahrens * c minus p represents the MFU space in the cache, since p is the size of the 2118fa9e4066Sahrens * cache that is dedicated to the MRU. In this situation there's still space on 2119fa9e4066Sahrens * the MFU side, so the MRU side needs to be victimized. 2120fa9e4066Sahrens * 212144cb6abcSbmc * 4. Insert for MFU (c - p) < sizeof(arc_mfu) -> 2122fa9e4066Sahrens * MFU's resident set is consuming more space than it has been allotted. In 2123fa9e4066Sahrens * this situation, we must victimize our own cache, the MFU, for this insertion. 2124fa9e4066Sahrens */ 2125fa9e4066Sahrens static void 212644eda4d7Smaybee arc_get_data_buf(arc_buf_t *buf) 2127fa9e4066Sahrens { 2128ad23a2dbSjohansen arc_state_t *state = buf->b_hdr->b_state; 2129ad23a2dbSjohansen uint64_t size = buf->b_hdr->b_size; 2130ad23a2dbSjohansen arc_buf_contents_t type = buf->b_hdr->b_type; 2131fa9e4066Sahrens 213244eda4d7Smaybee arc_adapt(size, state); 2133fa9e4066Sahrens 213444eda4d7Smaybee /* 213544eda4d7Smaybee * We have not yet reached cache maximum size, 213644eda4d7Smaybee * just allocate a new buffer. 213744eda4d7Smaybee */ 21380e8c6158Smaybee if (!arc_evict_needed(type)) { 2139ad23a2dbSjohansen if (type == ARC_BUFC_METADATA) { 2140ad23a2dbSjohansen buf->b_data = zio_buf_alloc(size); 2141*5a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(size, ARC_SPACE_DATA); 2142ad23a2dbSjohansen } else { 2143ad23a2dbSjohansen ASSERT(type == ARC_BUFC_DATA); 2144ad23a2dbSjohansen buf->b_data = zio_data_buf_alloc(size); 2145*5a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, size); 21460e8c6158Smaybee atomic_add_64(&arc_size, size); 2147ad23a2dbSjohansen } 214844eda4d7Smaybee goto out; 214944eda4d7Smaybee } 215044eda4d7Smaybee 215144eda4d7Smaybee /* 215244eda4d7Smaybee * If we are prefetching from the mfu ghost list, this buffer 215344eda4d7Smaybee * will end up on the mru list; so steal space from there. 215444eda4d7Smaybee */ 215544cb6abcSbmc if (state == arc_mfu_ghost) 215644cb6abcSbmc state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu; 215744cb6abcSbmc else if (state == arc_mru_ghost) 215844cb6abcSbmc state = arc_mru; 215944cb6abcSbmc 216044cb6abcSbmc if (state == arc_mru || state == arc_anon) { 216144cb6abcSbmc uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size; 2162*5a98e54bSBrendan Gregg - Sun Microsystems state = (arc_mfu->arcs_lsize[type] >= size && 21630e8c6158Smaybee arc_p > mru_used) ? arc_mfu : arc_mru; 2164fa9e4066Sahrens } else { 216544eda4d7Smaybee /* MFU cases */ 216644cb6abcSbmc uint64_t mfu_space = arc_c - arc_p; 2167*5a98e54bSBrendan Gregg - Sun Microsystems state = (arc_mru->arcs_lsize[type] >= size && 21680e8c6158Smaybee mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu; 216944eda4d7Smaybee } 2170874395d5Smaybee if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) { 2171ad23a2dbSjohansen if (type == ARC_BUFC_METADATA) { 2172ad23a2dbSjohansen buf->b_data = zio_buf_alloc(size); 2173*5a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(size, ARC_SPACE_DATA); 2174ad23a2dbSjohansen } else { 2175ad23a2dbSjohansen ASSERT(type == ARC_BUFC_DATA); 2176ad23a2dbSjohansen buf->b_data = zio_data_buf_alloc(size); 2177*5a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, size); 21780e8c6158Smaybee atomic_add_64(&arc_size, size); 2179ad23a2dbSjohansen } 218044cb6abcSbmc ARCSTAT_BUMP(arcstat_recycle_miss); 218144eda4d7Smaybee } 218244eda4d7Smaybee ASSERT(buf->b_data != NULL); 218344eda4d7Smaybee out: 218444eda4d7Smaybee /* 218544eda4d7Smaybee * Update the state size. Note that ghost states have a 218644eda4d7Smaybee * "ghost size" and so don't need to be updated. 218744eda4d7Smaybee */ 218844eda4d7Smaybee if (!GHOST_STATE(buf->b_hdr->b_state)) { 218944eda4d7Smaybee arc_buf_hdr_t *hdr = buf->b_hdr; 219044eda4d7Smaybee 219144cb6abcSbmc atomic_add_64(&hdr->b_state->arcs_size, size); 219244eda4d7Smaybee if (list_link_active(&hdr->b_arc_node)) { 219344eda4d7Smaybee ASSERT(refcount_is_zero(&hdr->b_refcnt)); 21940e8c6158Smaybee atomic_add_64(&hdr->b_state->arcs_lsize[type], size); 2195fa9e4066Sahrens } 2196641fbdaeSmaybee /* 2197641fbdaeSmaybee * If we are growing the cache, and we are adding anonymous 219844cb6abcSbmc * data, and we have outgrown arc_p, update arc_p 2199641fbdaeSmaybee */ 220044cb6abcSbmc if (arc_size < arc_c && hdr->b_state == arc_anon && 220144cb6abcSbmc arc_anon->arcs_size + arc_mru->arcs_size > arc_p) 220244cb6abcSbmc arc_p = MIN(arc_c, arc_p + size); 2203fa9e4066Sahrens } 2204fa9e4066Sahrens } 2205fa9e4066Sahrens 2206fa9e4066Sahrens /* 2207fa9e4066Sahrens * This routine is called whenever a buffer is accessed. 2208ea8dc4b6Seschrock * NOTE: the hash lock is dropped in this function. 2209fa9e4066Sahrens */ 2210fa9e4066Sahrens static void 221144eda4d7Smaybee arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock) 2212fa9e4066Sahrens { 2213fa9e4066Sahrens ASSERT(MUTEX_HELD(hash_lock)); 2214fa9e4066Sahrens 221544cb6abcSbmc if (buf->b_state == arc_anon) { 2216fa9e4066Sahrens /* 2217fa9e4066Sahrens * This buffer is not in the cache, and does not 2218fa9e4066Sahrens * appear in our "ghost" list. Add the new buffer 2219fa9e4066Sahrens * to the MRU state. 2220fa9e4066Sahrens */ 2221fa9e4066Sahrens 2222fa9e4066Sahrens ASSERT(buf->b_arc_access == 0); 2223fa9e4066Sahrens buf->b_arc_access = lbolt; 2224ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 222544cb6abcSbmc arc_change_state(arc_mru, buf, hash_lock); 2226fa9e4066Sahrens 222744cb6abcSbmc } else if (buf->b_state == arc_mru) { 2228fa9e4066Sahrens /* 222913506d1eSmaybee * If this buffer is here because of a prefetch, then either: 223013506d1eSmaybee * - clear the flag if this is a "referencing" read 223113506d1eSmaybee * (any subsequent access will bump this into the MFU state). 223213506d1eSmaybee * or 223313506d1eSmaybee * - move the buffer to the head of the list if this is 223413506d1eSmaybee * another prefetch (to make it less likely to be evicted). 2235fa9e4066Sahrens */ 2236fa9e4066Sahrens if ((buf->b_flags & ARC_PREFETCH) != 0) { 223713506d1eSmaybee if (refcount_count(&buf->b_refcnt) == 0) { 223813506d1eSmaybee ASSERT(list_link_active(&buf->b_arc_node)); 223913506d1eSmaybee } else { 224013506d1eSmaybee buf->b_flags &= ~ARC_PREFETCH; 224144cb6abcSbmc ARCSTAT_BUMP(arcstat_mru_hits); 224213506d1eSmaybee } 224313506d1eSmaybee buf->b_arc_access = lbolt; 2244fa9e4066Sahrens return; 2245fa9e4066Sahrens } 2246fa9e4066Sahrens 2247fa9e4066Sahrens /* 2248fa9e4066Sahrens * This buffer has been "accessed" only once so far, 2249fa9e4066Sahrens * but it is still in the cache. Move it to the MFU 2250fa9e4066Sahrens * state. 2251fa9e4066Sahrens */ 2252fa9e4066Sahrens if (lbolt > buf->b_arc_access + ARC_MINTIME) { 2253fa9e4066Sahrens /* 2254fa9e4066Sahrens * More than 125ms have passed since we 2255fa9e4066Sahrens * instantiated this buffer. Move it to the 2256fa9e4066Sahrens * most frequently used state. 2257fa9e4066Sahrens */ 2258fa9e4066Sahrens buf->b_arc_access = lbolt; 2259ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 226044cb6abcSbmc arc_change_state(arc_mfu, buf, hash_lock); 2261fa9e4066Sahrens } 226244cb6abcSbmc ARCSTAT_BUMP(arcstat_mru_hits); 226344cb6abcSbmc } else if (buf->b_state == arc_mru_ghost) { 2264fa9e4066Sahrens arc_state_t *new_state; 2265fa9e4066Sahrens /* 2266fa9e4066Sahrens * This buffer has been "accessed" recently, but 2267fa9e4066Sahrens * was evicted from the cache. Move it to the 2268fa9e4066Sahrens * MFU state. 2269fa9e4066Sahrens */ 2270fa9e4066Sahrens 2271fa9e4066Sahrens if (buf->b_flags & ARC_PREFETCH) { 227244cb6abcSbmc new_state = arc_mru; 227313506d1eSmaybee if (refcount_count(&buf->b_refcnt) > 0) 227413506d1eSmaybee buf->b_flags &= ~ARC_PREFETCH; 2275ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 2276fa9e4066Sahrens } else { 227744cb6abcSbmc new_state = arc_mfu; 2278ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2279fa9e4066Sahrens } 2280fa9e4066Sahrens 2281fa9e4066Sahrens buf->b_arc_access = lbolt; 2282fa9e4066Sahrens arc_change_state(new_state, buf, hash_lock); 2283fa9e4066Sahrens 228444cb6abcSbmc ARCSTAT_BUMP(arcstat_mru_ghost_hits); 228544cb6abcSbmc } else if (buf->b_state == arc_mfu) { 2286fa9e4066Sahrens /* 2287fa9e4066Sahrens * This buffer has been accessed more than once and is 2288fa9e4066Sahrens * still in the cache. Keep it in the MFU state. 2289fa9e4066Sahrens * 229013506d1eSmaybee * NOTE: an add_reference() that occurred when we did 229113506d1eSmaybee * the arc_read() will have kicked this off the list. 229213506d1eSmaybee * If it was a prefetch, we will explicitly move it to 229313506d1eSmaybee * the head of the list now. 2294fa9e4066Sahrens */ 229513506d1eSmaybee if ((buf->b_flags & ARC_PREFETCH) != 0) { 229613506d1eSmaybee ASSERT(refcount_count(&buf->b_refcnt) == 0); 229713506d1eSmaybee ASSERT(list_link_active(&buf->b_arc_node)); 229813506d1eSmaybee } 229944cb6abcSbmc ARCSTAT_BUMP(arcstat_mfu_hits); 230013506d1eSmaybee buf->b_arc_access = lbolt; 230144cb6abcSbmc } else if (buf->b_state == arc_mfu_ghost) { 230244cb6abcSbmc arc_state_t *new_state = arc_mfu; 2303fa9e4066Sahrens /* 2304fa9e4066Sahrens * This buffer has been accessed more than once but has 2305fa9e4066Sahrens * been evicted from the cache. Move it back to the 2306fa9e4066Sahrens * MFU state. 2307fa9e4066Sahrens */ 2308fa9e4066Sahrens 230913506d1eSmaybee if (buf->b_flags & ARC_PREFETCH) { 231013506d1eSmaybee /* 231113506d1eSmaybee * This is a prefetch access... 231213506d1eSmaybee * move this block back to the MRU state. 231313506d1eSmaybee */ 231413506d1eSmaybee ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0); 231544cb6abcSbmc new_state = arc_mru; 231613506d1eSmaybee } 231713506d1eSmaybee 2318fa9e4066Sahrens buf->b_arc_access = lbolt; 2319ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 232013506d1eSmaybee arc_change_state(new_state, buf, hash_lock); 2321fa9e4066Sahrens 232244cb6abcSbmc ARCSTAT_BUMP(arcstat_mfu_ghost_hits); 2323fa94a07fSbrendan } else if (buf->b_state == arc_l2c_only) { 2324fa94a07fSbrendan /* 2325fa94a07fSbrendan * This buffer is on the 2nd Level ARC. 2326fa94a07fSbrendan */ 2327fa94a07fSbrendan 2328fa94a07fSbrendan buf->b_arc_access = lbolt; 2329fa94a07fSbrendan DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2330fa94a07fSbrendan arc_change_state(arc_mfu, buf, hash_lock); 2331fa9e4066Sahrens } else { 2332fa9e4066Sahrens ASSERT(!"invalid arc state"); 2333fa9e4066Sahrens } 2334fa9e4066Sahrens } 2335fa9e4066Sahrens 2336fa9e4066Sahrens /* a generic arc_done_func_t which you can use */ 2337fa9e4066Sahrens /* ARGSUSED */ 2338fa9e4066Sahrens void 2339fa9e4066Sahrens arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) 2340fa9e4066Sahrens { 2341fa9e4066Sahrens bcopy(buf->b_data, arg, buf->b_hdr->b_size); 2342ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, arg) == 1); 2343fa9e4066Sahrens } 2344fa9e4066Sahrens 23450e8c6158Smaybee /* a generic arc_done_func_t */ 2346fa9e4066Sahrens void 2347fa9e4066Sahrens arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) 2348fa9e4066Sahrens { 2349fa9e4066Sahrens arc_buf_t **bufp = arg; 2350fa9e4066Sahrens if (zio && zio->io_error) { 2351ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, arg) == 1); 2352fa9e4066Sahrens *bufp = NULL; 2353fa9e4066Sahrens } else { 2354fa9e4066Sahrens *bufp = buf; 2355fa9e4066Sahrens } 2356fa9e4066Sahrens } 2357fa9e4066Sahrens 2358fa9e4066Sahrens static void 2359fa9e4066Sahrens arc_read_done(zio_t *zio) 2360fa9e4066Sahrens { 2361bbf4a8dfSmaybee arc_buf_hdr_t *hdr, *found; 2362fa9e4066Sahrens arc_buf_t *buf; 2363fa9e4066Sahrens arc_buf_t *abuf; /* buffer we're assigning to callback */ 2364fa9e4066Sahrens kmutex_t *hash_lock; 2365fa9e4066Sahrens arc_callback_t *callback_list, *acb; 2366fa9e4066Sahrens int freeable = FALSE; 2367fa9e4066Sahrens 2368fa9e4066Sahrens buf = zio->io_private; 2369fa9e4066Sahrens hdr = buf->b_hdr; 2370fa9e4066Sahrens 2371bbf4a8dfSmaybee /* 2372bbf4a8dfSmaybee * The hdr was inserted into hash-table and removed from lists 2373bbf4a8dfSmaybee * prior to starting I/O. We should find this header, since 2374bbf4a8dfSmaybee * it's in the hash table, and it should be legit since it's 2375bbf4a8dfSmaybee * not possible to evict it during the I/O. The only possible 2376bbf4a8dfSmaybee * reason for it not to be found is if we were freed during the 2377bbf4a8dfSmaybee * read. 2378bbf4a8dfSmaybee */ 2379bbf4a8dfSmaybee found = buf_hash_find(zio->io_spa, &hdr->b_dva, hdr->b_birth, 23806b4acc8bSahrens &hash_lock); 2381fa9e4066Sahrens 2382bbf4a8dfSmaybee ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) || 2383fa94a07fSbrendan (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || 2384fa94a07fSbrendan (found == hdr && HDR_L2_READING(hdr))); 2385fa94a07fSbrendan 23863a737e0dSbrendan hdr->b_flags &= ~ARC_L2_EVICTED; 2387fa94a07fSbrendan if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH)) 23883baa08fcSek hdr->b_flags &= ~ARC_L2CACHE; 2389fa9e4066Sahrens 2390fa9e4066Sahrens /* byteswap if necessary */ 2391fa9e4066Sahrens callback_list = hdr->b_acb; 2392fa9e4066Sahrens ASSERT(callback_list != NULL); 2393088f3894Sahrens if (BP_SHOULD_BYTESWAP(zio->io_bp)) { 2394088f3894Sahrens arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ? 2395088f3894Sahrens byteswap_uint64_array : 2396088f3894Sahrens dmu_ot[BP_GET_TYPE(zio->io_bp)].ot_byteswap; 2397088f3894Sahrens func(buf->b_data, hdr->b_size); 2398088f3894Sahrens } 2399fa9e4066Sahrens 2400fa94a07fSbrendan arc_cksum_compute(buf, B_FALSE); 24016b4acc8bSahrens 2402fa9e4066Sahrens /* create copies of the data buffer for the callers */ 2403fa9e4066Sahrens abuf = buf; 2404fa9e4066Sahrens for (acb = callback_list; acb; acb = acb->acb_next) { 2405fa9e4066Sahrens if (acb->acb_done) { 240644eda4d7Smaybee if (abuf == NULL) 240744eda4d7Smaybee abuf = arc_buf_clone(buf); 2408fa9e4066Sahrens acb->acb_buf = abuf; 2409fa9e4066Sahrens abuf = NULL; 2410fa9e4066Sahrens } 2411fa9e4066Sahrens } 2412fa9e4066Sahrens hdr->b_acb = NULL; 2413fa9e4066Sahrens hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2414ea8dc4b6Seschrock ASSERT(!HDR_BUF_AVAILABLE(hdr)); 2415ea8dc4b6Seschrock if (abuf == buf) 2416ea8dc4b6Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 2417fa9e4066Sahrens 2418fa9e4066Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL); 2419fa9e4066Sahrens 2420fa9e4066Sahrens if (zio->io_error != 0) { 2421fa9e4066Sahrens hdr->b_flags |= ARC_IO_ERROR; 242244cb6abcSbmc if (hdr->b_state != arc_anon) 242344cb6abcSbmc arc_change_state(arc_anon, hdr, hash_lock); 2424ea8dc4b6Seschrock if (HDR_IN_HASH_TABLE(hdr)) 2425ea8dc4b6Seschrock buf_hash_remove(hdr); 2426fa9e4066Sahrens freeable = refcount_is_zero(&hdr->b_refcnt); 2427fa9e4066Sahrens } 2428fa9e4066Sahrens 2429ea8dc4b6Seschrock /* 243013506d1eSmaybee * Broadcast before we drop the hash_lock to avoid the possibility 243113506d1eSmaybee * that the hdr (and hence the cv) might be freed before we get to 243213506d1eSmaybee * the cv_broadcast(). 2433ea8dc4b6Seschrock */ 2434ea8dc4b6Seschrock cv_broadcast(&hdr->b_cv); 2435ea8dc4b6Seschrock 2436bbf4a8dfSmaybee if (hash_lock) { 2437fa9e4066Sahrens /* 2438fa9e4066Sahrens * Only call arc_access on anonymous buffers. This is because 2439fa9e4066Sahrens * if we've issued an I/O for an evicted buffer, we've already 2440fa9e4066Sahrens * called arc_access (to prevent any simultaneous readers from 2441fa9e4066Sahrens * getting confused). 2442fa9e4066Sahrens */ 244344cb6abcSbmc if (zio->io_error == 0 && hdr->b_state == arc_anon) 244444eda4d7Smaybee arc_access(hdr, hash_lock); 244544eda4d7Smaybee mutex_exit(hash_lock); 2446fa9e4066Sahrens } else { 2447fa9e4066Sahrens /* 2448fa9e4066Sahrens * This block was freed while we waited for the read to 2449fa9e4066Sahrens * complete. It has been removed from the hash table and 2450fa9e4066Sahrens * moved to the anonymous state (so that it won't show up 2451fa9e4066Sahrens * in the cache). 2452fa9e4066Sahrens */ 245344cb6abcSbmc ASSERT3P(hdr->b_state, ==, arc_anon); 2454fa9e4066Sahrens freeable = refcount_is_zero(&hdr->b_refcnt); 2455fa9e4066Sahrens } 2456fa9e4066Sahrens 2457fa9e4066Sahrens /* execute each callback and free its structure */ 2458fa9e4066Sahrens while ((acb = callback_list) != NULL) { 2459fa9e4066Sahrens if (acb->acb_done) 2460fa9e4066Sahrens acb->acb_done(zio, acb->acb_buf, acb->acb_private); 2461fa9e4066Sahrens 2462fa9e4066Sahrens if (acb->acb_zio_dummy != NULL) { 2463fa9e4066Sahrens acb->acb_zio_dummy->io_error = zio->io_error; 2464fa9e4066Sahrens zio_nowait(acb->acb_zio_dummy); 2465fa9e4066Sahrens } 2466fa9e4066Sahrens 2467fa9e4066Sahrens callback_list = acb->acb_next; 2468fa9e4066Sahrens kmem_free(acb, sizeof (arc_callback_t)); 2469fa9e4066Sahrens } 2470fa9e4066Sahrens 2471fa9e4066Sahrens if (freeable) 2472ea8dc4b6Seschrock arc_hdr_destroy(hdr); 2473fa9e4066Sahrens } 2474fa9e4066Sahrens 2475fa9e4066Sahrens /* 2476fa9e4066Sahrens * "Read" the block block at the specified DVA (in bp) via the 2477fa9e4066Sahrens * cache. If the block is found in the cache, invoke the provided 2478fa9e4066Sahrens * callback immediately and return. Note that the `zio' parameter 2479fa9e4066Sahrens * in the callback will be NULL in this case, since no IO was 2480fa9e4066Sahrens * required. If the block is not in the cache pass the read request 2481fa9e4066Sahrens * on to the spa with a substitute callback function, so that the 2482fa9e4066Sahrens * requested block will be added to the cache. 2483fa9e4066Sahrens * 2484fa9e4066Sahrens * If a read request arrives for a block that has a read in-progress, 2485fa9e4066Sahrens * either wait for the in-progress read to complete (and return the 2486fa9e4066Sahrens * results); or, if this is a read with a "done" func, add a record 2487fa9e4066Sahrens * to the read to invoke the "done" func when the read completes, 2488fa9e4066Sahrens * and return; or just return. 2489fa9e4066Sahrens * 2490fa9e4066Sahrens * arc_read_done() will invoke all the requested "done" functions 2491fa9e4066Sahrens * for readers of this block. 2492088f3894Sahrens * 2493088f3894Sahrens * Normal callers should use arc_read and pass the arc buffer and offset 2494088f3894Sahrens * for the bp. But if you know you don't need locking, you can use 24958ad10dceSSuhasini Peddada * arc_read_bp. 2496fa9e4066Sahrens */ 2497fa9e4066Sahrens int 2498088f3894Sahrens arc_read(zio_t *pio, spa_t *spa, blkptr_t *bp, arc_buf_t *pbuf, 24993baa08fcSek arc_done_func_t *done, void *private, int priority, int zio_flags, 2500088f3894Sahrens uint32_t *arc_flags, const zbookmark_t *zb) 2501088f3894Sahrens { 2502088f3894Sahrens int err; 2503bb0ade09Sahrens arc_buf_hdr_t *hdr = pbuf->b_hdr; 2504088f3894Sahrens 2505088f3894Sahrens ASSERT(!refcount_is_zero(&pbuf->b_hdr->b_refcnt)); 2506088f3894Sahrens ASSERT3U((char *)bp - (char *)pbuf->b_data, <, pbuf->b_hdr->b_size); 25076f83844dSMark Maybee rw_enter(&pbuf->b_lock, RW_READER); 2508088f3894Sahrens 2509088f3894Sahrens err = arc_read_nolock(pio, spa, bp, done, private, priority, 25103baa08fcSek zio_flags, arc_flags, zb); 2511088f3894Sahrens 2512bb0ade09Sahrens ASSERT3P(hdr, ==, pbuf->b_hdr); 25136f83844dSMark Maybee rw_exit(&pbuf->b_lock); 2514088f3894Sahrens return (err); 2515088f3894Sahrens } 2516088f3894Sahrens 2517088f3894Sahrens int 2518088f3894Sahrens arc_read_nolock(zio_t *pio, spa_t *spa, blkptr_t *bp, 25193baa08fcSek arc_done_func_t *done, void *private, int priority, int zio_flags, 2520088f3894Sahrens uint32_t *arc_flags, const zbookmark_t *zb) 2521fa9e4066Sahrens { 2522fa9e4066Sahrens arc_buf_hdr_t *hdr; 2523fa9e4066Sahrens arc_buf_t *buf; 2524fa9e4066Sahrens kmutex_t *hash_lock; 2525fa94a07fSbrendan zio_t *rzio; 2526fa9e4066Sahrens 2527fa9e4066Sahrens top: 2528fa9e4066Sahrens hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock); 2529ea8dc4b6Seschrock if (hdr && hdr->b_datacnt > 0) { 2530fa9e4066Sahrens 253113506d1eSmaybee *arc_flags |= ARC_CACHED; 253213506d1eSmaybee 2533fa9e4066Sahrens if (HDR_IO_IN_PROGRESS(hdr)) { 253413506d1eSmaybee 253513506d1eSmaybee if (*arc_flags & ARC_WAIT) { 253613506d1eSmaybee cv_wait(&hdr->b_cv, hash_lock); 253713506d1eSmaybee mutex_exit(hash_lock); 253813506d1eSmaybee goto top; 253913506d1eSmaybee } 254013506d1eSmaybee ASSERT(*arc_flags & ARC_NOWAIT); 254113506d1eSmaybee 254213506d1eSmaybee if (done) { 2543fa9e4066Sahrens arc_callback_t *acb = NULL; 2544fa9e4066Sahrens 2545fa9e4066Sahrens acb = kmem_zalloc(sizeof (arc_callback_t), 2546fa9e4066Sahrens KM_SLEEP); 2547fa9e4066Sahrens acb->acb_done = done; 2548fa9e4066Sahrens acb->acb_private = private; 2549fa9e4066Sahrens if (pio != NULL) 2550fa9e4066Sahrens acb->acb_zio_dummy = zio_null(pio, 25513baa08fcSek spa, NULL, NULL, zio_flags); 2552fa9e4066Sahrens 2553fa9e4066Sahrens ASSERT(acb->acb_done != NULL); 2554fa9e4066Sahrens acb->acb_next = hdr->b_acb; 2555fa9e4066Sahrens hdr->b_acb = acb; 2556fa9e4066Sahrens add_reference(hdr, hash_lock, private); 2557fa9e4066Sahrens mutex_exit(hash_lock); 2558fa9e4066Sahrens return (0); 2559fa9e4066Sahrens } 2560fa9e4066Sahrens mutex_exit(hash_lock); 2561fa9e4066Sahrens return (0); 2562fa9e4066Sahrens } 2563fa9e4066Sahrens 256444cb6abcSbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2565fa9e4066Sahrens 2566ea8dc4b6Seschrock if (done) { 256744eda4d7Smaybee add_reference(hdr, hash_lock, private); 2568ea8dc4b6Seschrock /* 2569ea8dc4b6Seschrock * If this block is already in use, create a new 2570ea8dc4b6Seschrock * copy of the data so that we will be guaranteed 2571ea8dc4b6Seschrock * that arc_release() will always succeed. 2572ea8dc4b6Seschrock */ 2573fa9e4066Sahrens buf = hdr->b_buf; 2574ea8dc4b6Seschrock ASSERT(buf); 2575ea8dc4b6Seschrock ASSERT(buf->b_data); 257644eda4d7Smaybee if (HDR_BUF_AVAILABLE(hdr)) { 2577ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 2578ea8dc4b6Seschrock hdr->b_flags &= ~ARC_BUF_AVAILABLE; 257944eda4d7Smaybee } else { 258044eda4d7Smaybee buf = arc_buf_clone(buf); 2581ea8dc4b6Seschrock } 258213506d1eSmaybee } else if (*arc_flags & ARC_PREFETCH && 258313506d1eSmaybee refcount_count(&hdr->b_refcnt) == 0) { 258413506d1eSmaybee hdr->b_flags |= ARC_PREFETCH; 2585fa9e4066Sahrens } 2586fa9e4066Sahrens DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 258744eda4d7Smaybee arc_access(hdr, hash_lock); 25883baa08fcSek if (*arc_flags & ARC_L2CACHE) 25893baa08fcSek hdr->b_flags |= ARC_L2CACHE; 259044eda4d7Smaybee mutex_exit(hash_lock); 259144cb6abcSbmc ARCSTAT_BUMP(arcstat_hits); 259244cb6abcSbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 259344cb6abcSbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 259444cb6abcSbmc data, metadata, hits); 259544cb6abcSbmc 2596fa9e4066Sahrens if (done) 2597fa9e4066Sahrens done(NULL, buf, private); 2598fa9e4066Sahrens } else { 2599fa9e4066Sahrens uint64_t size = BP_GET_LSIZE(bp); 2600fa9e4066Sahrens arc_callback_t *acb; 26013a737e0dSbrendan vdev_t *vd = NULL; 26023a737e0dSbrendan daddr_t addr; 2603*5a98e54bSBrendan Gregg - Sun Microsystems boolean_t devw = B_FALSE; 2604fa9e4066Sahrens 2605fa9e4066Sahrens if (hdr == NULL) { 2606fa9e4066Sahrens /* this block is not in the cache */ 2607fa9e4066Sahrens arc_buf_hdr_t *exists; 2608ad23a2dbSjohansen arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp); 2609ad23a2dbSjohansen buf = arc_buf_alloc(spa, size, private, type); 2610fa9e4066Sahrens hdr = buf->b_hdr; 2611fa9e4066Sahrens hdr->b_dva = *BP_IDENTITY(bp); 2612fa9e4066Sahrens hdr->b_birth = bp->blk_birth; 2613fa9e4066Sahrens hdr->b_cksum0 = bp->blk_cksum.zc_word[0]; 2614fa9e4066Sahrens exists = buf_hash_insert(hdr, &hash_lock); 2615fa9e4066Sahrens if (exists) { 2616fa9e4066Sahrens /* somebody beat us to the hash insert */ 2617fa9e4066Sahrens mutex_exit(hash_lock); 2618fa9e4066Sahrens bzero(&hdr->b_dva, sizeof (dva_t)); 2619fa9e4066Sahrens hdr->b_birth = 0; 2620fa9e4066Sahrens hdr->b_cksum0 = 0; 2621ea8dc4b6Seschrock (void) arc_buf_remove_ref(buf, private); 2622fa9e4066Sahrens goto top; /* restart the IO request */ 2623fa9e4066Sahrens } 262413506d1eSmaybee /* if this is a prefetch, we don't have a reference */ 262513506d1eSmaybee if (*arc_flags & ARC_PREFETCH) { 262613506d1eSmaybee (void) remove_reference(hdr, hash_lock, 262713506d1eSmaybee private); 262813506d1eSmaybee hdr->b_flags |= ARC_PREFETCH; 262913506d1eSmaybee } 26303baa08fcSek if (*arc_flags & ARC_L2CACHE) 26313baa08fcSek hdr->b_flags |= ARC_L2CACHE; 263213506d1eSmaybee if (BP_GET_LEVEL(bp) > 0) 263313506d1eSmaybee hdr->b_flags |= ARC_INDIRECT; 2634fa9e4066Sahrens } else { 2635fa9e4066Sahrens /* this block is in the ghost cache */ 2636ea8dc4b6Seschrock ASSERT(GHOST_STATE(hdr->b_state)); 2637ea8dc4b6Seschrock ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 263813506d1eSmaybee ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0); 2639ea8dc4b6Seschrock ASSERT(hdr->b_buf == NULL); 264013506d1eSmaybee 264113506d1eSmaybee /* if this is a prefetch, we don't have a reference */ 264213506d1eSmaybee if (*arc_flags & ARC_PREFETCH) 264313506d1eSmaybee hdr->b_flags |= ARC_PREFETCH; 264413506d1eSmaybee else 264513506d1eSmaybee add_reference(hdr, hash_lock, private); 26463baa08fcSek if (*arc_flags & ARC_L2CACHE) 26473baa08fcSek hdr->b_flags |= ARC_L2CACHE; 26481ab7f2deSmaybee buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 2649fa9e4066Sahrens buf->b_hdr = hdr; 265044eda4d7Smaybee buf->b_data = NULL; 2651ea8dc4b6Seschrock buf->b_efunc = NULL; 2652ea8dc4b6Seschrock buf->b_private = NULL; 2653fa9e4066Sahrens buf->b_next = NULL; 2654fa9e4066Sahrens hdr->b_buf = buf; 265544eda4d7Smaybee arc_get_data_buf(buf); 2656ea8dc4b6Seschrock ASSERT(hdr->b_datacnt == 0); 2657ea8dc4b6Seschrock hdr->b_datacnt = 1; 265813506d1eSmaybee 2659fa9e4066Sahrens } 2660fa9e4066Sahrens 2661fa9e4066Sahrens acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 2662fa9e4066Sahrens acb->acb_done = done; 2663fa9e4066Sahrens acb->acb_private = private; 2664fa9e4066Sahrens 2665fa9e4066Sahrens ASSERT(hdr->b_acb == NULL); 2666fa9e4066Sahrens hdr->b_acb = acb; 2667fa9e4066Sahrens hdr->b_flags |= ARC_IO_IN_PROGRESS; 2668fa9e4066Sahrens 2669fa9e4066Sahrens /* 2670fa9e4066Sahrens * If the buffer has been evicted, migrate it to a present state 2671fa9e4066Sahrens * before issuing the I/O. Once we drop the hash-table lock, 2672fa9e4066Sahrens * the header will be marked as I/O in progress and have an 2673fa9e4066Sahrens * attached buffer. At this point, anybody who finds this 2674fa9e4066Sahrens * buffer ought to notice that it's legit but has a pending I/O. 2675fa9e4066Sahrens */ 2676fa9e4066Sahrens 2677ea8dc4b6Seschrock if (GHOST_STATE(hdr->b_state)) 267844eda4d7Smaybee arc_access(hdr, hash_lock); 2679fa9e4066Sahrens 2680e14bb325SJeff Bonwick if (HDR_L2CACHE(hdr) && hdr->b_l2hdr != NULL && 2681e14bb325SJeff Bonwick (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) { 2682*5a98e54bSBrendan Gregg - Sun Microsystems devw = hdr->b_l2hdr->b_dev->l2ad_writing; 26833a737e0dSbrendan addr = hdr->b_l2hdr->b_daddr; 2684e14bb325SJeff Bonwick /* 2685e14bb325SJeff Bonwick * Lock out device removal. 2686e14bb325SJeff Bonwick */ 2687e14bb325SJeff Bonwick if (vdev_is_dead(vd) || 2688e14bb325SJeff Bonwick !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER)) 2689e14bb325SJeff Bonwick vd = NULL; 26903a737e0dSbrendan } 26913a737e0dSbrendan 26923a737e0dSbrendan mutex_exit(hash_lock); 26933a737e0dSbrendan 2694fa9e4066Sahrens ASSERT3U(hdr->b_size, ==, size); 2695c543ec06Sahrens DTRACE_PROBE3(arc__miss, blkptr_t *, bp, uint64_t, size, 2696c543ec06Sahrens zbookmark_t *, zb); 269744cb6abcSbmc ARCSTAT_BUMP(arcstat_misses); 269844cb6abcSbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 269944cb6abcSbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 270044cb6abcSbmc data, metadata, misses); 2701ea8dc4b6Seschrock 2702*5a98e54bSBrendan Gregg - Sun Microsystems if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) { 2703fa94a07fSbrendan /* 2704fa94a07fSbrendan * Read from the L2ARC if the following are true: 27053a737e0dSbrendan * 1. The L2ARC vdev was previously cached. 27063a737e0dSbrendan * 2. This buffer still has L2ARC metadata. 27073a737e0dSbrendan * 3. This buffer isn't currently writing to the L2ARC. 27083a737e0dSbrendan * 4. The L2ARC entry wasn't evicted, which may 27093a737e0dSbrendan * also have invalidated the vdev. 2710*5a98e54bSBrendan Gregg - Sun Microsystems * 5. This isn't prefetch and l2arc_noprefetch is set. 2711fa94a07fSbrendan */ 2712e14bb325SJeff Bonwick if (hdr->b_l2hdr != NULL && 2713*5a98e54bSBrendan Gregg - Sun Microsystems !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) && 2714*5a98e54bSBrendan Gregg - Sun Microsystems !(l2arc_noprefetch && HDR_PREFETCH(hdr))) { 2715fa94a07fSbrendan l2arc_read_callback_t *cb; 2716fa94a07fSbrendan 2717c5904d13Seschrock DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr); 2718c5904d13Seschrock ARCSTAT_BUMP(arcstat_l2_hits); 2719c5904d13Seschrock 2720fa94a07fSbrendan cb = kmem_zalloc(sizeof (l2arc_read_callback_t), 2721fa94a07fSbrendan KM_SLEEP); 2722fa94a07fSbrendan cb->l2rcb_buf = buf; 2723fa94a07fSbrendan cb->l2rcb_spa = spa; 2724fa94a07fSbrendan cb->l2rcb_bp = *bp; 2725fa94a07fSbrendan cb->l2rcb_zb = *zb; 27263baa08fcSek cb->l2rcb_flags = zio_flags; 2727fa94a07fSbrendan 2728fa94a07fSbrendan /* 2729e14bb325SJeff Bonwick * l2arc read. The SCL_L2ARC lock will be 2730e14bb325SJeff Bonwick * released by l2arc_read_done(). 2731fa94a07fSbrendan */ 2732fa94a07fSbrendan rzio = zio_read_phys(pio, vd, addr, size, 2733fa94a07fSbrendan buf->b_data, ZIO_CHECKSUM_OFF, 27343baa08fcSek l2arc_read_done, cb, priority, zio_flags | 273549cf58c0SBrendan Gregg - Sun Microsystems ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL | 2736e14bb325SJeff Bonwick ZIO_FLAG_DONT_PROPAGATE | 2737e14bb325SJeff Bonwick ZIO_FLAG_DONT_RETRY, B_FALSE); 2738fa94a07fSbrendan DTRACE_PROBE2(l2arc__read, vdev_t *, vd, 2739fa94a07fSbrendan zio_t *, rzio); 2740*5a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_l2_read_bytes, size); 2741fa94a07fSbrendan 27423a737e0dSbrendan if (*arc_flags & ARC_NOWAIT) { 27433a737e0dSbrendan zio_nowait(rzio); 27443a737e0dSbrendan return (0); 27453a737e0dSbrendan } 2746fa94a07fSbrendan 27473a737e0dSbrendan ASSERT(*arc_flags & ARC_WAIT); 27483a737e0dSbrendan if (zio_wait(rzio) == 0) 27493a737e0dSbrendan return (0); 27503a737e0dSbrendan 27513a737e0dSbrendan /* l2arc read error; goto zio_read() */ 2752fa94a07fSbrendan } else { 2753fa94a07fSbrendan DTRACE_PROBE1(l2arc__miss, 2754fa94a07fSbrendan arc_buf_hdr_t *, hdr); 2755fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_misses); 2756fa94a07fSbrendan if (HDR_L2_WRITING(hdr)) 2757fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_rw_clash); 2758e14bb325SJeff Bonwick spa_config_exit(spa, SCL_L2ARC, vd); 2759fa94a07fSbrendan } 2760*5a98e54bSBrendan Gregg - Sun Microsystems } else { 2761*5a98e54bSBrendan Gregg - Sun Microsystems if (l2arc_ndev != 0) { 2762*5a98e54bSBrendan Gregg - Sun Microsystems DTRACE_PROBE1(l2arc__miss, 2763*5a98e54bSBrendan Gregg - Sun Microsystems arc_buf_hdr_t *, hdr); 2764*5a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_BUMP(arcstat_l2_misses); 2765*5a98e54bSBrendan Gregg - Sun Microsystems } 2766fa94a07fSbrendan } 2767c5904d13Seschrock 2768fa9e4066Sahrens rzio = zio_read(pio, spa, bp, buf->b_data, size, 27693baa08fcSek arc_read_done, buf, priority, zio_flags, zb); 2770fa9e4066Sahrens 277113506d1eSmaybee if (*arc_flags & ARC_WAIT) 2772fa9e4066Sahrens return (zio_wait(rzio)); 2773fa9e4066Sahrens 277413506d1eSmaybee ASSERT(*arc_flags & ARC_NOWAIT); 2775fa9e4066Sahrens zio_nowait(rzio); 2776fa9e4066Sahrens } 2777fa9e4066Sahrens return (0); 2778fa9e4066Sahrens } 2779fa9e4066Sahrens 2780fa9e4066Sahrens /* 2781fa9e4066Sahrens * arc_read() variant to support pool traversal. If the block is already 2782fa9e4066Sahrens * in the ARC, make a copy of it; otherwise, the caller will do the I/O. 2783fa9e4066Sahrens * The idea is that we don't want pool traversal filling up memory, but 2784fa9e4066Sahrens * if the ARC already has the data anyway, we shouldn't pay for the I/O. 2785fa9e4066Sahrens */ 2786fa9e4066Sahrens int 2787fa9e4066Sahrens arc_tryread(spa_t *spa, blkptr_t *bp, void *data) 2788fa9e4066Sahrens { 2789fa9e4066Sahrens arc_buf_hdr_t *hdr; 2790fa9e4066Sahrens kmutex_t *hash_mtx; 2791fa9e4066Sahrens int rc = 0; 2792fa9e4066Sahrens 2793fa9e4066Sahrens hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_mtx); 2794fa9e4066Sahrens 2795ea8dc4b6Seschrock if (hdr && hdr->b_datacnt > 0 && !HDR_IO_IN_PROGRESS(hdr)) { 2796ea8dc4b6Seschrock arc_buf_t *buf = hdr->b_buf; 2797ea8dc4b6Seschrock 2798ea8dc4b6Seschrock ASSERT(buf); 2799ea8dc4b6Seschrock while (buf->b_data == NULL) { 2800ea8dc4b6Seschrock buf = buf->b_next; 2801ea8dc4b6Seschrock ASSERT(buf); 2802ea8dc4b6Seschrock } 2803ea8dc4b6Seschrock bcopy(buf->b_data, data, hdr->b_size); 2804ea8dc4b6Seschrock } else { 2805fa9e4066Sahrens rc = ENOENT; 2806ea8dc4b6Seschrock } 2807fa9e4066Sahrens 2808fa9e4066Sahrens if (hash_mtx) 2809fa9e4066Sahrens mutex_exit(hash_mtx); 2810fa9e4066Sahrens 2811fa9e4066Sahrens return (rc); 2812fa9e4066Sahrens } 2813fa9e4066Sahrens 2814ea8dc4b6Seschrock void 2815ea8dc4b6Seschrock arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private) 2816ea8dc4b6Seschrock { 2817ea8dc4b6Seschrock ASSERT(buf->b_hdr != NULL); 281844cb6abcSbmc ASSERT(buf->b_hdr->b_state != arc_anon); 2819ea8dc4b6Seschrock ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL); 2820ea8dc4b6Seschrock buf->b_efunc = func; 2821ea8dc4b6Seschrock buf->b_private = private; 2822ea8dc4b6Seschrock } 2823ea8dc4b6Seschrock 2824ea8dc4b6Seschrock /* 2825ea8dc4b6Seschrock * This is used by the DMU to let the ARC know that a buffer is 2826ea8dc4b6Seschrock * being evicted, so the ARC should clean up. If this arc buf 2827ea8dc4b6Seschrock * is not yet in the evicted state, it will be put there. 2828ea8dc4b6Seschrock */ 2829ea8dc4b6Seschrock int 2830ea8dc4b6Seschrock arc_buf_evict(arc_buf_t *buf) 2831ea8dc4b6Seschrock { 283240d7d650Smaybee arc_buf_hdr_t *hdr; 2833ea8dc4b6Seschrock kmutex_t *hash_lock; 2834ea8dc4b6Seschrock arc_buf_t **bufp; 2835ea8dc4b6Seschrock 28366f83844dSMark Maybee rw_enter(&buf->b_lock, RW_WRITER); 283740d7d650Smaybee hdr = buf->b_hdr; 2838ea8dc4b6Seschrock if (hdr == NULL) { 2839ea8dc4b6Seschrock /* 2840ea8dc4b6Seschrock * We are in arc_do_user_evicts(). 2841ea8dc4b6Seschrock */ 2842ea8dc4b6Seschrock ASSERT(buf->b_data == NULL); 28436f83844dSMark Maybee rw_exit(&buf->b_lock); 2844ea8dc4b6Seschrock return (0); 28456f83844dSMark Maybee } else if (buf->b_data == NULL) { 28466f83844dSMark Maybee arc_buf_t copy = *buf; /* structure assignment */ 28479b23f181Smaybee /* 28486f83844dSMark Maybee * We are on the eviction list; process this buffer now 28496f83844dSMark Maybee * but let arc_do_user_evicts() do the reaping. 28509b23f181Smaybee */ 28516f83844dSMark Maybee buf->b_efunc = NULL; 28526f83844dSMark Maybee rw_exit(&buf->b_lock); 28536f83844dSMark Maybee VERIFY(copy.b_efunc(©) == 0); 28546f83844dSMark Maybee return (1); 28559b23f181Smaybee } 28566f83844dSMark Maybee hash_lock = HDR_LOCK(hdr); 28576f83844dSMark Maybee mutex_enter(hash_lock); 28589b23f181Smaybee 28599b23f181Smaybee ASSERT(buf->b_hdr == hdr); 28609b23f181Smaybee ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt); 286144cb6abcSbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2862ea8dc4b6Seschrock 2863ea8dc4b6Seschrock /* 2864ea8dc4b6Seschrock * Pull this buffer off of the hdr 2865ea8dc4b6Seschrock */ 2866ea8dc4b6Seschrock bufp = &hdr->b_buf; 2867ea8dc4b6Seschrock while (*bufp != buf) 2868ea8dc4b6Seschrock bufp = &(*bufp)->b_next; 2869ea8dc4b6Seschrock *bufp = buf->b_next; 2870ea8dc4b6Seschrock 2871ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 287244eda4d7Smaybee arc_buf_destroy(buf, FALSE, FALSE); 2873ea8dc4b6Seschrock 2874ea8dc4b6Seschrock if (hdr->b_datacnt == 0) { 2875ea8dc4b6Seschrock arc_state_t *old_state = hdr->b_state; 2876ea8dc4b6Seschrock arc_state_t *evicted_state; 2877ea8dc4b6Seschrock 2878ea8dc4b6Seschrock ASSERT(refcount_is_zero(&hdr->b_refcnt)); 2879ea8dc4b6Seschrock 2880ea8dc4b6Seschrock evicted_state = 288144cb6abcSbmc (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 2882ea8dc4b6Seschrock 288344cb6abcSbmc mutex_enter(&old_state->arcs_mtx); 288444cb6abcSbmc mutex_enter(&evicted_state->arcs_mtx); 2885ea8dc4b6Seschrock 2886ea8dc4b6Seschrock arc_change_state(evicted_state, hdr, hash_lock); 2887ea8dc4b6Seschrock ASSERT(HDR_IN_HASH_TABLE(hdr)); 2888fa94a07fSbrendan hdr->b_flags |= ARC_IN_HASH_TABLE; 2889fa94a07fSbrendan hdr->b_flags &= ~ARC_BUF_AVAILABLE; 2890ea8dc4b6Seschrock 289144cb6abcSbmc mutex_exit(&evicted_state->arcs_mtx); 289244cb6abcSbmc mutex_exit(&old_state->arcs_mtx); 2893ea8dc4b6Seschrock } 2894ea8dc4b6Seschrock mutex_exit(hash_lock); 28956f83844dSMark Maybee rw_exit(&buf->b_lock); 2896dd6ef538Smaybee 2897ea8dc4b6Seschrock VERIFY(buf->b_efunc(buf) == 0); 2898ea8dc4b6Seschrock buf->b_efunc = NULL; 2899ea8dc4b6Seschrock buf->b_private = NULL; 2900ea8dc4b6Seschrock buf->b_hdr = NULL; 2901ea8dc4b6Seschrock kmem_cache_free(buf_cache, buf); 2902ea8dc4b6Seschrock return (1); 2903ea8dc4b6Seschrock } 2904ea8dc4b6Seschrock 2905fa9e4066Sahrens /* 2906fa9e4066Sahrens * Release this buffer from the cache. This must be done 2907fa9e4066Sahrens * after a read and prior to modifying the buffer contents. 2908fa9e4066Sahrens * If the buffer has more than one reference, we must make 2909088f3894Sahrens * a new hdr for the buffer. 2910fa9e4066Sahrens */ 2911fa9e4066Sahrens void 2912fa9e4066Sahrens arc_release(arc_buf_t *buf, void *tag) 2913fa9e4066Sahrens { 29146f83844dSMark Maybee arc_buf_hdr_t *hdr; 29156f83844dSMark Maybee kmutex_t *hash_lock; 29166f83844dSMark Maybee l2arc_buf_hdr_t *l2hdr; 2917fa94a07fSbrendan uint64_t buf_size; 2918fa9e4066Sahrens 29196f83844dSMark Maybee rw_enter(&buf->b_lock, RW_WRITER); 29206f83844dSMark Maybee hdr = buf->b_hdr; 29216f83844dSMark Maybee 2922fa9e4066Sahrens /* this buffer is not on any list */ 2923fa9e4066Sahrens ASSERT(refcount_count(&hdr->b_refcnt) > 0); 2924088f3894Sahrens ASSERT(!(hdr->b_flags & ARC_STORED)); 2925fa9e4066Sahrens 292644cb6abcSbmc if (hdr->b_state == arc_anon) { 2927fa9e4066Sahrens /* this buffer is already released */ 2928fa9e4066Sahrens ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 1); 2929fa9e4066Sahrens ASSERT(BUF_EMPTY(hdr)); 2930ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 29316b4acc8bSahrens arc_buf_thaw(buf); 29326f83844dSMark Maybee rw_exit(&buf->b_lock); 2933fa9e4066Sahrens return; 2934fa9e4066Sahrens } 2935fa9e4066Sahrens 29366f83844dSMark Maybee hash_lock = HDR_LOCK(hdr); 2937fa9e4066Sahrens mutex_enter(hash_lock); 2938fa9e4066Sahrens 29396f83844dSMark Maybee l2hdr = hdr->b_l2hdr; 29406f83844dSMark Maybee if (l2hdr) { 29416f83844dSMark Maybee mutex_enter(&l2arc_buflist_mtx); 29426f83844dSMark Maybee hdr->b_l2hdr = NULL; 29436f83844dSMark Maybee buf_size = hdr->b_size; 29446f83844dSMark Maybee } 29456f83844dSMark Maybee 2946ea8dc4b6Seschrock /* 2947ea8dc4b6Seschrock * Do we have more than one buf? 2948ea8dc4b6Seschrock */ 29496f83844dSMark Maybee if (hdr->b_datacnt > 1) { 2950fa9e4066Sahrens arc_buf_hdr_t *nhdr; 2951fa9e4066Sahrens arc_buf_t **bufp; 2952fa9e4066Sahrens uint64_t blksz = hdr->b_size; 2953fa9e4066Sahrens spa_t *spa = hdr->b_spa; 2954ad23a2dbSjohansen arc_buf_contents_t type = hdr->b_type; 2955fa94a07fSbrendan uint32_t flags = hdr->b_flags; 2956fa9e4066Sahrens 29576f83844dSMark Maybee ASSERT(hdr->b_buf != buf || buf->b_next != NULL); 2958fa9e4066Sahrens /* 2959fa9e4066Sahrens * Pull the data off of this buf and attach it to 2960fa9e4066Sahrens * a new anonymous buf. 2961fa9e4066Sahrens */ 2962ea8dc4b6Seschrock (void) remove_reference(hdr, hash_lock, tag); 2963fa9e4066Sahrens bufp = &hdr->b_buf; 2964ea8dc4b6Seschrock while (*bufp != buf) 2965fa9e4066Sahrens bufp = &(*bufp)->b_next; 2966fa9e4066Sahrens *bufp = (*bufp)->b_next; 2967af2c4821Smaybee buf->b_next = NULL; 2968ea8dc4b6Seschrock 296944cb6abcSbmc ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size); 297044cb6abcSbmc atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size); 2971ea8dc4b6Seschrock if (refcount_is_zero(&hdr->b_refcnt)) { 29720e8c6158Smaybee uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type]; 29730e8c6158Smaybee ASSERT3U(*size, >=, hdr->b_size); 29740e8c6158Smaybee atomic_add_64(size, -hdr->b_size); 2975ea8dc4b6Seschrock } 2976ea8dc4b6Seschrock hdr->b_datacnt -= 1; 2977c717a561Smaybee arc_cksum_verify(buf); 2978ea8dc4b6Seschrock 2979fa9e4066Sahrens mutex_exit(hash_lock); 2980fa9e4066Sahrens 29811ab7f2deSmaybee nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 2982fa9e4066Sahrens nhdr->b_size = blksz; 2983fa9e4066Sahrens nhdr->b_spa = spa; 2984ad23a2dbSjohansen nhdr->b_type = type; 2985fa9e4066Sahrens nhdr->b_buf = buf; 298644cb6abcSbmc nhdr->b_state = arc_anon; 2987fa9e4066Sahrens nhdr->b_arc_access = 0; 2988fa94a07fSbrendan nhdr->b_flags = flags & ARC_L2_WRITING; 2989fa94a07fSbrendan nhdr->b_l2hdr = NULL; 2990ea8dc4b6Seschrock nhdr->b_datacnt = 1; 2991c717a561Smaybee nhdr->b_freeze_cksum = NULL; 2992fa9e4066Sahrens (void) refcount_add(&nhdr->b_refcnt, tag); 2993af2c4821Smaybee buf->b_hdr = nhdr; 29946f83844dSMark Maybee rw_exit(&buf->b_lock); 299544cb6abcSbmc atomic_add_64(&arc_anon->arcs_size, blksz); 2996fa9e4066Sahrens } else { 29976f83844dSMark Maybee rw_exit(&buf->b_lock); 2998ea8dc4b6Seschrock ASSERT(refcount_count(&hdr->b_refcnt) == 1); 2999fa9e4066Sahrens ASSERT(!list_link_active(&hdr->b_arc_node)); 3000fa9e4066Sahrens ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 300144cb6abcSbmc arc_change_state(arc_anon, hdr, hash_lock); 3002fa9e4066Sahrens hdr->b_arc_access = 0; 3003fa9e4066Sahrens mutex_exit(hash_lock); 3004fa94a07fSbrendan 3005fa9e4066Sahrens bzero(&hdr->b_dva, sizeof (dva_t)); 3006fa9e4066Sahrens hdr->b_birth = 0; 3007fa9e4066Sahrens hdr->b_cksum0 = 0; 3008c717a561Smaybee arc_buf_thaw(buf); 3009fa9e4066Sahrens } 3010ea8dc4b6Seschrock buf->b_efunc = NULL; 3011ea8dc4b6Seschrock buf->b_private = NULL; 3012fa94a07fSbrendan 3013fa94a07fSbrendan if (l2hdr) { 3014fa94a07fSbrendan list_remove(l2hdr->b_dev->l2ad_buflist, hdr); 3015fa94a07fSbrendan kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t)); 3016fa94a07fSbrendan ARCSTAT_INCR(arcstat_l2_size, -buf_size); 3017fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 30186f83844dSMark Maybee } 3019fa9e4066Sahrens } 3020fa9e4066Sahrens 3021fa9e4066Sahrens int 3022fa9e4066Sahrens arc_released(arc_buf_t *buf) 3023fa9e4066Sahrens { 30246f83844dSMark Maybee int released; 30256f83844dSMark Maybee 30266f83844dSMark Maybee rw_enter(&buf->b_lock, RW_READER); 30276f83844dSMark Maybee released = (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon); 30286f83844dSMark Maybee rw_exit(&buf->b_lock); 30296f83844dSMark Maybee return (released); 3030ea8dc4b6Seschrock } 3031ea8dc4b6Seschrock 3032ea8dc4b6Seschrock int 3033ea8dc4b6Seschrock arc_has_callback(arc_buf_t *buf) 3034ea8dc4b6Seschrock { 30356f83844dSMark Maybee int callback; 30366f83844dSMark Maybee 30376f83844dSMark Maybee rw_enter(&buf->b_lock, RW_READER); 30386f83844dSMark Maybee callback = (buf->b_efunc != NULL); 30396f83844dSMark Maybee rw_exit(&buf->b_lock); 30406f83844dSMark Maybee return (callback); 3041fa9e4066Sahrens } 3042fa9e4066Sahrens 3043ea8dc4b6Seschrock #ifdef ZFS_DEBUG 3044ea8dc4b6Seschrock int 3045ea8dc4b6Seschrock arc_referenced(arc_buf_t *buf) 3046ea8dc4b6Seschrock { 30476f83844dSMark Maybee int referenced; 30486f83844dSMark Maybee 30496f83844dSMark Maybee rw_enter(&buf->b_lock, RW_READER); 30506f83844dSMark Maybee referenced = (refcount_count(&buf->b_hdr->b_refcnt)); 30516f83844dSMark Maybee rw_exit(&buf->b_lock); 30526f83844dSMark Maybee return (referenced); 3053ea8dc4b6Seschrock } 3054ea8dc4b6Seschrock #endif 3055ea8dc4b6Seschrock 3056c717a561Smaybee static void 3057c717a561Smaybee arc_write_ready(zio_t *zio) 3058c717a561Smaybee { 3059c717a561Smaybee arc_write_callback_t *callback = zio->io_private; 3060c717a561Smaybee arc_buf_t *buf = callback->awcb_buf; 30610a4e9518Sgw arc_buf_hdr_t *hdr = buf->b_hdr; 3062c717a561Smaybee 3063e14bb325SJeff Bonwick ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt)); 3064e14bb325SJeff Bonwick callback->awcb_ready(zio, buf, callback->awcb_private); 3065e14bb325SJeff Bonwick 30660a4e9518Sgw /* 30670a4e9518Sgw * If the IO is already in progress, then this is a re-write 3068e14bb325SJeff Bonwick * attempt, so we need to thaw and re-compute the cksum. 3069e14bb325SJeff Bonwick * It is the responsibility of the callback to handle the 3070e14bb325SJeff Bonwick * accounting for any re-write attempt. 30710a4e9518Sgw */ 30720a4e9518Sgw if (HDR_IO_IN_PROGRESS(hdr)) { 30730a4e9518Sgw mutex_enter(&hdr->b_freeze_lock); 30740a4e9518Sgw if (hdr->b_freeze_cksum != NULL) { 30750a4e9518Sgw kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 30760a4e9518Sgw hdr->b_freeze_cksum = NULL; 30770a4e9518Sgw } 30780a4e9518Sgw mutex_exit(&hdr->b_freeze_lock); 30790a4e9518Sgw } 3080fa94a07fSbrendan arc_cksum_compute(buf, B_FALSE); 30810a4e9518Sgw hdr->b_flags |= ARC_IO_IN_PROGRESS; 3082c717a561Smaybee } 3083c717a561Smaybee 3084fa9e4066Sahrens static void 3085fa9e4066Sahrens arc_write_done(zio_t *zio) 3086fa9e4066Sahrens { 3087c717a561Smaybee arc_write_callback_t *callback = zio->io_private; 3088c717a561Smaybee arc_buf_t *buf = callback->awcb_buf; 3089c717a561Smaybee arc_buf_hdr_t *hdr = buf->b_hdr; 3090fa9e4066Sahrens 3091fa9e4066Sahrens hdr->b_acb = NULL; 3092fa9e4066Sahrens 3093fa9e4066Sahrens hdr->b_dva = *BP_IDENTITY(zio->io_bp); 3094fa9e4066Sahrens hdr->b_birth = zio->io_bp->blk_birth; 3095fa9e4066Sahrens hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0]; 3096ea8dc4b6Seschrock /* 3097ea8dc4b6Seschrock * If the block to be written was all-zero, we may have 3098ea8dc4b6Seschrock * compressed it away. In this case no write was performed 3099ea8dc4b6Seschrock * so there will be no dva/birth-date/checksum. The buffer 3100ea8dc4b6Seschrock * must therefor remain anonymous (and uncached). 3101ea8dc4b6Seschrock */ 3102fa9e4066Sahrens if (!BUF_EMPTY(hdr)) { 3103fa9e4066Sahrens arc_buf_hdr_t *exists; 3104fa9e4066Sahrens kmutex_t *hash_lock; 3105fa9e4066Sahrens 31066b4acc8bSahrens arc_cksum_verify(buf); 31076b4acc8bSahrens 3108fa9e4066Sahrens exists = buf_hash_insert(hdr, &hash_lock); 3109fa9e4066Sahrens if (exists) { 3110fa9e4066Sahrens /* 3111fa9e4066Sahrens * This can only happen if we overwrite for 3112fa9e4066Sahrens * sync-to-convergence, because we remove 3113fa9e4066Sahrens * buffers from the hash table when we arc_free(). 3114fa9e4066Sahrens */ 3115e14bb325SJeff Bonwick ASSERT(zio->io_flags & ZIO_FLAG_IO_REWRITE); 3116fa9e4066Sahrens ASSERT(DVA_EQUAL(BP_IDENTITY(&zio->io_bp_orig), 3117fa9e4066Sahrens BP_IDENTITY(zio->io_bp))); 3118fa9e4066Sahrens ASSERT3U(zio->io_bp_orig.blk_birth, ==, 3119fa9e4066Sahrens zio->io_bp->blk_birth); 3120fa9e4066Sahrens 3121fa9e4066Sahrens ASSERT(refcount_is_zero(&exists->b_refcnt)); 312244cb6abcSbmc arc_change_state(arc_anon, exists, hash_lock); 3123fa9e4066Sahrens mutex_exit(hash_lock); 3124ea8dc4b6Seschrock arc_hdr_destroy(exists); 3125fa9e4066Sahrens exists = buf_hash_insert(hdr, &hash_lock); 3126fa9e4066Sahrens ASSERT3P(exists, ==, NULL); 3127fa9e4066Sahrens } 3128ea8dc4b6Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 3129088f3894Sahrens /* if it's not anon, we are doing a scrub */ 3130088f3894Sahrens if (hdr->b_state == arc_anon) 3131088f3894Sahrens arc_access(hdr, hash_lock); 313244eda4d7Smaybee mutex_exit(hash_lock); 3133c717a561Smaybee } else if (callback->awcb_done == NULL) { 3134ea8dc4b6Seschrock int destroy_hdr; 3135ea8dc4b6Seschrock /* 3136ea8dc4b6Seschrock * This is an anonymous buffer with no user callback, 3137ea8dc4b6Seschrock * destroy it if there are no active references. 3138ea8dc4b6Seschrock */ 3139ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 3140ea8dc4b6Seschrock destroy_hdr = refcount_is_zero(&hdr->b_refcnt); 3141ea8dc4b6Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 3142ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 3143ea8dc4b6Seschrock if (destroy_hdr) 3144ea8dc4b6Seschrock arc_hdr_destroy(hdr); 3145ea8dc4b6Seschrock } else { 3146ea8dc4b6Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 3147fa9e4066Sahrens } 3148088f3894Sahrens hdr->b_flags &= ~ARC_STORED; 3149ea8dc4b6Seschrock 3150c717a561Smaybee if (callback->awcb_done) { 3151fa9e4066Sahrens ASSERT(!refcount_is_zero(&hdr->b_refcnt)); 3152c717a561Smaybee callback->awcb_done(zio, buf, callback->awcb_private); 3153fa9e4066Sahrens } 3154fa9e4066Sahrens 3155c717a561Smaybee kmem_free(callback, sizeof (arc_write_callback_t)); 3156fa9e4066Sahrens } 3157fa9e4066Sahrens 315882c9918fSTim Haley void 3159e14bb325SJeff Bonwick write_policy(spa_t *spa, const writeprops_t *wp, zio_prop_t *zp) 3160088f3894Sahrens { 3161088f3894Sahrens boolean_t ismd = (wp->wp_level > 0 || dmu_ot[wp->wp_type].ot_metadata); 3162088f3894Sahrens 3163088f3894Sahrens /* Determine checksum setting */ 3164088f3894Sahrens if (ismd) { 3165088f3894Sahrens /* 3166088f3894Sahrens * Metadata always gets checksummed. If the data 3167088f3894Sahrens * checksum is multi-bit correctable, and it's not a 3168088f3894Sahrens * ZBT-style checksum, then it's suitable for metadata 3169088f3894Sahrens * as well. Otherwise, the metadata checksum defaults 3170088f3894Sahrens * to fletcher4. 3171088f3894Sahrens */ 3172088f3894Sahrens if (zio_checksum_table[wp->wp_oschecksum].ci_correctable && 3173088f3894Sahrens !zio_checksum_table[wp->wp_oschecksum].ci_zbt) 3174e14bb325SJeff Bonwick zp->zp_checksum = wp->wp_oschecksum; 3175088f3894Sahrens else 3176e14bb325SJeff Bonwick zp->zp_checksum = ZIO_CHECKSUM_FLETCHER_4; 3177088f3894Sahrens } else { 3178e14bb325SJeff Bonwick zp->zp_checksum = zio_checksum_select(wp->wp_dnchecksum, 3179088f3894Sahrens wp->wp_oschecksum); 3180088f3894Sahrens } 3181088f3894Sahrens 3182088f3894Sahrens /* Determine compression setting */ 3183088f3894Sahrens if (ismd) { 3184088f3894Sahrens /* 3185088f3894Sahrens * XXX -- we should design a compression algorithm 3186088f3894Sahrens * that specializes in arrays of bps. 3187088f3894Sahrens */ 3188e14bb325SJeff Bonwick zp->zp_compress = zfs_mdcomp_disable ? ZIO_COMPRESS_EMPTY : 3189088f3894Sahrens ZIO_COMPRESS_LZJB; 3190088f3894Sahrens } else { 3191e14bb325SJeff Bonwick zp->zp_compress = zio_compress_select(wp->wp_dncompress, 3192088f3894Sahrens wp->wp_oscompress); 3193088f3894Sahrens } 3194e14bb325SJeff Bonwick 3195e14bb325SJeff Bonwick zp->zp_type = wp->wp_type; 3196e14bb325SJeff Bonwick zp->zp_level = wp->wp_level; 3197e14bb325SJeff Bonwick zp->zp_ndvas = MIN(wp->wp_copies + ismd, spa_max_replication(spa)); 3198088f3894Sahrens } 3199088f3894Sahrens 3200c717a561Smaybee zio_t * 3201088f3894Sahrens arc_write(zio_t *pio, spa_t *spa, const writeprops_t *wp, 32023baa08fcSek boolean_t l2arc, uint64_t txg, blkptr_t *bp, arc_buf_t *buf, 3203c717a561Smaybee arc_done_func_t *ready, arc_done_func_t *done, void *private, int priority, 32043baa08fcSek int zio_flags, const zbookmark_t *zb) 3205fa9e4066Sahrens { 3206fa9e4066Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 3207c717a561Smaybee arc_write_callback_t *callback; 3208e14bb325SJeff Bonwick zio_t *zio; 3209e14bb325SJeff Bonwick zio_prop_t zp; 3210fa9e4066Sahrens 3211e14bb325SJeff Bonwick ASSERT(ready != NULL); 3212fa9e4066Sahrens ASSERT(!HDR_IO_ERROR(hdr)); 3213c5c6ffa0Smaybee ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0); 3214c5c6ffa0Smaybee ASSERT(hdr->b_acb == 0); 32153baa08fcSek if (l2arc) 32163baa08fcSek hdr->b_flags |= ARC_L2CACHE; 3217c717a561Smaybee callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP); 3218c717a561Smaybee callback->awcb_ready = ready; 3219c717a561Smaybee callback->awcb_done = done; 3220c717a561Smaybee callback->awcb_private = private; 3221c717a561Smaybee callback->awcb_buf = buf; 3222088f3894Sahrens 3223e14bb325SJeff Bonwick write_policy(spa, wp, &zp); 3224e14bb325SJeff Bonwick zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, &zp, 3225e14bb325SJeff Bonwick arc_write_ready, arc_write_done, callback, priority, zio_flags, zb); 3226fa9e4066Sahrens 3227c717a561Smaybee return (zio); 3228fa9e4066Sahrens } 3229fa9e4066Sahrens 3230fa9e4066Sahrens int 3231fa9e4066Sahrens arc_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 3232fa9e4066Sahrens zio_done_func_t *done, void *private, uint32_t arc_flags) 3233fa9e4066Sahrens { 3234fa9e4066Sahrens arc_buf_hdr_t *ab; 3235fa9e4066Sahrens kmutex_t *hash_lock; 3236fa9e4066Sahrens zio_t *zio; 3237fa9e4066Sahrens 3238fa9e4066Sahrens /* 3239fa9e4066Sahrens * If this buffer is in the cache, release it, so it 3240fa9e4066Sahrens * can be re-used. 3241fa9e4066Sahrens */ 3242fa9e4066Sahrens ab = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock); 3243fa9e4066Sahrens if (ab != NULL) { 3244fa9e4066Sahrens /* 3245fa9e4066Sahrens * The checksum of blocks to free is not always 3246fa9e4066Sahrens * preserved (eg. on the deadlist). However, if it is 3247fa9e4066Sahrens * nonzero, it should match what we have in the cache. 3248fa9e4066Sahrens */ 3249fa9e4066Sahrens ASSERT(bp->blk_cksum.zc_word[0] == 0 || 3250e14bb325SJeff Bonwick bp->blk_cksum.zc_word[0] == ab->b_cksum0 || 3251e14bb325SJeff Bonwick bp->blk_fill == BLK_FILL_ALREADY_FREED); 3252e14bb325SJeff Bonwick 325344cb6abcSbmc if (ab->b_state != arc_anon) 325444cb6abcSbmc arc_change_state(arc_anon, ab, hash_lock); 325513506d1eSmaybee if (HDR_IO_IN_PROGRESS(ab)) { 325613506d1eSmaybee /* 325713506d1eSmaybee * This should only happen when we prefetch. 325813506d1eSmaybee */ 325913506d1eSmaybee ASSERT(ab->b_flags & ARC_PREFETCH); 326013506d1eSmaybee ASSERT3U(ab->b_datacnt, ==, 1); 326113506d1eSmaybee ab->b_flags |= ARC_FREED_IN_READ; 326213506d1eSmaybee if (HDR_IN_HASH_TABLE(ab)) 326313506d1eSmaybee buf_hash_remove(ab); 326413506d1eSmaybee ab->b_arc_access = 0; 326513506d1eSmaybee bzero(&ab->b_dva, sizeof (dva_t)); 326613506d1eSmaybee ab->b_birth = 0; 326713506d1eSmaybee ab->b_cksum0 = 0; 326813506d1eSmaybee ab->b_buf->b_efunc = NULL; 326913506d1eSmaybee ab->b_buf->b_private = NULL; 327013506d1eSmaybee mutex_exit(hash_lock); 327113506d1eSmaybee } else if (refcount_is_zero(&ab->b_refcnt)) { 3272fa94a07fSbrendan ab->b_flags |= ARC_FREE_IN_PROGRESS; 3273fa9e4066Sahrens mutex_exit(hash_lock); 3274ea8dc4b6Seschrock arc_hdr_destroy(ab); 327544cb6abcSbmc ARCSTAT_BUMP(arcstat_deleted); 3276fa9e4066Sahrens } else { 3277bbf4a8dfSmaybee /* 327813506d1eSmaybee * We still have an active reference on this 327913506d1eSmaybee * buffer. This can happen, e.g., from 328013506d1eSmaybee * dbuf_unoverride(). 3281bbf4a8dfSmaybee */ 328213506d1eSmaybee ASSERT(!HDR_IN_HASH_TABLE(ab)); 3283fa9e4066Sahrens ab->b_arc_access = 0; 3284fa9e4066Sahrens bzero(&ab->b_dva, sizeof (dva_t)); 3285fa9e4066Sahrens ab->b_birth = 0; 3286fa9e4066Sahrens ab->b_cksum0 = 0; 3287ea8dc4b6Seschrock ab->b_buf->b_efunc = NULL; 3288ea8dc4b6Seschrock ab->b_buf->b_private = NULL; 3289fa9e4066Sahrens mutex_exit(hash_lock); 3290fa9e4066Sahrens } 3291fa9e4066Sahrens } 3292fa9e4066Sahrens 3293e14bb325SJeff Bonwick zio = zio_free(pio, spa, txg, bp, done, private, ZIO_FLAG_MUSTSUCCEED); 3294fa9e4066Sahrens 3295fa9e4066Sahrens if (arc_flags & ARC_WAIT) 3296fa9e4066Sahrens return (zio_wait(zio)); 3297fa9e4066Sahrens 3298fa9e4066Sahrens ASSERT(arc_flags & ARC_NOWAIT); 3299fa9e4066Sahrens zio_nowait(zio); 3300fa9e4066Sahrens 3301fa9e4066Sahrens return (0); 3302fa9e4066Sahrens } 3303fa9e4066Sahrens 33041ab7f2deSmaybee static int 33051ab7f2deSmaybee arc_memory_throttle(uint64_t reserve, uint64_t txg) 33061ab7f2deSmaybee { 33071ab7f2deSmaybee #ifdef _KERNEL 33081ab7f2deSmaybee uint64_t inflight_data = arc_anon->arcs_size; 33091ab7f2deSmaybee uint64_t available_memory = ptob(freemem); 33101ab7f2deSmaybee static uint64_t page_load = 0; 33111ab7f2deSmaybee static uint64_t last_txg = 0; 33121ab7f2deSmaybee 33131ab7f2deSmaybee #if defined(__i386) 33141ab7f2deSmaybee available_memory = 33151ab7f2deSmaybee MIN(available_memory, vmem_size(heap_arena, VMEM_FREE)); 33161ab7f2deSmaybee #endif 33171ab7f2deSmaybee if (available_memory >= zfs_write_limit_max) 33181ab7f2deSmaybee return (0); 33191ab7f2deSmaybee 33201ab7f2deSmaybee if (txg > last_txg) { 33211ab7f2deSmaybee last_txg = txg; 33221ab7f2deSmaybee page_load = 0; 33231ab7f2deSmaybee } 33241ab7f2deSmaybee /* 33251ab7f2deSmaybee * If we are in pageout, we know that memory is already tight, 33261ab7f2deSmaybee * the arc is already going to be evicting, so we just want to 33271ab7f2deSmaybee * continue to let page writes occur as quickly as possible. 33281ab7f2deSmaybee */ 33291ab7f2deSmaybee if (curproc == proc_pageout) { 33301ab7f2deSmaybee if (page_load > MAX(ptob(minfree), available_memory) / 4) 33311ab7f2deSmaybee return (ERESTART); 33321ab7f2deSmaybee /* Note: reserve is inflated, so we deflate */ 33331ab7f2deSmaybee page_load += reserve / 8; 33341ab7f2deSmaybee return (0); 33351ab7f2deSmaybee } else if (page_load > 0 && arc_reclaim_needed()) { 33361ab7f2deSmaybee /* memory is low, delay before restarting */ 33371ab7f2deSmaybee ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 33381ab7f2deSmaybee return (EAGAIN); 33391ab7f2deSmaybee } 33401ab7f2deSmaybee page_load = 0; 33411ab7f2deSmaybee 33421ab7f2deSmaybee if (arc_size > arc_c_min) { 33431ab7f2deSmaybee uint64_t evictable_memory = 33441ab7f2deSmaybee arc_mru->arcs_lsize[ARC_BUFC_DATA] + 33451ab7f2deSmaybee arc_mru->arcs_lsize[ARC_BUFC_METADATA] + 33461ab7f2deSmaybee arc_mfu->arcs_lsize[ARC_BUFC_DATA] + 33471ab7f2deSmaybee arc_mfu->arcs_lsize[ARC_BUFC_METADATA]; 33481ab7f2deSmaybee available_memory += MIN(evictable_memory, arc_size - arc_c_min); 33491ab7f2deSmaybee } 33501ab7f2deSmaybee 33511ab7f2deSmaybee if (inflight_data > available_memory / 4) { 33521ab7f2deSmaybee ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 33531ab7f2deSmaybee return (ERESTART); 33541ab7f2deSmaybee } 33551ab7f2deSmaybee #endif 33561ab7f2deSmaybee return (0); 33571ab7f2deSmaybee } 33581ab7f2deSmaybee 3359fa9e4066Sahrens void 33601ab7f2deSmaybee arc_tempreserve_clear(uint64_t reserve) 3361fa9e4066Sahrens { 33621ab7f2deSmaybee atomic_add_64(&arc_tempreserve, -reserve); 3363fa9e4066Sahrens ASSERT((int64_t)arc_tempreserve >= 0); 3364fa9e4066Sahrens } 3365fa9e4066Sahrens 3366fa9e4066Sahrens int 33671ab7f2deSmaybee arc_tempreserve_space(uint64_t reserve, uint64_t txg) 3368fa9e4066Sahrens { 33691ab7f2deSmaybee int error; 33701ab7f2deSmaybee 3371fa9e4066Sahrens #ifdef ZFS_DEBUG 3372fa9e4066Sahrens /* 3373fa9e4066Sahrens * Once in a while, fail for no reason. Everything should cope. 3374fa9e4066Sahrens */ 3375fa9e4066Sahrens if (spa_get_random(10000) == 0) { 3376fa9e4066Sahrens dprintf("forcing random failure\n"); 3377fa9e4066Sahrens return (ERESTART); 3378fa9e4066Sahrens } 3379fa9e4066Sahrens #endif 33801ab7f2deSmaybee if (reserve > arc_c/4 && !arc_no_grow) 33811ab7f2deSmaybee arc_c = MIN(arc_c_max, reserve * 4); 33821ab7f2deSmaybee if (reserve > arc_c) 3383112fe045Smaybee return (ENOMEM); 3384112fe045Smaybee 33851ab7f2deSmaybee /* 33861ab7f2deSmaybee * Writes will, almost always, require additional memory allocations 33871ab7f2deSmaybee * in order to compress/encrypt/etc the data. We therefor need to 33881ab7f2deSmaybee * make sure that there is sufficient available memory for this. 33891ab7f2deSmaybee */ 33901ab7f2deSmaybee if (error = arc_memory_throttle(reserve, txg)) 33911ab7f2deSmaybee return (error); 33921ab7f2deSmaybee 3393fa9e4066Sahrens /* 3394112fe045Smaybee * Throttle writes when the amount of dirty data in the cache 3395112fe045Smaybee * gets too large. We try to keep the cache less than half full 3396112fe045Smaybee * of dirty blocks so that our sync times don't grow too large. 3397112fe045Smaybee * Note: if two requests come in concurrently, we might let them 3398112fe045Smaybee * both succeed, when one of them should fail. Not a huge deal. 3399fa9e4066Sahrens */ 34001ab7f2deSmaybee if (reserve + arc_tempreserve + arc_anon->arcs_size > arc_c / 2 && 34011ab7f2deSmaybee arc_anon->arcs_size > arc_c / 4) { 34020e8c6158Smaybee dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK " 34030e8c6158Smaybee "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n", 34040e8c6158Smaybee arc_tempreserve>>10, 34050e8c6158Smaybee arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10, 34060e8c6158Smaybee arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10, 34071ab7f2deSmaybee reserve>>10, arc_c>>10); 3408fa9e4066Sahrens return (ERESTART); 3409fa9e4066Sahrens } 34101ab7f2deSmaybee atomic_add_64(&arc_tempreserve, reserve); 3411fa9e4066Sahrens return (0); 3412fa9e4066Sahrens } 3413fa9e4066Sahrens 3414fa9e4066Sahrens void 3415fa9e4066Sahrens arc_init(void) 3416fa9e4066Sahrens { 3417fa9e4066Sahrens mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL); 3418fa9e4066Sahrens cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL); 3419fa9e4066Sahrens 342013506d1eSmaybee /* Convert seconds to clock ticks */ 3421b19a79ecSperrin arc_min_prefetch_lifespan = 1 * hz; 342213506d1eSmaybee 3423fa9e4066Sahrens /* Start out with 1/8 of all memory */ 342444cb6abcSbmc arc_c = physmem * PAGESIZE / 8; 3425fa9e4066Sahrens 3426fa9e4066Sahrens #ifdef _KERNEL 3427fa9e4066Sahrens /* 3428fa9e4066Sahrens * On architectures where the physical memory can be larger 3429fa9e4066Sahrens * than the addressable space (intel in 32-bit mode), we may 3430fa9e4066Sahrens * need to limit the cache to 1/8 of VM size. 3431fa9e4066Sahrens */ 343244cb6abcSbmc arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8); 3433fa9e4066Sahrens #endif 3434fa9e4066Sahrens 3435112fe045Smaybee /* set min cache to 1/32 of all memory, or 64MB, whichever is more */ 343644cb6abcSbmc arc_c_min = MAX(arc_c / 4, 64<<20); 3437112fe045Smaybee /* set max to 3/4 of all memory, or all but 1GB, whichever is more */ 343844cb6abcSbmc if (arc_c * 8 >= 1<<30) 343944cb6abcSbmc arc_c_max = (arc_c * 8) - (1<<30); 3440fa9e4066Sahrens else 344144cb6abcSbmc arc_c_max = arc_c_min; 344244cb6abcSbmc arc_c_max = MAX(arc_c * 6, arc_c_max); 3443a2eea2e1Sahrens 3444a2eea2e1Sahrens /* 3445a2eea2e1Sahrens * Allow the tunables to override our calculations if they are 3446a2eea2e1Sahrens * reasonable (ie. over 64MB) 3447a2eea2e1Sahrens */ 3448a2eea2e1Sahrens if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE) 344944cb6abcSbmc arc_c_max = zfs_arc_max; 345044cb6abcSbmc if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max) 345144cb6abcSbmc arc_c_min = zfs_arc_min; 3452a2eea2e1Sahrens 345344cb6abcSbmc arc_c = arc_c_max; 345444cb6abcSbmc arc_p = (arc_c >> 1); 3455fa9e4066Sahrens 34560e8c6158Smaybee /* limit meta-data to 1/4 of the arc capacity */ 34570e8c6158Smaybee arc_meta_limit = arc_c_max / 4; 34581116048bSek 34591116048bSek /* Allow the tunable to override if it is reasonable */ 34601116048bSek if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max) 34611116048bSek arc_meta_limit = zfs_arc_meta_limit; 34621116048bSek 34630e8c6158Smaybee if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0) 34640e8c6158Smaybee arc_c_min = arc_meta_limit / 2; 34650e8c6158Smaybee 3466*5a98e54bSBrendan Gregg - Sun Microsystems if (zfs_arc_grow_retry > 0) 3467*5a98e54bSBrendan Gregg - Sun Microsystems arc_grow_retry = zfs_arc_grow_retry; 3468*5a98e54bSBrendan Gregg - Sun Microsystems 3469*5a98e54bSBrendan Gregg - Sun Microsystems if (zfs_arc_shrink_shift > 0) 3470*5a98e54bSBrendan Gregg - Sun Microsystems arc_shrink_shift = zfs_arc_shrink_shift; 3471*5a98e54bSBrendan Gregg - Sun Microsystems 3472*5a98e54bSBrendan Gregg - Sun Microsystems if (zfs_arc_p_min_shift > 0) 3473*5a98e54bSBrendan Gregg - Sun Microsystems arc_p_min_shift = zfs_arc_p_min_shift; 3474*5a98e54bSBrendan Gregg - Sun Microsystems 3475fa9e4066Sahrens /* if kmem_flags are set, lets try to use less memory */ 3476fa9e4066Sahrens if (kmem_debugging()) 347744cb6abcSbmc arc_c = arc_c / 2; 347844cb6abcSbmc if (arc_c < arc_c_min) 347944cb6abcSbmc arc_c = arc_c_min; 348044cb6abcSbmc 348144cb6abcSbmc arc_anon = &ARC_anon; 348244cb6abcSbmc arc_mru = &ARC_mru; 348344cb6abcSbmc arc_mru_ghost = &ARC_mru_ghost; 348444cb6abcSbmc arc_mfu = &ARC_mfu; 348544cb6abcSbmc arc_mfu_ghost = &ARC_mfu_ghost; 3486fa94a07fSbrendan arc_l2c_only = &ARC_l2c_only; 348744cb6abcSbmc arc_size = 0; 348844cb6abcSbmc 348944cb6abcSbmc mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 349044cb6abcSbmc mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 349144cb6abcSbmc mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 349244cb6abcSbmc mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 349344cb6abcSbmc mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3494fa94a07fSbrendan mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 349544cb6abcSbmc 34960e8c6158Smaybee list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA], 34970e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 34980e8c6158Smaybee list_create(&arc_mru->arcs_list[ARC_BUFC_DATA], 34990e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 35000e8c6158Smaybee list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA], 35010e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 35020e8c6158Smaybee list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA], 35030e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 35040e8c6158Smaybee list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA], 35050e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 35060e8c6158Smaybee list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA], 35070e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 35080e8c6158Smaybee list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA], 35090e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 35100e8c6158Smaybee list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA], 35110e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3512fa94a07fSbrendan list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA], 3513fa94a07fSbrendan sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3514fa94a07fSbrendan list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA], 3515fa94a07fSbrendan sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3516fa9e4066Sahrens 3517fa9e4066Sahrens buf_init(); 3518fa9e4066Sahrens 3519fa9e4066Sahrens arc_thread_exit = 0; 3520ea8dc4b6Seschrock arc_eviction_list = NULL; 3521ea8dc4b6Seschrock mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL); 352240d7d650Smaybee bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t)); 3523fa9e4066Sahrens 352444cb6abcSbmc arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED, 352544cb6abcSbmc sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 352644cb6abcSbmc 352744cb6abcSbmc if (arc_ksp != NULL) { 352844cb6abcSbmc arc_ksp->ks_data = &arc_stats; 352944cb6abcSbmc kstat_install(arc_ksp); 353044cb6abcSbmc } 353144cb6abcSbmc 3532fa9e4066Sahrens (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0, 3533fa9e4066Sahrens TS_RUN, minclsyspri); 353449e3519aSmaybee 353549e3519aSmaybee arc_dead = FALSE; 35363a737e0dSbrendan arc_warm = B_FALSE; 35371ab7f2deSmaybee 35381ab7f2deSmaybee if (zfs_write_limit_max == 0) 353905715f94SMark Maybee zfs_write_limit_max = ptob(physmem) >> zfs_write_limit_shift; 35401ab7f2deSmaybee else 35411ab7f2deSmaybee zfs_write_limit_shift = 0; 354205715f94SMark Maybee mutex_init(&zfs_write_limit_lock, NULL, MUTEX_DEFAULT, NULL); 3543fa9e4066Sahrens } 3544fa9e4066Sahrens 3545fa9e4066Sahrens void 3546fa9e4066Sahrens arc_fini(void) 3547fa9e4066Sahrens { 3548fa9e4066Sahrens mutex_enter(&arc_reclaim_thr_lock); 3549fa9e4066Sahrens arc_thread_exit = 1; 3550fa9e4066Sahrens while (arc_thread_exit != 0) 3551fa9e4066Sahrens cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock); 3552fa9e4066Sahrens mutex_exit(&arc_reclaim_thr_lock); 3553fa9e4066Sahrens 3554874395d5Smaybee arc_flush(NULL); 3555fa9e4066Sahrens 3556fa9e4066Sahrens arc_dead = TRUE; 3557fa9e4066Sahrens 355844cb6abcSbmc if (arc_ksp != NULL) { 355944cb6abcSbmc kstat_delete(arc_ksp); 356044cb6abcSbmc arc_ksp = NULL; 356144cb6abcSbmc } 356244cb6abcSbmc 3563ea8dc4b6Seschrock mutex_destroy(&arc_eviction_mtx); 3564fa9e4066Sahrens mutex_destroy(&arc_reclaim_thr_lock); 3565fa9e4066Sahrens cv_destroy(&arc_reclaim_thr_cv); 3566fa9e4066Sahrens 35670e8c6158Smaybee list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]); 35680e8c6158Smaybee list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]); 35690e8c6158Smaybee list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]); 35700e8c6158Smaybee list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]); 35710e8c6158Smaybee list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]); 35720e8c6158Smaybee list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]); 35730e8c6158Smaybee list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]); 35740e8c6158Smaybee list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]); 3575fa9e4066Sahrens 357644cb6abcSbmc mutex_destroy(&arc_anon->arcs_mtx); 357744cb6abcSbmc mutex_destroy(&arc_mru->arcs_mtx); 357844cb6abcSbmc mutex_destroy(&arc_mru_ghost->arcs_mtx); 357944cb6abcSbmc mutex_destroy(&arc_mfu->arcs_mtx); 358044cb6abcSbmc mutex_destroy(&arc_mfu_ghost->arcs_mtx); 3581b5e70f97SRicardo M. Correia mutex_destroy(&arc_l2c_only->arcs_mtx); 35825ad82045Snd 358305715f94SMark Maybee mutex_destroy(&zfs_write_limit_lock); 358405715f94SMark Maybee 3585fa9e4066Sahrens buf_fini(); 3586fa9e4066Sahrens } 3587fa94a07fSbrendan 3588fa94a07fSbrendan /* 3589fa94a07fSbrendan * Level 2 ARC 3590fa94a07fSbrendan * 3591fa94a07fSbrendan * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk. 3592fa94a07fSbrendan * It uses dedicated storage devices to hold cached data, which are populated 3593fa94a07fSbrendan * using large infrequent writes. The main role of this cache is to boost 3594fa94a07fSbrendan * the performance of random read workloads. The intended L2ARC devices 3595fa94a07fSbrendan * include short-stroked disks, solid state disks, and other media with 3596fa94a07fSbrendan * substantially faster read latency than disk. 3597fa94a07fSbrendan * 3598fa94a07fSbrendan * +-----------------------+ 3599fa94a07fSbrendan * | ARC | 3600fa94a07fSbrendan * +-----------------------+ 3601fa94a07fSbrendan * | ^ ^ 3602fa94a07fSbrendan * | | | 3603fa94a07fSbrendan * l2arc_feed_thread() arc_read() 3604fa94a07fSbrendan * | | | 3605fa94a07fSbrendan * | l2arc read | 3606fa94a07fSbrendan * V | | 3607fa94a07fSbrendan * +---------------+ | 3608fa94a07fSbrendan * | L2ARC | | 3609fa94a07fSbrendan * +---------------+ | 3610fa94a07fSbrendan * | ^ | 3611fa94a07fSbrendan * l2arc_write() | | 3612fa94a07fSbrendan * | | | 3613fa94a07fSbrendan * V | | 3614fa94a07fSbrendan * +-------+ +-------+ 3615fa94a07fSbrendan * | vdev | | vdev | 3616fa94a07fSbrendan * | cache | | cache | 3617fa94a07fSbrendan * +-------+ +-------+ 3618fa94a07fSbrendan * +=========+ .-----. 3619fa94a07fSbrendan * : L2ARC : |-_____-| 3620fa94a07fSbrendan * : devices : | Disks | 3621fa94a07fSbrendan * +=========+ `-_____-' 3622fa94a07fSbrendan * 3623fa94a07fSbrendan * Read requests are satisfied from the following sources, in order: 3624fa94a07fSbrendan * 3625fa94a07fSbrendan * 1) ARC 3626fa94a07fSbrendan * 2) vdev cache of L2ARC devices 3627fa94a07fSbrendan * 3) L2ARC devices 3628fa94a07fSbrendan * 4) vdev cache of disks 3629fa94a07fSbrendan * 5) disks 3630fa94a07fSbrendan * 3631fa94a07fSbrendan * Some L2ARC device types exhibit extremely slow write performance. 3632fa94a07fSbrendan * To accommodate for this there are some significant differences between 3633fa94a07fSbrendan * the L2ARC and traditional cache design: 3634fa94a07fSbrendan * 3635fa94a07fSbrendan * 1. There is no eviction path from the ARC to the L2ARC. Evictions from 3636fa94a07fSbrendan * the ARC behave as usual, freeing buffers and placing headers on ghost 3637fa94a07fSbrendan * lists. The ARC does not send buffers to the L2ARC during eviction as 3638fa94a07fSbrendan * this would add inflated write latencies for all ARC memory pressure. 3639fa94a07fSbrendan * 3640fa94a07fSbrendan * 2. The L2ARC attempts to cache data from the ARC before it is evicted. 3641fa94a07fSbrendan * It does this by periodically scanning buffers from the eviction-end of 3642fa94a07fSbrendan * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are 3643fa94a07fSbrendan * not already there. It scans until a headroom of buffers is satisfied, 3644fa94a07fSbrendan * which itself is a buffer for ARC eviction. The thread that does this is 3645fa94a07fSbrendan * l2arc_feed_thread(), illustrated below; example sizes are included to 3646fa94a07fSbrendan * provide a better sense of ratio than this diagram: 3647fa94a07fSbrendan * 3648fa94a07fSbrendan * head --> tail 3649fa94a07fSbrendan * +---------------------+----------+ 3650fa94a07fSbrendan * ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->. # already on L2ARC 3651fa94a07fSbrendan * +---------------------+----------+ | o L2ARC eligible 3652fa94a07fSbrendan * ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->| : ARC buffer 3653fa94a07fSbrendan * +---------------------+----------+ | 3654fa94a07fSbrendan * 15.9 Gbytes ^ 32 Mbytes | 3655fa94a07fSbrendan * headroom | 3656fa94a07fSbrendan * l2arc_feed_thread() 3657fa94a07fSbrendan * | 3658fa94a07fSbrendan * l2arc write hand <--[oooo]--' 3659fa94a07fSbrendan * | 8 Mbyte 3660fa94a07fSbrendan * | write max 3661fa94a07fSbrendan * V 3662fa94a07fSbrendan * +==============================+ 3663fa94a07fSbrendan * L2ARC dev |####|#|###|###| |####| ... | 3664fa94a07fSbrendan * +==============================+ 3665fa94a07fSbrendan * 32 Gbytes 3666fa94a07fSbrendan * 3667fa94a07fSbrendan * 3. If an ARC buffer is copied to the L2ARC but then hit instead of 3668fa94a07fSbrendan * evicted, then the L2ARC has cached a buffer much sooner than it probably 3669fa94a07fSbrendan * needed to, potentially wasting L2ARC device bandwidth and storage. It is 3670fa94a07fSbrendan * safe to say that this is an uncommon case, since buffers at the end of 3671fa94a07fSbrendan * the ARC lists have moved there due to inactivity. 3672fa94a07fSbrendan * 3673fa94a07fSbrendan * 4. If the ARC evicts faster than the L2ARC can maintain a headroom, 3674fa94a07fSbrendan * then the L2ARC simply misses copying some buffers. This serves as a 3675fa94a07fSbrendan * pressure valve to prevent heavy read workloads from both stalling the ARC 3676fa94a07fSbrendan * with waits and clogging the L2ARC with writes. This also helps prevent 3677fa94a07fSbrendan * the potential for the L2ARC to churn if it attempts to cache content too 3678fa94a07fSbrendan * quickly, such as during backups of the entire pool. 3679fa94a07fSbrendan * 36803a737e0dSbrendan * 5. After system boot and before the ARC has filled main memory, there are 36813a737e0dSbrendan * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru 36823a737e0dSbrendan * lists can remain mostly static. Instead of searching from tail of these 36833a737e0dSbrendan * lists as pictured, the l2arc_feed_thread() will search from the list heads 36843a737e0dSbrendan * for eligible buffers, greatly increasing its chance of finding them. 36853a737e0dSbrendan * 36863a737e0dSbrendan * The L2ARC device write speed is also boosted during this time so that 36873a737e0dSbrendan * the L2ARC warms up faster. Since there have been no ARC evictions yet, 36883a737e0dSbrendan * there are no L2ARC reads, and no fear of degrading read performance 36893a737e0dSbrendan * through increased writes. 36903a737e0dSbrendan * 36913a737e0dSbrendan * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that 3692fa94a07fSbrendan * the vdev queue can aggregate them into larger and fewer writes. Each 3693fa94a07fSbrendan * device is written to in a rotor fashion, sweeping writes through 3694fa94a07fSbrendan * available space then repeating. 3695fa94a07fSbrendan * 36963a737e0dSbrendan * 7. The L2ARC does not store dirty content. It never needs to flush 3697fa94a07fSbrendan * write buffers back to disk based storage. 3698fa94a07fSbrendan * 36993a737e0dSbrendan * 8. If an ARC buffer is written (and dirtied) which also exists in the 3700fa94a07fSbrendan * L2ARC, the now stale L2ARC buffer is immediately dropped. 3701fa94a07fSbrendan * 3702fa94a07fSbrendan * The performance of the L2ARC can be tweaked by a number of tunables, which 3703fa94a07fSbrendan * may be necessary for different workloads: 3704fa94a07fSbrendan * 3705fa94a07fSbrendan * l2arc_write_max max write bytes per interval 37063a737e0dSbrendan * l2arc_write_boost extra write bytes during device warmup 3707fa94a07fSbrendan * l2arc_noprefetch skip caching prefetched buffers 3708fa94a07fSbrendan * l2arc_headroom number of max device writes to precache 3709fa94a07fSbrendan * l2arc_feed_secs seconds between L2ARC writing 3710fa94a07fSbrendan * 3711fa94a07fSbrendan * Tunables may be removed or added as future performance improvements are 3712fa94a07fSbrendan * integrated, and also may become zpool properties. 3713*5a98e54bSBrendan Gregg - Sun Microsystems * 3714*5a98e54bSBrendan Gregg - Sun Microsystems * There are three key functions that control how the L2ARC warms up: 3715*5a98e54bSBrendan Gregg - Sun Microsystems * 3716*5a98e54bSBrendan Gregg - Sun Microsystems * l2arc_write_eligible() check if a buffer is eligible to cache 3717*5a98e54bSBrendan Gregg - Sun Microsystems * l2arc_write_size() calculate how much to write 3718*5a98e54bSBrendan Gregg - Sun Microsystems * l2arc_write_interval() calculate sleep delay between writes 3719*5a98e54bSBrendan Gregg - Sun Microsystems * 3720*5a98e54bSBrendan Gregg - Sun Microsystems * These three functions determine what to write, how much, and how quickly 3721*5a98e54bSBrendan Gregg - Sun Microsystems * to send writes. 3722fa94a07fSbrendan */ 3723fa94a07fSbrendan 3724*5a98e54bSBrendan Gregg - Sun Microsystems static boolean_t 3725*5a98e54bSBrendan Gregg - Sun Microsystems l2arc_write_eligible(spa_t *spa, arc_buf_hdr_t *ab) 3726*5a98e54bSBrendan Gregg - Sun Microsystems { 3727*5a98e54bSBrendan Gregg - Sun Microsystems /* 3728*5a98e54bSBrendan Gregg - Sun Microsystems * A buffer is *not* eligible for the L2ARC if it: 3729*5a98e54bSBrendan Gregg - Sun Microsystems * 1. belongs to a different spa. 3730*5a98e54bSBrendan Gregg - Sun Microsystems * 2. has no attached buffer. 3731*5a98e54bSBrendan Gregg - Sun Microsystems * 3. is already cached on the L2ARC. 3732*5a98e54bSBrendan Gregg - Sun Microsystems * 4. has an I/O in progress (it may be an incomplete read). 3733*5a98e54bSBrendan Gregg - Sun Microsystems * 5. is flagged not eligible (zfs property). 3734*5a98e54bSBrendan Gregg - Sun Microsystems */ 3735*5a98e54bSBrendan Gregg - Sun Microsystems if (ab->b_spa != spa || ab->b_buf == NULL || ab->b_l2hdr != NULL || 3736*5a98e54bSBrendan Gregg - Sun Microsystems HDR_IO_IN_PROGRESS(ab) || !HDR_L2CACHE(ab)) 3737*5a98e54bSBrendan Gregg - Sun Microsystems return (B_FALSE); 3738*5a98e54bSBrendan Gregg - Sun Microsystems 3739*5a98e54bSBrendan Gregg - Sun Microsystems return (B_TRUE); 3740*5a98e54bSBrendan Gregg - Sun Microsystems } 3741*5a98e54bSBrendan Gregg - Sun Microsystems 3742*5a98e54bSBrendan Gregg - Sun Microsystems static uint64_t 3743*5a98e54bSBrendan Gregg - Sun Microsystems l2arc_write_size(l2arc_dev_t *dev) 3744*5a98e54bSBrendan Gregg - Sun Microsystems { 3745*5a98e54bSBrendan Gregg - Sun Microsystems uint64_t size; 3746*5a98e54bSBrendan Gregg - Sun Microsystems 3747*5a98e54bSBrendan Gregg - Sun Microsystems size = dev->l2ad_write; 3748*5a98e54bSBrendan Gregg - Sun Microsystems 3749*5a98e54bSBrendan Gregg - Sun Microsystems if (arc_warm == B_FALSE) 3750*5a98e54bSBrendan Gregg - Sun Microsystems size += dev->l2ad_boost; 3751*5a98e54bSBrendan Gregg - Sun Microsystems 3752*5a98e54bSBrendan Gregg - Sun Microsystems return (size); 3753*5a98e54bSBrendan Gregg - Sun Microsystems 3754*5a98e54bSBrendan Gregg - Sun Microsystems } 3755*5a98e54bSBrendan Gregg - Sun Microsystems 3756*5a98e54bSBrendan Gregg - Sun Microsystems static clock_t 3757*5a98e54bSBrendan Gregg - Sun Microsystems l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote) 3758*5a98e54bSBrendan Gregg - Sun Microsystems { 3759*5a98e54bSBrendan Gregg - Sun Microsystems clock_t interval, next; 3760*5a98e54bSBrendan Gregg - Sun Microsystems 3761*5a98e54bSBrendan Gregg - Sun Microsystems /* 3762*5a98e54bSBrendan Gregg - Sun Microsystems * If the ARC lists are busy, increase our write rate; if the 3763*5a98e54bSBrendan Gregg - Sun Microsystems * lists are stale, idle back. This is achieved by checking 3764*5a98e54bSBrendan Gregg - Sun Microsystems * how much we previously wrote - if it was more than half of 3765*5a98e54bSBrendan Gregg - Sun Microsystems * what we wanted, schedule the next write much sooner. 3766*5a98e54bSBrendan Gregg - Sun Microsystems */ 3767*5a98e54bSBrendan Gregg - Sun Microsystems if (l2arc_feed_again && wrote > (wanted / 2)) 3768*5a98e54bSBrendan Gregg - Sun Microsystems interval = (hz * l2arc_feed_min_ms) / 1000; 3769*5a98e54bSBrendan Gregg - Sun Microsystems else 3770*5a98e54bSBrendan Gregg - Sun Microsystems interval = hz * l2arc_feed_secs; 3771*5a98e54bSBrendan Gregg - Sun Microsystems 3772*5a98e54bSBrendan Gregg - Sun Microsystems next = MAX(lbolt, MIN(lbolt + interval, began + interval)); 3773*5a98e54bSBrendan Gregg - Sun Microsystems 3774*5a98e54bSBrendan Gregg - Sun Microsystems return (next); 3775*5a98e54bSBrendan Gregg - Sun Microsystems } 3776*5a98e54bSBrendan Gregg - Sun Microsystems 3777fa94a07fSbrendan static void 3778fa94a07fSbrendan l2arc_hdr_stat_add(void) 3779fa94a07fSbrendan { 3780e6c728e1Sbrendan ARCSTAT_INCR(arcstat_l2_hdr_size, HDR_SIZE + L2HDR_SIZE); 3781e6c728e1Sbrendan ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE); 3782fa94a07fSbrendan } 3783fa94a07fSbrendan 3784fa94a07fSbrendan static void 3785fa94a07fSbrendan l2arc_hdr_stat_remove(void) 3786fa94a07fSbrendan { 3787e6c728e1Sbrendan ARCSTAT_INCR(arcstat_l2_hdr_size, -(HDR_SIZE + L2HDR_SIZE)); 3788e6c728e1Sbrendan ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE); 3789fa94a07fSbrendan } 3790fa94a07fSbrendan 3791fa94a07fSbrendan /* 3792fa94a07fSbrendan * Cycle through L2ARC devices. This is how L2ARC load balances. 37933a737e0dSbrendan * If a device is returned, this also returns holding the spa config lock. 3794fa94a07fSbrendan */ 3795fa94a07fSbrendan static l2arc_dev_t * 3796fa94a07fSbrendan l2arc_dev_get_next(void) 3797fa94a07fSbrendan { 37983a737e0dSbrendan l2arc_dev_t *first, *next = NULL; 37993a737e0dSbrendan 38003a737e0dSbrendan /* 38013a737e0dSbrendan * Lock out the removal of spas (spa_namespace_lock), then removal 38023a737e0dSbrendan * of cache devices (l2arc_dev_mtx). Once a device has been selected, 38033a737e0dSbrendan * both locks will be dropped and a spa config lock held instead. 38043a737e0dSbrendan */ 38053a737e0dSbrendan mutex_enter(&spa_namespace_lock); 38063a737e0dSbrendan mutex_enter(&l2arc_dev_mtx); 3807fa94a07fSbrendan 3808c5904d13Seschrock /* if there are no vdevs, there is nothing to do */ 3809c5904d13Seschrock if (l2arc_ndev == 0) 38103a737e0dSbrendan goto out; 3811c5904d13Seschrock 3812c5904d13Seschrock first = NULL; 3813c5904d13Seschrock next = l2arc_dev_last; 3814c5904d13Seschrock do { 3815c5904d13Seschrock /* loop around the list looking for a non-faulted vdev */ 3816c5904d13Seschrock if (next == NULL) { 3817fa94a07fSbrendan next = list_head(l2arc_dev_list); 3818c5904d13Seschrock } else { 3819c5904d13Seschrock next = list_next(l2arc_dev_list, next); 3820c5904d13Seschrock if (next == NULL) 3821c5904d13Seschrock next = list_head(l2arc_dev_list); 3822c5904d13Seschrock } 3823c5904d13Seschrock 3824c5904d13Seschrock /* if we have come back to the start, bail out */ 3825c5904d13Seschrock if (first == NULL) 3826c5904d13Seschrock first = next; 3827c5904d13Seschrock else if (next == first) 3828c5904d13Seschrock break; 3829c5904d13Seschrock 3830c5904d13Seschrock } while (vdev_is_dead(next->l2ad_vdev)); 3831c5904d13Seschrock 3832c5904d13Seschrock /* if we were unable to find any usable vdevs, return NULL */ 3833c5904d13Seschrock if (vdev_is_dead(next->l2ad_vdev)) 38343a737e0dSbrendan next = NULL; 3835fa94a07fSbrendan 3836fa94a07fSbrendan l2arc_dev_last = next; 3837fa94a07fSbrendan 38383a737e0dSbrendan out: 38393a737e0dSbrendan mutex_exit(&l2arc_dev_mtx); 38403a737e0dSbrendan 38413a737e0dSbrendan /* 38423a737e0dSbrendan * Grab the config lock to prevent the 'next' device from being 38433a737e0dSbrendan * removed while we are writing to it. 38443a737e0dSbrendan */ 38453a737e0dSbrendan if (next != NULL) 3846e14bb325SJeff Bonwick spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER); 38473a737e0dSbrendan mutex_exit(&spa_namespace_lock); 38483a737e0dSbrendan 3849fa94a07fSbrendan return (next); 3850fa94a07fSbrendan } 3851fa94a07fSbrendan 38523a737e0dSbrendan /* 38533a737e0dSbrendan * Free buffers that were tagged for destruction. 38543a737e0dSbrendan */ 38553a737e0dSbrendan static void 38563a737e0dSbrendan l2arc_do_free_on_write() 38573a737e0dSbrendan { 38583a737e0dSbrendan list_t *buflist; 38593a737e0dSbrendan l2arc_data_free_t *df, *df_prev; 38603a737e0dSbrendan 38613a737e0dSbrendan mutex_enter(&l2arc_free_on_write_mtx); 38623a737e0dSbrendan buflist = l2arc_free_on_write; 38633a737e0dSbrendan 38643a737e0dSbrendan for (df = list_tail(buflist); df; df = df_prev) { 38653a737e0dSbrendan df_prev = list_prev(buflist, df); 38663a737e0dSbrendan ASSERT(df->l2df_data != NULL); 38673a737e0dSbrendan ASSERT(df->l2df_func != NULL); 38683a737e0dSbrendan df->l2df_func(df->l2df_data, df->l2df_size); 38693a737e0dSbrendan list_remove(buflist, df); 38703a737e0dSbrendan kmem_free(df, sizeof (l2arc_data_free_t)); 38713a737e0dSbrendan } 38723a737e0dSbrendan 38733a737e0dSbrendan mutex_exit(&l2arc_free_on_write_mtx); 38743a737e0dSbrendan } 38753a737e0dSbrendan 3876fa94a07fSbrendan /* 3877fa94a07fSbrendan * A write to a cache device has completed. Update all headers to allow 3878fa94a07fSbrendan * reads from these buffers to begin. 3879fa94a07fSbrendan */ 3880fa94a07fSbrendan static void 3881fa94a07fSbrendan l2arc_write_done(zio_t *zio) 3882fa94a07fSbrendan { 3883fa94a07fSbrendan l2arc_write_callback_t *cb; 3884fa94a07fSbrendan l2arc_dev_t *dev; 3885fa94a07fSbrendan list_t *buflist; 3886fa94a07fSbrendan arc_buf_hdr_t *head, *ab, *ab_prev; 38873a737e0dSbrendan l2arc_buf_hdr_t *abl2; 3888fa94a07fSbrendan kmutex_t *hash_lock; 3889fa94a07fSbrendan 3890fa94a07fSbrendan cb = zio->io_private; 3891fa94a07fSbrendan ASSERT(cb != NULL); 3892fa94a07fSbrendan dev = cb->l2wcb_dev; 3893fa94a07fSbrendan ASSERT(dev != NULL); 3894fa94a07fSbrendan head = cb->l2wcb_head; 3895fa94a07fSbrendan ASSERT(head != NULL); 3896fa94a07fSbrendan buflist = dev->l2ad_buflist; 3897fa94a07fSbrendan ASSERT(buflist != NULL); 3898fa94a07fSbrendan DTRACE_PROBE2(l2arc__iodone, zio_t *, zio, 3899fa94a07fSbrendan l2arc_write_callback_t *, cb); 3900fa94a07fSbrendan 3901fa94a07fSbrendan if (zio->io_error != 0) 3902fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_writes_error); 3903fa94a07fSbrendan 3904fa94a07fSbrendan mutex_enter(&l2arc_buflist_mtx); 3905fa94a07fSbrendan 3906fa94a07fSbrendan /* 3907fa94a07fSbrendan * All writes completed, or an error was hit. 3908fa94a07fSbrendan */ 3909fa94a07fSbrendan for (ab = list_prev(buflist, head); ab; ab = ab_prev) { 3910fa94a07fSbrendan ab_prev = list_prev(buflist, ab); 3911fa94a07fSbrendan 3912fa94a07fSbrendan hash_lock = HDR_LOCK(ab); 3913fa94a07fSbrendan if (!mutex_tryenter(hash_lock)) { 3914fa94a07fSbrendan /* 3915fa94a07fSbrendan * This buffer misses out. It may be in a stage 3916fa94a07fSbrendan * of eviction. Its ARC_L2_WRITING flag will be 3917fa94a07fSbrendan * left set, denying reads to this buffer. 3918fa94a07fSbrendan */ 3919fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss); 3920fa94a07fSbrendan continue; 3921fa94a07fSbrendan } 3922fa94a07fSbrendan 3923fa94a07fSbrendan if (zio->io_error != 0) { 3924fa94a07fSbrendan /* 39253a737e0dSbrendan * Error - drop L2ARC entry. 3926fa94a07fSbrendan */ 39273a737e0dSbrendan list_remove(buflist, ab); 39283a737e0dSbrendan abl2 = ab->b_l2hdr; 3929fa94a07fSbrendan ab->b_l2hdr = NULL; 39303a737e0dSbrendan kmem_free(abl2, sizeof (l2arc_buf_hdr_t)); 39313a737e0dSbrendan ARCSTAT_INCR(arcstat_l2_size, -ab->b_size); 3932fa94a07fSbrendan } 3933fa94a07fSbrendan 3934fa94a07fSbrendan /* 3935fa94a07fSbrendan * Allow ARC to begin reads to this L2ARC entry. 3936fa94a07fSbrendan */ 3937fa94a07fSbrendan ab->b_flags &= ~ARC_L2_WRITING; 3938fa94a07fSbrendan 3939fa94a07fSbrendan mutex_exit(hash_lock); 3940fa94a07fSbrendan } 3941fa94a07fSbrendan 3942fa94a07fSbrendan atomic_inc_64(&l2arc_writes_done); 3943fa94a07fSbrendan list_remove(buflist, head); 3944fa94a07fSbrendan kmem_cache_free(hdr_cache, head); 3945fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 3946fa94a07fSbrendan 39473a737e0dSbrendan l2arc_do_free_on_write(); 3948fa94a07fSbrendan 3949fa94a07fSbrendan kmem_free(cb, sizeof (l2arc_write_callback_t)); 3950fa94a07fSbrendan } 3951fa94a07fSbrendan 3952fa94a07fSbrendan /* 3953fa94a07fSbrendan * A read to a cache device completed. Validate buffer contents before 3954fa94a07fSbrendan * handing over to the regular ARC routines. 3955fa94a07fSbrendan */ 3956fa94a07fSbrendan static void 3957fa94a07fSbrendan l2arc_read_done(zio_t *zio) 3958fa94a07fSbrendan { 3959fa94a07fSbrendan l2arc_read_callback_t *cb; 3960fa94a07fSbrendan arc_buf_hdr_t *hdr; 3961fa94a07fSbrendan arc_buf_t *buf; 3962fa94a07fSbrendan kmutex_t *hash_lock; 39633a737e0dSbrendan int equal; 3964fa94a07fSbrendan 3965e14bb325SJeff Bonwick ASSERT(zio->io_vd != NULL); 3966e14bb325SJeff Bonwick ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE); 3967e14bb325SJeff Bonwick 3968e14bb325SJeff Bonwick spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd); 3969e14bb325SJeff Bonwick 3970fa94a07fSbrendan cb = zio->io_private; 3971fa94a07fSbrendan ASSERT(cb != NULL); 3972fa94a07fSbrendan buf = cb->l2rcb_buf; 3973fa94a07fSbrendan ASSERT(buf != NULL); 3974fa94a07fSbrendan hdr = buf->b_hdr; 3975fa94a07fSbrendan ASSERT(hdr != NULL); 3976fa94a07fSbrendan 3977fa94a07fSbrendan hash_lock = HDR_LOCK(hdr); 3978fa94a07fSbrendan mutex_enter(hash_lock); 3979fa94a07fSbrendan 3980fa94a07fSbrendan /* 3981fa94a07fSbrendan * Check this survived the L2ARC journey. 3982fa94a07fSbrendan */ 3983fa94a07fSbrendan equal = arc_cksum_equal(buf); 3984fa94a07fSbrendan if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) { 3985fa94a07fSbrendan mutex_exit(hash_lock); 3986fa94a07fSbrendan zio->io_private = buf; 3987e14bb325SJeff Bonwick zio->io_bp_copy = cb->l2rcb_bp; /* XXX fix in L2ARC 2.0 */ 3988e14bb325SJeff Bonwick zio->io_bp = &zio->io_bp_copy; /* XXX fix in L2ARC 2.0 */ 3989fa94a07fSbrendan arc_read_done(zio); 3990fa94a07fSbrendan } else { 3991fa94a07fSbrendan mutex_exit(hash_lock); 3992fa94a07fSbrendan /* 3993fa94a07fSbrendan * Buffer didn't survive caching. Increment stats and 3994fa94a07fSbrendan * reissue to the original storage device. 3995fa94a07fSbrendan */ 39963a737e0dSbrendan if (zio->io_error != 0) { 3997fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_io_error); 39983a737e0dSbrendan } else { 39993a737e0dSbrendan zio->io_error = EIO; 40003a737e0dSbrendan } 4001fa94a07fSbrendan if (!equal) 4002fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_cksum_bad); 4003fa94a07fSbrendan 4004e14bb325SJeff Bonwick /* 4005e14bb325SJeff Bonwick * If there's no waiter, issue an async i/o to the primary 4006e14bb325SJeff Bonwick * storage now. If there *is* a waiter, the caller must 4007e14bb325SJeff Bonwick * issue the i/o in a context where it's OK to block. 4008e14bb325SJeff Bonwick */ 4009e14bb325SJeff Bonwick if (zio->io_waiter == NULL) 4010e14bb325SJeff Bonwick zio_nowait(zio_read(zio->io_parent, 4011e14bb325SJeff Bonwick cb->l2rcb_spa, &cb->l2rcb_bp, 4012e14bb325SJeff Bonwick buf->b_data, zio->io_size, arc_read_done, buf, 4013e14bb325SJeff Bonwick zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb)); 4014fa94a07fSbrendan } 4015fa94a07fSbrendan 4016fa94a07fSbrendan kmem_free(cb, sizeof (l2arc_read_callback_t)); 4017fa94a07fSbrendan } 4018fa94a07fSbrendan 4019fa94a07fSbrendan /* 4020fa94a07fSbrendan * This is the list priority from which the L2ARC will search for pages to 4021fa94a07fSbrendan * cache. This is used within loops (0..3) to cycle through lists in the 4022fa94a07fSbrendan * desired order. This order can have a significant effect on cache 4023fa94a07fSbrendan * performance. 4024fa94a07fSbrendan * 4025fa94a07fSbrendan * Currently the metadata lists are hit first, MFU then MRU, followed by 4026fa94a07fSbrendan * the data lists. This function returns a locked list, and also returns 4027fa94a07fSbrendan * the lock pointer. 4028fa94a07fSbrendan */ 4029fa94a07fSbrendan static list_t * 4030fa94a07fSbrendan l2arc_list_locked(int list_num, kmutex_t **lock) 4031fa94a07fSbrendan { 4032fa94a07fSbrendan list_t *list; 4033fa94a07fSbrendan 4034fa94a07fSbrendan ASSERT(list_num >= 0 && list_num <= 3); 4035fa94a07fSbrendan 4036fa94a07fSbrendan switch (list_num) { 4037fa94a07fSbrendan case 0: 4038fa94a07fSbrendan list = &arc_mfu->arcs_list[ARC_BUFC_METADATA]; 4039fa94a07fSbrendan *lock = &arc_mfu->arcs_mtx; 4040fa94a07fSbrendan break; 4041fa94a07fSbrendan case 1: 4042fa94a07fSbrendan list = &arc_mru->arcs_list[ARC_BUFC_METADATA]; 4043fa94a07fSbrendan *lock = &arc_mru->arcs_mtx; 4044fa94a07fSbrendan break; 4045fa94a07fSbrendan case 2: 4046fa94a07fSbrendan list = &arc_mfu->arcs_list[ARC_BUFC_DATA]; 4047fa94a07fSbrendan *lock = &arc_mfu->arcs_mtx; 4048fa94a07fSbrendan break; 4049fa94a07fSbrendan case 3: 4050fa94a07fSbrendan list = &arc_mru->arcs_list[ARC_BUFC_DATA]; 4051fa94a07fSbrendan *lock = &arc_mru->arcs_mtx; 4052fa94a07fSbrendan break; 4053fa94a07fSbrendan } 4054fa94a07fSbrendan 4055fa94a07fSbrendan ASSERT(!(MUTEX_HELD(*lock))); 4056fa94a07fSbrendan mutex_enter(*lock); 4057fa94a07fSbrendan return (list); 4058fa94a07fSbrendan } 4059fa94a07fSbrendan 4060fa94a07fSbrendan /* 4061fa94a07fSbrendan * Evict buffers from the device write hand to the distance specified in 4062fa94a07fSbrendan * bytes. This distance may span populated buffers, it may span nothing. 4063fa94a07fSbrendan * This is clearing a region on the L2ARC device ready for writing. 4064fa94a07fSbrendan * If the 'all' boolean is set, every buffer is evicted. 4065fa94a07fSbrendan */ 4066fa94a07fSbrendan static void 4067fa94a07fSbrendan l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all) 4068fa94a07fSbrendan { 4069fa94a07fSbrendan list_t *buflist; 4070fa94a07fSbrendan l2arc_buf_hdr_t *abl2; 4071fa94a07fSbrendan arc_buf_hdr_t *ab, *ab_prev; 4072fa94a07fSbrendan kmutex_t *hash_lock; 4073fa94a07fSbrendan uint64_t taddr; 4074fa94a07fSbrendan 4075fa94a07fSbrendan buflist = dev->l2ad_buflist; 4076fa94a07fSbrendan 4077fa94a07fSbrendan if (buflist == NULL) 4078fa94a07fSbrendan return; 4079fa94a07fSbrendan 4080fa94a07fSbrendan if (!all && dev->l2ad_first) { 4081fa94a07fSbrendan /* 4082fa94a07fSbrendan * This is the first sweep through the device. There is 4083fa94a07fSbrendan * nothing to evict. 4084fa94a07fSbrendan */ 4085fa94a07fSbrendan return; 4086fa94a07fSbrendan } 4087fa94a07fSbrendan 40883a737e0dSbrendan if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) { 4089fa94a07fSbrendan /* 4090fa94a07fSbrendan * When nearing the end of the device, evict to the end 4091fa94a07fSbrendan * before the device write hand jumps to the start. 4092fa94a07fSbrendan */ 4093fa94a07fSbrendan taddr = dev->l2ad_end; 4094fa94a07fSbrendan } else { 4095fa94a07fSbrendan taddr = dev->l2ad_hand + distance; 4096fa94a07fSbrendan } 4097fa94a07fSbrendan DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist, 4098fa94a07fSbrendan uint64_t, taddr, boolean_t, all); 4099fa94a07fSbrendan 4100fa94a07fSbrendan top: 4101fa94a07fSbrendan mutex_enter(&l2arc_buflist_mtx); 4102fa94a07fSbrendan for (ab = list_tail(buflist); ab; ab = ab_prev) { 4103fa94a07fSbrendan ab_prev = list_prev(buflist, ab); 4104fa94a07fSbrendan 4105fa94a07fSbrendan hash_lock = HDR_LOCK(ab); 4106fa94a07fSbrendan if (!mutex_tryenter(hash_lock)) { 4107fa94a07fSbrendan /* 4108fa94a07fSbrendan * Missed the hash lock. Retry. 4109fa94a07fSbrendan */ 4110fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_evict_lock_retry); 4111fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 4112fa94a07fSbrendan mutex_enter(hash_lock); 4113fa94a07fSbrendan mutex_exit(hash_lock); 4114fa94a07fSbrendan goto top; 4115fa94a07fSbrendan } 4116fa94a07fSbrendan 4117fa94a07fSbrendan if (HDR_L2_WRITE_HEAD(ab)) { 4118fa94a07fSbrendan /* 4119fa94a07fSbrendan * We hit a write head node. Leave it for 4120fa94a07fSbrendan * l2arc_write_done(). 4121fa94a07fSbrendan */ 4122fa94a07fSbrendan list_remove(buflist, ab); 4123fa94a07fSbrendan mutex_exit(hash_lock); 4124fa94a07fSbrendan continue; 4125fa94a07fSbrendan } 4126fa94a07fSbrendan 4127fa94a07fSbrendan if (!all && ab->b_l2hdr != NULL && 4128fa94a07fSbrendan (ab->b_l2hdr->b_daddr > taddr || 4129fa94a07fSbrendan ab->b_l2hdr->b_daddr < dev->l2ad_hand)) { 4130fa94a07fSbrendan /* 4131fa94a07fSbrendan * We've evicted to the target address, 4132fa94a07fSbrendan * or the end of the device. 4133fa94a07fSbrendan */ 4134fa94a07fSbrendan mutex_exit(hash_lock); 4135fa94a07fSbrendan break; 4136fa94a07fSbrendan } 4137fa94a07fSbrendan 4138fa94a07fSbrendan if (HDR_FREE_IN_PROGRESS(ab)) { 4139fa94a07fSbrendan /* 4140fa94a07fSbrendan * Already on the path to destruction. 4141fa94a07fSbrendan */ 4142fa94a07fSbrendan mutex_exit(hash_lock); 4143fa94a07fSbrendan continue; 4144fa94a07fSbrendan } 4145fa94a07fSbrendan 4146fa94a07fSbrendan if (ab->b_state == arc_l2c_only) { 4147fa94a07fSbrendan ASSERT(!HDR_L2_READING(ab)); 4148fa94a07fSbrendan /* 4149fa94a07fSbrendan * This doesn't exist in the ARC. Destroy. 4150fa94a07fSbrendan * arc_hdr_destroy() will call list_remove() 4151fa94a07fSbrendan * and decrement arcstat_l2_size. 4152fa94a07fSbrendan */ 4153fa94a07fSbrendan arc_change_state(arc_anon, ab, hash_lock); 4154fa94a07fSbrendan arc_hdr_destroy(ab); 4155fa94a07fSbrendan } else { 41563a737e0dSbrendan /* 41573a737e0dSbrendan * Invalidate issued or about to be issued 41583a737e0dSbrendan * reads, since we may be about to write 41593a737e0dSbrendan * over this location. 41603a737e0dSbrendan */ 41613a737e0dSbrendan if (HDR_L2_READING(ab)) { 41623a737e0dSbrendan ARCSTAT_BUMP(arcstat_l2_evict_reading); 41633a737e0dSbrendan ab->b_flags |= ARC_L2_EVICTED; 41643a737e0dSbrendan } 41653a737e0dSbrendan 4166fa94a07fSbrendan /* 4167fa94a07fSbrendan * Tell ARC this no longer exists in L2ARC. 4168fa94a07fSbrendan */ 4169fa94a07fSbrendan if (ab->b_l2hdr != NULL) { 4170fa94a07fSbrendan abl2 = ab->b_l2hdr; 4171fa94a07fSbrendan ab->b_l2hdr = NULL; 4172fa94a07fSbrendan kmem_free(abl2, sizeof (l2arc_buf_hdr_t)); 4173fa94a07fSbrendan ARCSTAT_INCR(arcstat_l2_size, -ab->b_size); 4174fa94a07fSbrendan } 4175fa94a07fSbrendan list_remove(buflist, ab); 4176fa94a07fSbrendan 4177fa94a07fSbrendan /* 4178fa94a07fSbrendan * This may have been leftover after a 4179fa94a07fSbrendan * failed write. 4180fa94a07fSbrendan */ 4181fa94a07fSbrendan ab->b_flags &= ~ARC_L2_WRITING; 4182fa94a07fSbrendan } 4183fa94a07fSbrendan mutex_exit(hash_lock); 4184fa94a07fSbrendan } 4185fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 4186fa94a07fSbrendan 4187fa94a07fSbrendan spa_l2cache_space_update(dev->l2ad_vdev, 0, -(taddr - dev->l2ad_evict)); 4188fa94a07fSbrendan dev->l2ad_evict = taddr; 4189fa94a07fSbrendan } 4190fa94a07fSbrendan 4191fa94a07fSbrendan /* 4192fa94a07fSbrendan * Find and write ARC buffers to the L2ARC device. 4193fa94a07fSbrendan * 4194fa94a07fSbrendan * An ARC_L2_WRITING flag is set so that the L2ARC buffers are not valid 4195fa94a07fSbrendan * for reading until they have completed writing. 4196fa94a07fSbrendan */ 4197*5a98e54bSBrendan Gregg - Sun Microsystems static uint64_t 41983a737e0dSbrendan l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz) 4199fa94a07fSbrendan { 4200fa94a07fSbrendan arc_buf_hdr_t *ab, *ab_prev, *head; 4201fa94a07fSbrendan l2arc_buf_hdr_t *hdrl2; 4202fa94a07fSbrendan list_t *list; 42033a737e0dSbrendan uint64_t passed_sz, write_sz, buf_sz, headroom; 4204fa94a07fSbrendan void *buf_data; 4205fa94a07fSbrendan kmutex_t *hash_lock, *list_lock; 4206fa94a07fSbrendan boolean_t have_lock, full; 4207fa94a07fSbrendan l2arc_write_callback_t *cb; 4208fa94a07fSbrendan zio_t *pio, *wzio; 4209fa94a07fSbrendan 4210fa94a07fSbrendan ASSERT(dev->l2ad_vdev != NULL); 4211fa94a07fSbrendan 4212fa94a07fSbrendan pio = NULL; 4213fa94a07fSbrendan write_sz = 0; 4214fa94a07fSbrendan full = B_FALSE; 42151ab7f2deSmaybee head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 4216fa94a07fSbrendan head->b_flags |= ARC_L2_WRITE_HEAD; 4217fa94a07fSbrendan 4218fa94a07fSbrendan /* 4219fa94a07fSbrendan * Copy buffers for L2ARC writing. 4220fa94a07fSbrendan */ 4221fa94a07fSbrendan mutex_enter(&l2arc_buflist_mtx); 4222fa94a07fSbrendan for (int try = 0; try <= 3; try++) { 4223fa94a07fSbrendan list = l2arc_list_locked(try, &list_lock); 4224fa94a07fSbrendan passed_sz = 0; 4225fa94a07fSbrendan 42263a737e0dSbrendan /* 42273a737e0dSbrendan * L2ARC fast warmup. 42283a737e0dSbrendan * 42293a737e0dSbrendan * Until the ARC is warm and starts to evict, read from the 42303a737e0dSbrendan * head of the ARC lists rather than the tail. 42313a737e0dSbrendan */ 42323a737e0dSbrendan headroom = target_sz * l2arc_headroom; 42333a737e0dSbrendan if (arc_warm == B_FALSE) 42343a737e0dSbrendan ab = list_head(list); 42353a737e0dSbrendan else 42363a737e0dSbrendan ab = list_tail(list); 42373a737e0dSbrendan 42383a737e0dSbrendan for (; ab; ab = ab_prev) { 42393a737e0dSbrendan if (arc_warm == B_FALSE) 42403a737e0dSbrendan ab_prev = list_next(list, ab); 42413a737e0dSbrendan else 42423a737e0dSbrendan ab_prev = list_prev(list, ab); 4243fa94a07fSbrendan 4244fa94a07fSbrendan hash_lock = HDR_LOCK(ab); 4245fa94a07fSbrendan have_lock = MUTEX_HELD(hash_lock); 4246fa94a07fSbrendan if (!have_lock && !mutex_tryenter(hash_lock)) { 4247fa94a07fSbrendan /* 4248fa94a07fSbrendan * Skip this buffer rather than waiting. 4249fa94a07fSbrendan */ 4250fa94a07fSbrendan continue; 4251fa94a07fSbrendan } 4252fa94a07fSbrendan 4253fa94a07fSbrendan passed_sz += ab->b_size; 4254fa94a07fSbrendan if (passed_sz > headroom) { 4255fa94a07fSbrendan /* 4256fa94a07fSbrendan * Searched too far. 4257fa94a07fSbrendan */ 4258fa94a07fSbrendan mutex_exit(hash_lock); 4259fa94a07fSbrendan break; 4260fa94a07fSbrendan } 4261fa94a07fSbrendan 4262*5a98e54bSBrendan Gregg - Sun Microsystems if (!l2arc_write_eligible(spa, ab)) { 4263fa94a07fSbrendan mutex_exit(hash_lock); 4264fa94a07fSbrendan continue; 4265fa94a07fSbrendan } 4266fa94a07fSbrendan 4267fa94a07fSbrendan if ((write_sz + ab->b_size) > target_sz) { 4268fa94a07fSbrendan full = B_TRUE; 4269fa94a07fSbrendan mutex_exit(hash_lock); 4270fa94a07fSbrendan break; 4271fa94a07fSbrendan } 4272fa94a07fSbrendan 4273fa94a07fSbrendan if (pio == NULL) { 4274fa94a07fSbrendan /* 4275fa94a07fSbrendan * Insert a dummy header on the buflist so 4276fa94a07fSbrendan * l2arc_write_done() can find where the 4277fa94a07fSbrendan * write buffers begin without searching. 4278fa94a07fSbrendan */ 4279fa94a07fSbrendan list_insert_head(dev->l2ad_buflist, head); 4280fa94a07fSbrendan 4281fa94a07fSbrendan cb = kmem_alloc( 4282fa94a07fSbrendan sizeof (l2arc_write_callback_t), KM_SLEEP); 4283fa94a07fSbrendan cb->l2wcb_dev = dev; 4284fa94a07fSbrendan cb->l2wcb_head = head; 4285fa94a07fSbrendan pio = zio_root(spa, l2arc_write_done, cb, 4286fa94a07fSbrendan ZIO_FLAG_CANFAIL); 4287fa94a07fSbrendan } 4288fa94a07fSbrendan 4289fa94a07fSbrendan /* 4290fa94a07fSbrendan * Create and add a new L2ARC header. 4291fa94a07fSbrendan */ 4292fa94a07fSbrendan hdrl2 = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP); 4293fa94a07fSbrendan hdrl2->b_dev = dev; 4294fa94a07fSbrendan hdrl2->b_daddr = dev->l2ad_hand; 4295fa94a07fSbrendan 4296fa94a07fSbrendan ab->b_flags |= ARC_L2_WRITING; 4297fa94a07fSbrendan ab->b_l2hdr = hdrl2; 4298fa94a07fSbrendan list_insert_head(dev->l2ad_buflist, ab); 4299fa94a07fSbrendan buf_data = ab->b_buf->b_data; 4300fa94a07fSbrendan buf_sz = ab->b_size; 4301fa94a07fSbrendan 4302fa94a07fSbrendan /* 4303fa94a07fSbrendan * Compute and store the buffer cksum before 4304fa94a07fSbrendan * writing. On debug the cksum is verified first. 4305fa94a07fSbrendan */ 4306fa94a07fSbrendan arc_cksum_verify(ab->b_buf); 4307fa94a07fSbrendan arc_cksum_compute(ab->b_buf, B_TRUE); 4308fa94a07fSbrendan 4309fa94a07fSbrendan mutex_exit(hash_lock); 4310fa94a07fSbrendan 4311fa94a07fSbrendan wzio = zio_write_phys(pio, dev->l2ad_vdev, 4312fa94a07fSbrendan dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF, 4313fa94a07fSbrendan NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE, 4314fa94a07fSbrendan ZIO_FLAG_CANFAIL, B_FALSE); 4315fa94a07fSbrendan 4316fa94a07fSbrendan DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, 4317fa94a07fSbrendan zio_t *, wzio); 4318fa94a07fSbrendan (void) zio_nowait(wzio); 4319fa94a07fSbrendan 4320e14bb325SJeff Bonwick /* 4321e14bb325SJeff Bonwick * Keep the clock hand suitably device-aligned. 4322e14bb325SJeff Bonwick */ 4323e14bb325SJeff Bonwick buf_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz); 4324e14bb325SJeff Bonwick 4325fa94a07fSbrendan write_sz += buf_sz; 4326fa94a07fSbrendan dev->l2ad_hand += buf_sz; 4327fa94a07fSbrendan } 4328fa94a07fSbrendan 4329fa94a07fSbrendan mutex_exit(list_lock); 4330fa94a07fSbrendan 4331fa94a07fSbrendan if (full == B_TRUE) 4332fa94a07fSbrendan break; 4333fa94a07fSbrendan } 4334fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 4335fa94a07fSbrendan 4336fa94a07fSbrendan if (pio == NULL) { 4337fa94a07fSbrendan ASSERT3U(write_sz, ==, 0); 4338fa94a07fSbrendan kmem_cache_free(hdr_cache, head); 4339*5a98e54bSBrendan Gregg - Sun Microsystems return (0); 4340fa94a07fSbrendan } 4341fa94a07fSbrendan 4342fa94a07fSbrendan ASSERT3U(write_sz, <=, target_sz); 4343fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_writes_sent); 4344*5a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_l2_write_bytes, write_sz); 4345fa94a07fSbrendan ARCSTAT_INCR(arcstat_l2_size, write_sz); 4346fa94a07fSbrendan spa_l2cache_space_update(dev->l2ad_vdev, 0, write_sz); 4347fa94a07fSbrendan 4348fa94a07fSbrendan /* 4349fa94a07fSbrendan * Bump device hand to the device start if it is approaching the end. 4350fa94a07fSbrendan * l2arc_evict() will already have evicted ahead for this case. 4351fa94a07fSbrendan */ 43523a737e0dSbrendan if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) { 4353fa94a07fSbrendan spa_l2cache_space_update(dev->l2ad_vdev, 0, 4354fa94a07fSbrendan dev->l2ad_end - dev->l2ad_hand); 4355fa94a07fSbrendan dev->l2ad_hand = dev->l2ad_start; 4356fa94a07fSbrendan dev->l2ad_evict = dev->l2ad_start; 4357fa94a07fSbrendan dev->l2ad_first = B_FALSE; 4358fa94a07fSbrendan } 4359fa94a07fSbrendan 4360*5a98e54bSBrendan Gregg - Sun Microsystems dev->l2ad_writing = B_TRUE; 4361fa94a07fSbrendan (void) zio_wait(pio); 4362*5a98e54bSBrendan Gregg - Sun Microsystems dev->l2ad_writing = B_FALSE; 4363*5a98e54bSBrendan Gregg - Sun Microsystems 4364*5a98e54bSBrendan Gregg - Sun Microsystems return (write_sz); 4365fa94a07fSbrendan } 4366fa94a07fSbrendan 4367fa94a07fSbrendan /* 4368fa94a07fSbrendan * This thread feeds the L2ARC at regular intervals. This is the beating 4369fa94a07fSbrendan * heart of the L2ARC. 4370fa94a07fSbrendan */ 4371fa94a07fSbrendan static void 4372fa94a07fSbrendan l2arc_feed_thread(void) 4373fa94a07fSbrendan { 4374fa94a07fSbrendan callb_cpr_t cpr; 4375fa94a07fSbrendan l2arc_dev_t *dev; 4376fa94a07fSbrendan spa_t *spa; 4377*5a98e54bSBrendan Gregg - Sun Microsystems uint64_t size, wrote; 4378*5a98e54bSBrendan Gregg - Sun Microsystems clock_t begin, next = lbolt; 4379fa94a07fSbrendan 4380fa94a07fSbrendan CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG); 4381fa94a07fSbrendan 4382fa94a07fSbrendan mutex_enter(&l2arc_feed_thr_lock); 4383fa94a07fSbrendan 4384fa94a07fSbrendan while (l2arc_thread_exit == 0) { 4385fa94a07fSbrendan CALLB_CPR_SAFE_BEGIN(&cpr); 4386fa94a07fSbrendan (void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock, 4387*5a98e54bSBrendan Gregg - Sun Microsystems next); 4388fa94a07fSbrendan CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock); 4389*5a98e54bSBrendan Gregg - Sun Microsystems next = lbolt + hz; 4390fa94a07fSbrendan 43913a737e0dSbrendan /* 43923a737e0dSbrendan * Quick check for L2ARC devices. 43933a737e0dSbrendan */ 4394c5904d13Seschrock mutex_enter(&l2arc_dev_mtx); 43953a737e0dSbrendan if (l2arc_ndev == 0) { 43963a737e0dSbrendan mutex_exit(&l2arc_dev_mtx); 43973a737e0dSbrendan continue; 43983a737e0dSbrendan } 43993a737e0dSbrendan mutex_exit(&l2arc_dev_mtx); 4400*5a98e54bSBrendan Gregg - Sun Microsystems begin = lbolt; 4401c5904d13Seschrock 4402fa94a07fSbrendan /* 4403c5904d13Seschrock * This selects the next l2arc device to write to, and in 4404c5904d13Seschrock * doing so the next spa to feed from: dev->l2ad_spa. This 44053a737e0dSbrendan * will return NULL if there are now no l2arc devices or if 44063a737e0dSbrendan * they are all faulted. 44073a737e0dSbrendan * 44083a737e0dSbrendan * If a device is returned, its spa's config lock is also 44093a737e0dSbrendan * held to prevent device removal. l2arc_dev_get_next() 44103a737e0dSbrendan * will grab and release l2arc_dev_mtx. 4411fa94a07fSbrendan */ 44123a737e0dSbrendan if ((dev = l2arc_dev_get_next()) == NULL) 4413fa94a07fSbrendan continue; 44143a737e0dSbrendan 44153a737e0dSbrendan spa = dev->l2ad_spa; 44163a737e0dSbrendan ASSERT(spa != NULL); 4417fa94a07fSbrendan 4418fa94a07fSbrendan /* 4419fa94a07fSbrendan * Avoid contributing to memory pressure. 4420fa94a07fSbrendan */ 4421fa94a07fSbrendan if (arc_reclaim_needed()) { 4422fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_abort_lowmem); 4423e14bb325SJeff Bonwick spa_config_exit(spa, SCL_L2ARC, dev); 4424fa94a07fSbrendan continue; 4425fa94a07fSbrendan } 4426fa94a07fSbrendan 4427fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_feeds); 4428fa94a07fSbrendan 4429*5a98e54bSBrendan Gregg - Sun Microsystems size = l2arc_write_size(dev); 44303a737e0dSbrendan 4431fa94a07fSbrendan /* 4432fa94a07fSbrendan * Evict L2ARC buffers that will be overwritten. 4433fa94a07fSbrendan */ 44343a737e0dSbrendan l2arc_evict(dev, size, B_FALSE); 4435fa94a07fSbrendan 4436fa94a07fSbrendan /* 4437fa94a07fSbrendan * Write ARC buffers. 4438fa94a07fSbrendan */ 4439*5a98e54bSBrendan Gregg - Sun Microsystems wrote = l2arc_write_buffers(spa, dev, size); 4440*5a98e54bSBrendan Gregg - Sun Microsystems 4441*5a98e54bSBrendan Gregg - Sun Microsystems /* 4442*5a98e54bSBrendan Gregg - Sun Microsystems * Calculate interval between writes. 4443*5a98e54bSBrendan Gregg - Sun Microsystems */ 4444*5a98e54bSBrendan Gregg - Sun Microsystems next = l2arc_write_interval(begin, size, wrote); 4445e14bb325SJeff Bonwick spa_config_exit(spa, SCL_L2ARC, dev); 4446fa94a07fSbrendan } 4447fa94a07fSbrendan 4448fa94a07fSbrendan l2arc_thread_exit = 0; 4449fa94a07fSbrendan cv_broadcast(&l2arc_feed_thr_cv); 4450fa94a07fSbrendan CALLB_CPR_EXIT(&cpr); /* drops l2arc_feed_thr_lock */ 4451fa94a07fSbrendan thread_exit(); 4452fa94a07fSbrendan } 4453fa94a07fSbrendan 4454c5904d13Seschrock boolean_t 4455c5904d13Seschrock l2arc_vdev_present(vdev_t *vd) 4456c5904d13Seschrock { 4457c5904d13Seschrock l2arc_dev_t *dev; 4458c5904d13Seschrock 4459c5904d13Seschrock mutex_enter(&l2arc_dev_mtx); 4460c5904d13Seschrock for (dev = list_head(l2arc_dev_list); dev != NULL; 4461c5904d13Seschrock dev = list_next(l2arc_dev_list, dev)) { 4462c5904d13Seschrock if (dev->l2ad_vdev == vd) 4463c5904d13Seschrock break; 4464c5904d13Seschrock } 4465c5904d13Seschrock mutex_exit(&l2arc_dev_mtx); 4466c5904d13Seschrock 4467c5904d13Seschrock return (dev != NULL); 4468c5904d13Seschrock } 4469c5904d13Seschrock 4470fa94a07fSbrendan /* 4471fa94a07fSbrendan * Add a vdev for use by the L2ARC. By this point the spa has already 4472fa94a07fSbrendan * validated the vdev and opened it. 4473fa94a07fSbrendan */ 4474fa94a07fSbrendan void 4475fa94a07fSbrendan l2arc_add_vdev(spa_t *spa, vdev_t *vd, uint64_t start, uint64_t end) 4476fa94a07fSbrendan { 4477fa94a07fSbrendan l2arc_dev_t *adddev; 4478fa94a07fSbrendan 4479c5904d13Seschrock ASSERT(!l2arc_vdev_present(vd)); 4480c5904d13Seschrock 4481fa94a07fSbrendan /* 4482fa94a07fSbrendan * Create a new l2arc device entry. 4483fa94a07fSbrendan */ 4484fa94a07fSbrendan adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP); 4485fa94a07fSbrendan adddev->l2ad_spa = spa; 4486fa94a07fSbrendan adddev->l2ad_vdev = vd; 4487fa94a07fSbrendan adddev->l2ad_write = l2arc_write_max; 44883a737e0dSbrendan adddev->l2ad_boost = l2arc_write_boost; 4489fa94a07fSbrendan adddev->l2ad_start = start; 4490fa94a07fSbrendan adddev->l2ad_end = end; 4491fa94a07fSbrendan adddev->l2ad_hand = adddev->l2ad_start; 4492fa94a07fSbrendan adddev->l2ad_evict = adddev->l2ad_start; 4493fa94a07fSbrendan adddev->l2ad_first = B_TRUE; 4494*5a98e54bSBrendan Gregg - Sun Microsystems adddev->l2ad_writing = B_FALSE; 4495fa94a07fSbrendan ASSERT3U(adddev->l2ad_write, >, 0); 4496fa94a07fSbrendan 4497fa94a07fSbrendan /* 4498fa94a07fSbrendan * This is a list of all ARC buffers that are still valid on the 4499fa94a07fSbrendan * device. 4500fa94a07fSbrendan */ 4501fa94a07fSbrendan adddev->l2ad_buflist = kmem_zalloc(sizeof (list_t), KM_SLEEP); 4502fa94a07fSbrendan list_create(adddev->l2ad_buflist, sizeof (arc_buf_hdr_t), 4503fa94a07fSbrendan offsetof(arc_buf_hdr_t, b_l2node)); 4504fa94a07fSbrendan 4505fa94a07fSbrendan spa_l2cache_space_update(vd, adddev->l2ad_end - adddev->l2ad_hand, 0); 4506fa94a07fSbrendan 4507fa94a07fSbrendan /* 4508fa94a07fSbrendan * Add device to global list 4509fa94a07fSbrendan */ 4510fa94a07fSbrendan mutex_enter(&l2arc_dev_mtx); 4511fa94a07fSbrendan list_insert_head(l2arc_dev_list, adddev); 4512fa94a07fSbrendan atomic_inc_64(&l2arc_ndev); 4513fa94a07fSbrendan mutex_exit(&l2arc_dev_mtx); 4514fa94a07fSbrendan } 4515fa94a07fSbrendan 4516fa94a07fSbrendan /* 4517fa94a07fSbrendan * Remove a vdev from the L2ARC. 4518fa94a07fSbrendan */ 4519fa94a07fSbrendan void 4520fa94a07fSbrendan l2arc_remove_vdev(vdev_t *vd) 4521fa94a07fSbrendan { 4522fa94a07fSbrendan l2arc_dev_t *dev, *nextdev, *remdev = NULL; 4523fa94a07fSbrendan 4524fa94a07fSbrendan /* 4525fa94a07fSbrendan * Find the device by vdev 4526fa94a07fSbrendan */ 4527fa94a07fSbrendan mutex_enter(&l2arc_dev_mtx); 4528fa94a07fSbrendan for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) { 4529fa94a07fSbrendan nextdev = list_next(l2arc_dev_list, dev); 4530fa94a07fSbrendan if (vd == dev->l2ad_vdev) { 4531fa94a07fSbrendan remdev = dev; 4532fa94a07fSbrendan break; 4533fa94a07fSbrendan } 4534fa94a07fSbrendan } 4535fa94a07fSbrendan ASSERT(remdev != NULL); 4536fa94a07fSbrendan 4537fa94a07fSbrendan /* 4538fa94a07fSbrendan * Remove device from global list 4539fa94a07fSbrendan */ 4540fa94a07fSbrendan list_remove(l2arc_dev_list, remdev); 4541fa94a07fSbrendan l2arc_dev_last = NULL; /* may have been invalidated */ 45423a737e0dSbrendan atomic_dec_64(&l2arc_ndev); 45433a737e0dSbrendan mutex_exit(&l2arc_dev_mtx); 4544fa94a07fSbrendan 4545fa94a07fSbrendan /* 4546fa94a07fSbrendan * Clear all buflists and ARC references. L2ARC device flush. 4547fa94a07fSbrendan */ 4548fa94a07fSbrendan l2arc_evict(remdev, 0, B_TRUE); 4549fa94a07fSbrendan list_destroy(remdev->l2ad_buflist); 4550fa94a07fSbrendan kmem_free(remdev->l2ad_buflist, sizeof (list_t)); 4551fa94a07fSbrendan kmem_free(remdev, sizeof (l2arc_dev_t)); 4552fa94a07fSbrendan } 4553fa94a07fSbrendan 4554fa94a07fSbrendan void 4555e14bb325SJeff Bonwick l2arc_init(void) 4556fa94a07fSbrendan { 4557fa94a07fSbrendan l2arc_thread_exit = 0; 4558fa94a07fSbrendan l2arc_ndev = 0; 4559fa94a07fSbrendan l2arc_writes_sent = 0; 4560fa94a07fSbrendan l2arc_writes_done = 0; 4561fa94a07fSbrendan 4562fa94a07fSbrendan mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL); 4563fa94a07fSbrendan cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL); 4564fa94a07fSbrendan mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL); 4565fa94a07fSbrendan mutex_init(&l2arc_buflist_mtx, NULL, MUTEX_DEFAULT, NULL); 4566fa94a07fSbrendan mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL); 4567fa94a07fSbrendan 4568fa94a07fSbrendan l2arc_dev_list = &L2ARC_dev_list; 4569fa94a07fSbrendan l2arc_free_on_write = &L2ARC_free_on_write; 4570fa94a07fSbrendan list_create(l2arc_dev_list, sizeof (l2arc_dev_t), 4571fa94a07fSbrendan offsetof(l2arc_dev_t, l2ad_node)); 4572fa94a07fSbrendan list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t), 4573fa94a07fSbrendan offsetof(l2arc_data_free_t, l2df_list_node)); 4574fa94a07fSbrendan } 4575fa94a07fSbrendan 4576fa94a07fSbrendan void 4577e14bb325SJeff Bonwick l2arc_fini(void) 4578fa94a07fSbrendan { 45793a737e0dSbrendan /* 45803a737e0dSbrendan * This is called from dmu_fini(), which is called from spa_fini(); 45813a737e0dSbrendan * Because of this, we can assume that all l2arc devices have 45823a737e0dSbrendan * already been removed when the pools themselves were removed. 45833a737e0dSbrendan */ 45843a737e0dSbrendan 45853a737e0dSbrendan l2arc_do_free_on_write(); 45863a737e0dSbrendan 4587fa94a07fSbrendan mutex_destroy(&l2arc_feed_thr_lock); 4588fa94a07fSbrendan cv_destroy(&l2arc_feed_thr_cv); 4589fa94a07fSbrendan mutex_destroy(&l2arc_dev_mtx); 4590fa94a07fSbrendan mutex_destroy(&l2arc_buflist_mtx); 4591fa94a07fSbrendan mutex_destroy(&l2arc_free_on_write_mtx); 4592fa94a07fSbrendan 4593fa94a07fSbrendan list_destroy(l2arc_dev_list); 4594fa94a07fSbrendan list_destroy(l2arc_free_on_write); 4595fa94a07fSbrendan } 4596e14bb325SJeff Bonwick 4597e14bb325SJeff Bonwick void 4598e14bb325SJeff Bonwick l2arc_start(void) 4599e14bb325SJeff Bonwick { 46008ad4d6ddSJeff Bonwick if (!(spa_mode_global & FWRITE)) 4601e14bb325SJeff Bonwick return; 4602e14bb325SJeff Bonwick 4603e14bb325SJeff Bonwick (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0, 4604e14bb325SJeff Bonwick TS_RUN, minclsyspri); 4605e14bb325SJeff Bonwick } 4606e14bb325SJeff Bonwick 4607e14bb325SJeff Bonwick void 4608e14bb325SJeff Bonwick l2arc_stop(void) 4609e14bb325SJeff Bonwick { 46108ad4d6ddSJeff Bonwick if (!(spa_mode_global & FWRITE)) 4611e14bb325SJeff Bonwick return; 4612e14bb325SJeff Bonwick 4613e14bb325SJeff Bonwick mutex_enter(&l2arc_feed_thr_lock); 4614e14bb325SJeff Bonwick cv_signal(&l2arc_feed_thr_cv); /* kick thread out of startup */ 4615e14bb325SJeff Bonwick l2arc_thread_exit = 1; 4616e14bb325SJeff Bonwick while (l2arc_thread_exit != 0) 4617e14bb325SJeff Bonwick cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock); 4618e14bb325SJeff Bonwick mutex_exit(&l2arc_feed_thr_lock); 4619e14bb325SJeff Bonwick } 4620