1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5033f9833Sek * Common Development and Distribution License (the "License"). 6033f9833Sek * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 22*44cb6abcSbmc * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23fa9e4066Sahrens * Use is subject to license terms. 24fa9e4066Sahrens */ 25fa9e4066Sahrens 26fa9e4066Sahrens #pragma ident "%Z%%M% %I% %E% SMI" 27fa9e4066Sahrens 28fa9e4066Sahrens /* 29*44cb6abcSbmc * DVA-based Adjustable Replacement Cache 30fa9e4066Sahrens * 31ea8dc4b6Seschrock * While much of the theory of operation used here is 32ea8dc4b6Seschrock * based on the self-tuning, low overhead replacement cache 33fa9e4066Sahrens * presented by Megiddo and Modha at FAST 2003, there are some 34fa9e4066Sahrens * significant differences: 35fa9e4066Sahrens * 36fa9e4066Sahrens * 1. The Megiddo and Modha model assumes any page is evictable. 37fa9e4066Sahrens * Pages in its cache cannot be "locked" into memory. This makes 38fa9e4066Sahrens * the eviction algorithm simple: evict the last page in the list. 39fa9e4066Sahrens * This also make the performance characteristics easy to reason 40fa9e4066Sahrens * about. Our cache is not so simple. At any given moment, some 41fa9e4066Sahrens * subset of the blocks in the cache are un-evictable because we 42fa9e4066Sahrens * have handed out a reference to them. Blocks are only evictable 43fa9e4066Sahrens * when there are no external references active. This makes 44fa9e4066Sahrens * eviction far more problematic: we choose to evict the evictable 45fa9e4066Sahrens * blocks that are the "lowest" in the list. 46fa9e4066Sahrens * 47fa9e4066Sahrens * There are times when it is not possible to evict the requested 48fa9e4066Sahrens * space. In these circumstances we are unable to adjust the cache 49fa9e4066Sahrens * size. To prevent the cache growing unbounded at these times we 50fa9e4066Sahrens * implement a "cache throttle" that slowes the flow of new data 51fa9e4066Sahrens * into the cache until we can make space avaiable. 52fa9e4066Sahrens * 53fa9e4066Sahrens * 2. The Megiddo and Modha model assumes a fixed cache size. 54fa9e4066Sahrens * Pages are evicted when the cache is full and there is a cache 55fa9e4066Sahrens * miss. Our model has a variable sized cache. It grows with 56fa9e4066Sahrens * high use, but also tries to react to memory preasure from the 57fa9e4066Sahrens * operating system: decreasing its size when system memory is 58fa9e4066Sahrens * tight. 59fa9e4066Sahrens * 60fa9e4066Sahrens * 3. The Megiddo and Modha model assumes a fixed page size. All 61fa9e4066Sahrens * elements of the cache are therefor exactly the same size. So 62fa9e4066Sahrens * when adjusting the cache size following a cache miss, its simply 63fa9e4066Sahrens * a matter of choosing a single page to evict. In our model, we 64fa9e4066Sahrens * have variable sized cache blocks (rangeing from 512 bytes to 65fa9e4066Sahrens * 128K bytes). We therefor choose a set of blocks to evict to make 66fa9e4066Sahrens * space for a cache miss that approximates as closely as possible 67fa9e4066Sahrens * the space used by the new block. 68fa9e4066Sahrens * 69fa9e4066Sahrens * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache" 70fa9e4066Sahrens * by N. Megiddo & D. Modha, FAST 2003 71fa9e4066Sahrens */ 72fa9e4066Sahrens 73fa9e4066Sahrens /* 74fa9e4066Sahrens * The locking model: 75fa9e4066Sahrens * 76fa9e4066Sahrens * A new reference to a cache buffer can be obtained in two 77fa9e4066Sahrens * ways: 1) via a hash table lookup using the DVA as a key, 78fa9e4066Sahrens * or 2) via one of the ARC lists. The arc_read() inerface 79fa9e4066Sahrens * uses method 1, while the internal arc algorithms for 80fa9e4066Sahrens * adjusting the cache use method 2. We therefor provide two 81fa9e4066Sahrens * types of locks: 1) the hash table lock array, and 2) the 82fa9e4066Sahrens * arc list locks. 83fa9e4066Sahrens * 84fa9e4066Sahrens * Buffers do not have their own mutexs, rather they rely on the 85fa9e4066Sahrens * hash table mutexs for the bulk of their protection (i.e. most 86fa9e4066Sahrens * fields in the arc_buf_hdr_t are protected by these mutexs). 87fa9e4066Sahrens * 88fa9e4066Sahrens * buf_hash_find() returns the appropriate mutex (held) when it 89fa9e4066Sahrens * locates the requested buffer in the hash table. It returns 90fa9e4066Sahrens * NULL for the mutex if the buffer was not in the table. 91fa9e4066Sahrens * 92fa9e4066Sahrens * buf_hash_remove() expects the appropriate hash mutex to be 93fa9e4066Sahrens * already held before it is invoked. 94fa9e4066Sahrens * 95fa9e4066Sahrens * Each arc state also has a mutex which is used to protect the 96fa9e4066Sahrens * buffer list associated with the state. When attempting to 97fa9e4066Sahrens * obtain a hash table lock while holding an arc list lock you 98fa9e4066Sahrens * must use: mutex_tryenter() to avoid deadlock. Also note that 9944eda4d7Smaybee * the active state mutex must be held before the ghost state mutex. 100fa9e4066Sahrens * 101ea8dc4b6Seschrock * Arc buffers may have an associated eviction callback function. 102ea8dc4b6Seschrock * This function will be invoked prior to removing the buffer (e.g. 103ea8dc4b6Seschrock * in arc_do_user_evicts()). Note however that the data associated 104ea8dc4b6Seschrock * with the buffer may be evicted prior to the callback. The callback 105ea8dc4b6Seschrock * must be made with *no locks held* (to prevent deadlock). Additionally, 106ea8dc4b6Seschrock * the users of callbacks must ensure that their private data is 107ea8dc4b6Seschrock * protected from simultaneous callbacks from arc_buf_evict() 108ea8dc4b6Seschrock * and arc_do_user_evicts(). 109ea8dc4b6Seschrock * 110fa9e4066Sahrens * Note that the majority of the performance stats are manipulated 111fa9e4066Sahrens * with atomic operations. 112fa9e4066Sahrens */ 113fa9e4066Sahrens 114fa9e4066Sahrens #include <sys/spa.h> 115fa9e4066Sahrens #include <sys/zio.h> 1166b4acc8bSahrens #include <sys/zio_checksum.h> 117fa9e4066Sahrens #include <sys/zfs_context.h> 118fa9e4066Sahrens #include <sys/arc.h> 119fa9e4066Sahrens #include <sys/refcount.h> 120fa9e4066Sahrens #ifdef _KERNEL 121fa9e4066Sahrens #include <sys/vmsystm.h> 122fa9e4066Sahrens #include <vm/anon.h> 123fa9e4066Sahrens #include <sys/fs/swapnode.h> 124033f9833Sek #include <sys/dnlc.h> 125fa9e4066Sahrens #endif 126fa9e4066Sahrens #include <sys/callb.h> 127*44cb6abcSbmc #include <sys/kstat.h> 128fa9e4066Sahrens 129fa9e4066Sahrens static kmutex_t arc_reclaim_thr_lock; 130fa9e4066Sahrens static kcondvar_t arc_reclaim_thr_cv; /* used to signal reclaim thr */ 131fa9e4066Sahrens static uint8_t arc_thread_exit; 132fa9e4066Sahrens 133033f9833Sek #define ARC_REDUCE_DNLC_PERCENT 3 134033f9833Sek uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT; 135033f9833Sek 136fa9e4066Sahrens typedef enum arc_reclaim_strategy { 137fa9e4066Sahrens ARC_RECLAIM_AGGR, /* Aggressive reclaim strategy */ 138fa9e4066Sahrens ARC_RECLAIM_CONS /* Conservative reclaim strategy */ 139fa9e4066Sahrens } arc_reclaim_strategy_t; 140fa9e4066Sahrens 141fa9e4066Sahrens /* number of seconds before growing cache again */ 142fa9e4066Sahrens static int arc_grow_retry = 60; 143fa9e4066Sahrens 14413506d1eSmaybee /* 145b19a79ecSperrin * minimum lifespan of a prefetch block in clock ticks 146b19a79ecSperrin * (initialized in arc_init()) 14713506d1eSmaybee */ 148b19a79ecSperrin static int arc_min_prefetch_lifespan; 14913506d1eSmaybee 150fa9e4066Sahrens static int arc_dead; 151fa9e4066Sahrens 152a2eea2e1Sahrens /* 153a2eea2e1Sahrens * These tunables are for performance analysis. 154a2eea2e1Sahrens */ 155a2eea2e1Sahrens uint64_t zfs_arc_max; 156a2eea2e1Sahrens uint64_t zfs_arc_min; 157a2eea2e1Sahrens 158fa9e4066Sahrens /* 159fa9e4066Sahrens * Note that buffers can be on one of 5 states: 160fa9e4066Sahrens * ARC_anon - anonymous (discussed below) 161ea8dc4b6Seschrock * ARC_mru - recently used, currently cached 162ea8dc4b6Seschrock * ARC_mru_ghost - recentely used, no longer in cache 163ea8dc4b6Seschrock * ARC_mfu - frequently used, currently cached 164ea8dc4b6Seschrock * ARC_mfu_ghost - frequently used, no longer in cache 165fa9e4066Sahrens * When there are no active references to the buffer, they 166fa9e4066Sahrens * are linked onto one of the lists in arc. These are the 167fa9e4066Sahrens * only buffers that can be evicted or deleted. 168fa9e4066Sahrens * 169fa9e4066Sahrens * Anonymous buffers are buffers that are not associated with 170fa9e4066Sahrens * a DVA. These are buffers that hold dirty block copies 171fa9e4066Sahrens * before they are written to stable storage. By definition, 172ea8dc4b6Seschrock * they are "ref'd" and are considered part of arc_mru 173fa9e4066Sahrens * that cannot be freed. Generally, they will aquire a DVA 174ea8dc4b6Seschrock * as they are written and migrate onto the arc_mru list. 175fa9e4066Sahrens */ 176fa9e4066Sahrens 177fa9e4066Sahrens typedef struct arc_state { 178*44cb6abcSbmc list_t arcs_list; /* linked list of evictable buffer in state */ 179*44cb6abcSbmc uint64_t arcs_lsize; /* total size of buffers in the linked list */ 180*44cb6abcSbmc uint64_t arcs_size; /* total size of all buffers in this state */ 181*44cb6abcSbmc kmutex_t arcs_mtx; 182fa9e4066Sahrens } arc_state_t; 183fa9e4066Sahrens 184fa9e4066Sahrens /* The 5 states: */ 185fa9e4066Sahrens static arc_state_t ARC_anon; 186ea8dc4b6Seschrock static arc_state_t ARC_mru; 187ea8dc4b6Seschrock static arc_state_t ARC_mru_ghost; 188ea8dc4b6Seschrock static arc_state_t ARC_mfu; 189ea8dc4b6Seschrock static arc_state_t ARC_mfu_ghost; 190fa9e4066Sahrens 191*44cb6abcSbmc typedef struct arc_stats { 192*44cb6abcSbmc kstat_named_t arcstat_hits; 193*44cb6abcSbmc kstat_named_t arcstat_misses; 194*44cb6abcSbmc kstat_named_t arcstat_demand_data_hits; 195*44cb6abcSbmc kstat_named_t arcstat_demand_data_misses; 196*44cb6abcSbmc kstat_named_t arcstat_demand_metadata_hits; 197*44cb6abcSbmc kstat_named_t arcstat_demand_metadata_misses; 198*44cb6abcSbmc kstat_named_t arcstat_prefetch_data_hits; 199*44cb6abcSbmc kstat_named_t arcstat_prefetch_data_misses; 200*44cb6abcSbmc kstat_named_t arcstat_prefetch_metadata_hits; 201*44cb6abcSbmc kstat_named_t arcstat_prefetch_metadata_misses; 202*44cb6abcSbmc kstat_named_t arcstat_mru_hits; 203*44cb6abcSbmc kstat_named_t arcstat_mru_ghost_hits; 204*44cb6abcSbmc kstat_named_t arcstat_mfu_hits; 205*44cb6abcSbmc kstat_named_t arcstat_mfu_ghost_hits; 206*44cb6abcSbmc kstat_named_t arcstat_deleted; 207*44cb6abcSbmc kstat_named_t arcstat_recycle_miss; 208*44cb6abcSbmc kstat_named_t arcstat_mutex_miss; 209*44cb6abcSbmc kstat_named_t arcstat_evict_skip; 210*44cb6abcSbmc kstat_named_t arcstat_hash_elements; 211*44cb6abcSbmc kstat_named_t arcstat_hash_elements_max; 212*44cb6abcSbmc kstat_named_t arcstat_hash_collisions; 213*44cb6abcSbmc kstat_named_t arcstat_hash_chains; 214*44cb6abcSbmc kstat_named_t arcstat_hash_chain_max; 215*44cb6abcSbmc kstat_named_t arcstat_p; 216*44cb6abcSbmc kstat_named_t arcstat_c; 217*44cb6abcSbmc kstat_named_t arcstat_c_min; 218*44cb6abcSbmc kstat_named_t arcstat_c_max; 219*44cb6abcSbmc kstat_named_t arcstat_size; 220*44cb6abcSbmc } arc_stats_t; 221*44cb6abcSbmc 222*44cb6abcSbmc static arc_stats_t arc_stats = { 223*44cb6abcSbmc { "hits", KSTAT_DATA_UINT64 }, 224*44cb6abcSbmc { "misses", KSTAT_DATA_UINT64 }, 225*44cb6abcSbmc { "demand_data_hits", KSTAT_DATA_UINT64 }, 226*44cb6abcSbmc { "demand_data_misses", KSTAT_DATA_UINT64 }, 227*44cb6abcSbmc { "demand_metadata_hits", KSTAT_DATA_UINT64 }, 228*44cb6abcSbmc { "demand_metadata_misses", KSTAT_DATA_UINT64 }, 229*44cb6abcSbmc { "prefetch_data_hits", KSTAT_DATA_UINT64 }, 230*44cb6abcSbmc { "prefetch_data_misses", KSTAT_DATA_UINT64 }, 231*44cb6abcSbmc { "prefetch_metadata_hits", KSTAT_DATA_UINT64 }, 232*44cb6abcSbmc { "prefetch_metadata_misses", KSTAT_DATA_UINT64 }, 233*44cb6abcSbmc { "mru_hits", KSTAT_DATA_UINT64 }, 234*44cb6abcSbmc { "mru_ghost_hits", KSTAT_DATA_UINT64 }, 235*44cb6abcSbmc { "mfu_hits", KSTAT_DATA_UINT64 }, 236*44cb6abcSbmc { "mfu_ghost_hits", KSTAT_DATA_UINT64 }, 237*44cb6abcSbmc { "deleted", KSTAT_DATA_UINT64 }, 238*44cb6abcSbmc { "recycle_miss", KSTAT_DATA_UINT64 }, 239*44cb6abcSbmc { "mutex_miss", KSTAT_DATA_UINT64 }, 240*44cb6abcSbmc { "evict_skip", KSTAT_DATA_UINT64 }, 241*44cb6abcSbmc { "hash_elements", KSTAT_DATA_UINT64 }, 242*44cb6abcSbmc { "hash_elements_max", KSTAT_DATA_UINT64 }, 243*44cb6abcSbmc { "hash_collisions", KSTAT_DATA_UINT64 }, 244*44cb6abcSbmc { "hash_chains", KSTAT_DATA_UINT64 }, 245*44cb6abcSbmc { "hash_chain_max", KSTAT_DATA_UINT64 }, 246*44cb6abcSbmc { "p", KSTAT_DATA_UINT64 }, 247*44cb6abcSbmc { "c", KSTAT_DATA_UINT64 }, 248*44cb6abcSbmc { "c_min", KSTAT_DATA_UINT64 }, 249*44cb6abcSbmc { "c_max", KSTAT_DATA_UINT64 }, 250*44cb6abcSbmc { "size", KSTAT_DATA_UINT64 } 251*44cb6abcSbmc }; 252*44cb6abcSbmc 253*44cb6abcSbmc #define ARCSTAT(stat) (arc_stats.stat.value.ui64) 254*44cb6abcSbmc 255*44cb6abcSbmc #define ARCSTAT_INCR(stat, val) \ 256*44cb6abcSbmc atomic_add_64(&arc_stats.stat.value.ui64, (val)); 257*44cb6abcSbmc 258*44cb6abcSbmc #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) 259*44cb6abcSbmc #define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1) 260*44cb6abcSbmc 261*44cb6abcSbmc #define ARCSTAT_MAX(stat, val) { \ 262*44cb6abcSbmc uint64_t m; \ 263*44cb6abcSbmc while ((val) > (m = arc_stats.stat.value.ui64) && \ 264*44cb6abcSbmc (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \ 265*44cb6abcSbmc continue; \ 266*44cb6abcSbmc } 267*44cb6abcSbmc 268*44cb6abcSbmc #define ARCSTAT_MAXSTAT(stat) \ 269*44cb6abcSbmc ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64) 270*44cb6abcSbmc 271*44cb6abcSbmc /* 272*44cb6abcSbmc * We define a macro to allow ARC hits/misses to be easily broken down by 273*44cb6abcSbmc * two separate conditions, giving a total of four different subtypes for 274*44cb6abcSbmc * each of hits and misses (so eight statistics total). 275*44cb6abcSbmc */ 276*44cb6abcSbmc #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \ 277*44cb6abcSbmc if (cond1) { \ 278*44cb6abcSbmc if (cond2) { \ 279*44cb6abcSbmc ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \ 280*44cb6abcSbmc } else { \ 281*44cb6abcSbmc ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \ 282*44cb6abcSbmc } \ 283*44cb6abcSbmc } else { \ 284*44cb6abcSbmc if (cond2) { \ 285*44cb6abcSbmc ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \ 286*44cb6abcSbmc } else { \ 287*44cb6abcSbmc ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\ 288*44cb6abcSbmc } \ 289*44cb6abcSbmc } 290*44cb6abcSbmc 291*44cb6abcSbmc kstat_t *arc_ksp; 292*44cb6abcSbmc static arc_state_t *arc_anon; 293*44cb6abcSbmc static arc_state_t *arc_mru; 294*44cb6abcSbmc static arc_state_t *arc_mru_ghost; 295*44cb6abcSbmc static arc_state_t *arc_mfu; 296*44cb6abcSbmc static arc_state_t *arc_mfu_ghost; 297*44cb6abcSbmc 298*44cb6abcSbmc /* 299*44cb6abcSbmc * There are several ARC variables that are critical to export as kstats -- 300*44cb6abcSbmc * but we don't want to have to grovel around in the kstat whenever we wish to 301*44cb6abcSbmc * manipulate them. For these variables, we therefore define them to be in 302*44cb6abcSbmc * terms of the statistic variable. This assures that we are not introducing 303*44cb6abcSbmc * the possibility of inconsistency by having shadow copies of the variables, 304*44cb6abcSbmc * while still allowing the code to be readable. 305*44cb6abcSbmc */ 306*44cb6abcSbmc #define arc_size ARCSTAT(arcstat_size) /* actual total arc size */ 307*44cb6abcSbmc #define arc_p ARCSTAT(arcstat_p) /* target size of MRU */ 308*44cb6abcSbmc #define arc_c ARCSTAT(arcstat_c) /* target size of cache */ 309*44cb6abcSbmc #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ 310*44cb6abcSbmc #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ 311*44cb6abcSbmc 312*44cb6abcSbmc static int arc_no_grow; /* Don't try to grow cache size */ 313*44cb6abcSbmc static uint64_t arc_tempreserve; 314fa9e4066Sahrens 315fa9e4066Sahrens typedef struct arc_callback arc_callback_t; 316fa9e4066Sahrens 317fa9e4066Sahrens struct arc_callback { 318fa9e4066Sahrens arc_done_func_t *acb_done; 319fa9e4066Sahrens void *acb_private; 320fa9e4066Sahrens arc_byteswap_func_t *acb_byteswap; 321fa9e4066Sahrens arc_buf_t *acb_buf; 322fa9e4066Sahrens zio_t *acb_zio_dummy; 323fa9e4066Sahrens arc_callback_t *acb_next; 324fa9e4066Sahrens }; 325fa9e4066Sahrens 326fa9e4066Sahrens struct arc_buf_hdr { 327fa9e4066Sahrens /* protected by hash lock */ 328fa9e4066Sahrens dva_t b_dva; 329fa9e4066Sahrens uint64_t b_birth; 330fa9e4066Sahrens uint64_t b_cksum0; 331fa9e4066Sahrens 3326b4acc8bSahrens kmutex_t b_freeze_lock; 3336b4acc8bSahrens zio_cksum_t *b_freeze_cksum; 3346b4acc8bSahrens 335fa9e4066Sahrens arc_buf_hdr_t *b_hash_next; 336fa9e4066Sahrens arc_buf_t *b_buf; 337fa9e4066Sahrens uint32_t b_flags; 338ea8dc4b6Seschrock uint32_t b_datacnt; 339fa9e4066Sahrens 340fa9e4066Sahrens arc_callback_t *b_acb; 341ad23a2dbSjohansen kcondvar_t b_cv; 342ad23a2dbSjohansen 343ad23a2dbSjohansen /* immutable */ 344ad23a2dbSjohansen arc_buf_contents_t b_type; 345ad23a2dbSjohansen uint64_t b_size; 346ad23a2dbSjohansen spa_t *b_spa; 347fa9e4066Sahrens 348fa9e4066Sahrens /* protected by arc state mutex */ 349fa9e4066Sahrens arc_state_t *b_state; 350fa9e4066Sahrens list_node_t b_arc_node; 351fa9e4066Sahrens 352fa9e4066Sahrens /* updated atomically */ 353fa9e4066Sahrens clock_t b_arc_access; 354fa9e4066Sahrens 355fa9e4066Sahrens /* self protecting */ 356fa9e4066Sahrens refcount_t b_refcnt; 357fa9e4066Sahrens }; 358fa9e4066Sahrens 359ea8dc4b6Seschrock static arc_buf_t *arc_eviction_list; 360ea8dc4b6Seschrock static kmutex_t arc_eviction_mtx; 36140d7d650Smaybee static arc_buf_hdr_t arc_eviction_hdr; 36244eda4d7Smaybee static void arc_get_data_buf(arc_buf_t *buf); 36344eda4d7Smaybee static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock); 364ea8dc4b6Seschrock 365ea8dc4b6Seschrock #define GHOST_STATE(state) \ 366*44cb6abcSbmc ((state) == arc_mru_ghost || (state) == arc_mfu_ghost) 367ea8dc4b6Seschrock 368fa9e4066Sahrens /* 369fa9e4066Sahrens * Private ARC flags. These flags are private ARC only flags that will show up 370fa9e4066Sahrens * in b_flags in the arc_hdr_buf_t. Some flags are publicly declared, and can 371fa9e4066Sahrens * be passed in as arc_flags in things like arc_read. However, these flags 372fa9e4066Sahrens * should never be passed and should only be set by ARC code. When adding new 373fa9e4066Sahrens * public flags, make sure not to smash the private ones. 374fa9e4066Sahrens */ 375fa9e4066Sahrens 376ea8dc4b6Seschrock #define ARC_IN_HASH_TABLE (1 << 9) /* this buffer is hashed */ 377fa9e4066Sahrens #define ARC_IO_IN_PROGRESS (1 << 10) /* I/O in progress for buf */ 378fa9e4066Sahrens #define ARC_IO_ERROR (1 << 11) /* I/O failed for buf */ 379fa9e4066Sahrens #define ARC_FREED_IN_READ (1 << 12) /* buf freed while in read */ 380ea8dc4b6Seschrock #define ARC_BUF_AVAILABLE (1 << 13) /* block not in active use */ 38113506d1eSmaybee #define ARC_INDIRECT (1 << 14) /* this is an indirect block */ 382fa9e4066Sahrens 383ea8dc4b6Seschrock #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_IN_HASH_TABLE) 384fa9e4066Sahrens #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS) 385fa9e4066Sahrens #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_IO_ERROR) 386fa9e4066Sahrens #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FREED_IN_READ) 387ea8dc4b6Seschrock #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_BUF_AVAILABLE) 388fa9e4066Sahrens 389fa9e4066Sahrens /* 390fa9e4066Sahrens * Hash table routines 391fa9e4066Sahrens */ 392fa9e4066Sahrens 393fa9e4066Sahrens #define HT_LOCK_PAD 64 394fa9e4066Sahrens 395fa9e4066Sahrens struct ht_lock { 396fa9e4066Sahrens kmutex_t ht_lock; 397fa9e4066Sahrens #ifdef _KERNEL 398fa9e4066Sahrens unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))]; 399fa9e4066Sahrens #endif 400fa9e4066Sahrens }; 401fa9e4066Sahrens 402fa9e4066Sahrens #define BUF_LOCKS 256 403fa9e4066Sahrens typedef struct buf_hash_table { 404fa9e4066Sahrens uint64_t ht_mask; 405fa9e4066Sahrens arc_buf_hdr_t **ht_table; 406fa9e4066Sahrens struct ht_lock ht_locks[BUF_LOCKS]; 407fa9e4066Sahrens } buf_hash_table_t; 408fa9e4066Sahrens 409fa9e4066Sahrens static buf_hash_table_t buf_hash_table; 410fa9e4066Sahrens 411fa9e4066Sahrens #define BUF_HASH_INDEX(spa, dva, birth) \ 412fa9e4066Sahrens (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask) 413fa9e4066Sahrens #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)]) 414fa9e4066Sahrens #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock)) 415fa9e4066Sahrens #define HDR_LOCK(buf) \ 416fa9e4066Sahrens (BUF_HASH_LOCK(BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth))) 417fa9e4066Sahrens 418fa9e4066Sahrens uint64_t zfs_crc64_table[256]; 419fa9e4066Sahrens 420fa9e4066Sahrens static uint64_t 421fa9e4066Sahrens buf_hash(spa_t *spa, dva_t *dva, uint64_t birth) 422fa9e4066Sahrens { 423fa9e4066Sahrens uintptr_t spav = (uintptr_t)spa; 424fa9e4066Sahrens uint8_t *vdva = (uint8_t *)dva; 425fa9e4066Sahrens uint64_t crc = -1ULL; 426fa9e4066Sahrens int i; 427fa9e4066Sahrens 428fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 429fa9e4066Sahrens 430fa9e4066Sahrens for (i = 0; i < sizeof (dva_t); i++) 431fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF]; 432fa9e4066Sahrens 433fa9e4066Sahrens crc ^= (spav>>8) ^ birth; 434fa9e4066Sahrens 435fa9e4066Sahrens return (crc); 436fa9e4066Sahrens } 437fa9e4066Sahrens 438fa9e4066Sahrens #define BUF_EMPTY(buf) \ 439fa9e4066Sahrens ((buf)->b_dva.dva_word[0] == 0 && \ 440fa9e4066Sahrens (buf)->b_dva.dva_word[1] == 0 && \ 441fa9e4066Sahrens (buf)->b_birth == 0) 442fa9e4066Sahrens 443fa9e4066Sahrens #define BUF_EQUAL(spa, dva, birth, buf) \ 444fa9e4066Sahrens ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \ 445fa9e4066Sahrens ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \ 446fa9e4066Sahrens ((buf)->b_birth == birth) && ((buf)->b_spa == spa) 447fa9e4066Sahrens 448fa9e4066Sahrens static arc_buf_hdr_t * 449fa9e4066Sahrens buf_hash_find(spa_t *spa, dva_t *dva, uint64_t birth, kmutex_t **lockp) 450fa9e4066Sahrens { 451fa9e4066Sahrens uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); 452fa9e4066Sahrens kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 453fa9e4066Sahrens arc_buf_hdr_t *buf; 454fa9e4066Sahrens 455fa9e4066Sahrens mutex_enter(hash_lock); 456fa9e4066Sahrens for (buf = buf_hash_table.ht_table[idx]; buf != NULL; 457fa9e4066Sahrens buf = buf->b_hash_next) { 458fa9e4066Sahrens if (BUF_EQUAL(spa, dva, birth, buf)) { 459fa9e4066Sahrens *lockp = hash_lock; 460fa9e4066Sahrens return (buf); 461fa9e4066Sahrens } 462fa9e4066Sahrens } 463fa9e4066Sahrens mutex_exit(hash_lock); 464fa9e4066Sahrens *lockp = NULL; 465fa9e4066Sahrens return (NULL); 466fa9e4066Sahrens } 467fa9e4066Sahrens 468fa9e4066Sahrens /* 469fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 470fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 471fa9e4066Sahrens * will be returned and the new element will not be inserted. 472fa9e4066Sahrens * Otherwise returns NULL. 473fa9e4066Sahrens */ 474fa9e4066Sahrens static arc_buf_hdr_t * 475fa9e4066Sahrens buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp) 476fa9e4066Sahrens { 477fa9e4066Sahrens uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 478fa9e4066Sahrens kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 479fa9e4066Sahrens arc_buf_hdr_t *fbuf; 480*44cb6abcSbmc uint32_t i; 481fa9e4066Sahrens 482ea8dc4b6Seschrock ASSERT(!HDR_IN_HASH_TABLE(buf)); 483fa9e4066Sahrens *lockp = hash_lock; 484fa9e4066Sahrens mutex_enter(hash_lock); 485fa9e4066Sahrens for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL; 486fa9e4066Sahrens fbuf = fbuf->b_hash_next, i++) { 487fa9e4066Sahrens if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf)) 488fa9e4066Sahrens return (fbuf); 489fa9e4066Sahrens } 490fa9e4066Sahrens 491fa9e4066Sahrens buf->b_hash_next = buf_hash_table.ht_table[idx]; 492fa9e4066Sahrens buf_hash_table.ht_table[idx] = buf; 493ea8dc4b6Seschrock buf->b_flags |= ARC_IN_HASH_TABLE; 494fa9e4066Sahrens 495fa9e4066Sahrens /* collect some hash table performance data */ 496fa9e4066Sahrens if (i > 0) { 497*44cb6abcSbmc ARCSTAT_BUMP(arcstat_hash_collisions); 498fa9e4066Sahrens if (i == 1) 499*44cb6abcSbmc ARCSTAT_BUMP(arcstat_hash_chains); 500*44cb6abcSbmc 501*44cb6abcSbmc ARCSTAT_MAX(arcstat_hash_chain_max, i); 502fa9e4066Sahrens } 503*44cb6abcSbmc 504*44cb6abcSbmc ARCSTAT_BUMP(arcstat_hash_elements); 505*44cb6abcSbmc ARCSTAT_MAXSTAT(arcstat_hash_elements); 506fa9e4066Sahrens 507fa9e4066Sahrens return (NULL); 508fa9e4066Sahrens } 509fa9e4066Sahrens 510fa9e4066Sahrens static void 511fa9e4066Sahrens buf_hash_remove(arc_buf_hdr_t *buf) 512fa9e4066Sahrens { 513fa9e4066Sahrens arc_buf_hdr_t *fbuf, **bufp; 514fa9e4066Sahrens uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 515fa9e4066Sahrens 516fa9e4066Sahrens ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx))); 517ea8dc4b6Seschrock ASSERT(HDR_IN_HASH_TABLE(buf)); 518fa9e4066Sahrens 519fa9e4066Sahrens bufp = &buf_hash_table.ht_table[idx]; 520fa9e4066Sahrens while ((fbuf = *bufp) != buf) { 521fa9e4066Sahrens ASSERT(fbuf != NULL); 522fa9e4066Sahrens bufp = &fbuf->b_hash_next; 523fa9e4066Sahrens } 524fa9e4066Sahrens *bufp = buf->b_hash_next; 525fa9e4066Sahrens buf->b_hash_next = NULL; 526ea8dc4b6Seschrock buf->b_flags &= ~ARC_IN_HASH_TABLE; 527fa9e4066Sahrens 528fa9e4066Sahrens /* collect some hash table performance data */ 529*44cb6abcSbmc ARCSTAT_BUMPDOWN(arcstat_hash_elements); 530*44cb6abcSbmc 531fa9e4066Sahrens if (buf_hash_table.ht_table[idx] && 532fa9e4066Sahrens buf_hash_table.ht_table[idx]->b_hash_next == NULL) 533*44cb6abcSbmc ARCSTAT_BUMPDOWN(arcstat_hash_chains); 534fa9e4066Sahrens } 535fa9e4066Sahrens 536fa9e4066Sahrens /* 537fa9e4066Sahrens * Global data structures and functions for the buf kmem cache. 538fa9e4066Sahrens */ 539fa9e4066Sahrens static kmem_cache_t *hdr_cache; 540fa9e4066Sahrens static kmem_cache_t *buf_cache; 541fa9e4066Sahrens 542fa9e4066Sahrens static void 543fa9e4066Sahrens buf_fini(void) 544fa9e4066Sahrens { 545fa9e4066Sahrens int i; 546fa9e4066Sahrens 547fa9e4066Sahrens kmem_free(buf_hash_table.ht_table, 548fa9e4066Sahrens (buf_hash_table.ht_mask + 1) * sizeof (void *)); 549fa9e4066Sahrens for (i = 0; i < BUF_LOCKS; i++) 550fa9e4066Sahrens mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock); 551fa9e4066Sahrens kmem_cache_destroy(hdr_cache); 552fa9e4066Sahrens kmem_cache_destroy(buf_cache); 553fa9e4066Sahrens } 554fa9e4066Sahrens 555fa9e4066Sahrens /* 556fa9e4066Sahrens * Constructor callback - called when the cache is empty 557fa9e4066Sahrens * and a new buf is requested. 558fa9e4066Sahrens */ 559fa9e4066Sahrens /* ARGSUSED */ 560fa9e4066Sahrens static int 561fa9e4066Sahrens hdr_cons(void *vbuf, void *unused, int kmflag) 562fa9e4066Sahrens { 563fa9e4066Sahrens arc_buf_hdr_t *buf = vbuf; 564fa9e4066Sahrens 565fa9e4066Sahrens bzero(buf, sizeof (arc_buf_hdr_t)); 566fa9e4066Sahrens refcount_create(&buf->b_refcnt); 567fa9e4066Sahrens cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL); 568fa9e4066Sahrens return (0); 569fa9e4066Sahrens } 570fa9e4066Sahrens 571fa9e4066Sahrens /* 572fa9e4066Sahrens * Destructor callback - called when a cached buf is 573fa9e4066Sahrens * no longer required. 574fa9e4066Sahrens */ 575fa9e4066Sahrens /* ARGSUSED */ 576fa9e4066Sahrens static void 577fa9e4066Sahrens hdr_dest(void *vbuf, void *unused) 578fa9e4066Sahrens { 579fa9e4066Sahrens arc_buf_hdr_t *buf = vbuf; 580fa9e4066Sahrens 581fa9e4066Sahrens refcount_destroy(&buf->b_refcnt); 582fa9e4066Sahrens cv_destroy(&buf->b_cv); 583fa9e4066Sahrens } 584fa9e4066Sahrens 585fa9e4066Sahrens /* 586fa9e4066Sahrens * Reclaim callback -- invoked when memory is low. 587fa9e4066Sahrens */ 588fa9e4066Sahrens /* ARGSUSED */ 589fa9e4066Sahrens static void 590fa9e4066Sahrens hdr_recl(void *unused) 591fa9e4066Sahrens { 592fa9e4066Sahrens dprintf("hdr_recl called\n"); 59349e3519aSmaybee /* 59449e3519aSmaybee * umem calls the reclaim func when we destroy the buf cache, 59549e3519aSmaybee * which is after we do arc_fini(). 59649e3519aSmaybee */ 59749e3519aSmaybee if (!arc_dead) 59849e3519aSmaybee cv_signal(&arc_reclaim_thr_cv); 599fa9e4066Sahrens } 600fa9e4066Sahrens 601fa9e4066Sahrens static void 602fa9e4066Sahrens buf_init(void) 603fa9e4066Sahrens { 604fa9e4066Sahrens uint64_t *ct; 605ea8dc4b6Seschrock uint64_t hsize = 1ULL << 12; 606fa9e4066Sahrens int i, j; 607fa9e4066Sahrens 608fa9e4066Sahrens /* 609fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 610ea8dc4b6Seschrock * with an average 64K block size. The table will take up 611ea8dc4b6Seschrock * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers). 612fa9e4066Sahrens */ 613ea8dc4b6Seschrock while (hsize * 65536 < physmem * PAGESIZE) 614fa9e4066Sahrens hsize <<= 1; 615ea8dc4b6Seschrock retry: 616fa9e4066Sahrens buf_hash_table.ht_mask = hsize - 1; 617ea8dc4b6Seschrock buf_hash_table.ht_table = 618ea8dc4b6Seschrock kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP); 619ea8dc4b6Seschrock if (buf_hash_table.ht_table == NULL) { 620ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 8)); 621ea8dc4b6Seschrock hsize >>= 1; 622ea8dc4b6Seschrock goto retry; 623ea8dc4b6Seschrock } 624fa9e4066Sahrens 625fa9e4066Sahrens hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t), 626fa9e4066Sahrens 0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0); 627fa9e4066Sahrens buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t), 628fa9e4066Sahrens 0, NULL, NULL, NULL, NULL, NULL, 0); 629fa9e4066Sahrens 630fa9e4066Sahrens for (i = 0; i < 256; i++) 631fa9e4066Sahrens for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--) 632fa9e4066Sahrens *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY); 633fa9e4066Sahrens 634fa9e4066Sahrens for (i = 0; i < BUF_LOCKS; i++) { 635fa9e4066Sahrens mutex_init(&buf_hash_table.ht_locks[i].ht_lock, 636fa9e4066Sahrens NULL, MUTEX_DEFAULT, NULL); 637fa9e4066Sahrens } 638fa9e4066Sahrens } 639fa9e4066Sahrens 640fa9e4066Sahrens #define ARC_MINTIME (hz>>4) /* 62 ms */ 641fa9e4066Sahrens 6426b4acc8bSahrens static void 6436b4acc8bSahrens arc_cksum_verify(arc_buf_t *buf) 6446b4acc8bSahrens { 6456b4acc8bSahrens zio_cksum_t zc; 6466b4acc8bSahrens 647cc60fd72Sahrens if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 6486b4acc8bSahrens return; 6496b4acc8bSahrens 6506b4acc8bSahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 6513ccfa83cSahrens if (buf->b_hdr->b_freeze_cksum == NULL || 6523ccfa83cSahrens (buf->b_hdr->b_flags & ARC_IO_ERROR)) { 6536b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 6546b4acc8bSahrens return; 6556b4acc8bSahrens } 6566b4acc8bSahrens fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 6576b4acc8bSahrens if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc)) 6586b4acc8bSahrens panic("buffer modified while frozen!"); 6596b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 6606b4acc8bSahrens } 6616b4acc8bSahrens 6626b4acc8bSahrens static void 6636b4acc8bSahrens arc_cksum_compute(arc_buf_t *buf) 6646b4acc8bSahrens { 665cc60fd72Sahrens if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 6666b4acc8bSahrens return; 6676b4acc8bSahrens 6686b4acc8bSahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 6696b4acc8bSahrens if (buf->b_hdr->b_freeze_cksum != NULL) { 6706b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 6716b4acc8bSahrens return; 6726b4acc8bSahrens } 6736b4acc8bSahrens buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP); 6746b4acc8bSahrens fletcher_2_native(buf->b_data, buf->b_hdr->b_size, 6756b4acc8bSahrens buf->b_hdr->b_freeze_cksum); 6766b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 6776b4acc8bSahrens } 6786b4acc8bSahrens 6796b4acc8bSahrens void 6806b4acc8bSahrens arc_buf_thaw(arc_buf_t *buf) 6816b4acc8bSahrens { 682cc60fd72Sahrens if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 6836b4acc8bSahrens return; 6846b4acc8bSahrens 685*44cb6abcSbmc if (buf->b_hdr->b_state != arc_anon) 6866b4acc8bSahrens panic("modifying non-anon buffer!"); 6876b4acc8bSahrens if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS) 6886b4acc8bSahrens panic("modifying buffer while i/o in progress!"); 6896b4acc8bSahrens arc_cksum_verify(buf); 6906b4acc8bSahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 6916b4acc8bSahrens if (buf->b_hdr->b_freeze_cksum != NULL) { 6926b4acc8bSahrens kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 6936b4acc8bSahrens buf->b_hdr->b_freeze_cksum = NULL; 6946b4acc8bSahrens } 6956b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 6966b4acc8bSahrens } 6976b4acc8bSahrens 6986b4acc8bSahrens void 6996b4acc8bSahrens arc_buf_freeze(arc_buf_t *buf) 7006b4acc8bSahrens { 701cc60fd72Sahrens if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 702cc60fd72Sahrens return; 703cc60fd72Sahrens 7046b4acc8bSahrens ASSERT(buf->b_hdr->b_freeze_cksum != NULL || 705*44cb6abcSbmc buf->b_hdr->b_state == arc_anon); 7066b4acc8bSahrens arc_cksum_compute(buf); 7076b4acc8bSahrens } 7086b4acc8bSahrens 709fa9e4066Sahrens static void 710fa9e4066Sahrens add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 711fa9e4066Sahrens { 712fa9e4066Sahrens ASSERT(MUTEX_HELD(hash_lock)); 713fa9e4066Sahrens 714fa9e4066Sahrens if ((refcount_add(&ab->b_refcnt, tag) == 1) && 715*44cb6abcSbmc (ab->b_state != arc_anon)) { 716ea8dc4b6Seschrock int delta = ab->b_size * ab->b_datacnt; 717fa9e4066Sahrens 718*44cb6abcSbmc ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx)); 719*44cb6abcSbmc mutex_enter(&ab->b_state->arcs_mtx); 720fa9e4066Sahrens ASSERT(list_link_active(&ab->b_arc_node)); 721*44cb6abcSbmc list_remove(&ab->b_state->arcs_list, ab); 722ea8dc4b6Seschrock if (GHOST_STATE(ab->b_state)) { 723ea8dc4b6Seschrock ASSERT3U(ab->b_datacnt, ==, 0); 724ea8dc4b6Seschrock ASSERT3P(ab->b_buf, ==, NULL); 725ea8dc4b6Seschrock delta = ab->b_size; 726ea8dc4b6Seschrock } 727ea8dc4b6Seschrock ASSERT(delta > 0); 728*44cb6abcSbmc ASSERT3U(ab->b_state->arcs_lsize, >=, delta); 729*44cb6abcSbmc atomic_add_64(&ab->b_state->arcs_lsize, -delta); 730*44cb6abcSbmc mutex_exit(&ab->b_state->arcs_mtx); 73113506d1eSmaybee /* remove the prefetch flag is we get a reference */ 73213506d1eSmaybee if (ab->b_flags & ARC_PREFETCH) 73313506d1eSmaybee ab->b_flags &= ~ARC_PREFETCH; 734fa9e4066Sahrens } 735fa9e4066Sahrens } 736fa9e4066Sahrens 737fa9e4066Sahrens static int 738fa9e4066Sahrens remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 739fa9e4066Sahrens { 740fa9e4066Sahrens int cnt; 741*44cb6abcSbmc arc_state_t *state = ab->b_state; 742fa9e4066Sahrens 743*44cb6abcSbmc ASSERT(state == arc_anon || MUTEX_HELD(hash_lock)); 744*44cb6abcSbmc ASSERT(!GHOST_STATE(state)); 745fa9e4066Sahrens 746fa9e4066Sahrens if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) && 747*44cb6abcSbmc (state != arc_anon)) { 748*44cb6abcSbmc ASSERT(!MUTEX_HELD(&state->arcs_mtx)); 749*44cb6abcSbmc mutex_enter(&state->arcs_mtx); 750fa9e4066Sahrens ASSERT(!list_link_active(&ab->b_arc_node)); 751*44cb6abcSbmc list_insert_head(&state->arcs_list, ab); 752ea8dc4b6Seschrock ASSERT(ab->b_datacnt > 0); 753*44cb6abcSbmc atomic_add_64(&state->arcs_lsize, ab->b_size * ab->b_datacnt); 754*44cb6abcSbmc ASSERT3U(state->arcs_size, >=, state->arcs_lsize); 755*44cb6abcSbmc mutex_exit(&state->arcs_mtx); 756fa9e4066Sahrens } 757fa9e4066Sahrens return (cnt); 758fa9e4066Sahrens } 759fa9e4066Sahrens 760fa9e4066Sahrens /* 761fa9e4066Sahrens * Move the supplied buffer to the indicated state. The mutex 762fa9e4066Sahrens * for the buffer must be held by the caller. 763fa9e4066Sahrens */ 764fa9e4066Sahrens static void 765ea8dc4b6Seschrock arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock) 766fa9e4066Sahrens { 767ea8dc4b6Seschrock arc_state_t *old_state = ab->b_state; 768ea8dc4b6Seschrock int refcnt = refcount_count(&ab->b_refcnt); 769ea8dc4b6Seschrock int from_delta, to_delta; 770fa9e4066Sahrens 771fa9e4066Sahrens ASSERT(MUTEX_HELD(hash_lock)); 772ea8dc4b6Seschrock ASSERT(new_state != old_state); 773ea8dc4b6Seschrock ASSERT(refcnt == 0 || ab->b_datacnt > 0); 774ea8dc4b6Seschrock ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state)); 775ea8dc4b6Seschrock 776ea8dc4b6Seschrock from_delta = to_delta = ab->b_datacnt * ab->b_size; 777fa9e4066Sahrens 778fa9e4066Sahrens /* 779fa9e4066Sahrens * If this buffer is evictable, transfer it from the 780fa9e4066Sahrens * old state list to the new state list. 781fa9e4066Sahrens */ 782ea8dc4b6Seschrock if (refcnt == 0) { 783*44cb6abcSbmc if (old_state != arc_anon) { 784*44cb6abcSbmc int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx); 785ea8dc4b6Seschrock 786ea8dc4b6Seschrock if (use_mutex) 787*44cb6abcSbmc mutex_enter(&old_state->arcs_mtx); 788fa9e4066Sahrens 789fa9e4066Sahrens ASSERT(list_link_active(&ab->b_arc_node)); 790*44cb6abcSbmc list_remove(&old_state->arcs_list, ab); 791ea8dc4b6Seschrock 79213506d1eSmaybee /* 79313506d1eSmaybee * If prefetching out of the ghost cache, 79413506d1eSmaybee * we will have a non-null datacnt. 79513506d1eSmaybee */ 79613506d1eSmaybee if (GHOST_STATE(old_state) && ab->b_datacnt == 0) { 79713506d1eSmaybee /* ghost elements have a ghost size */ 798ea8dc4b6Seschrock ASSERT(ab->b_buf == NULL); 799ea8dc4b6Seschrock from_delta = ab->b_size; 800ea8dc4b6Seschrock } 801*44cb6abcSbmc ASSERT3U(old_state->arcs_lsize, >=, from_delta); 802*44cb6abcSbmc atomic_add_64(&old_state->arcs_lsize, -from_delta); 803ea8dc4b6Seschrock 804ea8dc4b6Seschrock if (use_mutex) 805*44cb6abcSbmc mutex_exit(&old_state->arcs_mtx); 806fa9e4066Sahrens } 807*44cb6abcSbmc if (new_state != arc_anon) { 808*44cb6abcSbmc int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx); 809fa9e4066Sahrens 810ea8dc4b6Seschrock if (use_mutex) 811*44cb6abcSbmc mutex_enter(&new_state->arcs_mtx); 812ea8dc4b6Seschrock 813*44cb6abcSbmc list_insert_head(&new_state->arcs_list, ab); 814ea8dc4b6Seschrock 815ea8dc4b6Seschrock /* ghost elements have a ghost size */ 816ea8dc4b6Seschrock if (GHOST_STATE(new_state)) { 817ea8dc4b6Seschrock ASSERT(ab->b_datacnt == 0); 818ea8dc4b6Seschrock ASSERT(ab->b_buf == NULL); 819ea8dc4b6Seschrock to_delta = ab->b_size; 820ea8dc4b6Seschrock } 821*44cb6abcSbmc atomic_add_64(&new_state->arcs_lsize, to_delta); 822*44cb6abcSbmc ASSERT3U(new_state->arcs_size + to_delta, >=, 823*44cb6abcSbmc new_state->arcs_lsize); 824ea8dc4b6Seschrock 825ea8dc4b6Seschrock if (use_mutex) 826*44cb6abcSbmc mutex_exit(&new_state->arcs_mtx); 827fa9e4066Sahrens } 828fa9e4066Sahrens } 829fa9e4066Sahrens 830fa9e4066Sahrens ASSERT(!BUF_EMPTY(ab)); 831*44cb6abcSbmc if (new_state == arc_anon && old_state != arc_anon) { 832fa9e4066Sahrens buf_hash_remove(ab); 833fa9e4066Sahrens } 834fa9e4066Sahrens 835ea8dc4b6Seschrock /* adjust state sizes */ 836ea8dc4b6Seschrock if (to_delta) 837*44cb6abcSbmc atomic_add_64(&new_state->arcs_size, to_delta); 838ea8dc4b6Seschrock if (from_delta) { 839*44cb6abcSbmc ASSERT3U(old_state->arcs_size, >=, from_delta); 840*44cb6abcSbmc atomic_add_64(&old_state->arcs_size, -from_delta); 841fa9e4066Sahrens } 842fa9e4066Sahrens ab->b_state = new_state; 843fa9e4066Sahrens } 844fa9e4066Sahrens 845fa9e4066Sahrens arc_buf_t * 846ad23a2dbSjohansen arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type) 847fa9e4066Sahrens { 848fa9e4066Sahrens arc_buf_hdr_t *hdr; 849fa9e4066Sahrens arc_buf_t *buf; 850fa9e4066Sahrens 851fa9e4066Sahrens ASSERT3U(size, >, 0); 852fa9e4066Sahrens hdr = kmem_cache_alloc(hdr_cache, KM_SLEEP); 853fa9e4066Sahrens ASSERT(BUF_EMPTY(hdr)); 854fa9e4066Sahrens hdr->b_size = size; 855ad23a2dbSjohansen hdr->b_type = type; 856fa9e4066Sahrens hdr->b_spa = spa; 857*44cb6abcSbmc hdr->b_state = arc_anon; 858fa9e4066Sahrens hdr->b_arc_access = 0; 859fa9e4066Sahrens buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 860fa9e4066Sahrens buf->b_hdr = hdr; 86144eda4d7Smaybee buf->b_data = NULL; 862ea8dc4b6Seschrock buf->b_efunc = NULL; 863ea8dc4b6Seschrock buf->b_private = NULL; 864fa9e4066Sahrens buf->b_next = NULL; 865fa9e4066Sahrens hdr->b_buf = buf; 86644eda4d7Smaybee arc_get_data_buf(buf); 867ea8dc4b6Seschrock hdr->b_datacnt = 1; 868fa9e4066Sahrens hdr->b_flags = 0; 869fa9e4066Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt)); 870fa9e4066Sahrens (void) refcount_add(&hdr->b_refcnt, tag); 871fa9e4066Sahrens 872fa9e4066Sahrens return (buf); 873fa9e4066Sahrens } 874fa9e4066Sahrens 87544eda4d7Smaybee static arc_buf_t * 87644eda4d7Smaybee arc_buf_clone(arc_buf_t *from) 877ea8dc4b6Seschrock { 87844eda4d7Smaybee arc_buf_t *buf; 87944eda4d7Smaybee arc_buf_hdr_t *hdr = from->b_hdr; 88044eda4d7Smaybee uint64_t size = hdr->b_size; 881ea8dc4b6Seschrock 88244eda4d7Smaybee buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 88344eda4d7Smaybee buf->b_hdr = hdr; 88444eda4d7Smaybee buf->b_data = NULL; 88544eda4d7Smaybee buf->b_efunc = NULL; 88644eda4d7Smaybee buf->b_private = NULL; 88744eda4d7Smaybee buf->b_next = hdr->b_buf; 88844eda4d7Smaybee hdr->b_buf = buf; 88944eda4d7Smaybee arc_get_data_buf(buf); 89044eda4d7Smaybee bcopy(from->b_data, buf->b_data, size); 89144eda4d7Smaybee hdr->b_datacnt += 1; 89244eda4d7Smaybee return (buf); 893ea8dc4b6Seschrock } 894ea8dc4b6Seschrock 895ea8dc4b6Seschrock void 896ea8dc4b6Seschrock arc_buf_add_ref(arc_buf_t *buf, void* tag) 897ea8dc4b6Seschrock { 89840d7d650Smaybee arc_buf_hdr_t *hdr; 899ea8dc4b6Seschrock kmutex_t *hash_lock; 900ea8dc4b6Seschrock 9019b23f181Smaybee /* 9029b23f181Smaybee * Check to see if this buffer is currently being evicted via 90340d7d650Smaybee * arc_do_user_evicts(). 9049b23f181Smaybee */ 90540d7d650Smaybee mutex_enter(&arc_eviction_mtx); 90640d7d650Smaybee hdr = buf->b_hdr; 90740d7d650Smaybee if (hdr == NULL) { 90840d7d650Smaybee mutex_exit(&arc_eviction_mtx); 9099b23f181Smaybee return; 91040d7d650Smaybee } 9119b23f181Smaybee hash_lock = HDR_LOCK(hdr); 91240d7d650Smaybee mutex_exit(&arc_eviction_mtx); 91340d7d650Smaybee 9149b23f181Smaybee mutex_enter(hash_lock); 915ea8dc4b6Seschrock if (buf->b_data == NULL) { 916ea8dc4b6Seschrock /* 917ea8dc4b6Seschrock * This buffer is evicted. 918ea8dc4b6Seschrock */ 9199b23f181Smaybee mutex_exit(hash_lock); 920ea8dc4b6Seschrock return; 921ea8dc4b6Seschrock } 922ea8dc4b6Seschrock 9239b23f181Smaybee ASSERT(buf->b_hdr == hdr); 924*44cb6abcSbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 925ea8dc4b6Seschrock add_reference(hdr, hash_lock, tag); 92644eda4d7Smaybee arc_access(hdr, hash_lock); 92744eda4d7Smaybee mutex_exit(hash_lock); 928*44cb6abcSbmc ARCSTAT_BUMP(arcstat_hits); 929*44cb6abcSbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 930*44cb6abcSbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 931*44cb6abcSbmc data, metadata, hits); 932ea8dc4b6Seschrock } 933ea8dc4b6Seschrock 934ea8dc4b6Seschrock static void 93544eda4d7Smaybee arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all) 936ea8dc4b6Seschrock { 937ea8dc4b6Seschrock arc_buf_t **bufp; 938ea8dc4b6Seschrock 939ea8dc4b6Seschrock /* free up data associated with the buf */ 940ea8dc4b6Seschrock if (buf->b_data) { 941ea8dc4b6Seschrock arc_state_t *state = buf->b_hdr->b_state; 942ea8dc4b6Seschrock uint64_t size = buf->b_hdr->b_size; 943ad23a2dbSjohansen arc_buf_contents_t type = buf->b_hdr->b_type; 944ea8dc4b6Seschrock 9456b4acc8bSahrens arc_cksum_verify(buf); 94644eda4d7Smaybee if (!recycle) { 947ad23a2dbSjohansen if (type == ARC_BUFC_METADATA) { 948ad23a2dbSjohansen zio_buf_free(buf->b_data, size); 949ad23a2dbSjohansen } else { 950ad23a2dbSjohansen ASSERT(type == ARC_BUFC_DATA); 951ad23a2dbSjohansen zio_data_buf_free(buf->b_data, size); 952ad23a2dbSjohansen } 953*44cb6abcSbmc atomic_add_64(&arc_size, -size); 95444eda4d7Smaybee } 955ea8dc4b6Seschrock if (list_link_active(&buf->b_hdr->b_arc_node)) { 956ea8dc4b6Seschrock ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt)); 957*44cb6abcSbmc ASSERT(state != arc_anon); 958*44cb6abcSbmc ASSERT3U(state->arcs_lsize, >=, size); 959*44cb6abcSbmc atomic_add_64(&state->arcs_lsize, -size); 960ea8dc4b6Seschrock } 961*44cb6abcSbmc ASSERT3U(state->arcs_size, >=, size); 962*44cb6abcSbmc atomic_add_64(&state->arcs_size, -size); 963ea8dc4b6Seschrock buf->b_data = NULL; 964ea8dc4b6Seschrock ASSERT(buf->b_hdr->b_datacnt > 0); 965ea8dc4b6Seschrock buf->b_hdr->b_datacnt -= 1; 966ea8dc4b6Seschrock } 967ea8dc4b6Seschrock 968ea8dc4b6Seschrock /* only remove the buf if requested */ 969ea8dc4b6Seschrock if (!all) 970ea8dc4b6Seschrock return; 971ea8dc4b6Seschrock 972ea8dc4b6Seschrock /* remove the buf from the hdr list */ 973ea8dc4b6Seschrock for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next) 974ea8dc4b6Seschrock continue; 975ea8dc4b6Seschrock *bufp = buf->b_next; 976ea8dc4b6Seschrock 977ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 978ea8dc4b6Seschrock 979ea8dc4b6Seschrock /* clean up the buf */ 980ea8dc4b6Seschrock buf->b_hdr = NULL; 981ea8dc4b6Seschrock kmem_cache_free(buf_cache, buf); 982ea8dc4b6Seschrock } 983ea8dc4b6Seschrock 984fa9e4066Sahrens static void 985ea8dc4b6Seschrock arc_hdr_destroy(arc_buf_hdr_t *hdr) 986fa9e4066Sahrens { 987fa9e4066Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt)); 988*44cb6abcSbmc ASSERT3P(hdr->b_state, ==, arc_anon); 989ea8dc4b6Seschrock ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 990fa9e4066Sahrens 991fa9e4066Sahrens if (!BUF_EMPTY(hdr)) { 992ea8dc4b6Seschrock ASSERT(!HDR_IN_HASH_TABLE(hdr)); 993fa9e4066Sahrens bzero(&hdr->b_dva, sizeof (dva_t)); 994fa9e4066Sahrens hdr->b_birth = 0; 995fa9e4066Sahrens hdr->b_cksum0 = 0; 996fa9e4066Sahrens } 997ea8dc4b6Seschrock while (hdr->b_buf) { 998fa9e4066Sahrens arc_buf_t *buf = hdr->b_buf; 999fa9e4066Sahrens 1000ea8dc4b6Seschrock if (buf->b_efunc) { 1001ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 1002ea8dc4b6Seschrock ASSERT(buf->b_hdr != NULL); 100344eda4d7Smaybee arc_buf_destroy(hdr->b_buf, FALSE, FALSE); 1004ea8dc4b6Seschrock hdr->b_buf = buf->b_next; 100540d7d650Smaybee buf->b_hdr = &arc_eviction_hdr; 1006ea8dc4b6Seschrock buf->b_next = arc_eviction_list; 1007ea8dc4b6Seschrock arc_eviction_list = buf; 1008ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1009ea8dc4b6Seschrock } else { 101044eda4d7Smaybee arc_buf_destroy(hdr->b_buf, FALSE, TRUE); 1011ea8dc4b6Seschrock } 1012fa9e4066Sahrens } 10136b4acc8bSahrens if (hdr->b_freeze_cksum != NULL) { 10146b4acc8bSahrens kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 10156b4acc8bSahrens hdr->b_freeze_cksum = NULL; 10166b4acc8bSahrens } 1017ea8dc4b6Seschrock 1018fa9e4066Sahrens ASSERT(!list_link_active(&hdr->b_arc_node)); 1019fa9e4066Sahrens ASSERT3P(hdr->b_hash_next, ==, NULL); 1020fa9e4066Sahrens ASSERT3P(hdr->b_acb, ==, NULL); 1021fa9e4066Sahrens kmem_cache_free(hdr_cache, hdr); 1022fa9e4066Sahrens } 1023fa9e4066Sahrens 1024fa9e4066Sahrens void 1025fa9e4066Sahrens arc_buf_free(arc_buf_t *buf, void *tag) 1026fa9e4066Sahrens { 1027fa9e4066Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 1028*44cb6abcSbmc int hashed = hdr->b_state != arc_anon; 1029fa9e4066Sahrens 1030ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 1031ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 1032ea8dc4b6Seschrock 1033ea8dc4b6Seschrock if (hashed) { 1034ea8dc4b6Seschrock kmutex_t *hash_lock = HDR_LOCK(hdr); 1035ea8dc4b6Seschrock 1036ea8dc4b6Seschrock mutex_enter(hash_lock); 1037ea8dc4b6Seschrock (void) remove_reference(hdr, hash_lock, tag); 1038ea8dc4b6Seschrock if (hdr->b_datacnt > 1) 103944eda4d7Smaybee arc_buf_destroy(buf, FALSE, TRUE); 1040ea8dc4b6Seschrock else 1041ea8dc4b6Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 1042fa9e4066Sahrens mutex_exit(hash_lock); 1043ea8dc4b6Seschrock } else if (HDR_IO_IN_PROGRESS(hdr)) { 1044ea8dc4b6Seschrock int destroy_hdr; 1045ea8dc4b6Seschrock /* 1046ea8dc4b6Seschrock * We are in the middle of an async write. Don't destroy 1047ea8dc4b6Seschrock * this buffer unless the write completes before we finish 1048ea8dc4b6Seschrock * decrementing the reference count. 1049ea8dc4b6Seschrock */ 1050ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 1051ea8dc4b6Seschrock (void) remove_reference(hdr, NULL, tag); 1052ea8dc4b6Seschrock ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1053ea8dc4b6Seschrock destroy_hdr = !HDR_IO_IN_PROGRESS(hdr); 1054ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1055ea8dc4b6Seschrock if (destroy_hdr) 1056ea8dc4b6Seschrock arc_hdr_destroy(hdr); 1057ea8dc4b6Seschrock } else { 1058ea8dc4b6Seschrock if (remove_reference(hdr, NULL, tag) > 0) { 1059ea8dc4b6Seschrock ASSERT(HDR_IO_ERROR(hdr)); 106044eda4d7Smaybee arc_buf_destroy(buf, FALSE, TRUE); 1061ea8dc4b6Seschrock } else { 1062ea8dc4b6Seschrock arc_hdr_destroy(hdr); 1063ea8dc4b6Seschrock } 1064fa9e4066Sahrens } 1065ea8dc4b6Seschrock } 1066fa9e4066Sahrens 1067ea8dc4b6Seschrock int 1068ea8dc4b6Seschrock arc_buf_remove_ref(arc_buf_t *buf, void* tag) 1069ea8dc4b6Seschrock { 1070ea8dc4b6Seschrock arc_buf_hdr_t *hdr = buf->b_hdr; 1071ea8dc4b6Seschrock kmutex_t *hash_lock = HDR_LOCK(hdr); 1072ea8dc4b6Seschrock int no_callback = (buf->b_efunc == NULL); 1073fa9e4066Sahrens 1074*44cb6abcSbmc if (hdr->b_state == arc_anon) { 1075ea8dc4b6Seschrock arc_buf_free(buf, tag); 1076ea8dc4b6Seschrock return (no_callback); 1077ea8dc4b6Seschrock } 1078ea8dc4b6Seschrock 1079ea8dc4b6Seschrock mutex_enter(hash_lock); 1080*44cb6abcSbmc ASSERT(hdr->b_state != arc_anon); 1081ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 1082ea8dc4b6Seschrock 1083ea8dc4b6Seschrock (void) remove_reference(hdr, hash_lock, tag); 1084ea8dc4b6Seschrock if (hdr->b_datacnt > 1) { 1085ea8dc4b6Seschrock if (no_callback) 108644eda4d7Smaybee arc_buf_destroy(buf, FALSE, TRUE); 1087ea8dc4b6Seschrock } else if (no_callback) { 1088ea8dc4b6Seschrock ASSERT(hdr->b_buf == buf && buf->b_next == NULL); 1089ea8dc4b6Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 1090ea8dc4b6Seschrock } 1091ea8dc4b6Seschrock ASSERT(no_callback || hdr->b_datacnt > 1 || 1092ea8dc4b6Seschrock refcount_is_zero(&hdr->b_refcnt)); 1093ea8dc4b6Seschrock mutex_exit(hash_lock); 1094ea8dc4b6Seschrock return (no_callback); 1095fa9e4066Sahrens } 1096fa9e4066Sahrens 1097fa9e4066Sahrens int 1098fa9e4066Sahrens arc_buf_size(arc_buf_t *buf) 1099fa9e4066Sahrens { 1100fa9e4066Sahrens return (buf->b_hdr->b_size); 1101fa9e4066Sahrens } 1102fa9e4066Sahrens 1103fa9e4066Sahrens /* 1104fa9e4066Sahrens * Evict buffers from list until we've removed the specified number of 1105fa9e4066Sahrens * bytes. Move the removed buffers to the appropriate evict state. 110644eda4d7Smaybee * If the recycle flag is set, then attempt to "recycle" a buffer: 110744eda4d7Smaybee * - look for a buffer to evict that is `bytes' long. 110844eda4d7Smaybee * - return the data block from this buffer rather than freeing it. 110944eda4d7Smaybee * This flag is used by callers that are trying to make space for a 111044eda4d7Smaybee * new buffer in a full arc cache. 1111fa9e4066Sahrens */ 111244eda4d7Smaybee static void * 1113ad23a2dbSjohansen arc_evict(arc_state_t *state, int64_t bytes, boolean_t recycle, 1114ad23a2dbSjohansen arc_buf_contents_t type) 1115fa9e4066Sahrens { 1116fa9e4066Sahrens arc_state_t *evicted_state; 111744eda4d7Smaybee uint64_t bytes_evicted = 0, skipped = 0, missed = 0; 11183fa51506Smaybee arc_buf_hdr_t *ab, *ab_prev = NULL; 1119fa9e4066Sahrens kmutex_t *hash_lock; 112044eda4d7Smaybee boolean_t have_lock; 11213fa51506Smaybee void *stolen = NULL; 1122fa9e4066Sahrens 1123*44cb6abcSbmc ASSERT(state == arc_mru || state == arc_mfu); 1124fa9e4066Sahrens 1125*44cb6abcSbmc evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 1126fa9e4066Sahrens 1127*44cb6abcSbmc mutex_enter(&state->arcs_mtx); 1128*44cb6abcSbmc mutex_enter(&evicted_state->arcs_mtx); 1129fa9e4066Sahrens 1130*44cb6abcSbmc for (ab = list_tail(&state->arcs_list); ab; ab = ab_prev) { 1131*44cb6abcSbmc ab_prev = list_prev(&state->arcs_list, ab); 113213506d1eSmaybee /* prefetch buffers have a minimum lifespan */ 113344eda4d7Smaybee if (HDR_IO_IN_PROGRESS(ab) || 113444eda4d7Smaybee (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) && 113544eda4d7Smaybee lbolt - ab->b_arc_access < arc_min_prefetch_lifespan)) { 113613506d1eSmaybee skipped++; 113713506d1eSmaybee continue; 113813506d1eSmaybee } 11393fa51506Smaybee /* "lookahead" for better eviction candidate */ 11403fa51506Smaybee if (recycle && ab->b_size != bytes && 11413fa51506Smaybee ab_prev && ab_prev->b_size == bytes) 114244eda4d7Smaybee continue; 1143fa9e4066Sahrens hash_lock = HDR_LOCK(ab); 114444eda4d7Smaybee have_lock = MUTEX_HELD(hash_lock); 114544eda4d7Smaybee if (have_lock || mutex_tryenter(hash_lock)) { 1146fa9e4066Sahrens ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0); 1147ea8dc4b6Seschrock ASSERT(ab->b_datacnt > 0); 1148ea8dc4b6Seschrock while (ab->b_buf) { 1149ea8dc4b6Seschrock arc_buf_t *buf = ab->b_buf; 115044eda4d7Smaybee if (buf->b_data) { 1151ea8dc4b6Seschrock bytes_evicted += ab->b_size; 1152ad23a2dbSjohansen if (recycle && ab->b_type == type && 1153ad23a2dbSjohansen ab->b_size == bytes) { 11543fa51506Smaybee stolen = buf->b_data; 11553fa51506Smaybee recycle = FALSE; 11563fa51506Smaybee } 115744eda4d7Smaybee } 1158ea8dc4b6Seschrock if (buf->b_efunc) { 1159ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 11603fa51506Smaybee arc_buf_destroy(buf, 11613fa51506Smaybee buf->b_data == stolen, FALSE); 1162ea8dc4b6Seschrock ab->b_buf = buf->b_next; 116340d7d650Smaybee buf->b_hdr = &arc_eviction_hdr; 1164ea8dc4b6Seschrock buf->b_next = arc_eviction_list; 1165ea8dc4b6Seschrock arc_eviction_list = buf; 1166ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1167ea8dc4b6Seschrock } else { 11683fa51506Smaybee arc_buf_destroy(buf, 11693fa51506Smaybee buf->b_data == stolen, TRUE); 1170ea8dc4b6Seschrock } 1171ea8dc4b6Seschrock } 1172ea8dc4b6Seschrock ASSERT(ab->b_datacnt == 0); 1173fa9e4066Sahrens arc_change_state(evicted_state, ab, hash_lock); 1174ea8dc4b6Seschrock ASSERT(HDR_IN_HASH_TABLE(ab)); 1175ea8dc4b6Seschrock ab->b_flags = ARC_IN_HASH_TABLE; 1176fa9e4066Sahrens DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab); 117744eda4d7Smaybee if (!have_lock) 117844eda4d7Smaybee mutex_exit(hash_lock); 1179ea8dc4b6Seschrock if (bytes >= 0 && bytes_evicted >= bytes) 1180fa9e4066Sahrens break; 1181fa9e4066Sahrens } else { 118244eda4d7Smaybee missed += 1; 1183fa9e4066Sahrens } 1184fa9e4066Sahrens } 1185*44cb6abcSbmc 1186*44cb6abcSbmc mutex_exit(&evicted_state->arcs_mtx); 1187*44cb6abcSbmc mutex_exit(&state->arcs_mtx); 1188fa9e4066Sahrens 1189fa9e4066Sahrens if (bytes_evicted < bytes) 1190fa9e4066Sahrens dprintf("only evicted %lld bytes from %x", 1191fa9e4066Sahrens (longlong_t)bytes_evicted, state); 1192fa9e4066Sahrens 119344eda4d7Smaybee if (skipped) 1194*44cb6abcSbmc ARCSTAT_INCR(arcstat_evict_skip, skipped); 1195*44cb6abcSbmc 119644eda4d7Smaybee if (missed) 1197*44cb6abcSbmc ARCSTAT_INCR(arcstat_mutex_miss, missed); 1198*44cb6abcSbmc 11993fa51506Smaybee return (stolen); 1200fa9e4066Sahrens } 1201fa9e4066Sahrens 1202fa9e4066Sahrens /* 1203fa9e4066Sahrens * Remove buffers from list until we've removed the specified number of 1204fa9e4066Sahrens * bytes. Destroy the buffers that are removed. 1205fa9e4066Sahrens */ 1206fa9e4066Sahrens static void 1207ea8dc4b6Seschrock arc_evict_ghost(arc_state_t *state, int64_t bytes) 1208fa9e4066Sahrens { 1209fa9e4066Sahrens arc_buf_hdr_t *ab, *ab_prev; 1210fa9e4066Sahrens kmutex_t *hash_lock; 1211ea8dc4b6Seschrock uint64_t bytes_deleted = 0; 1212ea8dc4b6Seschrock uint_t bufs_skipped = 0; 1213fa9e4066Sahrens 1214ea8dc4b6Seschrock ASSERT(GHOST_STATE(state)); 1215fa9e4066Sahrens top: 1216*44cb6abcSbmc mutex_enter(&state->arcs_mtx); 1217*44cb6abcSbmc for (ab = list_tail(&state->arcs_list); ab; ab = ab_prev) { 1218*44cb6abcSbmc ab_prev = list_prev(&state->arcs_list, ab); 1219fa9e4066Sahrens hash_lock = HDR_LOCK(ab); 1220fa9e4066Sahrens if (mutex_tryenter(hash_lock)) { 122113506d1eSmaybee ASSERT(!HDR_IO_IN_PROGRESS(ab)); 1222ea8dc4b6Seschrock ASSERT(ab->b_buf == NULL); 1223*44cb6abcSbmc arc_change_state(arc_anon, ab, hash_lock); 1224fa9e4066Sahrens mutex_exit(hash_lock); 1225*44cb6abcSbmc ARCSTAT_BUMP(arcstat_deleted); 1226fa9e4066Sahrens bytes_deleted += ab->b_size; 1227ea8dc4b6Seschrock arc_hdr_destroy(ab); 1228ea8dc4b6Seschrock DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab); 1229fa9e4066Sahrens if (bytes >= 0 && bytes_deleted >= bytes) 1230fa9e4066Sahrens break; 1231fa9e4066Sahrens } else { 1232fa9e4066Sahrens if (bytes < 0) { 1233*44cb6abcSbmc mutex_exit(&state->arcs_mtx); 1234fa9e4066Sahrens mutex_enter(hash_lock); 1235fa9e4066Sahrens mutex_exit(hash_lock); 1236fa9e4066Sahrens goto top; 1237fa9e4066Sahrens } 1238fa9e4066Sahrens bufs_skipped += 1; 1239fa9e4066Sahrens } 1240fa9e4066Sahrens } 1241*44cb6abcSbmc mutex_exit(&state->arcs_mtx); 1242fa9e4066Sahrens 1243fa9e4066Sahrens if (bufs_skipped) { 1244*44cb6abcSbmc ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped); 1245fa9e4066Sahrens ASSERT(bytes >= 0); 1246fa9e4066Sahrens } 1247fa9e4066Sahrens 1248fa9e4066Sahrens if (bytes_deleted < bytes) 1249fa9e4066Sahrens dprintf("only deleted %lld bytes from %p", 1250fa9e4066Sahrens (longlong_t)bytes_deleted, state); 1251fa9e4066Sahrens } 1252fa9e4066Sahrens 1253fa9e4066Sahrens static void 1254fa9e4066Sahrens arc_adjust(void) 1255fa9e4066Sahrens { 1256*44cb6abcSbmc int64_t top_sz, mru_over, arc_over, todelete; 1257fa9e4066Sahrens 1258*44cb6abcSbmc top_sz = arc_anon->arcs_size + arc_mru->arcs_size; 1259fa9e4066Sahrens 1260*44cb6abcSbmc if (top_sz > arc_p && arc_mru->arcs_lsize > 0) { 1261*44cb6abcSbmc int64_t toevict = MIN(arc_mru->arcs_lsize, top_sz - arc_p); 1262*44cb6abcSbmc (void) arc_evict(arc_mru, toevict, FALSE, ARC_BUFC_UNDEF); 1263*44cb6abcSbmc top_sz = arc_anon->arcs_size + arc_mru->arcs_size; 1264fa9e4066Sahrens } 1265fa9e4066Sahrens 1266*44cb6abcSbmc mru_over = top_sz + arc_mru_ghost->arcs_size - arc_c; 1267fa9e4066Sahrens 1268fa9e4066Sahrens if (mru_over > 0) { 1269*44cb6abcSbmc if (arc_mru_ghost->arcs_lsize > 0) { 1270*44cb6abcSbmc todelete = MIN(arc_mru_ghost->arcs_lsize, mru_over); 1271*44cb6abcSbmc arc_evict_ghost(arc_mru_ghost, todelete); 1272fa9e4066Sahrens } 1273fa9e4066Sahrens } 1274fa9e4066Sahrens 1275*44cb6abcSbmc if ((arc_over = arc_size - arc_c) > 0) { 1276ea8dc4b6Seschrock int64_t tbl_over; 1277fa9e4066Sahrens 1278*44cb6abcSbmc if (arc_mfu->arcs_lsize > 0) { 1279*44cb6abcSbmc int64_t toevict = MIN(arc_mfu->arcs_lsize, arc_over); 1280*44cb6abcSbmc (void) arc_evict(arc_mfu, toevict, FALSE, 1281ad23a2dbSjohansen ARC_BUFC_UNDEF); 1282fa9e4066Sahrens } 1283fa9e4066Sahrens 1284*44cb6abcSbmc tbl_over = arc_size + arc_mru_ghost->arcs_lsize + 1285*44cb6abcSbmc arc_mfu_ghost->arcs_lsize - arc_c*2; 1286fa9e4066Sahrens 1287*44cb6abcSbmc if (tbl_over > 0 && arc_mfu_ghost->arcs_lsize > 0) { 1288*44cb6abcSbmc todelete = MIN(arc_mfu_ghost->arcs_lsize, tbl_over); 1289*44cb6abcSbmc arc_evict_ghost(arc_mfu_ghost, todelete); 1290fa9e4066Sahrens } 1291fa9e4066Sahrens } 1292fa9e4066Sahrens } 1293fa9e4066Sahrens 1294ea8dc4b6Seschrock static void 1295ea8dc4b6Seschrock arc_do_user_evicts(void) 1296ea8dc4b6Seschrock { 1297ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 1298ea8dc4b6Seschrock while (arc_eviction_list != NULL) { 1299ea8dc4b6Seschrock arc_buf_t *buf = arc_eviction_list; 1300ea8dc4b6Seschrock arc_eviction_list = buf->b_next; 1301ea8dc4b6Seschrock buf->b_hdr = NULL; 1302ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1303ea8dc4b6Seschrock 1304dd6ef538Smaybee if (buf->b_efunc != NULL) 1305dd6ef538Smaybee VERIFY(buf->b_efunc(buf) == 0); 1306ea8dc4b6Seschrock 1307ea8dc4b6Seschrock buf->b_efunc = NULL; 1308ea8dc4b6Seschrock buf->b_private = NULL; 1309ea8dc4b6Seschrock kmem_cache_free(buf_cache, buf); 1310ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 1311ea8dc4b6Seschrock } 1312ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1313ea8dc4b6Seschrock } 1314ea8dc4b6Seschrock 1315fa9e4066Sahrens /* 1316fa9e4066Sahrens * Flush all *evictable* data from the cache. 1317fa9e4066Sahrens * NOTE: this will not touch "active" (i.e. referenced) data. 1318fa9e4066Sahrens */ 1319fa9e4066Sahrens void 1320fa9e4066Sahrens arc_flush(void) 1321fa9e4066Sahrens { 1322*44cb6abcSbmc while (list_head(&arc_mru->arcs_list)) 1323*44cb6abcSbmc (void) arc_evict(arc_mru, -1, FALSE, ARC_BUFC_UNDEF); 1324*44cb6abcSbmc while (list_head(&arc_mfu->arcs_list)) 1325*44cb6abcSbmc (void) arc_evict(arc_mfu, -1, FALSE, ARC_BUFC_UNDEF); 1326fa9e4066Sahrens 1327*44cb6abcSbmc arc_evict_ghost(arc_mru_ghost, -1); 1328*44cb6abcSbmc arc_evict_ghost(arc_mfu_ghost, -1); 1329ea8dc4b6Seschrock 1330ea8dc4b6Seschrock mutex_enter(&arc_reclaim_thr_lock); 1331ea8dc4b6Seschrock arc_do_user_evicts(); 1332ea8dc4b6Seschrock mutex_exit(&arc_reclaim_thr_lock); 1333ea8dc4b6Seschrock ASSERT(arc_eviction_list == NULL); 1334fa9e4066Sahrens } 1335fa9e4066Sahrens 133649e3519aSmaybee int arc_shrink_shift = 5; /* log2(fraction of arc to reclaim) */ 133713506d1eSmaybee 1338fa9e4066Sahrens void 133949e3519aSmaybee arc_shrink(void) 1340fa9e4066Sahrens { 1341*44cb6abcSbmc if (arc_c > arc_c_min) { 134249e3519aSmaybee uint64_t to_free; 1343fa9e4066Sahrens 13443cff2f43Sstans #ifdef _KERNEL 1345*44cb6abcSbmc to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree)); 13463cff2f43Sstans #else 1347*44cb6abcSbmc to_free = arc_c >> arc_shrink_shift; 13483cff2f43Sstans #endif 1349*44cb6abcSbmc if (arc_c > arc_c_min + to_free) 1350*44cb6abcSbmc atomic_add_64(&arc_c, -to_free); 135149e3519aSmaybee else 1352*44cb6abcSbmc arc_c = arc_c_min; 1353*44cb6abcSbmc 1354*44cb6abcSbmc atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift)); 1355*44cb6abcSbmc if (arc_c > arc_size) 1356*44cb6abcSbmc arc_c = MAX(arc_size, arc_c_min); 1357*44cb6abcSbmc if (arc_p > arc_c) 1358*44cb6abcSbmc arc_p = (arc_c >> 1); 1359*44cb6abcSbmc ASSERT(arc_c >= arc_c_min); 1360*44cb6abcSbmc ASSERT((int64_t)arc_p >= 0); 136149e3519aSmaybee } 1362fa9e4066Sahrens 1363*44cb6abcSbmc if (arc_size > arc_c) 136449e3519aSmaybee arc_adjust(); 1365fa9e4066Sahrens } 1366fa9e4066Sahrens 1367fa9e4066Sahrens static int 1368fa9e4066Sahrens arc_reclaim_needed(void) 1369fa9e4066Sahrens { 1370fa9e4066Sahrens uint64_t extra; 1371fa9e4066Sahrens 1372fa9e4066Sahrens #ifdef _KERNEL 13733cff2f43Sstans 13743cff2f43Sstans if (needfree) 13753cff2f43Sstans return (1); 13763cff2f43Sstans 1377fa9e4066Sahrens /* 1378fa9e4066Sahrens * take 'desfree' extra pages, so we reclaim sooner, rather than later 1379fa9e4066Sahrens */ 1380fa9e4066Sahrens extra = desfree; 1381fa9e4066Sahrens 1382fa9e4066Sahrens /* 1383fa9e4066Sahrens * check that we're out of range of the pageout scanner. It starts to 1384fa9e4066Sahrens * schedule paging if freemem is less than lotsfree and needfree. 1385fa9e4066Sahrens * lotsfree is the high-water mark for pageout, and needfree is the 1386fa9e4066Sahrens * number of needed free pages. We add extra pages here to make sure 1387fa9e4066Sahrens * the scanner doesn't start up while we're freeing memory. 1388fa9e4066Sahrens */ 1389fa9e4066Sahrens if (freemem < lotsfree + needfree + extra) 1390fa9e4066Sahrens return (1); 1391fa9e4066Sahrens 1392fa9e4066Sahrens /* 1393fa9e4066Sahrens * check to make sure that swapfs has enough space so that anon 1394fa9e4066Sahrens * reservations can still succeeed. anon_resvmem() checks that the 1395fa9e4066Sahrens * availrmem is greater than swapfs_minfree, and the number of reserved 1396fa9e4066Sahrens * swap pages. We also add a bit of extra here just to prevent 1397fa9e4066Sahrens * circumstances from getting really dire. 1398fa9e4066Sahrens */ 1399fa9e4066Sahrens if (availrmem < swapfs_minfree + swapfs_reserve + extra) 1400fa9e4066Sahrens return (1); 1401fa9e4066Sahrens 140290e485acSjohansen /* 140390e485acSjohansen * If zio data pages are being allocated out of a separate heap segment, 140490e485acSjohansen * then check that the size of available vmem for this area remains 140590e485acSjohansen * above 1/4th free. This needs to be done since the size of the 140690e485acSjohansen * non-default segment is smaller than physical memory, so we could 140790e485acSjohansen * conceivably run out of VA in that segment before running out of 140890e485acSjohansen * physical memory. 140990e485acSjohansen */ 141090e485acSjohansen if ((zio_arena != NULL) && (btop(vmem_size(zio_arena, VMEM_FREE)) < 141190e485acSjohansen (btop(vmem_size(zio_arena, VMEM_FREE | VMEM_ALLOC)) >> 2))) 141290e485acSjohansen return (1); 141390e485acSjohansen 14145dc8af33Smaybee #if defined(__i386) 1415fa9e4066Sahrens /* 1416fa9e4066Sahrens * If we're on an i386 platform, it's possible that we'll exhaust the 1417fa9e4066Sahrens * kernel heap space before we ever run out of available physical 1418fa9e4066Sahrens * memory. Most checks of the size of the heap_area compare against 1419fa9e4066Sahrens * tune.t_minarmem, which is the minimum available real memory that we 1420fa9e4066Sahrens * can have in the system. However, this is generally fixed at 25 pages 1421fa9e4066Sahrens * which is so low that it's useless. In this comparison, we seek to 1422fa9e4066Sahrens * calculate the total heap-size, and reclaim if more than 3/4ths of the 1423fa9e4066Sahrens * heap is allocated. (Or, in the caclulation, if less than 1/4th is 1424fa9e4066Sahrens * free) 1425fa9e4066Sahrens */ 1426fa9e4066Sahrens if (btop(vmem_size(heap_arena, VMEM_FREE)) < 1427fa9e4066Sahrens (btop(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2)) 1428fa9e4066Sahrens return (1); 1429fa9e4066Sahrens #endif 1430fa9e4066Sahrens 1431fa9e4066Sahrens #else 1432fa9e4066Sahrens if (spa_get_random(100) == 0) 1433fa9e4066Sahrens return (1); 1434fa9e4066Sahrens #endif 1435fa9e4066Sahrens return (0); 1436fa9e4066Sahrens } 1437fa9e4066Sahrens 1438fa9e4066Sahrens static void 1439fa9e4066Sahrens arc_kmem_reap_now(arc_reclaim_strategy_t strat) 1440fa9e4066Sahrens { 1441fa9e4066Sahrens size_t i; 1442fa9e4066Sahrens kmem_cache_t *prev_cache = NULL; 1443ad23a2dbSjohansen kmem_cache_t *prev_data_cache = NULL; 1444fa9e4066Sahrens extern kmem_cache_t *zio_buf_cache[]; 1445ad23a2dbSjohansen extern kmem_cache_t *zio_data_buf_cache[]; 1446fa9e4066Sahrens 1447033f9833Sek #ifdef _KERNEL 1448033f9833Sek /* 1449033f9833Sek * First purge some DNLC entries, in case the DNLC is using 1450033f9833Sek * up too much memory. 1451033f9833Sek */ 1452cee972f8Sek dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); 14535dc8af33Smaybee 14545dc8af33Smaybee #if defined(__i386) 14555dc8af33Smaybee /* 14565dc8af33Smaybee * Reclaim unused memory from all kmem caches. 14575dc8af33Smaybee */ 14585dc8af33Smaybee kmem_reap(); 14595dc8af33Smaybee #endif 1460033f9833Sek #endif 1461033f9833Sek 1462fa9e4066Sahrens /* 1463ea8dc4b6Seschrock * An agressive reclamation will shrink the cache size as well as 1464ea8dc4b6Seschrock * reap free buffers from the arc kmem caches. 1465fa9e4066Sahrens */ 1466fa9e4066Sahrens if (strat == ARC_RECLAIM_AGGR) 146749e3519aSmaybee arc_shrink(); 1468fa9e4066Sahrens 1469fa9e4066Sahrens for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { 1470fa9e4066Sahrens if (zio_buf_cache[i] != prev_cache) { 1471fa9e4066Sahrens prev_cache = zio_buf_cache[i]; 1472fa9e4066Sahrens kmem_cache_reap_now(zio_buf_cache[i]); 1473fa9e4066Sahrens } 1474ad23a2dbSjohansen if (zio_data_buf_cache[i] != prev_data_cache) { 1475ad23a2dbSjohansen prev_data_cache = zio_data_buf_cache[i]; 1476ad23a2dbSjohansen kmem_cache_reap_now(zio_data_buf_cache[i]); 1477ad23a2dbSjohansen } 1478fa9e4066Sahrens } 1479ea8dc4b6Seschrock kmem_cache_reap_now(buf_cache); 1480ea8dc4b6Seschrock kmem_cache_reap_now(hdr_cache); 1481fa9e4066Sahrens } 1482fa9e4066Sahrens 1483fa9e4066Sahrens static void 1484fa9e4066Sahrens arc_reclaim_thread(void) 1485fa9e4066Sahrens { 1486fa9e4066Sahrens clock_t growtime = 0; 1487fa9e4066Sahrens arc_reclaim_strategy_t last_reclaim = ARC_RECLAIM_CONS; 1488fa9e4066Sahrens callb_cpr_t cpr; 1489fa9e4066Sahrens 1490fa9e4066Sahrens CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG); 1491fa9e4066Sahrens 1492fa9e4066Sahrens mutex_enter(&arc_reclaim_thr_lock); 1493fa9e4066Sahrens while (arc_thread_exit == 0) { 1494fa9e4066Sahrens if (arc_reclaim_needed()) { 1495fa9e4066Sahrens 1496*44cb6abcSbmc if (arc_no_grow) { 1497fa9e4066Sahrens if (last_reclaim == ARC_RECLAIM_CONS) { 1498fa9e4066Sahrens last_reclaim = ARC_RECLAIM_AGGR; 1499fa9e4066Sahrens } else { 1500fa9e4066Sahrens last_reclaim = ARC_RECLAIM_CONS; 1501fa9e4066Sahrens } 1502fa9e4066Sahrens } else { 1503*44cb6abcSbmc arc_no_grow = TRUE; 1504fa9e4066Sahrens last_reclaim = ARC_RECLAIM_AGGR; 1505fa9e4066Sahrens membar_producer(); 1506fa9e4066Sahrens } 1507fa9e4066Sahrens 1508fa9e4066Sahrens /* reset the growth delay for every reclaim */ 1509fa9e4066Sahrens growtime = lbolt + (arc_grow_retry * hz); 15105ad82045Snd ASSERT(growtime > 0); 1511fa9e4066Sahrens 1512fa9e4066Sahrens arc_kmem_reap_now(last_reclaim); 1513fa9e4066Sahrens 1514fa9e4066Sahrens } else if ((growtime > 0) && ((growtime - lbolt) <= 0)) { 1515*44cb6abcSbmc arc_no_grow = FALSE; 1516fa9e4066Sahrens } 1517fa9e4066Sahrens 1518*44cb6abcSbmc if (2 * arc_c < arc_size + 1519*44cb6abcSbmc arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size) 1520641fbdaeSmaybee arc_adjust(); 1521641fbdaeSmaybee 1522ea8dc4b6Seschrock if (arc_eviction_list != NULL) 1523ea8dc4b6Seschrock arc_do_user_evicts(); 1524ea8dc4b6Seschrock 1525fa9e4066Sahrens /* block until needed, or one second, whichever is shorter */ 1526fa9e4066Sahrens CALLB_CPR_SAFE_BEGIN(&cpr); 1527fa9e4066Sahrens (void) cv_timedwait(&arc_reclaim_thr_cv, 1528fa9e4066Sahrens &arc_reclaim_thr_lock, (lbolt + hz)); 1529fa9e4066Sahrens CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock); 1530fa9e4066Sahrens } 1531fa9e4066Sahrens 1532fa9e4066Sahrens arc_thread_exit = 0; 1533fa9e4066Sahrens cv_broadcast(&arc_reclaim_thr_cv); 1534fa9e4066Sahrens CALLB_CPR_EXIT(&cpr); /* drops arc_reclaim_thr_lock */ 1535fa9e4066Sahrens thread_exit(); 1536fa9e4066Sahrens } 1537fa9e4066Sahrens 1538ea8dc4b6Seschrock /* 1539ea8dc4b6Seschrock * Adapt arc info given the number of bytes we are trying to add and 1540ea8dc4b6Seschrock * the state that we are comming from. This function is only called 1541ea8dc4b6Seschrock * when we are adding new content to the cache. 1542ea8dc4b6Seschrock */ 1543fa9e4066Sahrens static void 1544ea8dc4b6Seschrock arc_adapt(int bytes, arc_state_t *state) 1545fa9e4066Sahrens { 1546ea8dc4b6Seschrock int mult; 1547ea8dc4b6Seschrock 1548ea8dc4b6Seschrock ASSERT(bytes > 0); 1549fa9e4066Sahrens /* 1550ea8dc4b6Seschrock * Adapt the target size of the MRU list: 1551ea8dc4b6Seschrock * - if we just hit in the MRU ghost list, then increase 1552ea8dc4b6Seschrock * the target size of the MRU list. 1553ea8dc4b6Seschrock * - if we just hit in the MFU ghost list, then increase 1554ea8dc4b6Seschrock * the target size of the MFU list by decreasing the 1555ea8dc4b6Seschrock * target size of the MRU list. 1556fa9e4066Sahrens */ 1557*44cb6abcSbmc if (state == arc_mru_ghost) { 1558*44cb6abcSbmc mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ? 1559*44cb6abcSbmc 1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size)); 1560ea8dc4b6Seschrock 1561*44cb6abcSbmc arc_p = MIN(arc_c, arc_p + bytes * mult); 1562*44cb6abcSbmc } else if (state == arc_mfu_ghost) { 1563*44cb6abcSbmc mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ? 1564*44cb6abcSbmc 1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size)); 1565ea8dc4b6Seschrock 1566*44cb6abcSbmc arc_p = MAX(0, (int64_t)arc_p - bytes * mult); 1567ea8dc4b6Seschrock } 1568*44cb6abcSbmc ASSERT((int64_t)arc_p >= 0); 1569fa9e4066Sahrens 1570fa9e4066Sahrens if (arc_reclaim_needed()) { 1571fa9e4066Sahrens cv_signal(&arc_reclaim_thr_cv); 1572fa9e4066Sahrens return; 1573fa9e4066Sahrens } 1574fa9e4066Sahrens 1575*44cb6abcSbmc if (arc_no_grow) 1576fa9e4066Sahrens return; 1577fa9e4066Sahrens 1578*44cb6abcSbmc if (arc_c >= arc_c_max) 1579ea8dc4b6Seschrock return; 1580ea8dc4b6Seschrock 1581fa9e4066Sahrens /* 1582ea8dc4b6Seschrock * If we're within (2 * maxblocksize) bytes of the target 1583ea8dc4b6Seschrock * cache size, increment the target cache size 1584fa9e4066Sahrens */ 1585*44cb6abcSbmc if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) { 1586*44cb6abcSbmc atomic_add_64(&arc_c, (int64_t)bytes); 1587*44cb6abcSbmc if (arc_c > arc_c_max) 1588*44cb6abcSbmc arc_c = arc_c_max; 1589*44cb6abcSbmc else if (state == arc_anon) 1590*44cb6abcSbmc atomic_add_64(&arc_p, (int64_t)bytes); 1591*44cb6abcSbmc if (arc_p > arc_c) 1592*44cb6abcSbmc arc_p = arc_c; 1593fa9e4066Sahrens } 1594*44cb6abcSbmc ASSERT((int64_t)arc_p >= 0); 1595fa9e4066Sahrens } 1596fa9e4066Sahrens 1597fa9e4066Sahrens /* 1598ea8dc4b6Seschrock * Check if the cache has reached its limits and eviction is required 1599ea8dc4b6Seschrock * prior to insert. 1600fa9e4066Sahrens */ 1601fa9e4066Sahrens static int 1602fa9e4066Sahrens arc_evict_needed() 1603fa9e4066Sahrens { 1604fa9e4066Sahrens if (arc_reclaim_needed()) 1605fa9e4066Sahrens return (1); 1606fa9e4066Sahrens 1607*44cb6abcSbmc return (arc_size > arc_c); 1608fa9e4066Sahrens } 1609fa9e4066Sahrens 1610fa9e4066Sahrens /* 161144eda4d7Smaybee * The buffer, supplied as the first argument, needs a data block. 161244eda4d7Smaybee * So, if we are at cache max, determine which cache should be victimized. 161344eda4d7Smaybee * We have the following cases: 1614fa9e4066Sahrens * 1615*44cb6abcSbmc * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) -> 1616fa9e4066Sahrens * In this situation if we're out of space, but the resident size of the MFU is 1617fa9e4066Sahrens * under the limit, victimize the MFU cache to satisfy this insertion request. 1618fa9e4066Sahrens * 1619*44cb6abcSbmc * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) -> 1620fa9e4066Sahrens * Here, we've used up all of the available space for the MRU, so we need to 1621fa9e4066Sahrens * evict from our own cache instead. Evict from the set of resident MRU 1622fa9e4066Sahrens * entries. 1623fa9e4066Sahrens * 1624*44cb6abcSbmc * 3. Insert for MFU (c - p) > sizeof(arc_mfu) -> 1625fa9e4066Sahrens * c minus p represents the MFU space in the cache, since p is the size of the 1626fa9e4066Sahrens * cache that is dedicated to the MRU. In this situation there's still space on 1627fa9e4066Sahrens * the MFU side, so the MRU side needs to be victimized. 1628fa9e4066Sahrens * 1629*44cb6abcSbmc * 4. Insert for MFU (c - p) < sizeof(arc_mfu) -> 1630fa9e4066Sahrens * MFU's resident set is consuming more space than it has been allotted. In 1631fa9e4066Sahrens * this situation, we must victimize our own cache, the MFU, for this insertion. 1632fa9e4066Sahrens */ 1633fa9e4066Sahrens static void 163444eda4d7Smaybee arc_get_data_buf(arc_buf_t *buf) 1635fa9e4066Sahrens { 1636ad23a2dbSjohansen arc_state_t *state = buf->b_hdr->b_state; 1637ad23a2dbSjohansen uint64_t size = buf->b_hdr->b_size; 1638ad23a2dbSjohansen arc_buf_contents_t type = buf->b_hdr->b_type; 1639fa9e4066Sahrens 164044eda4d7Smaybee arc_adapt(size, state); 1641fa9e4066Sahrens 164244eda4d7Smaybee /* 164344eda4d7Smaybee * We have not yet reached cache maximum size, 164444eda4d7Smaybee * just allocate a new buffer. 164544eda4d7Smaybee */ 164644eda4d7Smaybee if (!arc_evict_needed()) { 1647ad23a2dbSjohansen if (type == ARC_BUFC_METADATA) { 1648ad23a2dbSjohansen buf->b_data = zio_buf_alloc(size); 1649ad23a2dbSjohansen } else { 1650ad23a2dbSjohansen ASSERT(type == ARC_BUFC_DATA); 1651ad23a2dbSjohansen buf->b_data = zio_data_buf_alloc(size); 1652ad23a2dbSjohansen } 1653*44cb6abcSbmc atomic_add_64(&arc_size, size); 165444eda4d7Smaybee goto out; 165544eda4d7Smaybee } 165644eda4d7Smaybee 165744eda4d7Smaybee /* 165844eda4d7Smaybee * If we are prefetching from the mfu ghost list, this buffer 165944eda4d7Smaybee * will end up on the mru list; so steal space from there. 166044eda4d7Smaybee */ 1661*44cb6abcSbmc if (state == arc_mfu_ghost) 1662*44cb6abcSbmc state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu; 1663*44cb6abcSbmc else if (state == arc_mru_ghost) 1664*44cb6abcSbmc state = arc_mru; 1665*44cb6abcSbmc 1666*44cb6abcSbmc if (state == arc_mru || state == arc_anon) { 1667*44cb6abcSbmc uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size; 1668*44cb6abcSbmc state = (arc_p > mru_used) ? arc_mfu : arc_mru; 1669fa9e4066Sahrens } else { 167044eda4d7Smaybee /* MFU cases */ 1671*44cb6abcSbmc uint64_t mfu_space = arc_c - arc_p; 1672*44cb6abcSbmc state = (mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu; 167344eda4d7Smaybee } 1674ad23a2dbSjohansen if ((buf->b_data = arc_evict(state, size, TRUE, type)) == NULL) { 1675ad23a2dbSjohansen if (type == ARC_BUFC_METADATA) { 1676ad23a2dbSjohansen buf->b_data = zio_buf_alloc(size); 1677ad23a2dbSjohansen } else { 1678ad23a2dbSjohansen ASSERT(type == ARC_BUFC_DATA); 1679ad23a2dbSjohansen buf->b_data = zio_data_buf_alloc(size); 1680ad23a2dbSjohansen } 1681*44cb6abcSbmc atomic_add_64(&arc_size, size); 1682*44cb6abcSbmc ARCSTAT_BUMP(arcstat_recycle_miss); 168344eda4d7Smaybee } 168444eda4d7Smaybee ASSERT(buf->b_data != NULL); 168544eda4d7Smaybee out: 168644eda4d7Smaybee /* 168744eda4d7Smaybee * Update the state size. Note that ghost states have a 168844eda4d7Smaybee * "ghost size" and so don't need to be updated. 168944eda4d7Smaybee */ 169044eda4d7Smaybee if (!GHOST_STATE(buf->b_hdr->b_state)) { 169144eda4d7Smaybee arc_buf_hdr_t *hdr = buf->b_hdr; 169244eda4d7Smaybee 1693*44cb6abcSbmc atomic_add_64(&hdr->b_state->arcs_size, size); 169444eda4d7Smaybee if (list_link_active(&hdr->b_arc_node)) { 169544eda4d7Smaybee ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1696*44cb6abcSbmc atomic_add_64(&hdr->b_state->arcs_lsize, size); 1697fa9e4066Sahrens } 1698641fbdaeSmaybee /* 1699641fbdaeSmaybee * If we are growing the cache, and we are adding anonymous 1700*44cb6abcSbmc * data, and we have outgrown arc_p, update arc_p 1701641fbdaeSmaybee */ 1702*44cb6abcSbmc if (arc_size < arc_c && hdr->b_state == arc_anon && 1703*44cb6abcSbmc arc_anon->arcs_size + arc_mru->arcs_size > arc_p) 1704*44cb6abcSbmc arc_p = MIN(arc_c, arc_p + size); 1705fa9e4066Sahrens } 1706fa9e4066Sahrens } 1707fa9e4066Sahrens 1708fa9e4066Sahrens /* 1709fa9e4066Sahrens * This routine is called whenever a buffer is accessed. 1710ea8dc4b6Seschrock * NOTE: the hash lock is dropped in this function. 1711fa9e4066Sahrens */ 1712fa9e4066Sahrens static void 171344eda4d7Smaybee arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock) 1714fa9e4066Sahrens { 1715fa9e4066Sahrens ASSERT(MUTEX_HELD(hash_lock)); 1716fa9e4066Sahrens 1717*44cb6abcSbmc if (buf->b_state == arc_anon) { 1718fa9e4066Sahrens /* 1719fa9e4066Sahrens * This buffer is not in the cache, and does not 1720fa9e4066Sahrens * appear in our "ghost" list. Add the new buffer 1721fa9e4066Sahrens * to the MRU state. 1722fa9e4066Sahrens */ 1723fa9e4066Sahrens 1724fa9e4066Sahrens ASSERT(buf->b_arc_access == 0); 1725fa9e4066Sahrens buf->b_arc_access = lbolt; 1726ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 1727*44cb6abcSbmc arc_change_state(arc_mru, buf, hash_lock); 1728fa9e4066Sahrens 1729*44cb6abcSbmc } else if (buf->b_state == arc_mru) { 1730fa9e4066Sahrens /* 173113506d1eSmaybee * If this buffer is here because of a prefetch, then either: 173213506d1eSmaybee * - clear the flag if this is a "referencing" read 173313506d1eSmaybee * (any subsequent access will bump this into the MFU state). 173413506d1eSmaybee * or 173513506d1eSmaybee * - move the buffer to the head of the list if this is 173613506d1eSmaybee * another prefetch (to make it less likely to be evicted). 1737fa9e4066Sahrens */ 1738fa9e4066Sahrens if ((buf->b_flags & ARC_PREFETCH) != 0) { 173913506d1eSmaybee if (refcount_count(&buf->b_refcnt) == 0) { 174013506d1eSmaybee ASSERT(list_link_active(&buf->b_arc_node)); 1741*44cb6abcSbmc mutex_enter(&arc_mru->arcs_mtx); 1742*44cb6abcSbmc list_remove(&arc_mru->arcs_list, buf); 1743*44cb6abcSbmc list_insert_head(&arc_mru->arcs_list, buf); 1744*44cb6abcSbmc mutex_exit(&arc_mru->arcs_mtx); 174513506d1eSmaybee } else { 174613506d1eSmaybee buf->b_flags &= ~ARC_PREFETCH; 1747*44cb6abcSbmc ARCSTAT_BUMP(arcstat_mru_hits); 174813506d1eSmaybee } 174913506d1eSmaybee buf->b_arc_access = lbolt; 1750fa9e4066Sahrens return; 1751fa9e4066Sahrens } 1752fa9e4066Sahrens 1753fa9e4066Sahrens /* 1754fa9e4066Sahrens * This buffer has been "accessed" only once so far, 1755fa9e4066Sahrens * but it is still in the cache. Move it to the MFU 1756fa9e4066Sahrens * state. 1757fa9e4066Sahrens */ 1758fa9e4066Sahrens if (lbolt > buf->b_arc_access + ARC_MINTIME) { 1759fa9e4066Sahrens /* 1760fa9e4066Sahrens * More than 125ms have passed since we 1761fa9e4066Sahrens * instantiated this buffer. Move it to the 1762fa9e4066Sahrens * most frequently used state. 1763fa9e4066Sahrens */ 1764fa9e4066Sahrens buf->b_arc_access = lbolt; 1765ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 1766*44cb6abcSbmc arc_change_state(arc_mfu, buf, hash_lock); 1767fa9e4066Sahrens } 1768*44cb6abcSbmc ARCSTAT_BUMP(arcstat_mru_hits); 1769*44cb6abcSbmc } else if (buf->b_state == arc_mru_ghost) { 1770fa9e4066Sahrens arc_state_t *new_state; 1771fa9e4066Sahrens /* 1772fa9e4066Sahrens * This buffer has been "accessed" recently, but 1773fa9e4066Sahrens * was evicted from the cache. Move it to the 1774fa9e4066Sahrens * MFU state. 1775fa9e4066Sahrens */ 1776fa9e4066Sahrens 1777fa9e4066Sahrens if (buf->b_flags & ARC_PREFETCH) { 1778*44cb6abcSbmc new_state = arc_mru; 177913506d1eSmaybee if (refcount_count(&buf->b_refcnt) > 0) 178013506d1eSmaybee buf->b_flags &= ~ARC_PREFETCH; 1781ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 1782fa9e4066Sahrens } else { 1783*44cb6abcSbmc new_state = arc_mfu; 1784ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 1785fa9e4066Sahrens } 1786fa9e4066Sahrens 1787fa9e4066Sahrens buf->b_arc_access = lbolt; 1788fa9e4066Sahrens arc_change_state(new_state, buf, hash_lock); 1789fa9e4066Sahrens 1790*44cb6abcSbmc ARCSTAT_BUMP(arcstat_mru_ghost_hits); 1791*44cb6abcSbmc } else if (buf->b_state == arc_mfu) { 1792fa9e4066Sahrens /* 1793fa9e4066Sahrens * This buffer has been accessed more than once and is 1794fa9e4066Sahrens * still in the cache. Keep it in the MFU state. 1795fa9e4066Sahrens * 179613506d1eSmaybee * NOTE: an add_reference() that occurred when we did 179713506d1eSmaybee * the arc_read() will have kicked this off the list. 179813506d1eSmaybee * If it was a prefetch, we will explicitly move it to 179913506d1eSmaybee * the head of the list now. 1800fa9e4066Sahrens */ 180113506d1eSmaybee if ((buf->b_flags & ARC_PREFETCH) != 0) { 180213506d1eSmaybee ASSERT(refcount_count(&buf->b_refcnt) == 0); 180313506d1eSmaybee ASSERT(list_link_active(&buf->b_arc_node)); 1804*44cb6abcSbmc mutex_enter(&arc_mfu->arcs_mtx); 1805*44cb6abcSbmc list_remove(&arc_mfu->arcs_list, buf); 1806*44cb6abcSbmc list_insert_head(&arc_mfu->arcs_list, buf); 1807*44cb6abcSbmc mutex_exit(&arc_mfu->arcs_mtx); 180813506d1eSmaybee } 1809*44cb6abcSbmc ARCSTAT_BUMP(arcstat_mfu_hits); 181013506d1eSmaybee buf->b_arc_access = lbolt; 1811*44cb6abcSbmc } else if (buf->b_state == arc_mfu_ghost) { 1812*44cb6abcSbmc arc_state_t *new_state = arc_mfu; 1813fa9e4066Sahrens /* 1814fa9e4066Sahrens * This buffer has been accessed more than once but has 1815fa9e4066Sahrens * been evicted from the cache. Move it back to the 1816fa9e4066Sahrens * MFU state. 1817fa9e4066Sahrens */ 1818fa9e4066Sahrens 181913506d1eSmaybee if (buf->b_flags & ARC_PREFETCH) { 182013506d1eSmaybee /* 182113506d1eSmaybee * This is a prefetch access... 182213506d1eSmaybee * move this block back to the MRU state. 182313506d1eSmaybee */ 182413506d1eSmaybee ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0); 1825*44cb6abcSbmc new_state = arc_mru; 182613506d1eSmaybee } 182713506d1eSmaybee 1828fa9e4066Sahrens buf->b_arc_access = lbolt; 1829ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 183013506d1eSmaybee arc_change_state(new_state, buf, hash_lock); 1831fa9e4066Sahrens 1832*44cb6abcSbmc ARCSTAT_BUMP(arcstat_mfu_ghost_hits); 1833fa9e4066Sahrens } else { 1834fa9e4066Sahrens ASSERT(!"invalid arc state"); 1835fa9e4066Sahrens } 1836fa9e4066Sahrens } 1837fa9e4066Sahrens 1838fa9e4066Sahrens /* a generic arc_done_func_t which you can use */ 1839fa9e4066Sahrens /* ARGSUSED */ 1840fa9e4066Sahrens void 1841fa9e4066Sahrens arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) 1842fa9e4066Sahrens { 1843fa9e4066Sahrens bcopy(buf->b_data, arg, buf->b_hdr->b_size); 1844ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, arg) == 1); 1845fa9e4066Sahrens } 1846fa9e4066Sahrens 1847fa9e4066Sahrens /* a generic arc_done_func_t which you can use */ 1848fa9e4066Sahrens void 1849fa9e4066Sahrens arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) 1850fa9e4066Sahrens { 1851fa9e4066Sahrens arc_buf_t **bufp = arg; 1852fa9e4066Sahrens if (zio && zio->io_error) { 1853ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, arg) == 1); 1854fa9e4066Sahrens *bufp = NULL; 1855fa9e4066Sahrens } else { 1856fa9e4066Sahrens *bufp = buf; 1857fa9e4066Sahrens } 1858fa9e4066Sahrens } 1859fa9e4066Sahrens 1860fa9e4066Sahrens static void 1861fa9e4066Sahrens arc_read_done(zio_t *zio) 1862fa9e4066Sahrens { 1863bbf4a8dfSmaybee arc_buf_hdr_t *hdr, *found; 1864fa9e4066Sahrens arc_buf_t *buf; 1865fa9e4066Sahrens arc_buf_t *abuf; /* buffer we're assigning to callback */ 1866fa9e4066Sahrens kmutex_t *hash_lock; 1867fa9e4066Sahrens arc_callback_t *callback_list, *acb; 1868fa9e4066Sahrens int freeable = FALSE; 1869fa9e4066Sahrens 1870fa9e4066Sahrens buf = zio->io_private; 1871fa9e4066Sahrens hdr = buf->b_hdr; 1872fa9e4066Sahrens 1873bbf4a8dfSmaybee /* 1874bbf4a8dfSmaybee * The hdr was inserted into hash-table and removed from lists 1875bbf4a8dfSmaybee * prior to starting I/O. We should find this header, since 1876bbf4a8dfSmaybee * it's in the hash table, and it should be legit since it's 1877bbf4a8dfSmaybee * not possible to evict it during the I/O. The only possible 1878bbf4a8dfSmaybee * reason for it not to be found is if we were freed during the 1879bbf4a8dfSmaybee * read. 1880bbf4a8dfSmaybee */ 1881bbf4a8dfSmaybee found = buf_hash_find(zio->io_spa, &hdr->b_dva, hdr->b_birth, 18826b4acc8bSahrens &hash_lock); 1883fa9e4066Sahrens 1884bbf4a8dfSmaybee ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) || 1885bbf4a8dfSmaybee (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp)))); 1886fa9e4066Sahrens 1887fa9e4066Sahrens /* byteswap if necessary */ 1888fa9e4066Sahrens callback_list = hdr->b_acb; 1889fa9e4066Sahrens ASSERT(callback_list != NULL); 1890fa9e4066Sahrens if (BP_SHOULD_BYTESWAP(zio->io_bp) && callback_list->acb_byteswap) 1891fa9e4066Sahrens callback_list->acb_byteswap(buf->b_data, hdr->b_size); 1892fa9e4066Sahrens 18936b4acc8bSahrens arc_cksum_compute(buf); 18946b4acc8bSahrens 1895fa9e4066Sahrens /* create copies of the data buffer for the callers */ 1896fa9e4066Sahrens abuf = buf; 1897fa9e4066Sahrens for (acb = callback_list; acb; acb = acb->acb_next) { 1898fa9e4066Sahrens if (acb->acb_done) { 189944eda4d7Smaybee if (abuf == NULL) 190044eda4d7Smaybee abuf = arc_buf_clone(buf); 1901fa9e4066Sahrens acb->acb_buf = abuf; 1902fa9e4066Sahrens abuf = NULL; 1903fa9e4066Sahrens } 1904fa9e4066Sahrens } 1905fa9e4066Sahrens hdr->b_acb = NULL; 1906fa9e4066Sahrens hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 1907ea8dc4b6Seschrock ASSERT(!HDR_BUF_AVAILABLE(hdr)); 1908ea8dc4b6Seschrock if (abuf == buf) 1909ea8dc4b6Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 1910fa9e4066Sahrens 1911fa9e4066Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL); 1912fa9e4066Sahrens 1913fa9e4066Sahrens if (zio->io_error != 0) { 1914fa9e4066Sahrens hdr->b_flags |= ARC_IO_ERROR; 1915*44cb6abcSbmc if (hdr->b_state != arc_anon) 1916*44cb6abcSbmc arc_change_state(arc_anon, hdr, hash_lock); 1917ea8dc4b6Seschrock if (HDR_IN_HASH_TABLE(hdr)) 1918ea8dc4b6Seschrock buf_hash_remove(hdr); 1919fa9e4066Sahrens freeable = refcount_is_zero(&hdr->b_refcnt); 192013506d1eSmaybee /* convert checksum errors into IO errors */ 1921ea8dc4b6Seschrock if (zio->io_error == ECKSUM) 1922ea8dc4b6Seschrock zio->io_error = EIO; 1923fa9e4066Sahrens } 1924fa9e4066Sahrens 1925ea8dc4b6Seschrock /* 192613506d1eSmaybee * Broadcast before we drop the hash_lock to avoid the possibility 192713506d1eSmaybee * that the hdr (and hence the cv) might be freed before we get to 192813506d1eSmaybee * the cv_broadcast(). 1929ea8dc4b6Seschrock */ 1930ea8dc4b6Seschrock cv_broadcast(&hdr->b_cv); 1931ea8dc4b6Seschrock 1932bbf4a8dfSmaybee if (hash_lock) { 1933fa9e4066Sahrens /* 1934fa9e4066Sahrens * Only call arc_access on anonymous buffers. This is because 1935fa9e4066Sahrens * if we've issued an I/O for an evicted buffer, we've already 1936fa9e4066Sahrens * called arc_access (to prevent any simultaneous readers from 1937fa9e4066Sahrens * getting confused). 1938fa9e4066Sahrens */ 1939*44cb6abcSbmc if (zio->io_error == 0 && hdr->b_state == arc_anon) 194044eda4d7Smaybee arc_access(hdr, hash_lock); 194144eda4d7Smaybee mutex_exit(hash_lock); 1942fa9e4066Sahrens } else { 1943fa9e4066Sahrens /* 1944fa9e4066Sahrens * This block was freed while we waited for the read to 1945fa9e4066Sahrens * complete. It has been removed from the hash table and 1946fa9e4066Sahrens * moved to the anonymous state (so that it won't show up 1947fa9e4066Sahrens * in the cache). 1948fa9e4066Sahrens */ 1949*44cb6abcSbmc ASSERT3P(hdr->b_state, ==, arc_anon); 1950fa9e4066Sahrens freeable = refcount_is_zero(&hdr->b_refcnt); 1951fa9e4066Sahrens } 1952fa9e4066Sahrens 1953fa9e4066Sahrens /* execute each callback and free its structure */ 1954fa9e4066Sahrens while ((acb = callback_list) != NULL) { 1955fa9e4066Sahrens if (acb->acb_done) 1956fa9e4066Sahrens acb->acb_done(zio, acb->acb_buf, acb->acb_private); 1957fa9e4066Sahrens 1958fa9e4066Sahrens if (acb->acb_zio_dummy != NULL) { 1959fa9e4066Sahrens acb->acb_zio_dummy->io_error = zio->io_error; 1960fa9e4066Sahrens zio_nowait(acb->acb_zio_dummy); 1961fa9e4066Sahrens } 1962fa9e4066Sahrens 1963fa9e4066Sahrens callback_list = acb->acb_next; 1964fa9e4066Sahrens kmem_free(acb, sizeof (arc_callback_t)); 1965fa9e4066Sahrens } 1966fa9e4066Sahrens 1967fa9e4066Sahrens if (freeable) 1968ea8dc4b6Seschrock arc_hdr_destroy(hdr); 1969fa9e4066Sahrens } 1970fa9e4066Sahrens 1971fa9e4066Sahrens /* 1972fa9e4066Sahrens * "Read" the block block at the specified DVA (in bp) via the 1973fa9e4066Sahrens * cache. If the block is found in the cache, invoke the provided 1974fa9e4066Sahrens * callback immediately and return. Note that the `zio' parameter 1975fa9e4066Sahrens * in the callback will be NULL in this case, since no IO was 1976fa9e4066Sahrens * required. If the block is not in the cache pass the read request 1977fa9e4066Sahrens * on to the spa with a substitute callback function, so that the 1978fa9e4066Sahrens * requested block will be added to the cache. 1979fa9e4066Sahrens * 1980fa9e4066Sahrens * If a read request arrives for a block that has a read in-progress, 1981fa9e4066Sahrens * either wait for the in-progress read to complete (and return the 1982fa9e4066Sahrens * results); or, if this is a read with a "done" func, add a record 1983fa9e4066Sahrens * to the read to invoke the "done" func when the read completes, 1984fa9e4066Sahrens * and return; or just return. 1985fa9e4066Sahrens * 1986fa9e4066Sahrens * arc_read_done() will invoke all the requested "done" functions 1987fa9e4066Sahrens * for readers of this block. 1988fa9e4066Sahrens */ 1989fa9e4066Sahrens int 1990fa9e4066Sahrens arc_read(zio_t *pio, spa_t *spa, blkptr_t *bp, arc_byteswap_func_t *swap, 1991fa9e4066Sahrens arc_done_func_t *done, void *private, int priority, int flags, 199213506d1eSmaybee uint32_t *arc_flags, zbookmark_t *zb) 1993fa9e4066Sahrens { 1994fa9e4066Sahrens arc_buf_hdr_t *hdr; 1995fa9e4066Sahrens arc_buf_t *buf; 1996fa9e4066Sahrens kmutex_t *hash_lock; 1997fa9e4066Sahrens zio_t *rzio; 1998fa9e4066Sahrens 1999fa9e4066Sahrens top: 2000fa9e4066Sahrens hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock); 2001ea8dc4b6Seschrock if (hdr && hdr->b_datacnt > 0) { 2002fa9e4066Sahrens 200313506d1eSmaybee *arc_flags |= ARC_CACHED; 200413506d1eSmaybee 2005fa9e4066Sahrens if (HDR_IO_IN_PROGRESS(hdr)) { 200613506d1eSmaybee 200713506d1eSmaybee if (*arc_flags & ARC_WAIT) { 200813506d1eSmaybee cv_wait(&hdr->b_cv, hash_lock); 200913506d1eSmaybee mutex_exit(hash_lock); 201013506d1eSmaybee goto top; 201113506d1eSmaybee } 201213506d1eSmaybee ASSERT(*arc_flags & ARC_NOWAIT); 201313506d1eSmaybee 201413506d1eSmaybee if (done) { 2015fa9e4066Sahrens arc_callback_t *acb = NULL; 2016fa9e4066Sahrens 2017fa9e4066Sahrens acb = kmem_zalloc(sizeof (arc_callback_t), 2018fa9e4066Sahrens KM_SLEEP); 2019fa9e4066Sahrens acb->acb_done = done; 2020fa9e4066Sahrens acb->acb_private = private; 2021fa9e4066Sahrens acb->acb_byteswap = swap; 2022fa9e4066Sahrens if (pio != NULL) 2023fa9e4066Sahrens acb->acb_zio_dummy = zio_null(pio, 2024fa9e4066Sahrens spa, NULL, NULL, flags); 2025fa9e4066Sahrens 2026fa9e4066Sahrens ASSERT(acb->acb_done != NULL); 2027fa9e4066Sahrens acb->acb_next = hdr->b_acb; 2028fa9e4066Sahrens hdr->b_acb = acb; 2029fa9e4066Sahrens add_reference(hdr, hash_lock, private); 2030fa9e4066Sahrens mutex_exit(hash_lock); 2031fa9e4066Sahrens return (0); 2032fa9e4066Sahrens } 2033fa9e4066Sahrens mutex_exit(hash_lock); 2034fa9e4066Sahrens return (0); 2035fa9e4066Sahrens } 2036fa9e4066Sahrens 2037*44cb6abcSbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2038fa9e4066Sahrens 2039ea8dc4b6Seschrock if (done) { 204044eda4d7Smaybee add_reference(hdr, hash_lock, private); 2041ea8dc4b6Seschrock /* 2042ea8dc4b6Seschrock * If this block is already in use, create a new 2043ea8dc4b6Seschrock * copy of the data so that we will be guaranteed 2044ea8dc4b6Seschrock * that arc_release() will always succeed. 2045ea8dc4b6Seschrock */ 2046fa9e4066Sahrens buf = hdr->b_buf; 2047ea8dc4b6Seschrock ASSERT(buf); 2048ea8dc4b6Seschrock ASSERT(buf->b_data); 204944eda4d7Smaybee if (HDR_BUF_AVAILABLE(hdr)) { 2050ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 2051ea8dc4b6Seschrock hdr->b_flags &= ~ARC_BUF_AVAILABLE; 205244eda4d7Smaybee } else { 205344eda4d7Smaybee buf = arc_buf_clone(buf); 2054ea8dc4b6Seschrock } 205513506d1eSmaybee } else if (*arc_flags & ARC_PREFETCH && 205613506d1eSmaybee refcount_count(&hdr->b_refcnt) == 0) { 205713506d1eSmaybee hdr->b_flags |= ARC_PREFETCH; 2058fa9e4066Sahrens } 2059fa9e4066Sahrens DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 206044eda4d7Smaybee arc_access(hdr, hash_lock); 206144eda4d7Smaybee mutex_exit(hash_lock); 2062*44cb6abcSbmc ARCSTAT_BUMP(arcstat_hits); 2063*44cb6abcSbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 2064*44cb6abcSbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 2065*44cb6abcSbmc data, metadata, hits); 2066*44cb6abcSbmc 2067fa9e4066Sahrens if (done) 2068fa9e4066Sahrens done(NULL, buf, private); 2069fa9e4066Sahrens } else { 2070fa9e4066Sahrens uint64_t size = BP_GET_LSIZE(bp); 2071fa9e4066Sahrens arc_callback_t *acb; 2072fa9e4066Sahrens 2073fa9e4066Sahrens if (hdr == NULL) { 2074fa9e4066Sahrens /* this block is not in the cache */ 2075fa9e4066Sahrens arc_buf_hdr_t *exists; 2076ad23a2dbSjohansen arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp); 2077ad23a2dbSjohansen buf = arc_buf_alloc(spa, size, private, type); 2078fa9e4066Sahrens hdr = buf->b_hdr; 2079fa9e4066Sahrens hdr->b_dva = *BP_IDENTITY(bp); 2080fa9e4066Sahrens hdr->b_birth = bp->blk_birth; 2081fa9e4066Sahrens hdr->b_cksum0 = bp->blk_cksum.zc_word[0]; 2082fa9e4066Sahrens exists = buf_hash_insert(hdr, &hash_lock); 2083fa9e4066Sahrens if (exists) { 2084fa9e4066Sahrens /* somebody beat us to the hash insert */ 2085fa9e4066Sahrens mutex_exit(hash_lock); 2086fa9e4066Sahrens bzero(&hdr->b_dva, sizeof (dva_t)); 2087fa9e4066Sahrens hdr->b_birth = 0; 2088fa9e4066Sahrens hdr->b_cksum0 = 0; 2089ea8dc4b6Seschrock (void) arc_buf_remove_ref(buf, private); 2090fa9e4066Sahrens goto top; /* restart the IO request */ 2091fa9e4066Sahrens } 209213506d1eSmaybee /* if this is a prefetch, we don't have a reference */ 209313506d1eSmaybee if (*arc_flags & ARC_PREFETCH) { 209413506d1eSmaybee (void) remove_reference(hdr, hash_lock, 209513506d1eSmaybee private); 209613506d1eSmaybee hdr->b_flags |= ARC_PREFETCH; 209713506d1eSmaybee } 209813506d1eSmaybee if (BP_GET_LEVEL(bp) > 0) 209913506d1eSmaybee hdr->b_flags |= ARC_INDIRECT; 2100fa9e4066Sahrens } else { 2101fa9e4066Sahrens /* this block is in the ghost cache */ 2102ea8dc4b6Seschrock ASSERT(GHOST_STATE(hdr->b_state)); 2103ea8dc4b6Seschrock ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 210413506d1eSmaybee ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0); 2105ea8dc4b6Seschrock ASSERT(hdr->b_buf == NULL); 210613506d1eSmaybee 210713506d1eSmaybee /* if this is a prefetch, we don't have a reference */ 210813506d1eSmaybee if (*arc_flags & ARC_PREFETCH) 210913506d1eSmaybee hdr->b_flags |= ARC_PREFETCH; 211013506d1eSmaybee else 211113506d1eSmaybee add_reference(hdr, hash_lock, private); 2112fa9e4066Sahrens buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 2113fa9e4066Sahrens buf->b_hdr = hdr; 211444eda4d7Smaybee buf->b_data = NULL; 2115ea8dc4b6Seschrock buf->b_efunc = NULL; 2116ea8dc4b6Seschrock buf->b_private = NULL; 2117fa9e4066Sahrens buf->b_next = NULL; 2118fa9e4066Sahrens hdr->b_buf = buf; 211944eda4d7Smaybee arc_get_data_buf(buf); 2120ea8dc4b6Seschrock ASSERT(hdr->b_datacnt == 0); 2121ea8dc4b6Seschrock hdr->b_datacnt = 1; 212213506d1eSmaybee 2123fa9e4066Sahrens } 2124fa9e4066Sahrens 2125fa9e4066Sahrens acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 2126fa9e4066Sahrens acb->acb_done = done; 2127fa9e4066Sahrens acb->acb_private = private; 2128fa9e4066Sahrens acb->acb_byteswap = swap; 2129fa9e4066Sahrens 2130fa9e4066Sahrens ASSERT(hdr->b_acb == NULL); 2131fa9e4066Sahrens hdr->b_acb = acb; 2132fa9e4066Sahrens hdr->b_flags |= ARC_IO_IN_PROGRESS; 2133fa9e4066Sahrens 2134fa9e4066Sahrens /* 2135fa9e4066Sahrens * If the buffer has been evicted, migrate it to a present state 2136fa9e4066Sahrens * before issuing the I/O. Once we drop the hash-table lock, 2137fa9e4066Sahrens * the header will be marked as I/O in progress and have an 2138fa9e4066Sahrens * attached buffer. At this point, anybody who finds this 2139fa9e4066Sahrens * buffer ought to notice that it's legit but has a pending I/O. 2140fa9e4066Sahrens */ 2141fa9e4066Sahrens 2142ea8dc4b6Seschrock if (GHOST_STATE(hdr->b_state)) 214344eda4d7Smaybee arc_access(hdr, hash_lock); 214444eda4d7Smaybee mutex_exit(hash_lock); 2145fa9e4066Sahrens 2146fa9e4066Sahrens ASSERT3U(hdr->b_size, ==, size); 2147c543ec06Sahrens DTRACE_PROBE3(arc__miss, blkptr_t *, bp, uint64_t, size, 2148c543ec06Sahrens zbookmark_t *, zb); 2149*44cb6abcSbmc ARCSTAT_BUMP(arcstat_misses); 2150*44cb6abcSbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 2151*44cb6abcSbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 2152*44cb6abcSbmc data, metadata, misses); 2153ea8dc4b6Seschrock 2154fa9e4066Sahrens rzio = zio_read(pio, spa, bp, buf->b_data, size, 2155ea8dc4b6Seschrock arc_read_done, buf, priority, flags, zb); 2156fa9e4066Sahrens 215713506d1eSmaybee if (*arc_flags & ARC_WAIT) 2158fa9e4066Sahrens return (zio_wait(rzio)); 2159fa9e4066Sahrens 216013506d1eSmaybee ASSERT(*arc_flags & ARC_NOWAIT); 2161fa9e4066Sahrens zio_nowait(rzio); 2162fa9e4066Sahrens } 2163fa9e4066Sahrens return (0); 2164fa9e4066Sahrens } 2165fa9e4066Sahrens 2166fa9e4066Sahrens /* 2167fa9e4066Sahrens * arc_read() variant to support pool traversal. If the block is already 2168fa9e4066Sahrens * in the ARC, make a copy of it; otherwise, the caller will do the I/O. 2169fa9e4066Sahrens * The idea is that we don't want pool traversal filling up memory, but 2170fa9e4066Sahrens * if the ARC already has the data anyway, we shouldn't pay for the I/O. 2171fa9e4066Sahrens */ 2172fa9e4066Sahrens int 2173fa9e4066Sahrens arc_tryread(spa_t *spa, blkptr_t *bp, void *data) 2174fa9e4066Sahrens { 2175fa9e4066Sahrens arc_buf_hdr_t *hdr; 2176fa9e4066Sahrens kmutex_t *hash_mtx; 2177fa9e4066Sahrens int rc = 0; 2178fa9e4066Sahrens 2179fa9e4066Sahrens hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_mtx); 2180fa9e4066Sahrens 2181ea8dc4b6Seschrock if (hdr && hdr->b_datacnt > 0 && !HDR_IO_IN_PROGRESS(hdr)) { 2182ea8dc4b6Seschrock arc_buf_t *buf = hdr->b_buf; 2183ea8dc4b6Seschrock 2184ea8dc4b6Seschrock ASSERT(buf); 2185ea8dc4b6Seschrock while (buf->b_data == NULL) { 2186ea8dc4b6Seschrock buf = buf->b_next; 2187ea8dc4b6Seschrock ASSERT(buf); 2188ea8dc4b6Seschrock } 2189ea8dc4b6Seschrock bcopy(buf->b_data, data, hdr->b_size); 2190ea8dc4b6Seschrock } else { 2191fa9e4066Sahrens rc = ENOENT; 2192ea8dc4b6Seschrock } 2193fa9e4066Sahrens 2194fa9e4066Sahrens if (hash_mtx) 2195fa9e4066Sahrens mutex_exit(hash_mtx); 2196fa9e4066Sahrens 2197fa9e4066Sahrens return (rc); 2198fa9e4066Sahrens } 2199fa9e4066Sahrens 2200ea8dc4b6Seschrock void 2201ea8dc4b6Seschrock arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private) 2202ea8dc4b6Seschrock { 2203ea8dc4b6Seschrock ASSERT(buf->b_hdr != NULL); 2204*44cb6abcSbmc ASSERT(buf->b_hdr->b_state != arc_anon); 2205ea8dc4b6Seschrock ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL); 2206ea8dc4b6Seschrock buf->b_efunc = func; 2207ea8dc4b6Seschrock buf->b_private = private; 2208ea8dc4b6Seschrock } 2209ea8dc4b6Seschrock 2210ea8dc4b6Seschrock /* 2211ea8dc4b6Seschrock * This is used by the DMU to let the ARC know that a buffer is 2212ea8dc4b6Seschrock * being evicted, so the ARC should clean up. If this arc buf 2213ea8dc4b6Seschrock * is not yet in the evicted state, it will be put there. 2214ea8dc4b6Seschrock */ 2215ea8dc4b6Seschrock int 2216ea8dc4b6Seschrock arc_buf_evict(arc_buf_t *buf) 2217ea8dc4b6Seschrock { 221840d7d650Smaybee arc_buf_hdr_t *hdr; 2219ea8dc4b6Seschrock kmutex_t *hash_lock; 2220ea8dc4b6Seschrock arc_buf_t **bufp; 2221ea8dc4b6Seschrock 222240d7d650Smaybee mutex_enter(&arc_eviction_mtx); 222340d7d650Smaybee hdr = buf->b_hdr; 2224ea8dc4b6Seschrock if (hdr == NULL) { 2225ea8dc4b6Seschrock /* 2226ea8dc4b6Seschrock * We are in arc_do_user_evicts(). 2227ea8dc4b6Seschrock */ 2228ea8dc4b6Seschrock ASSERT(buf->b_data == NULL); 222940d7d650Smaybee mutex_exit(&arc_eviction_mtx); 2230ea8dc4b6Seschrock return (0); 2231ea8dc4b6Seschrock } 2232ea8dc4b6Seschrock hash_lock = HDR_LOCK(hdr); 223340d7d650Smaybee mutex_exit(&arc_eviction_mtx); 223440d7d650Smaybee 2235ea8dc4b6Seschrock mutex_enter(hash_lock); 2236ea8dc4b6Seschrock 22379b23f181Smaybee if (buf->b_data == NULL) { 22389b23f181Smaybee /* 22399b23f181Smaybee * We are on the eviction list. 22409b23f181Smaybee */ 22419b23f181Smaybee mutex_exit(hash_lock); 22429b23f181Smaybee mutex_enter(&arc_eviction_mtx); 22439b23f181Smaybee if (buf->b_hdr == NULL) { 22449b23f181Smaybee /* 22459b23f181Smaybee * We are already in arc_do_user_evicts(). 22469b23f181Smaybee */ 22479b23f181Smaybee mutex_exit(&arc_eviction_mtx); 22489b23f181Smaybee return (0); 22499b23f181Smaybee } else { 22509b23f181Smaybee arc_buf_t copy = *buf; /* structure assignment */ 22519b23f181Smaybee /* 22529b23f181Smaybee * Process this buffer now 22539b23f181Smaybee * but let arc_do_user_evicts() do the reaping. 22549b23f181Smaybee */ 22559b23f181Smaybee buf->b_efunc = NULL; 22569b23f181Smaybee mutex_exit(&arc_eviction_mtx); 22579b23f181Smaybee VERIFY(copy.b_efunc(©) == 0); 22589b23f181Smaybee return (1); 22599b23f181Smaybee } 22609b23f181Smaybee } 22619b23f181Smaybee 22629b23f181Smaybee ASSERT(buf->b_hdr == hdr); 22639b23f181Smaybee ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt); 2264*44cb6abcSbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2265ea8dc4b6Seschrock 2266ea8dc4b6Seschrock /* 2267ea8dc4b6Seschrock * Pull this buffer off of the hdr 2268ea8dc4b6Seschrock */ 2269ea8dc4b6Seschrock bufp = &hdr->b_buf; 2270ea8dc4b6Seschrock while (*bufp != buf) 2271ea8dc4b6Seschrock bufp = &(*bufp)->b_next; 2272ea8dc4b6Seschrock *bufp = buf->b_next; 2273ea8dc4b6Seschrock 2274ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 227544eda4d7Smaybee arc_buf_destroy(buf, FALSE, FALSE); 2276ea8dc4b6Seschrock 2277ea8dc4b6Seschrock if (hdr->b_datacnt == 0) { 2278ea8dc4b6Seschrock arc_state_t *old_state = hdr->b_state; 2279ea8dc4b6Seschrock arc_state_t *evicted_state; 2280ea8dc4b6Seschrock 2281ea8dc4b6Seschrock ASSERT(refcount_is_zero(&hdr->b_refcnt)); 2282ea8dc4b6Seschrock 2283ea8dc4b6Seschrock evicted_state = 2284*44cb6abcSbmc (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 2285ea8dc4b6Seschrock 2286*44cb6abcSbmc mutex_enter(&old_state->arcs_mtx); 2287*44cb6abcSbmc mutex_enter(&evicted_state->arcs_mtx); 2288ea8dc4b6Seschrock 2289ea8dc4b6Seschrock arc_change_state(evicted_state, hdr, hash_lock); 2290ea8dc4b6Seschrock ASSERT(HDR_IN_HASH_TABLE(hdr)); 2291ea8dc4b6Seschrock hdr->b_flags = ARC_IN_HASH_TABLE; 2292ea8dc4b6Seschrock 2293*44cb6abcSbmc mutex_exit(&evicted_state->arcs_mtx); 2294*44cb6abcSbmc mutex_exit(&old_state->arcs_mtx); 2295ea8dc4b6Seschrock } 2296ea8dc4b6Seschrock mutex_exit(hash_lock); 2297dd6ef538Smaybee 2298ea8dc4b6Seschrock VERIFY(buf->b_efunc(buf) == 0); 2299ea8dc4b6Seschrock buf->b_efunc = NULL; 2300ea8dc4b6Seschrock buf->b_private = NULL; 2301ea8dc4b6Seschrock buf->b_hdr = NULL; 2302ea8dc4b6Seschrock kmem_cache_free(buf_cache, buf); 2303ea8dc4b6Seschrock return (1); 2304ea8dc4b6Seschrock } 2305ea8dc4b6Seschrock 2306fa9e4066Sahrens /* 2307fa9e4066Sahrens * Release this buffer from the cache. This must be done 2308fa9e4066Sahrens * after a read and prior to modifying the buffer contents. 2309fa9e4066Sahrens * If the buffer has more than one reference, we must make 2310fa9e4066Sahrens * make a new hdr for the buffer. 2311fa9e4066Sahrens */ 2312fa9e4066Sahrens void 2313fa9e4066Sahrens arc_release(arc_buf_t *buf, void *tag) 2314fa9e4066Sahrens { 2315fa9e4066Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 2316fa9e4066Sahrens kmutex_t *hash_lock = HDR_LOCK(hdr); 2317fa9e4066Sahrens 2318fa9e4066Sahrens /* this buffer is not on any list */ 2319fa9e4066Sahrens ASSERT(refcount_count(&hdr->b_refcnt) > 0); 2320fa9e4066Sahrens 2321*44cb6abcSbmc if (hdr->b_state == arc_anon) { 2322fa9e4066Sahrens /* this buffer is already released */ 2323fa9e4066Sahrens ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 1); 2324fa9e4066Sahrens ASSERT(BUF_EMPTY(hdr)); 2325ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 23266b4acc8bSahrens arc_buf_thaw(buf); 2327fa9e4066Sahrens return; 2328fa9e4066Sahrens } 2329fa9e4066Sahrens 2330fa9e4066Sahrens mutex_enter(hash_lock); 2331fa9e4066Sahrens 2332ea8dc4b6Seschrock /* 2333ea8dc4b6Seschrock * Do we have more than one buf? 2334ea8dc4b6Seschrock */ 2335ea8dc4b6Seschrock if (hdr->b_buf != buf || buf->b_next != NULL) { 2336fa9e4066Sahrens arc_buf_hdr_t *nhdr; 2337fa9e4066Sahrens arc_buf_t **bufp; 2338fa9e4066Sahrens uint64_t blksz = hdr->b_size; 2339fa9e4066Sahrens spa_t *spa = hdr->b_spa; 2340ad23a2dbSjohansen arc_buf_contents_t type = hdr->b_type; 2341fa9e4066Sahrens 2342ea8dc4b6Seschrock ASSERT(hdr->b_datacnt > 1); 2343fa9e4066Sahrens /* 2344fa9e4066Sahrens * Pull the data off of this buf and attach it to 2345fa9e4066Sahrens * a new anonymous buf. 2346fa9e4066Sahrens */ 2347ea8dc4b6Seschrock (void) remove_reference(hdr, hash_lock, tag); 2348fa9e4066Sahrens bufp = &hdr->b_buf; 2349ea8dc4b6Seschrock while (*bufp != buf) 2350fa9e4066Sahrens bufp = &(*bufp)->b_next; 2351fa9e4066Sahrens *bufp = (*bufp)->b_next; 2352ea8dc4b6Seschrock 2353*44cb6abcSbmc ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size); 2354*44cb6abcSbmc atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size); 2355ea8dc4b6Seschrock if (refcount_is_zero(&hdr->b_refcnt)) { 2356*44cb6abcSbmc ASSERT3U(hdr->b_state->arcs_lsize, >=, hdr->b_size); 2357*44cb6abcSbmc atomic_add_64(&hdr->b_state->arcs_lsize, -hdr->b_size); 2358ea8dc4b6Seschrock } 2359ea8dc4b6Seschrock hdr->b_datacnt -= 1; 2360ea8dc4b6Seschrock 2361fa9e4066Sahrens mutex_exit(hash_lock); 2362fa9e4066Sahrens 2363fa9e4066Sahrens nhdr = kmem_cache_alloc(hdr_cache, KM_SLEEP); 2364fa9e4066Sahrens nhdr->b_size = blksz; 2365fa9e4066Sahrens nhdr->b_spa = spa; 2366ad23a2dbSjohansen nhdr->b_type = type; 2367fa9e4066Sahrens nhdr->b_buf = buf; 2368*44cb6abcSbmc nhdr->b_state = arc_anon; 2369fa9e4066Sahrens nhdr->b_arc_access = 0; 2370fa9e4066Sahrens nhdr->b_flags = 0; 2371ea8dc4b6Seschrock nhdr->b_datacnt = 1; 2372cc60fd72Sahrens if (hdr->b_freeze_cksum != NULL) { 2373cc60fd72Sahrens nhdr->b_freeze_cksum = 2374cc60fd72Sahrens kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP); 2375cc60fd72Sahrens *nhdr->b_freeze_cksum = *hdr->b_freeze_cksum; 2376cc60fd72Sahrens } 2377fa9e4066Sahrens buf->b_hdr = nhdr; 2378fa9e4066Sahrens buf->b_next = NULL; 2379fa9e4066Sahrens (void) refcount_add(&nhdr->b_refcnt, tag); 2380*44cb6abcSbmc atomic_add_64(&arc_anon->arcs_size, blksz); 2381fa9e4066Sahrens 2382fa9e4066Sahrens hdr = nhdr; 2383fa9e4066Sahrens } else { 2384ea8dc4b6Seschrock ASSERT(refcount_count(&hdr->b_refcnt) == 1); 2385fa9e4066Sahrens ASSERT(!list_link_active(&hdr->b_arc_node)); 2386fa9e4066Sahrens ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 2387*44cb6abcSbmc arc_change_state(arc_anon, hdr, hash_lock); 2388fa9e4066Sahrens hdr->b_arc_access = 0; 2389fa9e4066Sahrens mutex_exit(hash_lock); 2390fa9e4066Sahrens bzero(&hdr->b_dva, sizeof (dva_t)); 2391fa9e4066Sahrens hdr->b_birth = 0; 2392fa9e4066Sahrens hdr->b_cksum0 = 0; 2393fa9e4066Sahrens } 2394ea8dc4b6Seschrock buf->b_efunc = NULL; 2395ea8dc4b6Seschrock buf->b_private = NULL; 23966b4acc8bSahrens arc_buf_thaw(buf); 2397fa9e4066Sahrens } 2398fa9e4066Sahrens 2399fa9e4066Sahrens int 2400fa9e4066Sahrens arc_released(arc_buf_t *buf) 2401fa9e4066Sahrens { 2402*44cb6abcSbmc return (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon); 2403ea8dc4b6Seschrock } 2404ea8dc4b6Seschrock 2405ea8dc4b6Seschrock int 2406ea8dc4b6Seschrock arc_has_callback(arc_buf_t *buf) 2407ea8dc4b6Seschrock { 2408ea8dc4b6Seschrock return (buf->b_efunc != NULL); 2409fa9e4066Sahrens } 2410fa9e4066Sahrens 2411ea8dc4b6Seschrock #ifdef ZFS_DEBUG 2412ea8dc4b6Seschrock int 2413ea8dc4b6Seschrock arc_referenced(arc_buf_t *buf) 2414ea8dc4b6Seschrock { 2415ea8dc4b6Seschrock return (refcount_count(&buf->b_hdr->b_refcnt)); 2416ea8dc4b6Seschrock } 2417ea8dc4b6Seschrock #endif 2418ea8dc4b6Seschrock 2419fa9e4066Sahrens static void 2420fa9e4066Sahrens arc_write_done(zio_t *zio) 2421fa9e4066Sahrens { 2422fa9e4066Sahrens arc_buf_t *buf; 2423fa9e4066Sahrens arc_buf_hdr_t *hdr; 2424fa9e4066Sahrens arc_callback_t *acb; 2425fa9e4066Sahrens 2426fa9e4066Sahrens buf = zio->io_private; 2427fa9e4066Sahrens hdr = buf->b_hdr; 2428fa9e4066Sahrens acb = hdr->b_acb; 2429fa9e4066Sahrens hdr->b_acb = NULL; 2430ea8dc4b6Seschrock ASSERT(acb != NULL); 2431fa9e4066Sahrens 2432fa9e4066Sahrens /* this buffer is on no lists and is not in the hash table */ 2433*44cb6abcSbmc ASSERT3P(hdr->b_state, ==, arc_anon); 2434fa9e4066Sahrens 2435fa9e4066Sahrens hdr->b_dva = *BP_IDENTITY(zio->io_bp); 2436fa9e4066Sahrens hdr->b_birth = zio->io_bp->blk_birth; 2437fa9e4066Sahrens hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0]; 2438ea8dc4b6Seschrock /* 2439ea8dc4b6Seschrock * If the block to be written was all-zero, we may have 2440ea8dc4b6Seschrock * compressed it away. In this case no write was performed 2441ea8dc4b6Seschrock * so there will be no dva/birth-date/checksum. The buffer 2442ea8dc4b6Seschrock * must therefor remain anonymous (and uncached). 2443ea8dc4b6Seschrock */ 2444fa9e4066Sahrens if (!BUF_EMPTY(hdr)) { 2445fa9e4066Sahrens arc_buf_hdr_t *exists; 2446fa9e4066Sahrens kmutex_t *hash_lock; 2447fa9e4066Sahrens 24486b4acc8bSahrens arc_cksum_verify(buf); 24496b4acc8bSahrens 2450fa9e4066Sahrens exists = buf_hash_insert(hdr, &hash_lock); 2451fa9e4066Sahrens if (exists) { 2452fa9e4066Sahrens /* 2453fa9e4066Sahrens * This can only happen if we overwrite for 2454fa9e4066Sahrens * sync-to-convergence, because we remove 2455fa9e4066Sahrens * buffers from the hash table when we arc_free(). 2456fa9e4066Sahrens */ 2457fa9e4066Sahrens ASSERT(DVA_EQUAL(BP_IDENTITY(&zio->io_bp_orig), 2458fa9e4066Sahrens BP_IDENTITY(zio->io_bp))); 2459fa9e4066Sahrens ASSERT3U(zio->io_bp_orig.blk_birth, ==, 2460fa9e4066Sahrens zio->io_bp->blk_birth); 2461fa9e4066Sahrens 2462fa9e4066Sahrens ASSERT(refcount_is_zero(&exists->b_refcnt)); 2463*44cb6abcSbmc arc_change_state(arc_anon, exists, hash_lock); 2464fa9e4066Sahrens mutex_exit(hash_lock); 2465ea8dc4b6Seschrock arc_hdr_destroy(exists); 2466fa9e4066Sahrens exists = buf_hash_insert(hdr, &hash_lock); 2467fa9e4066Sahrens ASSERT3P(exists, ==, NULL); 2468fa9e4066Sahrens } 2469ea8dc4b6Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 247044eda4d7Smaybee arc_access(hdr, hash_lock); 247144eda4d7Smaybee mutex_exit(hash_lock); 2472ea8dc4b6Seschrock } else if (acb->acb_done == NULL) { 2473ea8dc4b6Seschrock int destroy_hdr; 2474ea8dc4b6Seschrock /* 2475ea8dc4b6Seschrock * This is an anonymous buffer with no user callback, 2476ea8dc4b6Seschrock * destroy it if there are no active references. 2477ea8dc4b6Seschrock */ 2478ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 2479ea8dc4b6Seschrock destroy_hdr = refcount_is_zero(&hdr->b_refcnt); 2480ea8dc4b6Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2481ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 2482ea8dc4b6Seschrock if (destroy_hdr) 2483ea8dc4b6Seschrock arc_hdr_destroy(hdr); 2484ea8dc4b6Seschrock } else { 2485ea8dc4b6Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2486fa9e4066Sahrens } 2487ea8dc4b6Seschrock 2488ea8dc4b6Seschrock if (acb->acb_done) { 2489fa9e4066Sahrens ASSERT(!refcount_is_zero(&hdr->b_refcnt)); 2490fa9e4066Sahrens acb->acb_done(zio, buf, acb->acb_private); 2491fa9e4066Sahrens } 2492fa9e4066Sahrens 2493ea8dc4b6Seschrock kmem_free(acb, sizeof (arc_callback_t)); 2494fa9e4066Sahrens } 2495fa9e4066Sahrens 2496fa9e4066Sahrens int 249744cd46caSbillm arc_write(zio_t *pio, spa_t *spa, int checksum, int compress, int ncopies, 2498fa9e4066Sahrens uint64_t txg, blkptr_t *bp, arc_buf_t *buf, 2499fa9e4066Sahrens arc_done_func_t *done, void *private, int priority, int flags, 2500ea8dc4b6Seschrock uint32_t arc_flags, zbookmark_t *zb) 2501fa9e4066Sahrens { 2502fa9e4066Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 2503fa9e4066Sahrens arc_callback_t *acb; 2504fa9e4066Sahrens zio_t *rzio; 2505fa9e4066Sahrens 2506fa9e4066Sahrens /* this is a private buffer - no locking required */ 2507*44cb6abcSbmc ASSERT3P(hdr->b_state, ==, arc_anon); 2508fa9e4066Sahrens ASSERT(BUF_EMPTY(hdr)); 2509fa9e4066Sahrens ASSERT(!HDR_IO_ERROR(hdr)); 2510c5c6ffa0Smaybee ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0); 2511c5c6ffa0Smaybee ASSERT(hdr->b_acb == 0); 2512fa9e4066Sahrens acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 2513fa9e4066Sahrens acb->acb_done = done; 2514fa9e4066Sahrens acb->acb_private = private; 2515fa9e4066Sahrens acb->acb_byteswap = (arc_byteswap_func_t *)-1; 2516fa9e4066Sahrens hdr->b_acb = acb; 2517ea8dc4b6Seschrock hdr->b_flags |= ARC_IO_IN_PROGRESS; 25186b4acc8bSahrens arc_cksum_compute(buf); 251944cd46caSbillm rzio = zio_write(pio, spa, checksum, compress, ncopies, txg, bp, 2520ea8dc4b6Seschrock buf->b_data, hdr->b_size, arc_write_done, buf, priority, flags, zb); 2521fa9e4066Sahrens 2522fa9e4066Sahrens if (arc_flags & ARC_WAIT) 2523fa9e4066Sahrens return (zio_wait(rzio)); 2524fa9e4066Sahrens 2525fa9e4066Sahrens ASSERT(arc_flags & ARC_NOWAIT); 2526fa9e4066Sahrens zio_nowait(rzio); 2527fa9e4066Sahrens 2528fa9e4066Sahrens return (0); 2529fa9e4066Sahrens } 2530fa9e4066Sahrens 2531fa9e4066Sahrens int 2532fa9e4066Sahrens arc_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 2533fa9e4066Sahrens zio_done_func_t *done, void *private, uint32_t arc_flags) 2534fa9e4066Sahrens { 2535fa9e4066Sahrens arc_buf_hdr_t *ab; 2536fa9e4066Sahrens kmutex_t *hash_lock; 2537fa9e4066Sahrens zio_t *zio; 2538fa9e4066Sahrens 2539fa9e4066Sahrens /* 2540fa9e4066Sahrens * If this buffer is in the cache, release it, so it 2541fa9e4066Sahrens * can be re-used. 2542fa9e4066Sahrens */ 2543fa9e4066Sahrens ab = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock); 2544fa9e4066Sahrens if (ab != NULL) { 2545fa9e4066Sahrens /* 2546fa9e4066Sahrens * The checksum of blocks to free is not always 2547fa9e4066Sahrens * preserved (eg. on the deadlist). However, if it is 2548fa9e4066Sahrens * nonzero, it should match what we have in the cache. 2549fa9e4066Sahrens */ 2550fa9e4066Sahrens ASSERT(bp->blk_cksum.zc_word[0] == 0 || 2551fa9e4066Sahrens ab->b_cksum0 == bp->blk_cksum.zc_word[0]); 2552*44cb6abcSbmc if (ab->b_state != arc_anon) 2553*44cb6abcSbmc arc_change_state(arc_anon, ab, hash_lock); 255413506d1eSmaybee if (HDR_IO_IN_PROGRESS(ab)) { 255513506d1eSmaybee /* 255613506d1eSmaybee * This should only happen when we prefetch. 255713506d1eSmaybee */ 255813506d1eSmaybee ASSERT(ab->b_flags & ARC_PREFETCH); 255913506d1eSmaybee ASSERT3U(ab->b_datacnt, ==, 1); 256013506d1eSmaybee ab->b_flags |= ARC_FREED_IN_READ; 256113506d1eSmaybee if (HDR_IN_HASH_TABLE(ab)) 256213506d1eSmaybee buf_hash_remove(ab); 256313506d1eSmaybee ab->b_arc_access = 0; 256413506d1eSmaybee bzero(&ab->b_dva, sizeof (dva_t)); 256513506d1eSmaybee ab->b_birth = 0; 256613506d1eSmaybee ab->b_cksum0 = 0; 256713506d1eSmaybee ab->b_buf->b_efunc = NULL; 256813506d1eSmaybee ab->b_buf->b_private = NULL; 256913506d1eSmaybee mutex_exit(hash_lock); 257013506d1eSmaybee } else if (refcount_is_zero(&ab->b_refcnt)) { 2571fa9e4066Sahrens mutex_exit(hash_lock); 2572ea8dc4b6Seschrock arc_hdr_destroy(ab); 2573*44cb6abcSbmc ARCSTAT_BUMP(arcstat_deleted); 2574fa9e4066Sahrens } else { 2575bbf4a8dfSmaybee /* 257613506d1eSmaybee * We still have an active reference on this 257713506d1eSmaybee * buffer. This can happen, e.g., from 257813506d1eSmaybee * dbuf_unoverride(). 2579bbf4a8dfSmaybee */ 258013506d1eSmaybee ASSERT(!HDR_IN_HASH_TABLE(ab)); 2581fa9e4066Sahrens ab->b_arc_access = 0; 2582fa9e4066Sahrens bzero(&ab->b_dva, sizeof (dva_t)); 2583fa9e4066Sahrens ab->b_birth = 0; 2584fa9e4066Sahrens ab->b_cksum0 = 0; 2585ea8dc4b6Seschrock ab->b_buf->b_efunc = NULL; 2586ea8dc4b6Seschrock ab->b_buf->b_private = NULL; 2587fa9e4066Sahrens mutex_exit(hash_lock); 2588fa9e4066Sahrens } 2589fa9e4066Sahrens } 2590fa9e4066Sahrens 2591fa9e4066Sahrens zio = zio_free(pio, spa, txg, bp, done, private); 2592fa9e4066Sahrens 2593fa9e4066Sahrens if (arc_flags & ARC_WAIT) 2594fa9e4066Sahrens return (zio_wait(zio)); 2595fa9e4066Sahrens 2596fa9e4066Sahrens ASSERT(arc_flags & ARC_NOWAIT); 2597fa9e4066Sahrens zio_nowait(zio); 2598fa9e4066Sahrens 2599fa9e4066Sahrens return (0); 2600fa9e4066Sahrens } 2601fa9e4066Sahrens 2602fa9e4066Sahrens void 2603fa9e4066Sahrens arc_tempreserve_clear(uint64_t tempreserve) 2604fa9e4066Sahrens { 2605fa9e4066Sahrens atomic_add_64(&arc_tempreserve, -tempreserve); 2606fa9e4066Sahrens ASSERT((int64_t)arc_tempreserve >= 0); 2607fa9e4066Sahrens } 2608fa9e4066Sahrens 2609fa9e4066Sahrens int 2610fa9e4066Sahrens arc_tempreserve_space(uint64_t tempreserve) 2611fa9e4066Sahrens { 2612fa9e4066Sahrens #ifdef ZFS_DEBUG 2613fa9e4066Sahrens /* 2614fa9e4066Sahrens * Once in a while, fail for no reason. Everything should cope. 2615fa9e4066Sahrens */ 2616fa9e4066Sahrens if (spa_get_random(10000) == 0) { 2617fa9e4066Sahrens dprintf("forcing random failure\n"); 2618fa9e4066Sahrens return (ERESTART); 2619fa9e4066Sahrens } 2620fa9e4066Sahrens #endif 2621*44cb6abcSbmc if (tempreserve > arc_c/4 && !arc_no_grow) 2622*44cb6abcSbmc arc_c = MIN(arc_c_max, tempreserve * 4); 2623*44cb6abcSbmc if (tempreserve > arc_c) 2624112fe045Smaybee return (ENOMEM); 2625112fe045Smaybee 2626fa9e4066Sahrens /* 2627112fe045Smaybee * Throttle writes when the amount of dirty data in the cache 2628112fe045Smaybee * gets too large. We try to keep the cache less than half full 2629112fe045Smaybee * of dirty blocks so that our sync times don't grow too large. 2630112fe045Smaybee * Note: if two requests come in concurrently, we might let them 2631112fe045Smaybee * both succeed, when one of them should fail. Not a huge deal. 2632112fe045Smaybee * 2633112fe045Smaybee * XXX The limit should be adjusted dynamically to keep the time 2634112fe045Smaybee * to sync a dataset fixed (around 1-5 seconds?). 2635fa9e4066Sahrens */ 2636fa9e4066Sahrens 2637*44cb6abcSbmc if (tempreserve + arc_tempreserve + arc_anon->arcs_size > arc_c / 2 && 2638*44cb6abcSbmc arc_tempreserve + arc_anon->arcs_size > arc_c / 4) { 2639fa9e4066Sahrens dprintf("failing, arc_tempreserve=%lluK anon=%lluK " 2640*44cb6abcSbmc "tempreserve=%lluK arc_c=%lluK\n", 2641*44cb6abcSbmc arc_tempreserve>>10, arc_anon->arcs_lsize>>10, 2642*44cb6abcSbmc tempreserve>>10, arc_c>>10); 2643fa9e4066Sahrens return (ERESTART); 2644fa9e4066Sahrens } 2645fa9e4066Sahrens atomic_add_64(&arc_tempreserve, tempreserve); 2646fa9e4066Sahrens return (0); 2647fa9e4066Sahrens } 2648fa9e4066Sahrens 2649fa9e4066Sahrens void 2650fa9e4066Sahrens arc_init(void) 2651fa9e4066Sahrens { 2652fa9e4066Sahrens mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL); 2653fa9e4066Sahrens cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL); 2654fa9e4066Sahrens 265513506d1eSmaybee /* Convert seconds to clock ticks */ 2656b19a79ecSperrin arc_min_prefetch_lifespan = 1 * hz; 265713506d1eSmaybee 2658fa9e4066Sahrens /* Start out with 1/8 of all memory */ 2659*44cb6abcSbmc arc_c = physmem * PAGESIZE / 8; 2660fa9e4066Sahrens 2661fa9e4066Sahrens #ifdef _KERNEL 2662fa9e4066Sahrens /* 2663fa9e4066Sahrens * On architectures where the physical memory can be larger 2664fa9e4066Sahrens * than the addressable space (intel in 32-bit mode), we may 2665fa9e4066Sahrens * need to limit the cache to 1/8 of VM size. 2666fa9e4066Sahrens */ 2667*44cb6abcSbmc arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8); 2668fa9e4066Sahrens #endif 2669fa9e4066Sahrens 2670112fe045Smaybee /* set min cache to 1/32 of all memory, or 64MB, whichever is more */ 2671*44cb6abcSbmc arc_c_min = MAX(arc_c / 4, 64<<20); 2672112fe045Smaybee /* set max to 3/4 of all memory, or all but 1GB, whichever is more */ 2673*44cb6abcSbmc if (arc_c * 8 >= 1<<30) 2674*44cb6abcSbmc arc_c_max = (arc_c * 8) - (1<<30); 2675fa9e4066Sahrens else 2676*44cb6abcSbmc arc_c_max = arc_c_min; 2677*44cb6abcSbmc arc_c_max = MAX(arc_c * 6, arc_c_max); 2678a2eea2e1Sahrens 2679a2eea2e1Sahrens /* 2680a2eea2e1Sahrens * Allow the tunables to override our calculations if they are 2681a2eea2e1Sahrens * reasonable (ie. over 64MB) 2682a2eea2e1Sahrens */ 2683a2eea2e1Sahrens if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE) 2684*44cb6abcSbmc arc_c_max = zfs_arc_max; 2685*44cb6abcSbmc if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max) 2686*44cb6abcSbmc arc_c_min = zfs_arc_min; 2687a2eea2e1Sahrens 2688*44cb6abcSbmc arc_c = arc_c_max; 2689*44cb6abcSbmc arc_p = (arc_c >> 1); 2690fa9e4066Sahrens 2691fa9e4066Sahrens /* if kmem_flags are set, lets try to use less memory */ 2692fa9e4066Sahrens if (kmem_debugging()) 2693*44cb6abcSbmc arc_c = arc_c / 2; 2694*44cb6abcSbmc if (arc_c < arc_c_min) 2695*44cb6abcSbmc arc_c = arc_c_min; 2696*44cb6abcSbmc 2697*44cb6abcSbmc arc_anon = &ARC_anon; 2698*44cb6abcSbmc arc_mru = &ARC_mru; 2699*44cb6abcSbmc arc_mru_ghost = &ARC_mru_ghost; 2700*44cb6abcSbmc arc_mfu = &ARC_mfu; 2701*44cb6abcSbmc arc_mfu_ghost = &ARC_mfu_ghost; 2702*44cb6abcSbmc arc_size = 0; 2703*44cb6abcSbmc 2704*44cb6abcSbmc mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 2705*44cb6abcSbmc mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 2706*44cb6abcSbmc mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 2707*44cb6abcSbmc mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 2708*44cb6abcSbmc mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 2709*44cb6abcSbmc 2710*44cb6abcSbmc list_create(&arc_mru->arcs_list, sizeof (arc_buf_hdr_t), 2711fa9e4066Sahrens offsetof(arc_buf_hdr_t, b_arc_node)); 2712*44cb6abcSbmc list_create(&arc_mru_ghost->arcs_list, sizeof (arc_buf_hdr_t), 2713fa9e4066Sahrens offsetof(arc_buf_hdr_t, b_arc_node)); 2714*44cb6abcSbmc list_create(&arc_mfu->arcs_list, sizeof (arc_buf_hdr_t), 2715fa9e4066Sahrens offsetof(arc_buf_hdr_t, b_arc_node)); 2716*44cb6abcSbmc list_create(&arc_mfu_ghost->arcs_list, sizeof (arc_buf_hdr_t), 2717fa9e4066Sahrens offsetof(arc_buf_hdr_t, b_arc_node)); 2718fa9e4066Sahrens 2719fa9e4066Sahrens buf_init(); 2720fa9e4066Sahrens 2721fa9e4066Sahrens arc_thread_exit = 0; 2722ea8dc4b6Seschrock arc_eviction_list = NULL; 2723ea8dc4b6Seschrock mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL); 272440d7d650Smaybee bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t)); 2725fa9e4066Sahrens 2726*44cb6abcSbmc arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED, 2727*44cb6abcSbmc sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 2728*44cb6abcSbmc 2729*44cb6abcSbmc if (arc_ksp != NULL) { 2730*44cb6abcSbmc arc_ksp->ks_data = &arc_stats; 2731*44cb6abcSbmc kstat_install(arc_ksp); 2732*44cb6abcSbmc } 2733*44cb6abcSbmc 2734fa9e4066Sahrens (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0, 2735fa9e4066Sahrens TS_RUN, minclsyspri); 273649e3519aSmaybee 273749e3519aSmaybee arc_dead = FALSE; 2738fa9e4066Sahrens } 2739fa9e4066Sahrens 2740fa9e4066Sahrens void 2741fa9e4066Sahrens arc_fini(void) 2742fa9e4066Sahrens { 2743fa9e4066Sahrens mutex_enter(&arc_reclaim_thr_lock); 2744fa9e4066Sahrens arc_thread_exit = 1; 2745fa9e4066Sahrens while (arc_thread_exit != 0) 2746fa9e4066Sahrens cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock); 2747fa9e4066Sahrens mutex_exit(&arc_reclaim_thr_lock); 2748fa9e4066Sahrens 2749fa9e4066Sahrens arc_flush(); 2750fa9e4066Sahrens 2751fa9e4066Sahrens arc_dead = TRUE; 2752fa9e4066Sahrens 2753*44cb6abcSbmc if (arc_ksp != NULL) { 2754*44cb6abcSbmc kstat_delete(arc_ksp); 2755*44cb6abcSbmc arc_ksp = NULL; 2756*44cb6abcSbmc } 2757*44cb6abcSbmc 2758ea8dc4b6Seschrock mutex_destroy(&arc_eviction_mtx); 2759fa9e4066Sahrens mutex_destroy(&arc_reclaim_thr_lock); 2760fa9e4066Sahrens cv_destroy(&arc_reclaim_thr_cv); 2761fa9e4066Sahrens 2762*44cb6abcSbmc list_destroy(&arc_mru->arcs_list); 2763*44cb6abcSbmc list_destroy(&arc_mru_ghost->arcs_list); 2764*44cb6abcSbmc list_destroy(&arc_mfu->arcs_list); 2765*44cb6abcSbmc list_destroy(&arc_mfu_ghost->arcs_list); 2766fa9e4066Sahrens 2767*44cb6abcSbmc mutex_destroy(&arc_anon->arcs_mtx); 2768*44cb6abcSbmc mutex_destroy(&arc_mru->arcs_mtx); 2769*44cb6abcSbmc mutex_destroy(&arc_mru_ghost->arcs_mtx); 2770*44cb6abcSbmc mutex_destroy(&arc_mfu->arcs_mtx); 2771*44cb6abcSbmc mutex_destroy(&arc_mfu_ghost->arcs_mtx); 27725ad82045Snd 2773fa9e4066Sahrens buf_fini(); 2774fa9e4066Sahrens } 2775