1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5033f9833Sek * Common Development and Distribution License (the "License"). 6033f9833Sek * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 223f9d6ad7SLin Ling * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23e9103aaeSGarrett D'Amore * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 2494dd93aeSGeorge Wilson * Copyright (c) 2012 by Delphix. All rights reserved. 25fa9e4066Sahrens */ 26fa9e4066Sahrens 27fa9e4066Sahrens /* 2844cb6abcSbmc * DVA-based Adjustable Replacement Cache 29fa9e4066Sahrens * 30ea8dc4b6Seschrock * While much of the theory of operation used here is 31ea8dc4b6Seschrock * based on the self-tuning, low overhead replacement cache 32fa9e4066Sahrens * presented by Megiddo and Modha at FAST 2003, there are some 33fa9e4066Sahrens * significant differences: 34fa9e4066Sahrens * 35fa9e4066Sahrens * 1. The Megiddo and Modha model assumes any page is evictable. 36fa9e4066Sahrens * Pages in its cache cannot be "locked" into memory. This makes 37fa9e4066Sahrens * the eviction algorithm simple: evict the last page in the list. 38fa9e4066Sahrens * This also make the performance characteristics easy to reason 39fa9e4066Sahrens * about. Our cache is not so simple. At any given moment, some 40fa9e4066Sahrens * subset of the blocks in the cache are un-evictable because we 41fa9e4066Sahrens * have handed out a reference to them. Blocks are only evictable 42fa9e4066Sahrens * when there are no external references active. This makes 43fa9e4066Sahrens * eviction far more problematic: we choose to evict the evictable 44fa9e4066Sahrens * blocks that are the "lowest" in the list. 45fa9e4066Sahrens * 46fa9e4066Sahrens * There are times when it is not possible to evict the requested 47fa9e4066Sahrens * space. In these circumstances we are unable to adjust the cache 48fa9e4066Sahrens * size. To prevent the cache growing unbounded at these times we 49fa94a07fSbrendan * implement a "cache throttle" that slows the flow of new data 50fa94a07fSbrendan * into the cache until we can make space available. 51fa9e4066Sahrens * 52fa9e4066Sahrens * 2. The Megiddo and Modha model assumes a fixed cache size. 53fa9e4066Sahrens * Pages are evicted when the cache is full and there is a cache 54fa9e4066Sahrens * miss. Our model has a variable sized cache. It grows with 55fa94a07fSbrendan * high use, but also tries to react to memory pressure from the 56fa9e4066Sahrens * operating system: decreasing its size when system memory is 57fa9e4066Sahrens * tight. 58fa9e4066Sahrens * 59fa9e4066Sahrens * 3. The Megiddo and Modha model assumes a fixed page size. All 60fa9e4066Sahrens * elements of the cache are therefor exactly the same size. So 61fa9e4066Sahrens * when adjusting the cache size following a cache miss, its simply 62fa9e4066Sahrens * a matter of choosing a single page to evict. In our model, we 63fa9e4066Sahrens * have variable sized cache blocks (rangeing from 512 bytes to 64fa9e4066Sahrens * 128K bytes). We therefor choose a set of blocks to evict to make 65fa9e4066Sahrens * space for a cache miss that approximates as closely as possible 66fa9e4066Sahrens * the space used by the new block. 67fa9e4066Sahrens * 68fa9e4066Sahrens * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache" 69fa9e4066Sahrens * by N. Megiddo & D. Modha, FAST 2003 70fa9e4066Sahrens */ 71fa9e4066Sahrens 72fa9e4066Sahrens /* 73fa9e4066Sahrens * The locking model: 74fa9e4066Sahrens * 75fa9e4066Sahrens * A new reference to a cache buffer can be obtained in two 76fa9e4066Sahrens * ways: 1) via a hash table lookup using the DVA as a key, 77fa94a07fSbrendan * or 2) via one of the ARC lists. The arc_read() interface 78fa9e4066Sahrens * uses method 1, while the internal arc algorithms for 79fa9e4066Sahrens * adjusting the cache use method 2. We therefor provide two 80fa9e4066Sahrens * types of locks: 1) the hash table lock array, and 2) the 81fa9e4066Sahrens * arc list locks. 82fa9e4066Sahrens * 83fc98fea5SBart Coddens * Buffers do not have their own mutexes, rather they rely on the 84fc98fea5SBart Coddens * hash table mutexes for the bulk of their protection (i.e. most 85fc98fea5SBart Coddens * fields in the arc_buf_hdr_t are protected by these mutexes). 86fa9e4066Sahrens * 87fa9e4066Sahrens * buf_hash_find() returns the appropriate mutex (held) when it 88fa9e4066Sahrens * locates the requested buffer in the hash table. It returns 89fa9e4066Sahrens * NULL for the mutex if the buffer was not in the table. 90fa9e4066Sahrens * 91fa9e4066Sahrens * buf_hash_remove() expects the appropriate hash mutex to be 92fa9e4066Sahrens * already held before it is invoked. 93fa9e4066Sahrens * 94fa9e4066Sahrens * Each arc state also has a mutex which is used to protect the 95fa9e4066Sahrens * buffer list associated with the state. When attempting to 96fa9e4066Sahrens * obtain a hash table lock while holding an arc list lock you 97fa9e4066Sahrens * must use: mutex_tryenter() to avoid deadlock. Also note that 9844eda4d7Smaybee * the active state mutex must be held before the ghost state mutex. 99fa9e4066Sahrens * 100ea8dc4b6Seschrock * Arc buffers may have an associated eviction callback function. 101ea8dc4b6Seschrock * This function will be invoked prior to removing the buffer (e.g. 102ea8dc4b6Seschrock * in arc_do_user_evicts()). Note however that the data associated 103ea8dc4b6Seschrock * with the buffer may be evicted prior to the callback. The callback 104ea8dc4b6Seschrock * must be made with *no locks held* (to prevent deadlock). Additionally, 105ea8dc4b6Seschrock * the users of callbacks must ensure that their private data is 106ea8dc4b6Seschrock * protected from simultaneous callbacks from arc_buf_evict() 107ea8dc4b6Seschrock * and arc_do_user_evicts(). 108ea8dc4b6Seschrock * 109fa9e4066Sahrens * Note that the majority of the performance stats are manipulated 110fa9e4066Sahrens * with atomic operations. 111fa94a07fSbrendan * 112fa94a07fSbrendan * The L2ARC uses the l2arc_buflist_mtx global mutex for the following: 113fa94a07fSbrendan * 114fa94a07fSbrendan * - L2ARC buflist creation 115fa94a07fSbrendan * - L2ARC buflist eviction 116fa94a07fSbrendan * - L2ARC write completion, which walks L2ARC buflists 117fa94a07fSbrendan * - ARC header destruction, as it removes from L2ARC buflists 118fa94a07fSbrendan * - ARC header release, as it removes from L2ARC buflists 119fa9e4066Sahrens */ 120fa9e4066Sahrens 121fa9e4066Sahrens #include <sys/spa.h> 122fa9e4066Sahrens #include <sys/zio.h> 123fa9e4066Sahrens #include <sys/zfs_context.h> 124fa9e4066Sahrens #include <sys/arc.h> 125fa9e4066Sahrens #include <sys/refcount.h> 126c5904d13Seschrock #include <sys/vdev.h> 127573ca77eSGeorge Wilson #include <sys/vdev_impl.h> 128fa9e4066Sahrens #ifdef _KERNEL 129fa9e4066Sahrens #include <sys/vmsystm.h> 130fa9e4066Sahrens #include <vm/anon.h> 131fa9e4066Sahrens #include <sys/fs/swapnode.h> 132033f9833Sek #include <sys/dnlc.h> 133fa9e4066Sahrens #endif 134fa9e4066Sahrens #include <sys/callb.h> 13544cb6abcSbmc #include <sys/kstat.h> 136b24ab676SJeff Bonwick #include <zfs_fletcher.h> 137fa9e4066Sahrens 138*cd1c8b85SMatthew Ahrens #ifndef _KERNEL 139*cd1c8b85SMatthew Ahrens /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */ 140*cd1c8b85SMatthew Ahrens boolean_t arc_watch = B_FALSE; 141*cd1c8b85SMatthew Ahrens int arc_procfd; 142*cd1c8b85SMatthew Ahrens #endif 143*cd1c8b85SMatthew Ahrens 144fa9e4066Sahrens static kmutex_t arc_reclaim_thr_lock; 145fa9e4066Sahrens static kcondvar_t arc_reclaim_thr_cv; /* used to signal reclaim thr */ 146fa9e4066Sahrens static uint8_t arc_thread_exit; 147fa9e4066Sahrens 1481ab7f2deSmaybee extern int zfs_write_limit_shift; 1491ab7f2deSmaybee extern uint64_t zfs_write_limit_max; 15005715f94SMark Maybee extern kmutex_t zfs_write_limit_lock; 1511ab7f2deSmaybee 152033f9833Sek #define ARC_REDUCE_DNLC_PERCENT 3 153033f9833Sek uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT; 154033f9833Sek 155fa9e4066Sahrens typedef enum arc_reclaim_strategy { 156fa9e4066Sahrens ARC_RECLAIM_AGGR, /* Aggressive reclaim strategy */ 157fa9e4066Sahrens ARC_RECLAIM_CONS /* Conservative reclaim strategy */ 158fa9e4066Sahrens } arc_reclaim_strategy_t; 159fa9e4066Sahrens 160fa9e4066Sahrens /* number of seconds before growing cache again */ 161fa9e4066Sahrens static int arc_grow_retry = 60; 162fa9e4066Sahrens 1635a98e54bSBrendan Gregg - Sun Microsystems /* shift of arc_c for calculating both min and max arc_p */ 1645a98e54bSBrendan Gregg - Sun Microsystems static int arc_p_min_shift = 4; 1655a98e54bSBrendan Gregg - Sun Microsystems 1665a98e54bSBrendan Gregg - Sun Microsystems /* log2(fraction of arc to reclaim) */ 1675a98e54bSBrendan Gregg - Sun Microsystems static int arc_shrink_shift = 5; 1685a98e54bSBrendan Gregg - Sun Microsystems 16913506d1eSmaybee /* 170b19a79ecSperrin * minimum lifespan of a prefetch block in clock ticks 171b19a79ecSperrin * (initialized in arc_init()) 17213506d1eSmaybee */ 173b19a79ecSperrin static int arc_min_prefetch_lifespan; 17413506d1eSmaybee 175fa9e4066Sahrens static int arc_dead; 176fa9e4066Sahrens 1773a737e0dSbrendan /* 1783a737e0dSbrendan * The arc has filled available memory and has now warmed up. 1793a737e0dSbrendan */ 1803a737e0dSbrendan static boolean_t arc_warm; 1813a737e0dSbrendan 182a2eea2e1Sahrens /* 183a2eea2e1Sahrens * These tunables are for performance analysis. 184a2eea2e1Sahrens */ 185a2eea2e1Sahrens uint64_t zfs_arc_max; 186a2eea2e1Sahrens uint64_t zfs_arc_min; 1871116048bSek uint64_t zfs_arc_meta_limit = 0; 1885a98e54bSBrendan Gregg - Sun Microsystems int zfs_arc_grow_retry = 0; 1895a98e54bSBrendan Gregg - Sun Microsystems int zfs_arc_shrink_shift = 0; 1905a98e54bSBrendan Gregg - Sun Microsystems int zfs_arc_p_min_shift = 0; 191a2eea2e1Sahrens 192fa9e4066Sahrens /* 193fa94a07fSbrendan * Note that buffers can be in one of 6 states: 194fa9e4066Sahrens * ARC_anon - anonymous (discussed below) 195ea8dc4b6Seschrock * ARC_mru - recently used, currently cached 196ea8dc4b6Seschrock * ARC_mru_ghost - recentely used, no longer in cache 197ea8dc4b6Seschrock * ARC_mfu - frequently used, currently cached 198ea8dc4b6Seschrock * ARC_mfu_ghost - frequently used, no longer in cache 199fa94a07fSbrendan * ARC_l2c_only - exists in L2ARC but not other states 2000e8c6158Smaybee * When there are no active references to the buffer, they are 2010e8c6158Smaybee * are linked onto a list in one of these arc states. These are 2020e8c6158Smaybee * the only buffers that can be evicted or deleted. Within each 2030e8c6158Smaybee * state there are multiple lists, one for meta-data and one for 2040e8c6158Smaybee * non-meta-data. Meta-data (indirect blocks, blocks of dnodes, 2050e8c6158Smaybee * etc.) is tracked separately so that it can be managed more 206fa94a07fSbrendan * explicitly: favored over data, limited explicitly. 207fa9e4066Sahrens * 208fa9e4066Sahrens * Anonymous buffers are buffers that are not associated with 209fa9e4066Sahrens * a DVA. These are buffers that hold dirty block copies 210fa9e4066Sahrens * before they are written to stable storage. By definition, 211ea8dc4b6Seschrock * they are "ref'd" and are considered part of arc_mru 212fa9e4066Sahrens * that cannot be freed. Generally, they will aquire a DVA 213ea8dc4b6Seschrock * as they are written and migrate onto the arc_mru list. 214fa94a07fSbrendan * 215fa94a07fSbrendan * The ARC_l2c_only state is for buffers that are in the second 216fa94a07fSbrendan * level ARC but no longer in any of the ARC_m* lists. The second 217fa94a07fSbrendan * level ARC itself may also contain buffers that are in any of 218fa94a07fSbrendan * the ARC_m* states - meaning that a buffer can exist in two 219fa94a07fSbrendan * places. The reason for the ARC_l2c_only state is to keep the 220fa94a07fSbrendan * buffer header in the hash table, so that reads that hit the 221fa94a07fSbrendan * second level ARC benefit from these fast lookups. 222fa9e4066Sahrens */ 223fa9e4066Sahrens 224fa9e4066Sahrens typedef struct arc_state { 2250e8c6158Smaybee list_t arcs_list[ARC_BUFC_NUMTYPES]; /* list of evictable buffers */ 2260e8c6158Smaybee uint64_t arcs_lsize[ARC_BUFC_NUMTYPES]; /* amount of evictable data */ 2270e8c6158Smaybee uint64_t arcs_size; /* total amount of data in this state */ 22844cb6abcSbmc kmutex_t arcs_mtx; 229fa9e4066Sahrens } arc_state_t; 230fa9e4066Sahrens 231fa94a07fSbrendan /* The 6 states: */ 232fa9e4066Sahrens static arc_state_t ARC_anon; 233ea8dc4b6Seschrock static arc_state_t ARC_mru; 234ea8dc4b6Seschrock static arc_state_t ARC_mru_ghost; 235ea8dc4b6Seschrock static arc_state_t ARC_mfu; 236ea8dc4b6Seschrock static arc_state_t ARC_mfu_ghost; 237fa94a07fSbrendan static arc_state_t ARC_l2c_only; 238fa9e4066Sahrens 23944cb6abcSbmc typedef struct arc_stats { 24044cb6abcSbmc kstat_named_t arcstat_hits; 24144cb6abcSbmc kstat_named_t arcstat_misses; 24244cb6abcSbmc kstat_named_t arcstat_demand_data_hits; 24344cb6abcSbmc kstat_named_t arcstat_demand_data_misses; 24444cb6abcSbmc kstat_named_t arcstat_demand_metadata_hits; 24544cb6abcSbmc kstat_named_t arcstat_demand_metadata_misses; 24644cb6abcSbmc kstat_named_t arcstat_prefetch_data_hits; 24744cb6abcSbmc kstat_named_t arcstat_prefetch_data_misses; 24844cb6abcSbmc kstat_named_t arcstat_prefetch_metadata_hits; 24944cb6abcSbmc kstat_named_t arcstat_prefetch_metadata_misses; 25044cb6abcSbmc kstat_named_t arcstat_mru_hits; 25144cb6abcSbmc kstat_named_t arcstat_mru_ghost_hits; 25244cb6abcSbmc kstat_named_t arcstat_mfu_hits; 25344cb6abcSbmc kstat_named_t arcstat_mfu_ghost_hits; 25444cb6abcSbmc kstat_named_t arcstat_deleted; 25544cb6abcSbmc kstat_named_t arcstat_recycle_miss; 25644cb6abcSbmc kstat_named_t arcstat_mutex_miss; 25744cb6abcSbmc kstat_named_t arcstat_evict_skip; 2585ea40c06SBrendan Gregg - Sun Microsystems kstat_named_t arcstat_evict_l2_cached; 2595ea40c06SBrendan Gregg - Sun Microsystems kstat_named_t arcstat_evict_l2_eligible; 2605ea40c06SBrendan Gregg - Sun Microsystems kstat_named_t arcstat_evict_l2_ineligible; 26144cb6abcSbmc kstat_named_t arcstat_hash_elements; 26244cb6abcSbmc kstat_named_t arcstat_hash_elements_max; 26344cb6abcSbmc kstat_named_t arcstat_hash_collisions; 26444cb6abcSbmc kstat_named_t arcstat_hash_chains; 26544cb6abcSbmc kstat_named_t arcstat_hash_chain_max; 26644cb6abcSbmc kstat_named_t arcstat_p; 26744cb6abcSbmc kstat_named_t arcstat_c; 26844cb6abcSbmc kstat_named_t arcstat_c_min; 26944cb6abcSbmc kstat_named_t arcstat_c_max; 27044cb6abcSbmc kstat_named_t arcstat_size; 271fa94a07fSbrendan kstat_named_t arcstat_hdr_size; 2725a98e54bSBrendan Gregg - Sun Microsystems kstat_named_t arcstat_data_size; 2735a98e54bSBrendan Gregg - Sun Microsystems kstat_named_t arcstat_other_size; 274fa94a07fSbrendan kstat_named_t arcstat_l2_hits; 275fa94a07fSbrendan kstat_named_t arcstat_l2_misses; 276fa94a07fSbrendan kstat_named_t arcstat_l2_feeds; 277fa94a07fSbrendan kstat_named_t arcstat_l2_rw_clash; 2785a98e54bSBrendan Gregg - Sun Microsystems kstat_named_t arcstat_l2_read_bytes; 2795a98e54bSBrendan Gregg - Sun Microsystems kstat_named_t arcstat_l2_write_bytes; 280fa94a07fSbrendan kstat_named_t arcstat_l2_writes_sent; 281fa94a07fSbrendan kstat_named_t arcstat_l2_writes_done; 282fa94a07fSbrendan kstat_named_t arcstat_l2_writes_error; 283fa94a07fSbrendan kstat_named_t arcstat_l2_writes_hdr_miss; 284fa94a07fSbrendan kstat_named_t arcstat_l2_evict_lock_retry; 285fa94a07fSbrendan kstat_named_t arcstat_l2_evict_reading; 286fa94a07fSbrendan kstat_named_t arcstat_l2_free_on_write; 287fa94a07fSbrendan kstat_named_t arcstat_l2_abort_lowmem; 288fa94a07fSbrendan kstat_named_t arcstat_l2_cksum_bad; 289fa94a07fSbrendan kstat_named_t arcstat_l2_io_error; 290fa94a07fSbrendan kstat_named_t arcstat_l2_size; 291fa94a07fSbrendan kstat_named_t arcstat_l2_hdr_size; 2921ab7f2deSmaybee kstat_named_t arcstat_memory_throttle_count; 29344cb6abcSbmc } arc_stats_t; 29444cb6abcSbmc 29544cb6abcSbmc static arc_stats_t arc_stats = { 29644cb6abcSbmc { "hits", KSTAT_DATA_UINT64 }, 29744cb6abcSbmc { "misses", KSTAT_DATA_UINT64 }, 29844cb6abcSbmc { "demand_data_hits", KSTAT_DATA_UINT64 }, 29944cb6abcSbmc { "demand_data_misses", KSTAT_DATA_UINT64 }, 30044cb6abcSbmc { "demand_metadata_hits", KSTAT_DATA_UINT64 }, 30144cb6abcSbmc { "demand_metadata_misses", KSTAT_DATA_UINT64 }, 30244cb6abcSbmc { "prefetch_data_hits", KSTAT_DATA_UINT64 }, 30344cb6abcSbmc { "prefetch_data_misses", KSTAT_DATA_UINT64 }, 30444cb6abcSbmc { "prefetch_metadata_hits", KSTAT_DATA_UINT64 }, 30544cb6abcSbmc { "prefetch_metadata_misses", KSTAT_DATA_UINT64 }, 30644cb6abcSbmc { "mru_hits", KSTAT_DATA_UINT64 }, 30744cb6abcSbmc { "mru_ghost_hits", KSTAT_DATA_UINT64 }, 30844cb6abcSbmc { "mfu_hits", KSTAT_DATA_UINT64 }, 30944cb6abcSbmc { "mfu_ghost_hits", KSTAT_DATA_UINT64 }, 31044cb6abcSbmc { "deleted", KSTAT_DATA_UINT64 }, 31144cb6abcSbmc { "recycle_miss", KSTAT_DATA_UINT64 }, 31244cb6abcSbmc { "mutex_miss", KSTAT_DATA_UINT64 }, 31344cb6abcSbmc { "evict_skip", KSTAT_DATA_UINT64 }, 3145ea40c06SBrendan Gregg - Sun Microsystems { "evict_l2_cached", KSTAT_DATA_UINT64 }, 3155ea40c06SBrendan Gregg - Sun Microsystems { "evict_l2_eligible", KSTAT_DATA_UINT64 }, 3165ea40c06SBrendan Gregg - Sun Microsystems { "evict_l2_ineligible", KSTAT_DATA_UINT64 }, 31744cb6abcSbmc { "hash_elements", KSTAT_DATA_UINT64 }, 31844cb6abcSbmc { "hash_elements_max", KSTAT_DATA_UINT64 }, 31944cb6abcSbmc { "hash_collisions", KSTAT_DATA_UINT64 }, 32044cb6abcSbmc { "hash_chains", KSTAT_DATA_UINT64 }, 32144cb6abcSbmc { "hash_chain_max", KSTAT_DATA_UINT64 }, 32244cb6abcSbmc { "p", KSTAT_DATA_UINT64 }, 32344cb6abcSbmc { "c", KSTAT_DATA_UINT64 }, 32444cb6abcSbmc { "c_min", KSTAT_DATA_UINT64 }, 32544cb6abcSbmc { "c_max", KSTAT_DATA_UINT64 }, 326fa94a07fSbrendan { "size", KSTAT_DATA_UINT64 }, 327fa94a07fSbrendan { "hdr_size", KSTAT_DATA_UINT64 }, 3285a98e54bSBrendan Gregg - Sun Microsystems { "data_size", KSTAT_DATA_UINT64 }, 3295a98e54bSBrendan Gregg - Sun Microsystems { "other_size", KSTAT_DATA_UINT64 }, 330fa94a07fSbrendan { "l2_hits", KSTAT_DATA_UINT64 }, 331fa94a07fSbrendan { "l2_misses", KSTAT_DATA_UINT64 }, 332fa94a07fSbrendan { "l2_feeds", KSTAT_DATA_UINT64 }, 333fa94a07fSbrendan { "l2_rw_clash", KSTAT_DATA_UINT64 }, 3345a98e54bSBrendan Gregg - Sun Microsystems { "l2_read_bytes", KSTAT_DATA_UINT64 }, 3355a98e54bSBrendan Gregg - Sun Microsystems { "l2_write_bytes", KSTAT_DATA_UINT64 }, 336fa94a07fSbrendan { "l2_writes_sent", KSTAT_DATA_UINT64 }, 337fa94a07fSbrendan { "l2_writes_done", KSTAT_DATA_UINT64 }, 338fa94a07fSbrendan { "l2_writes_error", KSTAT_DATA_UINT64 }, 339fa94a07fSbrendan { "l2_writes_hdr_miss", KSTAT_DATA_UINT64 }, 340fa94a07fSbrendan { "l2_evict_lock_retry", KSTAT_DATA_UINT64 }, 341fa94a07fSbrendan { "l2_evict_reading", KSTAT_DATA_UINT64 }, 342fa94a07fSbrendan { "l2_free_on_write", KSTAT_DATA_UINT64 }, 343fa94a07fSbrendan { "l2_abort_lowmem", KSTAT_DATA_UINT64 }, 344fa94a07fSbrendan { "l2_cksum_bad", KSTAT_DATA_UINT64 }, 345fa94a07fSbrendan { "l2_io_error", KSTAT_DATA_UINT64 }, 346fa94a07fSbrendan { "l2_size", KSTAT_DATA_UINT64 }, 3471ab7f2deSmaybee { "l2_hdr_size", KSTAT_DATA_UINT64 }, 3481ab7f2deSmaybee { "memory_throttle_count", KSTAT_DATA_UINT64 } 34944cb6abcSbmc }; 35044cb6abcSbmc 35144cb6abcSbmc #define ARCSTAT(stat) (arc_stats.stat.value.ui64) 35244cb6abcSbmc 35344cb6abcSbmc #define ARCSTAT_INCR(stat, val) \ 35444cb6abcSbmc atomic_add_64(&arc_stats.stat.value.ui64, (val)); 35544cb6abcSbmc 356b24ab676SJeff Bonwick #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) 35744cb6abcSbmc #define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1) 35844cb6abcSbmc 35944cb6abcSbmc #define ARCSTAT_MAX(stat, val) { \ 36044cb6abcSbmc uint64_t m; \ 36144cb6abcSbmc while ((val) > (m = arc_stats.stat.value.ui64) && \ 36244cb6abcSbmc (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \ 36344cb6abcSbmc continue; \ 36444cb6abcSbmc } 36544cb6abcSbmc 36644cb6abcSbmc #define ARCSTAT_MAXSTAT(stat) \ 36744cb6abcSbmc ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64) 36844cb6abcSbmc 36944cb6abcSbmc /* 37044cb6abcSbmc * We define a macro to allow ARC hits/misses to be easily broken down by 37144cb6abcSbmc * two separate conditions, giving a total of four different subtypes for 37244cb6abcSbmc * each of hits and misses (so eight statistics total). 37344cb6abcSbmc */ 37444cb6abcSbmc #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \ 37544cb6abcSbmc if (cond1) { \ 37644cb6abcSbmc if (cond2) { \ 37744cb6abcSbmc ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \ 37844cb6abcSbmc } else { \ 37944cb6abcSbmc ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \ 38044cb6abcSbmc } \ 38144cb6abcSbmc } else { \ 38244cb6abcSbmc if (cond2) { \ 38344cb6abcSbmc ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \ 38444cb6abcSbmc } else { \ 38544cb6abcSbmc ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\ 38644cb6abcSbmc } \ 38744cb6abcSbmc } 38844cb6abcSbmc 38944cb6abcSbmc kstat_t *arc_ksp; 390b24ab676SJeff Bonwick static arc_state_t *arc_anon; 39144cb6abcSbmc static arc_state_t *arc_mru; 39244cb6abcSbmc static arc_state_t *arc_mru_ghost; 39344cb6abcSbmc static arc_state_t *arc_mfu; 39444cb6abcSbmc static arc_state_t *arc_mfu_ghost; 395fa94a07fSbrendan static arc_state_t *arc_l2c_only; 39644cb6abcSbmc 39744cb6abcSbmc /* 39844cb6abcSbmc * There are several ARC variables that are critical to export as kstats -- 39944cb6abcSbmc * but we don't want to have to grovel around in the kstat whenever we wish to 40044cb6abcSbmc * manipulate them. For these variables, we therefore define them to be in 40144cb6abcSbmc * terms of the statistic variable. This assures that we are not introducing 40244cb6abcSbmc * the possibility of inconsistency by having shadow copies of the variables, 40344cb6abcSbmc * while still allowing the code to be readable. 40444cb6abcSbmc */ 40544cb6abcSbmc #define arc_size ARCSTAT(arcstat_size) /* actual total arc size */ 40644cb6abcSbmc #define arc_p ARCSTAT(arcstat_p) /* target size of MRU */ 40744cb6abcSbmc #define arc_c ARCSTAT(arcstat_c) /* target size of cache */ 40844cb6abcSbmc #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ 40944cb6abcSbmc #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ 41044cb6abcSbmc 41144cb6abcSbmc static int arc_no_grow; /* Don't try to grow cache size */ 41244cb6abcSbmc static uint64_t arc_tempreserve; 4132fdbea25SAleksandr Guzovskiy static uint64_t arc_loaned_bytes; 4140e8c6158Smaybee static uint64_t arc_meta_used; 4150e8c6158Smaybee static uint64_t arc_meta_limit; 4160e8c6158Smaybee static uint64_t arc_meta_max = 0; 417fa9e4066Sahrens 418fa94a07fSbrendan typedef struct l2arc_buf_hdr l2arc_buf_hdr_t; 419fa94a07fSbrendan 420fa9e4066Sahrens typedef struct arc_callback arc_callback_t; 421fa9e4066Sahrens 422fa9e4066Sahrens struct arc_callback { 423fa9e4066Sahrens void *acb_private; 424c717a561Smaybee arc_done_func_t *acb_done; 425fa9e4066Sahrens arc_buf_t *acb_buf; 426fa9e4066Sahrens zio_t *acb_zio_dummy; 427fa9e4066Sahrens arc_callback_t *acb_next; 428fa9e4066Sahrens }; 429fa9e4066Sahrens 430c717a561Smaybee typedef struct arc_write_callback arc_write_callback_t; 431c717a561Smaybee 432c717a561Smaybee struct arc_write_callback { 433c717a561Smaybee void *awcb_private; 434c717a561Smaybee arc_done_func_t *awcb_ready; 435c717a561Smaybee arc_done_func_t *awcb_done; 436c717a561Smaybee arc_buf_t *awcb_buf; 437c717a561Smaybee }; 438c717a561Smaybee 439fa9e4066Sahrens struct arc_buf_hdr { 440fa9e4066Sahrens /* protected by hash lock */ 441fa9e4066Sahrens dva_t b_dva; 442fa9e4066Sahrens uint64_t b_birth; 443fa9e4066Sahrens uint64_t b_cksum0; 444fa9e4066Sahrens 4456b4acc8bSahrens kmutex_t b_freeze_lock; 4466b4acc8bSahrens zio_cksum_t *b_freeze_cksum; 4473f9d6ad7SLin Ling void *b_thawed; 4486b4acc8bSahrens 449fa9e4066Sahrens arc_buf_hdr_t *b_hash_next; 450fa9e4066Sahrens arc_buf_t *b_buf; 451fa9e4066Sahrens uint32_t b_flags; 452ea8dc4b6Seschrock uint32_t b_datacnt; 453fa9e4066Sahrens 454fa9e4066Sahrens arc_callback_t *b_acb; 455ad23a2dbSjohansen kcondvar_t b_cv; 456ad23a2dbSjohansen 457ad23a2dbSjohansen /* immutable */ 458ad23a2dbSjohansen arc_buf_contents_t b_type; 459ad23a2dbSjohansen uint64_t b_size; 460ac05c741SMark Maybee uint64_t b_spa; 461fa9e4066Sahrens 462fa9e4066Sahrens /* protected by arc state mutex */ 463fa9e4066Sahrens arc_state_t *b_state; 464fa9e4066Sahrens list_node_t b_arc_node; 465fa9e4066Sahrens 466fa9e4066Sahrens /* updated atomically */ 467fa9e4066Sahrens clock_t b_arc_access; 468fa9e4066Sahrens 469fa9e4066Sahrens /* self protecting */ 470fa9e4066Sahrens refcount_t b_refcnt; 471fa94a07fSbrendan 472fa94a07fSbrendan l2arc_buf_hdr_t *b_l2hdr; 473fa94a07fSbrendan list_node_t b_l2node; 474fa9e4066Sahrens }; 475fa9e4066Sahrens 476ea8dc4b6Seschrock static arc_buf_t *arc_eviction_list; 477ea8dc4b6Seschrock static kmutex_t arc_eviction_mtx; 47840d7d650Smaybee static arc_buf_hdr_t arc_eviction_hdr; 47944eda4d7Smaybee static void arc_get_data_buf(arc_buf_t *buf); 48044eda4d7Smaybee static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock); 4810e8c6158Smaybee static int arc_evict_needed(arc_buf_contents_t type); 482ac05c741SMark Maybee static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes); 483*cd1c8b85SMatthew Ahrens static void arc_buf_watch(arc_buf_t *buf); 484ea8dc4b6Seschrock 4855ea40c06SBrendan Gregg - Sun Microsystems static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab); 4865ea40c06SBrendan Gregg - Sun Microsystems 487ea8dc4b6Seschrock #define GHOST_STATE(state) \ 488fa94a07fSbrendan ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \ 489fa94a07fSbrendan (state) == arc_l2c_only) 490ea8dc4b6Seschrock 491fa9e4066Sahrens /* 492fa9e4066Sahrens * Private ARC flags. These flags are private ARC only flags that will show up 493fa9e4066Sahrens * in b_flags in the arc_hdr_buf_t. Some flags are publicly declared, and can 494fa9e4066Sahrens * be passed in as arc_flags in things like arc_read. However, these flags 495fa9e4066Sahrens * should never be passed and should only be set by ARC code. When adding new 496fa9e4066Sahrens * public flags, make sure not to smash the private ones. 497fa9e4066Sahrens */ 498fa9e4066Sahrens 499ea8dc4b6Seschrock #define ARC_IN_HASH_TABLE (1 << 9) /* this buffer is hashed */ 500fa9e4066Sahrens #define ARC_IO_IN_PROGRESS (1 << 10) /* I/O in progress for buf */ 501fa9e4066Sahrens #define ARC_IO_ERROR (1 << 11) /* I/O failed for buf */ 502fa9e4066Sahrens #define ARC_FREED_IN_READ (1 << 12) /* buf freed while in read */ 503ea8dc4b6Seschrock #define ARC_BUF_AVAILABLE (1 << 13) /* block not in active use */ 50413506d1eSmaybee #define ARC_INDIRECT (1 << 14) /* this is an indirect block */ 505fa94a07fSbrendan #define ARC_FREE_IN_PROGRESS (1 << 15) /* hdr about to be freed */ 5063baa08fcSek #define ARC_L2_WRITING (1 << 16) /* L2ARC write in progress */ 5073baa08fcSek #define ARC_L2_EVICTED (1 << 17) /* evicted during I/O */ 5083baa08fcSek #define ARC_L2_WRITE_HEAD (1 << 18) /* head of write list */ 509fa9e4066Sahrens 510ea8dc4b6Seschrock #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_IN_HASH_TABLE) 511fa9e4066Sahrens #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS) 512fa9e4066Sahrens #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_IO_ERROR) 5135a98e54bSBrendan Gregg - Sun Microsystems #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_PREFETCH) 514fa9e4066Sahrens #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FREED_IN_READ) 515ea8dc4b6Seschrock #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_BUF_AVAILABLE) 516fa94a07fSbrendan #define HDR_FREE_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FREE_IN_PROGRESS) 5173baa08fcSek #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_L2CACHE) 5183a737e0dSbrendan #define HDR_L2_READING(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS && \ 5193a737e0dSbrendan (hdr)->b_l2hdr != NULL) 520fa94a07fSbrendan #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_L2_WRITING) 521fa94a07fSbrendan #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_L2_EVICTED) 522fa94a07fSbrendan #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_L2_WRITE_HEAD) 523fa9e4066Sahrens 524e6c728e1Sbrendan /* 525e6c728e1Sbrendan * Other sizes 526e6c728e1Sbrendan */ 527e6c728e1Sbrendan 528e6c728e1Sbrendan #define HDR_SIZE ((int64_t)sizeof (arc_buf_hdr_t)) 529e6c728e1Sbrendan #define L2HDR_SIZE ((int64_t)sizeof (l2arc_buf_hdr_t)) 530e6c728e1Sbrendan 531fa9e4066Sahrens /* 532fa9e4066Sahrens * Hash table routines 533fa9e4066Sahrens */ 534fa9e4066Sahrens 535fa9e4066Sahrens #define HT_LOCK_PAD 64 536fa9e4066Sahrens 537fa9e4066Sahrens struct ht_lock { 538fa9e4066Sahrens kmutex_t ht_lock; 539fa9e4066Sahrens #ifdef _KERNEL 540fa9e4066Sahrens unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))]; 541fa9e4066Sahrens #endif 542fa9e4066Sahrens }; 543fa9e4066Sahrens 544fa9e4066Sahrens #define BUF_LOCKS 256 545fa9e4066Sahrens typedef struct buf_hash_table { 546fa9e4066Sahrens uint64_t ht_mask; 547fa9e4066Sahrens arc_buf_hdr_t **ht_table; 548fa9e4066Sahrens struct ht_lock ht_locks[BUF_LOCKS]; 549fa9e4066Sahrens } buf_hash_table_t; 550fa9e4066Sahrens 551fa9e4066Sahrens static buf_hash_table_t buf_hash_table; 552fa9e4066Sahrens 553fa9e4066Sahrens #define BUF_HASH_INDEX(spa, dva, birth) \ 554fa9e4066Sahrens (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask) 555fa9e4066Sahrens #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)]) 556fa9e4066Sahrens #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock)) 5573f9d6ad7SLin Ling #define HDR_LOCK(hdr) \ 5583f9d6ad7SLin Ling (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth))) 559fa9e4066Sahrens 560fa9e4066Sahrens uint64_t zfs_crc64_table[256]; 561fa9e4066Sahrens 562fa94a07fSbrendan /* 563fa94a07fSbrendan * Level 2 ARC 564fa94a07fSbrendan */ 565fa94a07fSbrendan 566fa94a07fSbrendan #define L2ARC_WRITE_SIZE (8 * 1024 * 1024) /* initial write max */ 5675a98e54bSBrendan Gregg - Sun Microsystems #define L2ARC_HEADROOM 2 /* num of writes */ 5685a98e54bSBrendan Gregg - Sun Microsystems #define L2ARC_FEED_SECS 1 /* caching interval secs */ 5695a98e54bSBrendan Gregg - Sun Microsystems #define L2ARC_FEED_MIN_MS 200 /* min caching interval ms */ 570fa94a07fSbrendan 571fa94a07fSbrendan #define l2arc_writes_sent ARCSTAT(arcstat_l2_writes_sent) 572fa94a07fSbrendan #define l2arc_writes_done ARCSTAT(arcstat_l2_writes_done) 573fa94a07fSbrendan 574fa94a07fSbrendan /* 575fa94a07fSbrendan * L2ARC Performance Tunables 576fa94a07fSbrendan */ 577fa94a07fSbrendan uint64_t l2arc_write_max = L2ARC_WRITE_SIZE; /* default max write size */ 5783a737e0dSbrendan uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE; /* extra write during warmup */ 579fa94a07fSbrendan uint64_t l2arc_headroom = L2ARC_HEADROOM; /* number of dev writes */ 580fa94a07fSbrendan uint64_t l2arc_feed_secs = L2ARC_FEED_SECS; /* interval seconds */ 5815a98e54bSBrendan Gregg - Sun Microsystems uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval milliseconds */ 582fa94a07fSbrendan boolean_t l2arc_noprefetch = B_TRUE; /* don't cache prefetch bufs */ 5835a98e54bSBrendan Gregg - Sun Microsystems boolean_t l2arc_feed_again = B_TRUE; /* turbo warmup */ 5845a98e54bSBrendan Gregg - Sun Microsystems boolean_t l2arc_norw = B_TRUE; /* no reads during writes */ 585fa94a07fSbrendan 586fa94a07fSbrendan /* 587fa94a07fSbrendan * L2ARC Internals 588fa94a07fSbrendan */ 589fa94a07fSbrendan typedef struct l2arc_dev { 590fa94a07fSbrendan vdev_t *l2ad_vdev; /* vdev */ 591fa94a07fSbrendan spa_t *l2ad_spa; /* spa */ 592fa94a07fSbrendan uint64_t l2ad_hand; /* next write location */ 593fa94a07fSbrendan uint64_t l2ad_write; /* desired write size, bytes */ 5943a737e0dSbrendan uint64_t l2ad_boost; /* warmup write boost, bytes */ 595fa94a07fSbrendan uint64_t l2ad_start; /* first addr on device */ 596fa94a07fSbrendan uint64_t l2ad_end; /* last addr on device */ 597fa94a07fSbrendan uint64_t l2ad_evict; /* last addr eviction reached */ 598fa94a07fSbrendan boolean_t l2ad_first; /* first sweep through */ 5995a98e54bSBrendan Gregg - Sun Microsystems boolean_t l2ad_writing; /* currently writing */ 600fa94a07fSbrendan list_t *l2ad_buflist; /* buffer list */ 601fa94a07fSbrendan list_node_t l2ad_node; /* device list node */ 602fa94a07fSbrendan } l2arc_dev_t; 603fa94a07fSbrendan 604fa94a07fSbrendan static list_t L2ARC_dev_list; /* device list */ 605fa94a07fSbrendan static list_t *l2arc_dev_list; /* device list pointer */ 606fa94a07fSbrendan static kmutex_t l2arc_dev_mtx; /* device list mutex */ 607fa94a07fSbrendan static l2arc_dev_t *l2arc_dev_last; /* last device used */ 608fa94a07fSbrendan static kmutex_t l2arc_buflist_mtx; /* mutex for all buflists */ 609fa94a07fSbrendan static list_t L2ARC_free_on_write; /* free after write buf list */ 610fa94a07fSbrendan static list_t *l2arc_free_on_write; /* free after write list ptr */ 611fa94a07fSbrendan static kmutex_t l2arc_free_on_write_mtx; /* mutex for list */ 612fa94a07fSbrendan static uint64_t l2arc_ndev; /* number of devices */ 613fa94a07fSbrendan 614fa94a07fSbrendan typedef struct l2arc_read_callback { 615fa94a07fSbrendan arc_buf_t *l2rcb_buf; /* read buffer */ 616fa94a07fSbrendan spa_t *l2rcb_spa; /* spa */ 617fa94a07fSbrendan blkptr_t l2rcb_bp; /* original blkptr */ 618fa94a07fSbrendan zbookmark_t l2rcb_zb; /* original bookmark */ 619fa94a07fSbrendan int l2rcb_flags; /* original flags */ 620fa94a07fSbrendan } l2arc_read_callback_t; 621fa94a07fSbrendan 622fa94a07fSbrendan typedef struct l2arc_write_callback { 623fa94a07fSbrendan l2arc_dev_t *l2wcb_dev; /* device info */ 624fa94a07fSbrendan arc_buf_hdr_t *l2wcb_head; /* head of write buflist */ 625fa94a07fSbrendan } l2arc_write_callback_t; 626fa94a07fSbrendan 627fa94a07fSbrendan struct l2arc_buf_hdr { 628fa94a07fSbrendan /* protected by arc_buf_hdr mutex */ 629fa94a07fSbrendan l2arc_dev_t *b_dev; /* L2ARC device */ 630f9d8334fSGeorge Wilson uint64_t b_daddr; /* disk address, offset byte */ 631fa94a07fSbrendan }; 632fa94a07fSbrendan 633fa94a07fSbrendan typedef struct l2arc_data_free { 634fa94a07fSbrendan /* protected by l2arc_free_on_write_mtx */ 635fa94a07fSbrendan void *l2df_data; 636fa94a07fSbrendan size_t l2df_size; 637fa94a07fSbrendan void (*l2df_func)(void *, size_t); 638fa94a07fSbrendan list_node_t l2df_list_node; 639fa94a07fSbrendan } l2arc_data_free_t; 640fa94a07fSbrendan 641fa94a07fSbrendan static kmutex_t l2arc_feed_thr_lock; 642fa94a07fSbrendan static kcondvar_t l2arc_feed_thr_cv; 643fa94a07fSbrendan static uint8_t l2arc_thread_exit; 644fa94a07fSbrendan 645fa94a07fSbrendan static void l2arc_read_done(zio_t *zio); 646fa94a07fSbrendan static void l2arc_hdr_stat_add(void); 647fa94a07fSbrendan static void l2arc_hdr_stat_remove(void); 648fa94a07fSbrendan 649fa9e4066Sahrens static uint64_t 650ac05c741SMark Maybee buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth) 651fa9e4066Sahrens { 652fa9e4066Sahrens uint8_t *vdva = (uint8_t *)dva; 653fa9e4066Sahrens uint64_t crc = -1ULL; 654fa9e4066Sahrens int i; 655fa9e4066Sahrens 656fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 657fa9e4066Sahrens 658fa9e4066Sahrens for (i = 0; i < sizeof (dva_t); i++) 659fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF]; 660fa9e4066Sahrens 661ac05c741SMark Maybee crc ^= (spa>>8) ^ birth; 662fa9e4066Sahrens 663fa9e4066Sahrens return (crc); 664fa9e4066Sahrens } 665fa9e4066Sahrens 666fa9e4066Sahrens #define BUF_EMPTY(buf) \ 667fa9e4066Sahrens ((buf)->b_dva.dva_word[0] == 0 && \ 668fa9e4066Sahrens (buf)->b_dva.dva_word[1] == 0 && \ 669fa9e4066Sahrens (buf)->b_birth == 0) 670fa9e4066Sahrens 671fa9e4066Sahrens #define BUF_EQUAL(spa, dva, birth, buf) \ 672fa9e4066Sahrens ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \ 673fa9e4066Sahrens ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \ 674fa9e4066Sahrens ((buf)->b_birth == birth) && ((buf)->b_spa == spa) 675fa9e4066Sahrens 6763f9d6ad7SLin Ling static void 6773f9d6ad7SLin Ling buf_discard_identity(arc_buf_hdr_t *hdr) 6783f9d6ad7SLin Ling { 6793f9d6ad7SLin Ling hdr->b_dva.dva_word[0] = 0; 6803f9d6ad7SLin Ling hdr->b_dva.dva_word[1] = 0; 6813f9d6ad7SLin Ling hdr->b_birth = 0; 6823f9d6ad7SLin Ling hdr->b_cksum0 = 0; 6833f9d6ad7SLin Ling } 6843f9d6ad7SLin Ling 685fa9e4066Sahrens static arc_buf_hdr_t * 686ac05c741SMark Maybee buf_hash_find(uint64_t spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp) 687fa9e4066Sahrens { 688fa9e4066Sahrens uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); 689fa9e4066Sahrens kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 690fa9e4066Sahrens arc_buf_hdr_t *buf; 691fa9e4066Sahrens 692fa9e4066Sahrens mutex_enter(hash_lock); 693fa9e4066Sahrens for (buf = buf_hash_table.ht_table[idx]; buf != NULL; 694fa9e4066Sahrens buf = buf->b_hash_next) { 695fa9e4066Sahrens if (BUF_EQUAL(spa, dva, birth, buf)) { 696fa9e4066Sahrens *lockp = hash_lock; 697fa9e4066Sahrens return (buf); 698fa9e4066Sahrens } 699fa9e4066Sahrens } 700fa9e4066Sahrens mutex_exit(hash_lock); 701fa9e4066Sahrens *lockp = NULL; 702fa9e4066Sahrens return (NULL); 703fa9e4066Sahrens } 704fa9e4066Sahrens 705fa9e4066Sahrens /* 706fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 707fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 708fa9e4066Sahrens * will be returned and the new element will not be inserted. 709fa9e4066Sahrens * Otherwise returns NULL. 710fa9e4066Sahrens */ 711fa9e4066Sahrens static arc_buf_hdr_t * 712fa9e4066Sahrens buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp) 713fa9e4066Sahrens { 714fa9e4066Sahrens uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 715fa9e4066Sahrens kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 716fa9e4066Sahrens arc_buf_hdr_t *fbuf; 71744cb6abcSbmc uint32_t i; 718fa9e4066Sahrens 719ea8dc4b6Seschrock ASSERT(!HDR_IN_HASH_TABLE(buf)); 720fa9e4066Sahrens *lockp = hash_lock; 721fa9e4066Sahrens mutex_enter(hash_lock); 722fa9e4066Sahrens for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL; 723fa9e4066Sahrens fbuf = fbuf->b_hash_next, i++) { 724fa9e4066Sahrens if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf)) 725fa9e4066Sahrens return (fbuf); 726fa9e4066Sahrens } 727fa9e4066Sahrens 728fa9e4066Sahrens buf->b_hash_next = buf_hash_table.ht_table[idx]; 729fa9e4066Sahrens buf_hash_table.ht_table[idx] = buf; 730ea8dc4b6Seschrock buf->b_flags |= ARC_IN_HASH_TABLE; 731fa9e4066Sahrens 732fa9e4066Sahrens /* collect some hash table performance data */ 733fa9e4066Sahrens if (i > 0) { 73444cb6abcSbmc ARCSTAT_BUMP(arcstat_hash_collisions); 735fa9e4066Sahrens if (i == 1) 73644cb6abcSbmc ARCSTAT_BUMP(arcstat_hash_chains); 73744cb6abcSbmc 73844cb6abcSbmc ARCSTAT_MAX(arcstat_hash_chain_max, i); 739fa9e4066Sahrens } 74044cb6abcSbmc 74144cb6abcSbmc ARCSTAT_BUMP(arcstat_hash_elements); 74244cb6abcSbmc ARCSTAT_MAXSTAT(arcstat_hash_elements); 743fa9e4066Sahrens 744fa9e4066Sahrens return (NULL); 745fa9e4066Sahrens } 746fa9e4066Sahrens 747fa9e4066Sahrens static void 748fa9e4066Sahrens buf_hash_remove(arc_buf_hdr_t *buf) 749fa9e4066Sahrens { 750fa9e4066Sahrens arc_buf_hdr_t *fbuf, **bufp; 751fa9e4066Sahrens uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 752fa9e4066Sahrens 753fa9e4066Sahrens ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx))); 754ea8dc4b6Seschrock ASSERT(HDR_IN_HASH_TABLE(buf)); 755fa9e4066Sahrens 756fa9e4066Sahrens bufp = &buf_hash_table.ht_table[idx]; 757fa9e4066Sahrens while ((fbuf = *bufp) != buf) { 758fa9e4066Sahrens ASSERT(fbuf != NULL); 759fa9e4066Sahrens bufp = &fbuf->b_hash_next; 760fa9e4066Sahrens } 761fa9e4066Sahrens *bufp = buf->b_hash_next; 762fa9e4066Sahrens buf->b_hash_next = NULL; 763ea8dc4b6Seschrock buf->b_flags &= ~ARC_IN_HASH_TABLE; 764fa9e4066Sahrens 765fa9e4066Sahrens /* collect some hash table performance data */ 76644cb6abcSbmc ARCSTAT_BUMPDOWN(arcstat_hash_elements); 76744cb6abcSbmc 768fa9e4066Sahrens if (buf_hash_table.ht_table[idx] && 769fa9e4066Sahrens buf_hash_table.ht_table[idx]->b_hash_next == NULL) 77044cb6abcSbmc ARCSTAT_BUMPDOWN(arcstat_hash_chains); 771fa9e4066Sahrens } 772fa9e4066Sahrens 773fa9e4066Sahrens /* 774fa9e4066Sahrens * Global data structures and functions for the buf kmem cache. 775fa9e4066Sahrens */ 776fa9e4066Sahrens static kmem_cache_t *hdr_cache; 777fa9e4066Sahrens static kmem_cache_t *buf_cache; 778fa9e4066Sahrens 779fa9e4066Sahrens static void 780fa9e4066Sahrens buf_fini(void) 781fa9e4066Sahrens { 782fa9e4066Sahrens int i; 783fa9e4066Sahrens 784fa9e4066Sahrens kmem_free(buf_hash_table.ht_table, 785fa9e4066Sahrens (buf_hash_table.ht_mask + 1) * sizeof (void *)); 786fa9e4066Sahrens for (i = 0; i < BUF_LOCKS; i++) 787fa9e4066Sahrens mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock); 788fa9e4066Sahrens kmem_cache_destroy(hdr_cache); 789fa9e4066Sahrens kmem_cache_destroy(buf_cache); 790fa9e4066Sahrens } 791fa9e4066Sahrens 792fa9e4066Sahrens /* 793fa9e4066Sahrens * Constructor callback - called when the cache is empty 794fa9e4066Sahrens * and a new buf is requested. 795fa9e4066Sahrens */ 796fa9e4066Sahrens /* ARGSUSED */ 797fa9e4066Sahrens static int 798fa9e4066Sahrens hdr_cons(void *vbuf, void *unused, int kmflag) 799fa9e4066Sahrens { 800fa9e4066Sahrens arc_buf_hdr_t *buf = vbuf; 801fa9e4066Sahrens 802fa9e4066Sahrens bzero(buf, sizeof (arc_buf_hdr_t)); 803fa9e4066Sahrens refcount_create(&buf->b_refcnt); 804fa9e4066Sahrens cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL); 805c25056deSgw mutex_init(&buf->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL); 8065a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS); 807fa94a07fSbrendan 808fa9e4066Sahrens return (0); 809fa9e4066Sahrens } 810fa9e4066Sahrens 8116f83844dSMark Maybee /* ARGSUSED */ 8126f83844dSMark Maybee static int 8136f83844dSMark Maybee buf_cons(void *vbuf, void *unused, int kmflag) 8146f83844dSMark Maybee { 8156f83844dSMark Maybee arc_buf_t *buf = vbuf; 8166f83844dSMark Maybee 8176f83844dSMark Maybee bzero(buf, sizeof (arc_buf_t)); 8183f9d6ad7SLin Ling mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL); 8193f9d6ad7SLin Ling rw_init(&buf->b_data_lock, NULL, RW_DEFAULT, NULL); 8205a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS); 8215a98e54bSBrendan Gregg - Sun Microsystems 8226f83844dSMark Maybee return (0); 8236f83844dSMark Maybee } 8246f83844dSMark Maybee 825fa9e4066Sahrens /* 826fa9e4066Sahrens * Destructor callback - called when a cached buf is 827fa9e4066Sahrens * no longer required. 828fa9e4066Sahrens */ 829fa9e4066Sahrens /* ARGSUSED */ 830fa9e4066Sahrens static void 831fa9e4066Sahrens hdr_dest(void *vbuf, void *unused) 832fa9e4066Sahrens { 833fa9e4066Sahrens arc_buf_hdr_t *buf = vbuf; 834fa9e4066Sahrens 835b24ab676SJeff Bonwick ASSERT(BUF_EMPTY(buf)); 836fa9e4066Sahrens refcount_destroy(&buf->b_refcnt); 837fa9e4066Sahrens cv_destroy(&buf->b_cv); 838c25056deSgw mutex_destroy(&buf->b_freeze_lock); 8395a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS); 840fa9e4066Sahrens } 841fa9e4066Sahrens 8426f83844dSMark Maybee /* ARGSUSED */ 8436f83844dSMark Maybee static void 8446f83844dSMark Maybee buf_dest(void *vbuf, void *unused) 8456f83844dSMark Maybee { 8466f83844dSMark Maybee arc_buf_t *buf = vbuf; 8476f83844dSMark Maybee 8483f9d6ad7SLin Ling mutex_destroy(&buf->b_evict_lock); 8493f9d6ad7SLin Ling rw_destroy(&buf->b_data_lock); 8505a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS); 8516f83844dSMark Maybee } 8526f83844dSMark Maybee 853fa9e4066Sahrens /* 854fa9e4066Sahrens * Reclaim callback -- invoked when memory is low. 855fa9e4066Sahrens */ 856fa9e4066Sahrens /* ARGSUSED */ 857fa9e4066Sahrens static void 858fa9e4066Sahrens hdr_recl(void *unused) 859fa9e4066Sahrens { 860fa9e4066Sahrens dprintf("hdr_recl called\n"); 86149e3519aSmaybee /* 86249e3519aSmaybee * umem calls the reclaim func when we destroy the buf cache, 86349e3519aSmaybee * which is after we do arc_fini(). 86449e3519aSmaybee */ 86549e3519aSmaybee if (!arc_dead) 86649e3519aSmaybee cv_signal(&arc_reclaim_thr_cv); 867fa9e4066Sahrens } 868fa9e4066Sahrens 869fa9e4066Sahrens static void 870fa9e4066Sahrens buf_init(void) 871fa9e4066Sahrens { 872fa9e4066Sahrens uint64_t *ct; 873ea8dc4b6Seschrock uint64_t hsize = 1ULL << 12; 874fa9e4066Sahrens int i, j; 875fa9e4066Sahrens 876fa9e4066Sahrens /* 877fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 878ea8dc4b6Seschrock * with an average 64K block size. The table will take up 879ea8dc4b6Seschrock * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers). 880fa9e4066Sahrens */ 881ea8dc4b6Seschrock while (hsize * 65536 < physmem * PAGESIZE) 882fa9e4066Sahrens hsize <<= 1; 883ea8dc4b6Seschrock retry: 884fa9e4066Sahrens buf_hash_table.ht_mask = hsize - 1; 885ea8dc4b6Seschrock buf_hash_table.ht_table = 886ea8dc4b6Seschrock kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP); 887ea8dc4b6Seschrock if (buf_hash_table.ht_table == NULL) { 888ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 8)); 889ea8dc4b6Seschrock hsize >>= 1; 890ea8dc4b6Seschrock goto retry; 891ea8dc4b6Seschrock } 892fa9e4066Sahrens 893fa9e4066Sahrens hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t), 894fa9e4066Sahrens 0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0); 895fa9e4066Sahrens buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t), 8966f83844dSMark Maybee 0, buf_cons, buf_dest, NULL, NULL, NULL, 0); 897fa9e4066Sahrens 898fa9e4066Sahrens for (i = 0; i < 256; i++) 899fa9e4066Sahrens for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--) 900fa9e4066Sahrens *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY); 901fa9e4066Sahrens 902fa9e4066Sahrens for (i = 0; i < BUF_LOCKS; i++) { 903fa9e4066Sahrens mutex_init(&buf_hash_table.ht_locks[i].ht_lock, 904fa9e4066Sahrens NULL, MUTEX_DEFAULT, NULL); 905fa9e4066Sahrens } 906fa9e4066Sahrens } 907fa9e4066Sahrens 908fa9e4066Sahrens #define ARC_MINTIME (hz>>4) /* 62 ms */ 909fa9e4066Sahrens 9106b4acc8bSahrens static void 9116b4acc8bSahrens arc_cksum_verify(arc_buf_t *buf) 9126b4acc8bSahrens { 9136b4acc8bSahrens zio_cksum_t zc; 9146b4acc8bSahrens 915cc60fd72Sahrens if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 9166b4acc8bSahrens return; 9176b4acc8bSahrens 9186b4acc8bSahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 9193ccfa83cSahrens if (buf->b_hdr->b_freeze_cksum == NULL || 9203ccfa83cSahrens (buf->b_hdr->b_flags & ARC_IO_ERROR)) { 9216b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 9226b4acc8bSahrens return; 9236b4acc8bSahrens } 9246b4acc8bSahrens fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 9256b4acc8bSahrens if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc)) 9266b4acc8bSahrens panic("buffer modified while frozen!"); 9276b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 9286b4acc8bSahrens } 9296b4acc8bSahrens 930fa94a07fSbrendan static int 931fa94a07fSbrendan arc_cksum_equal(arc_buf_t *buf) 932fa94a07fSbrendan { 933fa94a07fSbrendan zio_cksum_t zc; 934fa94a07fSbrendan int equal; 935fa94a07fSbrendan 936fa94a07fSbrendan mutex_enter(&buf->b_hdr->b_freeze_lock); 937fa94a07fSbrendan fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 938fa94a07fSbrendan equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc); 939fa94a07fSbrendan mutex_exit(&buf->b_hdr->b_freeze_lock); 940fa94a07fSbrendan 941fa94a07fSbrendan return (equal); 942fa94a07fSbrendan } 943fa94a07fSbrendan 9446b4acc8bSahrens static void 945fa94a07fSbrendan arc_cksum_compute(arc_buf_t *buf, boolean_t force) 9466b4acc8bSahrens { 947fa94a07fSbrendan if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY)) 9486b4acc8bSahrens return; 9496b4acc8bSahrens 9506b4acc8bSahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 9516b4acc8bSahrens if (buf->b_hdr->b_freeze_cksum != NULL) { 9526b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 9536b4acc8bSahrens return; 9546b4acc8bSahrens } 9556b4acc8bSahrens buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP); 9566b4acc8bSahrens fletcher_2_native(buf->b_data, buf->b_hdr->b_size, 9576b4acc8bSahrens buf->b_hdr->b_freeze_cksum); 9586b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 959*cd1c8b85SMatthew Ahrens arc_buf_watch(buf); 960*cd1c8b85SMatthew Ahrens } 961*cd1c8b85SMatthew Ahrens 962*cd1c8b85SMatthew Ahrens #ifndef _KERNEL 963*cd1c8b85SMatthew Ahrens typedef struct procctl { 964*cd1c8b85SMatthew Ahrens long cmd; 965*cd1c8b85SMatthew Ahrens prwatch_t prwatch; 966*cd1c8b85SMatthew Ahrens } procctl_t; 967*cd1c8b85SMatthew Ahrens #endif 968*cd1c8b85SMatthew Ahrens 969*cd1c8b85SMatthew Ahrens /* ARGSUSED */ 970*cd1c8b85SMatthew Ahrens static void 971*cd1c8b85SMatthew Ahrens arc_buf_unwatch(arc_buf_t *buf) 972*cd1c8b85SMatthew Ahrens { 973*cd1c8b85SMatthew Ahrens #ifndef _KERNEL 974*cd1c8b85SMatthew Ahrens if (arc_watch) { 975*cd1c8b85SMatthew Ahrens int result; 976*cd1c8b85SMatthew Ahrens procctl_t ctl; 977*cd1c8b85SMatthew Ahrens ctl.cmd = PCWATCH; 978*cd1c8b85SMatthew Ahrens ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data; 979*cd1c8b85SMatthew Ahrens ctl.prwatch.pr_size = 0; 980*cd1c8b85SMatthew Ahrens ctl.prwatch.pr_wflags = 0; 981*cd1c8b85SMatthew Ahrens result = write(arc_procfd, &ctl, sizeof (ctl)); 982*cd1c8b85SMatthew Ahrens ASSERT3U(result, ==, sizeof (ctl)); 983*cd1c8b85SMatthew Ahrens } 984*cd1c8b85SMatthew Ahrens #endif 985*cd1c8b85SMatthew Ahrens } 986*cd1c8b85SMatthew Ahrens 987*cd1c8b85SMatthew Ahrens /* ARGSUSED */ 988*cd1c8b85SMatthew Ahrens static void 989*cd1c8b85SMatthew Ahrens arc_buf_watch(arc_buf_t *buf) 990*cd1c8b85SMatthew Ahrens { 991*cd1c8b85SMatthew Ahrens #ifndef _KERNEL 992*cd1c8b85SMatthew Ahrens if (arc_watch) { 993*cd1c8b85SMatthew Ahrens int result; 994*cd1c8b85SMatthew Ahrens procctl_t ctl; 995*cd1c8b85SMatthew Ahrens ctl.cmd = PCWATCH; 996*cd1c8b85SMatthew Ahrens ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data; 997*cd1c8b85SMatthew Ahrens ctl.prwatch.pr_size = buf->b_hdr->b_size; 998*cd1c8b85SMatthew Ahrens ctl.prwatch.pr_wflags = WA_WRITE; 999*cd1c8b85SMatthew Ahrens result = write(arc_procfd, &ctl, sizeof (ctl)); 1000*cd1c8b85SMatthew Ahrens ASSERT3U(result, ==, sizeof (ctl)); 1001*cd1c8b85SMatthew Ahrens } 1002*cd1c8b85SMatthew Ahrens #endif 10036b4acc8bSahrens } 10046b4acc8bSahrens 10056b4acc8bSahrens void 10066b4acc8bSahrens arc_buf_thaw(arc_buf_t *buf) 10076b4acc8bSahrens { 1008fa94a07fSbrendan if (zfs_flags & ZFS_DEBUG_MODIFY) { 1009fa94a07fSbrendan if (buf->b_hdr->b_state != arc_anon) 1010fa94a07fSbrendan panic("modifying non-anon buffer!"); 1011fa94a07fSbrendan if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS) 1012fa94a07fSbrendan panic("modifying buffer while i/o in progress!"); 1013fa94a07fSbrendan arc_cksum_verify(buf); 1014fa94a07fSbrendan } 10156b4acc8bSahrens 10166b4acc8bSahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 10176b4acc8bSahrens if (buf->b_hdr->b_freeze_cksum != NULL) { 10186b4acc8bSahrens kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 10196b4acc8bSahrens buf->b_hdr->b_freeze_cksum = NULL; 10206b4acc8bSahrens } 10213f9d6ad7SLin Ling 10223f9d6ad7SLin Ling if (zfs_flags & ZFS_DEBUG_MODIFY) { 10233f9d6ad7SLin Ling if (buf->b_hdr->b_thawed) 10243f9d6ad7SLin Ling kmem_free(buf->b_hdr->b_thawed, 1); 10253f9d6ad7SLin Ling buf->b_hdr->b_thawed = kmem_alloc(1, KM_SLEEP); 10263f9d6ad7SLin Ling } 10273f9d6ad7SLin Ling 10286b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 1029*cd1c8b85SMatthew Ahrens 1030*cd1c8b85SMatthew Ahrens arc_buf_unwatch(buf); 10316b4acc8bSahrens } 10326b4acc8bSahrens 10336b4acc8bSahrens void 10346b4acc8bSahrens arc_buf_freeze(arc_buf_t *buf) 10356b4acc8bSahrens { 10363f9d6ad7SLin Ling kmutex_t *hash_lock; 10373f9d6ad7SLin Ling 1038cc60fd72Sahrens if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 1039cc60fd72Sahrens return; 1040cc60fd72Sahrens 10413f9d6ad7SLin Ling hash_lock = HDR_LOCK(buf->b_hdr); 10423f9d6ad7SLin Ling mutex_enter(hash_lock); 10433f9d6ad7SLin Ling 10446b4acc8bSahrens ASSERT(buf->b_hdr->b_freeze_cksum != NULL || 104544cb6abcSbmc buf->b_hdr->b_state == arc_anon); 1046fa94a07fSbrendan arc_cksum_compute(buf, B_FALSE); 10473f9d6ad7SLin Ling mutex_exit(hash_lock); 1048*cd1c8b85SMatthew Ahrens 10496b4acc8bSahrens } 10506b4acc8bSahrens 1051fa9e4066Sahrens static void 1052fa9e4066Sahrens add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 1053fa9e4066Sahrens { 1054fa9e4066Sahrens ASSERT(MUTEX_HELD(hash_lock)); 1055fa9e4066Sahrens 1056fa9e4066Sahrens if ((refcount_add(&ab->b_refcnt, tag) == 1) && 105744cb6abcSbmc (ab->b_state != arc_anon)) { 1058c0a81264Sek uint64_t delta = ab->b_size * ab->b_datacnt; 10590e8c6158Smaybee list_t *list = &ab->b_state->arcs_list[ab->b_type]; 10600e8c6158Smaybee uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type]; 1061fa9e4066Sahrens 106244cb6abcSbmc ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx)); 106344cb6abcSbmc mutex_enter(&ab->b_state->arcs_mtx); 1064fa9e4066Sahrens ASSERT(list_link_active(&ab->b_arc_node)); 10650e8c6158Smaybee list_remove(list, ab); 1066ea8dc4b6Seschrock if (GHOST_STATE(ab->b_state)) { 1067b420f3adSRichard Lowe ASSERT3U(ab->b_datacnt, ==, 0); 1068ea8dc4b6Seschrock ASSERT3P(ab->b_buf, ==, NULL); 1069ea8dc4b6Seschrock delta = ab->b_size; 1070ea8dc4b6Seschrock } 1071ea8dc4b6Seschrock ASSERT(delta > 0); 10720e8c6158Smaybee ASSERT3U(*size, >=, delta); 10730e8c6158Smaybee atomic_add_64(size, -delta); 107444cb6abcSbmc mutex_exit(&ab->b_state->arcs_mtx); 1075088f3894Sahrens /* remove the prefetch flag if we get a reference */ 107613506d1eSmaybee if (ab->b_flags & ARC_PREFETCH) 107713506d1eSmaybee ab->b_flags &= ~ARC_PREFETCH; 1078fa9e4066Sahrens } 1079fa9e4066Sahrens } 1080fa9e4066Sahrens 1081fa9e4066Sahrens static int 1082fa9e4066Sahrens remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 1083fa9e4066Sahrens { 1084fa9e4066Sahrens int cnt; 108544cb6abcSbmc arc_state_t *state = ab->b_state; 1086fa9e4066Sahrens 108744cb6abcSbmc ASSERT(state == arc_anon || MUTEX_HELD(hash_lock)); 108844cb6abcSbmc ASSERT(!GHOST_STATE(state)); 1089fa9e4066Sahrens 1090fa9e4066Sahrens if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) && 109144cb6abcSbmc (state != arc_anon)) { 10920e8c6158Smaybee uint64_t *size = &state->arcs_lsize[ab->b_type]; 10930e8c6158Smaybee 109444cb6abcSbmc ASSERT(!MUTEX_HELD(&state->arcs_mtx)); 109544cb6abcSbmc mutex_enter(&state->arcs_mtx); 1096fa9e4066Sahrens ASSERT(!list_link_active(&ab->b_arc_node)); 10970e8c6158Smaybee list_insert_head(&state->arcs_list[ab->b_type], ab); 1098ea8dc4b6Seschrock ASSERT(ab->b_datacnt > 0); 10990e8c6158Smaybee atomic_add_64(size, ab->b_size * ab->b_datacnt); 110044cb6abcSbmc mutex_exit(&state->arcs_mtx); 1101fa9e4066Sahrens } 1102fa9e4066Sahrens return (cnt); 1103fa9e4066Sahrens } 1104fa9e4066Sahrens 1105fa9e4066Sahrens /* 1106fa9e4066Sahrens * Move the supplied buffer to the indicated state. The mutex 1107fa9e4066Sahrens * for the buffer must be held by the caller. 1108fa9e4066Sahrens */ 1109fa9e4066Sahrens static void 1110ea8dc4b6Seschrock arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock) 1111fa9e4066Sahrens { 1112ea8dc4b6Seschrock arc_state_t *old_state = ab->b_state; 1113c0a81264Sek int64_t refcnt = refcount_count(&ab->b_refcnt); 1114c0a81264Sek uint64_t from_delta, to_delta; 1115fa9e4066Sahrens 1116fa9e4066Sahrens ASSERT(MUTEX_HELD(hash_lock)); 1117ea8dc4b6Seschrock ASSERT(new_state != old_state); 1118ea8dc4b6Seschrock ASSERT(refcnt == 0 || ab->b_datacnt > 0); 1119ea8dc4b6Seschrock ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state)); 1120b24ab676SJeff Bonwick ASSERT(ab->b_datacnt <= 1 || old_state != arc_anon); 1121ea8dc4b6Seschrock 1122ea8dc4b6Seschrock from_delta = to_delta = ab->b_datacnt * ab->b_size; 1123fa9e4066Sahrens 1124fa9e4066Sahrens /* 1125fa9e4066Sahrens * If this buffer is evictable, transfer it from the 1126fa9e4066Sahrens * old state list to the new state list. 1127fa9e4066Sahrens */ 1128ea8dc4b6Seschrock if (refcnt == 0) { 112944cb6abcSbmc if (old_state != arc_anon) { 113044cb6abcSbmc int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx); 11310e8c6158Smaybee uint64_t *size = &old_state->arcs_lsize[ab->b_type]; 1132ea8dc4b6Seschrock 1133ea8dc4b6Seschrock if (use_mutex) 113444cb6abcSbmc mutex_enter(&old_state->arcs_mtx); 1135fa9e4066Sahrens 1136fa9e4066Sahrens ASSERT(list_link_active(&ab->b_arc_node)); 11370e8c6158Smaybee list_remove(&old_state->arcs_list[ab->b_type], ab); 1138ea8dc4b6Seschrock 113913506d1eSmaybee /* 114013506d1eSmaybee * If prefetching out of the ghost cache, 11413f9d6ad7SLin Ling * we will have a non-zero datacnt. 114213506d1eSmaybee */ 114313506d1eSmaybee if (GHOST_STATE(old_state) && ab->b_datacnt == 0) { 114413506d1eSmaybee /* ghost elements have a ghost size */ 1145ea8dc4b6Seschrock ASSERT(ab->b_buf == NULL); 1146ea8dc4b6Seschrock from_delta = ab->b_size; 1147ea8dc4b6Seschrock } 11480e8c6158Smaybee ASSERT3U(*size, >=, from_delta); 11490e8c6158Smaybee atomic_add_64(size, -from_delta); 1150ea8dc4b6Seschrock 1151ea8dc4b6Seschrock if (use_mutex) 115244cb6abcSbmc mutex_exit(&old_state->arcs_mtx); 1153fa9e4066Sahrens } 115444cb6abcSbmc if (new_state != arc_anon) { 115544cb6abcSbmc int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx); 11560e8c6158Smaybee uint64_t *size = &new_state->arcs_lsize[ab->b_type]; 1157fa9e4066Sahrens 1158ea8dc4b6Seschrock if (use_mutex) 115944cb6abcSbmc mutex_enter(&new_state->arcs_mtx); 1160ea8dc4b6Seschrock 11610e8c6158Smaybee list_insert_head(&new_state->arcs_list[ab->b_type], ab); 1162ea8dc4b6Seschrock 1163ea8dc4b6Seschrock /* ghost elements have a ghost size */ 1164ea8dc4b6Seschrock if (GHOST_STATE(new_state)) { 1165ea8dc4b6Seschrock ASSERT(ab->b_datacnt == 0); 1166ea8dc4b6Seschrock ASSERT(ab->b_buf == NULL); 1167ea8dc4b6Seschrock to_delta = ab->b_size; 1168ea8dc4b6Seschrock } 11690e8c6158Smaybee atomic_add_64(size, to_delta); 1170ea8dc4b6Seschrock 1171ea8dc4b6Seschrock if (use_mutex) 117244cb6abcSbmc mutex_exit(&new_state->arcs_mtx); 1173fa9e4066Sahrens } 1174fa9e4066Sahrens } 1175fa9e4066Sahrens 1176fa9e4066Sahrens ASSERT(!BUF_EMPTY(ab)); 11773f9d6ad7SLin Ling if (new_state == arc_anon && HDR_IN_HASH_TABLE(ab)) 1178fa9e4066Sahrens buf_hash_remove(ab); 1179fa9e4066Sahrens 1180ea8dc4b6Seschrock /* adjust state sizes */ 1181ea8dc4b6Seschrock if (to_delta) 118244cb6abcSbmc atomic_add_64(&new_state->arcs_size, to_delta); 1183ea8dc4b6Seschrock if (from_delta) { 118444cb6abcSbmc ASSERT3U(old_state->arcs_size, >=, from_delta); 118544cb6abcSbmc atomic_add_64(&old_state->arcs_size, -from_delta); 1186fa9e4066Sahrens } 1187fa9e4066Sahrens ab->b_state = new_state; 1188fa94a07fSbrendan 1189fa94a07fSbrendan /* adjust l2arc hdr stats */ 1190fa94a07fSbrendan if (new_state == arc_l2c_only) 1191fa94a07fSbrendan l2arc_hdr_stat_add(); 1192fa94a07fSbrendan else if (old_state == arc_l2c_only) 1193fa94a07fSbrendan l2arc_hdr_stat_remove(); 1194fa9e4066Sahrens } 1195fa9e4066Sahrens 11960e8c6158Smaybee void 11975a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(uint64_t space, arc_space_type_t type) 11980e8c6158Smaybee { 11995a98e54bSBrendan Gregg - Sun Microsystems ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 12005a98e54bSBrendan Gregg - Sun Microsystems 12015a98e54bSBrendan Gregg - Sun Microsystems switch (type) { 12025a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_DATA: 12035a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, space); 12045a98e54bSBrendan Gregg - Sun Microsystems break; 12055a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_OTHER: 12065a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_other_size, space); 12075a98e54bSBrendan Gregg - Sun Microsystems break; 12085a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_HDRS: 12095a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_hdr_size, space); 12105a98e54bSBrendan Gregg - Sun Microsystems break; 12115a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_L2HDRS: 12125a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_l2_hdr_size, space); 12135a98e54bSBrendan Gregg - Sun Microsystems break; 12145a98e54bSBrendan Gregg - Sun Microsystems } 12155a98e54bSBrendan Gregg - Sun Microsystems 12160e8c6158Smaybee atomic_add_64(&arc_meta_used, space); 12170e8c6158Smaybee atomic_add_64(&arc_size, space); 12180e8c6158Smaybee } 12190e8c6158Smaybee 12200e8c6158Smaybee void 12215a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(uint64_t space, arc_space_type_t type) 12220e8c6158Smaybee { 12235a98e54bSBrendan Gregg - Sun Microsystems ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 12245a98e54bSBrendan Gregg - Sun Microsystems 12255a98e54bSBrendan Gregg - Sun Microsystems switch (type) { 12265a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_DATA: 12275a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, -space); 12285a98e54bSBrendan Gregg - Sun Microsystems break; 12295a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_OTHER: 12305a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_other_size, -space); 12315a98e54bSBrendan Gregg - Sun Microsystems break; 12325a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_HDRS: 12335a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_hdr_size, -space); 12345a98e54bSBrendan Gregg - Sun Microsystems break; 12355a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_L2HDRS: 12365a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_l2_hdr_size, -space); 12375a98e54bSBrendan Gregg - Sun Microsystems break; 12385a98e54bSBrendan Gregg - Sun Microsystems } 12395a98e54bSBrendan Gregg - Sun Microsystems 12400e8c6158Smaybee ASSERT(arc_meta_used >= space); 12410e8c6158Smaybee if (arc_meta_max < arc_meta_used) 12420e8c6158Smaybee arc_meta_max = arc_meta_used; 12430e8c6158Smaybee atomic_add_64(&arc_meta_used, -space); 12440e8c6158Smaybee ASSERT(arc_size >= space); 12450e8c6158Smaybee atomic_add_64(&arc_size, -space); 12460e8c6158Smaybee } 12470e8c6158Smaybee 12480e8c6158Smaybee void * 12490e8c6158Smaybee arc_data_buf_alloc(uint64_t size) 12500e8c6158Smaybee { 12510e8c6158Smaybee if (arc_evict_needed(ARC_BUFC_DATA)) 12520e8c6158Smaybee cv_signal(&arc_reclaim_thr_cv); 12530e8c6158Smaybee atomic_add_64(&arc_size, size); 12540e8c6158Smaybee return (zio_data_buf_alloc(size)); 12550e8c6158Smaybee } 12560e8c6158Smaybee 12570e8c6158Smaybee void 12580e8c6158Smaybee arc_data_buf_free(void *buf, uint64_t size) 12590e8c6158Smaybee { 12600e8c6158Smaybee zio_data_buf_free(buf, size); 12610e8c6158Smaybee ASSERT(arc_size >= size); 12620e8c6158Smaybee atomic_add_64(&arc_size, -size); 12630e8c6158Smaybee } 12640e8c6158Smaybee 1265fa9e4066Sahrens arc_buf_t * 1266ad23a2dbSjohansen arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type) 1267fa9e4066Sahrens { 1268fa9e4066Sahrens arc_buf_hdr_t *hdr; 1269fa9e4066Sahrens arc_buf_t *buf; 1270fa9e4066Sahrens 1271fa9e4066Sahrens ASSERT3U(size, >, 0); 12721ab7f2deSmaybee hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 1273fa9e4066Sahrens ASSERT(BUF_EMPTY(hdr)); 1274fa9e4066Sahrens hdr->b_size = size; 1275ad23a2dbSjohansen hdr->b_type = type; 1276e9103aaeSGarrett D'Amore hdr->b_spa = spa_load_guid(spa); 127744cb6abcSbmc hdr->b_state = arc_anon; 1278fa9e4066Sahrens hdr->b_arc_access = 0; 12791ab7f2deSmaybee buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 1280fa9e4066Sahrens buf->b_hdr = hdr; 128144eda4d7Smaybee buf->b_data = NULL; 1282ea8dc4b6Seschrock buf->b_efunc = NULL; 1283ea8dc4b6Seschrock buf->b_private = NULL; 1284fa9e4066Sahrens buf->b_next = NULL; 1285fa9e4066Sahrens hdr->b_buf = buf; 128644eda4d7Smaybee arc_get_data_buf(buf); 1287ea8dc4b6Seschrock hdr->b_datacnt = 1; 1288fa9e4066Sahrens hdr->b_flags = 0; 1289fa9e4066Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1290fa9e4066Sahrens (void) refcount_add(&hdr->b_refcnt, tag); 1291fa9e4066Sahrens 1292fa9e4066Sahrens return (buf); 1293fa9e4066Sahrens } 1294fa9e4066Sahrens 12952fdbea25SAleksandr Guzovskiy static char *arc_onloan_tag = "onloan"; 12962fdbea25SAleksandr Guzovskiy 12972fdbea25SAleksandr Guzovskiy /* 12982fdbea25SAleksandr Guzovskiy * Loan out an anonymous arc buffer. Loaned buffers are not counted as in 12992fdbea25SAleksandr Guzovskiy * flight data by arc_tempreserve_space() until they are "returned". Loaned 13002fdbea25SAleksandr Guzovskiy * buffers must be returned to the arc before they can be used by the DMU or 13012fdbea25SAleksandr Guzovskiy * freed. 13022fdbea25SAleksandr Guzovskiy */ 13032fdbea25SAleksandr Guzovskiy arc_buf_t * 13042fdbea25SAleksandr Guzovskiy arc_loan_buf(spa_t *spa, int size) 13052fdbea25SAleksandr Guzovskiy { 13062fdbea25SAleksandr Guzovskiy arc_buf_t *buf; 13072fdbea25SAleksandr Guzovskiy 13082fdbea25SAleksandr Guzovskiy buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA); 13092fdbea25SAleksandr Guzovskiy 13102fdbea25SAleksandr Guzovskiy atomic_add_64(&arc_loaned_bytes, size); 13112fdbea25SAleksandr Guzovskiy return (buf); 13122fdbea25SAleksandr Guzovskiy } 13132fdbea25SAleksandr Guzovskiy 13142fdbea25SAleksandr Guzovskiy /* 13152fdbea25SAleksandr Guzovskiy * Return a loaned arc buffer to the arc. 13162fdbea25SAleksandr Guzovskiy */ 13172fdbea25SAleksandr Guzovskiy void 13182fdbea25SAleksandr Guzovskiy arc_return_buf(arc_buf_t *buf, void *tag) 13192fdbea25SAleksandr Guzovskiy { 13202fdbea25SAleksandr Guzovskiy arc_buf_hdr_t *hdr = buf->b_hdr; 13212fdbea25SAleksandr Guzovskiy 13222fdbea25SAleksandr Guzovskiy ASSERT(buf->b_data != NULL); 1323c242f9a0Schunli zhang - Sun Microsystems - Irvine United States (void) refcount_add(&hdr->b_refcnt, tag); 1324c242f9a0Schunli zhang - Sun Microsystems - Irvine United States (void) refcount_remove(&hdr->b_refcnt, arc_onloan_tag); 13252fdbea25SAleksandr Guzovskiy 13262fdbea25SAleksandr Guzovskiy atomic_add_64(&arc_loaned_bytes, -hdr->b_size); 13272fdbea25SAleksandr Guzovskiy } 13282fdbea25SAleksandr Guzovskiy 1329c242f9a0Schunli zhang - Sun Microsystems - Irvine United States /* Detach an arc_buf from a dbuf (tag) */ 1330c242f9a0Schunli zhang - Sun Microsystems - Irvine United States void 1331c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_loan_inuse_buf(arc_buf_t *buf, void *tag) 1332c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 1333c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_hdr_t *hdr; 1334c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1335c242f9a0Schunli zhang - Sun Microsystems - Irvine United States ASSERT(buf->b_data != NULL); 1336c242f9a0Schunli zhang - Sun Microsystems - Irvine United States hdr = buf->b_hdr; 1337c242f9a0Schunli zhang - Sun Microsystems - Irvine United States (void) refcount_add(&hdr->b_refcnt, arc_onloan_tag); 1338c242f9a0Schunli zhang - Sun Microsystems - Irvine United States (void) refcount_remove(&hdr->b_refcnt, tag); 1339c242f9a0Schunli zhang - Sun Microsystems - Irvine United States buf->b_efunc = NULL; 1340c242f9a0Schunli zhang - Sun Microsystems - Irvine United States buf->b_private = NULL; 1341c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1342c242f9a0Schunli zhang - Sun Microsystems - Irvine United States atomic_add_64(&arc_loaned_bytes, hdr->b_size); 1343c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 1344c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 134544eda4d7Smaybee static arc_buf_t * 134644eda4d7Smaybee arc_buf_clone(arc_buf_t *from) 1347ea8dc4b6Seschrock { 134844eda4d7Smaybee arc_buf_t *buf; 134944eda4d7Smaybee arc_buf_hdr_t *hdr = from->b_hdr; 135044eda4d7Smaybee uint64_t size = hdr->b_size; 1351ea8dc4b6Seschrock 1352b24ab676SJeff Bonwick ASSERT(hdr->b_state != arc_anon); 1353b24ab676SJeff Bonwick 13541ab7f2deSmaybee buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 135544eda4d7Smaybee buf->b_hdr = hdr; 135644eda4d7Smaybee buf->b_data = NULL; 135744eda4d7Smaybee buf->b_efunc = NULL; 135844eda4d7Smaybee buf->b_private = NULL; 135944eda4d7Smaybee buf->b_next = hdr->b_buf; 136044eda4d7Smaybee hdr->b_buf = buf; 136144eda4d7Smaybee arc_get_data_buf(buf); 136244eda4d7Smaybee bcopy(from->b_data, buf->b_data, size); 136344eda4d7Smaybee hdr->b_datacnt += 1; 136444eda4d7Smaybee return (buf); 1365ea8dc4b6Seschrock } 1366ea8dc4b6Seschrock 1367ea8dc4b6Seschrock void 1368ea8dc4b6Seschrock arc_buf_add_ref(arc_buf_t *buf, void* tag) 1369ea8dc4b6Seschrock { 137040d7d650Smaybee arc_buf_hdr_t *hdr; 1371ea8dc4b6Seschrock kmutex_t *hash_lock; 1372ea8dc4b6Seschrock 13739b23f181Smaybee /* 13746f83844dSMark Maybee * Check to see if this buffer is evicted. Callers 13756f83844dSMark Maybee * must verify b_data != NULL to know if the add_ref 13766f83844dSMark Maybee * was successful. 13779b23f181Smaybee */ 13783f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 13796f83844dSMark Maybee if (buf->b_data == NULL) { 13803f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 13819b23f181Smaybee return; 138240d7d650Smaybee } 13833f9d6ad7SLin Ling hash_lock = HDR_LOCK(buf->b_hdr); 13849b23f181Smaybee mutex_enter(hash_lock); 13853f9d6ad7SLin Ling hdr = buf->b_hdr; 13863f9d6ad7SLin Ling ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 13873f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 1388ea8dc4b6Seschrock 138944cb6abcSbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 1390ea8dc4b6Seschrock add_reference(hdr, hash_lock, tag); 13915a98e54bSBrendan Gregg - Sun Microsystems DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 139244eda4d7Smaybee arc_access(hdr, hash_lock); 139344eda4d7Smaybee mutex_exit(hash_lock); 139444cb6abcSbmc ARCSTAT_BUMP(arcstat_hits); 139544cb6abcSbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 139644cb6abcSbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 139744cb6abcSbmc data, metadata, hits); 1398ea8dc4b6Seschrock } 1399ea8dc4b6Seschrock 1400fa94a07fSbrendan /* 1401fa94a07fSbrendan * Free the arc data buffer. If it is an l2arc write in progress, 1402fa94a07fSbrendan * the buffer is placed on l2arc_free_on_write to be freed later. 1403fa94a07fSbrendan */ 1404fa94a07fSbrendan static void 1405*cd1c8b85SMatthew Ahrens arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t)) 1406fa94a07fSbrendan { 1407*cd1c8b85SMatthew Ahrens arc_buf_hdr_t *hdr = buf->b_hdr; 1408*cd1c8b85SMatthew Ahrens 1409fa94a07fSbrendan if (HDR_L2_WRITING(hdr)) { 1410fa94a07fSbrendan l2arc_data_free_t *df; 1411fa94a07fSbrendan df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP); 1412*cd1c8b85SMatthew Ahrens df->l2df_data = buf->b_data; 1413*cd1c8b85SMatthew Ahrens df->l2df_size = hdr->b_size; 1414fa94a07fSbrendan df->l2df_func = free_func; 1415fa94a07fSbrendan mutex_enter(&l2arc_free_on_write_mtx); 1416fa94a07fSbrendan list_insert_head(l2arc_free_on_write, df); 1417fa94a07fSbrendan mutex_exit(&l2arc_free_on_write_mtx); 1418fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_free_on_write); 1419fa94a07fSbrendan } else { 1420*cd1c8b85SMatthew Ahrens free_func(buf->b_data, hdr->b_size); 1421fa94a07fSbrendan } 1422fa94a07fSbrendan } 1423fa94a07fSbrendan 1424ea8dc4b6Seschrock static void 142544eda4d7Smaybee arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all) 1426ea8dc4b6Seschrock { 1427ea8dc4b6Seschrock arc_buf_t **bufp; 1428ea8dc4b6Seschrock 1429ea8dc4b6Seschrock /* free up data associated with the buf */ 1430ea8dc4b6Seschrock if (buf->b_data) { 1431ea8dc4b6Seschrock arc_state_t *state = buf->b_hdr->b_state; 1432ea8dc4b6Seschrock uint64_t size = buf->b_hdr->b_size; 1433ad23a2dbSjohansen arc_buf_contents_t type = buf->b_hdr->b_type; 1434ea8dc4b6Seschrock 14356b4acc8bSahrens arc_cksum_verify(buf); 1436*cd1c8b85SMatthew Ahrens arc_buf_unwatch(buf); 1437b24ab676SJeff Bonwick 143844eda4d7Smaybee if (!recycle) { 1439ad23a2dbSjohansen if (type == ARC_BUFC_METADATA) { 1440*cd1c8b85SMatthew Ahrens arc_buf_data_free(buf, zio_buf_free); 14415a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(size, ARC_SPACE_DATA); 1442ad23a2dbSjohansen } else { 1443ad23a2dbSjohansen ASSERT(type == ARC_BUFC_DATA); 1444*cd1c8b85SMatthew Ahrens arc_buf_data_free(buf, zio_data_buf_free); 14455a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, -size); 14460e8c6158Smaybee atomic_add_64(&arc_size, -size); 1447ad23a2dbSjohansen } 144844eda4d7Smaybee } 1449ea8dc4b6Seschrock if (list_link_active(&buf->b_hdr->b_arc_node)) { 14500e8c6158Smaybee uint64_t *cnt = &state->arcs_lsize[type]; 14510e8c6158Smaybee 1452ea8dc4b6Seschrock ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt)); 145344cb6abcSbmc ASSERT(state != arc_anon); 14540e8c6158Smaybee 14550e8c6158Smaybee ASSERT3U(*cnt, >=, size); 14560e8c6158Smaybee atomic_add_64(cnt, -size); 1457ea8dc4b6Seschrock } 145844cb6abcSbmc ASSERT3U(state->arcs_size, >=, size); 145944cb6abcSbmc atomic_add_64(&state->arcs_size, -size); 1460ea8dc4b6Seschrock buf->b_data = NULL; 1461ea8dc4b6Seschrock ASSERT(buf->b_hdr->b_datacnt > 0); 1462ea8dc4b6Seschrock buf->b_hdr->b_datacnt -= 1; 1463ea8dc4b6Seschrock } 1464ea8dc4b6Seschrock 1465ea8dc4b6Seschrock /* only remove the buf if requested */ 1466ea8dc4b6Seschrock if (!all) 1467ea8dc4b6Seschrock return; 1468ea8dc4b6Seschrock 1469ea8dc4b6Seschrock /* remove the buf from the hdr list */ 1470ea8dc4b6Seschrock for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next) 1471ea8dc4b6Seschrock continue; 1472ea8dc4b6Seschrock *bufp = buf->b_next; 14733f9d6ad7SLin Ling buf->b_next = NULL; 1474ea8dc4b6Seschrock 1475ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 1476ea8dc4b6Seschrock 1477ea8dc4b6Seschrock /* clean up the buf */ 1478ea8dc4b6Seschrock buf->b_hdr = NULL; 1479ea8dc4b6Seschrock kmem_cache_free(buf_cache, buf); 1480ea8dc4b6Seschrock } 1481ea8dc4b6Seschrock 1482fa9e4066Sahrens static void 1483ea8dc4b6Seschrock arc_hdr_destroy(arc_buf_hdr_t *hdr) 1484fa9e4066Sahrens { 1485fa9e4066Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt)); 148644cb6abcSbmc ASSERT3P(hdr->b_state, ==, arc_anon); 1487ea8dc4b6Seschrock ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 1488b24ab676SJeff Bonwick l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr; 1489fa9e4066Sahrens 1490b24ab676SJeff Bonwick if (l2hdr != NULL) { 1491b24ab676SJeff Bonwick boolean_t buflist_held = MUTEX_HELD(&l2arc_buflist_mtx); 1492b24ab676SJeff Bonwick /* 1493b24ab676SJeff Bonwick * To prevent arc_free() and l2arc_evict() from 1494b24ab676SJeff Bonwick * attempting to free the same buffer at the same time, 1495b24ab676SJeff Bonwick * a FREE_IN_PROGRESS flag is given to arc_free() to 1496b24ab676SJeff Bonwick * give it priority. l2arc_evict() can't destroy this 1497b24ab676SJeff Bonwick * header while we are waiting on l2arc_buflist_mtx. 1498b24ab676SJeff Bonwick * 1499b24ab676SJeff Bonwick * The hdr may be removed from l2ad_buflist before we 1500b24ab676SJeff Bonwick * grab l2arc_buflist_mtx, so b_l2hdr is rechecked. 1501b24ab676SJeff Bonwick */ 1502b24ab676SJeff Bonwick if (!buflist_held) { 1503fa94a07fSbrendan mutex_enter(&l2arc_buflist_mtx); 1504b24ab676SJeff Bonwick l2hdr = hdr->b_l2hdr; 1505fa94a07fSbrendan } 1506b24ab676SJeff Bonwick 1507b24ab676SJeff Bonwick if (l2hdr != NULL) { 1508b24ab676SJeff Bonwick list_remove(l2hdr->b_dev->l2ad_buflist, hdr); 1509b24ab676SJeff Bonwick ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size); 1510b24ab676SJeff Bonwick kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t)); 1511b24ab676SJeff Bonwick if (hdr->b_state == arc_l2c_only) 1512b24ab676SJeff Bonwick l2arc_hdr_stat_remove(); 1513b24ab676SJeff Bonwick hdr->b_l2hdr = NULL; 1514b24ab676SJeff Bonwick } 1515b24ab676SJeff Bonwick 1516b24ab676SJeff Bonwick if (!buflist_held) 1517b24ab676SJeff Bonwick mutex_exit(&l2arc_buflist_mtx); 1518fa94a07fSbrendan } 1519fa94a07fSbrendan 1520fa9e4066Sahrens if (!BUF_EMPTY(hdr)) { 1521ea8dc4b6Seschrock ASSERT(!HDR_IN_HASH_TABLE(hdr)); 15223f9d6ad7SLin Ling buf_discard_identity(hdr); 1523fa9e4066Sahrens } 1524ea8dc4b6Seschrock while (hdr->b_buf) { 1525fa9e4066Sahrens arc_buf_t *buf = hdr->b_buf; 1526fa9e4066Sahrens 1527ea8dc4b6Seschrock if (buf->b_efunc) { 1528ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 15293f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 1530ea8dc4b6Seschrock ASSERT(buf->b_hdr != NULL); 153144eda4d7Smaybee arc_buf_destroy(hdr->b_buf, FALSE, FALSE); 1532ea8dc4b6Seschrock hdr->b_buf = buf->b_next; 153340d7d650Smaybee buf->b_hdr = &arc_eviction_hdr; 1534ea8dc4b6Seschrock buf->b_next = arc_eviction_list; 1535ea8dc4b6Seschrock arc_eviction_list = buf; 15363f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 1537ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1538ea8dc4b6Seschrock } else { 153944eda4d7Smaybee arc_buf_destroy(hdr->b_buf, FALSE, TRUE); 1540ea8dc4b6Seschrock } 1541fa9e4066Sahrens } 15426b4acc8bSahrens if (hdr->b_freeze_cksum != NULL) { 15436b4acc8bSahrens kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 15446b4acc8bSahrens hdr->b_freeze_cksum = NULL; 15456b4acc8bSahrens } 15463f9d6ad7SLin Ling if (hdr->b_thawed) { 15473f9d6ad7SLin Ling kmem_free(hdr->b_thawed, 1); 15483f9d6ad7SLin Ling hdr->b_thawed = NULL; 15493f9d6ad7SLin Ling } 1550ea8dc4b6Seschrock 1551fa9e4066Sahrens ASSERT(!list_link_active(&hdr->b_arc_node)); 1552fa9e4066Sahrens ASSERT3P(hdr->b_hash_next, ==, NULL); 1553fa9e4066Sahrens ASSERT3P(hdr->b_acb, ==, NULL); 1554fa9e4066Sahrens kmem_cache_free(hdr_cache, hdr); 1555fa9e4066Sahrens } 1556fa9e4066Sahrens 1557fa9e4066Sahrens void 1558fa9e4066Sahrens arc_buf_free(arc_buf_t *buf, void *tag) 1559fa9e4066Sahrens { 1560fa9e4066Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 156144cb6abcSbmc int hashed = hdr->b_state != arc_anon; 1562fa9e4066Sahrens 1563ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 1564ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 1565ea8dc4b6Seschrock 1566ea8dc4b6Seschrock if (hashed) { 1567ea8dc4b6Seschrock kmutex_t *hash_lock = HDR_LOCK(hdr); 1568ea8dc4b6Seschrock 1569ea8dc4b6Seschrock mutex_enter(hash_lock); 15703f9d6ad7SLin Ling hdr = buf->b_hdr; 15713f9d6ad7SLin Ling ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 15723f9d6ad7SLin Ling 1573ea8dc4b6Seschrock (void) remove_reference(hdr, hash_lock, tag); 1574b24ab676SJeff Bonwick if (hdr->b_datacnt > 1) { 157544eda4d7Smaybee arc_buf_destroy(buf, FALSE, TRUE); 1576b24ab676SJeff Bonwick } else { 1577b24ab676SJeff Bonwick ASSERT(buf == hdr->b_buf); 1578b24ab676SJeff Bonwick ASSERT(buf->b_efunc == NULL); 1579ea8dc4b6Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 1580b24ab676SJeff Bonwick } 1581fa9e4066Sahrens mutex_exit(hash_lock); 1582ea8dc4b6Seschrock } else if (HDR_IO_IN_PROGRESS(hdr)) { 1583ea8dc4b6Seschrock int destroy_hdr; 1584ea8dc4b6Seschrock /* 1585ea8dc4b6Seschrock * We are in the middle of an async write. Don't destroy 1586ea8dc4b6Seschrock * this buffer unless the write completes before we finish 1587ea8dc4b6Seschrock * decrementing the reference count. 1588ea8dc4b6Seschrock */ 1589ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 1590ea8dc4b6Seschrock (void) remove_reference(hdr, NULL, tag); 1591ea8dc4b6Seschrock ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1592ea8dc4b6Seschrock destroy_hdr = !HDR_IO_IN_PROGRESS(hdr); 1593ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1594ea8dc4b6Seschrock if (destroy_hdr) 1595ea8dc4b6Seschrock arc_hdr_destroy(hdr); 1596ea8dc4b6Seschrock } else { 15973f9d6ad7SLin Ling if (remove_reference(hdr, NULL, tag) > 0) 159844eda4d7Smaybee arc_buf_destroy(buf, FALSE, TRUE); 15993f9d6ad7SLin Ling else 1600ea8dc4b6Seschrock arc_hdr_destroy(hdr); 1601fa9e4066Sahrens } 1602ea8dc4b6Seschrock } 1603fa9e4066Sahrens 1604ea8dc4b6Seschrock int 1605ea8dc4b6Seschrock arc_buf_remove_ref(arc_buf_t *buf, void* tag) 1606ea8dc4b6Seschrock { 1607ea8dc4b6Seschrock arc_buf_hdr_t *hdr = buf->b_hdr; 1608ea8dc4b6Seschrock kmutex_t *hash_lock = HDR_LOCK(hdr); 1609ea8dc4b6Seschrock int no_callback = (buf->b_efunc == NULL); 1610fa9e4066Sahrens 161144cb6abcSbmc if (hdr->b_state == arc_anon) { 1612b24ab676SJeff Bonwick ASSERT(hdr->b_datacnt == 1); 1613ea8dc4b6Seschrock arc_buf_free(buf, tag); 1614ea8dc4b6Seschrock return (no_callback); 1615ea8dc4b6Seschrock } 1616ea8dc4b6Seschrock 1617ea8dc4b6Seschrock mutex_enter(hash_lock); 16183f9d6ad7SLin Ling hdr = buf->b_hdr; 16193f9d6ad7SLin Ling ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 162044cb6abcSbmc ASSERT(hdr->b_state != arc_anon); 1621ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 1622ea8dc4b6Seschrock 1623ea8dc4b6Seschrock (void) remove_reference(hdr, hash_lock, tag); 1624ea8dc4b6Seschrock if (hdr->b_datacnt > 1) { 1625ea8dc4b6Seschrock if (no_callback) 162644eda4d7Smaybee arc_buf_destroy(buf, FALSE, TRUE); 1627ea8dc4b6Seschrock } else if (no_callback) { 1628ea8dc4b6Seschrock ASSERT(hdr->b_buf == buf && buf->b_next == NULL); 1629b24ab676SJeff Bonwick ASSERT(buf->b_efunc == NULL); 1630ea8dc4b6Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 1631ea8dc4b6Seschrock } 1632ea8dc4b6Seschrock ASSERT(no_callback || hdr->b_datacnt > 1 || 1633ea8dc4b6Seschrock refcount_is_zero(&hdr->b_refcnt)); 1634ea8dc4b6Seschrock mutex_exit(hash_lock); 1635ea8dc4b6Seschrock return (no_callback); 1636fa9e4066Sahrens } 1637fa9e4066Sahrens 1638fa9e4066Sahrens int 1639fa9e4066Sahrens arc_buf_size(arc_buf_t *buf) 1640fa9e4066Sahrens { 1641fa9e4066Sahrens return (buf->b_hdr->b_size); 1642fa9e4066Sahrens } 1643fa9e4066Sahrens 1644fa9e4066Sahrens /* 1645fa9e4066Sahrens * Evict buffers from list until we've removed the specified number of 1646fa9e4066Sahrens * bytes. Move the removed buffers to the appropriate evict state. 164744eda4d7Smaybee * If the recycle flag is set, then attempt to "recycle" a buffer: 164844eda4d7Smaybee * - look for a buffer to evict that is `bytes' long. 164944eda4d7Smaybee * - return the data block from this buffer rather than freeing it. 165044eda4d7Smaybee * This flag is used by callers that are trying to make space for a 165144eda4d7Smaybee * new buffer in a full arc cache. 1652874395d5Smaybee * 1653874395d5Smaybee * This function makes a "best effort". It skips over any buffers 1654874395d5Smaybee * it can't get a hash_lock on, and so may not catch all candidates. 1655874395d5Smaybee * It may also return without evicting as much space as requested. 1656fa9e4066Sahrens */ 165744eda4d7Smaybee static void * 1658ac05c741SMark Maybee arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle, 1659ad23a2dbSjohansen arc_buf_contents_t type) 1660fa9e4066Sahrens { 1661fa9e4066Sahrens arc_state_t *evicted_state; 166244eda4d7Smaybee uint64_t bytes_evicted = 0, skipped = 0, missed = 0; 16633fa51506Smaybee arc_buf_hdr_t *ab, *ab_prev = NULL; 16640e8c6158Smaybee list_t *list = &state->arcs_list[type]; 1665fa9e4066Sahrens kmutex_t *hash_lock; 166644eda4d7Smaybee boolean_t have_lock; 16673fa51506Smaybee void *stolen = NULL; 1668fa9e4066Sahrens 166944cb6abcSbmc ASSERT(state == arc_mru || state == arc_mfu); 1670fa9e4066Sahrens 167144cb6abcSbmc evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 1672fa9e4066Sahrens 167344cb6abcSbmc mutex_enter(&state->arcs_mtx); 167444cb6abcSbmc mutex_enter(&evicted_state->arcs_mtx); 1675fa9e4066Sahrens 16760e8c6158Smaybee for (ab = list_tail(list); ab; ab = ab_prev) { 16770e8c6158Smaybee ab_prev = list_prev(list, ab); 167813506d1eSmaybee /* prefetch buffers have a minimum lifespan */ 167944eda4d7Smaybee if (HDR_IO_IN_PROGRESS(ab) || 1680874395d5Smaybee (spa && ab->b_spa != spa) || 168144eda4d7Smaybee (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) && 1682d3d50737SRafael Vanoni ddi_get_lbolt() - ab->b_arc_access < 1683d3d50737SRafael Vanoni arc_min_prefetch_lifespan)) { 168413506d1eSmaybee skipped++; 168513506d1eSmaybee continue; 168613506d1eSmaybee } 16873fa51506Smaybee /* "lookahead" for better eviction candidate */ 16883fa51506Smaybee if (recycle && ab->b_size != bytes && 16893fa51506Smaybee ab_prev && ab_prev->b_size == bytes) 169044eda4d7Smaybee continue; 1691fa9e4066Sahrens hash_lock = HDR_LOCK(ab); 169244eda4d7Smaybee have_lock = MUTEX_HELD(hash_lock); 169344eda4d7Smaybee if (have_lock || mutex_tryenter(hash_lock)) { 1694b420f3adSRichard Lowe ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0); 1695ea8dc4b6Seschrock ASSERT(ab->b_datacnt > 0); 1696ea8dc4b6Seschrock while (ab->b_buf) { 1697ea8dc4b6Seschrock arc_buf_t *buf = ab->b_buf; 16983f9d6ad7SLin Ling if (!mutex_tryenter(&buf->b_evict_lock)) { 16996f83844dSMark Maybee missed += 1; 17006f83844dSMark Maybee break; 17016f83844dSMark Maybee } 170244eda4d7Smaybee if (buf->b_data) { 1703ea8dc4b6Seschrock bytes_evicted += ab->b_size; 1704ad23a2dbSjohansen if (recycle && ab->b_type == type && 1705fa94a07fSbrendan ab->b_size == bytes && 1706fa94a07fSbrendan !HDR_L2_WRITING(ab)) { 17073fa51506Smaybee stolen = buf->b_data; 17083fa51506Smaybee recycle = FALSE; 17093fa51506Smaybee } 171044eda4d7Smaybee } 1711ea8dc4b6Seschrock if (buf->b_efunc) { 1712ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 17133fa51506Smaybee arc_buf_destroy(buf, 17143fa51506Smaybee buf->b_data == stolen, FALSE); 1715ea8dc4b6Seschrock ab->b_buf = buf->b_next; 171640d7d650Smaybee buf->b_hdr = &arc_eviction_hdr; 1717ea8dc4b6Seschrock buf->b_next = arc_eviction_list; 1718ea8dc4b6Seschrock arc_eviction_list = buf; 1719ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 17203f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 1721ea8dc4b6Seschrock } else { 17223f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 17233fa51506Smaybee arc_buf_destroy(buf, 17243fa51506Smaybee buf->b_data == stolen, TRUE); 1725ea8dc4b6Seschrock } 1726ea8dc4b6Seschrock } 17275ea40c06SBrendan Gregg - Sun Microsystems 17285ea40c06SBrendan Gregg - Sun Microsystems if (ab->b_l2hdr) { 17295ea40c06SBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_evict_l2_cached, 17305ea40c06SBrendan Gregg - Sun Microsystems ab->b_size); 17315ea40c06SBrendan Gregg - Sun Microsystems } else { 17325ea40c06SBrendan Gregg - Sun Microsystems if (l2arc_write_eligible(ab->b_spa, ab)) { 17335ea40c06SBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_evict_l2_eligible, 17345ea40c06SBrendan Gregg - Sun Microsystems ab->b_size); 17355ea40c06SBrendan Gregg - Sun Microsystems } else { 17365ea40c06SBrendan Gregg - Sun Microsystems ARCSTAT_INCR( 17375ea40c06SBrendan Gregg - Sun Microsystems arcstat_evict_l2_ineligible, 17385ea40c06SBrendan Gregg - Sun Microsystems ab->b_size); 17395ea40c06SBrendan Gregg - Sun Microsystems } 17405ea40c06SBrendan Gregg - Sun Microsystems } 17415ea40c06SBrendan Gregg - Sun Microsystems 17426f83844dSMark Maybee if (ab->b_datacnt == 0) { 17436f83844dSMark Maybee arc_change_state(evicted_state, ab, hash_lock); 17446f83844dSMark Maybee ASSERT(HDR_IN_HASH_TABLE(ab)); 17456f83844dSMark Maybee ab->b_flags |= ARC_IN_HASH_TABLE; 17466f83844dSMark Maybee ab->b_flags &= ~ARC_BUF_AVAILABLE; 17476f83844dSMark Maybee DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab); 17486f83844dSMark Maybee } 174944eda4d7Smaybee if (!have_lock) 175044eda4d7Smaybee mutex_exit(hash_lock); 1751ea8dc4b6Seschrock if (bytes >= 0 && bytes_evicted >= bytes) 1752fa9e4066Sahrens break; 1753fa9e4066Sahrens } else { 175444eda4d7Smaybee missed += 1; 1755fa9e4066Sahrens } 1756fa9e4066Sahrens } 175744cb6abcSbmc 175844cb6abcSbmc mutex_exit(&evicted_state->arcs_mtx); 175944cb6abcSbmc mutex_exit(&state->arcs_mtx); 1760fa9e4066Sahrens 1761fa9e4066Sahrens if (bytes_evicted < bytes) 1762fa9e4066Sahrens dprintf("only evicted %lld bytes from %x", 1763fa9e4066Sahrens (longlong_t)bytes_evicted, state); 1764fa9e4066Sahrens 176544eda4d7Smaybee if (skipped) 176644cb6abcSbmc ARCSTAT_INCR(arcstat_evict_skip, skipped); 176744cb6abcSbmc 176844eda4d7Smaybee if (missed) 176944cb6abcSbmc ARCSTAT_INCR(arcstat_mutex_miss, missed); 1770f4d2e9e6Smaybee 1771f4d2e9e6Smaybee /* 1772f4d2e9e6Smaybee * We have just evicted some date into the ghost state, make 1773f4d2e9e6Smaybee * sure we also adjust the ghost state size if necessary. 1774f4d2e9e6Smaybee */ 1775f4d2e9e6Smaybee if (arc_no_grow && 1776f4d2e9e6Smaybee arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size > arc_c) { 1777f4d2e9e6Smaybee int64_t mru_over = arc_anon->arcs_size + arc_mru->arcs_size + 1778f4d2e9e6Smaybee arc_mru_ghost->arcs_size - arc_c; 1779f4d2e9e6Smaybee 1780f4d2e9e6Smaybee if (mru_over > 0 && arc_mru_ghost->arcs_lsize[type] > 0) { 1781f4d2e9e6Smaybee int64_t todelete = 1782f4d2e9e6Smaybee MIN(arc_mru_ghost->arcs_lsize[type], mru_over); 1783874395d5Smaybee arc_evict_ghost(arc_mru_ghost, NULL, todelete); 1784f4d2e9e6Smaybee } else if (arc_mfu_ghost->arcs_lsize[type] > 0) { 1785f4d2e9e6Smaybee int64_t todelete = MIN(arc_mfu_ghost->arcs_lsize[type], 1786f4d2e9e6Smaybee arc_mru_ghost->arcs_size + 1787f4d2e9e6Smaybee arc_mfu_ghost->arcs_size - arc_c); 1788874395d5Smaybee arc_evict_ghost(arc_mfu_ghost, NULL, todelete); 1789f4d2e9e6Smaybee } 1790f4d2e9e6Smaybee } 179144cb6abcSbmc 17923fa51506Smaybee return (stolen); 1793fa9e4066Sahrens } 1794fa9e4066Sahrens 1795fa9e4066Sahrens /* 1796fa9e4066Sahrens * Remove buffers from list until we've removed the specified number of 1797fa9e4066Sahrens * bytes. Destroy the buffers that are removed. 1798fa9e4066Sahrens */ 1799fa9e4066Sahrens static void 1800ac05c741SMark Maybee arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes) 1801fa9e4066Sahrens { 1802fa9e4066Sahrens arc_buf_hdr_t *ab, *ab_prev; 1803b802aa8cSSanjeev Bagewadi arc_buf_hdr_t marker = { 0 }; 18040e8c6158Smaybee list_t *list = &state->arcs_list[ARC_BUFC_DATA]; 1805fa9e4066Sahrens kmutex_t *hash_lock; 1806ea8dc4b6Seschrock uint64_t bytes_deleted = 0; 1807c0a81264Sek uint64_t bufs_skipped = 0; 1808fa9e4066Sahrens 1809ea8dc4b6Seschrock ASSERT(GHOST_STATE(state)); 1810fa9e4066Sahrens top: 181144cb6abcSbmc mutex_enter(&state->arcs_mtx); 18120e8c6158Smaybee for (ab = list_tail(list); ab; ab = ab_prev) { 18130e8c6158Smaybee ab_prev = list_prev(list, ab); 1814874395d5Smaybee if (spa && ab->b_spa != spa) 1815874395d5Smaybee continue; 1816b802aa8cSSanjeev Bagewadi 1817b802aa8cSSanjeev Bagewadi /* ignore markers */ 1818b802aa8cSSanjeev Bagewadi if (ab->b_spa == 0) 1819b802aa8cSSanjeev Bagewadi continue; 1820b802aa8cSSanjeev Bagewadi 1821fa9e4066Sahrens hash_lock = HDR_LOCK(ab); 18227e453561SWilliam Gorrell /* caller may be trying to modify this buffer, skip it */ 18237e453561SWilliam Gorrell if (MUTEX_HELD(hash_lock)) 18247e453561SWilliam Gorrell continue; 18257e453561SWilliam Gorrell if (mutex_tryenter(hash_lock)) { 182613506d1eSmaybee ASSERT(!HDR_IO_IN_PROGRESS(ab)); 1827ea8dc4b6Seschrock ASSERT(ab->b_buf == NULL); 182844cb6abcSbmc ARCSTAT_BUMP(arcstat_deleted); 1829fa9e4066Sahrens bytes_deleted += ab->b_size; 1830fa94a07fSbrendan 1831fa94a07fSbrendan if (ab->b_l2hdr != NULL) { 1832fa94a07fSbrendan /* 1833fa94a07fSbrendan * This buffer is cached on the 2nd Level ARC; 1834fa94a07fSbrendan * don't destroy the header. 1835fa94a07fSbrendan */ 1836fa94a07fSbrendan arc_change_state(arc_l2c_only, ab, hash_lock); 18377e453561SWilliam Gorrell mutex_exit(hash_lock); 1838fa94a07fSbrendan } else { 1839fa94a07fSbrendan arc_change_state(arc_anon, ab, hash_lock); 18407e453561SWilliam Gorrell mutex_exit(hash_lock); 1841fa94a07fSbrendan arc_hdr_destroy(ab); 1842fa94a07fSbrendan } 1843fa94a07fSbrendan 1844ea8dc4b6Seschrock DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab); 1845fa9e4066Sahrens if (bytes >= 0 && bytes_deleted >= bytes) 1846fa9e4066Sahrens break; 1847b802aa8cSSanjeev Bagewadi } else if (bytes < 0) { 1848b802aa8cSSanjeev Bagewadi /* 1849b802aa8cSSanjeev Bagewadi * Insert a list marker and then wait for the 1850b802aa8cSSanjeev Bagewadi * hash lock to become available. Once its 1851b802aa8cSSanjeev Bagewadi * available, restart from where we left off. 1852b802aa8cSSanjeev Bagewadi */ 1853b802aa8cSSanjeev Bagewadi list_insert_after(list, ab, &marker); 1854b802aa8cSSanjeev Bagewadi mutex_exit(&state->arcs_mtx); 1855b802aa8cSSanjeev Bagewadi mutex_enter(hash_lock); 1856b802aa8cSSanjeev Bagewadi mutex_exit(hash_lock); 1857b802aa8cSSanjeev Bagewadi mutex_enter(&state->arcs_mtx); 1858b802aa8cSSanjeev Bagewadi ab_prev = list_prev(list, &marker); 1859b802aa8cSSanjeev Bagewadi list_remove(list, &marker); 1860b802aa8cSSanjeev Bagewadi } else 1861fa9e4066Sahrens bufs_skipped += 1; 1862fa9e4066Sahrens } 186344cb6abcSbmc mutex_exit(&state->arcs_mtx); 1864fa9e4066Sahrens 18650e8c6158Smaybee if (list == &state->arcs_list[ARC_BUFC_DATA] && 18660e8c6158Smaybee (bytes < 0 || bytes_deleted < bytes)) { 18670e8c6158Smaybee list = &state->arcs_list[ARC_BUFC_METADATA]; 18680e8c6158Smaybee goto top; 18690e8c6158Smaybee } 18700e8c6158Smaybee 1871fa9e4066Sahrens if (bufs_skipped) { 187244cb6abcSbmc ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped); 1873fa9e4066Sahrens ASSERT(bytes >= 0); 1874fa9e4066Sahrens } 1875fa9e4066Sahrens 1876fa9e4066Sahrens if (bytes_deleted < bytes) 1877fa9e4066Sahrens dprintf("only deleted %lld bytes from %p", 1878fa9e4066Sahrens (longlong_t)bytes_deleted, state); 1879fa9e4066Sahrens } 1880fa9e4066Sahrens 1881fa9e4066Sahrens static void 1882fa9e4066Sahrens arc_adjust(void) 1883fa9e4066Sahrens { 18845a98e54bSBrendan Gregg - Sun Microsystems int64_t adjustment, delta; 1885fa9e4066Sahrens 18865a98e54bSBrendan Gregg - Sun Microsystems /* 18875a98e54bSBrendan Gregg - Sun Microsystems * Adjust MRU size 18885a98e54bSBrendan Gregg - Sun Microsystems */ 18895a98e54bSBrendan Gregg - Sun Microsystems 18903e4e8481STom Erickson adjustment = MIN((int64_t)(arc_size - arc_c), 18913e4e8481STom Erickson (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used - 18923e4e8481STom Erickson arc_p)); 1893fa9e4066Sahrens 18945a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) { 18955a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment); 18965a98e54bSBrendan Gregg - Sun Microsystems (void) arc_evict(arc_mru, NULL, delta, FALSE, ARC_BUFC_DATA); 18975a98e54bSBrendan Gregg - Sun Microsystems adjustment -= delta; 18980e8c6158Smaybee } 18990e8c6158Smaybee 19005a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) { 19015a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment); 19025a98e54bSBrendan Gregg - Sun Microsystems (void) arc_evict(arc_mru, NULL, delta, FALSE, 1903874395d5Smaybee ARC_BUFC_METADATA); 1904fa9e4066Sahrens } 1905fa9e4066Sahrens 19065a98e54bSBrendan Gregg - Sun Microsystems /* 19075a98e54bSBrendan Gregg - Sun Microsystems * Adjust MFU size 19085a98e54bSBrendan Gregg - Sun Microsystems */ 1909fa9e4066Sahrens 19105a98e54bSBrendan Gregg - Sun Microsystems adjustment = arc_size - arc_c; 19115a98e54bSBrendan Gregg - Sun Microsystems 19125a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) { 19135a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]); 19145a98e54bSBrendan Gregg - Sun Microsystems (void) arc_evict(arc_mfu, NULL, delta, FALSE, ARC_BUFC_DATA); 19155a98e54bSBrendan Gregg - Sun Microsystems adjustment -= delta; 1916fa9e4066Sahrens } 1917fa9e4066Sahrens 19185a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) { 19195a98e54bSBrendan Gregg - Sun Microsystems int64_t delta = MIN(adjustment, 19205a98e54bSBrendan Gregg - Sun Microsystems arc_mfu->arcs_lsize[ARC_BUFC_METADATA]); 19215a98e54bSBrendan Gregg - Sun Microsystems (void) arc_evict(arc_mfu, NULL, delta, FALSE, 19225a98e54bSBrendan Gregg - Sun Microsystems ARC_BUFC_METADATA); 19235a98e54bSBrendan Gregg - Sun Microsystems } 1924fa9e4066Sahrens 19255a98e54bSBrendan Gregg - Sun Microsystems /* 19265a98e54bSBrendan Gregg - Sun Microsystems * Adjust ghost lists 19275a98e54bSBrendan Gregg - Sun Microsystems */ 1928fa9e4066Sahrens 19295a98e54bSBrendan Gregg - Sun Microsystems adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c; 1930fa9e4066Sahrens 19315a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) { 19325a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(arc_mru_ghost->arcs_size, adjustment); 19335a98e54bSBrendan Gregg - Sun Microsystems arc_evict_ghost(arc_mru_ghost, NULL, delta); 19345a98e54bSBrendan Gregg - Sun Microsystems } 19350e8c6158Smaybee 19365a98e54bSBrendan Gregg - Sun Microsystems adjustment = 19375a98e54bSBrendan Gregg - Sun Microsystems arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c; 19385a98e54bSBrendan Gregg - Sun Microsystems 19395a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) { 19405a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(arc_mfu_ghost->arcs_size, adjustment); 19415a98e54bSBrendan Gregg - Sun Microsystems arc_evict_ghost(arc_mfu_ghost, NULL, delta); 1942fa9e4066Sahrens } 1943fa9e4066Sahrens } 1944fa9e4066Sahrens 1945ea8dc4b6Seschrock static void 1946ea8dc4b6Seschrock arc_do_user_evicts(void) 1947ea8dc4b6Seschrock { 1948ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 1949ea8dc4b6Seschrock while (arc_eviction_list != NULL) { 1950ea8dc4b6Seschrock arc_buf_t *buf = arc_eviction_list; 1951ea8dc4b6Seschrock arc_eviction_list = buf->b_next; 19523f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 1953ea8dc4b6Seschrock buf->b_hdr = NULL; 19543f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 1955ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1956ea8dc4b6Seschrock 1957dd6ef538Smaybee if (buf->b_efunc != NULL) 1958dd6ef538Smaybee VERIFY(buf->b_efunc(buf) == 0); 1959ea8dc4b6Seschrock 1960ea8dc4b6Seschrock buf->b_efunc = NULL; 1961ea8dc4b6Seschrock buf->b_private = NULL; 1962ea8dc4b6Seschrock kmem_cache_free(buf_cache, buf); 1963ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 1964ea8dc4b6Seschrock } 1965ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1966ea8dc4b6Seschrock } 1967ea8dc4b6Seschrock 1968fa9e4066Sahrens /* 1969874395d5Smaybee * Flush all *evictable* data from the cache for the given spa. 1970fa9e4066Sahrens * NOTE: this will not touch "active" (i.e. referenced) data. 1971fa9e4066Sahrens */ 1972fa9e4066Sahrens void 1973874395d5Smaybee arc_flush(spa_t *spa) 1974fa9e4066Sahrens { 1975ac05c741SMark Maybee uint64_t guid = 0; 1976ac05c741SMark Maybee 1977ac05c741SMark Maybee if (spa) 1978e9103aaeSGarrett D'Amore guid = spa_load_guid(spa); 1979ac05c741SMark Maybee 1980874395d5Smaybee while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) { 1981ac05c741SMark Maybee (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA); 1982874395d5Smaybee if (spa) 1983874395d5Smaybee break; 1984874395d5Smaybee } 1985874395d5Smaybee while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) { 1986ac05c741SMark Maybee (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA); 1987874395d5Smaybee if (spa) 1988874395d5Smaybee break; 1989874395d5Smaybee } 1990874395d5Smaybee while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) { 1991ac05c741SMark Maybee (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA); 1992874395d5Smaybee if (spa) 1993874395d5Smaybee break; 1994874395d5Smaybee } 1995874395d5Smaybee while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) { 1996ac05c741SMark Maybee (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA); 1997874395d5Smaybee if (spa) 1998874395d5Smaybee break; 1999874395d5Smaybee } 2000874395d5Smaybee 2001ac05c741SMark Maybee arc_evict_ghost(arc_mru_ghost, guid, -1); 2002ac05c741SMark Maybee arc_evict_ghost(arc_mfu_ghost, guid, -1); 2003ea8dc4b6Seschrock 2004ea8dc4b6Seschrock mutex_enter(&arc_reclaim_thr_lock); 2005ea8dc4b6Seschrock arc_do_user_evicts(); 2006ea8dc4b6Seschrock mutex_exit(&arc_reclaim_thr_lock); 2007874395d5Smaybee ASSERT(spa || arc_eviction_list == NULL); 2008fa9e4066Sahrens } 2009fa9e4066Sahrens 2010fa9e4066Sahrens void 201149e3519aSmaybee arc_shrink(void) 2012fa9e4066Sahrens { 201344cb6abcSbmc if (arc_c > arc_c_min) { 201449e3519aSmaybee uint64_t to_free; 2015fa9e4066Sahrens 20163cff2f43Sstans #ifdef _KERNEL 201744cb6abcSbmc to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree)); 20183cff2f43Sstans #else 201944cb6abcSbmc to_free = arc_c >> arc_shrink_shift; 20203cff2f43Sstans #endif 202144cb6abcSbmc if (arc_c > arc_c_min + to_free) 202244cb6abcSbmc atomic_add_64(&arc_c, -to_free); 202349e3519aSmaybee else 202444cb6abcSbmc arc_c = arc_c_min; 202544cb6abcSbmc 202644cb6abcSbmc atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift)); 202744cb6abcSbmc if (arc_c > arc_size) 202844cb6abcSbmc arc_c = MAX(arc_size, arc_c_min); 202944cb6abcSbmc if (arc_p > arc_c) 203044cb6abcSbmc arc_p = (arc_c >> 1); 203144cb6abcSbmc ASSERT(arc_c >= arc_c_min); 203244cb6abcSbmc ASSERT((int64_t)arc_p >= 0); 203349e3519aSmaybee } 2034fa9e4066Sahrens 203544cb6abcSbmc if (arc_size > arc_c) 203649e3519aSmaybee arc_adjust(); 2037fa9e4066Sahrens } 2038fa9e4066Sahrens 203994dd93aeSGeorge Wilson /* 204094dd93aeSGeorge Wilson * Determine if the system is under memory pressure and is asking 204194dd93aeSGeorge Wilson * to reclaim memory. A return value of 1 indicates that the system 204294dd93aeSGeorge Wilson * is under memory pressure and that the arc should adjust accordingly. 204394dd93aeSGeorge Wilson */ 2044fa9e4066Sahrens static int 2045fa9e4066Sahrens arc_reclaim_needed(void) 2046fa9e4066Sahrens { 2047fa9e4066Sahrens uint64_t extra; 2048fa9e4066Sahrens 2049fa9e4066Sahrens #ifdef _KERNEL 20503cff2f43Sstans 20513cff2f43Sstans if (needfree) 20523cff2f43Sstans return (1); 20533cff2f43Sstans 2054fa9e4066Sahrens /* 2055fa9e4066Sahrens * take 'desfree' extra pages, so we reclaim sooner, rather than later 2056fa9e4066Sahrens */ 2057fa9e4066Sahrens extra = desfree; 2058fa9e4066Sahrens 2059fa9e4066Sahrens /* 2060fa9e4066Sahrens * check that we're out of range of the pageout scanner. It starts to 2061fa9e4066Sahrens * schedule paging if freemem is less than lotsfree and needfree. 2062fa9e4066Sahrens * lotsfree is the high-water mark for pageout, and needfree is the 2063fa9e4066Sahrens * number of needed free pages. We add extra pages here to make sure 2064fa9e4066Sahrens * the scanner doesn't start up while we're freeing memory. 2065fa9e4066Sahrens */ 2066fa9e4066Sahrens if (freemem < lotsfree + needfree + extra) 2067fa9e4066Sahrens return (1); 2068fa9e4066Sahrens 2069fa9e4066Sahrens /* 2070fa9e4066Sahrens * check to make sure that swapfs has enough space so that anon 2071fa94a07fSbrendan * reservations can still succeed. anon_resvmem() checks that the 2072fa9e4066Sahrens * availrmem is greater than swapfs_minfree, and the number of reserved 2073fa9e4066Sahrens * swap pages. We also add a bit of extra here just to prevent 2074fa9e4066Sahrens * circumstances from getting really dire. 2075fa9e4066Sahrens */ 2076fa9e4066Sahrens if (availrmem < swapfs_minfree + swapfs_reserve + extra) 2077fa9e4066Sahrens return (1); 2078fa9e4066Sahrens 20795dc8af33Smaybee #if defined(__i386) 2080fa9e4066Sahrens /* 2081fa9e4066Sahrens * If we're on an i386 platform, it's possible that we'll exhaust the 2082fa9e4066Sahrens * kernel heap space before we ever run out of available physical 2083fa9e4066Sahrens * memory. Most checks of the size of the heap_area compare against 2084fa9e4066Sahrens * tune.t_minarmem, which is the minimum available real memory that we 2085fa9e4066Sahrens * can have in the system. However, this is generally fixed at 25 pages 2086fa9e4066Sahrens * which is so low that it's useless. In this comparison, we seek to 2087fa9e4066Sahrens * calculate the total heap-size, and reclaim if more than 3/4ths of the 2088fa94a07fSbrendan * heap is allocated. (Or, in the calculation, if less than 1/4th is 2089fa9e4066Sahrens * free) 2090fa9e4066Sahrens */ 209194dd93aeSGeorge Wilson if (vmem_size(heap_arena, VMEM_FREE) < 209294dd93aeSGeorge Wilson (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2)) 2093fa9e4066Sahrens return (1); 2094fa9e4066Sahrens #endif 2095fa9e4066Sahrens 209694dd93aeSGeorge Wilson /* 209794dd93aeSGeorge Wilson * If zio data pages are being allocated out of a separate heap segment, 209894dd93aeSGeorge Wilson * then enforce that the size of available vmem for this arena remains 209994dd93aeSGeorge Wilson * above about 1/16th free. 210094dd93aeSGeorge Wilson * 210194dd93aeSGeorge Wilson * Note: The 1/16th arena free requirement was put in place 210294dd93aeSGeorge Wilson * to aggressively evict memory from the arc in order to avoid 210394dd93aeSGeorge Wilson * memory fragmentation issues. 210494dd93aeSGeorge Wilson */ 210594dd93aeSGeorge Wilson if (zio_arena != NULL && 210694dd93aeSGeorge Wilson vmem_size(zio_arena, VMEM_FREE) < 210794dd93aeSGeorge Wilson (vmem_size(zio_arena, VMEM_ALLOC) >> 4)) 210894dd93aeSGeorge Wilson return (1); 2109fa9e4066Sahrens #else 2110fa9e4066Sahrens if (spa_get_random(100) == 0) 2111fa9e4066Sahrens return (1); 2112fa9e4066Sahrens #endif 2113fa9e4066Sahrens return (0); 2114fa9e4066Sahrens } 2115fa9e4066Sahrens 2116fa9e4066Sahrens static void 2117fa9e4066Sahrens arc_kmem_reap_now(arc_reclaim_strategy_t strat) 2118fa9e4066Sahrens { 2119fa9e4066Sahrens size_t i; 2120fa9e4066Sahrens kmem_cache_t *prev_cache = NULL; 2121ad23a2dbSjohansen kmem_cache_t *prev_data_cache = NULL; 2122fa9e4066Sahrens extern kmem_cache_t *zio_buf_cache[]; 2123ad23a2dbSjohansen extern kmem_cache_t *zio_data_buf_cache[]; 2124fa9e4066Sahrens 2125033f9833Sek #ifdef _KERNEL 21260e8c6158Smaybee if (arc_meta_used >= arc_meta_limit) { 21270e8c6158Smaybee /* 21280e8c6158Smaybee * We are exceeding our meta-data cache limit. 21290e8c6158Smaybee * Purge some DNLC entries to release holds on meta-data. 21300e8c6158Smaybee */ 21310e8c6158Smaybee dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); 21320e8c6158Smaybee } 21335dc8af33Smaybee #if defined(__i386) 21345dc8af33Smaybee /* 21355dc8af33Smaybee * Reclaim unused memory from all kmem caches. 21365dc8af33Smaybee */ 21375dc8af33Smaybee kmem_reap(); 21385dc8af33Smaybee #endif 2139033f9833Sek #endif 2140033f9833Sek 2141fa9e4066Sahrens /* 2142fa94a07fSbrendan * An aggressive reclamation will shrink the cache size as well as 2143ea8dc4b6Seschrock * reap free buffers from the arc kmem caches. 2144fa9e4066Sahrens */ 2145fa9e4066Sahrens if (strat == ARC_RECLAIM_AGGR) 214649e3519aSmaybee arc_shrink(); 2147fa9e4066Sahrens 2148fa9e4066Sahrens for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { 2149fa9e4066Sahrens if (zio_buf_cache[i] != prev_cache) { 2150fa9e4066Sahrens prev_cache = zio_buf_cache[i]; 2151fa9e4066Sahrens kmem_cache_reap_now(zio_buf_cache[i]); 2152fa9e4066Sahrens } 2153ad23a2dbSjohansen if (zio_data_buf_cache[i] != prev_data_cache) { 2154ad23a2dbSjohansen prev_data_cache = zio_data_buf_cache[i]; 2155ad23a2dbSjohansen kmem_cache_reap_now(zio_data_buf_cache[i]); 2156ad23a2dbSjohansen } 2157fa9e4066Sahrens } 2158ea8dc4b6Seschrock kmem_cache_reap_now(buf_cache); 2159ea8dc4b6Seschrock kmem_cache_reap_now(hdr_cache); 216094dd93aeSGeorge Wilson 216194dd93aeSGeorge Wilson /* 216294dd93aeSGeorge Wilson * Ask the vmem areana to reclaim unused memory from its 216394dd93aeSGeorge Wilson * quantum caches. 216494dd93aeSGeorge Wilson */ 216594dd93aeSGeorge Wilson if (zio_arena != NULL && strat == ARC_RECLAIM_AGGR) 216694dd93aeSGeorge Wilson vmem_qcache_reap(zio_arena); 2167fa9e4066Sahrens } 2168fa9e4066Sahrens 2169fa9e4066Sahrens static void 2170fa9e4066Sahrens arc_reclaim_thread(void) 2171fa9e4066Sahrens { 2172fa9e4066Sahrens clock_t growtime = 0; 2173fa9e4066Sahrens arc_reclaim_strategy_t last_reclaim = ARC_RECLAIM_CONS; 2174fa9e4066Sahrens callb_cpr_t cpr; 2175fa9e4066Sahrens 2176fa9e4066Sahrens CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG); 2177fa9e4066Sahrens 2178fa9e4066Sahrens mutex_enter(&arc_reclaim_thr_lock); 2179fa9e4066Sahrens while (arc_thread_exit == 0) { 2180fa9e4066Sahrens if (arc_reclaim_needed()) { 2181fa9e4066Sahrens 218244cb6abcSbmc if (arc_no_grow) { 2183fa9e4066Sahrens if (last_reclaim == ARC_RECLAIM_CONS) { 2184fa9e4066Sahrens last_reclaim = ARC_RECLAIM_AGGR; 2185fa9e4066Sahrens } else { 2186fa9e4066Sahrens last_reclaim = ARC_RECLAIM_CONS; 2187fa9e4066Sahrens } 2188fa9e4066Sahrens } else { 218944cb6abcSbmc arc_no_grow = TRUE; 2190fa9e4066Sahrens last_reclaim = ARC_RECLAIM_AGGR; 2191fa9e4066Sahrens membar_producer(); 2192fa9e4066Sahrens } 2193fa9e4066Sahrens 2194fa9e4066Sahrens /* reset the growth delay for every reclaim */ 2195d3d50737SRafael Vanoni growtime = ddi_get_lbolt() + (arc_grow_retry * hz); 2196fa9e4066Sahrens 2197fa9e4066Sahrens arc_kmem_reap_now(last_reclaim); 21983a737e0dSbrendan arc_warm = B_TRUE; 2199fa9e4066Sahrens 2200d3d50737SRafael Vanoni } else if (arc_no_grow && ddi_get_lbolt() >= growtime) { 220144cb6abcSbmc arc_no_grow = FALSE; 2202fa9e4066Sahrens } 2203fa9e4066Sahrens 22043e4e8481STom Erickson arc_adjust(); 2205641fbdaeSmaybee 2206ea8dc4b6Seschrock if (arc_eviction_list != NULL) 2207ea8dc4b6Seschrock arc_do_user_evicts(); 2208ea8dc4b6Seschrock 2209fa9e4066Sahrens /* block until needed, or one second, whichever is shorter */ 2210fa9e4066Sahrens CALLB_CPR_SAFE_BEGIN(&cpr); 2211fa9e4066Sahrens (void) cv_timedwait(&arc_reclaim_thr_cv, 2212d3d50737SRafael Vanoni &arc_reclaim_thr_lock, (ddi_get_lbolt() + hz)); 2213fa9e4066Sahrens CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock); 2214fa9e4066Sahrens } 2215fa9e4066Sahrens 2216fa9e4066Sahrens arc_thread_exit = 0; 2217fa9e4066Sahrens cv_broadcast(&arc_reclaim_thr_cv); 2218fa9e4066Sahrens CALLB_CPR_EXIT(&cpr); /* drops arc_reclaim_thr_lock */ 2219fa9e4066Sahrens thread_exit(); 2220fa9e4066Sahrens } 2221fa9e4066Sahrens 2222ea8dc4b6Seschrock /* 2223ea8dc4b6Seschrock * Adapt arc info given the number of bytes we are trying to add and 2224ea8dc4b6Seschrock * the state that we are comming from. This function is only called 2225ea8dc4b6Seschrock * when we are adding new content to the cache. 2226ea8dc4b6Seschrock */ 2227fa9e4066Sahrens static void 2228ea8dc4b6Seschrock arc_adapt(int bytes, arc_state_t *state) 2229fa9e4066Sahrens { 2230ea8dc4b6Seschrock int mult; 22315a98e54bSBrendan Gregg - Sun Microsystems uint64_t arc_p_min = (arc_c >> arc_p_min_shift); 2232ea8dc4b6Seschrock 2233fa94a07fSbrendan if (state == arc_l2c_only) 2234fa94a07fSbrendan return; 2235fa94a07fSbrendan 2236ea8dc4b6Seschrock ASSERT(bytes > 0); 2237fa9e4066Sahrens /* 2238ea8dc4b6Seschrock * Adapt the target size of the MRU list: 2239ea8dc4b6Seschrock * - if we just hit in the MRU ghost list, then increase 2240ea8dc4b6Seschrock * the target size of the MRU list. 2241ea8dc4b6Seschrock * - if we just hit in the MFU ghost list, then increase 2242ea8dc4b6Seschrock * the target size of the MFU list by decreasing the 2243ea8dc4b6Seschrock * target size of the MRU list. 2244fa9e4066Sahrens */ 224544cb6abcSbmc if (state == arc_mru_ghost) { 224644cb6abcSbmc mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ? 224744cb6abcSbmc 1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size)); 22483e4e8481STom Erickson mult = MIN(mult, 10); /* avoid wild arc_p adjustment */ 2249ea8dc4b6Seschrock 22505a98e54bSBrendan Gregg - Sun Microsystems arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult); 225144cb6abcSbmc } else if (state == arc_mfu_ghost) { 22525a98e54bSBrendan Gregg - Sun Microsystems uint64_t delta; 22535a98e54bSBrendan Gregg - Sun Microsystems 225444cb6abcSbmc mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ? 225544cb6abcSbmc 1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size)); 22563e4e8481STom Erickson mult = MIN(mult, 10); 2257ea8dc4b6Seschrock 22585a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(bytes * mult, arc_p); 22595a98e54bSBrendan Gregg - Sun Microsystems arc_p = MAX(arc_p_min, arc_p - delta); 2260ea8dc4b6Seschrock } 226144cb6abcSbmc ASSERT((int64_t)arc_p >= 0); 2262fa9e4066Sahrens 2263fa9e4066Sahrens if (arc_reclaim_needed()) { 2264fa9e4066Sahrens cv_signal(&arc_reclaim_thr_cv); 2265fa9e4066Sahrens return; 2266fa9e4066Sahrens } 2267fa9e4066Sahrens 226844cb6abcSbmc if (arc_no_grow) 2269fa9e4066Sahrens return; 2270fa9e4066Sahrens 227144cb6abcSbmc if (arc_c >= arc_c_max) 2272ea8dc4b6Seschrock return; 2273ea8dc4b6Seschrock 2274fa9e4066Sahrens /* 2275ea8dc4b6Seschrock * If we're within (2 * maxblocksize) bytes of the target 2276ea8dc4b6Seschrock * cache size, increment the target cache size 2277fa9e4066Sahrens */ 227844cb6abcSbmc if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) { 227944cb6abcSbmc atomic_add_64(&arc_c, (int64_t)bytes); 228044cb6abcSbmc if (arc_c > arc_c_max) 228144cb6abcSbmc arc_c = arc_c_max; 228244cb6abcSbmc else if (state == arc_anon) 228344cb6abcSbmc atomic_add_64(&arc_p, (int64_t)bytes); 228444cb6abcSbmc if (arc_p > arc_c) 228544cb6abcSbmc arc_p = arc_c; 2286fa9e4066Sahrens } 228744cb6abcSbmc ASSERT((int64_t)arc_p >= 0); 2288fa9e4066Sahrens } 2289fa9e4066Sahrens 2290fa9e4066Sahrens /* 2291ea8dc4b6Seschrock * Check if the cache has reached its limits and eviction is required 2292ea8dc4b6Seschrock * prior to insert. 2293fa9e4066Sahrens */ 2294fa9e4066Sahrens static int 22950e8c6158Smaybee arc_evict_needed(arc_buf_contents_t type) 2296fa9e4066Sahrens { 22970e8c6158Smaybee if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit) 22980e8c6158Smaybee return (1); 22990e8c6158Smaybee 2300fa9e4066Sahrens if (arc_reclaim_needed()) 2301fa9e4066Sahrens return (1); 2302fa9e4066Sahrens 230344cb6abcSbmc return (arc_size > arc_c); 2304fa9e4066Sahrens } 2305fa9e4066Sahrens 2306fa9e4066Sahrens /* 230744eda4d7Smaybee * The buffer, supplied as the first argument, needs a data block. 230844eda4d7Smaybee * So, if we are at cache max, determine which cache should be victimized. 230944eda4d7Smaybee * We have the following cases: 2310fa9e4066Sahrens * 231144cb6abcSbmc * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) -> 2312fa9e4066Sahrens * In this situation if we're out of space, but the resident size of the MFU is 2313fa9e4066Sahrens * under the limit, victimize the MFU cache to satisfy this insertion request. 2314fa9e4066Sahrens * 231544cb6abcSbmc * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) -> 2316fa9e4066Sahrens * Here, we've used up all of the available space for the MRU, so we need to 2317fa9e4066Sahrens * evict from our own cache instead. Evict from the set of resident MRU 2318fa9e4066Sahrens * entries. 2319fa9e4066Sahrens * 232044cb6abcSbmc * 3. Insert for MFU (c - p) > sizeof(arc_mfu) -> 2321fa9e4066Sahrens * c minus p represents the MFU space in the cache, since p is the size of the 2322fa9e4066Sahrens * cache that is dedicated to the MRU. In this situation there's still space on 2323fa9e4066Sahrens * the MFU side, so the MRU side needs to be victimized. 2324fa9e4066Sahrens * 232544cb6abcSbmc * 4. Insert for MFU (c - p) < sizeof(arc_mfu) -> 2326fa9e4066Sahrens * MFU's resident set is consuming more space than it has been allotted. In 2327fa9e4066Sahrens * this situation, we must victimize our own cache, the MFU, for this insertion. 2328fa9e4066Sahrens */ 2329fa9e4066Sahrens static void 233044eda4d7Smaybee arc_get_data_buf(arc_buf_t *buf) 2331fa9e4066Sahrens { 2332ad23a2dbSjohansen arc_state_t *state = buf->b_hdr->b_state; 2333ad23a2dbSjohansen uint64_t size = buf->b_hdr->b_size; 2334ad23a2dbSjohansen arc_buf_contents_t type = buf->b_hdr->b_type; 2335fa9e4066Sahrens 233644eda4d7Smaybee arc_adapt(size, state); 2337fa9e4066Sahrens 233844eda4d7Smaybee /* 233944eda4d7Smaybee * We have not yet reached cache maximum size, 234044eda4d7Smaybee * just allocate a new buffer. 234144eda4d7Smaybee */ 23420e8c6158Smaybee if (!arc_evict_needed(type)) { 2343ad23a2dbSjohansen if (type == ARC_BUFC_METADATA) { 2344ad23a2dbSjohansen buf->b_data = zio_buf_alloc(size); 23455a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(size, ARC_SPACE_DATA); 2346ad23a2dbSjohansen } else { 2347ad23a2dbSjohansen ASSERT(type == ARC_BUFC_DATA); 2348ad23a2dbSjohansen buf->b_data = zio_data_buf_alloc(size); 23495a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, size); 23500e8c6158Smaybee atomic_add_64(&arc_size, size); 2351ad23a2dbSjohansen } 235244eda4d7Smaybee goto out; 235344eda4d7Smaybee } 235444eda4d7Smaybee 235544eda4d7Smaybee /* 235644eda4d7Smaybee * If we are prefetching from the mfu ghost list, this buffer 235744eda4d7Smaybee * will end up on the mru list; so steal space from there. 235844eda4d7Smaybee */ 235944cb6abcSbmc if (state == arc_mfu_ghost) 236044cb6abcSbmc state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu; 236144cb6abcSbmc else if (state == arc_mru_ghost) 236244cb6abcSbmc state = arc_mru; 236344cb6abcSbmc 236444cb6abcSbmc if (state == arc_mru || state == arc_anon) { 236544cb6abcSbmc uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size; 23665a98e54bSBrendan Gregg - Sun Microsystems state = (arc_mfu->arcs_lsize[type] >= size && 23670e8c6158Smaybee arc_p > mru_used) ? arc_mfu : arc_mru; 2368fa9e4066Sahrens } else { 236944eda4d7Smaybee /* MFU cases */ 237044cb6abcSbmc uint64_t mfu_space = arc_c - arc_p; 23715a98e54bSBrendan Gregg - Sun Microsystems state = (arc_mru->arcs_lsize[type] >= size && 23720e8c6158Smaybee mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu; 237344eda4d7Smaybee } 2374874395d5Smaybee if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) { 2375ad23a2dbSjohansen if (type == ARC_BUFC_METADATA) { 2376ad23a2dbSjohansen buf->b_data = zio_buf_alloc(size); 23775a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(size, ARC_SPACE_DATA); 2378ad23a2dbSjohansen } else { 2379ad23a2dbSjohansen ASSERT(type == ARC_BUFC_DATA); 2380ad23a2dbSjohansen buf->b_data = zio_data_buf_alloc(size); 23815a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, size); 23820e8c6158Smaybee atomic_add_64(&arc_size, size); 2383ad23a2dbSjohansen } 238444cb6abcSbmc ARCSTAT_BUMP(arcstat_recycle_miss); 238544eda4d7Smaybee } 238644eda4d7Smaybee ASSERT(buf->b_data != NULL); 238744eda4d7Smaybee out: 238844eda4d7Smaybee /* 238944eda4d7Smaybee * Update the state size. Note that ghost states have a 239044eda4d7Smaybee * "ghost size" and so don't need to be updated. 239144eda4d7Smaybee */ 239244eda4d7Smaybee if (!GHOST_STATE(buf->b_hdr->b_state)) { 239344eda4d7Smaybee arc_buf_hdr_t *hdr = buf->b_hdr; 239444eda4d7Smaybee 239544cb6abcSbmc atomic_add_64(&hdr->b_state->arcs_size, size); 239644eda4d7Smaybee if (list_link_active(&hdr->b_arc_node)) { 239744eda4d7Smaybee ASSERT(refcount_is_zero(&hdr->b_refcnt)); 23980e8c6158Smaybee atomic_add_64(&hdr->b_state->arcs_lsize[type], size); 2399fa9e4066Sahrens } 2400641fbdaeSmaybee /* 2401641fbdaeSmaybee * If we are growing the cache, and we are adding anonymous 240244cb6abcSbmc * data, and we have outgrown arc_p, update arc_p 2403641fbdaeSmaybee */ 240444cb6abcSbmc if (arc_size < arc_c && hdr->b_state == arc_anon && 240544cb6abcSbmc arc_anon->arcs_size + arc_mru->arcs_size > arc_p) 240644cb6abcSbmc arc_p = MIN(arc_c, arc_p + size); 2407fa9e4066Sahrens } 2408fa9e4066Sahrens } 2409fa9e4066Sahrens 2410fa9e4066Sahrens /* 2411fa9e4066Sahrens * This routine is called whenever a buffer is accessed. 2412ea8dc4b6Seschrock * NOTE: the hash lock is dropped in this function. 2413fa9e4066Sahrens */ 2414fa9e4066Sahrens static void 241544eda4d7Smaybee arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock) 2416fa9e4066Sahrens { 2417d3d50737SRafael Vanoni clock_t now; 2418d3d50737SRafael Vanoni 2419fa9e4066Sahrens ASSERT(MUTEX_HELD(hash_lock)); 2420fa9e4066Sahrens 242144cb6abcSbmc if (buf->b_state == arc_anon) { 2422fa9e4066Sahrens /* 2423fa9e4066Sahrens * This buffer is not in the cache, and does not 2424fa9e4066Sahrens * appear in our "ghost" list. Add the new buffer 2425fa9e4066Sahrens * to the MRU state. 2426fa9e4066Sahrens */ 2427fa9e4066Sahrens 2428fa9e4066Sahrens ASSERT(buf->b_arc_access == 0); 2429d3d50737SRafael Vanoni buf->b_arc_access = ddi_get_lbolt(); 2430ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 243144cb6abcSbmc arc_change_state(arc_mru, buf, hash_lock); 2432fa9e4066Sahrens 243344cb6abcSbmc } else if (buf->b_state == arc_mru) { 2434d3d50737SRafael Vanoni now = ddi_get_lbolt(); 2435d3d50737SRafael Vanoni 2436fa9e4066Sahrens /* 243713506d1eSmaybee * If this buffer is here because of a prefetch, then either: 243813506d1eSmaybee * - clear the flag if this is a "referencing" read 243913506d1eSmaybee * (any subsequent access will bump this into the MFU state). 244013506d1eSmaybee * or 244113506d1eSmaybee * - move the buffer to the head of the list if this is 244213506d1eSmaybee * another prefetch (to make it less likely to be evicted). 2443fa9e4066Sahrens */ 2444fa9e4066Sahrens if ((buf->b_flags & ARC_PREFETCH) != 0) { 244513506d1eSmaybee if (refcount_count(&buf->b_refcnt) == 0) { 244613506d1eSmaybee ASSERT(list_link_active(&buf->b_arc_node)); 244713506d1eSmaybee } else { 244813506d1eSmaybee buf->b_flags &= ~ARC_PREFETCH; 244944cb6abcSbmc ARCSTAT_BUMP(arcstat_mru_hits); 245013506d1eSmaybee } 2451d3d50737SRafael Vanoni buf->b_arc_access = now; 2452fa9e4066Sahrens return; 2453fa9e4066Sahrens } 2454fa9e4066Sahrens 2455fa9e4066Sahrens /* 2456fa9e4066Sahrens * This buffer has been "accessed" only once so far, 2457fa9e4066Sahrens * but it is still in the cache. Move it to the MFU 2458fa9e4066Sahrens * state. 2459fa9e4066Sahrens */ 2460d3d50737SRafael Vanoni if (now > buf->b_arc_access + ARC_MINTIME) { 2461fa9e4066Sahrens /* 2462fa9e4066Sahrens * More than 125ms have passed since we 2463fa9e4066Sahrens * instantiated this buffer. Move it to the 2464fa9e4066Sahrens * most frequently used state. 2465fa9e4066Sahrens */ 2466d3d50737SRafael Vanoni buf->b_arc_access = now; 2467ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 246844cb6abcSbmc arc_change_state(arc_mfu, buf, hash_lock); 2469fa9e4066Sahrens } 247044cb6abcSbmc ARCSTAT_BUMP(arcstat_mru_hits); 247144cb6abcSbmc } else if (buf->b_state == arc_mru_ghost) { 2472fa9e4066Sahrens arc_state_t *new_state; 2473fa9e4066Sahrens /* 2474fa9e4066Sahrens * This buffer has been "accessed" recently, but 2475fa9e4066Sahrens * was evicted from the cache. Move it to the 2476fa9e4066Sahrens * MFU state. 2477fa9e4066Sahrens */ 2478fa9e4066Sahrens 2479fa9e4066Sahrens if (buf->b_flags & ARC_PREFETCH) { 248044cb6abcSbmc new_state = arc_mru; 248113506d1eSmaybee if (refcount_count(&buf->b_refcnt) > 0) 248213506d1eSmaybee buf->b_flags &= ~ARC_PREFETCH; 2483ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 2484fa9e4066Sahrens } else { 248544cb6abcSbmc new_state = arc_mfu; 2486ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2487fa9e4066Sahrens } 2488fa9e4066Sahrens 2489d3d50737SRafael Vanoni buf->b_arc_access = ddi_get_lbolt(); 2490fa9e4066Sahrens arc_change_state(new_state, buf, hash_lock); 2491fa9e4066Sahrens 249244cb6abcSbmc ARCSTAT_BUMP(arcstat_mru_ghost_hits); 249344cb6abcSbmc } else if (buf->b_state == arc_mfu) { 2494fa9e4066Sahrens /* 2495fa9e4066Sahrens * This buffer has been accessed more than once and is 2496fa9e4066Sahrens * still in the cache. Keep it in the MFU state. 2497fa9e4066Sahrens * 249813506d1eSmaybee * NOTE: an add_reference() that occurred when we did 249913506d1eSmaybee * the arc_read() will have kicked this off the list. 250013506d1eSmaybee * If it was a prefetch, we will explicitly move it to 250113506d1eSmaybee * the head of the list now. 2502fa9e4066Sahrens */ 250313506d1eSmaybee if ((buf->b_flags & ARC_PREFETCH) != 0) { 250413506d1eSmaybee ASSERT(refcount_count(&buf->b_refcnt) == 0); 250513506d1eSmaybee ASSERT(list_link_active(&buf->b_arc_node)); 250613506d1eSmaybee } 250744cb6abcSbmc ARCSTAT_BUMP(arcstat_mfu_hits); 2508d3d50737SRafael Vanoni buf->b_arc_access = ddi_get_lbolt(); 250944cb6abcSbmc } else if (buf->b_state == arc_mfu_ghost) { 251044cb6abcSbmc arc_state_t *new_state = arc_mfu; 2511fa9e4066Sahrens /* 2512fa9e4066Sahrens * This buffer has been accessed more than once but has 2513fa9e4066Sahrens * been evicted from the cache. Move it back to the 2514fa9e4066Sahrens * MFU state. 2515fa9e4066Sahrens */ 2516fa9e4066Sahrens 251713506d1eSmaybee if (buf->b_flags & ARC_PREFETCH) { 251813506d1eSmaybee /* 251913506d1eSmaybee * This is a prefetch access... 252013506d1eSmaybee * move this block back to the MRU state. 252113506d1eSmaybee */ 2522b420f3adSRichard Lowe ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0); 252344cb6abcSbmc new_state = arc_mru; 252413506d1eSmaybee } 252513506d1eSmaybee 2526d3d50737SRafael Vanoni buf->b_arc_access = ddi_get_lbolt(); 2527ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 252813506d1eSmaybee arc_change_state(new_state, buf, hash_lock); 2529fa9e4066Sahrens 253044cb6abcSbmc ARCSTAT_BUMP(arcstat_mfu_ghost_hits); 2531fa94a07fSbrendan } else if (buf->b_state == arc_l2c_only) { 2532fa94a07fSbrendan /* 2533fa94a07fSbrendan * This buffer is on the 2nd Level ARC. 2534fa94a07fSbrendan */ 2535fa94a07fSbrendan 2536d3d50737SRafael Vanoni buf->b_arc_access = ddi_get_lbolt(); 2537fa94a07fSbrendan DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2538fa94a07fSbrendan arc_change_state(arc_mfu, buf, hash_lock); 2539fa9e4066Sahrens } else { 2540fa9e4066Sahrens ASSERT(!"invalid arc state"); 2541fa9e4066Sahrens } 2542fa9e4066Sahrens } 2543fa9e4066Sahrens 2544fa9e4066Sahrens /* a generic arc_done_func_t which you can use */ 2545fa9e4066Sahrens /* ARGSUSED */ 2546fa9e4066Sahrens void 2547fa9e4066Sahrens arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) 2548fa9e4066Sahrens { 25493f9d6ad7SLin Ling if (zio == NULL || zio->io_error == 0) 25503f9d6ad7SLin Ling bcopy(buf->b_data, arg, buf->b_hdr->b_size); 2551ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, arg) == 1); 2552fa9e4066Sahrens } 2553fa9e4066Sahrens 25540e8c6158Smaybee /* a generic arc_done_func_t */ 2555fa9e4066Sahrens void 2556fa9e4066Sahrens arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) 2557fa9e4066Sahrens { 2558fa9e4066Sahrens arc_buf_t **bufp = arg; 2559fa9e4066Sahrens if (zio && zio->io_error) { 2560ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, arg) == 1); 2561fa9e4066Sahrens *bufp = NULL; 2562fa9e4066Sahrens } else { 2563fa9e4066Sahrens *bufp = buf; 25643f9d6ad7SLin Ling ASSERT(buf->b_data); 2565fa9e4066Sahrens } 2566fa9e4066Sahrens } 2567fa9e4066Sahrens 2568fa9e4066Sahrens static void 2569fa9e4066Sahrens arc_read_done(zio_t *zio) 2570fa9e4066Sahrens { 2571bbf4a8dfSmaybee arc_buf_hdr_t *hdr, *found; 2572fa9e4066Sahrens arc_buf_t *buf; 2573fa9e4066Sahrens arc_buf_t *abuf; /* buffer we're assigning to callback */ 2574fa9e4066Sahrens kmutex_t *hash_lock; 2575fa9e4066Sahrens arc_callback_t *callback_list, *acb; 2576fa9e4066Sahrens int freeable = FALSE; 2577fa9e4066Sahrens 2578fa9e4066Sahrens buf = zio->io_private; 2579fa9e4066Sahrens hdr = buf->b_hdr; 2580fa9e4066Sahrens 2581bbf4a8dfSmaybee /* 2582bbf4a8dfSmaybee * The hdr was inserted into hash-table and removed from lists 2583bbf4a8dfSmaybee * prior to starting I/O. We should find this header, since 2584bbf4a8dfSmaybee * it's in the hash table, and it should be legit since it's 2585bbf4a8dfSmaybee * not possible to evict it during the I/O. The only possible 2586bbf4a8dfSmaybee * reason for it not to be found is if we were freed during the 2587bbf4a8dfSmaybee * read. 2588bbf4a8dfSmaybee */ 2589ac05c741SMark Maybee found = buf_hash_find(hdr->b_spa, &hdr->b_dva, hdr->b_birth, 25906b4acc8bSahrens &hash_lock); 2591fa9e4066Sahrens 2592bbf4a8dfSmaybee ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) || 2593fa94a07fSbrendan (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || 2594fa94a07fSbrendan (found == hdr && HDR_L2_READING(hdr))); 2595fa94a07fSbrendan 25963a737e0dSbrendan hdr->b_flags &= ~ARC_L2_EVICTED; 2597fa94a07fSbrendan if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH)) 25983baa08fcSek hdr->b_flags &= ~ARC_L2CACHE; 2599fa9e4066Sahrens 2600fa9e4066Sahrens /* byteswap if necessary */ 2601fa9e4066Sahrens callback_list = hdr->b_acb; 2602fa9e4066Sahrens ASSERT(callback_list != NULL); 26038e0f0d3dSWilliam Gorrell if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) { 2604ad135b5dSChristopher Siden dmu_object_byteswap_t bswap = 2605ad135b5dSChristopher Siden DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp)); 2606088f3894Sahrens arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ? 2607088f3894Sahrens byteswap_uint64_array : 2608ad135b5dSChristopher Siden dmu_ot_byteswap[bswap].ob_func; 2609088f3894Sahrens func(buf->b_data, hdr->b_size); 2610088f3894Sahrens } 2611fa9e4066Sahrens 2612fa94a07fSbrendan arc_cksum_compute(buf, B_FALSE); 2613*cd1c8b85SMatthew Ahrens arc_buf_watch(buf); 26146b4acc8bSahrens 2615b24ab676SJeff Bonwick if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) { 2616b24ab676SJeff Bonwick /* 2617b24ab676SJeff Bonwick * Only call arc_access on anonymous buffers. This is because 2618b24ab676SJeff Bonwick * if we've issued an I/O for an evicted buffer, we've already 2619b24ab676SJeff Bonwick * called arc_access (to prevent any simultaneous readers from 2620b24ab676SJeff Bonwick * getting confused). 2621b24ab676SJeff Bonwick */ 2622b24ab676SJeff Bonwick arc_access(hdr, hash_lock); 2623b24ab676SJeff Bonwick } 2624b24ab676SJeff Bonwick 2625fa9e4066Sahrens /* create copies of the data buffer for the callers */ 2626fa9e4066Sahrens abuf = buf; 2627fa9e4066Sahrens for (acb = callback_list; acb; acb = acb->acb_next) { 2628fa9e4066Sahrens if (acb->acb_done) { 262944eda4d7Smaybee if (abuf == NULL) 263044eda4d7Smaybee abuf = arc_buf_clone(buf); 2631fa9e4066Sahrens acb->acb_buf = abuf; 2632fa9e4066Sahrens abuf = NULL; 2633fa9e4066Sahrens } 2634fa9e4066Sahrens } 2635fa9e4066Sahrens hdr->b_acb = NULL; 2636fa9e4066Sahrens hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2637ea8dc4b6Seschrock ASSERT(!HDR_BUF_AVAILABLE(hdr)); 2638b24ab676SJeff Bonwick if (abuf == buf) { 2639b24ab676SJeff Bonwick ASSERT(buf->b_efunc == NULL); 2640b24ab676SJeff Bonwick ASSERT(hdr->b_datacnt == 1); 2641ea8dc4b6Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 2642b24ab676SJeff Bonwick } 2643fa9e4066Sahrens 2644fa9e4066Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL); 2645fa9e4066Sahrens 2646fa9e4066Sahrens if (zio->io_error != 0) { 2647fa9e4066Sahrens hdr->b_flags |= ARC_IO_ERROR; 264844cb6abcSbmc if (hdr->b_state != arc_anon) 264944cb6abcSbmc arc_change_state(arc_anon, hdr, hash_lock); 2650ea8dc4b6Seschrock if (HDR_IN_HASH_TABLE(hdr)) 2651ea8dc4b6Seschrock buf_hash_remove(hdr); 2652fa9e4066Sahrens freeable = refcount_is_zero(&hdr->b_refcnt); 2653fa9e4066Sahrens } 2654fa9e4066Sahrens 2655ea8dc4b6Seschrock /* 265613506d1eSmaybee * Broadcast before we drop the hash_lock to avoid the possibility 265713506d1eSmaybee * that the hdr (and hence the cv) might be freed before we get to 265813506d1eSmaybee * the cv_broadcast(). 2659ea8dc4b6Seschrock */ 2660ea8dc4b6Seschrock cv_broadcast(&hdr->b_cv); 2661ea8dc4b6Seschrock 2662bbf4a8dfSmaybee if (hash_lock) { 266344eda4d7Smaybee mutex_exit(hash_lock); 2664fa9e4066Sahrens } else { 2665fa9e4066Sahrens /* 2666fa9e4066Sahrens * This block was freed while we waited for the read to 2667fa9e4066Sahrens * complete. It has been removed from the hash table and 2668fa9e4066Sahrens * moved to the anonymous state (so that it won't show up 2669fa9e4066Sahrens * in the cache). 2670fa9e4066Sahrens */ 267144cb6abcSbmc ASSERT3P(hdr->b_state, ==, arc_anon); 2672fa9e4066Sahrens freeable = refcount_is_zero(&hdr->b_refcnt); 2673fa9e4066Sahrens } 2674fa9e4066Sahrens 2675fa9e4066Sahrens /* execute each callback and free its structure */ 2676fa9e4066Sahrens while ((acb = callback_list) != NULL) { 2677fa9e4066Sahrens if (acb->acb_done) 2678fa9e4066Sahrens acb->acb_done(zio, acb->acb_buf, acb->acb_private); 2679fa9e4066Sahrens 2680fa9e4066Sahrens if (acb->acb_zio_dummy != NULL) { 2681fa9e4066Sahrens acb->acb_zio_dummy->io_error = zio->io_error; 2682fa9e4066Sahrens zio_nowait(acb->acb_zio_dummy); 2683fa9e4066Sahrens } 2684fa9e4066Sahrens 2685fa9e4066Sahrens callback_list = acb->acb_next; 2686fa9e4066Sahrens kmem_free(acb, sizeof (arc_callback_t)); 2687fa9e4066Sahrens } 2688fa9e4066Sahrens 2689fa9e4066Sahrens if (freeable) 2690ea8dc4b6Seschrock arc_hdr_destroy(hdr); 2691fa9e4066Sahrens } 2692fa9e4066Sahrens 2693fa9e4066Sahrens /* 2694fc98fea5SBart Coddens * "Read" the block at the specified DVA (in bp) via the 2695fa9e4066Sahrens * cache. If the block is found in the cache, invoke the provided 2696fa9e4066Sahrens * callback immediately and return. Note that the `zio' parameter 2697fa9e4066Sahrens * in the callback will be NULL in this case, since no IO was 2698fa9e4066Sahrens * required. If the block is not in the cache pass the read request 2699fa9e4066Sahrens * on to the spa with a substitute callback function, so that the 2700fa9e4066Sahrens * requested block will be added to the cache. 2701fa9e4066Sahrens * 2702fa9e4066Sahrens * If a read request arrives for a block that has a read in-progress, 2703fa9e4066Sahrens * either wait for the in-progress read to complete (and return the 2704fa9e4066Sahrens * results); or, if this is a read with a "done" func, add a record 2705fa9e4066Sahrens * to the read to invoke the "done" func when the read completes, 2706fa9e4066Sahrens * and return; or just return. 2707fa9e4066Sahrens * 2708fa9e4066Sahrens * arc_read_done() will invoke all the requested "done" functions 2709fa9e4066Sahrens * for readers of this block. 2710088f3894Sahrens * 2711088f3894Sahrens * Normal callers should use arc_read and pass the arc buffer and offset 2712088f3894Sahrens * for the bp. But if you know you don't need locking, you can use 27138ad10dceSSuhasini Peddada * arc_read_bp. 2714fa9e4066Sahrens */ 2715fa9e4066Sahrens int 2716b24ab676SJeff Bonwick arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_buf_t *pbuf, 27173baa08fcSek arc_done_func_t *done, void *private, int priority, int zio_flags, 2718088f3894Sahrens uint32_t *arc_flags, const zbookmark_t *zb) 2719088f3894Sahrens { 2720088f3894Sahrens int err; 2721088f3894Sahrens 27223f9d6ad7SLin Ling if (pbuf == NULL) { 27233f9d6ad7SLin Ling /* 27243f9d6ad7SLin Ling * XXX This happens from traverse callback funcs, for 27253f9d6ad7SLin Ling * the objset_phys_t block. 27263f9d6ad7SLin Ling */ 27273f9d6ad7SLin Ling return (arc_read_nolock(pio, spa, bp, done, private, priority, 27283f9d6ad7SLin Ling zio_flags, arc_flags, zb)); 27293f9d6ad7SLin Ling } 27303f9d6ad7SLin Ling 2731088f3894Sahrens ASSERT(!refcount_is_zero(&pbuf->b_hdr->b_refcnt)); 2732088f3894Sahrens ASSERT3U((char *)bp - (char *)pbuf->b_data, <, pbuf->b_hdr->b_size); 27333f9d6ad7SLin Ling rw_enter(&pbuf->b_data_lock, RW_READER); 2734088f3894Sahrens 2735088f3894Sahrens err = arc_read_nolock(pio, spa, bp, done, private, priority, 27363baa08fcSek zio_flags, arc_flags, zb); 27373f9d6ad7SLin Ling rw_exit(&pbuf->b_data_lock); 273814843421SMatthew Ahrens 2739088f3894Sahrens return (err); 2740088f3894Sahrens } 2741088f3894Sahrens 2742088f3894Sahrens int 2743b24ab676SJeff Bonwick arc_read_nolock(zio_t *pio, spa_t *spa, const blkptr_t *bp, 27443baa08fcSek arc_done_func_t *done, void *private, int priority, int zio_flags, 2745088f3894Sahrens uint32_t *arc_flags, const zbookmark_t *zb) 2746fa9e4066Sahrens { 2747fa9e4066Sahrens arc_buf_hdr_t *hdr; 2748fa9e4066Sahrens arc_buf_t *buf; 2749fa9e4066Sahrens kmutex_t *hash_lock; 2750fa94a07fSbrendan zio_t *rzio; 2751e9103aaeSGarrett D'Amore uint64_t guid = spa_load_guid(spa); 2752fa9e4066Sahrens 2753fa9e4066Sahrens top: 2754b24ab676SJeff Bonwick hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp), 2755b24ab676SJeff Bonwick &hash_lock); 2756ea8dc4b6Seschrock if (hdr && hdr->b_datacnt > 0) { 2757fa9e4066Sahrens 275813506d1eSmaybee *arc_flags |= ARC_CACHED; 275913506d1eSmaybee 2760fa9e4066Sahrens if (HDR_IO_IN_PROGRESS(hdr)) { 276113506d1eSmaybee 276213506d1eSmaybee if (*arc_flags & ARC_WAIT) { 276313506d1eSmaybee cv_wait(&hdr->b_cv, hash_lock); 276413506d1eSmaybee mutex_exit(hash_lock); 276513506d1eSmaybee goto top; 276613506d1eSmaybee } 276713506d1eSmaybee ASSERT(*arc_flags & ARC_NOWAIT); 276813506d1eSmaybee 276913506d1eSmaybee if (done) { 2770fa9e4066Sahrens arc_callback_t *acb = NULL; 2771fa9e4066Sahrens 2772fa9e4066Sahrens acb = kmem_zalloc(sizeof (arc_callback_t), 2773fa9e4066Sahrens KM_SLEEP); 2774fa9e4066Sahrens acb->acb_done = done; 2775fa9e4066Sahrens acb->acb_private = private; 2776fa9e4066Sahrens if (pio != NULL) 2777fa9e4066Sahrens acb->acb_zio_dummy = zio_null(pio, 2778a3f829aeSBill Moore spa, NULL, NULL, NULL, zio_flags); 2779fa9e4066Sahrens 2780fa9e4066Sahrens ASSERT(acb->acb_done != NULL); 2781fa9e4066Sahrens acb->acb_next = hdr->b_acb; 2782fa9e4066Sahrens hdr->b_acb = acb; 2783fa9e4066Sahrens add_reference(hdr, hash_lock, private); 2784fa9e4066Sahrens mutex_exit(hash_lock); 2785fa9e4066Sahrens return (0); 2786fa9e4066Sahrens } 2787fa9e4066Sahrens mutex_exit(hash_lock); 2788fa9e4066Sahrens return (0); 2789fa9e4066Sahrens } 2790fa9e4066Sahrens 279144cb6abcSbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2792fa9e4066Sahrens 2793ea8dc4b6Seschrock if (done) { 279444eda4d7Smaybee add_reference(hdr, hash_lock, private); 2795ea8dc4b6Seschrock /* 2796ea8dc4b6Seschrock * If this block is already in use, create a new 2797ea8dc4b6Seschrock * copy of the data so that we will be guaranteed 2798ea8dc4b6Seschrock * that arc_release() will always succeed. 2799ea8dc4b6Seschrock */ 2800fa9e4066Sahrens buf = hdr->b_buf; 2801ea8dc4b6Seschrock ASSERT(buf); 2802ea8dc4b6Seschrock ASSERT(buf->b_data); 280344eda4d7Smaybee if (HDR_BUF_AVAILABLE(hdr)) { 2804ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 2805ea8dc4b6Seschrock hdr->b_flags &= ~ARC_BUF_AVAILABLE; 280644eda4d7Smaybee } else { 280744eda4d7Smaybee buf = arc_buf_clone(buf); 2808ea8dc4b6Seschrock } 2809b24ab676SJeff Bonwick 281013506d1eSmaybee } else if (*arc_flags & ARC_PREFETCH && 281113506d1eSmaybee refcount_count(&hdr->b_refcnt) == 0) { 281213506d1eSmaybee hdr->b_flags |= ARC_PREFETCH; 2813fa9e4066Sahrens } 2814fa9e4066Sahrens DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 281544eda4d7Smaybee arc_access(hdr, hash_lock); 28163baa08fcSek if (*arc_flags & ARC_L2CACHE) 28173baa08fcSek hdr->b_flags |= ARC_L2CACHE; 281844eda4d7Smaybee mutex_exit(hash_lock); 281944cb6abcSbmc ARCSTAT_BUMP(arcstat_hits); 282044cb6abcSbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 282144cb6abcSbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 282244cb6abcSbmc data, metadata, hits); 282344cb6abcSbmc 2824fa9e4066Sahrens if (done) 2825fa9e4066Sahrens done(NULL, buf, private); 2826fa9e4066Sahrens } else { 2827fa9e4066Sahrens uint64_t size = BP_GET_LSIZE(bp); 2828fa9e4066Sahrens arc_callback_t *acb; 28293a737e0dSbrendan vdev_t *vd = NULL; 2830f9d8334fSGeorge Wilson uint64_t addr; 28315a98e54bSBrendan Gregg - Sun Microsystems boolean_t devw = B_FALSE; 2832fa9e4066Sahrens 2833fa9e4066Sahrens if (hdr == NULL) { 2834fa9e4066Sahrens /* this block is not in the cache */ 2835fa9e4066Sahrens arc_buf_hdr_t *exists; 2836ad23a2dbSjohansen arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp); 2837ad23a2dbSjohansen buf = arc_buf_alloc(spa, size, private, type); 2838fa9e4066Sahrens hdr = buf->b_hdr; 2839fa9e4066Sahrens hdr->b_dva = *BP_IDENTITY(bp); 2840b24ab676SJeff Bonwick hdr->b_birth = BP_PHYSICAL_BIRTH(bp); 2841fa9e4066Sahrens hdr->b_cksum0 = bp->blk_cksum.zc_word[0]; 2842fa9e4066Sahrens exists = buf_hash_insert(hdr, &hash_lock); 2843fa9e4066Sahrens if (exists) { 2844fa9e4066Sahrens /* somebody beat us to the hash insert */ 2845fa9e4066Sahrens mutex_exit(hash_lock); 28463f9d6ad7SLin Ling buf_discard_identity(hdr); 2847ea8dc4b6Seschrock (void) arc_buf_remove_ref(buf, private); 2848fa9e4066Sahrens goto top; /* restart the IO request */ 2849fa9e4066Sahrens } 285013506d1eSmaybee /* if this is a prefetch, we don't have a reference */ 285113506d1eSmaybee if (*arc_flags & ARC_PREFETCH) { 285213506d1eSmaybee (void) remove_reference(hdr, hash_lock, 285313506d1eSmaybee private); 285413506d1eSmaybee hdr->b_flags |= ARC_PREFETCH; 285513506d1eSmaybee } 28563baa08fcSek if (*arc_flags & ARC_L2CACHE) 28573baa08fcSek hdr->b_flags |= ARC_L2CACHE; 285813506d1eSmaybee if (BP_GET_LEVEL(bp) > 0) 285913506d1eSmaybee hdr->b_flags |= ARC_INDIRECT; 2860fa9e4066Sahrens } else { 2861fa9e4066Sahrens /* this block is in the ghost cache */ 2862ea8dc4b6Seschrock ASSERT(GHOST_STATE(hdr->b_state)); 2863ea8dc4b6Seschrock ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 2864b420f3adSRichard Lowe ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0); 2865ea8dc4b6Seschrock ASSERT(hdr->b_buf == NULL); 286613506d1eSmaybee 286713506d1eSmaybee /* if this is a prefetch, we don't have a reference */ 286813506d1eSmaybee if (*arc_flags & ARC_PREFETCH) 286913506d1eSmaybee hdr->b_flags |= ARC_PREFETCH; 287013506d1eSmaybee else 287113506d1eSmaybee add_reference(hdr, hash_lock, private); 28723baa08fcSek if (*arc_flags & ARC_L2CACHE) 28733baa08fcSek hdr->b_flags |= ARC_L2CACHE; 28741ab7f2deSmaybee buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 2875fa9e4066Sahrens buf->b_hdr = hdr; 287644eda4d7Smaybee buf->b_data = NULL; 2877ea8dc4b6Seschrock buf->b_efunc = NULL; 2878ea8dc4b6Seschrock buf->b_private = NULL; 2879fa9e4066Sahrens buf->b_next = NULL; 2880fa9e4066Sahrens hdr->b_buf = buf; 2881ea8dc4b6Seschrock ASSERT(hdr->b_datacnt == 0); 2882ea8dc4b6Seschrock hdr->b_datacnt = 1; 28835614b00aSWilliam Gorrell arc_get_data_buf(buf); 28847e453561SWilliam Gorrell arc_access(hdr, hash_lock); 2885fa9e4066Sahrens } 2886fa9e4066Sahrens 28875614b00aSWilliam Gorrell ASSERT(!GHOST_STATE(hdr->b_state)); 28885614b00aSWilliam Gorrell 2889fa9e4066Sahrens acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 2890fa9e4066Sahrens acb->acb_done = done; 2891fa9e4066Sahrens acb->acb_private = private; 2892fa9e4066Sahrens 2893fa9e4066Sahrens ASSERT(hdr->b_acb == NULL); 2894fa9e4066Sahrens hdr->b_acb = acb; 2895fa9e4066Sahrens hdr->b_flags |= ARC_IO_IN_PROGRESS; 2896fa9e4066Sahrens 2897e14bb325SJeff Bonwick if (HDR_L2CACHE(hdr) && hdr->b_l2hdr != NULL && 2898e14bb325SJeff Bonwick (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) { 28995a98e54bSBrendan Gregg - Sun Microsystems devw = hdr->b_l2hdr->b_dev->l2ad_writing; 29003a737e0dSbrendan addr = hdr->b_l2hdr->b_daddr; 2901e14bb325SJeff Bonwick /* 2902e14bb325SJeff Bonwick * Lock out device removal. 2903e14bb325SJeff Bonwick */ 2904e14bb325SJeff Bonwick if (vdev_is_dead(vd) || 2905e14bb325SJeff Bonwick !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER)) 2906e14bb325SJeff Bonwick vd = NULL; 29073a737e0dSbrendan } 29083a737e0dSbrendan 29093a737e0dSbrendan mutex_exit(hash_lock); 29103a737e0dSbrendan 2911fa9e4066Sahrens ASSERT3U(hdr->b_size, ==, size); 29125c28183bSBrendan Gregg - Sun Microsystems DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp, 29135c28183bSBrendan Gregg - Sun Microsystems uint64_t, size, zbookmark_t *, zb); 291444cb6abcSbmc ARCSTAT_BUMP(arcstat_misses); 291544cb6abcSbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 291644cb6abcSbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 291744cb6abcSbmc data, metadata, misses); 2918ea8dc4b6Seschrock 29195a98e54bSBrendan Gregg - Sun Microsystems if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) { 2920fa94a07fSbrendan /* 2921fa94a07fSbrendan * Read from the L2ARC if the following are true: 29223a737e0dSbrendan * 1. The L2ARC vdev was previously cached. 29233a737e0dSbrendan * 2. This buffer still has L2ARC metadata. 29243a737e0dSbrendan * 3. This buffer isn't currently writing to the L2ARC. 29253a737e0dSbrendan * 4. The L2ARC entry wasn't evicted, which may 29263a737e0dSbrendan * also have invalidated the vdev. 29275a98e54bSBrendan Gregg - Sun Microsystems * 5. This isn't prefetch and l2arc_noprefetch is set. 2928fa94a07fSbrendan */ 2929e14bb325SJeff Bonwick if (hdr->b_l2hdr != NULL && 29305a98e54bSBrendan Gregg - Sun Microsystems !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) && 29315a98e54bSBrendan Gregg - Sun Microsystems !(l2arc_noprefetch && HDR_PREFETCH(hdr))) { 2932fa94a07fSbrendan l2arc_read_callback_t *cb; 2933fa94a07fSbrendan 2934c5904d13Seschrock DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr); 2935c5904d13Seschrock ARCSTAT_BUMP(arcstat_l2_hits); 2936c5904d13Seschrock 2937fa94a07fSbrendan cb = kmem_zalloc(sizeof (l2arc_read_callback_t), 2938fa94a07fSbrendan KM_SLEEP); 2939fa94a07fSbrendan cb->l2rcb_buf = buf; 2940fa94a07fSbrendan cb->l2rcb_spa = spa; 2941fa94a07fSbrendan cb->l2rcb_bp = *bp; 2942fa94a07fSbrendan cb->l2rcb_zb = *zb; 29433baa08fcSek cb->l2rcb_flags = zio_flags; 2944fa94a07fSbrendan 2945fa94a07fSbrendan /* 2946e14bb325SJeff Bonwick * l2arc read. The SCL_L2ARC lock will be 2947e14bb325SJeff Bonwick * released by l2arc_read_done(). 2948fa94a07fSbrendan */ 2949fa94a07fSbrendan rzio = zio_read_phys(pio, vd, addr, size, 2950fa94a07fSbrendan buf->b_data, ZIO_CHECKSUM_OFF, 29513baa08fcSek l2arc_read_done, cb, priority, zio_flags | 295249cf58c0SBrendan Gregg - Sun Microsystems ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL | 2953e14bb325SJeff Bonwick ZIO_FLAG_DONT_PROPAGATE | 2954e14bb325SJeff Bonwick ZIO_FLAG_DONT_RETRY, B_FALSE); 2955fa94a07fSbrendan DTRACE_PROBE2(l2arc__read, vdev_t *, vd, 2956fa94a07fSbrendan zio_t *, rzio); 29575a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_l2_read_bytes, size); 2958fa94a07fSbrendan 29593a737e0dSbrendan if (*arc_flags & ARC_NOWAIT) { 29603a737e0dSbrendan zio_nowait(rzio); 29613a737e0dSbrendan return (0); 29623a737e0dSbrendan } 2963fa94a07fSbrendan 29643a737e0dSbrendan ASSERT(*arc_flags & ARC_WAIT); 29653a737e0dSbrendan if (zio_wait(rzio) == 0) 29663a737e0dSbrendan return (0); 29673a737e0dSbrendan 29683a737e0dSbrendan /* l2arc read error; goto zio_read() */ 2969fa94a07fSbrendan } else { 2970fa94a07fSbrendan DTRACE_PROBE1(l2arc__miss, 2971fa94a07fSbrendan arc_buf_hdr_t *, hdr); 2972fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_misses); 2973fa94a07fSbrendan if (HDR_L2_WRITING(hdr)) 2974fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_rw_clash); 2975e14bb325SJeff Bonwick spa_config_exit(spa, SCL_L2ARC, vd); 2976fa94a07fSbrendan } 29775a98e54bSBrendan Gregg - Sun Microsystems } else { 297876a25fafSBill Moore if (vd != NULL) 297976a25fafSBill Moore spa_config_exit(spa, SCL_L2ARC, vd); 29805a98e54bSBrendan Gregg - Sun Microsystems if (l2arc_ndev != 0) { 29815a98e54bSBrendan Gregg - Sun Microsystems DTRACE_PROBE1(l2arc__miss, 29825a98e54bSBrendan Gregg - Sun Microsystems arc_buf_hdr_t *, hdr); 29835a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_BUMP(arcstat_l2_misses); 29845a98e54bSBrendan Gregg - Sun Microsystems } 2985fa94a07fSbrendan } 2986c5904d13Seschrock 2987fa9e4066Sahrens rzio = zio_read(pio, spa, bp, buf->b_data, size, 29883baa08fcSek arc_read_done, buf, priority, zio_flags, zb); 2989fa9e4066Sahrens 299013506d1eSmaybee if (*arc_flags & ARC_WAIT) 2991fa9e4066Sahrens return (zio_wait(rzio)); 2992fa9e4066Sahrens 299313506d1eSmaybee ASSERT(*arc_flags & ARC_NOWAIT); 2994fa9e4066Sahrens zio_nowait(rzio); 2995fa9e4066Sahrens } 2996fa9e4066Sahrens return (0); 2997fa9e4066Sahrens } 2998fa9e4066Sahrens 2999ea8dc4b6Seschrock void 3000ea8dc4b6Seschrock arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private) 3001ea8dc4b6Seschrock { 3002ea8dc4b6Seschrock ASSERT(buf->b_hdr != NULL); 300344cb6abcSbmc ASSERT(buf->b_hdr->b_state != arc_anon); 3004ea8dc4b6Seschrock ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL); 3005b24ab676SJeff Bonwick ASSERT(buf->b_efunc == NULL); 3006b24ab676SJeff Bonwick ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr)); 3007b24ab676SJeff Bonwick 3008ea8dc4b6Seschrock buf->b_efunc = func; 3009ea8dc4b6Seschrock buf->b_private = private; 3010ea8dc4b6Seschrock } 3011ea8dc4b6Seschrock 3012ea8dc4b6Seschrock /* 3013ea8dc4b6Seschrock * This is used by the DMU to let the ARC know that a buffer is 3014ea8dc4b6Seschrock * being evicted, so the ARC should clean up. If this arc buf 3015ea8dc4b6Seschrock * is not yet in the evicted state, it will be put there. 3016ea8dc4b6Seschrock */ 3017ea8dc4b6Seschrock int 3018ea8dc4b6Seschrock arc_buf_evict(arc_buf_t *buf) 3019ea8dc4b6Seschrock { 302040d7d650Smaybee arc_buf_hdr_t *hdr; 3021ea8dc4b6Seschrock kmutex_t *hash_lock; 3022ea8dc4b6Seschrock arc_buf_t **bufp; 3023ea8dc4b6Seschrock 30243f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 302540d7d650Smaybee hdr = buf->b_hdr; 3026ea8dc4b6Seschrock if (hdr == NULL) { 3027ea8dc4b6Seschrock /* 3028ea8dc4b6Seschrock * We are in arc_do_user_evicts(). 3029ea8dc4b6Seschrock */ 3030ea8dc4b6Seschrock ASSERT(buf->b_data == NULL); 30313f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 3032ea8dc4b6Seschrock return (0); 30336f83844dSMark Maybee } else if (buf->b_data == NULL) { 30346f83844dSMark Maybee arc_buf_t copy = *buf; /* structure assignment */ 30359b23f181Smaybee /* 30366f83844dSMark Maybee * We are on the eviction list; process this buffer now 30376f83844dSMark Maybee * but let arc_do_user_evicts() do the reaping. 30389b23f181Smaybee */ 30396f83844dSMark Maybee buf->b_efunc = NULL; 30403f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 30416f83844dSMark Maybee VERIFY(copy.b_efunc(©) == 0); 30426f83844dSMark Maybee return (1); 30439b23f181Smaybee } 30446f83844dSMark Maybee hash_lock = HDR_LOCK(hdr); 30456f83844dSMark Maybee mutex_enter(hash_lock); 30463f9d6ad7SLin Ling hdr = buf->b_hdr; 30473f9d6ad7SLin Ling ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 30489b23f181Smaybee 30499b23f181Smaybee ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt); 305044cb6abcSbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 3051ea8dc4b6Seschrock 3052ea8dc4b6Seschrock /* 3053ea8dc4b6Seschrock * Pull this buffer off of the hdr 3054ea8dc4b6Seschrock */ 3055ea8dc4b6Seschrock bufp = &hdr->b_buf; 3056ea8dc4b6Seschrock while (*bufp != buf) 3057ea8dc4b6Seschrock bufp = &(*bufp)->b_next; 3058ea8dc4b6Seschrock *bufp = buf->b_next; 3059ea8dc4b6Seschrock 3060ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 306144eda4d7Smaybee arc_buf_destroy(buf, FALSE, FALSE); 3062ea8dc4b6Seschrock 3063ea8dc4b6Seschrock if (hdr->b_datacnt == 0) { 3064ea8dc4b6Seschrock arc_state_t *old_state = hdr->b_state; 3065ea8dc4b6Seschrock arc_state_t *evicted_state; 3066ea8dc4b6Seschrock 30673f9d6ad7SLin Ling ASSERT(hdr->b_buf == NULL); 3068ea8dc4b6Seschrock ASSERT(refcount_is_zero(&hdr->b_refcnt)); 3069ea8dc4b6Seschrock 3070ea8dc4b6Seschrock evicted_state = 307144cb6abcSbmc (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 3072ea8dc4b6Seschrock 307344cb6abcSbmc mutex_enter(&old_state->arcs_mtx); 307444cb6abcSbmc mutex_enter(&evicted_state->arcs_mtx); 3075ea8dc4b6Seschrock 3076ea8dc4b6Seschrock arc_change_state(evicted_state, hdr, hash_lock); 3077ea8dc4b6Seschrock ASSERT(HDR_IN_HASH_TABLE(hdr)); 3078fa94a07fSbrendan hdr->b_flags |= ARC_IN_HASH_TABLE; 3079fa94a07fSbrendan hdr->b_flags &= ~ARC_BUF_AVAILABLE; 3080ea8dc4b6Seschrock 308144cb6abcSbmc mutex_exit(&evicted_state->arcs_mtx); 308244cb6abcSbmc mutex_exit(&old_state->arcs_mtx); 3083ea8dc4b6Seschrock } 3084ea8dc4b6Seschrock mutex_exit(hash_lock); 30853f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 3086dd6ef538Smaybee 3087ea8dc4b6Seschrock VERIFY(buf->b_efunc(buf) == 0); 3088ea8dc4b6Seschrock buf->b_efunc = NULL; 3089ea8dc4b6Seschrock buf->b_private = NULL; 3090ea8dc4b6Seschrock buf->b_hdr = NULL; 30913f9d6ad7SLin Ling buf->b_next = NULL; 3092ea8dc4b6Seschrock kmem_cache_free(buf_cache, buf); 3093ea8dc4b6Seschrock return (1); 3094ea8dc4b6Seschrock } 3095ea8dc4b6Seschrock 3096fa9e4066Sahrens /* 3097fa9e4066Sahrens * Release this buffer from the cache. This must be done 3098fa9e4066Sahrens * after a read and prior to modifying the buffer contents. 3099fa9e4066Sahrens * If the buffer has more than one reference, we must make 3100088f3894Sahrens * a new hdr for the buffer. 3101fa9e4066Sahrens */ 3102fa9e4066Sahrens void 3103fa9e4066Sahrens arc_release(arc_buf_t *buf, void *tag) 3104fa9e4066Sahrens { 31056f83844dSMark Maybee arc_buf_hdr_t *hdr; 31063f9d6ad7SLin Ling kmutex_t *hash_lock = NULL; 31076f83844dSMark Maybee l2arc_buf_hdr_t *l2hdr; 3108fa94a07fSbrendan uint64_t buf_size; 3109fa9e4066Sahrens 31103f9d6ad7SLin Ling /* 31113f9d6ad7SLin Ling * It would be nice to assert that if it's DMU metadata (level > 31123f9d6ad7SLin Ling * 0 || it's the dnode file), then it must be syncing context. 31133f9d6ad7SLin Ling * But we don't know that information at this level. 31143f9d6ad7SLin Ling */ 31153f9d6ad7SLin Ling 31163f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 31176f83844dSMark Maybee hdr = buf->b_hdr; 31186f83844dSMark Maybee 3119fa9e4066Sahrens /* this buffer is not on any list */ 3120fa9e4066Sahrens ASSERT(refcount_count(&hdr->b_refcnt) > 0); 3121fa9e4066Sahrens 312244cb6abcSbmc if (hdr->b_state == arc_anon) { 3123fa9e4066Sahrens /* this buffer is already released */ 3124ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 31250a95608cSBrendan Gregg - Sun Microsystems } else { 31260a95608cSBrendan Gregg - Sun Microsystems hash_lock = HDR_LOCK(hdr); 31270a95608cSBrendan Gregg - Sun Microsystems mutex_enter(hash_lock); 31283f9d6ad7SLin Ling hdr = buf->b_hdr; 31293f9d6ad7SLin Ling ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 3130fa9e4066Sahrens } 3131fa9e4066Sahrens 31326f83844dSMark Maybee l2hdr = hdr->b_l2hdr; 31336f83844dSMark Maybee if (l2hdr) { 31346f83844dSMark Maybee mutex_enter(&l2arc_buflist_mtx); 31356f83844dSMark Maybee hdr->b_l2hdr = NULL; 31366f83844dSMark Maybee buf_size = hdr->b_size; 31376f83844dSMark Maybee } 31386f83844dSMark Maybee 3139ea8dc4b6Seschrock /* 3140ea8dc4b6Seschrock * Do we have more than one buf? 3141ea8dc4b6Seschrock */ 31426f83844dSMark Maybee if (hdr->b_datacnt > 1) { 3143fa9e4066Sahrens arc_buf_hdr_t *nhdr; 3144fa9e4066Sahrens arc_buf_t **bufp; 3145fa9e4066Sahrens uint64_t blksz = hdr->b_size; 3146ac05c741SMark Maybee uint64_t spa = hdr->b_spa; 3147ad23a2dbSjohansen arc_buf_contents_t type = hdr->b_type; 3148fa94a07fSbrendan uint32_t flags = hdr->b_flags; 3149fa9e4066Sahrens 31506f83844dSMark Maybee ASSERT(hdr->b_buf != buf || buf->b_next != NULL); 3151fa9e4066Sahrens /* 31523f9d6ad7SLin Ling * Pull the data off of this hdr and attach it to 31533f9d6ad7SLin Ling * a new anonymous hdr. 3154fa9e4066Sahrens */ 3155ea8dc4b6Seschrock (void) remove_reference(hdr, hash_lock, tag); 3156fa9e4066Sahrens bufp = &hdr->b_buf; 3157ea8dc4b6Seschrock while (*bufp != buf) 3158fa9e4066Sahrens bufp = &(*bufp)->b_next; 31593f9d6ad7SLin Ling *bufp = buf->b_next; 3160af2c4821Smaybee buf->b_next = NULL; 3161ea8dc4b6Seschrock 316244cb6abcSbmc ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size); 316344cb6abcSbmc atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size); 3164ea8dc4b6Seschrock if (refcount_is_zero(&hdr->b_refcnt)) { 31650e8c6158Smaybee uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type]; 31660e8c6158Smaybee ASSERT3U(*size, >=, hdr->b_size); 31670e8c6158Smaybee atomic_add_64(size, -hdr->b_size); 3168ea8dc4b6Seschrock } 3169ea8dc4b6Seschrock hdr->b_datacnt -= 1; 3170c717a561Smaybee arc_cksum_verify(buf); 3171*cd1c8b85SMatthew Ahrens arc_buf_unwatch(buf); 3172ea8dc4b6Seschrock 3173fa9e4066Sahrens mutex_exit(hash_lock); 3174fa9e4066Sahrens 31751ab7f2deSmaybee nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 3176fa9e4066Sahrens nhdr->b_size = blksz; 3177fa9e4066Sahrens nhdr->b_spa = spa; 3178ad23a2dbSjohansen nhdr->b_type = type; 3179fa9e4066Sahrens nhdr->b_buf = buf; 318044cb6abcSbmc nhdr->b_state = arc_anon; 3181fa9e4066Sahrens nhdr->b_arc_access = 0; 3182fa94a07fSbrendan nhdr->b_flags = flags & ARC_L2_WRITING; 3183fa94a07fSbrendan nhdr->b_l2hdr = NULL; 3184ea8dc4b6Seschrock nhdr->b_datacnt = 1; 3185c717a561Smaybee nhdr->b_freeze_cksum = NULL; 3186fa9e4066Sahrens (void) refcount_add(&nhdr->b_refcnt, tag); 3187af2c4821Smaybee buf->b_hdr = nhdr; 31883f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 318944cb6abcSbmc atomic_add_64(&arc_anon->arcs_size, blksz); 3190fa9e4066Sahrens } else { 31913f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 3192ea8dc4b6Seschrock ASSERT(refcount_count(&hdr->b_refcnt) == 1); 3193fa9e4066Sahrens ASSERT(!list_link_active(&hdr->b_arc_node)); 3194fa9e4066Sahrens ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 31953f9d6ad7SLin Ling if (hdr->b_state != arc_anon) 31963f9d6ad7SLin Ling arc_change_state(arc_anon, hdr, hash_lock); 3197fa9e4066Sahrens hdr->b_arc_access = 0; 31983f9d6ad7SLin Ling if (hash_lock) 31993f9d6ad7SLin Ling mutex_exit(hash_lock); 3200fa94a07fSbrendan 32013f9d6ad7SLin Ling buf_discard_identity(hdr); 3202c717a561Smaybee arc_buf_thaw(buf); 3203fa9e4066Sahrens } 3204ea8dc4b6Seschrock buf->b_efunc = NULL; 3205ea8dc4b6Seschrock buf->b_private = NULL; 3206fa94a07fSbrendan 3207fa94a07fSbrendan if (l2hdr) { 3208fa94a07fSbrendan list_remove(l2hdr->b_dev->l2ad_buflist, hdr); 3209fa94a07fSbrendan kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t)); 3210fa94a07fSbrendan ARCSTAT_INCR(arcstat_l2_size, -buf_size); 3211fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 32126f83844dSMark Maybee } 3213fa9e4066Sahrens } 3214fa9e4066Sahrens 32153f9d6ad7SLin Ling /* 32163f9d6ad7SLin Ling * Release this buffer. If it does not match the provided BP, fill it 32173f9d6ad7SLin Ling * with that block's contents. 32183f9d6ad7SLin Ling */ 32193f9d6ad7SLin Ling /* ARGSUSED */ 32203f9d6ad7SLin Ling int 32213f9d6ad7SLin Ling arc_release_bp(arc_buf_t *buf, void *tag, blkptr_t *bp, spa_t *spa, 32223f9d6ad7SLin Ling zbookmark_t *zb) 32233f9d6ad7SLin Ling { 32243f9d6ad7SLin Ling arc_release(buf, tag); 32253f9d6ad7SLin Ling return (0); 32263f9d6ad7SLin Ling } 32273f9d6ad7SLin Ling 3228fa9e4066Sahrens int 3229fa9e4066Sahrens arc_released(arc_buf_t *buf) 3230fa9e4066Sahrens { 32316f83844dSMark Maybee int released; 32326f83844dSMark Maybee 32333f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 32346f83844dSMark Maybee released = (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon); 32353f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 32366f83844dSMark Maybee return (released); 3237ea8dc4b6Seschrock } 3238ea8dc4b6Seschrock 3239ea8dc4b6Seschrock int 3240ea8dc4b6Seschrock arc_has_callback(arc_buf_t *buf) 3241ea8dc4b6Seschrock { 32426f83844dSMark Maybee int callback; 32436f83844dSMark Maybee 32443f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 32456f83844dSMark Maybee callback = (buf->b_efunc != NULL); 32463f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 32476f83844dSMark Maybee return (callback); 3248fa9e4066Sahrens } 3249fa9e4066Sahrens 3250ea8dc4b6Seschrock #ifdef ZFS_DEBUG 3251ea8dc4b6Seschrock int 3252ea8dc4b6Seschrock arc_referenced(arc_buf_t *buf) 3253ea8dc4b6Seschrock { 32546f83844dSMark Maybee int referenced; 32556f83844dSMark Maybee 32563f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 32576f83844dSMark Maybee referenced = (refcount_count(&buf->b_hdr->b_refcnt)); 32583f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 32596f83844dSMark Maybee return (referenced); 3260ea8dc4b6Seschrock } 3261ea8dc4b6Seschrock #endif 3262ea8dc4b6Seschrock 3263c717a561Smaybee static void 3264c717a561Smaybee arc_write_ready(zio_t *zio) 3265c717a561Smaybee { 3266c717a561Smaybee arc_write_callback_t *callback = zio->io_private; 3267c717a561Smaybee arc_buf_t *buf = callback->awcb_buf; 32680a4e9518Sgw arc_buf_hdr_t *hdr = buf->b_hdr; 3269c717a561Smaybee 3270e14bb325SJeff Bonwick ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt)); 3271e14bb325SJeff Bonwick callback->awcb_ready(zio, buf, callback->awcb_private); 3272e14bb325SJeff Bonwick 32730a4e9518Sgw /* 32740a4e9518Sgw * If the IO is already in progress, then this is a re-write 3275e14bb325SJeff Bonwick * attempt, so we need to thaw and re-compute the cksum. 3276e14bb325SJeff Bonwick * It is the responsibility of the callback to handle the 3277e14bb325SJeff Bonwick * accounting for any re-write attempt. 32780a4e9518Sgw */ 32790a4e9518Sgw if (HDR_IO_IN_PROGRESS(hdr)) { 32800a4e9518Sgw mutex_enter(&hdr->b_freeze_lock); 32810a4e9518Sgw if (hdr->b_freeze_cksum != NULL) { 32820a4e9518Sgw kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 32830a4e9518Sgw hdr->b_freeze_cksum = NULL; 32840a4e9518Sgw } 32850a4e9518Sgw mutex_exit(&hdr->b_freeze_lock); 32860a4e9518Sgw } 3287fa94a07fSbrendan arc_cksum_compute(buf, B_FALSE); 32880a4e9518Sgw hdr->b_flags |= ARC_IO_IN_PROGRESS; 3289c717a561Smaybee } 3290c717a561Smaybee 3291fa9e4066Sahrens static void 3292fa9e4066Sahrens arc_write_done(zio_t *zio) 3293fa9e4066Sahrens { 3294c717a561Smaybee arc_write_callback_t *callback = zio->io_private; 3295c717a561Smaybee arc_buf_t *buf = callback->awcb_buf; 3296c717a561Smaybee arc_buf_hdr_t *hdr = buf->b_hdr; 3297fa9e4066Sahrens 3298b24ab676SJeff Bonwick ASSERT(hdr->b_acb == NULL); 3299b24ab676SJeff Bonwick 3300b24ab676SJeff Bonwick if (zio->io_error == 0) { 3301b24ab676SJeff Bonwick hdr->b_dva = *BP_IDENTITY(zio->io_bp); 3302b24ab676SJeff Bonwick hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp); 3303b24ab676SJeff Bonwick hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0]; 3304b24ab676SJeff Bonwick } else { 3305b24ab676SJeff Bonwick ASSERT(BUF_EMPTY(hdr)); 3306b24ab676SJeff Bonwick } 3307fa9e4066Sahrens 3308ea8dc4b6Seschrock /* 3309ea8dc4b6Seschrock * If the block to be written was all-zero, we may have 3310ea8dc4b6Seschrock * compressed it away. In this case no write was performed 33113f9d6ad7SLin Ling * so there will be no dva/birth/checksum. The buffer must 33123f9d6ad7SLin Ling * therefore remain anonymous (and uncached). 3313ea8dc4b6Seschrock */ 3314fa9e4066Sahrens if (!BUF_EMPTY(hdr)) { 3315fa9e4066Sahrens arc_buf_hdr_t *exists; 3316fa9e4066Sahrens kmutex_t *hash_lock; 3317fa9e4066Sahrens 3318b24ab676SJeff Bonwick ASSERT(zio->io_error == 0); 3319b24ab676SJeff Bonwick 33206b4acc8bSahrens arc_cksum_verify(buf); 33216b4acc8bSahrens 3322fa9e4066Sahrens exists = buf_hash_insert(hdr, &hash_lock); 3323fa9e4066Sahrens if (exists) { 3324fa9e4066Sahrens /* 3325fa9e4066Sahrens * This can only happen if we overwrite for 3326fa9e4066Sahrens * sync-to-convergence, because we remove 3327fa9e4066Sahrens * buffers from the hash table when we arc_free(). 3328fa9e4066Sahrens */ 3329b24ab676SJeff Bonwick if (zio->io_flags & ZIO_FLAG_IO_REWRITE) { 3330b24ab676SJeff Bonwick if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp)) 3331b24ab676SJeff Bonwick panic("bad overwrite, hdr=%p exists=%p", 3332b24ab676SJeff Bonwick (void *)hdr, (void *)exists); 3333b24ab676SJeff Bonwick ASSERT(refcount_is_zero(&exists->b_refcnt)); 3334b24ab676SJeff Bonwick arc_change_state(arc_anon, exists, hash_lock); 3335b24ab676SJeff Bonwick mutex_exit(hash_lock); 3336b24ab676SJeff Bonwick arc_hdr_destroy(exists); 3337b24ab676SJeff Bonwick exists = buf_hash_insert(hdr, &hash_lock); 3338b24ab676SJeff Bonwick ASSERT3P(exists, ==, NULL); 3339b24ab676SJeff Bonwick } else { 3340b24ab676SJeff Bonwick /* Dedup */ 3341b24ab676SJeff Bonwick ASSERT(hdr->b_datacnt == 1); 3342b24ab676SJeff Bonwick ASSERT(hdr->b_state == arc_anon); 3343b24ab676SJeff Bonwick ASSERT(BP_GET_DEDUP(zio->io_bp)); 3344b24ab676SJeff Bonwick ASSERT(BP_GET_LEVEL(zio->io_bp) == 0); 3345ae46e4c7SMatthew Ahrens } 3346fa9e4066Sahrens } 3347ea8dc4b6Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 3348088f3894Sahrens /* if it's not anon, we are doing a scrub */ 3349b24ab676SJeff Bonwick if (!exists && hdr->b_state == arc_anon) 3350088f3894Sahrens arc_access(hdr, hash_lock); 335144eda4d7Smaybee mutex_exit(hash_lock); 3352ea8dc4b6Seschrock } else { 3353ea8dc4b6Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 3354fa9e4066Sahrens } 3355ea8dc4b6Seschrock 3356b24ab676SJeff Bonwick ASSERT(!refcount_is_zero(&hdr->b_refcnt)); 3357b24ab676SJeff Bonwick callback->awcb_done(zio, buf, callback->awcb_private); 3358fa9e4066Sahrens 3359c717a561Smaybee kmem_free(callback, sizeof (arc_write_callback_t)); 3360fa9e4066Sahrens } 3361fa9e4066Sahrens 3362c717a561Smaybee zio_t * 3363b24ab676SJeff Bonwick arc_write(zio_t *pio, spa_t *spa, uint64_t txg, 3364b24ab676SJeff Bonwick blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, const zio_prop_t *zp, 3365b24ab676SJeff Bonwick arc_done_func_t *ready, arc_done_func_t *done, void *private, 3366b24ab676SJeff Bonwick int priority, int zio_flags, const zbookmark_t *zb) 3367fa9e4066Sahrens { 3368fa9e4066Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 3369c717a561Smaybee arc_write_callback_t *callback; 3370e14bb325SJeff Bonwick zio_t *zio; 3371fa9e4066Sahrens 3372e14bb325SJeff Bonwick ASSERT(ready != NULL); 3373b24ab676SJeff Bonwick ASSERT(done != NULL); 3374fa9e4066Sahrens ASSERT(!HDR_IO_ERROR(hdr)); 3375c5c6ffa0Smaybee ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0); 3376b24ab676SJeff Bonwick ASSERT(hdr->b_acb == NULL); 33773baa08fcSek if (l2arc) 33783baa08fcSek hdr->b_flags |= ARC_L2CACHE; 3379c717a561Smaybee callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP); 3380c717a561Smaybee callback->awcb_ready = ready; 3381c717a561Smaybee callback->awcb_done = done; 3382c717a561Smaybee callback->awcb_private = private; 3383c717a561Smaybee callback->awcb_buf = buf; 3384088f3894Sahrens 3385b24ab676SJeff Bonwick zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp, 3386e14bb325SJeff Bonwick arc_write_ready, arc_write_done, callback, priority, zio_flags, zb); 3387fa9e4066Sahrens 3388c717a561Smaybee return (zio); 3389fa9e4066Sahrens } 3390fa9e4066Sahrens 33911ab7f2deSmaybee static int 33922fdbea25SAleksandr Guzovskiy arc_memory_throttle(uint64_t reserve, uint64_t inflight_data, uint64_t txg) 33931ab7f2deSmaybee { 33941ab7f2deSmaybee #ifdef _KERNEL 33951ab7f2deSmaybee uint64_t available_memory = ptob(freemem); 33961ab7f2deSmaybee static uint64_t page_load = 0; 33971ab7f2deSmaybee static uint64_t last_txg = 0; 33981ab7f2deSmaybee 33991ab7f2deSmaybee #if defined(__i386) 34001ab7f2deSmaybee available_memory = 34011ab7f2deSmaybee MIN(available_memory, vmem_size(heap_arena, VMEM_FREE)); 34021ab7f2deSmaybee #endif 34031ab7f2deSmaybee if (available_memory >= zfs_write_limit_max) 34041ab7f2deSmaybee return (0); 34051ab7f2deSmaybee 34061ab7f2deSmaybee if (txg > last_txg) { 34071ab7f2deSmaybee last_txg = txg; 34081ab7f2deSmaybee page_load = 0; 34091ab7f2deSmaybee } 34101ab7f2deSmaybee /* 34111ab7f2deSmaybee * If we are in pageout, we know that memory is already tight, 34121ab7f2deSmaybee * the arc is already going to be evicting, so we just want to 34131ab7f2deSmaybee * continue to let page writes occur as quickly as possible. 34141ab7f2deSmaybee */ 34151ab7f2deSmaybee if (curproc == proc_pageout) { 34161ab7f2deSmaybee if (page_load > MAX(ptob(minfree), available_memory) / 4) 34171ab7f2deSmaybee return (ERESTART); 34181ab7f2deSmaybee /* Note: reserve is inflated, so we deflate */ 34191ab7f2deSmaybee page_load += reserve / 8; 34201ab7f2deSmaybee return (0); 34211ab7f2deSmaybee } else if (page_load > 0 && arc_reclaim_needed()) { 34221ab7f2deSmaybee /* memory is low, delay before restarting */ 34231ab7f2deSmaybee ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 34241ab7f2deSmaybee return (EAGAIN); 34251ab7f2deSmaybee } 34261ab7f2deSmaybee page_load = 0; 34271ab7f2deSmaybee 34281ab7f2deSmaybee if (arc_size > arc_c_min) { 34291ab7f2deSmaybee uint64_t evictable_memory = 34301ab7f2deSmaybee arc_mru->arcs_lsize[ARC_BUFC_DATA] + 34311ab7f2deSmaybee arc_mru->arcs_lsize[ARC_BUFC_METADATA] + 34321ab7f2deSmaybee arc_mfu->arcs_lsize[ARC_BUFC_DATA] + 34331ab7f2deSmaybee arc_mfu->arcs_lsize[ARC_BUFC_METADATA]; 34341ab7f2deSmaybee available_memory += MIN(evictable_memory, arc_size - arc_c_min); 34351ab7f2deSmaybee } 34361ab7f2deSmaybee 34371ab7f2deSmaybee if (inflight_data > available_memory / 4) { 34381ab7f2deSmaybee ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 34391ab7f2deSmaybee return (ERESTART); 34401ab7f2deSmaybee } 34411ab7f2deSmaybee #endif 34421ab7f2deSmaybee return (0); 34431ab7f2deSmaybee } 34441ab7f2deSmaybee 3445fa9e4066Sahrens void 34461ab7f2deSmaybee arc_tempreserve_clear(uint64_t reserve) 3447fa9e4066Sahrens { 34481ab7f2deSmaybee atomic_add_64(&arc_tempreserve, -reserve); 3449fa9e4066Sahrens ASSERT((int64_t)arc_tempreserve >= 0); 3450fa9e4066Sahrens } 3451fa9e4066Sahrens 3452fa9e4066Sahrens int 34531ab7f2deSmaybee arc_tempreserve_space(uint64_t reserve, uint64_t txg) 3454fa9e4066Sahrens { 34551ab7f2deSmaybee int error; 34562fdbea25SAleksandr Guzovskiy uint64_t anon_size; 34571ab7f2deSmaybee 3458fa9e4066Sahrens #ifdef ZFS_DEBUG 3459fa9e4066Sahrens /* 3460fa9e4066Sahrens * Once in a while, fail for no reason. Everything should cope. 3461fa9e4066Sahrens */ 3462fa9e4066Sahrens if (spa_get_random(10000) == 0) { 3463fa9e4066Sahrens dprintf("forcing random failure\n"); 3464fa9e4066Sahrens return (ERESTART); 3465fa9e4066Sahrens } 3466fa9e4066Sahrens #endif 34671ab7f2deSmaybee if (reserve > arc_c/4 && !arc_no_grow) 34681ab7f2deSmaybee arc_c = MIN(arc_c_max, reserve * 4); 34691ab7f2deSmaybee if (reserve > arc_c) 3470112fe045Smaybee return (ENOMEM); 3471112fe045Smaybee 34722fdbea25SAleksandr Guzovskiy /* 34732fdbea25SAleksandr Guzovskiy * Don't count loaned bufs as in flight dirty data to prevent long 34742fdbea25SAleksandr Guzovskiy * network delays from blocking transactions that are ready to be 34752fdbea25SAleksandr Guzovskiy * assigned to a txg. 34762fdbea25SAleksandr Guzovskiy */ 34772fdbea25SAleksandr Guzovskiy anon_size = MAX((int64_t)(arc_anon->arcs_size - arc_loaned_bytes), 0); 34782fdbea25SAleksandr Guzovskiy 34791ab7f2deSmaybee /* 34801ab7f2deSmaybee * Writes will, almost always, require additional memory allocations 34811ab7f2deSmaybee * in order to compress/encrypt/etc the data. We therefor need to 34821ab7f2deSmaybee * make sure that there is sufficient available memory for this. 34831ab7f2deSmaybee */ 34842fdbea25SAleksandr Guzovskiy if (error = arc_memory_throttle(reserve, anon_size, txg)) 34851ab7f2deSmaybee return (error); 34861ab7f2deSmaybee 3487fa9e4066Sahrens /* 3488112fe045Smaybee * Throttle writes when the amount of dirty data in the cache 3489112fe045Smaybee * gets too large. We try to keep the cache less than half full 3490112fe045Smaybee * of dirty blocks so that our sync times don't grow too large. 3491112fe045Smaybee * Note: if two requests come in concurrently, we might let them 3492112fe045Smaybee * both succeed, when one of them should fail. Not a huge deal. 3493fa9e4066Sahrens */ 34942fdbea25SAleksandr Guzovskiy 34952fdbea25SAleksandr Guzovskiy if (reserve + arc_tempreserve + anon_size > arc_c / 2 && 34962fdbea25SAleksandr Guzovskiy anon_size > arc_c / 4) { 34970e8c6158Smaybee dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK " 34980e8c6158Smaybee "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n", 34990e8c6158Smaybee arc_tempreserve>>10, 35000e8c6158Smaybee arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10, 35010e8c6158Smaybee arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10, 35021ab7f2deSmaybee reserve>>10, arc_c>>10); 3503fa9e4066Sahrens return (ERESTART); 3504fa9e4066Sahrens } 35051ab7f2deSmaybee atomic_add_64(&arc_tempreserve, reserve); 3506fa9e4066Sahrens return (0); 3507fa9e4066Sahrens } 3508fa9e4066Sahrens 3509fa9e4066Sahrens void 3510fa9e4066Sahrens arc_init(void) 3511fa9e4066Sahrens { 3512fa9e4066Sahrens mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL); 3513fa9e4066Sahrens cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL); 3514fa9e4066Sahrens 351513506d1eSmaybee /* Convert seconds to clock ticks */ 3516b19a79ecSperrin arc_min_prefetch_lifespan = 1 * hz; 351713506d1eSmaybee 3518fa9e4066Sahrens /* Start out with 1/8 of all memory */ 351944cb6abcSbmc arc_c = physmem * PAGESIZE / 8; 3520fa9e4066Sahrens 3521fa9e4066Sahrens #ifdef _KERNEL 3522fa9e4066Sahrens /* 3523fa9e4066Sahrens * On architectures where the physical memory can be larger 3524fa9e4066Sahrens * than the addressable space (intel in 32-bit mode), we may 3525fa9e4066Sahrens * need to limit the cache to 1/8 of VM size. 3526fa9e4066Sahrens */ 352744cb6abcSbmc arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8); 3528fa9e4066Sahrens #endif 3529fa9e4066Sahrens 3530112fe045Smaybee /* set min cache to 1/32 of all memory, or 64MB, whichever is more */ 353144cb6abcSbmc arc_c_min = MAX(arc_c / 4, 64<<20); 3532112fe045Smaybee /* set max to 3/4 of all memory, or all but 1GB, whichever is more */ 353344cb6abcSbmc if (arc_c * 8 >= 1<<30) 353444cb6abcSbmc arc_c_max = (arc_c * 8) - (1<<30); 3535fa9e4066Sahrens else 353644cb6abcSbmc arc_c_max = arc_c_min; 353744cb6abcSbmc arc_c_max = MAX(arc_c * 6, arc_c_max); 3538a2eea2e1Sahrens 3539a2eea2e1Sahrens /* 3540a2eea2e1Sahrens * Allow the tunables to override our calculations if they are 3541a2eea2e1Sahrens * reasonable (ie. over 64MB) 3542a2eea2e1Sahrens */ 3543a2eea2e1Sahrens if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE) 354444cb6abcSbmc arc_c_max = zfs_arc_max; 354544cb6abcSbmc if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max) 354644cb6abcSbmc arc_c_min = zfs_arc_min; 3547a2eea2e1Sahrens 354844cb6abcSbmc arc_c = arc_c_max; 354944cb6abcSbmc arc_p = (arc_c >> 1); 3550fa9e4066Sahrens 35510e8c6158Smaybee /* limit meta-data to 1/4 of the arc capacity */ 35520e8c6158Smaybee arc_meta_limit = arc_c_max / 4; 35531116048bSek 35541116048bSek /* Allow the tunable to override if it is reasonable */ 35551116048bSek if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max) 35561116048bSek arc_meta_limit = zfs_arc_meta_limit; 35571116048bSek 35580e8c6158Smaybee if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0) 35590e8c6158Smaybee arc_c_min = arc_meta_limit / 2; 35600e8c6158Smaybee 35615a98e54bSBrendan Gregg - Sun Microsystems if (zfs_arc_grow_retry > 0) 35625a98e54bSBrendan Gregg - Sun Microsystems arc_grow_retry = zfs_arc_grow_retry; 35635a98e54bSBrendan Gregg - Sun Microsystems 35645a98e54bSBrendan Gregg - Sun Microsystems if (zfs_arc_shrink_shift > 0) 35655a98e54bSBrendan Gregg - Sun Microsystems arc_shrink_shift = zfs_arc_shrink_shift; 35665a98e54bSBrendan Gregg - Sun Microsystems 35675a98e54bSBrendan Gregg - Sun Microsystems if (zfs_arc_p_min_shift > 0) 35685a98e54bSBrendan Gregg - Sun Microsystems arc_p_min_shift = zfs_arc_p_min_shift; 35695a98e54bSBrendan Gregg - Sun Microsystems 3570fa9e4066Sahrens /* if kmem_flags are set, lets try to use less memory */ 3571fa9e4066Sahrens if (kmem_debugging()) 357244cb6abcSbmc arc_c = arc_c / 2; 357344cb6abcSbmc if (arc_c < arc_c_min) 357444cb6abcSbmc arc_c = arc_c_min; 357544cb6abcSbmc 357644cb6abcSbmc arc_anon = &ARC_anon; 357744cb6abcSbmc arc_mru = &ARC_mru; 357844cb6abcSbmc arc_mru_ghost = &ARC_mru_ghost; 357944cb6abcSbmc arc_mfu = &ARC_mfu; 358044cb6abcSbmc arc_mfu_ghost = &ARC_mfu_ghost; 3581fa94a07fSbrendan arc_l2c_only = &ARC_l2c_only; 358244cb6abcSbmc arc_size = 0; 358344cb6abcSbmc 358444cb6abcSbmc mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 358544cb6abcSbmc mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 358644cb6abcSbmc mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 358744cb6abcSbmc mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 358844cb6abcSbmc mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3589fa94a07fSbrendan mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 359044cb6abcSbmc 35910e8c6158Smaybee list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA], 35920e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 35930e8c6158Smaybee list_create(&arc_mru->arcs_list[ARC_BUFC_DATA], 35940e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 35950e8c6158Smaybee list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA], 35960e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 35970e8c6158Smaybee list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA], 35980e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 35990e8c6158Smaybee list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA], 36000e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 36010e8c6158Smaybee list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA], 36020e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 36030e8c6158Smaybee list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA], 36040e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 36050e8c6158Smaybee list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA], 36060e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3607fa94a07fSbrendan list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA], 3608fa94a07fSbrendan sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3609fa94a07fSbrendan list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA], 3610fa94a07fSbrendan sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3611fa9e4066Sahrens 3612fa9e4066Sahrens buf_init(); 3613fa9e4066Sahrens 3614fa9e4066Sahrens arc_thread_exit = 0; 3615ea8dc4b6Seschrock arc_eviction_list = NULL; 3616ea8dc4b6Seschrock mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL); 361740d7d650Smaybee bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t)); 3618fa9e4066Sahrens 361944cb6abcSbmc arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED, 362044cb6abcSbmc sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 362144cb6abcSbmc 362244cb6abcSbmc if (arc_ksp != NULL) { 362344cb6abcSbmc arc_ksp->ks_data = &arc_stats; 362444cb6abcSbmc kstat_install(arc_ksp); 362544cb6abcSbmc } 362644cb6abcSbmc 3627fa9e4066Sahrens (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0, 3628fa9e4066Sahrens TS_RUN, minclsyspri); 362949e3519aSmaybee 363049e3519aSmaybee arc_dead = FALSE; 36313a737e0dSbrendan arc_warm = B_FALSE; 36321ab7f2deSmaybee 36331ab7f2deSmaybee if (zfs_write_limit_max == 0) 363405715f94SMark Maybee zfs_write_limit_max = ptob(physmem) >> zfs_write_limit_shift; 36351ab7f2deSmaybee else 36361ab7f2deSmaybee zfs_write_limit_shift = 0; 363705715f94SMark Maybee mutex_init(&zfs_write_limit_lock, NULL, MUTEX_DEFAULT, NULL); 3638fa9e4066Sahrens } 3639fa9e4066Sahrens 3640fa9e4066Sahrens void 3641fa9e4066Sahrens arc_fini(void) 3642fa9e4066Sahrens { 3643fa9e4066Sahrens mutex_enter(&arc_reclaim_thr_lock); 3644fa9e4066Sahrens arc_thread_exit = 1; 3645fa9e4066Sahrens while (arc_thread_exit != 0) 3646fa9e4066Sahrens cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock); 3647fa9e4066Sahrens mutex_exit(&arc_reclaim_thr_lock); 3648fa9e4066Sahrens 3649874395d5Smaybee arc_flush(NULL); 3650fa9e4066Sahrens 3651fa9e4066Sahrens arc_dead = TRUE; 3652fa9e4066Sahrens 365344cb6abcSbmc if (arc_ksp != NULL) { 365444cb6abcSbmc kstat_delete(arc_ksp); 365544cb6abcSbmc arc_ksp = NULL; 365644cb6abcSbmc } 365744cb6abcSbmc 3658ea8dc4b6Seschrock mutex_destroy(&arc_eviction_mtx); 3659fa9e4066Sahrens mutex_destroy(&arc_reclaim_thr_lock); 3660fa9e4066Sahrens cv_destroy(&arc_reclaim_thr_cv); 3661fa9e4066Sahrens 36620e8c6158Smaybee list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]); 36630e8c6158Smaybee list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]); 36640e8c6158Smaybee list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]); 36650e8c6158Smaybee list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]); 36660e8c6158Smaybee list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]); 36670e8c6158Smaybee list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]); 36680e8c6158Smaybee list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]); 36690e8c6158Smaybee list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]); 3670fa9e4066Sahrens 367144cb6abcSbmc mutex_destroy(&arc_anon->arcs_mtx); 367244cb6abcSbmc mutex_destroy(&arc_mru->arcs_mtx); 367344cb6abcSbmc mutex_destroy(&arc_mru_ghost->arcs_mtx); 367444cb6abcSbmc mutex_destroy(&arc_mfu->arcs_mtx); 367544cb6abcSbmc mutex_destroy(&arc_mfu_ghost->arcs_mtx); 3676b5e70f97SRicardo M. Correia mutex_destroy(&arc_l2c_only->arcs_mtx); 36775ad82045Snd 367805715f94SMark Maybee mutex_destroy(&zfs_write_limit_lock); 367905715f94SMark Maybee 3680fa9e4066Sahrens buf_fini(); 36812fdbea25SAleksandr Guzovskiy 36822fdbea25SAleksandr Guzovskiy ASSERT(arc_loaned_bytes == 0); 3683fa9e4066Sahrens } 3684fa94a07fSbrendan 3685fa94a07fSbrendan /* 3686fa94a07fSbrendan * Level 2 ARC 3687fa94a07fSbrendan * 3688fa94a07fSbrendan * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk. 3689fa94a07fSbrendan * It uses dedicated storage devices to hold cached data, which are populated 3690fa94a07fSbrendan * using large infrequent writes. The main role of this cache is to boost 3691fa94a07fSbrendan * the performance of random read workloads. The intended L2ARC devices 3692fa94a07fSbrendan * include short-stroked disks, solid state disks, and other media with 3693fa94a07fSbrendan * substantially faster read latency than disk. 3694fa94a07fSbrendan * 3695fa94a07fSbrendan * +-----------------------+ 3696fa94a07fSbrendan * | ARC | 3697fa94a07fSbrendan * +-----------------------+ 3698fa94a07fSbrendan * | ^ ^ 3699fa94a07fSbrendan * | | | 3700fa94a07fSbrendan * l2arc_feed_thread() arc_read() 3701fa94a07fSbrendan * | | | 3702fa94a07fSbrendan * | l2arc read | 3703fa94a07fSbrendan * V | | 3704fa94a07fSbrendan * +---------------+ | 3705fa94a07fSbrendan * | L2ARC | | 3706fa94a07fSbrendan * +---------------+ | 3707fa94a07fSbrendan * | ^ | 3708fa94a07fSbrendan * l2arc_write() | | 3709fa94a07fSbrendan * | | | 3710fa94a07fSbrendan * V | | 3711fa94a07fSbrendan * +-------+ +-------+ 3712fa94a07fSbrendan * | vdev | | vdev | 3713fa94a07fSbrendan * | cache | | cache | 3714fa94a07fSbrendan * +-------+ +-------+ 3715fa94a07fSbrendan * +=========+ .-----. 3716fa94a07fSbrendan * : L2ARC : |-_____-| 3717fa94a07fSbrendan * : devices : | Disks | 3718fa94a07fSbrendan * +=========+ `-_____-' 3719fa94a07fSbrendan * 3720fa94a07fSbrendan * Read requests are satisfied from the following sources, in order: 3721fa94a07fSbrendan * 3722fa94a07fSbrendan * 1) ARC 3723fa94a07fSbrendan * 2) vdev cache of L2ARC devices 3724fa94a07fSbrendan * 3) L2ARC devices 3725fa94a07fSbrendan * 4) vdev cache of disks 3726fa94a07fSbrendan * 5) disks 3727fa94a07fSbrendan * 3728fa94a07fSbrendan * Some L2ARC device types exhibit extremely slow write performance. 3729fa94a07fSbrendan * To accommodate for this there are some significant differences between 3730fa94a07fSbrendan * the L2ARC and traditional cache design: 3731fa94a07fSbrendan * 3732fa94a07fSbrendan * 1. There is no eviction path from the ARC to the L2ARC. Evictions from 3733fa94a07fSbrendan * the ARC behave as usual, freeing buffers and placing headers on ghost 3734fa94a07fSbrendan * lists. The ARC does not send buffers to the L2ARC during eviction as 3735fa94a07fSbrendan * this would add inflated write latencies for all ARC memory pressure. 3736fa94a07fSbrendan * 3737fa94a07fSbrendan * 2. The L2ARC attempts to cache data from the ARC before it is evicted. 3738fa94a07fSbrendan * It does this by periodically scanning buffers from the eviction-end of 3739fa94a07fSbrendan * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are 3740fa94a07fSbrendan * not already there. It scans until a headroom of buffers is satisfied, 3741fa94a07fSbrendan * which itself is a buffer for ARC eviction. The thread that does this is 3742fa94a07fSbrendan * l2arc_feed_thread(), illustrated below; example sizes are included to 3743fa94a07fSbrendan * provide a better sense of ratio than this diagram: 3744fa94a07fSbrendan * 3745fa94a07fSbrendan * head --> tail 3746fa94a07fSbrendan * +---------------------+----------+ 3747fa94a07fSbrendan * ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->. # already on L2ARC 3748fa94a07fSbrendan * +---------------------+----------+ | o L2ARC eligible 3749fa94a07fSbrendan * ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->| : ARC buffer 3750fa94a07fSbrendan * +---------------------+----------+ | 3751fa94a07fSbrendan * 15.9 Gbytes ^ 32 Mbytes | 3752fa94a07fSbrendan * headroom | 3753fa94a07fSbrendan * l2arc_feed_thread() 3754fa94a07fSbrendan * | 3755fa94a07fSbrendan * l2arc write hand <--[oooo]--' 3756fa94a07fSbrendan * | 8 Mbyte 3757fa94a07fSbrendan * | write max 3758fa94a07fSbrendan * V 3759fa94a07fSbrendan * +==============================+ 3760fa94a07fSbrendan * L2ARC dev |####|#|###|###| |####| ... | 3761fa94a07fSbrendan * +==============================+ 3762fa94a07fSbrendan * 32 Gbytes 3763fa94a07fSbrendan * 3764fa94a07fSbrendan * 3. If an ARC buffer is copied to the L2ARC but then hit instead of 3765fa94a07fSbrendan * evicted, then the L2ARC has cached a buffer much sooner than it probably 3766fa94a07fSbrendan * needed to, potentially wasting L2ARC device bandwidth and storage. It is 3767fa94a07fSbrendan * safe to say that this is an uncommon case, since buffers at the end of 3768fa94a07fSbrendan * the ARC lists have moved there due to inactivity. 3769fa94a07fSbrendan * 3770fa94a07fSbrendan * 4. If the ARC evicts faster than the L2ARC can maintain a headroom, 3771fa94a07fSbrendan * then the L2ARC simply misses copying some buffers. This serves as a 3772fa94a07fSbrendan * pressure valve to prevent heavy read workloads from both stalling the ARC 3773fa94a07fSbrendan * with waits and clogging the L2ARC with writes. This also helps prevent 3774fa94a07fSbrendan * the potential for the L2ARC to churn if it attempts to cache content too 3775fa94a07fSbrendan * quickly, such as during backups of the entire pool. 3776fa94a07fSbrendan * 37773a737e0dSbrendan * 5. After system boot and before the ARC has filled main memory, there are 37783a737e0dSbrendan * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru 37793a737e0dSbrendan * lists can remain mostly static. Instead of searching from tail of these 37803a737e0dSbrendan * lists as pictured, the l2arc_feed_thread() will search from the list heads 37813a737e0dSbrendan * for eligible buffers, greatly increasing its chance of finding them. 37823a737e0dSbrendan * 37833a737e0dSbrendan * The L2ARC device write speed is also boosted during this time so that 37843a737e0dSbrendan * the L2ARC warms up faster. Since there have been no ARC evictions yet, 37853a737e0dSbrendan * there are no L2ARC reads, and no fear of degrading read performance 37863a737e0dSbrendan * through increased writes. 37873a737e0dSbrendan * 37883a737e0dSbrendan * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that 3789fa94a07fSbrendan * the vdev queue can aggregate them into larger and fewer writes. Each 3790fa94a07fSbrendan * device is written to in a rotor fashion, sweeping writes through 3791fa94a07fSbrendan * available space then repeating. 3792fa94a07fSbrendan * 37933a737e0dSbrendan * 7. The L2ARC does not store dirty content. It never needs to flush 3794fa94a07fSbrendan * write buffers back to disk based storage. 3795fa94a07fSbrendan * 37963a737e0dSbrendan * 8. If an ARC buffer is written (and dirtied) which also exists in the 3797fa94a07fSbrendan * L2ARC, the now stale L2ARC buffer is immediately dropped. 3798fa94a07fSbrendan * 3799fa94a07fSbrendan * The performance of the L2ARC can be tweaked by a number of tunables, which 3800fa94a07fSbrendan * may be necessary for different workloads: 3801fa94a07fSbrendan * 3802fa94a07fSbrendan * l2arc_write_max max write bytes per interval 38033a737e0dSbrendan * l2arc_write_boost extra write bytes during device warmup 3804fa94a07fSbrendan * l2arc_noprefetch skip caching prefetched buffers 3805fa94a07fSbrendan * l2arc_headroom number of max device writes to precache 3806fa94a07fSbrendan * l2arc_feed_secs seconds between L2ARC writing 3807fa94a07fSbrendan * 3808fa94a07fSbrendan * Tunables may be removed or added as future performance improvements are 3809fa94a07fSbrendan * integrated, and also may become zpool properties. 38105a98e54bSBrendan Gregg - Sun Microsystems * 38115a98e54bSBrendan Gregg - Sun Microsystems * There are three key functions that control how the L2ARC warms up: 38125a98e54bSBrendan Gregg - Sun Microsystems * 38135a98e54bSBrendan Gregg - Sun Microsystems * l2arc_write_eligible() check if a buffer is eligible to cache 38145a98e54bSBrendan Gregg - Sun Microsystems * l2arc_write_size() calculate how much to write 38155a98e54bSBrendan Gregg - Sun Microsystems * l2arc_write_interval() calculate sleep delay between writes 38165a98e54bSBrendan Gregg - Sun Microsystems * 38175a98e54bSBrendan Gregg - Sun Microsystems * These three functions determine what to write, how much, and how quickly 38185a98e54bSBrendan Gregg - Sun Microsystems * to send writes. 3819fa94a07fSbrendan */ 3820fa94a07fSbrendan 38215a98e54bSBrendan Gregg - Sun Microsystems static boolean_t 3822ac05c741SMark Maybee l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab) 38235a98e54bSBrendan Gregg - Sun Microsystems { 38245a98e54bSBrendan Gregg - Sun Microsystems /* 38255a98e54bSBrendan Gregg - Sun Microsystems * A buffer is *not* eligible for the L2ARC if it: 38265a98e54bSBrendan Gregg - Sun Microsystems * 1. belongs to a different spa. 38275ea40c06SBrendan Gregg - Sun Microsystems * 2. is already cached on the L2ARC. 38285ea40c06SBrendan Gregg - Sun Microsystems * 3. has an I/O in progress (it may be an incomplete read). 38295ea40c06SBrendan Gregg - Sun Microsystems * 4. is flagged not eligible (zfs property). 38305a98e54bSBrendan Gregg - Sun Microsystems */ 38315ea40c06SBrendan Gregg - Sun Microsystems if (ab->b_spa != spa_guid || ab->b_l2hdr != NULL || 38325a98e54bSBrendan Gregg - Sun Microsystems HDR_IO_IN_PROGRESS(ab) || !HDR_L2CACHE(ab)) 38335a98e54bSBrendan Gregg - Sun Microsystems return (B_FALSE); 38345a98e54bSBrendan Gregg - Sun Microsystems 38355a98e54bSBrendan Gregg - Sun Microsystems return (B_TRUE); 38365a98e54bSBrendan Gregg - Sun Microsystems } 38375a98e54bSBrendan Gregg - Sun Microsystems 38385a98e54bSBrendan Gregg - Sun Microsystems static uint64_t 38395a98e54bSBrendan Gregg - Sun Microsystems l2arc_write_size(l2arc_dev_t *dev) 38405a98e54bSBrendan Gregg - Sun Microsystems { 38415a98e54bSBrendan Gregg - Sun Microsystems uint64_t size; 38425a98e54bSBrendan Gregg - Sun Microsystems 38435a98e54bSBrendan Gregg - Sun Microsystems size = dev->l2ad_write; 38445a98e54bSBrendan Gregg - Sun Microsystems 38455a98e54bSBrendan Gregg - Sun Microsystems if (arc_warm == B_FALSE) 38465a98e54bSBrendan Gregg - Sun Microsystems size += dev->l2ad_boost; 38475a98e54bSBrendan Gregg - Sun Microsystems 38485a98e54bSBrendan Gregg - Sun Microsystems return (size); 38495a98e54bSBrendan Gregg - Sun Microsystems 38505a98e54bSBrendan Gregg - Sun Microsystems } 38515a98e54bSBrendan Gregg - Sun Microsystems 38525a98e54bSBrendan Gregg - Sun Microsystems static clock_t 38535a98e54bSBrendan Gregg - Sun Microsystems l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote) 38545a98e54bSBrendan Gregg - Sun Microsystems { 3855d3d50737SRafael Vanoni clock_t interval, next, now; 38565a98e54bSBrendan Gregg - Sun Microsystems 38575a98e54bSBrendan Gregg - Sun Microsystems /* 38585a98e54bSBrendan Gregg - Sun Microsystems * If the ARC lists are busy, increase our write rate; if the 38595a98e54bSBrendan Gregg - Sun Microsystems * lists are stale, idle back. This is achieved by checking 38605a98e54bSBrendan Gregg - Sun Microsystems * how much we previously wrote - if it was more than half of 38615a98e54bSBrendan Gregg - Sun Microsystems * what we wanted, schedule the next write much sooner. 38625a98e54bSBrendan Gregg - Sun Microsystems */ 38635a98e54bSBrendan Gregg - Sun Microsystems if (l2arc_feed_again && wrote > (wanted / 2)) 38645a98e54bSBrendan Gregg - Sun Microsystems interval = (hz * l2arc_feed_min_ms) / 1000; 38655a98e54bSBrendan Gregg - Sun Microsystems else 38665a98e54bSBrendan Gregg - Sun Microsystems interval = hz * l2arc_feed_secs; 38675a98e54bSBrendan Gregg - Sun Microsystems 3868d3d50737SRafael Vanoni now = ddi_get_lbolt(); 3869d3d50737SRafael Vanoni next = MAX(now, MIN(now + interval, began + interval)); 38705a98e54bSBrendan Gregg - Sun Microsystems 38715a98e54bSBrendan Gregg - Sun Microsystems return (next); 38725a98e54bSBrendan Gregg - Sun Microsystems } 38735a98e54bSBrendan Gregg - Sun Microsystems 3874fa94a07fSbrendan static void 3875fa94a07fSbrendan l2arc_hdr_stat_add(void) 3876fa94a07fSbrendan { 3877e6c728e1Sbrendan ARCSTAT_INCR(arcstat_l2_hdr_size, HDR_SIZE + L2HDR_SIZE); 3878e6c728e1Sbrendan ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE); 3879fa94a07fSbrendan } 3880fa94a07fSbrendan 3881fa94a07fSbrendan static void 3882fa94a07fSbrendan l2arc_hdr_stat_remove(void) 3883fa94a07fSbrendan { 3884e6c728e1Sbrendan ARCSTAT_INCR(arcstat_l2_hdr_size, -(HDR_SIZE + L2HDR_SIZE)); 3885e6c728e1Sbrendan ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE); 3886fa94a07fSbrendan } 3887fa94a07fSbrendan 3888fa94a07fSbrendan /* 3889fa94a07fSbrendan * Cycle through L2ARC devices. This is how L2ARC load balances. 38903a737e0dSbrendan * If a device is returned, this also returns holding the spa config lock. 3891fa94a07fSbrendan */ 3892fa94a07fSbrendan static l2arc_dev_t * 3893fa94a07fSbrendan l2arc_dev_get_next(void) 3894fa94a07fSbrendan { 38953a737e0dSbrendan l2arc_dev_t *first, *next = NULL; 38963a737e0dSbrendan 38973a737e0dSbrendan /* 38983a737e0dSbrendan * Lock out the removal of spas (spa_namespace_lock), then removal 38993a737e0dSbrendan * of cache devices (l2arc_dev_mtx). Once a device has been selected, 39003a737e0dSbrendan * both locks will be dropped and a spa config lock held instead. 39013a737e0dSbrendan */ 39023a737e0dSbrendan mutex_enter(&spa_namespace_lock); 39033a737e0dSbrendan mutex_enter(&l2arc_dev_mtx); 3904fa94a07fSbrendan 3905c5904d13Seschrock /* if there are no vdevs, there is nothing to do */ 3906c5904d13Seschrock if (l2arc_ndev == 0) 39073a737e0dSbrendan goto out; 3908c5904d13Seschrock 3909c5904d13Seschrock first = NULL; 3910c5904d13Seschrock next = l2arc_dev_last; 3911c5904d13Seschrock do { 3912c5904d13Seschrock /* loop around the list looking for a non-faulted vdev */ 3913c5904d13Seschrock if (next == NULL) { 3914fa94a07fSbrendan next = list_head(l2arc_dev_list); 3915c5904d13Seschrock } else { 3916c5904d13Seschrock next = list_next(l2arc_dev_list, next); 3917c5904d13Seschrock if (next == NULL) 3918c5904d13Seschrock next = list_head(l2arc_dev_list); 3919c5904d13Seschrock } 3920c5904d13Seschrock 3921c5904d13Seschrock /* if we have come back to the start, bail out */ 3922c5904d13Seschrock if (first == NULL) 3923c5904d13Seschrock first = next; 3924c5904d13Seschrock else if (next == first) 3925c5904d13Seschrock break; 3926c5904d13Seschrock 3927c5904d13Seschrock } while (vdev_is_dead(next->l2ad_vdev)); 3928c5904d13Seschrock 3929c5904d13Seschrock /* if we were unable to find any usable vdevs, return NULL */ 3930c5904d13Seschrock if (vdev_is_dead(next->l2ad_vdev)) 39313a737e0dSbrendan next = NULL; 3932fa94a07fSbrendan 3933fa94a07fSbrendan l2arc_dev_last = next; 3934fa94a07fSbrendan 39353a737e0dSbrendan out: 39363a737e0dSbrendan mutex_exit(&l2arc_dev_mtx); 39373a737e0dSbrendan 39383a737e0dSbrendan /* 39393a737e0dSbrendan * Grab the config lock to prevent the 'next' device from being 39403a737e0dSbrendan * removed while we are writing to it. 39413a737e0dSbrendan */ 39423a737e0dSbrendan if (next != NULL) 3943e14bb325SJeff Bonwick spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER); 39443a737e0dSbrendan mutex_exit(&spa_namespace_lock); 39453a737e0dSbrendan 3946fa94a07fSbrendan return (next); 3947fa94a07fSbrendan } 3948fa94a07fSbrendan 39493a737e0dSbrendan /* 39503a737e0dSbrendan * Free buffers that were tagged for destruction. 39513a737e0dSbrendan */ 39523a737e0dSbrendan static void 39533a737e0dSbrendan l2arc_do_free_on_write() 39543a737e0dSbrendan { 39553a737e0dSbrendan list_t *buflist; 39563a737e0dSbrendan l2arc_data_free_t *df, *df_prev; 39573a737e0dSbrendan 39583a737e0dSbrendan mutex_enter(&l2arc_free_on_write_mtx); 39593a737e0dSbrendan buflist = l2arc_free_on_write; 39603a737e0dSbrendan 39613a737e0dSbrendan for (df = list_tail(buflist); df; df = df_prev) { 39623a737e0dSbrendan df_prev = list_prev(buflist, df); 39633a737e0dSbrendan ASSERT(df->l2df_data != NULL); 39643a737e0dSbrendan ASSERT(df->l2df_func != NULL); 39653a737e0dSbrendan df->l2df_func(df->l2df_data, df->l2df_size); 39663a737e0dSbrendan list_remove(buflist, df); 39673a737e0dSbrendan kmem_free(df, sizeof (l2arc_data_free_t)); 39683a737e0dSbrendan } 39693a737e0dSbrendan 39703a737e0dSbrendan mutex_exit(&l2arc_free_on_write_mtx); 39713a737e0dSbrendan } 39723a737e0dSbrendan 3973fa94a07fSbrendan /* 3974fa94a07fSbrendan * A write to a cache device has completed. Update all headers to allow 3975fa94a07fSbrendan * reads from these buffers to begin. 3976fa94a07fSbrendan */ 3977fa94a07fSbrendan static void 3978fa94a07fSbrendan l2arc_write_done(zio_t *zio) 3979fa94a07fSbrendan { 3980fa94a07fSbrendan l2arc_write_callback_t *cb; 3981fa94a07fSbrendan l2arc_dev_t *dev; 3982fa94a07fSbrendan list_t *buflist; 3983fa94a07fSbrendan arc_buf_hdr_t *head, *ab, *ab_prev; 39843a737e0dSbrendan l2arc_buf_hdr_t *abl2; 3985fa94a07fSbrendan kmutex_t *hash_lock; 3986fa94a07fSbrendan 3987fa94a07fSbrendan cb = zio->io_private; 3988fa94a07fSbrendan ASSERT(cb != NULL); 3989fa94a07fSbrendan dev = cb->l2wcb_dev; 3990fa94a07fSbrendan ASSERT(dev != NULL); 3991fa94a07fSbrendan head = cb->l2wcb_head; 3992fa94a07fSbrendan ASSERT(head != NULL); 3993fa94a07fSbrendan buflist = dev->l2ad_buflist; 3994fa94a07fSbrendan ASSERT(buflist != NULL); 3995fa94a07fSbrendan DTRACE_PROBE2(l2arc__iodone, zio_t *, zio, 3996fa94a07fSbrendan l2arc_write_callback_t *, cb); 3997fa94a07fSbrendan 3998fa94a07fSbrendan if (zio->io_error != 0) 3999fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_writes_error); 4000fa94a07fSbrendan 4001fa94a07fSbrendan mutex_enter(&l2arc_buflist_mtx); 4002fa94a07fSbrendan 4003fa94a07fSbrendan /* 4004fa94a07fSbrendan * All writes completed, or an error was hit. 4005fa94a07fSbrendan */ 4006fa94a07fSbrendan for (ab = list_prev(buflist, head); ab; ab = ab_prev) { 4007fa94a07fSbrendan ab_prev = list_prev(buflist, ab); 4008fa94a07fSbrendan 4009fa94a07fSbrendan hash_lock = HDR_LOCK(ab); 4010fa94a07fSbrendan if (!mutex_tryenter(hash_lock)) { 4011fa94a07fSbrendan /* 4012fa94a07fSbrendan * This buffer misses out. It may be in a stage 4013fa94a07fSbrendan * of eviction. Its ARC_L2_WRITING flag will be 4014fa94a07fSbrendan * left set, denying reads to this buffer. 4015fa94a07fSbrendan */ 4016fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss); 4017fa94a07fSbrendan continue; 4018fa94a07fSbrendan } 4019fa94a07fSbrendan 4020fa94a07fSbrendan if (zio->io_error != 0) { 4021fa94a07fSbrendan /* 40223a737e0dSbrendan * Error - drop L2ARC entry. 4023fa94a07fSbrendan */ 40243a737e0dSbrendan list_remove(buflist, ab); 40253a737e0dSbrendan abl2 = ab->b_l2hdr; 4026fa94a07fSbrendan ab->b_l2hdr = NULL; 40273a737e0dSbrendan kmem_free(abl2, sizeof (l2arc_buf_hdr_t)); 40283a737e0dSbrendan ARCSTAT_INCR(arcstat_l2_size, -ab->b_size); 4029fa94a07fSbrendan } 4030fa94a07fSbrendan 4031fa94a07fSbrendan /* 4032fa94a07fSbrendan * Allow ARC to begin reads to this L2ARC entry. 4033fa94a07fSbrendan */ 4034fa94a07fSbrendan ab->b_flags &= ~ARC_L2_WRITING; 4035fa94a07fSbrendan 4036fa94a07fSbrendan mutex_exit(hash_lock); 4037fa94a07fSbrendan } 4038fa94a07fSbrendan 4039fa94a07fSbrendan atomic_inc_64(&l2arc_writes_done); 4040fa94a07fSbrendan list_remove(buflist, head); 4041fa94a07fSbrendan kmem_cache_free(hdr_cache, head); 4042fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 4043fa94a07fSbrendan 40443a737e0dSbrendan l2arc_do_free_on_write(); 4045fa94a07fSbrendan 4046fa94a07fSbrendan kmem_free(cb, sizeof (l2arc_write_callback_t)); 4047fa94a07fSbrendan } 4048fa94a07fSbrendan 4049fa94a07fSbrendan /* 4050fa94a07fSbrendan * A read to a cache device completed. Validate buffer contents before 4051fa94a07fSbrendan * handing over to the regular ARC routines. 4052fa94a07fSbrendan */ 4053fa94a07fSbrendan static void 4054fa94a07fSbrendan l2arc_read_done(zio_t *zio) 4055fa94a07fSbrendan { 4056fa94a07fSbrendan l2arc_read_callback_t *cb; 4057fa94a07fSbrendan arc_buf_hdr_t *hdr; 4058fa94a07fSbrendan arc_buf_t *buf; 4059fa94a07fSbrendan kmutex_t *hash_lock; 40603a737e0dSbrendan int equal; 4061fa94a07fSbrendan 4062e14bb325SJeff Bonwick ASSERT(zio->io_vd != NULL); 4063e14bb325SJeff Bonwick ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE); 4064e14bb325SJeff Bonwick 4065e14bb325SJeff Bonwick spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd); 4066e14bb325SJeff Bonwick 4067fa94a07fSbrendan cb = zio->io_private; 4068fa94a07fSbrendan ASSERT(cb != NULL); 4069fa94a07fSbrendan buf = cb->l2rcb_buf; 4070fa94a07fSbrendan ASSERT(buf != NULL); 4071fa94a07fSbrendan 40723f9d6ad7SLin Ling hash_lock = HDR_LOCK(buf->b_hdr); 4073fa94a07fSbrendan mutex_enter(hash_lock); 40743f9d6ad7SLin Ling hdr = buf->b_hdr; 40753f9d6ad7SLin Ling ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 4076fa94a07fSbrendan 4077fa94a07fSbrendan /* 4078fa94a07fSbrendan * Check this survived the L2ARC journey. 4079fa94a07fSbrendan */ 4080fa94a07fSbrendan equal = arc_cksum_equal(buf); 4081fa94a07fSbrendan if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) { 4082fa94a07fSbrendan mutex_exit(hash_lock); 4083fa94a07fSbrendan zio->io_private = buf; 4084e14bb325SJeff Bonwick zio->io_bp_copy = cb->l2rcb_bp; /* XXX fix in L2ARC 2.0 */ 4085e14bb325SJeff Bonwick zio->io_bp = &zio->io_bp_copy; /* XXX fix in L2ARC 2.0 */ 4086fa94a07fSbrendan arc_read_done(zio); 4087fa94a07fSbrendan } else { 4088fa94a07fSbrendan mutex_exit(hash_lock); 4089fa94a07fSbrendan /* 4090fa94a07fSbrendan * Buffer didn't survive caching. Increment stats and 4091fa94a07fSbrendan * reissue to the original storage device. 4092fa94a07fSbrendan */ 40933a737e0dSbrendan if (zio->io_error != 0) { 4094fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_io_error); 40953a737e0dSbrendan } else { 40963a737e0dSbrendan zio->io_error = EIO; 40973a737e0dSbrendan } 4098fa94a07fSbrendan if (!equal) 4099fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_cksum_bad); 4100fa94a07fSbrendan 4101e14bb325SJeff Bonwick /* 4102e14bb325SJeff Bonwick * If there's no waiter, issue an async i/o to the primary 4103e14bb325SJeff Bonwick * storage now. If there *is* a waiter, the caller must 4104e14bb325SJeff Bonwick * issue the i/o in a context where it's OK to block. 4105e14bb325SJeff Bonwick */ 4106a3f829aeSBill Moore if (zio->io_waiter == NULL) { 4107a3f829aeSBill Moore zio_t *pio = zio_unique_parent(zio); 4108a3f829aeSBill Moore 4109a3f829aeSBill Moore ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL); 4110a3f829aeSBill Moore 4111a3f829aeSBill Moore zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp, 4112e14bb325SJeff Bonwick buf->b_data, zio->io_size, arc_read_done, buf, 4113e14bb325SJeff Bonwick zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb)); 4114a3f829aeSBill Moore } 4115fa94a07fSbrendan } 4116fa94a07fSbrendan 4117fa94a07fSbrendan kmem_free(cb, sizeof (l2arc_read_callback_t)); 4118fa94a07fSbrendan } 4119fa94a07fSbrendan 4120fa94a07fSbrendan /* 4121fa94a07fSbrendan * This is the list priority from which the L2ARC will search for pages to 4122fa94a07fSbrendan * cache. This is used within loops (0..3) to cycle through lists in the 4123fa94a07fSbrendan * desired order. This order can have a significant effect on cache 4124fa94a07fSbrendan * performance. 4125fa94a07fSbrendan * 4126fa94a07fSbrendan * Currently the metadata lists are hit first, MFU then MRU, followed by 4127fa94a07fSbrendan * the data lists. This function returns a locked list, and also returns 4128fa94a07fSbrendan * the lock pointer. 4129fa94a07fSbrendan */ 4130fa94a07fSbrendan static list_t * 4131fa94a07fSbrendan l2arc_list_locked(int list_num, kmutex_t **lock) 4132fa94a07fSbrendan { 4133fa94a07fSbrendan list_t *list; 4134fa94a07fSbrendan 4135fa94a07fSbrendan ASSERT(list_num >= 0 && list_num <= 3); 4136fa94a07fSbrendan 4137fa94a07fSbrendan switch (list_num) { 4138fa94a07fSbrendan case 0: 4139fa94a07fSbrendan list = &arc_mfu->arcs_list[ARC_BUFC_METADATA]; 4140fa94a07fSbrendan *lock = &arc_mfu->arcs_mtx; 4141fa94a07fSbrendan break; 4142fa94a07fSbrendan case 1: 4143fa94a07fSbrendan list = &arc_mru->arcs_list[ARC_BUFC_METADATA]; 4144fa94a07fSbrendan *lock = &arc_mru->arcs_mtx; 4145fa94a07fSbrendan break; 4146fa94a07fSbrendan case 2: 4147fa94a07fSbrendan list = &arc_mfu->arcs_list[ARC_BUFC_DATA]; 4148fa94a07fSbrendan *lock = &arc_mfu->arcs_mtx; 4149fa94a07fSbrendan break; 4150fa94a07fSbrendan case 3: 4151fa94a07fSbrendan list = &arc_mru->arcs_list[ARC_BUFC_DATA]; 4152fa94a07fSbrendan *lock = &arc_mru->arcs_mtx; 4153fa94a07fSbrendan break; 4154fa94a07fSbrendan } 4155fa94a07fSbrendan 4156fa94a07fSbrendan ASSERT(!(MUTEX_HELD(*lock))); 4157fa94a07fSbrendan mutex_enter(*lock); 4158fa94a07fSbrendan return (list); 4159fa94a07fSbrendan } 4160fa94a07fSbrendan 4161fa94a07fSbrendan /* 4162fa94a07fSbrendan * Evict buffers from the device write hand to the distance specified in 4163fa94a07fSbrendan * bytes. This distance may span populated buffers, it may span nothing. 4164fa94a07fSbrendan * This is clearing a region on the L2ARC device ready for writing. 4165fa94a07fSbrendan * If the 'all' boolean is set, every buffer is evicted. 4166fa94a07fSbrendan */ 4167fa94a07fSbrendan static void 4168fa94a07fSbrendan l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all) 4169fa94a07fSbrendan { 4170fa94a07fSbrendan list_t *buflist; 4171fa94a07fSbrendan l2arc_buf_hdr_t *abl2; 4172fa94a07fSbrendan arc_buf_hdr_t *ab, *ab_prev; 4173fa94a07fSbrendan kmutex_t *hash_lock; 4174fa94a07fSbrendan uint64_t taddr; 4175fa94a07fSbrendan 4176fa94a07fSbrendan buflist = dev->l2ad_buflist; 4177fa94a07fSbrendan 4178fa94a07fSbrendan if (buflist == NULL) 4179fa94a07fSbrendan return; 4180fa94a07fSbrendan 4181fa94a07fSbrendan if (!all && dev->l2ad_first) { 4182fa94a07fSbrendan /* 4183fa94a07fSbrendan * This is the first sweep through the device. There is 4184fa94a07fSbrendan * nothing to evict. 4185fa94a07fSbrendan */ 4186fa94a07fSbrendan return; 4187fa94a07fSbrendan } 4188fa94a07fSbrendan 41893a737e0dSbrendan if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) { 4190fa94a07fSbrendan /* 4191fa94a07fSbrendan * When nearing the end of the device, evict to the end 4192fa94a07fSbrendan * before the device write hand jumps to the start. 4193fa94a07fSbrendan */ 4194fa94a07fSbrendan taddr = dev->l2ad_end; 4195fa94a07fSbrendan } else { 4196fa94a07fSbrendan taddr = dev->l2ad_hand + distance; 4197fa94a07fSbrendan } 4198fa94a07fSbrendan DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist, 4199fa94a07fSbrendan uint64_t, taddr, boolean_t, all); 4200fa94a07fSbrendan 4201fa94a07fSbrendan top: 4202fa94a07fSbrendan mutex_enter(&l2arc_buflist_mtx); 4203fa94a07fSbrendan for (ab = list_tail(buflist); ab; ab = ab_prev) { 4204fa94a07fSbrendan ab_prev = list_prev(buflist, ab); 4205fa94a07fSbrendan 4206fa94a07fSbrendan hash_lock = HDR_LOCK(ab); 4207fa94a07fSbrendan if (!mutex_tryenter(hash_lock)) { 4208fa94a07fSbrendan /* 4209fa94a07fSbrendan * Missed the hash lock. Retry. 4210fa94a07fSbrendan */ 4211fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_evict_lock_retry); 4212fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 4213fa94a07fSbrendan mutex_enter(hash_lock); 4214fa94a07fSbrendan mutex_exit(hash_lock); 4215fa94a07fSbrendan goto top; 4216fa94a07fSbrendan } 4217fa94a07fSbrendan 4218fa94a07fSbrendan if (HDR_L2_WRITE_HEAD(ab)) { 4219fa94a07fSbrendan /* 4220fa94a07fSbrendan * We hit a write head node. Leave it for 4221fa94a07fSbrendan * l2arc_write_done(). 4222fa94a07fSbrendan */ 4223fa94a07fSbrendan list_remove(buflist, ab); 4224fa94a07fSbrendan mutex_exit(hash_lock); 4225fa94a07fSbrendan continue; 4226fa94a07fSbrendan } 4227fa94a07fSbrendan 4228fa94a07fSbrendan if (!all && ab->b_l2hdr != NULL && 4229fa94a07fSbrendan (ab->b_l2hdr->b_daddr > taddr || 4230fa94a07fSbrendan ab->b_l2hdr->b_daddr < dev->l2ad_hand)) { 4231fa94a07fSbrendan /* 4232fa94a07fSbrendan * We've evicted to the target address, 4233fa94a07fSbrendan * or the end of the device. 4234fa94a07fSbrendan */ 4235fa94a07fSbrendan mutex_exit(hash_lock); 4236fa94a07fSbrendan break; 4237fa94a07fSbrendan } 4238fa94a07fSbrendan 4239fa94a07fSbrendan if (HDR_FREE_IN_PROGRESS(ab)) { 4240fa94a07fSbrendan /* 4241fa94a07fSbrendan * Already on the path to destruction. 4242fa94a07fSbrendan */ 4243fa94a07fSbrendan mutex_exit(hash_lock); 4244fa94a07fSbrendan continue; 4245fa94a07fSbrendan } 4246fa94a07fSbrendan 4247fa94a07fSbrendan if (ab->b_state == arc_l2c_only) { 4248fa94a07fSbrendan ASSERT(!HDR_L2_READING(ab)); 4249fa94a07fSbrendan /* 4250fa94a07fSbrendan * This doesn't exist in the ARC. Destroy. 4251fa94a07fSbrendan * arc_hdr_destroy() will call list_remove() 4252fa94a07fSbrendan * and decrement arcstat_l2_size. 4253fa94a07fSbrendan */ 4254fa94a07fSbrendan arc_change_state(arc_anon, ab, hash_lock); 4255fa94a07fSbrendan arc_hdr_destroy(ab); 4256fa94a07fSbrendan } else { 42573a737e0dSbrendan /* 42583a737e0dSbrendan * Invalidate issued or about to be issued 42593a737e0dSbrendan * reads, since we may be about to write 42603a737e0dSbrendan * over this location. 42613a737e0dSbrendan */ 42623a737e0dSbrendan if (HDR_L2_READING(ab)) { 42633a737e0dSbrendan ARCSTAT_BUMP(arcstat_l2_evict_reading); 42643a737e0dSbrendan ab->b_flags |= ARC_L2_EVICTED; 42653a737e0dSbrendan } 42663a737e0dSbrendan 4267fa94a07fSbrendan /* 4268fa94a07fSbrendan * Tell ARC this no longer exists in L2ARC. 4269fa94a07fSbrendan */ 4270fa94a07fSbrendan if (ab->b_l2hdr != NULL) { 4271fa94a07fSbrendan abl2 = ab->b_l2hdr; 4272fa94a07fSbrendan ab->b_l2hdr = NULL; 4273fa94a07fSbrendan kmem_free(abl2, sizeof (l2arc_buf_hdr_t)); 4274fa94a07fSbrendan ARCSTAT_INCR(arcstat_l2_size, -ab->b_size); 4275fa94a07fSbrendan } 4276fa94a07fSbrendan list_remove(buflist, ab); 4277fa94a07fSbrendan 4278fa94a07fSbrendan /* 4279fa94a07fSbrendan * This may have been leftover after a 4280fa94a07fSbrendan * failed write. 4281fa94a07fSbrendan */ 4282fa94a07fSbrendan ab->b_flags &= ~ARC_L2_WRITING; 4283fa94a07fSbrendan } 4284fa94a07fSbrendan mutex_exit(hash_lock); 4285fa94a07fSbrendan } 4286fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 4287fa94a07fSbrendan 4288b24ab676SJeff Bonwick vdev_space_update(dev->l2ad_vdev, -(taddr - dev->l2ad_evict), 0, 0); 4289fa94a07fSbrendan dev->l2ad_evict = taddr; 4290fa94a07fSbrendan } 4291fa94a07fSbrendan 4292fa94a07fSbrendan /* 4293fa94a07fSbrendan * Find and write ARC buffers to the L2ARC device. 4294fa94a07fSbrendan * 4295fa94a07fSbrendan * An ARC_L2_WRITING flag is set so that the L2ARC buffers are not valid 4296fa94a07fSbrendan * for reading until they have completed writing. 4297fa94a07fSbrendan */ 42985a98e54bSBrendan Gregg - Sun Microsystems static uint64_t 42993a737e0dSbrendan l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz) 4300fa94a07fSbrendan { 4301fa94a07fSbrendan arc_buf_hdr_t *ab, *ab_prev, *head; 4302fa94a07fSbrendan l2arc_buf_hdr_t *hdrl2; 4303fa94a07fSbrendan list_t *list; 43043a737e0dSbrendan uint64_t passed_sz, write_sz, buf_sz, headroom; 4305fa94a07fSbrendan void *buf_data; 4306fa94a07fSbrendan kmutex_t *hash_lock, *list_lock; 4307fa94a07fSbrendan boolean_t have_lock, full; 4308fa94a07fSbrendan l2arc_write_callback_t *cb; 4309fa94a07fSbrendan zio_t *pio, *wzio; 4310e9103aaeSGarrett D'Amore uint64_t guid = spa_load_guid(spa); 4311fa94a07fSbrendan 4312fa94a07fSbrendan ASSERT(dev->l2ad_vdev != NULL); 4313fa94a07fSbrendan 4314fa94a07fSbrendan pio = NULL; 4315fa94a07fSbrendan write_sz = 0; 4316fa94a07fSbrendan full = B_FALSE; 43171ab7f2deSmaybee head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 4318fa94a07fSbrendan head->b_flags |= ARC_L2_WRITE_HEAD; 4319fa94a07fSbrendan 4320fa94a07fSbrendan /* 4321fa94a07fSbrendan * Copy buffers for L2ARC writing. 4322fa94a07fSbrendan */ 4323fa94a07fSbrendan mutex_enter(&l2arc_buflist_mtx); 4324fa94a07fSbrendan for (int try = 0; try <= 3; try++) { 4325fa94a07fSbrendan list = l2arc_list_locked(try, &list_lock); 4326fa94a07fSbrendan passed_sz = 0; 4327fa94a07fSbrendan 43283a737e0dSbrendan /* 43293a737e0dSbrendan * L2ARC fast warmup. 43303a737e0dSbrendan * 43313a737e0dSbrendan * Until the ARC is warm and starts to evict, read from the 43323a737e0dSbrendan * head of the ARC lists rather than the tail. 43333a737e0dSbrendan */ 43343a737e0dSbrendan headroom = target_sz * l2arc_headroom; 43353a737e0dSbrendan if (arc_warm == B_FALSE) 43363a737e0dSbrendan ab = list_head(list); 43373a737e0dSbrendan else 43383a737e0dSbrendan ab = list_tail(list); 43393a737e0dSbrendan 43403a737e0dSbrendan for (; ab; ab = ab_prev) { 43413a737e0dSbrendan if (arc_warm == B_FALSE) 43423a737e0dSbrendan ab_prev = list_next(list, ab); 43433a737e0dSbrendan else 43443a737e0dSbrendan ab_prev = list_prev(list, ab); 4345fa94a07fSbrendan 4346fa94a07fSbrendan hash_lock = HDR_LOCK(ab); 4347fa94a07fSbrendan have_lock = MUTEX_HELD(hash_lock); 4348fa94a07fSbrendan if (!have_lock && !mutex_tryenter(hash_lock)) { 4349fa94a07fSbrendan /* 4350fa94a07fSbrendan * Skip this buffer rather than waiting. 4351fa94a07fSbrendan */ 4352fa94a07fSbrendan continue; 4353fa94a07fSbrendan } 4354fa94a07fSbrendan 4355fa94a07fSbrendan passed_sz += ab->b_size; 4356fa94a07fSbrendan if (passed_sz > headroom) { 4357fa94a07fSbrendan /* 4358fa94a07fSbrendan * Searched too far. 4359fa94a07fSbrendan */ 4360fa94a07fSbrendan mutex_exit(hash_lock); 4361fa94a07fSbrendan break; 4362fa94a07fSbrendan } 4363fa94a07fSbrendan 4364ac05c741SMark Maybee if (!l2arc_write_eligible(guid, ab)) { 4365fa94a07fSbrendan mutex_exit(hash_lock); 4366fa94a07fSbrendan continue; 4367fa94a07fSbrendan } 4368fa94a07fSbrendan 4369fa94a07fSbrendan if ((write_sz + ab->b_size) > target_sz) { 4370fa94a07fSbrendan full = B_TRUE; 4371fa94a07fSbrendan mutex_exit(hash_lock); 4372fa94a07fSbrendan break; 4373fa94a07fSbrendan } 4374fa94a07fSbrendan 4375fa94a07fSbrendan if (pio == NULL) { 4376fa94a07fSbrendan /* 4377fa94a07fSbrendan * Insert a dummy header on the buflist so 4378fa94a07fSbrendan * l2arc_write_done() can find where the 4379fa94a07fSbrendan * write buffers begin without searching. 4380fa94a07fSbrendan */ 4381fa94a07fSbrendan list_insert_head(dev->l2ad_buflist, head); 4382fa94a07fSbrendan 4383fa94a07fSbrendan cb = kmem_alloc( 4384fa94a07fSbrendan sizeof (l2arc_write_callback_t), KM_SLEEP); 4385fa94a07fSbrendan cb->l2wcb_dev = dev; 4386fa94a07fSbrendan cb->l2wcb_head = head; 4387fa94a07fSbrendan pio = zio_root(spa, l2arc_write_done, cb, 4388fa94a07fSbrendan ZIO_FLAG_CANFAIL); 4389fa94a07fSbrendan } 4390fa94a07fSbrendan 4391fa94a07fSbrendan /* 4392fa94a07fSbrendan * Create and add a new L2ARC header. 4393fa94a07fSbrendan */ 4394fa94a07fSbrendan hdrl2 = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP); 4395fa94a07fSbrendan hdrl2->b_dev = dev; 4396fa94a07fSbrendan hdrl2->b_daddr = dev->l2ad_hand; 4397fa94a07fSbrendan 4398fa94a07fSbrendan ab->b_flags |= ARC_L2_WRITING; 4399fa94a07fSbrendan ab->b_l2hdr = hdrl2; 4400fa94a07fSbrendan list_insert_head(dev->l2ad_buflist, ab); 4401fa94a07fSbrendan buf_data = ab->b_buf->b_data; 4402fa94a07fSbrendan buf_sz = ab->b_size; 4403fa94a07fSbrendan 4404fa94a07fSbrendan /* 4405fa94a07fSbrendan * Compute and store the buffer cksum before 4406fa94a07fSbrendan * writing. On debug the cksum is verified first. 4407fa94a07fSbrendan */ 4408fa94a07fSbrendan arc_cksum_verify(ab->b_buf); 4409fa94a07fSbrendan arc_cksum_compute(ab->b_buf, B_TRUE); 4410fa94a07fSbrendan 4411fa94a07fSbrendan mutex_exit(hash_lock); 4412fa94a07fSbrendan 4413fa94a07fSbrendan wzio = zio_write_phys(pio, dev->l2ad_vdev, 4414fa94a07fSbrendan dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF, 4415fa94a07fSbrendan NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE, 4416fa94a07fSbrendan ZIO_FLAG_CANFAIL, B_FALSE); 4417fa94a07fSbrendan 4418fa94a07fSbrendan DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, 4419fa94a07fSbrendan zio_t *, wzio); 4420fa94a07fSbrendan (void) zio_nowait(wzio); 4421fa94a07fSbrendan 4422e14bb325SJeff Bonwick /* 4423e14bb325SJeff Bonwick * Keep the clock hand suitably device-aligned. 4424e14bb325SJeff Bonwick */ 4425e14bb325SJeff Bonwick buf_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz); 4426e14bb325SJeff Bonwick 4427fa94a07fSbrendan write_sz += buf_sz; 4428fa94a07fSbrendan dev->l2ad_hand += buf_sz; 4429fa94a07fSbrendan } 4430fa94a07fSbrendan 4431fa94a07fSbrendan mutex_exit(list_lock); 4432fa94a07fSbrendan 4433fa94a07fSbrendan if (full == B_TRUE) 4434fa94a07fSbrendan break; 4435fa94a07fSbrendan } 4436fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 4437fa94a07fSbrendan 4438fa94a07fSbrendan if (pio == NULL) { 4439b420f3adSRichard Lowe ASSERT3U(write_sz, ==, 0); 4440fa94a07fSbrendan kmem_cache_free(hdr_cache, head); 44415a98e54bSBrendan Gregg - Sun Microsystems return (0); 4442fa94a07fSbrendan } 4443fa94a07fSbrendan 4444fa94a07fSbrendan ASSERT3U(write_sz, <=, target_sz); 4445fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_writes_sent); 44465a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_l2_write_bytes, write_sz); 4447fa94a07fSbrendan ARCSTAT_INCR(arcstat_l2_size, write_sz); 4448b24ab676SJeff Bonwick vdev_space_update(dev->l2ad_vdev, write_sz, 0, 0); 4449fa94a07fSbrendan 4450fa94a07fSbrendan /* 4451fa94a07fSbrendan * Bump device hand to the device start if it is approaching the end. 4452fa94a07fSbrendan * l2arc_evict() will already have evicted ahead for this case. 4453fa94a07fSbrendan */ 44543a737e0dSbrendan if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) { 4455b24ab676SJeff Bonwick vdev_space_update(dev->l2ad_vdev, 4456b24ab676SJeff Bonwick dev->l2ad_end - dev->l2ad_hand, 0, 0); 4457fa94a07fSbrendan dev->l2ad_hand = dev->l2ad_start; 4458fa94a07fSbrendan dev->l2ad_evict = dev->l2ad_start; 4459fa94a07fSbrendan dev->l2ad_first = B_FALSE; 4460fa94a07fSbrendan } 4461fa94a07fSbrendan 44625a98e54bSBrendan Gregg - Sun Microsystems dev->l2ad_writing = B_TRUE; 4463fa94a07fSbrendan (void) zio_wait(pio); 44645a98e54bSBrendan Gregg - Sun Microsystems dev->l2ad_writing = B_FALSE; 44655a98e54bSBrendan Gregg - Sun Microsystems 44665a98e54bSBrendan Gregg - Sun Microsystems return (write_sz); 4467fa94a07fSbrendan } 4468fa94a07fSbrendan 4469fa94a07fSbrendan /* 4470fa94a07fSbrendan * This thread feeds the L2ARC at regular intervals. This is the beating 4471fa94a07fSbrendan * heart of the L2ARC. 4472fa94a07fSbrendan */ 4473fa94a07fSbrendan static void 4474fa94a07fSbrendan l2arc_feed_thread(void) 4475fa94a07fSbrendan { 4476fa94a07fSbrendan callb_cpr_t cpr; 4477fa94a07fSbrendan l2arc_dev_t *dev; 4478fa94a07fSbrendan spa_t *spa; 44795a98e54bSBrendan Gregg - Sun Microsystems uint64_t size, wrote; 4480d3d50737SRafael Vanoni clock_t begin, next = ddi_get_lbolt(); 4481fa94a07fSbrendan 4482fa94a07fSbrendan CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG); 4483fa94a07fSbrendan 4484fa94a07fSbrendan mutex_enter(&l2arc_feed_thr_lock); 4485fa94a07fSbrendan 4486fa94a07fSbrendan while (l2arc_thread_exit == 0) { 4487fa94a07fSbrendan CALLB_CPR_SAFE_BEGIN(&cpr); 4488fa94a07fSbrendan (void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock, 44895a98e54bSBrendan Gregg - Sun Microsystems next); 4490fa94a07fSbrendan CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock); 4491d3d50737SRafael Vanoni next = ddi_get_lbolt() + hz; 4492fa94a07fSbrendan 44933a737e0dSbrendan /* 44943a737e0dSbrendan * Quick check for L2ARC devices. 44953a737e0dSbrendan */ 4496c5904d13Seschrock mutex_enter(&l2arc_dev_mtx); 44973a737e0dSbrendan if (l2arc_ndev == 0) { 44983a737e0dSbrendan mutex_exit(&l2arc_dev_mtx); 44993a737e0dSbrendan continue; 45003a737e0dSbrendan } 45013a737e0dSbrendan mutex_exit(&l2arc_dev_mtx); 4502d3d50737SRafael Vanoni begin = ddi_get_lbolt(); 4503c5904d13Seschrock 4504fa94a07fSbrendan /* 4505c5904d13Seschrock * This selects the next l2arc device to write to, and in 4506c5904d13Seschrock * doing so the next spa to feed from: dev->l2ad_spa. This 45073a737e0dSbrendan * will return NULL if there are now no l2arc devices or if 45083a737e0dSbrendan * they are all faulted. 45093a737e0dSbrendan * 45103a737e0dSbrendan * If a device is returned, its spa's config lock is also 45113a737e0dSbrendan * held to prevent device removal. l2arc_dev_get_next() 45123a737e0dSbrendan * will grab and release l2arc_dev_mtx. 4513fa94a07fSbrendan */ 45143a737e0dSbrendan if ((dev = l2arc_dev_get_next()) == NULL) 4515fa94a07fSbrendan continue; 45163a737e0dSbrendan 45173a737e0dSbrendan spa = dev->l2ad_spa; 45183a737e0dSbrendan ASSERT(spa != NULL); 4519fa94a07fSbrendan 4520f9af39baSGeorge Wilson /* 4521f9af39baSGeorge Wilson * If the pool is read-only then force the feed thread to 4522f9af39baSGeorge Wilson * sleep a little longer. 4523f9af39baSGeorge Wilson */ 4524f9af39baSGeorge Wilson if (!spa_writeable(spa)) { 4525f9af39baSGeorge Wilson next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz; 4526f9af39baSGeorge Wilson spa_config_exit(spa, SCL_L2ARC, dev); 4527f9af39baSGeorge Wilson continue; 4528f9af39baSGeorge Wilson } 4529f9af39baSGeorge Wilson 4530fa94a07fSbrendan /* 4531fa94a07fSbrendan * Avoid contributing to memory pressure. 4532fa94a07fSbrendan */ 4533fa94a07fSbrendan if (arc_reclaim_needed()) { 4534fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_abort_lowmem); 4535e14bb325SJeff Bonwick spa_config_exit(spa, SCL_L2ARC, dev); 4536fa94a07fSbrendan continue; 4537fa94a07fSbrendan } 4538fa94a07fSbrendan 4539fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_feeds); 4540fa94a07fSbrendan 45415a98e54bSBrendan Gregg - Sun Microsystems size = l2arc_write_size(dev); 45423a737e0dSbrendan 4543fa94a07fSbrendan /* 4544fa94a07fSbrendan * Evict L2ARC buffers that will be overwritten. 4545fa94a07fSbrendan */ 45463a737e0dSbrendan l2arc_evict(dev, size, B_FALSE); 4547fa94a07fSbrendan 4548fa94a07fSbrendan /* 4549fa94a07fSbrendan * Write ARC buffers. 4550fa94a07fSbrendan */ 45515a98e54bSBrendan Gregg - Sun Microsystems wrote = l2arc_write_buffers(spa, dev, size); 45525a98e54bSBrendan Gregg - Sun Microsystems 45535a98e54bSBrendan Gregg - Sun Microsystems /* 45545a98e54bSBrendan Gregg - Sun Microsystems * Calculate interval between writes. 45555a98e54bSBrendan Gregg - Sun Microsystems */ 45565a98e54bSBrendan Gregg - Sun Microsystems next = l2arc_write_interval(begin, size, wrote); 4557e14bb325SJeff Bonwick spa_config_exit(spa, SCL_L2ARC, dev); 4558fa94a07fSbrendan } 4559fa94a07fSbrendan 4560fa94a07fSbrendan l2arc_thread_exit = 0; 4561fa94a07fSbrendan cv_broadcast(&l2arc_feed_thr_cv); 4562fa94a07fSbrendan CALLB_CPR_EXIT(&cpr); /* drops l2arc_feed_thr_lock */ 4563fa94a07fSbrendan thread_exit(); 4564fa94a07fSbrendan } 4565fa94a07fSbrendan 4566c5904d13Seschrock boolean_t 4567c5904d13Seschrock l2arc_vdev_present(vdev_t *vd) 4568c5904d13Seschrock { 4569c5904d13Seschrock l2arc_dev_t *dev; 4570c5904d13Seschrock 4571c5904d13Seschrock mutex_enter(&l2arc_dev_mtx); 4572c5904d13Seschrock for (dev = list_head(l2arc_dev_list); dev != NULL; 4573c5904d13Seschrock dev = list_next(l2arc_dev_list, dev)) { 4574c5904d13Seschrock if (dev->l2ad_vdev == vd) 4575c5904d13Seschrock break; 4576c5904d13Seschrock } 4577c5904d13Seschrock mutex_exit(&l2arc_dev_mtx); 4578c5904d13Seschrock 4579c5904d13Seschrock return (dev != NULL); 4580c5904d13Seschrock } 4581c5904d13Seschrock 4582fa94a07fSbrendan /* 4583fa94a07fSbrendan * Add a vdev for use by the L2ARC. By this point the spa has already 4584fa94a07fSbrendan * validated the vdev and opened it. 4585fa94a07fSbrendan */ 4586fa94a07fSbrendan void 4587573ca77eSGeorge Wilson l2arc_add_vdev(spa_t *spa, vdev_t *vd) 4588fa94a07fSbrendan { 4589fa94a07fSbrendan l2arc_dev_t *adddev; 4590fa94a07fSbrendan 4591c5904d13Seschrock ASSERT(!l2arc_vdev_present(vd)); 4592c5904d13Seschrock 4593fa94a07fSbrendan /* 4594fa94a07fSbrendan * Create a new l2arc device entry. 4595fa94a07fSbrendan */ 4596fa94a07fSbrendan adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP); 4597fa94a07fSbrendan adddev->l2ad_spa = spa; 4598fa94a07fSbrendan adddev->l2ad_vdev = vd; 4599fa94a07fSbrendan adddev->l2ad_write = l2arc_write_max; 46003a737e0dSbrendan adddev->l2ad_boost = l2arc_write_boost; 4601573ca77eSGeorge Wilson adddev->l2ad_start = VDEV_LABEL_START_SIZE; 4602573ca77eSGeorge Wilson adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd); 4603fa94a07fSbrendan adddev->l2ad_hand = adddev->l2ad_start; 4604fa94a07fSbrendan adddev->l2ad_evict = adddev->l2ad_start; 4605fa94a07fSbrendan adddev->l2ad_first = B_TRUE; 46065a98e54bSBrendan Gregg - Sun Microsystems adddev->l2ad_writing = B_FALSE; 4607fa94a07fSbrendan ASSERT3U(adddev->l2ad_write, >, 0); 4608fa94a07fSbrendan 4609fa94a07fSbrendan /* 4610fa94a07fSbrendan * This is a list of all ARC buffers that are still valid on the 4611fa94a07fSbrendan * device. 4612fa94a07fSbrendan */ 4613fa94a07fSbrendan adddev->l2ad_buflist = kmem_zalloc(sizeof (list_t), KM_SLEEP); 4614fa94a07fSbrendan list_create(adddev->l2ad_buflist, sizeof (arc_buf_hdr_t), 4615fa94a07fSbrendan offsetof(arc_buf_hdr_t, b_l2node)); 4616fa94a07fSbrendan 4617b24ab676SJeff Bonwick vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand); 4618fa94a07fSbrendan 4619fa94a07fSbrendan /* 4620fa94a07fSbrendan * Add device to global list 4621fa94a07fSbrendan */ 4622fa94a07fSbrendan mutex_enter(&l2arc_dev_mtx); 4623fa94a07fSbrendan list_insert_head(l2arc_dev_list, adddev); 4624fa94a07fSbrendan atomic_inc_64(&l2arc_ndev); 4625fa94a07fSbrendan mutex_exit(&l2arc_dev_mtx); 4626fa94a07fSbrendan } 4627fa94a07fSbrendan 4628fa94a07fSbrendan /* 4629fa94a07fSbrendan * Remove a vdev from the L2ARC. 4630fa94a07fSbrendan */ 4631fa94a07fSbrendan void 4632fa94a07fSbrendan l2arc_remove_vdev(vdev_t *vd) 4633fa94a07fSbrendan { 4634fa94a07fSbrendan l2arc_dev_t *dev, *nextdev, *remdev = NULL; 4635fa94a07fSbrendan 4636fa94a07fSbrendan /* 4637fa94a07fSbrendan * Find the device by vdev 4638fa94a07fSbrendan */ 4639fa94a07fSbrendan mutex_enter(&l2arc_dev_mtx); 4640fa94a07fSbrendan for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) { 4641fa94a07fSbrendan nextdev = list_next(l2arc_dev_list, dev); 4642fa94a07fSbrendan if (vd == dev->l2ad_vdev) { 4643fa94a07fSbrendan remdev = dev; 4644fa94a07fSbrendan break; 4645fa94a07fSbrendan } 4646fa94a07fSbrendan } 4647fa94a07fSbrendan ASSERT(remdev != NULL); 4648fa94a07fSbrendan 4649fa94a07fSbrendan /* 4650fa94a07fSbrendan * Remove device from global list 4651fa94a07fSbrendan */ 4652fa94a07fSbrendan list_remove(l2arc_dev_list, remdev); 4653fa94a07fSbrendan l2arc_dev_last = NULL; /* may have been invalidated */ 46543a737e0dSbrendan atomic_dec_64(&l2arc_ndev); 46553a737e0dSbrendan mutex_exit(&l2arc_dev_mtx); 4656fa94a07fSbrendan 4657fa94a07fSbrendan /* 4658fa94a07fSbrendan * Clear all buflists and ARC references. L2ARC device flush. 4659fa94a07fSbrendan */ 4660fa94a07fSbrendan l2arc_evict(remdev, 0, B_TRUE); 4661fa94a07fSbrendan list_destroy(remdev->l2ad_buflist); 4662fa94a07fSbrendan kmem_free(remdev->l2ad_buflist, sizeof (list_t)); 4663fa94a07fSbrendan kmem_free(remdev, sizeof (l2arc_dev_t)); 4664fa94a07fSbrendan } 4665fa94a07fSbrendan 4666fa94a07fSbrendan void 4667e14bb325SJeff Bonwick l2arc_init(void) 4668fa94a07fSbrendan { 4669fa94a07fSbrendan l2arc_thread_exit = 0; 4670fa94a07fSbrendan l2arc_ndev = 0; 4671fa94a07fSbrendan l2arc_writes_sent = 0; 4672fa94a07fSbrendan l2arc_writes_done = 0; 4673fa94a07fSbrendan 4674fa94a07fSbrendan mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL); 4675fa94a07fSbrendan cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL); 4676fa94a07fSbrendan mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL); 4677fa94a07fSbrendan mutex_init(&l2arc_buflist_mtx, NULL, MUTEX_DEFAULT, NULL); 4678fa94a07fSbrendan mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL); 4679fa94a07fSbrendan 4680fa94a07fSbrendan l2arc_dev_list = &L2ARC_dev_list; 4681fa94a07fSbrendan l2arc_free_on_write = &L2ARC_free_on_write; 4682fa94a07fSbrendan list_create(l2arc_dev_list, sizeof (l2arc_dev_t), 4683fa94a07fSbrendan offsetof(l2arc_dev_t, l2ad_node)); 4684fa94a07fSbrendan list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t), 4685fa94a07fSbrendan offsetof(l2arc_data_free_t, l2df_list_node)); 4686fa94a07fSbrendan } 4687fa94a07fSbrendan 4688fa94a07fSbrendan void 4689e14bb325SJeff Bonwick l2arc_fini(void) 4690fa94a07fSbrendan { 46913a737e0dSbrendan /* 46923a737e0dSbrendan * This is called from dmu_fini(), which is called from spa_fini(); 46933a737e0dSbrendan * Because of this, we can assume that all l2arc devices have 46943a737e0dSbrendan * already been removed when the pools themselves were removed. 46953a737e0dSbrendan */ 46963a737e0dSbrendan 46973a737e0dSbrendan l2arc_do_free_on_write(); 46983a737e0dSbrendan 4699fa94a07fSbrendan mutex_destroy(&l2arc_feed_thr_lock); 4700fa94a07fSbrendan cv_destroy(&l2arc_feed_thr_cv); 4701fa94a07fSbrendan mutex_destroy(&l2arc_dev_mtx); 4702fa94a07fSbrendan mutex_destroy(&l2arc_buflist_mtx); 4703fa94a07fSbrendan mutex_destroy(&l2arc_free_on_write_mtx); 4704fa94a07fSbrendan 4705fa94a07fSbrendan list_destroy(l2arc_dev_list); 4706fa94a07fSbrendan list_destroy(l2arc_free_on_write); 4707fa94a07fSbrendan } 4708e14bb325SJeff Bonwick 4709e14bb325SJeff Bonwick void 4710e14bb325SJeff Bonwick l2arc_start(void) 4711e14bb325SJeff Bonwick { 47128ad4d6ddSJeff Bonwick if (!(spa_mode_global & FWRITE)) 4713e14bb325SJeff Bonwick return; 4714e14bb325SJeff Bonwick 4715e14bb325SJeff Bonwick (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0, 4716e14bb325SJeff Bonwick TS_RUN, minclsyspri); 4717e14bb325SJeff Bonwick } 4718e14bb325SJeff Bonwick 4719e14bb325SJeff Bonwick void 4720e14bb325SJeff Bonwick l2arc_stop(void) 4721e14bb325SJeff Bonwick { 47228ad4d6ddSJeff Bonwick if (!(spa_mode_global & FWRITE)) 4723e14bb325SJeff Bonwick return; 4724e14bb325SJeff Bonwick 4725e14bb325SJeff Bonwick mutex_enter(&l2arc_feed_thr_lock); 4726e14bb325SJeff Bonwick cv_signal(&l2arc_feed_thr_cv); /* kick thread out of startup */ 4727e14bb325SJeff Bonwick l2arc_thread_exit = 1; 4728e14bb325SJeff Bonwick while (l2arc_thread_exit != 0) 4729e14bb325SJeff Bonwick cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock); 4730e14bb325SJeff Bonwick mutex_exit(&l2arc_feed_thr_lock); 4731e14bb325SJeff Bonwick } 4732