1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5033f9833Sek * Common Development and Distribution License (the "License"). 6033f9833Sek * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 223f9d6ad7SLin Ling * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23e9103aaeSGarrett D'Amore * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 24be6fd75aSMatthew Ahrens * Copyright (c) 2013 by Delphix. All rights reserved. 25aad02571SSaso Kiselkov * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 26fa9e4066Sahrens */ 27fa9e4066Sahrens 28fa9e4066Sahrens /* 2944cb6abcSbmc * DVA-based Adjustable Replacement Cache 30fa9e4066Sahrens * 31ea8dc4b6Seschrock * While much of the theory of operation used here is 32ea8dc4b6Seschrock * based on the self-tuning, low overhead replacement cache 33fa9e4066Sahrens * presented by Megiddo and Modha at FAST 2003, there are some 34fa9e4066Sahrens * significant differences: 35fa9e4066Sahrens * 36fa9e4066Sahrens * 1. The Megiddo and Modha model assumes any page is evictable. 37fa9e4066Sahrens * Pages in its cache cannot be "locked" into memory. This makes 38fa9e4066Sahrens * the eviction algorithm simple: evict the last page in the list. 39fa9e4066Sahrens * This also make the performance characteristics easy to reason 40fa9e4066Sahrens * about. Our cache is not so simple. At any given moment, some 41fa9e4066Sahrens * subset of the blocks in the cache are un-evictable because we 42fa9e4066Sahrens * have handed out a reference to them. Blocks are only evictable 43fa9e4066Sahrens * when there are no external references active. This makes 44fa9e4066Sahrens * eviction far more problematic: we choose to evict the evictable 45fa9e4066Sahrens * blocks that are the "lowest" in the list. 46fa9e4066Sahrens * 47fa9e4066Sahrens * There are times when it is not possible to evict the requested 48fa9e4066Sahrens * space. In these circumstances we are unable to adjust the cache 49fa9e4066Sahrens * size. To prevent the cache growing unbounded at these times we 50fa94a07fSbrendan * implement a "cache throttle" that slows the flow of new data 51fa94a07fSbrendan * into the cache until we can make space available. 52fa9e4066Sahrens * 53fa9e4066Sahrens * 2. The Megiddo and Modha model assumes a fixed cache size. 54fa9e4066Sahrens * Pages are evicted when the cache is full and there is a cache 55fa9e4066Sahrens * miss. Our model has a variable sized cache. It grows with 56fa94a07fSbrendan * high use, but also tries to react to memory pressure from the 57fa9e4066Sahrens * operating system: decreasing its size when system memory is 58fa9e4066Sahrens * tight. 59fa9e4066Sahrens * 60fa9e4066Sahrens * 3. The Megiddo and Modha model assumes a fixed page size. All 61*f7170741SWill Andrews * elements of the cache are therefore exactly the same size. So 62fa9e4066Sahrens * when adjusting the cache size following a cache miss, its simply 63fa9e4066Sahrens * a matter of choosing a single page to evict. In our model, we 64fa9e4066Sahrens * have variable sized cache blocks (rangeing from 512 bytes to 65*f7170741SWill Andrews * 128K bytes). We therefore choose a set of blocks to evict to make 66fa9e4066Sahrens * space for a cache miss that approximates as closely as possible 67fa9e4066Sahrens * the space used by the new block. 68fa9e4066Sahrens * 69fa9e4066Sahrens * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache" 70fa9e4066Sahrens * by N. Megiddo & D. Modha, FAST 2003 71fa9e4066Sahrens */ 72fa9e4066Sahrens 73fa9e4066Sahrens /* 74fa9e4066Sahrens * The locking model: 75fa9e4066Sahrens * 76fa9e4066Sahrens * A new reference to a cache buffer can be obtained in two 77fa9e4066Sahrens * ways: 1) via a hash table lookup using the DVA as a key, 78fa94a07fSbrendan * or 2) via one of the ARC lists. The arc_read() interface 79fa9e4066Sahrens * uses method 1, while the internal arc algorithms for 80*f7170741SWill Andrews * adjusting the cache use method 2. We therefore provide two 81fa9e4066Sahrens * types of locks: 1) the hash table lock array, and 2) the 82fa9e4066Sahrens * arc list locks. 83fa9e4066Sahrens * 84fc98fea5SBart Coddens * Buffers do not have their own mutexes, rather they rely on the 85fc98fea5SBart Coddens * hash table mutexes for the bulk of their protection (i.e. most 86fc98fea5SBart Coddens * fields in the arc_buf_hdr_t are protected by these mutexes). 87fa9e4066Sahrens * 88fa9e4066Sahrens * buf_hash_find() returns the appropriate mutex (held) when it 89fa9e4066Sahrens * locates the requested buffer in the hash table. It returns 90fa9e4066Sahrens * NULL for the mutex if the buffer was not in the table. 91fa9e4066Sahrens * 92fa9e4066Sahrens * buf_hash_remove() expects the appropriate hash mutex to be 93fa9e4066Sahrens * already held before it is invoked. 94fa9e4066Sahrens * 95fa9e4066Sahrens * Each arc state also has a mutex which is used to protect the 96fa9e4066Sahrens * buffer list associated with the state. When attempting to 97fa9e4066Sahrens * obtain a hash table lock while holding an arc list lock you 98fa9e4066Sahrens * must use: mutex_tryenter() to avoid deadlock. Also note that 9944eda4d7Smaybee * the active state mutex must be held before the ghost state mutex. 100fa9e4066Sahrens * 101ea8dc4b6Seschrock * Arc buffers may have an associated eviction callback function. 102ea8dc4b6Seschrock * This function will be invoked prior to removing the buffer (e.g. 103ea8dc4b6Seschrock * in arc_do_user_evicts()). Note however that the data associated 104ea8dc4b6Seschrock * with the buffer may be evicted prior to the callback. The callback 105ea8dc4b6Seschrock * must be made with *no locks held* (to prevent deadlock). Additionally, 106ea8dc4b6Seschrock * the users of callbacks must ensure that their private data is 107ea8dc4b6Seschrock * protected from simultaneous callbacks from arc_buf_evict() 108ea8dc4b6Seschrock * and arc_do_user_evicts(). 109ea8dc4b6Seschrock * 110fa9e4066Sahrens * Note that the majority of the performance stats are manipulated 111fa9e4066Sahrens * with atomic operations. 112fa94a07fSbrendan * 113fa94a07fSbrendan * The L2ARC uses the l2arc_buflist_mtx global mutex for the following: 114fa94a07fSbrendan * 115fa94a07fSbrendan * - L2ARC buflist creation 116fa94a07fSbrendan * - L2ARC buflist eviction 117fa94a07fSbrendan * - L2ARC write completion, which walks L2ARC buflists 118fa94a07fSbrendan * - ARC header destruction, as it removes from L2ARC buflists 119fa94a07fSbrendan * - ARC header release, as it removes from L2ARC buflists 120fa9e4066Sahrens */ 121fa9e4066Sahrens 122fa9e4066Sahrens #include <sys/spa.h> 123fa9e4066Sahrens #include <sys/zio.h> 124aad02571SSaso Kiselkov #include <sys/zio_compress.h> 125fa9e4066Sahrens #include <sys/zfs_context.h> 126fa9e4066Sahrens #include <sys/arc.h> 127fa9e4066Sahrens #include <sys/refcount.h> 128c5904d13Seschrock #include <sys/vdev.h> 129573ca77eSGeorge Wilson #include <sys/vdev_impl.h> 130fa9e4066Sahrens #ifdef _KERNEL 131fa9e4066Sahrens #include <sys/vmsystm.h> 132fa9e4066Sahrens #include <vm/anon.h> 133fa9e4066Sahrens #include <sys/fs/swapnode.h> 134033f9833Sek #include <sys/dnlc.h> 135fa9e4066Sahrens #endif 136fa9e4066Sahrens #include <sys/callb.h> 13744cb6abcSbmc #include <sys/kstat.h> 138b24ab676SJeff Bonwick #include <zfs_fletcher.h> 139fa9e4066Sahrens 140cd1c8b85SMatthew Ahrens #ifndef _KERNEL 141cd1c8b85SMatthew Ahrens /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */ 142cd1c8b85SMatthew Ahrens boolean_t arc_watch = B_FALSE; 143cd1c8b85SMatthew Ahrens int arc_procfd; 144cd1c8b85SMatthew Ahrens #endif 145cd1c8b85SMatthew Ahrens 146fa9e4066Sahrens static kmutex_t arc_reclaim_thr_lock; 147fa9e4066Sahrens static kcondvar_t arc_reclaim_thr_cv; /* used to signal reclaim thr */ 148fa9e4066Sahrens static uint8_t arc_thread_exit; 149fa9e4066Sahrens 1501ab7f2deSmaybee extern int zfs_write_limit_shift; 1511ab7f2deSmaybee extern uint64_t zfs_write_limit_max; 15205715f94SMark Maybee extern kmutex_t zfs_write_limit_lock; 1531ab7f2deSmaybee 154033f9833Sek #define ARC_REDUCE_DNLC_PERCENT 3 155033f9833Sek uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT; 156033f9833Sek 157fa9e4066Sahrens typedef enum arc_reclaim_strategy { 158fa9e4066Sahrens ARC_RECLAIM_AGGR, /* Aggressive reclaim strategy */ 159fa9e4066Sahrens ARC_RECLAIM_CONS /* Conservative reclaim strategy */ 160fa9e4066Sahrens } arc_reclaim_strategy_t; 161fa9e4066Sahrens 162fa9e4066Sahrens /* number of seconds before growing cache again */ 163fa9e4066Sahrens static int arc_grow_retry = 60; 164fa9e4066Sahrens 1655a98e54bSBrendan Gregg - Sun Microsystems /* shift of arc_c for calculating both min and max arc_p */ 1665a98e54bSBrendan Gregg - Sun Microsystems static int arc_p_min_shift = 4; 1675a98e54bSBrendan Gregg - Sun Microsystems 1685a98e54bSBrendan Gregg - Sun Microsystems /* log2(fraction of arc to reclaim) */ 1695a98e54bSBrendan Gregg - Sun Microsystems static int arc_shrink_shift = 5; 1705a98e54bSBrendan Gregg - Sun Microsystems 17113506d1eSmaybee /* 172b19a79ecSperrin * minimum lifespan of a prefetch block in clock ticks 173b19a79ecSperrin * (initialized in arc_init()) 17413506d1eSmaybee */ 175b19a79ecSperrin static int arc_min_prefetch_lifespan; 17613506d1eSmaybee 177fa9e4066Sahrens static int arc_dead; 178fa9e4066Sahrens 1793a737e0dSbrendan /* 1803a737e0dSbrendan * The arc has filled available memory and has now warmed up. 1813a737e0dSbrendan */ 1823a737e0dSbrendan static boolean_t arc_warm; 1833a737e0dSbrendan 184a2eea2e1Sahrens /* 185a2eea2e1Sahrens * These tunables are for performance analysis. 186a2eea2e1Sahrens */ 187a2eea2e1Sahrens uint64_t zfs_arc_max; 188a2eea2e1Sahrens uint64_t zfs_arc_min; 1891116048bSek uint64_t zfs_arc_meta_limit = 0; 1905a98e54bSBrendan Gregg - Sun Microsystems int zfs_arc_grow_retry = 0; 1915a98e54bSBrendan Gregg - Sun Microsystems int zfs_arc_shrink_shift = 0; 1925a98e54bSBrendan Gregg - Sun Microsystems int zfs_arc_p_min_shift = 0; 1939253d63dSGeorge Wilson int zfs_disable_dup_eviction = 0; 194a2eea2e1Sahrens 195fa9e4066Sahrens /* 196fa94a07fSbrendan * Note that buffers can be in one of 6 states: 197fa9e4066Sahrens * ARC_anon - anonymous (discussed below) 198ea8dc4b6Seschrock * ARC_mru - recently used, currently cached 199ea8dc4b6Seschrock * ARC_mru_ghost - recentely used, no longer in cache 200ea8dc4b6Seschrock * ARC_mfu - frequently used, currently cached 201ea8dc4b6Seschrock * ARC_mfu_ghost - frequently used, no longer in cache 202fa94a07fSbrendan * ARC_l2c_only - exists in L2ARC but not other states 2030e8c6158Smaybee * When there are no active references to the buffer, they are 2040e8c6158Smaybee * are linked onto a list in one of these arc states. These are 2050e8c6158Smaybee * the only buffers that can be evicted or deleted. Within each 2060e8c6158Smaybee * state there are multiple lists, one for meta-data and one for 2070e8c6158Smaybee * non-meta-data. Meta-data (indirect blocks, blocks of dnodes, 2080e8c6158Smaybee * etc.) is tracked separately so that it can be managed more 209fa94a07fSbrendan * explicitly: favored over data, limited explicitly. 210fa9e4066Sahrens * 211fa9e4066Sahrens * Anonymous buffers are buffers that are not associated with 212fa9e4066Sahrens * a DVA. These are buffers that hold dirty block copies 213fa9e4066Sahrens * before they are written to stable storage. By definition, 214ea8dc4b6Seschrock * they are "ref'd" and are considered part of arc_mru 215fa9e4066Sahrens * that cannot be freed. Generally, they will aquire a DVA 216ea8dc4b6Seschrock * as they are written and migrate onto the arc_mru list. 217fa94a07fSbrendan * 218fa94a07fSbrendan * The ARC_l2c_only state is for buffers that are in the second 219fa94a07fSbrendan * level ARC but no longer in any of the ARC_m* lists. The second 220fa94a07fSbrendan * level ARC itself may also contain buffers that are in any of 221fa94a07fSbrendan * the ARC_m* states - meaning that a buffer can exist in two 222fa94a07fSbrendan * places. The reason for the ARC_l2c_only state is to keep the 223fa94a07fSbrendan * buffer header in the hash table, so that reads that hit the 224fa94a07fSbrendan * second level ARC benefit from these fast lookups. 225fa9e4066Sahrens */ 226fa9e4066Sahrens 227fa9e4066Sahrens typedef struct arc_state { 2280e8c6158Smaybee list_t arcs_list[ARC_BUFC_NUMTYPES]; /* list of evictable buffers */ 2290e8c6158Smaybee uint64_t arcs_lsize[ARC_BUFC_NUMTYPES]; /* amount of evictable data */ 2300e8c6158Smaybee uint64_t arcs_size; /* total amount of data in this state */ 23144cb6abcSbmc kmutex_t arcs_mtx; 232fa9e4066Sahrens } arc_state_t; 233fa9e4066Sahrens 234fa94a07fSbrendan /* The 6 states: */ 235fa9e4066Sahrens static arc_state_t ARC_anon; 236ea8dc4b6Seschrock static arc_state_t ARC_mru; 237ea8dc4b6Seschrock static arc_state_t ARC_mru_ghost; 238ea8dc4b6Seschrock static arc_state_t ARC_mfu; 239ea8dc4b6Seschrock static arc_state_t ARC_mfu_ghost; 240fa94a07fSbrendan static arc_state_t ARC_l2c_only; 241fa9e4066Sahrens 24244cb6abcSbmc typedef struct arc_stats { 24344cb6abcSbmc kstat_named_t arcstat_hits; 24444cb6abcSbmc kstat_named_t arcstat_misses; 24544cb6abcSbmc kstat_named_t arcstat_demand_data_hits; 24644cb6abcSbmc kstat_named_t arcstat_demand_data_misses; 24744cb6abcSbmc kstat_named_t arcstat_demand_metadata_hits; 24844cb6abcSbmc kstat_named_t arcstat_demand_metadata_misses; 24944cb6abcSbmc kstat_named_t arcstat_prefetch_data_hits; 25044cb6abcSbmc kstat_named_t arcstat_prefetch_data_misses; 25144cb6abcSbmc kstat_named_t arcstat_prefetch_metadata_hits; 25244cb6abcSbmc kstat_named_t arcstat_prefetch_metadata_misses; 25344cb6abcSbmc kstat_named_t arcstat_mru_hits; 25444cb6abcSbmc kstat_named_t arcstat_mru_ghost_hits; 25544cb6abcSbmc kstat_named_t arcstat_mfu_hits; 25644cb6abcSbmc kstat_named_t arcstat_mfu_ghost_hits; 25744cb6abcSbmc kstat_named_t arcstat_deleted; 25844cb6abcSbmc kstat_named_t arcstat_recycle_miss; 2593e30c24aSWill Andrews /* 2603e30c24aSWill Andrews * Number of buffers that could not be evicted because the hash lock 2613e30c24aSWill Andrews * was held by another thread. The lock may not necessarily be held 2623e30c24aSWill Andrews * by something using the same buffer, since hash locks are shared 2633e30c24aSWill Andrews * by multiple buffers. 2643e30c24aSWill Andrews */ 26544cb6abcSbmc kstat_named_t arcstat_mutex_miss; 2663e30c24aSWill Andrews /* 2673e30c24aSWill Andrews * Number of buffers skipped because they have I/O in progress, are 2683e30c24aSWill Andrews * indrect prefetch buffers that have not lived long enough, or are 2693e30c24aSWill Andrews * not from the spa we're trying to evict from. 2703e30c24aSWill Andrews */ 27144cb6abcSbmc kstat_named_t arcstat_evict_skip; 2725ea40c06SBrendan Gregg - Sun Microsystems kstat_named_t arcstat_evict_l2_cached; 2735ea40c06SBrendan Gregg - Sun Microsystems kstat_named_t arcstat_evict_l2_eligible; 2745ea40c06SBrendan Gregg - Sun Microsystems kstat_named_t arcstat_evict_l2_ineligible; 27544cb6abcSbmc kstat_named_t arcstat_hash_elements; 27644cb6abcSbmc kstat_named_t arcstat_hash_elements_max; 27744cb6abcSbmc kstat_named_t arcstat_hash_collisions; 27844cb6abcSbmc kstat_named_t arcstat_hash_chains; 27944cb6abcSbmc kstat_named_t arcstat_hash_chain_max; 28044cb6abcSbmc kstat_named_t arcstat_p; 28144cb6abcSbmc kstat_named_t arcstat_c; 28244cb6abcSbmc kstat_named_t arcstat_c_min; 28344cb6abcSbmc kstat_named_t arcstat_c_max; 28444cb6abcSbmc kstat_named_t arcstat_size; 285fa94a07fSbrendan kstat_named_t arcstat_hdr_size; 2865a98e54bSBrendan Gregg - Sun Microsystems kstat_named_t arcstat_data_size; 2875a98e54bSBrendan Gregg - Sun Microsystems kstat_named_t arcstat_other_size; 288fa94a07fSbrendan kstat_named_t arcstat_l2_hits; 289fa94a07fSbrendan kstat_named_t arcstat_l2_misses; 290fa94a07fSbrendan kstat_named_t arcstat_l2_feeds; 291fa94a07fSbrendan kstat_named_t arcstat_l2_rw_clash; 2925a98e54bSBrendan Gregg - Sun Microsystems kstat_named_t arcstat_l2_read_bytes; 2935a98e54bSBrendan Gregg - Sun Microsystems kstat_named_t arcstat_l2_write_bytes; 294fa94a07fSbrendan kstat_named_t arcstat_l2_writes_sent; 295fa94a07fSbrendan kstat_named_t arcstat_l2_writes_done; 296fa94a07fSbrendan kstat_named_t arcstat_l2_writes_error; 297fa94a07fSbrendan kstat_named_t arcstat_l2_writes_hdr_miss; 298fa94a07fSbrendan kstat_named_t arcstat_l2_evict_lock_retry; 299fa94a07fSbrendan kstat_named_t arcstat_l2_evict_reading; 300fa94a07fSbrendan kstat_named_t arcstat_l2_free_on_write; 301fa94a07fSbrendan kstat_named_t arcstat_l2_abort_lowmem; 302fa94a07fSbrendan kstat_named_t arcstat_l2_cksum_bad; 303fa94a07fSbrendan kstat_named_t arcstat_l2_io_error; 304fa94a07fSbrendan kstat_named_t arcstat_l2_size; 305aad02571SSaso Kiselkov kstat_named_t arcstat_l2_asize; 306fa94a07fSbrendan kstat_named_t arcstat_l2_hdr_size; 307aad02571SSaso Kiselkov kstat_named_t arcstat_l2_compress_successes; 308aad02571SSaso Kiselkov kstat_named_t arcstat_l2_compress_zeros; 309aad02571SSaso Kiselkov kstat_named_t arcstat_l2_compress_failures; 3101ab7f2deSmaybee kstat_named_t arcstat_memory_throttle_count; 3119253d63dSGeorge Wilson kstat_named_t arcstat_duplicate_buffers; 3129253d63dSGeorge Wilson kstat_named_t arcstat_duplicate_buffers_size; 3139253d63dSGeorge Wilson kstat_named_t arcstat_duplicate_reads; 31420128a08SGeorge Wilson kstat_named_t arcstat_meta_used; 31520128a08SGeorge Wilson kstat_named_t arcstat_meta_limit; 31620128a08SGeorge Wilson kstat_named_t arcstat_meta_max; 31744cb6abcSbmc } arc_stats_t; 31844cb6abcSbmc 31944cb6abcSbmc static arc_stats_t arc_stats = { 32044cb6abcSbmc { "hits", KSTAT_DATA_UINT64 }, 32144cb6abcSbmc { "misses", KSTAT_DATA_UINT64 }, 32244cb6abcSbmc { "demand_data_hits", KSTAT_DATA_UINT64 }, 32344cb6abcSbmc { "demand_data_misses", KSTAT_DATA_UINT64 }, 32444cb6abcSbmc { "demand_metadata_hits", KSTAT_DATA_UINT64 }, 32544cb6abcSbmc { "demand_metadata_misses", KSTAT_DATA_UINT64 }, 32644cb6abcSbmc { "prefetch_data_hits", KSTAT_DATA_UINT64 }, 32744cb6abcSbmc { "prefetch_data_misses", KSTAT_DATA_UINT64 }, 32844cb6abcSbmc { "prefetch_metadata_hits", KSTAT_DATA_UINT64 }, 32944cb6abcSbmc { "prefetch_metadata_misses", KSTAT_DATA_UINT64 }, 33044cb6abcSbmc { "mru_hits", KSTAT_DATA_UINT64 }, 33144cb6abcSbmc { "mru_ghost_hits", KSTAT_DATA_UINT64 }, 33244cb6abcSbmc { "mfu_hits", KSTAT_DATA_UINT64 }, 33344cb6abcSbmc { "mfu_ghost_hits", KSTAT_DATA_UINT64 }, 33444cb6abcSbmc { "deleted", KSTAT_DATA_UINT64 }, 33544cb6abcSbmc { "recycle_miss", KSTAT_DATA_UINT64 }, 33644cb6abcSbmc { "mutex_miss", KSTAT_DATA_UINT64 }, 33744cb6abcSbmc { "evict_skip", KSTAT_DATA_UINT64 }, 3385ea40c06SBrendan Gregg - Sun Microsystems { "evict_l2_cached", KSTAT_DATA_UINT64 }, 3395ea40c06SBrendan Gregg - Sun Microsystems { "evict_l2_eligible", KSTAT_DATA_UINT64 }, 3405ea40c06SBrendan Gregg - Sun Microsystems { "evict_l2_ineligible", KSTAT_DATA_UINT64 }, 34144cb6abcSbmc { "hash_elements", KSTAT_DATA_UINT64 }, 34244cb6abcSbmc { "hash_elements_max", KSTAT_DATA_UINT64 }, 34344cb6abcSbmc { "hash_collisions", KSTAT_DATA_UINT64 }, 34444cb6abcSbmc { "hash_chains", KSTAT_DATA_UINT64 }, 34544cb6abcSbmc { "hash_chain_max", KSTAT_DATA_UINT64 }, 34644cb6abcSbmc { "p", KSTAT_DATA_UINT64 }, 34744cb6abcSbmc { "c", KSTAT_DATA_UINT64 }, 34844cb6abcSbmc { "c_min", KSTAT_DATA_UINT64 }, 34944cb6abcSbmc { "c_max", KSTAT_DATA_UINT64 }, 350fa94a07fSbrendan { "size", KSTAT_DATA_UINT64 }, 351fa94a07fSbrendan { "hdr_size", KSTAT_DATA_UINT64 }, 3525a98e54bSBrendan Gregg - Sun Microsystems { "data_size", KSTAT_DATA_UINT64 }, 3535a98e54bSBrendan Gregg - Sun Microsystems { "other_size", KSTAT_DATA_UINT64 }, 354fa94a07fSbrendan { "l2_hits", KSTAT_DATA_UINT64 }, 355fa94a07fSbrendan { "l2_misses", KSTAT_DATA_UINT64 }, 356fa94a07fSbrendan { "l2_feeds", KSTAT_DATA_UINT64 }, 357fa94a07fSbrendan { "l2_rw_clash", KSTAT_DATA_UINT64 }, 3585a98e54bSBrendan Gregg - Sun Microsystems { "l2_read_bytes", KSTAT_DATA_UINT64 }, 3595a98e54bSBrendan Gregg - Sun Microsystems { "l2_write_bytes", KSTAT_DATA_UINT64 }, 360fa94a07fSbrendan { "l2_writes_sent", KSTAT_DATA_UINT64 }, 361fa94a07fSbrendan { "l2_writes_done", KSTAT_DATA_UINT64 }, 362fa94a07fSbrendan { "l2_writes_error", KSTAT_DATA_UINT64 }, 363fa94a07fSbrendan { "l2_writes_hdr_miss", KSTAT_DATA_UINT64 }, 364fa94a07fSbrendan { "l2_evict_lock_retry", KSTAT_DATA_UINT64 }, 365fa94a07fSbrendan { "l2_evict_reading", KSTAT_DATA_UINT64 }, 366fa94a07fSbrendan { "l2_free_on_write", KSTAT_DATA_UINT64 }, 367fa94a07fSbrendan { "l2_abort_lowmem", KSTAT_DATA_UINT64 }, 368fa94a07fSbrendan { "l2_cksum_bad", KSTAT_DATA_UINT64 }, 369fa94a07fSbrendan { "l2_io_error", KSTAT_DATA_UINT64 }, 370fa94a07fSbrendan { "l2_size", KSTAT_DATA_UINT64 }, 371aad02571SSaso Kiselkov { "l2_asize", KSTAT_DATA_UINT64 }, 3721ab7f2deSmaybee { "l2_hdr_size", KSTAT_DATA_UINT64 }, 373aad02571SSaso Kiselkov { "l2_compress_successes", KSTAT_DATA_UINT64 }, 374aad02571SSaso Kiselkov { "l2_compress_zeros", KSTAT_DATA_UINT64 }, 375aad02571SSaso Kiselkov { "l2_compress_failures", KSTAT_DATA_UINT64 }, 3769253d63dSGeorge Wilson { "memory_throttle_count", KSTAT_DATA_UINT64 }, 3779253d63dSGeorge Wilson { "duplicate_buffers", KSTAT_DATA_UINT64 }, 3789253d63dSGeorge Wilson { "duplicate_buffers_size", KSTAT_DATA_UINT64 }, 37920128a08SGeorge Wilson { "duplicate_reads", KSTAT_DATA_UINT64 }, 38020128a08SGeorge Wilson { "arc_meta_used", KSTAT_DATA_UINT64 }, 38120128a08SGeorge Wilson { "arc_meta_limit", KSTAT_DATA_UINT64 }, 38220128a08SGeorge Wilson { "arc_meta_max", KSTAT_DATA_UINT64 } 38344cb6abcSbmc }; 38444cb6abcSbmc 38544cb6abcSbmc #define ARCSTAT(stat) (arc_stats.stat.value.ui64) 38644cb6abcSbmc 38744cb6abcSbmc #define ARCSTAT_INCR(stat, val) \ 388*f7170741SWill Andrews atomic_add_64(&arc_stats.stat.value.ui64, (val)) 38944cb6abcSbmc 390b24ab676SJeff Bonwick #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) 39144cb6abcSbmc #define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1) 39244cb6abcSbmc 39344cb6abcSbmc #define ARCSTAT_MAX(stat, val) { \ 39444cb6abcSbmc uint64_t m; \ 39544cb6abcSbmc while ((val) > (m = arc_stats.stat.value.ui64) && \ 39644cb6abcSbmc (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \ 39744cb6abcSbmc continue; \ 39844cb6abcSbmc } 39944cb6abcSbmc 40044cb6abcSbmc #define ARCSTAT_MAXSTAT(stat) \ 40144cb6abcSbmc ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64) 40244cb6abcSbmc 40344cb6abcSbmc /* 40444cb6abcSbmc * We define a macro to allow ARC hits/misses to be easily broken down by 40544cb6abcSbmc * two separate conditions, giving a total of four different subtypes for 40644cb6abcSbmc * each of hits and misses (so eight statistics total). 40744cb6abcSbmc */ 40844cb6abcSbmc #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \ 40944cb6abcSbmc if (cond1) { \ 41044cb6abcSbmc if (cond2) { \ 41144cb6abcSbmc ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \ 41244cb6abcSbmc } else { \ 41344cb6abcSbmc ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \ 41444cb6abcSbmc } \ 41544cb6abcSbmc } else { \ 41644cb6abcSbmc if (cond2) { \ 41744cb6abcSbmc ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \ 41844cb6abcSbmc } else { \ 41944cb6abcSbmc ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\ 42044cb6abcSbmc } \ 42144cb6abcSbmc } 42244cb6abcSbmc 42344cb6abcSbmc kstat_t *arc_ksp; 424b24ab676SJeff Bonwick static arc_state_t *arc_anon; 42544cb6abcSbmc static arc_state_t *arc_mru; 42644cb6abcSbmc static arc_state_t *arc_mru_ghost; 42744cb6abcSbmc static arc_state_t *arc_mfu; 42844cb6abcSbmc static arc_state_t *arc_mfu_ghost; 429fa94a07fSbrendan static arc_state_t *arc_l2c_only; 43044cb6abcSbmc 43144cb6abcSbmc /* 43244cb6abcSbmc * There are several ARC variables that are critical to export as kstats -- 43344cb6abcSbmc * but we don't want to have to grovel around in the kstat whenever we wish to 43444cb6abcSbmc * manipulate them. For these variables, we therefore define them to be in 43544cb6abcSbmc * terms of the statistic variable. This assures that we are not introducing 43644cb6abcSbmc * the possibility of inconsistency by having shadow copies of the variables, 43744cb6abcSbmc * while still allowing the code to be readable. 43844cb6abcSbmc */ 43944cb6abcSbmc #define arc_size ARCSTAT(arcstat_size) /* actual total arc size */ 44044cb6abcSbmc #define arc_p ARCSTAT(arcstat_p) /* target size of MRU */ 44144cb6abcSbmc #define arc_c ARCSTAT(arcstat_c) /* target size of cache */ 44244cb6abcSbmc #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ 44344cb6abcSbmc #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ 44420128a08SGeorge Wilson #define arc_meta_limit ARCSTAT(arcstat_meta_limit) /* max size for metadata */ 44520128a08SGeorge Wilson #define arc_meta_used ARCSTAT(arcstat_meta_used) /* size of metadata */ 44620128a08SGeorge Wilson #define arc_meta_max ARCSTAT(arcstat_meta_max) /* max size of metadata */ 44744cb6abcSbmc 448aad02571SSaso Kiselkov #define L2ARC_IS_VALID_COMPRESS(_c_) \ 449aad02571SSaso Kiselkov ((_c_) == ZIO_COMPRESS_LZ4 || (_c_) == ZIO_COMPRESS_EMPTY) 450aad02571SSaso Kiselkov 45144cb6abcSbmc static int arc_no_grow; /* Don't try to grow cache size */ 45244cb6abcSbmc static uint64_t arc_tempreserve; 4532fdbea25SAleksandr Guzovskiy static uint64_t arc_loaned_bytes; 454fa9e4066Sahrens 455fa94a07fSbrendan typedef struct l2arc_buf_hdr l2arc_buf_hdr_t; 456fa94a07fSbrendan 457fa9e4066Sahrens typedef struct arc_callback arc_callback_t; 458fa9e4066Sahrens 459fa9e4066Sahrens struct arc_callback { 460fa9e4066Sahrens void *acb_private; 461c717a561Smaybee arc_done_func_t *acb_done; 462fa9e4066Sahrens arc_buf_t *acb_buf; 463fa9e4066Sahrens zio_t *acb_zio_dummy; 464fa9e4066Sahrens arc_callback_t *acb_next; 465fa9e4066Sahrens }; 466fa9e4066Sahrens 467c717a561Smaybee typedef struct arc_write_callback arc_write_callback_t; 468c717a561Smaybee 469c717a561Smaybee struct arc_write_callback { 470c717a561Smaybee void *awcb_private; 471c717a561Smaybee arc_done_func_t *awcb_ready; 472c717a561Smaybee arc_done_func_t *awcb_done; 473c717a561Smaybee arc_buf_t *awcb_buf; 474c717a561Smaybee }; 475c717a561Smaybee 476fa9e4066Sahrens struct arc_buf_hdr { 477fa9e4066Sahrens /* protected by hash lock */ 478fa9e4066Sahrens dva_t b_dva; 479fa9e4066Sahrens uint64_t b_birth; 480fa9e4066Sahrens uint64_t b_cksum0; 481fa9e4066Sahrens 4826b4acc8bSahrens kmutex_t b_freeze_lock; 4836b4acc8bSahrens zio_cksum_t *b_freeze_cksum; 4843f9d6ad7SLin Ling void *b_thawed; 4856b4acc8bSahrens 486fa9e4066Sahrens arc_buf_hdr_t *b_hash_next; 487fa9e4066Sahrens arc_buf_t *b_buf; 488fa9e4066Sahrens uint32_t b_flags; 489ea8dc4b6Seschrock uint32_t b_datacnt; 490fa9e4066Sahrens 491fa9e4066Sahrens arc_callback_t *b_acb; 492ad23a2dbSjohansen kcondvar_t b_cv; 493ad23a2dbSjohansen 494ad23a2dbSjohansen /* immutable */ 495ad23a2dbSjohansen arc_buf_contents_t b_type; 496ad23a2dbSjohansen uint64_t b_size; 497ac05c741SMark Maybee uint64_t b_spa; 498fa9e4066Sahrens 499fa9e4066Sahrens /* protected by arc state mutex */ 500fa9e4066Sahrens arc_state_t *b_state; 501fa9e4066Sahrens list_node_t b_arc_node; 502fa9e4066Sahrens 503fa9e4066Sahrens /* updated atomically */ 504fa9e4066Sahrens clock_t b_arc_access; 505fa9e4066Sahrens 506fa9e4066Sahrens /* self protecting */ 507fa9e4066Sahrens refcount_t b_refcnt; 508fa94a07fSbrendan 509fa94a07fSbrendan l2arc_buf_hdr_t *b_l2hdr; 510fa94a07fSbrendan list_node_t b_l2node; 511fa9e4066Sahrens }; 512fa9e4066Sahrens 513ea8dc4b6Seschrock static arc_buf_t *arc_eviction_list; 514ea8dc4b6Seschrock static kmutex_t arc_eviction_mtx; 51540d7d650Smaybee static arc_buf_hdr_t arc_eviction_hdr; 51644eda4d7Smaybee static void arc_get_data_buf(arc_buf_t *buf); 51744eda4d7Smaybee static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock); 5180e8c6158Smaybee static int arc_evict_needed(arc_buf_contents_t type); 519ac05c741SMark Maybee static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes); 520cd1c8b85SMatthew Ahrens static void arc_buf_watch(arc_buf_t *buf); 521ea8dc4b6Seschrock 5225ea40c06SBrendan Gregg - Sun Microsystems static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab); 5235ea40c06SBrendan Gregg - Sun Microsystems 524ea8dc4b6Seschrock #define GHOST_STATE(state) \ 525fa94a07fSbrendan ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \ 526fa94a07fSbrendan (state) == arc_l2c_only) 527ea8dc4b6Seschrock 528fa9e4066Sahrens /* 529fa9e4066Sahrens * Private ARC flags. These flags are private ARC only flags that will show up 530fa9e4066Sahrens * in b_flags in the arc_hdr_buf_t. Some flags are publicly declared, and can 531fa9e4066Sahrens * be passed in as arc_flags in things like arc_read. However, these flags 532fa9e4066Sahrens * should never be passed and should only be set by ARC code. When adding new 533fa9e4066Sahrens * public flags, make sure not to smash the private ones. 534fa9e4066Sahrens */ 535fa9e4066Sahrens 536ea8dc4b6Seschrock #define ARC_IN_HASH_TABLE (1 << 9) /* this buffer is hashed */ 537fa9e4066Sahrens #define ARC_IO_IN_PROGRESS (1 << 10) /* I/O in progress for buf */ 538fa9e4066Sahrens #define ARC_IO_ERROR (1 << 11) /* I/O failed for buf */ 539fa9e4066Sahrens #define ARC_FREED_IN_READ (1 << 12) /* buf freed while in read */ 540ea8dc4b6Seschrock #define ARC_BUF_AVAILABLE (1 << 13) /* block not in active use */ 54113506d1eSmaybee #define ARC_INDIRECT (1 << 14) /* this is an indirect block */ 542fa94a07fSbrendan #define ARC_FREE_IN_PROGRESS (1 << 15) /* hdr about to be freed */ 5433baa08fcSek #define ARC_L2_WRITING (1 << 16) /* L2ARC write in progress */ 5443baa08fcSek #define ARC_L2_EVICTED (1 << 17) /* evicted during I/O */ 5453baa08fcSek #define ARC_L2_WRITE_HEAD (1 << 18) /* head of write list */ 546fa9e4066Sahrens 547ea8dc4b6Seschrock #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_IN_HASH_TABLE) 548fa9e4066Sahrens #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS) 549fa9e4066Sahrens #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_IO_ERROR) 5505a98e54bSBrendan Gregg - Sun Microsystems #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_PREFETCH) 551fa9e4066Sahrens #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FREED_IN_READ) 552ea8dc4b6Seschrock #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_BUF_AVAILABLE) 553fa94a07fSbrendan #define HDR_FREE_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FREE_IN_PROGRESS) 5543baa08fcSek #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_L2CACHE) 5553a737e0dSbrendan #define HDR_L2_READING(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS && \ 5563a737e0dSbrendan (hdr)->b_l2hdr != NULL) 557fa94a07fSbrendan #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_L2_WRITING) 558fa94a07fSbrendan #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_L2_EVICTED) 559fa94a07fSbrendan #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_L2_WRITE_HEAD) 560fa9e4066Sahrens 561e6c728e1Sbrendan /* 562e6c728e1Sbrendan * Other sizes 563e6c728e1Sbrendan */ 564e6c728e1Sbrendan 565e6c728e1Sbrendan #define HDR_SIZE ((int64_t)sizeof (arc_buf_hdr_t)) 566e6c728e1Sbrendan #define L2HDR_SIZE ((int64_t)sizeof (l2arc_buf_hdr_t)) 567e6c728e1Sbrendan 568fa9e4066Sahrens /* 569fa9e4066Sahrens * Hash table routines 570fa9e4066Sahrens */ 571fa9e4066Sahrens 572fa9e4066Sahrens #define HT_LOCK_PAD 64 573fa9e4066Sahrens 574fa9e4066Sahrens struct ht_lock { 575fa9e4066Sahrens kmutex_t ht_lock; 576fa9e4066Sahrens #ifdef _KERNEL 577fa9e4066Sahrens unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))]; 578fa9e4066Sahrens #endif 579fa9e4066Sahrens }; 580fa9e4066Sahrens 581fa9e4066Sahrens #define BUF_LOCKS 256 582fa9e4066Sahrens typedef struct buf_hash_table { 583fa9e4066Sahrens uint64_t ht_mask; 584fa9e4066Sahrens arc_buf_hdr_t **ht_table; 585fa9e4066Sahrens struct ht_lock ht_locks[BUF_LOCKS]; 586fa9e4066Sahrens } buf_hash_table_t; 587fa9e4066Sahrens 588fa9e4066Sahrens static buf_hash_table_t buf_hash_table; 589fa9e4066Sahrens 590fa9e4066Sahrens #define BUF_HASH_INDEX(spa, dva, birth) \ 591fa9e4066Sahrens (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask) 592fa9e4066Sahrens #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)]) 593fa9e4066Sahrens #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock)) 5943f9d6ad7SLin Ling #define HDR_LOCK(hdr) \ 5953f9d6ad7SLin Ling (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth))) 596fa9e4066Sahrens 597fa9e4066Sahrens uint64_t zfs_crc64_table[256]; 598fa9e4066Sahrens 599fa94a07fSbrendan /* 600fa94a07fSbrendan * Level 2 ARC 601fa94a07fSbrendan */ 602fa94a07fSbrendan 603fa94a07fSbrendan #define L2ARC_WRITE_SIZE (8 * 1024 * 1024) /* initial write max */ 604aad02571SSaso Kiselkov #define L2ARC_HEADROOM 2 /* num of writes */ 605aad02571SSaso Kiselkov /* 606aad02571SSaso Kiselkov * If we discover during ARC scan any buffers to be compressed, we boost 607aad02571SSaso Kiselkov * our headroom for the next scanning cycle by this percentage multiple. 608aad02571SSaso Kiselkov */ 609aad02571SSaso Kiselkov #define L2ARC_HEADROOM_BOOST 200 6105a98e54bSBrendan Gregg - Sun Microsystems #define L2ARC_FEED_SECS 1 /* caching interval secs */ 6115a98e54bSBrendan Gregg - Sun Microsystems #define L2ARC_FEED_MIN_MS 200 /* min caching interval ms */ 612fa94a07fSbrendan 613fa94a07fSbrendan #define l2arc_writes_sent ARCSTAT(arcstat_l2_writes_sent) 614fa94a07fSbrendan #define l2arc_writes_done ARCSTAT(arcstat_l2_writes_done) 615fa94a07fSbrendan 616*f7170741SWill Andrews /* L2ARC Performance Tunables */ 617fa94a07fSbrendan uint64_t l2arc_write_max = L2ARC_WRITE_SIZE; /* default max write size */ 6183a737e0dSbrendan uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE; /* extra write during warmup */ 619fa94a07fSbrendan uint64_t l2arc_headroom = L2ARC_HEADROOM; /* number of dev writes */ 620aad02571SSaso Kiselkov uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST; 621fa94a07fSbrendan uint64_t l2arc_feed_secs = L2ARC_FEED_SECS; /* interval seconds */ 6225a98e54bSBrendan Gregg - Sun Microsystems uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval milliseconds */ 623fa94a07fSbrendan boolean_t l2arc_noprefetch = B_TRUE; /* don't cache prefetch bufs */ 6245a98e54bSBrendan Gregg - Sun Microsystems boolean_t l2arc_feed_again = B_TRUE; /* turbo warmup */ 6255a98e54bSBrendan Gregg - Sun Microsystems boolean_t l2arc_norw = B_TRUE; /* no reads during writes */ 626fa94a07fSbrendan 627fa94a07fSbrendan /* 628fa94a07fSbrendan * L2ARC Internals 629fa94a07fSbrendan */ 630fa94a07fSbrendan typedef struct l2arc_dev { 631fa94a07fSbrendan vdev_t *l2ad_vdev; /* vdev */ 632fa94a07fSbrendan spa_t *l2ad_spa; /* spa */ 633fa94a07fSbrendan uint64_t l2ad_hand; /* next write location */ 634fa94a07fSbrendan uint64_t l2ad_start; /* first addr on device */ 635fa94a07fSbrendan uint64_t l2ad_end; /* last addr on device */ 636fa94a07fSbrendan uint64_t l2ad_evict; /* last addr eviction reached */ 637fa94a07fSbrendan boolean_t l2ad_first; /* first sweep through */ 6385a98e54bSBrendan Gregg - Sun Microsystems boolean_t l2ad_writing; /* currently writing */ 639fa94a07fSbrendan list_t *l2ad_buflist; /* buffer list */ 640fa94a07fSbrendan list_node_t l2ad_node; /* device list node */ 641fa94a07fSbrendan } l2arc_dev_t; 642fa94a07fSbrendan 643fa94a07fSbrendan static list_t L2ARC_dev_list; /* device list */ 644fa94a07fSbrendan static list_t *l2arc_dev_list; /* device list pointer */ 645fa94a07fSbrendan static kmutex_t l2arc_dev_mtx; /* device list mutex */ 646fa94a07fSbrendan static l2arc_dev_t *l2arc_dev_last; /* last device used */ 647fa94a07fSbrendan static kmutex_t l2arc_buflist_mtx; /* mutex for all buflists */ 648fa94a07fSbrendan static list_t L2ARC_free_on_write; /* free after write buf list */ 649fa94a07fSbrendan static list_t *l2arc_free_on_write; /* free after write list ptr */ 650fa94a07fSbrendan static kmutex_t l2arc_free_on_write_mtx; /* mutex for list */ 651fa94a07fSbrendan static uint64_t l2arc_ndev; /* number of devices */ 652fa94a07fSbrendan 653fa94a07fSbrendan typedef struct l2arc_read_callback { 654aad02571SSaso Kiselkov arc_buf_t *l2rcb_buf; /* read buffer */ 655aad02571SSaso Kiselkov spa_t *l2rcb_spa; /* spa */ 656aad02571SSaso Kiselkov blkptr_t l2rcb_bp; /* original blkptr */ 657aad02571SSaso Kiselkov zbookmark_t l2rcb_zb; /* original bookmark */ 658aad02571SSaso Kiselkov int l2rcb_flags; /* original flags */ 659aad02571SSaso Kiselkov enum zio_compress l2rcb_compress; /* applied compress */ 660fa94a07fSbrendan } l2arc_read_callback_t; 661fa94a07fSbrendan 662fa94a07fSbrendan typedef struct l2arc_write_callback { 663fa94a07fSbrendan l2arc_dev_t *l2wcb_dev; /* device info */ 664fa94a07fSbrendan arc_buf_hdr_t *l2wcb_head; /* head of write buflist */ 665fa94a07fSbrendan } l2arc_write_callback_t; 666fa94a07fSbrendan 667fa94a07fSbrendan struct l2arc_buf_hdr { 668fa94a07fSbrendan /* protected by arc_buf_hdr mutex */ 669aad02571SSaso Kiselkov l2arc_dev_t *b_dev; /* L2ARC device */ 670aad02571SSaso Kiselkov uint64_t b_daddr; /* disk address, offset byte */ 671aad02571SSaso Kiselkov /* compression applied to buffer data */ 672aad02571SSaso Kiselkov enum zio_compress b_compress; 673aad02571SSaso Kiselkov /* real alloc'd buffer size depending on b_compress applied */ 674aad02571SSaso Kiselkov int b_asize; 675aad02571SSaso Kiselkov /* temporary buffer holder for in-flight compressed data */ 676aad02571SSaso Kiselkov void *b_tmp_cdata; 677fa94a07fSbrendan }; 678fa94a07fSbrendan 679fa94a07fSbrendan typedef struct l2arc_data_free { 680fa94a07fSbrendan /* protected by l2arc_free_on_write_mtx */ 681fa94a07fSbrendan void *l2df_data; 682fa94a07fSbrendan size_t l2df_size; 683fa94a07fSbrendan void (*l2df_func)(void *, size_t); 684fa94a07fSbrendan list_node_t l2df_list_node; 685fa94a07fSbrendan } l2arc_data_free_t; 686fa94a07fSbrendan 687fa94a07fSbrendan static kmutex_t l2arc_feed_thr_lock; 688fa94a07fSbrendan static kcondvar_t l2arc_feed_thr_cv; 689fa94a07fSbrendan static uint8_t l2arc_thread_exit; 690fa94a07fSbrendan 691fa94a07fSbrendan static void l2arc_read_done(zio_t *zio); 692fa94a07fSbrendan static void l2arc_hdr_stat_add(void); 693fa94a07fSbrendan static void l2arc_hdr_stat_remove(void); 694fa94a07fSbrendan 695aad02571SSaso Kiselkov static boolean_t l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr); 696aad02571SSaso Kiselkov static void l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, 697aad02571SSaso Kiselkov enum zio_compress c); 698aad02571SSaso Kiselkov static void l2arc_release_cdata_buf(arc_buf_hdr_t *ab); 699aad02571SSaso Kiselkov 700fa9e4066Sahrens static uint64_t 701ac05c741SMark Maybee buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth) 702fa9e4066Sahrens { 703fa9e4066Sahrens uint8_t *vdva = (uint8_t *)dva; 704fa9e4066Sahrens uint64_t crc = -1ULL; 705fa9e4066Sahrens int i; 706fa9e4066Sahrens 707fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 708fa9e4066Sahrens 709fa9e4066Sahrens for (i = 0; i < sizeof (dva_t); i++) 710fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF]; 711fa9e4066Sahrens 712ac05c741SMark Maybee crc ^= (spa>>8) ^ birth; 713fa9e4066Sahrens 714fa9e4066Sahrens return (crc); 715fa9e4066Sahrens } 716fa9e4066Sahrens 717fa9e4066Sahrens #define BUF_EMPTY(buf) \ 718fa9e4066Sahrens ((buf)->b_dva.dva_word[0] == 0 && \ 719fa9e4066Sahrens (buf)->b_dva.dva_word[1] == 0 && \ 720fa9e4066Sahrens (buf)->b_birth == 0) 721fa9e4066Sahrens 722fa9e4066Sahrens #define BUF_EQUAL(spa, dva, birth, buf) \ 723fa9e4066Sahrens ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \ 724fa9e4066Sahrens ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \ 725fa9e4066Sahrens ((buf)->b_birth == birth) && ((buf)->b_spa == spa) 726fa9e4066Sahrens 7273f9d6ad7SLin Ling static void 7283f9d6ad7SLin Ling buf_discard_identity(arc_buf_hdr_t *hdr) 7293f9d6ad7SLin Ling { 7303f9d6ad7SLin Ling hdr->b_dva.dva_word[0] = 0; 7313f9d6ad7SLin Ling hdr->b_dva.dva_word[1] = 0; 7323f9d6ad7SLin Ling hdr->b_birth = 0; 7333f9d6ad7SLin Ling hdr->b_cksum0 = 0; 7343f9d6ad7SLin Ling } 7353f9d6ad7SLin Ling 736fa9e4066Sahrens static arc_buf_hdr_t * 737ac05c741SMark Maybee buf_hash_find(uint64_t spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp) 738fa9e4066Sahrens { 739fa9e4066Sahrens uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); 740fa9e4066Sahrens kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 741fa9e4066Sahrens arc_buf_hdr_t *buf; 742fa9e4066Sahrens 743fa9e4066Sahrens mutex_enter(hash_lock); 744fa9e4066Sahrens for (buf = buf_hash_table.ht_table[idx]; buf != NULL; 745fa9e4066Sahrens buf = buf->b_hash_next) { 746fa9e4066Sahrens if (BUF_EQUAL(spa, dva, birth, buf)) { 747fa9e4066Sahrens *lockp = hash_lock; 748fa9e4066Sahrens return (buf); 749fa9e4066Sahrens } 750fa9e4066Sahrens } 751fa9e4066Sahrens mutex_exit(hash_lock); 752fa9e4066Sahrens *lockp = NULL; 753fa9e4066Sahrens return (NULL); 754fa9e4066Sahrens } 755fa9e4066Sahrens 756fa9e4066Sahrens /* 757fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 758fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 759fa9e4066Sahrens * will be returned and the new element will not be inserted. 760fa9e4066Sahrens * Otherwise returns NULL. 761fa9e4066Sahrens */ 762fa9e4066Sahrens static arc_buf_hdr_t * 763fa9e4066Sahrens buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp) 764fa9e4066Sahrens { 765fa9e4066Sahrens uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 766fa9e4066Sahrens kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 767fa9e4066Sahrens arc_buf_hdr_t *fbuf; 76844cb6abcSbmc uint32_t i; 769fa9e4066Sahrens 770ea8dc4b6Seschrock ASSERT(!HDR_IN_HASH_TABLE(buf)); 771fa9e4066Sahrens *lockp = hash_lock; 772fa9e4066Sahrens mutex_enter(hash_lock); 773fa9e4066Sahrens for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL; 774fa9e4066Sahrens fbuf = fbuf->b_hash_next, i++) { 775fa9e4066Sahrens if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf)) 776fa9e4066Sahrens return (fbuf); 777fa9e4066Sahrens } 778fa9e4066Sahrens 779fa9e4066Sahrens buf->b_hash_next = buf_hash_table.ht_table[idx]; 780fa9e4066Sahrens buf_hash_table.ht_table[idx] = buf; 781ea8dc4b6Seschrock buf->b_flags |= ARC_IN_HASH_TABLE; 782fa9e4066Sahrens 783fa9e4066Sahrens /* collect some hash table performance data */ 784fa9e4066Sahrens if (i > 0) { 78544cb6abcSbmc ARCSTAT_BUMP(arcstat_hash_collisions); 786fa9e4066Sahrens if (i == 1) 78744cb6abcSbmc ARCSTAT_BUMP(arcstat_hash_chains); 78844cb6abcSbmc 78944cb6abcSbmc ARCSTAT_MAX(arcstat_hash_chain_max, i); 790fa9e4066Sahrens } 79144cb6abcSbmc 79244cb6abcSbmc ARCSTAT_BUMP(arcstat_hash_elements); 79344cb6abcSbmc ARCSTAT_MAXSTAT(arcstat_hash_elements); 794fa9e4066Sahrens 795fa9e4066Sahrens return (NULL); 796fa9e4066Sahrens } 797fa9e4066Sahrens 798fa9e4066Sahrens static void 799fa9e4066Sahrens buf_hash_remove(arc_buf_hdr_t *buf) 800fa9e4066Sahrens { 801fa9e4066Sahrens arc_buf_hdr_t *fbuf, **bufp; 802fa9e4066Sahrens uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 803fa9e4066Sahrens 804fa9e4066Sahrens ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx))); 805ea8dc4b6Seschrock ASSERT(HDR_IN_HASH_TABLE(buf)); 806fa9e4066Sahrens 807fa9e4066Sahrens bufp = &buf_hash_table.ht_table[idx]; 808fa9e4066Sahrens while ((fbuf = *bufp) != buf) { 809fa9e4066Sahrens ASSERT(fbuf != NULL); 810fa9e4066Sahrens bufp = &fbuf->b_hash_next; 811fa9e4066Sahrens } 812fa9e4066Sahrens *bufp = buf->b_hash_next; 813fa9e4066Sahrens buf->b_hash_next = NULL; 814ea8dc4b6Seschrock buf->b_flags &= ~ARC_IN_HASH_TABLE; 815fa9e4066Sahrens 816fa9e4066Sahrens /* collect some hash table performance data */ 81744cb6abcSbmc ARCSTAT_BUMPDOWN(arcstat_hash_elements); 81844cb6abcSbmc 819fa9e4066Sahrens if (buf_hash_table.ht_table[idx] && 820fa9e4066Sahrens buf_hash_table.ht_table[idx]->b_hash_next == NULL) 82144cb6abcSbmc ARCSTAT_BUMPDOWN(arcstat_hash_chains); 822fa9e4066Sahrens } 823fa9e4066Sahrens 824fa9e4066Sahrens /* 825fa9e4066Sahrens * Global data structures and functions for the buf kmem cache. 826fa9e4066Sahrens */ 827fa9e4066Sahrens static kmem_cache_t *hdr_cache; 828fa9e4066Sahrens static kmem_cache_t *buf_cache; 829fa9e4066Sahrens 830fa9e4066Sahrens static void 831fa9e4066Sahrens buf_fini(void) 832fa9e4066Sahrens { 833fa9e4066Sahrens int i; 834fa9e4066Sahrens 835fa9e4066Sahrens kmem_free(buf_hash_table.ht_table, 836fa9e4066Sahrens (buf_hash_table.ht_mask + 1) * sizeof (void *)); 837fa9e4066Sahrens for (i = 0; i < BUF_LOCKS; i++) 838fa9e4066Sahrens mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock); 839fa9e4066Sahrens kmem_cache_destroy(hdr_cache); 840fa9e4066Sahrens kmem_cache_destroy(buf_cache); 841fa9e4066Sahrens } 842fa9e4066Sahrens 843fa9e4066Sahrens /* 844fa9e4066Sahrens * Constructor callback - called when the cache is empty 845fa9e4066Sahrens * and a new buf is requested. 846fa9e4066Sahrens */ 847fa9e4066Sahrens /* ARGSUSED */ 848fa9e4066Sahrens static int 849fa9e4066Sahrens hdr_cons(void *vbuf, void *unused, int kmflag) 850fa9e4066Sahrens { 851fa9e4066Sahrens arc_buf_hdr_t *buf = vbuf; 852fa9e4066Sahrens 853fa9e4066Sahrens bzero(buf, sizeof (arc_buf_hdr_t)); 854fa9e4066Sahrens refcount_create(&buf->b_refcnt); 855fa9e4066Sahrens cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL); 856c25056deSgw mutex_init(&buf->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL); 8575a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS); 858fa94a07fSbrendan 859fa9e4066Sahrens return (0); 860fa9e4066Sahrens } 861fa9e4066Sahrens 8626f83844dSMark Maybee /* ARGSUSED */ 8636f83844dSMark Maybee static int 8646f83844dSMark Maybee buf_cons(void *vbuf, void *unused, int kmflag) 8656f83844dSMark Maybee { 8666f83844dSMark Maybee arc_buf_t *buf = vbuf; 8676f83844dSMark Maybee 8686f83844dSMark Maybee bzero(buf, sizeof (arc_buf_t)); 8693f9d6ad7SLin Ling mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL); 8705a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS); 8715a98e54bSBrendan Gregg - Sun Microsystems 8726f83844dSMark Maybee return (0); 8736f83844dSMark Maybee } 8746f83844dSMark Maybee 875fa9e4066Sahrens /* 876fa9e4066Sahrens * Destructor callback - called when a cached buf is 877fa9e4066Sahrens * no longer required. 878fa9e4066Sahrens */ 879fa9e4066Sahrens /* ARGSUSED */ 880fa9e4066Sahrens static void 881fa9e4066Sahrens hdr_dest(void *vbuf, void *unused) 882fa9e4066Sahrens { 883fa9e4066Sahrens arc_buf_hdr_t *buf = vbuf; 884fa9e4066Sahrens 885b24ab676SJeff Bonwick ASSERT(BUF_EMPTY(buf)); 886fa9e4066Sahrens refcount_destroy(&buf->b_refcnt); 887fa9e4066Sahrens cv_destroy(&buf->b_cv); 888c25056deSgw mutex_destroy(&buf->b_freeze_lock); 8895a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS); 890fa9e4066Sahrens } 891fa9e4066Sahrens 8926f83844dSMark Maybee /* ARGSUSED */ 8936f83844dSMark Maybee static void 8946f83844dSMark Maybee buf_dest(void *vbuf, void *unused) 8956f83844dSMark Maybee { 8966f83844dSMark Maybee arc_buf_t *buf = vbuf; 8976f83844dSMark Maybee 8983f9d6ad7SLin Ling mutex_destroy(&buf->b_evict_lock); 8995a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS); 9006f83844dSMark Maybee } 9016f83844dSMark Maybee 902fa9e4066Sahrens /* 903fa9e4066Sahrens * Reclaim callback -- invoked when memory is low. 904fa9e4066Sahrens */ 905fa9e4066Sahrens /* ARGSUSED */ 906fa9e4066Sahrens static void 907fa9e4066Sahrens hdr_recl(void *unused) 908fa9e4066Sahrens { 909fa9e4066Sahrens dprintf("hdr_recl called\n"); 91049e3519aSmaybee /* 91149e3519aSmaybee * umem calls the reclaim func when we destroy the buf cache, 91249e3519aSmaybee * which is after we do arc_fini(). 91349e3519aSmaybee */ 91449e3519aSmaybee if (!arc_dead) 91549e3519aSmaybee cv_signal(&arc_reclaim_thr_cv); 916fa9e4066Sahrens } 917fa9e4066Sahrens 918fa9e4066Sahrens static void 919fa9e4066Sahrens buf_init(void) 920fa9e4066Sahrens { 921fa9e4066Sahrens uint64_t *ct; 922ea8dc4b6Seschrock uint64_t hsize = 1ULL << 12; 923fa9e4066Sahrens int i, j; 924fa9e4066Sahrens 925fa9e4066Sahrens /* 926fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 927ea8dc4b6Seschrock * with an average 64K block size. The table will take up 928ea8dc4b6Seschrock * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers). 929fa9e4066Sahrens */ 930ea8dc4b6Seschrock while (hsize * 65536 < physmem * PAGESIZE) 931fa9e4066Sahrens hsize <<= 1; 932ea8dc4b6Seschrock retry: 933fa9e4066Sahrens buf_hash_table.ht_mask = hsize - 1; 934ea8dc4b6Seschrock buf_hash_table.ht_table = 935ea8dc4b6Seschrock kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP); 936ea8dc4b6Seschrock if (buf_hash_table.ht_table == NULL) { 937ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 8)); 938ea8dc4b6Seschrock hsize >>= 1; 939ea8dc4b6Seschrock goto retry; 940ea8dc4b6Seschrock } 941fa9e4066Sahrens 942fa9e4066Sahrens hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t), 943fa9e4066Sahrens 0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0); 944fa9e4066Sahrens buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t), 9456f83844dSMark Maybee 0, buf_cons, buf_dest, NULL, NULL, NULL, 0); 946fa9e4066Sahrens 947fa9e4066Sahrens for (i = 0; i < 256; i++) 948fa9e4066Sahrens for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--) 949fa9e4066Sahrens *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY); 950fa9e4066Sahrens 951fa9e4066Sahrens for (i = 0; i < BUF_LOCKS; i++) { 952fa9e4066Sahrens mutex_init(&buf_hash_table.ht_locks[i].ht_lock, 953fa9e4066Sahrens NULL, MUTEX_DEFAULT, NULL); 954fa9e4066Sahrens } 955fa9e4066Sahrens } 956fa9e4066Sahrens 957fa9e4066Sahrens #define ARC_MINTIME (hz>>4) /* 62 ms */ 958fa9e4066Sahrens 9596b4acc8bSahrens static void 9606b4acc8bSahrens arc_cksum_verify(arc_buf_t *buf) 9616b4acc8bSahrens { 9626b4acc8bSahrens zio_cksum_t zc; 9636b4acc8bSahrens 964cc60fd72Sahrens if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 9656b4acc8bSahrens return; 9666b4acc8bSahrens 9676b4acc8bSahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 9683ccfa83cSahrens if (buf->b_hdr->b_freeze_cksum == NULL || 9693ccfa83cSahrens (buf->b_hdr->b_flags & ARC_IO_ERROR)) { 9706b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 9716b4acc8bSahrens return; 9726b4acc8bSahrens } 9736b4acc8bSahrens fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 9746b4acc8bSahrens if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc)) 9756b4acc8bSahrens panic("buffer modified while frozen!"); 9766b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 9776b4acc8bSahrens } 9786b4acc8bSahrens 979fa94a07fSbrendan static int 980fa94a07fSbrendan arc_cksum_equal(arc_buf_t *buf) 981fa94a07fSbrendan { 982fa94a07fSbrendan zio_cksum_t zc; 983fa94a07fSbrendan int equal; 984fa94a07fSbrendan 985fa94a07fSbrendan mutex_enter(&buf->b_hdr->b_freeze_lock); 986fa94a07fSbrendan fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 987fa94a07fSbrendan equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc); 988fa94a07fSbrendan mutex_exit(&buf->b_hdr->b_freeze_lock); 989fa94a07fSbrendan 990fa94a07fSbrendan return (equal); 991fa94a07fSbrendan } 992fa94a07fSbrendan 9936b4acc8bSahrens static void 994fa94a07fSbrendan arc_cksum_compute(arc_buf_t *buf, boolean_t force) 9956b4acc8bSahrens { 996fa94a07fSbrendan if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY)) 9976b4acc8bSahrens return; 9986b4acc8bSahrens 9996b4acc8bSahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 10006b4acc8bSahrens if (buf->b_hdr->b_freeze_cksum != NULL) { 10016b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 10026b4acc8bSahrens return; 10036b4acc8bSahrens } 10046b4acc8bSahrens buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP); 10056b4acc8bSahrens fletcher_2_native(buf->b_data, buf->b_hdr->b_size, 10066b4acc8bSahrens buf->b_hdr->b_freeze_cksum); 10076b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 1008cd1c8b85SMatthew Ahrens arc_buf_watch(buf); 1009cd1c8b85SMatthew Ahrens } 1010cd1c8b85SMatthew Ahrens 1011cd1c8b85SMatthew Ahrens #ifndef _KERNEL 1012cd1c8b85SMatthew Ahrens typedef struct procctl { 1013cd1c8b85SMatthew Ahrens long cmd; 1014cd1c8b85SMatthew Ahrens prwatch_t prwatch; 1015cd1c8b85SMatthew Ahrens } procctl_t; 1016cd1c8b85SMatthew Ahrens #endif 1017cd1c8b85SMatthew Ahrens 1018cd1c8b85SMatthew Ahrens /* ARGSUSED */ 1019cd1c8b85SMatthew Ahrens static void 1020cd1c8b85SMatthew Ahrens arc_buf_unwatch(arc_buf_t *buf) 1021cd1c8b85SMatthew Ahrens { 1022cd1c8b85SMatthew Ahrens #ifndef _KERNEL 1023cd1c8b85SMatthew Ahrens if (arc_watch) { 1024cd1c8b85SMatthew Ahrens int result; 1025cd1c8b85SMatthew Ahrens procctl_t ctl; 1026cd1c8b85SMatthew Ahrens ctl.cmd = PCWATCH; 1027cd1c8b85SMatthew Ahrens ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data; 1028cd1c8b85SMatthew Ahrens ctl.prwatch.pr_size = 0; 1029cd1c8b85SMatthew Ahrens ctl.prwatch.pr_wflags = 0; 1030cd1c8b85SMatthew Ahrens result = write(arc_procfd, &ctl, sizeof (ctl)); 1031cd1c8b85SMatthew Ahrens ASSERT3U(result, ==, sizeof (ctl)); 1032cd1c8b85SMatthew Ahrens } 1033cd1c8b85SMatthew Ahrens #endif 1034cd1c8b85SMatthew Ahrens } 1035cd1c8b85SMatthew Ahrens 1036cd1c8b85SMatthew Ahrens /* ARGSUSED */ 1037cd1c8b85SMatthew Ahrens static void 1038cd1c8b85SMatthew Ahrens arc_buf_watch(arc_buf_t *buf) 1039cd1c8b85SMatthew Ahrens { 1040cd1c8b85SMatthew Ahrens #ifndef _KERNEL 1041cd1c8b85SMatthew Ahrens if (arc_watch) { 1042cd1c8b85SMatthew Ahrens int result; 1043cd1c8b85SMatthew Ahrens procctl_t ctl; 1044cd1c8b85SMatthew Ahrens ctl.cmd = PCWATCH; 1045cd1c8b85SMatthew Ahrens ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data; 1046cd1c8b85SMatthew Ahrens ctl.prwatch.pr_size = buf->b_hdr->b_size; 1047cd1c8b85SMatthew Ahrens ctl.prwatch.pr_wflags = WA_WRITE; 1048cd1c8b85SMatthew Ahrens result = write(arc_procfd, &ctl, sizeof (ctl)); 1049cd1c8b85SMatthew Ahrens ASSERT3U(result, ==, sizeof (ctl)); 1050cd1c8b85SMatthew Ahrens } 1051cd1c8b85SMatthew Ahrens #endif 10526b4acc8bSahrens } 10536b4acc8bSahrens 10546b4acc8bSahrens void 10556b4acc8bSahrens arc_buf_thaw(arc_buf_t *buf) 10566b4acc8bSahrens { 1057fa94a07fSbrendan if (zfs_flags & ZFS_DEBUG_MODIFY) { 1058fa94a07fSbrendan if (buf->b_hdr->b_state != arc_anon) 1059fa94a07fSbrendan panic("modifying non-anon buffer!"); 1060fa94a07fSbrendan if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS) 1061fa94a07fSbrendan panic("modifying buffer while i/o in progress!"); 1062fa94a07fSbrendan arc_cksum_verify(buf); 1063fa94a07fSbrendan } 10646b4acc8bSahrens 10656b4acc8bSahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 10666b4acc8bSahrens if (buf->b_hdr->b_freeze_cksum != NULL) { 10676b4acc8bSahrens kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 10686b4acc8bSahrens buf->b_hdr->b_freeze_cksum = NULL; 10696b4acc8bSahrens } 10703f9d6ad7SLin Ling 10713f9d6ad7SLin Ling if (zfs_flags & ZFS_DEBUG_MODIFY) { 10723f9d6ad7SLin Ling if (buf->b_hdr->b_thawed) 10733f9d6ad7SLin Ling kmem_free(buf->b_hdr->b_thawed, 1); 10743f9d6ad7SLin Ling buf->b_hdr->b_thawed = kmem_alloc(1, KM_SLEEP); 10753f9d6ad7SLin Ling } 10763f9d6ad7SLin Ling 10776b4acc8bSahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 1078cd1c8b85SMatthew Ahrens 1079cd1c8b85SMatthew Ahrens arc_buf_unwatch(buf); 10806b4acc8bSahrens } 10816b4acc8bSahrens 10826b4acc8bSahrens void 10836b4acc8bSahrens arc_buf_freeze(arc_buf_t *buf) 10846b4acc8bSahrens { 10853f9d6ad7SLin Ling kmutex_t *hash_lock; 10863f9d6ad7SLin Ling 1087cc60fd72Sahrens if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 1088cc60fd72Sahrens return; 1089cc60fd72Sahrens 10903f9d6ad7SLin Ling hash_lock = HDR_LOCK(buf->b_hdr); 10913f9d6ad7SLin Ling mutex_enter(hash_lock); 10923f9d6ad7SLin Ling 10936b4acc8bSahrens ASSERT(buf->b_hdr->b_freeze_cksum != NULL || 109444cb6abcSbmc buf->b_hdr->b_state == arc_anon); 1095fa94a07fSbrendan arc_cksum_compute(buf, B_FALSE); 10963f9d6ad7SLin Ling mutex_exit(hash_lock); 1097cd1c8b85SMatthew Ahrens 10986b4acc8bSahrens } 10996b4acc8bSahrens 1100fa9e4066Sahrens static void 1101fa9e4066Sahrens add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 1102fa9e4066Sahrens { 1103fa9e4066Sahrens ASSERT(MUTEX_HELD(hash_lock)); 1104fa9e4066Sahrens 1105fa9e4066Sahrens if ((refcount_add(&ab->b_refcnt, tag) == 1) && 110644cb6abcSbmc (ab->b_state != arc_anon)) { 1107c0a81264Sek uint64_t delta = ab->b_size * ab->b_datacnt; 11080e8c6158Smaybee list_t *list = &ab->b_state->arcs_list[ab->b_type]; 11090e8c6158Smaybee uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type]; 1110fa9e4066Sahrens 111144cb6abcSbmc ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx)); 111244cb6abcSbmc mutex_enter(&ab->b_state->arcs_mtx); 1113fa9e4066Sahrens ASSERT(list_link_active(&ab->b_arc_node)); 11140e8c6158Smaybee list_remove(list, ab); 1115ea8dc4b6Seschrock if (GHOST_STATE(ab->b_state)) { 1116fb09f5aaSMadhav Suresh ASSERT0(ab->b_datacnt); 1117ea8dc4b6Seschrock ASSERT3P(ab->b_buf, ==, NULL); 1118ea8dc4b6Seschrock delta = ab->b_size; 1119ea8dc4b6Seschrock } 1120ea8dc4b6Seschrock ASSERT(delta > 0); 11210e8c6158Smaybee ASSERT3U(*size, >=, delta); 11220e8c6158Smaybee atomic_add_64(size, -delta); 112344cb6abcSbmc mutex_exit(&ab->b_state->arcs_mtx); 1124088f3894Sahrens /* remove the prefetch flag if we get a reference */ 112513506d1eSmaybee if (ab->b_flags & ARC_PREFETCH) 112613506d1eSmaybee ab->b_flags &= ~ARC_PREFETCH; 1127fa9e4066Sahrens } 1128fa9e4066Sahrens } 1129fa9e4066Sahrens 1130fa9e4066Sahrens static int 1131fa9e4066Sahrens remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 1132fa9e4066Sahrens { 1133fa9e4066Sahrens int cnt; 113444cb6abcSbmc arc_state_t *state = ab->b_state; 1135fa9e4066Sahrens 113644cb6abcSbmc ASSERT(state == arc_anon || MUTEX_HELD(hash_lock)); 113744cb6abcSbmc ASSERT(!GHOST_STATE(state)); 1138fa9e4066Sahrens 1139fa9e4066Sahrens if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) && 114044cb6abcSbmc (state != arc_anon)) { 11410e8c6158Smaybee uint64_t *size = &state->arcs_lsize[ab->b_type]; 11420e8c6158Smaybee 114344cb6abcSbmc ASSERT(!MUTEX_HELD(&state->arcs_mtx)); 114444cb6abcSbmc mutex_enter(&state->arcs_mtx); 1145fa9e4066Sahrens ASSERT(!list_link_active(&ab->b_arc_node)); 11460e8c6158Smaybee list_insert_head(&state->arcs_list[ab->b_type], ab); 1147ea8dc4b6Seschrock ASSERT(ab->b_datacnt > 0); 11480e8c6158Smaybee atomic_add_64(size, ab->b_size * ab->b_datacnt); 114944cb6abcSbmc mutex_exit(&state->arcs_mtx); 1150fa9e4066Sahrens } 1151fa9e4066Sahrens return (cnt); 1152fa9e4066Sahrens } 1153fa9e4066Sahrens 1154fa9e4066Sahrens /* 1155fa9e4066Sahrens * Move the supplied buffer to the indicated state. The mutex 1156fa9e4066Sahrens * for the buffer must be held by the caller. 1157fa9e4066Sahrens */ 1158fa9e4066Sahrens static void 1159ea8dc4b6Seschrock arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock) 1160fa9e4066Sahrens { 1161ea8dc4b6Seschrock arc_state_t *old_state = ab->b_state; 1162c0a81264Sek int64_t refcnt = refcount_count(&ab->b_refcnt); 1163c0a81264Sek uint64_t from_delta, to_delta; 1164fa9e4066Sahrens 1165fa9e4066Sahrens ASSERT(MUTEX_HELD(hash_lock)); 1166ea8dc4b6Seschrock ASSERT(new_state != old_state); 1167ea8dc4b6Seschrock ASSERT(refcnt == 0 || ab->b_datacnt > 0); 1168ea8dc4b6Seschrock ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state)); 1169b24ab676SJeff Bonwick ASSERT(ab->b_datacnt <= 1 || old_state != arc_anon); 1170ea8dc4b6Seschrock 1171ea8dc4b6Seschrock from_delta = to_delta = ab->b_datacnt * ab->b_size; 1172fa9e4066Sahrens 1173fa9e4066Sahrens /* 1174fa9e4066Sahrens * If this buffer is evictable, transfer it from the 1175fa9e4066Sahrens * old state list to the new state list. 1176fa9e4066Sahrens */ 1177ea8dc4b6Seschrock if (refcnt == 0) { 117844cb6abcSbmc if (old_state != arc_anon) { 117944cb6abcSbmc int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx); 11800e8c6158Smaybee uint64_t *size = &old_state->arcs_lsize[ab->b_type]; 1181ea8dc4b6Seschrock 1182ea8dc4b6Seschrock if (use_mutex) 118344cb6abcSbmc mutex_enter(&old_state->arcs_mtx); 1184fa9e4066Sahrens 1185fa9e4066Sahrens ASSERT(list_link_active(&ab->b_arc_node)); 11860e8c6158Smaybee list_remove(&old_state->arcs_list[ab->b_type], ab); 1187ea8dc4b6Seschrock 118813506d1eSmaybee /* 118913506d1eSmaybee * If prefetching out of the ghost cache, 11903f9d6ad7SLin Ling * we will have a non-zero datacnt. 119113506d1eSmaybee */ 119213506d1eSmaybee if (GHOST_STATE(old_state) && ab->b_datacnt == 0) { 119313506d1eSmaybee /* ghost elements have a ghost size */ 1194ea8dc4b6Seschrock ASSERT(ab->b_buf == NULL); 1195ea8dc4b6Seschrock from_delta = ab->b_size; 1196ea8dc4b6Seschrock } 11970e8c6158Smaybee ASSERT3U(*size, >=, from_delta); 11980e8c6158Smaybee atomic_add_64(size, -from_delta); 1199ea8dc4b6Seschrock 1200ea8dc4b6Seschrock if (use_mutex) 120144cb6abcSbmc mutex_exit(&old_state->arcs_mtx); 1202fa9e4066Sahrens } 120344cb6abcSbmc if (new_state != arc_anon) { 120444cb6abcSbmc int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx); 12050e8c6158Smaybee uint64_t *size = &new_state->arcs_lsize[ab->b_type]; 1206fa9e4066Sahrens 1207ea8dc4b6Seschrock if (use_mutex) 120844cb6abcSbmc mutex_enter(&new_state->arcs_mtx); 1209ea8dc4b6Seschrock 12100e8c6158Smaybee list_insert_head(&new_state->arcs_list[ab->b_type], ab); 1211ea8dc4b6Seschrock 1212ea8dc4b6Seschrock /* ghost elements have a ghost size */ 1213ea8dc4b6Seschrock if (GHOST_STATE(new_state)) { 1214ea8dc4b6Seschrock ASSERT(ab->b_datacnt == 0); 1215ea8dc4b6Seschrock ASSERT(ab->b_buf == NULL); 1216ea8dc4b6Seschrock to_delta = ab->b_size; 1217ea8dc4b6Seschrock } 12180e8c6158Smaybee atomic_add_64(size, to_delta); 1219ea8dc4b6Seschrock 1220ea8dc4b6Seschrock if (use_mutex) 122144cb6abcSbmc mutex_exit(&new_state->arcs_mtx); 1222fa9e4066Sahrens } 1223fa9e4066Sahrens } 1224fa9e4066Sahrens 1225fa9e4066Sahrens ASSERT(!BUF_EMPTY(ab)); 12263f9d6ad7SLin Ling if (new_state == arc_anon && HDR_IN_HASH_TABLE(ab)) 1227fa9e4066Sahrens buf_hash_remove(ab); 1228fa9e4066Sahrens 1229ea8dc4b6Seschrock /* adjust state sizes */ 1230ea8dc4b6Seschrock if (to_delta) 123144cb6abcSbmc atomic_add_64(&new_state->arcs_size, to_delta); 1232ea8dc4b6Seschrock if (from_delta) { 123344cb6abcSbmc ASSERT3U(old_state->arcs_size, >=, from_delta); 123444cb6abcSbmc atomic_add_64(&old_state->arcs_size, -from_delta); 1235fa9e4066Sahrens } 1236fa9e4066Sahrens ab->b_state = new_state; 1237fa94a07fSbrendan 1238fa94a07fSbrendan /* adjust l2arc hdr stats */ 1239fa94a07fSbrendan if (new_state == arc_l2c_only) 1240fa94a07fSbrendan l2arc_hdr_stat_add(); 1241fa94a07fSbrendan else if (old_state == arc_l2c_only) 1242fa94a07fSbrendan l2arc_hdr_stat_remove(); 1243fa9e4066Sahrens } 1244fa9e4066Sahrens 12450e8c6158Smaybee void 12465a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(uint64_t space, arc_space_type_t type) 12470e8c6158Smaybee { 12485a98e54bSBrendan Gregg - Sun Microsystems ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 12495a98e54bSBrendan Gregg - Sun Microsystems 12505a98e54bSBrendan Gregg - Sun Microsystems switch (type) { 12515a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_DATA: 12525a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, space); 12535a98e54bSBrendan Gregg - Sun Microsystems break; 12545a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_OTHER: 12555a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_other_size, space); 12565a98e54bSBrendan Gregg - Sun Microsystems break; 12575a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_HDRS: 12585a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_hdr_size, space); 12595a98e54bSBrendan Gregg - Sun Microsystems break; 12605a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_L2HDRS: 12615a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_l2_hdr_size, space); 12625a98e54bSBrendan Gregg - Sun Microsystems break; 12635a98e54bSBrendan Gregg - Sun Microsystems } 12645a98e54bSBrendan Gregg - Sun Microsystems 126520128a08SGeorge Wilson ARCSTAT_INCR(arcstat_meta_used, space); 12660e8c6158Smaybee atomic_add_64(&arc_size, space); 12670e8c6158Smaybee } 12680e8c6158Smaybee 12690e8c6158Smaybee void 12705a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(uint64_t space, arc_space_type_t type) 12710e8c6158Smaybee { 12725a98e54bSBrendan Gregg - Sun Microsystems ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 12735a98e54bSBrendan Gregg - Sun Microsystems 12745a98e54bSBrendan Gregg - Sun Microsystems switch (type) { 12755a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_DATA: 12765a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, -space); 12775a98e54bSBrendan Gregg - Sun Microsystems break; 12785a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_OTHER: 12795a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_other_size, -space); 12805a98e54bSBrendan Gregg - Sun Microsystems break; 12815a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_HDRS: 12825a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_hdr_size, -space); 12835a98e54bSBrendan Gregg - Sun Microsystems break; 12845a98e54bSBrendan Gregg - Sun Microsystems case ARC_SPACE_L2HDRS: 12855a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_l2_hdr_size, -space); 12865a98e54bSBrendan Gregg - Sun Microsystems break; 12875a98e54bSBrendan Gregg - Sun Microsystems } 12885a98e54bSBrendan Gregg - Sun Microsystems 12890e8c6158Smaybee ASSERT(arc_meta_used >= space); 12900e8c6158Smaybee if (arc_meta_max < arc_meta_used) 12910e8c6158Smaybee arc_meta_max = arc_meta_used; 129220128a08SGeorge Wilson ARCSTAT_INCR(arcstat_meta_used, -space); 12930e8c6158Smaybee ASSERT(arc_size >= space); 12940e8c6158Smaybee atomic_add_64(&arc_size, -space); 12950e8c6158Smaybee } 12960e8c6158Smaybee 12970e8c6158Smaybee void * 12980e8c6158Smaybee arc_data_buf_alloc(uint64_t size) 12990e8c6158Smaybee { 13000e8c6158Smaybee if (arc_evict_needed(ARC_BUFC_DATA)) 13010e8c6158Smaybee cv_signal(&arc_reclaim_thr_cv); 13020e8c6158Smaybee atomic_add_64(&arc_size, size); 13030e8c6158Smaybee return (zio_data_buf_alloc(size)); 13040e8c6158Smaybee } 13050e8c6158Smaybee 13060e8c6158Smaybee void 13070e8c6158Smaybee arc_data_buf_free(void *buf, uint64_t size) 13080e8c6158Smaybee { 13090e8c6158Smaybee zio_data_buf_free(buf, size); 13100e8c6158Smaybee ASSERT(arc_size >= size); 13110e8c6158Smaybee atomic_add_64(&arc_size, -size); 13120e8c6158Smaybee } 13130e8c6158Smaybee 1314fa9e4066Sahrens arc_buf_t * 1315ad23a2dbSjohansen arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type) 1316fa9e4066Sahrens { 1317fa9e4066Sahrens arc_buf_hdr_t *hdr; 1318fa9e4066Sahrens arc_buf_t *buf; 1319fa9e4066Sahrens 1320fa9e4066Sahrens ASSERT3U(size, >, 0); 13211ab7f2deSmaybee hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 1322fa9e4066Sahrens ASSERT(BUF_EMPTY(hdr)); 1323fa9e4066Sahrens hdr->b_size = size; 1324ad23a2dbSjohansen hdr->b_type = type; 1325e9103aaeSGarrett D'Amore hdr->b_spa = spa_load_guid(spa); 132644cb6abcSbmc hdr->b_state = arc_anon; 1327fa9e4066Sahrens hdr->b_arc_access = 0; 13281ab7f2deSmaybee buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 1329fa9e4066Sahrens buf->b_hdr = hdr; 133044eda4d7Smaybee buf->b_data = NULL; 1331ea8dc4b6Seschrock buf->b_efunc = NULL; 1332ea8dc4b6Seschrock buf->b_private = NULL; 1333fa9e4066Sahrens buf->b_next = NULL; 1334fa9e4066Sahrens hdr->b_buf = buf; 133544eda4d7Smaybee arc_get_data_buf(buf); 1336ea8dc4b6Seschrock hdr->b_datacnt = 1; 1337fa9e4066Sahrens hdr->b_flags = 0; 1338fa9e4066Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1339fa9e4066Sahrens (void) refcount_add(&hdr->b_refcnt, tag); 1340fa9e4066Sahrens 1341fa9e4066Sahrens return (buf); 1342fa9e4066Sahrens } 1343fa9e4066Sahrens 13442fdbea25SAleksandr Guzovskiy static char *arc_onloan_tag = "onloan"; 13452fdbea25SAleksandr Guzovskiy 13462fdbea25SAleksandr Guzovskiy /* 13472fdbea25SAleksandr Guzovskiy * Loan out an anonymous arc buffer. Loaned buffers are not counted as in 13482fdbea25SAleksandr Guzovskiy * flight data by arc_tempreserve_space() until they are "returned". Loaned 13492fdbea25SAleksandr Guzovskiy * buffers must be returned to the arc before they can be used by the DMU or 13502fdbea25SAleksandr Guzovskiy * freed. 13512fdbea25SAleksandr Guzovskiy */ 13522fdbea25SAleksandr Guzovskiy arc_buf_t * 13532fdbea25SAleksandr Guzovskiy arc_loan_buf(spa_t *spa, int size) 13542fdbea25SAleksandr Guzovskiy { 13552fdbea25SAleksandr Guzovskiy arc_buf_t *buf; 13562fdbea25SAleksandr Guzovskiy 13572fdbea25SAleksandr Guzovskiy buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA); 13582fdbea25SAleksandr Guzovskiy 13592fdbea25SAleksandr Guzovskiy atomic_add_64(&arc_loaned_bytes, size); 13602fdbea25SAleksandr Guzovskiy return (buf); 13612fdbea25SAleksandr Guzovskiy } 13622fdbea25SAleksandr Guzovskiy 13632fdbea25SAleksandr Guzovskiy /* 13642fdbea25SAleksandr Guzovskiy * Return a loaned arc buffer to the arc. 13652fdbea25SAleksandr Guzovskiy */ 13662fdbea25SAleksandr Guzovskiy void 13672fdbea25SAleksandr Guzovskiy arc_return_buf(arc_buf_t *buf, void *tag) 13682fdbea25SAleksandr Guzovskiy { 13692fdbea25SAleksandr Guzovskiy arc_buf_hdr_t *hdr = buf->b_hdr; 13702fdbea25SAleksandr Guzovskiy 13712fdbea25SAleksandr Guzovskiy ASSERT(buf->b_data != NULL); 1372c242f9a0Schunli zhang - Sun Microsystems - Irvine United States (void) refcount_add(&hdr->b_refcnt, tag); 1373c242f9a0Schunli zhang - Sun Microsystems - Irvine United States (void) refcount_remove(&hdr->b_refcnt, arc_onloan_tag); 13742fdbea25SAleksandr Guzovskiy 13752fdbea25SAleksandr Guzovskiy atomic_add_64(&arc_loaned_bytes, -hdr->b_size); 13762fdbea25SAleksandr Guzovskiy } 13772fdbea25SAleksandr Guzovskiy 1378c242f9a0Schunli zhang - Sun Microsystems - Irvine United States /* Detach an arc_buf from a dbuf (tag) */ 1379c242f9a0Schunli zhang - Sun Microsystems - Irvine United States void 1380c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_loan_inuse_buf(arc_buf_t *buf, void *tag) 1381c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 1382c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_hdr_t *hdr; 1383c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1384c242f9a0Schunli zhang - Sun Microsystems - Irvine United States ASSERT(buf->b_data != NULL); 1385c242f9a0Schunli zhang - Sun Microsystems - Irvine United States hdr = buf->b_hdr; 1386c242f9a0Schunli zhang - Sun Microsystems - Irvine United States (void) refcount_add(&hdr->b_refcnt, arc_onloan_tag); 1387c242f9a0Schunli zhang - Sun Microsystems - Irvine United States (void) refcount_remove(&hdr->b_refcnt, tag); 1388c242f9a0Schunli zhang - Sun Microsystems - Irvine United States buf->b_efunc = NULL; 1389c242f9a0Schunli zhang - Sun Microsystems - Irvine United States buf->b_private = NULL; 1390c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1391c242f9a0Schunli zhang - Sun Microsystems - Irvine United States atomic_add_64(&arc_loaned_bytes, hdr->b_size); 1392c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 1393c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 139444eda4d7Smaybee static arc_buf_t * 139544eda4d7Smaybee arc_buf_clone(arc_buf_t *from) 1396ea8dc4b6Seschrock { 139744eda4d7Smaybee arc_buf_t *buf; 139844eda4d7Smaybee arc_buf_hdr_t *hdr = from->b_hdr; 139944eda4d7Smaybee uint64_t size = hdr->b_size; 1400ea8dc4b6Seschrock 1401b24ab676SJeff Bonwick ASSERT(hdr->b_state != arc_anon); 1402b24ab676SJeff Bonwick 14031ab7f2deSmaybee buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 140444eda4d7Smaybee buf->b_hdr = hdr; 140544eda4d7Smaybee buf->b_data = NULL; 140644eda4d7Smaybee buf->b_efunc = NULL; 140744eda4d7Smaybee buf->b_private = NULL; 140844eda4d7Smaybee buf->b_next = hdr->b_buf; 140944eda4d7Smaybee hdr->b_buf = buf; 141044eda4d7Smaybee arc_get_data_buf(buf); 141144eda4d7Smaybee bcopy(from->b_data, buf->b_data, size); 14129253d63dSGeorge Wilson 14139253d63dSGeorge Wilson /* 14149253d63dSGeorge Wilson * This buffer already exists in the arc so create a duplicate 14159253d63dSGeorge Wilson * copy for the caller. If the buffer is associated with user data 14169253d63dSGeorge Wilson * then track the size and number of duplicates. These stats will be 14179253d63dSGeorge Wilson * updated as duplicate buffers are created and destroyed. 14189253d63dSGeorge Wilson */ 14199253d63dSGeorge Wilson if (hdr->b_type == ARC_BUFC_DATA) { 14209253d63dSGeorge Wilson ARCSTAT_BUMP(arcstat_duplicate_buffers); 14219253d63dSGeorge Wilson ARCSTAT_INCR(arcstat_duplicate_buffers_size, size); 14229253d63dSGeorge Wilson } 142344eda4d7Smaybee hdr->b_datacnt += 1; 142444eda4d7Smaybee return (buf); 1425ea8dc4b6Seschrock } 1426ea8dc4b6Seschrock 1427ea8dc4b6Seschrock void 1428ea8dc4b6Seschrock arc_buf_add_ref(arc_buf_t *buf, void* tag) 1429ea8dc4b6Seschrock { 143040d7d650Smaybee arc_buf_hdr_t *hdr; 1431ea8dc4b6Seschrock kmutex_t *hash_lock; 1432ea8dc4b6Seschrock 14339b23f181Smaybee /* 14346f83844dSMark Maybee * Check to see if this buffer is evicted. Callers 14356f83844dSMark Maybee * must verify b_data != NULL to know if the add_ref 14366f83844dSMark Maybee * was successful. 14379b23f181Smaybee */ 14383f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 14396f83844dSMark Maybee if (buf->b_data == NULL) { 14403f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 14419b23f181Smaybee return; 144240d7d650Smaybee } 14433f9d6ad7SLin Ling hash_lock = HDR_LOCK(buf->b_hdr); 14449b23f181Smaybee mutex_enter(hash_lock); 14453f9d6ad7SLin Ling hdr = buf->b_hdr; 14463f9d6ad7SLin Ling ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 14473f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 1448ea8dc4b6Seschrock 144944cb6abcSbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 1450ea8dc4b6Seschrock add_reference(hdr, hash_lock, tag); 14515a98e54bSBrendan Gregg - Sun Microsystems DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 145244eda4d7Smaybee arc_access(hdr, hash_lock); 145344eda4d7Smaybee mutex_exit(hash_lock); 145444cb6abcSbmc ARCSTAT_BUMP(arcstat_hits); 145544cb6abcSbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 145644cb6abcSbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 145744cb6abcSbmc data, metadata, hits); 1458ea8dc4b6Seschrock } 1459ea8dc4b6Seschrock 1460fa94a07fSbrendan /* 1461fa94a07fSbrendan * Free the arc data buffer. If it is an l2arc write in progress, 1462fa94a07fSbrendan * the buffer is placed on l2arc_free_on_write to be freed later. 1463fa94a07fSbrendan */ 1464fa94a07fSbrendan static void 1465cd1c8b85SMatthew Ahrens arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t)) 1466fa94a07fSbrendan { 1467cd1c8b85SMatthew Ahrens arc_buf_hdr_t *hdr = buf->b_hdr; 1468cd1c8b85SMatthew Ahrens 1469fa94a07fSbrendan if (HDR_L2_WRITING(hdr)) { 1470fa94a07fSbrendan l2arc_data_free_t *df; 1471fa94a07fSbrendan df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP); 1472cd1c8b85SMatthew Ahrens df->l2df_data = buf->b_data; 1473cd1c8b85SMatthew Ahrens df->l2df_size = hdr->b_size; 1474fa94a07fSbrendan df->l2df_func = free_func; 1475fa94a07fSbrendan mutex_enter(&l2arc_free_on_write_mtx); 1476fa94a07fSbrendan list_insert_head(l2arc_free_on_write, df); 1477fa94a07fSbrendan mutex_exit(&l2arc_free_on_write_mtx); 1478fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_free_on_write); 1479fa94a07fSbrendan } else { 1480cd1c8b85SMatthew Ahrens free_func(buf->b_data, hdr->b_size); 1481fa94a07fSbrendan } 1482fa94a07fSbrendan } 1483fa94a07fSbrendan 1484ea8dc4b6Seschrock static void 148544eda4d7Smaybee arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all) 1486ea8dc4b6Seschrock { 1487ea8dc4b6Seschrock arc_buf_t **bufp; 1488ea8dc4b6Seschrock 1489ea8dc4b6Seschrock /* free up data associated with the buf */ 1490ea8dc4b6Seschrock if (buf->b_data) { 1491ea8dc4b6Seschrock arc_state_t *state = buf->b_hdr->b_state; 1492ea8dc4b6Seschrock uint64_t size = buf->b_hdr->b_size; 1493ad23a2dbSjohansen arc_buf_contents_t type = buf->b_hdr->b_type; 1494ea8dc4b6Seschrock 14956b4acc8bSahrens arc_cksum_verify(buf); 1496cd1c8b85SMatthew Ahrens arc_buf_unwatch(buf); 1497b24ab676SJeff Bonwick 149844eda4d7Smaybee if (!recycle) { 1499ad23a2dbSjohansen if (type == ARC_BUFC_METADATA) { 1500cd1c8b85SMatthew Ahrens arc_buf_data_free(buf, zio_buf_free); 15015a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(size, ARC_SPACE_DATA); 1502ad23a2dbSjohansen } else { 1503ad23a2dbSjohansen ASSERT(type == ARC_BUFC_DATA); 1504cd1c8b85SMatthew Ahrens arc_buf_data_free(buf, zio_data_buf_free); 15055a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, -size); 15060e8c6158Smaybee atomic_add_64(&arc_size, -size); 1507ad23a2dbSjohansen } 150844eda4d7Smaybee } 1509ea8dc4b6Seschrock if (list_link_active(&buf->b_hdr->b_arc_node)) { 15100e8c6158Smaybee uint64_t *cnt = &state->arcs_lsize[type]; 15110e8c6158Smaybee 1512ea8dc4b6Seschrock ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt)); 151344cb6abcSbmc ASSERT(state != arc_anon); 15140e8c6158Smaybee 15150e8c6158Smaybee ASSERT3U(*cnt, >=, size); 15160e8c6158Smaybee atomic_add_64(cnt, -size); 1517ea8dc4b6Seschrock } 151844cb6abcSbmc ASSERT3U(state->arcs_size, >=, size); 151944cb6abcSbmc atomic_add_64(&state->arcs_size, -size); 1520ea8dc4b6Seschrock buf->b_data = NULL; 15219253d63dSGeorge Wilson 15229253d63dSGeorge Wilson /* 15239253d63dSGeorge Wilson * If we're destroying a duplicate buffer make sure 15249253d63dSGeorge Wilson * that the appropriate statistics are updated. 15259253d63dSGeorge Wilson */ 15269253d63dSGeorge Wilson if (buf->b_hdr->b_datacnt > 1 && 15279253d63dSGeorge Wilson buf->b_hdr->b_type == ARC_BUFC_DATA) { 15289253d63dSGeorge Wilson ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers); 15299253d63dSGeorge Wilson ARCSTAT_INCR(arcstat_duplicate_buffers_size, -size); 15309253d63dSGeorge Wilson } 1531ea8dc4b6Seschrock ASSERT(buf->b_hdr->b_datacnt > 0); 1532ea8dc4b6Seschrock buf->b_hdr->b_datacnt -= 1; 1533ea8dc4b6Seschrock } 1534ea8dc4b6Seschrock 1535ea8dc4b6Seschrock /* only remove the buf if requested */ 1536ea8dc4b6Seschrock if (!all) 1537ea8dc4b6Seschrock return; 1538ea8dc4b6Seschrock 1539ea8dc4b6Seschrock /* remove the buf from the hdr list */ 1540ea8dc4b6Seschrock for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next) 1541ea8dc4b6Seschrock continue; 1542ea8dc4b6Seschrock *bufp = buf->b_next; 15433f9d6ad7SLin Ling buf->b_next = NULL; 1544ea8dc4b6Seschrock 1545ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 1546ea8dc4b6Seschrock 1547ea8dc4b6Seschrock /* clean up the buf */ 1548ea8dc4b6Seschrock buf->b_hdr = NULL; 1549ea8dc4b6Seschrock kmem_cache_free(buf_cache, buf); 1550ea8dc4b6Seschrock } 1551ea8dc4b6Seschrock 1552fa9e4066Sahrens static void 1553ea8dc4b6Seschrock arc_hdr_destroy(arc_buf_hdr_t *hdr) 1554fa9e4066Sahrens { 1555fa9e4066Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt)); 155644cb6abcSbmc ASSERT3P(hdr->b_state, ==, arc_anon); 1557ea8dc4b6Seschrock ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 1558b24ab676SJeff Bonwick l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr; 1559fa9e4066Sahrens 1560b24ab676SJeff Bonwick if (l2hdr != NULL) { 1561b24ab676SJeff Bonwick boolean_t buflist_held = MUTEX_HELD(&l2arc_buflist_mtx); 1562b24ab676SJeff Bonwick /* 1563b24ab676SJeff Bonwick * To prevent arc_free() and l2arc_evict() from 1564b24ab676SJeff Bonwick * attempting to free the same buffer at the same time, 1565b24ab676SJeff Bonwick * a FREE_IN_PROGRESS flag is given to arc_free() to 1566b24ab676SJeff Bonwick * give it priority. l2arc_evict() can't destroy this 1567b24ab676SJeff Bonwick * header while we are waiting on l2arc_buflist_mtx. 1568b24ab676SJeff Bonwick * 1569b24ab676SJeff Bonwick * The hdr may be removed from l2ad_buflist before we 1570b24ab676SJeff Bonwick * grab l2arc_buflist_mtx, so b_l2hdr is rechecked. 1571b24ab676SJeff Bonwick */ 1572b24ab676SJeff Bonwick if (!buflist_held) { 1573fa94a07fSbrendan mutex_enter(&l2arc_buflist_mtx); 1574b24ab676SJeff Bonwick l2hdr = hdr->b_l2hdr; 1575fa94a07fSbrendan } 1576b24ab676SJeff Bonwick 1577b24ab676SJeff Bonwick if (l2hdr != NULL) { 1578b24ab676SJeff Bonwick list_remove(l2hdr->b_dev->l2ad_buflist, hdr); 1579b24ab676SJeff Bonwick ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size); 1580aad02571SSaso Kiselkov ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize); 1581b24ab676SJeff Bonwick kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t)); 1582b24ab676SJeff Bonwick if (hdr->b_state == arc_l2c_only) 1583b24ab676SJeff Bonwick l2arc_hdr_stat_remove(); 1584b24ab676SJeff Bonwick hdr->b_l2hdr = NULL; 1585b24ab676SJeff Bonwick } 1586b24ab676SJeff Bonwick 1587b24ab676SJeff Bonwick if (!buflist_held) 1588b24ab676SJeff Bonwick mutex_exit(&l2arc_buflist_mtx); 1589fa94a07fSbrendan } 1590fa94a07fSbrendan 1591fa9e4066Sahrens if (!BUF_EMPTY(hdr)) { 1592ea8dc4b6Seschrock ASSERT(!HDR_IN_HASH_TABLE(hdr)); 15933f9d6ad7SLin Ling buf_discard_identity(hdr); 1594fa9e4066Sahrens } 1595ea8dc4b6Seschrock while (hdr->b_buf) { 1596fa9e4066Sahrens arc_buf_t *buf = hdr->b_buf; 1597fa9e4066Sahrens 1598ea8dc4b6Seschrock if (buf->b_efunc) { 1599ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 16003f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 1601ea8dc4b6Seschrock ASSERT(buf->b_hdr != NULL); 160244eda4d7Smaybee arc_buf_destroy(hdr->b_buf, FALSE, FALSE); 1603ea8dc4b6Seschrock hdr->b_buf = buf->b_next; 160440d7d650Smaybee buf->b_hdr = &arc_eviction_hdr; 1605ea8dc4b6Seschrock buf->b_next = arc_eviction_list; 1606ea8dc4b6Seschrock arc_eviction_list = buf; 16073f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 1608ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1609ea8dc4b6Seschrock } else { 161044eda4d7Smaybee arc_buf_destroy(hdr->b_buf, FALSE, TRUE); 1611ea8dc4b6Seschrock } 1612fa9e4066Sahrens } 16136b4acc8bSahrens if (hdr->b_freeze_cksum != NULL) { 16146b4acc8bSahrens kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 16156b4acc8bSahrens hdr->b_freeze_cksum = NULL; 16166b4acc8bSahrens } 16173f9d6ad7SLin Ling if (hdr->b_thawed) { 16183f9d6ad7SLin Ling kmem_free(hdr->b_thawed, 1); 16193f9d6ad7SLin Ling hdr->b_thawed = NULL; 16203f9d6ad7SLin Ling } 1621ea8dc4b6Seschrock 1622fa9e4066Sahrens ASSERT(!list_link_active(&hdr->b_arc_node)); 1623fa9e4066Sahrens ASSERT3P(hdr->b_hash_next, ==, NULL); 1624fa9e4066Sahrens ASSERT3P(hdr->b_acb, ==, NULL); 1625fa9e4066Sahrens kmem_cache_free(hdr_cache, hdr); 1626fa9e4066Sahrens } 1627fa9e4066Sahrens 1628fa9e4066Sahrens void 1629fa9e4066Sahrens arc_buf_free(arc_buf_t *buf, void *tag) 1630fa9e4066Sahrens { 1631fa9e4066Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 163244cb6abcSbmc int hashed = hdr->b_state != arc_anon; 1633fa9e4066Sahrens 1634ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 1635ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 1636ea8dc4b6Seschrock 1637ea8dc4b6Seschrock if (hashed) { 1638ea8dc4b6Seschrock kmutex_t *hash_lock = HDR_LOCK(hdr); 1639ea8dc4b6Seschrock 1640ea8dc4b6Seschrock mutex_enter(hash_lock); 16413f9d6ad7SLin Ling hdr = buf->b_hdr; 16423f9d6ad7SLin Ling ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 16433f9d6ad7SLin Ling 1644ea8dc4b6Seschrock (void) remove_reference(hdr, hash_lock, tag); 1645b24ab676SJeff Bonwick if (hdr->b_datacnt > 1) { 164644eda4d7Smaybee arc_buf_destroy(buf, FALSE, TRUE); 1647b24ab676SJeff Bonwick } else { 1648b24ab676SJeff Bonwick ASSERT(buf == hdr->b_buf); 1649b24ab676SJeff Bonwick ASSERT(buf->b_efunc == NULL); 1650ea8dc4b6Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 1651b24ab676SJeff Bonwick } 1652fa9e4066Sahrens mutex_exit(hash_lock); 1653ea8dc4b6Seschrock } else if (HDR_IO_IN_PROGRESS(hdr)) { 1654ea8dc4b6Seschrock int destroy_hdr; 1655ea8dc4b6Seschrock /* 1656ea8dc4b6Seschrock * We are in the middle of an async write. Don't destroy 1657ea8dc4b6Seschrock * this buffer unless the write completes before we finish 1658ea8dc4b6Seschrock * decrementing the reference count. 1659ea8dc4b6Seschrock */ 1660ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 1661ea8dc4b6Seschrock (void) remove_reference(hdr, NULL, tag); 1662ea8dc4b6Seschrock ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1663ea8dc4b6Seschrock destroy_hdr = !HDR_IO_IN_PROGRESS(hdr); 1664ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 1665ea8dc4b6Seschrock if (destroy_hdr) 1666ea8dc4b6Seschrock arc_hdr_destroy(hdr); 1667ea8dc4b6Seschrock } else { 16683f9d6ad7SLin Ling if (remove_reference(hdr, NULL, tag) > 0) 166944eda4d7Smaybee arc_buf_destroy(buf, FALSE, TRUE); 16703f9d6ad7SLin Ling else 1671ea8dc4b6Seschrock arc_hdr_destroy(hdr); 1672fa9e4066Sahrens } 1673ea8dc4b6Seschrock } 1674fa9e4066Sahrens 16753b2aab18SMatthew Ahrens boolean_t 1676ea8dc4b6Seschrock arc_buf_remove_ref(arc_buf_t *buf, void* tag) 1677ea8dc4b6Seschrock { 1678ea8dc4b6Seschrock arc_buf_hdr_t *hdr = buf->b_hdr; 1679ea8dc4b6Seschrock kmutex_t *hash_lock = HDR_LOCK(hdr); 16803b2aab18SMatthew Ahrens boolean_t no_callback = (buf->b_efunc == NULL); 1681fa9e4066Sahrens 168244cb6abcSbmc if (hdr->b_state == arc_anon) { 1683b24ab676SJeff Bonwick ASSERT(hdr->b_datacnt == 1); 1684ea8dc4b6Seschrock arc_buf_free(buf, tag); 1685ea8dc4b6Seschrock return (no_callback); 1686ea8dc4b6Seschrock } 1687ea8dc4b6Seschrock 1688ea8dc4b6Seschrock mutex_enter(hash_lock); 16893f9d6ad7SLin Ling hdr = buf->b_hdr; 16903f9d6ad7SLin Ling ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 169144cb6abcSbmc ASSERT(hdr->b_state != arc_anon); 1692ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 1693ea8dc4b6Seschrock 1694ea8dc4b6Seschrock (void) remove_reference(hdr, hash_lock, tag); 1695ea8dc4b6Seschrock if (hdr->b_datacnt > 1) { 1696ea8dc4b6Seschrock if (no_callback) 169744eda4d7Smaybee arc_buf_destroy(buf, FALSE, TRUE); 1698ea8dc4b6Seschrock } else if (no_callback) { 1699ea8dc4b6Seschrock ASSERT(hdr->b_buf == buf && buf->b_next == NULL); 1700b24ab676SJeff Bonwick ASSERT(buf->b_efunc == NULL); 1701ea8dc4b6Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 1702ea8dc4b6Seschrock } 1703ea8dc4b6Seschrock ASSERT(no_callback || hdr->b_datacnt > 1 || 1704ea8dc4b6Seschrock refcount_is_zero(&hdr->b_refcnt)); 1705ea8dc4b6Seschrock mutex_exit(hash_lock); 1706ea8dc4b6Seschrock return (no_callback); 1707fa9e4066Sahrens } 1708fa9e4066Sahrens 1709fa9e4066Sahrens int 1710fa9e4066Sahrens arc_buf_size(arc_buf_t *buf) 1711fa9e4066Sahrens { 1712fa9e4066Sahrens return (buf->b_hdr->b_size); 1713fa9e4066Sahrens } 1714fa9e4066Sahrens 17159253d63dSGeorge Wilson /* 17169253d63dSGeorge Wilson * Called from the DMU to determine if the current buffer should be 17179253d63dSGeorge Wilson * evicted. In order to ensure proper locking, the eviction must be initiated 17189253d63dSGeorge Wilson * from the DMU. Return true if the buffer is associated with user data and 17199253d63dSGeorge Wilson * duplicate buffers still exist. 17209253d63dSGeorge Wilson */ 17219253d63dSGeorge Wilson boolean_t 17229253d63dSGeorge Wilson arc_buf_eviction_needed(arc_buf_t *buf) 17239253d63dSGeorge Wilson { 17249253d63dSGeorge Wilson arc_buf_hdr_t *hdr; 17259253d63dSGeorge Wilson boolean_t evict_needed = B_FALSE; 17269253d63dSGeorge Wilson 17279253d63dSGeorge Wilson if (zfs_disable_dup_eviction) 17289253d63dSGeorge Wilson return (B_FALSE); 17299253d63dSGeorge Wilson 17309253d63dSGeorge Wilson mutex_enter(&buf->b_evict_lock); 17319253d63dSGeorge Wilson hdr = buf->b_hdr; 17329253d63dSGeorge Wilson if (hdr == NULL) { 17339253d63dSGeorge Wilson /* 17349253d63dSGeorge Wilson * We are in arc_do_user_evicts(); let that function 17359253d63dSGeorge Wilson * perform the eviction. 17369253d63dSGeorge Wilson */ 17379253d63dSGeorge Wilson ASSERT(buf->b_data == NULL); 17389253d63dSGeorge Wilson mutex_exit(&buf->b_evict_lock); 17399253d63dSGeorge Wilson return (B_FALSE); 17409253d63dSGeorge Wilson } else if (buf->b_data == NULL) { 17419253d63dSGeorge Wilson /* 17429253d63dSGeorge Wilson * We have already been added to the arc eviction list; 17439253d63dSGeorge Wilson * recommend eviction. 17449253d63dSGeorge Wilson */ 17459253d63dSGeorge Wilson ASSERT3P(hdr, ==, &arc_eviction_hdr); 17469253d63dSGeorge Wilson mutex_exit(&buf->b_evict_lock); 17479253d63dSGeorge Wilson return (B_TRUE); 17489253d63dSGeorge Wilson } 17499253d63dSGeorge Wilson 17509253d63dSGeorge Wilson if (hdr->b_datacnt > 1 && hdr->b_type == ARC_BUFC_DATA) 17519253d63dSGeorge Wilson evict_needed = B_TRUE; 17529253d63dSGeorge Wilson 17539253d63dSGeorge Wilson mutex_exit(&buf->b_evict_lock); 17549253d63dSGeorge Wilson return (evict_needed); 17559253d63dSGeorge Wilson } 17569253d63dSGeorge Wilson 1757fa9e4066Sahrens /* 1758fa9e4066Sahrens * Evict buffers from list until we've removed the specified number of 1759fa9e4066Sahrens * bytes. Move the removed buffers to the appropriate evict state. 176044eda4d7Smaybee * If the recycle flag is set, then attempt to "recycle" a buffer: 176144eda4d7Smaybee * - look for a buffer to evict that is `bytes' long. 176244eda4d7Smaybee * - return the data block from this buffer rather than freeing it. 176344eda4d7Smaybee * This flag is used by callers that are trying to make space for a 176444eda4d7Smaybee * new buffer in a full arc cache. 1765874395d5Smaybee * 1766874395d5Smaybee * This function makes a "best effort". It skips over any buffers 1767874395d5Smaybee * it can't get a hash_lock on, and so may not catch all candidates. 1768874395d5Smaybee * It may also return without evicting as much space as requested. 1769fa9e4066Sahrens */ 177044eda4d7Smaybee static void * 1771ac05c741SMark Maybee arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle, 1772ad23a2dbSjohansen arc_buf_contents_t type) 1773fa9e4066Sahrens { 1774fa9e4066Sahrens arc_state_t *evicted_state; 177544eda4d7Smaybee uint64_t bytes_evicted = 0, skipped = 0, missed = 0; 17763fa51506Smaybee arc_buf_hdr_t *ab, *ab_prev = NULL; 17770e8c6158Smaybee list_t *list = &state->arcs_list[type]; 1778fa9e4066Sahrens kmutex_t *hash_lock; 177944eda4d7Smaybee boolean_t have_lock; 17803fa51506Smaybee void *stolen = NULL; 1781fa9e4066Sahrens 178244cb6abcSbmc ASSERT(state == arc_mru || state == arc_mfu); 1783fa9e4066Sahrens 178444cb6abcSbmc evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 1785fa9e4066Sahrens 178644cb6abcSbmc mutex_enter(&state->arcs_mtx); 178744cb6abcSbmc mutex_enter(&evicted_state->arcs_mtx); 1788fa9e4066Sahrens 17890e8c6158Smaybee for (ab = list_tail(list); ab; ab = ab_prev) { 17900e8c6158Smaybee ab_prev = list_prev(list, ab); 179113506d1eSmaybee /* prefetch buffers have a minimum lifespan */ 179244eda4d7Smaybee if (HDR_IO_IN_PROGRESS(ab) || 1793874395d5Smaybee (spa && ab->b_spa != spa) || 179444eda4d7Smaybee (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) && 1795d3d50737SRafael Vanoni ddi_get_lbolt() - ab->b_arc_access < 1796d3d50737SRafael Vanoni arc_min_prefetch_lifespan)) { 179713506d1eSmaybee skipped++; 179813506d1eSmaybee continue; 179913506d1eSmaybee } 18003fa51506Smaybee /* "lookahead" for better eviction candidate */ 18013fa51506Smaybee if (recycle && ab->b_size != bytes && 18023fa51506Smaybee ab_prev && ab_prev->b_size == bytes) 180344eda4d7Smaybee continue; 1804fa9e4066Sahrens hash_lock = HDR_LOCK(ab); 180544eda4d7Smaybee have_lock = MUTEX_HELD(hash_lock); 180644eda4d7Smaybee if (have_lock || mutex_tryenter(hash_lock)) { 1807fb09f5aaSMadhav Suresh ASSERT0(refcount_count(&ab->b_refcnt)); 1808ea8dc4b6Seschrock ASSERT(ab->b_datacnt > 0); 1809ea8dc4b6Seschrock while (ab->b_buf) { 1810ea8dc4b6Seschrock arc_buf_t *buf = ab->b_buf; 18113f9d6ad7SLin Ling if (!mutex_tryenter(&buf->b_evict_lock)) { 18126f83844dSMark Maybee missed += 1; 18136f83844dSMark Maybee break; 18146f83844dSMark Maybee } 181544eda4d7Smaybee if (buf->b_data) { 1816ea8dc4b6Seschrock bytes_evicted += ab->b_size; 1817ad23a2dbSjohansen if (recycle && ab->b_type == type && 1818fa94a07fSbrendan ab->b_size == bytes && 1819fa94a07fSbrendan !HDR_L2_WRITING(ab)) { 18203fa51506Smaybee stolen = buf->b_data; 18213fa51506Smaybee recycle = FALSE; 18223fa51506Smaybee } 182344eda4d7Smaybee } 1824ea8dc4b6Seschrock if (buf->b_efunc) { 1825ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 18263fa51506Smaybee arc_buf_destroy(buf, 18273fa51506Smaybee buf->b_data == stolen, FALSE); 1828ea8dc4b6Seschrock ab->b_buf = buf->b_next; 182940d7d650Smaybee buf->b_hdr = &arc_eviction_hdr; 1830ea8dc4b6Seschrock buf->b_next = arc_eviction_list; 1831ea8dc4b6Seschrock arc_eviction_list = buf; 1832ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 18333f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 1834ea8dc4b6Seschrock } else { 18353f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 18363fa51506Smaybee arc_buf_destroy(buf, 18373fa51506Smaybee buf->b_data == stolen, TRUE); 1838ea8dc4b6Seschrock } 1839ea8dc4b6Seschrock } 18405ea40c06SBrendan Gregg - Sun Microsystems 18415ea40c06SBrendan Gregg - Sun Microsystems if (ab->b_l2hdr) { 18425ea40c06SBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_evict_l2_cached, 18435ea40c06SBrendan Gregg - Sun Microsystems ab->b_size); 18445ea40c06SBrendan Gregg - Sun Microsystems } else { 18455ea40c06SBrendan Gregg - Sun Microsystems if (l2arc_write_eligible(ab->b_spa, ab)) { 18465ea40c06SBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_evict_l2_eligible, 18475ea40c06SBrendan Gregg - Sun Microsystems ab->b_size); 18485ea40c06SBrendan Gregg - Sun Microsystems } else { 18495ea40c06SBrendan Gregg - Sun Microsystems ARCSTAT_INCR( 18505ea40c06SBrendan Gregg - Sun Microsystems arcstat_evict_l2_ineligible, 18515ea40c06SBrendan Gregg - Sun Microsystems ab->b_size); 18525ea40c06SBrendan Gregg - Sun Microsystems } 18535ea40c06SBrendan Gregg - Sun Microsystems } 18545ea40c06SBrendan Gregg - Sun Microsystems 18556f83844dSMark Maybee if (ab->b_datacnt == 0) { 18566f83844dSMark Maybee arc_change_state(evicted_state, ab, hash_lock); 18576f83844dSMark Maybee ASSERT(HDR_IN_HASH_TABLE(ab)); 18586f83844dSMark Maybee ab->b_flags |= ARC_IN_HASH_TABLE; 18596f83844dSMark Maybee ab->b_flags &= ~ARC_BUF_AVAILABLE; 18606f83844dSMark Maybee DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab); 18616f83844dSMark Maybee } 186244eda4d7Smaybee if (!have_lock) 186344eda4d7Smaybee mutex_exit(hash_lock); 1864ea8dc4b6Seschrock if (bytes >= 0 && bytes_evicted >= bytes) 1865fa9e4066Sahrens break; 1866fa9e4066Sahrens } else { 186744eda4d7Smaybee missed += 1; 1868fa9e4066Sahrens } 1869fa9e4066Sahrens } 187044cb6abcSbmc 187144cb6abcSbmc mutex_exit(&evicted_state->arcs_mtx); 187244cb6abcSbmc mutex_exit(&state->arcs_mtx); 1873fa9e4066Sahrens 1874fa9e4066Sahrens if (bytes_evicted < bytes) 1875fa9e4066Sahrens dprintf("only evicted %lld bytes from %x", 1876fa9e4066Sahrens (longlong_t)bytes_evicted, state); 1877fa9e4066Sahrens 187844eda4d7Smaybee if (skipped) 187944cb6abcSbmc ARCSTAT_INCR(arcstat_evict_skip, skipped); 188044cb6abcSbmc 188144eda4d7Smaybee if (missed) 188244cb6abcSbmc ARCSTAT_INCR(arcstat_mutex_miss, missed); 1883f4d2e9e6Smaybee 1884f4d2e9e6Smaybee /* 18853b2aab18SMatthew Ahrens * We have just evicted some data into the ghost state, make 1886f4d2e9e6Smaybee * sure we also adjust the ghost state size if necessary. 1887f4d2e9e6Smaybee */ 1888f4d2e9e6Smaybee if (arc_no_grow && 1889f4d2e9e6Smaybee arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size > arc_c) { 1890f4d2e9e6Smaybee int64_t mru_over = arc_anon->arcs_size + arc_mru->arcs_size + 1891f4d2e9e6Smaybee arc_mru_ghost->arcs_size - arc_c; 1892f4d2e9e6Smaybee 1893f4d2e9e6Smaybee if (mru_over > 0 && arc_mru_ghost->arcs_lsize[type] > 0) { 1894f4d2e9e6Smaybee int64_t todelete = 1895f4d2e9e6Smaybee MIN(arc_mru_ghost->arcs_lsize[type], mru_over); 1896874395d5Smaybee arc_evict_ghost(arc_mru_ghost, NULL, todelete); 1897f4d2e9e6Smaybee } else if (arc_mfu_ghost->arcs_lsize[type] > 0) { 1898f4d2e9e6Smaybee int64_t todelete = MIN(arc_mfu_ghost->arcs_lsize[type], 1899f4d2e9e6Smaybee arc_mru_ghost->arcs_size + 1900f4d2e9e6Smaybee arc_mfu_ghost->arcs_size - arc_c); 1901874395d5Smaybee arc_evict_ghost(arc_mfu_ghost, NULL, todelete); 1902f4d2e9e6Smaybee } 1903f4d2e9e6Smaybee } 190444cb6abcSbmc 19053fa51506Smaybee return (stolen); 1906fa9e4066Sahrens } 1907fa9e4066Sahrens 1908fa9e4066Sahrens /* 1909fa9e4066Sahrens * Remove buffers from list until we've removed the specified number of 1910fa9e4066Sahrens * bytes. Destroy the buffers that are removed. 1911fa9e4066Sahrens */ 1912fa9e4066Sahrens static void 1913ac05c741SMark Maybee arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes) 1914fa9e4066Sahrens { 1915fa9e4066Sahrens arc_buf_hdr_t *ab, *ab_prev; 1916b802aa8cSSanjeev Bagewadi arc_buf_hdr_t marker = { 0 }; 19170e8c6158Smaybee list_t *list = &state->arcs_list[ARC_BUFC_DATA]; 1918fa9e4066Sahrens kmutex_t *hash_lock; 1919ea8dc4b6Seschrock uint64_t bytes_deleted = 0; 1920c0a81264Sek uint64_t bufs_skipped = 0; 1921fa9e4066Sahrens 1922ea8dc4b6Seschrock ASSERT(GHOST_STATE(state)); 1923fa9e4066Sahrens top: 192444cb6abcSbmc mutex_enter(&state->arcs_mtx); 19250e8c6158Smaybee for (ab = list_tail(list); ab; ab = ab_prev) { 19260e8c6158Smaybee ab_prev = list_prev(list, ab); 1927874395d5Smaybee if (spa && ab->b_spa != spa) 1928874395d5Smaybee continue; 1929b802aa8cSSanjeev Bagewadi 1930b802aa8cSSanjeev Bagewadi /* ignore markers */ 1931b802aa8cSSanjeev Bagewadi if (ab->b_spa == 0) 1932b802aa8cSSanjeev Bagewadi continue; 1933b802aa8cSSanjeev Bagewadi 1934fa9e4066Sahrens hash_lock = HDR_LOCK(ab); 19357e453561SWilliam Gorrell /* caller may be trying to modify this buffer, skip it */ 19367e453561SWilliam Gorrell if (MUTEX_HELD(hash_lock)) 19377e453561SWilliam Gorrell continue; 19387e453561SWilliam Gorrell if (mutex_tryenter(hash_lock)) { 193913506d1eSmaybee ASSERT(!HDR_IO_IN_PROGRESS(ab)); 1940ea8dc4b6Seschrock ASSERT(ab->b_buf == NULL); 194144cb6abcSbmc ARCSTAT_BUMP(arcstat_deleted); 1942fa9e4066Sahrens bytes_deleted += ab->b_size; 1943fa94a07fSbrendan 1944fa94a07fSbrendan if (ab->b_l2hdr != NULL) { 1945fa94a07fSbrendan /* 1946fa94a07fSbrendan * This buffer is cached on the 2nd Level ARC; 1947fa94a07fSbrendan * don't destroy the header. 1948fa94a07fSbrendan */ 1949fa94a07fSbrendan arc_change_state(arc_l2c_only, ab, hash_lock); 19507e453561SWilliam Gorrell mutex_exit(hash_lock); 1951fa94a07fSbrendan } else { 1952fa94a07fSbrendan arc_change_state(arc_anon, ab, hash_lock); 19537e453561SWilliam Gorrell mutex_exit(hash_lock); 1954fa94a07fSbrendan arc_hdr_destroy(ab); 1955fa94a07fSbrendan } 1956fa94a07fSbrendan 1957ea8dc4b6Seschrock DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab); 1958fa9e4066Sahrens if (bytes >= 0 && bytes_deleted >= bytes) 1959fa9e4066Sahrens break; 1960b802aa8cSSanjeev Bagewadi } else if (bytes < 0) { 1961b802aa8cSSanjeev Bagewadi /* 1962b802aa8cSSanjeev Bagewadi * Insert a list marker and then wait for the 1963b802aa8cSSanjeev Bagewadi * hash lock to become available. Once its 1964b802aa8cSSanjeev Bagewadi * available, restart from where we left off. 1965b802aa8cSSanjeev Bagewadi */ 1966b802aa8cSSanjeev Bagewadi list_insert_after(list, ab, &marker); 1967b802aa8cSSanjeev Bagewadi mutex_exit(&state->arcs_mtx); 1968b802aa8cSSanjeev Bagewadi mutex_enter(hash_lock); 1969b802aa8cSSanjeev Bagewadi mutex_exit(hash_lock); 1970b802aa8cSSanjeev Bagewadi mutex_enter(&state->arcs_mtx); 1971b802aa8cSSanjeev Bagewadi ab_prev = list_prev(list, &marker); 1972b802aa8cSSanjeev Bagewadi list_remove(list, &marker); 1973b802aa8cSSanjeev Bagewadi } else 1974fa9e4066Sahrens bufs_skipped += 1; 1975fa9e4066Sahrens } 197644cb6abcSbmc mutex_exit(&state->arcs_mtx); 1977fa9e4066Sahrens 19780e8c6158Smaybee if (list == &state->arcs_list[ARC_BUFC_DATA] && 19790e8c6158Smaybee (bytes < 0 || bytes_deleted < bytes)) { 19800e8c6158Smaybee list = &state->arcs_list[ARC_BUFC_METADATA]; 19810e8c6158Smaybee goto top; 19820e8c6158Smaybee } 19830e8c6158Smaybee 1984fa9e4066Sahrens if (bufs_skipped) { 198544cb6abcSbmc ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped); 1986fa9e4066Sahrens ASSERT(bytes >= 0); 1987fa9e4066Sahrens } 1988fa9e4066Sahrens 1989fa9e4066Sahrens if (bytes_deleted < bytes) 1990fa9e4066Sahrens dprintf("only deleted %lld bytes from %p", 1991fa9e4066Sahrens (longlong_t)bytes_deleted, state); 1992fa9e4066Sahrens } 1993fa9e4066Sahrens 1994fa9e4066Sahrens static void 1995fa9e4066Sahrens arc_adjust(void) 1996fa9e4066Sahrens { 19975a98e54bSBrendan Gregg - Sun Microsystems int64_t adjustment, delta; 1998fa9e4066Sahrens 19995a98e54bSBrendan Gregg - Sun Microsystems /* 20005a98e54bSBrendan Gregg - Sun Microsystems * Adjust MRU size 20015a98e54bSBrendan Gregg - Sun Microsystems */ 20025a98e54bSBrendan Gregg - Sun Microsystems 20033e4e8481STom Erickson adjustment = MIN((int64_t)(arc_size - arc_c), 20043e4e8481STom Erickson (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used - 20053e4e8481STom Erickson arc_p)); 2006fa9e4066Sahrens 20075a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) { 20085a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment); 20095a98e54bSBrendan Gregg - Sun Microsystems (void) arc_evict(arc_mru, NULL, delta, FALSE, ARC_BUFC_DATA); 20105a98e54bSBrendan Gregg - Sun Microsystems adjustment -= delta; 20110e8c6158Smaybee } 20120e8c6158Smaybee 20135a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) { 20145a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment); 20155a98e54bSBrendan Gregg - Sun Microsystems (void) arc_evict(arc_mru, NULL, delta, FALSE, 2016874395d5Smaybee ARC_BUFC_METADATA); 2017fa9e4066Sahrens } 2018fa9e4066Sahrens 20195a98e54bSBrendan Gregg - Sun Microsystems /* 20205a98e54bSBrendan Gregg - Sun Microsystems * Adjust MFU size 20215a98e54bSBrendan Gregg - Sun Microsystems */ 2022fa9e4066Sahrens 20235a98e54bSBrendan Gregg - Sun Microsystems adjustment = arc_size - arc_c; 20245a98e54bSBrendan Gregg - Sun Microsystems 20255a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) { 20265a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]); 20275a98e54bSBrendan Gregg - Sun Microsystems (void) arc_evict(arc_mfu, NULL, delta, FALSE, ARC_BUFC_DATA); 20285a98e54bSBrendan Gregg - Sun Microsystems adjustment -= delta; 2029fa9e4066Sahrens } 2030fa9e4066Sahrens 20315a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) { 20325a98e54bSBrendan Gregg - Sun Microsystems int64_t delta = MIN(adjustment, 20335a98e54bSBrendan Gregg - Sun Microsystems arc_mfu->arcs_lsize[ARC_BUFC_METADATA]); 20345a98e54bSBrendan Gregg - Sun Microsystems (void) arc_evict(arc_mfu, NULL, delta, FALSE, 20355a98e54bSBrendan Gregg - Sun Microsystems ARC_BUFC_METADATA); 20365a98e54bSBrendan Gregg - Sun Microsystems } 2037fa9e4066Sahrens 20385a98e54bSBrendan Gregg - Sun Microsystems /* 20395a98e54bSBrendan Gregg - Sun Microsystems * Adjust ghost lists 20405a98e54bSBrendan Gregg - Sun Microsystems */ 2041fa9e4066Sahrens 20425a98e54bSBrendan Gregg - Sun Microsystems adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c; 2043fa9e4066Sahrens 20445a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) { 20455a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(arc_mru_ghost->arcs_size, adjustment); 20465a98e54bSBrendan Gregg - Sun Microsystems arc_evict_ghost(arc_mru_ghost, NULL, delta); 20475a98e54bSBrendan Gregg - Sun Microsystems } 20480e8c6158Smaybee 20495a98e54bSBrendan Gregg - Sun Microsystems adjustment = 20505a98e54bSBrendan Gregg - Sun Microsystems arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c; 20515a98e54bSBrendan Gregg - Sun Microsystems 20525a98e54bSBrendan Gregg - Sun Microsystems if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) { 20535a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(arc_mfu_ghost->arcs_size, adjustment); 20545a98e54bSBrendan Gregg - Sun Microsystems arc_evict_ghost(arc_mfu_ghost, NULL, delta); 2055fa9e4066Sahrens } 2056fa9e4066Sahrens } 2057fa9e4066Sahrens 2058ea8dc4b6Seschrock static void 2059ea8dc4b6Seschrock arc_do_user_evicts(void) 2060ea8dc4b6Seschrock { 2061ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 2062ea8dc4b6Seschrock while (arc_eviction_list != NULL) { 2063ea8dc4b6Seschrock arc_buf_t *buf = arc_eviction_list; 2064ea8dc4b6Seschrock arc_eviction_list = buf->b_next; 20653f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 2066ea8dc4b6Seschrock buf->b_hdr = NULL; 20673f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 2068ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 2069ea8dc4b6Seschrock 2070dd6ef538Smaybee if (buf->b_efunc != NULL) 2071dd6ef538Smaybee VERIFY(buf->b_efunc(buf) == 0); 2072ea8dc4b6Seschrock 2073ea8dc4b6Seschrock buf->b_efunc = NULL; 2074ea8dc4b6Seschrock buf->b_private = NULL; 2075ea8dc4b6Seschrock kmem_cache_free(buf_cache, buf); 2076ea8dc4b6Seschrock mutex_enter(&arc_eviction_mtx); 2077ea8dc4b6Seschrock } 2078ea8dc4b6Seschrock mutex_exit(&arc_eviction_mtx); 2079ea8dc4b6Seschrock } 2080ea8dc4b6Seschrock 2081fa9e4066Sahrens /* 2082874395d5Smaybee * Flush all *evictable* data from the cache for the given spa. 2083fa9e4066Sahrens * NOTE: this will not touch "active" (i.e. referenced) data. 2084fa9e4066Sahrens */ 2085fa9e4066Sahrens void 2086874395d5Smaybee arc_flush(spa_t *spa) 2087fa9e4066Sahrens { 2088ac05c741SMark Maybee uint64_t guid = 0; 2089ac05c741SMark Maybee 2090ac05c741SMark Maybee if (spa) 2091e9103aaeSGarrett D'Amore guid = spa_load_guid(spa); 2092ac05c741SMark Maybee 2093874395d5Smaybee while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) { 2094ac05c741SMark Maybee (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA); 2095874395d5Smaybee if (spa) 2096874395d5Smaybee break; 2097874395d5Smaybee } 2098874395d5Smaybee while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) { 2099ac05c741SMark Maybee (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA); 2100874395d5Smaybee if (spa) 2101874395d5Smaybee break; 2102874395d5Smaybee } 2103874395d5Smaybee while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) { 2104ac05c741SMark Maybee (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA); 2105874395d5Smaybee if (spa) 2106874395d5Smaybee break; 2107874395d5Smaybee } 2108874395d5Smaybee while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) { 2109ac05c741SMark Maybee (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA); 2110874395d5Smaybee if (spa) 2111874395d5Smaybee break; 2112874395d5Smaybee } 2113874395d5Smaybee 2114ac05c741SMark Maybee arc_evict_ghost(arc_mru_ghost, guid, -1); 2115ac05c741SMark Maybee arc_evict_ghost(arc_mfu_ghost, guid, -1); 2116ea8dc4b6Seschrock 2117ea8dc4b6Seschrock mutex_enter(&arc_reclaim_thr_lock); 2118ea8dc4b6Seschrock arc_do_user_evicts(); 2119ea8dc4b6Seschrock mutex_exit(&arc_reclaim_thr_lock); 2120874395d5Smaybee ASSERT(spa || arc_eviction_list == NULL); 2121fa9e4066Sahrens } 2122fa9e4066Sahrens 2123fa9e4066Sahrens void 212449e3519aSmaybee arc_shrink(void) 2125fa9e4066Sahrens { 212644cb6abcSbmc if (arc_c > arc_c_min) { 212749e3519aSmaybee uint64_t to_free; 2128fa9e4066Sahrens 21293cff2f43Sstans #ifdef _KERNEL 213044cb6abcSbmc to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree)); 21313cff2f43Sstans #else 213244cb6abcSbmc to_free = arc_c >> arc_shrink_shift; 21333cff2f43Sstans #endif 213444cb6abcSbmc if (arc_c > arc_c_min + to_free) 213544cb6abcSbmc atomic_add_64(&arc_c, -to_free); 213649e3519aSmaybee else 213744cb6abcSbmc arc_c = arc_c_min; 213844cb6abcSbmc 213944cb6abcSbmc atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift)); 214044cb6abcSbmc if (arc_c > arc_size) 214144cb6abcSbmc arc_c = MAX(arc_size, arc_c_min); 214244cb6abcSbmc if (arc_p > arc_c) 214344cb6abcSbmc arc_p = (arc_c >> 1); 214444cb6abcSbmc ASSERT(arc_c >= arc_c_min); 214544cb6abcSbmc ASSERT((int64_t)arc_p >= 0); 214649e3519aSmaybee } 2147fa9e4066Sahrens 214844cb6abcSbmc if (arc_size > arc_c) 214949e3519aSmaybee arc_adjust(); 2150fa9e4066Sahrens } 2151fa9e4066Sahrens 215294dd93aeSGeorge Wilson /* 215394dd93aeSGeorge Wilson * Determine if the system is under memory pressure and is asking 215494dd93aeSGeorge Wilson * to reclaim memory. A return value of 1 indicates that the system 215594dd93aeSGeorge Wilson * is under memory pressure and that the arc should adjust accordingly. 215694dd93aeSGeorge Wilson */ 2157fa9e4066Sahrens static int 2158fa9e4066Sahrens arc_reclaim_needed(void) 2159fa9e4066Sahrens { 2160fa9e4066Sahrens uint64_t extra; 2161fa9e4066Sahrens 2162fa9e4066Sahrens #ifdef _KERNEL 21633cff2f43Sstans 21643cff2f43Sstans if (needfree) 21653cff2f43Sstans return (1); 21663cff2f43Sstans 2167fa9e4066Sahrens /* 2168fa9e4066Sahrens * take 'desfree' extra pages, so we reclaim sooner, rather than later 2169fa9e4066Sahrens */ 2170fa9e4066Sahrens extra = desfree; 2171fa9e4066Sahrens 2172fa9e4066Sahrens /* 2173fa9e4066Sahrens * check that we're out of range of the pageout scanner. It starts to 2174fa9e4066Sahrens * schedule paging if freemem is less than lotsfree and needfree. 2175fa9e4066Sahrens * lotsfree is the high-water mark for pageout, and needfree is the 2176fa9e4066Sahrens * number of needed free pages. We add extra pages here to make sure 2177fa9e4066Sahrens * the scanner doesn't start up while we're freeing memory. 2178fa9e4066Sahrens */ 2179fa9e4066Sahrens if (freemem < lotsfree + needfree + extra) 2180fa9e4066Sahrens return (1); 2181fa9e4066Sahrens 2182fa9e4066Sahrens /* 2183fa9e4066Sahrens * check to make sure that swapfs has enough space so that anon 2184fa94a07fSbrendan * reservations can still succeed. anon_resvmem() checks that the 2185fa9e4066Sahrens * availrmem is greater than swapfs_minfree, and the number of reserved 2186fa9e4066Sahrens * swap pages. We also add a bit of extra here just to prevent 2187fa9e4066Sahrens * circumstances from getting really dire. 2188fa9e4066Sahrens */ 2189fa9e4066Sahrens if (availrmem < swapfs_minfree + swapfs_reserve + extra) 2190fa9e4066Sahrens return (1); 2191fa9e4066Sahrens 21925dc8af33Smaybee #if defined(__i386) 2193fa9e4066Sahrens /* 2194fa9e4066Sahrens * If we're on an i386 platform, it's possible that we'll exhaust the 2195fa9e4066Sahrens * kernel heap space before we ever run out of available physical 2196fa9e4066Sahrens * memory. Most checks of the size of the heap_area compare against 2197fa9e4066Sahrens * tune.t_minarmem, which is the minimum available real memory that we 2198fa9e4066Sahrens * can have in the system. However, this is generally fixed at 25 pages 2199fa9e4066Sahrens * which is so low that it's useless. In this comparison, we seek to 2200fa9e4066Sahrens * calculate the total heap-size, and reclaim if more than 3/4ths of the 2201fa94a07fSbrendan * heap is allocated. (Or, in the calculation, if less than 1/4th is 2202fa9e4066Sahrens * free) 2203fa9e4066Sahrens */ 220494dd93aeSGeorge Wilson if (vmem_size(heap_arena, VMEM_FREE) < 220594dd93aeSGeorge Wilson (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2)) 2206fa9e4066Sahrens return (1); 2207fa9e4066Sahrens #endif 2208fa9e4066Sahrens 220994dd93aeSGeorge Wilson /* 221094dd93aeSGeorge Wilson * If zio data pages are being allocated out of a separate heap segment, 221194dd93aeSGeorge Wilson * then enforce that the size of available vmem for this arena remains 221294dd93aeSGeorge Wilson * above about 1/16th free. 221394dd93aeSGeorge Wilson * 221494dd93aeSGeorge Wilson * Note: The 1/16th arena free requirement was put in place 221594dd93aeSGeorge Wilson * to aggressively evict memory from the arc in order to avoid 221694dd93aeSGeorge Wilson * memory fragmentation issues. 221794dd93aeSGeorge Wilson */ 221894dd93aeSGeorge Wilson if (zio_arena != NULL && 221994dd93aeSGeorge Wilson vmem_size(zio_arena, VMEM_FREE) < 222094dd93aeSGeorge Wilson (vmem_size(zio_arena, VMEM_ALLOC) >> 4)) 222194dd93aeSGeorge Wilson return (1); 2222fa9e4066Sahrens #else 2223fa9e4066Sahrens if (spa_get_random(100) == 0) 2224fa9e4066Sahrens return (1); 2225fa9e4066Sahrens #endif 2226fa9e4066Sahrens return (0); 2227fa9e4066Sahrens } 2228fa9e4066Sahrens 2229fa9e4066Sahrens static void 2230fa9e4066Sahrens arc_kmem_reap_now(arc_reclaim_strategy_t strat) 2231fa9e4066Sahrens { 2232fa9e4066Sahrens size_t i; 2233fa9e4066Sahrens kmem_cache_t *prev_cache = NULL; 2234ad23a2dbSjohansen kmem_cache_t *prev_data_cache = NULL; 2235fa9e4066Sahrens extern kmem_cache_t *zio_buf_cache[]; 2236ad23a2dbSjohansen extern kmem_cache_t *zio_data_buf_cache[]; 2237fa9e4066Sahrens 2238033f9833Sek #ifdef _KERNEL 22390e8c6158Smaybee if (arc_meta_used >= arc_meta_limit) { 22400e8c6158Smaybee /* 22410e8c6158Smaybee * We are exceeding our meta-data cache limit. 22420e8c6158Smaybee * Purge some DNLC entries to release holds on meta-data. 22430e8c6158Smaybee */ 22440e8c6158Smaybee dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); 22450e8c6158Smaybee } 22465dc8af33Smaybee #if defined(__i386) 22475dc8af33Smaybee /* 22485dc8af33Smaybee * Reclaim unused memory from all kmem caches. 22495dc8af33Smaybee */ 22505dc8af33Smaybee kmem_reap(); 22515dc8af33Smaybee #endif 2252033f9833Sek #endif 2253033f9833Sek 2254fa9e4066Sahrens /* 2255fa94a07fSbrendan * An aggressive reclamation will shrink the cache size as well as 2256ea8dc4b6Seschrock * reap free buffers from the arc kmem caches. 2257fa9e4066Sahrens */ 2258fa9e4066Sahrens if (strat == ARC_RECLAIM_AGGR) 225949e3519aSmaybee arc_shrink(); 2260fa9e4066Sahrens 2261fa9e4066Sahrens for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { 2262fa9e4066Sahrens if (zio_buf_cache[i] != prev_cache) { 2263fa9e4066Sahrens prev_cache = zio_buf_cache[i]; 2264fa9e4066Sahrens kmem_cache_reap_now(zio_buf_cache[i]); 2265fa9e4066Sahrens } 2266ad23a2dbSjohansen if (zio_data_buf_cache[i] != prev_data_cache) { 2267ad23a2dbSjohansen prev_data_cache = zio_data_buf_cache[i]; 2268ad23a2dbSjohansen kmem_cache_reap_now(zio_data_buf_cache[i]); 2269ad23a2dbSjohansen } 2270fa9e4066Sahrens } 2271ea8dc4b6Seschrock kmem_cache_reap_now(buf_cache); 2272ea8dc4b6Seschrock kmem_cache_reap_now(hdr_cache); 227394dd93aeSGeorge Wilson 227494dd93aeSGeorge Wilson /* 227594dd93aeSGeorge Wilson * Ask the vmem areana to reclaim unused memory from its 227694dd93aeSGeorge Wilson * quantum caches. 227794dd93aeSGeorge Wilson */ 227894dd93aeSGeorge Wilson if (zio_arena != NULL && strat == ARC_RECLAIM_AGGR) 227994dd93aeSGeorge Wilson vmem_qcache_reap(zio_arena); 2280fa9e4066Sahrens } 2281fa9e4066Sahrens 2282fa9e4066Sahrens static void 2283fa9e4066Sahrens arc_reclaim_thread(void) 2284fa9e4066Sahrens { 2285fa9e4066Sahrens clock_t growtime = 0; 2286fa9e4066Sahrens arc_reclaim_strategy_t last_reclaim = ARC_RECLAIM_CONS; 2287fa9e4066Sahrens callb_cpr_t cpr; 2288fa9e4066Sahrens 2289fa9e4066Sahrens CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG); 2290fa9e4066Sahrens 2291fa9e4066Sahrens mutex_enter(&arc_reclaim_thr_lock); 2292fa9e4066Sahrens while (arc_thread_exit == 0) { 2293fa9e4066Sahrens if (arc_reclaim_needed()) { 2294fa9e4066Sahrens 229544cb6abcSbmc if (arc_no_grow) { 2296fa9e4066Sahrens if (last_reclaim == ARC_RECLAIM_CONS) { 2297fa9e4066Sahrens last_reclaim = ARC_RECLAIM_AGGR; 2298fa9e4066Sahrens } else { 2299fa9e4066Sahrens last_reclaim = ARC_RECLAIM_CONS; 2300fa9e4066Sahrens } 2301fa9e4066Sahrens } else { 230244cb6abcSbmc arc_no_grow = TRUE; 2303fa9e4066Sahrens last_reclaim = ARC_RECLAIM_AGGR; 2304fa9e4066Sahrens membar_producer(); 2305fa9e4066Sahrens } 2306fa9e4066Sahrens 2307fa9e4066Sahrens /* reset the growth delay for every reclaim */ 2308d3d50737SRafael Vanoni growtime = ddi_get_lbolt() + (arc_grow_retry * hz); 2309fa9e4066Sahrens 2310fa9e4066Sahrens arc_kmem_reap_now(last_reclaim); 23113a737e0dSbrendan arc_warm = B_TRUE; 2312fa9e4066Sahrens 2313d3d50737SRafael Vanoni } else if (arc_no_grow && ddi_get_lbolt() >= growtime) { 231444cb6abcSbmc arc_no_grow = FALSE; 2315fa9e4066Sahrens } 2316fa9e4066Sahrens 23173e4e8481STom Erickson arc_adjust(); 2318641fbdaeSmaybee 2319ea8dc4b6Seschrock if (arc_eviction_list != NULL) 2320ea8dc4b6Seschrock arc_do_user_evicts(); 2321ea8dc4b6Seschrock 2322fa9e4066Sahrens /* block until needed, or one second, whichever is shorter */ 2323fa9e4066Sahrens CALLB_CPR_SAFE_BEGIN(&cpr); 2324fa9e4066Sahrens (void) cv_timedwait(&arc_reclaim_thr_cv, 2325d3d50737SRafael Vanoni &arc_reclaim_thr_lock, (ddi_get_lbolt() + hz)); 2326fa9e4066Sahrens CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock); 2327fa9e4066Sahrens } 2328fa9e4066Sahrens 2329fa9e4066Sahrens arc_thread_exit = 0; 2330fa9e4066Sahrens cv_broadcast(&arc_reclaim_thr_cv); 2331fa9e4066Sahrens CALLB_CPR_EXIT(&cpr); /* drops arc_reclaim_thr_lock */ 2332fa9e4066Sahrens thread_exit(); 2333fa9e4066Sahrens } 2334fa9e4066Sahrens 2335ea8dc4b6Seschrock /* 2336ea8dc4b6Seschrock * Adapt arc info given the number of bytes we are trying to add and 2337ea8dc4b6Seschrock * the state that we are comming from. This function is only called 2338ea8dc4b6Seschrock * when we are adding new content to the cache. 2339ea8dc4b6Seschrock */ 2340fa9e4066Sahrens static void 2341ea8dc4b6Seschrock arc_adapt(int bytes, arc_state_t *state) 2342fa9e4066Sahrens { 2343ea8dc4b6Seschrock int mult; 23445a98e54bSBrendan Gregg - Sun Microsystems uint64_t arc_p_min = (arc_c >> arc_p_min_shift); 2345ea8dc4b6Seschrock 2346fa94a07fSbrendan if (state == arc_l2c_only) 2347fa94a07fSbrendan return; 2348fa94a07fSbrendan 2349ea8dc4b6Seschrock ASSERT(bytes > 0); 2350fa9e4066Sahrens /* 2351ea8dc4b6Seschrock * Adapt the target size of the MRU list: 2352ea8dc4b6Seschrock * - if we just hit in the MRU ghost list, then increase 2353ea8dc4b6Seschrock * the target size of the MRU list. 2354ea8dc4b6Seschrock * - if we just hit in the MFU ghost list, then increase 2355ea8dc4b6Seschrock * the target size of the MFU list by decreasing the 2356ea8dc4b6Seschrock * target size of the MRU list. 2357fa9e4066Sahrens */ 235844cb6abcSbmc if (state == arc_mru_ghost) { 235944cb6abcSbmc mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ? 236044cb6abcSbmc 1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size)); 23613e4e8481STom Erickson mult = MIN(mult, 10); /* avoid wild arc_p adjustment */ 2362ea8dc4b6Seschrock 23635a98e54bSBrendan Gregg - Sun Microsystems arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult); 236444cb6abcSbmc } else if (state == arc_mfu_ghost) { 23655a98e54bSBrendan Gregg - Sun Microsystems uint64_t delta; 23665a98e54bSBrendan Gregg - Sun Microsystems 236744cb6abcSbmc mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ? 236844cb6abcSbmc 1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size)); 23693e4e8481STom Erickson mult = MIN(mult, 10); 2370ea8dc4b6Seschrock 23715a98e54bSBrendan Gregg - Sun Microsystems delta = MIN(bytes * mult, arc_p); 23725a98e54bSBrendan Gregg - Sun Microsystems arc_p = MAX(arc_p_min, arc_p - delta); 2373ea8dc4b6Seschrock } 237444cb6abcSbmc ASSERT((int64_t)arc_p >= 0); 2375fa9e4066Sahrens 2376fa9e4066Sahrens if (arc_reclaim_needed()) { 2377fa9e4066Sahrens cv_signal(&arc_reclaim_thr_cv); 2378fa9e4066Sahrens return; 2379fa9e4066Sahrens } 2380fa9e4066Sahrens 238144cb6abcSbmc if (arc_no_grow) 2382fa9e4066Sahrens return; 2383fa9e4066Sahrens 238444cb6abcSbmc if (arc_c >= arc_c_max) 2385ea8dc4b6Seschrock return; 2386ea8dc4b6Seschrock 2387fa9e4066Sahrens /* 2388ea8dc4b6Seschrock * If we're within (2 * maxblocksize) bytes of the target 2389ea8dc4b6Seschrock * cache size, increment the target cache size 2390fa9e4066Sahrens */ 239144cb6abcSbmc if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) { 239244cb6abcSbmc atomic_add_64(&arc_c, (int64_t)bytes); 239344cb6abcSbmc if (arc_c > arc_c_max) 239444cb6abcSbmc arc_c = arc_c_max; 239544cb6abcSbmc else if (state == arc_anon) 239644cb6abcSbmc atomic_add_64(&arc_p, (int64_t)bytes); 239744cb6abcSbmc if (arc_p > arc_c) 239844cb6abcSbmc arc_p = arc_c; 2399fa9e4066Sahrens } 240044cb6abcSbmc ASSERT((int64_t)arc_p >= 0); 2401fa9e4066Sahrens } 2402fa9e4066Sahrens 2403fa9e4066Sahrens /* 2404ea8dc4b6Seschrock * Check if the cache has reached its limits and eviction is required 2405ea8dc4b6Seschrock * prior to insert. 2406fa9e4066Sahrens */ 2407fa9e4066Sahrens static int 24080e8c6158Smaybee arc_evict_needed(arc_buf_contents_t type) 2409fa9e4066Sahrens { 24100e8c6158Smaybee if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit) 24110e8c6158Smaybee return (1); 24120e8c6158Smaybee 2413fa9e4066Sahrens if (arc_reclaim_needed()) 2414fa9e4066Sahrens return (1); 2415fa9e4066Sahrens 241644cb6abcSbmc return (arc_size > arc_c); 2417fa9e4066Sahrens } 2418fa9e4066Sahrens 2419fa9e4066Sahrens /* 242044eda4d7Smaybee * The buffer, supplied as the first argument, needs a data block. 242144eda4d7Smaybee * So, if we are at cache max, determine which cache should be victimized. 242244eda4d7Smaybee * We have the following cases: 2423fa9e4066Sahrens * 242444cb6abcSbmc * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) -> 2425fa9e4066Sahrens * In this situation if we're out of space, but the resident size of the MFU is 2426fa9e4066Sahrens * under the limit, victimize the MFU cache to satisfy this insertion request. 2427fa9e4066Sahrens * 242844cb6abcSbmc * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) -> 2429fa9e4066Sahrens * Here, we've used up all of the available space for the MRU, so we need to 2430fa9e4066Sahrens * evict from our own cache instead. Evict from the set of resident MRU 2431fa9e4066Sahrens * entries. 2432fa9e4066Sahrens * 243344cb6abcSbmc * 3. Insert for MFU (c - p) > sizeof(arc_mfu) -> 2434fa9e4066Sahrens * c minus p represents the MFU space in the cache, since p is the size of the 2435fa9e4066Sahrens * cache that is dedicated to the MRU. In this situation there's still space on 2436fa9e4066Sahrens * the MFU side, so the MRU side needs to be victimized. 2437fa9e4066Sahrens * 243844cb6abcSbmc * 4. Insert for MFU (c - p) < sizeof(arc_mfu) -> 2439fa9e4066Sahrens * MFU's resident set is consuming more space than it has been allotted. In 2440fa9e4066Sahrens * this situation, we must victimize our own cache, the MFU, for this insertion. 2441fa9e4066Sahrens */ 2442fa9e4066Sahrens static void 244344eda4d7Smaybee arc_get_data_buf(arc_buf_t *buf) 2444fa9e4066Sahrens { 2445ad23a2dbSjohansen arc_state_t *state = buf->b_hdr->b_state; 2446ad23a2dbSjohansen uint64_t size = buf->b_hdr->b_size; 2447ad23a2dbSjohansen arc_buf_contents_t type = buf->b_hdr->b_type; 2448fa9e4066Sahrens 244944eda4d7Smaybee arc_adapt(size, state); 2450fa9e4066Sahrens 245144eda4d7Smaybee /* 245244eda4d7Smaybee * We have not yet reached cache maximum size, 245344eda4d7Smaybee * just allocate a new buffer. 245444eda4d7Smaybee */ 24550e8c6158Smaybee if (!arc_evict_needed(type)) { 2456ad23a2dbSjohansen if (type == ARC_BUFC_METADATA) { 2457ad23a2dbSjohansen buf->b_data = zio_buf_alloc(size); 24585a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(size, ARC_SPACE_DATA); 2459ad23a2dbSjohansen } else { 2460ad23a2dbSjohansen ASSERT(type == ARC_BUFC_DATA); 2461ad23a2dbSjohansen buf->b_data = zio_data_buf_alloc(size); 24625a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, size); 24630e8c6158Smaybee atomic_add_64(&arc_size, size); 2464ad23a2dbSjohansen } 246544eda4d7Smaybee goto out; 246644eda4d7Smaybee } 246744eda4d7Smaybee 246844eda4d7Smaybee /* 246944eda4d7Smaybee * If we are prefetching from the mfu ghost list, this buffer 247044eda4d7Smaybee * will end up on the mru list; so steal space from there. 247144eda4d7Smaybee */ 247244cb6abcSbmc if (state == arc_mfu_ghost) 247344cb6abcSbmc state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu; 247444cb6abcSbmc else if (state == arc_mru_ghost) 247544cb6abcSbmc state = arc_mru; 247644cb6abcSbmc 247744cb6abcSbmc if (state == arc_mru || state == arc_anon) { 247844cb6abcSbmc uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size; 24795a98e54bSBrendan Gregg - Sun Microsystems state = (arc_mfu->arcs_lsize[type] >= size && 24800e8c6158Smaybee arc_p > mru_used) ? arc_mfu : arc_mru; 2481fa9e4066Sahrens } else { 248244eda4d7Smaybee /* MFU cases */ 248344cb6abcSbmc uint64_t mfu_space = arc_c - arc_p; 24845a98e54bSBrendan Gregg - Sun Microsystems state = (arc_mru->arcs_lsize[type] >= size && 24850e8c6158Smaybee mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu; 248644eda4d7Smaybee } 2487874395d5Smaybee if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) { 2488ad23a2dbSjohansen if (type == ARC_BUFC_METADATA) { 2489ad23a2dbSjohansen buf->b_data = zio_buf_alloc(size); 24905a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(size, ARC_SPACE_DATA); 2491ad23a2dbSjohansen } else { 2492ad23a2dbSjohansen ASSERT(type == ARC_BUFC_DATA); 2493ad23a2dbSjohansen buf->b_data = zio_data_buf_alloc(size); 24945a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_INCR(arcstat_data_size, size); 24950e8c6158Smaybee atomic_add_64(&arc_size, size); 2496ad23a2dbSjohansen } 249744cb6abcSbmc ARCSTAT_BUMP(arcstat_recycle_miss); 249844eda4d7Smaybee } 249944eda4d7Smaybee ASSERT(buf->b_data != NULL); 250044eda4d7Smaybee out: 250144eda4d7Smaybee /* 250244eda4d7Smaybee * Update the state size. Note that ghost states have a 250344eda4d7Smaybee * "ghost size" and so don't need to be updated. 250444eda4d7Smaybee */ 250544eda4d7Smaybee if (!GHOST_STATE(buf->b_hdr->b_state)) { 250644eda4d7Smaybee arc_buf_hdr_t *hdr = buf->b_hdr; 250744eda4d7Smaybee 250844cb6abcSbmc atomic_add_64(&hdr->b_state->arcs_size, size); 250944eda4d7Smaybee if (list_link_active(&hdr->b_arc_node)) { 251044eda4d7Smaybee ASSERT(refcount_is_zero(&hdr->b_refcnt)); 25110e8c6158Smaybee atomic_add_64(&hdr->b_state->arcs_lsize[type], size); 2512fa9e4066Sahrens } 2513641fbdaeSmaybee /* 2514641fbdaeSmaybee * If we are growing the cache, and we are adding anonymous 251544cb6abcSbmc * data, and we have outgrown arc_p, update arc_p 2516641fbdaeSmaybee */ 251744cb6abcSbmc if (arc_size < arc_c && hdr->b_state == arc_anon && 251844cb6abcSbmc arc_anon->arcs_size + arc_mru->arcs_size > arc_p) 251944cb6abcSbmc arc_p = MIN(arc_c, arc_p + size); 2520fa9e4066Sahrens } 2521fa9e4066Sahrens } 2522fa9e4066Sahrens 2523fa9e4066Sahrens /* 2524fa9e4066Sahrens * This routine is called whenever a buffer is accessed. 2525ea8dc4b6Seschrock * NOTE: the hash lock is dropped in this function. 2526fa9e4066Sahrens */ 2527fa9e4066Sahrens static void 252844eda4d7Smaybee arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock) 2529fa9e4066Sahrens { 2530d3d50737SRafael Vanoni clock_t now; 2531d3d50737SRafael Vanoni 2532fa9e4066Sahrens ASSERT(MUTEX_HELD(hash_lock)); 2533fa9e4066Sahrens 253444cb6abcSbmc if (buf->b_state == arc_anon) { 2535fa9e4066Sahrens /* 2536fa9e4066Sahrens * This buffer is not in the cache, and does not 2537fa9e4066Sahrens * appear in our "ghost" list. Add the new buffer 2538fa9e4066Sahrens * to the MRU state. 2539fa9e4066Sahrens */ 2540fa9e4066Sahrens 2541fa9e4066Sahrens ASSERT(buf->b_arc_access == 0); 2542d3d50737SRafael Vanoni buf->b_arc_access = ddi_get_lbolt(); 2543ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 254444cb6abcSbmc arc_change_state(arc_mru, buf, hash_lock); 2545fa9e4066Sahrens 254644cb6abcSbmc } else if (buf->b_state == arc_mru) { 2547d3d50737SRafael Vanoni now = ddi_get_lbolt(); 2548d3d50737SRafael Vanoni 2549fa9e4066Sahrens /* 255013506d1eSmaybee * If this buffer is here because of a prefetch, then either: 255113506d1eSmaybee * - clear the flag if this is a "referencing" read 255213506d1eSmaybee * (any subsequent access will bump this into the MFU state). 255313506d1eSmaybee * or 255413506d1eSmaybee * - move the buffer to the head of the list if this is 255513506d1eSmaybee * another prefetch (to make it less likely to be evicted). 2556fa9e4066Sahrens */ 2557fa9e4066Sahrens if ((buf->b_flags & ARC_PREFETCH) != 0) { 255813506d1eSmaybee if (refcount_count(&buf->b_refcnt) == 0) { 255913506d1eSmaybee ASSERT(list_link_active(&buf->b_arc_node)); 256013506d1eSmaybee } else { 256113506d1eSmaybee buf->b_flags &= ~ARC_PREFETCH; 256244cb6abcSbmc ARCSTAT_BUMP(arcstat_mru_hits); 256313506d1eSmaybee } 2564d3d50737SRafael Vanoni buf->b_arc_access = now; 2565fa9e4066Sahrens return; 2566fa9e4066Sahrens } 2567fa9e4066Sahrens 2568fa9e4066Sahrens /* 2569fa9e4066Sahrens * This buffer has been "accessed" only once so far, 2570fa9e4066Sahrens * but it is still in the cache. Move it to the MFU 2571fa9e4066Sahrens * state. 2572fa9e4066Sahrens */ 2573d3d50737SRafael Vanoni if (now > buf->b_arc_access + ARC_MINTIME) { 2574fa9e4066Sahrens /* 2575fa9e4066Sahrens * More than 125ms have passed since we 2576fa9e4066Sahrens * instantiated this buffer. Move it to the 2577fa9e4066Sahrens * most frequently used state. 2578fa9e4066Sahrens */ 2579d3d50737SRafael Vanoni buf->b_arc_access = now; 2580ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 258144cb6abcSbmc arc_change_state(arc_mfu, buf, hash_lock); 2582fa9e4066Sahrens } 258344cb6abcSbmc ARCSTAT_BUMP(arcstat_mru_hits); 258444cb6abcSbmc } else if (buf->b_state == arc_mru_ghost) { 2585fa9e4066Sahrens arc_state_t *new_state; 2586fa9e4066Sahrens /* 2587fa9e4066Sahrens * This buffer has been "accessed" recently, but 2588fa9e4066Sahrens * was evicted from the cache. Move it to the 2589fa9e4066Sahrens * MFU state. 2590fa9e4066Sahrens */ 2591fa9e4066Sahrens 2592fa9e4066Sahrens if (buf->b_flags & ARC_PREFETCH) { 259344cb6abcSbmc new_state = arc_mru; 259413506d1eSmaybee if (refcount_count(&buf->b_refcnt) > 0) 259513506d1eSmaybee buf->b_flags &= ~ARC_PREFETCH; 2596ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 2597fa9e4066Sahrens } else { 259844cb6abcSbmc new_state = arc_mfu; 2599ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2600fa9e4066Sahrens } 2601fa9e4066Sahrens 2602d3d50737SRafael Vanoni buf->b_arc_access = ddi_get_lbolt(); 2603fa9e4066Sahrens arc_change_state(new_state, buf, hash_lock); 2604fa9e4066Sahrens 260544cb6abcSbmc ARCSTAT_BUMP(arcstat_mru_ghost_hits); 260644cb6abcSbmc } else if (buf->b_state == arc_mfu) { 2607fa9e4066Sahrens /* 2608fa9e4066Sahrens * This buffer has been accessed more than once and is 2609fa9e4066Sahrens * still in the cache. Keep it in the MFU state. 2610fa9e4066Sahrens * 261113506d1eSmaybee * NOTE: an add_reference() that occurred when we did 261213506d1eSmaybee * the arc_read() will have kicked this off the list. 261313506d1eSmaybee * If it was a prefetch, we will explicitly move it to 261413506d1eSmaybee * the head of the list now. 2615fa9e4066Sahrens */ 261613506d1eSmaybee if ((buf->b_flags & ARC_PREFETCH) != 0) { 261713506d1eSmaybee ASSERT(refcount_count(&buf->b_refcnt) == 0); 261813506d1eSmaybee ASSERT(list_link_active(&buf->b_arc_node)); 261913506d1eSmaybee } 262044cb6abcSbmc ARCSTAT_BUMP(arcstat_mfu_hits); 2621d3d50737SRafael Vanoni buf->b_arc_access = ddi_get_lbolt(); 262244cb6abcSbmc } else if (buf->b_state == arc_mfu_ghost) { 262344cb6abcSbmc arc_state_t *new_state = arc_mfu; 2624fa9e4066Sahrens /* 2625fa9e4066Sahrens * This buffer has been accessed more than once but has 2626fa9e4066Sahrens * been evicted from the cache. Move it back to the 2627fa9e4066Sahrens * MFU state. 2628fa9e4066Sahrens */ 2629fa9e4066Sahrens 263013506d1eSmaybee if (buf->b_flags & ARC_PREFETCH) { 263113506d1eSmaybee /* 263213506d1eSmaybee * This is a prefetch access... 263313506d1eSmaybee * move this block back to the MRU state. 263413506d1eSmaybee */ 2635fb09f5aaSMadhav Suresh ASSERT0(refcount_count(&buf->b_refcnt)); 263644cb6abcSbmc new_state = arc_mru; 263713506d1eSmaybee } 263813506d1eSmaybee 2639d3d50737SRafael Vanoni buf->b_arc_access = ddi_get_lbolt(); 2640ea8dc4b6Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 264113506d1eSmaybee arc_change_state(new_state, buf, hash_lock); 2642fa9e4066Sahrens 264344cb6abcSbmc ARCSTAT_BUMP(arcstat_mfu_ghost_hits); 2644fa94a07fSbrendan } else if (buf->b_state == arc_l2c_only) { 2645fa94a07fSbrendan /* 2646fa94a07fSbrendan * This buffer is on the 2nd Level ARC. 2647fa94a07fSbrendan */ 2648fa94a07fSbrendan 2649d3d50737SRafael Vanoni buf->b_arc_access = ddi_get_lbolt(); 2650fa94a07fSbrendan DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2651fa94a07fSbrendan arc_change_state(arc_mfu, buf, hash_lock); 2652fa9e4066Sahrens } else { 2653fa9e4066Sahrens ASSERT(!"invalid arc state"); 2654fa9e4066Sahrens } 2655fa9e4066Sahrens } 2656fa9e4066Sahrens 2657fa9e4066Sahrens /* a generic arc_done_func_t which you can use */ 2658fa9e4066Sahrens /* ARGSUSED */ 2659fa9e4066Sahrens void 2660fa9e4066Sahrens arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) 2661fa9e4066Sahrens { 26623f9d6ad7SLin Ling if (zio == NULL || zio->io_error == 0) 26633f9d6ad7SLin Ling bcopy(buf->b_data, arg, buf->b_hdr->b_size); 26643b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(buf, arg)); 2665fa9e4066Sahrens } 2666fa9e4066Sahrens 26670e8c6158Smaybee /* a generic arc_done_func_t */ 2668fa9e4066Sahrens void 2669fa9e4066Sahrens arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) 2670fa9e4066Sahrens { 2671fa9e4066Sahrens arc_buf_t **bufp = arg; 2672fa9e4066Sahrens if (zio && zio->io_error) { 26733b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(buf, arg)); 2674fa9e4066Sahrens *bufp = NULL; 2675fa9e4066Sahrens } else { 2676fa9e4066Sahrens *bufp = buf; 26773f9d6ad7SLin Ling ASSERT(buf->b_data); 2678fa9e4066Sahrens } 2679fa9e4066Sahrens } 2680fa9e4066Sahrens 2681fa9e4066Sahrens static void 2682fa9e4066Sahrens arc_read_done(zio_t *zio) 2683fa9e4066Sahrens { 2684bbf4a8dfSmaybee arc_buf_hdr_t *hdr, *found; 2685fa9e4066Sahrens arc_buf_t *buf; 2686fa9e4066Sahrens arc_buf_t *abuf; /* buffer we're assigning to callback */ 2687fa9e4066Sahrens kmutex_t *hash_lock; 2688fa9e4066Sahrens arc_callback_t *callback_list, *acb; 2689fa9e4066Sahrens int freeable = FALSE; 2690fa9e4066Sahrens 2691fa9e4066Sahrens buf = zio->io_private; 2692fa9e4066Sahrens hdr = buf->b_hdr; 2693fa9e4066Sahrens 2694bbf4a8dfSmaybee /* 2695bbf4a8dfSmaybee * The hdr was inserted into hash-table and removed from lists 2696bbf4a8dfSmaybee * prior to starting I/O. We should find this header, since 2697bbf4a8dfSmaybee * it's in the hash table, and it should be legit since it's 2698bbf4a8dfSmaybee * not possible to evict it during the I/O. The only possible 2699bbf4a8dfSmaybee * reason for it not to be found is if we were freed during the 2700bbf4a8dfSmaybee * read. 2701bbf4a8dfSmaybee */ 2702ac05c741SMark Maybee found = buf_hash_find(hdr->b_spa, &hdr->b_dva, hdr->b_birth, 27036b4acc8bSahrens &hash_lock); 2704fa9e4066Sahrens 2705bbf4a8dfSmaybee ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) || 2706fa94a07fSbrendan (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || 2707fa94a07fSbrendan (found == hdr && HDR_L2_READING(hdr))); 2708fa94a07fSbrendan 27093a737e0dSbrendan hdr->b_flags &= ~ARC_L2_EVICTED; 2710fa94a07fSbrendan if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH)) 27113baa08fcSek hdr->b_flags &= ~ARC_L2CACHE; 2712fa9e4066Sahrens 2713fa9e4066Sahrens /* byteswap if necessary */ 2714fa9e4066Sahrens callback_list = hdr->b_acb; 2715fa9e4066Sahrens ASSERT(callback_list != NULL); 27168e0f0d3dSWilliam Gorrell if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) { 2717ad135b5dSChristopher Siden dmu_object_byteswap_t bswap = 2718ad135b5dSChristopher Siden DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp)); 2719088f3894Sahrens arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ? 2720088f3894Sahrens byteswap_uint64_array : 2721ad135b5dSChristopher Siden dmu_ot_byteswap[bswap].ob_func; 2722088f3894Sahrens func(buf->b_data, hdr->b_size); 2723088f3894Sahrens } 2724fa9e4066Sahrens 2725fa94a07fSbrendan arc_cksum_compute(buf, B_FALSE); 2726cd1c8b85SMatthew Ahrens arc_buf_watch(buf); 27276b4acc8bSahrens 2728b24ab676SJeff Bonwick if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) { 2729b24ab676SJeff Bonwick /* 2730b24ab676SJeff Bonwick * Only call arc_access on anonymous buffers. This is because 2731b24ab676SJeff Bonwick * if we've issued an I/O for an evicted buffer, we've already 2732b24ab676SJeff Bonwick * called arc_access (to prevent any simultaneous readers from 2733b24ab676SJeff Bonwick * getting confused). 2734b24ab676SJeff Bonwick */ 2735b24ab676SJeff Bonwick arc_access(hdr, hash_lock); 2736b24ab676SJeff Bonwick } 2737b24ab676SJeff Bonwick 2738fa9e4066Sahrens /* create copies of the data buffer for the callers */ 2739fa9e4066Sahrens abuf = buf; 2740fa9e4066Sahrens for (acb = callback_list; acb; acb = acb->acb_next) { 2741fa9e4066Sahrens if (acb->acb_done) { 27429253d63dSGeorge Wilson if (abuf == NULL) { 27439253d63dSGeorge Wilson ARCSTAT_BUMP(arcstat_duplicate_reads); 274444eda4d7Smaybee abuf = arc_buf_clone(buf); 27459253d63dSGeorge Wilson } 2746fa9e4066Sahrens acb->acb_buf = abuf; 2747fa9e4066Sahrens abuf = NULL; 2748fa9e4066Sahrens } 2749fa9e4066Sahrens } 2750fa9e4066Sahrens hdr->b_acb = NULL; 2751fa9e4066Sahrens hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2752ea8dc4b6Seschrock ASSERT(!HDR_BUF_AVAILABLE(hdr)); 2753b24ab676SJeff Bonwick if (abuf == buf) { 2754b24ab676SJeff Bonwick ASSERT(buf->b_efunc == NULL); 2755b24ab676SJeff Bonwick ASSERT(hdr->b_datacnt == 1); 2756ea8dc4b6Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 2757b24ab676SJeff Bonwick } 2758fa9e4066Sahrens 2759fa9e4066Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL); 2760fa9e4066Sahrens 2761fa9e4066Sahrens if (zio->io_error != 0) { 2762fa9e4066Sahrens hdr->b_flags |= ARC_IO_ERROR; 276344cb6abcSbmc if (hdr->b_state != arc_anon) 276444cb6abcSbmc arc_change_state(arc_anon, hdr, hash_lock); 2765ea8dc4b6Seschrock if (HDR_IN_HASH_TABLE(hdr)) 2766ea8dc4b6Seschrock buf_hash_remove(hdr); 2767fa9e4066Sahrens freeable = refcount_is_zero(&hdr->b_refcnt); 2768fa9e4066Sahrens } 2769fa9e4066Sahrens 2770ea8dc4b6Seschrock /* 277113506d1eSmaybee * Broadcast before we drop the hash_lock to avoid the possibility 277213506d1eSmaybee * that the hdr (and hence the cv) might be freed before we get to 277313506d1eSmaybee * the cv_broadcast(). 2774ea8dc4b6Seschrock */ 2775ea8dc4b6Seschrock cv_broadcast(&hdr->b_cv); 2776ea8dc4b6Seschrock 2777bbf4a8dfSmaybee if (hash_lock) { 277844eda4d7Smaybee mutex_exit(hash_lock); 2779fa9e4066Sahrens } else { 2780fa9e4066Sahrens /* 2781fa9e4066Sahrens * This block was freed while we waited for the read to 2782fa9e4066Sahrens * complete. It has been removed from the hash table and 2783fa9e4066Sahrens * moved to the anonymous state (so that it won't show up 2784fa9e4066Sahrens * in the cache). 2785fa9e4066Sahrens */ 278644cb6abcSbmc ASSERT3P(hdr->b_state, ==, arc_anon); 2787fa9e4066Sahrens freeable = refcount_is_zero(&hdr->b_refcnt); 2788fa9e4066Sahrens } 2789fa9e4066Sahrens 2790fa9e4066Sahrens /* execute each callback and free its structure */ 2791fa9e4066Sahrens while ((acb = callback_list) != NULL) { 2792fa9e4066Sahrens if (acb->acb_done) 2793fa9e4066Sahrens acb->acb_done(zio, acb->acb_buf, acb->acb_private); 2794fa9e4066Sahrens 2795fa9e4066Sahrens if (acb->acb_zio_dummy != NULL) { 2796fa9e4066Sahrens acb->acb_zio_dummy->io_error = zio->io_error; 2797fa9e4066Sahrens zio_nowait(acb->acb_zio_dummy); 2798fa9e4066Sahrens } 2799fa9e4066Sahrens 2800fa9e4066Sahrens callback_list = acb->acb_next; 2801fa9e4066Sahrens kmem_free(acb, sizeof (arc_callback_t)); 2802fa9e4066Sahrens } 2803fa9e4066Sahrens 2804fa9e4066Sahrens if (freeable) 2805ea8dc4b6Seschrock arc_hdr_destroy(hdr); 2806fa9e4066Sahrens } 2807fa9e4066Sahrens 2808fa9e4066Sahrens /* 2809fc98fea5SBart Coddens * "Read" the block at the specified DVA (in bp) via the 2810fa9e4066Sahrens * cache. If the block is found in the cache, invoke the provided 2811fa9e4066Sahrens * callback immediately and return. Note that the `zio' parameter 2812fa9e4066Sahrens * in the callback will be NULL in this case, since no IO was 2813fa9e4066Sahrens * required. If the block is not in the cache pass the read request 2814fa9e4066Sahrens * on to the spa with a substitute callback function, so that the 2815fa9e4066Sahrens * requested block will be added to the cache. 2816fa9e4066Sahrens * 2817fa9e4066Sahrens * If a read request arrives for a block that has a read in-progress, 2818fa9e4066Sahrens * either wait for the in-progress read to complete (and return the 2819fa9e4066Sahrens * results); or, if this is a read with a "done" func, add a record 2820fa9e4066Sahrens * to the read to invoke the "done" func when the read completes, 2821fa9e4066Sahrens * and return; or just return. 2822fa9e4066Sahrens * 2823fa9e4066Sahrens * arc_read_done() will invoke all the requested "done" functions 2824fa9e4066Sahrens * for readers of this block. 2825fa9e4066Sahrens */ 2826fa9e4066Sahrens int 28271b912ec7SGeorge Wilson arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done, 28281b912ec7SGeorge Wilson void *private, int priority, int zio_flags, uint32_t *arc_flags, 28291b912ec7SGeorge Wilson const zbookmark_t *zb) 2830fa9e4066Sahrens { 2831fa9e4066Sahrens arc_buf_hdr_t *hdr; 2832d5285caeSGeorge Wilson arc_buf_t *buf = NULL; 2833fa9e4066Sahrens kmutex_t *hash_lock; 2834fa94a07fSbrendan zio_t *rzio; 2835e9103aaeSGarrett D'Amore uint64_t guid = spa_load_guid(spa); 2836fa9e4066Sahrens 2837fa9e4066Sahrens top: 2838b24ab676SJeff Bonwick hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp), 2839b24ab676SJeff Bonwick &hash_lock); 2840ea8dc4b6Seschrock if (hdr && hdr->b_datacnt > 0) { 2841fa9e4066Sahrens 284213506d1eSmaybee *arc_flags |= ARC_CACHED; 284313506d1eSmaybee 2844fa9e4066Sahrens if (HDR_IO_IN_PROGRESS(hdr)) { 284513506d1eSmaybee 284613506d1eSmaybee if (*arc_flags & ARC_WAIT) { 284713506d1eSmaybee cv_wait(&hdr->b_cv, hash_lock); 284813506d1eSmaybee mutex_exit(hash_lock); 284913506d1eSmaybee goto top; 285013506d1eSmaybee } 285113506d1eSmaybee ASSERT(*arc_flags & ARC_NOWAIT); 285213506d1eSmaybee 285313506d1eSmaybee if (done) { 2854fa9e4066Sahrens arc_callback_t *acb = NULL; 2855fa9e4066Sahrens 2856fa9e4066Sahrens acb = kmem_zalloc(sizeof (arc_callback_t), 2857fa9e4066Sahrens KM_SLEEP); 2858fa9e4066Sahrens acb->acb_done = done; 2859fa9e4066Sahrens acb->acb_private = private; 2860fa9e4066Sahrens if (pio != NULL) 2861fa9e4066Sahrens acb->acb_zio_dummy = zio_null(pio, 2862a3f829aeSBill Moore spa, NULL, NULL, NULL, zio_flags); 2863fa9e4066Sahrens 2864fa9e4066Sahrens ASSERT(acb->acb_done != NULL); 2865fa9e4066Sahrens acb->acb_next = hdr->b_acb; 2866fa9e4066Sahrens hdr->b_acb = acb; 2867fa9e4066Sahrens add_reference(hdr, hash_lock, private); 2868fa9e4066Sahrens mutex_exit(hash_lock); 2869fa9e4066Sahrens return (0); 2870fa9e4066Sahrens } 2871fa9e4066Sahrens mutex_exit(hash_lock); 2872fa9e4066Sahrens return (0); 2873fa9e4066Sahrens } 2874fa9e4066Sahrens 287544cb6abcSbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2876fa9e4066Sahrens 2877ea8dc4b6Seschrock if (done) { 287844eda4d7Smaybee add_reference(hdr, hash_lock, private); 2879ea8dc4b6Seschrock /* 2880ea8dc4b6Seschrock * If this block is already in use, create a new 2881ea8dc4b6Seschrock * copy of the data so that we will be guaranteed 2882ea8dc4b6Seschrock * that arc_release() will always succeed. 2883ea8dc4b6Seschrock */ 2884fa9e4066Sahrens buf = hdr->b_buf; 2885ea8dc4b6Seschrock ASSERT(buf); 2886ea8dc4b6Seschrock ASSERT(buf->b_data); 288744eda4d7Smaybee if (HDR_BUF_AVAILABLE(hdr)) { 2888ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 2889ea8dc4b6Seschrock hdr->b_flags &= ~ARC_BUF_AVAILABLE; 289044eda4d7Smaybee } else { 289144eda4d7Smaybee buf = arc_buf_clone(buf); 2892ea8dc4b6Seschrock } 2893b24ab676SJeff Bonwick 289413506d1eSmaybee } else if (*arc_flags & ARC_PREFETCH && 289513506d1eSmaybee refcount_count(&hdr->b_refcnt) == 0) { 289613506d1eSmaybee hdr->b_flags |= ARC_PREFETCH; 2897fa9e4066Sahrens } 2898fa9e4066Sahrens DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 289944eda4d7Smaybee arc_access(hdr, hash_lock); 29003baa08fcSek if (*arc_flags & ARC_L2CACHE) 29013baa08fcSek hdr->b_flags |= ARC_L2CACHE; 2902aad02571SSaso Kiselkov if (*arc_flags & ARC_L2COMPRESS) 2903aad02571SSaso Kiselkov hdr->b_flags |= ARC_L2COMPRESS; 290444eda4d7Smaybee mutex_exit(hash_lock); 290544cb6abcSbmc ARCSTAT_BUMP(arcstat_hits); 290644cb6abcSbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 290744cb6abcSbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 290844cb6abcSbmc data, metadata, hits); 290944cb6abcSbmc 2910fa9e4066Sahrens if (done) 2911fa9e4066Sahrens done(NULL, buf, private); 2912fa9e4066Sahrens } else { 2913fa9e4066Sahrens uint64_t size = BP_GET_LSIZE(bp); 2914fa9e4066Sahrens arc_callback_t *acb; 29153a737e0dSbrendan vdev_t *vd = NULL; 2916d5285caeSGeorge Wilson uint64_t addr = 0; 29175a98e54bSBrendan Gregg - Sun Microsystems boolean_t devw = B_FALSE; 2918fa9e4066Sahrens 2919fa9e4066Sahrens if (hdr == NULL) { 2920fa9e4066Sahrens /* this block is not in the cache */ 2921fa9e4066Sahrens arc_buf_hdr_t *exists; 2922ad23a2dbSjohansen arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp); 2923ad23a2dbSjohansen buf = arc_buf_alloc(spa, size, private, type); 2924fa9e4066Sahrens hdr = buf->b_hdr; 2925fa9e4066Sahrens hdr->b_dva = *BP_IDENTITY(bp); 2926b24ab676SJeff Bonwick hdr->b_birth = BP_PHYSICAL_BIRTH(bp); 2927fa9e4066Sahrens hdr->b_cksum0 = bp->blk_cksum.zc_word[0]; 2928fa9e4066Sahrens exists = buf_hash_insert(hdr, &hash_lock); 2929fa9e4066Sahrens if (exists) { 2930fa9e4066Sahrens /* somebody beat us to the hash insert */ 2931fa9e4066Sahrens mutex_exit(hash_lock); 29323f9d6ad7SLin Ling buf_discard_identity(hdr); 2933ea8dc4b6Seschrock (void) arc_buf_remove_ref(buf, private); 2934fa9e4066Sahrens goto top; /* restart the IO request */ 2935fa9e4066Sahrens } 293613506d1eSmaybee /* if this is a prefetch, we don't have a reference */ 293713506d1eSmaybee if (*arc_flags & ARC_PREFETCH) { 293813506d1eSmaybee (void) remove_reference(hdr, hash_lock, 293913506d1eSmaybee private); 294013506d1eSmaybee hdr->b_flags |= ARC_PREFETCH; 294113506d1eSmaybee } 29423baa08fcSek if (*arc_flags & ARC_L2CACHE) 29433baa08fcSek hdr->b_flags |= ARC_L2CACHE; 2944aad02571SSaso Kiselkov if (*arc_flags & ARC_L2COMPRESS) 2945aad02571SSaso Kiselkov hdr->b_flags |= ARC_L2COMPRESS; 294613506d1eSmaybee if (BP_GET_LEVEL(bp) > 0) 294713506d1eSmaybee hdr->b_flags |= ARC_INDIRECT; 2948fa9e4066Sahrens } else { 2949fa9e4066Sahrens /* this block is in the ghost cache */ 2950ea8dc4b6Seschrock ASSERT(GHOST_STATE(hdr->b_state)); 2951ea8dc4b6Seschrock ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 2952fb09f5aaSMadhav Suresh ASSERT0(refcount_count(&hdr->b_refcnt)); 2953ea8dc4b6Seschrock ASSERT(hdr->b_buf == NULL); 295413506d1eSmaybee 295513506d1eSmaybee /* if this is a prefetch, we don't have a reference */ 295613506d1eSmaybee if (*arc_flags & ARC_PREFETCH) 295713506d1eSmaybee hdr->b_flags |= ARC_PREFETCH; 295813506d1eSmaybee else 295913506d1eSmaybee add_reference(hdr, hash_lock, private); 29603baa08fcSek if (*arc_flags & ARC_L2CACHE) 29613baa08fcSek hdr->b_flags |= ARC_L2CACHE; 2962aad02571SSaso Kiselkov if (*arc_flags & ARC_L2COMPRESS) 2963aad02571SSaso Kiselkov hdr->b_flags |= ARC_L2COMPRESS; 29641ab7f2deSmaybee buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 2965fa9e4066Sahrens buf->b_hdr = hdr; 296644eda4d7Smaybee buf->b_data = NULL; 2967ea8dc4b6Seschrock buf->b_efunc = NULL; 2968ea8dc4b6Seschrock buf->b_private = NULL; 2969fa9e4066Sahrens buf->b_next = NULL; 2970fa9e4066Sahrens hdr->b_buf = buf; 2971ea8dc4b6Seschrock ASSERT(hdr->b_datacnt == 0); 2972ea8dc4b6Seschrock hdr->b_datacnt = 1; 29735614b00aSWilliam Gorrell arc_get_data_buf(buf); 29747e453561SWilliam Gorrell arc_access(hdr, hash_lock); 2975fa9e4066Sahrens } 2976fa9e4066Sahrens 29775614b00aSWilliam Gorrell ASSERT(!GHOST_STATE(hdr->b_state)); 29785614b00aSWilliam Gorrell 2979fa9e4066Sahrens acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 2980fa9e4066Sahrens acb->acb_done = done; 2981fa9e4066Sahrens acb->acb_private = private; 2982fa9e4066Sahrens 2983fa9e4066Sahrens ASSERT(hdr->b_acb == NULL); 2984fa9e4066Sahrens hdr->b_acb = acb; 2985fa9e4066Sahrens hdr->b_flags |= ARC_IO_IN_PROGRESS; 2986fa9e4066Sahrens 2987e14bb325SJeff Bonwick if (HDR_L2CACHE(hdr) && hdr->b_l2hdr != NULL && 2988e14bb325SJeff Bonwick (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) { 29895a98e54bSBrendan Gregg - Sun Microsystems devw = hdr->b_l2hdr->b_dev->l2ad_writing; 29903a737e0dSbrendan addr = hdr->b_l2hdr->b_daddr; 2991e14bb325SJeff Bonwick /* 2992e14bb325SJeff Bonwick * Lock out device removal. 2993e14bb325SJeff Bonwick */ 2994e14bb325SJeff Bonwick if (vdev_is_dead(vd) || 2995e14bb325SJeff Bonwick !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER)) 2996e14bb325SJeff Bonwick vd = NULL; 29973a737e0dSbrendan } 29983a737e0dSbrendan 29993a737e0dSbrendan mutex_exit(hash_lock); 30003a737e0dSbrendan 30013e30c24aSWill Andrews /* 30023e30c24aSWill Andrews * At this point, we have a level 1 cache miss. Try again in 30033e30c24aSWill Andrews * L2ARC if possible. 30043e30c24aSWill Andrews */ 3005fa9e4066Sahrens ASSERT3U(hdr->b_size, ==, size); 30065c28183bSBrendan Gregg - Sun Microsystems DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp, 30075c28183bSBrendan Gregg - Sun Microsystems uint64_t, size, zbookmark_t *, zb); 300844cb6abcSbmc ARCSTAT_BUMP(arcstat_misses); 300944cb6abcSbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 301044cb6abcSbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 301144cb6abcSbmc data, metadata, misses); 3012ea8dc4b6Seschrock 30135a98e54bSBrendan Gregg - Sun Microsystems if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) { 3014fa94a07fSbrendan /* 3015fa94a07fSbrendan * Read from the L2ARC if the following are true: 30163a737e0dSbrendan * 1. The L2ARC vdev was previously cached. 30173a737e0dSbrendan * 2. This buffer still has L2ARC metadata. 30183a737e0dSbrendan * 3. This buffer isn't currently writing to the L2ARC. 30193a737e0dSbrendan * 4. The L2ARC entry wasn't evicted, which may 30203a737e0dSbrendan * also have invalidated the vdev. 30215a98e54bSBrendan Gregg - Sun Microsystems * 5. This isn't prefetch and l2arc_noprefetch is set. 3022fa94a07fSbrendan */ 3023e14bb325SJeff Bonwick if (hdr->b_l2hdr != NULL && 30245a98e54bSBrendan Gregg - Sun Microsystems !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) && 30255a98e54bSBrendan Gregg - Sun Microsystems !(l2arc_noprefetch && HDR_PREFETCH(hdr))) { 3026fa94a07fSbrendan l2arc_read_callback_t *cb; 3027fa94a07fSbrendan 3028c5904d13Seschrock DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr); 3029c5904d13Seschrock ARCSTAT_BUMP(arcstat_l2_hits); 3030c5904d13Seschrock 3031fa94a07fSbrendan cb = kmem_zalloc(sizeof (l2arc_read_callback_t), 3032fa94a07fSbrendan KM_SLEEP); 3033fa94a07fSbrendan cb->l2rcb_buf = buf; 3034fa94a07fSbrendan cb->l2rcb_spa = spa; 3035fa94a07fSbrendan cb->l2rcb_bp = *bp; 3036fa94a07fSbrendan cb->l2rcb_zb = *zb; 30373baa08fcSek cb->l2rcb_flags = zio_flags; 3038aad02571SSaso Kiselkov cb->l2rcb_compress = hdr->b_l2hdr->b_compress; 3039fa94a07fSbrendan 3040d5285caeSGeorge Wilson ASSERT(addr >= VDEV_LABEL_START_SIZE && 3041d5285caeSGeorge Wilson addr + size < vd->vdev_psize - 3042d5285caeSGeorge Wilson VDEV_LABEL_END_SIZE); 3043d5285caeSGeorge Wilson 3044fa94a07fSbrendan /* 3045e14bb325SJeff Bonwick * l2arc read. The SCL_L2ARC lock will be 3046e14bb325SJeff Bonwick * released by l2arc_read_done(). 3047aad02571SSaso Kiselkov * Issue a null zio if the underlying buffer 3048aad02571SSaso Kiselkov * was squashed to zero size by compression. 3049fa94a07fSbrendan */ 3050aad02571SSaso Kiselkov if (hdr->b_l2hdr->b_compress == 3051aad02571SSaso Kiselkov ZIO_COMPRESS_EMPTY) { 3052aad02571SSaso Kiselkov rzio = zio_null(pio, spa, vd, 3053aad02571SSaso Kiselkov l2arc_read_done, cb, 3054aad02571SSaso Kiselkov zio_flags | ZIO_FLAG_DONT_CACHE | 3055aad02571SSaso Kiselkov ZIO_FLAG_CANFAIL | 3056aad02571SSaso Kiselkov ZIO_FLAG_DONT_PROPAGATE | 3057aad02571SSaso Kiselkov ZIO_FLAG_DONT_RETRY); 3058aad02571SSaso Kiselkov } else { 3059aad02571SSaso Kiselkov rzio = zio_read_phys(pio, vd, addr, 3060aad02571SSaso Kiselkov hdr->b_l2hdr->b_asize, 3061aad02571SSaso Kiselkov buf->b_data, ZIO_CHECKSUM_OFF, 3062aad02571SSaso Kiselkov l2arc_read_done, cb, priority, 3063aad02571SSaso Kiselkov zio_flags | ZIO_FLAG_DONT_CACHE | 3064aad02571SSaso Kiselkov ZIO_FLAG_CANFAIL | 3065aad02571SSaso Kiselkov ZIO_FLAG_DONT_PROPAGATE | 3066aad02571SSaso Kiselkov ZIO_FLAG_DONT_RETRY, B_FALSE); 3067aad02571SSaso Kiselkov } 3068fa94a07fSbrendan DTRACE_PROBE2(l2arc__read, vdev_t *, vd, 3069fa94a07fSbrendan zio_t *, rzio); 3070aad02571SSaso Kiselkov ARCSTAT_INCR(arcstat_l2_read_bytes, 3071aad02571SSaso Kiselkov hdr->b_l2hdr->b_asize); 3072fa94a07fSbrendan 30733a737e0dSbrendan if (*arc_flags & ARC_NOWAIT) { 30743a737e0dSbrendan zio_nowait(rzio); 30753a737e0dSbrendan return (0); 30763a737e0dSbrendan } 3077fa94a07fSbrendan 30783a737e0dSbrendan ASSERT(*arc_flags & ARC_WAIT); 30793a737e0dSbrendan if (zio_wait(rzio) == 0) 30803a737e0dSbrendan return (0); 30813a737e0dSbrendan 30823a737e0dSbrendan /* l2arc read error; goto zio_read() */ 3083fa94a07fSbrendan } else { 3084fa94a07fSbrendan DTRACE_PROBE1(l2arc__miss, 3085fa94a07fSbrendan arc_buf_hdr_t *, hdr); 3086fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_misses); 3087fa94a07fSbrendan if (HDR_L2_WRITING(hdr)) 3088fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_rw_clash); 3089e14bb325SJeff Bonwick spa_config_exit(spa, SCL_L2ARC, vd); 3090fa94a07fSbrendan } 30915a98e54bSBrendan Gregg - Sun Microsystems } else { 309276a25fafSBill Moore if (vd != NULL) 309376a25fafSBill Moore spa_config_exit(spa, SCL_L2ARC, vd); 30945a98e54bSBrendan Gregg - Sun Microsystems if (l2arc_ndev != 0) { 30955a98e54bSBrendan Gregg - Sun Microsystems DTRACE_PROBE1(l2arc__miss, 30965a98e54bSBrendan Gregg - Sun Microsystems arc_buf_hdr_t *, hdr); 30975a98e54bSBrendan Gregg - Sun Microsystems ARCSTAT_BUMP(arcstat_l2_misses); 30985a98e54bSBrendan Gregg - Sun Microsystems } 3099fa94a07fSbrendan } 3100c5904d13Seschrock 3101fa9e4066Sahrens rzio = zio_read(pio, spa, bp, buf->b_data, size, 31023baa08fcSek arc_read_done, buf, priority, zio_flags, zb); 3103fa9e4066Sahrens 310413506d1eSmaybee if (*arc_flags & ARC_WAIT) 3105fa9e4066Sahrens return (zio_wait(rzio)); 3106fa9e4066Sahrens 310713506d1eSmaybee ASSERT(*arc_flags & ARC_NOWAIT); 3108fa9e4066Sahrens zio_nowait(rzio); 3109fa9e4066Sahrens } 3110fa9e4066Sahrens return (0); 3111fa9e4066Sahrens } 3112fa9e4066Sahrens 3113ea8dc4b6Seschrock void 3114ea8dc4b6Seschrock arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private) 3115ea8dc4b6Seschrock { 3116ea8dc4b6Seschrock ASSERT(buf->b_hdr != NULL); 311744cb6abcSbmc ASSERT(buf->b_hdr->b_state != arc_anon); 3118ea8dc4b6Seschrock ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL); 3119b24ab676SJeff Bonwick ASSERT(buf->b_efunc == NULL); 3120b24ab676SJeff Bonwick ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr)); 3121b24ab676SJeff Bonwick 3122ea8dc4b6Seschrock buf->b_efunc = func; 3123ea8dc4b6Seschrock buf->b_private = private; 3124ea8dc4b6Seschrock } 3125ea8dc4b6Seschrock 31266e6d5868SMatthew Ahrens /* 31276e6d5868SMatthew Ahrens * Notify the arc that a block was freed, and thus will never be used again. 31286e6d5868SMatthew Ahrens */ 31296e6d5868SMatthew Ahrens void 31306e6d5868SMatthew Ahrens arc_freed(spa_t *spa, const blkptr_t *bp) 31316e6d5868SMatthew Ahrens { 31326e6d5868SMatthew Ahrens arc_buf_hdr_t *hdr; 31336e6d5868SMatthew Ahrens kmutex_t *hash_lock; 31346e6d5868SMatthew Ahrens uint64_t guid = spa_load_guid(spa); 31356e6d5868SMatthew Ahrens 31366e6d5868SMatthew Ahrens hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp), 31376e6d5868SMatthew Ahrens &hash_lock); 31386e6d5868SMatthew Ahrens if (hdr == NULL) 31396e6d5868SMatthew Ahrens return; 31406e6d5868SMatthew Ahrens if (HDR_BUF_AVAILABLE(hdr)) { 31416e6d5868SMatthew Ahrens arc_buf_t *buf = hdr->b_buf; 31426e6d5868SMatthew Ahrens add_reference(hdr, hash_lock, FTAG); 31436e6d5868SMatthew Ahrens hdr->b_flags &= ~ARC_BUF_AVAILABLE; 31446e6d5868SMatthew Ahrens mutex_exit(hash_lock); 31456e6d5868SMatthew Ahrens 31466e6d5868SMatthew Ahrens arc_release(buf, FTAG); 31476e6d5868SMatthew Ahrens (void) arc_buf_remove_ref(buf, FTAG); 31486e6d5868SMatthew Ahrens } else { 31496e6d5868SMatthew Ahrens mutex_exit(hash_lock); 31506e6d5868SMatthew Ahrens } 31516e6d5868SMatthew Ahrens 31526e6d5868SMatthew Ahrens } 31536e6d5868SMatthew Ahrens 3154ea8dc4b6Seschrock /* 3155ea8dc4b6Seschrock * This is used by the DMU to let the ARC know that a buffer is 3156ea8dc4b6Seschrock * being evicted, so the ARC should clean up. If this arc buf 3157ea8dc4b6Seschrock * is not yet in the evicted state, it will be put there. 3158ea8dc4b6Seschrock */ 3159ea8dc4b6Seschrock int 3160ea8dc4b6Seschrock arc_buf_evict(arc_buf_t *buf) 3161ea8dc4b6Seschrock { 316240d7d650Smaybee arc_buf_hdr_t *hdr; 3163ea8dc4b6Seschrock kmutex_t *hash_lock; 3164ea8dc4b6Seschrock arc_buf_t **bufp; 3165ea8dc4b6Seschrock 31663f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 316740d7d650Smaybee hdr = buf->b_hdr; 3168ea8dc4b6Seschrock if (hdr == NULL) { 3169ea8dc4b6Seschrock /* 3170ea8dc4b6Seschrock * We are in arc_do_user_evicts(). 3171ea8dc4b6Seschrock */ 3172ea8dc4b6Seschrock ASSERT(buf->b_data == NULL); 31733f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 3174ea8dc4b6Seschrock return (0); 31756f83844dSMark Maybee } else if (buf->b_data == NULL) { 31766f83844dSMark Maybee arc_buf_t copy = *buf; /* structure assignment */ 31779b23f181Smaybee /* 31786f83844dSMark Maybee * We are on the eviction list; process this buffer now 31796f83844dSMark Maybee * but let arc_do_user_evicts() do the reaping. 31809b23f181Smaybee */ 31816f83844dSMark Maybee buf->b_efunc = NULL; 31823f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 31836f83844dSMark Maybee VERIFY(copy.b_efunc(©) == 0); 31846f83844dSMark Maybee return (1); 31859b23f181Smaybee } 31866f83844dSMark Maybee hash_lock = HDR_LOCK(hdr); 31876f83844dSMark Maybee mutex_enter(hash_lock); 31883f9d6ad7SLin Ling hdr = buf->b_hdr; 31893f9d6ad7SLin Ling ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 31909b23f181Smaybee 31919b23f181Smaybee ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt); 319244cb6abcSbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 3193ea8dc4b6Seschrock 3194ea8dc4b6Seschrock /* 3195ea8dc4b6Seschrock * Pull this buffer off of the hdr 3196ea8dc4b6Seschrock */ 3197ea8dc4b6Seschrock bufp = &hdr->b_buf; 3198ea8dc4b6Seschrock while (*bufp != buf) 3199ea8dc4b6Seschrock bufp = &(*bufp)->b_next; 3200ea8dc4b6Seschrock *bufp = buf->b_next; 3201ea8dc4b6Seschrock 3202ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 320344eda4d7Smaybee arc_buf_destroy(buf, FALSE, FALSE); 3204ea8dc4b6Seschrock 3205ea8dc4b6Seschrock if (hdr->b_datacnt == 0) { 3206ea8dc4b6Seschrock arc_state_t *old_state = hdr->b_state; 3207ea8dc4b6Seschrock arc_state_t *evicted_state; 3208ea8dc4b6Seschrock 32093f9d6ad7SLin Ling ASSERT(hdr->b_buf == NULL); 3210ea8dc4b6Seschrock ASSERT(refcount_is_zero(&hdr->b_refcnt)); 3211ea8dc4b6Seschrock 3212ea8dc4b6Seschrock evicted_state = 321344cb6abcSbmc (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 3214ea8dc4b6Seschrock 321544cb6abcSbmc mutex_enter(&old_state->arcs_mtx); 321644cb6abcSbmc mutex_enter(&evicted_state->arcs_mtx); 3217ea8dc4b6Seschrock 3218ea8dc4b6Seschrock arc_change_state(evicted_state, hdr, hash_lock); 3219ea8dc4b6Seschrock ASSERT(HDR_IN_HASH_TABLE(hdr)); 3220fa94a07fSbrendan hdr->b_flags |= ARC_IN_HASH_TABLE; 3221fa94a07fSbrendan hdr->b_flags &= ~ARC_BUF_AVAILABLE; 3222ea8dc4b6Seschrock 322344cb6abcSbmc mutex_exit(&evicted_state->arcs_mtx); 322444cb6abcSbmc mutex_exit(&old_state->arcs_mtx); 3225ea8dc4b6Seschrock } 3226ea8dc4b6Seschrock mutex_exit(hash_lock); 32273f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 3228dd6ef538Smaybee 3229ea8dc4b6Seschrock VERIFY(buf->b_efunc(buf) == 0); 3230ea8dc4b6Seschrock buf->b_efunc = NULL; 3231ea8dc4b6Seschrock buf->b_private = NULL; 3232ea8dc4b6Seschrock buf->b_hdr = NULL; 32333f9d6ad7SLin Ling buf->b_next = NULL; 3234ea8dc4b6Seschrock kmem_cache_free(buf_cache, buf); 3235ea8dc4b6Seschrock return (1); 3236ea8dc4b6Seschrock } 3237ea8dc4b6Seschrock 3238fa9e4066Sahrens /* 32393e30c24aSWill Andrews * Release this buffer from the cache, making it an anonymous buffer. This 32403e30c24aSWill Andrews * must be done after a read and prior to modifying the buffer contents. 3241fa9e4066Sahrens * If the buffer has more than one reference, we must make 3242088f3894Sahrens * a new hdr for the buffer. 3243fa9e4066Sahrens */ 3244fa9e4066Sahrens void 3245fa9e4066Sahrens arc_release(arc_buf_t *buf, void *tag) 3246fa9e4066Sahrens { 32476f83844dSMark Maybee arc_buf_hdr_t *hdr; 32483f9d6ad7SLin Ling kmutex_t *hash_lock = NULL; 32496f83844dSMark Maybee l2arc_buf_hdr_t *l2hdr; 3250fa94a07fSbrendan uint64_t buf_size; 3251fa9e4066Sahrens 32523f9d6ad7SLin Ling /* 32533f9d6ad7SLin Ling * It would be nice to assert that if it's DMU metadata (level > 32543f9d6ad7SLin Ling * 0 || it's the dnode file), then it must be syncing context. 32553f9d6ad7SLin Ling * But we don't know that information at this level. 32563f9d6ad7SLin Ling */ 32573f9d6ad7SLin Ling 32583f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 32596f83844dSMark Maybee hdr = buf->b_hdr; 32606f83844dSMark Maybee 3261fa9e4066Sahrens /* this buffer is not on any list */ 3262fa9e4066Sahrens ASSERT(refcount_count(&hdr->b_refcnt) > 0); 3263fa9e4066Sahrens 326444cb6abcSbmc if (hdr->b_state == arc_anon) { 3265fa9e4066Sahrens /* this buffer is already released */ 3266ea8dc4b6Seschrock ASSERT(buf->b_efunc == NULL); 32670a95608cSBrendan Gregg - Sun Microsystems } else { 32680a95608cSBrendan Gregg - Sun Microsystems hash_lock = HDR_LOCK(hdr); 32690a95608cSBrendan Gregg - Sun Microsystems mutex_enter(hash_lock); 32703f9d6ad7SLin Ling hdr = buf->b_hdr; 32713f9d6ad7SLin Ling ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 3272fa9e4066Sahrens } 3273fa9e4066Sahrens 32746f83844dSMark Maybee l2hdr = hdr->b_l2hdr; 32756f83844dSMark Maybee if (l2hdr) { 32766f83844dSMark Maybee mutex_enter(&l2arc_buflist_mtx); 32776f83844dSMark Maybee hdr->b_l2hdr = NULL; 32786f83844dSMark Maybee } 3279d5285caeSGeorge Wilson buf_size = hdr->b_size; 32806f83844dSMark Maybee 3281ea8dc4b6Seschrock /* 3282ea8dc4b6Seschrock * Do we have more than one buf? 3283ea8dc4b6Seschrock */ 32846f83844dSMark Maybee if (hdr->b_datacnt > 1) { 3285fa9e4066Sahrens arc_buf_hdr_t *nhdr; 3286fa9e4066Sahrens arc_buf_t **bufp; 3287fa9e4066Sahrens uint64_t blksz = hdr->b_size; 3288ac05c741SMark Maybee uint64_t spa = hdr->b_spa; 3289ad23a2dbSjohansen arc_buf_contents_t type = hdr->b_type; 3290fa94a07fSbrendan uint32_t flags = hdr->b_flags; 3291fa9e4066Sahrens 32926f83844dSMark Maybee ASSERT(hdr->b_buf != buf || buf->b_next != NULL); 3293fa9e4066Sahrens /* 32943f9d6ad7SLin Ling * Pull the data off of this hdr and attach it to 32953f9d6ad7SLin Ling * a new anonymous hdr. 3296fa9e4066Sahrens */ 3297ea8dc4b6Seschrock (void) remove_reference(hdr, hash_lock, tag); 3298fa9e4066Sahrens bufp = &hdr->b_buf; 3299ea8dc4b6Seschrock while (*bufp != buf) 3300fa9e4066Sahrens bufp = &(*bufp)->b_next; 33013f9d6ad7SLin Ling *bufp = buf->b_next; 3302af2c4821Smaybee buf->b_next = NULL; 3303ea8dc4b6Seschrock 330444cb6abcSbmc ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size); 330544cb6abcSbmc atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size); 3306ea8dc4b6Seschrock if (refcount_is_zero(&hdr->b_refcnt)) { 33070e8c6158Smaybee uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type]; 33080e8c6158Smaybee ASSERT3U(*size, >=, hdr->b_size); 33090e8c6158Smaybee atomic_add_64(size, -hdr->b_size); 3310ea8dc4b6Seschrock } 33119253d63dSGeorge Wilson 33129253d63dSGeorge Wilson /* 33139253d63dSGeorge Wilson * We're releasing a duplicate user data buffer, update 33149253d63dSGeorge Wilson * our statistics accordingly. 33159253d63dSGeorge Wilson */ 33169253d63dSGeorge Wilson if (hdr->b_type == ARC_BUFC_DATA) { 33179253d63dSGeorge Wilson ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers); 33189253d63dSGeorge Wilson ARCSTAT_INCR(arcstat_duplicate_buffers_size, 33199253d63dSGeorge Wilson -hdr->b_size); 33209253d63dSGeorge Wilson } 3321ea8dc4b6Seschrock hdr->b_datacnt -= 1; 3322c717a561Smaybee arc_cksum_verify(buf); 3323cd1c8b85SMatthew Ahrens arc_buf_unwatch(buf); 3324ea8dc4b6Seschrock 3325fa9e4066Sahrens mutex_exit(hash_lock); 3326fa9e4066Sahrens 33271ab7f2deSmaybee nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 3328fa9e4066Sahrens nhdr->b_size = blksz; 3329fa9e4066Sahrens nhdr->b_spa = spa; 3330ad23a2dbSjohansen nhdr->b_type = type; 3331fa9e4066Sahrens nhdr->b_buf = buf; 333244cb6abcSbmc nhdr->b_state = arc_anon; 3333fa9e4066Sahrens nhdr->b_arc_access = 0; 3334fa94a07fSbrendan nhdr->b_flags = flags & ARC_L2_WRITING; 3335fa94a07fSbrendan nhdr->b_l2hdr = NULL; 3336ea8dc4b6Seschrock nhdr->b_datacnt = 1; 3337c717a561Smaybee nhdr->b_freeze_cksum = NULL; 3338fa9e4066Sahrens (void) refcount_add(&nhdr->b_refcnt, tag); 3339af2c4821Smaybee buf->b_hdr = nhdr; 33403f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 334144cb6abcSbmc atomic_add_64(&arc_anon->arcs_size, blksz); 3342fa9e4066Sahrens } else { 33433f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 3344ea8dc4b6Seschrock ASSERT(refcount_count(&hdr->b_refcnt) == 1); 3345fa9e4066Sahrens ASSERT(!list_link_active(&hdr->b_arc_node)); 3346fa9e4066Sahrens ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 33473f9d6ad7SLin Ling if (hdr->b_state != arc_anon) 33483f9d6ad7SLin Ling arc_change_state(arc_anon, hdr, hash_lock); 3349fa9e4066Sahrens hdr->b_arc_access = 0; 33503f9d6ad7SLin Ling if (hash_lock) 33513f9d6ad7SLin Ling mutex_exit(hash_lock); 3352fa94a07fSbrendan 33533f9d6ad7SLin Ling buf_discard_identity(hdr); 3354c717a561Smaybee arc_buf_thaw(buf); 3355fa9e4066Sahrens } 3356ea8dc4b6Seschrock buf->b_efunc = NULL; 3357ea8dc4b6Seschrock buf->b_private = NULL; 3358fa94a07fSbrendan 3359fa94a07fSbrendan if (l2hdr) { 3360aad02571SSaso Kiselkov ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize); 3361fa94a07fSbrendan list_remove(l2hdr->b_dev->l2ad_buflist, hdr); 3362fa94a07fSbrendan kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t)); 3363fa94a07fSbrendan ARCSTAT_INCR(arcstat_l2_size, -buf_size); 3364fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 33656f83844dSMark Maybee } 3366fa9e4066Sahrens } 3367fa9e4066Sahrens 3368fa9e4066Sahrens int 3369fa9e4066Sahrens arc_released(arc_buf_t *buf) 3370fa9e4066Sahrens { 33716f83844dSMark Maybee int released; 33726f83844dSMark Maybee 33733f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 33746f83844dSMark Maybee released = (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon); 33753f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 33766f83844dSMark Maybee return (released); 3377ea8dc4b6Seschrock } 3378ea8dc4b6Seschrock 3379ea8dc4b6Seschrock int 3380ea8dc4b6Seschrock arc_has_callback(arc_buf_t *buf) 3381ea8dc4b6Seschrock { 33826f83844dSMark Maybee int callback; 33836f83844dSMark Maybee 33843f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 33856f83844dSMark Maybee callback = (buf->b_efunc != NULL); 33863f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 33876f83844dSMark Maybee return (callback); 3388fa9e4066Sahrens } 3389fa9e4066Sahrens 3390ea8dc4b6Seschrock #ifdef ZFS_DEBUG 3391ea8dc4b6Seschrock int 3392ea8dc4b6Seschrock arc_referenced(arc_buf_t *buf) 3393ea8dc4b6Seschrock { 33946f83844dSMark Maybee int referenced; 33956f83844dSMark Maybee 33963f9d6ad7SLin Ling mutex_enter(&buf->b_evict_lock); 33976f83844dSMark Maybee referenced = (refcount_count(&buf->b_hdr->b_refcnt)); 33983f9d6ad7SLin Ling mutex_exit(&buf->b_evict_lock); 33996f83844dSMark Maybee return (referenced); 3400ea8dc4b6Seschrock } 3401ea8dc4b6Seschrock #endif 3402ea8dc4b6Seschrock 3403c717a561Smaybee static void 3404c717a561Smaybee arc_write_ready(zio_t *zio) 3405c717a561Smaybee { 3406c717a561Smaybee arc_write_callback_t *callback = zio->io_private; 3407c717a561Smaybee arc_buf_t *buf = callback->awcb_buf; 34080a4e9518Sgw arc_buf_hdr_t *hdr = buf->b_hdr; 3409c717a561Smaybee 3410e14bb325SJeff Bonwick ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt)); 3411e14bb325SJeff Bonwick callback->awcb_ready(zio, buf, callback->awcb_private); 3412e14bb325SJeff Bonwick 34130a4e9518Sgw /* 34140a4e9518Sgw * If the IO is already in progress, then this is a re-write 3415e14bb325SJeff Bonwick * attempt, so we need to thaw and re-compute the cksum. 3416e14bb325SJeff Bonwick * It is the responsibility of the callback to handle the 3417e14bb325SJeff Bonwick * accounting for any re-write attempt. 34180a4e9518Sgw */ 34190a4e9518Sgw if (HDR_IO_IN_PROGRESS(hdr)) { 34200a4e9518Sgw mutex_enter(&hdr->b_freeze_lock); 34210a4e9518Sgw if (hdr->b_freeze_cksum != NULL) { 34220a4e9518Sgw kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 34230a4e9518Sgw hdr->b_freeze_cksum = NULL; 34240a4e9518Sgw } 34250a4e9518Sgw mutex_exit(&hdr->b_freeze_lock); 34260a4e9518Sgw } 3427fa94a07fSbrendan arc_cksum_compute(buf, B_FALSE); 34280a4e9518Sgw hdr->b_flags |= ARC_IO_IN_PROGRESS; 3429c717a561Smaybee } 3430c717a561Smaybee 3431fa9e4066Sahrens static void 3432fa9e4066Sahrens arc_write_done(zio_t *zio) 3433fa9e4066Sahrens { 3434c717a561Smaybee arc_write_callback_t *callback = zio->io_private; 3435c717a561Smaybee arc_buf_t *buf = callback->awcb_buf; 3436c717a561Smaybee arc_buf_hdr_t *hdr = buf->b_hdr; 3437fa9e4066Sahrens 3438b24ab676SJeff Bonwick ASSERT(hdr->b_acb == NULL); 3439b24ab676SJeff Bonwick 3440b24ab676SJeff Bonwick if (zio->io_error == 0) { 3441b24ab676SJeff Bonwick hdr->b_dva = *BP_IDENTITY(zio->io_bp); 3442b24ab676SJeff Bonwick hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp); 3443b24ab676SJeff Bonwick hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0]; 3444b24ab676SJeff Bonwick } else { 3445b24ab676SJeff Bonwick ASSERT(BUF_EMPTY(hdr)); 3446b24ab676SJeff Bonwick } 3447fa9e4066Sahrens 3448ea8dc4b6Seschrock /* 3449ea8dc4b6Seschrock * If the block to be written was all-zero, we may have 3450ea8dc4b6Seschrock * compressed it away. In this case no write was performed 34513f9d6ad7SLin Ling * so there will be no dva/birth/checksum. The buffer must 34523f9d6ad7SLin Ling * therefore remain anonymous (and uncached). 3453ea8dc4b6Seschrock */ 3454fa9e4066Sahrens if (!BUF_EMPTY(hdr)) { 3455fa9e4066Sahrens arc_buf_hdr_t *exists; 3456fa9e4066Sahrens kmutex_t *hash_lock; 3457fa9e4066Sahrens 3458b24ab676SJeff Bonwick ASSERT(zio->io_error == 0); 3459b24ab676SJeff Bonwick 34606b4acc8bSahrens arc_cksum_verify(buf); 34616b4acc8bSahrens 3462fa9e4066Sahrens exists = buf_hash_insert(hdr, &hash_lock); 3463fa9e4066Sahrens if (exists) { 3464fa9e4066Sahrens /* 3465fa9e4066Sahrens * This can only happen if we overwrite for 3466fa9e4066Sahrens * sync-to-convergence, because we remove 3467fa9e4066Sahrens * buffers from the hash table when we arc_free(). 3468fa9e4066Sahrens */ 3469b24ab676SJeff Bonwick if (zio->io_flags & ZIO_FLAG_IO_REWRITE) { 3470b24ab676SJeff Bonwick if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp)) 3471b24ab676SJeff Bonwick panic("bad overwrite, hdr=%p exists=%p", 3472b24ab676SJeff Bonwick (void *)hdr, (void *)exists); 3473b24ab676SJeff Bonwick ASSERT(refcount_is_zero(&exists->b_refcnt)); 3474b24ab676SJeff Bonwick arc_change_state(arc_anon, exists, hash_lock); 3475b24ab676SJeff Bonwick mutex_exit(hash_lock); 3476b24ab676SJeff Bonwick arc_hdr_destroy(exists); 3477b24ab676SJeff Bonwick exists = buf_hash_insert(hdr, &hash_lock); 3478b24ab676SJeff Bonwick ASSERT3P(exists, ==, NULL); 347980901aeaSGeorge Wilson } else if (zio->io_flags & ZIO_FLAG_NOPWRITE) { 348080901aeaSGeorge Wilson /* nopwrite */ 348180901aeaSGeorge Wilson ASSERT(zio->io_prop.zp_nopwrite); 348280901aeaSGeorge Wilson if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp)) 348380901aeaSGeorge Wilson panic("bad nopwrite, hdr=%p exists=%p", 348480901aeaSGeorge Wilson (void *)hdr, (void *)exists); 3485b24ab676SJeff Bonwick } else { 3486b24ab676SJeff Bonwick /* Dedup */ 3487b24ab676SJeff Bonwick ASSERT(hdr->b_datacnt == 1); 3488b24ab676SJeff Bonwick ASSERT(hdr->b_state == arc_anon); 3489b24ab676SJeff Bonwick ASSERT(BP_GET_DEDUP(zio->io_bp)); 3490b24ab676SJeff Bonwick ASSERT(BP_GET_LEVEL(zio->io_bp) == 0); 3491ae46e4c7SMatthew Ahrens } 3492fa9e4066Sahrens } 3493ea8dc4b6Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 3494088f3894Sahrens /* if it's not anon, we are doing a scrub */ 3495b24ab676SJeff Bonwick if (!exists && hdr->b_state == arc_anon) 3496088f3894Sahrens arc_access(hdr, hash_lock); 349744eda4d7Smaybee mutex_exit(hash_lock); 3498ea8dc4b6Seschrock } else { 3499ea8dc4b6Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 3500fa9e4066Sahrens } 3501ea8dc4b6Seschrock 3502b24ab676SJeff Bonwick ASSERT(!refcount_is_zero(&hdr->b_refcnt)); 3503b24ab676SJeff Bonwick callback->awcb_done(zio, buf, callback->awcb_private); 3504fa9e4066Sahrens 3505c717a561Smaybee kmem_free(callback, sizeof (arc_write_callback_t)); 3506fa9e4066Sahrens } 3507fa9e4066Sahrens 3508c717a561Smaybee zio_t * 3509b24ab676SJeff Bonwick arc_write(zio_t *pio, spa_t *spa, uint64_t txg, 3510aad02571SSaso Kiselkov blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, boolean_t l2arc_compress, 3511aad02571SSaso Kiselkov const zio_prop_t *zp, arc_done_func_t *ready, arc_done_func_t *done, 3512aad02571SSaso Kiselkov void *private, int priority, int zio_flags, const zbookmark_t *zb) 3513fa9e4066Sahrens { 3514fa9e4066Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 3515c717a561Smaybee arc_write_callback_t *callback; 3516e14bb325SJeff Bonwick zio_t *zio; 3517fa9e4066Sahrens 3518e14bb325SJeff Bonwick ASSERT(ready != NULL); 3519b24ab676SJeff Bonwick ASSERT(done != NULL); 3520fa9e4066Sahrens ASSERT(!HDR_IO_ERROR(hdr)); 3521c5c6ffa0Smaybee ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0); 3522b24ab676SJeff Bonwick ASSERT(hdr->b_acb == NULL); 35233baa08fcSek if (l2arc) 35243baa08fcSek hdr->b_flags |= ARC_L2CACHE; 3525aad02571SSaso Kiselkov if (l2arc_compress) 3526aad02571SSaso Kiselkov hdr->b_flags |= ARC_L2COMPRESS; 3527c717a561Smaybee callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP); 3528c717a561Smaybee callback->awcb_ready = ready; 3529c717a561Smaybee callback->awcb_done = done; 3530c717a561Smaybee callback->awcb_private = private; 3531c717a561Smaybee callback->awcb_buf = buf; 3532088f3894Sahrens 3533b24ab676SJeff Bonwick zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp, 3534e14bb325SJeff Bonwick arc_write_ready, arc_write_done, callback, priority, zio_flags, zb); 3535fa9e4066Sahrens 3536c717a561Smaybee return (zio); 3537fa9e4066Sahrens } 3538fa9e4066Sahrens 35391ab7f2deSmaybee static int 35402fdbea25SAleksandr Guzovskiy arc_memory_throttle(uint64_t reserve, uint64_t inflight_data, uint64_t txg) 35411ab7f2deSmaybee { 35421ab7f2deSmaybee #ifdef _KERNEL 35431ab7f2deSmaybee uint64_t available_memory = ptob(freemem); 35441ab7f2deSmaybee static uint64_t page_load = 0; 35451ab7f2deSmaybee static uint64_t last_txg = 0; 35461ab7f2deSmaybee 35471ab7f2deSmaybee #if defined(__i386) 35481ab7f2deSmaybee available_memory = 35491ab7f2deSmaybee MIN(available_memory, vmem_size(heap_arena, VMEM_FREE)); 35501ab7f2deSmaybee #endif 35511ab7f2deSmaybee if (available_memory >= zfs_write_limit_max) 35521ab7f2deSmaybee return (0); 35531ab7f2deSmaybee 35541ab7f2deSmaybee if (txg > last_txg) { 35551ab7f2deSmaybee last_txg = txg; 35561ab7f2deSmaybee page_load = 0; 35571ab7f2deSmaybee } 35581ab7f2deSmaybee /* 35591ab7f2deSmaybee * If we are in pageout, we know that memory is already tight, 35601ab7f2deSmaybee * the arc is already going to be evicting, so we just want to 35611ab7f2deSmaybee * continue to let page writes occur as quickly as possible. 35621ab7f2deSmaybee */ 35631ab7f2deSmaybee if (curproc == proc_pageout) { 35641ab7f2deSmaybee if (page_load > MAX(ptob(minfree), available_memory) / 4) 3565be6fd75aSMatthew Ahrens return (SET_ERROR(ERESTART)); 35661ab7f2deSmaybee /* Note: reserve is inflated, so we deflate */ 35671ab7f2deSmaybee page_load += reserve / 8; 35681ab7f2deSmaybee return (0); 35691ab7f2deSmaybee } else if (page_load > 0 && arc_reclaim_needed()) { 35701ab7f2deSmaybee /* memory is low, delay before restarting */ 35711ab7f2deSmaybee ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 3572be6fd75aSMatthew Ahrens return (SET_ERROR(EAGAIN)); 35731ab7f2deSmaybee } 35741ab7f2deSmaybee page_load = 0; 35751ab7f2deSmaybee 35761ab7f2deSmaybee if (arc_size > arc_c_min) { 35771ab7f2deSmaybee uint64_t evictable_memory = 35781ab7f2deSmaybee arc_mru->arcs_lsize[ARC_BUFC_DATA] + 35791ab7f2deSmaybee arc_mru->arcs_lsize[ARC_BUFC_METADATA] + 35801ab7f2deSmaybee arc_mfu->arcs_lsize[ARC_BUFC_DATA] + 35811ab7f2deSmaybee arc_mfu->arcs_lsize[ARC_BUFC_METADATA]; 35821ab7f2deSmaybee available_memory += MIN(evictable_memory, arc_size - arc_c_min); 35831ab7f2deSmaybee } 35841ab7f2deSmaybee 35851ab7f2deSmaybee if (inflight_data > available_memory / 4) { 35861ab7f2deSmaybee ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 3587be6fd75aSMatthew Ahrens return (SET_ERROR(ERESTART)); 35881ab7f2deSmaybee } 35891ab7f2deSmaybee #endif 35901ab7f2deSmaybee return (0); 35911ab7f2deSmaybee } 35921ab7f2deSmaybee 3593fa9e4066Sahrens void 35941ab7f2deSmaybee arc_tempreserve_clear(uint64_t reserve) 3595fa9e4066Sahrens { 35961ab7f2deSmaybee atomic_add_64(&arc_tempreserve, -reserve); 3597fa9e4066Sahrens ASSERT((int64_t)arc_tempreserve >= 0); 3598fa9e4066Sahrens } 3599fa9e4066Sahrens 3600fa9e4066Sahrens int 36011ab7f2deSmaybee arc_tempreserve_space(uint64_t reserve, uint64_t txg) 3602fa9e4066Sahrens { 36031ab7f2deSmaybee int error; 36042fdbea25SAleksandr Guzovskiy uint64_t anon_size; 36051ab7f2deSmaybee 3606fa9e4066Sahrens #ifdef ZFS_DEBUG 3607fa9e4066Sahrens /* 3608fa9e4066Sahrens * Once in a while, fail for no reason. Everything should cope. 3609fa9e4066Sahrens */ 3610fa9e4066Sahrens if (spa_get_random(10000) == 0) { 3611fa9e4066Sahrens dprintf("forcing random failure\n"); 3612be6fd75aSMatthew Ahrens return (SET_ERROR(ERESTART)); 3613fa9e4066Sahrens } 3614fa9e4066Sahrens #endif 36151ab7f2deSmaybee if (reserve > arc_c/4 && !arc_no_grow) 36161ab7f2deSmaybee arc_c = MIN(arc_c_max, reserve * 4); 36171ab7f2deSmaybee if (reserve > arc_c) 3618be6fd75aSMatthew Ahrens return (SET_ERROR(ENOMEM)); 3619112fe045Smaybee 36202fdbea25SAleksandr Guzovskiy /* 36212fdbea25SAleksandr Guzovskiy * Don't count loaned bufs as in flight dirty data to prevent long 36222fdbea25SAleksandr Guzovskiy * network delays from blocking transactions that are ready to be 36232fdbea25SAleksandr Guzovskiy * assigned to a txg. 36242fdbea25SAleksandr Guzovskiy */ 36252fdbea25SAleksandr Guzovskiy anon_size = MAX((int64_t)(arc_anon->arcs_size - arc_loaned_bytes), 0); 36262fdbea25SAleksandr Guzovskiy 36271ab7f2deSmaybee /* 36281ab7f2deSmaybee * Writes will, almost always, require additional memory allocations 3629*f7170741SWill Andrews * in order to compress/encrypt/etc the data. We therefore need to 36301ab7f2deSmaybee * make sure that there is sufficient available memory for this. 36311ab7f2deSmaybee */ 36322fdbea25SAleksandr Guzovskiy if (error = arc_memory_throttle(reserve, anon_size, txg)) 36331ab7f2deSmaybee return (error); 36341ab7f2deSmaybee 3635fa9e4066Sahrens /* 3636112fe045Smaybee * Throttle writes when the amount of dirty data in the cache 3637112fe045Smaybee * gets too large. We try to keep the cache less than half full 3638112fe045Smaybee * of dirty blocks so that our sync times don't grow too large. 3639112fe045Smaybee * Note: if two requests come in concurrently, we might let them 3640112fe045Smaybee * both succeed, when one of them should fail. Not a huge deal. 3641fa9e4066Sahrens */ 36422fdbea25SAleksandr Guzovskiy 36432fdbea25SAleksandr Guzovskiy if (reserve + arc_tempreserve + anon_size > arc_c / 2 && 36442fdbea25SAleksandr Guzovskiy anon_size > arc_c / 4) { 36450e8c6158Smaybee dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK " 36460e8c6158Smaybee "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n", 36470e8c6158Smaybee arc_tempreserve>>10, 36480e8c6158Smaybee arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10, 36490e8c6158Smaybee arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10, 36501ab7f2deSmaybee reserve>>10, arc_c>>10); 3651be6fd75aSMatthew Ahrens return (SET_ERROR(ERESTART)); 3652fa9e4066Sahrens } 36531ab7f2deSmaybee atomic_add_64(&arc_tempreserve, reserve); 3654fa9e4066Sahrens return (0); 3655fa9e4066Sahrens } 3656fa9e4066Sahrens 3657fa9e4066Sahrens void 3658fa9e4066Sahrens arc_init(void) 3659fa9e4066Sahrens { 3660fa9e4066Sahrens mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL); 3661fa9e4066Sahrens cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL); 3662fa9e4066Sahrens 366313506d1eSmaybee /* Convert seconds to clock ticks */ 3664b19a79ecSperrin arc_min_prefetch_lifespan = 1 * hz; 366513506d1eSmaybee 3666fa9e4066Sahrens /* Start out with 1/8 of all memory */ 366744cb6abcSbmc arc_c = physmem * PAGESIZE / 8; 3668fa9e4066Sahrens 3669fa9e4066Sahrens #ifdef _KERNEL 3670fa9e4066Sahrens /* 3671fa9e4066Sahrens * On architectures where the physical memory can be larger 3672fa9e4066Sahrens * than the addressable space (intel in 32-bit mode), we may 3673fa9e4066Sahrens * need to limit the cache to 1/8 of VM size. 3674fa9e4066Sahrens */ 367544cb6abcSbmc arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8); 3676fa9e4066Sahrens #endif 3677fa9e4066Sahrens 3678112fe045Smaybee /* set min cache to 1/32 of all memory, or 64MB, whichever is more */ 367944cb6abcSbmc arc_c_min = MAX(arc_c / 4, 64<<20); 3680112fe045Smaybee /* set max to 3/4 of all memory, or all but 1GB, whichever is more */ 368144cb6abcSbmc if (arc_c * 8 >= 1<<30) 368244cb6abcSbmc arc_c_max = (arc_c * 8) - (1<<30); 3683fa9e4066Sahrens else 368444cb6abcSbmc arc_c_max = arc_c_min; 368544cb6abcSbmc arc_c_max = MAX(arc_c * 6, arc_c_max); 3686a2eea2e1Sahrens 3687a2eea2e1Sahrens /* 3688a2eea2e1Sahrens * Allow the tunables to override our calculations if they are 3689a2eea2e1Sahrens * reasonable (ie. over 64MB) 3690a2eea2e1Sahrens */ 3691a2eea2e1Sahrens if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE) 369244cb6abcSbmc arc_c_max = zfs_arc_max; 369344cb6abcSbmc if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max) 369444cb6abcSbmc arc_c_min = zfs_arc_min; 3695a2eea2e1Sahrens 369644cb6abcSbmc arc_c = arc_c_max; 369744cb6abcSbmc arc_p = (arc_c >> 1); 3698fa9e4066Sahrens 36990e8c6158Smaybee /* limit meta-data to 1/4 of the arc capacity */ 37000e8c6158Smaybee arc_meta_limit = arc_c_max / 4; 37011116048bSek 37021116048bSek /* Allow the tunable to override if it is reasonable */ 37031116048bSek if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max) 37041116048bSek arc_meta_limit = zfs_arc_meta_limit; 37051116048bSek 37060e8c6158Smaybee if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0) 37070e8c6158Smaybee arc_c_min = arc_meta_limit / 2; 37080e8c6158Smaybee 37095a98e54bSBrendan Gregg - Sun Microsystems if (zfs_arc_grow_retry > 0) 37105a98e54bSBrendan Gregg - Sun Microsystems arc_grow_retry = zfs_arc_grow_retry; 37115a98e54bSBrendan Gregg - Sun Microsystems 37125a98e54bSBrendan Gregg - Sun Microsystems if (zfs_arc_shrink_shift > 0) 37135a98e54bSBrendan Gregg - Sun Microsystems arc_shrink_shift = zfs_arc_shrink_shift; 37145a98e54bSBrendan Gregg - Sun Microsystems 37155a98e54bSBrendan Gregg - Sun Microsystems if (zfs_arc_p_min_shift > 0) 37165a98e54bSBrendan Gregg - Sun Microsystems arc_p_min_shift = zfs_arc_p_min_shift; 37175a98e54bSBrendan Gregg - Sun Microsystems 3718fa9e4066Sahrens /* if kmem_flags are set, lets try to use less memory */ 3719fa9e4066Sahrens if (kmem_debugging()) 372044cb6abcSbmc arc_c = arc_c / 2; 372144cb6abcSbmc if (arc_c < arc_c_min) 372244cb6abcSbmc arc_c = arc_c_min; 372344cb6abcSbmc 372444cb6abcSbmc arc_anon = &ARC_anon; 372544cb6abcSbmc arc_mru = &ARC_mru; 372644cb6abcSbmc arc_mru_ghost = &ARC_mru_ghost; 372744cb6abcSbmc arc_mfu = &ARC_mfu; 372844cb6abcSbmc arc_mfu_ghost = &ARC_mfu_ghost; 3729fa94a07fSbrendan arc_l2c_only = &ARC_l2c_only; 373044cb6abcSbmc arc_size = 0; 373144cb6abcSbmc 373244cb6abcSbmc mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 373344cb6abcSbmc mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 373444cb6abcSbmc mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 373544cb6abcSbmc mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 373644cb6abcSbmc mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3737fa94a07fSbrendan mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 373844cb6abcSbmc 37390e8c6158Smaybee list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA], 37400e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 37410e8c6158Smaybee list_create(&arc_mru->arcs_list[ARC_BUFC_DATA], 37420e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 37430e8c6158Smaybee list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA], 37440e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 37450e8c6158Smaybee list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA], 37460e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 37470e8c6158Smaybee list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA], 37480e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 37490e8c6158Smaybee list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA], 37500e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 37510e8c6158Smaybee list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA], 37520e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 37530e8c6158Smaybee list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA], 37540e8c6158Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3755fa94a07fSbrendan list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA], 3756fa94a07fSbrendan sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3757fa94a07fSbrendan list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA], 3758fa94a07fSbrendan sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3759fa9e4066Sahrens 3760fa9e4066Sahrens buf_init(); 3761fa9e4066Sahrens 3762fa9e4066Sahrens arc_thread_exit = 0; 3763ea8dc4b6Seschrock arc_eviction_list = NULL; 3764ea8dc4b6Seschrock mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL); 376540d7d650Smaybee bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t)); 3766fa9e4066Sahrens 376744cb6abcSbmc arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED, 376844cb6abcSbmc sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 376944cb6abcSbmc 377044cb6abcSbmc if (arc_ksp != NULL) { 377144cb6abcSbmc arc_ksp->ks_data = &arc_stats; 377244cb6abcSbmc kstat_install(arc_ksp); 377344cb6abcSbmc } 377444cb6abcSbmc 3775fa9e4066Sahrens (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0, 3776fa9e4066Sahrens TS_RUN, minclsyspri); 377749e3519aSmaybee 377849e3519aSmaybee arc_dead = FALSE; 37793a737e0dSbrendan arc_warm = B_FALSE; 37801ab7f2deSmaybee 37811ab7f2deSmaybee if (zfs_write_limit_max == 0) 378205715f94SMark Maybee zfs_write_limit_max = ptob(physmem) >> zfs_write_limit_shift; 37831ab7f2deSmaybee else 37841ab7f2deSmaybee zfs_write_limit_shift = 0; 378505715f94SMark Maybee mutex_init(&zfs_write_limit_lock, NULL, MUTEX_DEFAULT, NULL); 3786fa9e4066Sahrens } 3787fa9e4066Sahrens 3788fa9e4066Sahrens void 3789fa9e4066Sahrens arc_fini(void) 3790fa9e4066Sahrens { 3791fa9e4066Sahrens mutex_enter(&arc_reclaim_thr_lock); 3792fa9e4066Sahrens arc_thread_exit = 1; 3793fa9e4066Sahrens while (arc_thread_exit != 0) 3794fa9e4066Sahrens cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock); 3795fa9e4066Sahrens mutex_exit(&arc_reclaim_thr_lock); 3796fa9e4066Sahrens 3797874395d5Smaybee arc_flush(NULL); 3798fa9e4066Sahrens 3799fa9e4066Sahrens arc_dead = TRUE; 3800fa9e4066Sahrens 380144cb6abcSbmc if (arc_ksp != NULL) { 380244cb6abcSbmc kstat_delete(arc_ksp); 380344cb6abcSbmc arc_ksp = NULL; 380444cb6abcSbmc } 380544cb6abcSbmc 3806ea8dc4b6Seschrock mutex_destroy(&arc_eviction_mtx); 3807fa9e4066Sahrens mutex_destroy(&arc_reclaim_thr_lock); 3808fa9e4066Sahrens cv_destroy(&arc_reclaim_thr_cv); 3809fa9e4066Sahrens 38100e8c6158Smaybee list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]); 38110e8c6158Smaybee list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]); 38120e8c6158Smaybee list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]); 38130e8c6158Smaybee list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]); 38140e8c6158Smaybee list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]); 38150e8c6158Smaybee list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]); 38160e8c6158Smaybee list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]); 38170e8c6158Smaybee list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]); 3818fa9e4066Sahrens 381944cb6abcSbmc mutex_destroy(&arc_anon->arcs_mtx); 382044cb6abcSbmc mutex_destroy(&arc_mru->arcs_mtx); 382144cb6abcSbmc mutex_destroy(&arc_mru_ghost->arcs_mtx); 382244cb6abcSbmc mutex_destroy(&arc_mfu->arcs_mtx); 382344cb6abcSbmc mutex_destroy(&arc_mfu_ghost->arcs_mtx); 3824b5e70f97SRicardo M. Correia mutex_destroy(&arc_l2c_only->arcs_mtx); 38255ad82045Snd 382605715f94SMark Maybee mutex_destroy(&zfs_write_limit_lock); 382705715f94SMark Maybee 3828fa9e4066Sahrens buf_fini(); 38292fdbea25SAleksandr Guzovskiy 38302fdbea25SAleksandr Guzovskiy ASSERT(arc_loaned_bytes == 0); 3831fa9e4066Sahrens } 3832fa94a07fSbrendan 3833fa94a07fSbrendan /* 3834fa94a07fSbrendan * Level 2 ARC 3835fa94a07fSbrendan * 3836fa94a07fSbrendan * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk. 3837fa94a07fSbrendan * It uses dedicated storage devices to hold cached data, which are populated 3838fa94a07fSbrendan * using large infrequent writes. The main role of this cache is to boost 3839fa94a07fSbrendan * the performance of random read workloads. The intended L2ARC devices 3840fa94a07fSbrendan * include short-stroked disks, solid state disks, and other media with 3841fa94a07fSbrendan * substantially faster read latency than disk. 3842fa94a07fSbrendan * 3843fa94a07fSbrendan * +-----------------------+ 3844fa94a07fSbrendan * | ARC | 3845fa94a07fSbrendan * +-----------------------+ 3846fa94a07fSbrendan * | ^ ^ 3847fa94a07fSbrendan * | | | 3848fa94a07fSbrendan * l2arc_feed_thread() arc_read() 3849fa94a07fSbrendan * | | | 3850fa94a07fSbrendan * | l2arc read | 3851fa94a07fSbrendan * V | | 3852fa94a07fSbrendan * +---------------+ | 3853fa94a07fSbrendan * | L2ARC | | 3854fa94a07fSbrendan * +---------------+ | 3855fa94a07fSbrendan * | ^ | 3856fa94a07fSbrendan * l2arc_write() | | 3857fa94a07fSbrendan * | | | 3858fa94a07fSbrendan * V | | 3859fa94a07fSbrendan * +-------+ +-------+ 3860fa94a07fSbrendan * | vdev | | vdev | 3861fa94a07fSbrendan * | cache | | cache | 3862fa94a07fSbrendan * +-------+ +-------+ 3863fa94a07fSbrendan * +=========+ .-----. 3864fa94a07fSbrendan * : L2ARC : |-_____-| 3865fa94a07fSbrendan * : devices : | Disks | 3866fa94a07fSbrendan * +=========+ `-_____-' 3867fa94a07fSbrendan * 3868fa94a07fSbrendan * Read requests are satisfied from the following sources, in order: 3869fa94a07fSbrendan * 3870fa94a07fSbrendan * 1) ARC 3871fa94a07fSbrendan * 2) vdev cache of L2ARC devices 3872fa94a07fSbrendan * 3) L2ARC devices 3873fa94a07fSbrendan * 4) vdev cache of disks 3874fa94a07fSbrendan * 5) disks 3875fa94a07fSbrendan * 3876fa94a07fSbrendan * Some L2ARC device types exhibit extremely slow write performance. 3877fa94a07fSbrendan * To accommodate for this there are some significant differences between 3878fa94a07fSbrendan * the L2ARC and traditional cache design: 3879fa94a07fSbrendan * 3880fa94a07fSbrendan * 1. There is no eviction path from the ARC to the L2ARC. Evictions from 3881fa94a07fSbrendan * the ARC behave as usual, freeing buffers and placing headers on ghost 3882fa94a07fSbrendan * lists. The ARC does not send buffers to the L2ARC during eviction as 3883fa94a07fSbrendan * this would add inflated write latencies for all ARC memory pressure. 3884fa94a07fSbrendan * 3885fa94a07fSbrendan * 2. The L2ARC attempts to cache data from the ARC before it is evicted. 3886fa94a07fSbrendan * It does this by periodically scanning buffers from the eviction-end of 3887fa94a07fSbrendan * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are 3888aad02571SSaso Kiselkov * not already there. It scans until a headroom of buffers is satisfied, 3889aad02571SSaso Kiselkov * which itself is a buffer for ARC eviction. If a compressible buffer is 3890aad02571SSaso Kiselkov * found during scanning and selected for writing to an L2ARC device, we 3891aad02571SSaso Kiselkov * temporarily boost scanning headroom during the next scan cycle to make 3892aad02571SSaso Kiselkov * sure we adapt to compression effects (which might significantly reduce 3893aad02571SSaso Kiselkov * the data volume we write to L2ARC). The thread that does this is 3894fa94a07fSbrendan * l2arc_feed_thread(), illustrated below; example sizes are included to 3895fa94a07fSbrendan * provide a better sense of ratio than this diagram: 3896fa94a07fSbrendan * 3897fa94a07fSbrendan * head --> tail 3898fa94a07fSbrendan * +---------------------+----------+ 3899fa94a07fSbrendan * ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->. # already on L2ARC 3900fa94a07fSbrendan * +---------------------+----------+ | o L2ARC eligible 3901fa94a07fSbrendan * ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->| : ARC buffer 3902fa94a07fSbrendan * +---------------------+----------+ | 3903fa94a07fSbrendan * 15.9 Gbytes ^ 32 Mbytes | 3904fa94a07fSbrendan * headroom | 3905fa94a07fSbrendan * l2arc_feed_thread() 3906fa94a07fSbrendan * | 3907fa94a07fSbrendan * l2arc write hand <--[oooo]--' 3908fa94a07fSbrendan * | 8 Mbyte 3909fa94a07fSbrendan * | write max 3910fa94a07fSbrendan * V 3911fa94a07fSbrendan * +==============================+ 3912fa94a07fSbrendan * L2ARC dev |####|#|###|###| |####| ... | 3913fa94a07fSbrendan * +==============================+ 3914fa94a07fSbrendan * 32 Gbytes 3915fa94a07fSbrendan * 3916fa94a07fSbrendan * 3. If an ARC buffer is copied to the L2ARC but then hit instead of 3917fa94a07fSbrendan * evicted, then the L2ARC has cached a buffer much sooner than it probably 3918fa94a07fSbrendan * needed to, potentially wasting L2ARC device bandwidth and storage. It is 3919fa94a07fSbrendan * safe to say that this is an uncommon case, since buffers at the end of 3920fa94a07fSbrendan * the ARC lists have moved there due to inactivity. 3921fa94a07fSbrendan * 3922fa94a07fSbrendan * 4. If the ARC evicts faster than the L2ARC can maintain a headroom, 3923fa94a07fSbrendan * then the L2ARC simply misses copying some buffers. This serves as a 3924fa94a07fSbrendan * pressure valve to prevent heavy read workloads from both stalling the ARC 3925fa94a07fSbrendan * with waits and clogging the L2ARC with writes. This also helps prevent 3926fa94a07fSbrendan * the potential for the L2ARC to churn if it attempts to cache content too 3927fa94a07fSbrendan * quickly, such as during backups of the entire pool. 3928fa94a07fSbrendan * 39293a737e0dSbrendan * 5. After system boot and before the ARC has filled main memory, there are 39303a737e0dSbrendan * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru 39313a737e0dSbrendan * lists can remain mostly static. Instead of searching from tail of these 39323a737e0dSbrendan * lists as pictured, the l2arc_feed_thread() will search from the list heads 39333a737e0dSbrendan * for eligible buffers, greatly increasing its chance of finding them. 39343a737e0dSbrendan * 39353a737e0dSbrendan * The L2ARC device write speed is also boosted during this time so that 39363a737e0dSbrendan * the L2ARC warms up faster. Since there have been no ARC evictions yet, 39373a737e0dSbrendan * there are no L2ARC reads, and no fear of degrading read performance 39383a737e0dSbrendan * through increased writes. 39393a737e0dSbrendan * 39403a737e0dSbrendan * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that 3941fa94a07fSbrendan * the vdev queue can aggregate them into larger and fewer writes. Each 3942fa94a07fSbrendan * device is written to in a rotor fashion, sweeping writes through 3943fa94a07fSbrendan * available space then repeating. 3944fa94a07fSbrendan * 39453a737e0dSbrendan * 7. The L2ARC does not store dirty content. It never needs to flush 3946fa94a07fSbrendan * write buffers back to disk based storage. 3947fa94a07fSbrendan * 39483a737e0dSbrendan * 8. If an ARC buffer is written (and dirtied) which also exists in the 3949fa94a07fSbrendan * L2ARC, the now stale L2ARC buffer is immediately dropped. 3950fa94a07fSbrendan * 3951fa94a07fSbrendan * The performance of the L2ARC can be tweaked by a number of tunables, which 3952fa94a07fSbrendan * may be necessary for different workloads: 3953fa94a07fSbrendan * 3954fa94a07fSbrendan * l2arc_write_max max write bytes per interval 39553a737e0dSbrendan * l2arc_write_boost extra write bytes during device warmup 3956fa94a07fSbrendan * l2arc_noprefetch skip caching prefetched buffers 3957fa94a07fSbrendan * l2arc_headroom number of max device writes to precache 3958aad02571SSaso Kiselkov * l2arc_headroom_boost when we find compressed buffers during ARC 3959aad02571SSaso Kiselkov * scanning, we multiply headroom by this 3960aad02571SSaso Kiselkov * percentage factor for the next scan cycle, 3961aad02571SSaso Kiselkov * since more compressed buffers are likely to 3962aad02571SSaso Kiselkov * be present 3963fa94a07fSbrendan * l2arc_feed_secs seconds between L2ARC writing 3964fa94a07fSbrendan * 3965fa94a07fSbrendan * Tunables may be removed or added as future performance improvements are 3966fa94a07fSbrendan * integrated, and also may become zpool properties. 39675a98e54bSBrendan Gregg - Sun Microsystems * 39685a98e54bSBrendan Gregg - Sun Microsystems * There are three key functions that control how the L2ARC warms up: 39695a98e54bSBrendan Gregg - Sun Microsystems * 39705a98e54bSBrendan Gregg - Sun Microsystems * l2arc_write_eligible() check if a buffer is eligible to cache 39715a98e54bSBrendan Gregg - Sun Microsystems * l2arc_write_size() calculate how much to write 39725a98e54bSBrendan Gregg - Sun Microsystems * l2arc_write_interval() calculate sleep delay between writes 39735a98e54bSBrendan Gregg - Sun Microsystems * 39745a98e54bSBrendan Gregg - Sun Microsystems * These three functions determine what to write, how much, and how quickly 39755a98e54bSBrendan Gregg - Sun Microsystems * to send writes. 3976fa94a07fSbrendan */ 3977fa94a07fSbrendan 39785a98e54bSBrendan Gregg - Sun Microsystems static boolean_t 3979ac05c741SMark Maybee l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab) 39805a98e54bSBrendan Gregg - Sun Microsystems { 39815a98e54bSBrendan Gregg - Sun Microsystems /* 39825a98e54bSBrendan Gregg - Sun Microsystems * A buffer is *not* eligible for the L2ARC if it: 39835a98e54bSBrendan Gregg - Sun Microsystems * 1. belongs to a different spa. 39845ea40c06SBrendan Gregg - Sun Microsystems * 2. is already cached on the L2ARC. 39855ea40c06SBrendan Gregg - Sun Microsystems * 3. has an I/O in progress (it may be an incomplete read). 39865ea40c06SBrendan Gregg - Sun Microsystems * 4. is flagged not eligible (zfs property). 39875a98e54bSBrendan Gregg - Sun Microsystems */ 39885ea40c06SBrendan Gregg - Sun Microsystems if (ab->b_spa != spa_guid || ab->b_l2hdr != NULL || 39895a98e54bSBrendan Gregg - Sun Microsystems HDR_IO_IN_PROGRESS(ab) || !HDR_L2CACHE(ab)) 39905a98e54bSBrendan Gregg - Sun Microsystems return (B_FALSE); 39915a98e54bSBrendan Gregg - Sun Microsystems 39925a98e54bSBrendan Gregg - Sun Microsystems return (B_TRUE); 39935a98e54bSBrendan Gregg - Sun Microsystems } 39945a98e54bSBrendan Gregg - Sun Microsystems 39955a98e54bSBrendan Gregg - Sun Microsystems static uint64_t 3996aad02571SSaso Kiselkov l2arc_write_size(void) 39975a98e54bSBrendan Gregg - Sun Microsystems { 39985a98e54bSBrendan Gregg - Sun Microsystems uint64_t size; 39995a98e54bSBrendan Gregg - Sun Microsystems 4000aad02571SSaso Kiselkov /* 4001aad02571SSaso Kiselkov * Make sure our globals have meaningful values in case the user 4002aad02571SSaso Kiselkov * altered them. 4003aad02571SSaso Kiselkov */ 4004aad02571SSaso Kiselkov size = l2arc_write_max; 4005aad02571SSaso Kiselkov if (size == 0) { 4006aad02571SSaso Kiselkov cmn_err(CE_NOTE, "Bad value for l2arc_write_max, value must " 4007aad02571SSaso Kiselkov "be greater than zero, resetting it to the default (%d)", 4008aad02571SSaso Kiselkov L2ARC_WRITE_SIZE); 4009aad02571SSaso Kiselkov size = l2arc_write_max = L2ARC_WRITE_SIZE; 4010aad02571SSaso Kiselkov } 40115a98e54bSBrendan Gregg - Sun Microsystems 40125a98e54bSBrendan Gregg - Sun Microsystems if (arc_warm == B_FALSE) 4013aad02571SSaso Kiselkov size += l2arc_write_boost; 40145a98e54bSBrendan Gregg - Sun Microsystems 40155a98e54bSBrendan Gregg - Sun Microsystems return (size); 40165a98e54bSBrendan Gregg - Sun Microsystems 40175a98e54bSBrendan Gregg - Sun Microsystems } 40185a98e54bSBrendan Gregg - Sun Microsystems 40195a98e54bSBrendan Gregg - Sun Microsystems static clock_t 40205a98e54bSBrendan Gregg - Sun Microsystems l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote) 40215a98e54bSBrendan Gregg - Sun Microsystems { 4022d3d50737SRafael Vanoni clock_t interval, next, now; 40235a98e54bSBrendan Gregg - Sun Microsystems 40245a98e54bSBrendan Gregg - Sun Microsystems /* 40255a98e54bSBrendan Gregg - Sun Microsystems * If the ARC lists are busy, increase our write rate; if the 40265a98e54bSBrendan Gregg - Sun Microsystems * lists are stale, idle back. This is achieved by checking 40275a98e54bSBrendan Gregg - Sun Microsystems * how much we previously wrote - if it was more than half of 40285a98e54bSBrendan Gregg - Sun Microsystems * what we wanted, schedule the next write much sooner. 40295a98e54bSBrendan Gregg - Sun Microsystems */ 40305a98e54bSBrendan Gregg - Sun Microsystems if (l2arc_feed_again && wrote > (wanted / 2)) 40315a98e54bSBrendan Gregg - Sun Microsystems interval = (hz * l2arc_feed_min_ms) / 1000; 40325a98e54bSBrendan Gregg - Sun Microsystems else 40335a98e54bSBrendan Gregg - Sun Microsystems interval = hz * l2arc_feed_secs; 40345a98e54bSBrendan Gregg - Sun Microsystems 4035d3d50737SRafael Vanoni now = ddi_get_lbolt(); 4036d3d50737SRafael Vanoni next = MAX(now, MIN(now + interval, began + interval)); 40375a98e54bSBrendan Gregg - Sun Microsystems 40385a98e54bSBrendan Gregg - Sun Microsystems return (next); 40395a98e54bSBrendan Gregg - Sun Microsystems } 40405a98e54bSBrendan Gregg - Sun Microsystems 4041fa94a07fSbrendan static void 4042fa94a07fSbrendan l2arc_hdr_stat_add(void) 4043fa94a07fSbrendan { 4044e6c728e1Sbrendan ARCSTAT_INCR(arcstat_l2_hdr_size, HDR_SIZE + L2HDR_SIZE); 4045e6c728e1Sbrendan ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE); 4046fa94a07fSbrendan } 4047fa94a07fSbrendan 4048fa94a07fSbrendan static void 4049fa94a07fSbrendan l2arc_hdr_stat_remove(void) 4050fa94a07fSbrendan { 4051e6c728e1Sbrendan ARCSTAT_INCR(arcstat_l2_hdr_size, -(HDR_SIZE + L2HDR_SIZE)); 4052e6c728e1Sbrendan ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE); 4053fa94a07fSbrendan } 4054fa94a07fSbrendan 4055fa94a07fSbrendan /* 4056fa94a07fSbrendan * Cycle through L2ARC devices. This is how L2ARC load balances. 40573a737e0dSbrendan * If a device is returned, this also returns holding the spa config lock. 4058fa94a07fSbrendan */ 4059fa94a07fSbrendan static l2arc_dev_t * 4060fa94a07fSbrendan l2arc_dev_get_next(void) 4061fa94a07fSbrendan { 40623a737e0dSbrendan l2arc_dev_t *first, *next = NULL; 40633a737e0dSbrendan 40643a737e0dSbrendan /* 40653a737e0dSbrendan * Lock out the removal of spas (spa_namespace_lock), then removal 40663a737e0dSbrendan * of cache devices (l2arc_dev_mtx). Once a device has been selected, 40673a737e0dSbrendan * both locks will be dropped and a spa config lock held instead. 40683a737e0dSbrendan */ 40693a737e0dSbrendan mutex_enter(&spa_namespace_lock); 40703a737e0dSbrendan mutex_enter(&l2arc_dev_mtx); 4071fa94a07fSbrendan 4072c5904d13Seschrock /* if there are no vdevs, there is nothing to do */ 4073c5904d13Seschrock if (l2arc_ndev == 0) 40743a737e0dSbrendan goto out; 4075c5904d13Seschrock 4076c5904d13Seschrock first = NULL; 4077c5904d13Seschrock next = l2arc_dev_last; 4078c5904d13Seschrock do { 4079c5904d13Seschrock /* loop around the list looking for a non-faulted vdev */ 4080c5904d13Seschrock if (next == NULL) { 4081fa94a07fSbrendan next = list_head(l2arc_dev_list); 4082c5904d13Seschrock } else { 4083c5904d13Seschrock next = list_next(l2arc_dev_list, next); 4084c5904d13Seschrock if (next == NULL) 4085c5904d13Seschrock next = list_head(l2arc_dev_list); 4086c5904d13Seschrock } 4087c5904d13Seschrock 4088c5904d13Seschrock /* if we have come back to the start, bail out */ 4089c5904d13Seschrock if (first == NULL) 4090c5904d13Seschrock first = next; 4091c5904d13Seschrock else if (next == first) 4092c5904d13Seschrock break; 4093c5904d13Seschrock 4094c5904d13Seschrock } while (vdev_is_dead(next->l2ad_vdev)); 4095c5904d13Seschrock 4096c5904d13Seschrock /* if we were unable to find any usable vdevs, return NULL */ 4097c5904d13Seschrock if (vdev_is_dead(next->l2ad_vdev)) 40983a737e0dSbrendan next = NULL; 4099fa94a07fSbrendan 4100fa94a07fSbrendan l2arc_dev_last = next; 4101fa94a07fSbrendan 41023a737e0dSbrendan out: 41033a737e0dSbrendan mutex_exit(&l2arc_dev_mtx); 41043a737e0dSbrendan 41053a737e0dSbrendan /* 41063a737e0dSbrendan * Grab the config lock to prevent the 'next' device from being 41073a737e0dSbrendan * removed while we are writing to it. 41083a737e0dSbrendan */ 41093a737e0dSbrendan if (next != NULL) 4110e14bb325SJeff Bonwick spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER); 41113a737e0dSbrendan mutex_exit(&spa_namespace_lock); 41123a737e0dSbrendan 4113fa94a07fSbrendan return (next); 4114fa94a07fSbrendan } 4115fa94a07fSbrendan 41163a737e0dSbrendan /* 41173a737e0dSbrendan * Free buffers that were tagged for destruction. 41183a737e0dSbrendan */ 41193a737e0dSbrendan static void 41203a737e0dSbrendan l2arc_do_free_on_write() 41213a737e0dSbrendan { 41223a737e0dSbrendan list_t *buflist; 41233a737e0dSbrendan l2arc_data_free_t *df, *df_prev; 41243a737e0dSbrendan 41253a737e0dSbrendan mutex_enter(&l2arc_free_on_write_mtx); 41263a737e0dSbrendan buflist = l2arc_free_on_write; 41273a737e0dSbrendan 41283a737e0dSbrendan for (df = list_tail(buflist); df; df = df_prev) { 41293a737e0dSbrendan df_prev = list_prev(buflist, df); 41303a737e0dSbrendan ASSERT(df->l2df_data != NULL); 41313a737e0dSbrendan ASSERT(df->l2df_func != NULL); 41323a737e0dSbrendan df->l2df_func(df->l2df_data, df->l2df_size); 41333a737e0dSbrendan list_remove(buflist, df); 41343a737e0dSbrendan kmem_free(df, sizeof (l2arc_data_free_t)); 41353a737e0dSbrendan } 41363a737e0dSbrendan 41373a737e0dSbrendan mutex_exit(&l2arc_free_on_write_mtx); 41383a737e0dSbrendan } 41393a737e0dSbrendan 4140fa94a07fSbrendan /* 4141fa94a07fSbrendan * A write to a cache device has completed. Update all headers to allow 4142fa94a07fSbrendan * reads from these buffers to begin. 4143fa94a07fSbrendan */ 4144fa94a07fSbrendan static void 4145fa94a07fSbrendan l2arc_write_done(zio_t *zio) 4146fa94a07fSbrendan { 4147fa94a07fSbrendan l2arc_write_callback_t *cb; 4148fa94a07fSbrendan l2arc_dev_t *dev; 4149fa94a07fSbrendan list_t *buflist; 4150fa94a07fSbrendan arc_buf_hdr_t *head, *ab, *ab_prev; 41513a737e0dSbrendan l2arc_buf_hdr_t *abl2; 4152fa94a07fSbrendan kmutex_t *hash_lock; 4153fa94a07fSbrendan 4154fa94a07fSbrendan cb = zio->io_private; 4155fa94a07fSbrendan ASSERT(cb != NULL); 4156fa94a07fSbrendan dev = cb->l2wcb_dev; 4157fa94a07fSbrendan ASSERT(dev != NULL); 4158fa94a07fSbrendan head = cb->l2wcb_head; 4159fa94a07fSbrendan ASSERT(head != NULL); 4160fa94a07fSbrendan buflist = dev->l2ad_buflist; 4161fa94a07fSbrendan ASSERT(buflist != NULL); 4162fa94a07fSbrendan DTRACE_PROBE2(l2arc__iodone, zio_t *, zio, 4163fa94a07fSbrendan l2arc_write_callback_t *, cb); 4164fa94a07fSbrendan 4165fa94a07fSbrendan if (zio->io_error != 0) 4166fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_writes_error); 4167fa94a07fSbrendan 4168fa94a07fSbrendan mutex_enter(&l2arc_buflist_mtx); 4169fa94a07fSbrendan 4170fa94a07fSbrendan /* 4171fa94a07fSbrendan * All writes completed, or an error was hit. 4172fa94a07fSbrendan */ 4173fa94a07fSbrendan for (ab = list_prev(buflist, head); ab; ab = ab_prev) { 4174fa94a07fSbrendan ab_prev = list_prev(buflist, ab); 4175fa94a07fSbrendan 4176fa94a07fSbrendan hash_lock = HDR_LOCK(ab); 4177fa94a07fSbrendan if (!mutex_tryenter(hash_lock)) { 4178fa94a07fSbrendan /* 4179fa94a07fSbrendan * This buffer misses out. It may be in a stage 4180fa94a07fSbrendan * of eviction. Its ARC_L2_WRITING flag will be 4181fa94a07fSbrendan * left set, denying reads to this buffer. 4182fa94a07fSbrendan */ 4183fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss); 4184fa94a07fSbrendan continue; 4185fa94a07fSbrendan } 4186fa94a07fSbrendan 4187aad02571SSaso Kiselkov abl2 = ab->b_l2hdr; 4188aad02571SSaso Kiselkov 4189aad02571SSaso Kiselkov /* 4190aad02571SSaso Kiselkov * Release the temporary compressed buffer as soon as possible. 4191aad02571SSaso Kiselkov */ 4192aad02571SSaso Kiselkov if (abl2->b_compress != ZIO_COMPRESS_OFF) 4193aad02571SSaso Kiselkov l2arc_release_cdata_buf(ab); 4194aad02571SSaso Kiselkov 4195fa94a07fSbrendan if (zio->io_error != 0) { 4196fa94a07fSbrendan /* 41973a737e0dSbrendan * Error - drop L2ARC entry. 4198fa94a07fSbrendan */ 41993a737e0dSbrendan list_remove(buflist, ab); 4200aad02571SSaso Kiselkov ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize); 4201fa94a07fSbrendan ab->b_l2hdr = NULL; 42023a737e0dSbrendan kmem_free(abl2, sizeof (l2arc_buf_hdr_t)); 42033a737e0dSbrendan ARCSTAT_INCR(arcstat_l2_size, -ab->b_size); 4204fa94a07fSbrendan } 4205fa94a07fSbrendan 4206fa94a07fSbrendan /* 4207fa94a07fSbrendan * Allow ARC to begin reads to this L2ARC entry. 4208fa94a07fSbrendan */ 4209fa94a07fSbrendan ab->b_flags &= ~ARC_L2_WRITING; 4210fa94a07fSbrendan 4211fa94a07fSbrendan mutex_exit(hash_lock); 4212fa94a07fSbrendan } 4213fa94a07fSbrendan 4214fa94a07fSbrendan atomic_inc_64(&l2arc_writes_done); 4215fa94a07fSbrendan list_remove(buflist, head); 4216fa94a07fSbrendan kmem_cache_free(hdr_cache, head); 4217fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 4218fa94a07fSbrendan 42193a737e0dSbrendan l2arc_do_free_on_write(); 4220fa94a07fSbrendan 4221fa94a07fSbrendan kmem_free(cb, sizeof (l2arc_write_callback_t)); 4222fa94a07fSbrendan } 4223fa94a07fSbrendan 4224fa94a07fSbrendan /* 4225fa94a07fSbrendan * A read to a cache device completed. Validate buffer contents before 4226fa94a07fSbrendan * handing over to the regular ARC routines. 4227fa94a07fSbrendan */ 4228fa94a07fSbrendan static void 4229fa94a07fSbrendan l2arc_read_done(zio_t *zio) 4230fa94a07fSbrendan { 4231fa94a07fSbrendan l2arc_read_callback_t *cb; 4232fa94a07fSbrendan arc_buf_hdr_t *hdr; 4233fa94a07fSbrendan arc_buf_t *buf; 4234fa94a07fSbrendan kmutex_t *hash_lock; 42353a737e0dSbrendan int equal; 4236fa94a07fSbrendan 4237e14bb325SJeff Bonwick ASSERT(zio->io_vd != NULL); 4238e14bb325SJeff Bonwick ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE); 4239e14bb325SJeff Bonwick 4240e14bb325SJeff Bonwick spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd); 4241e14bb325SJeff Bonwick 4242fa94a07fSbrendan cb = zio->io_private; 4243fa94a07fSbrendan ASSERT(cb != NULL); 4244fa94a07fSbrendan buf = cb->l2rcb_buf; 4245fa94a07fSbrendan ASSERT(buf != NULL); 4246fa94a07fSbrendan 42473f9d6ad7SLin Ling hash_lock = HDR_LOCK(buf->b_hdr); 4248fa94a07fSbrendan mutex_enter(hash_lock); 42493f9d6ad7SLin Ling hdr = buf->b_hdr; 42503f9d6ad7SLin Ling ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 4251fa94a07fSbrendan 4252aad02571SSaso Kiselkov /* 4253aad02571SSaso Kiselkov * If the buffer was compressed, decompress it first. 4254aad02571SSaso Kiselkov */ 4255aad02571SSaso Kiselkov if (cb->l2rcb_compress != ZIO_COMPRESS_OFF) 4256aad02571SSaso Kiselkov l2arc_decompress_zio(zio, hdr, cb->l2rcb_compress); 4257aad02571SSaso Kiselkov ASSERT(zio->io_data != NULL); 4258aad02571SSaso Kiselkov 4259fa94a07fSbrendan /* 4260fa94a07fSbrendan * Check this survived the L2ARC journey. 4261fa94a07fSbrendan */ 4262fa94a07fSbrendan equal = arc_cksum_equal(buf); 4263fa94a07fSbrendan if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) { 4264fa94a07fSbrendan mutex_exit(hash_lock); 4265fa94a07fSbrendan zio->io_private = buf; 4266e14bb325SJeff Bonwick zio->io_bp_copy = cb->l2rcb_bp; /* XXX fix in L2ARC 2.0 */ 4267e14bb325SJeff Bonwick zio->io_bp = &zio->io_bp_copy; /* XXX fix in L2ARC 2.0 */ 4268fa94a07fSbrendan arc_read_done(zio); 4269fa94a07fSbrendan } else { 4270fa94a07fSbrendan mutex_exit(hash_lock); 4271fa94a07fSbrendan /* 4272fa94a07fSbrendan * Buffer didn't survive caching. Increment stats and 4273fa94a07fSbrendan * reissue to the original storage device. 4274fa94a07fSbrendan */ 42753a737e0dSbrendan if (zio->io_error != 0) { 4276fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_io_error); 42773a737e0dSbrendan } else { 4278be6fd75aSMatthew Ahrens zio->io_error = SET_ERROR(EIO); 42793a737e0dSbrendan } 4280fa94a07fSbrendan if (!equal) 4281fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_cksum_bad); 4282fa94a07fSbrendan 4283e14bb325SJeff Bonwick /* 4284e14bb325SJeff Bonwick * If there's no waiter, issue an async i/o to the primary 4285e14bb325SJeff Bonwick * storage now. If there *is* a waiter, the caller must 4286e14bb325SJeff Bonwick * issue the i/o in a context where it's OK to block. 4287e14bb325SJeff Bonwick */ 4288a3f829aeSBill Moore if (zio->io_waiter == NULL) { 4289a3f829aeSBill Moore zio_t *pio = zio_unique_parent(zio); 4290a3f829aeSBill Moore 4291a3f829aeSBill Moore ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL); 4292a3f829aeSBill Moore 4293a3f829aeSBill Moore zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp, 4294e14bb325SJeff Bonwick buf->b_data, zio->io_size, arc_read_done, buf, 4295e14bb325SJeff Bonwick zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb)); 4296a3f829aeSBill Moore } 4297fa94a07fSbrendan } 4298fa94a07fSbrendan 4299fa94a07fSbrendan kmem_free(cb, sizeof (l2arc_read_callback_t)); 4300fa94a07fSbrendan } 4301fa94a07fSbrendan 4302fa94a07fSbrendan /* 4303fa94a07fSbrendan * This is the list priority from which the L2ARC will search for pages to 4304fa94a07fSbrendan * cache. This is used within loops (0..3) to cycle through lists in the 4305fa94a07fSbrendan * desired order. This order can have a significant effect on cache 4306fa94a07fSbrendan * performance. 4307fa94a07fSbrendan * 4308fa94a07fSbrendan * Currently the metadata lists are hit first, MFU then MRU, followed by 4309fa94a07fSbrendan * the data lists. This function returns a locked list, and also returns 4310fa94a07fSbrendan * the lock pointer. 4311fa94a07fSbrendan */ 4312fa94a07fSbrendan static list_t * 4313fa94a07fSbrendan l2arc_list_locked(int list_num, kmutex_t **lock) 4314fa94a07fSbrendan { 4315d5285caeSGeorge Wilson list_t *list = NULL; 4316fa94a07fSbrendan 4317fa94a07fSbrendan ASSERT(list_num >= 0 && list_num <= 3); 4318fa94a07fSbrendan 4319fa94a07fSbrendan switch (list_num) { 4320fa94a07fSbrendan case 0: 4321fa94a07fSbrendan list = &arc_mfu->arcs_list[ARC_BUFC_METADATA]; 4322fa94a07fSbrendan *lock = &arc_mfu->arcs_mtx; 4323fa94a07fSbrendan break; 4324fa94a07fSbrendan case 1: 4325fa94a07fSbrendan list = &arc_mru->arcs_list[ARC_BUFC_METADATA]; 4326fa94a07fSbrendan *lock = &arc_mru->arcs_mtx; 4327fa94a07fSbrendan break; 4328fa94a07fSbrendan case 2: 4329fa94a07fSbrendan list = &arc_mfu->arcs_list[ARC_BUFC_DATA]; 4330fa94a07fSbrendan *lock = &arc_mfu->arcs_mtx; 4331fa94a07fSbrendan break; 4332fa94a07fSbrendan case 3: 4333fa94a07fSbrendan list = &arc_mru->arcs_list[ARC_BUFC_DATA]; 4334fa94a07fSbrendan *lock = &arc_mru->arcs_mtx; 4335fa94a07fSbrendan break; 4336fa94a07fSbrendan } 4337fa94a07fSbrendan 4338fa94a07fSbrendan ASSERT(!(MUTEX_HELD(*lock))); 4339fa94a07fSbrendan mutex_enter(*lock); 4340fa94a07fSbrendan return (list); 4341fa94a07fSbrendan } 4342fa94a07fSbrendan 4343fa94a07fSbrendan /* 4344fa94a07fSbrendan * Evict buffers from the device write hand to the distance specified in 4345fa94a07fSbrendan * bytes. This distance may span populated buffers, it may span nothing. 4346fa94a07fSbrendan * This is clearing a region on the L2ARC device ready for writing. 4347fa94a07fSbrendan * If the 'all' boolean is set, every buffer is evicted. 4348fa94a07fSbrendan */ 4349fa94a07fSbrendan static void 4350fa94a07fSbrendan l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all) 4351fa94a07fSbrendan { 4352fa94a07fSbrendan list_t *buflist; 4353fa94a07fSbrendan l2arc_buf_hdr_t *abl2; 4354fa94a07fSbrendan arc_buf_hdr_t *ab, *ab_prev; 4355fa94a07fSbrendan kmutex_t *hash_lock; 4356fa94a07fSbrendan uint64_t taddr; 4357fa94a07fSbrendan 4358fa94a07fSbrendan buflist = dev->l2ad_buflist; 4359fa94a07fSbrendan 4360fa94a07fSbrendan if (buflist == NULL) 4361fa94a07fSbrendan return; 4362fa94a07fSbrendan 4363fa94a07fSbrendan if (!all && dev->l2ad_first) { 4364fa94a07fSbrendan /* 4365fa94a07fSbrendan * This is the first sweep through the device. There is 4366fa94a07fSbrendan * nothing to evict. 4367fa94a07fSbrendan */ 4368fa94a07fSbrendan return; 4369fa94a07fSbrendan } 4370fa94a07fSbrendan 43713a737e0dSbrendan if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) { 4372fa94a07fSbrendan /* 4373fa94a07fSbrendan * When nearing the end of the device, evict to the end 4374fa94a07fSbrendan * before the device write hand jumps to the start. 4375fa94a07fSbrendan */ 4376fa94a07fSbrendan taddr = dev->l2ad_end; 4377fa94a07fSbrendan } else { 4378fa94a07fSbrendan taddr = dev->l2ad_hand + distance; 4379fa94a07fSbrendan } 4380fa94a07fSbrendan DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist, 4381fa94a07fSbrendan uint64_t, taddr, boolean_t, all); 4382fa94a07fSbrendan 4383fa94a07fSbrendan top: 4384fa94a07fSbrendan mutex_enter(&l2arc_buflist_mtx); 4385fa94a07fSbrendan for (ab = list_tail(buflist); ab; ab = ab_prev) { 4386fa94a07fSbrendan ab_prev = list_prev(buflist, ab); 4387fa94a07fSbrendan 4388fa94a07fSbrendan hash_lock = HDR_LOCK(ab); 4389fa94a07fSbrendan if (!mutex_tryenter(hash_lock)) { 4390fa94a07fSbrendan /* 4391fa94a07fSbrendan * Missed the hash lock. Retry. 4392fa94a07fSbrendan */ 4393fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_evict_lock_retry); 4394fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 4395fa94a07fSbrendan mutex_enter(hash_lock); 4396fa94a07fSbrendan mutex_exit(hash_lock); 4397fa94a07fSbrendan goto top; 4398fa94a07fSbrendan } 4399fa94a07fSbrendan 4400fa94a07fSbrendan if (HDR_L2_WRITE_HEAD(ab)) { 4401fa94a07fSbrendan /* 4402fa94a07fSbrendan * We hit a write head node. Leave it for 4403fa94a07fSbrendan * l2arc_write_done(). 4404fa94a07fSbrendan */ 4405fa94a07fSbrendan list_remove(buflist, ab); 4406fa94a07fSbrendan mutex_exit(hash_lock); 4407fa94a07fSbrendan continue; 4408fa94a07fSbrendan } 4409fa94a07fSbrendan 4410fa94a07fSbrendan if (!all && ab->b_l2hdr != NULL && 4411fa94a07fSbrendan (ab->b_l2hdr->b_daddr > taddr || 4412fa94a07fSbrendan ab->b_l2hdr->b_daddr < dev->l2ad_hand)) { 4413fa94a07fSbrendan /* 4414fa94a07fSbrendan * We've evicted to the target address, 4415fa94a07fSbrendan * or the end of the device. 4416fa94a07fSbrendan */ 4417fa94a07fSbrendan mutex_exit(hash_lock); 4418fa94a07fSbrendan break; 4419fa94a07fSbrendan } 4420fa94a07fSbrendan 4421fa94a07fSbrendan if (HDR_FREE_IN_PROGRESS(ab)) { 4422fa94a07fSbrendan /* 4423fa94a07fSbrendan * Already on the path to destruction. 4424fa94a07fSbrendan */ 4425fa94a07fSbrendan mutex_exit(hash_lock); 4426fa94a07fSbrendan continue; 4427fa94a07fSbrendan } 4428fa94a07fSbrendan 4429fa94a07fSbrendan if (ab->b_state == arc_l2c_only) { 4430fa94a07fSbrendan ASSERT(!HDR_L2_READING(ab)); 4431fa94a07fSbrendan /* 4432fa94a07fSbrendan * This doesn't exist in the ARC. Destroy. 4433fa94a07fSbrendan * arc_hdr_destroy() will call list_remove() 4434fa94a07fSbrendan * and decrement arcstat_l2_size. 4435fa94a07fSbrendan */ 4436fa94a07fSbrendan arc_change_state(arc_anon, ab, hash_lock); 4437fa94a07fSbrendan arc_hdr_destroy(ab); 4438fa94a07fSbrendan } else { 44393a737e0dSbrendan /* 44403a737e0dSbrendan * Invalidate issued or about to be issued 44413a737e0dSbrendan * reads, since we may be about to write 44423a737e0dSbrendan * over this location. 44433a737e0dSbrendan */ 44443a737e0dSbrendan if (HDR_L2_READING(ab)) { 44453a737e0dSbrendan ARCSTAT_BUMP(arcstat_l2_evict_reading); 44463a737e0dSbrendan ab->b_flags |= ARC_L2_EVICTED; 44473a737e0dSbrendan } 44483a737e0dSbrendan 4449fa94a07fSbrendan /* 4450fa94a07fSbrendan * Tell ARC this no longer exists in L2ARC. 4451fa94a07fSbrendan */ 4452fa94a07fSbrendan if (ab->b_l2hdr != NULL) { 4453fa94a07fSbrendan abl2 = ab->b_l2hdr; 4454aad02571SSaso Kiselkov ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize); 4455fa94a07fSbrendan ab->b_l2hdr = NULL; 4456fa94a07fSbrendan kmem_free(abl2, sizeof (l2arc_buf_hdr_t)); 4457fa94a07fSbrendan ARCSTAT_INCR(arcstat_l2_size, -ab->b_size); 4458fa94a07fSbrendan } 4459fa94a07fSbrendan list_remove(buflist, ab); 4460fa94a07fSbrendan 4461fa94a07fSbrendan /* 4462fa94a07fSbrendan * This may have been leftover after a 4463fa94a07fSbrendan * failed write. 4464fa94a07fSbrendan */ 4465fa94a07fSbrendan ab->b_flags &= ~ARC_L2_WRITING; 4466fa94a07fSbrendan } 4467fa94a07fSbrendan mutex_exit(hash_lock); 4468fa94a07fSbrendan } 4469fa94a07fSbrendan mutex_exit(&l2arc_buflist_mtx); 4470fa94a07fSbrendan 4471b24ab676SJeff Bonwick vdev_space_update(dev->l2ad_vdev, -(taddr - dev->l2ad_evict), 0, 0); 4472fa94a07fSbrendan dev->l2ad_evict = taddr; 4473fa94a07fSbrendan } 4474fa94a07fSbrendan 4475fa94a07fSbrendan /* 4476fa94a07fSbrendan * Find and write ARC buffers to the L2ARC device. 4477fa94a07fSbrendan * 4478fa94a07fSbrendan * An ARC_L2_WRITING flag is set so that the L2ARC buffers are not valid 4479fa94a07fSbrendan * for reading until they have completed writing. 4480aad02571SSaso Kiselkov * The headroom_boost is an in-out parameter used to maintain headroom boost 4481aad02571SSaso Kiselkov * state between calls to this function. 4482aad02571SSaso Kiselkov * 4483aad02571SSaso Kiselkov * Returns the number of bytes actually written (which may be smaller than 4484aad02571SSaso Kiselkov * the delta by which the device hand has changed due to alignment). 4485fa94a07fSbrendan */ 44865a98e54bSBrendan Gregg - Sun Microsystems static uint64_t 4487aad02571SSaso Kiselkov l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz, 4488aad02571SSaso Kiselkov boolean_t *headroom_boost) 4489fa94a07fSbrendan { 4490fa94a07fSbrendan arc_buf_hdr_t *ab, *ab_prev, *head; 4491fa94a07fSbrendan list_t *list; 4492aad02571SSaso Kiselkov uint64_t write_asize, write_psize, write_sz, headroom, 4493aad02571SSaso Kiselkov buf_compress_minsz; 4494fa94a07fSbrendan void *buf_data; 4495aad02571SSaso Kiselkov kmutex_t *list_lock; 4496aad02571SSaso Kiselkov boolean_t full; 4497fa94a07fSbrendan l2arc_write_callback_t *cb; 4498fa94a07fSbrendan zio_t *pio, *wzio; 4499e9103aaeSGarrett D'Amore uint64_t guid = spa_load_guid(spa); 4500aad02571SSaso Kiselkov const boolean_t do_headroom_boost = *headroom_boost; 4501fa94a07fSbrendan 4502fa94a07fSbrendan ASSERT(dev->l2ad_vdev != NULL); 4503fa94a07fSbrendan 4504aad02571SSaso Kiselkov /* Lower the flag now, we might want to raise it again later. */ 4505aad02571SSaso Kiselkov *headroom_boost = B_FALSE; 4506aad02571SSaso Kiselkov 4507fa94a07fSbrendan pio = NULL; 4508aad02571SSaso Kiselkov write_sz = write_asize = write_psize = 0; 4509fa94a07fSbrendan full = B_FALSE; 45101ab7f2deSmaybee head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 4511fa94a07fSbrendan head->b_flags |= ARC_L2_WRITE_HEAD; 4512fa94a07fSbrendan 4513aad02571SSaso Kiselkov /* 4514aad02571SSaso Kiselkov * We will want to try to compress buffers that are at least 2x the 4515aad02571SSaso Kiselkov * device sector size. 4516aad02571SSaso Kiselkov */ 4517aad02571SSaso Kiselkov buf_compress_minsz = 2 << dev->l2ad_vdev->vdev_ashift; 4518aad02571SSaso Kiselkov 4519fa94a07fSbrendan /* 4520fa94a07fSbrendan * Copy buffers for L2ARC writing. 4521fa94a07fSbrendan */ 4522fa94a07fSbrendan mutex_enter(&l2arc_buflist_mtx); 4523fa94a07fSbrendan for (int try = 0; try <= 3; try++) { 4524aad02571SSaso Kiselkov uint64_t passed_sz = 0; 4525aad02571SSaso Kiselkov 4526fa94a07fSbrendan list = l2arc_list_locked(try, &list_lock); 4527fa94a07fSbrendan 45283a737e0dSbrendan /* 45293a737e0dSbrendan * L2ARC fast warmup. 45303a737e0dSbrendan * 45313a737e0dSbrendan * Until the ARC is warm and starts to evict, read from the 45323a737e0dSbrendan * head of the ARC lists rather than the tail. 45333a737e0dSbrendan */ 45343a737e0dSbrendan if (arc_warm == B_FALSE) 45353a737e0dSbrendan ab = list_head(list); 45363a737e0dSbrendan else 45373a737e0dSbrendan ab = list_tail(list); 45383a737e0dSbrendan 4539aad02571SSaso Kiselkov headroom = target_sz * l2arc_headroom; 4540aad02571SSaso Kiselkov if (do_headroom_boost) 4541aad02571SSaso Kiselkov headroom = (headroom * l2arc_headroom_boost) / 100; 4542aad02571SSaso Kiselkov 45433a737e0dSbrendan for (; ab; ab = ab_prev) { 4544aad02571SSaso Kiselkov l2arc_buf_hdr_t *l2hdr; 4545aad02571SSaso Kiselkov kmutex_t *hash_lock; 4546aad02571SSaso Kiselkov uint64_t buf_sz; 4547aad02571SSaso Kiselkov 45483a737e0dSbrendan if (arc_warm == B_FALSE) 45493a737e0dSbrendan ab_prev = list_next(list, ab); 45503a737e0dSbrendan else 45513a737e0dSbrendan ab_prev = list_prev(list, ab); 4552fa94a07fSbrendan 4553fa94a07fSbrendan hash_lock = HDR_LOCK(ab); 4554aad02571SSaso Kiselkov if (!mutex_tryenter(hash_lock)) { 4555fa94a07fSbrendan /* 4556fa94a07fSbrendan * Skip this buffer rather than waiting. 4557fa94a07fSbrendan */ 4558fa94a07fSbrendan continue; 4559fa94a07fSbrendan } 4560fa94a07fSbrendan 4561fa94a07fSbrendan passed_sz += ab->b_size; 4562fa94a07fSbrendan if (passed_sz > headroom) { 4563fa94a07fSbrendan /* 4564fa94a07fSbrendan * Searched too far. 4565fa94a07fSbrendan */ 4566fa94a07fSbrendan mutex_exit(hash_lock); 4567fa94a07fSbrendan break; 4568fa94a07fSbrendan } 4569fa94a07fSbrendan 4570ac05c741SMark Maybee if (!l2arc_write_eligible(guid, ab)) { 4571fa94a07fSbrendan mutex_exit(hash_lock); 4572fa94a07fSbrendan continue; 4573fa94a07fSbrendan } 4574fa94a07fSbrendan 4575fa94a07fSbrendan if ((write_sz + ab->b_size) > target_sz) { 4576fa94a07fSbrendan full = B_TRUE; 4577fa94a07fSbrendan mutex_exit(hash_lock); 4578fa94a07fSbrendan break; 4579fa94a07fSbrendan } 4580fa94a07fSbrendan 4581fa94a07fSbrendan if (pio == NULL) { 4582fa94a07fSbrendan /* 4583fa94a07fSbrendan * Insert a dummy header on the buflist so 4584fa94a07fSbrendan * l2arc_write_done() can find where the 4585fa94a07fSbrendan * write buffers begin without searching. 4586fa94a07fSbrendan */ 4587fa94a07fSbrendan list_insert_head(dev->l2ad_buflist, head); 4588fa94a07fSbrendan 4589fa94a07fSbrendan cb = kmem_alloc( 4590fa94a07fSbrendan sizeof (l2arc_write_callback_t), KM_SLEEP); 4591fa94a07fSbrendan cb->l2wcb_dev = dev; 4592fa94a07fSbrendan cb->l2wcb_head = head; 4593fa94a07fSbrendan pio = zio_root(spa, l2arc_write_done, cb, 4594fa94a07fSbrendan ZIO_FLAG_CANFAIL); 4595fa94a07fSbrendan } 4596fa94a07fSbrendan 4597fa94a07fSbrendan /* 4598fa94a07fSbrendan * Create and add a new L2ARC header. 4599fa94a07fSbrendan */ 4600aad02571SSaso Kiselkov l2hdr = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP); 4601aad02571SSaso Kiselkov l2hdr->b_dev = dev; 4602fa94a07fSbrendan ab->b_flags |= ARC_L2_WRITING; 4603aad02571SSaso Kiselkov 4604aad02571SSaso Kiselkov /* 4605aad02571SSaso Kiselkov * Temporarily stash the data buffer in b_tmp_cdata. 4606aad02571SSaso Kiselkov * The subsequent write step will pick it up from 4607aad02571SSaso Kiselkov * there. This is because can't access ab->b_buf 4608aad02571SSaso Kiselkov * without holding the hash_lock, which we in turn 4609aad02571SSaso Kiselkov * can't access without holding the ARC list locks 4610aad02571SSaso Kiselkov * (which we want to avoid during compression/writing). 4611aad02571SSaso Kiselkov */ 4612aad02571SSaso Kiselkov l2hdr->b_compress = ZIO_COMPRESS_OFF; 4613aad02571SSaso Kiselkov l2hdr->b_asize = ab->b_size; 4614aad02571SSaso Kiselkov l2hdr->b_tmp_cdata = ab->b_buf->b_data; 4615aad02571SSaso Kiselkov 4616fa94a07fSbrendan buf_sz = ab->b_size; 4617aad02571SSaso Kiselkov ab->b_l2hdr = l2hdr; 4618aad02571SSaso Kiselkov 4619aad02571SSaso Kiselkov list_insert_head(dev->l2ad_buflist, ab); 4620fa94a07fSbrendan 4621fa94a07fSbrendan /* 4622fa94a07fSbrendan * Compute and store the buffer cksum before 4623fa94a07fSbrendan * writing. On debug the cksum is verified first. 4624fa94a07fSbrendan */ 4625fa94a07fSbrendan arc_cksum_verify(ab->b_buf); 4626fa94a07fSbrendan arc_cksum_compute(ab->b_buf, B_TRUE); 4627fa94a07fSbrendan 4628fa94a07fSbrendan mutex_exit(hash_lock); 4629fa94a07fSbrendan 4630aad02571SSaso Kiselkov write_sz += buf_sz; 4631aad02571SSaso Kiselkov } 4632aad02571SSaso Kiselkov 4633aad02571SSaso Kiselkov mutex_exit(list_lock); 4634aad02571SSaso Kiselkov 4635aad02571SSaso Kiselkov if (full == B_TRUE) 4636aad02571SSaso Kiselkov break; 4637aad02571SSaso Kiselkov } 4638aad02571SSaso Kiselkov 4639aad02571SSaso Kiselkov /* No buffers selected for writing? */ 4640aad02571SSaso Kiselkov if (pio == NULL) { 4641aad02571SSaso Kiselkov ASSERT0(write_sz); 4642aad02571SSaso Kiselkov mutex_exit(&l2arc_buflist_mtx); 4643aad02571SSaso Kiselkov kmem_cache_free(hdr_cache, head); 4644aad02571SSaso Kiselkov return (0); 4645aad02571SSaso Kiselkov } 4646aad02571SSaso Kiselkov 4647aad02571SSaso Kiselkov /* 4648aad02571SSaso Kiselkov * Now start writing the buffers. We're starting at the write head 4649aad02571SSaso Kiselkov * and work backwards, retracing the course of the buffer selector 4650aad02571SSaso Kiselkov * loop above. 4651aad02571SSaso Kiselkov */ 4652aad02571SSaso Kiselkov for (ab = list_prev(dev->l2ad_buflist, head); ab; 4653aad02571SSaso Kiselkov ab = list_prev(dev->l2ad_buflist, ab)) { 4654aad02571SSaso Kiselkov l2arc_buf_hdr_t *l2hdr; 4655aad02571SSaso Kiselkov uint64_t buf_sz; 4656aad02571SSaso Kiselkov 4657aad02571SSaso Kiselkov /* 4658aad02571SSaso Kiselkov * We shouldn't need to lock the buffer here, since we flagged 4659aad02571SSaso Kiselkov * it as ARC_L2_WRITING in the previous step, but we must take 4660aad02571SSaso Kiselkov * care to only access its L2 cache parameters. In particular, 4661aad02571SSaso Kiselkov * ab->b_buf may be invalid by now due to ARC eviction. 4662aad02571SSaso Kiselkov */ 4663aad02571SSaso Kiselkov l2hdr = ab->b_l2hdr; 4664aad02571SSaso Kiselkov l2hdr->b_daddr = dev->l2ad_hand; 4665aad02571SSaso Kiselkov 4666aad02571SSaso Kiselkov if ((ab->b_flags & ARC_L2COMPRESS) && 4667aad02571SSaso Kiselkov l2hdr->b_asize >= buf_compress_minsz) { 4668aad02571SSaso Kiselkov if (l2arc_compress_buf(l2hdr)) { 4669aad02571SSaso Kiselkov /* 4670aad02571SSaso Kiselkov * If compression succeeded, enable headroom 4671aad02571SSaso Kiselkov * boost on the next scan cycle. 4672aad02571SSaso Kiselkov */ 4673aad02571SSaso Kiselkov *headroom_boost = B_TRUE; 4674aad02571SSaso Kiselkov } 4675aad02571SSaso Kiselkov } 4676aad02571SSaso Kiselkov 4677aad02571SSaso Kiselkov /* 4678aad02571SSaso Kiselkov * Pick up the buffer data we had previously stashed away 4679aad02571SSaso Kiselkov * (and now potentially also compressed). 4680aad02571SSaso Kiselkov */ 4681aad02571SSaso Kiselkov buf_data = l2hdr->b_tmp_cdata; 4682aad02571SSaso Kiselkov buf_sz = l2hdr->b_asize; 4683aad02571SSaso Kiselkov 4684aad02571SSaso Kiselkov /* Compression may have squashed the buffer to zero length. */ 4685aad02571SSaso Kiselkov if (buf_sz != 0) { 4686aad02571SSaso Kiselkov uint64_t buf_p_sz; 4687aad02571SSaso Kiselkov 4688fa94a07fSbrendan wzio = zio_write_phys(pio, dev->l2ad_vdev, 4689fa94a07fSbrendan dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF, 4690fa94a07fSbrendan NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE, 4691fa94a07fSbrendan ZIO_FLAG_CANFAIL, B_FALSE); 4692fa94a07fSbrendan 4693fa94a07fSbrendan DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, 4694fa94a07fSbrendan zio_t *, wzio); 4695fa94a07fSbrendan (void) zio_nowait(wzio); 4696fa94a07fSbrendan 4697aad02571SSaso Kiselkov write_asize += buf_sz; 4698e14bb325SJeff Bonwick /* 4699e14bb325SJeff Bonwick * Keep the clock hand suitably device-aligned. 4700e14bb325SJeff Bonwick */ 4701aad02571SSaso Kiselkov buf_p_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz); 4702aad02571SSaso Kiselkov write_psize += buf_p_sz; 4703aad02571SSaso Kiselkov dev->l2ad_hand += buf_p_sz; 4704fa94a07fSbrendan } 4705fa94a07fSbrendan } 4706fa94a07fSbrendan 4707aad02571SSaso Kiselkov mutex_exit(&l2arc_buflist_mtx); 4708fa94a07fSbrendan 4709aad02571SSaso Kiselkov ASSERT3U(write_asize, <=, target_sz); 4710fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_writes_sent); 4711aad02571SSaso Kiselkov ARCSTAT_INCR(arcstat_l2_write_bytes, write_asize); 4712fa94a07fSbrendan ARCSTAT_INCR(arcstat_l2_size, write_sz); 4713aad02571SSaso Kiselkov ARCSTAT_INCR(arcstat_l2_asize, write_asize); 4714aad02571SSaso Kiselkov vdev_space_update(dev->l2ad_vdev, write_psize, 0, 0); 4715fa94a07fSbrendan 4716fa94a07fSbrendan /* 4717fa94a07fSbrendan * Bump device hand to the device start if it is approaching the end. 4718fa94a07fSbrendan * l2arc_evict() will already have evicted ahead for this case. 4719fa94a07fSbrendan */ 47203a737e0dSbrendan if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) { 4721b24ab676SJeff Bonwick vdev_space_update(dev->l2ad_vdev, 4722b24ab676SJeff Bonwick dev->l2ad_end - dev->l2ad_hand, 0, 0); 4723fa94a07fSbrendan dev->l2ad_hand = dev->l2ad_start; 4724fa94a07fSbrendan dev->l2ad_evict = dev->l2ad_start; 4725fa94a07fSbrendan dev->l2ad_first = B_FALSE; 4726fa94a07fSbrendan } 4727fa94a07fSbrendan 47285a98e54bSBrendan Gregg - Sun Microsystems dev->l2ad_writing = B_TRUE; 4729fa94a07fSbrendan (void) zio_wait(pio); 47305a98e54bSBrendan Gregg - Sun Microsystems dev->l2ad_writing = B_FALSE; 47315a98e54bSBrendan Gregg - Sun Microsystems 4732aad02571SSaso Kiselkov return (write_asize); 4733aad02571SSaso Kiselkov } 4734aad02571SSaso Kiselkov 4735aad02571SSaso Kiselkov /* 4736aad02571SSaso Kiselkov * Compresses an L2ARC buffer. 4737aad02571SSaso Kiselkov * The data to be compressed must be prefilled in l2hdr->b_tmp_cdata and its 4738aad02571SSaso Kiselkov * size in l2hdr->b_asize. This routine tries to compress the data and 4739aad02571SSaso Kiselkov * depending on the compression result there are three possible outcomes: 4740aad02571SSaso Kiselkov * *) The buffer was incompressible. The original l2hdr contents were left 4741aad02571SSaso Kiselkov * untouched and are ready for writing to an L2 device. 4742aad02571SSaso Kiselkov * *) The buffer was all-zeros, so there is no need to write it to an L2 4743aad02571SSaso Kiselkov * device. To indicate this situation b_tmp_cdata is NULL'ed, b_asize is 4744aad02571SSaso Kiselkov * set to zero and b_compress is set to ZIO_COMPRESS_EMPTY. 4745aad02571SSaso Kiselkov * *) Compression succeeded and b_tmp_cdata was replaced with a temporary 4746aad02571SSaso Kiselkov * data buffer which holds the compressed data to be written, and b_asize 4747aad02571SSaso Kiselkov * tells us how much data there is. b_compress is set to the appropriate 4748aad02571SSaso Kiselkov * compression algorithm. Once writing is done, invoke 4749aad02571SSaso Kiselkov * l2arc_release_cdata_buf on this l2hdr to free this temporary buffer. 4750aad02571SSaso Kiselkov * 4751aad02571SSaso Kiselkov * Returns B_TRUE if compression succeeded, or B_FALSE if it didn't (the 4752aad02571SSaso Kiselkov * buffer was incompressible). 4753aad02571SSaso Kiselkov */ 4754aad02571SSaso Kiselkov static boolean_t 4755aad02571SSaso Kiselkov l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr) 4756aad02571SSaso Kiselkov { 4757aad02571SSaso Kiselkov void *cdata; 4758aad02571SSaso Kiselkov size_t csize, len; 4759aad02571SSaso Kiselkov 4760aad02571SSaso Kiselkov ASSERT(l2hdr->b_compress == ZIO_COMPRESS_OFF); 4761aad02571SSaso Kiselkov ASSERT(l2hdr->b_tmp_cdata != NULL); 4762aad02571SSaso Kiselkov 4763aad02571SSaso Kiselkov len = l2hdr->b_asize; 4764aad02571SSaso Kiselkov cdata = zio_data_buf_alloc(len); 4765aad02571SSaso Kiselkov csize = zio_compress_data(ZIO_COMPRESS_LZ4, l2hdr->b_tmp_cdata, 4766aad02571SSaso Kiselkov cdata, l2hdr->b_asize); 4767aad02571SSaso Kiselkov 4768aad02571SSaso Kiselkov if (csize == 0) { 4769aad02571SSaso Kiselkov /* zero block, indicate that there's nothing to write */ 4770aad02571SSaso Kiselkov zio_data_buf_free(cdata, len); 4771aad02571SSaso Kiselkov l2hdr->b_compress = ZIO_COMPRESS_EMPTY; 4772aad02571SSaso Kiselkov l2hdr->b_asize = 0; 4773aad02571SSaso Kiselkov l2hdr->b_tmp_cdata = NULL; 4774aad02571SSaso Kiselkov ARCSTAT_BUMP(arcstat_l2_compress_zeros); 4775aad02571SSaso Kiselkov return (B_TRUE); 4776aad02571SSaso Kiselkov } else if (csize > 0 && csize < len) { 4777aad02571SSaso Kiselkov /* 4778aad02571SSaso Kiselkov * Compression succeeded, we'll keep the cdata around for 4779aad02571SSaso Kiselkov * writing and release it afterwards. 4780aad02571SSaso Kiselkov */ 4781aad02571SSaso Kiselkov l2hdr->b_compress = ZIO_COMPRESS_LZ4; 4782aad02571SSaso Kiselkov l2hdr->b_asize = csize; 4783aad02571SSaso Kiselkov l2hdr->b_tmp_cdata = cdata; 4784aad02571SSaso Kiselkov ARCSTAT_BUMP(arcstat_l2_compress_successes); 4785aad02571SSaso Kiselkov return (B_TRUE); 4786aad02571SSaso Kiselkov } else { 4787aad02571SSaso Kiselkov /* 4788aad02571SSaso Kiselkov * Compression failed, release the compressed buffer. 4789aad02571SSaso Kiselkov * l2hdr will be left unmodified. 4790aad02571SSaso Kiselkov */ 4791aad02571SSaso Kiselkov zio_data_buf_free(cdata, len); 4792aad02571SSaso Kiselkov ARCSTAT_BUMP(arcstat_l2_compress_failures); 4793aad02571SSaso Kiselkov return (B_FALSE); 4794aad02571SSaso Kiselkov } 4795aad02571SSaso Kiselkov } 4796aad02571SSaso Kiselkov 4797aad02571SSaso Kiselkov /* 4798aad02571SSaso Kiselkov * Decompresses a zio read back from an l2arc device. On success, the 4799aad02571SSaso Kiselkov * underlying zio's io_data buffer is overwritten by the uncompressed 4800aad02571SSaso Kiselkov * version. On decompression error (corrupt compressed stream), the 4801aad02571SSaso Kiselkov * zio->io_error value is set to signal an I/O error. 4802aad02571SSaso Kiselkov * 4803aad02571SSaso Kiselkov * Please note that the compressed data stream is not checksummed, so 4804aad02571SSaso Kiselkov * if the underlying device is experiencing data corruption, we may feed 4805aad02571SSaso Kiselkov * corrupt data to the decompressor, so the decompressor needs to be 4806aad02571SSaso Kiselkov * able to handle this situation (LZ4 does). 4807aad02571SSaso Kiselkov */ 4808aad02571SSaso Kiselkov static void 4809aad02571SSaso Kiselkov l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c) 4810aad02571SSaso Kiselkov { 4811aad02571SSaso Kiselkov ASSERT(L2ARC_IS_VALID_COMPRESS(c)); 4812aad02571SSaso Kiselkov 4813aad02571SSaso Kiselkov if (zio->io_error != 0) { 4814aad02571SSaso Kiselkov /* 4815aad02571SSaso Kiselkov * An io error has occured, just restore the original io 4816aad02571SSaso Kiselkov * size in preparation for a main pool read. 4817aad02571SSaso Kiselkov */ 4818aad02571SSaso Kiselkov zio->io_orig_size = zio->io_size = hdr->b_size; 4819aad02571SSaso Kiselkov return; 4820aad02571SSaso Kiselkov } 4821aad02571SSaso Kiselkov 4822aad02571SSaso Kiselkov if (c == ZIO_COMPRESS_EMPTY) { 4823aad02571SSaso Kiselkov /* 4824aad02571SSaso Kiselkov * An empty buffer results in a null zio, which means we 4825aad02571SSaso Kiselkov * need to fill its io_data after we're done restoring the 4826aad02571SSaso Kiselkov * buffer's contents. 4827aad02571SSaso Kiselkov */ 4828aad02571SSaso Kiselkov ASSERT(hdr->b_buf != NULL); 4829aad02571SSaso Kiselkov bzero(hdr->b_buf->b_data, hdr->b_size); 4830aad02571SSaso Kiselkov zio->io_data = zio->io_orig_data = hdr->b_buf->b_data; 4831aad02571SSaso Kiselkov } else { 4832aad02571SSaso Kiselkov ASSERT(zio->io_data != NULL); 4833aad02571SSaso Kiselkov /* 4834aad02571SSaso Kiselkov * We copy the compressed data from the start of the arc buffer 4835aad02571SSaso Kiselkov * (the zio_read will have pulled in only what we need, the 4836aad02571SSaso Kiselkov * rest is garbage which we will overwrite at decompression) 4837aad02571SSaso Kiselkov * and then decompress back to the ARC data buffer. This way we 4838aad02571SSaso Kiselkov * can minimize copying by simply decompressing back over the 4839aad02571SSaso Kiselkov * original compressed data (rather than decompressing to an 4840aad02571SSaso Kiselkov * aux buffer and then copying back the uncompressed buffer, 4841aad02571SSaso Kiselkov * which is likely to be much larger). 4842aad02571SSaso Kiselkov */ 4843aad02571SSaso Kiselkov uint64_t csize; 4844aad02571SSaso Kiselkov void *cdata; 4845aad02571SSaso Kiselkov 4846aad02571SSaso Kiselkov csize = zio->io_size; 4847aad02571SSaso Kiselkov cdata = zio_data_buf_alloc(csize); 4848aad02571SSaso Kiselkov bcopy(zio->io_data, cdata, csize); 4849aad02571SSaso Kiselkov if (zio_decompress_data(c, cdata, zio->io_data, csize, 4850aad02571SSaso Kiselkov hdr->b_size) != 0) 4851aad02571SSaso Kiselkov zio->io_error = EIO; 4852aad02571SSaso Kiselkov zio_data_buf_free(cdata, csize); 4853aad02571SSaso Kiselkov } 4854aad02571SSaso Kiselkov 4855aad02571SSaso Kiselkov /* Restore the expected uncompressed IO size. */ 4856aad02571SSaso Kiselkov zio->io_orig_size = zio->io_size = hdr->b_size; 4857aad02571SSaso Kiselkov } 4858aad02571SSaso Kiselkov 4859aad02571SSaso Kiselkov /* 4860aad02571SSaso Kiselkov * Releases the temporary b_tmp_cdata buffer in an l2arc header structure. 4861aad02571SSaso Kiselkov * This buffer serves as a temporary holder of compressed data while 4862aad02571SSaso Kiselkov * the buffer entry is being written to an l2arc device. Once that is 4863aad02571SSaso Kiselkov * done, we can dispose of it. 4864aad02571SSaso Kiselkov */ 4865aad02571SSaso Kiselkov static void 4866aad02571SSaso Kiselkov l2arc_release_cdata_buf(arc_buf_hdr_t *ab) 4867aad02571SSaso Kiselkov { 4868aad02571SSaso Kiselkov l2arc_buf_hdr_t *l2hdr = ab->b_l2hdr; 4869aad02571SSaso Kiselkov 4870aad02571SSaso Kiselkov if (l2hdr->b_compress == ZIO_COMPRESS_LZ4) { 4871aad02571SSaso Kiselkov /* 4872aad02571SSaso Kiselkov * If the data was compressed, then we've allocated a 4873aad02571SSaso Kiselkov * temporary buffer for it, so now we need to release it. 4874aad02571SSaso Kiselkov */ 4875aad02571SSaso Kiselkov ASSERT(l2hdr->b_tmp_cdata != NULL); 4876aad02571SSaso Kiselkov zio_data_buf_free(l2hdr->b_tmp_cdata, ab->b_size); 4877aad02571SSaso Kiselkov } 4878aad02571SSaso Kiselkov l2hdr->b_tmp_cdata = NULL; 4879fa94a07fSbrendan } 4880fa94a07fSbrendan 4881fa94a07fSbrendan /* 4882fa94a07fSbrendan * This thread feeds the L2ARC at regular intervals. This is the beating 4883fa94a07fSbrendan * heart of the L2ARC. 4884fa94a07fSbrendan */ 4885fa94a07fSbrendan static void 4886fa94a07fSbrendan l2arc_feed_thread(void) 4887fa94a07fSbrendan { 4888fa94a07fSbrendan callb_cpr_t cpr; 4889fa94a07fSbrendan l2arc_dev_t *dev; 4890fa94a07fSbrendan spa_t *spa; 48915a98e54bSBrendan Gregg - Sun Microsystems uint64_t size, wrote; 4892d3d50737SRafael Vanoni clock_t begin, next = ddi_get_lbolt(); 4893aad02571SSaso Kiselkov boolean_t headroom_boost = B_FALSE; 4894fa94a07fSbrendan 4895fa94a07fSbrendan CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG); 4896fa94a07fSbrendan 4897fa94a07fSbrendan mutex_enter(&l2arc_feed_thr_lock); 4898fa94a07fSbrendan 4899fa94a07fSbrendan while (l2arc_thread_exit == 0) { 4900fa94a07fSbrendan CALLB_CPR_SAFE_BEGIN(&cpr); 4901fa94a07fSbrendan (void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock, 49025a98e54bSBrendan Gregg - Sun Microsystems next); 4903fa94a07fSbrendan CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock); 4904d3d50737SRafael Vanoni next = ddi_get_lbolt() + hz; 4905fa94a07fSbrendan 49063a737e0dSbrendan /* 49073a737e0dSbrendan * Quick check for L2ARC devices. 49083a737e0dSbrendan */ 4909c5904d13Seschrock mutex_enter(&l2arc_dev_mtx); 49103a737e0dSbrendan if (l2arc_ndev == 0) { 49113a737e0dSbrendan mutex_exit(&l2arc_dev_mtx); 49123a737e0dSbrendan continue; 49133a737e0dSbrendan } 49143a737e0dSbrendan mutex_exit(&l2arc_dev_mtx); 4915d3d50737SRafael Vanoni begin = ddi_get_lbolt(); 4916c5904d13Seschrock 4917fa94a07fSbrendan /* 4918c5904d13Seschrock * This selects the next l2arc device to write to, and in 4919c5904d13Seschrock * doing so the next spa to feed from: dev->l2ad_spa. This 49203a737e0dSbrendan * will return NULL if there are now no l2arc devices or if 49213a737e0dSbrendan * they are all faulted. 49223a737e0dSbrendan * 49233a737e0dSbrendan * If a device is returned, its spa's config lock is also 49243a737e0dSbrendan * held to prevent device removal. l2arc_dev_get_next() 49253a737e0dSbrendan * will grab and release l2arc_dev_mtx. 4926fa94a07fSbrendan */ 49273a737e0dSbrendan if ((dev = l2arc_dev_get_next()) == NULL) 4928fa94a07fSbrendan continue; 49293a737e0dSbrendan 49303a737e0dSbrendan spa = dev->l2ad_spa; 49313a737e0dSbrendan ASSERT(spa != NULL); 4932fa94a07fSbrendan 4933f9af39baSGeorge Wilson /* 4934f9af39baSGeorge Wilson * If the pool is read-only then force the feed thread to 4935f9af39baSGeorge Wilson * sleep a little longer. 4936f9af39baSGeorge Wilson */ 4937f9af39baSGeorge Wilson if (!spa_writeable(spa)) { 4938f9af39baSGeorge Wilson next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz; 4939f9af39baSGeorge Wilson spa_config_exit(spa, SCL_L2ARC, dev); 4940f9af39baSGeorge Wilson continue; 4941f9af39baSGeorge Wilson } 4942f9af39baSGeorge Wilson 4943fa94a07fSbrendan /* 4944fa94a07fSbrendan * Avoid contributing to memory pressure. 4945fa94a07fSbrendan */ 4946fa94a07fSbrendan if (arc_reclaim_needed()) { 4947fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_abort_lowmem); 4948e14bb325SJeff Bonwick spa_config_exit(spa, SCL_L2ARC, dev); 4949fa94a07fSbrendan continue; 4950fa94a07fSbrendan } 4951fa94a07fSbrendan 4952fa94a07fSbrendan ARCSTAT_BUMP(arcstat_l2_feeds); 4953fa94a07fSbrendan 4954aad02571SSaso Kiselkov size = l2arc_write_size(); 49553a737e0dSbrendan 4956fa94a07fSbrendan /* 4957fa94a07fSbrendan * Evict L2ARC buffers that will be overwritten. 4958fa94a07fSbrendan */ 49593a737e0dSbrendan l2arc_evict(dev, size, B_FALSE); 4960fa94a07fSbrendan 4961fa94a07fSbrendan /* 4962fa94a07fSbrendan * Write ARC buffers. 4963fa94a07fSbrendan */ 4964aad02571SSaso Kiselkov wrote = l2arc_write_buffers(spa, dev, size, &headroom_boost); 49655a98e54bSBrendan Gregg - Sun Microsystems 49665a98e54bSBrendan Gregg - Sun Microsystems /* 49675a98e54bSBrendan Gregg - Sun Microsystems * Calculate interval between writes. 49685a98e54bSBrendan Gregg - Sun Microsystems */ 49695a98e54bSBrendan Gregg - Sun Microsystems next = l2arc_write_interval(begin, size, wrote); 4970e14bb325SJeff Bonwick spa_config_exit(spa, SCL_L2ARC, dev); 4971fa94a07fSbrendan } 4972fa94a07fSbrendan 4973fa94a07fSbrendan l2arc_thread_exit = 0; 4974fa94a07fSbrendan cv_broadcast(&l2arc_feed_thr_cv); 4975fa94a07fSbrendan CALLB_CPR_EXIT(&cpr); /* drops l2arc_feed_thr_lock */ 4976fa94a07fSbrendan thread_exit(); 4977fa94a07fSbrendan } 4978fa94a07fSbrendan 4979c5904d13Seschrock boolean_t 4980c5904d13Seschrock l2arc_vdev_present(vdev_t *vd) 4981c5904d13Seschrock { 4982c5904d13Seschrock l2arc_dev_t *dev; 4983c5904d13Seschrock 4984c5904d13Seschrock mutex_enter(&l2arc_dev_mtx); 4985c5904d13Seschrock for (dev = list_head(l2arc_dev_list); dev != NULL; 4986c5904d13Seschrock dev = list_next(l2arc_dev_list, dev)) { 4987c5904d13Seschrock if (dev->l2ad_vdev == vd) 4988c5904d13Seschrock break; 4989c5904d13Seschrock } 4990c5904d13Seschrock mutex_exit(&l2arc_dev_mtx); 4991c5904d13Seschrock 4992c5904d13Seschrock return (dev != NULL); 4993c5904d13Seschrock } 4994c5904d13Seschrock 4995fa94a07fSbrendan /* 4996fa94a07fSbrendan * Add a vdev for use by the L2ARC. By this point the spa has already 4997fa94a07fSbrendan * validated the vdev and opened it. 4998fa94a07fSbrendan */ 4999fa94a07fSbrendan void 5000573ca77eSGeorge Wilson l2arc_add_vdev(spa_t *spa, vdev_t *vd) 5001fa94a07fSbrendan { 5002fa94a07fSbrendan l2arc_dev_t *adddev; 5003fa94a07fSbrendan 5004c5904d13Seschrock ASSERT(!l2arc_vdev_present(vd)); 5005c5904d13Seschrock 5006fa94a07fSbrendan /* 5007fa94a07fSbrendan * Create a new l2arc device entry. 5008fa94a07fSbrendan */ 5009fa94a07fSbrendan adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP); 5010fa94a07fSbrendan adddev->l2ad_spa = spa; 5011fa94a07fSbrendan adddev->l2ad_vdev = vd; 5012573ca77eSGeorge Wilson adddev->l2ad_start = VDEV_LABEL_START_SIZE; 5013573ca77eSGeorge Wilson adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd); 5014fa94a07fSbrendan adddev->l2ad_hand = adddev->l2ad_start; 5015fa94a07fSbrendan adddev->l2ad_evict = adddev->l2ad_start; 5016fa94a07fSbrendan adddev->l2ad_first = B_TRUE; 50175a98e54bSBrendan Gregg - Sun Microsystems adddev->l2ad_writing = B_FALSE; 5018fa94a07fSbrendan 5019fa94a07fSbrendan /* 5020fa94a07fSbrendan * This is a list of all ARC buffers that are still valid on the 5021fa94a07fSbrendan * device. 5022fa94a07fSbrendan */ 5023fa94a07fSbrendan adddev->l2ad_buflist = kmem_zalloc(sizeof (list_t), KM_SLEEP); 5024fa94a07fSbrendan list_create(adddev->l2ad_buflist, sizeof (arc_buf_hdr_t), 5025fa94a07fSbrendan offsetof(arc_buf_hdr_t, b_l2node)); 5026fa94a07fSbrendan 5027b24ab676SJeff Bonwick vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand); 5028fa94a07fSbrendan 5029fa94a07fSbrendan /* 5030fa94a07fSbrendan * Add device to global list 5031fa94a07fSbrendan */ 5032fa94a07fSbrendan mutex_enter(&l2arc_dev_mtx); 5033fa94a07fSbrendan list_insert_head(l2arc_dev_list, adddev); 5034fa94a07fSbrendan atomic_inc_64(&l2arc_ndev); 5035fa94a07fSbrendan mutex_exit(&l2arc_dev_mtx); 5036fa94a07fSbrendan } 5037fa94a07fSbrendan 5038fa94a07fSbrendan /* 5039fa94a07fSbrendan * Remove a vdev from the L2ARC. 5040fa94a07fSbrendan */ 5041fa94a07fSbrendan void 5042fa94a07fSbrendan l2arc_remove_vdev(vdev_t *vd) 5043fa94a07fSbrendan { 5044fa94a07fSbrendan l2arc_dev_t *dev, *nextdev, *remdev = NULL; 5045fa94a07fSbrendan 5046fa94a07fSbrendan /* 5047fa94a07fSbrendan * Find the device by vdev 5048fa94a07fSbrendan */ 5049fa94a07fSbrendan mutex_enter(&l2arc_dev_mtx); 5050fa94a07fSbrendan for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) { 5051fa94a07fSbrendan nextdev = list_next(l2arc_dev_list, dev); 5052fa94a07fSbrendan if (vd == dev->l2ad_vdev) { 5053fa94a07fSbrendan remdev = dev; 5054fa94a07fSbrendan break; 5055fa94a07fSbrendan } 5056fa94a07fSbrendan } 5057fa94a07fSbrendan ASSERT(remdev != NULL); 5058fa94a07fSbrendan 5059fa94a07fSbrendan /* 5060fa94a07fSbrendan * Remove device from global list 5061fa94a07fSbrendan */ 5062fa94a07fSbrendan list_remove(l2arc_dev_list, remdev); 5063fa94a07fSbrendan l2arc_dev_last = NULL; /* may have been invalidated */ 50643a737e0dSbrendan atomic_dec_64(&l2arc_ndev); 50653a737e0dSbrendan mutex_exit(&l2arc_dev_mtx); 5066fa94a07fSbrendan 5067fa94a07fSbrendan /* 5068fa94a07fSbrendan * Clear all buflists and ARC references. L2ARC device flush. 5069fa94a07fSbrendan */ 5070fa94a07fSbrendan l2arc_evict(remdev, 0, B_TRUE); 5071fa94a07fSbrendan list_destroy(remdev->l2ad_buflist); 5072fa94a07fSbrendan kmem_free(remdev->l2ad_buflist, sizeof (list_t)); 5073fa94a07fSbrendan kmem_free(remdev, sizeof (l2arc_dev_t)); 5074fa94a07fSbrendan } 5075fa94a07fSbrendan 5076fa94a07fSbrendan void 5077e14bb325SJeff Bonwick l2arc_init(void) 5078fa94a07fSbrendan { 5079fa94a07fSbrendan l2arc_thread_exit = 0; 5080fa94a07fSbrendan l2arc_ndev = 0; 5081fa94a07fSbrendan l2arc_writes_sent = 0; 5082fa94a07fSbrendan l2arc_writes_done = 0; 5083fa94a07fSbrendan 5084fa94a07fSbrendan mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL); 5085fa94a07fSbrendan cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL); 5086fa94a07fSbrendan mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL); 5087fa94a07fSbrendan mutex_init(&l2arc_buflist_mtx, NULL, MUTEX_DEFAULT, NULL); 5088fa94a07fSbrendan mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL); 5089fa94a07fSbrendan 5090fa94a07fSbrendan l2arc_dev_list = &L2ARC_dev_list; 5091fa94a07fSbrendan l2arc_free_on_write = &L2ARC_free_on_write; 5092fa94a07fSbrendan list_create(l2arc_dev_list, sizeof (l2arc_dev_t), 5093fa94a07fSbrendan offsetof(l2arc_dev_t, l2ad_node)); 5094fa94a07fSbrendan list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t), 5095fa94a07fSbrendan offsetof(l2arc_data_free_t, l2df_list_node)); 5096fa94a07fSbrendan } 5097fa94a07fSbrendan 5098fa94a07fSbrendan void 5099e14bb325SJeff Bonwick l2arc_fini(void) 5100fa94a07fSbrendan { 51013a737e0dSbrendan /* 51023a737e0dSbrendan * This is called from dmu_fini(), which is called from spa_fini(); 51033a737e0dSbrendan * Because of this, we can assume that all l2arc devices have 51043a737e0dSbrendan * already been removed when the pools themselves were removed. 51053a737e0dSbrendan */ 51063a737e0dSbrendan 51073a737e0dSbrendan l2arc_do_free_on_write(); 51083a737e0dSbrendan 5109fa94a07fSbrendan mutex_destroy(&l2arc_feed_thr_lock); 5110fa94a07fSbrendan cv_destroy(&l2arc_feed_thr_cv); 5111fa94a07fSbrendan mutex_destroy(&l2arc_dev_mtx); 5112fa94a07fSbrendan mutex_destroy(&l2arc_buflist_mtx); 5113fa94a07fSbrendan mutex_destroy(&l2arc_free_on_write_mtx); 5114fa94a07fSbrendan 5115fa94a07fSbrendan list_destroy(l2arc_dev_list); 5116fa94a07fSbrendan list_destroy(l2arc_free_on_write); 5117fa94a07fSbrendan } 5118e14bb325SJeff Bonwick 5119e14bb325SJeff Bonwick void 5120e14bb325SJeff Bonwick l2arc_start(void) 5121e14bb325SJeff Bonwick { 51228ad4d6ddSJeff Bonwick if (!(spa_mode_global & FWRITE)) 5123e14bb325SJeff Bonwick return; 5124e14bb325SJeff Bonwick 5125e14bb325SJeff Bonwick (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0, 5126e14bb325SJeff Bonwick TS_RUN, minclsyspri); 5127e14bb325SJeff Bonwick } 5128e14bb325SJeff Bonwick 5129e14bb325SJeff Bonwick void 5130e14bb325SJeff Bonwick l2arc_stop(void) 5131e14bb325SJeff Bonwick { 51328ad4d6ddSJeff Bonwick if (!(spa_mode_global & FWRITE)) 5133e14bb325SJeff Bonwick return; 5134e14bb325SJeff Bonwick 5135e14bb325SJeff Bonwick mutex_enter(&l2arc_feed_thr_lock); 5136e14bb325SJeff Bonwick cv_signal(&l2arc_feed_thr_cv); /* kick thread out of startup */ 5137e14bb325SJeff Bonwick l2arc_thread_exit = 1; 5138e14bb325SJeff Bonwick while (l2arc_thread_exit != 0) 5139e14bb325SJeff Bonwick cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock); 5140e14bb325SJeff Bonwick mutex_exit(&l2arc_feed_thr_lock); 5141e14bb325SJeff Bonwick } 5142