1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * DVA-based Adjustable Replacement Cache 28 * 29 * While much of the theory of operation used here is 30 * based on the self-tuning, low overhead replacement cache 31 * presented by Megiddo and Modha at FAST 2003, there are some 32 * significant differences: 33 * 34 * 1. The Megiddo and Modha model assumes any page is evictable. 35 * Pages in its cache cannot be "locked" into memory. This makes 36 * the eviction algorithm simple: evict the last page in the list. 37 * This also make the performance characteristics easy to reason 38 * about. Our cache is not so simple. At any given moment, some 39 * subset of the blocks in the cache are un-evictable because we 40 * have handed out a reference to them. Blocks are only evictable 41 * when there are no external references active. This makes 42 * eviction far more problematic: we choose to evict the evictable 43 * blocks that are the "lowest" in the list. 44 * 45 * There are times when it is not possible to evict the requested 46 * space. In these circumstances we are unable to adjust the cache 47 * size. To prevent the cache growing unbounded at these times we 48 * implement a "cache throttle" that slows the flow of new data 49 * into the cache until we can make space available. 50 * 51 * 2. The Megiddo and Modha model assumes a fixed cache size. 52 * Pages are evicted when the cache is full and there is a cache 53 * miss. Our model has a variable sized cache. It grows with 54 * high use, but also tries to react to memory pressure from the 55 * operating system: decreasing its size when system memory is 56 * tight. 57 * 58 * 3. The Megiddo and Modha model assumes a fixed page size. All 59 * elements of the cache are therefor exactly the same size. So 60 * when adjusting the cache size following a cache miss, its simply 61 * a matter of choosing a single page to evict. In our model, we 62 * have variable sized cache blocks (rangeing from 512 bytes to 63 * 128K bytes). We therefor choose a set of blocks to evict to make 64 * space for a cache miss that approximates as closely as possible 65 * the space used by the new block. 66 * 67 * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache" 68 * by N. Megiddo & D. Modha, FAST 2003 69 */ 70 71 /* 72 * The locking model: 73 * 74 * A new reference to a cache buffer can be obtained in two 75 * ways: 1) via a hash table lookup using the DVA as a key, 76 * or 2) via one of the ARC lists. The arc_read() interface 77 * uses method 1, while the internal arc algorithms for 78 * adjusting the cache use method 2. We therefor provide two 79 * types of locks: 1) the hash table lock array, and 2) the 80 * arc list locks. 81 * 82 * Buffers do not have their own mutexs, rather they rely on the 83 * hash table mutexs for the bulk of their protection (i.e. most 84 * fields in the arc_buf_hdr_t are protected by these mutexs). 85 * 86 * buf_hash_find() returns the appropriate mutex (held) when it 87 * locates the requested buffer in the hash table. It returns 88 * NULL for the mutex if the buffer was not in the table. 89 * 90 * buf_hash_remove() expects the appropriate hash mutex to be 91 * already held before it is invoked. 92 * 93 * Each arc state also has a mutex which is used to protect the 94 * buffer list associated with the state. When attempting to 95 * obtain a hash table lock while holding an arc list lock you 96 * must use: mutex_tryenter() to avoid deadlock. Also note that 97 * the active state mutex must be held before the ghost state mutex. 98 * 99 * Arc buffers may have an associated eviction callback function. 100 * This function will be invoked prior to removing the buffer (e.g. 101 * in arc_do_user_evicts()). Note however that the data associated 102 * with the buffer may be evicted prior to the callback. The callback 103 * must be made with *no locks held* (to prevent deadlock). Additionally, 104 * the users of callbacks must ensure that their private data is 105 * protected from simultaneous callbacks from arc_buf_evict() 106 * and arc_do_user_evicts(). 107 * 108 * Note that the majority of the performance stats are manipulated 109 * with atomic operations. 110 * 111 * The L2ARC uses the l2arc_buflist_mtx global mutex for the following: 112 * 113 * - L2ARC buflist creation 114 * - L2ARC buflist eviction 115 * - L2ARC write completion, which walks L2ARC buflists 116 * - ARC header destruction, as it removes from L2ARC buflists 117 * - ARC header release, as it removes from L2ARC buflists 118 */ 119 120 #include <sys/spa.h> 121 #include <sys/zio.h> 122 #include <sys/zfs_context.h> 123 #include <sys/arc.h> 124 #include <sys/refcount.h> 125 #include <sys/vdev.h> 126 #include <sys/vdev_impl.h> 127 #ifdef _KERNEL 128 #include <sys/vmsystm.h> 129 #include <vm/anon.h> 130 #include <sys/fs/swapnode.h> 131 #include <sys/dnlc.h> 132 #endif 133 #include <sys/callb.h> 134 #include <sys/kstat.h> 135 #include <zfs_fletcher.h> 136 137 static kmutex_t arc_reclaim_thr_lock; 138 static kcondvar_t arc_reclaim_thr_cv; /* used to signal reclaim thr */ 139 static uint8_t arc_thread_exit; 140 141 extern int zfs_write_limit_shift; 142 extern uint64_t zfs_write_limit_max; 143 extern kmutex_t zfs_write_limit_lock; 144 145 #define ARC_REDUCE_DNLC_PERCENT 3 146 uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT; 147 148 typedef enum arc_reclaim_strategy { 149 ARC_RECLAIM_AGGR, /* Aggressive reclaim strategy */ 150 ARC_RECLAIM_CONS /* Conservative reclaim strategy */ 151 } arc_reclaim_strategy_t; 152 153 /* number of seconds before growing cache again */ 154 static int arc_grow_retry = 60; 155 156 /* shift of arc_c for calculating both min and max arc_p */ 157 static int arc_p_min_shift = 4; 158 159 /* log2(fraction of arc to reclaim) */ 160 static int arc_shrink_shift = 5; 161 162 /* 163 * minimum lifespan of a prefetch block in clock ticks 164 * (initialized in arc_init()) 165 */ 166 static int arc_min_prefetch_lifespan; 167 168 static int arc_dead; 169 170 /* 171 * The arc has filled available memory and has now warmed up. 172 */ 173 static boolean_t arc_warm; 174 175 /* 176 * These tunables are for performance analysis. 177 */ 178 uint64_t zfs_arc_max; 179 uint64_t zfs_arc_min; 180 uint64_t zfs_arc_meta_limit = 0; 181 int zfs_arc_grow_retry = 0; 182 int zfs_arc_shrink_shift = 0; 183 int zfs_arc_p_min_shift = 0; 184 185 /* 186 * Note that buffers can be in one of 6 states: 187 * ARC_anon - anonymous (discussed below) 188 * ARC_mru - recently used, currently cached 189 * ARC_mru_ghost - recentely used, no longer in cache 190 * ARC_mfu - frequently used, currently cached 191 * ARC_mfu_ghost - frequently used, no longer in cache 192 * ARC_l2c_only - exists in L2ARC but not other states 193 * When there are no active references to the buffer, they are 194 * are linked onto a list in one of these arc states. These are 195 * the only buffers that can be evicted or deleted. Within each 196 * state there are multiple lists, one for meta-data and one for 197 * non-meta-data. Meta-data (indirect blocks, blocks of dnodes, 198 * etc.) is tracked separately so that it can be managed more 199 * explicitly: favored over data, limited explicitly. 200 * 201 * Anonymous buffers are buffers that are not associated with 202 * a DVA. These are buffers that hold dirty block copies 203 * before they are written to stable storage. By definition, 204 * they are "ref'd" and are considered part of arc_mru 205 * that cannot be freed. Generally, they will aquire a DVA 206 * as they are written and migrate onto the arc_mru list. 207 * 208 * The ARC_l2c_only state is for buffers that are in the second 209 * level ARC but no longer in any of the ARC_m* lists. The second 210 * level ARC itself may also contain buffers that are in any of 211 * the ARC_m* states - meaning that a buffer can exist in two 212 * places. The reason for the ARC_l2c_only state is to keep the 213 * buffer header in the hash table, so that reads that hit the 214 * second level ARC benefit from these fast lookups. 215 */ 216 217 typedef struct arc_state { 218 list_t arcs_list[ARC_BUFC_NUMTYPES]; /* list of evictable buffers */ 219 uint64_t arcs_lsize[ARC_BUFC_NUMTYPES]; /* amount of evictable data */ 220 uint64_t arcs_size; /* total amount of data in this state */ 221 kmutex_t arcs_mtx; 222 } arc_state_t; 223 224 /* The 6 states: */ 225 static arc_state_t ARC_anon; 226 static arc_state_t ARC_mru; 227 static arc_state_t ARC_mru_ghost; 228 static arc_state_t ARC_mfu; 229 static arc_state_t ARC_mfu_ghost; 230 static arc_state_t ARC_l2c_only; 231 232 typedef struct arc_stats { 233 kstat_named_t arcstat_hits; 234 kstat_named_t arcstat_misses; 235 kstat_named_t arcstat_demand_data_hits; 236 kstat_named_t arcstat_demand_data_misses; 237 kstat_named_t arcstat_demand_metadata_hits; 238 kstat_named_t arcstat_demand_metadata_misses; 239 kstat_named_t arcstat_prefetch_data_hits; 240 kstat_named_t arcstat_prefetch_data_misses; 241 kstat_named_t arcstat_prefetch_metadata_hits; 242 kstat_named_t arcstat_prefetch_metadata_misses; 243 kstat_named_t arcstat_mru_hits; 244 kstat_named_t arcstat_mru_ghost_hits; 245 kstat_named_t arcstat_mfu_hits; 246 kstat_named_t arcstat_mfu_ghost_hits; 247 kstat_named_t arcstat_deleted; 248 kstat_named_t arcstat_recycle_miss; 249 kstat_named_t arcstat_mutex_miss; 250 kstat_named_t arcstat_evict_skip; 251 kstat_named_t arcstat_evict_l2_cached; 252 kstat_named_t arcstat_evict_l2_eligible; 253 kstat_named_t arcstat_evict_l2_ineligible; 254 kstat_named_t arcstat_hash_elements; 255 kstat_named_t arcstat_hash_elements_max; 256 kstat_named_t arcstat_hash_collisions; 257 kstat_named_t arcstat_hash_chains; 258 kstat_named_t arcstat_hash_chain_max; 259 kstat_named_t arcstat_p; 260 kstat_named_t arcstat_c; 261 kstat_named_t arcstat_c_min; 262 kstat_named_t arcstat_c_max; 263 kstat_named_t arcstat_size; 264 kstat_named_t arcstat_hdr_size; 265 kstat_named_t arcstat_data_size; 266 kstat_named_t arcstat_other_size; 267 kstat_named_t arcstat_l2_hits; 268 kstat_named_t arcstat_l2_misses; 269 kstat_named_t arcstat_l2_feeds; 270 kstat_named_t arcstat_l2_rw_clash; 271 kstat_named_t arcstat_l2_read_bytes; 272 kstat_named_t arcstat_l2_write_bytes; 273 kstat_named_t arcstat_l2_writes_sent; 274 kstat_named_t arcstat_l2_writes_done; 275 kstat_named_t arcstat_l2_writes_error; 276 kstat_named_t arcstat_l2_writes_hdr_miss; 277 kstat_named_t arcstat_l2_evict_lock_retry; 278 kstat_named_t arcstat_l2_evict_reading; 279 kstat_named_t arcstat_l2_free_on_write; 280 kstat_named_t arcstat_l2_abort_lowmem; 281 kstat_named_t arcstat_l2_cksum_bad; 282 kstat_named_t arcstat_l2_io_error; 283 kstat_named_t arcstat_l2_size; 284 kstat_named_t arcstat_l2_hdr_size; 285 kstat_named_t arcstat_memory_throttle_count; 286 } arc_stats_t; 287 288 static arc_stats_t arc_stats = { 289 { "hits", KSTAT_DATA_UINT64 }, 290 { "misses", KSTAT_DATA_UINT64 }, 291 { "demand_data_hits", KSTAT_DATA_UINT64 }, 292 { "demand_data_misses", KSTAT_DATA_UINT64 }, 293 { "demand_metadata_hits", KSTAT_DATA_UINT64 }, 294 { "demand_metadata_misses", KSTAT_DATA_UINT64 }, 295 { "prefetch_data_hits", KSTAT_DATA_UINT64 }, 296 { "prefetch_data_misses", KSTAT_DATA_UINT64 }, 297 { "prefetch_metadata_hits", KSTAT_DATA_UINT64 }, 298 { "prefetch_metadata_misses", KSTAT_DATA_UINT64 }, 299 { "mru_hits", KSTAT_DATA_UINT64 }, 300 { "mru_ghost_hits", KSTAT_DATA_UINT64 }, 301 { "mfu_hits", KSTAT_DATA_UINT64 }, 302 { "mfu_ghost_hits", KSTAT_DATA_UINT64 }, 303 { "deleted", KSTAT_DATA_UINT64 }, 304 { "recycle_miss", KSTAT_DATA_UINT64 }, 305 { "mutex_miss", KSTAT_DATA_UINT64 }, 306 { "evict_skip", KSTAT_DATA_UINT64 }, 307 { "evict_l2_cached", KSTAT_DATA_UINT64 }, 308 { "evict_l2_eligible", KSTAT_DATA_UINT64 }, 309 { "evict_l2_ineligible", KSTAT_DATA_UINT64 }, 310 { "hash_elements", KSTAT_DATA_UINT64 }, 311 { "hash_elements_max", KSTAT_DATA_UINT64 }, 312 { "hash_collisions", KSTAT_DATA_UINT64 }, 313 { "hash_chains", KSTAT_DATA_UINT64 }, 314 { "hash_chain_max", KSTAT_DATA_UINT64 }, 315 { "p", KSTAT_DATA_UINT64 }, 316 { "c", KSTAT_DATA_UINT64 }, 317 { "c_min", KSTAT_DATA_UINT64 }, 318 { "c_max", KSTAT_DATA_UINT64 }, 319 { "size", KSTAT_DATA_UINT64 }, 320 { "hdr_size", KSTAT_DATA_UINT64 }, 321 { "data_size", KSTAT_DATA_UINT64 }, 322 { "other_size", KSTAT_DATA_UINT64 }, 323 { "l2_hits", KSTAT_DATA_UINT64 }, 324 { "l2_misses", KSTAT_DATA_UINT64 }, 325 { "l2_feeds", KSTAT_DATA_UINT64 }, 326 { "l2_rw_clash", KSTAT_DATA_UINT64 }, 327 { "l2_read_bytes", KSTAT_DATA_UINT64 }, 328 { "l2_write_bytes", KSTAT_DATA_UINT64 }, 329 { "l2_writes_sent", KSTAT_DATA_UINT64 }, 330 { "l2_writes_done", KSTAT_DATA_UINT64 }, 331 { "l2_writes_error", KSTAT_DATA_UINT64 }, 332 { "l2_writes_hdr_miss", KSTAT_DATA_UINT64 }, 333 { "l2_evict_lock_retry", KSTAT_DATA_UINT64 }, 334 { "l2_evict_reading", KSTAT_DATA_UINT64 }, 335 { "l2_free_on_write", KSTAT_DATA_UINT64 }, 336 { "l2_abort_lowmem", KSTAT_DATA_UINT64 }, 337 { "l2_cksum_bad", KSTAT_DATA_UINT64 }, 338 { "l2_io_error", KSTAT_DATA_UINT64 }, 339 { "l2_size", KSTAT_DATA_UINT64 }, 340 { "l2_hdr_size", KSTAT_DATA_UINT64 }, 341 { "memory_throttle_count", KSTAT_DATA_UINT64 } 342 }; 343 344 #define ARCSTAT(stat) (arc_stats.stat.value.ui64) 345 346 #define ARCSTAT_INCR(stat, val) \ 347 atomic_add_64(&arc_stats.stat.value.ui64, (val)); 348 349 #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) 350 #define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1) 351 352 #define ARCSTAT_MAX(stat, val) { \ 353 uint64_t m; \ 354 while ((val) > (m = arc_stats.stat.value.ui64) && \ 355 (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \ 356 continue; \ 357 } 358 359 #define ARCSTAT_MAXSTAT(stat) \ 360 ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64) 361 362 /* 363 * We define a macro to allow ARC hits/misses to be easily broken down by 364 * two separate conditions, giving a total of four different subtypes for 365 * each of hits and misses (so eight statistics total). 366 */ 367 #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \ 368 if (cond1) { \ 369 if (cond2) { \ 370 ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \ 371 } else { \ 372 ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \ 373 } \ 374 } else { \ 375 if (cond2) { \ 376 ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \ 377 } else { \ 378 ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\ 379 } \ 380 } 381 382 kstat_t *arc_ksp; 383 static arc_state_t *arc_anon; 384 static arc_state_t *arc_mru; 385 static arc_state_t *arc_mru_ghost; 386 static arc_state_t *arc_mfu; 387 static arc_state_t *arc_mfu_ghost; 388 static arc_state_t *arc_l2c_only; 389 390 /* 391 * There are several ARC variables that are critical to export as kstats -- 392 * but we don't want to have to grovel around in the kstat whenever we wish to 393 * manipulate them. For these variables, we therefore define them to be in 394 * terms of the statistic variable. This assures that we are not introducing 395 * the possibility of inconsistency by having shadow copies of the variables, 396 * while still allowing the code to be readable. 397 */ 398 #define arc_size ARCSTAT(arcstat_size) /* actual total arc size */ 399 #define arc_p ARCSTAT(arcstat_p) /* target size of MRU */ 400 #define arc_c ARCSTAT(arcstat_c) /* target size of cache */ 401 #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ 402 #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ 403 404 static int arc_no_grow; /* Don't try to grow cache size */ 405 static uint64_t arc_tempreserve; 406 static uint64_t arc_loaned_bytes; 407 static uint64_t arc_meta_used; 408 static uint64_t arc_meta_limit; 409 static uint64_t arc_meta_max = 0; 410 411 typedef struct l2arc_buf_hdr l2arc_buf_hdr_t; 412 413 typedef struct arc_callback arc_callback_t; 414 415 struct arc_callback { 416 void *acb_private; 417 arc_done_func_t *acb_done; 418 arc_buf_t *acb_buf; 419 zio_t *acb_zio_dummy; 420 arc_callback_t *acb_next; 421 }; 422 423 typedef struct arc_write_callback arc_write_callback_t; 424 425 struct arc_write_callback { 426 void *awcb_private; 427 arc_done_func_t *awcb_ready; 428 arc_done_func_t *awcb_done; 429 arc_buf_t *awcb_buf; 430 }; 431 432 struct arc_buf_hdr { 433 /* protected by hash lock */ 434 dva_t b_dva; 435 uint64_t b_birth; 436 uint64_t b_cksum0; 437 438 kmutex_t b_freeze_lock; 439 zio_cksum_t *b_freeze_cksum; 440 441 arc_buf_hdr_t *b_hash_next; 442 arc_buf_t *b_buf; 443 uint32_t b_flags; 444 uint32_t b_datacnt; 445 446 arc_callback_t *b_acb; 447 kcondvar_t b_cv; 448 449 /* immutable */ 450 arc_buf_contents_t b_type; 451 uint64_t b_size; 452 uint64_t b_spa; 453 454 /* protected by arc state mutex */ 455 arc_state_t *b_state; 456 list_node_t b_arc_node; 457 458 /* updated atomically */ 459 clock_t b_arc_access; 460 461 /* self protecting */ 462 refcount_t b_refcnt; 463 464 l2arc_buf_hdr_t *b_l2hdr; 465 list_node_t b_l2node; 466 }; 467 468 static arc_buf_t *arc_eviction_list; 469 static kmutex_t arc_eviction_mtx; 470 static arc_buf_hdr_t arc_eviction_hdr; 471 static void arc_get_data_buf(arc_buf_t *buf); 472 static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock); 473 static int arc_evict_needed(arc_buf_contents_t type); 474 static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes); 475 476 static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab); 477 478 #define GHOST_STATE(state) \ 479 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \ 480 (state) == arc_l2c_only) 481 482 /* 483 * Private ARC flags. These flags are private ARC only flags that will show up 484 * in b_flags in the arc_hdr_buf_t. Some flags are publicly declared, and can 485 * be passed in as arc_flags in things like arc_read. However, these flags 486 * should never be passed and should only be set by ARC code. When adding new 487 * public flags, make sure not to smash the private ones. 488 */ 489 490 #define ARC_IN_HASH_TABLE (1 << 9) /* this buffer is hashed */ 491 #define ARC_IO_IN_PROGRESS (1 << 10) /* I/O in progress for buf */ 492 #define ARC_IO_ERROR (1 << 11) /* I/O failed for buf */ 493 #define ARC_FREED_IN_READ (1 << 12) /* buf freed while in read */ 494 #define ARC_BUF_AVAILABLE (1 << 13) /* block not in active use */ 495 #define ARC_INDIRECT (1 << 14) /* this is an indirect block */ 496 #define ARC_FREE_IN_PROGRESS (1 << 15) /* hdr about to be freed */ 497 #define ARC_L2_WRITING (1 << 16) /* L2ARC write in progress */ 498 #define ARC_L2_EVICTED (1 << 17) /* evicted during I/O */ 499 #define ARC_L2_WRITE_HEAD (1 << 18) /* head of write list */ 500 501 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_IN_HASH_TABLE) 502 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS) 503 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_IO_ERROR) 504 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_PREFETCH) 505 #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FREED_IN_READ) 506 #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_BUF_AVAILABLE) 507 #define HDR_FREE_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FREE_IN_PROGRESS) 508 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_L2CACHE) 509 #define HDR_L2_READING(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS && \ 510 (hdr)->b_l2hdr != NULL) 511 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_L2_WRITING) 512 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_L2_EVICTED) 513 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_L2_WRITE_HEAD) 514 515 /* 516 * Other sizes 517 */ 518 519 #define HDR_SIZE ((int64_t)sizeof (arc_buf_hdr_t)) 520 #define L2HDR_SIZE ((int64_t)sizeof (l2arc_buf_hdr_t)) 521 522 /* 523 * Hash table routines 524 */ 525 526 #define HT_LOCK_PAD 64 527 528 struct ht_lock { 529 kmutex_t ht_lock; 530 #ifdef _KERNEL 531 unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))]; 532 #endif 533 }; 534 535 #define BUF_LOCKS 256 536 typedef struct buf_hash_table { 537 uint64_t ht_mask; 538 arc_buf_hdr_t **ht_table; 539 struct ht_lock ht_locks[BUF_LOCKS]; 540 } buf_hash_table_t; 541 542 static buf_hash_table_t buf_hash_table; 543 544 #define BUF_HASH_INDEX(spa, dva, birth) \ 545 (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask) 546 #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)]) 547 #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock)) 548 #define HDR_LOCK(buf) \ 549 (BUF_HASH_LOCK(BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth))) 550 551 uint64_t zfs_crc64_table[256]; 552 553 /* 554 * Level 2 ARC 555 */ 556 557 #define L2ARC_WRITE_SIZE (8 * 1024 * 1024) /* initial write max */ 558 #define L2ARC_HEADROOM 2 /* num of writes */ 559 #define L2ARC_FEED_SECS 1 /* caching interval secs */ 560 #define L2ARC_FEED_MIN_MS 200 /* min caching interval ms */ 561 562 #define l2arc_writes_sent ARCSTAT(arcstat_l2_writes_sent) 563 #define l2arc_writes_done ARCSTAT(arcstat_l2_writes_done) 564 565 /* 566 * L2ARC Performance Tunables 567 */ 568 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE; /* default max write size */ 569 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE; /* extra write during warmup */ 570 uint64_t l2arc_headroom = L2ARC_HEADROOM; /* number of dev writes */ 571 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS; /* interval seconds */ 572 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval milliseconds */ 573 boolean_t l2arc_noprefetch = B_TRUE; /* don't cache prefetch bufs */ 574 boolean_t l2arc_feed_again = B_TRUE; /* turbo warmup */ 575 boolean_t l2arc_norw = B_TRUE; /* no reads during writes */ 576 577 /* 578 * L2ARC Internals 579 */ 580 typedef struct l2arc_dev { 581 vdev_t *l2ad_vdev; /* vdev */ 582 spa_t *l2ad_spa; /* spa */ 583 uint64_t l2ad_hand; /* next write location */ 584 uint64_t l2ad_write; /* desired write size, bytes */ 585 uint64_t l2ad_boost; /* warmup write boost, bytes */ 586 uint64_t l2ad_start; /* first addr on device */ 587 uint64_t l2ad_end; /* last addr on device */ 588 uint64_t l2ad_evict; /* last addr eviction reached */ 589 boolean_t l2ad_first; /* first sweep through */ 590 boolean_t l2ad_writing; /* currently writing */ 591 list_t *l2ad_buflist; /* buffer list */ 592 list_node_t l2ad_node; /* device list node */ 593 } l2arc_dev_t; 594 595 static list_t L2ARC_dev_list; /* device list */ 596 static list_t *l2arc_dev_list; /* device list pointer */ 597 static kmutex_t l2arc_dev_mtx; /* device list mutex */ 598 static l2arc_dev_t *l2arc_dev_last; /* last device used */ 599 static kmutex_t l2arc_buflist_mtx; /* mutex for all buflists */ 600 static list_t L2ARC_free_on_write; /* free after write buf list */ 601 static list_t *l2arc_free_on_write; /* free after write list ptr */ 602 static kmutex_t l2arc_free_on_write_mtx; /* mutex for list */ 603 static uint64_t l2arc_ndev; /* number of devices */ 604 605 typedef struct l2arc_read_callback { 606 arc_buf_t *l2rcb_buf; /* read buffer */ 607 spa_t *l2rcb_spa; /* spa */ 608 blkptr_t l2rcb_bp; /* original blkptr */ 609 zbookmark_t l2rcb_zb; /* original bookmark */ 610 int l2rcb_flags; /* original flags */ 611 } l2arc_read_callback_t; 612 613 typedef struct l2arc_write_callback { 614 l2arc_dev_t *l2wcb_dev; /* device info */ 615 arc_buf_hdr_t *l2wcb_head; /* head of write buflist */ 616 } l2arc_write_callback_t; 617 618 struct l2arc_buf_hdr { 619 /* protected by arc_buf_hdr mutex */ 620 l2arc_dev_t *b_dev; /* L2ARC device */ 621 uint64_t b_daddr; /* disk address, offset byte */ 622 }; 623 624 typedef struct l2arc_data_free { 625 /* protected by l2arc_free_on_write_mtx */ 626 void *l2df_data; 627 size_t l2df_size; 628 void (*l2df_func)(void *, size_t); 629 list_node_t l2df_list_node; 630 } l2arc_data_free_t; 631 632 static kmutex_t l2arc_feed_thr_lock; 633 static kcondvar_t l2arc_feed_thr_cv; 634 static uint8_t l2arc_thread_exit; 635 636 static void l2arc_read_done(zio_t *zio); 637 static void l2arc_hdr_stat_add(void); 638 static void l2arc_hdr_stat_remove(void); 639 640 static uint64_t 641 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth) 642 { 643 uint8_t *vdva = (uint8_t *)dva; 644 uint64_t crc = -1ULL; 645 int i; 646 647 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 648 649 for (i = 0; i < sizeof (dva_t); i++) 650 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF]; 651 652 crc ^= (spa>>8) ^ birth; 653 654 return (crc); 655 } 656 657 #define BUF_EMPTY(buf) \ 658 ((buf)->b_dva.dva_word[0] == 0 && \ 659 (buf)->b_dva.dva_word[1] == 0 && \ 660 (buf)->b_birth == 0) 661 662 #define BUF_EQUAL(spa, dva, birth, buf) \ 663 ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \ 664 ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \ 665 ((buf)->b_birth == birth) && ((buf)->b_spa == spa) 666 667 static arc_buf_hdr_t * 668 buf_hash_find(uint64_t spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp) 669 { 670 uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); 671 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 672 arc_buf_hdr_t *buf; 673 674 mutex_enter(hash_lock); 675 for (buf = buf_hash_table.ht_table[idx]; buf != NULL; 676 buf = buf->b_hash_next) { 677 if (BUF_EQUAL(spa, dva, birth, buf)) { 678 *lockp = hash_lock; 679 return (buf); 680 } 681 } 682 mutex_exit(hash_lock); 683 *lockp = NULL; 684 return (NULL); 685 } 686 687 /* 688 * Insert an entry into the hash table. If there is already an element 689 * equal to elem in the hash table, then the already existing element 690 * will be returned and the new element will not be inserted. 691 * Otherwise returns NULL. 692 */ 693 static arc_buf_hdr_t * 694 buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp) 695 { 696 uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 697 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 698 arc_buf_hdr_t *fbuf; 699 uint32_t i; 700 701 ASSERT(!HDR_IN_HASH_TABLE(buf)); 702 *lockp = hash_lock; 703 mutex_enter(hash_lock); 704 for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL; 705 fbuf = fbuf->b_hash_next, i++) { 706 if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf)) 707 return (fbuf); 708 } 709 710 buf->b_hash_next = buf_hash_table.ht_table[idx]; 711 buf_hash_table.ht_table[idx] = buf; 712 buf->b_flags |= ARC_IN_HASH_TABLE; 713 714 /* collect some hash table performance data */ 715 if (i > 0) { 716 ARCSTAT_BUMP(arcstat_hash_collisions); 717 if (i == 1) 718 ARCSTAT_BUMP(arcstat_hash_chains); 719 720 ARCSTAT_MAX(arcstat_hash_chain_max, i); 721 } 722 723 ARCSTAT_BUMP(arcstat_hash_elements); 724 ARCSTAT_MAXSTAT(arcstat_hash_elements); 725 726 return (NULL); 727 } 728 729 static void 730 buf_hash_remove(arc_buf_hdr_t *buf) 731 { 732 arc_buf_hdr_t *fbuf, **bufp; 733 uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 734 735 ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx))); 736 ASSERT(HDR_IN_HASH_TABLE(buf)); 737 738 bufp = &buf_hash_table.ht_table[idx]; 739 while ((fbuf = *bufp) != buf) { 740 ASSERT(fbuf != NULL); 741 bufp = &fbuf->b_hash_next; 742 } 743 *bufp = buf->b_hash_next; 744 buf->b_hash_next = NULL; 745 buf->b_flags &= ~ARC_IN_HASH_TABLE; 746 747 /* collect some hash table performance data */ 748 ARCSTAT_BUMPDOWN(arcstat_hash_elements); 749 750 if (buf_hash_table.ht_table[idx] && 751 buf_hash_table.ht_table[idx]->b_hash_next == NULL) 752 ARCSTAT_BUMPDOWN(arcstat_hash_chains); 753 } 754 755 /* 756 * Global data structures and functions for the buf kmem cache. 757 */ 758 static kmem_cache_t *hdr_cache; 759 static kmem_cache_t *buf_cache; 760 761 static void 762 buf_fini(void) 763 { 764 int i; 765 766 kmem_free(buf_hash_table.ht_table, 767 (buf_hash_table.ht_mask + 1) * sizeof (void *)); 768 for (i = 0; i < BUF_LOCKS; i++) 769 mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock); 770 kmem_cache_destroy(hdr_cache); 771 kmem_cache_destroy(buf_cache); 772 } 773 774 /* 775 * Constructor callback - called when the cache is empty 776 * and a new buf is requested. 777 */ 778 /* ARGSUSED */ 779 static int 780 hdr_cons(void *vbuf, void *unused, int kmflag) 781 { 782 arc_buf_hdr_t *buf = vbuf; 783 784 bzero(buf, sizeof (arc_buf_hdr_t)); 785 refcount_create(&buf->b_refcnt); 786 cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL); 787 mutex_init(&buf->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL); 788 arc_space_consume(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS); 789 790 return (0); 791 } 792 793 /* ARGSUSED */ 794 static int 795 buf_cons(void *vbuf, void *unused, int kmflag) 796 { 797 arc_buf_t *buf = vbuf; 798 799 bzero(buf, sizeof (arc_buf_t)); 800 rw_init(&buf->b_lock, NULL, RW_DEFAULT, NULL); 801 arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS); 802 803 return (0); 804 } 805 806 /* 807 * Destructor callback - called when a cached buf is 808 * no longer required. 809 */ 810 /* ARGSUSED */ 811 static void 812 hdr_dest(void *vbuf, void *unused) 813 { 814 arc_buf_hdr_t *buf = vbuf; 815 816 ASSERT(BUF_EMPTY(buf)); 817 refcount_destroy(&buf->b_refcnt); 818 cv_destroy(&buf->b_cv); 819 mutex_destroy(&buf->b_freeze_lock); 820 arc_space_return(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS); 821 } 822 823 /* ARGSUSED */ 824 static void 825 buf_dest(void *vbuf, void *unused) 826 { 827 arc_buf_t *buf = vbuf; 828 829 rw_destroy(&buf->b_lock); 830 arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS); 831 } 832 833 /* 834 * Reclaim callback -- invoked when memory is low. 835 */ 836 /* ARGSUSED */ 837 static void 838 hdr_recl(void *unused) 839 { 840 dprintf("hdr_recl called\n"); 841 /* 842 * umem calls the reclaim func when we destroy the buf cache, 843 * which is after we do arc_fini(). 844 */ 845 if (!arc_dead) 846 cv_signal(&arc_reclaim_thr_cv); 847 } 848 849 static void 850 buf_init(void) 851 { 852 uint64_t *ct; 853 uint64_t hsize = 1ULL << 12; 854 int i, j; 855 856 /* 857 * The hash table is big enough to fill all of physical memory 858 * with an average 64K block size. The table will take up 859 * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers). 860 */ 861 while (hsize * 65536 < physmem * PAGESIZE) 862 hsize <<= 1; 863 retry: 864 buf_hash_table.ht_mask = hsize - 1; 865 buf_hash_table.ht_table = 866 kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP); 867 if (buf_hash_table.ht_table == NULL) { 868 ASSERT(hsize > (1ULL << 8)); 869 hsize >>= 1; 870 goto retry; 871 } 872 873 hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t), 874 0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0); 875 buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t), 876 0, buf_cons, buf_dest, NULL, NULL, NULL, 0); 877 878 for (i = 0; i < 256; i++) 879 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--) 880 *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY); 881 882 for (i = 0; i < BUF_LOCKS; i++) { 883 mutex_init(&buf_hash_table.ht_locks[i].ht_lock, 884 NULL, MUTEX_DEFAULT, NULL); 885 } 886 } 887 888 #define ARC_MINTIME (hz>>4) /* 62 ms */ 889 890 static void 891 arc_cksum_verify(arc_buf_t *buf) 892 { 893 zio_cksum_t zc; 894 895 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 896 return; 897 898 mutex_enter(&buf->b_hdr->b_freeze_lock); 899 if (buf->b_hdr->b_freeze_cksum == NULL || 900 (buf->b_hdr->b_flags & ARC_IO_ERROR)) { 901 mutex_exit(&buf->b_hdr->b_freeze_lock); 902 return; 903 } 904 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 905 if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc)) 906 panic("buffer modified while frozen!"); 907 mutex_exit(&buf->b_hdr->b_freeze_lock); 908 } 909 910 static int 911 arc_cksum_equal(arc_buf_t *buf) 912 { 913 zio_cksum_t zc; 914 int equal; 915 916 mutex_enter(&buf->b_hdr->b_freeze_lock); 917 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 918 equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc); 919 mutex_exit(&buf->b_hdr->b_freeze_lock); 920 921 return (equal); 922 } 923 924 static void 925 arc_cksum_compute(arc_buf_t *buf, boolean_t force) 926 { 927 if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY)) 928 return; 929 930 mutex_enter(&buf->b_hdr->b_freeze_lock); 931 if (buf->b_hdr->b_freeze_cksum != NULL) { 932 mutex_exit(&buf->b_hdr->b_freeze_lock); 933 return; 934 } 935 buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP); 936 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, 937 buf->b_hdr->b_freeze_cksum); 938 mutex_exit(&buf->b_hdr->b_freeze_lock); 939 } 940 941 void 942 arc_buf_thaw(arc_buf_t *buf) 943 { 944 if (zfs_flags & ZFS_DEBUG_MODIFY) { 945 if (buf->b_hdr->b_state != arc_anon) 946 panic("modifying non-anon buffer!"); 947 if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS) 948 panic("modifying buffer while i/o in progress!"); 949 arc_cksum_verify(buf); 950 } 951 952 mutex_enter(&buf->b_hdr->b_freeze_lock); 953 if (buf->b_hdr->b_freeze_cksum != NULL) { 954 kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 955 buf->b_hdr->b_freeze_cksum = NULL; 956 } 957 mutex_exit(&buf->b_hdr->b_freeze_lock); 958 } 959 960 void 961 arc_buf_freeze(arc_buf_t *buf) 962 { 963 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 964 return; 965 966 ASSERT(buf->b_hdr->b_freeze_cksum != NULL || 967 buf->b_hdr->b_state == arc_anon); 968 arc_cksum_compute(buf, B_FALSE); 969 } 970 971 static void 972 add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 973 { 974 ASSERT(MUTEX_HELD(hash_lock)); 975 976 if ((refcount_add(&ab->b_refcnt, tag) == 1) && 977 (ab->b_state != arc_anon)) { 978 uint64_t delta = ab->b_size * ab->b_datacnt; 979 list_t *list = &ab->b_state->arcs_list[ab->b_type]; 980 uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type]; 981 982 ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx)); 983 mutex_enter(&ab->b_state->arcs_mtx); 984 ASSERT(list_link_active(&ab->b_arc_node)); 985 list_remove(list, ab); 986 if (GHOST_STATE(ab->b_state)) { 987 ASSERT3U(ab->b_datacnt, ==, 0); 988 ASSERT3P(ab->b_buf, ==, NULL); 989 delta = ab->b_size; 990 } 991 ASSERT(delta > 0); 992 ASSERT3U(*size, >=, delta); 993 atomic_add_64(size, -delta); 994 mutex_exit(&ab->b_state->arcs_mtx); 995 /* remove the prefetch flag if we get a reference */ 996 if (ab->b_flags & ARC_PREFETCH) 997 ab->b_flags &= ~ARC_PREFETCH; 998 } 999 } 1000 1001 static int 1002 remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 1003 { 1004 int cnt; 1005 arc_state_t *state = ab->b_state; 1006 1007 ASSERT(state == arc_anon || MUTEX_HELD(hash_lock)); 1008 ASSERT(!GHOST_STATE(state)); 1009 1010 if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) && 1011 (state != arc_anon)) { 1012 uint64_t *size = &state->arcs_lsize[ab->b_type]; 1013 1014 ASSERT(!MUTEX_HELD(&state->arcs_mtx)); 1015 mutex_enter(&state->arcs_mtx); 1016 ASSERT(!list_link_active(&ab->b_arc_node)); 1017 list_insert_head(&state->arcs_list[ab->b_type], ab); 1018 ASSERT(ab->b_datacnt > 0); 1019 atomic_add_64(size, ab->b_size * ab->b_datacnt); 1020 mutex_exit(&state->arcs_mtx); 1021 } 1022 return (cnt); 1023 } 1024 1025 /* 1026 * Move the supplied buffer to the indicated state. The mutex 1027 * for the buffer must be held by the caller. 1028 */ 1029 static void 1030 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock) 1031 { 1032 arc_state_t *old_state = ab->b_state; 1033 int64_t refcnt = refcount_count(&ab->b_refcnt); 1034 uint64_t from_delta, to_delta; 1035 1036 ASSERT(MUTEX_HELD(hash_lock)); 1037 ASSERT(new_state != old_state); 1038 ASSERT(refcnt == 0 || ab->b_datacnt > 0); 1039 ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state)); 1040 ASSERT(ab->b_datacnt <= 1 || new_state != arc_anon); 1041 ASSERT(ab->b_datacnt <= 1 || old_state != arc_anon); 1042 1043 from_delta = to_delta = ab->b_datacnt * ab->b_size; 1044 1045 /* 1046 * If this buffer is evictable, transfer it from the 1047 * old state list to the new state list. 1048 */ 1049 if (refcnt == 0) { 1050 if (old_state != arc_anon) { 1051 int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx); 1052 uint64_t *size = &old_state->arcs_lsize[ab->b_type]; 1053 1054 if (use_mutex) 1055 mutex_enter(&old_state->arcs_mtx); 1056 1057 ASSERT(list_link_active(&ab->b_arc_node)); 1058 list_remove(&old_state->arcs_list[ab->b_type], ab); 1059 1060 /* 1061 * If prefetching out of the ghost cache, 1062 * we will have a non-null datacnt. 1063 */ 1064 if (GHOST_STATE(old_state) && ab->b_datacnt == 0) { 1065 /* ghost elements have a ghost size */ 1066 ASSERT(ab->b_buf == NULL); 1067 from_delta = ab->b_size; 1068 } 1069 ASSERT3U(*size, >=, from_delta); 1070 atomic_add_64(size, -from_delta); 1071 1072 if (use_mutex) 1073 mutex_exit(&old_state->arcs_mtx); 1074 } 1075 if (new_state != arc_anon) { 1076 int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx); 1077 uint64_t *size = &new_state->arcs_lsize[ab->b_type]; 1078 1079 if (use_mutex) 1080 mutex_enter(&new_state->arcs_mtx); 1081 1082 list_insert_head(&new_state->arcs_list[ab->b_type], ab); 1083 1084 /* ghost elements have a ghost size */ 1085 if (GHOST_STATE(new_state)) { 1086 ASSERT(ab->b_datacnt == 0); 1087 ASSERT(ab->b_buf == NULL); 1088 to_delta = ab->b_size; 1089 } 1090 atomic_add_64(size, to_delta); 1091 1092 if (use_mutex) 1093 mutex_exit(&new_state->arcs_mtx); 1094 } 1095 } 1096 1097 ASSERT(!BUF_EMPTY(ab)); 1098 if (new_state == arc_anon) { 1099 buf_hash_remove(ab); 1100 } 1101 1102 /* adjust state sizes */ 1103 if (to_delta) 1104 atomic_add_64(&new_state->arcs_size, to_delta); 1105 if (from_delta) { 1106 ASSERT3U(old_state->arcs_size, >=, from_delta); 1107 atomic_add_64(&old_state->arcs_size, -from_delta); 1108 } 1109 ab->b_state = new_state; 1110 1111 /* adjust l2arc hdr stats */ 1112 if (new_state == arc_l2c_only) 1113 l2arc_hdr_stat_add(); 1114 else if (old_state == arc_l2c_only) 1115 l2arc_hdr_stat_remove(); 1116 } 1117 1118 void 1119 arc_space_consume(uint64_t space, arc_space_type_t type) 1120 { 1121 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 1122 1123 switch (type) { 1124 case ARC_SPACE_DATA: 1125 ARCSTAT_INCR(arcstat_data_size, space); 1126 break; 1127 case ARC_SPACE_OTHER: 1128 ARCSTAT_INCR(arcstat_other_size, space); 1129 break; 1130 case ARC_SPACE_HDRS: 1131 ARCSTAT_INCR(arcstat_hdr_size, space); 1132 break; 1133 case ARC_SPACE_L2HDRS: 1134 ARCSTAT_INCR(arcstat_l2_hdr_size, space); 1135 break; 1136 } 1137 1138 atomic_add_64(&arc_meta_used, space); 1139 atomic_add_64(&arc_size, space); 1140 } 1141 1142 void 1143 arc_space_return(uint64_t space, arc_space_type_t type) 1144 { 1145 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 1146 1147 switch (type) { 1148 case ARC_SPACE_DATA: 1149 ARCSTAT_INCR(arcstat_data_size, -space); 1150 break; 1151 case ARC_SPACE_OTHER: 1152 ARCSTAT_INCR(arcstat_other_size, -space); 1153 break; 1154 case ARC_SPACE_HDRS: 1155 ARCSTAT_INCR(arcstat_hdr_size, -space); 1156 break; 1157 case ARC_SPACE_L2HDRS: 1158 ARCSTAT_INCR(arcstat_l2_hdr_size, -space); 1159 break; 1160 } 1161 1162 ASSERT(arc_meta_used >= space); 1163 if (arc_meta_max < arc_meta_used) 1164 arc_meta_max = arc_meta_used; 1165 atomic_add_64(&arc_meta_used, -space); 1166 ASSERT(arc_size >= space); 1167 atomic_add_64(&arc_size, -space); 1168 } 1169 1170 void * 1171 arc_data_buf_alloc(uint64_t size) 1172 { 1173 if (arc_evict_needed(ARC_BUFC_DATA)) 1174 cv_signal(&arc_reclaim_thr_cv); 1175 atomic_add_64(&arc_size, size); 1176 return (zio_data_buf_alloc(size)); 1177 } 1178 1179 void 1180 arc_data_buf_free(void *buf, uint64_t size) 1181 { 1182 zio_data_buf_free(buf, size); 1183 ASSERT(arc_size >= size); 1184 atomic_add_64(&arc_size, -size); 1185 } 1186 1187 arc_buf_t * 1188 arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type) 1189 { 1190 arc_buf_hdr_t *hdr; 1191 arc_buf_t *buf; 1192 1193 ASSERT3U(size, >, 0); 1194 hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 1195 ASSERT(BUF_EMPTY(hdr)); 1196 hdr->b_size = size; 1197 hdr->b_type = type; 1198 hdr->b_spa = spa_guid(spa); 1199 hdr->b_state = arc_anon; 1200 hdr->b_arc_access = 0; 1201 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 1202 buf->b_hdr = hdr; 1203 buf->b_data = NULL; 1204 buf->b_efunc = NULL; 1205 buf->b_private = NULL; 1206 buf->b_next = NULL; 1207 hdr->b_buf = buf; 1208 arc_get_data_buf(buf); 1209 hdr->b_datacnt = 1; 1210 hdr->b_flags = 0; 1211 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1212 (void) refcount_add(&hdr->b_refcnt, tag); 1213 1214 return (buf); 1215 } 1216 1217 static char *arc_onloan_tag = "onloan"; 1218 1219 /* 1220 * Loan out an anonymous arc buffer. Loaned buffers are not counted as in 1221 * flight data by arc_tempreserve_space() until they are "returned". Loaned 1222 * buffers must be returned to the arc before they can be used by the DMU or 1223 * freed. 1224 */ 1225 arc_buf_t * 1226 arc_loan_buf(spa_t *spa, int size) 1227 { 1228 arc_buf_t *buf; 1229 1230 buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA); 1231 1232 atomic_add_64(&arc_loaned_bytes, size); 1233 return (buf); 1234 } 1235 1236 /* 1237 * Return a loaned arc buffer to the arc. 1238 */ 1239 void 1240 arc_return_buf(arc_buf_t *buf, void *tag) 1241 { 1242 arc_buf_hdr_t *hdr = buf->b_hdr; 1243 1244 ASSERT(hdr->b_state == arc_anon); 1245 ASSERT(buf->b_data != NULL); 1246 VERIFY(refcount_remove(&hdr->b_refcnt, arc_onloan_tag) == 0); 1247 VERIFY(refcount_add(&hdr->b_refcnt, tag) == 1); 1248 1249 atomic_add_64(&arc_loaned_bytes, -hdr->b_size); 1250 } 1251 1252 static arc_buf_t * 1253 arc_buf_clone(arc_buf_t *from) 1254 { 1255 arc_buf_t *buf; 1256 arc_buf_hdr_t *hdr = from->b_hdr; 1257 uint64_t size = hdr->b_size; 1258 1259 ASSERT(hdr->b_state != arc_anon); 1260 1261 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 1262 buf->b_hdr = hdr; 1263 buf->b_data = NULL; 1264 buf->b_efunc = NULL; 1265 buf->b_private = NULL; 1266 buf->b_next = hdr->b_buf; 1267 hdr->b_buf = buf; 1268 arc_get_data_buf(buf); 1269 bcopy(from->b_data, buf->b_data, size); 1270 hdr->b_datacnt += 1; 1271 return (buf); 1272 } 1273 1274 void 1275 arc_buf_add_ref(arc_buf_t *buf, void* tag) 1276 { 1277 arc_buf_hdr_t *hdr; 1278 kmutex_t *hash_lock; 1279 1280 /* 1281 * Check to see if this buffer is evicted. Callers 1282 * must verify b_data != NULL to know if the add_ref 1283 * was successful. 1284 */ 1285 rw_enter(&buf->b_lock, RW_READER); 1286 if (buf->b_data == NULL) { 1287 rw_exit(&buf->b_lock); 1288 return; 1289 } 1290 hdr = buf->b_hdr; 1291 ASSERT(hdr != NULL); 1292 hash_lock = HDR_LOCK(hdr); 1293 mutex_enter(hash_lock); 1294 rw_exit(&buf->b_lock); 1295 1296 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 1297 add_reference(hdr, hash_lock, tag); 1298 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 1299 arc_access(hdr, hash_lock); 1300 mutex_exit(hash_lock); 1301 ARCSTAT_BUMP(arcstat_hits); 1302 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 1303 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 1304 data, metadata, hits); 1305 } 1306 1307 /* 1308 * Free the arc data buffer. If it is an l2arc write in progress, 1309 * the buffer is placed on l2arc_free_on_write to be freed later. 1310 */ 1311 static void 1312 arc_buf_data_free(arc_buf_hdr_t *hdr, void (*free_func)(void *, size_t), 1313 void *data, size_t size) 1314 { 1315 if (HDR_L2_WRITING(hdr)) { 1316 l2arc_data_free_t *df; 1317 df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP); 1318 df->l2df_data = data; 1319 df->l2df_size = size; 1320 df->l2df_func = free_func; 1321 mutex_enter(&l2arc_free_on_write_mtx); 1322 list_insert_head(l2arc_free_on_write, df); 1323 mutex_exit(&l2arc_free_on_write_mtx); 1324 ARCSTAT_BUMP(arcstat_l2_free_on_write); 1325 } else { 1326 free_func(data, size); 1327 } 1328 } 1329 1330 static void 1331 arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all) 1332 { 1333 arc_buf_t **bufp; 1334 1335 /* free up data associated with the buf */ 1336 if (buf->b_data) { 1337 arc_state_t *state = buf->b_hdr->b_state; 1338 uint64_t size = buf->b_hdr->b_size; 1339 arc_buf_contents_t type = buf->b_hdr->b_type; 1340 1341 arc_cksum_verify(buf); 1342 1343 if (!recycle) { 1344 if (type == ARC_BUFC_METADATA) { 1345 arc_buf_data_free(buf->b_hdr, zio_buf_free, 1346 buf->b_data, size); 1347 arc_space_return(size, ARC_SPACE_DATA); 1348 } else { 1349 ASSERT(type == ARC_BUFC_DATA); 1350 arc_buf_data_free(buf->b_hdr, 1351 zio_data_buf_free, buf->b_data, size); 1352 ARCSTAT_INCR(arcstat_data_size, -size); 1353 atomic_add_64(&arc_size, -size); 1354 } 1355 } 1356 if (list_link_active(&buf->b_hdr->b_arc_node)) { 1357 uint64_t *cnt = &state->arcs_lsize[type]; 1358 1359 ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt)); 1360 ASSERT(state != arc_anon); 1361 1362 ASSERT3U(*cnt, >=, size); 1363 atomic_add_64(cnt, -size); 1364 } 1365 ASSERT3U(state->arcs_size, >=, size); 1366 atomic_add_64(&state->arcs_size, -size); 1367 buf->b_data = NULL; 1368 ASSERT(buf->b_hdr->b_datacnt > 0); 1369 buf->b_hdr->b_datacnt -= 1; 1370 } 1371 1372 /* only remove the buf if requested */ 1373 if (!all) 1374 return; 1375 1376 /* remove the buf from the hdr list */ 1377 for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next) 1378 continue; 1379 *bufp = buf->b_next; 1380 1381 ASSERT(buf->b_efunc == NULL); 1382 1383 /* clean up the buf */ 1384 buf->b_hdr = NULL; 1385 kmem_cache_free(buf_cache, buf); 1386 } 1387 1388 static void 1389 arc_hdr_destroy(arc_buf_hdr_t *hdr) 1390 { 1391 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1392 ASSERT3P(hdr->b_state, ==, arc_anon); 1393 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 1394 l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr; 1395 1396 if (l2hdr != NULL) { 1397 boolean_t buflist_held = MUTEX_HELD(&l2arc_buflist_mtx); 1398 /* 1399 * To prevent arc_free() and l2arc_evict() from 1400 * attempting to free the same buffer at the same time, 1401 * a FREE_IN_PROGRESS flag is given to arc_free() to 1402 * give it priority. l2arc_evict() can't destroy this 1403 * header while we are waiting on l2arc_buflist_mtx. 1404 * 1405 * The hdr may be removed from l2ad_buflist before we 1406 * grab l2arc_buflist_mtx, so b_l2hdr is rechecked. 1407 */ 1408 if (!buflist_held) { 1409 mutex_enter(&l2arc_buflist_mtx); 1410 l2hdr = hdr->b_l2hdr; 1411 } 1412 1413 if (l2hdr != NULL) { 1414 list_remove(l2hdr->b_dev->l2ad_buflist, hdr); 1415 ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size); 1416 kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t)); 1417 if (hdr->b_state == arc_l2c_only) 1418 l2arc_hdr_stat_remove(); 1419 hdr->b_l2hdr = NULL; 1420 } 1421 1422 if (!buflist_held) 1423 mutex_exit(&l2arc_buflist_mtx); 1424 } 1425 1426 if (!BUF_EMPTY(hdr)) { 1427 ASSERT(!HDR_IN_HASH_TABLE(hdr)); 1428 bzero(&hdr->b_dva, sizeof (dva_t)); 1429 hdr->b_birth = 0; 1430 hdr->b_cksum0 = 0; 1431 } 1432 while (hdr->b_buf) { 1433 arc_buf_t *buf = hdr->b_buf; 1434 1435 if (buf->b_efunc) { 1436 mutex_enter(&arc_eviction_mtx); 1437 rw_enter(&buf->b_lock, RW_WRITER); 1438 ASSERT(buf->b_hdr != NULL); 1439 arc_buf_destroy(hdr->b_buf, FALSE, FALSE); 1440 hdr->b_buf = buf->b_next; 1441 buf->b_hdr = &arc_eviction_hdr; 1442 buf->b_next = arc_eviction_list; 1443 arc_eviction_list = buf; 1444 rw_exit(&buf->b_lock); 1445 mutex_exit(&arc_eviction_mtx); 1446 } else { 1447 arc_buf_destroy(hdr->b_buf, FALSE, TRUE); 1448 } 1449 } 1450 if (hdr->b_freeze_cksum != NULL) { 1451 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 1452 hdr->b_freeze_cksum = NULL; 1453 } 1454 1455 ASSERT(!list_link_active(&hdr->b_arc_node)); 1456 ASSERT3P(hdr->b_hash_next, ==, NULL); 1457 ASSERT3P(hdr->b_acb, ==, NULL); 1458 kmem_cache_free(hdr_cache, hdr); 1459 } 1460 1461 void 1462 arc_buf_free(arc_buf_t *buf, void *tag) 1463 { 1464 arc_buf_hdr_t *hdr = buf->b_hdr; 1465 int hashed = hdr->b_state != arc_anon; 1466 1467 ASSERT(buf->b_efunc == NULL); 1468 ASSERT(buf->b_data != NULL); 1469 1470 if (hashed) { 1471 kmutex_t *hash_lock = HDR_LOCK(hdr); 1472 1473 mutex_enter(hash_lock); 1474 (void) remove_reference(hdr, hash_lock, tag); 1475 if (hdr->b_datacnt > 1) { 1476 arc_buf_destroy(buf, FALSE, TRUE); 1477 } else { 1478 ASSERT(buf == hdr->b_buf); 1479 ASSERT(buf->b_efunc == NULL); 1480 hdr->b_flags |= ARC_BUF_AVAILABLE; 1481 } 1482 mutex_exit(hash_lock); 1483 } else if (HDR_IO_IN_PROGRESS(hdr)) { 1484 int destroy_hdr; 1485 /* 1486 * We are in the middle of an async write. Don't destroy 1487 * this buffer unless the write completes before we finish 1488 * decrementing the reference count. 1489 */ 1490 mutex_enter(&arc_eviction_mtx); 1491 (void) remove_reference(hdr, NULL, tag); 1492 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1493 destroy_hdr = !HDR_IO_IN_PROGRESS(hdr); 1494 mutex_exit(&arc_eviction_mtx); 1495 if (destroy_hdr) 1496 arc_hdr_destroy(hdr); 1497 } else { 1498 if (remove_reference(hdr, NULL, tag) > 0) { 1499 ASSERT(HDR_IO_ERROR(hdr)); 1500 arc_buf_destroy(buf, FALSE, TRUE); 1501 } else { 1502 arc_hdr_destroy(hdr); 1503 } 1504 } 1505 } 1506 1507 int 1508 arc_buf_remove_ref(arc_buf_t *buf, void* tag) 1509 { 1510 arc_buf_hdr_t *hdr = buf->b_hdr; 1511 kmutex_t *hash_lock = HDR_LOCK(hdr); 1512 int no_callback = (buf->b_efunc == NULL); 1513 1514 if (hdr->b_state == arc_anon) { 1515 ASSERT(hdr->b_datacnt == 1); 1516 arc_buf_free(buf, tag); 1517 return (no_callback); 1518 } 1519 1520 mutex_enter(hash_lock); 1521 ASSERT(hdr->b_state != arc_anon); 1522 ASSERT(buf->b_data != NULL); 1523 1524 (void) remove_reference(hdr, hash_lock, tag); 1525 if (hdr->b_datacnt > 1) { 1526 if (no_callback) 1527 arc_buf_destroy(buf, FALSE, TRUE); 1528 } else if (no_callback) { 1529 ASSERT(hdr->b_buf == buf && buf->b_next == NULL); 1530 ASSERT(buf->b_efunc == NULL); 1531 hdr->b_flags |= ARC_BUF_AVAILABLE; 1532 } 1533 ASSERT(no_callback || hdr->b_datacnt > 1 || 1534 refcount_is_zero(&hdr->b_refcnt)); 1535 mutex_exit(hash_lock); 1536 return (no_callback); 1537 } 1538 1539 int 1540 arc_buf_size(arc_buf_t *buf) 1541 { 1542 return (buf->b_hdr->b_size); 1543 } 1544 1545 /* 1546 * Evict buffers from list until we've removed the specified number of 1547 * bytes. Move the removed buffers to the appropriate evict state. 1548 * If the recycle flag is set, then attempt to "recycle" a buffer: 1549 * - look for a buffer to evict that is `bytes' long. 1550 * - return the data block from this buffer rather than freeing it. 1551 * This flag is used by callers that are trying to make space for a 1552 * new buffer in a full arc cache. 1553 * 1554 * This function makes a "best effort". It skips over any buffers 1555 * it can't get a hash_lock on, and so may not catch all candidates. 1556 * It may also return without evicting as much space as requested. 1557 */ 1558 static void * 1559 arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle, 1560 arc_buf_contents_t type) 1561 { 1562 arc_state_t *evicted_state; 1563 uint64_t bytes_evicted = 0, skipped = 0, missed = 0; 1564 arc_buf_hdr_t *ab, *ab_prev = NULL; 1565 list_t *list = &state->arcs_list[type]; 1566 kmutex_t *hash_lock; 1567 boolean_t have_lock; 1568 void *stolen = NULL; 1569 1570 ASSERT(state == arc_mru || state == arc_mfu); 1571 1572 evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 1573 1574 mutex_enter(&state->arcs_mtx); 1575 mutex_enter(&evicted_state->arcs_mtx); 1576 1577 for (ab = list_tail(list); ab; ab = ab_prev) { 1578 ab_prev = list_prev(list, ab); 1579 /* prefetch buffers have a minimum lifespan */ 1580 if (HDR_IO_IN_PROGRESS(ab) || 1581 (spa && ab->b_spa != spa) || 1582 (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) && 1583 ddi_get_lbolt() - ab->b_arc_access < 1584 arc_min_prefetch_lifespan)) { 1585 skipped++; 1586 continue; 1587 } 1588 /* "lookahead" for better eviction candidate */ 1589 if (recycle && ab->b_size != bytes && 1590 ab_prev && ab_prev->b_size == bytes) 1591 continue; 1592 hash_lock = HDR_LOCK(ab); 1593 have_lock = MUTEX_HELD(hash_lock); 1594 if (have_lock || mutex_tryenter(hash_lock)) { 1595 ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0); 1596 ASSERT(ab->b_datacnt > 0); 1597 while (ab->b_buf) { 1598 arc_buf_t *buf = ab->b_buf; 1599 if (!rw_tryenter(&buf->b_lock, RW_WRITER)) { 1600 missed += 1; 1601 break; 1602 } 1603 if (buf->b_data) { 1604 bytes_evicted += ab->b_size; 1605 if (recycle && ab->b_type == type && 1606 ab->b_size == bytes && 1607 !HDR_L2_WRITING(ab)) { 1608 stolen = buf->b_data; 1609 recycle = FALSE; 1610 } 1611 } 1612 if (buf->b_efunc) { 1613 mutex_enter(&arc_eviction_mtx); 1614 arc_buf_destroy(buf, 1615 buf->b_data == stolen, FALSE); 1616 ab->b_buf = buf->b_next; 1617 buf->b_hdr = &arc_eviction_hdr; 1618 buf->b_next = arc_eviction_list; 1619 arc_eviction_list = buf; 1620 mutex_exit(&arc_eviction_mtx); 1621 rw_exit(&buf->b_lock); 1622 } else { 1623 rw_exit(&buf->b_lock); 1624 arc_buf_destroy(buf, 1625 buf->b_data == stolen, TRUE); 1626 } 1627 } 1628 1629 if (ab->b_l2hdr) { 1630 ARCSTAT_INCR(arcstat_evict_l2_cached, 1631 ab->b_size); 1632 } else { 1633 if (l2arc_write_eligible(ab->b_spa, ab)) { 1634 ARCSTAT_INCR(arcstat_evict_l2_eligible, 1635 ab->b_size); 1636 } else { 1637 ARCSTAT_INCR( 1638 arcstat_evict_l2_ineligible, 1639 ab->b_size); 1640 } 1641 } 1642 1643 if (ab->b_datacnt == 0) { 1644 arc_change_state(evicted_state, ab, hash_lock); 1645 ASSERT(HDR_IN_HASH_TABLE(ab)); 1646 ab->b_flags |= ARC_IN_HASH_TABLE; 1647 ab->b_flags &= ~ARC_BUF_AVAILABLE; 1648 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab); 1649 } 1650 if (!have_lock) 1651 mutex_exit(hash_lock); 1652 if (bytes >= 0 && bytes_evicted >= bytes) 1653 break; 1654 } else { 1655 missed += 1; 1656 } 1657 } 1658 1659 mutex_exit(&evicted_state->arcs_mtx); 1660 mutex_exit(&state->arcs_mtx); 1661 1662 if (bytes_evicted < bytes) 1663 dprintf("only evicted %lld bytes from %x", 1664 (longlong_t)bytes_evicted, state); 1665 1666 if (skipped) 1667 ARCSTAT_INCR(arcstat_evict_skip, skipped); 1668 1669 if (missed) 1670 ARCSTAT_INCR(arcstat_mutex_miss, missed); 1671 1672 /* 1673 * We have just evicted some date into the ghost state, make 1674 * sure we also adjust the ghost state size if necessary. 1675 */ 1676 if (arc_no_grow && 1677 arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size > arc_c) { 1678 int64_t mru_over = arc_anon->arcs_size + arc_mru->arcs_size + 1679 arc_mru_ghost->arcs_size - arc_c; 1680 1681 if (mru_over > 0 && arc_mru_ghost->arcs_lsize[type] > 0) { 1682 int64_t todelete = 1683 MIN(arc_mru_ghost->arcs_lsize[type], mru_over); 1684 arc_evict_ghost(arc_mru_ghost, NULL, todelete); 1685 } else if (arc_mfu_ghost->arcs_lsize[type] > 0) { 1686 int64_t todelete = MIN(arc_mfu_ghost->arcs_lsize[type], 1687 arc_mru_ghost->arcs_size + 1688 arc_mfu_ghost->arcs_size - arc_c); 1689 arc_evict_ghost(arc_mfu_ghost, NULL, todelete); 1690 } 1691 } 1692 1693 return (stolen); 1694 } 1695 1696 /* 1697 * Remove buffers from list until we've removed the specified number of 1698 * bytes. Destroy the buffers that are removed. 1699 */ 1700 static void 1701 arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes) 1702 { 1703 arc_buf_hdr_t *ab, *ab_prev; 1704 list_t *list = &state->arcs_list[ARC_BUFC_DATA]; 1705 kmutex_t *hash_lock; 1706 uint64_t bytes_deleted = 0; 1707 uint64_t bufs_skipped = 0; 1708 1709 ASSERT(GHOST_STATE(state)); 1710 top: 1711 mutex_enter(&state->arcs_mtx); 1712 for (ab = list_tail(list); ab; ab = ab_prev) { 1713 ab_prev = list_prev(list, ab); 1714 if (spa && ab->b_spa != spa) 1715 continue; 1716 hash_lock = HDR_LOCK(ab); 1717 if (mutex_tryenter(hash_lock)) { 1718 ASSERT(!HDR_IO_IN_PROGRESS(ab)); 1719 ASSERT(ab->b_buf == NULL); 1720 ARCSTAT_BUMP(arcstat_deleted); 1721 bytes_deleted += ab->b_size; 1722 1723 if (ab->b_l2hdr != NULL) { 1724 /* 1725 * This buffer is cached on the 2nd Level ARC; 1726 * don't destroy the header. 1727 */ 1728 arc_change_state(arc_l2c_only, ab, hash_lock); 1729 mutex_exit(hash_lock); 1730 } else { 1731 arc_change_state(arc_anon, ab, hash_lock); 1732 mutex_exit(hash_lock); 1733 arc_hdr_destroy(ab); 1734 } 1735 1736 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab); 1737 if (bytes >= 0 && bytes_deleted >= bytes) 1738 break; 1739 } else { 1740 if (bytes < 0) { 1741 mutex_exit(&state->arcs_mtx); 1742 mutex_enter(hash_lock); 1743 mutex_exit(hash_lock); 1744 goto top; 1745 } 1746 bufs_skipped += 1; 1747 } 1748 } 1749 mutex_exit(&state->arcs_mtx); 1750 1751 if (list == &state->arcs_list[ARC_BUFC_DATA] && 1752 (bytes < 0 || bytes_deleted < bytes)) { 1753 list = &state->arcs_list[ARC_BUFC_METADATA]; 1754 goto top; 1755 } 1756 1757 if (bufs_skipped) { 1758 ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped); 1759 ASSERT(bytes >= 0); 1760 } 1761 1762 if (bytes_deleted < bytes) 1763 dprintf("only deleted %lld bytes from %p", 1764 (longlong_t)bytes_deleted, state); 1765 } 1766 1767 static void 1768 arc_adjust(void) 1769 { 1770 int64_t adjustment, delta; 1771 1772 /* 1773 * Adjust MRU size 1774 */ 1775 1776 adjustment = MIN(arc_size - arc_c, 1777 arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used - arc_p); 1778 1779 if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) { 1780 delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment); 1781 (void) arc_evict(arc_mru, NULL, delta, FALSE, ARC_BUFC_DATA); 1782 adjustment -= delta; 1783 } 1784 1785 if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) { 1786 delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment); 1787 (void) arc_evict(arc_mru, NULL, delta, FALSE, 1788 ARC_BUFC_METADATA); 1789 } 1790 1791 /* 1792 * Adjust MFU size 1793 */ 1794 1795 adjustment = arc_size - arc_c; 1796 1797 if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) { 1798 delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]); 1799 (void) arc_evict(arc_mfu, NULL, delta, FALSE, ARC_BUFC_DATA); 1800 adjustment -= delta; 1801 } 1802 1803 if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) { 1804 int64_t delta = MIN(adjustment, 1805 arc_mfu->arcs_lsize[ARC_BUFC_METADATA]); 1806 (void) arc_evict(arc_mfu, NULL, delta, FALSE, 1807 ARC_BUFC_METADATA); 1808 } 1809 1810 /* 1811 * Adjust ghost lists 1812 */ 1813 1814 adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c; 1815 1816 if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) { 1817 delta = MIN(arc_mru_ghost->arcs_size, adjustment); 1818 arc_evict_ghost(arc_mru_ghost, NULL, delta); 1819 } 1820 1821 adjustment = 1822 arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c; 1823 1824 if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) { 1825 delta = MIN(arc_mfu_ghost->arcs_size, adjustment); 1826 arc_evict_ghost(arc_mfu_ghost, NULL, delta); 1827 } 1828 } 1829 1830 static void 1831 arc_do_user_evicts(void) 1832 { 1833 mutex_enter(&arc_eviction_mtx); 1834 while (arc_eviction_list != NULL) { 1835 arc_buf_t *buf = arc_eviction_list; 1836 arc_eviction_list = buf->b_next; 1837 rw_enter(&buf->b_lock, RW_WRITER); 1838 buf->b_hdr = NULL; 1839 rw_exit(&buf->b_lock); 1840 mutex_exit(&arc_eviction_mtx); 1841 1842 if (buf->b_efunc != NULL) 1843 VERIFY(buf->b_efunc(buf) == 0); 1844 1845 buf->b_efunc = NULL; 1846 buf->b_private = NULL; 1847 kmem_cache_free(buf_cache, buf); 1848 mutex_enter(&arc_eviction_mtx); 1849 } 1850 mutex_exit(&arc_eviction_mtx); 1851 } 1852 1853 /* 1854 * Flush all *evictable* data from the cache for the given spa. 1855 * NOTE: this will not touch "active" (i.e. referenced) data. 1856 */ 1857 void 1858 arc_flush(spa_t *spa) 1859 { 1860 uint64_t guid = 0; 1861 1862 if (spa) 1863 guid = spa_guid(spa); 1864 1865 while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) { 1866 (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA); 1867 if (spa) 1868 break; 1869 } 1870 while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) { 1871 (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA); 1872 if (spa) 1873 break; 1874 } 1875 while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) { 1876 (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA); 1877 if (spa) 1878 break; 1879 } 1880 while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) { 1881 (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA); 1882 if (spa) 1883 break; 1884 } 1885 1886 arc_evict_ghost(arc_mru_ghost, guid, -1); 1887 arc_evict_ghost(arc_mfu_ghost, guid, -1); 1888 1889 mutex_enter(&arc_reclaim_thr_lock); 1890 arc_do_user_evicts(); 1891 mutex_exit(&arc_reclaim_thr_lock); 1892 ASSERT(spa || arc_eviction_list == NULL); 1893 } 1894 1895 void 1896 arc_shrink(void) 1897 { 1898 if (arc_c > arc_c_min) { 1899 uint64_t to_free; 1900 1901 #ifdef _KERNEL 1902 to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree)); 1903 #else 1904 to_free = arc_c >> arc_shrink_shift; 1905 #endif 1906 if (arc_c > arc_c_min + to_free) 1907 atomic_add_64(&arc_c, -to_free); 1908 else 1909 arc_c = arc_c_min; 1910 1911 atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift)); 1912 if (arc_c > arc_size) 1913 arc_c = MAX(arc_size, arc_c_min); 1914 if (arc_p > arc_c) 1915 arc_p = (arc_c >> 1); 1916 ASSERT(arc_c >= arc_c_min); 1917 ASSERT((int64_t)arc_p >= 0); 1918 } 1919 1920 if (arc_size > arc_c) 1921 arc_adjust(); 1922 } 1923 1924 static int 1925 arc_reclaim_needed(void) 1926 { 1927 uint64_t extra; 1928 1929 #ifdef _KERNEL 1930 1931 if (needfree) 1932 return (1); 1933 1934 /* 1935 * take 'desfree' extra pages, so we reclaim sooner, rather than later 1936 */ 1937 extra = desfree; 1938 1939 /* 1940 * check that we're out of range of the pageout scanner. It starts to 1941 * schedule paging if freemem is less than lotsfree and needfree. 1942 * lotsfree is the high-water mark for pageout, and needfree is the 1943 * number of needed free pages. We add extra pages here to make sure 1944 * the scanner doesn't start up while we're freeing memory. 1945 */ 1946 if (freemem < lotsfree + needfree + extra) 1947 return (1); 1948 1949 /* 1950 * check to make sure that swapfs has enough space so that anon 1951 * reservations can still succeed. anon_resvmem() checks that the 1952 * availrmem is greater than swapfs_minfree, and the number of reserved 1953 * swap pages. We also add a bit of extra here just to prevent 1954 * circumstances from getting really dire. 1955 */ 1956 if (availrmem < swapfs_minfree + swapfs_reserve + extra) 1957 return (1); 1958 1959 #if defined(__i386) 1960 /* 1961 * If we're on an i386 platform, it's possible that we'll exhaust the 1962 * kernel heap space before we ever run out of available physical 1963 * memory. Most checks of the size of the heap_area compare against 1964 * tune.t_minarmem, which is the minimum available real memory that we 1965 * can have in the system. However, this is generally fixed at 25 pages 1966 * which is so low that it's useless. In this comparison, we seek to 1967 * calculate the total heap-size, and reclaim if more than 3/4ths of the 1968 * heap is allocated. (Or, in the calculation, if less than 1/4th is 1969 * free) 1970 */ 1971 if (btop(vmem_size(heap_arena, VMEM_FREE)) < 1972 (btop(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2)) 1973 return (1); 1974 #endif 1975 1976 #else 1977 if (spa_get_random(100) == 0) 1978 return (1); 1979 #endif 1980 return (0); 1981 } 1982 1983 static void 1984 arc_kmem_reap_now(arc_reclaim_strategy_t strat) 1985 { 1986 size_t i; 1987 kmem_cache_t *prev_cache = NULL; 1988 kmem_cache_t *prev_data_cache = NULL; 1989 extern kmem_cache_t *zio_buf_cache[]; 1990 extern kmem_cache_t *zio_data_buf_cache[]; 1991 1992 #ifdef _KERNEL 1993 if (arc_meta_used >= arc_meta_limit) { 1994 /* 1995 * We are exceeding our meta-data cache limit. 1996 * Purge some DNLC entries to release holds on meta-data. 1997 */ 1998 dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); 1999 } 2000 #if defined(__i386) 2001 /* 2002 * Reclaim unused memory from all kmem caches. 2003 */ 2004 kmem_reap(); 2005 #endif 2006 #endif 2007 2008 /* 2009 * An aggressive reclamation will shrink the cache size as well as 2010 * reap free buffers from the arc kmem caches. 2011 */ 2012 if (strat == ARC_RECLAIM_AGGR) 2013 arc_shrink(); 2014 2015 for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { 2016 if (zio_buf_cache[i] != prev_cache) { 2017 prev_cache = zio_buf_cache[i]; 2018 kmem_cache_reap_now(zio_buf_cache[i]); 2019 } 2020 if (zio_data_buf_cache[i] != prev_data_cache) { 2021 prev_data_cache = zio_data_buf_cache[i]; 2022 kmem_cache_reap_now(zio_data_buf_cache[i]); 2023 } 2024 } 2025 kmem_cache_reap_now(buf_cache); 2026 kmem_cache_reap_now(hdr_cache); 2027 } 2028 2029 static void 2030 arc_reclaim_thread(void) 2031 { 2032 clock_t growtime = 0; 2033 arc_reclaim_strategy_t last_reclaim = ARC_RECLAIM_CONS; 2034 callb_cpr_t cpr; 2035 2036 CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG); 2037 2038 mutex_enter(&arc_reclaim_thr_lock); 2039 while (arc_thread_exit == 0) { 2040 if (arc_reclaim_needed()) { 2041 2042 if (arc_no_grow) { 2043 if (last_reclaim == ARC_RECLAIM_CONS) { 2044 last_reclaim = ARC_RECLAIM_AGGR; 2045 } else { 2046 last_reclaim = ARC_RECLAIM_CONS; 2047 } 2048 } else { 2049 arc_no_grow = TRUE; 2050 last_reclaim = ARC_RECLAIM_AGGR; 2051 membar_producer(); 2052 } 2053 2054 /* reset the growth delay for every reclaim */ 2055 growtime = ddi_get_lbolt() + (arc_grow_retry * hz); 2056 2057 arc_kmem_reap_now(last_reclaim); 2058 arc_warm = B_TRUE; 2059 2060 } else if (arc_no_grow && ddi_get_lbolt() >= growtime) { 2061 arc_no_grow = FALSE; 2062 } 2063 2064 if (2 * arc_c < arc_size + 2065 arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size) 2066 arc_adjust(); 2067 2068 if (arc_eviction_list != NULL) 2069 arc_do_user_evicts(); 2070 2071 /* block until needed, or one second, whichever is shorter */ 2072 CALLB_CPR_SAFE_BEGIN(&cpr); 2073 (void) cv_timedwait(&arc_reclaim_thr_cv, 2074 &arc_reclaim_thr_lock, (ddi_get_lbolt() + hz)); 2075 CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock); 2076 } 2077 2078 arc_thread_exit = 0; 2079 cv_broadcast(&arc_reclaim_thr_cv); 2080 CALLB_CPR_EXIT(&cpr); /* drops arc_reclaim_thr_lock */ 2081 thread_exit(); 2082 } 2083 2084 /* 2085 * Adapt arc info given the number of bytes we are trying to add and 2086 * the state that we are comming from. This function is only called 2087 * when we are adding new content to the cache. 2088 */ 2089 static void 2090 arc_adapt(int bytes, arc_state_t *state) 2091 { 2092 int mult; 2093 uint64_t arc_p_min = (arc_c >> arc_p_min_shift); 2094 2095 if (state == arc_l2c_only) 2096 return; 2097 2098 ASSERT(bytes > 0); 2099 /* 2100 * Adapt the target size of the MRU list: 2101 * - if we just hit in the MRU ghost list, then increase 2102 * the target size of the MRU list. 2103 * - if we just hit in the MFU ghost list, then increase 2104 * the target size of the MFU list by decreasing the 2105 * target size of the MRU list. 2106 */ 2107 if (state == arc_mru_ghost) { 2108 mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ? 2109 1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size)); 2110 2111 arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult); 2112 } else if (state == arc_mfu_ghost) { 2113 uint64_t delta; 2114 2115 mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ? 2116 1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size)); 2117 2118 delta = MIN(bytes * mult, arc_p); 2119 arc_p = MAX(arc_p_min, arc_p - delta); 2120 } 2121 ASSERT((int64_t)arc_p >= 0); 2122 2123 if (arc_reclaim_needed()) { 2124 cv_signal(&arc_reclaim_thr_cv); 2125 return; 2126 } 2127 2128 if (arc_no_grow) 2129 return; 2130 2131 if (arc_c >= arc_c_max) 2132 return; 2133 2134 /* 2135 * If we're within (2 * maxblocksize) bytes of the target 2136 * cache size, increment the target cache size 2137 */ 2138 if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) { 2139 atomic_add_64(&arc_c, (int64_t)bytes); 2140 if (arc_c > arc_c_max) 2141 arc_c = arc_c_max; 2142 else if (state == arc_anon) 2143 atomic_add_64(&arc_p, (int64_t)bytes); 2144 if (arc_p > arc_c) 2145 arc_p = arc_c; 2146 } 2147 ASSERT((int64_t)arc_p >= 0); 2148 } 2149 2150 /* 2151 * Check if the cache has reached its limits and eviction is required 2152 * prior to insert. 2153 */ 2154 static int 2155 arc_evict_needed(arc_buf_contents_t type) 2156 { 2157 if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit) 2158 return (1); 2159 2160 #ifdef _KERNEL 2161 /* 2162 * If zio data pages are being allocated out of a separate heap segment, 2163 * then enforce that the size of available vmem for this area remains 2164 * above about 1/32nd free. 2165 */ 2166 if (type == ARC_BUFC_DATA && zio_arena != NULL && 2167 vmem_size(zio_arena, VMEM_FREE) < 2168 (vmem_size(zio_arena, VMEM_ALLOC) >> 5)) 2169 return (1); 2170 #endif 2171 2172 if (arc_reclaim_needed()) 2173 return (1); 2174 2175 return (arc_size > arc_c); 2176 } 2177 2178 /* 2179 * The buffer, supplied as the first argument, needs a data block. 2180 * So, if we are at cache max, determine which cache should be victimized. 2181 * We have the following cases: 2182 * 2183 * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) -> 2184 * In this situation if we're out of space, but the resident size of the MFU is 2185 * under the limit, victimize the MFU cache to satisfy this insertion request. 2186 * 2187 * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) -> 2188 * Here, we've used up all of the available space for the MRU, so we need to 2189 * evict from our own cache instead. Evict from the set of resident MRU 2190 * entries. 2191 * 2192 * 3. Insert for MFU (c - p) > sizeof(arc_mfu) -> 2193 * c minus p represents the MFU space in the cache, since p is the size of the 2194 * cache that is dedicated to the MRU. In this situation there's still space on 2195 * the MFU side, so the MRU side needs to be victimized. 2196 * 2197 * 4. Insert for MFU (c - p) < sizeof(arc_mfu) -> 2198 * MFU's resident set is consuming more space than it has been allotted. In 2199 * this situation, we must victimize our own cache, the MFU, for this insertion. 2200 */ 2201 static void 2202 arc_get_data_buf(arc_buf_t *buf) 2203 { 2204 arc_state_t *state = buf->b_hdr->b_state; 2205 uint64_t size = buf->b_hdr->b_size; 2206 arc_buf_contents_t type = buf->b_hdr->b_type; 2207 2208 arc_adapt(size, state); 2209 2210 /* 2211 * We have not yet reached cache maximum size, 2212 * just allocate a new buffer. 2213 */ 2214 if (!arc_evict_needed(type)) { 2215 if (type == ARC_BUFC_METADATA) { 2216 buf->b_data = zio_buf_alloc(size); 2217 arc_space_consume(size, ARC_SPACE_DATA); 2218 } else { 2219 ASSERT(type == ARC_BUFC_DATA); 2220 buf->b_data = zio_data_buf_alloc(size); 2221 ARCSTAT_INCR(arcstat_data_size, size); 2222 atomic_add_64(&arc_size, size); 2223 } 2224 goto out; 2225 } 2226 2227 /* 2228 * If we are prefetching from the mfu ghost list, this buffer 2229 * will end up on the mru list; so steal space from there. 2230 */ 2231 if (state == arc_mfu_ghost) 2232 state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu; 2233 else if (state == arc_mru_ghost) 2234 state = arc_mru; 2235 2236 if (state == arc_mru || state == arc_anon) { 2237 uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size; 2238 state = (arc_mfu->arcs_lsize[type] >= size && 2239 arc_p > mru_used) ? arc_mfu : arc_mru; 2240 } else { 2241 /* MFU cases */ 2242 uint64_t mfu_space = arc_c - arc_p; 2243 state = (arc_mru->arcs_lsize[type] >= size && 2244 mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu; 2245 } 2246 if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) { 2247 if (type == ARC_BUFC_METADATA) { 2248 buf->b_data = zio_buf_alloc(size); 2249 arc_space_consume(size, ARC_SPACE_DATA); 2250 } else { 2251 ASSERT(type == ARC_BUFC_DATA); 2252 buf->b_data = zio_data_buf_alloc(size); 2253 ARCSTAT_INCR(arcstat_data_size, size); 2254 atomic_add_64(&arc_size, size); 2255 } 2256 ARCSTAT_BUMP(arcstat_recycle_miss); 2257 } 2258 ASSERT(buf->b_data != NULL); 2259 out: 2260 /* 2261 * Update the state size. Note that ghost states have a 2262 * "ghost size" and so don't need to be updated. 2263 */ 2264 if (!GHOST_STATE(buf->b_hdr->b_state)) { 2265 arc_buf_hdr_t *hdr = buf->b_hdr; 2266 2267 atomic_add_64(&hdr->b_state->arcs_size, size); 2268 if (list_link_active(&hdr->b_arc_node)) { 2269 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 2270 atomic_add_64(&hdr->b_state->arcs_lsize[type], size); 2271 } 2272 /* 2273 * If we are growing the cache, and we are adding anonymous 2274 * data, and we have outgrown arc_p, update arc_p 2275 */ 2276 if (arc_size < arc_c && hdr->b_state == arc_anon && 2277 arc_anon->arcs_size + arc_mru->arcs_size > arc_p) 2278 arc_p = MIN(arc_c, arc_p + size); 2279 } 2280 } 2281 2282 /* 2283 * This routine is called whenever a buffer is accessed. 2284 * NOTE: the hash lock is dropped in this function. 2285 */ 2286 static void 2287 arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock) 2288 { 2289 clock_t now; 2290 2291 ASSERT(MUTEX_HELD(hash_lock)); 2292 2293 if (buf->b_state == arc_anon) { 2294 /* 2295 * This buffer is not in the cache, and does not 2296 * appear in our "ghost" list. Add the new buffer 2297 * to the MRU state. 2298 */ 2299 2300 ASSERT(buf->b_arc_access == 0); 2301 buf->b_arc_access = ddi_get_lbolt(); 2302 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 2303 arc_change_state(arc_mru, buf, hash_lock); 2304 2305 } else if (buf->b_state == arc_mru) { 2306 now = ddi_get_lbolt(); 2307 2308 /* 2309 * If this buffer is here because of a prefetch, then either: 2310 * - clear the flag if this is a "referencing" read 2311 * (any subsequent access will bump this into the MFU state). 2312 * or 2313 * - move the buffer to the head of the list if this is 2314 * another prefetch (to make it less likely to be evicted). 2315 */ 2316 if ((buf->b_flags & ARC_PREFETCH) != 0) { 2317 if (refcount_count(&buf->b_refcnt) == 0) { 2318 ASSERT(list_link_active(&buf->b_arc_node)); 2319 } else { 2320 buf->b_flags &= ~ARC_PREFETCH; 2321 ARCSTAT_BUMP(arcstat_mru_hits); 2322 } 2323 buf->b_arc_access = now; 2324 return; 2325 } 2326 2327 /* 2328 * This buffer has been "accessed" only once so far, 2329 * but it is still in the cache. Move it to the MFU 2330 * state. 2331 */ 2332 if (now > buf->b_arc_access + ARC_MINTIME) { 2333 /* 2334 * More than 125ms have passed since we 2335 * instantiated this buffer. Move it to the 2336 * most frequently used state. 2337 */ 2338 buf->b_arc_access = now; 2339 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2340 arc_change_state(arc_mfu, buf, hash_lock); 2341 } 2342 ARCSTAT_BUMP(arcstat_mru_hits); 2343 } else if (buf->b_state == arc_mru_ghost) { 2344 arc_state_t *new_state; 2345 /* 2346 * This buffer has been "accessed" recently, but 2347 * was evicted from the cache. Move it to the 2348 * MFU state. 2349 */ 2350 2351 if (buf->b_flags & ARC_PREFETCH) { 2352 new_state = arc_mru; 2353 if (refcount_count(&buf->b_refcnt) > 0) 2354 buf->b_flags &= ~ARC_PREFETCH; 2355 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 2356 } else { 2357 new_state = arc_mfu; 2358 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2359 } 2360 2361 buf->b_arc_access = ddi_get_lbolt(); 2362 arc_change_state(new_state, buf, hash_lock); 2363 2364 ARCSTAT_BUMP(arcstat_mru_ghost_hits); 2365 } else if (buf->b_state == arc_mfu) { 2366 /* 2367 * This buffer has been accessed more than once and is 2368 * still in the cache. Keep it in the MFU state. 2369 * 2370 * NOTE: an add_reference() that occurred when we did 2371 * the arc_read() will have kicked this off the list. 2372 * If it was a prefetch, we will explicitly move it to 2373 * the head of the list now. 2374 */ 2375 if ((buf->b_flags & ARC_PREFETCH) != 0) { 2376 ASSERT(refcount_count(&buf->b_refcnt) == 0); 2377 ASSERT(list_link_active(&buf->b_arc_node)); 2378 } 2379 ARCSTAT_BUMP(arcstat_mfu_hits); 2380 buf->b_arc_access = ddi_get_lbolt(); 2381 } else if (buf->b_state == arc_mfu_ghost) { 2382 arc_state_t *new_state = arc_mfu; 2383 /* 2384 * This buffer has been accessed more than once but has 2385 * been evicted from the cache. Move it back to the 2386 * MFU state. 2387 */ 2388 2389 if (buf->b_flags & ARC_PREFETCH) { 2390 /* 2391 * This is a prefetch access... 2392 * move this block back to the MRU state. 2393 */ 2394 ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0); 2395 new_state = arc_mru; 2396 } 2397 2398 buf->b_arc_access = ddi_get_lbolt(); 2399 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2400 arc_change_state(new_state, buf, hash_lock); 2401 2402 ARCSTAT_BUMP(arcstat_mfu_ghost_hits); 2403 } else if (buf->b_state == arc_l2c_only) { 2404 /* 2405 * This buffer is on the 2nd Level ARC. 2406 */ 2407 2408 buf->b_arc_access = ddi_get_lbolt(); 2409 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2410 arc_change_state(arc_mfu, buf, hash_lock); 2411 } else { 2412 ASSERT(!"invalid arc state"); 2413 } 2414 } 2415 2416 /* a generic arc_done_func_t which you can use */ 2417 /* ARGSUSED */ 2418 void 2419 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) 2420 { 2421 bcopy(buf->b_data, arg, buf->b_hdr->b_size); 2422 VERIFY(arc_buf_remove_ref(buf, arg) == 1); 2423 } 2424 2425 /* a generic arc_done_func_t */ 2426 void 2427 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) 2428 { 2429 arc_buf_t **bufp = arg; 2430 if (zio && zio->io_error) { 2431 VERIFY(arc_buf_remove_ref(buf, arg) == 1); 2432 *bufp = NULL; 2433 } else { 2434 *bufp = buf; 2435 } 2436 } 2437 2438 static void 2439 arc_read_done(zio_t *zio) 2440 { 2441 arc_buf_hdr_t *hdr, *found; 2442 arc_buf_t *buf; 2443 arc_buf_t *abuf; /* buffer we're assigning to callback */ 2444 kmutex_t *hash_lock; 2445 arc_callback_t *callback_list, *acb; 2446 int freeable = FALSE; 2447 2448 buf = zio->io_private; 2449 hdr = buf->b_hdr; 2450 2451 /* 2452 * The hdr was inserted into hash-table and removed from lists 2453 * prior to starting I/O. We should find this header, since 2454 * it's in the hash table, and it should be legit since it's 2455 * not possible to evict it during the I/O. The only possible 2456 * reason for it not to be found is if we were freed during the 2457 * read. 2458 */ 2459 found = buf_hash_find(hdr->b_spa, &hdr->b_dva, hdr->b_birth, 2460 &hash_lock); 2461 2462 ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) || 2463 (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || 2464 (found == hdr && HDR_L2_READING(hdr))); 2465 2466 hdr->b_flags &= ~ARC_L2_EVICTED; 2467 if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH)) 2468 hdr->b_flags &= ~ARC_L2CACHE; 2469 2470 /* byteswap if necessary */ 2471 callback_list = hdr->b_acb; 2472 ASSERT(callback_list != NULL); 2473 if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) { 2474 arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ? 2475 byteswap_uint64_array : 2476 dmu_ot[BP_GET_TYPE(zio->io_bp)].ot_byteswap; 2477 func(buf->b_data, hdr->b_size); 2478 } 2479 2480 arc_cksum_compute(buf, B_FALSE); 2481 2482 if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) { 2483 /* 2484 * Only call arc_access on anonymous buffers. This is because 2485 * if we've issued an I/O for an evicted buffer, we've already 2486 * called arc_access (to prevent any simultaneous readers from 2487 * getting confused). 2488 */ 2489 arc_access(hdr, hash_lock); 2490 } 2491 2492 /* create copies of the data buffer for the callers */ 2493 abuf = buf; 2494 for (acb = callback_list; acb; acb = acb->acb_next) { 2495 if (acb->acb_done) { 2496 if (abuf == NULL) 2497 abuf = arc_buf_clone(buf); 2498 acb->acb_buf = abuf; 2499 abuf = NULL; 2500 } 2501 } 2502 hdr->b_acb = NULL; 2503 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2504 ASSERT(!HDR_BUF_AVAILABLE(hdr)); 2505 if (abuf == buf) { 2506 ASSERT(buf->b_efunc == NULL); 2507 ASSERT(hdr->b_datacnt == 1); 2508 hdr->b_flags |= ARC_BUF_AVAILABLE; 2509 } 2510 2511 ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL); 2512 2513 if (zio->io_error != 0) { 2514 hdr->b_flags |= ARC_IO_ERROR; 2515 if (hdr->b_state != arc_anon) 2516 arc_change_state(arc_anon, hdr, hash_lock); 2517 if (HDR_IN_HASH_TABLE(hdr)) 2518 buf_hash_remove(hdr); 2519 freeable = refcount_is_zero(&hdr->b_refcnt); 2520 } 2521 2522 /* 2523 * Broadcast before we drop the hash_lock to avoid the possibility 2524 * that the hdr (and hence the cv) might be freed before we get to 2525 * the cv_broadcast(). 2526 */ 2527 cv_broadcast(&hdr->b_cv); 2528 2529 if (hash_lock) { 2530 mutex_exit(hash_lock); 2531 } else { 2532 /* 2533 * This block was freed while we waited for the read to 2534 * complete. It has been removed from the hash table and 2535 * moved to the anonymous state (so that it won't show up 2536 * in the cache). 2537 */ 2538 ASSERT3P(hdr->b_state, ==, arc_anon); 2539 freeable = refcount_is_zero(&hdr->b_refcnt); 2540 } 2541 2542 /* execute each callback and free its structure */ 2543 while ((acb = callback_list) != NULL) { 2544 if (acb->acb_done) 2545 acb->acb_done(zio, acb->acb_buf, acb->acb_private); 2546 2547 if (acb->acb_zio_dummy != NULL) { 2548 acb->acb_zio_dummy->io_error = zio->io_error; 2549 zio_nowait(acb->acb_zio_dummy); 2550 } 2551 2552 callback_list = acb->acb_next; 2553 kmem_free(acb, sizeof (arc_callback_t)); 2554 } 2555 2556 if (freeable) 2557 arc_hdr_destroy(hdr); 2558 } 2559 2560 /* 2561 * "Read" the block block at the specified DVA (in bp) via the 2562 * cache. If the block is found in the cache, invoke the provided 2563 * callback immediately and return. Note that the `zio' parameter 2564 * in the callback will be NULL in this case, since no IO was 2565 * required. If the block is not in the cache pass the read request 2566 * on to the spa with a substitute callback function, so that the 2567 * requested block will be added to the cache. 2568 * 2569 * If a read request arrives for a block that has a read in-progress, 2570 * either wait for the in-progress read to complete (and return the 2571 * results); or, if this is a read with a "done" func, add a record 2572 * to the read to invoke the "done" func when the read completes, 2573 * and return; or just return. 2574 * 2575 * arc_read_done() will invoke all the requested "done" functions 2576 * for readers of this block. 2577 * 2578 * Normal callers should use arc_read and pass the arc buffer and offset 2579 * for the bp. But if you know you don't need locking, you can use 2580 * arc_read_bp. 2581 */ 2582 int 2583 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_buf_t *pbuf, 2584 arc_done_func_t *done, void *private, int priority, int zio_flags, 2585 uint32_t *arc_flags, const zbookmark_t *zb) 2586 { 2587 int err; 2588 2589 ASSERT(!refcount_is_zero(&pbuf->b_hdr->b_refcnt)); 2590 ASSERT3U((char *)bp - (char *)pbuf->b_data, <, pbuf->b_hdr->b_size); 2591 rw_enter(&pbuf->b_lock, RW_READER); 2592 2593 err = arc_read_nolock(pio, spa, bp, done, private, priority, 2594 zio_flags, arc_flags, zb); 2595 rw_exit(&pbuf->b_lock); 2596 2597 return (err); 2598 } 2599 2600 int 2601 arc_read_nolock(zio_t *pio, spa_t *spa, const blkptr_t *bp, 2602 arc_done_func_t *done, void *private, int priority, int zio_flags, 2603 uint32_t *arc_flags, const zbookmark_t *zb) 2604 { 2605 arc_buf_hdr_t *hdr; 2606 arc_buf_t *buf; 2607 kmutex_t *hash_lock; 2608 zio_t *rzio; 2609 uint64_t guid = spa_guid(spa); 2610 2611 top: 2612 hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp), 2613 &hash_lock); 2614 if (hdr && hdr->b_datacnt > 0) { 2615 2616 *arc_flags |= ARC_CACHED; 2617 2618 if (HDR_IO_IN_PROGRESS(hdr)) { 2619 2620 if (*arc_flags & ARC_WAIT) { 2621 cv_wait(&hdr->b_cv, hash_lock); 2622 mutex_exit(hash_lock); 2623 goto top; 2624 } 2625 ASSERT(*arc_flags & ARC_NOWAIT); 2626 2627 if (done) { 2628 arc_callback_t *acb = NULL; 2629 2630 acb = kmem_zalloc(sizeof (arc_callback_t), 2631 KM_SLEEP); 2632 acb->acb_done = done; 2633 acb->acb_private = private; 2634 if (pio != NULL) 2635 acb->acb_zio_dummy = zio_null(pio, 2636 spa, NULL, NULL, NULL, zio_flags); 2637 2638 ASSERT(acb->acb_done != NULL); 2639 acb->acb_next = hdr->b_acb; 2640 hdr->b_acb = acb; 2641 add_reference(hdr, hash_lock, private); 2642 mutex_exit(hash_lock); 2643 return (0); 2644 } 2645 mutex_exit(hash_lock); 2646 return (0); 2647 } 2648 2649 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2650 2651 if (done) { 2652 add_reference(hdr, hash_lock, private); 2653 /* 2654 * If this block is already in use, create a new 2655 * copy of the data so that we will be guaranteed 2656 * that arc_release() will always succeed. 2657 */ 2658 buf = hdr->b_buf; 2659 ASSERT(buf); 2660 ASSERT(buf->b_data); 2661 if (HDR_BUF_AVAILABLE(hdr)) { 2662 ASSERT(buf->b_efunc == NULL); 2663 hdr->b_flags &= ~ARC_BUF_AVAILABLE; 2664 } else { 2665 buf = arc_buf_clone(buf); 2666 } 2667 2668 } else if (*arc_flags & ARC_PREFETCH && 2669 refcount_count(&hdr->b_refcnt) == 0) { 2670 hdr->b_flags |= ARC_PREFETCH; 2671 } 2672 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 2673 arc_access(hdr, hash_lock); 2674 if (*arc_flags & ARC_L2CACHE) 2675 hdr->b_flags |= ARC_L2CACHE; 2676 mutex_exit(hash_lock); 2677 ARCSTAT_BUMP(arcstat_hits); 2678 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 2679 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 2680 data, metadata, hits); 2681 2682 if (done) 2683 done(NULL, buf, private); 2684 } else { 2685 uint64_t size = BP_GET_LSIZE(bp); 2686 arc_callback_t *acb; 2687 vdev_t *vd = NULL; 2688 uint64_t addr; 2689 boolean_t devw = B_FALSE; 2690 2691 if (hdr == NULL) { 2692 /* this block is not in the cache */ 2693 arc_buf_hdr_t *exists; 2694 arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp); 2695 buf = arc_buf_alloc(spa, size, private, type); 2696 hdr = buf->b_hdr; 2697 hdr->b_dva = *BP_IDENTITY(bp); 2698 hdr->b_birth = BP_PHYSICAL_BIRTH(bp); 2699 hdr->b_cksum0 = bp->blk_cksum.zc_word[0]; 2700 exists = buf_hash_insert(hdr, &hash_lock); 2701 if (exists) { 2702 /* somebody beat us to the hash insert */ 2703 mutex_exit(hash_lock); 2704 bzero(&hdr->b_dva, sizeof (dva_t)); 2705 hdr->b_birth = 0; 2706 hdr->b_cksum0 = 0; 2707 (void) arc_buf_remove_ref(buf, private); 2708 goto top; /* restart the IO request */ 2709 } 2710 /* if this is a prefetch, we don't have a reference */ 2711 if (*arc_flags & ARC_PREFETCH) { 2712 (void) remove_reference(hdr, hash_lock, 2713 private); 2714 hdr->b_flags |= ARC_PREFETCH; 2715 } 2716 if (*arc_flags & ARC_L2CACHE) 2717 hdr->b_flags |= ARC_L2CACHE; 2718 if (BP_GET_LEVEL(bp) > 0) 2719 hdr->b_flags |= ARC_INDIRECT; 2720 } else { 2721 /* this block is in the ghost cache */ 2722 ASSERT(GHOST_STATE(hdr->b_state)); 2723 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 2724 ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0); 2725 ASSERT(hdr->b_buf == NULL); 2726 2727 /* if this is a prefetch, we don't have a reference */ 2728 if (*arc_flags & ARC_PREFETCH) 2729 hdr->b_flags |= ARC_PREFETCH; 2730 else 2731 add_reference(hdr, hash_lock, private); 2732 if (*arc_flags & ARC_L2CACHE) 2733 hdr->b_flags |= ARC_L2CACHE; 2734 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 2735 buf->b_hdr = hdr; 2736 buf->b_data = NULL; 2737 buf->b_efunc = NULL; 2738 buf->b_private = NULL; 2739 buf->b_next = NULL; 2740 hdr->b_buf = buf; 2741 arc_get_data_buf(buf); 2742 ASSERT(hdr->b_datacnt == 0); 2743 hdr->b_datacnt = 1; 2744 } 2745 2746 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 2747 acb->acb_done = done; 2748 acb->acb_private = private; 2749 2750 ASSERT(hdr->b_acb == NULL); 2751 hdr->b_acb = acb; 2752 hdr->b_flags |= ARC_IO_IN_PROGRESS; 2753 2754 /* 2755 * If the buffer has been evicted, migrate it to a present state 2756 * before issuing the I/O. Once we drop the hash-table lock, 2757 * the header will be marked as I/O in progress and have an 2758 * attached buffer. At this point, anybody who finds this 2759 * buffer ought to notice that it's legit but has a pending I/O. 2760 */ 2761 2762 if (GHOST_STATE(hdr->b_state)) 2763 arc_access(hdr, hash_lock); 2764 2765 if (HDR_L2CACHE(hdr) && hdr->b_l2hdr != NULL && 2766 (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) { 2767 devw = hdr->b_l2hdr->b_dev->l2ad_writing; 2768 addr = hdr->b_l2hdr->b_daddr; 2769 /* 2770 * Lock out device removal. 2771 */ 2772 if (vdev_is_dead(vd) || 2773 !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER)) 2774 vd = NULL; 2775 } 2776 2777 mutex_exit(hash_lock); 2778 2779 ASSERT3U(hdr->b_size, ==, size); 2780 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp, 2781 uint64_t, size, zbookmark_t *, zb); 2782 ARCSTAT_BUMP(arcstat_misses); 2783 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 2784 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 2785 data, metadata, misses); 2786 2787 if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) { 2788 /* 2789 * Read from the L2ARC if the following are true: 2790 * 1. The L2ARC vdev was previously cached. 2791 * 2. This buffer still has L2ARC metadata. 2792 * 3. This buffer isn't currently writing to the L2ARC. 2793 * 4. The L2ARC entry wasn't evicted, which may 2794 * also have invalidated the vdev. 2795 * 5. This isn't prefetch and l2arc_noprefetch is set. 2796 */ 2797 if (hdr->b_l2hdr != NULL && 2798 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) && 2799 !(l2arc_noprefetch && HDR_PREFETCH(hdr))) { 2800 l2arc_read_callback_t *cb; 2801 2802 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr); 2803 ARCSTAT_BUMP(arcstat_l2_hits); 2804 2805 cb = kmem_zalloc(sizeof (l2arc_read_callback_t), 2806 KM_SLEEP); 2807 cb->l2rcb_buf = buf; 2808 cb->l2rcb_spa = spa; 2809 cb->l2rcb_bp = *bp; 2810 cb->l2rcb_zb = *zb; 2811 cb->l2rcb_flags = zio_flags; 2812 2813 /* 2814 * l2arc read. The SCL_L2ARC lock will be 2815 * released by l2arc_read_done(). 2816 */ 2817 rzio = zio_read_phys(pio, vd, addr, size, 2818 buf->b_data, ZIO_CHECKSUM_OFF, 2819 l2arc_read_done, cb, priority, zio_flags | 2820 ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL | 2821 ZIO_FLAG_DONT_PROPAGATE | 2822 ZIO_FLAG_DONT_RETRY, B_FALSE); 2823 DTRACE_PROBE2(l2arc__read, vdev_t *, vd, 2824 zio_t *, rzio); 2825 ARCSTAT_INCR(arcstat_l2_read_bytes, size); 2826 2827 if (*arc_flags & ARC_NOWAIT) { 2828 zio_nowait(rzio); 2829 return (0); 2830 } 2831 2832 ASSERT(*arc_flags & ARC_WAIT); 2833 if (zio_wait(rzio) == 0) 2834 return (0); 2835 2836 /* l2arc read error; goto zio_read() */ 2837 } else { 2838 DTRACE_PROBE1(l2arc__miss, 2839 arc_buf_hdr_t *, hdr); 2840 ARCSTAT_BUMP(arcstat_l2_misses); 2841 if (HDR_L2_WRITING(hdr)) 2842 ARCSTAT_BUMP(arcstat_l2_rw_clash); 2843 spa_config_exit(spa, SCL_L2ARC, vd); 2844 } 2845 } else { 2846 if (vd != NULL) 2847 spa_config_exit(spa, SCL_L2ARC, vd); 2848 if (l2arc_ndev != 0) { 2849 DTRACE_PROBE1(l2arc__miss, 2850 arc_buf_hdr_t *, hdr); 2851 ARCSTAT_BUMP(arcstat_l2_misses); 2852 } 2853 } 2854 2855 rzio = zio_read(pio, spa, bp, buf->b_data, size, 2856 arc_read_done, buf, priority, zio_flags, zb); 2857 2858 if (*arc_flags & ARC_WAIT) 2859 return (zio_wait(rzio)); 2860 2861 ASSERT(*arc_flags & ARC_NOWAIT); 2862 zio_nowait(rzio); 2863 } 2864 return (0); 2865 } 2866 2867 void 2868 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private) 2869 { 2870 ASSERT(buf->b_hdr != NULL); 2871 ASSERT(buf->b_hdr->b_state != arc_anon); 2872 ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL); 2873 ASSERT(buf->b_efunc == NULL); 2874 ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr)); 2875 2876 buf->b_efunc = func; 2877 buf->b_private = private; 2878 } 2879 2880 /* 2881 * This is used by the DMU to let the ARC know that a buffer is 2882 * being evicted, so the ARC should clean up. If this arc buf 2883 * is not yet in the evicted state, it will be put there. 2884 */ 2885 int 2886 arc_buf_evict(arc_buf_t *buf) 2887 { 2888 arc_buf_hdr_t *hdr; 2889 kmutex_t *hash_lock; 2890 arc_buf_t **bufp; 2891 2892 rw_enter(&buf->b_lock, RW_WRITER); 2893 hdr = buf->b_hdr; 2894 if (hdr == NULL) { 2895 /* 2896 * We are in arc_do_user_evicts(). 2897 */ 2898 ASSERT(buf->b_data == NULL); 2899 rw_exit(&buf->b_lock); 2900 return (0); 2901 } else if (buf->b_data == NULL) { 2902 arc_buf_t copy = *buf; /* structure assignment */ 2903 /* 2904 * We are on the eviction list; process this buffer now 2905 * but let arc_do_user_evicts() do the reaping. 2906 */ 2907 buf->b_efunc = NULL; 2908 rw_exit(&buf->b_lock); 2909 VERIFY(copy.b_efunc(©) == 0); 2910 return (1); 2911 } 2912 hash_lock = HDR_LOCK(hdr); 2913 mutex_enter(hash_lock); 2914 2915 ASSERT(buf->b_hdr == hdr); 2916 ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt); 2917 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2918 2919 /* 2920 * Pull this buffer off of the hdr 2921 */ 2922 bufp = &hdr->b_buf; 2923 while (*bufp != buf) 2924 bufp = &(*bufp)->b_next; 2925 *bufp = buf->b_next; 2926 2927 ASSERT(buf->b_data != NULL); 2928 arc_buf_destroy(buf, FALSE, FALSE); 2929 2930 if (hdr->b_datacnt == 0) { 2931 arc_state_t *old_state = hdr->b_state; 2932 arc_state_t *evicted_state; 2933 2934 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 2935 2936 evicted_state = 2937 (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 2938 2939 mutex_enter(&old_state->arcs_mtx); 2940 mutex_enter(&evicted_state->arcs_mtx); 2941 2942 arc_change_state(evicted_state, hdr, hash_lock); 2943 ASSERT(HDR_IN_HASH_TABLE(hdr)); 2944 hdr->b_flags |= ARC_IN_HASH_TABLE; 2945 hdr->b_flags &= ~ARC_BUF_AVAILABLE; 2946 2947 mutex_exit(&evicted_state->arcs_mtx); 2948 mutex_exit(&old_state->arcs_mtx); 2949 } 2950 mutex_exit(hash_lock); 2951 rw_exit(&buf->b_lock); 2952 2953 VERIFY(buf->b_efunc(buf) == 0); 2954 buf->b_efunc = NULL; 2955 buf->b_private = NULL; 2956 buf->b_hdr = NULL; 2957 kmem_cache_free(buf_cache, buf); 2958 return (1); 2959 } 2960 2961 /* 2962 * Release this buffer from the cache. This must be done 2963 * after a read and prior to modifying the buffer contents. 2964 * If the buffer has more than one reference, we must make 2965 * a new hdr for the buffer. 2966 */ 2967 void 2968 arc_release(arc_buf_t *buf, void *tag) 2969 { 2970 arc_buf_hdr_t *hdr; 2971 kmutex_t *hash_lock; 2972 l2arc_buf_hdr_t *l2hdr; 2973 uint64_t buf_size; 2974 boolean_t released = B_FALSE; 2975 2976 rw_enter(&buf->b_lock, RW_WRITER); 2977 hdr = buf->b_hdr; 2978 2979 /* this buffer is not on any list */ 2980 ASSERT(refcount_count(&hdr->b_refcnt) > 0); 2981 2982 if (hdr->b_state == arc_anon) { 2983 /* this buffer is already released */ 2984 ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 1); 2985 ASSERT(BUF_EMPTY(hdr)); 2986 ASSERT(buf->b_efunc == NULL); 2987 arc_buf_thaw(buf); 2988 rw_exit(&buf->b_lock); 2989 released = B_TRUE; 2990 } else { 2991 hash_lock = HDR_LOCK(hdr); 2992 mutex_enter(hash_lock); 2993 } 2994 2995 l2hdr = hdr->b_l2hdr; 2996 if (l2hdr) { 2997 mutex_enter(&l2arc_buflist_mtx); 2998 hdr->b_l2hdr = NULL; 2999 buf_size = hdr->b_size; 3000 } 3001 3002 if (released) 3003 goto out; 3004 3005 /* 3006 * Do we have more than one buf? 3007 */ 3008 if (hdr->b_datacnt > 1) { 3009 arc_buf_hdr_t *nhdr; 3010 arc_buf_t **bufp; 3011 uint64_t blksz = hdr->b_size; 3012 uint64_t spa = hdr->b_spa; 3013 arc_buf_contents_t type = hdr->b_type; 3014 uint32_t flags = hdr->b_flags; 3015 3016 ASSERT(hdr->b_buf != buf || buf->b_next != NULL); 3017 /* 3018 * Pull the data off of this buf and attach it to 3019 * a new anonymous buf. 3020 */ 3021 (void) remove_reference(hdr, hash_lock, tag); 3022 bufp = &hdr->b_buf; 3023 while (*bufp != buf) 3024 bufp = &(*bufp)->b_next; 3025 *bufp = (*bufp)->b_next; 3026 buf->b_next = NULL; 3027 3028 ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size); 3029 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size); 3030 if (refcount_is_zero(&hdr->b_refcnt)) { 3031 uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type]; 3032 ASSERT3U(*size, >=, hdr->b_size); 3033 atomic_add_64(size, -hdr->b_size); 3034 } 3035 hdr->b_datacnt -= 1; 3036 arc_cksum_verify(buf); 3037 3038 mutex_exit(hash_lock); 3039 3040 nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 3041 nhdr->b_size = blksz; 3042 nhdr->b_spa = spa; 3043 nhdr->b_type = type; 3044 nhdr->b_buf = buf; 3045 nhdr->b_state = arc_anon; 3046 nhdr->b_arc_access = 0; 3047 nhdr->b_flags = flags & ARC_L2_WRITING; 3048 nhdr->b_l2hdr = NULL; 3049 nhdr->b_datacnt = 1; 3050 nhdr->b_freeze_cksum = NULL; 3051 (void) refcount_add(&nhdr->b_refcnt, tag); 3052 buf->b_hdr = nhdr; 3053 rw_exit(&buf->b_lock); 3054 atomic_add_64(&arc_anon->arcs_size, blksz); 3055 } else { 3056 rw_exit(&buf->b_lock); 3057 ASSERT(refcount_count(&hdr->b_refcnt) == 1); 3058 ASSERT(!list_link_active(&hdr->b_arc_node)); 3059 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 3060 arc_change_state(arc_anon, hdr, hash_lock); 3061 hdr->b_arc_access = 0; 3062 mutex_exit(hash_lock); 3063 3064 bzero(&hdr->b_dva, sizeof (dva_t)); 3065 hdr->b_birth = 0; 3066 hdr->b_cksum0 = 0; 3067 arc_buf_thaw(buf); 3068 } 3069 buf->b_efunc = NULL; 3070 buf->b_private = NULL; 3071 3072 out: 3073 if (l2hdr) { 3074 list_remove(l2hdr->b_dev->l2ad_buflist, hdr); 3075 kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t)); 3076 ARCSTAT_INCR(arcstat_l2_size, -buf_size); 3077 mutex_exit(&l2arc_buflist_mtx); 3078 } 3079 } 3080 3081 int 3082 arc_released(arc_buf_t *buf) 3083 { 3084 int released; 3085 3086 rw_enter(&buf->b_lock, RW_READER); 3087 released = (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon); 3088 rw_exit(&buf->b_lock); 3089 return (released); 3090 } 3091 3092 int 3093 arc_has_callback(arc_buf_t *buf) 3094 { 3095 int callback; 3096 3097 rw_enter(&buf->b_lock, RW_READER); 3098 callback = (buf->b_efunc != NULL); 3099 rw_exit(&buf->b_lock); 3100 return (callback); 3101 } 3102 3103 #ifdef ZFS_DEBUG 3104 int 3105 arc_referenced(arc_buf_t *buf) 3106 { 3107 int referenced; 3108 3109 rw_enter(&buf->b_lock, RW_READER); 3110 referenced = (refcount_count(&buf->b_hdr->b_refcnt)); 3111 rw_exit(&buf->b_lock); 3112 return (referenced); 3113 } 3114 #endif 3115 3116 static void 3117 arc_write_ready(zio_t *zio) 3118 { 3119 arc_write_callback_t *callback = zio->io_private; 3120 arc_buf_t *buf = callback->awcb_buf; 3121 arc_buf_hdr_t *hdr = buf->b_hdr; 3122 3123 ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt)); 3124 callback->awcb_ready(zio, buf, callback->awcb_private); 3125 3126 /* 3127 * If the IO is already in progress, then this is a re-write 3128 * attempt, so we need to thaw and re-compute the cksum. 3129 * It is the responsibility of the callback to handle the 3130 * accounting for any re-write attempt. 3131 */ 3132 if (HDR_IO_IN_PROGRESS(hdr)) { 3133 mutex_enter(&hdr->b_freeze_lock); 3134 if (hdr->b_freeze_cksum != NULL) { 3135 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 3136 hdr->b_freeze_cksum = NULL; 3137 } 3138 mutex_exit(&hdr->b_freeze_lock); 3139 } 3140 arc_cksum_compute(buf, B_FALSE); 3141 hdr->b_flags |= ARC_IO_IN_PROGRESS; 3142 } 3143 3144 static void 3145 arc_write_done(zio_t *zio) 3146 { 3147 arc_write_callback_t *callback = zio->io_private; 3148 arc_buf_t *buf = callback->awcb_buf; 3149 arc_buf_hdr_t *hdr = buf->b_hdr; 3150 3151 ASSERT(hdr->b_acb == NULL); 3152 3153 if (zio->io_error == 0) { 3154 hdr->b_dva = *BP_IDENTITY(zio->io_bp); 3155 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp); 3156 hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0]; 3157 } else { 3158 ASSERT(BUF_EMPTY(hdr)); 3159 } 3160 3161 /* 3162 * If the block to be written was all-zero, we may have 3163 * compressed it away. In this case no write was performed 3164 * so there will be no dva/birth-date/checksum. The buffer 3165 * must therefor remain anonymous (and uncached). 3166 */ 3167 if (!BUF_EMPTY(hdr)) { 3168 arc_buf_hdr_t *exists; 3169 kmutex_t *hash_lock; 3170 3171 ASSERT(zio->io_error == 0); 3172 3173 arc_cksum_verify(buf); 3174 3175 exists = buf_hash_insert(hdr, &hash_lock); 3176 if (exists) { 3177 /* 3178 * This can only happen if we overwrite for 3179 * sync-to-convergence, because we remove 3180 * buffers from the hash table when we arc_free(). 3181 */ 3182 if (zio->io_flags & ZIO_FLAG_IO_REWRITE) { 3183 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp)) 3184 panic("bad overwrite, hdr=%p exists=%p", 3185 (void *)hdr, (void *)exists); 3186 ASSERT(refcount_is_zero(&exists->b_refcnt)); 3187 arc_change_state(arc_anon, exists, hash_lock); 3188 mutex_exit(hash_lock); 3189 arc_hdr_destroy(exists); 3190 exists = buf_hash_insert(hdr, &hash_lock); 3191 ASSERT3P(exists, ==, NULL); 3192 } else { 3193 /* Dedup */ 3194 ASSERT(hdr->b_datacnt == 1); 3195 ASSERT(hdr->b_state == arc_anon); 3196 ASSERT(BP_GET_DEDUP(zio->io_bp)); 3197 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0); 3198 } 3199 } 3200 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 3201 /* if it's not anon, we are doing a scrub */ 3202 if (!exists && hdr->b_state == arc_anon) 3203 arc_access(hdr, hash_lock); 3204 mutex_exit(hash_lock); 3205 } else { 3206 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 3207 } 3208 3209 ASSERT(!refcount_is_zero(&hdr->b_refcnt)); 3210 callback->awcb_done(zio, buf, callback->awcb_private); 3211 3212 kmem_free(callback, sizeof (arc_write_callback_t)); 3213 } 3214 3215 zio_t * 3216 arc_write(zio_t *pio, spa_t *spa, uint64_t txg, 3217 blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, const zio_prop_t *zp, 3218 arc_done_func_t *ready, arc_done_func_t *done, void *private, 3219 int priority, int zio_flags, const zbookmark_t *zb) 3220 { 3221 arc_buf_hdr_t *hdr = buf->b_hdr; 3222 arc_write_callback_t *callback; 3223 zio_t *zio; 3224 3225 ASSERT(ready != NULL); 3226 ASSERT(done != NULL); 3227 ASSERT(!HDR_IO_ERROR(hdr)); 3228 ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0); 3229 ASSERT(hdr->b_acb == NULL); 3230 if (l2arc) 3231 hdr->b_flags |= ARC_L2CACHE; 3232 callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP); 3233 callback->awcb_ready = ready; 3234 callback->awcb_done = done; 3235 callback->awcb_private = private; 3236 callback->awcb_buf = buf; 3237 3238 zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp, 3239 arc_write_ready, arc_write_done, callback, priority, zio_flags, zb); 3240 3241 return (zio); 3242 } 3243 3244 void 3245 arc_free(spa_t *spa, const blkptr_t *bp) 3246 { 3247 arc_buf_hdr_t *ab; 3248 kmutex_t *hash_lock; 3249 uint64_t guid = spa_guid(spa); 3250 3251 /* 3252 * If this buffer is in the cache, release it, so it can be re-used. 3253 */ 3254 ab = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp), 3255 &hash_lock); 3256 if (ab != NULL) { 3257 if (ab->b_state != arc_anon) 3258 arc_change_state(arc_anon, ab, hash_lock); 3259 if (HDR_IO_IN_PROGRESS(ab)) { 3260 /* 3261 * This should only happen when we prefetch. 3262 */ 3263 ASSERT(ab->b_flags & ARC_PREFETCH); 3264 ASSERT3U(ab->b_datacnt, ==, 1); 3265 ab->b_flags |= ARC_FREED_IN_READ; 3266 if (HDR_IN_HASH_TABLE(ab)) 3267 buf_hash_remove(ab); 3268 ab->b_arc_access = 0; 3269 bzero(&ab->b_dva, sizeof (dva_t)); 3270 ab->b_birth = 0; 3271 ab->b_cksum0 = 0; 3272 ab->b_buf->b_efunc = NULL; 3273 ab->b_buf->b_private = NULL; 3274 mutex_exit(hash_lock); 3275 } else { 3276 ASSERT(refcount_is_zero(&ab->b_refcnt)); 3277 ab->b_flags |= ARC_FREE_IN_PROGRESS; 3278 mutex_exit(hash_lock); 3279 arc_hdr_destroy(ab); 3280 ARCSTAT_BUMP(arcstat_deleted); 3281 } 3282 } 3283 } 3284 3285 static int 3286 arc_memory_throttle(uint64_t reserve, uint64_t inflight_data, uint64_t txg) 3287 { 3288 #ifdef _KERNEL 3289 uint64_t available_memory = ptob(freemem); 3290 static uint64_t page_load = 0; 3291 static uint64_t last_txg = 0; 3292 3293 #if defined(__i386) 3294 available_memory = 3295 MIN(available_memory, vmem_size(heap_arena, VMEM_FREE)); 3296 #endif 3297 if (available_memory >= zfs_write_limit_max) 3298 return (0); 3299 3300 if (txg > last_txg) { 3301 last_txg = txg; 3302 page_load = 0; 3303 } 3304 /* 3305 * If we are in pageout, we know that memory is already tight, 3306 * the arc is already going to be evicting, so we just want to 3307 * continue to let page writes occur as quickly as possible. 3308 */ 3309 if (curproc == proc_pageout) { 3310 if (page_load > MAX(ptob(minfree), available_memory) / 4) 3311 return (ERESTART); 3312 /* Note: reserve is inflated, so we deflate */ 3313 page_load += reserve / 8; 3314 return (0); 3315 } else if (page_load > 0 && arc_reclaim_needed()) { 3316 /* memory is low, delay before restarting */ 3317 ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 3318 return (EAGAIN); 3319 } 3320 page_load = 0; 3321 3322 if (arc_size > arc_c_min) { 3323 uint64_t evictable_memory = 3324 arc_mru->arcs_lsize[ARC_BUFC_DATA] + 3325 arc_mru->arcs_lsize[ARC_BUFC_METADATA] + 3326 arc_mfu->arcs_lsize[ARC_BUFC_DATA] + 3327 arc_mfu->arcs_lsize[ARC_BUFC_METADATA]; 3328 available_memory += MIN(evictable_memory, arc_size - arc_c_min); 3329 } 3330 3331 if (inflight_data > available_memory / 4) { 3332 ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 3333 return (ERESTART); 3334 } 3335 #endif 3336 return (0); 3337 } 3338 3339 void 3340 arc_tempreserve_clear(uint64_t reserve) 3341 { 3342 atomic_add_64(&arc_tempreserve, -reserve); 3343 ASSERT((int64_t)arc_tempreserve >= 0); 3344 } 3345 3346 int 3347 arc_tempreserve_space(uint64_t reserve, uint64_t txg) 3348 { 3349 int error; 3350 uint64_t anon_size; 3351 3352 #ifdef ZFS_DEBUG 3353 /* 3354 * Once in a while, fail for no reason. Everything should cope. 3355 */ 3356 if (spa_get_random(10000) == 0) { 3357 dprintf("forcing random failure\n"); 3358 return (ERESTART); 3359 } 3360 #endif 3361 if (reserve > arc_c/4 && !arc_no_grow) 3362 arc_c = MIN(arc_c_max, reserve * 4); 3363 if (reserve > arc_c) 3364 return (ENOMEM); 3365 3366 /* 3367 * Don't count loaned bufs as in flight dirty data to prevent long 3368 * network delays from blocking transactions that are ready to be 3369 * assigned to a txg. 3370 */ 3371 anon_size = MAX((int64_t)(arc_anon->arcs_size - arc_loaned_bytes), 0); 3372 3373 /* 3374 * Writes will, almost always, require additional memory allocations 3375 * in order to compress/encrypt/etc the data. We therefor need to 3376 * make sure that there is sufficient available memory for this. 3377 */ 3378 if (error = arc_memory_throttle(reserve, anon_size, txg)) 3379 return (error); 3380 3381 /* 3382 * Throttle writes when the amount of dirty data in the cache 3383 * gets too large. We try to keep the cache less than half full 3384 * of dirty blocks so that our sync times don't grow too large. 3385 * Note: if two requests come in concurrently, we might let them 3386 * both succeed, when one of them should fail. Not a huge deal. 3387 */ 3388 3389 if (reserve + arc_tempreserve + anon_size > arc_c / 2 && 3390 anon_size > arc_c / 4) { 3391 dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK " 3392 "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n", 3393 arc_tempreserve>>10, 3394 arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10, 3395 arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10, 3396 reserve>>10, arc_c>>10); 3397 return (ERESTART); 3398 } 3399 atomic_add_64(&arc_tempreserve, reserve); 3400 return (0); 3401 } 3402 3403 void 3404 arc_init(void) 3405 { 3406 mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL); 3407 cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL); 3408 3409 /* Convert seconds to clock ticks */ 3410 arc_min_prefetch_lifespan = 1 * hz; 3411 3412 /* Start out with 1/8 of all memory */ 3413 arc_c = physmem * PAGESIZE / 8; 3414 3415 #ifdef _KERNEL 3416 /* 3417 * On architectures where the physical memory can be larger 3418 * than the addressable space (intel in 32-bit mode), we may 3419 * need to limit the cache to 1/8 of VM size. 3420 */ 3421 arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8); 3422 #endif 3423 3424 /* set min cache to 1/32 of all memory, or 64MB, whichever is more */ 3425 arc_c_min = MAX(arc_c / 4, 64<<20); 3426 /* set max to 3/4 of all memory, or all but 1GB, whichever is more */ 3427 if (arc_c * 8 >= 1<<30) 3428 arc_c_max = (arc_c * 8) - (1<<30); 3429 else 3430 arc_c_max = arc_c_min; 3431 arc_c_max = MAX(arc_c * 6, arc_c_max); 3432 3433 /* 3434 * Allow the tunables to override our calculations if they are 3435 * reasonable (ie. over 64MB) 3436 */ 3437 if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE) 3438 arc_c_max = zfs_arc_max; 3439 if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max) 3440 arc_c_min = zfs_arc_min; 3441 3442 arc_c = arc_c_max; 3443 arc_p = (arc_c >> 1); 3444 3445 /* limit meta-data to 1/4 of the arc capacity */ 3446 arc_meta_limit = arc_c_max / 4; 3447 3448 /* Allow the tunable to override if it is reasonable */ 3449 if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max) 3450 arc_meta_limit = zfs_arc_meta_limit; 3451 3452 if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0) 3453 arc_c_min = arc_meta_limit / 2; 3454 3455 if (zfs_arc_grow_retry > 0) 3456 arc_grow_retry = zfs_arc_grow_retry; 3457 3458 if (zfs_arc_shrink_shift > 0) 3459 arc_shrink_shift = zfs_arc_shrink_shift; 3460 3461 if (zfs_arc_p_min_shift > 0) 3462 arc_p_min_shift = zfs_arc_p_min_shift; 3463 3464 /* if kmem_flags are set, lets try to use less memory */ 3465 if (kmem_debugging()) 3466 arc_c = arc_c / 2; 3467 if (arc_c < arc_c_min) 3468 arc_c = arc_c_min; 3469 3470 arc_anon = &ARC_anon; 3471 arc_mru = &ARC_mru; 3472 arc_mru_ghost = &ARC_mru_ghost; 3473 arc_mfu = &ARC_mfu; 3474 arc_mfu_ghost = &ARC_mfu_ghost; 3475 arc_l2c_only = &ARC_l2c_only; 3476 arc_size = 0; 3477 3478 mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3479 mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3480 mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3481 mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3482 mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3483 mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3484 3485 list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA], 3486 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3487 list_create(&arc_mru->arcs_list[ARC_BUFC_DATA], 3488 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3489 list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA], 3490 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3491 list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA], 3492 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3493 list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA], 3494 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3495 list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA], 3496 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3497 list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA], 3498 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3499 list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA], 3500 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3501 list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA], 3502 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3503 list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA], 3504 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3505 3506 buf_init(); 3507 3508 arc_thread_exit = 0; 3509 arc_eviction_list = NULL; 3510 mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL); 3511 bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t)); 3512 3513 arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED, 3514 sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 3515 3516 if (arc_ksp != NULL) { 3517 arc_ksp->ks_data = &arc_stats; 3518 kstat_install(arc_ksp); 3519 } 3520 3521 (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0, 3522 TS_RUN, minclsyspri); 3523 3524 arc_dead = FALSE; 3525 arc_warm = B_FALSE; 3526 3527 if (zfs_write_limit_max == 0) 3528 zfs_write_limit_max = ptob(physmem) >> zfs_write_limit_shift; 3529 else 3530 zfs_write_limit_shift = 0; 3531 mutex_init(&zfs_write_limit_lock, NULL, MUTEX_DEFAULT, NULL); 3532 } 3533 3534 void 3535 arc_fini(void) 3536 { 3537 mutex_enter(&arc_reclaim_thr_lock); 3538 arc_thread_exit = 1; 3539 while (arc_thread_exit != 0) 3540 cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock); 3541 mutex_exit(&arc_reclaim_thr_lock); 3542 3543 arc_flush(NULL); 3544 3545 arc_dead = TRUE; 3546 3547 if (arc_ksp != NULL) { 3548 kstat_delete(arc_ksp); 3549 arc_ksp = NULL; 3550 } 3551 3552 mutex_destroy(&arc_eviction_mtx); 3553 mutex_destroy(&arc_reclaim_thr_lock); 3554 cv_destroy(&arc_reclaim_thr_cv); 3555 3556 list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]); 3557 list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]); 3558 list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]); 3559 list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]); 3560 list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]); 3561 list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]); 3562 list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]); 3563 list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]); 3564 3565 mutex_destroy(&arc_anon->arcs_mtx); 3566 mutex_destroy(&arc_mru->arcs_mtx); 3567 mutex_destroy(&arc_mru_ghost->arcs_mtx); 3568 mutex_destroy(&arc_mfu->arcs_mtx); 3569 mutex_destroy(&arc_mfu_ghost->arcs_mtx); 3570 mutex_destroy(&arc_l2c_only->arcs_mtx); 3571 3572 mutex_destroy(&zfs_write_limit_lock); 3573 3574 buf_fini(); 3575 3576 ASSERT(arc_loaned_bytes == 0); 3577 } 3578 3579 /* 3580 * Level 2 ARC 3581 * 3582 * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk. 3583 * It uses dedicated storage devices to hold cached data, which are populated 3584 * using large infrequent writes. The main role of this cache is to boost 3585 * the performance of random read workloads. The intended L2ARC devices 3586 * include short-stroked disks, solid state disks, and other media with 3587 * substantially faster read latency than disk. 3588 * 3589 * +-----------------------+ 3590 * | ARC | 3591 * +-----------------------+ 3592 * | ^ ^ 3593 * | | | 3594 * l2arc_feed_thread() arc_read() 3595 * | | | 3596 * | l2arc read | 3597 * V | | 3598 * +---------------+ | 3599 * | L2ARC | | 3600 * +---------------+ | 3601 * | ^ | 3602 * l2arc_write() | | 3603 * | | | 3604 * V | | 3605 * +-------+ +-------+ 3606 * | vdev | | vdev | 3607 * | cache | | cache | 3608 * +-------+ +-------+ 3609 * +=========+ .-----. 3610 * : L2ARC : |-_____-| 3611 * : devices : | Disks | 3612 * +=========+ `-_____-' 3613 * 3614 * Read requests are satisfied from the following sources, in order: 3615 * 3616 * 1) ARC 3617 * 2) vdev cache of L2ARC devices 3618 * 3) L2ARC devices 3619 * 4) vdev cache of disks 3620 * 5) disks 3621 * 3622 * Some L2ARC device types exhibit extremely slow write performance. 3623 * To accommodate for this there are some significant differences between 3624 * the L2ARC and traditional cache design: 3625 * 3626 * 1. There is no eviction path from the ARC to the L2ARC. Evictions from 3627 * the ARC behave as usual, freeing buffers and placing headers on ghost 3628 * lists. The ARC does not send buffers to the L2ARC during eviction as 3629 * this would add inflated write latencies for all ARC memory pressure. 3630 * 3631 * 2. The L2ARC attempts to cache data from the ARC before it is evicted. 3632 * It does this by periodically scanning buffers from the eviction-end of 3633 * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are 3634 * not already there. It scans until a headroom of buffers is satisfied, 3635 * which itself is a buffer for ARC eviction. The thread that does this is 3636 * l2arc_feed_thread(), illustrated below; example sizes are included to 3637 * provide a better sense of ratio than this diagram: 3638 * 3639 * head --> tail 3640 * +---------------------+----------+ 3641 * ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->. # already on L2ARC 3642 * +---------------------+----------+ | o L2ARC eligible 3643 * ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->| : ARC buffer 3644 * +---------------------+----------+ | 3645 * 15.9 Gbytes ^ 32 Mbytes | 3646 * headroom | 3647 * l2arc_feed_thread() 3648 * | 3649 * l2arc write hand <--[oooo]--' 3650 * | 8 Mbyte 3651 * | write max 3652 * V 3653 * +==============================+ 3654 * L2ARC dev |####|#|###|###| |####| ... | 3655 * +==============================+ 3656 * 32 Gbytes 3657 * 3658 * 3. If an ARC buffer is copied to the L2ARC but then hit instead of 3659 * evicted, then the L2ARC has cached a buffer much sooner than it probably 3660 * needed to, potentially wasting L2ARC device bandwidth and storage. It is 3661 * safe to say that this is an uncommon case, since buffers at the end of 3662 * the ARC lists have moved there due to inactivity. 3663 * 3664 * 4. If the ARC evicts faster than the L2ARC can maintain a headroom, 3665 * then the L2ARC simply misses copying some buffers. This serves as a 3666 * pressure valve to prevent heavy read workloads from both stalling the ARC 3667 * with waits and clogging the L2ARC with writes. This also helps prevent 3668 * the potential for the L2ARC to churn if it attempts to cache content too 3669 * quickly, such as during backups of the entire pool. 3670 * 3671 * 5. After system boot and before the ARC has filled main memory, there are 3672 * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru 3673 * lists can remain mostly static. Instead of searching from tail of these 3674 * lists as pictured, the l2arc_feed_thread() will search from the list heads 3675 * for eligible buffers, greatly increasing its chance of finding them. 3676 * 3677 * The L2ARC device write speed is also boosted during this time so that 3678 * the L2ARC warms up faster. Since there have been no ARC evictions yet, 3679 * there are no L2ARC reads, and no fear of degrading read performance 3680 * through increased writes. 3681 * 3682 * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that 3683 * the vdev queue can aggregate them into larger and fewer writes. Each 3684 * device is written to in a rotor fashion, sweeping writes through 3685 * available space then repeating. 3686 * 3687 * 7. The L2ARC does not store dirty content. It never needs to flush 3688 * write buffers back to disk based storage. 3689 * 3690 * 8. If an ARC buffer is written (and dirtied) which also exists in the 3691 * L2ARC, the now stale L2ARC buffer is immediately dropped. 3692 * 3693 * The performance of the L2ARC can be tweaked by a number of tunables, which 3694 * may be necessary for different workloads: 3695 * 3696 * l2arc_write_max max write bytes per interval 3697 * l2arc_write_boost extra write bytes during device warmup 3698 * l2arc_noprefetch skip caching prefetched buffers 3699 * l2arc_headroom number of max device writes to precache 3700 * l2arc_feed_secs seconds between L2ARC writing 3701 * 3702 * Tunables may be removed or added as future performance improvements are 3703 * integrated, and also may become zpool properties. 3704 * 3705 * There are three key functions that control how the L2ARC warms up: 3706 * 3707 * l2arc_write_eligible() check if a buffer is eligible to cache 3708 * l2arc_write_size() calculate how much to write 3709 * l2arc_write_interval() calculate sleep delay between writes 3710 * 3711 * These three functions determine what to write, how much, and how quickly 3712 * to send writes. 3713 */ 3714 3715 static boolean_t 3716 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab) 3717 { 3718 /* 3719 * A buffer is *not* eligible for the L2ARC if it: 3720 * 1. belongs to a different spa. 3721 * 2. is already cached on the L2ARC. 3722 * 3. has an I/O in progress (it may be an incomplete read). 3723 * 4. is flagged not eligible (zfs property). 3724 */ 3725 if (ab->b_spa != spa_guid || ab->b_l2hdr != NULL || 3726 HDR_IO_IN_PROGRESS(ab) || !HDR_L2CACHE(ab)) 3727 return (B_FALSE); 3728 3729 return (B_TRUE); 3730 } 3731 3732 static uint64_t 3733 l2arc_write_size(l2arc_dev_t *dev) 3734 { 3735 uint64_t size; 3736 3737 size = dev->l2ad_write; 3738 3739 if (arc_warm == B_FALSE) 3740 size += dev->l2ad_boost; 3741 3742 return (size); 3743 3744 } 3745 3746 static clock_t 3747 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote) 3748 { 3749 clock_t interval, next, now; 3750 3751 /* 3752 * If the ARC lists are busy, increase our write rate; if the 3753 * lists are stale, idle back. This is achieved by checking 3754 * how much we previously wrote - if it was more than half of 3755 * what we wanted, schedule the next write much sooner. 3756 */ 3757 if (l2arc_feed_again && wrote > (wanted / 2)) 3758 interval = (hz * l2arc_feed_min_ms) / 1000; 3759 else 3760 interval = hz * l2arc_feed_secs; 3761 3762 now = ddi_get_lbolt(); 3763 next = MAX(now, MIN(now + interval, began + interval)); 3764 3765 return (next); 3766 } 3767 3768 static void 3769 l2arc_hdr_stat_add(void) 3770 { 3771 ARCSTAT_INCR(arcstat_l2_hdr_size, HDR_SIZE + L2HDR_SIZE); 3772 ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE); 3773 } 3774 3775 static void 3776 l2arc_hdr_stat_remove(void) 3777 { 3778 ARCSTAT_INCR(arcstat_l2_hdr_size, -(HDR_SIZE + L2HDR_SIZE)); 3779 ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE); 3780 } 3781 3782 /* 3783 * Cycle through L2ARC devices. This is how L2ARC load balances. 3784 * If a device is returned, this also returns holding the spa config lock. 3785 */ 3786 static l2arc_dev_t * 3787 l2arc_dev_get_next(void) 3788 { 3789 l2arc_dev_t *first, *next = NULL; 3790 3791 /* 3792 * Lock out the removal of spas (spa_namespace_lock), then removal 3793 * of cache devices (l2arc_dev_mtx). Once a device has been selected, 3794 * both locks will be dropped and a spa config lock held instead. 3795 */ 3796 mutex_enter(&spa_namespace_lock); 3797 mutex_enter(&l2arc_dev_mtx); 3798 3799 /* if there are no vdevs, there is nothing to do */ 3800 if (l2arc_ndev == 0) 3801 goto out; 3802 3803 first = NULL; 3804 next = l2arc_dev_last; 3805 do { 3806 /* loop around the list looking for a non-faulted vdev */ 3807 if (next == NULL) { 3808 next = list_head(l2arc_dev_list); 3809 } else { 3810 next = list_next(l2arc_dev_list, next); 3811 if (next == NULL) 3812 next = list_head(l2arc_dev_list); 3813 } 3814 3815 /* if we have come back to the start, bail out */ 3816 if (first == NULL) 3817 first = next; 3818 else if (next == first) 3819 break; 3820 3821 } while (vdev_is_dead(next->l2ad_vdev)); 3822 3823 /* if we were unable to find any usable vdevs, return NULL */ 3824 if (vdev_is_dead(next->l2ad_vdev)) 3825 next = NULL; 3826 3827 l2arc_dev_last = next; 3828 3829 out: 3830 mutex_exit(&l2arc_dev_mtx); 3831 3832 /* 3833 * Grab the config lock to prevent the 'next' device from being 3834 * removed while we are writing to it. 3835 */ 3836 if (next != NULL) 3837 spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER); 3838 mutex_exit(&spa_namespace_lock); 3839 3840 return (next); 3841 } 3842 3843 /* 3844 * Free buffers that were tagged for destruction. 3845 */ 3846 static void 3847 l2arc_do_free_on_write() 3848 { 3849 list_t *buflist; 3850 l2arc_data_free_t *df, *df_prev; 3851 3852 mutex_enter(&l2arc_free_on_write_mtx); 3853 buflist = l2arc_free_on_write; 3854 3855 for (df = list_tail(buflist); df; df = df_prev) { 3856 df_prev = list_prev(buflist, df); 3857 ASSERT(df->l2df_data != NULL); 3858 ASSERT(df->l2df_func != NULL); 3859 df->l2df_func(df->l2df_data, df->l2df_size); 3860 list_remove(buflist, df); 3861 kmem_free(df, sizeof (l2arc_data_free_t)); 3862 } 3863 3864 mutex_exit(&l2arc_free_on_write_mtx); 3865 } 3866 3867 /* 3868 * A write to a cache device has completed. Update all headers to allow 3869 * reads from these buffers to begin. 3870 */ 3871 static void 3872 l2arc_write_done(zio_t *zio) 3873 { 3874 l2arc_write_callback_t *cb; 3875 l2arc_dev_t *dev; 3876 list_t *buflist; 3877 arc_buf_hdr_t *head, *ab, *ab_prev; 3878 l2arc_buf_hdr_t *abl2; 3879 kmutex_t *hash_lock; 3880 3881 cb = zio->io_private; 3882 ASSERT(cb != NULL); 3883 dev = cb->l2wcb_dev; 3884 ASSERT(dev != NULL); 3885 head = cb->l2wcb_head; 3886 ASSERT(head != NULL); 3887 buflist = dev->l2ad_buflist; 3888 ASSERT(buflist != NULL); 3889 DTRACE_PROBE2(l2arc__iodone, zio_t *, zio, 3890 l2arc_write_callback_t *, cb); 3891 3892 if (zio->io_error != 0) 3893 ARCSTAT_BUMP(arcstat_l2_writes_error); 3894 3895 mutex_enter(&l2arc_buflist_mtx); 3896 3897 /* 3898 * All writes completed, or an error was hit. 3899 */ 3900 for (ab = list_prev(buflist, head); ab; ab = ab_prev) { 3901 ab_prev = list_prev(buflist, ab); 3902 3903 hash_lock = HDR_LOCK(ab); 3904 if (!mutex_tryenter(hash_lock)) { 3905 /* 3906 * This buffer misses out. It may be in a stage 3907 * of eviction. Its ARC_L2_WRITING flag will be 3908 * left set, denying reads to this buffer. 3909 */ 3910 ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss); 3911 continue; 3912 } 3913 3914 if (zio->io_error != 0) { 3915 /* 3916 * Error - drop L2ARC entry. 3917 */ 3918 list_remove(buflist, ab); 3919 abl2 = ab->b_l2hdr; 3920 ab->b_l2hdr = NULL; 3921 kmem_free(abl2, sizeof (l2arc_buf_hdr_t)); 3922 ARCSTAT_INCR(arcstat_l2_size, -ab->b_size); 3923 } 3924 3925 /* 3926 * Allow ARC to begin reads to this L2ARC entry. 3927 */ 3928 ab->b_flags &= ~ARC_L2_WRITING; 3929 3930 mutex_exit(hash_lock); 3931 } 3932 3933 atomic_inc_64(&l2arc_writes_done); 3934 list_remove(buflist, head); 3935 kmem_cache_free(hdr_cache, head); 3936 mutex_exit(&l2arc_buflist_mtx); 3937 3938 l2arc_do_free_on_write(); 3939 3940 kmem_free(cb, sizeof (l2arc_write_callback_t)); 3941 } 3942 3943 /* 3944 * A read to a cache device completed. Validate buffer contents before 3945 * handing over to the regular ARC routines. 3946 */ 3947 static void 3948 l2arc_read_done(zio_t *zio) 3949 { 3950 l2arc_read_callback_t *cb; 3951 arc_buf_hdr_t *hdr; 3952 arc_buf_t *buf; 3953 kmutex_t *hash_lock; 3954 int equal; 3955 3956 ASSERT(zio->io_vd != NULL); 3957 ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE); 3958 3959 spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd); 3960 3961 cb = zio->io_private; 3962 ASSERT(cb != NULL); 3963 buf = cb->l2rcb_buf; 3964 ASSERT(buf != NULL); 3965 hdr = buf->b_hdr; 3966 ASSERT(hdr != NULL); 3967 3968 hash_lock = HDR_LOCK(hdr); 3969 mutex_enter(hash_lock); 3970 3971 /* 3972 * Check this survived the L2ARC journey. 3973 */ 3974 equal = arc_cksum_equal(buf); 3975 if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) { 3976 mutex_exit(hash_lock); 3977 zio->io_private = buf; 3978 zio->io_bp_copy = cb->l2rcb_bp; /* XXX fix in L2ARC 2.0 */ 3979 zio->io_bp = &zio->io_bp_copy; /* XXX fix in L2ARC 2.0 */ 3980 arc_read_done(zio); 3981 } else { 3982 mutex_exit(hash_lock); 3983 /* 3984 * Buffer didn't survive caching. Increment stats and 3985 * reissue to the original storage device. 3986 */ 3987 if (zio->io_error != 0) { 3988 ARCSTAT_BUMP(arcstat_l2_io_error); 3989 } else { 3990 zio->io_error = EIO; 3991 } 3992 if (!equal) 3993 ARCSTAT_BUMP(arcstat_l2_cksum_bad); 3994 3995 /* 3996 * If there's no waiter, issue an async i/o to the primary 3997 * storage now. If there *is* a waiter, the caller must 3998 * issue the i/o in a context where it's OK to block. 3999 */ 4000 if (zio->io_waiter == NULL) { 4001 zio_t *pio = zio_unique_parent(zio); 4002 4003 ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL); 4004 4005 zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp, 4006 buf->b_data, zio->io_size, arc_read_done, buf, 4007 zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb)); 4008 } 4009 } 4010 4011 kmem_free(cb, sizeof (l2arc_read_callback_t)); 4012 } 4013 4014 /* 4015 * This is the list priority from which the L2ARC will search for pages to 4016 * cache. This is used within loops (0..3) to cycle through lists in the 4017 * desired order. This order can have a significant effect on cache 4018 * performance. 4019 * 4020 * Currently the metadata lists are hit first, MFU then MRU, followed by 4021 * the data lists. This function returns a locked list, and also returns 4022 * the lock pointer. 4023 */ 4024 static list_t * 4025 l2arc_list_locked(int list_num, kmutex_t **lock) 4026 { 4027 list_t *list; 4028 4029 ASSERT(list_num >= 0 && list_num <= 3); 4030 4031 switch (list_num) { 4032 case 0: 4033 list = &arc_mfu->arcs_list[ARC_BUFC_METADATA]; 4034 *lock = &arc_mfu->arcs_mtx; 4035 break; 4036 case 1: 4037 list = &arc_mru->arcs_list[ARC_BUFC_METADATA]; 4038 *lock = &arc_mru->arcs_mtx; 4039 break; 4040 case 2: 4041 list = &arc_mfu->arcs_list[ARC_BUFC_DATA]; 4042 *lock = &arc_mfu->arcs_mtx; 4043 break; 4044 case 3: 4045 list = &arc_mru->arcs_list[ARC_BUFC_DATA]; 4046 *lock = &arc_mru->arcs_mtx; 4047 break; 4048 } 4049 4050 ASSERT(!(MUTEX_HELD(*lock))); 4051 mutex_enter(*lock); 4052 return (list); 4053 } 4054 4055 /* 4056 * Evict buffers from the device write hand to the distance specified in 4057 * bytes. This distance may span populated buffers, it may span nothing. 4058 * This is clearing a region on the L2ARC device ready for writing. 4059 * If the 'all' boolean is set, every buffer is evicted. 4060 */ 4061 static void 4062 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all) 4063 { 4064 list_t *buflist; 4065 l2arc_buf_hdr_t *abl2; 4066 arc_buf_hdr_t *ab, *ab_prev; 4067 kmutex_t *hash_lock; 4068 uint64_t taddr; 4069 4070 buflist = dev->l2ad_buflist; 4071 4072 if (buflist == NULL) 4073 return; 4074 4075 if (!all && dev->l2ad_first) { 4076 /* 4077 * This is the first sweep through the device. There is 4078 * nothing to evict. 4079 */ 4080 return; 4081 } 4082 4083 if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) { 4084 /* 4085 * When nearing the end of the device, evict to the end 4086 * before the device write hand jumps to the start. 4087 */ 4088 taddr = dev->l2ad_end; 4089 } else { 4090 taddr = dev->l2ad_hand + distance; 4091 } 4092 DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist, 4093 uint64_t, taddr, boolean_t, all); 4094 4095 top: 4096 mutex_enter(&l2arc_buflist_mtx); 4097 for (ab = list_tail(buflist); ab; ab = ab_prev) { 4098 ab_prev = list_prev(buflist, ab); 4099 4100 hash_lock = HDR_LOCK(ab); 4101 if (!mutex_tryenter(hash_lock)) { 4102 /* 4103 * Missed the hash lock. Retry. 4104 */ 4105 ARCSTAT_BUMP(arcstat_l2_evict_lock_retry); 4106 mutex_exit(&l2arc_buflist_mtx); 4107 mutex_enter(hash_lock); 4108 mutex_exit(hash_lock); 4109 goto top; 4110 } 4111 4112 if (HDR_L2_WRITE_HEAD(ab)) { 4113 /* 4114 * We hit a write head node. Leave it for 4115 * l2arc_write_done(). 4116 */ 4117 list_remove(buflist, ab); 4118 mutex_exit(hash_lock); 4119 continue; 4120 } 4121 4122 if (!all && ab->b_l2hdr != NULL && 4123 (ab->b_l2hdr->b_daddr > taddr || 4124 ab->b_l2hdr->b_daddr < dev->l2ad_hand)) { 4125 /* 4126 * We've evicted to the target address, 4127 * or the end of the device. 4128 */ 4129 mutex_exit(hash_lock); 4130 break; 4131 } 4132 4133 if (HDR_FREE_IN_PROGRESS(ab)) { 4134 /* 4135 * Already on the path to destruction. 4136 */ 4137 mutex_exit(hash_lock); 4138 continue; 4139 } 4140 4141 if (ab->b_state == arc_l2c_only) { 4142 ASSERT(!HDR_L2_READING(ab)); 4143 /* 4144 * This doesn't exist in the ARC. Destroy. 4145 * arc_hdr_destroy() will call list_remove() 4146 * and decrement arcstat_l2_size. 4147 */ 4148 arc_change_state(arc_anon, ab, hash_lock); 4149 arc_hdr_destroy(ab); 4150 } else { 4151 /* 4152 * Invalidate issued or about to be issued 4153 * reads, since we may be about to write 4154 * over this location. 4155 */ 4156 if (HDR_L2_READING(ab)) { 4157 ARCSTAT_BUMP(arcstat_l2_evict_reading); 4158 ab->b_flags |= ARC_L2_EVICTED; 4159 } 4160 4161 /* 4162 * Tell ARC this no longer exists in L2ARC. 4163 */ 4164 if (ab->b_l2hdr != NULL) { 4165 abl2 = ab->b_l2hdr; 4166 ab->b_l2hdr = NULL; 4167 kmem_free(abl2, sizeof (l2arc_buf_hdr_t)); 4168 ARCSTAT_INCR(arcstat_l2_size, -ab->b_size); 4169 } 4170 list_remove(buflist, ab); 4171 4172 /* 4173 * This may have been leftover after a 4174 * failed write. 4175 */ 4176 ab->b_flags &= ~ARC_L2_WRITING; 4177 } 4178 mutex_exit(hash_lock); 4179 } 4180 mutex_exit(&l2arc_buflist_mtx); 4181 4182 vdev_space_update(dev->l2ad_vdev, -(taddr - dev->l2ad_evict), 0, 0); 4183 dev->l2ad_evict = taddr; 4184 } 4185 4186 /* 4187 * Find and write ARC buffers to the L2ARC device. 4188 * 4189 * An ARC_L2_WRITING flag is set so that the L2ARC buffers are not valid 4190 * for reading until they have completed writing. 4191 */ 4192 static uint64_t 4193 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz) 4194 { 4195 arc_buf_hdr_t *ab, *ab_prev, *head; 4196 l2arc_buf_hdr_t *hdrl2; 4197 list_t *list; 4198 uint64_t passed_sz, write_sz, buf_sz, headroom; 4199 void *buf_data; 4200 kmutex_t *hash_lock, *list_lock; 4201 boolean_t have_lock, full; 4202 l2arc_write_callback_t *cb; 4203 zio_t *pio, *wzio; 4204 uint64_t guid = spa_guid(spa); 4205 4206 ASSERT(dev->l2ad_vdev != NULL); 4207 4208 pio = NULL; 4209 write_sz = 0; 4210 full = B_FALSE; 4211 head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 4212 head->b_flags |= ARC_L2_WRITE_HEAD; 4213 4214 /* 4215 * Copy buffers for L2ARC writing. 4216 */ 4217 mutex_enter(&l2arc_buflist_mtx); 4218 for (int try = 0; try <= 3; try++) { 4219 list = l2arc_list_locked(try, &list_lock); 4220 passed_sz = 0; 4221 4222 /* 4223 * L2ARC fast warmup. 4224 * 4225 * Until the ARC is warm and starts to evict, read from the 4226 * head of the ARC lists rather than the tail. 4227 */ 4228 headroom = target_sz * l2arc_headroom; 4229 if (arc_warm == B_FALSE) 4230 ab = list_head(list); 4231 else 4232 ab = list_tail(list); 4233 4234 for (; ab; ab = ab_prev) { 4235 if (arc_warm == B_FALSE) 4236 ab_prev = list_next(list, ab); 4237 else 4238 ab_prev = list_prev(list, ab); 4239 4240 hash_lock = HDR_LOCK(ab); 4241 have_lock = MUTEX_HELD(hash_lock); 4242 if (!have_lock && !mutex_tryenter(hash_lock)) { 4243 /* 4244 * Skip this buffer rather than waiting. 4245 */ 4246 continue; 4247 } 4248 4249 passed_sz += ab->b_size; 4250 if (passed_sz > headroom) { 4251 /* 4252 * Searched too far. 4253 */ 4254 mutex_exit(hash_lock); 4255 break; 4256 } 4257 4258 if (!l2arc_write_eligible(guid, ab)) { 4259 mutex_exit(hash_lock); 4260 continue; 4261 } 4262 4263 if ((write_sz + ab->b_size) > target_sz) { 4264 full = B_TRUE; 4265 mutex_exit(hash_lock); 4266 break; 4267 } 4268 4269 if (pio == NULL) { 4270 /* 4271 * Insert a dummy header on the buflist so 4272 * l2arc_write_done() can find where the 4273 * write buffers begin without searching. 4274 */ 4275 list_insert_head(dev->l2ad_buflist, head); 4276 4277 cb = kmem_alloc( 4278 sizeof (l2arc_write_callback_t), KM_SLEEP); 4279 cb->l2wcb_dev = dev; 4280 cb->l2wcb_head = head; 4281 pio = zio_root(spa, l2arc_write_done, cb, 4282 ZIO_FLAG_CANFAIL); 4283 } 4284 4285 /* 4286 * Create and add a new L2ARC header. 4287 */ 4288 hdrl2 = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP); 4289 hdrl2->b_dev = dev; 4290 hdrl2->b_daddr = dev->l2ad_hand; 4291 4292 ab->b_flags |= ARC_L2_WRITING; 4293 ab->b_l2hdr = hdrl2; 4294 list_insert_head(dev->l2ad_buflist, ab); 4295 buf_data = ab->b_buf->b_data; 4296 buf_sz = ab->b_size; 4297 4298 /* 4299 * Compute and store the buffer cksum before 4300 * writing. On debug the cksum is verified first. 4301 */ 4302 arc_cksum_verify(ab->b_buf); 4303 arc_cksum_compute(ab->b_buf, B_TRUE); 4304 4305 mutex_exit(hash_lock); 4306 4307 wzio = zio_write_phys(pio, dev->l2ad_vdev, 4308 dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF, 4309 NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE, 4310 ZIO_FLAG_CANFAIL, B_FALSE); 4311 4312 DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, 4313 zio_t *, wzio); 4314 (void) zio_nowait(wzio); 4315 4316 /* 4317 * Keep the clock hand suitably device-aligned. 4318 */ 4319 buf_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz); 4320 4321 write_sz += buf_sz; 4322 dev->l2ad_hand += buf_sz; 4323 } 4324 4325 mutex_exit(list_lock); 4326 4327 if (full == B_TRUE) 4328 break; 4329 } 4330 mutex_exit(&l2arc_buflist_mtx); 4331 4332 if (pio == NULL) { 4333 ASSERT3U(write_sz, ==, 0); 4334 kmem_cache_free(hdr_cache, head); 4335 return (0); 4336 } 4337 4338 ASSERT3U(write_sz, <=, target_sz); 4339 ARCSTAT_BUMP(arcstat_l2_writes_sent); 4340 ARCSTAT_INCR(arcstat_l2_write_bytes, write_sz); 4341 ARCSTAT_INCR(arcstat_l2_size, write_sz); 4342 vdev_space_update(dev->l2ad_vdev, write_sz, 0, 0); 4343 4344 /* 4345 * Bump device hand to the device start if it is approaching the end. 4346 * l2arc_evict() will already have evicted ahead for this case. 4347 */ 4348 if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) { 4349 vdev_space_update(dev->l2ad_vdev, 4350 dev->l2ad_end - dev->l2ad_hand, 0, 0); 4351 dev->l2ad_hand = dev->l2ad_start; 4352 dev->l2ad_evict = dev->l2ad_start; 4353 dev->l2ad_first = B_FALSE; 4354 } 4355 4356 dev->l2ad_writing = B_TRUE; 4357 (void) zio_wait(pio); 4358 dev->l2ad_writing = B_FALSE; 4359 4360 return (write_sz); 4361 } 4362 4363 /* 4364 * This thread feeds the L2ARC at regular intervals. This is the beating 4365 * heart of the L2ARC. 4366 */ 4367 static void 4368 l2arc_feed_thread(void) 4369 { 4370 callb_cpr_t cpr; 4371 l2arc_dev_t *dev; 4372 spa_t *spa; 4373 uint64_t size, wrote; 4374 clock_t begin, next = ddi_get_lbolt(); 4375 4376 CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG); 4377 4378 mutex_enter(&l2arc_feed_thr_lock); 4379 4380 while (l2arc_thread_exit == 0) { 4381 CALLB_CPR_SAFE_BEGIN(&cpr); 4382 (void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock, 4383 next); 4384 CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock); 4385 next = ddi_get_lbolt() + hz; 4386 4387 /* 4388 * Quick check for L2ARC devices. 4389 */ 4390 mutex_enter(&l2arc_dev_mtx); 4391 if (l2arc_ndev == 0) { 4392 mutex_exit(&l2arc_dev_mtx); 4393 continue; 4394 } 4395 mutex_exit(&l2arc_dev_mtx); 4396 begin = ddi_get_lbolt(); 4397 4398 /* 4399 * This selects the next l2arc device to write to, and in 4400 * doing so the next spa to feed from: dev->l2ad_spa. This 4401 * will return NULL if there are now no l2arc devices or if 4402 * they are all faulted. 4403 * 4404 * If a device is returned, its spa's config lock is also 4405 * held to prevent device removal. l2arc_dev_get_next() 4406 * will grab and release l2arc_dev_mtx. 4407 */ 4408 if ((dev = l2arc_dev_get_next()) == NULL) 4409 continue; 4410 4411 spa = dev->l2ad_spa; 4412 ASSERT(spa != NULL); 4413 4414 /* 4415 * Avoid contributing to memory pressure. 4416 */ 4417 if (arc_reclaim_needed()) { 4418 ARCSTAT_BUMP(arcstat_l2_abort_lowmem); 4419 spa_config_exit(spa, SCL_L2ARC, dev); 4420 continue; 4421 } 4422 4423 ARCSTAT_BUMP(arcstat_l2_feeds); 4424 4425 size = l2arc_write_size(dev); 4426 4427 /* 4428 * Evict L2ARC buffers that will be overwritten. 4429 */ 4430 l2arc_evict(dev, size, B_FALSE); 4431 4432 /* 4433 * Write ARC buffers. 4434 */ 4435 wrote = l2arc_write_buffers(spa, dev, size); 4436 4437 /* 4438 * Calculate interval between writes. 4439 */ 4440 next = l2arc_write_interval(begin, size, wrote); 4441 spa_config_exit(spa, SCL_L2ARC, dev); 4442 } 4443 4444 l2arc_thread_exit = 0; 4445 cv_broadcast(&l2arc_feed_thr_cv); 4446 CALLB_CPR_EXIT(&cpr); /* drops l2arc_feed_thr_lock */ 4447 thread_exit(); 4448 } 4449 4450 boolean_t 4451 l2arc_vdev_present(vdev_t *vd) 4452 { 4453 l2arc_dev_t *dev; 4454 4455 mutex_enter(&l2arc_dev_mtx); 4456 for (dev = list_head(l2arc_dev_list); dev != NULL; 4457 dev = list_next(l2arc_dev_list, dev)) { 4458 if (dev->l2ad_vdev == vd) 4459 break; 4460 } 4461 mutex_exit(&l2arc_dev_mtx); 4462 4463 return (dev != NULL); 4464 } 4465 4466 /* 4467 * Add a vdev for use by the L2ARC. By this point the spa has already 4468 * validated the vdev and opened it. 4469 */ 4470 void 4471 l2arc_add_vdev(spa_t *spa, vdev_t *vd) 4472 { 4473 l2arc_dev_t *adddev; 4474 4475 ASSERT(!l2arc_vdev_present(vd)); 4476 4477 /* 4478 * Create a new l2arc device entry. 4479 */ 4480 adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP); 4481 adddev->l2ad_spa = spa; 4482 adddev->l2ad_vdev = vd; 4483 adddev->l2ad_write = l2arc_write_max; 4484 adddev->l2ad_boost = l2arc_write_boost; 4485 adddev->l2ad_start = VDEV_LABEL_START_SIZE; 4486 adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd); 4487 adddev->l2ad_hand = adddev->l2ad_start; 4488 adddev->l2ad_evict = adddev->l2ad_start; 4489 adddev->l2ad_first = B_TRUE; 4490 adddev->l2ad_writing = B_FALSE; 4491 ASSERT3U(adddev->l2ad_write, >, 0); 4492 4493 /* 4494 * This is a list of all ARC buffers that are still valid on the 4495 * device. 4496 */ 4497 adddev->l2ad_buflist = kmem_zalloc(sizeof (list_t), KM_SLEEP); 4498 list_create(adddev->l2ad_buflist, sizeof (arc_buf_hdr_t), 4499 offsetof(arc_buf_hdr_t, b_l2node)); 4500 4501 vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand); 4502 4503 /* 4504 * Add device to global list 4505 */ 4506 mutex_enter(&l2arc_dev_mtx); 4507 list_insert_head(l2arc_dev_list, adddev); 4508 atomic_inc_64(&l2arc_ndev); 4509 mutex_exit(&l2arc_dev_mtx); 4510 } 4511 4512 /* 4513 * Remove a vdev from the L2ARC. 4514 */ 4515 void 4516 l2arc_remove_vdev(vdev_t *vd) 4517 { 4518 l2arc_dev_t *dev, *nextdev, *remdev = NULL; 4519 4520 /* 4521 * Find the device by vdev 4522 */ 4523 mutex_enter(&l2arc_dev_mtx); 4524 for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) { 4525 nextdev = list_next(l2arc_dev_list, dev); 4526 if (vd == dev->l2ad_vdev) { 4527 remdev = dev; 4528 break; 4529 } 4530 } 4531 ASSERT(remdev != NULL); 4532 4533 /* 4534 * Remove device from global list 4535 */ 4536 list_remove(l2arc_dev_list, remdev); 4537 l2arc_dev_last = NULL; /* may have been invalidated */ 4538 atomic_dec_64(&l2arc_ndev); 4539 mutex_exit(&l2arc_dev_mtx); 4540 4541 /* 4542 * Clear all buflists and ARC references. L2ARC device flush. 4543 */ 4544 l2arc_evict(remdev, 0, B_TRUE); 4545 list_destroy(remdev->l2ad_buflist); 4546 kmem_free(remdev->l2ad_buflist, sizeof (list_t)); 4547 kmem_free(remdev, sizeof (l2arc_dev_t)); 4548 } 4549 4550 void 4551 l2arc_init(void) 4552 { 4553 l2arc_thread_exit = 0; 4554 l2arc_ndev = 0; 4555 l2arc_writes_sent = 0; 4556 l2arc_writes_done = 0; 4557 4558 mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL); 4559 cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL); 4560 mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL); 4561 mutex_init(&l2arc_buflist_mtx, NULL, MUTEX_DEFAULT, NULL); 4562 mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL); 4563 4564 l2arc_dev_list = &L2ARC_dev_list; 4565 l2arc_free_on_write = &L2ARC_free_on_write; 4566 list_create(l2arc_dev_list, sizeof (l2arc_dev_t), 4567 offsetof(l2arc_dev_t, l2ad_node)); 4568 list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t), 4569 offsetof(l2arc_data_free_t, l2df_list_node)); 4570 } 4571 4572 void 4573 l2arc_fini(void) 4574 { 4575 /* 4576 * This is called from dmu_fini(), which is called from spa_fini(); 4577 * Because of this, we can assume that all l2arc devices have 4578 * already been removed when the pools themselves were removed. 4579 */ 4580 4581 l2arc_do_free_on_write(); 4582 4583 mutex_destroy(&l2arc_feed_thr_lock); 4584 cv_destroy(&l2arc_feed_thr_cv); 4585 mutex_destroy(&l2arc_dev_mtx); 4586 mutex_destroy(&l2arc_buflist_mtx); 4587 mutex_destroy(&l2arc_free_on_write_mtx); 4588 4589 list_destroy(l2arc_dev_list); 4590 list_destroy(l2arc_free_on_write); 4591 } 4592 4593 void 4594 l2arc_start(void) 4595 { 4596 if (!(spa_mode_global & FWRITE)) 4597 return; 4598 4599 (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0, 4600 TS_RUN, minclsyspri); 4601 } 4602 4603 void 4604 l2arc_stop(void) 4605 { 4606 if (!(spa_mode_global & FWRITE)) 4607 return; 4608 4609 mutex_enter(&l2arc_feed_thr_lock); 4610 cv_signal(&l2arc_feed_thr_cv); /* kick thread out of startup */ 4611 l2arc_thread_exit = 1; 4612 while (l2arc_thread_exit != 0) 4613 cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock); 4614 mutex_exit(&l2arc_feed_thr_lock); 4615 } 4616