1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * DVA-based Adjustable Replacement Cache 30 * 31 * While much of the theory of operation used here is 32 * based on the self-tuning, low overhead replacement cache 33 * presented by Megiddo and Modha at FAST 2003, there are some 34 * significant differences: 35 * 36 * 1. The Megiddo and Modha model assumes any page is evictable. 37 * Pages in its cache cannot be "locked" into memory. This makes 38 * the eviction algorithm simple: evict the last page in the list. 39 * This also make the performance characteristics easy to reason 40 * about. Our cache is not so simple. At any given moment, some 41 * subset of the blocks in the cache are un-evictable because we 42 * have handed out a reference to them. Blocks are only evictable 43 * when there are no external references active. This makes 44 * eviction far more problematic: we choose to evict the evictable 45 * blocks that are the "lowest" in the list. 46 * 47 * There are times when it is not possible to evict the requested 48 * space. In these circumstances we are unable to adjust the cache 49 * size. To prevent the cache growing unbounded at these times we 50 * implement a "cache throttle" that slowes the flow of new data 51 * into the cache until we can make space avaiable. 52 * 53 * 2. The Megiddo and Modha model assumes a fixed cache size. 54 * Pages are evicted when the cache is full and there is a cache 55 * miss. Our model has a variable sized cache. It grows with 56 * high use, but also tries to react to memory preasure from the 57 * operating system: decreasing its size when system memory is 58 * tight. 59 * 60 * 3. The Megiddo and Modha model assumes a fixed page size. All 61 * elements of the cache are therefor exactly the same size. So 62 * when adjusting the cache size following a cache miss, its simply 63 * a matter of choosing a single page to evict. In our model, we 64 * have variable sized cache blocks (rangeing from 512 bytes to 65 * 128K bytes). We therefor choose a set of blocks to evict to make 66 * space for a cache miss that approximates as closely as possible 67 * the space used by the new block. 68 * 69 * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache" 70 * by N. Megiddo & D. Modha, FAST 2003 71 */ 72 73 /* 74 * The locking model: 75 * 76 * A new reference to a cache buffer can be obtained in two 77 * ways: 1) via a hash table lookup using the DVA as a key, 78 * or 2) via one of the ARC lists. The arc_read() inerface 79 * uses method 1, while the internal arc algorithms for 80 * adjusting the cache use method 2. We therefor provide two 81 * types of locks: 1) the hash table lock array, and 2) the 82 * arc list locks. 83 * 84 * Buffers do not have their own mutexs, rather they rely on the 85 * hash table mutexs for the bulk of their protection (i.e. most 86 * fields in the arc_buf_hdr_t are protected by these mutexs). 87 * 88 * buf_hash_find() returns the appropriate mutex (held) when it 89 * locates the requested buffer in the hash table. It returns 90 * NULL for the mutex if the buffer was not in the table. 91 * 92 * buf_hash_remove() expects the appropriate hash mutex to be 93 * already held before it is invoked. 94 * 95 * Each arc state also has a mutex which is used to protect the 96 * buffer list associated with the state. When attempting to 97 * obtain a hash table lock while holding an arc list lock you 98 * must use: mutex_tryenter() to avoid deadlock. Also note that 99 * the active state mutex must be held before the ghost state mutex. 100 * 101 * Arc buffers may have an associated eviction callback function. 102 * This function will be invoked prior to removing the buffer (e.g. 103 * in arc_do_user_evicts()). Note however that the data associated 104 * with the buffer may be evicted prior to the callback. The callback 105 * must be made with *no locks held* (to prevent deadlock). Additionally, 106 * the users of callbacks must ensure that their private data is 107 * protected from simultaneous callbacks from arc_buf_evict() 108 * and arc_do_user_evicts(). 109 * 110 * Note that the majority of the performance stats are manipulated 111 * with atomic operations. 112 */ 113 114 #include <sys/spa.h> 115 #include <sys/zio.h> 116 #include <sys/zio_checksum.h> 117 #include <sys/zfs_context.h> 118 #include <sys/arc.h> 119 #include <sys/refcount.h> 120 #ifdef _KERNEL 121 #include <sys/vmsystm.h> 122 #include <vm/anon.h> 123 #include <sys/fs/swapnode.h> 124 #include <sys/dnlc.h> 125 #endif 126 #include <sys/callb.h> 127 #include <sys/kstat.h> 128 129 static kmutex_t arc_reclaim_thr_lock; 130 static kcondvar_t arc_reclaim_thr_cv; /* used to signal reclaim thr */ 131 static uint8_t arc_thread_exit; 132 133 #define ARC_REDUCE_DNLC_PERCENT 3 134 uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT; 135 136 typedef enum arc_reclaim_strategy { 137 ARC_RECLAIM_AGGR, /* Aggressive reclaim strategy */ 138 ARC_RECLAIM_CONS /* Conservative reclaim strategy */ 139 } arc_reclaim_strategy_t; 140 141 /* number of seconds before growing cache again */ 142 static int arc_grow_retry = 60; 143 144 /* 145 * minimum lifespan of a prefetch block in clock ticks 146 * (initialized in arc_init()) 147 */ 148 static int arc_min_prefetch_lifespan; 149 150 static int arc_dead; 151 152 /* 153 * These tunables are for performance analysis. 154 */ 155 uint64_t zfs_arc_max; 156 uint64_t zfs_arc_min; 157 158 /* 159 * Note that buffers can be on one of 5 states: 160 * ARC_anon - anonymous (discussed below) 161 * ARC_mru - recently used, currently cached 162 * ARC_mru_ghost - recentely used, no longer in cache 163 * ARC_mfu - frequently used, currently cached 164 * ARC_mfu_ghost - frequently used, no longer in cache 165 * When there are no active references to the buffer, they 166 * are linked onto one of the lists in arc. These are the 167 * only buffers that can be evicted or deleted. 168 * 169 * Anonymous buffers are buffers that are not associated with 170 * a DVA. These are buffers that hold dirty block copies 171 * before they are written to stable storage. By definition, 172 * they are "ref'd" and are considered part of arc_mru 173 * that cannot be freed. Generally, they will aquire a DVA 174 * as they are written and migrate onto the arc_mru list. 175 */ 176 177 typedef struct arc_state { 178 list_t arcs_list; /* linked list of evictable buffer in state */ 179 uint64_t arcs_lsize; /* total size of buffers in the linked list */ 180 uint64_t arcs_size; /* total size of all buffers in this state */ 181 kmutex_t arcs_mtx; 182 } arc_state_t; 183 184 /* The 5 states: */ 185 static arc_state_t ARC_anon; 186 static arc_state_t ARC_mru; 187 static arc_state_t ARC_mru_ghost; 188 static arc_state_t ARC_mfu; 189 static arc_state_t ARC_mfu_ghost; 190 191 typedef struct arc_stats { 192 kstat_named_t arcstat_hits; 193 kstat_named_t arcstat_misses; 194 kstat_named_t arcstat_demand_data_hits; 195 kstat_named_t arcstat_demand_data_misses; 196 kstat_named_t arcstat_demand_metadata_hits; 197 kstat_named_t arcstat_demand_metadata_misses; 198 kstat_named_t arcstat_prefetch_data_hits; 199 kstat_named_t arcstat_prefetch_data_misses; 200 kstat_named_t arcstat_prefetch_metadata_hits; 201 kstat_named_t arcstat_prefetch_metadata_misses; 202 kstat_named_t arcstat_mru_hits; 203 kstat_named_t arcstat_mru_ghost_hits; 204 kstat_named_t arcstat_mfu_hits; 205 kstat_named_t arcstat_mfu_ghost_hits; 206 kstat_named_t arcstat_deleted; 207 kstat_named_t arcstat_recycle_miss; 208 kstat_named_t arcstat_mutex_miss; 209 kstat_named_t arcstat_evict_skip; 210 kstat_named_t arcstat_hash_elements; 211 kstat_named_t arcstat_hash_elements_max; 212 kstat_named_t arcstat_hash_collisions; 213 kstat_named_t arcstat_hash_chains; 214 kstat_named_t arcstat_hash_chain_max; 215 kstat_named_t arcstat_p; 216 kstat_named_t arcstat_c; 217 kstat_named_t arcstat_c_min; 218 kstat_named_t arcstat_c_max; 219 kstat_named_t arcstat_size; 220 } arc_stats_t; 221 222 static arc_stats_t arc_stats = { 223 { "hits", KSTAT_DATA_UINT64 }, 224 { "misses", KSTAT_DATA_UINT64 }, 225 { "demand_data_hits", KSTAT_DATA_UINT64 }, 226 { "demand_data_misses", KSTAT_DATA_UINT64 }, 227 { "demand_metadata_hits", KSTAT_DATA_UINT64 }, 228 { "demand_metadata_misses", KSTAT_DATA_UINT64 }, 229 { "prefetch_data_hits", KSTAT_DATA_UINT64 }, 230 { "prefetch_data_misses", KSTAT_DATA_UINT64 }, 231 { "prefetch_metadata_hits", KSTAT_DATA_UINT64 }, 232 { "prefetch_metadata_misses", KSTAT_DATA_UINT64 }, 233 { "mru_hits", KSTAT_DATA_UINT64 }, 234 { "mru_ghost_hits", KSTAT_DATA_UINT64 }, 235 { "mfu_hits", KSTAT_DATA_UINT64 }, 236 { "mfu_ghost_hits", KSTAT_DATA_UINT64 }, 237 { "deleted", KSTAT_DATA_UINT64 }, 238 { "recycle_miss", KSTAT_DATA_UINT64 }, 239 { "mutex_miss", KSTAT_DATA_UINT64 }, 240 { "evict_skip", KSTAT_DATA_UINT64 }, 241 { "hash_elements", KSTAT_DATA_UINT64 }, 242 { "hash_elements_max", KSTAT_DATA_UINT64 }, 243 { "hash_collisions", KSTAT_DATA_UINT64 }, 244 { "hash_chains", KSTAT_DATA_UINT64 }, 245 { "hash_chain_max", KSTAT_DATA_UINT64 }, 246 { "p", KSTAT_DATA_UINT64 }, 247 { "c", KSTAT_DATA_UINT64 }, 248 { "c_min", KSTAT_DATA_UINT64 }, 249 { "c_max", KSTAT_DATA_UINT64 }, 250 { "size", KSTAT_DATA_UINT64 } 251 }; 252 253 #define ARCSTAT(stat) (arc_stats.stat.value.ui64) 254 255 #define ARCSTAT_INCR(stat, val) \ 256 atomic_add_64(&arc_stats.stat.value.ui64, (val)); 257 258 #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) 259 #define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1) 260 261 #define ARCSTAT_MAX(stat, val) { \ 262 uint64_t m; \ 263 while ((val) > (m = arc_stats.stat.value.ui64) && \ 264 (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \ 265 continue; \ 266 } 267 268 #define ARCSTAT_MAXSTAT(stat) \ 269 ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64) 270 271 /* 272 * We define a macro to allow ARC hits/misses to be easily broken down by 273 * two separate conditions, giving a total of four different subtypes for 274 * each of hits and misses (so eight statistics total). 275 */ 276 #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \ 277 if (cond1) { \ 278 if (cond2) { \ 279 ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \ 280 } else { \ 281 ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \ 282 } \ 283 } else { \ 284 if (cond2) { \ 285 ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \ 286 } else { \ 287 ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\ 288 } \ 289 } 290 291 kstat_t *arc_ksp; 292 static arc_state_t *arc_anon; 293 static arc_state_t *arc_mru; 294 static arc_state_t *arc_mru_ghost; 295 static arc_state_t *arc_mfu; 296 static arc_state_t *arc_mfu_ghost; 297 298 /* 299 * There are several ARC variables that are critical to export as kstats -- 300 * but we don't want to have to grovel around in the kstat whenever we wish to 301 * manipulate them. For these variables, we therefore define them to be in 302 * terms of the statistic variable. This assures that we are not introducing 303 * the possibility of inconsistency by having shadow copies of the variables, 304 * while still allowing the code to be readable. 305 */ 306 #define arc_size ARCSTAT(arcstat_size) /* actual total arc size */ 307 #define arc_p ARCSTAT(arcstat_p) /* target size of MRU */ 308 #define arc_c ARCSTAT(arcstat_c) /* target size of cache */ 309 #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ 310 #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ 311 312 static int arc_no_grow; /* Don't try to grow cache size */ 313 static uint64_t arc_tempreserve; 314 315 typedef struct arc_callback arc_callback_t; 316 317 struct arc_callback { 318 void *acb_private; 319 arc_done_func_t *acb_done; 320 arc_byteswap_func_t *acb_byteswap; 321 arc_buf_t *acb_buf; 322 zio_t *acb_zio_dummy; 323 arc_callback_t *acb_next; 324 }; 325 326 typedef struct arc_write_callback arc_write_callback_t; 327 328 struct arc_write_callback { 329 void *awcb_private; 330 arc_done_func_t *awcb_ready; 331 arc_done_func_t *awcb_done; 332 arc_buf_t *awcb_buf; 333 }; 334 335 struct arc_buf_hdr { 336 /* protected by hash lock */ 337 dva_t b_dva; 338 uint64_t b_birth; 339 uint64_t b_cksum0; 340 341 kmutex_t b_freeze_lock; 342 zio_cksum_t *b_freeze_cksum; 343 344 arc_buf_hdr_t *b_hash_next; 345 arc_buf_t *b_buf; 346 uint32_t b_flags; 347 uint32_t b_datacnt; 348 349 arc_callback_t *b_acb; 350 kcondvar_t b_cv; 351 352 /* immutable */ 353 arc_buf_contents_t b_type; 354 uint64_t b_size; 355 spa_t *b_spa; 356 357 /* protected by arc state mutex */ 358 arc_state_t *b_state; 359 list_node_t b_arc_node; 360 361 /* updated atomically */ 362 clock_t b_arc_access; 363 364 /* self protecting */ 365 refcount_t b_refcnt; 366 }; 367 368 static arc_buf_t *arc_eviction_list; 369 static kmutex_t arc_eviction_mtx; 370 static arc_buf_hdr_t arc_eviction_hdr; 371 static void arc_get_data_buf(arc_buf_t *buf); 372 static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock); 373 374 #define GHOST_STATE(state) \ 375 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost) 376 377 /* 378 * Private ARC flags. These flags are private ARC only flags that will show up 379 * in b_flags in the arc_hdr_buf_t. Some flags are publicly declared, and can 380 * be passed in as arc_flags in things like arc_read. However, these flags 381 * should never be passed and should only be set by ARC code. When adding new 382 * public flags, make sure not to smash the private ones. 383 */ 384 385 #define ARC_IN_HASH_TABLE (1 << 9) /* this buffer is hashed */ 386 #define ARC_IO_IN_PROGRESS (1 << 10) /* I/O in progress for buf */ 387 #define ARC_IO_ERROR (1 << 11) /* I/O failed for buf */ 388 #define ARC_FREED_IN_READ (1 << 12) /* buf freed while in read */ 389 #define ARC_BUF_AVAILABLE (1 << 13) /* block not in active use */ 390 #define ARC_INDIRECT (1 << 14) /* this is an indirect block */ 391 392 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_IN_HASH_TABLE) 393 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS) 394 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_IO_ERROR) 395 #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FREED_IN_READ) 396 #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_BUF_AVAILABLE) 397 398 /* 399 * Hash table routines 400 */ 401 402 #define HT_LOCK_PAD 64 403 404 struct ht_lock { 405 kmutex_t ht_lock; 406 #ifdef _KERNEL 407 unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))]; 408 #endif 409 }; 410 411 #define BUF_LOCKS 256 412 typedef struct buf_hash_table { 413 uint64_t ht_mask; 414 arc_buf_hdr_t **ht_table; 415 struct ht_lock ht_locks[BUF_LOCKS]; 416 } buf_hash_table_t; 417 418 static buf_hash_table_t buf_hash_table; 419 420 #define BUF_HASH_INDEX(spa, dva, birth) \ 421 (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask) 422 #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)]) 423 #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock)) 424 #define HDR_LOCK(buf) \ 425 (BUF_HASH_LOCK(BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth))) 426 427 uint64_t zfs_crc64_table[256]; 428 429 static uint64_t 430 buf_hash(spa_t *spa, dva_t *dva, uint64_t birth) 431 { 432 uintptr_t spav = (uintptr_t)spa; 433 uint8_t *vdva = (uint8_t *)dva; 434 uint64_t crc = -1ULL; 435 int i; 436 437 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 438 439 for (i = 0; i < sizeof (dva_t); i++) 440 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF]; 441 442 crc ^= (spav>>8) ^ birth; 443 444 return (crc); 445 } 446 447 #define BUF_EMPTY(buf) \ 448 ((buf)->b_dva.dva_word[0] == 0 && \ 449 (buf)->b_dva.dva_word[1] == 0 && \ 450 (buf)->b_birth == 0) 451 452 #define BUF_EQUAL(spa, dva, birth, buf) \ 453 ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \ 454 ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \ 455 ((buf)->b_birth == birth) && ((buf)->b_spa == spa) 456 457 static arc_buf_hdr_t * 458 buf_hash_find(spa_t *spa, dva_t *dva, uint64_t birth, kmutex_t **lockp) 459 { 460 uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); 461 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 462 arc_buf_hdr_t *buf; 463 464 mutex_enter(hash_lock); 465 for (buf = buf_hash_table.ht_table[idx]; buf != NULL; 466 buf = buf->b_hash_next) { 467 if (BUF_EQUAL(spa, dva, birth, buf)) { 468 *lockp = hash_lock; 469 return (buf); 470 } 471 } 472 mutex_exit(hash_lock); 473 *lockp = NULL; 474 return (NULL); 475 } 476 477 /* 478 * Insert an entry into the hash table. If there is already an element 479 * equal to elem in the hash table, then the already existing element 480 * will be returned and the new element will not be inserted. 481 * Otherwise returns NULL. 482 */ 483 static arc_buf_hdr_t * 484 buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp) 485 { 486 uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 487 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 488 arc_buf_hdr_t *fbuf; 489 uint32_t i; 490 491 ASSERT(!HDR_IN_HASH_TABLE(buf)); 492 *lockp = hash_lock; 493 mutex_enter(hash_lock); 494 for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL; 495 fbuf = fbuf->b_hash_next, i++) { 496 if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf)) 497 return (fbuf); 498 } 499 500 buf->b_hash_next = buf_hash_table.ht_table[idx]; 501 buf_hash_table.ht_table[idx] = buf; 502 buf->b_flags |= ARC_IN_HASH_TABLE; 503 504 /* collect some hash table performance data */ 505 if (i > 0) { 506 ARCSTAT_BUMP(arcstat_hash_collisions); 507 if (i == 1) 508 ARCSTAT_BUMP(arcstat_hash_chains); 509 510 ARCSTAT_MAX(arcstat_hash_chain_max, i); 511 } 512 513 ARCSTAT_BUMP(arcstat_hash_elements); 514 ARCSTAT_MAXSTAT(arcstat_hash_elements); 515 516 return (NULL); 517 } 518 519 static void 520 buf_hash_remove(arc_buf_hdr_t *buf) 521 { 522 arc_buf_hdr_t *fbuf, **bufp; 523 uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 524 525 ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx))); 526 ASSERT(HDR_IN_HASH_TABLE(buf)); 527 528 bufp = &buf_hash_table.ht_table[idx]; 529 while ((fbuf = *bufp) != buf) { 530 ASSERT(fbuf != NULL); 531 bufp = &fbuf->b_hash_next; 532 } 533 *bufp = buf->b_hash_next; 534 buf->b_hash_next = NULL; 535 buf->b_flags &= ~ARC_IN_HASH_TABLE; 536 537 /* collect some hash table performance data */ 538 ARCSTAT_BUMPDOWN(arcstat_hash_elements); 539 540 if (buf_hash_table.ht_table[idx] && 541 buf_hash_table.ht_table[idx]->b_hash_next == NULL) 542 ARCSTAT_BUMPDOWN(arcstat_hash_chains); 543 } 544 545 /* 546 * Global data structures and functions for the buf kmem cache. 547 */ 548 static kmem_cache_t *hdr_cache; 549 static kmem_cache_t *buf_cache; 550 551 static void 552 buf_fini(void) 553 { 554 int i; 555 556 kmem_free(buf_hash_table.ht_table, 557 (buf_hash_table.ht_mask + 1) * sizeof (void *)); 558 for (i = 0; i < BUF_LOCKS; i++) 559 mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock); 560 kmem_cache_destroy(hdr_cache); 561 kmem_cache_destroy(buf_cache); 562 } 563 564 /* 565 * Constructor callback - called when the cache is empty 566 * and a new buf is requested. 567 */ 568 /* ARGSUSED */ 569 static int 570 hdr_cons(void *vbuf, void *unused, int kmflag) 571 { 572 arc_buf_hdr_t *buf = vbuf; 573 574 bzero(buf, sizeof (arc_buf_hdr_t)); 575 refcount_create(&buf->b_refcnt); 576 cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL); 577 return (0); 578 } 579 580 /* 581 * Destructor callback - called when a cached buf is 582 * no longer required. 583 */ 584 /* ARGSUSED */ 585 static void 586 hdr_dest(void *vbuf, void *unused) 587 { 588 arc_buf_hdr_t *buf = vbuf; 589 590 refcount_destroy(&buf->b_refcnt); 591 cv_destroy(&buf->b_cv); 592 } 593 594 /* 595 * Reclaim callback -- invoked when memory is low. 596 */ 597 /* ARGSUSED */ 598 static void 599 hdr_recl(void *unused) 600 { 601 dprintf("hdr_recl called\n"); 602 /* 603 * umem calls the reclaim func when we destroy the buf cache, 604 * which is after we do arc_fini(). 605 */ 606 if (!arc_dead) 607 cv_signal(&arc_reclaim_thr_cv); 608 } 609 610 static void 611 buf_init(void) 612 { 613 uint64_t *ct; 614 uint64_t hsize = 1ULL << 12; 615 int i, j; 616 617 /* 618 * The hash table is big enough to fill all of physical memory 619 * with an average 64K block size. The table will take up 620 * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers). 621 */ 622 while (hsize * 65536 < physmem * PAGESIZE) 623 hsize <<= 1; 624 retry: 625 buf_hash_table.ht_mask = hsize - 1; 626 buf_hash_table.ht_table = 627 kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP); 628 if (buf_hash_table.ht_table == NULL) { 629 ASSERT(hsize > (1ULL << 8)); 630 hsize >>= 1; 631 goto retry; 632 } 633 634 hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t), 635 0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0); 636 buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t), 637 0, NULL, NULL, NULL, NULL, NULL, 0); 638 639 for (i = 0; i < 256; i++) 640 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--) 641 *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY); 642 643 for (i = 0; i < BUF_LOCKS; i++) { 644 mutex_init(&buf_hash_table.ht_locks[i].ht_lock, 645 NULL, MUTEX_DEFAULT, NULL); 646 } 647 } 648 649 #define ARC_MINTIME (hz>>4) /* 62 ms */ 650 651 static void 652 arc_cksum_verify(arc_buf_t *buf) 653 { 654 zio_cksum_t zc; 655 656 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 657 return; 658 659 mutex_enter(&buf->b_hdr->b_freeze_lock); 660 if (buf->b_hdr->b_freeze_cksum == NULL || 661 (buf->b_hdr->b_flags & ARC_IO_ERROR)) { 662 mutex_exit(&buf->b_hdr->b_freeze_lock); 663 return; 664 } 665 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 666 if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc)) 667 panic("buffer modified while frozen!"); 668 mutex_exit(&buf->b_hdr->b_freeze_lock); 669 } 670 671 static void 672 arc_cksum_compute(arc_buf_t *buf) 673 { 674 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 675 return; 676 677 mutex_enter(&buf->b_hdr->b_freeze_lock); 678 if (buf->b_hdr->b_freeze_cksum != NULL) { 679 mutex_exit(&buf->b_hdr->b_freeze_lock); 680 return; 681 } 682 buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP); 683 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, 684 buf->b_hdr->b_freeze_cksum); 685 mutex_exit(&buf->b_hdr->b_freeze_lock); 686 } 687 688 void 689 arc_buf_thaw(arc_buf_t *buf) 690 { 691 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 692 return; 693 694 if (buf->b_hdr->b_state != arc_anon) 695 panic("modifying non-anon buffer!"); 696 if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS) 697 panic("modifying buffer while i/o in progress!"); 698 arc_cksum_verify(buf); 699 mutex_enter(&buf->b_hdr->b_freeze_lock); 700 if (buf->b_hdr->b_freeze_cksum != NULL) { 701 kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 702 buf->b_hdr->b_freeze_cksum = NULL; 703 } 704 mutex_exit(&buf->b_hdr->b_freeze_lock); 705 } 706 707 void 708 arc_buf_freeze(arc_buf_t *buf) 709 { 710 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 711 return; 712 713 ASSERT(buf->b_hdr->b_freeze_cksum != NULL || 714 buf->b_hdr->b_state == arc_anon); 715 arc_cksum_compute(buf); 716 } 717 718 static void 719 add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 720 { 721 ASSERT(MUTEX_HELD(hash_lock)); 722 723 if ((refcount_add(&ab->b_refcnt, tag) == 1) && 724 (ab->b_state != arc_anon)) { 725 int delta = ab->b_size * ab->b_datacnt; 726 727 ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx)); 728 mutex_enter(&ab->b_state->arcs_mtx); 729 ASSERT(list_link_active(&ab->b_arc_node)); 730 list_remove(&ab->b_state->arcs_list, ab); 731 if (GHOST_STATE(ab->b_state)) { 732 ASSERT3U(ab->b_datacnt, ==, 0); 733 ASSERT3P(ab->b_buf, ==, NULL); 734 delta = ab->b_size; 735 } 736 ASSERT(delta > 0); 737 ASSERT3U(ab->b_state->arcs_lsize, >=, delta); 738 atomic_add_64(&ab->b_state->arcs_lsize, -delta); 739 mutex_exit(&ab->b_state->arcs_mtx); 740 /* remove the prefetch flag is we get a reference */ 741 if (ab->b_flags & ARC_PREFETCH) 742 ab->b_flags &= ~ARC_PREFETCH; 743 } 744 } 745 746 static int 747 remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 748 { 749 int cnt; 750 arc_state_t *state = ab->b_state; 751 752 ASSERT(state == arc_anon || MUTEX_HELD(hash_lock)); 753 ASSERT(!GHOST_STATE(state)); 754 755 if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) && 756 (state != arc_anon)) { 757 ASSERT(!MUTEX_HELD(&state->arcs_mtx)); 758 mutex_enter(&state->arcs_mtx); 759 ASSERT(!list_link_active(&ab->b_arc_node)); 760 list_insert_head(&state->arcs_list, ab); 761 ASSERT(ab->b_datacnt > 0); 762 atomic_add_64(&state->arcs_lsize, ab->b_size * ab->b_datacnt); 763 ASSERT3U(state->arcs_size, >=, state->arcs_lsize); 764 mutex_exit(&state->arcs_mtx); 765 } 766 return (cnt); 767 } 768 769 /* 770 * Move the supplied buffer to the indicated state. The mutex 771 * for the buffer must be held by the caller. 772 */ 773 static void 774 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock) 775 { 776 arc_state_t *old_state = ab->b_state; 777 int refcnt = refcount_count(&ab->b_refcnt); 778 int from_delta, to_delta; 779 780 ASSERT(MUTEX_HELD(hash_lock)); 781 ASSERT(new_state != old_state); 782 ASSERT(refcnt == 0 || ab->b_datacnt > 0); 783 ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state)); 784 785 from_delta = to_delta = ab->b_datacnt * ab->b_size; 786 787 /* 788 * If this buffer is evictable, transfer it from the 789 * old state list to the new state list. 790 */ 791 if (refcnt == 0) { 792 if (old_state != arc_anon) { 793 int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx); 794 795 if (use_mutex) 796 mutex_enter(&old_state->arcs_mtx); 797 798 ASSERT(list_link_active(&ab->b_arc_node)); 799 list_remove(&old_state->arcs_list, ab); 800 801 /* 802 * If prefetching out of the ghost cache, 803 * we will have a non-null datacnt. 804 */ 805 if (GHOST_STATE(old_state) && ab->b_datacnt == 0) { 806 /* ghost elements have a ghost size */ 807 ASSERT(ab->b_buf == NULL); 808 from_delta = ab->b_size; 809 } 810 ASSERT3U(old_state->arcs_lsize, >=, from_delta); 811 atomic_add_64(&old_state->arcs_lsize, -from_delta); 812 813 if (use_mutex) 814 mutex_exit(&old_state->arcs_mtx); 815 } 816 if (new_state != arc_anon) { 817 int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx); 818 819 if (use_mutex) 820 mutex_enter(&new_state->arcs_mtx); 821 822 list_insert_head(&new_state->arcs_list, ab); 823 824 /* ghost elements have a ghost size */ 825 if (GHOST_STATE(new_state)) { 826 ASSERT(ab->b_datacnt == 0); 827 ASSERT(ab->b_buf == NULL); 828 to_delta = ab->b_size; 829 } 830 atomic_add_64(&new_state->arcs_lsize, to_delta); 831 ASSERT3U(new_state->arcs_size + to_delta, >=, 832 new_state->arcs_lsize); 833 834 if (use_mutex) 835 mutex_exit(&new_state->arcs_mtx); 836 } 837 } 838 839 ASSERT(!BUF_EMPTY(ab)); 840 if (new_state == arc_anon && old_state != arc_anon) { 841 buf_hash_remove(ab); 842 } 843 844 /* adjust state sizes */ 845 if (to_delta) 846 atomic_add_64(&new_state->arcs_size, to_delta); 847 if (from_delta) { 848 ASSERT3U(old_state->arcs_size, >=, from_delta); 849 atomic_add_64(&old_state->arcs_size, -from_delta); 850 } 851 ab->b_state = new_state; 852 } 853 854 arc_buf_t * 855 arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type) 856 { 857 arc_buf_hdr_t *hdr; 858 arc_buf_t *buf; 859 860 ASSERT3U(size, >, 0); 861 hdr = kmem_cache_alloc(hdr_cache, KM_SLEEP); 862 ASSERT(BUF_EMPTY(hdr)); 863 hdr->b_size = size; 864 hdr->b_type = type; 865 hdr->b_spa = spa; 866 hdr->b_state = arc_anon; 867 hdr->b_arc_access = 0; 868 buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 869 buf->b_hdr = hdr; 870 buf->b_data = NULL; 871 buf->b_efunc = NULL; 872 buf->b_private = NULL; 873 buf->b_next = NULL; 874 hdr->b_buf = buf; 875 arc_get_data_buf(buf); 876 hdr->b_datacnt = 1; 877 hdr->b_flags = 0; 878 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 879 (void) refcount_add(&hdr->b_refcnt, tag); 880 881 return (buf); 882 } 883 884 static arc_buf_t * 885 arc_buf_clone(arc_buf_t *from) 886 { 887 arc_buf_t *buf; 888 arc_buf_hdr_t *hdr = from->b_hdr; 889 uint64_t size = hdr->b_size; 890 891 buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 892 buf->b_hdr = hdr; 893 buf->b_data = NULL; 894 buf->b_efunc = NULL; 895 buf->b_private = NULL; 896 buf->b_next = hdr->b_buf; 897 hdr->b_buf = buf; 898 arc_get_data_buf(buf); 899 bcopy(from->b_data, buf->b_data, size); 900 hdr->b_datacnt += 1; 901 return (buf); 902 } 903 904 void 905 arc_buf_add_ref(arc_buf_t *buf, void* tag) 906 { 907 arc_buf_hdr_t *hdr; 908 kmutex_t *hash_lock; 909 910 /* 911 * Check to see if this buffer is currently being evicted via 912 * arc_do_user_evicts(). 913 */ 914 mutex_enter(&arc_eviction_mtx); 915 hdr = buf->b_hdr; 916 if (hdr == NULL) { 917 mutex_exit(&arc_eviction_mtx); 918 return; 919 } 920 hash_lock = HDR_LOCK(hdr); 921 mutex_exit(&arc_eviction_mtx); 922 923 mutex_enter(hash_lock); 924 if (buf->b_data == NULL) { 925 /* 926 * This buffer is evicted. 927 */ 928 mutex_exit(hash_lock); 929 return; 930 } 931 932 ASSERT(buf->b_hdr == hdr); 933 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 934 add_reference(hdr, hash_lock, tag); 935 arc_access(hdr, hash_lock); 936 mutex_exit(hash_lock); 937 ARCSTAT_BUMP(arcstat_hits); 938 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 939 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 940 data, metadata, hits); 941 } 942 943 static void 944 arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all) 945 { 946 arc_buf_t **bufp; 947 948 /* free up data associated with the buf */ 949 if (buf->b_data) { 950 arc_state_t *state = buf->b_hdr->b_state; 951 uint64_t size = buf->b_hdr->b_size; 952 arc_buf_contents_t type = buf->b_hdr->b_type; 953 954 arc_cksum_verify(buf); 955 if (!recycle) { 956 if (type == ARC_BUFC_METADATA) { 957 zio_buf_free(buf->b_data, size); 958 } else { 959 ASSERT(type == ARC_BUFC_DATA); 960 zio_data_buf_free(buf->b_data, size); 961 } 962 atomic_add_64(&arc_size, -size); 963 } 964 if (list_link_active(&buf->b_hdr->b_arc_node)) { 965 ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt)); 966 ASSERT(state != arc_anon); 967 ASSERT3U(state->arcs_lsize, >=, size); 968 atomic_add_64(&state->arcs_lsize, -size); 969 } 970 ASSERT3U(state->arcs_size, >=, size); 971 atomic_add_64(&state->arcs_size, -size); 972 buf->b_data = NULL; 973 ASSERT(buf->b_hdr->b_datacnt > 0); 974 buf->b_hdr->b_datacnt -= 1; 975 } 976 977 /* only remove the buf if requested */ 978 if (!all) 979 return; 980 981 /* remove the buf from the hdr list */ 982 for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next) 983 continue; 984 *bufp = buf->b_next; 985 986 ASSERT(buf->b_efunc == NULL); 987 988 /* clean up the buf */ 989 buf->b_hdr = NULL; 990 kmem_cache_free(buf_cache, buf); 991 } 992 993 static void 994 arc_hdr_destroy(arc_buf_hdr_t *hdr) 995 { 996 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 997 ASSERT3P(hdr->b_state, ==, arc_anon); 998 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 999 1000 if (!BUF_EMPTY(hdr)) { 1001 ASSERT(!HDR_IN_HASH_TABLE(hdr)); 1002 bzero(&hdr->b_dva, sizeof (dva_t)); 1003 hdr->b_birth = 0; 1004 hdr->b_cksum0 = 0; 1005 } 1006 while (hdr->b_buf) { 1007 arc_buf_t *buf = hdr->b_buf; 1008 1009 if (buf->b_efunc) { 1010 mutex_enter(&arc_eviction_mtx); 1011 ASSERT(buf->b_hdr != NULL); 1012 arc_buf_destroy(hdr->b_buf, FALSE, FALSE); 1013 hdr->b_buf = buf->b_next; 1014 buf->b_hdr = &arc_eviction_hdr; 1015 buf->b_next = arc_eviction_list; 1016 arc_eviction_list = buf; 1017 mutex_exit(&arc_eviction_mtx); 1018 } else { 1019 arc_buf_destroy(hdr->b_buf, FALSE, TRUE); 1020 } 1021 } 1022 if (hdr->b_freeze_cksum != NULL) { 1023 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 1024 hdr->b_freeze_cksum = NULL; 1025 } 1026 1027 ASSERT(!list_link_active(&hdr->b_arc_node)); 1028 ASSERT3P(hdr->b_hash_next, ==, NULL); 1029 ASSERT3P(hdr->b_acb, ==, NULL); 1030 kmem_cache_free(hdr_cache, hdr); 1031 } 1032 1033 void 1034 arc_buf_free(arc_buf_t *buf, void *tag) 1035 { 1036 arc_buf_hdr_t *hdr = buf->b_hdr; 1037 int hashed = hdr->b_state != arc_anon; 1038 1039 ASSERT(buf->b_efunc == NULL); 1040 ASSERT(buf->b_data != NULL); 1041 1042 if (hashed) { 1043 kmutex_t *hash_lock = HDR_LOCK(hdr); 1044 1045 mutex_enter(hash_lock); 1046 (void) remove_reference(hdr, hash_lock, tag); 1047 if (hdr->b_datacnt > 1) 1048 arc_buf_destroy(buf, FALSE, TRUE); 1049 else 1050 hdr->b_flags |= ARC_BUF_AVAILABLE; 1051 mutex_exit(hash_lock); 1052 } else if (HDR_IO_IN_PROGRESS(hdr)) { 1053 int destroy_hdr; 1054 /* 1055 * We are in the middle of an async write. Don't destroy 1056 * this buffer unless the write completes before we finish 1057 * decrementing the reference count. 1058 */ 1059 mutex_enter(&arc_eviction_mtx); 1060 (void) remove_reference(hdr, NULL, tag); 1061 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1062 destroy_hdr = !HDR_IO_IN_PROGRESS(hdr); 1063 mutex_exit(&arc_eviction_mtx); 1064 if (destroy_hdr) 1065 arc_hdr_destroy(hdr); 1066 } else { 1067 if (remove_reference(hdr, NULL, tag) > 0) { 1068 ASSERT(HDR_IO_ERROR(hdr)); 1069 arc_buf_destroy(buf, FALSE, TRUE); 1070 } else { 1071 arc_hdr_destroy(hdr); 1072 } 1073 } 1074 } 1075 1076 int 1077 arc_buf_remove_ref(arc_buf_t *buf, void* tag) 1078 { 1079 arc_buf_hdr_t *hdr = buf->b_hdr; 1080 kmutex_t *hash_lock = HDR_LOCK(hdr); 1081 int no_callback = (buf->b_efunc == NULL); 1082 1083 if (hdr->b_state == arc_anon) { 1084 arc_buf_free(buf, tag); 1085 return (no_callback); 1086 } 1087 1088 mutex_enter(hash_lock); 1089 ASSERT(hdr->b_state != arc_anon); 1090 ASSERT(buf->b_data != NULL); 1091 1092 (void) remove_reference(hdr, hash_lock, tag); 1093 if (hdr->b_datacnt > 1) { 1094 if (no_callback) 1095 arc_buf_destroy(buf, FALSE, TRUE); 1096 } else if (no_callback) { 1097 ASSERT(hdr->b_buf == buf && buf->b_next == NULL); 1098 hdr->b_flags |= ARC_BUF_AVAILABLE; 1099 } 1100 ASSERT(no_callback || hdr->b_datacnt > 1 || 1101 refcount_is_zero(&hdr->b_refcnt)); 1102 mutex_exit(hash_lock); 1103 return (no_callback); 1104 } 1105 1106 int 1107 arc_buf_size(arc_buf_t *buf) 1108 { 1109 return (buf->b_hdr->b_size); 1110 } 1111 1112 /* 1113 * Evict buffers from list until we've removed the specified number of 1114 * bytes. Move the removed buffers to the appropriate evict state. 1115 * If the recycle flag is set, then attempt to "recycle" a buffer: 1116 * - look for a buffer to evict that is `bytes' long. 1117 * - return the data block from this buffer rather than freeing it. 1118 * This flag is used by callers that are trying to make space for a 1119 * new buffer in a full arc cache. 1120 */ 1121 static void * 1122 arc_evict(arc_state_t *state, int64_t bytes, boolean_t recycle, 1123 arc_buf_contents_t type) 1124 { 1125 arc_state_t *evicted_state; 1126 uint64_t bytes_evicted = 0, skipped = 0, missed = 0; 1127 arc_buf_hdr_t *ab, *ab_prev = NULL; 1128 kmutex_t *hash_lock; 1129 boolean_t have_lock; 1130 void *stolen = NULL; 1131 1132 ASSERT(state == arc_mru || state == arc_mfu); 1133 1134 evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 1135 1136 mutex_enter(&state->arcs_mtx); 1137 mutex_enter(&evicted_state->arcs_mtx); 1138 1139 for (ab = list_tail(&state->arcs_list); ab; ab = ab_prev) { 1140 ab_prev = list_prev(&state->arcs_list, ab); 1141 /* prefetch buffers have a minimum lifespan */ 1142 if (HDR_IO_IN_PROGRESS(ab) || 1143 (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) && 1144 lbolt - ab->b_arc_access < arc_min_prefetch_lifespan)) { 1145 skipped++; 1146 continue; 1147 } 1148 /* "lookahead" for better eviction candidate */ 1149 if (recycle && ab->b_size != bytes && 1150 ab_prev && ab_prev->b_size == bytes) 1151 continue; 1152 hash_lock = HDR_LOCK(ab); 1153 have_lock = MUTEX_HELD(hash_lock); 1154 if (have_lock || mutex_tryenter(hash_lock)) { 1155 ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0); 1156 ASSERT(ab->b_datacnt > 0); 1157 while (ab->b_buf) { 1158 arc_buf_t *buf = ab->b_buf; 1159 if (buf->b_data) { 1160 bytes_evicted += ab->b_size; 1161 if (recycle && ab->b_type == type && 1162 ab->b_size == bytes) { 1163 stolen = buf->b_data; 1164 recycle = FALSE; 1165 } 1166 } 1167 if (buf->b_efunc) { 1168 mutex_enter(&arc_eviction_mtx); 1169 arc_buf_destroy(buf, 1170 buf->b_data == stolen, FALSE); 1171 ab->b_buf = buf->b_next; 1172 buf->b_hdr = &arc_eviction_hdr; 1173 buf->b_next = arc_eviction_list; 1174 arc_eviction_list = buf; 1175 mutex_exit(&arc_eviction_mtx); 1176 } else { 1177 arc_buf_destroy(buf, 1178 buf->b_data == stolen, TRUE); 1179 } 1180 } 1181 ASSERT(ab->b_datacnt == 0); 1182 arc_change_state(evicted_state, ab, hash_lock); 1183 ASSERT(HDR_IN_HASH_TABLE(ab)); 1184 ab->b_flags = ARC_IN_HASH_TABLE; 1185 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab); 1186 if (!have_lock) 1187 mutex_exit(hash_lock); 1188 if (bytes >= 0 && bytes_evicted >= bytes) 1189 break; 1190 } else { 1191 missed += 1; 1192 } 1193 } 1194 1195 mutex_exit(&evicted_state->arcs_mtx); 1196 mutex_exit(&state->arcs_mtx); 1197 1198 if (bytes_evicted < bytes) 1199 dprintf("only evicted %lld bytes from %x", 1200 (longlong_t)bytes_evicted, state); 1201 1202 if (skipped) 1203 ARCSTAT_INCR(arcstat_evict_skip, skipped); 1204 1205 if (missed) 1206 ARCSTAT_INCR(arcstat_mutex_miss, missed); 1207 1208 return (stolen); 1209 } 1210 1211 /* 1212 * Remove buffers from list until we've removed the specified number of 1213 * bytes. Destroy the buffers that are removed. 1214 */ 1215 static void 1216 arc_evict_ghost(arc_state_t *state, int64_t bytes) 1217 { 1218 arc_buf_hdr_t *ab, *ab_prev; 1219 kmutex_t *hash_lock; 1220 uint64_t bytes_deleted = 0; 1221 uint_t bufs_skipped = 0; 1222 1223 ASSERT(GHOST_STATE(state)); 1224 top: 1225 mutex_enter(&state->arcs_mtx); 1226 for (ab = list_tail(&state->arcs_list); ab; ab = ab_prev) { 1227 ab_prev = list_prev(&state->arcs_list, ab); 1228 hash_lock = HDR_LOCK(ab); 1229 if (mutex_tryenter(hash_lock)) { 1230 ASSERT(!HDR_IO_IN_PROGRESS(ab)); 1231 ASSERT(ab->b_buf == NULL); 1232 arc_change_state(arc_anon, ab, hash_lock); 1233 mutex_exit(hash_lock); 1234 ARCSTAT_BUMP(arcstat_deleted); 1235 bytes_deleted += ab->b_size; 1236 arc_hdr_destroy(ab); 1237 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab); 1238 if (bytes >= 0 && bytes_deleted >= bytes) 1239 break; 1240 } else { 1241 if (bytes < 0) { 1242 mutex_exit(&state->arcs_mtx); 1243 mutex_enter(hash_lock); 1244 mutex_exit(hash_lock); 1245 goto top; 1246 } 1247 bufs_skipped += 1; 1248 } 1249 } 1250 mutex_exit(&state->arcs_mtx); 1251 1252 if (bufs_skipped) { 1253 ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped); 1254 ASSERT(bytes >= 0); 1255 } 1256 1257 if (bytes_deleted < bytes) 1258 dprintf("only deleted %lld bytes from %p", 1259 (longlong_t)bytes_deleted, state); 1260 } 1261 1262 static void 1263 arc_adjust(void) 1264 { 1265 int64_t top_sz, mru_over, arc_over, todelete; 1266 1267 top_sz = arc_anon->arcs_size + arc_mru->arcs_size; 1268 1269 if (top_sz > arc_p && arc_mru->arcs_lsize > 0) { 1270 int64_t toevict = MIN(arc_mru->arcs_lsize, top_sz - arc_p); 1271 (void) arc_evict(arc_mru, toevict, FALSE, ARC_BUFC_UNDEF); 1272 top_sz = arc_anon->arcs_size + arc_mru->arcs_size; 1273 } 1274 1275 mru_over = top_sz + arc_mru_ghost->arcs_size - arc_c; 1276 1277 if (mru_over > 0) { 1278 if (arc_mru_ghost->arcs_lsize > 0) { 1279 todelete = MIN(arc_mru_ghost->arcs_lsize, mru_over); 1280 arc_evict_ghost(arc_mru_ghost, todelete); 1281 } 1282 } 1283 1284 if ((arc_over = arc_size - arc_c) > 0) { 1285 int64_t tbl_over; 1286 1287 if (arc_mfu->arcs_lsize > 0) { 1288 int64_t toevict = MIN(arc_mfu->arcs_lsize, arc_over); 1289 (void) arc_evict(arc_mfu, toevict, FALSE, 1290 ARC_BUFC_UNDEF); 1291 } 1292 1293 tbl_over = arc_size + arc_mru_ghost->arcs_lsize + 1294 arc_mfu_ghost->arcs_lsize - arc_c*2; 1295 1296 if (tbl_over > 0 && arc_mfu_ghost->arcs_lsize > 0) { 1297 todelete = MIN(arc_mfu_ghost->arcs_lsize, tbl_over); 1298 arc_evict_ghost(arc_mfu_ghost, todelete); 1299 } 1300 } 1301 } 1302 1303 static void 1304 arc_do_user_evicts(void) 1305 { 1306 mutex_enter(&arc_eviction_mtx); 1307 while (arc_eviction_list != NULL) { 1308 arc_buf_t *buf = arc_eviction_list; 1309 arc_eviction_list = buf->b_next; 1310 buf->b_hdr = NULL; 1311 mutex_exit(&arc_eviction_mtx); 1312 1313 if (buf->b_efunc != NULL) 1314 VERIFY(buf->b_efunc(buf) == 0); 1315 1316 buf->b_efunc = NULL; 1317 buf->b_private = NULL; 1318 kmem_cache_free(buf_cache, buf); 1319 mutex_enter(&arc_eviction_mtx); 1320 } 1321 mutex_exit(&arc_eviction_mtx); 1322 } 1323 1324 /* 1325 * Flush all *evictable* data from the cache. 1326 * NOTE: this will not touch "active" (i.e. referenced) data. 1327 */ 1328 void 1329 arc_flush(void) 1330 { 1331 while (list_head(&arc_mru->arcs_list)) 1332 (void) arc_evict(arc_mru, -1, FALSE, ARC_BUFC_UNDEF); 1333 while (list_head(&arc_mfu->arcs_list)) 1334 (void) arc_evict(arc_mfu, -1, FALSE, ARC_BUFC_UNDEF); 1335 1336 arc_evict_ghost(arc_mru_ghost, -1); 1337 arc_evict_ghost(arc_mfu_ghost, -1); 1338 1339 mutex_enter(&arc_reclaim_thr_lock); 1340 arc_do_user_evicts(); 1341 mutex_exit(&arc_reclaim_thr_lock); 1342 ASSERT(arc_eviction_list == NULL); 1343 } 1344 1345 int arc_shrink_shift = 5; /* log2(fraction of arc to reclaim) */ 1346 1347 void 1348 arc_shrink(void) 1349 { 1350 if (arc_c > arc_c_min) { 1351 uint64_t to_free; 1352 1353 #ifdef _KERNEL 1354 to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree)); 1355 #else 1356 to_free = arc_c >> arc_shrink_shift; 1357 #endif 1358 if (arc_c > arc_c_min + to_free) 1359 atomic_add_64(&arc_c, -to_free); 1360 else 1361 arc_c = arc_c_min; 1362 1363 atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift)); 1364 if (arc_c > arc_size) 1365 arc_c = MAX(arc_size, arc_c_min); 1366 if (arc_p > arc_c) 1367 arc_p = (arc_c >> 1); 1368 ASSERT(arc_c >= arc_c_min); 1369 ASSERT((int64_t)arc_p >= 0); 1370 } 1371 1372 if (arc_size > arc_c) 1373 arc_adjust(); 1374 } 1375 1376 static int 1377 arc_reclaim_needed(void) 1378 { 1379 uint64_t extra; 1380 1381 #ifdef _KERNEL 1382 1383 if (needfree) 1384 return (1); 1385 1386 /* 1387 * take 'desfree' extra pages, so we reclaim sooner, rather than later 1388 */ 1389 extra = desfree; 1390 1391 /* 1392 * check that we're out of range of the pageout scanner. It starts to 1393 * schedule paging if freemem is less than lotsfree and needfree. 1394 * lotsfree is the high-water mark for pageout, and needfree is the 1395 * number of needed free pages. We add extra pages here to make sure 1396 * the scanner doesn't start up while we're freeing memory. 1397 */ 1398 if (freemem < lotsfree + needfree + extra) 1399 return (1); 1400 1401 /* 1402 * check to make sure that swapfs has enough space so that anon 1403 * reservations can still succeeed. anon_resvmem() checks that the 1404 * availrmem is greater than swapfs_minfree, and the number of reserved 1405 * swap pages. We also add a bit of extra here just to prevent 1406 * circumstances from getting really dire. 1407 */ 1408 if (availrmem < swapfs_minfree + swapfs_reserve + extra) 1409 return (1); 1410 1411 /* 1412 * If zio data pages are being allocated out of a separate heap segment, 1413 * then check that the size of available vmem for this area remains 1414 * above 1/4th free. This needs to be done since the size of the 1415 * non-default segment is smaller than physical memory, so we could 1416 * conceivably run out of VA in that segment before running out of 1417 * physical memory. 1418 */ 1419 if ((zio_arena != NULL) && (btop(vmem_size(zio_arena, VMEM_FREE)) < 1420 (btop(vmem_size(zio_arena, VMEM_FREE | VMEM_ALLOC)) >> 2))) 1421 return (1); 1422 1423 #if defined(__i386) 1424 /* 1425 * If we're on an i386 platform, it's possible that we'll exhaust the 1426 * kernel heap space before we ever run out of available physical 1427 * memory. Most checks of the size of the heap_area compare against 1428 * tune.t_minarmem, which is the minimum available real memory that we 1429 * can have in the system. However, this is generally fixed at 25 pages 1430 * which is so low that it's useless. In this comparison, we seek to 1431 * calculate the total heap-size, and reclaim if more than 3/4ths of the 1432 * heap is allocated. (Or, in the caclulation, if less than 1/4th is 1433 * free) 1434 */ 1435 if (btop(vmem_size(heap_arena, VMEM_FREE)) < 1436 (btop(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2)) 1437 return (1); 1438 #endif 1439 1440 #else 1441 if (spa_get_random(100) == 0) 1442 return (1); 1443 #endif 1444 return (0); 1445 } 1446 1447 static void 1448 arc_kmem_reap_now(arc_reclaim_strategy_t strat) 1449 { 1450 size_t i; 1451 kmem_cache_t *prev_cache = NULL; 1452 kmem_cache_t *prev_data_cache = NULL; 1453 extern kmem_cache_t *zio_buf_cache[]; 1454 extern kmem_cache_t *zio_data_buf_cache[]; 1455 1456 #ifdef _KERNEL 1457 /* 1458 * First purge some DNLC entries, in case the DNLC is using 1459 * up too much memory. 1460 */ 1461 dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); 1462 1463 #if defined(__i386) 1464 /* 1465 * Reclaim unused memory from all kmem caches. 1466 */ 1467 kmem_reap(); 1468 #endif 1469 #endif 1470 1471 /* 1472 * An agressive reclamation will shrink the cache size as well as 1473 * reap free buffers from the arc kmem caches. 1474 */ 1475 if (strat == ARC_RECLAIM_AGGR) 1476 arc_shrink(); 1477 1478 for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { 1479 if (zio_buf_cache[i] != prev_cache) { 1480 prev_cache = zio_buf_cache[i]; 1481 kmem_cache_reap_now(zio_buf_cache[i]); 1482 } 1483 if (zio_data_buf_cache[i] != prev_data_cache) { 1484 prev_data_cache = zio_data_buf_cache[i]; 1485 kmem_cache_reap_now(zio_data_buf_cache[i]); 1486 } 1487 } 1488 kmem_cache_reap_now(buf_cache); 1489 kmem_cache_reap_now(hdr_cache); 1490 } 1491 1492 static void 1493 arc_reclaim_thread(void) 1494 { 1495 clock_t growtime = 0; 1496 arc_reclaim_strategy_t last_reclaim = ARC_RECLAIM_CONS; 1497 callb_cpr_t cpr; 1498 1499 CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG); 1500 1501 mutex_enter(&arc_reclaim_thr_lock); 1502 while (arc_thread_exit == 0) { 1503 if (arc_reclaim_needed()) { 1504 1505 if (arc_no_grow) { 1506 if (last_reclaim == ARC_RECLAIM_CONS) { 1507 last_reclaim = ARC_RECLAIM_AGGR; 1508 } else { 1509 last_reclaim = ARC_RECLAIM_CONS; 1510 } 1511 } else { 1512 arc_no_grow = TRUE; 1513 last_reclaim = ARC_RECLAIM_AGGR; 1514 membar_producer(); 1515 } 1516 1517 /* reset the growth delay for every reclaim */ 1518 growtime = lbolt + (arc_grow_retry * hz); 1519 ASSERT(growtime > 0); 1520 1521 arc_kmem_reap_now(last_reclaim); 1522 1523 } else if ((growtime > 0) && ((growtime - lbolt) <= 0)) { 1524 arc_no_grow = FALSE; 1525 } 1526 1527 if (2 * arc_c < arc_size + 1528 arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size) 1529 arc_adjust(); 1530 1531 if (arc_eviction_list != NULL) 1532 arc_do_user_evicts(); 1533 1534 /* block until needed, or one second, whichever is shorter */ 1535 CALLB_CPR_SAFE_BEGIN(&cpr); 1536 (void) cv_timedwait(&arc_reclaim_thr_cv, 1537 &arc_reclaim_thr_lock, (lbolt + hz)); 1538 CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock); 1539 } 1540 1541 arc_thread_exit = 0; 1542 cv_broadcast(&arc_reclaim_thr_cv); 1543 CALLB_CPR_EXIT(&cpr); /* drops arc_reclaim_thr_lock */ 1544 thread_exit(); 1545 } 1546 1547 /* 1548 * Adapt arc info given the number of bytes we are trying to add and 1549 * the state that we are comming from. This function is only called 1550 * when we are adding new content to the cache. 1551 */ 1552 static void 1553 arc_adapt(int bytes, arc_state_t *state) 1554 { 1555 int mult; 1556 1557 ASSERT(bytes > 0); 1558 /* 1559 * Adapt the target size of the MRU list: 1560 * - if we just hit in the MRU ghost list, then increase 1561 * the target size of the MRU list. 1562 * - if we just hit in the MFU ghost list, then increase 1563 * the target size of the MFU list by decreasing the 1564 * target size of the MRU list. 1565 */ 1566 if (state == arc_mru_ghost) { 1567 mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ? 1568 1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size)); 1569 1570 arc_p = MIN(arc_c, arc_p + bytes * mult); 1571 } else if (state == arc_mfu_ghost) { 1572 mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ? 1573 1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size)); 1574 1575 arc_p = MAX(0, (int64_t)arc_p - bytes * mult); 1576 } 1577 ASSERT((int64_t)arc_p >= 0); 1578 1579 if (arc_reclaim_needed()) { 1580 cv_signal(&arc_reclaim_thr_cv); 1581 return; 1582 } 1583 1584 if (arc_no_grow) 1585 return; 1586 1587 if (arc_c >= arc_c_max) 1588 return; 1589 1590 /* 1591 * If we're within (2 * maxblocksize) bytes of the target 1592 * cache size, increment the target cache size 1593 */ 1594 if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) { 1595 atomic_add_64(&arc_c, (int64_t)bytes); 1596 if (arc_c > arc_c_max) 1597 arc_c = arc_c_max; 1598 else if (state == arc_anon) 1599 atomic_add_64(&arc_p, (int64_t)bytes); 1600 if (arc_p > arc_c) 1601 arc_p = arc_c; 1602 } 1603 ASSERT((int64_t)arc_p >= 0); 1604 } 1605 1606 /* 1607 * Check if the cache has reached its limits and eviction is required 1608 * prior to insert. 1609 */ 1610 static int 1611 arc_evict_needed() 1612 { 1613 if (arc_reclaim_needed()) 1614 return (1); 1615 1616 return (arc_size > arc_c); 1617 } 1618 1619 /* 1620 * The buffer, supplied as the first argument, needs a data block. 1621 * So, if we are at cache max, determine which cache should be victimized. 1622 * We have the following cases: 1623 * 1624 * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) -> 1625 * In this situation if we're out of space, but the resident size of the MFU is 1626 * under the limit, victimize the MFU cache to satisfy this insertion request. 1627 * 1628 * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) -> 1629 * Here, we've used up all of the available space for the MRU, so we need to 1630 * evict from our own cache instead. Evict from the set of resident MRU 1631 * entries. 1632 * 1633 * 3. Insert for MFU (c - p) > sizeof(arc_mfu) -> 1634 * c minus p represents the MFU space in the cache, since p is the size of the 1635 * cache that is dedicated to the MRU. In this situation there's still space on 1636 * the MFU side, so the MRU side needs to be victimized. 1637 * 1638 * 4. Insert for MFU (c - p) < sizeof(arc_mfu) -> 1639 * MFU's resident set is consuming more space than it has been allotted. In 1640 * this situation, we must victimize our own cache, the MFU, for this insertion. 1641 */ 1642 static void 1643 arc_get_data_buf(arc_buf_t *buf) 1644 { 1645 arc_state_t *state = buf->b_hdr->b_state; 1646 uint64_t size = buf->b_hdr->b_size; 1647 arc_buf_contents_t type = buf->b_hdr->b_type; 1648 1649 arc_adapt(size, state); 1650 1651 /* 1652 * We have not yet reached cache maximum size, 1653 * just allocate a new buffer. 1654 */ 1655 if (!arc_evict_needed()) { 1656 if (type == ARC_BUFC_METADATA) { 1657 buf->b_data = zio_buf_alloc(size); 1658 } else { 1659 ASSERT(type == ARC_BUFC_DATA); 1660 buf->b_data = zio_data_buf_alloc(size); 1661 } 1662 atomic_add_64(&arc_size, size); 1663 goto out; 1664 } 1665 1666 /* 1667 * If we are prefetching from the mfu ghost list, this buffer 1668 * will end up on the mru list; so steal space from there. 1669 */ 1670 if (state == arc_mfu_ghost) 1671 state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu; 1672 else if (state == arc_mru_ghost) 1673 state = arc_mru; 1674 1675 if (state == arc_mru || state == arc_anon) { 1676 uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size; 1677 state = (arc_p > mru_used) ? arc_mfu : arc_mru; 1678 } else { 1679 /* MFU cases */ 1680 uint64_t mfu_space = arc_c - arc_p; 1681 state = (mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu; 1682 } 1683 if ((buf->b_data = arc_evict(state, size, TRUE, type)) == NULL) { 1684 if (type == ARC_BUFC_METADATA) { 1685 buf->b_data = zio_buf_alloc(size); 1686 } else { 1687 ASSERT(type == ARC_BUFC_DATA); 1688 buf->b_data = zio_data_buf_alloc(size); 1689 } 1690 atomic_add_64(&arc_size, size); 1691 ARCSTAT_BUMP(arcstat_recycle_miss); 1692 } 1693 ASSERT(buf->b_data != NULL); 1694 out: 1695 /* 1696 * Update the state size. Note that ghost states have a 1697 * "ghost size" and so don't need to be updated. 1698 */ 1699 if (!GHOST_STATE(buf->b_hdr->b_state)) { 1700 arc_buf_hdr_t *hdr = buf->b_hdr; 1701 1702 atomic_add_64(&hdr->b_state->arcs_size, size); 1703 if (list_link_active(&hdr->b_arc_node)) { 1704 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1705 atomic_add_64(&hdr->b_state->arcs_lsize, size); 1706 } 1707 /* 1708 * If we are growing the cache, and we are adding anonymous 1709 * data, and we have outgrown arc_p, update arc_p 1710 */ 1711 if (arc_size < arc_c && hdr->b_state == arc_anon && 1712 arc_anon->arcs_size + arc_mru->arcs_size > arc_p) 1713 arc_p = MIN(arc_c, arc_p + size); 1714 } 1715 } 1716 1717 /* 1718 * This routine is called whenever a buffer is accessed. 1719 * NOTE: the hash lock is dropped in this function. 1720 */ 1721 static void 1722 arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock) 1723 { 1724 ASSERT(MUTEX_HELD(hash_lock)); 1725 1726 if (buf->b_state == arc_anon) { 1727 /* 1728 * This buffer is not in the cache, and does not 1729 * appear in our "ghost" list. Add the new buffer 1730 * to the MRU state. 1731 */ 1732 1733 ASSERT(buf->b_arc_access == 0); 1734 buf->b_arc_access = lbolt; 1735 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 1736 arc_change_state(arc_mru, buf, hash_lock); 1737 1738 } else if (buf->b_state == arc_mru) { 1739 /* 1740 * If this buffer is here because of a prefetch, then either: 1741 * - clear the flag if this is a "referencing" read 1742 * (any subsequent access will bump this into the MFU state). 1743 * or 1744 * - move the buffer to the head of the list if this is 1745 * another prefetch (to make it less likely to be evicted). 1746 */ 1747 if ((buf->b_flags & ARC_PREFETCH) != 0) { 1748 if (refcount_count(&buf->b_refcnt) == 0) { 1749 ASSERT(list_link_active(&buf->b_arc_node)); 1750 mutex_enter(&arc_mru->arcs_mtx); 1751 list_remove(&arc_mru->arcs_list, buf); 1752 list_insert_head(&arc_mru->arcs_list, buf); 1753 mutex_exit(&arc_mru->arcs_mtx); 1754 } else { 1755 buf->b_flags &= ~ARC_PREFETCH; 1756 ARCSTAT_BUMP(arcstat_mru_hits); 1757 } 1758 buf->b_arc_access = lbolt; 1759 return; 1760 } 1761 1762 /* 1763 * This buffer has been "accessed" only once so far, 1764 * but it is still in the cache. Move it to the MFU 1765 * state. 1766 */ 1767 if (lbolt > buf->b_arc_access + ARC_MINTIME) { 1768 /* 1769 * More than 125ms have passed since we 1770 * instantiated this buffer. Move it to the 1771 * most frequently used state. 1772 */ 1773 buf->b_arc_access = lbolt; 1774 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 1775 arc_change_state(arc_mfu, buf, hash_lock); 1776 } 1777 ARCSTAT_BUMP(arcstat_mru_hits); 1778 } else if (buf->b_state == arc_mru_ghost) { 1779 arc_state_t *new_state; 1780 /* 1781 * This buffer has been "accessed" recently, but 1782 * was evicted from the cache. Move it to the 1783 * MFU state. 1784 */ 1785 1786 if (buf->b_flags & ARC_PREFETCH) { 1787 new_state = arc_mru; 1788 if (refcount_count(&buf->b_refcnt) > 0) 1789 buf->b_flags &= ~ARC_PREFETCH; 1790 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 1791 } else { 1792 new_state = arc_mfu; 1793 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 1794 } 1795 1796 buf->b_arc_access = lbolt; 1797 arc_change_state(new_state, buf, hash_lock); 1798 1799 ARCSTAT_BUMP(arcstat_mru_ghost_hits); 1800 } else if (buf->b_state == arc_mfu) { 1801 /* 1802 * This buffer has been accessed more than once and is 1803 * still in the cache. Keep it in the MFU state. 1804 * 1805 * NOTE: an add_reference() that occurred when we did 1806 * the arc_read() will have kicked this off the list. 1807 * If it was a prefetch, we will explicitly move it to 1808 * the head of the list now. 1809 */ 1810 if ((buf->b_flags & ARC_PREFETCH) != 0) { 1811 ASSERT(refcount_count(&buf->b_refcnt) == 0); 1812 ASSERT(list_link_active(&buf->b_arc_node)); 1813 mutex_enter(&arc_mfu->arcs_mtx); 1814 list_remove(&arc_mfu->arcs_list, buf); 1815 list_insert_head(&arc_mfu->arcs_list, buf); 1816 mutex_exit(&arc_mfu->arcs_mtx); 1817 } 1818 ARCSTAT_BUMP(arcstat_mfu_hits); 1819 buf->b_arc_access = lbolt; 1820 } else if (buf->b_state == arc_mfu_ghost) { 1821 arc_state_t *new_state = arc_mfu; 1822 /* 1823 * This buffer has been accessed more than once but has 1824 * been evicted from the cache. Move it back to the 1825 * MFU state. 1826 */ 1827 1828 if (buf->b_flags & ARC_PREFETCH) { 1829 /* 1830 * This is a prefetch access... 1831 * move this block back to the MRU state. 1832 */ 1833 ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0); 1834 new_state = arc_mru; 1835 } 1836 1837 buf->b_arc_access = lbolt; 1838 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 1839 arc_change_state(new_state, buf, hash_lock); 1840 1841 ARCSTAT_BUMP(arcstat_mfu_ghost_hits); 1842 } else { 1843 ASSERT(!"invalid arc state"); 1844 } 1845 } 1846 1847 /* a generic arc_done_func_t which you can use */ 1848 /* ARGSUSED */ 1849 void 1850 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) 1851 { 1852 bcopy(buf->b_data, arg, buf->b_hdr->b_size); 1853 VERIFY(arc_buf_remove_ref(buf, arg) == 1); 1854 } 1855 1856 /* a generic arc_done_func_t which you can use */ 1857 void 1858 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) 1859 { 1860 arc_buf_t **bufp = arg; 1861 if (zio && zio->io_error) { 1862 VERIFY(arc_buf_remove_ref(buf, arg) == 1); 1863 *bufp = NULL; 1864 } else { 1865 *bufp = buf; 1866 } 1867 } 1868 1869 static void 1870 arc_read_done(zio_t *zio) 1871 { 1872 arc_buf_hdr_t *hdr, *found; 1873 arc_buf_t *buf; 1874 arc_buf_t *abuf; /* buffer we're assigning to callback */ 1875 kmutex_t *hash_lock; 1876 arc_callback_t *callback_list, *acb; 1877 int freeable = FALSE; 1878 1879 buf = zio->io_private; 1880 hdr = buf->b_hdr; 1881 1882 /* 1883 * The hdr was inserted into hash-table and removed from lists 1884 * prior to starting I/O. We should find this header, since 1885 * it's in the hash table, and it should be legit since it's 1886 * not possible to evict it during the I/O. The only possible 1887 * reason for it not to be found is if we were freed during the 1888 * read. 1889 */ 1890 found = buf_hash_find(zio->io_spa, &hdr->b_dva, hdr->b_birth, 1891 &hash_lock); 1892 1893 ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) || 1894 (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp)))); 1895 1896 /* byteswap if necessary */ 1897 callback_list = hdr->b_acb; 1898 ASSERT(callback_list != NULL); 1899 if (BP_SHOULD_BYTESWAP(zio->io_bp) && callback_list->acb_byteswap) 1900 callback_list->acb_byteswap(buf->b_data, hdr->b_size); 1901 1902 arc_cksum_compute(buf); 1903 1904 /* create copies of the data buffer for the callers */ 1905 abuf = buf; 1906 for (acb = callback_list; acb; acb = acb->acb_next) { 1907 if (acb->acb_done) { 1908 if (abuf == NULL) 1909 abuf = arc_buf_clone(buf); 1910 acb->acb_buf = abuf; 1911 abuf = NULL; 1912 } 1913 } 1914 hdr->b_acb = NULL; 1915 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 1916 ASSERT(!HDR_BUF_AVAILABLE(hdr)); 1917 if (abuf == buf) 1918 hdr->b_flags |= ARC_BUF_AVAILABLE; 1919 1920 ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL); 1921 1922 if (zio->io_error != 0) { 1923 hdr->b_flags |= ARC_IO_ERROR; 1924 if (hdr->b_state != arc_anon) 1925 arc_change_state(arc_anon, hdr, hash_lock); 1926 if (HDR_IN_HASH_TABLE(hdr)) 1927 buf_hash_remove(hdr); 1928 freeable = refcount_is_zero(&hdr->b_refcnt); 1929 /* convert checksum errors into IO errors */ 1930 if (zio->io_error == ECKSUM) 1931 zio->io_error = EIO; 1932 } 1933 1934 /* 1935 * Broadcast before we drop the hash_lock to avoid the possibility 1936 * that the hdr (and hence the cv) might be freed before we get to 1937 * the cv_broadcast(). 1938 */ 1939 cv_broadcast(&hdr->b_cv); 1940 1941 if (hash_lock) { 1942 /* 1943 * Only call arc_access on anonymous buffers. This is because 1944 * if we've issued an I/O for an evicted buffer, we've already 1945 * called arc_access (to prevent any simultaneous readers from 1946 * getting confused). 1947 */ 1948 if (zio->io_error == 0 && hdr->b_state == arc_anon) 1949 arc_access(hdr, hash_lock); 1950 mutex_exit(hash_lock); 1951 } else { 1952 /* 1953 * This block was freed while we waited for the read to 1954 * complete. It has been removed from the hash table and 1955 * moved to the anonymous state (so that it won't show up 1956 * in the cache). 1957 */ 1958 ASSERT3P(hdr->b_state, ==, arc_anon); 1959 freeable = refcount_is_zero(&hdr->b_refcnt); 1960 } 1961 1962 /* execute each callback and free its structure */ 1963 while ((acb = callback_list) != NULL) { 1964 if (acb->acb_done) 1965 acb->acb_done(zio, acb->acb_buf, acb->acb_private); 1966 1967 if (acb->acb_zio_dummy != NULL) { 1968 acb->acb_zio_dummy->io_error = zio->io_error; 1969 zio_nowait(acb->acb_zio_dummy); 1970 } 1971 1972 callback_list = acb->acb_next; 1973 kmem_free(acb, sizeof (arc_callback_t)); 1974 } 1975 1976 if (freeable) 1977 arc_hdr_destroy(hdr); 1978 } 1979 1980 /* 1981 * "Read" the block block at the specified DVA (in bp) via the 1982 * cache. If the block is found in the cache, invoke the provided 1983 * callback immediately and return. Note that the `zio' parameter 1984 * in the callback will be NULL in this case, since no IO was 1985 * required. If the block is not in the cache pass the read request 1986 * on to the spa with a substitute callback function, so that the 1987 * requested block will be added to the cache. 1988 * 1989 * If a read request arrives for a block that has a read in-progress, 1990 * either wait for the in-progress read to complete (and return the 1991 * results); or, if this is a read with a "done" func, add a record 1992 * to the read to invoke the "done" func when the read completes, 1993 * and return; or just return. 1994 * 1995 * arc_read_done() will invoke all the requested "done" functions 1996 * for readers of this block. 1997 */ 1998 int 1999 arc_read(zio_t *pio, spa_t *spa, blkptr_t *bp, arc_byteswap_func_t *swap, 2000 arc_done_func_t *done, void *private, int priority, int flags, 2001 uint32_t *arc_flags, zbookmark_t *zb) 2002 { 2003 arc_buf_hdr_t *hdr; 2004 arc_buf_t *buf; 2005 kmutex_t *hash_lock; 2006 zio_t *rzio; 2007 2008 top: 2009 hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock); 2010 if (hdr && hdr->b_datacnt > 0) { 2011 2012 *arc_flags |= ARC_CACHED; 2013 2014 if (HDR_IO_IN_PROGRESS(hdr)) { 2015 2016 if (*arc_flags & ARC_WAIT) { 2017 cv_wait(&hdr->b_cv, hash_lock); 2018 mutex_exit(hash_lock); 2019 goto top; 2020 } 2021 ASSERT(*arc_flags & ARC_NOWAIT); 2022 2023 if (done) { 2024 arc_callback_t *acb = NULL; 2025 2026 acb = kmem_zalloc(sizeof (arc_callback_t), 2027 KM_SLEEP); 2028 acb->acb_done = done; 2029 acb->acb_private = private; 2030 acb->acb_byteswap = swap; 2031 if (pio != NULL) 2032 acb->acb_zio_dummy = zio_null(pio, 2033 spa, NULL, NULL, flags); 2034 2035 ASSERT(acb->acb_done != NULL); 2036 acb->acb_next = hdr->b_acb; 2037 hdr->b_acb = acb; 2038 add_reference(hdr, hash_lock, private); 2039 mutex_exit(hash_lock); 2040 return (0); 2041 } 2042 mutex_exit(hash_lock); 2043 return (0); 2044 } 2045 2046 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2047 2048 if (done) { 2049 add_reference(hdr, hash_lock, private); 2050 /* 2051 * If this block is already in use, create a new 2052 * copy of the data so that we will be guaranteed 2053 * that arc_release() will always succeed. 2054 */ 2055 buf = hdr->b_buf; 2056 ASSERT(buf); 2057 ASSERT(buf->b_data); 2058 if (HDR_BUF_AVAILABLE(hdr)) { 2059 ASSERT(buf->b_efunc == NULL); 2060 hdr->b_flags &= ~ARC_BUF_AVAILABLE; 2061 } else { 2062 buf = arc_buf_clone(buf); 2063 } 2064 } else if (*arc_flags & ARC_PREFETCH && 2065 refcount_count(&hdr->b_refcnt) == 0) { 2066 hdr->b_flags |= ARC_PREFETCH; 2067 } 2068 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 2069 arc_access(hdr, hash_lock); 2070 mutex_exit(hash_lock); 2071 ARCSTAT_BUMP(arcstat_hits); 2072 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 2073 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 2074 data, metadata, hits); 2075 2076 if (done) 2077 done(NULL, buf, private); 2078 } else { 2079 uint64_t size = BP_GET_LSIZE(bp); 2080 arc_callback_t *acb; 2081 2082 if (hdr == NULL) { 2083 /* this block is not in the cache */ 2084 arc_buf_hdr_t *exists; 2085 arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp); 2086 buf = arc_buf_alloc(spa, size, private, type); 2087 hdr = buf->b_hdr; 2088 hdr->b_dva = *BP_IDENTITY(bp); 2089 hdr->b_birth = bp->blk_birth; 2090 hdr->b_cksum0 = bp->blk_cksum.zc_word[0]; 2091 exists = buf_hash_insert(hdr, &hash_lock); 2092 if (exists) { 2093 /* somebody beat us to the hash insert */ 2094 mutex_exit(hash_lock); 2095 bzero(&hdr->b_dva, sizeof (dva_t)); 2096 hdr->b_birth = 0; 2097 hdr->b_cksum0 = 0; 2098 (void) arc_buf_remove_ref(buf, private); 2099 goto top; /* restart the IO request */ 2100 } 2101 /* if this is a prefetch, we don't have a reference */ 2102 if (*arc_flags & ARC_PREFETCH) { 2103 (void) remove_reference(hdr, hash_lock, 2104 private); 2105 hdr->b_flags |= ARC_PREFETCH; 2106 } 2107 if (BP_GET_LEVEL(bp) > 0) 2108 hdr->b_flags |= ARC_INDIRECT; 2109 } else { 2110 /* this block is in the ghost cache */ 2111 ASSERT(GHOST_STATE(hdr->b_state)); 2112 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 2113 ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0); 2114 ASSERT(hdr->b_buf == NULL); 2115 2116 /* if this is a prefetch, we don't have a reference */ 2117 if (*arc_flags & ARC_PREFETCH) 2118 hdr->b_flags |= ARC_PREFETCH; 2119 else 2120 add_reference(hdr, hash_lock, private); 2121 buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 2122 buf->b_hdr = hdr; 2123 buf->b_data = NULL; 2124 buf->b_efunc = NULL; 2125 buf->b_private = NULL; 2126 buf->b_next = NULL; 2127 hdr->b_buf = buf; 2128 arc_get_data_buf(buf); 2129 ASSERT(hdr->b_datacnt == 0); 2130 hdr->b_datacnt = 1; 2131 2132 } 2133 2134 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 2135 acb->acb_done = done; 2136 acb->acb_private = private; 2137 acb->acb_byteswap = swap; 2138 2139 ASSERT(hdr->b_acb == NULL); 2140 hdr->b_acb = acb; 2141 hdr->b_flags |= ARC_IO_IN_PROGRESS; 2142 2143 /* 2144 * If the buffer has been evicted, migrate it to a present state 2145 * before issuing the I/O. Once we drop the hash-table lock, 2146 * the header will be marked as I/O in progress and have an 2147 * attached buffer. At this point, anybody who finds this 2148 * buffer ought to notice that it's legit but has a pending I/O. 2149 */ 2150 2151 if (GHOST_STATE(hdr->b_state)) 2152 arc_access(hdr, hash_lock); 2153 mutex_exit(hash_lock); 2154 2155 ASSERT3U(hdr->b_size, ==, size); 2156 DTRACE_PROBE3(arc__miss, blkptr_t *, bp, uint64_t, size, 2157 zbookmark_t *, zb); 2158 ARCSTAT_BUMP(arcstat_misses); 2159 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 2160 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 2161 data, metadata, misses); 2162 2163 rzio = zio_read(pio, spa, bp, buf->b_data, size, 2164 arc_read_done, buf, priority, flags, zb); 2165 2166 if (*arc_flags & ARC_WAIT) 2167 return (zio_wait(rzio)); 2168 2169 ASSERT(*arc_flags & ARC_NOWAIT); 2170 zio_nowait(rzio); 2171 } 2172 return (0); 2173 } 2174 2175 /* 2176 * arc_read() variant to support pool traversal. If the block is already 2177 * in the ARC, make a copy of it; otherwise, the caller will do the I/O. 2178 * The idea is that we don't want pool traversal filling up memory, but 2179 * if the ARC already has the data anyway, we shouldn't pay for the I/O. 2180 */ 2181 int 2182 arc_tryread(spa_t *spa, blkptr_t *bp, void *data) 2183 { 2184 arc_buf_hdr_t *hdr; 2185 kmutex_t *hash_mtx; 2186 int rc = 0; 2187 2188 hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_mtx); 2189 2190 if (hdr && hdr->b_datacnt > 0 && !HDR_IO_IN_PROGRESS(hdr)) { 2191 arc_buf_t *buf = hdr->b_buf; 2192 2193 ASSERT(buf); 2194 while (buf->b_data == NULL) { 2195 buf = buf->b_next; 2196 ASSERT(buf); 2197 } 2198 bcopy(buf->b_data, data, hdr->b_size); 2199 } else { 2200 rc = ENOENT; 2201 } 2202 2203 if (hash_mtx) 2204 mutex_exit(hash_mtx); 2205 2206 return (rc); 2207 } 2208 2209 void 2210 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private) 2211 { 2212 ASSERT(buf->b_hdr != NULL); 2213 ASSERT(buf->b_hdr->b_state != arc_anon); 2214 ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL); 2215 buf->b_efunc = func; 2216 buf->b_private = private; 2217 } 2218 2219 /* 2220 * This is used by the DMU to let the ARC know that a buffer is 2221 * being evicted, so the ARC should clean up. If this arc buf 2222 * is not yet in the evicted state, it will be put there. 2223 */ 2224 int 2225 arc_buf_evict(arc_buf_t *buf) 2226 { 2227 arc_buf_hdr_t *hdr; 2228 kmutex_t *hash_lock; 2229 arc_buf_t **bufp; 2230 2231 mutex_enter(&arc_eviction_mtx); 2232 hdr = buf->b_hdr; 2233 if (hdr == NULL) { 2234 /* 2235 * We are in arc_do_user_evicts(). 2236 */ 2237 ASSERT(buf->b_data == NULL); 2238 mutex_exit(&arc_eviction_mtx); 2239 return (0); 2240 } 2241 hash_lock = HDR_LOCK(hdr); 2242 mutex_exit(&arc_eviction_mtx); 2243 2244 mutex_enter(hash_lock); 2245 2246 if (buf->b_data == NULL) { 2247 /* 2248 * We are on the eviction list. 2249 */ 2250 mutex_exit(hash_lock); 2251 mutex_enter(&arc_eviction_mtx); 2252 if (buf->b_hdr == NULL) { 2253 /* 2254 * We are already in arc_do_user_evicts(). 2255 */ 2256 mutex_exit(&arc_eviction_mtx); 2257 return (0); 2258 } else { 2259 arc_buf_t copy = *buf; /* structure assignment */ 2260 /* 2261 * Process this buffer now 2262 * but let arc_do_user_evicts() do the reaping. 2263 */ 2264 buf->b_efunc = NULL; 2265 mutex_exit(&arc_eviction_mtx); 2266 VERIFY(copy.b_efunc(©) == 0); 2267 return (1); 2268 } 2269 } 2270 2271 ASSERT(buf->b_hdr == hdr); 2272 ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt); 2273 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2274 2275 /* 2276 * Pull this buffer off of the hdr 2277 */ 2278 bufp = &hdr->b_buf; 2279 while (*bufp != buf) 2280 bufp = &(*bufp)->b_next; 2281 *bufp = buf->b_next; 2282 2283 ASSERT(buf->b_data != NULL); 2284 arc_buf_destroy(buf, FALSE, FALSE); 2285 2286 if (hdr->b_datacnt == 0) { 2287 arc_state_t *old_state = hdr->b_state; 2288 arc_state_t *evicted_state; 2289 2290 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 2291 2292 evicted_state = 2293 (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 2294 2295 mutex_enter(&old_state->arcs_mtx); 2296 mutex_enter(&evicted_state->arcs_mtx); 2297 2298 arc_change_state(evicted_state, hdr, hash_lock); 2299 ASSERT(HDR_IN_HASH_TABLE(hdr)); 2300 hdr->b_flags = ARC_IN_HASH_TABLE; 2301 2302 mutex_exit(&evicted_state->arcs_mtx); 2303 mutex_exit(&old_state->arcs_mtx); 2304 } 2305 mutex_exit(hash_lock); 2306 2307 VERIFY(buf->b_efunc(buf) == 0); 2308 buf->b_efunc = NULL; 2309 buf->b_private = NULL; 2310 buf->b_hdr = NULL; 2311 kmem_cache_free(buf_cache, buf); 2312 return (1); 2313 } 2314 2315 /* 2316 * Release this buffer from the cache. This must be done 2317 * after a read and prior to modifying the buffer contents. 2318 * If the buffer has more than one reference, we must make 2319 * make a new hdr for the buffer. 2320 */ 2321 void 2322 arc_release(arc_buf_t *buf, void *tag) 2323 { 2324 arc_buf_hdr_t *hdr = buf->b_hdr; 2325 kmutex_t *hash_lock = HDR_LOCK(hdr); 2326 2327 /* this buffer is not on any list */ 2328 ASSERT(refcount_count(&hdr->b_refcnt) > 0); 2329 2330 if (hdr->b_state == arc_anon) { 2331 /* this buffer is already released */ 2332 ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 1); 2333 ASSERT(BUF_EMPTY(hdr)); 2334 ASSERT(buf->b_efunc == NULL); 2335 arc_buf_thaw(buf); 2336 return; 2337 } 2338 2339 mutex_enter(hash_lock); 2340 2341 /* 2342 * Do we have more than one buf? 2343 */ 2344 if (hdr->b_buf != buf || buf->b_next != NULL) { 2345 arc_buf_hdr_t *nhdr; 2346 arc_buf_t **bufp; 2347 uint64_t blksz = hdr->b_size; 2348 spa_t *spa = hdr->b_spa; 2349 arc_buf_contents_t type = hdr->b_type; 2350 2351 ASSERT(hdr->b_datacnt > 1); 2352 /* 2353 * Pull the data off of this buf and attach it to 2354 * a new anonymous buf. 2355 */ 2356 (void) remove_reference(hdr, hash_lock, tag); 2357 bufp = &hdr->b_buf; 2358 while (*bufp != buf) 2359 bufp = &(*bufp)->b_next; 2360 *bufp = (*bufp)->b_next; 2361 2362 ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size); 2363 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size); 2364 if (refcount_is_zero(&hdr->b_refcnt)) { 2365 ASSERT3U(hdr->b_state->arcs_lsize, >=, hdr->b_size); 2366 atomic_add_64(&hdr->b_state->arcs_lsize, -hdr->b_size); 2367 } 2368 hdr->b_datacnt -= 1; 2369 arc_cksum_verify(buf); 2370 2371 mutex_exit(hash_lock); 2372 2373 nhdr = kmem_cache_alloc(hdr_cache, KM_SLEEP); 2374 nhdr->b_size = blksz; 2375 nhdr->b_spa = spa; 2376 nhdr->b_type = type; 2377 nhdr->b_buf = buf; 2378 nhdr->b_state = arc_anon; 2379 nhdr->b_arc_access = 0; 2380 nhdr->b_flags = 0; 2381 nhdr->b_datacnt = 1; 2382 nhdr->b_freeze_cksum = NULL; 2383 buf->b_hdr = nhdr; 2384 buf->b_next = NULL; 2385 (void) refcount_add(&nhdr->b_refcnt, tag); 2386 atomic_add_64(&arc_anon->arcs_size, blksz); 2387 2388 hdr = nhdr; 2389 } else { 2390 ASSERT(refcount_count(&hdr->b_refcnt) == 1); 2391 ASSERT(!list_link_active(&hdr->b_arc_node)); 2392 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 2393 arc_change_state(arc_anon, hdr, hash_lock); 2394 hdr->b_arc_access = 0; 2395 mutex_exit(hash_lock); 2396 bzero(&hdr->b_dva, sizeof (dva_t)); 2397 hdr->b_birth = 0; 2398 hdr->b_cksum0 = 0; 2399 arc_buf_thaw(buf); 2400 } 2401 buf->b_efunc = NULL; 2402 buf->b_private = NULL; 2403 } 2404 2405 int 2406 arc_released(arc_buf_t *buf) 2407 { 2408 return (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon); 2409 } 2410 2411 int 2412 arc_has_callback(arc_buf_t *buf) 2413 { 2414 return (buf->b_efunc != NULL); 2415 } 2416 2417 #ifdef ZFS_DEBUG 2418 int 2419 arc_referenced(arc_buf_t *buf) 2420 { 2421 return (refcount_count(&buf->b_hdr->b_refcnt)); 2422 } 2423 #endif 2424 2425 static void 2426 arc_write_ready(zio_t *zio) 2427 { 2428 arc_write_callback_t *callback = zio->io_private; 2429 arc_buf_t *buf = callback->awcb_buf; 2430 2431 if (callback->awcb_ready) { 2432 ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt)); 2433 callback->awcb_ready(zio, buf, callback->awcb_private); 2434 } 2435 arc_cksum_compute(buf); 2436 } 2437 2438 static void 2439 arc_write_done(zio_t *zio) 2440 { 2441 arc_write_callback_t *callback = zio->io_private; 2442 arc_buf_t *buf = callback->awcb_buf; 2443 arc_buf_hdr_t *hdr = buf->b_hdr; 2444 2445 hdr->b_acb = NULL; 2446 2447 /* this buffer is on no lists and is not in the hash table */ 2448 ASSERT3P(hdr->b_state, ==, arc_anon); 2449 2450 hdr->b_dva = *BP_IDENTITY(zio->io_bp); 2451 hdr->b_birth = zio->io_bp->blk_birth; 2452 hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0]; 2453 /* 2454 * If the block to be written was all-zero, we may have 2455 * compressed it away. In this case no write was performed 2456 * so there will be no dva/birth-date/checksum. The buffer 2457 * must therefor remain anonymous (and uncached). 2458 */ 2459 if (!BUF_EMPTY(hdr)) { 2460 arc_buf_hdr_t *exists; 2461 kmutex_t *hash_lock; 2462 2463 arc_cksum_verify(buf); 2464 2465 exists = buf_hash_insert(hdr, &hash_lock); 2466 if (exists) { 2467 /* 2468 * This can only happen if we overwrite for 2469 * sync-to-convergence, because we remove 2470 * buffers from the hash table when we arc_free(). 2471 */ 2472 ASSERT(DVA_EQUAL(BP_IDENTITY(&zio->io_bp_orig), 2473 BP_IDENTITY(zio->io_bp))); 2474 ASSERT3U(zio->io_bp_orig.blk_birth, ==, 2475 zio->io_bp->blk_birth); 2476 2477 ASSERT(refcount_is_zero(&exists->b_refcnt)); 2478 arc_change_state(arc_anon, exists, hash_lock); 2479 mutex_exit(hash_lock); 2480 arc_hdr_destroy(exists); 2481 exists = buf_hash_insert(hdr, &hash_lock); 2482 ASSERT3P(exists, ==, NULL); 2483 } 2484 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2485 arc_access(hdr, hash_lock); 2486 mutex_exit(hash_lock); 2487 } else if (callback->awcb_done == NULL) { 2488 int destroy_hdr; 2489 /* 2490 * This is an anonymous buffer with no user callback, 2491 * destroy it if there are no active references. 2492 */ 2493 mutex_enter(&arc_eviction_mtx); 2494 destroy_hdr = refcount_is_zero(&hdr->b_refcnt); 2495 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2496 mutex_exit(&arc_eviction_mtx); 2497 if (destroy_hdr) 2498 arc_hdr_destroy(hdr); 2499 } else { 2500 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2501 } 2502 2503 if (callback->awcb_done) { 2504 ASSERT(!refcount_is_zero(&hdr->b_refcnt)); 2505 callback->awcb_done(zio, buf, callback->awcb_private); 2506 } 2507 2508 kmem_free(callback, sizeof (arc_write_callback_t)); 2509 } 2510 2511 zio_t * 2512 arc_write(zio_t *pio, spa_t *spa, int checksum, int compress, int ncopies, 2513 uint64_t txg, blkptr_t *bp, arc_buf_t *buf, 2514 arc_done_func_t *ready, arc_done_func_t *done, void *private, int priority, 2515 int flags, zbookmark_t *zb) 2516 { 2517 arc_buf_hdr_t *hdr = buf->b_hdr; 2518 arc_write_callback_t *callback; 2519 zio_t *zio; 2520 2521 /* this is a private buffer - no locking required */ 2522 ASSERT3P(hdr->b_state, ==, arc_anon); 2523 ASSERT(BUF_EMPTY(hdr)); 2524 ASSERT(!HDR_IO_ERROR(hdr)); 2525 ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0); 2526 ASSERT(hdr->b_acb == 0); 2527 callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP); 2528 callback->awcb_ready = ready; 2529 callback->awcb_done = done; 2530 callback->awcb_private = private; 2531 callback->awcb_buf = buf; 2532 hdr->b_flags |= ARC_IO_IN_PROGRESS; 2533 zio = zio_write(pio, spa, checksum, compress, ncopies, txg, bp, 2534 buf->b_data, hdr->b_size, arc_write_ready, arc_write_done, callback, 2535 priority, flags, zb); 2536 2537 return (zio); 2538 } 2539 2540 int 2541 arc_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 2542 zio_done_func_t *done, void *private, uint32_t arc_flags) 2543 { 2544 arc_buf_hdr_t *ab; 2545 kmutex_t *hash_lock; 2546 zio_t *zio; 2547 2548 /* 2549 * If this buffer is in the cache, release it, so it 2550 * can be re-used. 2551 */ 2552 ab = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock); 2553 if (ab != NULL) { 2554 /* 2555 * The checksum of blocks to free is not always 2556 * preserved (eg. on the deadlist). However, if it is 2557 * nonzero, it should match what we have in the cache. 2558 */ 2559 ASSERT(bp->blk_cksum.zc_word[0] == 0 || 2560 ab->b_cksum0 == bp->blk_cksum.zc_word[0]); 2561 if (ab->b_state != arc_anon) 2562 arc_change_state(arc_anon, ab, hash_lock); 2563 if (HDR_IO_IN_PROGRESS(ab)) { 2564 /* 2565 * This should only happen when we prefetch. 2566 */ 2567 ASSERT(ab->b_flags & ARC_PREFETCH); 2568 ASSERT3U(ab->b_datacnt, ==, 1); 2569 ab->b_flags |= ARC_FREED_IN_READ; 2570 if (HDR_IN_HASH_TABLE(ab)) 2571 buf_hash_remove(ab); 2572 ab->b_arc_access = 0; 2573 bzero(&ab->b_dva, sizeof (dva_t)); 2574 ab->b_birth = 0; 2575 ab->b_cksum0 = 0; 2576 ab->b_buf->b_efunc = NULL; 2577 ab->b_buf->b_private = NULL; 2578 mutex_exit(hash_lock); 2579 } else if (refcount_is_zero(&ab->b_refcnt)) { 2580 mutex_exit(hash_lock); 2581 arc_hdr_destroy(ab); 2582 ARCSTAT_BUMP(arcstat_deleted); 2583 } else { 2584 /* 2585 * We still have an active reference on this 2586 * buffer. This can happen, e.g., from 2587 * dbuf_unoverride(). 2588 */ 2589 ASSERT(!HDR_IN_HASH_TABLE(ab)); 2590 ab->b_arc_access = 0; 2591 bzero(&ab->b_dva, sizeof (dva_t)); 2592 ab->b_birth = 0; 2593 ab->b_cksum0 = 0; 2594 ab->b_buf->b_efunc = NULL; 2595 ab->b_buf->b_private = NULL; 2596 mutex_exit(hash_lock); 2597 } 2598 } 2599 2600 zio = zio_free(pio, spa, txg, bp, done, private); 2601 2602 if (arc_flags & ARC_WAIT) 2603 return (zio_wait(zio)); 2604 2605 ASSERT(arc_flags & ARC_NOWAIT); 2606 zio_nowait(zio); 2607 2608 return (0); 2609 } 2610 2611 void 2612 arc_tempreserve_clear(uint64_t tempreserve) 2613 { 2614 atomic_add_64(&arc_tempreserve, -tempreserve); 2615 ASSERT((int64_t)arc_tempreserve >= 0); 2616 } 2617 2618 int 2619 arc_tempreserve_space(uint64_t tempreserve) 2620 { 2621 #ifdef ZFS_DEBUG 2622 /* 2623 * Once in a while, fail for no reason. Everything should cope. 2624 */ 2625 if (spa_get_random(10000) == 0) { 2626 dprintf("forcing random failure\n"); 2627 return (ERESTART); 2628 } 2629 #endif 2630 if (tempreserve > arc_c/4 && !arc_no_grow) 2631 arc_c = MIN(arc_c_max, tempreserve * 4); 2632 if (tempreserve > arc_c) 2633 return (ENOMEM); 2634 2635 /* 2636 * Throttle writes when the amount of dirty data in the cache 2637 * gets too large. We try to keep the cache less than half full 2638 * of dirty blocks so that our sync times don't grow too large. 2639 * Note: if two requests come in concurrently, we might let them 2640 * both succeed, when one of them should fail. Not a huge deal. 2641 * 2642 * XXX The limit should be adjusted dynamically to keep the time 2643 * to sync a dataset fixed (around 1-5 seconds?). 2644 */ 2645 2646 if (tempreserve + arc_tempreserve + arc_anon->arcs_size > arc_c / 2 && 2647 arc_tempreserve + arc_anon->arcs_size > arc_c / 4) { 2648 dprintf("failing, arc_tempreserve=%lluK anon=%lluK " 2649 "tempreserve=%lluK arc_c=%lluK\n", 2650 arc_tempreserve>>10, arc_anon->arcs_lsize>>10, 2651 tempreserve>>10, arc_c>>10); 2652 return (ERESTART); 2653 } 2654 atomic_add_64(&arc_tempreserve, tempreserve); 2655 return (0); 2656 } 2657 2658 void 2659 arc_init(void) 2660 { 2661 mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL); 2662 cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL); 2663 2664 /* Convert seconds to clock ticks */ 2665 arc_min_prefetch_lifespan = 1 * hz; 2666 2667 /* Start out with 1/8 of all memory */ 2668 arc_c = physmem * PAGESIZE / 8; 2669 2670 #ifdef _KERNEL 2671 /* 2672 * On architectures where the physical memory can be larger 2673 * than the addressable space (intel in 32-bit mode), we may 2674 * need to limit the cache to 1/8 of VM size. 2675 */ 2676 arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8); 2677 #endif 2678 2679 /* set min cache to 1/32 of all memory, or 64MB, whichever is more */ 2680 arc_c_min = MAX(arc_c / 4, 64<<20); 2681 /* set max to 3/4 of all memory, or all but 1GB, whichever is more */ 2682 if (arc_c * 8 >= 1<<30) 2683 arc_c_max = (arc_c * 8) - (1<<30); 2684 else 2685 arc_c_max = arc_c_min; 2686 arc_c_max = MAX(arc_c * 6, arc_c_max); 2687 2688 /* 2689 * Allow the tunables to override our calculations if they are 2690 * reasonable (ie. over 64MB) 2691 */ 2692 if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE) 2693 arc_c_max = zfs_arc_max; 2694 if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max) 2695 arc_c_min = zfs_arc_min; 2696 2697 arc_c = arc_c_max; 2698 arc_p = (arc_c >> 1); 2699 2700 /* if kmem_flags are set, lets try to use less memory */ 2701 if (kmem_debugging()) 2702 arc_c = arc_c / 2; 2703 if (arc_c < arc_c_min) 2704 arc_c = arc_c_min; 2705 2706 arc_anon = &ARC_anon; 2707 arc_mru = &ARC_mru; 2708 arc_mru_ghost = &ARC_mru_ghost; 2709 arc_mfu = &ARC_mfu; 2710 arc_mfu_ghost = &ARC_mfu_ghost; 2711 arc_size = 0; 2712 2713 mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 2714 mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 2715 mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 2716 mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 2717 mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 2718 2719 list_create(&arc_mru->arcs_list, sizeof (arc_buf_hdr_t), 2720 offsetof(arc_buf_hdr_t, b_arc_node)); 2721 list_create(&arc_mru_ghost->arcs_list, sizeof (arc_buf_hdr_t), 2722 offsetof(arc_buf_hdr_t, b_arc_node)); 2723 list_create(&arc_mfu->arcs_list, sizeof (arc_buf_hdr_t), 2724 offsetof(arc_buf_hdr_t, b_arc_node)); 2725 list_create(&arc_mfu_ghost->arcs_list, sizeof (arc_buf_hdr_t), 2726 offsetof(arc_buf_hdr_t, b_arc_node)); 2727 2728 buf_init(); 2729 2730 arc_thread_exit = 0; 2731 arc_eviction_list = NULL; 2732 mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL); 2733 bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t)); 2734 2735 arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED, 2736 sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 2737 2738 if (arc_ksp != NULL) { 2739 arc_ksp->ks_data = &arc_stats; 2740 kstat_install(arc_ksp); 2741 } 2742 2743 (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0, 2744 TS_RUN, minclsyspri); 2745 2746 arc_dead = FALSE; 2747 } 2748 2749 void 2750 arc_fini(void) 2751 { 2752 mutex_enter(&arc_reclaim_thr_lock); 2753 arc_thread_exit = 1; 2754 while (arc_thread_exit != 0) 2755 cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock); 2756 mutex_exit(&arc_reclaim_thr_lock); 2757 2758 arc_flush(); 2759 2760 arc_dead = TRUE; 2761 2762 if (arc_ksp != NULL) { 2763 kstat_delete(arc_ksp); 2764 arc_ksp = NULL; 2765 } 2766 2767 mutex_destroy(&arc_eviction_mtx); 2768 mutex_destroy(&arc_reclaim_thr_lock); 2769 cv_destroy(&arc_reclaim_thr_cv); 2770 2771 list_destroy(&arc_mru->arcs_list); 2772 list_destroy(&arc_mru_ghost->arcs_list); 2773 list_destroy(&arc_mfu->arcs_list); 2774 list_destroy(&arc_mfu_ghost->arcs_list); 2775 2776 mutex_destroy(&arc_anon->arcs_mtx); 2777 mutex_destroy(&arc_mru->arcs_mtx); 2778 mutex_destroy(&arc_mru_ghost->arcs_mtx); 2779 mutex_destroy(&arc_mfu->arcs_mtx); 2780 mutex_destroy(&arc_mfu_ghost->arcs_mtx); 2781 2782 buf_fini(); 2783 } 2784