1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2019, Joyent, Inc. 24 * Copyright (c) 2011, 2018 by Delphix. All rights reserved. 25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved. 26 * Copyright 2017 Nexenta Systems, Inc. All rights reserved. 27 */ 28 29/* 30 * DVA-based Adjustable Replacement Cache 31 * 32 * While much of the theory of operation used here is 33 * based on the self-tuning, low overhead replacement cache 34 * presented by Megiddo and Modha at FAST 2003, there are some 35 * significant differences: 36 * 37 * 1. The Megiddo and Modha model assumes any page is evictable. 38 * Pages in its cache cannot be "locked" into memory. This makes 39 * the eviction algorithm simple: evict the last page in the list. 40 * This also make the performance characteristics easy to reason 41 * about. Our cache is not so simple. At any given moment, some 42 * subset of the blocks in the cache are un-evictable because we 43 * have handed out a reference to them. Blocks are only evictable 44 * when there are no external references active. This makes 45 * eviction far more problematic: we choose to evict the evictable 46 * blocks that are the "lowest" in the list. 47 * 48 * There are times when it is not possible to evict the requested 49 * space. In these circumstances we are unable to adjust the cache 50 * size. To prevent the cache growing unbounded at these times we 51 * implement a "cache throttle" that slows the flow of new data 52 * into the cache until we can make space available. 53 * 54 * 2. The Megiddo and Modha model assumes a fixed cache size. 55 * Pages are evicted when the cache is full and there is a cache 56 * miss. Our model has a variable sized cache. It grows with 57 * high use, but also tries to react to memory pressure from the 58 * operating system: decreasing its size when system memory is 59 * tight. 60 * 61 * 3. The Megiddo and Modha model assumes a fixed page size. All 62 * elements of the cache are therefore exactly the same size. So 63 * when adjusting the cache size following a cache miss, its simply 64 * a matter of choosing a single page to evict. In our model, we 65 * have variable sized cache blocks (rangeing from 512 bytes to 66 * 128K bytes). We therefore choose a set of blocks to evict to make 67 * space for a cache miss that approximates as closely as possible 68 * the space used by the new block. 69 * 70 * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache" 71 * by N. Megiddo & D. Modha, FAST 2003 72 */ 73 74/* 75 * The locking model: 76 * 77 * A new reference to a cache buffer can be obtained in two 78 * ways: 1) via a hash table lookup using the DVA as a key, 79 * or 2) via one of the ARC lists. The arc_read() interface 80 * uses method 1, while the internal ARC algorithms for 81 * adjusting the cache use method 2. We therefore provide two 82 * types of locks: 1) the hash table lock array, and 2) the 83 * ARC list locks. 84 * 85 * Buffers do not have their own mutexes, rather they rely on the 86 * hash table mutexes for the bulk of their protection (i.e. most 87 * fields in the arc_buf_hdr_t are protected by these mutexes). 88 * 89 * buf_hash_find() returns the appropriate mutex (held) when it 90 * locates the requested buffer in the hash table. It returns 91 * NULL for the mutex if the buffer was not in the table. 92 * 93 * buf_hash_remove() expects the appropriate hash mutex to be 94 * already held before it is invoked. 95 * 96 * Each ARC state also has a mutex which is used to protect the 97 * buffer list associated with the state. When attempting to 98 * obtain a hash table lock while holding an ARC list lock you 99 * must use: mutex_tryenter() to avoid deadlock. Also note that 100 * the active state mutex must be held before the ghost state mutex. 101 * 102 * Note that the majority of the performance stats are manipulated 103 * with atomic operations. 104 * 105 * The L2ARC uses the l2ad_mtx on each vdev for the following: 106 * 107 * - L2ARC buflist creation 108 * - L2ARC buflist eviction 109 * - L2ARC write completion, which walks L2ARC buflists 110 * - ARC header destruction, as it removes from L2ARC buflists 111 * - ARC header release, as it removes from L2ARC buflists 112 */ 113 114/* 115 * ARC operation: 116 * 117 * Every block that is in the ARC is tracked by an arc_buf_hdr_t structure. 118 * This structure can point either to a block that is still in the cache or to 119 * one that is only accessible in an L2 ARC device, or it can provide 120 * information about a block that was recently evicted. If a block is 121 * only accessible in the L2ARC, then the arc_buf_hdr_t only has enough 122 * information to retrieve it from the L2ARC device. This information is 123 * stored in the l2arc_buf_hdr_t sub-structure of the arc_buf_hdr_t. A block 124 * that is in this state cannot access the data directly. 125 * 126 * Blocks that are actively being referenced or have not been evicted 127 * are cached in the L1ARC. The L1ARC (l1arc_buf_hdr_t) is a structure within 128 * the arc_buf_hdr_t that will point to the data block in memory. A block can 129 * only be read by a consumer if it has an l1arc_buf_hdr_t. The L1ARC 130 * caches data in two ways -- in a list of ARC buffers (arc_buf_t) and 131 * also in the arc_buf_hdr_t's private physical data block pointer (b_pabd). 132 * 133 * The L1ARC's data pointer may or may not be uncompressed. The ARC has the 134 * ability to store the physical data (b_pabd) associated with the DVA of the 135 * arc_buf_hdr_t. Since the b_pabd is a copy of the on-disk physical block, 136 * it will match its on-disk compression characteristics. This behavior can be 137 * disabled by setting 'zfs_compressed_arc_enabled' to B_FALSE. When the 138 * compressed ARC functionality is disabled, the b_pabd will point to an 139 * uncompressed version of the on-disk data. 140 * 141 * Data in the L1ARC is not accessed by consumers of the ARC directly. Each 142 * arc_buf_hdr_t can have multiple ARC buffers (arc_buf_t) which reference it. 143 * Each ARC buffer (arc_buf_t) is being actively accessed by a specific ARC 144 * consumer. The ARC will provide references to this data and will keep it 145 * cached until it is no longer in use. The ARC caches only the L1ARC's physical 146 * data block and will evict any arc_buf_t that is no longer referenced. The 147 * amount of memory consumed by the arc_buf_ts' data buffers can be seen via the 148 * "overhead_size" kstat. 149 * 150 * Depending on the consumer, an arc_buf_t can be requested in uncompressed or 151 * compressed form. The typical case is that consumers will want uncompressed 152 * data, and when that happens a new data buffer is allocated where the data is 153 * decompressed for them to use. Currently the only consumer who wants 154 * compressed arc_buf_t's is "zfs send", when it streams data exactly as it 155 * exists on disk. When this happens, the arc_buf_t's data buffer is shared 156 * with the arc_buf_hdr_t. 157 * 158 * Here is a diagram showing an arc_buf_hdr_t referenced by two arc_buf_t's. The 159 * first one is owned by a compressed send consumer (and therefore references 160 * the same compressed data buffer as the arc_buf_hdr_t) and the second could be 161 * used by any other consumer (and has its own uncompressed copy of the data 162 * buffer). 163 * 164 * arc_buf_hdr_t 165 * +-----------+ 166 * | fields | 167 * | common to | 168 * | L1- and | 169 * | L2ARC | 170 * +-----------+ 171 * | l2arc_buf_hdr_t 172 * | | 173 * +-----------+ 174 * | l1arc_buf_hdr_t 175 * | | arc_buf_t 176 * | b_buf +------------>+-----------+ arc_buf_t 177 * | b_pabd +-+ |b_next +---->+-----------+ 178 * +-----------+ | |-----------| |b_next +-->NULL 179 * | |b_comp = T | +-----------+ 180 * | |b_data +-+ |b_comp = F | 181 * | +-----------+ | |b_data +-+ 182 * +->+------+ | +-----------+ | 183 * compressed | | | | 184 * data | |<--------------+ | uncompressed 185 * +------+ compressed, | data 186 * shared +-->+------+ 187 * data | | 188 * | | 189 * +------+ 190 * 191 * When a consumer reads a block, the ARC must first look to see if the 192 * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new 193 * arc_buf_t and either copies uncompressed data into a new data buffer from an 194 * existing uncompressed arc_buf_t, decompresses the hdr's b_pabd buffer into a 195 * new data buffer, or shares the hdr's b_pabd buffer, depending on whether the 196 * hdr is compressed and the desired compression characteristics of the 197 * arc_buf_t consumer. If the arc_buf_t ends up sharing data with the 198 * arc_buf_hdr_t and both of them are uncompressed then the arc_buf_t must be 199 * the last buffer in the hdr's b_buf list, however a shared compressed buf can 200 * be anywhere in the hdr's list. 201 * 202 * The diagram below shows an example of an uncompressed ARC hdr that is 203 * sharing its data with an arc_buf_t (note that the shared uncompressed buf is 204 * the last element in the buf list): 205 * 206 * arc_buf_hdr_t 207 * +-----------+ 208 * | | 209 * | | 210 * | | 211 * +-----------+ 212 * l2arc_buf_hdr_t| | 213 * | | 214 * +-----------+ 215 * l1arc_buf_hdr_t| | 216 * | | arc_buf_t (shared) 217 * | b_buf +------------>+---------+ arc_buf_t 218 * | | |b_next +---->+---------+ 219 * | b_pabd +-+ |---------| |b_next +-->NULL 220 * +-----------+ | | | +---------+ 221 * | |b_data +-+ | | 222 * | +---------+ | |b_data +-+ 223 * +->+------+ | +---------+ | 224 * | | | | 225 * uncompressed | | | | 226 * data +------+ | | 227 * ^ +->+------+ | 228 * | uncompressed | | | 229 * | data | | | 230 * | +------+ | 231 * +---------------------------------+ 232 * 233 * Writing to the ARC requires that the ARC first discard the hdr's b_pabd 234 * since the physical block is about to be rewritten. The new data contents 235 * will be contained in the arc_buf_t. As the I/O pipeline performs the write, 236 * it may compress the data before writing it to disk. The ARC will be called 237 * with the transformed data and will bcopy the transformed on-disk block into 238 * a newly allocated b_pabd. Writes are always done into buffers which have 239 * either been loaned (and hence are new and don't have other readers) or 240 * buffers which have been released (and hence have their own hdr, if there 241 * were originally other readers of the buf's original hdr). This ensures that 242 * the ARC only needs to update a single buf and its hdr after a write occurs. 243 * 244 * When the L2ARC is in use, it will also take advantage of the b_pabd. The 245 * L2ARC will always write the contents of b_pabd to the L2ARC. This means 246 * that when compressed ARC is enabled that the L2ARC blocks are identical 247 * to the on-disk block in the main data pool. This provides a significant 248 * advantage since the ARC can leverage the bp's checksum when reading from the 249 * L2ARC to determine if the contents are valid. However, if the compressed 250 * ARC is disabled, then the L2ARC's block must be transformed to look 251 * like the physical block in the main data pool before comparing the 252 * checksum and determining its validity. 253 * 254 * The L1ARC has a slightly different system for storing encrypted data. 255 * Raw (encrypted + possibly compressed) data has a few subtle differences from 256 * data that is just compressed. The biggest difference is that it is not 257 * possible to decrypt encrypted data (or visa versa) if the keys aren't loaded. 258 * The other difference is that encryption cannot be treated as a suggestion. 259 * If a caller would prefer compressed data, but they actually wind up with 260 * uncompressed data the worst thing that could happen is there might be a 261 * performance hit. If the caller requests encrypted data, however, we must be 262 * sure they actually get it or else secret information could be leaked. Raw 263 * data is stored in hdr->b_crypt_hdr.b_rabd. An encrypted header, therefore, 264 * may have both an encrypted version and a decrypted version of its data at 265 * once. When a caller needs a raw arc_buf_t, it is allocated and the data is 266 * copied out of this header. To avoid complications with b_pabd, raw buffers 267 * cannot be shared. 268 */ 269 270#include <sys/spa.h> 271#include <sys/zio.h> 272#include <sys/spa_impl.h> 273#include <sys/zio_compress.h> 274#include <sys/zio_checksum.h> 275#include <sys/zfs_context.h> 276#include <sys/arc.h> 277#include <sys/refcount.h> 278#include <sys/vdev.h> 279#include <sys/vdev_impl.h> 280#include <sys/dsl_pool.h> 281#include <sys/zio_checksum.h> 282#include <sys/multilist.h> 283#include <sys/abd.h> 284#include <sys/zil.h> 285#include <sys/fm/fs/zfs.h> 286#ifdef _KERNEL 287#include <sys/vmsystm.h> 288#include <vm/anon.h> 289#include <sys/fs/swapnode.h> 290#include <sys/dnlc.h> 291#endif 292#include <sys/callb.h> 293#include <sys/kstat.h> 294#include <sys/zthr.h> 295#include <zfs_fletcher.h> 296#include <sys/aggsum.h> 297#include <sys/cityhash.h> 298#include <sys/param.h> 299 300#ifndef _KERNEL 301/* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */ 302boolean_t arc_watch = B_FALSE; 303int arc_procfd; 304#endif 305 306/* 307 * This thread's job is to keep enough free memory in the system, by 308 * calling arc_kmem_reap_now() plus arc_shrink(), which improves 309 * arc_available_memory(). 310 */ 311static zthr_t *arc_reap_zthr; 312 313/* 314 * This thread's job is to keep arc_size under arc_c, by calling 315 * arc_adjust(), which improves arc_is_overflowing(). 316 */ 317static zthr_t *arc_adjust_zthr; 318 319static kmutex_t arc_adjust_lock; 320static kcondvar_t arc_adjust_waiters_cv; 321static boolean_t arc_adjust_needed = B_FALSE; 322 323uint_t arc_reduce_dnlc_percent = 3; 324 325/* 326 * The number of headers to evict in arc_evict_state_impl() before 327 * dropping the sublist lock and evicting from another sublist. A lower 328 * value means we're more likely to evict the "correct" header (i.e. the 329 * oldest header in the arc state), but comes with higher overhead 330 * (i.e. more invocations of arc_evict_state_impl()). 331 */ 332int zfs_arc_evict_batch_limit = 10; 333 334/* number of seconds before growing cache again */ 335int arc_grow_retry = 60; 336 337/* 338 * Minimum time between calls to arc_kmem_reap_soon(). Note that this will 339 * be converted to ticks, so with the default hz=100, a setting of 15 ms 340 * will actually wait 2 ticks, or 20ms. 341 */ 342int arc_kmem_cache_reap_retry_ms = 1000; 343 344/* shift of arc_c for calculating overflow limit in arc_get_data_impl */ 345int zfs_arc_overflow_shift = 8; 346 347/* shift of arc_c for calculating both min and max arc_p */ 348int arc_p_min_shift = 4; 349 350/* log2(fraction of arc to reclaim) */ 351int arc_shrink_shift = 7; 352 353/* 354 * log2(fraction of ARC which must be free to allow growing). 355 * I.e. If there is less than arc_c >> arc_no_grow_shift free memory, 356 * when reading a new block into the ARC, we will evict an equal-sized block 357 * from the ARC. 358 * 359 * This must be less than arc_shrink_shift, so that when we shrink the ARC, 360 * we will still not allow it to grow. 361 */ 362int arc_no_grow_shift = 5; 363 364 365/* 366 * minimum lifespan of a prefetch block in clock ticks 367 * (initialized in arc_init()) 368 */ 369static int zfs_arc_min_prefetch_ms = 1; 370static int zfs_arc_min_prescient_prefetch_ms = 6; 371 372/* 373 * If this percent of memory is free, don't throttle. 374 */ 375int arc_lotsfree_percent = 10; 376 377static boolean_t arc_initialized; 378 379/* 380 * The arc has filled available memory and has now warmed up. 381 */ 382static boolean_t arc_warm; 383 384/* 385 * log2 fraction of the zio arena to keep free. 386 */ 387int arc_zio_arena_free_shift = 2; 388 389/* 390 * These tunables are for performance analysis. 391 */ 392uint64_t zfs_arc_max; 393uint64_t zfs_arc_min; 394uint64_t zfs_arc_meta_limit = 0; 395uint64_t zfs_arc_meta_min = 0; 396int zfs_arc_grow_retry = 0; 397int zfs_arc_shrink_shift = 0; 398int zfs_arc_p_min_shift = 0; 399int zfs_arc_average_blocksize = 8 * 1024; /* 8KB */ 400 401/* 402 * ARC dirty data constraints for arc_tempreserve_space() throttle 403 */ 404uint_t zfs_arc_dirty_limit_percent = 50; /* total dirty data limit */ 405uint_t zfs_arc_anon_limit_percent = 25; /* anon block dirty limit */ 406uint_t zfs_arc_pool_dirty_percent = 20; /* each pool's anon allowance */ 407 408boolean_t zfs_compressed_arc_enabled = B_TRUE; 409 410/* 411 * Note that buffers can be in one of 6 states: 412 * ARC_anon - anonymous (discussed below) 413 * ARC_mru - recently used, currently cached 414 * ARC_mru_ghost - recentely used, no longer in cache 415 * ARC_mfu - frequently used, currently cached 416 * ARC_mfu_ghost - frequently used, no longer in cache 417 * ARC_l2c_only - exists in L2ARC but not other states 418 * When there are no active references to the buffer, they are 419 * are linked onto a list in one of these arc states. These are 420 * the only buffers that can be evicted or deleted. Within each 421 * state there are multiple lists, one for meta-data and one for 422 * non-meta-data. Meta-data (indirect blocks, blocks of dnodes, 423 * etc.) is tracked separately so that it can be managed more 424 * explicitly: favored over data, limited explicitly. 425 * 426 * Anonymous buffers are buffers that are not associated with 427 * a DVA. These are buffers that hold dirty block copies 428 * before they are written to stable storage. By definition, 429 * they are "ref'd" and are considered part of arc_mru 430 * that cannot be freed. Generally, they will aquire a DVA 431 * as they are written and migrate onto the arc_mru list. 432 * 433 * The ARC_l2c_only state is for buffers that are in the second 434 * level ARC but no longer in any of the ARC_m* lists. The second 435 * level ARC itself may also contain buffers that are in any of 436 * the ARC_m* states - meaning that a buffer can exist in two 437 * places. The reason for the ARC_l2c_only state is to keep the 438 * buffer header in the hash table, so that reads that hit the 439 * second level ARC benefit from these fast lookups. 440 */ 441 442typedef struct arc_state { 443 /* 444 * list of evictable buffers 445 */ 446 multilist_t *arcs_list[ARC_BUFC_NUMTYPES]; 447 /* 448 * total amount of evictable data in this state 449 */ 450 zfs_refcount_t arcs_esize[ARC_BUFC_NUMTYPES]; 451 /* 452 * total amount of data in this state; this includes: evictable, 453 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA. 454 */ 455 zfs_refcount_t arcs_size; 456} arc_state_t; 457 458/* The 6 states: */ 459static arc_state_t ARC_anon; 460static arc_state_t ARC_mru; 461static arc_state_t ARC_mru_ghost; 462static arc_state_t ARC_mfu; 463static arc_state_t ARC_mfu_ghost; 464static arc_state_t ARC_l2c_only; 465 466typedef struct arc_stats { 467 kstat_named_t arcstat_hits; 468 kstat_named_t arcstat_misses; 469 kstat_named_t arcstat_demand_data_hits; 470 kstat_named_t arcstat_demand_data_misses; 471 kstat_named_t arcstat_demand_metadata_hits; 472 kstat_named_t arcstat_demand_metadata_misses; 473 kstat_named_t arcstat_prefetch_data_hits; 474 kstat_named_t arcstat_prefetch_data_misses; 475 kstat_named_t arcstat_prefetch_metadata_hits; 476 kstat_named_t arcstat_prefetch_metadata_misses; 477 kstat_named_t arcstat_mru_hits; 478 kstat_named_t arcstat_mru_ghost_hits; 479 kstat_named_t arcstat_mfu_hits; 480 kstat_named_t arcstat_mfu_ghost_hits; 481 kstat_named_t arcstat_deleted; 482 /* 483 * Number of buffers that could not be evicted because the hash lock 484 * was held by another thread. The lock may not necessarily be held 485 * by something using the same buffer, since hash locks are shared 486 * by multiple buffers. 487 */ 488 kstat_named_t arcstat_mutex_miss; 489 /* 490 * Number of buffers skipped when updating the access state due to the 491 * header having already been released after acquiring the hash lock. 492 */ 493 kstat_named_t arcstat_access_skip; 494 /* 495 * Number of buffers skipped because they have I/O in progress, are 496 * indirect prefetch buffers that have not lived long enough, or are 497 * not from the spa we're trying to evict from. 498 */ 499 kstat_named_t arcstat_evict_skip; 500 /* 501 * Number of times arc_evict_state() was unable to evict enough 502 * buffers to reach its target amount. 503 */ 504 kstat_named_t arcstat_evict_not_enough; 505 kstat_named_t arcstat_evict_l2_cached; 506 kstat_named_t arcstat_evict_l2_eligible; 507 kstat_named_t arcstat_evict_l2_ineligible; 508 kstat_named_t arcstat_evict_l2_skip; 509 kstat_named_t arcstat_hash_elements; 510 kstat_named_t arcstat_hash_elements_max; 511 kstat_named_t arcstat_hash_collisions; 512 kstat_named_t arcstat_hash_chains; 513 kstat_named_t arcstat_hash_chain_max; 514 kstat_named_t arcstat_p; 515 kstat_named_t arcstat_c; 516 kstat_named_t arcstat_c_min; 517 kstat_named_t arcstat_c_max; 518 /* Not updated directly; only synced in arc_kstat_update. */ 519 kstat_named_t arcstat_size; 520 /* 521 * Number of compressed bytes stored in the arc_buf_hdr_t's b_pabd. 522 * Note that the compressed bytes may match the uncompressed bytes 523 * if the block is either not compressed or compressed arc is disabled. 524 */ 525 kstat_named_t arcstat_compressed_size; 526 /* 527 * Uncompressed size of the data stored in b_pabd. If compressed 528 * arc is disabled then this value will be identical to the stat 529 * above. 530 */ 531 kstat_named_t arcstat_uncompressed_size; 532 /* 533 * Number of bytes stored in all the arc_buf_t's. This is classified 534 * as "overhead" since this data is typically short-lived and will 535 * be evicted from the arc when it becomes unreferenced unless the 536 * zfs_keep_uncompressed_metadata or zfs_keep_uncompressed_level 537 * values have been set (see comment in dbuf.c for more information). 538 */ 539 kstat_named_t arcstat_overhead_size; 540 /* 541 * Number of bytes consumed by internal ARC structures necessary 542 * for tracking purposes; these structures are not actually 543 * backed by ARC buffers. This includes arc_buf_hdr_t structures 544 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only 545 * caches), and arc_buf_t structures (allocated via arc_buf_t 546 * cache). 547 * Not updated directly; only synced in arc_kstat_update. 548 */ 549 kstat_named_t arcstat_hdr_size; 550 /* 551 * Number of bytes consumed by ARC buffers of type equal to 552 * ARC_BUFC_DATA. This is generally consumed by buffers backing 553 * on disk user data (e.g. plain file contents). 554 * Not updated directly; only synced in arc_kstat_update. 555 */ 556 kstat_named_t arcstat_data_size; 557 /* 558 * Number of bytes consumed by ARC buffers of type equal to 559 * ARC_BUFC_METADATA. This is generally consumed by buffers 560 * backing on disk data that is used for internal ZFS 561 * structures (e.g. ZAP, dnode, indirect blocks, etc). 562 * Not updated directly; only synced in arc_kstat_update. 563 */ 564 kstat_named_t arcstat_metadata_size; 565 /* 566 * Number of bytes consumed by various buffers and structures 567 * not actually backed with ARC buffers. This includes bonus 568 * buffers (allocated directly via zio_buf_* functions), 569 * dmu_buf_impl_t structures (allocated via dmu_buf_impl_t 570 * cache), and dnode_t structures (allocated via dnode_t cache). 571 * Not updated directly; only synced in arc_kstat_update. 572 */ 573 kstat_named_t arcstat_other_size; 574 /* 575 * Total number of bytes consumed by ARC buffers residing in the 576 * arc_anon state. This includes *all* buffers in the arc_anon 577 * state; e.g. data, metadata, evictable, and unevictable buffers 578 * are all included in this value. 579 * Not updated directly; only synced in arc_kstat_update. 580 */ 581 kstat_named_t arcstat_anon_size; 582 /* 583 * Number of bytes consumed by ARC buffers that meet the 584 * following criteria: backing buffers of type ARC_BUFC_DATA, 585 * residing in the arc_anon state, and are eligible for eviction 586 * (e.g. have no outstanding holds on the buffer). 587 * Not updated directly; only synced in arc_kstat_update. 588 */ 589 kstat_named_t arcstat_anon_evictable_data; 590 /* 591 * Number of bytes consumed by ARC buffers that meet the 592 * following criteria: backing buffers of type ARC_BUFC_METADATA, 593 * residing in the arc_anon state, and are eligible for eviction 594 * (e.g. have no outstanding holds on the buffer). 595 * Not updated directly; only synced in arc_kstat_update. 596 */ 597 kstat_named_t arcstat_anon_evictable_metadata; 598 /* 599 * Total number of bytes consumed by ARC buffers residing in the 600 * arc_mru state. This includes *all* buffers in the arc_mru 601 * state; e.g. data, metadata, evictable, and unevictable buffers 602 * are all included in this value. 603 * Not updated directly; only synced in arc_kstat_update. 604 */ 605 kstat_named_t arcstat_mru_size; 606 /* 607 * Number of bytes consumed by ARC buffers that meet the 608 * following criteria: backing buffers of type ARC_BUFC_DATA, 609 * residing in the arc_mru state, and are eligible for eviction 610 * (e.g. have no outstanding holds on the buffer). 611 * Not updated directly; only synced in arc_kstat_update. 612 */ 613 kstat_named_t arcstat_mru_evictable_data; 614 /* 615 * Number of bytes consumed by ARC buffers that meet the 616 * following criteria: backing buffers of type ARC_BUFC_METADATA, 617 * residing in the arc_mru state, and are eligible for eviction 618 * (e.g. have no outstanding holds on the buffer). 619 * Not updated directly; only synced in arc_kstat_update. 620 */ 621 kstat_named_t arcstat_mru_evictable_metadata; 622 /* 623 * Total number of bytes that *would have been* consumed by ARC 624 * buffers in the arc_mru_ghost state. The key thing to note 625 * here, is the fact that this size doesn't actually indicate 626 * RAM consumption. The ghost lists only consist of headers and 627 * don't actually have ARC buffers linked off of these headers. 628 * Thus, *if* the headers had associated ARC buffers, these 629 * buffers *would have* consumed this number of bytes. 630 * Not updated directly; only synced in arc_kstat_update. 631 */ 632 kstat_named_t arcstat_mru_ghost_size; 633 /* 634 * Number of bytes that *would have been* consumed by ARC 635 * buffers that are eligible for eviction, of type 636 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state. 637 * Not updated directly; only synced in arc_kstat_update. 638 */ 639 kstat_named_t arcstat_mru_ghost_evictable_data; 640 /* 641 * Number of bytes that *would have been* consumed by ARC 642 * buffers that are eligible for eviction, of type 643 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state. 644 * Not updated directly; only synced in arc_kstat_update. 645 */ 646 kstat_named_t arcstat_mru_ghost_evictable_metadata; 647 /* 648 * Total number of bytes consumed by ARC buffers residing in the 649 * arc_mfu state. This includes *all* buffers in the arc_mfu 650 * state; e.g. data, metadata, evictable, and unevictable buffers 651 * are all included in this value. 652 * Not updated directly; only synced in arc_kstat_update. 653 */ 654 kstat_named_t arcstat_mfu_size; 655 /* 656 * Number of bytes consumed by ARC buffers that are eligible for 657 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu 658 * state. 659 * Not updated directly; only synced in arc_kstat_update. 660 */ 661 kstat_named_t arcstat_mfu_evictable_data; 662 /* 663 * Number of bytes consumed by ARC buffers that are eligible for 664 * eviction, of type ARC_BUFC_METADATA, and reside in the 665 * arc_mfu state. 666 * Not updated directly; only synced in arc_kstat_update. 667 */ 668 kstat_named_t arcstat_mfu_evictable_metadata; 669 /* 670 * Total number of bytes that *would have been* consumed by ARC 671 * buffers in the arc_mfu_ghost state. See the comment above 672 * arcstat_mru_ghost_size for more details. 673 * Not updated directly; only synced in arc_kstat_update. 674 */ 675 kstat_named_t arcstat_mfu_ghost_size; 676 /* 677 * Number of bytes that *would have been* consumed by ARC 678 * buffers that are eligible for eviction, of type 679 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state. 680 * Not updated directly; only synced in arc_kstat_update. 681 */ 682 kstat_named_t arcstat_mfu_ghost_evictable_data; 683 /* 684 * Number of bytes that *would have been* consumed by ARC 685 * buffers that are eligible for eviction, of type 686 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state. 687 * Not updated directly; only synced in arc_kstat_update. 688 */ 689 kstat_named_t arcstat_mfu_ghost_evictable_metadata; 690 kstat_named_t arcstat_l2_hits; 691 kstat_named_t arcstat_l2_misses; 692 kstat_named_t arcstat_l2_feeds; 693 kstat_named_t arcstat_l2_rw_clash; 694 kstat_named_t arcstat_l2_read_bytes; 695 kstat_named_t arcstat_l2_write_bytes; 696 kstat_named_t arcstat_l2_writes_sent; 697 kstat_named_t arcstat_l2_writes_done; 698 kstat_named_t arcstat_l2_writes_error; 699 kstat_named_t arcstat_l2_writes_lock_retry; 700 kstat_named_t arcstat_l2_evict_lock_retry; 701 kstat_named_t arcstat_l2_evict_reading; 702 kstat_named_t arcstat_l2_evict_l1cached; 703 kstat_named_t arcstat_l2_free_on_write; 704 kstat_named_t arcstat_l2_abort_lowmem; 705 kstat_named_t arcstat_l2_cksum_bad; 706 kstat_named_t arcstat_l2_io_error; 707 kstat_named_t arcstat_l2_lsize; 708 kstat_named_t arcstat_l2_psize; 709 /* Not updated directly; only synced in arc_kstat_update. */ 710 kstat_named_t arcstat_l2_hdr_size; 711 kstat_named_t arcstat_memory_throttle_count; 712 /* Not updated directly; only synced in arc_kstat_update. */ 713 kstat_named_t arcstat_meta_used; 714 kstat_named_t arcstat_meta_limit; 715 kstat_named_t arcstat_meta_max; 716 kstat_named_t arcstat_meta_min; 717 kstat_named_t arcstat_async_upgrade_sync; 718 kstat_named_t arcstat_demand_hit_predictive_prefetch; 719 kstat_named_t arcstat_demand_hit_prescient_prefetch; 720} arc_stats_t; 721 722static arc_stats_t arc_stats = { 723 { "hits", KSTAT_DATA_UINT64 }, 724 { "misses", KSTAT_DATA_UINT64 }, 725 { "demand_data_hits", KSTAT_DATA_UINT64 }, 726 { "demand_data_misses", KSTAT_DATA_UINT64 }, 727 { "demand_metadata_hits", KSTAT_DATA_UINT64 }, 728 { "demand_metadata_misses", KSTAT_DATA_UINT64 }, 729 { "prefetch_data_hits", KSTAT_DATA_UINT64 }, 730 { "prefetch_data_misses", KSTAT_DATA_UINT64 }, 731 { "prefetch_metadata_hits", KSTAT_DATA_UINT64 }, 732 { "prefetch_metadata_misses", KSTAT_DATA_UINT64 }, 733 { "mru_hits", KSTAT_DATA_UINT64 }, 734 { "mru_ghost_hits", KSTAT_DATA_UINT64 }, 735 { "mfu_hits", KSTAT_DATA_UINT64 }, 736 { "mfu_ghost_hits", KSTAT_DATA_UINT64 }, 737 { "deleted", KSTAT_DATA_UINT64 }, 738 { "mutex_miss", KSTAT_DATA_UINT64 }, 739 { "access_skip", KSTAT_DATA_UINT64 }, 740 { "evict_skip", KSTAT_DATA_UINT64 }, 741 { "evict_not_enough", KSTAT_DATA_UINT64 }, 742 { "evict_l2_cached", KSTAT_DATA_UINT64 }, 743 { "evict_l2_eligible", KSTAT_DATA_UINT64 }, 744 { "evict_l2_ineligible", KSTAT_DATA_UINT64 }, 745 { "evict_l2_skip", KSTAT_DATA_UINT64 }, 746 { "hash_elements", KSTAT_DATA_UINT64 }, 747 { "hash_elements_max", KSTAT_DATA_UINT64 }, 748 { "hash_collisions", KSTAT_DATA_UINT64 }, 749 { "hash_chains", KSTAT_DATA_UINT64 }, 750 { "hash_chain_max", KSTAT_DATA_UINT64 }, 751 { "p", KSTAT_DATA_UINT64 }, 752 { "c", KSTAT_DATA_UINT64 }, 753 { "c_min", KSTAT_DATA_UINT64 }, 754 { "c_max", KSTAT_DATA_UINT64 }, 755 { "size", KSTAT_DATA_UINT64 }, 756 { "compressed_size", KSTAT_DATA_UINT64 }, 757 { "uncompressed_size", KSTAT_DATA_UINT64 }, 758 { "overhead_size", KSTAT_DATA_UINT64 }, 759 { "hdr_size", KSTAT_DATA_UINT64 }, 760 { "data_size", KSTAT_DATA_UINT64 }, 761 { "metadata_size", KSTAT_DATA_UINT64 }, 762 { "other_size", KSTAT_DATA_UINT64 }, 763 { "anon_size", KSTAT_DATA_UINT64 }, 764 { "anon_evictable_data", KSTAT_DATA_UINT64 }, 765 { "anon_evictable_metadata", KSTAT_DATA_UINT64 }, 766 { "mru_size", KSTAT_DATA_UINT64 }, 767 { "mru_evictable_data", KSTAT_DATA_UINT64 }, 768 { "mru_evictable_metadata", KSTAT_DATA_UINT64 }, 769 { "mru_ghost_size", KSTAT_DATA_UINT64 }, 770 { "mru_ghost_evictable_data", KSTAT_DATA_UINT64 }, 771 { "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 }, 772 { "mfu_size", KSTAT_DATA_UINT64 }, 773 { "mfu_evictable_data", KSTAT_DATA_UINT64 }, 774 { "mfu_evictable_metadata", KSTAT_DATA_UINT64 }, 775 { "mfu_ghost_size", KSTAT_DATA_UINT64 }, 776 { "mfu_ghost_evictable_data", KSTAT_DATA_UINT64 }, 777 { "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 }, 778 { "l2_hits", KSTAT_DATA_UINT64 }, 779 { "l2_misses", KSTAT_DATA_UINT64 }, 780 { "l2_feeds", KSTAT_DATA_UINT64 }, 781 { "l2_rw_clash", KSTAT_DATA_UINT64 }, 782 { "l2_read_bytes", KSTAT_DATA_UINT64 }, 783 { "l2_write_bytes", KSTAT_DATA_UINT64 }, 784 { "l2_writes_sent", KSTAT_DATA_UINT64 }, 785 { "l2_writes_done", KSTAT_DATA_UINT64 }, 786 { "l2_writes_error", KSTAT_DATA_UINT64 }, 787 { "l2_writes_lock_retry", KSTAT_DATA_UINT64 }, 788 { "l2_evict_lock_retry", KSTAT_DATA_UINT64 }, 789 { "l2_evict_reading", KSTAT_DATA_UINT64 }, 790 { "l2_evict_l1cached", KSTAT_DATA_UINT64 }, 791 { "l2_free_on_write", KSTAT_DATA_UINT64 }, 792 { "l2_abort_lowmem", KSTAT_DATA_UINT64 }, 793 { "l2_cksum_bad", KSTAT_DATA_UINT64 }, 794 { "l2_io_error", KSTAT_DATA_UINT64 }, 795 { "l2_size", KSTAT_DATA_UINT64 }, 796 { "l2_asize", KSTAT_DATA_UINT64 }, 797 { "l2_hdr_size", KSTAT_DATA_UINT64 }, 798 { "memory_throttle_count", KSTAT_DATA_UINT64 }, 799 { "arc_meta_used", KSTAT_DATA_UINT64 }, 800 { "arc_meta_limit", KSTAT_DATA_UINT64 }, 801 { "arc_meta_max", KSTAT_DATA_UINT64 }, 802 { "arc_meta_min", KSTAT_DATA_UINT64 }, 803 { "async_upgrade_sync", KSTAT_DATA_UINT64 }, 804 { "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 }, 805 { "demand_hit_prescient_prefetch", KSTAT_DATA_UINT64 }, 806}; 807 808#define ARCSTAT(stat) (arc_stats.stat.value.ui64) 809 810#define ARCSTAT_INCR(stat, val) \ 811 atomic_add_64(&arc_stats.stat.value.ui64, (val)) 812 813#define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) 814#define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1) 815 816#define ARCSTAT_MAX(stat, val) { \ 817 uint64_t m; \ 818 while ((val) > (m = arc_stats.stat.value.ui64) && \ 819 (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \ 820 continue; \ 821} 822 823#define ARCSTAT_MAXSTAT(stat) \ 824 ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64) 825 826/* 827 * We define a macro to allow ARC hits/misses to be easily broken down by 828 * two separate conditions, giving a total of four different subtypes for 829 * each of hits and misses (so eight statistics total). 830 */ 831#define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \ 832 if (cond1) { \ 833 if (cond2) { \ 834 ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \ 835 } else { \ 836 ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \ 837 } \ 838 } else { \ 839 if (cond2) { \ 840 ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \ 841 } else { \ 842 ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\ 843 } \ 844 } 845 846kstat_t *arc_ksp; 847static arc_state_t *arc_anon; 848static arc_state_t *arc_mru; 849static arc_state_t *arc_mru_ghost; 850static arc_state_t *arc_mfu; 851static arc_state_t *arc_mfu_ghost; 852static arc_state_t *arc_l2c_only; 853 854/* 855 * There are several ARC variables that are critical to export as kstats -- 856 * but we don't want to have to grovel around in the kstat whenever we wish to 857 * manipulate them. For these variables, we therefore define them to be in 858 * terms of the statistic variable. This assures that we are not introducing 859 * the possibility of inconsistency by having shadow copies of the variables, 860 * while still allowing the code to be readable. 861 */ 862#define arc_p ARCSTAT(arcstat_p) /* target size of MRU */ 863#define arc_c ARCSTAT(arcstat_c) /* target size of cache */ 864#define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ 865#define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ 866#define arc_meta_limit ARCSTAT(arcstat_meta_limit) /* max size for metadata */ 867#define arc_meta_min ARCSTAT(arcstat_meta_min) /* min size for metadata */ 868#define arc_meta_max ARCSTAT(arcstat_meta_max) /* max size of metadata */ 869 870/* compressed size of entire arc */ 871#define arc_compressed_size ARCSTAT(arcstat_compressed_size) 872/* uncompressed size of entire arc */ 873#define arc_uncompressed_size ARCSTAT(arcstat_uncompressed_size) 874/* number of bytes in the arc from arc_buf_t's */ 875#define arc_overhead_size ARCSTAT(arcstat_overhead_size) 876 877/* 878 * There are also some ARC variables that we want to export, but that are 879 * updated so often that having the canonical representation be the statistic 880 * variable causes a performance bottleneck. We want to use aggsum_t's for these 881 * instead, but still be able to export the kstat in the same way as before. 882 * The solution is to always use the aggsum version, except in the kstat update 883 * callback. 884 */ 885aggsum_t arc_size; 886aggsum_t arc_meta_used; 887aggsum_t astat_data_size; 888aggsum_t astat_metadata_size; 889aggsum_t astat_hdr_size; 890aggsum_t astat_other_size; 891aggsum_t astat_l2_hdr_size; 892 893static int arc_no_grow; /* Don't try to grow cache size */ 894static hrtime_t arc_growtime; 895static uint64_t arc_tempreserve; 896static uint64_t arc_loaned_bytes; 897 898typedef struct arc_callback arc_callback_t; 899 900struct arc_callback { 901 void *acb_private; 902 arc_read_done_func_t *acb_done; 903 arc_buf_t *acb_buf; 904 boolean_t acb_encrypted; 905 boolean_t acb_compressed; 906 boolean_t acb_noauth; 907 zbookmark_phys_t acb_zb; 908 zio_t *acb_zio_dummy; 909 zio_t *acb_zio_head; 910 arc_callback_t *acb_next; 911}; 912 913typedef struct arc_write_callback arc_write_callback_t; 914 915struct arc_write_callback { 916 void *awcb_private; 917 arc_write_done_func_t *awcb_ready; 918 arc_write_done_func_t *awcb_children_ready; 919 arc_write_done_func_t *awcb_physdone; 920 arc_write_done_func_t *awcb_done; 921 arc_buf_t *awcb_buf; 922}; 923 924/* 925 * ARC buffers are separated into multiple structs as a memory saving measure: 926 * - Common fields struct, always defined, and embedded within it: 927 * - L2-only fields, always allocated but undefined when not in L2ARC 928 * - L1-only fields, only allocated when in L1ARC 929 * 930 * Buffer in L1 Buffer only in L2 931 * +------------------------+ +------------------------+ 932 * | arc_buf_hdr_t | | arc_buf_hdr_t | 933 * | | | | 934 * | | | | 935 * | | | | 936 * +------------------------+ +------------------------+ 937 * | l2arc_buf_hdr_t | | l2arc_buf_hdr_t | 938 * | (undefined if L1-only) | | | 939 * +------------------------+ +------------------------+ 940 * | l1arc_buf_hdr_t | 941 * | | 942 * | | 943 * | | 944 * | | 945 * +------------------------+ 946 * 947 * Because it's possible for the L2ARC to become extremely large, we can wind 948 * up eating a lot of memory in L2ARC buffer headers, so the size of a header 949 * is minimized by only allocating the fields necessary for an L1-cached buffer 950 * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and 951 * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple 952 * words in pointers. arc_hdr_realloc() is used to switch a header between 953 * these two allocation states. 954 */ 955typedef struct l1arc_buf_hdr { 956 kmutex_t b_freeze_lock; 957 zio_cksum_t *b_freeze_cksum; 958#ifdef ZFS_DEBUG 959 /* 960 * Used for debugging with kmem_flags - by allocating and freeing 961 * b_thawed when the buffer is thawed, we get a record of the stack 962 * trace that thawed it. 963 */ 964 void *b_thawed; 965#endif 966 967 arc_buf_t *b_buf; 968 uint32_t b_bufcnt; 969 /* for waiting on writes to complete */ 970 kcondvar_t b_cv; 971 uint8_t b_byteswap; 972 973 /* protected by arc state mutex */ 974 arc_state_t *b_state; 975 multilist_node_t b_arc_node; 976 977 /* updated atomically */ 978 clock_t b_arc_access; 979 980 /* self protecting */ 981 zfs_refcount_t b_refcnt; 982 983 arc_callback_t *b_acb; 984 abd_t *b_pabd; 985} l1arc_buf_hdr_t; 986 987/* 988 * Encrypted blocks will need to be stored encrypted on the L2ARC 989 * disk as they appear in the main pool. In order for this to work we 990 * need to pass around the encryption parameters so they can be used 991 * to write data to the L2ARC. This struct is only defined in the 992 * arc_buf_hdr_t if the L1 header is defined and has the ARC_FLAG_ENCRYPTED 993 * flag set. 994 */ 995typedef struct arc_buf_hdr_crypt { 996 abd_t *b_rabd; /* raw encrypted data */ 997 dmu_object_type_t b_ot; /* object type */ 998 uint32_t b_ebufcnt; /* number or encryped buffers */ 999 1000 /* dsobj for looking up encryption key for l2arc encryption */ 1001 uint64_t b_dsobj; /* for looking up key */ 1002 1003 /* encryption parameters */ 1004 uint8_t b_salt[ZIO_DATA_SALT_LEN]; 1005 uint8_t b_iv[ZIO_DATA_IV_LEN]; 1006 1007 /* 1008 * Technically this could be removed since we will always be able to 1009 * get the mac from the bp when we need it. However, it is inconvenient 1010 * for callers of arc code to have to pass a bp in all the time. This 1011 * also allows us to assert that L2ARC data is properly encrypted to 1012 * match the data in the main storage pool. 1013 */ 1014 uint8_t b_mac[ZIO_DATA_MAC_LEN]; 1015} arc_buf_hdr_crypt_t; 1016 1017typedef struct l2arc_dev l2arc_dev_t; 1018 1019typedef struct l2arc_buf_hdr { 1020 /* protected by arc_buf_hdr mutex */ 1021 l2arc_dev_t *b_dev; /* L2ARC device */ 1022 uint64_t b_daddr; /* disk address, offset byte */ 1023 1024 list_node_t b_l2node; 1025} l2arc_buf_hdr_t; 1026 1027struct arc_buf_hdr { 1028 /* protected by hash lock */ 1029 dva_t b_dva; 1030 uint64_t b_birth; 1031 1032 arc_buf_contents_t b_type; 1033 arc_buf_hdr_t *b_hash_next; 1034 arc_flags_t b_flags; 1035 1036 /* 1037 * This field stores the size of the data buffer after 1038 * compression, and is set in the arc's zio completion handlers. 1039 * It is in units of SPA_MINBLOCKSIZE (e.g. 1 == 512 bytes). 1040 * 1041 * While the block pointers can store up to 32MB in their psize 1042 * field, we can only store up to 32MB minus 512B. This is due 1043 * to the bp using a bias of 1, whereas we use a bias of 0 (i.e. 1044 * a field of zeros represents 512B in the bp). We can't use a 1045 * bias of 1 since we need to reserve a psize of zero, here, to 1046 * represent holes and embedded blocks. 1047 * 1048 * This isn't a problem in practice, since the maximum size of a 1049 * buffer is limited to 16MB, so we never need to store 32MB in 1050 * this field. Even in the upstream illumos code base, the 1051 * maximum size of a buffer is limited to 16MB. 1052 */ 1053 uint16_t b_psize; 1054 1055 /* 1056 * This field stores the size of the data buffer before 1057 * compression, and cannot change once set. It is in units 1058 * of SPA_MINBLOCKSIZE (e.g. 2 == 1024 bytes) 1059 */ 1060 uint16_t b_lsize; /* immutable */ 1061 uint64_t b_spa; /* immutable */ 1062 1063 /* L2ARC fields. Undefined when not in L2ARC. */ 1064 l2arc_buf_hdr_t b_l2hdr; 1065 /* L1ARC fields. Undefined when in l2arc_only state */ 1066 l1arc_buf_hdr_t b_l1hdr; 1067 /* 1068 * Encryption parameters. Defined only when ARC_FLAG_ENCRYPTED 1069 * is set and the L1 header exists. 1070 */ 1071 arc_buf_hdr_crypt_t b_crypt_hdr; 1072}; 1073 1074#define GHOST_STATE(state) \ 1075 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \ 1076 (state) == arc_l2c_only) 1077 1078#define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE) 1079#define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) 1080#define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR) 1081#define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH) 1082#define HDR_PRESCIENT_PREFETCH(hdr) \ 1083 ((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH) 1084#define HDR_COMPRESSION_ENABLED(hdr) \ 1085 ((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC) 1086 1087#define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE) 1088#define HDR_L2_READING(hdr) \ 1089 (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \ 1090 ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)) 1091#define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING) 1092#define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED) 1093#define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD) 1094#define HDR_PROTECTED(hdr) ((hdr)->b_flags & ARC_FLAG_PROTECTED) 1095#define HDR_NOAUTH(hdr) ((hdr)->b_flags & ARC_FLAG_NOAUTH) 1096#define HDR_SHARED_DATA(hdr) ((hdr)->b_flags & ARC_FLAG_SHARED_DATA) 1097 1098#define HDR_ISTYPE_METADATA(hdr) \ 1099 ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA) 1100#define HDR_ISTYPE_DATA(hdr) (!HDR_ISTYPE_METADATA(hdr)) 1101 1102#define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR) 1103#define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR) 1104#define HDR_HAS_RABD(hdr) \ 1105 (HDR_HAS_L1HDR(hdr) && HDR_PROTECTED(hdr) && \ 1106 (hdr)->b_crypt_hdr.b_rabd != NULL) 1107#define HDR_ENCRYPTED(hdr) \ 1108 (HDR_PROTECTED(hdr) && DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot)) 1109#define HDR_AUTHENTICATED(hdr) \ 1110 (HDR_PROTECTED(hdr) && !DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot)) 1111 1112/* For storing compression mode in b_flags */ 1113#define HDR_COMPRESS_OFFSET (highbit64(ARC_FLAG_COMPRESS_0) - 1) 1114 1115#define HDR_GET_COMPRESS(hdr) ((enum zio_compress)BF32_GET((hdr)->b_flags, \ 1116 HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS)) 1117#define HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \ 1118 HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS, (cmp)); 1119 1120#define ARC_BUF_LAST(buf) ((buf)->b_next == NULL) 1121#define ARC_BUF_SHARED(buf) ((buf)->b_flags & ARC_BUF_FLAG_SHARED) 1122#define ARC_BUF_COMPRESSED(buf) ((buf)->b_flags & ARC_BUF_FLAG_COMPRESSED) 1123#define ARC_BUF_ENCRYPTED(buf) ((buf)->b_flags & ARC_BUF_FLAG_ENCRYPTED) 1124 1125/* 1126 * Other sizes 1127 */ 1128 1129#define HDR_FULL_CRYPT_SIZE ((int64_t)sizeof (arc_buf_hdr_t)) 1130#define HDR_FULL_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_crypt_hdr)) 1131#define HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr)) 1132 1133/* 1134 * Hash table routines 1135 */ 1136 1137#define HT_LOCK_PAD 64 1138 1139struct ht_lock { 1140 kmutex_t ht_lock; 1141#ifdef _KERNEL 1142 unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))]; 1143#endif 1144}; 1145 1146#define BUF_LOCKS 256 1147typedef struct buf_hash_table { 1148 uint64_t ht_mask; 1149 arc_buf_hdr_t **ht_table; 1150 struct ht_lock ht_locks[BUF_LOCKS]; 1151} buf_hash_table_t; 1152 1153static buf_hash_table_t buf_hash_table; 1154 1155#define BUF_HASH_INDEX(spa, dva, birth) \ 1156 (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask) 1157#define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)]) 1158#define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock)) 1159#define HDR_LOCK(hdr) \ 1160 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth))) 1161 1162uint64_t zfs_crc64_table[256]; 1163 1164/* 1165 * Level 2 ARC 1166 */ 1167 1168#define L2ARC_WRITE_SIZE (8 * 1024 * 1024) /* initial write max */ 1169#define L2ARC_HEADROOM 2 /* num of writes */ 1170/* 1171 * If we discover during ARC scan any buffers to be compressed, we boost 1172 * our headroom for the next scanning cycle by this percentage multiple. 1173 */ 1174#define L2ARC_HEADROOM_BOOST 200 1175#define L2ARC_FEED_SECS 1 /* caching interval secs */ 1176#define L2ARC_FEED_MIN_MS 200 /* min caching interval ms */ 1177 1178#define l2arc_writes_sent ARCSTAT(arcstat_l2_writes_sent) 1179#define l2arc_writes_done ARCSTAT(arcstat_l2_writes_done) 1180 1181/* L2ARC Performance Tunables */ 1182uint64_t l2arc_write_max = L2ARC_WRITE_SIZE; /* default max write size */ 1183uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE; /* extra write during warmup */ 1184uint64_t l2arc_headroom = L2ARC_HEADROOM; /* number of dev writes */ 1185uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST; 1186uint64_t l2arc_feed_secs = L2ARC_FEED_SECS; /* interval seconds */ 1187uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval milliseconds */ 1188boolean_t l2arc_noprefetch = B_TRUE; /* don't cache prefetch bufs */ 1189boolean_t l2arc_feed_again = B_TRUE; /* turbo warmup */ 1190boolean_t l2arc_norw = B_TRUE; /* no reads during writes */ 1191 1192/* 1193 * L2ARC Internals 1194 */ 1195struct l2arc_dev { 1196 vdev_t *l2ad_vdev; /* vdev */ 1197 spa_t *l2ad_spa; /* spa */ 1198 uint64_t l2ad_hand; /* next write location */ 1199 uint64_t l2ad_start; /* first addr on device */ 1200 uint64_t l2ad_end; /* last addr on device */ 1201 boolean_t l2ad_first; /* first sweep through */ 1202 boolean_t l2ad_writing; /* currently writing */ 1203 kmutex_t l2ad_mtx; /* lock for buffer list */ 1204 list_t l2ad_buflist; /* buffer list */ 1205 list_node_t l2ad_node; /* device list node */ 1206 zfs_refcount_t l2ad_alloc; /* allocated bytes */ 1207}; 1208 1209static list_t L2ARC_dev_list; /* device list */ 1210static list_t *l2arc_dev_list; /* device list pointer */ 1211static kmutex_t l2arc_dev_mtx; /* device list mutex */ 1212static l2arc_dev_t *l2arc_dev_last; /* last device used */ 1213static list_t L2ARC_free_on_write; /* free after write buf list */ 1214static list_t *l2arc_free_on_write; /* free after write list ptr */ 1215static kmutex_t l2arc_free_on_write_mtx; /* mutex for list */ 1216static uint64_t l2arc_ndev; /* number of devices */ 1217 1218typedef struct l2arc_read_callback { 1219 arc_buf_hdr_t *l2rcb_hdr; /* read header */ 1220 blkptr_t l2rcb_bp; /* original blkptr */ 1221 zbookmark_phys_t l2rcb_zb; /* original bookmark */ 1222 int l2rcb_flags; /* original flags */ 1223 abd_t *l2rcb_abd; /* temporary buffer */ 1224} l2arc_read_callback_t; 1225 1226typedef struct l2arc_write_callback { 1227 l2arc_dev_t *l2wcb_dev; /* device info */ 1228 arc_buf_hdr_t *l2wcb_head; /* head of write buflist */ 1229} l2arc_write_callback_t; 1230 1231typedef struct l2arc_data_free { 1232 /* protected by l2arc_free_on_write_mtx */ 1233 abd_t *l2df_abd; 1234 size_t l2df_size; 1235 arc_buf_contents_t l2df_type; 1236 list_node_t l2df_list_node; 1237} l2arc_data_free_t; 1238 1239static kmutex_t l2arc_feed_thr_lock; 1240static kcondvar_t l2arc_feed_thr_cv; 1241static uint8_t l2arc_thread_exit; 1242 1243static abd_t *arc_get_data_abd(arc_buf_hdr_t *, uint64_t, void *); 1244typedef enum arc_fill_flags { 1245 ARC_FILL_LOCKED = 1 << 0, /* hdr lock is held */ 1246 ARC_FILL_COMPRESSED = 1 << 1, /* fill with compressed data */ 1247 ARC_FILL_ENCRYPTED = 1 << 2, /* fill with encrypted data */ 1248 ARC_FILL_NOAUTH = 1 << 3, /* don't attempt to authenticate */ 1249 ARC_FILL_IN_PLACE = 1 << 4 /* fill in place (special case) */ 1250} arc_fill_flags_t; 1251 1252static void *arc_get_data_buf(arc_buf_hdr_t *, uint64_t, void *); 1253static void arc_get_data_impl(arc_buf_hdr_t *, uint64_t, void *); 1254static void arc_free_data_abd(arc_buf_hdr_t *, abd_t *, uint64_t, void *); 1255static void arc_free_data_buf(arc_buf_hdr_t *, void *, uint64_t, void *); 1256static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag); 1257static void arc_hdr_free_pabd(arc_buf_hdr_t *, boolean_t); 1258static void arc_hdr_alloc_pabd(arc_buf_hdr_t *, boolean_t); 1259static void arc_access(arc_buf_hdr_t *, kmutex_t *); 1260static boolean_t arc_is_overflowing(); 1261static void arc_buf_watch(arc_buf_t *); 1262 1263static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *); 1264static uint32_t arc_bufc_to_flags(arc_buf_contents_t); 1265static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags); 1266static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags); 1267 1268static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *); 1269static void l2arc_read_done(zio_t *); 1270 1271/* 1272 * The arc_all_memory function is a ZoL enhancement that lives in their OSL 1273 * code. In user-space code, which is used primarily for testing, we return 1274 * half of all memory. 1275 */ 1276uint64_t 1277arc_all_memory(void) 1278{ 1279#ifdef _KERNEL 1280 return (ptob(physmem)); 1281#else 1282 return ((sysconf(_SC_PAGESIZE) * sysconf(_SC_PHYS_PAGES)) / 2); 1283#endif 1284} 1285 1286/* 1287 * We use Cityhash for this. It's fast, and has good hash properties without 1288 * requiring any large static buffers. 1289 */ 1290static uint64_t 1291buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth) 1292{ 1293 return (cityhash4(spa, dva->dva_word[0], dva->dva_word[1], birth)); 1294} 1295 1296#define HDR_EMPTY(hdr) \ 1297 ((hdr)->b_dva.dva_word[0] == 0 && \ 1298 (hdr)->b_dva.dva_word[1] == 0) 1299 1300#define HDR_EQUAL(spa, dva, birth, hdr) \ 1301 ((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \ 1302 ((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \ 1303 ((hdr)->b_birth == birth) && ((hdr)->b_spa == spa) 1304 1305static void 1306buf_discard_identity(arc_buf_hdr_t *hdr) 1307{ 1308 hdr->b_dva.dva_word[0] = 0; 1309 hdr->b_dva.dva_word[1] = 0; 1310 hdr->b_birth = 0; 1311} 1312 1313static arc_buf_hdr_t * 1314buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp) 1315{ 1316 const dva_t *dva = BP_IDENTITY(bp); 1317 uint64_t birth = BP_PHYSICAL_BIRTH(bp); 1318 uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); 1319 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 1320 arc_buf_hdr_t *hdr; 1321 1322 mutex_enter(hash_lock); 1323 for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL; 1324 hdr = hdr->b_hash_next) { 1325 if (HDR_EQUAL(spa, dva, birth, hdr)) { 1326 *lockp = hash_lock; 1327 return (hdr); 1328 } 1329 } 1330 mutex_exit(hash_lock); 1331 *lockp = NULL; 1332 return (NULL); 1333} 1334 1335/* 1336 * Insert an entry into the hash table. If there is already an element 1337 * equal to elem in the hash table, then the already existing element 1338 * will be returned and the new element will not be inserted. 1339 * Otherwise returns NULL. 1340 * If lockp == NULL, the caller is assumed to already hold the hash lock. 1341 */ 1342static arc_buf_hdr_t * 1343buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp) 1344{ 1345 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth); 1346 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 1347 arc_buf_hdr_t *fhdr; 1348 uint32_t i; 1349 1350 ASSERT(!DVA_IS_EMPTY(&hdr->b_dva)); 1351 ASSERT(hdr->b_birth != 0); 1352 ASSERT(!HDR_IN_HASH_TABLE(hdr)); 1353 1354 if (lockp != NULL) { 1355 *lockp = hash_lock; 1356 mutex_enter(hash_lock); 1357 } else { 1358 ASSERT(MUTEX_HELD(hash_lock)); 1359 } 1360 1361 for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL; 1362 fhdr = fhdr->b_hash_next, i++) { 1363 if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr)) 1364 return (fhdr); 1365 } 1366 1367 hdr->b_hash_next = buf_hash_table.ht_table[idx]; 1368 buf_hash_table.ht_table[idx] = hdr; 1369 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE); 1370 1371 /* collect some hash table performance data */ 1372 if (i > 0) { 1373 ARCSTAT_BUMP(arcstat_hash_collisions); 1374 if (i == 1) 1375 ARCSTAT_BUMP(arcstat_hash_chains); 1376 1377 ARCSTAT_MAX(arcstat_hash_chain_max, i); 1378 } 1379 1380 ARCSTAT_BUMP(arcstat_hash_elements); 1381 ARCSTAT_MAXSTAT(arcstat_hash_elements); 1382 1383 return (NULL); 1384} 1385 1386static void 1387buf_hash_remove(arc_buf_hdr_t *hdr) 1388{ 1389 arc_buf_hdr_t *fhdr, **hdrp; 1390 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth); 1391 1392 ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx))); 1393 ASSERT(HDR_IN_HASH_TABLE(hdr)); 1394 1395 hdrp = &buf_hash_table.ht_table[idx]; 1396 while ((fhdr = *hdrp) != hdr) { 1397 ASSERT3P(fhdr, !=, NULL); 1398 hdrp = &fhdr->b_hash_next; 1399 } 1400 *hdrp = hdr->b_hash_next; 1401 hdr->b_hash_next = NULL; 1402 arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE); 1403 1404 /* collect some hash table performance data */ 1405 ARCSTAT_BUMPDOWN(arcstat_hash_elements); 1406 1407 if (buf_hash_table.ht_table[idx] && 1408 buf_hash_table.ht_table[idx]->b_hash_next == NULL) 1409 ARCSTAT_BUMPDOWN(arcstat_hash_chains); 1410} 1411 1412/* 1413 * Global data structures and functions for the buf kmem cache. 1414 */ 1415 1416static kmem_cache_t *hdr_full_cache; 1417static kmem_cache_t *hdr_full_crypt_cache; 1418static kmem_cache_t *hdr_l2only_cache; 1419static kmem_cache_t *buf_cache; 1420 1421static void 1422buf_fini(void) 1423{ 1424 int i; 1425 1426 kmem_free(buf_hash_table.ht_table, 1427 (buf_hash_table.ht_mask + 1) * sizeof (void *)); 1428 for (i = 0; i < BUF_LOCKS; i++) 1429 mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock); 1430 kmem_cache_destroy(hdr_full_cache); 1431 kmem_cache_destroy(hdr_full_crypt_cache); 1432 kmem_cache_destroy(hdr_l2only_cache); 1433 kmem_cache_destroy(buf_cache); 1434} 1435 1436/* 1437 * Constructor callback - called when the cache is empty 1438 * and a new buf is requested. 1439 */ 1440/* ARGSUSED */ 1441static int 1442hdr_full_cons(void *vbuf, void *unused, int kmflag) 1443{ 1444 arc_buf_hdr_t *hdr = vbuf; 1445 1446 bzero(hdr, HDR_FULL_SIZE); 1447 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; 1448 cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL); 1449 zfs_refcount_create(&hdr->b_l1hdr.b_refcnt); 1450 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL); 1451 multilist_link_init(&hdr->b_l1hdr.b_arc_node); 1452 arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS); 1453 1454 return (0); 1455} 1456 1457/* ARGSUSED */ 1458static int 1459hdr_full_crypt_cons(void *vbuf, void *unused, int kmflag) 1460{ 1461 arc_buf_hdr_t *hdr = vbuf; 1462 1463 (void) hdr_full_cons(vbuf, unused, kmflag); 1464 bzero(&hdr->b_crypt_hdr, sizeof (hdr->b_crypt_hdr)); 1465 arc_space_consume(sizeof (hdr->b_crypt_hdr), ARC_SPACE_HDRS); 1466 1467 return (0); 1468} 1469 1470/* ARGSUSED */ 1471static int 1472hdr_l2only_cons(void *vbuf, void *unused, int kmflag) 1473{ 1474 arc_buf_hdr_t *hdr = vbuf; 1475 1476 bzero(hdr, HDR_L2ONLY_SIZE); 1477 arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS); 1478 1479 return (0); 1480} 1481 1482/* ARGSUSED */ 1483static int 1484buf_cons(void *vbuf, void *unused, int kmflag) 1485{ 1486 arc_buf_t *buf = vbuf; 1487 1488 bzero(buf, sizeof (arc_buf_t)); 1489 mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL); 1490 arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS); 1491 1492 return (0); 1493} 1494 1495/* 1496 * Destructor callback - called when a cached buf is 1497 * no longer required. 1498 */ 1499/* ARGSUSED */ 1500static void 1501hdr_full_dest(void *vbuf, void *unused) 1502{ 1503 arc_buf_hdr_t *hdr = vbuf; 1504 1505 ASSERT(HDR_EMPTY(hdr)); 1506 cv_destroy(&hdr->b_l1hdr.b_cv); 1507 zfs_refcount_destroy(&hdr->b_l1hdr.b_refcnt); 1508 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock); 1509 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); 1510 arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS); 1511} 1512 1513/* ARGSUSED */ 1514static void 1515hdr_full_crypt_dest(void *vbuf, void *unused) 1516{ 1517 arc_buf_hdr_t *hdr = vbuf; 1518 1519 hdr_full_dest(hdr, unused); 1520 arc_space_return(sizeof (hdr->b_crypt_hdr), ARC_SPACE_HDRS); 1521} 1522 1523/* ARGSUSED */ 1524static void 1525hdr_l2only_dest(void *vbuf, void *unused) 1526{ 1527 arc_buf_hdr_t *hdr = vbuf; 1528 1529 ASSERT(HDR_EMPTY(hdr)); 1530 arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS); 1531} 1532 1533/* ARGSUSED */ 1534static void 1535buf_dest(void *vbuf, void *unused) 1536{ 1537 arc_buf_t *buf = vbuf; 1538 1539 mutex_destroy(&buf->b_evict_lock); 1540 arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS); 1541} 1542 1543/* 1544 * Reclaim callback -- invoked when memory is low. 1545 */ 1546/* ARGSUSED */ 1547static void 1548hdr_recl(void *unused) 1549{ 1550 dprintf("hdr_recl called\n"); 1551 /* 1552 * umem calls the reclaim func when we destroy the buf cache, 1553 * which is after we do arc_fini(). 1554 */ 1555 if (arc_initialized) 1556 zthr_wakeup(arc_reap_zthr); 1557} 1558 1559static void 1560buf_init(void) 1561{ 1562 uint64_t *ct; 1563 uint64_t hsize = 1ULL << 12; 1564 int i, j; 1565 1566 /* 1567 * The hash table is big enough to fill all of physical memory 1568 * with an average block size of zfs_arc_average_blocksize (default 8K). 1569 * By default, the table will take up 1570 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers). 1571 */ 1572 while (hsize * zfs_arc_average_blocksize < physmem * PAGESIZE) 1573 hsize <<= 1; 1574retry: 1575 buf_hash_table.ht_mask = hsize - 1; 1576 buf_hash_table.ht_table = 1577 kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP); 1578 if (buf_hash_table.ht_table == NULL) { 1579 ASSERT(hsize > (1ULL << 8)); 1580 hsize >>= 1; 1581 goto retry; 1582 } 1583 1584 hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE, 1585 0, hdr_full_cons, hdr_full_dest, hdr_recl, NULL, NULL, 0); 1586 hdr_full_crypt_cache = kmem_cache_create("arc_buf_hdr_t_full_crypt", 1587 HDR_FULL_CRYPT_SIZE, 0, hdr_full_crypt_cons, hdr_full_crypt_dest, 1588 hdr_recl, NULL, NULL, 0); 1589 hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only", 1590 HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, hdr_recl, 1591 NULL, NULL, 0); 1592 buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t), 1593 0, buf_cons, buf_dest, NULL, NULL, NULL, 0); 1594 1595 for (i = 0; i < 256; i++) 1596 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--) 1597 *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY); 1598 1599 for (i = 0; i < BUF_LOCKS; i++) { 1600 mutex_init(&buf_hash_table.ht_locks[i].ht_lock, 1601 NULL, MUTEX_DEFAULT, NULL); 1602 } 1603} 1604 1605/* 1606 * This is the size that the buf occupies in memory. If the buf is compressed, 1607 * it will correspond to the compressed size. You should use this method of 1608 * getting the buf size unless you explicitly need the logical size. 1609 */ 1610int32_t 1611arc_buf_size(arc_buf_t *buf) 1612{ 1613 return (ARC_BUF_COMPRESSED(buf) ? 1614 HDR_GET_PSIZE(buf->b_hdr) : HDR_GET_LSIZE(buf->b_hdr)); 1615} 1616 1617int32_t 1618arc_buf_lsize(arc_buf_t *buf) 1619{ 1620 return (HDR_GET_LSIZE(buf->b_hdr)); 1621} 1622 1623/* 1624 * This function will return B_TRUE if the buffer is encrypted in memory. 1625 * This buffer can be decrypted by calling arc_untransform(). 1626 */ 1627boolean_t 1628arc_is_encrypted(arc_buf_t *buf) 1629{ 1630 return (ARC_BUF_ENCRYPTED(buf) != 0); 1631} 1632 1633/* 1634 * Returns B_TRUE if the buffer represents data that has not had its MAC 1635 * verified yet. 1636 */ 1637boolean_t 1638arc_is_unauthenticated(arc_buf_t *buf) 1639{ 1640 return (HDR_NOAUTH(buf->b_hdr) != 0); 1641} 1642 1643void 1644arc_get_raw_params(arc_buf_t *buf, boolean_t *byteorder, uint8_t *salt, 1645 uint8_t *iv, uint8_t *mac) 1646{ 1647 arc_buf_hdr_t *hdr = buf->b_hdr; 1648 1649 ASSERT(HDR_PROTECTED(hdr)); 1650 1651 bcopy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN); 1652 bcopy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN); 1653 bcopy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN); 1654 *byteorder = (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ? 1655 /* CONSTCOND */ 1656 ZFS_HOST_BYTEORDER : !ZFS_HOST_BYTEORDER; 1657} 1658 1659/* 1660 * Indicates how this buffer is compressed in memory. If it is not compressed 1661 * the value will be ZIO_COMPRESS_OFF. It can be made normally readable with 1662 * arc_untransform() as long as it is also unencrypted. 1663 */ 1664enum zio_compress 1665arc_get_compression(arc_buf_t *buf) 1666{ 1667 return (ARC_BUF_COMPRESSED(buf) ? 1668 HDR_GET_COMPRESS(buf->b_hdr) : ZIO_COMPRESS_OFF); 1669} 1670 1671#define ARC_MINTIME (hz>>4) /* 62 ms */ 1672 1673/* 1674 * Return the compression algorithm used to store this data in the ARC. If ARC 1675 * compression is enabled or this is an encrypted block, this will be the same 1676 * as what's used to store it on-disk. Otherwise, this will be ZIO_COMPRESS_OFF. 1677 */ 1678static inline enum zio_compress 1679arc_hdr_get_compress(arc_buf_hdr_t *hdr) 1680{ 1681 return (HDR_COMPRESSION_ENABLED(hdr) ? 1682 HDR_GET_COMPRESS(hdr) : ZIO_COMPRESS_OFF); 1683} 1684 1685static inline boolean_t 1686arc_buf_is_shared(arc_buf_t *buf) 1687{ 1688 boolean_t shared = (buf->b_data != NULL && 1689 buf->b_hdr->b_l1hdr.b_pabd != NULL && 1690 abd_is_linear(buf->b_hdr->b_l1hdr.b_pabd) && 1691 buf->b_data == abd_to_buf(buf->b_hdr->b_l1hdr.b_pabd)); 1692 IMPLY(shared, HDR_SHARED_DATA(buf->b_hdr)); 1693 IMPLY(shared, ARC_BUF_SHARED(buf)); 1694 IMPLY(shared, ARC_BUF_COMPRESSED(buf) || ARC_BUF_LAST(buf)); 1695 1696 /* 1697 * It would be nice to assert arc_can_share() too, but the "hdr isn't 1698 * already being shared" requirement prevents us from doing that. 1699 */ 1700 1701 return (shared); 1702} 1703 1704/* 1705 * Free the checksum associated with this header. If there is no checksum, this 1706 * is a no-op. 1707 */ 1708static inline void 1709arc_cksum_free(arc_buf_hdr_t *hdr) 1710{ 1711 ASSERT(HDR_HAS_L1HDR(hdr)); 1712 1713 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); 1714 if (hdr->b_l1hdr.b_freeze_cksum != NULL) { 1715 kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t)); 1716 hdr->b_l1hdr.b_freeze_cksum = NULL; 1717 } 1718 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1719} 1720 1721/* 1722 * Return true iff at least one of the bufs on hdr is not compressed. 1723 * Encrypted buffers count as compressed. 1724 */ 1725static boolean_t 1726arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr) 1727{ 1728 ASSERT(hdr->b_l1hdr.b_state == arc_anon || 1729 MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 1730 1731 for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) { 1732 if (!ARC_BUF_COMPRESSED(b)) { 1733 return (B_TRUE); 1734 } 1735 } 1736 return (B_FALSE); 1737} 1738 1739/* 1740 * If we've turned on the ZFS_DEBUG_MODIFY flag, verify that the buf's data 1741 * matches the checksum that is stored in the hdr. If there is no checksum, 1742 * or if the buf is compressed, this is a no-op. 1743 */ 1744static void 1745arc_cksum_verify(arc_buf_t *buf) 1746{ 1747 arc_buf_hdr_t *hdr = buf->b_hdr; 1748 zio_cksum_t zc; 1749 1750 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 1751 return; 1752 1753 if (ARC_BUF_COMPRESSED(buf)) 1754 return; 1755 1756 ASSERT(HDR_HAS_L1HDR(hdr)); 1757 1758 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); 1759 1760 if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) { 1761 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1762 return; 1763 } 1764 1765 fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, &zc); 1766 if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc)) 1767 panic("buffer modified while frozen!"); 1768 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1769} 1770 1771/* 1772 * This function makes the assumption that data stored in the L2ARC 1773 * will be transformed exactly as it is in the main pool. Because of 1774 * this we can verify the checksum against the reading process's bp. 1775 */ 1776static boolean_t 1777arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio) 1778{ 1779 enum zio_compress compress = BP_GET_COMPRESS(zio->io_bp); 1780 boolean_t valid_cksum; 1781 1782 ASSERT(!BP_IS_EMBEDDED(zio->io_bp)); 1783 VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr)); 1784 1785 /* 1786 * We rely on the blkptr's checksum to determine if the block 1787 * is valid or not. When compressed arc is enabled, the l2arc 1788 * writes the block to the l2arc just as it appears in the pool. 1789 * This allows us to use the blkptr's checksum to validate the 1790 * data that we just read off of the l2arc without having to store 1791 * a separate checksum in the arc_buf_hdr_t. However, if compressed 1792 * arc is disabled, then the data written to the l2arc is always 1793 * uncompressed and won't match the block as it exists in the main 1794 * pool. When this is the case, we must first compress it if it is 1795 * compressed on the main pool before we can validate the checksum. 1796 */ 1797 if (!HDR_COMPRESSION_ENABLED(hdr) && compress != ZIO_COMPRESS_OFF) { 1798 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); 1799 uint64_t lsize = HDR_GET_LSIZE(hdr); 1800 uint64_t csize; 1801 1802 abd_t *cdata = abd_alloc_linear(HDR_GET_PSIZE(hdr), B_TRUE); 1803 csize = zio_compress_data(compress, zio->io_abd, 1804 abd_to_buf(cdata), lsize); 1805 1806 ASSERT3U(csize, <=, HDR_GET_PSIZE(hdr)); 1807 if (csize < HDR_GET_PSIZE(hdr)) { 1808 /* 1809 * Compressed blocks are always a multiple of the 1810 * smallest ashift in the pool. Ideally, we would 1811 * like to round up the csize to the next 1812 * spa_min_ashift but that value may have changed 1813 * since the block was last written. Instead, 1814 * we rely on the fact that the hdr's psize 1815 * was set to the psize of the block when it was 1816 * last written. We set the csize to that value 1817 * and zero out any part that should not contain 1818 * data. 1819 */ 1820 abd_zero_off(cdata, csize, HDR_GET_PSIZE(hdr) - csize); 1821 csize = HDR_GET_PSIZE(hdr); 1822 } 1823 zio_push_transform(zio, cdata, csize, HDR_GET_PSIZE(hdr), NULL); 1824 } 1825 1826 /* 1827 * Block pointers always store the checksum for the logical data. 1828 * If the block pointer has the gang bit set, then the checksum 1829 * it represents is for the reconstituted data and not for an 1830 * individual gang member. The zio pipeline, however, must be able to 1831 * determine the checksum of each of the gang constituents so it 1832 * treats the checksum comparison differently than what we need 1833 * for l2arc blocks. This prevents us from using the 1834 * zio_checksum_error() interface directly. Instead we must call the 1835 * zio_checksum_error_impl() so that we can ensure the checksum is 1836 * generated using the correct checksum algorithm and accounts for the 1837 * logical I/O size and not just a gang fragment. 1838 */ 1839 valid_cksum = (zio_checksum_error_impl(zio->io_spa, zio->io_bp, 1840 BP_GET_CHECKSUM(zio->io_bp), zio->io_abd, zio->io_size, 1841 zio->io_offset, NULL) == 0); 1842 zio_pop_transforms(zio); 1843 return (valid_cksum); 1844} 1845 1846/* 1847 * Given a buf full of data, if ZFS_DEBUG_MODIFY is enabled this computes a 1848 * checksum and attaches it to the buf's hdr so that we can ensure that the buf 1849 * isn't modified later on. If buf is compressed or there is already a checksum 1850 * on the hdr, this is a no-op (we only checksum uncompressed bufs). 1851 */ 1852static void 1853arc_cksum_compute(arc_buf_t *buf) 1854{ 1855 arc_buf_hdr_t *hdr = buf->b_hdr; 1856 1857 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 1858 return; 1859 1860 ASSERT(HDR_HAS_L1HDR(hdr)); 1861 1862 mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock); 1863 if (hdr->b_l1hdr.b_freeze_cksum != NULL || ARC_BUF_COMPRESSED(buf)) { 1864 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1865 return; 1866 } 1867 1868 ASSERT(!ARC_BUF_ENCRYPTED(buf)); 1869 ASSERT(!ARC_BUF_COMPRESSED(buf)); 1870 hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), 1871 KM_SLEEP); 1872 fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, 1873 hdr->b_l1hdr.b_freeze_cksum); 1874 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1875 arc_buf_watch(buf); 1876} 1877 1878#ifndef _KERNEL 1879typedef struct procctl { 1880 long cmd; 1881 prwatch_t prwatch; 1882} procctl_t; 1883#endif 1884 1885/* ARGSUSED */ 1886static void 1887arc_buf_unwatch(arc_buf_t *buf) 1888{ 1889#ifndef _KERNEL 1890 if (arc_watch) { 1891 int result; 1892 procctl_t ctl; 1893 ctl.cmd = PCWATCH; 1894 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data; 1895 ctl.prwatch.pr_size = 0; 1896 ctl.prwatch.pr_wflags = 0; 1897 result = write(arc_procfd, &ctl, sizeof (ctl)); 1898 ASSERT3U(result, ==, sizeof (ctl)); 1899 } 1900#endif 1901} 1902 1903/* ARGSUSED */ 1904static void 1905arc_buf_watch(arc_buf_t *buf) 1906{ 1907#ifndef _KERNEL 1908 if (arc_watch) { 1909 int result; 1910 procctl_t ctl; 1911 ctl.cmd = PCWATCH; 1912 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data; 1913 ctl.prwatch.pr_size = arc_buf_size(buf); 1914 ctl.prwatch.pr_wflags = WA_WRITE; 1915 result = write(arc_procfd, &ctl, sizeof (ctl)); 1916 ASSERT3U(result, ==, sizeof (ctl)); 1917 } 1918#endif 1919} 1920 1921static arc_buf_contents_t 1922arc_buf_type(arc_buf_hdr_t *hdr) 1923{ 1924 arc_buf_contents_t type; 1925 if (HDR_ISTYPE_METADATA(hdr)) { 1926 type = ARC_BUFC_METADATA; 1927 } else { 1928 type = ARC_BUFC_DATA; 1929 } 1930 VERIFY3U(hdr->b_type, ==, type); 1931 return (type); 1932} 1933 1934boolean_t 1935arc_is_metadata(arc_buf_t *buf) 1936{ 1937 return (HDR_ISTYPE_METADATA(buf->b_hdr) != 0); 1938} 1939 1940static uint32_t 1941arc_bufc_to_flags(arc_buf_contents_t type) 1942{ 1943 switch (type) { 1944 case ARC_BUFC_DATA: 1945 /* metadata field is 0 if buffer contains normal data */ 1946 return (0); 1947 case ARC_BUFC_METADATA: 1948 return (ARC_FLAG_BUFC_METADATA); 1949 default: 1950 break; 1951 } 1952 panic("undefined ARC buffer type!"); 1953 return ((uint32_t)-1); 1954} 1955 1956void 1957arc_buf_thaw(arc_buf_t *buf) 1958{ 1959 arc_buf_hdr_t *hdr = buf->b_hdr; 1960 1961 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); 1962 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 1963 1964 arc_cksum_verify(buf); 1965 1966 /* 1967 * Compressed buffers do not manipulate the b_freeze_cksum. 1968 */ 1969 if (ARC_BUF_COMPRESSED(buf)) 1970 return; 1971 1972 ASSERT(HDR_HAS_L1HDR(hdr)); 1973 arc_cksum_free(hdr); 1974 1975 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); 1976#ifdef ZFS_DEBUG 1977 if (zfs_flags & ZFS_DEBUG_MODIFY) { 1978 if (hdr->b_l1hdr.b_thawed != NULL) 1979 kmem_free(hdr->b_l1hdr.b_thawed, 1); 1980 hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP); 1981 } 1982#endif 1983 1984 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1985 1986 arc_buf_unwatch(buf); 1987} 1988 1989void 1990arc_buf_freeze(arc_buf_t *buf) 1991{ 1992 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 1993 return; 1994 1995 if (ARC_BUF_COMPRESSED(buf)) 1996 return; 1997 1998 ASSERT(HDR_HAS_L1HDR(buf->b_hdr)); 1999 arc_cksum_compute(buf); 2000} 2001 2002/* 2003 * The arc_buf_hdr_t's b_flags should never be modified directly. Instead, 2004 * the following functions should be used to ensure that the flags are 2005 * updated in a thread-safe way. When manipulating the flags either 2006 * the hash_lock must be held or the hdr must be undiscoverable. This 2007 * ensures that we're not racing with any other threads when updating 2008 * the flags. 2009 */ 2010static inline void 2011arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags) 2012{ 2013 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2014 hdr->b_flags |= flags; 2015} 2016 2017static inline void 2018arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags) 2019{ 2020 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2021 hdr->b_flags &= ~flags; 2022} 2023 2024/* 2025 * Setting the compression bits in the arc_buf_hdr_t's b_flags is 2026 * done in a special way since we have to clear and set bits 2027 * at the same time. Consumers that wish to set the compression bits 2028 * must use this function to ensure that the flags are updated in 2029 * thread-safe manner. 2030 */ 2031static void 2032arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp) 2033{ 2034 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2035 2036 /* 2037 * Holes and embedded blocks will always have a psize = 0 so 2038 * we ignore the compression of the blkptr and set the 2039 * arc_buf_hdr_t's compression to ZIO_COMPRESS_OFF. 2040 * Holes and embedded blocks remain anonymous so we don't 2041 * want to uncompress them. Mark them as uncompressed. 2042 */ 2043 if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) { 2044 arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC); 2045 ASSERT(!HDR_COMPRESSION_ENABLED(hdr)); 2046 } else { 2047 arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC); 2048 ASSERT(HDR_COMPRESSION_ENABLED(hdr)); 2049 } 2050 2051 HDR_SET_COMPRESS(hdr, cmp); 2052 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp); 2053} 2054 2055/* 2056 * Looks for another buf on the same hdr which has the data decompressed, copies 2057 * from it, and returns true. If no such buf exists, returns false. 2058 */ 2059static boolean_t 2060arc_buf_try_copy_decompressed_data(arc_buf_t *buf) 2061{ 2062 arc_buf_hdr_t *hdr = buf->b_hdr; 2063 boolean_t copied = B_FALSE; 2064 2065 ASSERT(HDR_HAS_L1HDR(hdr)); 2066 ASSERT3P(buf->b_data, !=, NULL); 2067 ASSERT(!ARC_BUF_COMPRESSED(buf)); 2068 2069 for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL; 2070 from = from->b_next) { 2071 /* can't use our own data buffer */ 2072 if (from == buf) { 2073 continue; 2074 } 2075 2076 if (!ARC_BUF_COMPRESSED(from)) { 2077 bcopy(from->b_data, buf->b_data, arc_buf_size(buf)); 2078 copied = B_TRUE; 2079 break; 2080 } 2081 } 2082 2083 /* 2084 * Note: With encryption support, the following assertion is no longer 2085 * necessarily valid. If we receive two back to back raw snapshots 2086 * (send -w), the second receive can use a hdr with a cksum already 2087 * calculated. This happens via: 2088 * dmu_recv_stream() -> receive_read_record() -> arc_loan_raw_buf() 2089 * The rsend/send_mixed_raw test case exercises this code path. 2090 * 2091 * There were no decompressed bufs, so there should not be a 2092 * checksum on the hdr either. 2093 * EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL); 2094 */ 2095 2096 return (copied); 2097} 2098 2099/* 2100 * Return the size of the block, b_pabd, that is stored in the arc_buf_hdr_t. 2101 */ 2102static uint64_t 2103arc_hdr_size(arc_buf_hdr_t *hdr) 2104{ 2105 uint64_t size; 2106 2107 if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF && 2108 HDR_GET_PSIZE(hdr) > 0) { 2109 size = HDR_GET_PSIZE(hdr); 2110 } else { 2111 ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0); 2112 size = HDR_GET_LSIZE(hdr); 2113 } 2114 return (size); 2115} 2116 2117static int 2118arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj) 2119{ 2120 int ret; 2121 uint64_t csize; 2122 uint64_t lsize = HDR_GET_LSIZE(hdr); 2123 uint64_t psize = HDR_GET_PSIZE(hdr); 2124 void *tmpbuf = NULL; 2125 abd_t *abd = hdr->b_l1hdr.b_pabd; 2126 2127 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2128 ASSERT(HDR_AUTHENTICATED(hdr)); 2129 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 2130 2131 /* 2132 * The MAC is calculated on the compressed data that is stored on disk. 2133 * However, if compressed arc is disabled we will only have the 2134 * decompressed data available to us now. Compress it into a temporary 2135 * abd so we can verify the MAC. The performance overhead of this will 2136 * be relatively low, since most objects in an encrypted objset will 2137 * be encrypted (instead of authenticated) anyway. 2138 */ 2139 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && 2140 !HDR_COMPRESSION_ENABLED(hdr)) { 2141 tmpbuf = zio_buf_alloc(lsize); 2142 abd = abd_get_from_buf(tmpbuf, lsize); 2143 abd_take_ownership_of_buf(abd, B_TRUE); 2144 2145 csize = zio_compress_data(HDR_GET_COMPRESS(hdr), 2146 hdr->b_l1hdr.b_pabd, tmpbuf, lsize); 2147 ASSERT3U(csize, <=, psize); 2148 abd_zero_off(abd, csize, psize - csize); 2149 } 2150 2151 /* 2152 * Authentication is best effort. We authenticate whenever the key is 2153 * available. If we succeed we clear ARC_FLAG_NOAUTH. 2154 */ 2155 if (hdr->b_crypt_hdr.b_ot == DMU_OT_OBJSET) { 2156 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); 2157 ASSERT3U(lsize, ==, psize); 2158 ret = spa_do_crypt_objset_mac_abd(B_FALSE, spa, dsobj, abd, 2159 psize, hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS); 2160 } else { 2161 ret = spa_do_crypt_mac_abd(B_FALSE, spa, dsobj, abd, psize, 2162 hdr->b_crypt_hdr.b_mac); 2163 } 2164 2165 if (ret == 0) 2166 arc_hdr_clear_flags(hdr, ARC_FLAG_NOAUTH); 2167 else if (ret != ENOENT) 2168 goto error; 2169 2170 if (tmpbuf != NULL) 2171 abd_free(abd); 2172 2173 return (0); 2174 2175error: 2176 if (tmpbuf != NULL) 2177 abd_free(abd); 2178 2179 return (ret); 2180} 2181 2182/* 2183 * This function will take a header that only has raw encrypted data in 2184 * b_crypt_hdr.b_rabd and decrypt it into a new buffer which is stored in 2185 * b_l1hdr.b_pabd. If designated in the header flags, this function will 2186 * also decompress the data. 2187 */ 2188static int 2189arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb) 2190{ 2191 int ret; 2192 abd_t *cabd = NULL; 2193 void *tmp = NULL; 2194 boolean_t no_crypt = B_FALSE; 2195 boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS); 2196 2197 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2198 ASSERT(HDR_ENCRYPTED(hdr)); 2199 2200 arc_hdr_alloc_pabd(hdr, B_FALSE); 2201 2202 ret = spa_do_crypt_abd(B_FALSE, spa, zb, hdr->b_crypt_hdr.b_ot, 2203 B_FALSE, bswap, hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_iv, 2204 hdr->b_crypt_hdr.b_mac, HDR_GET_PSIZE(hdr), hdr->b_l1hdr.b_pabd, 2205 hdr->b_crypt_hdr.b_rabd, &no_crypt); 2206 if (ret != 0) 2207 goto error; 2208 2209 if (no_crypt) { 2210 abd_copy(hdr->b_l1hdr.b_pabd, hdr->b_crypt_hdr.b_rabd, 2211 HDR_GET_PSIZE(hdr)); 2212 } 2213 2214 /* 2215 * If this header has disabled arc compression but the b_pabd is 2216 * compressed after decrypting it, we need to decompress the newly 2217 * decrypted data. 2218 */ 2219 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && 2220 !HDR_COMPRESSION_ENABLED(hdr)) { 2221 /* 2222 * We want to make sure that we are correctly honoring the 2223 * zfs_abd_scatter_enabled setting, so we allocate an abd here 2224 * and then loan a buffer from it, rather than allocating a 2225 * linear buffer and wrapping it in an abd later. 2226 */ 2227 cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr); 2228 tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr)); 2229 2230 ret = zio_decompress_data(HDR_GET_COMPRESS(hdr), 2231 hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr), 2232 HDR_GET_LSIZE(hdr)); 2233 if (ret != 0) { 2234 abd_return_buf(cabd, tmp, arc_hdr_size(hdr)); 2235 goto error; 2236 } 2237 2238 abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr)); 2239 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, 2240 arc_hdr_size(hdr), hdr); 2241 hdr->b_l1hdr.b_pabd = cabd; 2242 } 2243 2244 return (0); 2245 2246error: 2247 arc_hdr_free_pabd(hdr, B_FALSE); 2248 if (cabd != NULL) 2249 arc_free_data_buf(hdr, cabd, arc_hdr_size(hdr), hdr); 2250 2251 return (ret); 2252} 2253 2254/* 2255 * This function is called during arc_buf_fill() to prepare the header's 2256 * abd plaintext pointer for use. This involves authenticated protected 2257 * data and decrypting encrypted data into the plaintext abd. 2258 */ 2259static int 2260arc_fill_hdr_crypt(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, spa_t *spa, 2261 const zbookmark_phys_t *zb, boolean_t noauth) 2262{ 2263 int ret; 2264 2265 ASSERT(HDR_PROTECTED(hdr)); 2266 2267 if (hash_lock != NULL) 2268 mutex_enter(hash_lock); 2269 2270 if (HDR_NOAUTH(hdr) && !noauth) { 2271 /* 2272 * The caller requested authenticated data but our data has 2273 * not been authenticated yet. Verify the MAC now if we can. 2274 */ 2275 ret = arc_hdr_authenticate(hdr, spa, zb->zb_objset); 2276 if (ret != 0) 2277 goto error; 2278 } else if (HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd == NULL) { 2279 /* 2280 * If we only have the encrypted version of the data, but the 2281 * unencrypted version was requested we take this opportunity 2282 * to store the decrypted version in the header for future use. 2283 */ 2284 ret = arc_hdr_decrypt(hdr, spa, zb); 2285 if (ret != 0) 2286 goto error; 2287 } 2288 2289 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 2290 2291 if (hash_lock != NULL) 2292 mutex_exit(hash_lock); 2293 2294 return (0); 2295 2296error: 2297 if (hash_lock != NULL) 2298 mutex_exit(hash_lock); 2299 2300 return (ret); 2301} 2302 2303/* 2304 * This function is used by the dbuf code to decrypt bonus buffers in place. 2305 * The dbuf code itself doesn't have any locking for decrypting a shared dnode 2306 * block, so we use the hash lock here to protect against concurrent calls to 2307 * arc_buf_fill(). 2308 */ 2309/* ARGSUSED */ 2310static void 2311arc_buf_untransform_in_place(arc_buf_t *buf, kmutex_t *hash_lock) 2312{ 2313 arc_buf_hdr_t *hdr = buf->b_hdr; 2314 2315 ASSERT(HDR_ENCRYPTED(hdr)); 2316 ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE); 2317 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2318 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 2319 2320 zio_crypt_copy_dnode_bonus(hdr->b_l1hdr.b_pabd, buf->b_data, 2321 arc_buf_size(buf)); 2322 buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED; 2323 buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED; 2324 hdr->b_crypt_hdr.b_ebufcnt -= 1; 2325} 2326 2327/* 2328 * Given a buf that has a data buffer attached to it, this function will 2329 * efficiently fill the buf with data of the specified compression setting from 2330 * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr 2331 * are already sharing a data buf, no copy is performed. 2332 * 2333 * If the buf is marked as compressed but uncompressed data was requested, this 2334 * will allocate a new data buffer for the buf, remove that flag, and fill the 2335 * buf with uncompressed data. You can't request a compressed buf on a hdr with 2336 * uncompressed data, and (since we haven't added support for it yet) if you 2337 * want compressed data your buf must already be marked as compressed and have 2338 * the correct-sized data buffer. 2339 */ 2340static int 2341arc_buf_fill(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb, 2342 arc_fill_flags_t flags) 2343{ 2344 int error = 0; 2345 arc_buf_hdr_t *hdr = buf->b_hdr; 2346 boolean_t hdr_compressed = 2347 (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF); 2348 boolean_t compressed = (flags & ARC_FILL_COMPRESSED) != 0; 2349 boolean_t encrypted = (flags & ARC_FILL_ENCRYPTED) != 0; 2350 dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap; 2351 kmutex_t *hash_lock = (flags & ARC_FILL_LOCKED) ? NULL : HDR_LOCK(hdr); 2352 2353 ASSERT3P(buf->b_data, !=, NULL); 2354 IMPLY(compressed, hdr_compressed || ARC_BUF_ENCRYPTED(buf)); 2355 IMPLY(compressed, ARC_BUF_COMPRESSED(buf)); 2356 IMPLY(encrypted, HDR_ENCRYPTED(hdr)); 2357 IMPLY(encrypted, ARC_BUF_ENCRYPTED(buf)); 2358 IMPLY(encrypted, ARC_BUF_COMPRESSED(buf)); 2359 IMPLY(encrypted, !ARC_BUF_SHARED(buf)); 2360 2361 /* 2362 * If the caller wanted encrypted data we just need to copy it from 2363 * b_rabd and potentially byteswap it. We won't be able to do any 2364 * further transforms on it. 2365 */ 2366 if (encrypted) { 2367 ASSERT(HDR_HAS_RABD(hdr)); 2368 abd_copy_to_buf(buf->b_data, hdr->b_crypt_hdr.b_rabd, 2369 HDR_GET_PSIZE(hdr)); 2370 goto byteswap; 2371 } 2372 2373 /* 2374 * Adjust encrypted and authenticated headers to accomodate 2375 * the request if needed. Dnode blocks (ARC_FILL_IN_PLACE) are 2376 * allowed to fail decryption due to keys not being loaded 2377 * without being marked as an IO error. 2378 */ 2379 if (HDR_PROTECTED(hdr)) { 2380 error = arc_fill_hdr_crypt(hdr, hash_lock, spa, 2381 zb, !!(flags & ARC_FILL_NOAUTH)); 2382 if (error == EACCES && (flags & ARC_FILL_IN_PLACE) != 0) { 2383 return (error); 2384 } else if (error != 0) { 2385 if (hash_lock != NULL) 2386 mutex_enter(hash_lock); 2387 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); 2388 if (hash_lock != NULL) 2389 mutex_exit(hash_lock); 2390 return (error); 2391 } 2392 } 2393 2394 /* 2395 * There is a special case here for dnode blocks which are 2396 * decrypting their bonus buffers. These blocks may request to 2397 * be decrypted in-place. This is necessary because there may 2398 * be many dnodes pointing into this buffer and there is 2399 * currently no method to synchronize replacing the backing 2400 * b_data buffer and updating all of the pointers. Here we use 2401 * the hash lock to ensure there are no races. If the need 2402 * arises for other types to be decrypted in-place, they must 2403 * add handling here as well. 2404 */ 2405 if ((flags & ARC_FILL_IN_PLACE) != 0) { 2406 ASSERT(!hdr_compressed); 2407 ASSERT(!compressed); 2408 ASSERT(!encrypted); 2409 2410 if (HDR_ENCRYPTED(hdr) && ARC_BUF_ENCRYPTED(buf)) { 2411 ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE); 2412 2413 if (hash_lock != NULL) 2414 mutex_enter(hash_lock); 2415 arc_buf_untransform_in_place(buf, hash_lock); 2416 if (hash_lock != NULL) 2417 mutex_exit(hash_lock); 2418 2419 /* Compute the hdr's checksum if necessary */ 2420 arc_cksum_compute(buf); 2421 } 2422 2423 return (0); 2424 } 2425 2426 if (hdr_compressed == compressed) { 2427 if (!arc_buf_is_shared(buf)) { 2428 abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd, 2429 arc_buf_size(buf)); 2430 } 2431 } else { 2432 ASSERT(hdr_compressed); 2433 ASSERT(!compressed); 2434 ASSERT3U(HDR_GET_LSIZE(hdr), !=, HDR_GET_PSIZE(hdr)); 2435 2436 /* 2437 * If the buf is sharing its data with the hdr, unlink it and 2438 * allocate a new data buffer for the buf. 2439 */ 2440 if (arc_buf_is_shared(buf)) { 2441 ASSERT(ARC_BUF_COMPRESSED(buf)); 2442 2443 /* We need to give the buf its own b_data */ 2444 buf->b_flags &= ~ARC_BUF_FLAG_SHARED; 2445 buf->b_data = 2446 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); 2447 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); 2448 2449 /* Previously overhead was 0; just add new overhead */ 2450 ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr)); 2451 } else if (ARC_BUF_COMPRESSED(buf)) { 2452 /* We need to reallocate the buf's b_data */ 2453 arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr), 2454 buf); 2455 buf->b_data = 2456 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); 2457 2458 /* We increased the size of b_data; update overhead */ 2459 ARCSTAT_INCR(arcstat_overhead_size, 2460 HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr)); 2461 } 2462 2463 /* 2464 * Regardless of the buf's previous compression settings, it 2465 * should not be compressed at the end of this function. 2466 */ 2467 buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED; 2468 2469 /* 2470 * Try copying the data from another buf which already has a 2471 * decompressed version. If that's not possible, it's time to 2472 * bite the bullet and decompress the data from the hdr. 2473 */ 2474 if (arc_buf_try_copy_decompressed_data(buf)) { 2475 /* Skip byteswapping and checksumming (already done) */ 2476 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, !=, NULL); 2477 return (0); 2478 } else { 2479 error = zio_decompress_data(HDR_GET_COMPRESS(hdr), 2480 hdr->b_l1hdr.b_pabd, buf->b_data, 2481 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr)); 2482 2483 /* 2484 * Absent hardware errors or software bugs, this should 2485 * be impossible, but log it anyway so we can debug it. 2486 */ 2487 if (error != 0) { 2488 zfs_dbgmsg( 2489 "hdr %p, compress %d, psize %d, lsize %d", 2490 hdr, arc_hdr_get_compress(hdr), 2491 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr)); 2492 if (hash_lock != NULL) 2493 mutex_enter(hash_lock); 2494 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); 2495 if (hash_lock != NULL) 2496 mutex_exit(hash_lock); 2497 return (SET_ERROR(EIO)); 2498 } 2499 } 2500 } 2501 2502byteswap: 2503 /* Byteswap the buf's data if necessary */ 2504 if (bswap != DMU_BSWAP_NUMFUNCS) { 2505 ASSERT(!HDR_SHARED_DATA(hdr)); 2506 ASSERT3U(bswap, <, DMU_BSWAP_NUMFUNCS); 2507 dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr)); 2508 } 2509 2510 /* Compute the hdr's checksum if necessary */ 2511 arc_cksum_compute(buf); 2512 2513 return (0); 2514} 2515 2516/* 2517 * If this function is being called to decrypt an encrypted buffer or verify an 2518 * authenticated one, the key must be loaded and a mapping must be made 2519 * available in the keystore via spa_keystore_create_mapping() or one of its 2520 * callers. 2521 */ 2522int 2523arc_untransform(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb, 2524 boolean_t in_place) 2525{ 2526 int ret; 2527 arc_fill_flags_t flags = 0; 2528 2529 if (in_place) 2530 flags |= ARC_FILL_IN_PLACE; 2531 2532 ret = arc_buf_fill(buf, spa, zb, flags); 2533 if (ret == ECKSUM) { 2534 /* 2535 * Convert authentication and decryption errors to EIO 2536 * (and generate an ereport) before leaving the ARC. 2537 */ 2538 ret = SET_ERROR(EIO); 2539 spa_log_error(spa, zb); 2540 zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION, 2541 spa, NULL, zb, NULL, 0, 0); 2542 } 2543 2544 return (ret); 2545} 2546 2547/* 2548 * Increment the amount of evictable space in the arc_state_t's refcount. 2549 * We account for the space used by the hdr and the arc buf individually 2550 * so that we can add and remove them from the refcount individually. 2551 */ 2552static void 2553arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state) 2554{ 2555 arc_buf_contents_t type = arc_buf_type(hdr); 2556 2557 ASSERT(HDR_HAS_L1HDR(hdr)); 2558 2559 if (GHOST_STATE(state)) { 2560 ASSERT0(hdr->b_l1hdr.b_bufcnt); 2561 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2562 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2563 ASSERT(!HDR_HAS_RABD(hdr)); 2564 (void) zfs_refcount_add_many(&state->arcs_esize[type], 2565 HDR_GET_LSIZE(hdr), hdr); 2566 return; 2567 } 2568 2569 ASSERT(!GHOST_STATE(state)); 2570 if (hdr->b_l1hdr.b_pabd != NULL) { 2571 (void) zfs_refcount_add_many(&state->arcs_esize[type], 2572 arc_hdr_size(hdr), hdr); 2573 } 2574 if (HDR_HAS_RABD(hdr)) { 2575 (void) zfs_refcount_add_many(&state->arcs_esize[type], 2576 HDR_GET_PSIZE(hdr), hdr); 2577 } 2578 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; 2579 buf = buf->b_next) { 2580 if (arc_buf_is_shared(buf)) 2581 continue; 2582 (void) zfs_refcount_add_many(&state->arcs_esize[type], 2583 arc_buf_size(buf), buf); 2584 } 2585} 2586 2587/* 2588 * Decrement the amount of evictable space in the arc_state_t's refcount. 2589 * We account for the space used by the hdr and the arc buf individually 2590 * so that we can add and remove them from the refcount individually. 2591 */ 2592static void 2593arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state) 2594{ 2595 arc_buf_contents_t type = arc_buf_type(hdr); 2596 2597 ASSERT(HDR_HAS_L1HDR(hdr)); 2598 2599 if (GHOST_STATE(state)) { 2600 ASSERT0(hdr->b_l1hdr.b_bufcnt); 2601 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2602 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2603 ASSERT(!HDR_HAS_RABD(hdr)); 2604 (void) zfs_refcount_remove_many(&state->arcs_esize[type], 2605 HDR_GET_LSIZE(hdr), hdr); 2606 return; 2607 } 2608 2609 ASSERT(!GHOST_STATE(state)); 2610 if (hdr->b_l1hdr.b_pabd != NULL) { 2611 (void) zfs_refcount_remove_many(&state->arcs_esize[type], 2612 arc_hdr_size(hdr), hdr); 2613 } 2614 if (HDR_HAS_RABD(hdr)) { 2615 (void) zfs_refcount_remove_many(&state->arcs_esize[type], 2616 HDR_GET_PSIZE(hdr), hdr); 2617 } 2618 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; 2619 buf = buf->b_next) { 2620 if (arc_buf_is_shared(buf)) 2621 continue; 2622 (void) zfs_refcount_remove_many(&state->arcs_esize[type], 2623 arc_buf_size(buf), buf); 2624 } 2625} 2626 2627/* 2628 * Add a reference to this hdr indicating that someone is actively 2629 * referencing that memory. When the refcount transitions from 0 to 1, 2630 * we remove it from the respective arc_state_t list to indicate that 2631 * it is not evictable. 2632 */ 2633static void 2634add_reference(arc_buf_hdr_t *hdr, void *tag) 2635{ 2636 ASSERT(HDR_HAS_L1HDR(hdr)); 2637 if (!MUTEX_HELD(HDR_LOCK(hdr))) { 2638 ASSERT(hdr->b_l1hdr.b_state == arc_anon); 2639 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 2640 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2641 } 2642 2643 arc_state_t *state = hdr->b_l1hdr.b_state; 2644 2645 if ((zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) && 2646 (state != arc_anon)) { 2647 /* We don't use the L2-only state list. */ 2648 if (state != arc_l2c_only) { 2649 multilist_remove(state->arcs_list[arc_buf_type(hdr)], 2650 hdr); 2651 arc_evictable_space_decrement(hdr, state); 2652 } 2653 /* remove the prefetch flag if we get a reference */ 2654 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); 2655 } 2656} 2657 2658/* 2659 * Remove a reference from this hdr. When the reference transitions from 2660 * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's 2661 * list making it eligible for eviction. 2662 */ 2663static int 2664remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag) 2665{ 2666 int cnt; 2667 arc_state_t *state = hdr->b_l1hdr.b_state; 2668 2669 ASSERT(HDR_HAS_L1HDR(hdr)); 2670 ASSERT(state == arc_anon || MUTEX_HELD(hash_lock)); 2671 ASSERT(!GHOST_STATE(state)); 2672 2673 /* 2674 * arc_l2c_only counts as a ghost state so we don't need to explicitly 2675 * check to prevent usage of the arc_l2c_only list. 2676 */ 2677 if (((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) && 2678 (state != arc_anon)) { 2679 multilist_insert(state->arcs_list[arc_buf_type(hdr)], hdr); 2680 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0); 2681 arc_evictable_space_increment(hdr, state); 2682 } 2683 return (cnt); 2684} 2685 2686/* 2687 * Move the supplied buffer to the indicated state. The hash lock 2688 * for the buffer must be held by the caller. 2689 */ 2690static void 2691arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr, 2692 kmutex_t *hash_lock) 2693{ 2694 arc_state_t *old_state; 2695 int64_t refcnt; 2696 uint32_t bufcnt; 2697 boolean_t update_old, update_new; 2698 arc_buf_contents_t buftype = arc_buf_type(hdr); 2699 2700 /* 2701 * We almost always have an L1 hdr here, since we call arc_hdr_realloc() 2702 * in arc_read() when bringing a buffer out of the L2ARC. However, the 2703 * L1 hdr doesn't always exist when we change state to arc_anon before 2704 * destroying a header, in which case reallocating to add the L1 hdr is 2705 * pointless. 2706 */ 2707 if (HDR_HAS_L1HDR(hdr)) { 2708 old_state = hdr->b_l1hdr.b_state; 2709 refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt); 2710 bufcnt = hdr->b_l1hdr.b_bufcnt; 2711 2712 update_old = (bufcnt > 0 || hdr->b_l1hdr.b_pabd != NULL || 2713 HDR_HAS_RABD(hdr)); 2714 } else { 2715 old_state = arc_l2c_only; 2716 refcnt = 0; 2717 bufcnt = 0; 2718 update_old = B_FALSE; 2719 } 2720 update_new = update_old; 2721 2722 ASSERT(MUTEX_HELD(hash_lock)); 2723 ASSERT3P(new_state, !=, old_state); 2724 ASSERT(!GHOST_STATE(new_state) || bufcnt == 0); 2725 ASSERT(old_state != arc_anon || bufcnt <= 1); 2726 2727 /* 2728 * If this buffer is evictable, transfer it from the 2729 * old state list to the new state list. 2730 */ 2731 if (refcnt == 0) { 2732 if (old_state != arc_anon && old_state != arc_l2c_only) { 2733 ASSERT(HDR_HAS_L1HDR(hdr)); 2734 multilist_remove(old_state->arcs_list[buftype], hdr); 2735 2736 if (GHOST_STATE(old_state)) { 2737 ASSERT0(bufcnt); 2738 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2739 update_old = B_TRUE; 2740 } 2741 arc_evictable_space_decrement(hdr, old_state); 2742 } 2743 if (new_state != arc_anon && new_state != arc_l2c_only) { 2744 2745 /* 2746 * An L1 header always exists here, since if we're 2747 * moving to some L1-cached state (i.e. not l2c_only or 2748 * anonymous), we realloc the header to add an L1hdr 2749 * beforehand. 2750 */ 2751 ASSERT(HDR_HAS_L1HDR(hdr)); 2752 multilist_insert(new_state->arcs_list[buftype], hdr); 2753 2754 if (GHOST_STATE(new_state)) { 2755 ASSERT0(bufcnt); 2756 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2757 update_new = B_TRUE; 2758 } 2759 arc_evictable_space_increment(hdr, new_state); 2760 } 2761 } 2762 2763 ASSERT(!HDR_EMPTY(hdr)); 2764 if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr)) 2765 buf_hash_remove(hdr); 2766 2767 /* adjust state sizes (ignore arc_l2c_only) */ 2768 2769 if (update_new && new_state != arc_l2c_only) { 2770 ASSERT(HDR_HAS_L1HDR(hdr)); 2771 if (GHOST_STATE(new_state)) { 2772 ASSERT0(bufcnt); 2773 2774 /* 2775 * When moving a header to a ghost state, we first 2776 * remove all arc buffers. Thus, we'll have a 2777 * bufcnt of zero, and no arc buffer to use for 2778 * the reference. As a result, we use the arc 2779 * header pointer for the reference. 2780 */ 2781 (void) zfs_refcount_add_many(&new_state->arcs_size, 2782 HDR_GET_LSIZE(hdr), hdr); 2783 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2784 ASSERT(!HDR_HAS_RABD(hdr)); 2785 } else { 2786 uint32_t buffers = 0; 2787 2788 /* 2789 * Each individual buffer holds a unique reference, 2790 * thus we must remove each of these references one 2791 * at a time. 2792 */ 2793 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; 2794 buf = buf->b_next) { 2795 ASSERT3U(bufcnt, !=, 0); 2796 buffers++; 2797 2798 /* 2799 * When the arc_buf_t is sharing the data 2800 * block with the hdr, the owner of the 2801 * reference belongs to the hdr. Only 2802 * add to the refcount if the arc_buf_t is 2803 * not shared. 2804 */ 2805 if (arc_buf_is_shared(buf)) 2806 continue; 2807 2808 (void) zfs_refcount_add_many( 2809 &new_state->arcs_size, 2810 arc_buf_size(buf), buf); 2811 } 2812 ASSERT3U(bufcnt, ==, buffers); 2813 2814 if (hdr->b_l1hdr.b_pabd != NULL) { 2815 (void) zfs_refcount_add_many( 2816 &new_state->arcs_size, 2817 arc_hdr_size(hdr), hdr); 2818 } 2819 2820 if (HDR_HAS_RABD(hdr)) { 2821 (void) zfs_refcount_add_many( 2822 &new_state->arcs_size, 2823 HDR_GET_PSIZE(hdr), hdr); 2824 } 2825 } 2826 } 2827 2828 if (update_old && old_state != arc_l2c_only) { 2829 ASSERT(HDR_HAS_L1HDR(hdr)); 2830 if (GHOST_STATE(old_state)) { 2831 ASSERT0(bufcnt); 2832 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2833 ASSERT(!HDR_HAS_RABD(hdr)); 2834 2835 /* 2836 * When moving a header off of a ghost state, 2837 * the header will not contain any arc buffers. 2838 * We use the arc header pointer for the reference 2839 * which is exactly what we did when we put the 2840 * header on the ghost state. 2841 */ 2842 2843 (void) zfs_refcount_remove_many(&old_state->arcs_size, 2844 HDR_GET_LSIZE(hdr), hdr); 2845 } else { 2846 uint32_t buffers = 0; 2847 2848 /* 2849 * Each individual buffer holds a unique reference, 2850 * thus we must remove each of these references one 2851 * at a time. 2852 */ 2853 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; 2854 buf = buf->b_next) { 2855 ASSERT3U(bufcnt, !=, 0); 2856 buffers++; 2857 2858 /* 2859 * When the arc_buf_t is sharing the data 2860 * block with the hdr, the owner of the 2861 * reference belongs to the hdr. Only 2862 * add to the refcount if the arc_buf_t is 2863 * not shared. 2864 */ 2865 if (arc_buf_is_shared(buf)) 2866 continue; 2867 2868 (void) zfs_refcount_remove_many( 2869 &old_state->arcs_size, arc_buf_size(buf), 2870 buf); 2871 } 2872 ASSERT3U(bufcnt, ==, buffers); 2873 ASSERT(hdr->b_l1hdr.b_pabd != NULL || 2874 HDR_HAS_RABD(hdr)); 2875 2876 if (hdr->b_l1hdr.b_pabd != NULL) { 2877 (void) zfs_refcount_remove_many( 2878 &old_state->arcs_size, arc_hdr_size(hdr), 2879 hdr); 2880 } 2881 2882 if (HDR_HAS_RABD(hdr)) { 2883 (void) zfs_refcount_remove_many( 2884 &old_state->arcs_size, HDR_GET_PSIZE(hdr), 2885 hdr); 2886 } 2887 } 2888 } 2889 2890 if (HDR_HAS_L1HDR(hdr)) 2891 hdr->b_l1hdr.b_state = new_state; 2892 2893 /* 2894 * L2 headers should never be on the L2 state list since they don't 2895 * have L1 headers allocated. 2896 */ 2897 ASSERT(multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_DATA]) && 2898 multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_METADATA])); 2899} 2900 2901void 2902arc_space_consume(uint64_t space, arc_space_type_t type) 2903{ 2904 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 2905 2906 switch (type) { 2907 case ARC_SPACE_DATA: 2908 aggsum_add(&astat_data_size, space); 2909 break; 2910 case ARC_SPACE_META: 2911 aggsum_add(&astat_metadata_size, space); 2912 break; 2913 case ARC_SPACE_OTHER: 2914 aggsum_add(&astat_other_size, space); 2915 break; 2916 case ARC_SPACE_HDRS: 2917 aggsum_add(&astat_hdr_size, space); 2918 break; 2919 case ARC_SPACE_L2HDRS: 2920 aggsum_add(&astat_l2_hdr_size, space); 2921 break; 2922 } 2923 2924 if (type != ARC_SPACE_DATA) 2925 aggsum_add(&arc_meta_used, space); 2926 2927 aggsum_add(&arc_size, space); 2928} 2929 2930void 2931arc_space_return(uint64_t space, arc_space_type_t type) 2932{ 2933 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 2934 2935 switch (type) { 2936 case ARC_SPACE_DATA: 2937 aggsum_add(&astat_data_size, -space); 2938 break; 2939 case ARC_SPACE_META: 2940 aggsum_add(&astat_metadata_size, -space); 2941 break; 2942 case ARC_SPACE_OTHER: 2943 aggsum_add(&astat_other_size, -space); 2944 break; 2945 case ARC_SPACE_HDRS: 2946 aggsum_add(&astat_hdr_size, -space); 2947 break; 2948 case ARC_SPACE_L2HDRS: 2949 aggsum_add(&astat_l2_hdr_size, -space); 2950 break; 2951 } 2952 2953 if (type != ARC_SPACE_DATA) { 2954 ASSERT(aggsum_compare(&arc_meta_used, space) >= 0); 2955 /* 2956 * We use the upper bound here rather than the precise value 2957 * because the arc_meta_max value doesn't need to be 2958 * precise. It's only consumed by humans via arcstats. 2959 */ 2960 if (arc_meta_max < aggsum_upper_bound(&arc_meta_used)) 2961 arc_meta_max = aggsum_upper_bound(&arc_meta_used); 2962 aggsum_add(&arc_meta_used, -space); 2963 } 2964 2965 ASSERT(aggsum_compare(&arc_size, space) >= 0); 2966 aggsum_add(&arc_size, -space); 2967} 2968 2969/* 2970 * Given a hdr and a buf, returns whether that buf can share its b_data buffer 2971 * with the hdr's b_pabd. 2972 */ 2973static boolean_t 2974arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf) 2975{ 2976 /* 2977 * The criteria for sharing a hdr's data are: 2978 * 1. the buffer is not encrypted 2979 * 2. the hdr's compression matches the buf's compression 2980 * 3. the hdr doesn't need to be byteswapped 2981 * 4. the hdr isn't already being shared 2982 * 5. the buf is either compressed or it is the last buf in the hdr list 2983 * 2984 * Criterion #5 maintains the invariant that shared uncompressed 2985 * bufs must be the final buf in the hdr's b_buf list. Reading this, you 2986 * might ask, "if a compressed buf is allocated first, won't that be the 2987 * last thing in the list?", but in that case it's impossible to create 2988 * a shared uncompressed buf anyway (because the hdr must be compressed 2989 * to have the compressed buf). You might also think that #3 is 2990 * sufficient to make this guarantee, however it's possible 2991 * (specifically in the rare L2ARC write race mentioned in 2992 * arc_buf_alloc_impl()) there will be an existing uncompressed buf that 2993 * is sharable, but wasn't at the time of its allocation. Rather than 2994 * allow a new shared uncompressed buf to be created and then shuffle 2995 * the list around to make it the last element, this simply disallows 2996 * sharing if the new buf isn't the first to be added. 2997 */ 2998 ASSERT3P(buf->b_hdr, ==, hdr); 2999 boolean_t hdr_compressed = arc_hdr_get_compress(hdr) != 3000 ZIO_COMPRESS_OFF; 3001 boolean_t buf_compressed = ARC_BUF_COMPRESSED(buf) != 0; 3002 return (!ARC_BUF_ENCRYPTED(buf) && 3003 buf_compressed == hdr_compressed && 3004 hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS && 3005 !HDR_SHARED_DATA(hdr) && 3006 (ARC_BUF_LAST(buf) || ARC_BUF_COMPRESSED(buf))); 3007} 3008 3009/* 3010 * Allocate a buf for this hdr. If you care about the data that's in the hdr, 3011 * or if you want a compressed buffer, pass those flags in. Returns 0 if the 3012 * copy was made successfully, or an error code otherwise. 3013 */ 3014static int 3015arc_buf_alloc_impl(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb, 3016 void *tag, boolean_t encrypted, boolean_t compressed, boolean_t noauth, 3017 boolean_t fill, arc_buf_t **ret) 3018{ 3019 arc_buf_t *buf; 3020 arc_fill_flags_t flags = ARC_FILL_LOCKED; 3021 3022 ASSERT(HDR_HAS_L1HDR(hdr)); 3023 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0); 3024 VERIFY(hdr->b_type == ARC_BUFC_DATA || 3025 hdr->b_type == ARC_BUFC_METADATA); 3026 ASSERT3P(ret, !=, NULL); 3027 ASSERT3P(*ret, ==, NULL); 3028 IMPLY(encrypted, compressed); 3029 3030 buf = *ret = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 3031 buf->b_hdr = hdr; 3032 buf->b_data = NULL; 3033 buf->b_next = hdr->b_l1hdr.b_buf; 3034 buf->b_flags = 0; 3035 3036 add_reference(hdr, tag); 3037 3038 /* 3039 * We're about to change the hdr's b_flags. We must either 3040 * hold the hash_lock or be undiscoverable. 3041 */ 3042 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 3043 3044 /* 3045 * Only honor requests for compressed bufs if the hdr is actually 3046 * compressed. This must be overriden if the buffer is encrypted since 3047 * encrypted buffers cannot be decompressed. 3048 */ 3049 if (encrypted) { 3050 buf->b_flags |= ARC_BUF_FLAG_COMPRESSED; 3051 buf->b_flags |= ARC_BUF_FLAG_ENCRYPTED; 3052 flags |= ARC_FILL_COMPRESSED | ARC_FILL_ENCRYPTED; 3053 } else if (compressed && 3054 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) { 3055 buf->b_flags |= ARC_BUF_FLAG_COMPRESSED; 3056 flags |= ARC_FILL_COMPRESSED; 3057 } 3058 3059 if (noauth) { 3060 ASSERT0(encrypted); 3061 flags |= ARC_FILL_NOAUTH; 3062 } 3063 3064 /* 3065 * If the hdr's data can be shared then we share the data buffer and 3066 * set the appropriate bit in the hdr's b_flags to indicate the hdr is 3067 * allocate a new buffer to store the buf's data. 3068 * 3069 * There are two additional restrictions here because we're sharing 3070 * hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be 3071 * actively involved in an L2ARC write, because if this buf is used by 3072 * an arc_write() then the hdr's data buffer will be released when the 3073 * write completes, even though the L2ARC write might still be using it. 3074 * Second, the hdr's ABD must be linear so that the buf's user doesn't 3075 * need to be ABD-aware. 3076 */ 3077 boolean_t can_share = arc_can_share(hdr, buf) && !HDR_L2_WRITING(hdr) && 3078 hdr->b_l1hdr.b_pabd != NULL && abd_is_linear(hdr->b_l1hdr.b_pabd); 3079 3080 /* Set up b_data and sharing */ 3081 if (can_share) { 3082 buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd); 3083 buf->b_flags |= ARC_BUF_FLAG_SHARED; 3084 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA); 3085 } else { 3086 buf->b_data = 3087 arc_get_data_buf(hdr, arc_buf_size(buf), buf); 3088 ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf)); 3089 } 3090 VERIFY3P(buf->b_data, !=, NULL); 3091 3092 hdr->b_l1hdr.b_buf = buf; 3093 hdr->b_l1hdr.b_bufcnt += 1; 3094 if (encrypted) 3095 hdr->b_crypt_hdr.b_ebufcnt += 1; 3096 3097 /* 3098 * If the user wants the data from the hdr, we need to either copy or 3099 * decompress the data. 3100 */ 3101 if (fill) { 3102 ASSERT3P(zb, !=, NULL); 3103 return (arc_buf_fill(buf, spa, zb, flags)); 3104 } 3105 3106 return (0); 3107} 3108 3109static char *arc_onloan_tag = "onloan"; 3110 3111static inline void 3112arc_loaned_bytes_update(int64_t delta) 3113{ 3114 atomic_add_64(&arc_loaned_bytes, delta); 3115 3116 /* assert that it did not wrap around */ 3117 ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0); 3118} 3119 3120/* 3121 * Loan out an anonymous arc buffer. Loaned buffers are not counted as in 3122 * flight data by arc_tempreserve_space() until they are "returned". Loaned 3123 * buffers must be returned to the arc before they can be used by the DMU or 3124 * freed. 3125 */ 3126arc_buf_t * 3127arc_loan_buf(spa_t *spa, boolean_t is_metadata, int size) 3128{ 3129 arc_buf_t *buf = arc_alloc_buf(spa, arc_onloan_tag, 3130 is_metadata ? ARC_BUFC_METADATA : ARC_BUFC_DATA, size); 3131 3132 arc_loaned_bytes_update(arc_buf_size(buf)); 3133 3134 return (buf); 3135} 3136 3137arc_buf_t * 3138arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize, 3139 enum zio_compress compression_type) 3140{ 3141 arc_buf_t *buf = arc_alloc_compressed_buf(spa, arc_onloan_tag, 3142 psize, lsize, compression_type); 3143 3144 arc_loaned_bytes_update(arc_buf_size(buf)); 3145 3146 return (buf); 3147} 3148 3149arc_buf_t * 3150arc_loan_raw_buf(spa_t *spa, uint64_t dsobj, boolean_t byteorder, 3151 const uint8_t *salt, const uint8_t *iv, const uint8_t *mac, 3152 dmu_object_type_t ot, uint64_t psize, uint64_t lsize, 3153 enum zio_compress compression_type) 3154{ 3155 arc_buf_t *buf = arc_alloc_raw_buf(spa, arc_onloan_tag, dsobj, 3156 byteorder, salt, iv, mac, ot, psize, lsize, compression_type); 3157 3158 atomic_add_64(&arc_loaned_bytes, psize); 3159 return (buf); 3160} 3161 3162 3163/* 3164 * Return a loaned arc buffer to the arc. 3165 */ 3166void 3167arc_return_buf(arc_buf_t *buf, void *tag) 3168{ 3169 arc_buf_hdr_t *hdr = buf->b_hdr; 3170 3171 ASSERT3P(buf->b_data, !=, NULL); 3172 ASSERT(HDR_HAS_L1HDR(hdr)); 3173 (void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag); 3174 (void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); 3175 3176 arc_loaned_bytes_update(-arc_buf_size(buf)); 3177} 3178 3179/* Detach an arc_buf from a dbuf (tag) */ 3180void 3181arc_loan_inuse_buf(arc_buf_t *buf, void *tag) 3182{ 3183 arc_buf_hdr_t *hdr = buf->b_hdr; 3184 3185 ASSERT3P(buf->b_data, !=, NULL); 3186 ASSERT(HDR_HAS_L1HDR(hdr)); 3187 (void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); 3188 (void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag); 3189 3190 arc_loaned_bytes_update(arc_buf_size(buf)); 3191} 3192 3193static void 3194l2arc_free_abd_on_write(abd_t *abd, size_t size, arc_buf_contents_t type) 3195{ 3196 l2arc_data_free_t *df = kmem_alloc(sizeof (*df), KM_SLEEP); 3197 3198 df->l2df_abd = abd; 3199 df->l2df_size = size; 3200 df->l2df_type = type; 3201 mutex_enter(&l2arc_free_on_write_mtx); 3202 list_insert_head(l2arc_free_on_write, df); 3203 mutex_exit(&l2arc_free_on_write_mtx); 3204} 3205 3206static void 3207arc_hdr_free_on_write(arc_buf_hdr_t *hdr, boolean_t free_rdata) 3208{ 3209 arc_state_t *state = hdr->b_l1hdr.b_state; 3210 arc_buf_contents_t type = arc_buf_type(hdr); 3211 uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr); 3212 3213 /* protected by hash lock, if in the hash table */ 3214 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { 3215 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 3216 ASSERT(state != arc_anon && state != arc_l2c_only); 3217 3218 (void) zfs_refcount_remove_many(&state->arcs_esize[type], 3219 size, hdr); 3220 } 3221 (void) zfs_refcount_remove_many(&state->arcs_size, size, hdr); 3222 if (type == ARC_BUFC_METADATA) { 3223 arc_space_return(size, ARC_SPACE_META); 3224 } else { 3225 ASSERT(type == ARC_BUFC_DATA); 3226 arc_space_return(size, ARC_SPACE_DATA); 3227 } 3228 3229 if (free_rdata) { 3230 l2arc_free_abd_on_write(hdr->b_crypt_hdr.b_rabd, size, type); 3231 } else { 3232 l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type); 3233 } 3234} 3235 3236/* 3237 * Share the arc_buf_t's data with the hdr. Whenever we are sharing the 3238 * data buffer, we transfer the refcount ownership to the hdr and update 3239 * the appropriate kstats. 3240 */ 3241static void 3242arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf) 3243{ 3244 /* LINTED */ 3245 arc_state_t *state = hdr->b_l1hdr.b_state; 3246 3247 ASSERT(arc_can_share(hdr, buf)); 3248 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 3249 ASSERT(!ARC_BUF_ENCRYPTED(buf)); 3250 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 3251 3252 /* 3253 * Start sharing the data buffer. We transfer the 3254 * refcount ownership to the hdr since it always owns 3255 * the refcount whenever an arc_buf_t is shared. 3256 */ 3257 zfs_refcount_transfer_ownership_many(&hdr->b_l1hdr.b_state->arcs_size, 3258 arc_hdr_size(hdr), buf, hdr); 3259 hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf)); 3260 abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd, 3261 HDR_ISTYPE_METADATA(hdr)); 3262 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA); 3263 buf->b_flags |= ARC_BUF_FLAG_SHARED; 3264 3265 /* 3266 * Since we've transferred ownership to the hdr we need 3267 * to increment its compressed and uncompressed kstats and 3268 * decrement the overhead size. 3269 */ 3270 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr)); 3271 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr)); 3272 ARCSTAT_INCR(arcstat_overhead_size, -arc_buf_size(buf)); 3273} 3274 3275static void 3276arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf) 3277{ 3278 /* LINTED */ 3279 arc_state_t *state = hdr->b_l1hdr.b_state; 3280 3281 ASSERT(arc_buf_is_shared(buf)); 3282 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 3283 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 3284 3285 /* 3286 * We are no longer sharing this buffer so we need 3287 * to transfer its ownership to the rightful owner. 3288 */ 3289 zfs_refcount_transfer_ownership_many(&hdr->b_l1hdr.b_state->arcs_size, 3290 arc_hdr_size(hdr), hdr, buf); 3291 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); 3292 abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd); 3293 abd_put(hdr->b_l1hdr.b_pabd); 3294 hdr->b_l1hdr.b_pabd = NULL; 3295 buf->b_flags &= ~ARC_BUF_FLAG_SHARED; 3296 3297 /* 3298 * Since the buffer is no longer shared between 3299 * the arc buf and the hdr, count it as overhead. 3300 */ 3301 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr)); 3302 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr)); 3303 ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf)); 3304} 3305 3306/* 3307 * Remove an arc_buf_t from the hdr's buf list and return the last 3308 * arc_buf_t on the list. If no buffers remain on the list then return 3309 * NULL. 3310 */ 3311static arc_buf_t * 3312arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf) 3313{ 3314 arc_buf_t **bufp = &hdr->b_l1hdr.b_buf; 3315 arc_buf_t *lastbuf = NULL; 3316 3317 ASSERT(HDR_HAS_L1HDR(hdr)); 3318 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 3319 3320 /* 3321 * Remove the buf from the hdr list and locate the last 3322 * remaining buffer on the list. 3323 */ 3324 while (*bufp != NULL) { 3325 if (*bufp == buf) 3326 *bufp = buf->b_next; 3327 3328 /* 3329 * If we've removed a buffer in the middle of 3330 * the list then update the lastbuf and update 3331 * bufp. 3332 */ 3333 if (*bufp != NULL) { 3334 lastbuf = *bufp; 3335 bufp = &(*bufp)->b_next; 3336 } 3337 } 3338 buf->b_next = NULL; 3339 ASSERT3P(lastbuf, !=, buf); 3340 IMPLY(hdr->b_l1hdr.b_bufcnt > 0, lastbuf != NULL); 3341 IMPLY(hdr->b_l1hdr.b_bufcnt > 0, hdr->b_l1hdr.b_buf != NULL); 3342 IMPLY(lastbuf != NULL, ARC_BUF_LAST(lastbuf)); 3343 3344 return (lastbuf); 3345} 3346 3347/* 3348 * Free up buf->b_data and pull the arc_buf_t off of the the arc_buf_hdr_t's 3349 * list and free it. 3350 */ 3351static void 3352arc_buf_destroy_impl(arc_buf_t *buf) 3353{ 3354 arc_buf_hdr_t *hdr = buf->b_hdr; 3355 3356 /* 3357 * Free up the data associated with the buf but only if we're not 3358 * sharing this with the hdr. If we are sharing it with the hdr, the 3359 * hdr is responsible for doing the free. 3360 */ 3361 if (buf->b_data != NULL) { 3362 /* 3363 * We're about to change the hdr's b_flags. We must either 3364 * hold the hash_lock or be undiscoverable. 3365 */ 3366 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 3367 3368 arc_cksum_verify(buf); 3369 arc_buf_unwatch(buf); 3370 3371 if (arc_buf_is_shared(buf)) { 3372 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); 3373 } else { 3374 uint64_t size = arc_buf_size(buf); 3375 arc_free_data_buf(hdr, buf->b_data, size, buf); 3376 ARCSTAT_INCR(arcstat_overhead_size, -size); 3377 } 3378 buf->b_data = NULL; 3379 3380 ASSERT(hdr->b_l1hdr.b_bufcnt > 0); 3381 hdr->b_l1hdr.b_bufcnt -= 1; 3382 3383 if (ARC_BUF_ENCRYPTED(buf)) { 3384 hdr->b_crypt_hdr.b_ebufcnt -= 1; 3385 3386 /* 3387 * If we have no more encrypted buffers and we've 3388 * already gotten a copy of the decrypted data we can 3389 * free b_rabd to save some space. 3390 */ 3391 if (hdr->b_crypt_hdr.b_ebufcnt == 0 && 3392 HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd != NULL && 3393 !HDR_IO_IN_PROGRESS(hdr)) { 3394 arc_hdr_free_pabd(hdr, B_TRUE); 3395 } 3396 } 3397 } 3398 3399 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf); 3400 3401 if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) { 3402 /* 3403 * If the current arc_buf_t is sharing its data buffer with the 3404 * hdr, then reassign the hdr's b_pabd to share it with the new 3405 * buffer at the end of the list. The shared buffer is always 3406 * the last one on the hdr's buffer list. 3407 * 3408 * There is an equivalent case for compressed bufs, but since 3409 * they aren't guaranteed to be the last buf in the list and 3410 * that is an exceedingly rare case, we just allow that space be 3411 * wasted temporarily. We must also be careful not to share 3412 * encrypted buffers, since they cannot be shared. 3413 */ 3414 if (lastbuf != NULL && !ARC_BUF_ENCRYPTED(lastbuf)) { 3415 /* Only one buf can be shared at once */ 3416 VERIFY(!arc_buf_is_shared(lastbuf)); 3417 /* hdr is uncompressed so can't have compressed buf */ 3418 VERIFY(!ARC_BUF_COMPRESSED(lastbuf)); 3419 3420 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 3421 arc_hdr_free_pabd(hdr, B_FALSE); 3422 3423 /* 3424 * We must setup a new shared block between the 3425 * last buffer and the hdr. The data would have 3426 * been allocated by the arc buf so we need to transfer 3427 * ownership to the hdr since it's now being shared. 3428 */ 3429 arc_share_buf(hdr, lastbuf); 3430 } 3431 } else if (HDR_SHARED_DATA(hdr)) { 3432 /* 3433 * Uncompressed shared buffers are always at the end 3434 * of the list. Compressed buffers don't have the 3435 * same requirements. This makes it hard to 3436 * simply assert that the lastbuf is shared so 3437 * we rely on the hdr's compression flags to determine 3438 * if we have a compressed, shared buffer. 3439 */ 3440 ASSERT3P(lastbuf, !=, NULL); 3441 ASSERT(arc_buf_is_shared(lastbuf) || 3442 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF); 3443 } 3444 3445 /* 3446 * Free the checksum if we're removing the last uncompressed buf from 3447 * this hdr. 3448 */ 3449 if (!arc_hdr_has_uncompressed_buf(hdr)) { 3450 arc_cksum_free(hdr); 3451 } 3452 3453 /* clean up the buf */ 3454 buf->b_hdr = NULL; 3455 kmem_cache_free(buf_cache, buf); 3456} 3457 3458static void 3459arc_hdr_alloc_pabd(arc_buf_hdr_t *hdr, boolean_t alloc_rdata) 3460{ 3461 uint64_t size; 3462 3463 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0); 3464 ASSERT(HDR_HAS_L1HDR(hdr)); 3465 ASSERT(!HDR_SHARED_DATA(hdr) || alloc_rdata); 3466 IMPLY(alloc_rdata, HDR_PROTECTED(hdr)); 3467 3468 if (alloc_rdata) { 3469 size = HDR_GET_PSIZE(hdr); 3470 ASSERT3P(hdr->b_crypt_hdr.b_rabd, ==, NULL); 3471 hdr->b_crypt_hdr.b_rabd = arc_get_data_abd(hdr, size, hdr); 3472 ASSERT3P(hdr->b_crypt_hdr.b_rabd, !=, NULL); 3473 } else { 3474 size = arc_hdr_size(hdr); 3475 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 3476 hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, size, hdr); 3477 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 3478 } 3479 3480 ARCSTAT_INCR(arcstat_compressed_size, size); 3481 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr)); 3482} 3483 3484static void 3485arc_hdr_free_pabd(arc_buf_hdr_t *hdr, boolean_t free_rdata) 3486{ 3487 uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr); 3488 3489 ASSERT(HDR_HAS_L1HDR(hdr)); 3490 ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr)); 3491 IMPLY(free_rdata, HDR_HAS_RABD(hdr)); 3492 3493 3494 /* 3495 * If the hdr is currently being written to the l2arc then 3496 * we defer freeing the data by adding it to the l2arc_free_on_write 3497 * list. The l2arc will free the data once it's finished 3498 * writing it to the l2arc device. 3499 */ 3500 if (HDR_L2_WRITING(hdr)) { 3501 arc_hdr_free_on_write(hdr, free_rdata); 3502 ARCSTAT_BUMP(arcstat_l2_free_on_write); 3503 } else if (free_rdata) { 3504 arc_free_data_abd(hdr, hdr->b_crypt_hdr.b_rabd, size, hdr); 3505 } else { 3506 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, 3507 size, hdr); 3508 } 3509 3510 if (free_rdata) { 3511 hdr->b_crypt_hdr.b_rabd = NULL; 3512 } else { 3513 hdr->b_l1hdr.b_pabd = NULL; 3514 } 3515 3516 if (hdr->b_l1hdr.b_pabd == NULL && !HDR_HAS_RABD(hdr)) 3517 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; 3518 3519 ARCSTAT_INCR(arcstat_compressed_size, -size); 3520 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr)); 3521} 3522 3523static arc_buf_hdr_t * 3524arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize, 3525 boolean_t protected, enum zio_compress compression_type, 3526 arc_buf_contents_t type, boolean_t alloc_rdata) 3527{ 3528 arc_buf_hdr_t *hdr; 3529 3530 VERIFY(type == ARC_BUFC_DATA || type == ARC_BUFC_METADATA); 3531 if (protected) { 3532 hdr = kmem_cache_alloc(hdr_full_crypt_cache, KM_PUSHPAGE); 3533 } else { 3534 hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE); 3535 } 3536 ASSERT(HDR_EMPTY(hdr)); 3537 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); 3538 ASSERT3P(hdr->b_l1hdr.b_thawed, ==, NULL); 3539 HDR_SET_PSIZE(hdr, psize); 3540 HDR_SET_LSIZE(hdr, lsize); 3541 hdr->b_spa = spa; 3542 hdr->b_type = type; 3543 hdr->b_flags = 0; 3544 arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR); 3545 arc_hdr_set_compress(hdr, compression_type); 3546 if (protected) 3547 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED); 3548 3549 hdr->b_l1hdr.b_state = arc_anon; 3550 hdr->b_l1hdr.b_arc_access = 0; 3551 hdr->b_l1hdr.b_bufcnt = 0; 3552 hdr->b_l1hdr.b_buf = NULL; 3553 3554 /* 3555 * Allocate the hdr's buffer. This will contain either 3556 * the compressed or uncompressed data depending on the block 3557 * it references and compressed arc enablement. 3558 */ 3559 arc_hdr_alloc_pabd(hdr, alloc_rdata); 3560 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 3561 3562 return (hdr); 3563} 3564 3565/* 3566 * Transition between the two allocation states for the arc_buf_hdr struct. 3567 * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without 3568 * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller 3569 * version is used when a cache buffer is only in the L2ARC in order to reduce 3570 * memory usage. 3571 */ 3572static arc_buf_hdr_t * 3573arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new) 3574{ 3575 ASSERT(HDR_HAS_L2HDR(hdr)); 3576 3577 arc_buf_hdr_t *nhdr; 3578 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; 3579 3580 ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) || 3581 (old == hdr_l2only_cache && new == hdr_full_cache)); 3582 3583 /* 3584 * if the caller wanted a new full header and the header is to be 3585 * encrypted we will actually allocate the header from the full crypt 3586 * cache instead. The same applies to freeing from the old cache. 3587 */ 3588 if (HDR_PROTECTED(hdr) && new == hdr_full_cache) 3589 new = hdr_full_crypt_cache; 3590 if (HDR_PROTECTED(hdr) && old == hdr_full_cache) 3591 old = hdr_full_crypt_cache; 3592 3593 nhdr = kmem_cache_alloc(new, KM_PUSHPAGE); 3594 3595 ASSERT(MUTEX_HELD(HDR_LOCK(hdr))); 3596 buf_hash_remove(hdr); 3597 3598 bcopy(hdr, nhdr, HDR_L2ONLY_SIZE); 3599 3600 if (new == hdr_full_cache || new == hdr_full_crypt_cache) { 3601 arc_hdr_set_flags(nhdr, ARC_FLAG_HAS_L1HDR); 3602 /* 3603 * arc_access and arc_change_state need to be aware that a 3604 * header has just come out of L2ARC, so we set its state to 3605 * l2c_only even though it's about to change. 3606 */ 3607 nhdr->b_l1hdr.b_state = arc_l2c_only; 3608 3609 /* Verify previous threads set to NULL before freeing */ 3610 ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL); 3611 ASSERT(!HDR_HAS_RABD(hdr)); 3612 } else { 3613 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 3614 ASSERT0(hdr->b_l1hdr.b_bufcnt); 3615 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); 3616 3617 /* 3618 * If we've reached here, We must have been called from 3619 * arc_evict_hdr(), as such we should have already been 3620 * removed from any ghost list we were previously on 3621 * (which protects us from racing with arc_evict_state), 3622 * thus no locking is needed during this check. 3623 */ 3624 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); 3625 3626 /* 3627 * A buffer must not be moved into the arc_l2c_only 3628 * state if it's not finished being written out to the 3629 * l2arc device. Otherwise, the b_l1hdr.b_pabd field 3630 * might try to be accessed, even though it was removed. 3631 */ 3632 VERIFY(!HDR_L2_WRITING(hdr)); 3633 VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL); 3634 ASSERT(!HDR_HAS_RABD(hdr)); 3635 3636#ifdef ZFS_DEBUG 3637 if (hdr->b_l1hdr.b_thawed != NULL) { 3638 kmem_free(hdr->b_l1hdr.b_thawed, 1); 3639 hdr->b_l1hdr.b_thawed = NULL; 3640 } 3641#endif 3642 3643 arc_hdr_clear_flags(nhdr, ARC_FLAG_HAS_L1HDR); 3644 } 3645 /* 3646 * The header has been reallocated so we need to re-insert it into any 3647 * lists it was on. 3648 */ 3649 (void) buf_hash_insert(nhdr, NULL); 3650 3651 ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node)); 3652 3653 mutex_enter(&dev->l2ad_mtx); 3654 3655 /* 3656 * We must place the realloc'ed header back into the list at 3657 * the same spot. Otherwise, if it's placed earlier in the list, 3658 * l2arc_write_buffers() could find it during the function's 3659 * write phase, and try to write it out to the l2arc. 3660 */ 3661 list_insert_after(&dev->l2ad_buflist, hdr, nhdr); 3662 list_remove(&dev->l2ad_buflist, hdr); 3663 3664 mutex_exit(&dev->l2ad_mtx); 3665 3666 /* 3667 * Since we're using the pointer address as the tag when 3668 * incrementing and decrementing the l2ad_alloc refcount, we 3669 * must remove the old pointer (that we're about to destroy) and 3670 * add the new pointer to the refcount. Otherwise we'd remove 3671 * the wrong pointer address when calling arc_hdr_destroy() later. 3672 */ 3673 3674 (void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr), 3675 hdr); 3676 (void) zfs_refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(nhdr), 3677 nhdr); 3678 3679 buf_discard_identity(hdr); 3680 kmem_cache_free(old, hdr); 3681 3682 return (nhdr); 3683} 3684 3685/* 3686 * This function allows an L1 header to be reallocated as a crypt 3687 * header and vice versa. If we are going to a crypt header, the 3688 * new fields will be zeroed out. 3689 */ 3690static arc_buf_hdr_t * 3691arc_hdr_realloc_crypt(arc_buf_hdr_t *hdr, boolean_t need_crypt) 3692{ 3693 arc_buf_hdr_t *nhdr; 3694 arc_buf_t *buf; 3695 kmem_cache_t *ncache, *ocache; 3696 3697 ASSERT(HDR_HAS_L1HDR(hdr)); 3698 ASSERT3U(!!HDR_PROTECTED(hdr), !=, need_crypt); 3699 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); 3700 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); 3701 ASSERT(!list_link_active(&hdr->b_l2hdr.b_l2node)); 3702 ASSERT3P(hdr->b_hash_next, ==, NULL); 3703 3704 if (need_crypt) { 3705 ncache = hdr_full_crypt_cache; 3706 ocache = hdr_full_cache; 3707 } else { 3708 ncache = hdr_full_cache; 3709 ocache = hdr_full_crypt_cache; 3710 } 3711 3712 nhdr = kmem_cache_alloc(ncache, KM_PUSHPAGE); 3713 3714 /* 3715 * Copy all members that aren't locks or condvars to the new header. 3716 * No lists are pointing to us (as we asserted above), so we don't 3717 * need to worry about the list nodes. 3718 */ 3719 nhdr->b_dva = hdr->b_dva; 3720 nhdr->b_birth = hdr->b_birth; 3721 nhdr->b_type = hdr->b_type; 3722 nhdr->b_flags = hdr->b_flags; 3723 nhdr->b_psize = hdr->b_psize; 3724 nhdr->b_lsize = hdr->b_lsize; 3725 nhdr->b_spa = hdr->b_spa; 3726 nhdr->b_l2hdr.b_dev = hdr->b_l2hdr.b_dev; 3727 nhdr->b_l2hdr.b_daddr = hdr->b_l2hdr.b_daddr; 3728 nhdr->b_l1hdr.b_freeze_cksum = hdr->b_l1hdr.b_freeze_cksum; 3729 nhdr->b_l1hdr.b_bufcnt = hdr->b_l1hdr.b_bufcnt; 3730 nhdr->b_l1hdr.b_byteswap = hdr->b_l1hdr.b_byteswap; 3731 nhdr->b_l1hdr.b_state = hdr->b_l1hdr.b_state; 3732 nhdr->b_l1hdr.b_arc_access = hdr->b_l1hdr.b_arc_access; 3733 nhdr->b_l1hdr.b_acb = hdr->b_l1hdr.b_acb; 3734 nhdr->b_l1hdr.b_pabd = hdr->b_l1hdr.b_pabd; 3735#ifdef ZFS_DEBUG 3736 if (hdr->b_l1hdr.b_thawed != NULL) { 3737 nhdr->b_l1hdr.b_thawed = hdr->b_l1hdr.b_thawed; 3738 hdr->b_l1hdr.b_thawed = NULL; 3739 } 3740#endif 3741 3742 /* 3743 * This refcount_add() exists only to ensure that the individual 3744 * arc buffers always point to a header that is referenced, avoiding 3745 * a small race condition that could trigger ASSERTs. 3746 */ 3747 (void) zfs_refcount_add(&nhdr->b_l1hdr.b_refcnt, FTAG); 3748 nhdr->b_l1hdr.b_buf = hdr->b_l1hdr.b_buf; 3749 for (buf = nhdr->b_l1hdr.b_buf; buf != NULL; buf = buf->b_next) { 3750 mutex_enter(&buf->b_evict_lock); 3751 buf->b_hdr = nhdr; 3752 mutex_exit(&buf->b_evict_lock); 3753 } 3754 zfs_refcount_transfer(&nhdr->b_l1hdr.b_refcnt, &hdr->b_l1hdr.b_refcnt); 3755 (void) zfs_refcount_remove(&nhdr->b_l1hdr.b_refcnt, FTAG); 3756 ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt)); 3757 3758 if (need_crypt) { 3759 arc_hdr_set_flags(nhdr, ARC_FLAG_PROTECTED); 3760 } else { 3761 arc_hdr_clear_flags(nhdr, ARC_FLAG_PROTECTED); 3762 } 3763 3764 /* unset all members of the original hdr */ 3765 bzero(&hdr->b_dva, sizeof (dva_t)); 3766 hdr->b_birth = 0; 3767 hdr->b_type = ARC_BUFC_INVALID; 3768 hdr->b_flags = 0; 3769 hdr->b_psize = 0; 3770 hdr->b_lsize = 0; 3771 hdr->b_spa = 0; 3772 hdr->b_l2hdr.b_dev = NULL; 3773 hdr->b_l2hdr.b_daddr = 0; 3774 hdr->b_l1hdr.b_freeze_cksum = NULL; 3775 hdr->b_l1hdr.b_buf = NULL; 3776 hdr->b_l1hdr.b_bufcnt = 0; 3777 hdr->b_l1hdr.b_byteswap = 0; 3778 hdr->b_l1hdr.b_state = NULL; 3779 hdr->b_l1hdr.b_arc_access = 0; 3780 hdr->b_l1hdr.b_acb = NULL; 3781 hdr->b_l1hdr.b_pabd = NULL; 3782 3783 if (ocache == hdr_full_crypt_cache) { 3784 ASSERT(!HDR_HAS_RABD(hdr)); 3785 hdr->b_crypt_hdr.b_ot = DMU_OT_NONE; 3786 hdr->b_crypt_hdr.b_ebufcnt = 0; 3787 hdr->b_crypt_hdr.b_dsobj = 0; 3788 bzero(hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN); 3789 bzero(hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN); 3790 bzero(hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN); 3791 } 3792 3793 buf_discard_identity(hdr); 3794 kmem_cache_free(ocache, hdr); 3795 3796 return (nhdr); 3797} 3798 3799/* 3800 * This function is used by the send / receive code to convert a newly 3801 * allocated arc_buf_t to one that is suitable for a raw encrypted write. It 3802 * is also used to allow the root objset block to be uupdated without altering 3803 * its embedded MACs. Both block types will always be uncompressed so we do not 3804 * have to worry about compression type or psize. 3805 */ 3806void 3807arc_convert_to_raw(arc_buf_t *buf, uint64_t dsobj, boolean_t byteorder, 3808 dmu_object_type_t ot, const uint8_t *salt, const uint8_t *iv, 3809 const uint8_t *mac) 3810{ 3811 arc_buf_hdr_t *hdr = buf->b_hdr; 3812 3813 ASSERT(ot == DMU_OT_DNODE || ot == DMU_OT_OBJSET); 3814 ASSERT(HDR_HAS_L1HDR(hdr)); 3815 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); 3816 3817 buf->b_flags |= (ARC_BUF_FLAG_COMPRESSED | ARC_BUF_FLAG_ENCRYPTED); 3818 if (!HDR_PROTECTED(hdr)) 3819 hdr = arc_hdr_realloc_crypt(hdr, B_TRUE); 3820 hdr->b_crypt_hdr.b_dsobj = dsobj; 3821 hdr->b_crypt_hdr.b_ot = ot; 3822 hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ? 3823 DMU_BSWAP_NUMFUNCS : DMU_OT_BYTESWAP(ot); 3824 if (!arc_hdr_has_uncompressed_buf(hdr)) 3825 arc_cksum_free(hdr); 3826 3827 if (salt != NULL) 3828 bcopy(salt, hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN); 3829 if (iv != NULL) 3830 bcopy(iv, hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN); 3831 if (mac != NULL) 3832 bcopy(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN); 3833} 3834 3835/* 3836 * Allocate a new arc_buf_hdr_t and arc_buf_t and return the buf to the caller. 3837 * The buf is returned thawed since we expect the consumer to modify it. 3838 */ 3839arc_buf_t * 3840arc_alloc_buf(spa_t *spa, void *tag, arc_buf_contents_t type, int32_t size) 3841{ 3842 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size, 3843 B_FALSE, ZIO_COMPRESS_OFF, type, B_FALSE); 3844 ASSERT(!MUTEX_HELD(HDR_LOCK(hdr))); 3845 3846 arc_buf_t *buf = NULL; 3847 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, B_FALSE, 3848 B_FALSE, B_FALSE, &buf)); 3849 arc_buf_thaw(buf); 3850 3851 return (buf); 3852} 3853 3854/* 3855 * Allocate a compressed buf in the same manner as arc_alloc_buf. Don't use this 3856 * for bufs containing metadata. 3857 */ 3858arc_buf_t * 3859arc_alloc_compressed_buf(spa_t *spa, void *tag, uint64_t psize, uint64_t lsize, 3860 enum zio_compress compression_type) 3861{ 3862 ASSERT3U(lsize, >, 0); 3863 ASSERT3U(lsize, >=, psize); 3864 ASSERT3U(compression_type, >, ZIO_COMPRESS_OFF); 3865 ASSERT3U(compression_type, <, ZIO_COMPRESS_FUNCTIONS); 3866 3867 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, 3868 B_FALSE, compression_type, ARC_BUFC_DATA, B_FALSE); 3869 ASSERT(!MUTEX_HELD(HDR_LOCK(hdr))); 3870 3871 arc_buf_t *buf = NULL; 3872 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, 3873 B_TRUE, B_FALSE, B_FALSE, &buf)); 3874 arc_buf_thaw(buf); 3875 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); 3876 3877 if (!arc_buf_is_shared(buf)) { 3878 /* 3879 * To ensure that the hdr has the correct data in it if we call 3880 * arc_untransform() on this buf before it's been written to 3881 * disk, it's easiest if we just set up sharing between the 3882 * buf and the hdr. 3883 */ 3884 ASSERT(!abd_is_linear(hdr->b_l1hdr.b_pabd)); 3885 arc_hdr_free_pabd(hdr, B_FALSE); 3886 arc_share_buf(hdr, buf); 3887 } 3888 3889 return (buf); 3890} 3891 3892arc_buf_t * 3893arc_alloc_raw_buf(spa_t *spa, void *tag, uint64_t dsobj, boolean_t byteorder, 3894 const uint8_t *salt, const uint8_t *iv, const uint8_t *mac, 3895 dmu_object_type_t ot, uint64_t psize, uint64_t lsize, 3896 enum zio_compress compression_type) 3897{ 3898 arc_buf_hdr_t *hdr; 3899 arc_buf_t *buf; 3900 arc_buf_contents_t type = DMU_OT_IS_METADATA(ot) ? 3901 ARC_BUFC_METADATA : ARC_BUFC_DATA; 3902 3903 ASSERT3U(lsize, >, 0); 3904 ASSERT3U(lsize, >=, psize); 3905 ASSERT3U(compression_type, >=, ZIO_COMPRESS_OFF); 3906 ASSERT3U(compression_type, <, ZIO_COMPRESS_FUNCTIONS); 3907 3908 hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, B_TRUE, 3909 compression_type, type, B_TRUE); 3910 ASSERT(!MUTEX_HELD(HDR_LOCK(hdr))); 3911 3912 hdr->b_crypt_hdr.b_dsobj = dsobj; 3913 hdr->b_crypt_hdr.b_ot = ot; 3914 hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ? 3915 DMU_BSWAP_NUMFUNCS : DMU_OT_BYTESWAP(ot); 3916 bcopy(salt, hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN); 3917 bcopy(iv, hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN); 3918 bcopy(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN); 3919 3920 /* 3921 * This buffer will be considered encrypted even if the ot is not an 3922 * encrypted type. It will become authenticated instead in 3923 * arc_write_ready(). 3924 */ 3925 buf = NULL; 3926 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_TRUE, B_TRUE, 3927 B_FALSE, B_FALSE, &buf)); 3928 arc_buf_thaw(buf); 3929 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); 3930 3931 return (buf); 3932} 3933 3934static void 3935arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr) 3936{ 3937 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr; 3938 l2arc_dev_t *dev = l2hdr->b_dev; 3939 uint64_t psize = HDR_GET_PSIZE(hdr); 3940 uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev, psize); 3941 3942 ASSERT(MUTEX_HELD(&dev->l2ad_mtx)); 3943 ASSERT(HDR_HAS_L2HDR(hdr)); 3944 3945 list_remove(&dev->l2ad_buflist, hdr); 3946 3947 ARCSTAT_INCR(arcstat_l2_psize, -psize); 3948 ARCSTAT_INCR(arcstat_l2_lsize, -HDR_GET_LSIZE(hdr)); 3949 3950 vdev_space_update(dev->l2ad_vdev, -asize, 0, 0); 3951 3952 (void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr), 3953 hdr); 3954 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR); 3955} 3956 3957static void 3958arc_hdr_destroy(arc_buf_hdr_t *hdr) 3959{ 3960 if (HDR_HAS_L1HDR(hdr)) { 3961 ASSERT(hdr->b_l1hdr.b_buf == NULL || 3962 hdr->b_l1hdr.b_bufcnt > 0); 3963 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 3964 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); 3965 } 3966 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 3967 ASSERT(!HDR_IN_HASH_TABLE(hdr)); 3968 3969 if (!HDR_EMPTY(hdr)) 3970 buf_discard_identity(hdr); 3971 3972 if (HDR_HAS_L2HDR(hdr)) { 3973 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; 3974 boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx); 3975 3976 if (!buflist_held) 3977 mutex_enter(&dev->l2ad_mtx); 3978 3979 /* 3980 * Even though we checked this conditional above, we 3981 * need to check this again now that we have the 3982 * l2ad_mtx. This is because we could be racing with 3983 * another thread calling l2arc_evict() which might have 3984 * destroyed this header's L2 portion as we were waiting 3985 * to acquire the l2ad_mtx. If that happens, we don't 3986 * want to re-destroy the header's L2 portion. 3987 */ 3988 if (HDR_HAS_L2HDR(hdr)) 3989 arc_hdr_l2hdr_destroy(hdr); 3990 3991 if (!buflist_held) 3992 mutex_exit(&dev->l2ad_mtx); 3993 } 3994 3995 if (HDR_HAS_L1HDR(hdr)) { 3996 arc_cksum_free(hdr); 3997 3998 while (hdr->b_l1hdr.b_buf != NULL) 3999 arc_buf_destroy_impl(hdr->b_l1hdr.b_buf); 4000 4001#ifdef ZFS_DEBUG 4002 if (hdr->b_l1hdr.b_thawed != NULL) { 4003 kmem_free(hdr->b_l1hdr.b_thawed, 1); 4004 hdr->b_l1hdr.b_thawed = NULL; 4005 } 4006#endif 4007 4008 if (hdr->b_l1hdr.b_pabd != NULL) { 4009 arc_hdr_free_pabd(hdr, B_FALSE); 4010 } 4011 4012 if (HDR_HAS_RABD(hdr)) 4013 arc_hdr_free_pabd(hdr, B_TRUE); 4014 } 4015 4016 ASSERT3P(hdr->b_hash_next, ==, NULL); 4017 if (HDR_HAS_L1HDR(hdr)) { 4018 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); 4019 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); 4020 4021 if (!HDR_PROTECTED(hdr)) { 4022 kmem_cache_free(hdr_full_cache, hdr); 4023 } else { 4024 kmem_cache_free(hdr_full_crypt_cache, hdr); 4025 } 4026 } else { 4027 kmem_cache_free(hdr_l2only_cache, hdr); 4028 } 4029} 4030 4031void 4032arc_buf_destroy(arc_buf_t *buf, void* tag) 4033{ 4034 arc_buf_hdr_t *hdr = buf->b_hdr; 4035 kmutex_t *hash_lock = HDR_LOCK(hdr); 4036 4037 if (hdr->b_l1hdr.b_state == arc_anon) { 4038 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); 4039 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 4040 VERIFY0(remove_reference(hdr, NULL, tag)); 4041 arc_hdr_destroy(hdr); 4042 return; 4043 } 4044 4045 mutex_enter(hash_lock); 4046 ASSERT3P(hdr, ==, buf->b_hdr); 4047 ASSERT(hdr->b_l1hdr.b_bufcnt > 0); 4048 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 4049 ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon); 4050 ASSERT3P(buf->b_data, !=, NULL); 4051 4052 (void) remove_reference(hdr, hash_lock, tag); 4053 arc_buf_destroy_impl(buf); 4054 mutex_exit(hash_lock); 4055} 4056 4057/* 4058 * Evict the arc_buf_hdr that is provided as a parameter. The resultant 4059 * state of the header is dependent on its state prior to entering this 4060 * function. The following transitions are possible: 4061 * 4062 * - arc_mru -> arc_mru_ghost 4063 * - arc_mfu -> arc_mfu_ghost 4064 * - arc_mru_ghost -> arc_l2c_only 4065 * - arc_mru_ghost -> deleted 4066 * - arc_mfu_ghost -> arc_l2c_only 4067 * - arc_mfu_ghost -> deleted 4068 */ 4069static int64_t 4070arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) 4071{ 4072 arc_state_t *evicted_state, *state; 4073 int64_t bytes_evicted = 0; 4074 int min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ? 4075 zfs_arc_min_prescient_prefetch_ms : zfs_arc_min_prefetch_ms; 4076 4077 ASSERT(MUTEX_HELD(hash_lock)); 4078 ASSERT(HDR_HAS_L1HDR(hdr)); 4079 4080 state = hdr->b_l1hdr.b_state; 4081 if (GHOST_STATE(state)) { 4082 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 4083 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 4084 4085 /* 4086 * l2arc_write_buffers() relies on a header's L1 portion 4087 * (i.e. its b_pabd field) during its write phase. 4088 * Thus, we cannot push a header onto the arc_l2c_only 4089 * state (removing its L1 piece) until the header is 4090 * done being written to the l2arc. 4091 */ 4092 if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) { 4093 ARCSTAT_BUMP(arcstat_evict_l2_skip); 4094 return (bytes_evicted); 4095 } 4096 4097 ARCSTAT_BUMP(arcstat_deleted); 4098 bytes_evicted += HDR_GET_LSIZE(hdr); 4099 4100 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr); 4101 4102 if (HDR_HAS_L2HDR(hdr)) { 4103 ASSERT(hdr->b_l1hdr.b_pabd == NULL); 4104 ASSERT(!HDR_HAS_RABD(hdr)); 4105 /* 4106 * This buffer is cached on the 2nd Level ARC; 4107 * don't destroy the header. 4108 */ 4109 arc_change_state(arc_l2c_only, hdr, hash_lock); 4110 /* 4111 * dropping from L1+L2 cached to L2-only, 4112 * realloc to remove the L1 header. 4113 */ 4114 hdr = arc_hdr_realloc(hdr, hdr_full_cache, 4115 hdr_l2only_cache); 4116 } else { 4117 arc_change_state(arc_anon, hdr, hash_lock); 4118 arc_hdr_destroy(hdr); 4119 } 4120 return (bytes_evicted); 4121 } 4122 4123 ASSERT(state == arc_mru || state == arc_mfu); 4124 evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 4125 4126 /* prefetch buffers have a minimum lifespan */ 4127 if (HDR_IO_IN_PROGRESS(hdr) || 4128 ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) && 4129 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < min_lifetime * hz)) { 4130 ARCSTAT_BUMP(arcstat_evict_skip); 4131 return (bytes_evicted); 4132 } 4133 4134 ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt)); 4135 while (hdr->b_l1hdr.b_buf) { 4136 arc_buf_t *buf = hdr->b_l1hdr.b_buf; 4137 if (!mutex_tryenter(&buf->b_evict_lock)) { 4138 ARCSTAT_BUMP(arcstat_mutex_miss); 4139 break; 4140 } 4141 if (buf->b_data != NULL) 4142 bytes_evicted += HDR_GET_LSIZE(hdr); 4143 mutex_exit(&buf->b_evict_lock); 4144 arc_buf_destroy_impl(buf); 4145 } 4146 4147 if (HDR_HAS_L2HDR(hdr)) { 4148 ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr)); 4149 } else { 4150 if (l2arc_write_eligible(hdr->b_spa, hdr)) { 4151 ARCSTAT_INCR(arcstat_evict_l2_eligible, 4152 HDR_GET_LSIZE(hdr)); 4153 } else { 4154 ARCSTAT_INCR(arcstat_evict_l2_ineligible, 4155 HDR_GET_LSIZE(hdr)); 4156 } 4157 } 4158 4159 if (hdr->b_l1hdr.b_bufcnt == 0) { 4160 arc_cksum_free(hdr); 4161 4162 bytes_evicted += arc_hdr_size(hdr); 4163 4164 /* 4165 * If this hdr is being evicted and has a compressed 4166 * buffer then we discard it here before we change states. 4167 * This ensures that the accounting is updated correctly 4168 * in arc_free_data_impl(). 4169 */ 4170 if (hdr->b_l1hdr.b_pabd != NULL) 4171 arc_hdr_free_pabd(hdr, B_FALSE); 4172 4173 if (HDR_HAS_RABD(hdr)) 4174 arc_hdr_free_pabd(hdr, B_TRUE); 4175 4176 arc_change_state(evicted_state, hdr, hash_lock); 4177 ASSERT(HDR_IN_HASH_TABLE(hdr)); 4178 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE); 4179 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr); 4180 } 4181 4182 return (bytes_evicted); 4183} 4184 4185static uint64_t 4186arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker, 4187 uint64_t spa, int64_t bytes) 4188{ 4189 multilist_sublist_t *mls; 4190 uint64_t bytes_evicted = 0; 4191 arc_buf_hdr_t *hdr; 4192 kmutex_t *hash_lock; 4193 int evict_count = 0; 4194 4195 ASSERT3P(marker, !=, NULL); 4196 IMPLY(bytes < 0, bytes == ARC_EVICT_ALL); 4197 4198 mls = multilist_sublist_lock(ml, idx); 4199 4200 for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL; 4201 hdr = multilist_sublist_prev(mls, marker)) { 4202 if ((bytes != ARC_EVICT_ALL && bytes_evicted >= bytes) || 4203 (evict_count >= zfs_arc_evict_batch_limit)) 4204 break; 4205 4206 /* 4207 * To keep our iteration location, move the marker 4208 * forward. Since we're not holding hdr's hash lock, we 4209 * must be very careful and not remove 'hdr' from the 4210 * sublist. Otherwise, other consumers might mistake the 4211 * 'hdr' as not being on a sublist when they call the 4212 * multilist_link_active() function (they all rely on 4213 * the hash lock protecting concurrent insertions and 4214 * removals). multilist_sublist_move_forward() was 4215 * specifically implemented to ensure this is the case 4216 * (only 'marker' will be removed and re-inserted). 4217 */ 4218 multilist_sublist_move_forward(mls, marker); 4219 4220 /* 4221 * The only case where the b_spa field should ever be 4222 * zero, is the marker headers inserted by 4223 * arc_evict_state(). It's possible for multiple threads 4224 * to be calling arc_evict_state() concurrently (e.g. 4225 * dsl_pool_close() and zio_inject_fault()), so we must 4226 * skip any markers we see from these other threads. 4227 */ 4228 if (hdr->b_spa == 0) 4229 continue; 4230 4231 /* we're only interested in evicting buffers of a certain spa */ 4232 if (spa != 0 && hdr->b_spa != spa) { 4233 ARCSTAT_BUMP(arcstat_evict_skip); 4234 continue; 4235 } 4236 4237 hash_lock = HDR_LOCK(hdr); 4238 4239 /* 4240 * We aren't calling this function from any code path 4241 * that would already be holding a hash lock, so we're 4242 * asserting on this assumption to be defensive in case 4243 * this ever changes. Without this check, it would be 4244 * possible to incorrectly increment arcstat_mutex_miss 4245 * below (e.g. if the code changed such that we called 4246 * this function with a hash lock held). 4247 */ 4248 ASSERT(!MUTEX_HELD(hash_lock)); 4249 4250 if (mutex_tryenter(hash_lock)) { 4251 uint64_t evicted = arc_evict_hdr(hdr, hash_lock); 4252 mutex_exit(hash_lock); 4253 4254 bytes_evicted += evicted; 4255 4256 /* 4257 * If evicted is zero, arc_evict_hdr() must have 4258 * decided to skip this header, don't increment 4259 * evict_count in this case. 4260 */ 4261 if (evicted != 0) 4262 evict_count++; 4263 4264 /* 4265 * If arc_size isn't overflowing, signal any 4266 * threads that might happen to be waiting. 4267 * 4268 * For each header evicted, we wake up a single 4269 * thread. If we used cv_broadcast, we could 4270 * wake up "too many" threads causing arc_size 4271 * to significantly overflow arc_c; since 4272 * arc_get_data_impl() doesn't check for overflow 4273 * when it's woken up (it doesn't because it's 4274 * possible for the ARC to be overflowing while 4275 * full of un-evictable buffers, and the 4276 * function should proceed in this case). 4277 * 4278 * If threads are left sleeping, due to not 4279 * using cv_broadcast here, they will be woken 4280 * up via cv_broadcast in arc_adjust_cb() just 4281 * before arc_adjust_zthr sleeps. 4282 */ 4283 mutex_enter(&arc_adjust_lock); 4284 if (!arc_is_overflowing()) 4285 cv_signal(&arc_adjust_waiters_cv); 4286 mutex_exit(&arc_adjust_lock); 4287 } else { 4288 ARCSTAT_BUMP(arcstat_mutex_miss); 4289 } 4290 } 4291 4292 multilist_sublist_unlock(mls); 4293 4294 return (bytes_evicted); 4295} 4296 4297/* 4298 * Evict buffers from the given arc state, until we've removed the 4299 * specified number of bytes. Move the removed buffers to the 4300 * appropriate evict state. 4301 * 4302 * This function makes a "best effort". It skips over any buffers 4303 * it can't get a hash_lock on, and so, may not catch all candidates. 4304 * It may also return without evicting as much space as requested. 4305 * 4306 * If bytes is specified using the special value ARC_EVICT_ALL, this 4307 * will evict all available (i.e. unlocked and evictable) buffers from 4308 * the given arc state; which is used by arc_flush(). 4309 */ 4310static uint64_t 4311arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes, 4312 arc_buf_contents_t type) 4313{ 4314 uint64_t total_evicted = 0; 4315 multilist_t *ml = state->arcs_list[type]; 4316 int num_sublists; 4317 arc_buf_hdr_t **markers; 4318 4319 IMPLY(bytes < 0, bytes == ARC_EVICT_ALL); 4320 4321 num_sublists = multilist_get_num_sublists(ml); 4322 4323 /* 4324 * If we've tried to evict from each sublist, made some 4325 * progress, but still have not hit the target number of bytes 4326 * to evict, we want to keep trying. The markers allow us to 4327 * pick up where we left off for each individual sublist, rather 4328 * than starting from the tail each time. 4329 */ 4330 markers = kmem_zalloc(sizeof (*markers) * num_sublists, KM_SLEEP); 4331 for (int i = 0; i < num_sublists; i++) { 4332 markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP); 4333 4334 /* 4335 * A b_spa of 0 is used to indicate that this header is 4336 * a marker. This fact is used in arc_adjust_type() and 4337 * arc_evict_state_impl(). 4338 */ 4339 markers[i]->b_spa = 0; 4340 4341 multilist_sublist_t *mls = multilist_sublist_lock(ml, i); 4342 multilist_sublist_insert_tail(mls, markers[i]); 4343 multilist_sublist_unlock(mls); 4344 } 4345 4346 /* 4347 * While we haven't hit our target number of bytes to evict, or 4348 * we're evicting all available buffers. 4349 */ 4350 while (total_evicted < bytes || bytes == ARC_EVICT_ALL) { 4351 /* 4352 * Start eviction using a randomly selected sublist, 4353 * this is to try and evenly balance eviction across all 4354 * sublists. Always starting at the same sublist 4355 * (e.g. index 0) would cause evictions to favor certain 4356 * sublists over others. 4357 */ 4358 int sublist_idx = multilist_get_random_index(ml); 4359 uint64_t scan_evicted = 0; 4360 4361 for (int i = 0; i < num_sublists; i++) { 4362 uint64_t bytes_remaining; 4363 uint64_t bytes_evicted; 4364 4365 if (bytes == ARC_EVICT_ALL) 4366 bytes_remaining = ARC_EVICT_ALL; 4367 else if (total_evicted < bytes) 4368 bytes_remaining = bytes - total_evicted; 4369 else 4370 break; 4371 4372 bytes_evicted = arc_evict_state_impl(ml, sublist_idx, 4373 markers[sublist_idx], spa, bytes_remaining); 4374 4375 scan_evicted += bytes_evicted; 4376 total_evicted += bytes_evicted; 4377 4378 /* we've reached the end, wrap to the beginning */ 4379 if (++sublist_idx >= num_sublists) 4380 sublist_idx = 0; 4381 } 4382 4383 /* 4384 * If we didn't evict anything during this scan, we have 4385 * no reason to believe we'll evict more during another 4386 * scan, so break the loop. 4387 */ 4388 if (scan_evicted == 0) { 4389 /* This isn't possible, let's make that obvious */ 4390 ASSERT3S(bytes, !=, 0); 4391 4392 /* 4393 * When bytes is ARC_EVICT_ALL, the only way to 4394 * break the loop is when scan_evicted is zero. 4395 * In that case, we actually have evicted enough, 4396 * so we don't want to increment the kstat. 4397 */ 4398 if (bytes != ARC_EVICT_ALL) { 4399 ASSERT3S(total_evicted, <, bytes); 4400 ARCSTAT_BUMP(arcstat_evict_not_enough); 4401 } 4402 4403 break; 4404 } 4405 } 4406 4407 for (int i = 0; i < num_sublists; i++) { 4408 multilist_sublist_t *mls = multilist_sublist_lock(ml, i); 4409 multilist_sublist_remove(mls, markers[i]); 4410 multilist_sublist_unlock(mls); 4411 4412 kmem_cache_free(hdr_full_cache, markers[i]); 4413 } 4414 kmem_free(markers, sizeof (*markers) * num_sublists); 4415 4416 return (total_evicted); 4417} 4418 4419/* 4420 * Flush all "evictable" data of the given type from the arc state 4421 * specified. This will not evict any "active" buffers (i.e. referenced). 4422 * 4423 * When 'retry' is set to B_FALSE, the function will make a single pass 4424 * over the state and evict any buffers that it can. Since it doesn't 4425 * continually retry the eviction, it might end up leaving some buffers 4426 * in the ARC due to lock misses. 4427 * 4428 * When 'retry' is set to B_TRUE, the function will continually retry the 4429 * eviction until *all* evictable buffers have been removed from the 4430 * state. As a result, if concurrent insertions into the state are 4431 * allowed (e.g. if the ARC isn't shutting down), this function might 4432 * wind up in an infinite loop, continually trying to evict buffers. 4433 */ 4434static uint64_t 4435arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type, 4436 boolean_t retry) 4437{ 4438 uint64_t evicted = 0; 4439 4440 while (zfs_refcount_count(&state->arcs_esize[type]) != 0) { 4441 evicted += arc_evict_state(state, spa, ARC_EVICT_ALL, type); 4442 4443 if (!retry) 4444 break; 4445 } 4446 4447 return (evicted); 4448} 4449 4450/* 4451 * Evict the specified number of bytes from the state specified, 4452 * restricting eviction to the spa and type given. This function 4453 * prevents us from trying to evict more from a state's list than 4454 * is "evictable", and to skip evicting altogether when passed a 4455 * negative value for "bytes". In contrast, arc_evict_state() will 4456 * evict everything it can, when passed a negative value for "bytes". 4457 */ 4458static uint64_t 4459arc_adjust_impl(arc_state_t *state, uint64_t spa, int64_t bytes, 4460 arc_buf_contents_t type) 4461{ 4462 int64_t delta; 4463 4464 if (bytes > 0 && zfs_refcount_count(&state->arcs_esize[type]) > 0) { 4465 delta = MIN(zfs_refcount_count(&state->arcs_esize[type]), 4466 bytes); 4467 return (arc_evict_state(state, spa, delta, type)); 4468 } 4469 4470 return (0); 4471} 4472 4473/* 4474 * Evict metadata buffers from the cache, such that arc_meta_used is 4475 * capped by the arc_meta_limit tunable. 4476 */ 4477static uint64_t 4478arc_adjust_meta(uint64_t meta_used) 4479{ 4480 uint64_t total_evicted = 0; 4481 int64_t target; 4482 4483 /* 4484 * If we're over the meta limit, we want to evict enough 4485 * metadata to get back under the meta limit. We don't want to 4486 * evict so much that we drop the MRU below arc_p, though. If 4487 * we're over the meta limit more than we're over arc_p, we 4488 * evict some from the MRU here, and some from the MFU below. 4489 */ 4490 target = MIN((int64_t)(meta_used - arc_meta_limit), 4491 (int64_t)(zfs_refcount_count(&arc_anon->arcs_size) + 4492 zfs_refcount_count(&arc_mru->arcs_size) - arc_p)); 4493 4494 total_evicted += arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA); 4495 4496 /* 4497 * Similar to the above, we want to evict enough bytes to get us 4498 * below the meta limit, but not so much as to drop us below the 4499 * space allotted to the MFU (which is defined as arc_c - arc_p). 4500 */ 4501 target = MIN((int64_t)(meta_used - arc_meta_limit), 4502 (int64_t)(zfs_refcount_count(&arc_mfu->arcs_size) - 4503 (arc_c - arc_p))); 4504 4505 total_evicted += arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA); 4506 4507 return (total_evicted); 4508} 4509 4510/* 4511 * Return the type of the oldest buffer in the given arc state 4512 * 4513 * This function will select a random sublist of type ARC_BUFC_DATA and 4514 * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist 4515 * is compared, and the type which contains the "older" buffer will be 4516 * returned. 4517 */ 4518static arc_buf_contents_t 4519arc_adjust_type(arc_state_t *state) 4520{ 4521 multilist_t *data_ml = state->arcs_list[ARC_BUFC_DATA]; 4522 multilist_t *meta_ml = state->arcs_list[ARC_BUFC_METADATA]; 4523 int data_idx = multilist_get_random_index(data_ml); 4524 int meta_idx = multilist_get_random_index(meta_ml); 4525 multilist_sublist_t *data_mls; 4526 multilist_sublist_t *meta_mls; 4527 arc_buf_contents_t type; 4528 arc_buf_hdr_t *data_hdr; 4529 arc_buf_hdr_t *meta_hdr; 4530 4531 /* 4532 * We keep the sublist lock until we're finished, to prevent 4533 * the headers from being destroyed via arc_evict_state(). 4534 */ 4535 data_mls = multilist_sublist_lock(data_ml, data_idx); 4536 meta_mls = multilist_sublist_lock(meta_ml, meta_idx); 4537 4538 /* 4539 * These two loops are to ensure we skip any markers that 4540 * might be at the tail of the lists due to arc_evict_state(). 4541 */ 4542 4543 for (data_hdr = multilist_sublist_tail(data_mls); data_hdr != NULL; 4544 data_hdr = multilist_sublist_prev(data_mls, data_hdr)) { 4545 if (data_hdr->b_spa != 0) 4546 break; 4547 } 4548 4549 for (meta_hdr = multilist_sublist_tail(meta_mls); meta_hdr != NULL; 4550 meta_hdr = multilist_sublist_prev(meta_mls, meta_hdr)) { 4551 if (meta_hdr->b_spa != 0) 4552 break; 4553 } 4554 4555 if (data_hdr == NULL && meta_hdr == NULL) { 4556 type = ARC_BUFC_DATA; 4557 } else if (data_hdr == NULL) { 4558 ASSERT3P(meta_hdr, !=, NULL); 4559 type = ARC_BUFC_METADATA; 4560 } else if (meta_hdr == NULL) { 4561 ASSERT3P(data_hdr, !=, NULL); 4562 type = ARC_BUFC_DATA; 4563 } else { 4564 ASSERT3P(data_hdr, !=, NULL); 4565 ASSERT3P(meta_hdr, !=, NULL); 4566 4567 /* The headers can't be on the sublist without an L1 header */ 4568 ASSERT(HDR_HAS_L1HDR(data_hdr)); 4569 ASSERT(HDR_HAS_L1HDR(meta_hdr)); 4570 4571 if (data_hdr->b_l1hdr.b_arc_access < 4572 meta_hdr->b_l1hdr.b_arc_access) { 4573 type = ARC_BUFC_DATA; 4574 } else { 4575 type = ARC_BUFC_METADATA; 4576 } 4577 } 4578 4579 multilist_sublist_unlock(meta_mls); 4580 multilist_sublist_unlock(data_mls); 4581 4582 return (type); 4583} 4584 4585/* 4586 * Evict buffers from the cache, such that arc_size is capped by arc_c. 4587 */ 4588static uint64_t 4589arc_adjust(void) 4590{ 4591 uint64_t total_evicted = 0; 4592 uint64_t bytes; 4593 int64_t target; 4594 uint64_t asize = aggsum_value(&arc_size); 4595 uint64_t ameta = aggsum_value(&arc_meta_used); 4596 4597 /* 4598 * If we're over arc_meta_limit, we want to correct that before 4599 * potentially evicting data buffers below. 4600 */ 4601 total_evicted += arc_adjust_meta(ameta); 4602 4603 /* 4604 * Adjust MRU size 4605 * 4606 * If we're over the target cache size, we want to evict enough 4607 * from the list to get back to our target size. We don't want 4608 * to evict too much from the MRU, such that it drops below 4609 * arc_p. So, if we're over our target cache size more than 4610 * the MRU is over arc_p, we'll evict enough to get back to 4611 * arc_p here, and then evict more from the MFU below. 4612 */ 4613 target = MIN((int64_t)(asize - arc_c), 4614 (int64_t)(zfs_refcount_count(&arc_anon->arcs_size) + 4615 zfs_refcount_count(&arc_mru->arcs_size) + ameta - arc_p)); 4616 4617 /* 4618 * If we're below arc_meta_min, always prefer to evict data. 4619 * Otherwise, try to satisfy the requested number of bytes to 4620 * evict from the type which contains older buffers; in an 4621 * effort to keep newer buffers in the cache regardless of their 4622 * type. If we cannot satisfy the number of bytes from this 4623 * type, spill over into the next type. 4624 */ 4625 if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA && 4626 ameta > arc_meta_min) { 4627 bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA); 4628 total_evicted += bytes; 4629 4630 /* 4631 * If we couldn't evict our target number of bytes from 4632 * metadata, we try to get the rest from data. 4633 */ 4634 target -= bytes; 4635 4636 total_evicted += 4637 arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA); 4638 } else { 4639 bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA); 4640 total_evicted += bytes; 4641 4642 /* 4643 * If we couldn't evict our target number of bytes from 4644 * data, we try to get the rest from metadata. 4645 */ 4646 target -= bytes; 4647 4648 total_evicted += 4649 arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA); 4650 } 4651 4652 /* 4653 * Adjust MFU size 4654 * 4655 * Now that we've tried to evict enough from the MRU to get its 4656 * size back to arc_p, if we're still above the target cache 4657 * size, we evict the rest from the MFU. 4658 */ 4659 target = asize - arc_c; 4660 4661 if (arc_adjust_type(arc_mfu) == ARC_BUFC_METADATA && 4662 ameta > arc_meta_min) { 4663 bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA); 4664 total_evicted += bytes; 4665 4666 /* 4667 * If we couldn't evict our target number of bytes from 4668 * metadata, we try to get the rest from data. 4669 */ 4670 target -= bytes; 4671 4672 total_evicted += 4673 arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA); 4674 } else { 4675 bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA); 4676 total_evicted += bytes; 4677 4678 /* 4679 * If we couldn't evict our target number of bytes from 4680 * data, we try to get the rest from data. 4681 */ 4682 target -= bytes; 4683 4684 total_evicted += 4685 arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA); 4686 } 4687 4688 /* 4689 * Adjust ghost lists 4690 * 4691 * In addition to the above, the ARC also defines target values 4692 * for the ghost lists. The sum of the mru list and mru ghost 4693 * list should never exceed the target size of the cache, and 4694 * the sum of the mru list, mfu list, mru ghost list, and mfu 4695 * ghost list should never exceed twice the target size of the 4696 * cache. The following logic enforces these limits on the ghost 4697 * caches, and evicts from them as needed. 4698 */ 4699 target = zfs_refcount_count(&arc_mru->arcs_size) + 4700 zfs_refcount_count(&arc_mru_ghost->arcs_size) - arc_c; 4701 4702 bytes = arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_DATA); 4703 total_evicted += bytes; 4704 4705 target -= bytes; 4706 4707 total_evicted += 4708 arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_METADATA); 4709 4710 /* 4711 * We assume the sum of the mru list and mfu list is less than 4712 * or equal to arc_c (we enforced this above), which means we 4713 * can use the simpler of the two equations below: 4714 * 4715 * mru + mfu + mru ghost + mfu ghost <= 2 * arc_c 4716 * mru ghost + mfu ghost <= arc_c 4717 */ 4718 target = zfs_refcount_count(&arc_mru_ghost->arcs_size) + 4719 zfs_refcount_count(&arc_mfu_ghost->arcs_size) - arc_c; 4720 4721 bytes = arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_DATA); 4722 total_evicted += bytes; 4723 4724 target -= bytes; 4725 4726 total_evicted += 4727 arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_METADATA); 4728 4729 return (total_evicted); 4730} 4731 4732void 4733arc_flush(spa_t *spa, boolean_t retry) 4734{ 4735 uint64_t guid = 0; 4736 4737 /* 4738 * If retry is B_TRUE, a spa must not be specified since we have 4739 * no good way to determine if all of a spa's buffers have been 4740 * evicted from an arc state. 4741 */ 4742 ASSERT(!retry || spa == 0); 4743 4744 if (spa != NULL) 4745 guid = spa_load_guid(spa); 4746 4747 (void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry); 4748 (void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry); 4749 4750 (void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry); 4751 (void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry); 4752 4753 (void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry); 4754 (void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry); 4755 4756 (void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry); 4757 (void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry); 4758} 4759 4760static void 4761arc_reduce_target_size(int64_t to_free) 4762{ 4763 uint64_t asize = aggsum_value(&arc_size); 4764 if (arc_c > arc_c_min) { 4765 4766 if (arc_c > arc_c_min + to_free) 4767 atomic_add_64(&arc_c, -to_free); 4768 else 4769 arc_c = arc_c_min; 4770 4771 atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift)); 4772 if (asize < arc_c) 4773 arc_c = MAX(asize, arc_c_min); 4774 if (arc_p > arc_c) 4775 arc_p = (arc_c >> 1); 4776 ASSERT(arc_c >= arc_c_min); 4777 ASSERT((int64_t)arc_p >= 0); 4778 } 4779 4780 if (asize > arc_c) { 4781 /* See comment in arc_adjust_cb_check() on why lock+flag */ 4782 mutex_enter(&arc_adjust_lock); 4783 arc_adjust_needed = B_TRUE; 4784 mutex_exit(&arc_adjust_lock); 4785 zthr_wakeup(arc_adjust_zthr); 4786 } 4787} 4788 4789typedef enum free_memory_reason_t { 4790 FMR_UNKNOWN, 4791 FMR_NEEDFREE, 4792 FMR_LOTSFREE, 4793 FMR_SWAPFS_MINFREE, 4794 FMR_PAGES_PP_MAXIMUM, 4795 FMR_HEAP_ARENA, 4796 FMR_ZIO_ARENA, 4797} free_memory_reason_t; 4798 4799int64_t last_free_memory; 4800free_memory_reason_t last_free_reason; 4801 4802/* 4803 * Additional reserve of pages for pp_reserve. 4804 */ 4805int64_t arc_pages_pp_reserve = 64; 4806 4807/* 4808 * Additional reserve of pages for swapfs. 4809 */ 4810int64_t arc_swapfs_reserve = 64; 4811 4812/* 4813 * Return the amount of memory that can be consumed before reclaim will be 4814 * needed. Positive if there is sufficient free memory, negative indicates 4815 * the amount of memory that needs to be freed up. 4816 */ 4817static int64_t 4818arc_available_memory(void) 4819{ 4820 int64_t lowest = INT64_MAX; 4821 int64_t n; 4822 free_memory_reason_t r = FMR_UNKNOWN; 4823 4824#ifdef _KERNEL 4825 if (needfree > 0) { 4826 n = PAGESIZE * (-needfree); 4827 if (n < lowest) { 4828 lowest = n; 4829 r = FMR_NEEDFREE; 4830 } 4831 } 4832 4833 /* 4834 * check that we're out of range of the pageout scanner. It starts to 4835 * schedule paging if freemem is less than lotsfree and needfree. 4836 * lotsfree is the high-water mark for pageout, and needfree is the 4837 * number of needed free pages. We add extra pages here to make sure 4838 * the scanner doesn't start up while we're freeing memory. 4839 */ 4840 n = PAGESIZE * (freemem - lotsfree - needfree - desfree); 4841 if (n < lowest) { 4842 lowest = n; 4843 r = FMR_LOTSFREE; 4844 } 4845 4846 /* 4847 * check to make sure that swapfs has enough space so that anon 4848 * reservations can still succeed. anon_resvmem() checks that the 4849 * availrmem is greater than swapfs_minfree, and the number of reserved 4850 * swap pages. We also add a bit of extra here just to prevent 4851 * circumstances from getting really dire. 4852 */ 4853 n = PAGESIZE * (availrmem - swapfs_minfree - swapfs_reserve - 4854 desfree - arc_swapfs_reserve); 4855 if (n < lowest) { 4856 lowest = n; 4857 r = FMR_SWAPFS_MINFREE; 4858 } 4859 4860 4861 /* 4862 * Check that we have enough availrmem that memory locking (e.g., via 4863 * mlock(3C) or memcntl(2)) can still succeed. (pages_pp_maximum 4864 * stores the number of pages that cannot be locked; when availrmem 4865 * drops below pages_pp_maximum, page locking mechanisms such as 4866 * page_pp_lock() will fail.) 4867 */ 4868 n = PAGESIZE * (availrmem - pages_pp_maximum - 4869 arc_pages_pp_reserve); 4870 if (n < lowest) { 4871 lowest = n; 4872 r = FMR_PAGES_PP_MAXIMUM; 4873 } 4874 4875#if defined(__i386) 4876 /* 4877 * If we're on an i386 platform, it's possible that we'll exhaust the 4878 * kernel heap space before we ever run out of available physical 4879 * memory. Most checks of the size of the heap_area compare against 4880 * tune.t_minarmem, which is the minimum available real memory that we 4881 * can have in the system. However, this is generally fixed at 25 pages 4882 * which is so low that it's useless. In this comparison, we seek to 4883 * calculate the total heap-size, and reclaim if more than 3/4ths of the 4884 * heap is allocated. (Or, in the calculation, if less than 1/4th is 4885 * free) 4886 */ 4887 n = (int64_t)vmem_size(heap_arena, VMEM_FREE) - 4888 (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2); 4889 if (n < lowest) { 4890 lowest = n; 4891 r = FMR_HEAP_ARENA; 4892 } 4893#endif 4894 4895 /* 4896 * If zio data pages are being allocated out of a separate heap segment, 4897 * then enforce that the size of available vmem for this arena remains 4898 * above about 1/4th (1/(2^arc_zio_arena_free_shift)) free. 4899 * 4900 * Note that reducing the arc_zio_arena_free_shift keeps more virtual 4901 * memory (in the zio_arena) free, which can avoid memory 4902 * fragmentation issues. 4903 */ 4904 if (zio_arena != NULL) { 4905 n = (int64_t)vmem_size(zio_arena, VMEM_FREE) - 4906 (vmem_size(zio_arena, VMEM_ALLOC) >> 4907 arc_zio_arena_free_shift); 4908 if (n < lowest) { 4909 lowest = n; 4910 r = FMR_ZIO_ARENA; 4911 } 4912 } 4913#else 4914 /* Every 100 calls, free a small amount */ 4915 if (spa_get_random(100) == 0) 4916 lowest = -1024; 4917#endif 4918 4919 last_free_memory = lowest; 4920 last_free_reason = r; 4921 4922 return (lowest); 4923} 4924 4925 4926/* 4927 * Determine if the system is under memory pressure and is asking 4928 * to reclaim memory. A return value of B_TRUE indicates that the system 4929 * is under memory pressure and that the arc should adjust accordingly. 4930 */ 4931static boolean_t 4932arc_reclaim_needed(void) 4933{ 4934 return (arc_available_memory() < 0); 4935} 4936 4937static void 4938arc_kmem_reap_soon(void) 4939{ 4940 size_t i; 4941 kmem_cache_t *prev_cache = NULL; 4942 kmem_cache_t *prev_data_cache = NULL; 4943 extern kmem_cache_t *zio_buf_cache[]; 4944 extern kmem_cache_t *zio_data_buf_cache[]; 4945 extern kmem_cache_t *zfs_btree_leaf_cache; 4946 extern kmem_cache_t *abd_chunk_cache; 4947 4948#ifdef _KERNEL 4949 if (aggsum_compare(&arc_meta_used, arc_meta_limit) >= 0) { 4950 /* 4951 * We are exceeding our meta-data cache limit. 4952 * Purge some DNLC entries to release holds on meta-data. 4953 */ 4954 dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); 4955 } 4956#if defined(__i386) 4957 /* 4958 * Reclaim unused memory from all kmem caches. 4959 */ 4960 kmem_reap(); 4961#endif 4962#endif 4963 4964 for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { 4965 if (zio_buf_cache[i] != prev_cache) { 4966 prev_cache = zio_buf_cache[i]; 4967 kmem_cache_reap_soon(zio_buf_cache[i]); 4968 } 4969 if (zio_data_buf_cache[i] != prev_data_cache) { 4970 prev_data_cache = zio_data_buf_cache[i]; 4971 kmem_cache_reap_soon(zio_data_buf_cache[i]); 4972 } 4973 } 4974 kmem_cache_reap_soon(abd_chunk_cache); 4975 kmem_cache_reap_soon(buf_cache); 4976 kmem_cache_reap_soon(hdr_full_cache); 4977 kmem_cache_reap_soon(hdr_l2only_cache); 4978 kmem_cache_reap_soon(zfs_btree_leaf_cache); 4979 4980 if (zio_arena != NULL) { 4981 /* 4982 * Ask the vmem arena to reclaim unused memory from its 4983 * quantum caches. 4984 */ 4985 vmem_qcache_reap(zio_arena); 4986 } 4987} 4988 4989/* ARGSUSED */ 4990static boolean_t 4991arc_adjust_cb_check(void *arg, zthr_t *zthr) 4992{ 4993 /* 4994 * This is necessary in order for the mdb ::arc dcmd to 4995 * show up to date information. Since the ::arc command 4996 * does not call the kstat's update function, without 4997 * this call, the command may show stale stats for the 4998 * anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even 4999 * with this change, the data might be up to 1 second 5000 * out of date(the arc_adjust_zthr has a maximum sleep 5001 * time of 1 second); but that should suffice. The 5002 * arc_state_t structures can be queried directly if more 5003 * accurate information is needed. 5004 */ 5005 if (arc_ksp != NULL) 5006 arc_ksp->ks_update(arc_ksp, KSTAT_READ); 5007 5008 /* 5009 * We have to rely on arc_get_data_impl() to tell us when to adjust, 5010 * rather than checking if we are overflowing here, so that we are 5011 * sure to not leave arc_get_data_impl() waiting on 5012 * arc_adjust_waiters_cv. If we have become "not overflowing" since 5013 * arc_get_data_impl() checked, we need to wake it up. We could 5014 * broadcast the CV here, but arc_get_data_impl() may have not yet 5015 * gone to sleep. We would need to use a mutex to ensure that this 5016 * function doesn't broadcast until arc_get_data_impl() has gone to 5017 * sleep (e.g. the arc_adjust_lock). However, the lock ordering of 5018 * such a lock would necessarily be incorrect with respect to the 5019 * zthr_lock, which is held before this function is called, and is 5020 * held by arc_get_data_impl() when it calls zthr_wakeup(). 5021 */ 5022 return (arc_adjust_needed); 5023} 5024 5025/* 5026 * Keep arc_size under arc_c by running arc_adjust which evicts data 5027 * from the ARC. 5028 */ 5029/* ARGSUSED */ 5030static void 5031arc_adjust_cb(void *arg, zthr_t *zthr) 5032{ 5033 uint64_t evicted = 0; 5034 5035 /* Evict from cache */ 5036 evicted = arc_adjust(); 5037 5038 /* 5039 * If evicted is zero, we couldn't evict anything 5040 * via arc_adjust(). This could be due to hash lock 5041 * collisions, but more likely due to the majority of 5042 * arc buffers being unevictable. Therefore, even if 5043 * arc_size is above arc_c, another pass is unlikely to 5044 * be helpful and could potentially cause us to enter an 5045 * infinite loop. Additionally, zthr_iscancelled() is 5046 * checked here so that if the arc is shutting down, the 5047 * broadcast will wake any remaining arc adjust waiters. 5048 */ 5049 mutex_enter(&arc_adjust_lock); 5050 arc_adjust_needed = !zthr_iscancelled(arc_adjust_zthr) && 5051 evicted > 0 && aggsum_compare(&arc_size, arc_c) > 0; 5052 if (!arc_adjust_needed) { 5053 /* 5054 * We're either no longer overflowing, or we 5055 * can't evict anything more, so we should wake 5056 * up any waiters. 5057 */ 5058 cv_broadcast(&arc_adjust_waiters_cv); 5059 } 5060 mutex_exit(&arc_adjust_lock); 5061} 5062 5063/* ARGSUSED */ 5064static boolean_t 5065arc_reap_cb_check(void *arg, zthr_t *zthr) 5066{ 5067 int64_t free_memory = arc_available_memory(); 5068 5069 /* 5070 * If a kmem reap is already active, don't schedule more. We must 5071 * check for this because kmem_cache_reap_soon() won't actually 5072 * block on the cache being reaped (this is to prevent callers from 5073 * becoming implicitly blocked by a system-wide kmem reap -- which, 5074 * on a system with many, many full magazines, can take minutes). 5075 */ 5076 if (!kmem_cache_reap_active() && 5077 free_memory < 0) { 5078 arc_no_grow = B_TRUE; 5079 arc_warm = B_TRUE; 5080 /* 5081 * Wait at least zfs_grow_retry (default 60) seconds 5082 * before considering growing. 5083 */ 5084 arc_growtime = gethrtime() + SEC2NSEC(arc_grow_retry); 5085 return (B_TRUE); 5086 } else if (free_memory < arc_c >> arc_no_grow_shift) { 5087 arc_no_grow = B_TRUE; 5088 } else if (gethrtime() >= arc_growtime) { 5089 arc_no_grow = B_FALSE; 5090 } 5091 5092 return (B_FALSE); 5093} 5094 5095/* 5096 * Keep enough free memory in the system by reaping the ARC's kmem 5097 * caches. To cause more slabs to be reapable, we may reduce the 5098 * target size of the cache (arc_c), causing the arc_adjust_cb() 5099 * to free more buffers. 5100 */ 5101/* ARGSUSED */ 5102static void 5103arc_reap_cb(void *arg, zthr_t *zthr) 5104{ 5105 int64_t free_memory; 5106 5107 /* 5108 * Kick off asynchronous kmem_reap()'s of all our caches. 5109 */ 5110 arc_kmem_reap_soon(); 5111 5112 /* 5113 * Wait at least arc_kmem_cache_reap_retry_ms between 5114 * arc_kmem_reap_soon() calls. Without this check it is possible to 5115 * end up in a situation where we spend lots of time reaping 5116 * caches, while we're near arc_c_min. Waiting here also gives the 5117 * subsequent free memory check a chance of finding that the 5118 * asynchronous reap has already freed enough memory, and we don't 5119 * need to call arc_reduce_target_size(). 5120 */ 5121 delay((hz * arc_kmem_cache_reap_retry_ms + 999) / 1000); 5122 5123 /* 5124 * Reduce the target size as needed to maintain the amount of free 5125 * memory in the system at a fraction of the arc_size (1/128th by 5126 * default). If oversubscribed (free_memory < 0) then reduce the 5127 * target arc_size by the deficit amount plus the fractional 5128 * amount. If free memory is positive but less then the fractional 5129 * amount, reduce by what is needed to hit the fractional amount. 5130 */ 5131 free_memory = arc_available_memory(); 5132 5133 int64_t to_free = 5134 (arc_c >> arc_shrink_shift) - free_memory; 5135 if (to_free > 0) { 5136#ifdef _KERNEL 5137 to_free = MAX(to_free, ptob(needfree)); 5138#endif 5139 arc_reduce_target_size(to_free); 5140 } 5141} 5142 5143/* 5144 * Adapt arc info given the number of bytes we are trying to add and 5145 * the state that we are coming from. This function is only called 5146 * when we are adding new content to the cache. 5147 */ 5148static void 5149arc_adapt(int bytes, arc_state_t *state) 5150{ 5151 int mult; 5152 uint64_t arc_p_min = (arc_c >> arc_p_min_shift); 5153 int64_t mrug_size = zfs_refcount_count(&arc_mru_ghost->arcs_size); 5154 int64_t mfug_size = zfs_refcount_count(&arc_mfu_ghost->arcs_size); 5155 5156 if (state == arc_l2c_only) 5157 return; 5158 5159 ASSERT(bytes > 0); 5160 /* 5161 * Adapt the target size of the MRU list: 5162 * - if we just hit in the MRU ghost list, then increase 5163 * the target size of the MRU list. 5164 * - if we just hit in the MFU ghost list, then increase 5165 * the target size of the MFU list by decreasing the 5166 * target size of the MRU list. 5167 */ 5168 if (state == arc_mru_ghost) { 5169 mult = (mrug_size >= mfug_size) ? 1 : (mfug_size / mrug_size); 5170 mult = MIN(mult, 10); /* avoid wild arc_p adjustment */ 5171 5172 arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult); 5173 } else if (state == arc_mfu_ghost) { 5174 uint64_t delta; 5175 5176 mult = (mfug_size >= mrug_size) ? 1 : (mrug_size / mfug_size); 5177 mult = MIN(mult, 10); 5178 5179 delta = MIN(bytes * mult, arc_p); 5180 arc_p = MAX(arc_p_min, arc_p - delta); 5181 } 5182 ASSERT((int64_t)arc_p >= 0); 5183 5184 /* 5185 * Wake reap thread if we do not have any available memory 5186 */ 5187 if (arc_reclaim_needed()) { 5188 zthr_wakeup(arc_reap_zthr); 5189 return; 5190 } 5191 5192 5193 if (arc_no_grow) 5194 return; 5195 5196 if (arc_c >= arc_c_max) 5197 return; 5198 5199 /* 5200 * If we're within (2 * maxblocksize) bytes of the target 5201 * cache size, increment the target cache size 5202 */ 5203 if (aggsum_compare(&arc_size, arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) > 5204 0) { 5205 atomic_add_64(&arc_c, (int64_t)bytes); 5206 if (arc_c > arc_c_max) 5207 arc_c = arc_c_max; 5208 else if (state == arc_anon) 5209 atomic_add_64(&arc_p, (int64_t)bytes); 5210 if (arc_p > arc_c) 5211 arc_p = arc_c; 5212 } 5213 ASSERT((int64_t)arc_p >= 0); 5214} 5215 5216/* 5217 * Check if arc_size has grown past our upper threshold, determined by 5218 * zfs_arc_overflow_shift. 5219 */ 5220static boolean_t 5221arc_is_overflowing(void) 5222{ 5223 /* Always allow at least one block of overflow */ 5224 uint64_t overflow = MAX(SPA_MAXBLOCKSIZE, 5225 arc_c >> zfs_arc_overflow_shift); 5226 5227 /* 5228 * We just compare the lower bound here for performance reasons. Our 5229 * primary goals are to make sure that the arc never grows without 5230 * bound, and that it can reach its maximum size. This check 5231 * accomplishes both goals. The maximum amount we could run over by is 5232 * 2 * aggsum_borrow_multiplier * NUM_CPUS * the average size of a block 5233 * in the ARC. In practice, that's in the tens of MB, which is low 5234 * enough to be safe. 5235 */ 5236 return (aggsum_lower_bound(&arc_size) >= arc_c + overflow); 5237} 5238 5239static abd_t * 5240arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, void *tag) 5241{ 5242 arc_buf_contents_t type = arc_buf_type(hdr); 5243 5244 arc_get_data_impl(hdr, size, tag); 5245 if (type == ARC_BUFC_METADATA) { 5246 return (abd_alloc(size, B_TRUE)); 5247 } else { 5248 ASSERT(type == ARC_BUFC_DATA); 5249 return (abd_alloc(size, B_FALSE)); 5250 } 5251} 5252 5253static void * 5254arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, void *tag) 5255{ 5256 arc_buf_contents_t type = arc_buf_type(hdr); 5257 5258 arc_get_data_impl(hdr, size, tag); 5259 if (type == ARC_BUFC_METADATA) { 5260 return (zio_buf_alloc(size)); 5261 } else { 5262 ASSERT(type == ARC_BUFC_DATA); 5263 return (zio_data_buf_alloc(size)); 5264 } 5265} 5266 5267/* 5268 * Allocate a block and return it to the caller. If we are hitting the 5269 * hard limit for the cache size, we must sleep, waiting for the eviction 5270 * thread to catch up. If we're past the target size but below the hard 5271 * limit, we'll only signal the reclaim thread and continue on. 5272 */ 5273static void 5274arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag) 5275{ 5276 arc_state_t *state = hdr->b_l1hdr.b_state; 5277 arc_buf_contents_t type = arc_buf_type(hdr); 5278 5279 arc_adapt(size, state); 5280 5281 /* 5282 * If arc_size is currently overflowing, and has grown past our 5283 * upper limit, we must be adding data faster than the evict 5284 * thread can evict. Thus, to ensure we don't compound the 5285 * problem by adding more data and forcing arc_size to grow even 5286 * further past its target size, we halt and wait for the 5287 * eviction thread to catch up. 5288 * 5289 * It's also possible that the reclaim thread is unable to evict 5290 * enough buffers to get arc_size below the overflow limit (e.g. 5291 * due to buffers being un-evictable, or hash lock collisions). 5292 * In this case, we want to proceed regardless if we're 5293 * overflowing; thus we don't use a while loop here. 5294 */ 5295 if (arc_is_overflowing()) { 5296 mutex_enter(&arc_adjust_lock); 5297 5298 /* 5299 * Now that we've acquired the lock, we may no longer be 5300 * over the overflow limit, lets check. 5301 * 5302 * We're ignoring the case of spurious wake ups. If that 5303 * were to happen, it'd let this thread consume an ARC 5304 * buffer before it should have (i.e. before we're under 5305 * the overflow limit and were signalled by the reclaim 5306 * thread). As long as that is a rare occurrence, it 5307 * shouldn't cause any harm. 5308 */ 5309 if (arc_is_overflowing()) { 5310 arc_adjust_needed = B_TRUE; 5311 zthr_wakeup(arc_adjust_zthr); 5312 (void) cv_wait(&arc_adjust_waiters_cv, 5313 &arc_adjust_lock); 5314 } 5315 mutex_exit(&arc_adjust_lock); 5316 } 5317 5318 VERIFY3U(hdr->b_type, ==, type); 5319 if (type == ARC_BUFC_METADATA) { 5320 arc_space_consume(size, ARC_SPACE_META); 5321 } else { 5322 arc_space_consume(size, ARC_SPACE_DATA); 5323 } 5324 5325 /* 5326 * Update the state size. Note that ghost states have a 5327 * "ghost size" and so don't need to be updated. 5328 */ 5329 if (!GHOST_STATE(state)) { 5330 5331 (void) zfs_refcount_add_many(&state->arcs_size, size, tag); 5332 5333 /* 5334 * If this is reached via arc_read, the link is 5335 * protected by the hash lock. If reached via 5336 * arc_buf_alloc, the header should not be accessed by 5337 * any other thread. And, if reached via arc_read_done, 5338 * the hash lock will protect it if it's found in the 5339 * hash table; otherwise no other thread should be 5340 * trying to [add|remove]_reference it. 5341 */ 5342 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { 5343 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 5344 (void) zfs_refcount_add_many(&state->arcs_esize[type], 5345 size, tag); 5346 } 5347 5348 /* 5349 * If we are growing the cache, and we are adding anonymous 5350 * data, and we have outgrown arc_p, update arc_p 5351 */ 5352 if (aggsum_compare(&arc_size, arc_c) < 0 && 5353 hdr->b_l1hdr.b_state == arc_anon && 5354 (zfs_refcount_count(&arc_anon->arcs_size) + 5355 zfs_refcount_count(&arc_mru->arcs_size) > arc_p)) 5356 arc_p = MIN(arc_c, arc_p + size); 5357 } 5358} 5359 5360static void 5361arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size, void *tag) 5362{ 5363 arc_free_data_impl(hdr, size, tag); 5364 abd_free(abd); 5365} 5366 5367static void 5368arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, void *tag) 5369{ 5370 arc_buf_contents_t type = arc_buf_type(hdr); 5371 5372 arc_free_data_impl(hdr, size, tag); 5373 if (type == ARC_BUFC_METADATA) { 5374 zio_buf_free(buf, size); 5375 } else { 5376 ASSERT(type == ARC_BUFC_DATA); 5377 zio_data_buf_free(buf, size); 5378 } 5379} 5380 5381/* 5382 * Free the arc data buffer. 5383 */ 5384static void 5385arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag) 5386{ 5387 arc_state_t *state = hdr->b_l1hdr.b_state; 5388 arc_buf_contents_t type = arc_buf_type(hdr); 5389 5390 /* protected by hash lock, if in the hash table */ 5391 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { 5392 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 5393 ASSERT(state != arc_anon && state != arc_l2c_only); 5394 5395 (void) zfs_refcount_remove_many(&state->arcs_esize[type], 5396 size, tag); 5397 } 5398 (void) zfs_refcount_remove_many(&state->arcs_size, size, tag); 5399 5400 VERIFY3U(hdr->b_type, ==, type); 5401 if (type == ARC_BUFC_METADATA) { 5402 arc_space_return(size, ARC_SPACE_META); 5403 } else { 5404 ASSERT(type == ARC_BUFC_DATA); 5405 arc_space_return(size, ARC_SPACE_DATA); 5406 } 5407} 5408 5409/* 5410 * This routine is called whenever a buffer is accessed. 5411 * NOTE: the hash lock is dropped in this function. 5412 */ 5413static void 5414arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) 5415{ 5416 clock_t now; 5417 5418 ASSERT(MUTEX_HELD(hash_lock)); 5419 ASSERT(HDR_HAS_L1HDR(hdr)); 5420 5421 if (hdr->b_l1hdr.b_state == arc_anon) { 5422 /* 5423 * This buffer is not in the cache, and does not 5424 * appear in our "ghost" list. Add the new buffer 5425 * to the MRU state. 5426 */ 5427 5428 ASSERT0(hdr->b_l1hdr.b_arc_access); 5429 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 5430 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); 5431 arc_change_state(arc_mru, hdr, hash_lock); 5432 5433 } else if (hdr->b_l1hdr.b_state == arc_mru) { 5434 now = ddi_get_lbolt(); 5435 5436 /* 5437 * If this buffer is here because of a prefetch, then either: 5438 * - clear the flag if this is a "referencing" read 5439 * (any subsequent access will bump this into the MFU state). 5440 * or 5441 * - move the buffer to the head of the list if this is 5442 * another prefetch (to make it less likely to be evicted). 5443 */ 5444 if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) { 5445 if (zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { 5446 /* link protected by hash lock */ 5447 ASSERT(multilist_link_active( 5448 &hdr->b_l1hdr.b_arc_node)); 5449 } else { 5450 arc_hdr_clear_flags(hdr, 5451 ARC_FLAG_PREFETCH | 5452 ARC_FLAG_PRESCIENT_PREFETCH); 5453 ARCSTAT_BUMP(arcstat_mru_hits); 5454 } 5455 hdr->b_l1hdr.b_arc_access = now; 5456 return; 5457 } 5458 5459 /* 5460 * This buffer has been "accessed" only once so far, 5461 * but it is still in the cache. Move it to the MFU 5462 * state. 5463 */ 5464 if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) { 5465 /* 5466 * More than 125ms have passed since we 5467 * instantiated this buffer. Move it to the 5468 * most frequently used state. 5469 */ 5470 hdr->b_l1hdr.b_arc_access = now; 5471 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); 5472 arc_change_state(arc_mfu, hdr, hash_lock); 5473 } 5474 ARCSTAT_BUMP(arcstat_mru_hits); 5475 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) { 5476 arc_state_t *new_state; 5477 /* 5478 * This buffer has been "accessed" recently, but 5479 * was evicted from the cache. Move it to the 5480 * MFU state. 5481 */ 5482 5483 if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) { 5484 new_state = arc_mru; 5485 if (zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) > 0) { 5486 arc_hdr_clear_flags(hdr, 5487 ARC_FLAG_PREFETCH | 5488 ARC_FLAG_PRESCIENT_PREFETCH); 5489 } 5490 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); 5491 } else { 5492 new_state = arc_mfu; 5493 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); 5494 } 5495 5496 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 5497 arc_change_state(new_state, hdr, hash_lock); 5498 5499 ARCSTAT_BUMP(arcstat_mru_ghost_hits); 5500 } else if (hdr->b_l1hdr.b_state == arc_mfu) { 5501 /* 5502 * This buffer has been accessed more than once and is 5503 * still in the cache. Keep it in the MFU state. 5504 * 5505 * NOTE: an add_reference() that occurred when we did 5506 * the arc_read() will have kicked this off the list. 5507 * If it was a prefetch, we will explicitly move it to 5508 * the head of the list now. 5509 */ 5510 ARCSTAT_BUMP(arcstat_mfu_hits); 5511 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 5512 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) { 5513 arc_state_t *new_state = arc_mfu; 5514 /* 5515 * This buffer has been accessed more than once but has 5516 * been evicted from the cache. Move it back to the 5517 * MFU state. 5518 */ 5519 5520 if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) { 5521 /* 5522 * This is a prefetch access... 5523 * move this block back to the MRU state. 5524 */ 5525 new_state = arc_mru; 5526 } 5527 5528 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 5529 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); 5530 arc_change_state(new_state, hdr, hash_lock); 5531 5532 ARCSTAT_BUMP(arcstat_mfu_ghost_hits); 5533 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) { 5534 /* 5535 * This buffer is on the 2nd Level ARC. 5536 */ 5537 5538 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 5539 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); 5540 arc_change_state(arc_mfu, hdr, hash_lock); 5541 } else { 5542 ASSERT(!"invalid arc state"); 5543 } 5544} 5545 5546/* 5547 * This routine is called by dbuf_hold() to update the arc_access() state 5548 * which otherwise would be skipped for entries in the dbuf cache. 5549 */ 5550void 5551arc_buf_access(arc_buf_t *buf) 5552{ 5553 mutex_enter(&buf->b_evict_lock); 5554 arc_buf_hdr_t *hdr = buf->b_hdr; 5555 5556 /* 5557 * Avoid taking the hash_lock when possible as an optimization. 5558 * The header must be checked again under the hash_lock in order 5559 * to handle the case where it is concurrently being released. 5560 */ 5561 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) { 5562 mutex_exit(&buf->b_evict_lock); 5563 return; 5564 } 5565 5566 kmutex_t *hash_lock = HDR_LOCK(hdr); 5567 mutex_enter(hash_lock); 5568 5569 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) { 5570 mutex_exit(hash_lock); 5571 mutex_exit(&buf->b_evict_lock); 5572 ARCSTAT_BUMP(arcstat_access_skip); 5573 return; 5574 } 5575 5576 mutex_exit(&buf->b_evict_lock); 5577 5578 ASSERT(hdr->b_l1hdr.b_state == arc_mru || 5579 hdr->b_l1hdr.b_state == arc_mfu); 5580 5581 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 5582 arc_access(hdr, hash_lock); 5583 mutex_exit(hash_lock); 5584 5585 ARCSTAT_BUMP(arcstat_hits); 5586 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), 5587 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data, metadata, hits); 5588} 5589 5590/* a generic arc_read_done_func_t which you can use */ 5591/* ARGSUSED */ 5592void 5593arc_bcopy_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, 5594 arc_buf_t *buf, void *arg) 5595{ 5596 if (buf == NULL) 5597 return; 5598 5599 bcopy(buf->b_data, arg, arc_buf_size(buf)); 5600 arc_buf_destroy(buf, arg); 5601} 5602 5603/* a generic arc_read_done_func_t */ 5604void 5605arc_getbuf_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, 5606 arc_buf_t *buf, void *arg) 5607{ 5608 arc_buf_t **bufp = arg; 5609 5610 if (buf == NULL) { 5611 ASSERT(zio == NULL || zio->io_error != 0); 5612 *bufp = NULL; 5613 } else { 5614 ASSERT(zio == NULL || zio->io_error == 0); 5615 *bufp = buf; 5616 ASSERT(buf->b_data != NULL); 5617 } 5618} 5619 5620static void 5621arc_hdr_verify(arc_buf_hdr_t *hdr, const blkptr_t *bp) 5622{ 5623 if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) { 5624 ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0); 5625 ASSERT3U(arc_hdr_get_compress(hdr), ==, ZIO_COMPRESS_OFF); 5626 } else { 5627 if (HDR_COMPRESSION_ENABLED(hdr)) { 5628 ASSERT3U(arc_hdr_get_compress(hdr), ==, 5629 BP_GET_COMPRESS(bp)); 5630 } 5631 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp)); 5632 ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp)); 5633 ASSERT3U(!!HDR_PROTECTED(hdr), ==, BP_IS_PROTECTED(bp)); 5634 } 5635} 5636 5637/* 5638 * XXX this should be changed to return an error, and callers 5639 * re-read from disk on failure (on nondebug bits). 5640 */ 5641static void 5642arc_hdr_verify_checksum(spa_t *spa, arc_buf_hdr_t *hdr, const blkptr_t *bp) 5643{ 5644 arc_hdr_verify(hdr, bp); 5645 if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) 5646 return; 5647 int err = 0; 5648 abd_t *abd = NULL; 5649 if (BP_IS_ENCRYPTED(bp)) { 5650 if (HDR_HAS_RABD(hdr)) { 5651 abd = hdr->b_crypt_hdr.b_rabd; 5652 } 5653 } else if (HDR_COMPRESSION_ENABLED(hdr)) { 5654 abd = hdr->b_l1hdr.b_pabd; 5655 } 5656 if (abd != NULL) { 5657 /* 5658 * The offset is only used for labels, which are not 5659 * cached in the ARC, so it doesn't matter what we 5660 * pass for the offset parameter. 5661 */ 5662 int psize = HDR_GET_PSIZE(hdr); 5663 err = zio_checksum_error_impl(spa, bp, 5664 BP_GET_CHECKSUM(bp), abd, psize, 0, NULL); 5665 if (err != 0) { 5666 /* 5667 * Use abd_copy_to_buf() rather than 5668 * abd_borrow_buf_copy() so that we are sure to 5669 * include the buf in crash dumps. 5670 */ 5671 void *buf = kmem_alloc(psize, KM_SLEEP); 5672 abd_copy_to_buf(buf, abd, psize); 5673 panic("checksum of cached data doesn't match BP " 5674 "err=%u hdr=%p bp=%p abd=%p buf=%p", 5675 err, (void *)hdr, (void *)bp, (void *)abd, buf); 5676 } 5677 } 5678} 5679 5680static void 5681arc_read_done(zio_t *zio) 5682{ 5683 blkptr_t *bp = zio->io_bp; 5684 arc_buf_hdr_t *hdr = zio->io_private; 5685 kmutex_t *hash_lock = NULL; 5686 arc_callback_t *callback_list; 5687 arc_callback_t *acb; 5688 boolean_t freeable = B_FALSE; 5689 5690 /* 5691 * The hdr was inserted into hash-table and removed from lists 5692 * prior to starting I/O. We should find this header, since 5693 * it's in the hash table, and it should be legit since it's 5694 * not possible to evict it during the I/O. The only possible 5695 * reason for it not to be found is if we were freed during the 5696 * read. 5697 */ 5698 if (HDR_IN_HASH_TABLE(hdr)) { 5699 ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp)); 5700 ASSERT3U(hdr->b_dva.dva_word[0], ==, 5701 BP_IDENTITY(zio->io_bp)->dva_word[0]); 5702 ASSERT3U(hdr->b_dva.dva_word[1], ==, 5703 BP_IDENTITY(zio->io_bp)->dva_word[1]); 5704 5705 arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp, 5706 &hash_lock); 5707 5708 ASSERT((found == hdr && 5709 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || 5710 (found == hdr && HDR_L2_READING(hdr))); 5711 ASSERT3P(hash_lock, !=, NULL); 5712 } 5713 5714 if (BP_IS_PROTECTED(bp)) { 5715 hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp); 5716 hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset; 5717 zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt, 5718 hdr->b_crypt_hdr.b_iv); 5719 5720 if (BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG) { 5721 void *tmpbuf; 5722 5723 tmpbuf = abd_borrow_buf_copy(zio->io_abd, 5724 sizeof (zil_chain_t)); 5725 zio_crypt_decode_mac_zil(tmpbuf, 5726 hdr->b_crypt_hdr.b_mac); 5727 abd_return_buf(zio->io_abd, tmpbuf, 5728 sizeof (zil_chain_t)); 5729 } else { 5730 zio_crypt_decode_mac_bp(bp, hdr->b_crypt_hdr.b_mac); 5731 } 5732 } 5733 5734 if (zio->io_error == 0) { 5735 /* byteswap if necessary */ 5736 if (BP_SHOULD_BYTESWAP(zio->io_bp)) { 5737 if (BP_GET_LEVEL(zio->io_bp) > 0) { 5738 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64; 5739 } else { 5740 hdr->b_l1hdr.b_byteswap = 5741 DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp)); 5742 } 5743 } else { 5744 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; 5745 } 5746 } 5747 5748 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED); 5749 if (l2arc_noprefetch && HDR_PREFETCH(hdr)) 5750 arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE); 5751 5752 callback_list = hdr->b_l1hdr.b_acb; 5753 ASSERT3P(callback_list, !=, NULL); 5754 5755 if (hash_lock && zio->io_error == 0 && 5756 hdr->b_l1hdr.b_state == arc_anon) { 5757 /* 5758 * Only call arc_access on anonymous buffers. This is because 5759 * if we've issued an I/O for an evicted buffer, we've already 5760 * called arc_access (to prevent any simultaneous readers from 5761 * getting confused). 5762 */ 5763 arc_access(hdr, hash_lock); 5764 } 5765 5766 /* 5767 * If a read request has a callback (i.e. acb_done is not NULL), then we 5768 * make a buf containing the data according to the parameters which were 5769 * passed in. The implementation of arc_buf_alloc_impl() ensures that we 5770 * aren't needlessly decompressing the data multiple times. 5771 */ 5772 int callback_cnt = 0; 5773 for (acb = callback_list; acb != NULL; acb = acb->acb_next) { 5774 if (!acb->acb_done) 5775 continue; 5776 5777 callback_cnt++; 5778 5779 if (zio->io_error != 0) 5780 continue; 5781 5782 int error = arc_buf_alloc_impl(hdr, zio->io_spa, 5783 &acb->acb_zb, acb->acb_private, acb->acb_encrypted, 5784 acb->acb_compressed, acb->acb_noauth, B_TRUE, 5785 &acb->acb_buf); 5786 5787 /* 5788 * Assert non-speculative zios didn't fail because an 5789 * encryption key wasn't loaded 5790 */ 5791 ASSERT((zio->io_flags & ZIO_FLAG_SPECULATIVE) || 5792 error != EACCES); 5793 5794 /* 5795 * If we failed to decrypt, report an error now (as the zio 5796 * layer would have done if it had done the transforms). 5797 */ 5798 if (error == ECKSUM) { 5799 ASSERT(BP_IS_PROTECTED(bp)); 5800 error = SET_ERROR(EIO); 5801 if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) { 5802 spa_log_error(zio->io_spa, &acb->acb_zb); 5803 zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION, 5804 zio->io_spa, NULL, &acb->acb_zb, zio, 0, 0); 5805 } 5806 } 5807 5808 if (error != 0) { 5809 /* 5810 * Decompression failed. Set io_error 5811 * so that when we call acb_done (below), 5812 * we will indicate that the read failed. 5813 * Note that in the unusual case where one 5814 * callback is compressed and another 5815 * uncompressed, we will mark all of them 5816 * as failed, even though the uncompressed 5817 * one can't actually fail. In this case, 5818 * the hdr will not be anonymous, because 5819 * if there are multiple callbacks, it's 5820 * because multiple threads found the same 5821 * arc buf in the hash table. 5822 */ 5823 zio->io_error = error; 5824 } 5825 } 5826 5827 /* 5828 * If there are multiple callbacks, we must have the hash lock, 5829 * because the only way for multiple threads to find this hdr is 5830 * in the hash table. This ensures that if there are multiple 5831 * callbacks, the hdr is not anonymous. If it were anonymous, 5832 * we couldn't use arc_buf_destroy() in the error case below. 5833 */ 5834 ASSERT(callback_cnt < 2 || hash_lock != NULL); 5835 5836 hdr->b_l1hdr.b_acb = NULL; 5837 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); 5838 if (callback_cnt == 0) 5839 ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr)); 5840 5841 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt) || 5842 callback_list != NULL); 5843 5844 if (zio->io_error == 0) { 5845 arc_hdr_verify(hdr, zio->io_bp); 5846 } else { 5847 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); 5848 if (hdr->b_l1hdr.b_state != arc_anon) 5849 arc_change_state(arc_anon, hdr, hash_lock); 5850 if (HDR_IN_HASH_TABLE(hdr)) 5851 buf_hash_remove(hdr); 5852 freeable = zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt); 5853 } 5854 5855 /* 5856 * Broadcast before we drop the hash_lock to avoid the possibility 5857 * that the hdr (and hence the cv) might be freed before we get to 5858 * the cv_broadcast(). 5859 */ 5860 cv_broadcast(&hdr->b_l1hdr.b_cv); 5861 5862 if (hash_lock != NULL) { 5863 mutex_exit(hash_lock); 5864 } else { 5865 /* 5866 * This block was freed while we waited for the read to 5867 * complete. It has been removed from the hash table and 5868 * moved to the anonymous state (so that it won't show up 5869 * in the cache). 5870 */ 5871 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); 5872 freeable = zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt); 5873 } 5874 5875 /* execute each callback and free its structure */ 5876 while ((acb = callback_list) != NULL) { 5877 5878 if (acb->acb_done != NULL) { 5879 if (zio->io_error != 0 && acb->acb_buf != NULL) { 5880 /* 5881 * If arc_buf_alloc_impl() fails during 5882 * decompression, the buf will still be 5883 * allocated, and needs to be freed here. 5884 */ 5885 arc_buf_destroy(acb->acb_buf, acb->acb_private); 5886 acb->acb_buf = NULL; 5887 } 5888 acb->acb_done(zio, &zio->io_bookmark, zio->io_bp, 5889 acb->acb_buf, acb->acb_private); 5890 } 5891 5892 if (acb->acb_zio_dummy != NULL) { 5893 acb->acb_zio_dummy->io_error = zio->io_error; 5894 zio_nowait(acb->acb_zio_dummy); 5895 } 5896 5897 callback_list = acb->acb_next; 5898 kmem_free(acb, sizeof (arc_callback_t)); 5899 } 5900 5901 if (freeable) 5902 arc_hdr_destroy(hdr); 5903} 5904 5905/* 5906 * "Read" the block at the specified DVA (in bp) via the 5907 * cache. If the block is found in the cache, invoke the provided 5908 * callback immediately and return. Note that the `zio' parameter 5909 * in the callback will be NULL in this case, since no IO was 5910 * required. If the block is not in the cache pass the read request 5911 * on to the spa with a substitute callback function, so that the 5912 * requested block will be added to the cache. 5913 * 5914 * If a read request arrives for a block that has a read in-progress, 5915 * either wait for the in-progress read to complete (and return the 5916 * results); or, if this is a read with a "done" func, add a record 5917 * to the read to invoke the "done" func when the read completes, 5918 * and return; or just return. 5919 * 5920 * arc_read_done() will invoke all the requested "done" functions 5921 * for readers of this block. 5922 */ 5923int 5924arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_read_done_func_t *done, 5925 void *private, zio_priority_t priority, int zio_flags, 5926 arc_flags_t *arc_flags, const zbookmark_phys_t *zb) 5927{ 5928 arc_buf_hdr_t *hdr = NULL; 5929 kmutex_t *hash_lock = NULL; 5930 zio_t *rzio; 5931 uint64_t guid = spa_load_guid(spa); 5932 boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW_COMPRESS) != 0; 5933 boolean_t encrypted_read = BP_IS_ENCRYPTED(bp) && 5934 (zio_flags & ZIO_FLAG_RAW_ENCRYPT) != 0; 5935 boolean_t noauth_read = BP_IS_AUTHENTICATED(bp) && 5936 (zio_flags & ZIO_FLAG_RAW_ENCRYPT) != 0; 5937 int rc = 0; 5938 5939 ASSERT(!BP_IS_EMBEDDED(bp) || 5940 BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA); 5941 5942top: 5943 if (!BP_IS_EMBEDDED(bp)) { 5944 /* 5945 * Embedded BP's have no DVA and require no I/O to "read". 5946 * Create an anonymous arc buf to back it. 5947 */ 5948 hdr = buf_hash_find(guid, bp, &hash_lock); 5949 } 5950 5951 /* 5952 * Determine if we have an L1 cache hit or a cache miss. For simplicity 5953 * we maintain encrypted data seperately from compressed / uncompressed 5954 * data. If the user is requesting raw encrypted data and we don't have 5955 * that in the header we will read from disk to guarantee that we can 5956 * get it even if the encryption keys aren't loaded. 5957 */ 5958 if (hdr != NULL && HDR_HAS_L1HDR(hdr) && (HDR_HAS_RABD(hdr) || 5959 (hdr->b_l1hdr.b_pabd != NULL && !encrypted_read))) { 5960 arc_buf_t *buf = NULL; 5961 *arc_flags |= ARC_FLAG_CACHED; 5962 5963 if (HDR_IO_IN_PROGRESS(hdr)) { 5964 zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head; 5965 5966 ASSERT3P(head_zio, !=, NULL); 5967 if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) && 5968 priority == ZIO_PRIORITY_SYNC_READ) { 5969 /* 5970 * This is a sync read that needs to wait for 5971 * an in-flight async read. Request that the 5972 * zio have its priority upgraded. 5973 */ 5974 zio_change_priority(head_zio, priority); 5975 DTRACE_PROBE1(arc__async__upgrade__sync, 5976 arc_buf_hdr_t *, hdr); 5977 ARCSTAT_BUMP(arcstat_async_upgrade_sync); 5978 } 5979 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { 5980 arc_hdr_clear_flags(hdr, 5981 ARC_FLAG_PREDICTIVE_PREFETCH); 5982 } 5983 5984 if (*arc_flags & ARC_FLAG_WAIT) { 5985 cv_wait(&hdr->b_l1hdr.b_cv, hash_lock); 5986 mutex_exit(hash_lock); 5987 goto top; 5988 } 5989 ASSERT(*arc_flags & ARC_FLAG_NOWAIT); 5990 5991 if (done) { 5992 arc_callback_t *acb = NULL; 5993 5994 acb = kmem_zalloc(sizeof (arc_callback_t), 5995 KM_SLEEP); 5996 acb->acb_done = done; 5997 acb->acb_private = private; 5998 acb->acb_compressed = compressed_read; 5999 acb->acb_encrypted = encrypted_read; 6000 acb->acb_noauth = noauth_read; 6001 acb->acb_zb = *zb; 6002 if (pio != NULL) 6003 acb->acb_zio_dummy = zio_null(pio, 6004 spa, NULL, NULL, NULL, zio_flags); 6005 6006 ASSERT3P(acb->acb_done, !=, NULL); 6007 acb->acb_zio_head = head_zio; 6008 acb->acb_next = hdr->b_l1hdr.b_acb; 6009 hdr->b_l1hdr.b_acb = acb; 6010 mutex_exit(hash_lock); 6011 return (0); 6012 } 6013 mutex_exit(hash_lock); 6014 return (0); 6015 } 6016 6017 ASSERT(hdr->b_l1hdr.b_state == arc_mru || 6018 hdr->b_l1hdr.b_state == arc_mfu); 6019 6020 if (done) { 6021 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { 6022 /* 6023 * This is a demand read which does not have to 6024 * wait for i/o because we did a predictive 6025 * prefetch i/o for it, which has completed. 6026 */ 6027 DTRACE_PROBE1( 6028 arc__demand__hit__predictive__prefetch, 6029 arc_buf_hdr_t *, hdr); 6030 ARCSTAT_BUMP( 6031 arcstat_demand_hit_predictive_prefetch); 6032 arc_hdr_clear_flags(hdr, 6033 ARC_FLAG_PREDICTIVE_PREFETCH); 6034 } 6035 6036 if (hdr->b_flags & ARC_FLAG_PRESCIENT_PREFETCH) { 6037 ARCSTAT_BUMP( 6038 arcstat_demand_hit_prescient_prefetch); 6039 arc_hdr_clear_flags(hdr, 6040 ARC_FLAG_PRESCIENT_PREFETCH); 6041 } 6042 6043 ASSERT(!BP_IS_EMBEDDED(bp) || !BP_IS_HOLE(bp)); 6044 6045 arc_hdr_verify_checksum(spa, hdr, bp); 6046 6047 /* Get a buf with the desired data in it. */ 6048 rc = arc_buf_alloc_impl(hdr, spa, zb, private, 6049 encrypted_read, compressed_read, noauth_read, 6050 B_TRUE, &buf); 6051 if (rc == ECKSUM) { 6052 /* 6053 * Convert authentication and decryption errors 6054 * to EIO (and generate an ereport if needed) 6055 * before leaving the ARC. 6056 */ 6057 rc = SET_ERROR(EIO); 6058 if ((zio_flags & ZIO_FLAG_SPECULATIVE) == 0) { 6059 spa_log_error(spa, zb); 6060 zfs_ereport_post( 6061 FM_EREPORT_ZFS_AUTHENTICATION, 6062 spa, NULL, zb, NULL, 0, 0); 6063 } 6064 } 6065 if (rc != 0) { 6066 (void) remove_reference(hdr, hash_lock, 6067 private); 6068 arc_buf_destroy_impl(buf); 6069 buf = NULL; 6070 } 6071 /* assert any errors weren't due to unloaded keys */ 6072 ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) || 6073 rc != EACCES); 6074 } else if (*arc_flags & ARC_FLAG_PREFETCH && 6075 zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { 6076 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); 6077 } 6078 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 6079 arc_access(hdr, hash_lock); 6080 if (*arc_flags & ARC_FLAG_PRESCIENT_PREFETCH) 6081 arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH); 6082 if (*arc_flags & ARC_FLAG_L2CACHE) 6083 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); 6084 mutex_exit(hash_lock); 6085 ARCSTAT_BUMP(arcstat_hits); 6086 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), 6087 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), 6088 data, metadata, hits); 6089 6090 if (done) 6091 done(NULL, zb, bp, buf, private); 6092 } else { 6093 uint64_t lsize = BP_GET_LSIZE(bp); 6094 uint64_t psize = BP_GET_PSIZE(bp); 6095 arc_callback_t *acb; 6096 vdev_t *vd = NULL; 6097 uint64_t addr = 0; 6098 boolean_t devw = B_FALSE; 6099 uint64_t size; 6100 abd_t *hdr_abd; 6101 6102 if (hdr == NULL) { 6103 /* this block is not in the cache */ 6104 arc_buf_hdr_t *exists = NULL; 6105 arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp); 6106 hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, 6107 BP_IS_PROTECTED(bp), BP_GET_COMPRESS(bp), type, 6108 encrypted_read); 6109 6110 if (!BP_IS_EMBEDDED(bp)) { 6111 hdr->b_dva = *BP_IDENTITY(bp); 6112 hdr->b_birth = BP_PHYSICAL_BIRTH(bp); 6113 exists = buf_hash_insert(hdr, &hash_lock); 6114 } 6115 if (exists != NULL) { 6116 /* somebody beat us to the hash insert */ 6117 mutex_exit(hash_lock); 6118 buf_discard_identity(hdr); 6119 arc_hdr_destroy(hdr); 6120 goto top; /* restart the IO request */ 6121 } 6122 } else { 6123 /* 6124 * This block is in the ghost cache or encrypted data 6125 * was requested and we didn't have it. If it was 6126 * L2-only (and thus didn't have an L1 hdr), 6127 * we realloc the header to add an L1 hdr. 6128 */ 6129 if (!HDR_HAS_L1HDR(hdr)) { 6130 hdr = arc_hdr_realloc(hdr, hdr_l2only_cache, 6131 hdr_full_cache); 6132 } 6133 6134 if (GHOST_STATE(hdr->b_l1hdr.b_state)) { 6135 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 6136 ASSERT(!HDR_HAS_RABD(hdr)); 6137 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 6138 ASSERT0(zfs_refcount_count( 6139 &hdr->b_l1hdr.b_refcnt)); 6140 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 6141 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); 6142 } else if (HDR_IO_IN_PROGRESS(hdr)) { 6143 /* 6144 * If this header already had an IO in progress 6145 * and we are performing another IO to fetch 6146 * encrypted data we must wait until the first 6147 * IO completes so as not to confuse 6148 * arc_read_done(). This should be very rare 6149 * and so the performance impact shouldn't 6150 * matter. 6151 */ 6152 cv_wait(&hdr->b_l1hdr.b_cv, hash_lock); 6153 mutex_exit(hash_lock); 6154 goto top; 6155 } 6156 6157 /* 6158 * This is a delicate dance that we play here. 6159 * This hdr might be in the ghost list so we access 6160 * it to move it out of the ghost list before we 6161 * initiate the read. If it's a prefetch then 6162 * it won't have a callback so we'll remove the 6163 * reference that arc_buf_alloc_impl() created. We 6164 * do this after we've called arc_access() to 6165 * avoid hitting an assert in remove_reference(). 6166 */ 6167 arc_access(hdr, hash_lock); 6168 arc_hdr_alloc_pabd(hdr, encrypted_read); 6169 } 6170 6171 if (encrypted_read) { 6172 ASSERT(HDR_HAS_RABD(hdr)); 6173 size = HDR_GET_PSIZE(hdr); 6174 hdr_abd = hdr->b_crypt_hdr.b_rabd; 6175 zio_flags |= ZIO_FLAG_RAW; 6176 } else { 6177 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 6178 size = arc_hdr_size(hdr); 6179 hdr_abd = hdr->b_l1hdr.b_pabd; 6180 6181 if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) { 6182 zio_flags |= ZIO_FLAG_RAW_COMPRESS; 6183 } 6184 6185 /* 6186 * For authenticated bp's, we do not ask the ZIO layer 6187 * to authenticate them since this will cause the entire 6188 * IO to fail if the key isn't loaded. Instead, we 6189 * defer authentication until arc_buf_fill(), which will 6190 * verify the data when the key is available. 6191 */ 6192 if (BP_IS_AUTHENTICATED(bp)) 6193 zio_flags |= ZIO_FLAG_RAW_ENCRYPT; 6194 } 6195 6196 if (*arc_flags & ARC_FLAG_PREFETCH && 6197 zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) 6198 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); 6199 if (*arc_flags & ARC_FLAG_PRESCIENT_PREFETCH) 6200 arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH); 6201 6202 if (*arc_flags & ARC_FLAG_L2CACHE) 6203 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); 6204 if (BP_IS_AUTHENTICATED(bp)) 6205 arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH); 6206 if (BP_GET_LEVEL(bp) > 0) 6207 arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT); 6208 if (*arc_flags & ARC_FLAG_PREDICTIVE_PREFETCH) 6209 arc_hdr_set_flags(hdr, ARC_FLAG_PREDICTIVE_PREFETCH); 6210 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state)); 6211 6212 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 6213 acb->acb_done = done; 6214 acb->acb_private = private; 6215 acb->acb_compressed = compressed_read; 6216 acb->acb_encrypted = encrypted_read; 6217 acb->acb_noauth = noauth_read; 6218 acb->acb_zb = *zb; 6219 6220 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); 6221 hdr->b_l1hdr.b_acb = acb; 6222 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); 6223 6224 if (HDR_HAS_L2HDR(hdr) && 6225 (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) { 6226 devw = hdr->b_l2hdr.b_dev->l2ad_writing; 6227 addr = hdr->b_l2hdr.b_daddr; 6228 /* 6229 * Lock out L2ARC device removal. 6230 */ 6231 if (vdev_is_dead(vd) || 6232 !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER)) 6233 vd = NULL; 6234 } 6235 6236 /* 6237 * We count both async reads and scrub IOs as asynchronous so 6238 * that both can be upgraded in the event of a cache hit while 6239 * the read IO is still in-flight. 6240 */ 6241 if (priority == ZIO_PRIORITY_ASYNC_READ || 6242 priority == ZIO_PRIORITY_SCRUB) 6243 arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); 6244 else 6245 arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); 6246 6247 /* 6248 * At this point, we have a level 1 cache miss. Try again in 6249 * L2ARC if possible. 6250 */ 6251 ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize); 6252 6253 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp, 6254 uint64_t, lsize, zbookmark_phys_t *, zb); 6255 ARCSTAT_BUMP(arcstat_misses); 6256 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), 6257 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), 6258 data, metadata, misses); 6259 6260 if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) { 6261 /* 6262 * Read from the L2ARC if the following are true: 6263 * 1. The L2ARC vdev was previously cached. 6264 * 2. This buffer still has L2ARC metadata. 6265 * 3. This buffer isn't currently writing to the L2ARC. 6266 * 4. The L2ARC entry wasn't evicted, which may 6267 * also have invalidated the vdev. 6268 * 5. This isn't prefetch and l2arc_noprefetch is set. 6269 */ 6270 if (HDR_HAS_L2HDR(hdr) && 6271 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) && 6272 !(l2arc_noprefetch && HDR_PREFETCH(hdr))) { 6273 l2arc_read_callback_t *cb; 6274 abd_t *abd; 6275 uint64_t asize; 6276 6277 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr); 6278 ARCSTAT_BUMP(arcstat_l2_hits); 6279 6280 cb = kmem_zalloc(sizeof (l2arc_read_callback_t), 6281 KM_SLEEP); 6282 cb->l2rcb_hdr = hdr; 6283 cb->l2rcb_bp = *bp; 6284 cb->l2rcb_zb = *zb; 6285 cb->l2rcb_flags = zio_flags; 6286 6287 asize = vdev_psize_to_asize(vd, size); 6288 if (asize != size) { 6289 abd = abd_alloc_for_io(asize, 6290 HDR_ISTYPE_METADATA(hdr)); 6291 cb->l2rcb_abd = abd; 6292 } else { 6293 abd = hdr_abd; 6294 } 6295 6296 ASSERT(addr >= VDEV_LABEL_START_SIZE && 6297 addr + asize <= vd->vdev_psize - 6298 VDEV_LABEL_END_SIZE); 6299 6300 /* 6301 * l2arc read. The SCL_L2ARC lock will be 6302 * released by l2arc_read_done(). 6303 * Issue a null zio if the underlying buffer 6304 * was squashed to zero size by compression. 6305 */ 6306 ASSERT3U(arc_hdr_get_compress(hdr), !=, 6307 ZIO_COMPRESS_EMPTY); 6308 rzio = zio_read_phys(pio, vd, addr, 6309 asize, abd, 6310 ZIO_CHECKSUM_OFF, 6311 l2arc_read_done, cb, priority, 6312 zio_flags | ZIO_FLAG_DONT_CACHE | 6313 ZIO_FLAG_CANFAIL | 6314 ZIO_FLAG_DONT_PROPAGATE | 6315 ZIO_FLAG_DONT_RETRY, B_FALSE); 6316 acb->acb_zio_head = rzio; 6317 6318 if (hash_lock != NULL) 6319 mutex_exit(hash_lock); 6320 6321 DTRACE_PROBE2(l2arc__read, vdev_t *, vd, 6322 zio_t *, rzio); 6323 ARCSTAT_INCR(arcstat_l2_read_bytes, 6324 HDR_GET_PSIZE(hdr)); 6325 6326 if (*arc_flags & ARC_FLAG_NOWAIT) { 6327 zio_nowait(rzio); 6328 return (0); 6329 } 6330 6331 ASSERT(*arc_flags & ARC_FLAG_WAIT); 6332 if (zio_wait(rzio) == 0) 6333 return (0); 6334 6335 /* l2arc read error; goto zio_read() */ 6336 if (hash_lock != NULL) 6337 mutex_enter(hash_lock); 6338 } else { 6339 DTRACE_PROBE1(l2arc__miss, 6340 arc_buf_hdr_t *, hdr); 6341 ARCSTAT_BUMP(arcstat_l2_misses); 6342 if (HDR_L2_WRITING(hdr)) 6343 ARCSTAT_BUMP(arcstat_l2_rw_clash); 6344 spa_config_exit(spa, SCL_L2ARC, vd); 6345 } 6346 } else { 6347 if (vd != NULL) 6348 spa_config_exit(spa, SCL_L2ARC, vd); 6349 if (l2arc_ndev != 0) { 6350 DTRACE_PROBE1(l2arc__miss, 6351 arc_buf_hdr_t *, hdr); 6352 ARCSTAT_BUMP(arcstat_l2_misses); 6353 } 6354 } 6355 6356 rzio = zio_read(pio, spa, bp, hdr_abd, size, 6357 arc_read_done, hdr, priority, zio_flags, zb); 6358 acb->acb_zio_head = rzio; 6359 6360 if (hash_lock != NULL) 6361 mutex_exit(hash_lock); 6362 6363 if (*arc_flags & ARC_FLAG_WAIT) 6364 return (zio_wait(rzio)); 6365 6366 ASSERT(*arc_flags & ARC_FLAG_NOWAIT); 6367 zio_nowait(rzio); 6368 } 6369 return (rc); 6370} 6371 6372/* 6373 * Notify the arc that a block was freed, and thus will never be used again. 6374 */ 6375void 6376arc_freed(spa_t *spa, const blkptr_t *bp) 6377{ 6378 arc_buf_hdr_t *hdr; 6379 kmutex_t *hash_lock; 6380 uint64_t guid = spa_load_guid(spa); 6381 6382 ASSERT(!BP_IS_EMBEDDED(bp)); 6383 6384 hdr = buf_hash_find(guid, bp, &hash_lock); 6385 if (hdr == NULL) 6386 return; 6387 6388 /* 6389 * We might be trying to free a block that is still doing I/O 6390 * (i.e. prefetch) or has a reference (i.e. a dedup-ed, 6391 * dmu_sync-ed block). If this block is being prefetched, then it 6392 * would still have the ARC_FLAG_IO_IN_PROGRESS flag set on the hdr 6393 * until the I/O completes. A block may also have a reference if it is 6394 * part of a dedup-ed, dmu_synced write. The dmu_sync() function would 6395 * have written the new block to its final resting place on disk but 6396 * without the dedup flag set. This would have left the hdr in the MRU 6397 * state and discoverable. When the txg finally syncs it detects that 6398 * the block was overridden in open context and issues an override I/O. 6399 * Since this is a dedup block, the override I/O will determine if the 6400 * block is already in the DDT. If so, then it will replace the io_bp 6401 * with the bp from the DDT and allow the I/O to finish. When the I/O 6402 * reaches the done callback, dbuf_write_override_done, it will 6403 * check to see if the io_bp and io_bp_override are identical. 6404 * If they are not, then it indicates that the bp was replaced with 6405 * the bp in the DDT and the override bp is freed. This allows 6406 * us to arrive here with a reference on a block that is being 6407 * freed. So if we have an I/O in progress, or a reference to 6408 * this hdr, then we don't destroy the hdr. 6409 */ 6410 if (!HDR_HAS_L1HDR(hdr) || (!HDR_IO_IN_PROGRESS(hdr) && 6411 zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt))) { 6412 arc_change_state(arc_anon, hdr, hash_lock); 6413 arc_hdr_destroy(hdr); 6414 mutex_exit(hash_lock); 6415 } else { 6416 mutex_exit(hash_lock); 6417 } 6418 6419} 6420 6421/* 6422 * Release this buffer from the cache, making it an anonymous buffer. This 6423 * must be done after a read and prior to modifying the buffer contents. 6424 * If the buffer has more than one reference, we must make 6425 * a new hdr for the buffer. 6426 */ 6427void 6428arc_release(arc_buf_t *buf, void *tag) 6429{ 6430 arc_buf_hdr_t *hdr = buf->b_hdr; 6431 6432 /* 6433 * It would be nice to assert that if its DMU metadata (level > 6434 * 0 || it's the dnode file), then it must be syncing context. 6435 * But we don't know that information at this level. 6436 */ 6437 6438 mutex_enter(&buf->b_evict_lock); 6439 6440 ASSERT(HDR_HAS_L1HDR(hdr)); 6441 6442 /* 6443 * We don't grab the hash lock prior to this check, because if 6444 * the buffer's header is in the arc_anon state, it won't be 6445 * linked into the hash table. 6446 */ 6447 if (hdr->b_l1hdr.b_state == arc_anon) { 6448 mutex_exit(&buf->b_evict_lock); 6449 /* 6450 * If we are called from dmu_convert_mdn_block_to_raw(), 6451 * a write might be in progress. This is OK because 6452 * the caller won't change the content of this buffer, 6453 * only the flags (via arc_convert_to_raw()). 6454 */ 6455 /* ASSERT(!HDR_IO_IN_PROGRESS(hdr)); */ 6456 ASSERT(!HDR_IN_HASH_TABLE(hdr)); 6457 ASSERT(!HDR_HAS_L2HDR(hdr)); 6458 ASSERT(HDR_EMPTY(hdr)); 6459 6460 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); 6461 ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1); 6462 ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node)); 6463 6464 hdr->b_l1hdr.b_arc_access = 0; 6465 6466 /* 6467 * If the buf is being overridden then it may already 6468 * have a hdr that is not empty. 6469 */ 6470 buf_discard_identity(hdr); 6471 arc_buf_thaw(buf); 6472 6473 return; 6474 } 6475 6476 kmutex_t *