Lines Matching refs:hdr
616 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE) argument
617 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) argument
618 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR) argument
619 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH) argument
620 #define HDR_PRESCIENT_PREFETCH(hdr) \ argument
621 ((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH)
622 #define HDR_COMPRESSION_ENABLED(hdr) \ argument
623 ((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
625 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE) argument
626 #define HDR_L2_READING(hdr) \ argument
627 (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \
628 ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
629 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING) argument
630 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED) argument
631 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD) argument
632 #define HDR_PROTECTED(hdr) ((hdr)->b_flags & ARC_FLAG_PROTECTED) argument
633 #define HDR_NOAUTH(hdr) ((hdr)->b_flags & ARC_FLAG_NOAUTH) argument
634 #define HDR_SHARED_DATA(hdr) ((hdr)->b_flags & ARC_FLAG_SHARED_DATA) argument
636 #define HDR_ISTYPE_METADATA(hdr) \ argument
637 ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
638 #define HDR_ISTYPE_DATA(hdr) (!HDR_ISTYPE_METADATA(hdr)) argument
640 #define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR) argument
641 #define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR) argument
642 #define HDR_HAS_RABD(hdr) \ argument
643 (HDR_HAS_L1HDR(hdr) && HDR_PROTECTED(hdr) && \
644 (hdr)->b_crypt_hdr.b_rabd != NULL)
645 #define HDR_ENCRYPTED(hdr) \ argument
646 (HDR_PROTECTED(hdr) && DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
647 #define HDR_AUTHENTICATED(hdr) \ argument
648 (HDR_PROTECTED(hdr) && !DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
653 #define HDR_GET_COMPRESS(hdr) ((enum zio_compress)BF32_GET((hdr)->b_flags, \ argument
655 #define HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \ argument
697 #define HDR_LOCK(hdr) \ argument
698 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
792 static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag);
802 static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
803 static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
808 static void l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
811 #define l2arc_hdr_arcstats_increment(hdr) \ argument
812 l2arc_hdr_arcstats_update((hdr), B_TRUE, B_FALSE)
813 #define l2arc_hdr_arcstats_decrement(hdr) \ argument
814 l2arc_hdr_arcstats_update((hdr), B_FALSE, B_FALSE)
815 #define l2arc_hdr_arcstats_increment_state(hdr) \ argument
816 l2arc_hdr_arcstats_update((hdr), B_TRUE, B_TRUE)
817 #define l2arc_hdr_arcstats_decrement_state(hdr) \ argument
818 l2arc_hdr_arcstats_update((hdr), B_FALSE, B_TRUE)
845 #define HDR_EMPTY(hdr) \ argument
846 ((hdr)->b_dva.dva_word[0] == 0 && \
847 (hdr)->b_dva.dva_word[1] == 0)
849 #define HDR_EMPTY_OR_LOCKED(hdr) \ argument
850 (HDR_EMPTY(hdr) || MUTEX_HELD(HDR_LOCK(hdr)))
852 #define HDR_EQUAL(spa, dva, birth, hdr) \ argument
853 ((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \
854 ((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \
855 ((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
858 buf_discard_identity(arc_buf_hdr_t *hdr) in buf_discard_identity() argument
860 hdr->b_dva.dva_word[0] = 0; in buf_discard_identity()
861 hdr->b_dva.dva_word[1] = 0; in buf_discard_identity()
862 hdr->b_birth = 0; in buf_discard_identity()
872 arc_buf_hdr_t *hdr; in buf_hash_find() local
875 for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL; in buf_hash_find()
876 hdr = hdr->b_hash_next) { in buf_hash_find()
877 if (HDR_EQUAL(spa, dva, birth, hdr)) { in buf_hash_find()
879 return (hdr); in buf_hash_find()
895 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp) in buf_hash_insert() argument
897 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth); in buf_hash_insert()
902 ASSERT(!DVA_IS_EMPTY(&hdr->b_dva)); in buf_hash_insert()
903 ASSERT(hdr->b_birth != 0); in buf_hash_insert()
904 ASSERT(!HDR_IN_HASH_TABLE(hdr)); in buf_hash_insert()
915 if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr)) in buf_hash_insert()
919 hdr->b_hash_next = buf_hash_table.ht_table[idx]; in buf_hash_insert()
920 buf_hash_table.ht_table[idx] = hdr; in buf_hash_insert()
921 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE); in buf_hash_insert()
939 buf_hash_remove(arc_buf_hdr_t *hdr) in buf_hash_remove() argument
942 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth); in buf_hash_remove()
945 ASSERT(HDR_IN_HASH_TABLE(hdr)); in buf_hash_remove()
948 while ((fhdr = *hdrp) != hdr) { in buf_hash_remove()
952 *hdrp = hdr->b_hash_next; in buf_hash_remove()
953 hdr->b_hash_next = NULL; in buf_hash_remove()
954 arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE); in buf_hash_remove()
1002 arc_buf_hdr_t *hdr = vbuf; in hdr_full_cons() local
1004 bzero(hdr, HDR_FULL_SIZE); in hdr_full_cons()
1005 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; in hdr_full_cons()
1006 cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL); in hdr_full_cons()
1007 zfs_refcount_create(&hdr->b_l1hdr.b_refcnt); in hdr_full_cons()
1008 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL); in hdr_full_cons()
1009 multilist_link_init(&hdr->b_l1hdr.b_arc_node); in hdr_full_cons()
1019 arc_buf_hdr_t *hdr = vbuf; in hdr_full_crypt_cons() local
1022 bzero(&hdr->b_crypt_hdr, sizeof (hdr->b_crypt_hdr)); in hdr_full_crypt_cons()
1023 arc_space_consume(sizeof (hdr->b_crypt_hdr), ARC_SPACE_HDRS); in hdr_full_crypt_cons()
1032 arc_buf_hdr_t *hdr = vbuf; in hdr_l2only_cons() local
1034 bzero(hdr, HDR_L2ONLY_SIZE); in hdr_l2only_cons()
1061 arc_buf_hdr_t *hdr = vbuf; in hdr_full_dest() local
1063 ASSERT(HDR_EMPTY(hdr)); in hdr_full_dest()
1064 cv_destroy(&hdr->b_l1hdr.b_cv); in hdr_full_dest()
1065 zfs_refcount_destroy(&hdr->b_l1hdr.b_refcnt); in hdr_full_dest()
1066 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock); in hdr_full_dest()
1067 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in hdr_full_dest()
1075 arc_buf_hdr_t *hdr = vbuf; in hdr_full_crypt_dest() local
1077 hdr_full_dest(hdr, unused); in hdr_full_crypt_dest()
1078 arc_space_return(sizeof (hdr->b_crypt_hdr), ARC_SPACE_HDRS); in hdr_full_crypt_dest()
1085 arc_buf_hdr_t *hdr = vbuf; in hdr_l2only_dest() local
1087 ASSERT(HDR_EMPTY(hdr)); in hdr_l2only_dest()
1205 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_get_raw_params() local
1207 ASSERT(HDR_PROTECTED(hdr)); in arc_get_raw_params()
1209 bcopy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN); in arc_get_raw_params()
1210 bcopy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN); in arc_get_raw_params()
1211 bcopy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN); in arc_get_raw_params()
1212 *byteorder = (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ? in arc_get_raw_params()
1237 arc_hdr_get_compress(arc_buf_hdr_t *hdr) in arc_hdr_get_compress() argument
1239 return (HDR_COMPRESSION_ENABLED(hdr) ? in arc_hdr_get_compress()
1240 HDR_GET_COMPRESS(hdr) : ZIO_COMPRESS_OFF); in arc_hdr_get_compress()
1267 arc_cksum_free(arc_buf_hdr_t *hdr) in arc_cksum_free() argument
1269 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_cksum_free()
1271 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_free()
1272 if (hdr->b_l1hdr.b_freeze_cksum != NULL) { in arc_cksum_free()
1273 kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t)); in arc_cksum_free()
1274 hdr->b_l1hdr.b_freeze_cksum = NULL; in arc_cksum_free()
1276 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_free()
1284 arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr) in arc_hdr_has_uncompressed_buf() argument
1286 ASSERT(hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY_OR_LOCKED(hdr)); in arc_hdr_has_uncompressed_buf()
1288 for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) { in arc_hdr_has_uncompressed_buf()
1304 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_cksum_verify() local
1313 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_cksum_verify()
1315 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_verify()
1317 if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) { in arc_cksum_verify()
1318 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_verify()
1323 if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc)) in arc_cksum_verify()
1325 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_verify()
1334 arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio) in arc_cksum_is_equal() argument
1337 VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr)); in arc_cksum_is_equal()
1366 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_cksum_compute() local
1371 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_cksum_compute()
1374 if (hdr->b_l1hdr.b_freeze_cksum != NULL || ARC_BUF_COMPRESSED(buf)) { in arc_cksum_compute()
1375 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_compute()
1381 hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), in arc_cksum_compute()
1384 hdr->b_l1hdr.b_freeze_cksum); in arc_cksum_compute()
1385 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_compute()
1433 arc_buf_type(arc_buf_hdr_t *hdr) in arc_buf_type() argument
1436 if (HDR_ISTYPE_METADATA(hdr)) { in arc_buf_type()
1441 VERIFY3U(hdr->b_type, ==, type); in arc_buf_type()
1470 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_thaw() local
1472 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); in arc_buf_thaw()
1473 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_buf_thaw()
1483 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_buf_thaw()
1484 arc_cksum_free(hdr); in arc_buf_thaw()
1486 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); in arc_buf_thaw()
1489 if (hdr->b_l1hdr.b_thawed != NULL) in arc_buf_thaw()
1490 kmem_free(hdr->b_l1hdr.b_thawed, 1); in arc_buf_thaw()
1491 hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP); in arc_buf_thaw()
1495 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_buf_thaw()
1522 arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags) in arc_hdr_set_flags() argument
1524 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_hdr_set_flags()
1525 hdr->b_flags |= flags; in arc_hdr_set_flags()
1529 arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags) in arc_hdr_clear_flags() argument
1531 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_hdr_clear_flags()
1532 hdr->b_flags &= ~flags; in arc_hdr_clear_flags()
1543 arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp) in arc_hdr_set_compress() argument
1545 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_hdr_set_compress()
1554 if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) { in arc_hdr_set_compress()
1555 arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC); in arc_hdr_set_compress()
1556 ASSERT(!HDR_COMPRESSION_ENABLED(hdr)); in arc_hdr_set_compress()
1558 arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC); in arc_hdr_set_compress()
1559 ASSERT(HDR_COMPRESSION_ENABLED(hdr)); in arc_hdr_set_compress()
1562 HDR_SET_COMPRESS(hdr, cmp); in arc_hdr_set_compress()
1563 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp); in arc_hdr_set_compress()
1573 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_try_copy_decompressed_data() local
1576 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_buf_try_copy_decompressed_data()
1580 for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL; in arc_buf_try_copy_decompressed_data()
1614 arc_hdr_size(arc_buf_hdr_t *hdr) in arc_hdr_size() argument
1618 if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF && in arc_hdr_size()
1619 HDR_GET_PSIZE(hdr) > 0) { in arc_hdr_size()
1620 size = HDR_GET_PSIZE(hdr); in arc_hdr_size()
1622 ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0); in arc_hdr_size()
1623 size = HDR_GET_LSIZE(hdr); in arc_hdr_size()
1629 arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj) in arc_hdr_authenticate() argument
1633 uint64_t lsize = HDR_GET_LSIZE(hdr); in arc_hdr_authenticate()
1634 uint64_t psize = HDR_GET_PSIZE(hdr); in arc_hdr_authenticate()
1636 abd_t *abd = hdr->b_l1hdr.b_pabd; in arc_hdr_authenticate()
1638 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_hdr_authenticate()
1639 ASSERT(HDR_AUTHENTICATED(hdr)); in arc_hdr_authenticate()
1640 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in arc_hdr_authenticate()
1650 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && in arc_hdr_authenticate()
1651 !HDR_COMPRESSION_ENABLED(hdr)) { in arc_hdr_authenticate()
1656 csize = zio_compress_data(HDR_GET_COMPRESS(hdr), in arc_hdr_authenticate()
1657 hdr->b_l1hdr.b_pabd, tmpbuf, lsize); in arc_hdr_authenticate()
1666 if (hdr->b_crypt_hdr.b_ot == DMU_OT_OBJSET) { in arc_hdr_authenticate()
1667 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); in arc_hdr_authenticate()
1670 psize, hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS); in arc_hdr_authenticate()
1673 hdr->b_crypt_hdr.b_mac); in arc_hdr_authenticate()
1677 arc_hdr_clear_flags(hdr, ARC_FLAG_NOAUTH); in arc_hdr_authenticate()
1700 arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb) in arc_hdr_decrypt() argument
1706 boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS); in arc_hdr_decrypt()
1708 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_hdr_decrypt()
1709 ASSERT(HDR_ENCRYPTED(hdr)); in arc_hdr_decrypt()
1711 arc_hdr_alloc_pabd(hdr, ARC_HDR_DO_ADAPT); in arc_hdr_decrypt()
1713 ret = spa_do_crypt_abd(B_FALSE, spa, zb, hdr->b_crypt_hdr.b_ot, in arc_hdr_decrypt()
1714 B_FALSE, bswap, hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_iv, in arc_hdr_decrypt()
1715 hdr->b_crypt_hdr.b_mac, HDR_GET_PSIZE(hdr), hdr->b_l1hdr.b_pabd, in arc_hdr_decrypt()
1716 hdr->b_crypt_hdr.b_rabd, &no_crypt); in arc_hdr_decrypt()
1721 abd_copy(hdr->b_l1hdr.b_pabd, hdr->b_crypt_hdr.b_rabd, in arc_hdr_decrypt()
1722 HDR_GET_PSIZE(hdr)); in arc_hdr_decrypt()
1730 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && in arc_hdr_decrypt()
1731 !HDR_COMPRESSION_ENABLED(hdr)) { in arc_hdr_decrypt()
1738 cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, B_TRUE); in arc_hdr_decrypt()
1739 tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr)); in arc_hdr_decrypt()
1741 ret = zio_decompress_data(HDR_GET_COMPRESS(hdr), in arc_hdr_decrypt()
1742 hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr), in arc_hdr_decrypt()
1743 HDR_GET_LSIZE(hdr)); in arc_hdr_decrypt()
1745 abd_return_buf(cabd, tmp, arc_hdr_size(hdr)); in arc_hdr_decrypt()
1749 abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr)); in arc_hdr_decrypt()
1750 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, in arc_hdr_decrypt()
1751 arc_hdr_size(hdr), hdr); in arc_hdr_decrypt()
1752 hdr->b_l1hdr.b_pabd = cabd; in arc_hdr_decrypt()
1758 arc_hdr_free_pabd(hdr, B_FALSE); in arc_hdr_decrypt()
1760 arc_free_data_buf(hdr, cabd, arc_hdr_size(hdr), hdr); in arc_hdr_decrypt()
1771 arc_fill_hdr_crypt(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, spa_t *spa, in arc_fill_hdr_crypt() argument
1776 ASSERT(HDR_PROTECTED(hdr)); in arc_fill_hdr_crypt()
1781 if (HDR_NOAUTH(hdr) && !noauth) { in arc_fill_hdr_crypt()
1786 ret = arc_hdr_authenticate(hdr, spa, zb->zb_objset); in arc_fill_hdr_crypt()
1789 } else if (HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd == NULL) { in arc_fill_hdr_crypt()
1795 ret = arc_hdr_decrypt(hdr, spa, zb); in arc_fill_hdr_crypt()
1800 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in arc_fill_hdr_crypt()
1824 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_untransform_in_place() local
1826 ASSERT(HDR_ENCRYPTED(hdr)); in arc_buf_untransform_in_place()
1827 ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE); in arc_buf_untransform_in_place()
1828 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_buf_untransform_in_place()
1829 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in arc_buf_untransform_in_place()
1831 zio_crypt_copy_dnode_bonus(hdr->b_l1hdr.b_pabd, buf->b_data, in arc_buf_untransform_in_place()
1835 hdr->b_crypt_hdr.b_ebufcnt -= 1; in arc_buf_untransform_in_place()
1856 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_fill() local
1858 (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF); in arc_buf_fill()
1861 dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap; in arc_buf_fill()
1862 kmutex_t *hash_lock = (flags & ARC_FILL_LOCKED) ? NULL : HDR_LOCK(hdr); in arc_buf_fill()
1867 IMPLY(encrypted, HDR_ENCRYPTED(hdr)); in arc_buf_fill()
1878 ASSERT(HDR_HAS_RABD(hdr)); in arc_buf_fill()
1879 abd_copy_to_buf(buf->b_data, hdr->b_crypt_hdr.b_rabd, in arc_buf_fill()
1880 HDR_GET_PSIZE(hdr)); in arc_buf_fill()
1890 if (HDR_PROTECTED(hdr)) { in arc_buf_fill()
1891 error = arc_fill_hdr_crypt(hdr, hash_lock, spa, in arc_buf_fill()
1898 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); in arc_buf_fill()
1921 if (HDR_ENCRYPTED(hdr) && ARC_BUF_ENCRYPTED(buf)) { in arc_buf_fill()
1922 ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE); in arc_buf_fill()
1939 abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd, in arc_buf_fill()
1945 ASSERT3U(HDR_GET_LSIZE(hdr), !=, HDR_GET_PSIZE(hdr)); in arc_buf_fill()
1957 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); in arc_buf_fill()
1958 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_buf_fill()
1961 ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr)); in arc_buf_fill()
1964 arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr), in arc_buf_fill()
1967 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); in arc_buf_fill()
1971 HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr)); in arc_buf_fill()
1987 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, !=, NULL); in arc_buf_fill()
1990 error = zio_decompress_data(HDR_GET_COMPRESS(hdr), in arc_buf_fill()
1991 hdr->b_l1hdr.b_pabd, buf->b_data, in arc_buf_fill()
1992 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr)); in arc_buf_fill()
2001 hdr, arc_hdr_get_compress(hdr), in arc_buf_fill()
2002 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr)); in arc_buf_fill()
2005 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); in arc_buf_fill()
2016 ASSERT(!HDR_SHARED_DATA(hdr)); in arc_buf_fill()
2018 dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr)); in arc_buf_fill()
2064 arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state) in arc_evictable_space_increment() argument
2066 arc_buf_contents_t type = arc_buf_type(hdr); in arc_evictable_space_increment()
2068 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_evictable_space_increment()
2071 ASSERT0(hdr->b_l1hdr.b_bufcnt); in arc_evictable_space_increment()
2072 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_evictable_space_increment()
2073 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_evictable_space_increment()
2074 ASSERT(!HDR_HAS_RABD(hdr)); in arc_evictable_space_increment()
2076 HDR_GET_LSIZE(hdr), hdr); in arc_evictable_space_increment()
2081 if (hdr->b_l1hdr.b_pabd != NULL) { in arc_evictable_space_increment()
2083 arc_hdr_size(hdr), hdr); in arc_evictable_space_increment()
2085 if (HDR_HAS_RABD(hdr)) { in arc_evictable_space_increment()
2087 HDR_GET_PSIZE(hdr), hdr); in arc_evictable_space_increment()
2089 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; in arc_evictable_space_increment()
2104 arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state) in arc_evictable_space_decrement() argument
2106 arc_buf_contents_t type = arc_buf_type(hdr); in arc_evictable_space_decrement()
2108 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_evictable_space_decrement()
2111 ASSERT0(hdr->b_l1hdr.b_bufcnt); in arc_evictable_space_decrement()
2112 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_evictable_space_decrement()
2113 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_evictable_space_decrement()
2114 ASSERT(!HDR_HAS_RABD(hdr)); in arc_evictable_space_decrement()
2116 HDR_GET_LSIZE(hdr), hdr); in arc_evictable_space_decrement()
2121 if (hdr->b_l1hdr.b_pabd != NULL) { in arc_evictable_space_decrement()
2123 arc_hdr_size(hdr), hdr); in arc_evictable_space_decrement()
2125 if (HDR_HAS_RABD(hdr)) { in arc_evictable_space_decrement()
2127 HDR_GET_PSIZE(hdr), hdr); in arc_evictable_space_decrement()
2129 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; in arc_evictable_space_decrement()
2145 add_reference(arc_buf_hdr_t *hdr, void *tag) in add_reference() argument
2147 ASSERT(HDR_HAS_L1HDR(hdr)); in add_reference()
2148 if (!HDR_EMPTY(hdr) && !MUTEX_HELD(HDR_LOCK(hdr))) { in add_reference()
2149 ASSERT(hdr->b_l1hdr.b_state == arc_anon); in add_reference()
2150 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in add_reference()
2151 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in add_reference()
2154 arc_state_t *state = hdr->b_l1hdr.b_state; in add_reference()
2156 if ((zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) && in add_reference()
2160 multilist_remove(state->arcs_list[arc_buf_type(hdr)], in add_reference()
2161 hdr); in add_reference()
2162 arc_evictable_space_decrement(hdr, state); in add_reference()
2165 if (HDR_HAS_L2HDR(hdr)) in add_reference()
2166 l2arc_hdr_arcstats_decrement_state(hdr); in add_reference()
2167 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); in add_reference()
2168 if (HDR_HAS_L2HDR(hdr)) in add_reference()
2169 l2arc_hdr_arcstats_increment_state(hdr); in add_reference()
2179 remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag) in remove_reference() argument
2182 arc_state_t *state = hdr->b_l1hdr.b_state; in remove_reference()
2184 ASSERT(HDR_HAS_L1HDR(hdr)); in remove_reference()
2192 if (((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) && in remove_reference()
2194 multilist_insert(state->arcs_list[arc_buf_type(hdr)], hdr); in remove_reference()
2195 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0); in remove_reference()
2196 arc_evictable_space_increment(hdr, state); in remove_reference()
2206 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr, in arc_change_state() argument
2213 arc_buf_contents_t buftype = arc_buf_type(hdr); in arc_change_state()
2222 if (HDR_HAS_L1HDR(hdr)) { in arc_change_state()
2223 old_state = hdr->b_l1hdr.b_state; in arc_change_state()
2224 refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt); in arc_change_state()
2225 bufcnt = hdr->b_l1hdr.b_bufcnt; in arc_change_state()
2227 update_old = (bufcnt > 0 || hdr->b_l1hdr.b_pabd != NULL || in arc_change_state()
2228 HDR_HAS_RABD(hdr)); in arc_change_state()
2248 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
2249 multilist_remove(old_state->arcs_list[buftype], hdr); in arc_change_state()
2253 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_change_state()
2256 arc_evictable_space_decrement(hdr, old_state); in arc_change_state()
2266 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
2267 multilist_insert(new_state->arcs_list[buftype], hdr); in arc_change_state()
2271 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_change_state()
2274 arc_evictable_space_increment(hdr, new_state); in arc_change_state()
2278 ASSERT(!HDR_EMPTY(hdr)); in arc_change_state()
2279 if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr)) in arc_change_state()
2280 buf_hash_remove(hdr); in arc_change_state()
2285 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
2297 HDR_GET_LSIZE(hdr), hdr); in arc_change_state()
2298 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_change_state()
2299 ASSERT(!HDR_HAS_RABD(hdr)); in arc_change_state()
2308 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; in arc_change_state()
2329 if (hdr->b_l1hdr.b_pabd != NULL) { in arc_change_state()
2332 arc_hdr_size(hdr), hdr); in arc_change_state()
2335 if (HDR_HAS_RABD(hdr)) { in arc_change_state()
2338 HDR_GET_PSIZE(hdr), hdr); in arc_change_state()
2344 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
2347 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_change_state()
2348 ASSERT(!HDR_HAS_RABD(hdr)); in arc_change_state()
2359 HDR_GET_LSIZE(hdr), hdr); in arc_change_state()
2368 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; in arc_change_state()
2388 ASSERT(hdr->b_l1hdr.b_pabd != NULL || in arc_change_state()
2389 HDR_HAS_RABD(hdr)); in arc_change_state()
2391 if (hdr->b_l1hdr.b_pabd != NULL) { in arc_change_state()
2393 &old_state->arcs_size, arc_hdr_size(hdr), in arc_change_state()
2394 hdr); in arc_change_state()
2397 if (HDR_HAS_RABD(hdr)) { in arc_change_state()
2399 &old_state->arcs_size, HDR_GET_PSIZE(hdr), in arc_change_state()
2400 hdr); in arc_change_state()
2405 if (HDR_HAS_L1HDR(hdr)) { in arc_change_state()
2406 hdr->b_l1hdr.b_state = new_state; in arc_change_state()
2408 if (HDR_HAS_L2HDR(hdr) && new_state != arc_l2c_only) { in arc_change_state()
2409 l2arc_hdr_arcstats_decrement_state(hdr); in arc_change_state()
2410 hdr->b_l2hdr.b_arcs_state = new_state->arcs_state; in arc_change_state()
2411 l2arc_hdr_arcstats_increment_state(hdr); in arc_change_state()
2496 arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf) in arc_can_share() argument
2520 ASSERT3P(buf->b_hdr, ==, hdr); in arc_can_share()
2521 boolean_t hdr_compressed = arc_hdr_get_compress(hdr) != in arc_can_share()
2526 hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS && in arc_can_share()
2527 !HDR_SHARED_DATA(hdr) && in arc_can_share()
2537 arc_buf_alloc_impl(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb, in arc_buf_alloc_impl() argument
2544 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_buf_alloc_impl()
2545 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0); in arc_buf_alloc_impl()
2546 VERIFY(hdr->b_type == ARC_BUFC_DATA || in arc_buf_alloc_impl()
2547 hdr->b_type == ARC_BUFC_METADATA); in arc_buf_alloc_impl()
2553 buf->b_hdr = hdr; in arc_buf_alloc_impl()
2555 buf->b_next = hdr->b_l1hdr.b_buf; in arc_buf_alloc_impl()
2558 add_reference(hdr, tag); in arc_buf_alloc_impl()
2564 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_buf_alloc_impl()
2576 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) { in arc_buf_alloc_impl()
2599 boolean_t can_share = arc_can_share(hdr, buf) && !HDR_L2_WRITING(hdr) && in arc_buf_alloc_impl()
2600 hdr->b_l1hdr.b_pabd != NULL && abd_is_linear(hdr->b_l1hdr.b_pabd); in arc_buf_alloc_impl()
2604 buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd); in arc_buf_alloc_impl()
2606 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_buf_alloc_impl()
2609 arc_get_data_buf(hdr, arc_buf_size(buf), buf); in arc_buf_alloc_impl()
2614 hdr->b_l1hdr.b_buf = buf; in arc_buf_alloc_impl()
2615 hdr->b_l1hdr.b_bufcnt += 1; in arc_buf_alloc_impl()
2617 hdr->b_crypt_hdr.b_ebufcnt += 1; in arc_buf_alloc_impl()
2743 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_return_buf() local
2746 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_return_buf()
2747 (void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag); in arc_return_buf()
2748 (void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); in arc_return_buf()
2757 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_loan_inuse_buf() local
2760 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_loan_inuse_buf()
2761 (void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); in arc_loan_inuse_buf()
2762 (void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag); in arc_loan_inuse_buf()
2781 arc_hdr_free_on_write(arc_buf_hdr_t *hdr, boolean_t free_rdata) in arc_hdr_free_on_write() argument
2783 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_hdr_free_on_write()
2784 arc_buf_contents_t type = arc_buf_type(hdr); in arc_hdr_free_on_write()
2785 uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr); in arc_hdr_free_on_write()
2788 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { in arc_hdr_free_on_write()
2789 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_hdr_free_on_write()
2793 size, hdr); in arc_hdr_free_on_write()
2795 (void) zfs_refcount_remove_many(&state->arcs_size, size, hdr); in arc_hdr_free_on_write()
2804 l2arc_free_abd_on_write(hdr->b_crypt_hdr.b_rabd, size, type); in arc_hdr_free_on_write()
2806 l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type); in arc_hdr_free_on_write()
2816 arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf) in arc_share_buf() argument
2819 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_share_buf()
2821 ASSERT(arc_can_share(hdr, buf)); in arc_share_buf()
2822 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_share_buf()
2824 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_share_buf()
2831 zfs_refcount_transfer_ownership_many(&hdr->b_l1hdr.b_state->arcs_size, in arc_share_buf()
2832 arc_hdr_size(hdr), buf, hdr); in arc_share_buf()
2833 hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf)); in arc_share_buf()
2834 abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd, in arc_share_buf()
2835 HDR_ISTYPE_METADATA(hdr)); in arc_share_buf()
2836 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_share_buf()
2844 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr)); in arc_share_buf()
2845 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr)); in arc_share_buf()
2850 arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf) in arc_unshare_buf() argument
2853 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_unshare_buf()
2856 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in arc_unshare_buf()
2857 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_unshare_buf()
2863 zfs_refcount_transfer_ownership_many(&hdr->b_l1hdr.b_state->arcs_size, in arc_unshare_buf()
2864 arc_hdr_size(hdr), hdr, buf); in arc_unshare_buf()
2865 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_unshare_buf()
2866 abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd); in arc_unshare_buf()
2867 abd_put(hdr->b_l1hdr.b_pabd); in arc_unshare_buf()
2868 hdr->b_l1hdr.b_pabd = NULL; in arc_unshare_buf()
2875 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr)); in arc_unshare_buf()
2876 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr)); in arc_unshare_buf()
2886 arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf) in arc_buf_remove() argument
2888 arc_buf_t **bufp = &hdr->b_l1hdr.b_buf; in arc_buf_remove()
2891 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_buf_remove()
2892 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_buf_remove()
2914 IMPLY(hdr->b_l1hdr.b_bufcnt > 0, lastbuf != NULL); in arc_buf_remove()
2915 IMPLY(hdr->b_l1hdr.b_bufcnt > 0, hdr->b_l1hdr.b_buf != NULL); in arc_buf_remove()
2928 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_destroy_impl() local
2940 ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); in arc_buf_destroy_impl()
2946 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_buf_destroy_impl()
2949 arc_free_data_buf(hdr, buf->b_data, size, buf); in arc_buf_destroy_impl()
2954 ASSERT(hdr->b_l1hdr.b_bufcnt > 0); in arc_buf_destroy_impl()
2955 hdr->b_l1hdr.b_bufcnt -= 1; in arc_buf_destroy_impl()
2958 hdr->b_crypt_hdr.b_ebufcnt -= 1; in arc_buf_destroy_impl()
2965 if (hdr->b_crypt_hdr.b_ebufcnt == 0 && in arc_buf_destroy_impl()
2966 HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd != NULL && in arc_buf_destroy_impl()
2967 !HDR_IO_IN_PROGRESS(hdr)) { in arc_buf_destroy_impl()
2968 arc_hdr_free_pabd(hdr, B_TRUE); in arc_buf_destroy_impl()
2973 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf); in arc_buf_destroy_impl()
2994 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in arc_buf_destroy_impl()
2995 arc_hdr_free_pabd(hdr, B_FALSE); in arc_buf_destroy_impl()
3003 arc_share_buf(hdr, lastbuf); in arc_buf_destroy_impl()
3005 } else if (HDR_SHARED_DATA(hdr)) { in arc_buf_destroy_impl()
3016 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF); in arc_buf_destroy_impl()
3023 if (!arc_hdr_has_uncompressed_buf(hdr)) { in arc_buf_destroy_impl()
3024 arc_cksum_free(hdr); in arc_buf_destroy_impl()
3033 arc_hdr_alloc_pabd(arc_buf_hdr_t *hdr, int alloc_flags) in arc_hdr_alloc_pabd() argument
3039 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0); in arc_hdr_alloc_pabd()
3040 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_hdr_alloc_pabd()
3041 ASSERT(!HDR_SHARED_DATA(hdr) || alloc_rdata); in arc_hdr_alloc_pabd()
3042 IMPLY(alloc_rdata, HDR_PROTECTED(hdr)); in arc_hdr_alloc_pabd()
3045 size = HDR_GET_PSIZE(hdr); in arc_hdr_alloc_pabd()
3046 ASSERT3P(hdr->b_crypt_hdr.b_rabd, ==, NULL); in arc_hdr_alloc_pabd()
3047 hdr->b_crypt_hdr.b_rabd = arc_get_data_abd(hdr, size, hdr, in arc_hdr_alloc_pabd()
3049 ASSERT3P(hdr->b_crypt_hdr.b_rabd, !=, NULL); in arc_hdr_alloc_pabd()
3051 size = arc_hdr_size(hdr); in arc_hdr_alloc_pabd()
3052 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_hdr_alloc_pabd()
3053 hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, size, hdr, in arc_hdr_alloc_pabd()
3055 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in arc_hdr_alloc_pabd()
3059 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr)); in arc_hdr_alloc_pabd()
3063 arc_hdr_free_pabd(arc_buf_hdr_t *hdr, boolean_t free_rdata) in arc_hdr_free_pabd() argument
3065 uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr); in arc_hdr_free_pabd()
3067 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_hdr_free_pabd()
3068 ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr)); in arc_hdr_free_pabd()
3069 IMPLY(free_rdata, HDR_HAS_RABD(hdr)); in arc_hdr_free_pabd()
3078 if (HDR_L2_WRITING(hdr)) { in arc_hdr_free_pabd()
3079 arc_hdr_free_on_write(hdr, free_rdata); in arc_hdr_free_pabd()
3082 arc_free_data_abd(hdr, hdr->b_crypt_hdr.b_rabd, size, hdr); in arc_hdr_free_pabd()
3084 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, in arc_hdr_free_pabd()
3085 size, hdr); in arc_hdr_free_pabd()
3089 hdr->b_crypt_hdr.b_rabd = NULL; in arc_hdr_free_pabd()
3091 hdr->b_l1hdr.b_pabd = NULL; in arc_hdr_free_pabd()
3094 if (hdr->b_l1hdr.b_pabd == NULL && !HDR_HAS_RABD(hdr)) in arc_hdr_free_pabd()
3095 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; in arc_hdr_free_pabd()
3098 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr)); in arc_hdr_free_pabd()
3106 arc_buf_hdr_t *hdr; in arc_hdr_alloc() local
3111 hdr = kmem_cache_alloc(hdr_full_crypt_cache, KM_PUSHPAGE); in arc_hdr_alloc()
3113 hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE); in arc_hdr_alloc()
3116 ASSERT(HDR_EMPTY(hdr)); in arc_hdr_alloc()
3117 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); in arc_hdr_alloc()
3118 ASSERT3P(hdr->b_l1hdr.b_thawed, ==, NULL); in arc_hdr_alloc()
3119 HDR_SET_PSIZE(hdr, psize); in arc_hdr_alloc()
3120 HDR_SET_LSIZE(hdr, lsize); in arc_hdr_alloc()
3121 hdr->b_spa = spa; in arc_hdr_alloc()
3122 hdr->b_type = type; in arc_hdr_alloc()
3123 hdr->b_flags = 0; in arc_hdr_alloc()
3124 arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR); in arc_hdr_alloc()
3125 arc_hdr_set_compress(hdr, compression_type); in arc_hdr_alloc()
3127 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED); in arc_hdr_alloc()
3129 hdr->b_l1hdr.b_state = arc_anon; in arc_hdr_alloc()
3130 hdr->b_l1hdr.b_arc_access = 0; in arc_hdr_alloc()
3131 hdr->b_l1hdr.b_bufcnt = 0; in arc_hdr_alloc()
3132 hdr->b_l1hdr.b_buf = NULL; in arc_hdr_alloc()
3139 arc_hdr_alloc_pabd(hdr, flags); in arc_hdr_alloc()
3140 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_hdr_alloc()
3142 return (hdr); in arc_hdr_alloc()
3153 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new) in arc_hdr_realloc() argument
3155 ASSERT(HDR_HAS_L2HDR(hdr)); in arc_hdr_realloc()
3158 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; in arc_hdr_realloc()
3168 if (HDR_PROTECTED(hdr) && new == hdr_full_cache) in arc_hdr_realloc()
3170 if (HDR_PROTECTED(hdr) && old == hdr_full_cache) in arc_hdr_realloc()
3175 ASSERT(MUTEX_HELD(HDR_LOCK(hdr))); in arc_hdr_realloc()
3176 buf_hash_remove(hdr); in arc_hdr_realloc()
3178 bcopy(hdr, nhdr, HDR_L2ONLY_SIZE); in arc_hdr_realloc()
3191 ASSERT(!HDR_HAS_RABD(hdr)); in arc_hdr_realloc()
3193 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_hdr_realloc()
3194 ASSERT0(hdr->b_l1hdr.b_bufcnt); in arc_hdr_realloc()
3195 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); in arc_hdr_realloc()
3204 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_hdr_realloc()
3212 VERIFY(!HDR_L2_WRITING(hdr)); in arc_hdr_realloc()
3213 VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_hdr_realloc()
3214 ASSERT(!HDR_HAS_RABD(hdr)); in arc_hdr_realloc()
3217 if (hdr->b_l1hdr.b_thawed != NULL) { in arc_hdr_realloc()
3218 kmem_free(hdr->b_l1hdr.b_thawed, 1); in arc_hdr_realloc()
3219 hdr->b_l1hdr.b_thawed = NULL; in arc_hdr_realloc()
3231 ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node)); in arc_hdr_realloc()
3241 list_insert_after(&dev->l2ad_buflist, hdr, nhdr); in arc_hdr_realloc()
3242 list_remove(&dev->l2ad_buflist, hdr); in arc_hdr_realloc()
3254 (void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr), in arc_hdr_realloc()
3255 hdr); in arc_hdr_realloc()
3259 buf_discard_identity(hdr); in arc_hdr_realloc()
3260 kmem_cache_free(old, hdr); in arc_hdr_realloc()
3271 arc_hdr_realloc_crypt(arc_buf_hdr_t *hdr, boolean_t need_crypt) in arc_hdr_realloc_crypt() argument
3282 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_hdr_realloc_crypt()
3283 ASSERT(!HDR_HAS_L2HDR(hdr)); in arc_hdr_realloc_crypt()
3284 ASSERT3U(!!HDR_PROTECTED(hdr), !=, need_crypt); in arc_hdr_realloc_crypt()
3285 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); in arc_hdr_realloc_crypt()
3286 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_hdr_realloc_crypt()
3287 ASSERT(!list_link_active(&hdr->b_l2hdr.b_l2node)); in arc_hdr_realloc_crypt()
3288 ASSERT3P(hdr->b_hash_next, ==, NULL); in arc_hdr_realloc_crypt()
3305 nhdr->b_dva = hdr->b_dva; in arc_hdr_realloc_crypt()
3306 nhdr->b_birth = hdr->b_birth; in arc_hdr_realloc_crypt()
3307 nhdr->b_type = hdr->b_type; in arc_hdr_realloc_crypt()
3308 nhdr->b_flags = hdr->b_flags; in arc_hdr_realloc_crypt()
3309 nhdr->b_psize = hdr->b_psize; in arc_hdr_realloc_crypt()
3310 nhdr->b_lsize = hdr->b_lsize; in arc_hdr_realloc_crypt()
3311 nhdr->b_spa = hdr->b_spa; in arc_hdr_realloc_crypt()
3312 nhdr->b_l2hdr.b_dev = hdr->b_l2hdr.b_dev; in arc_hdr_realloc_crypt()
3313 nhdr->b_l2hdr.b_daddr = hdr->b_l2hdr.b_daddr; in arc_hdr_realloc_crypt()
3314 nhdr->b_l1hdr.b_freeze_cksum = hdr->b_l1hdr.b_freeze_cksum; in arc_hdr_realloc_crypt()
3315 nhdr->b_l1hdr.b_bufcnt = hdr->b_l1hdr.b_bufcnt; in arc_hdr_realloc_crypt()
3316 nhdr->b_l1hdr.b_byteswap = hdr->b_l1hdr.b_byteswap; in arc_hdr_realloc_crypt()
3317 nhdr->b_l1hdr.b_state = hdr->b_l1hdr.b_state; in arc_hdr_realloc_crypt()
3318 nhdr->b_l1hdr.b_arc_access = hdr->b_l1hdr.b_arc_access; in arc_hdr_realloc_crypt()
3319 nhdr->b_l1hdr.b_acb = hdr->b_l1hdr.b_acb; in arc_hdr_realloc_crypt()
3320 nhdr->b_l1hdr.b_pabd = hdr->b_l1hdr.b_pabd; in arc_hdr_realloc_crypt()
3322 if (hdr->b_l1hdr.b_thawed != NULL) { in arc_hdr_realloc_crypt()
3323 nhdr->b_l1hdr.b_thawed = hdr->b_l1hdr.b_thawed; in arc_hdr_realloc_crypt()
3324 hdr->b_l1hdr.b_thawed = NULL; in arc_hdr_realloc_crypt()
3334 nhdr->b_l1hdr.b_buf = hdr->b_l1hdr.b_buf; in arc_hdr_realloc_crypt()
3340 zfs_refcount_transfer(&nhdr->b_l1hdr.b_refcnt, &hdr->b_l1hdr.b_refcnt); in arc_hdr_realloc_crypt()
3342 ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt)); in arc_hdr_realloc_crypt()
3350 if (hdr->b_l1hdr.b_pabd != NULL) { in arc_hdr_realloc_crypt()
3352 &hdr->b_l1hdr.b_state->arcs_size, arc_hdr_size(hdr), in arc_hdr_realloc_crypt()
3353 hdr, nhdr); in arc_hdr_realloc_crypt()
3363 bzero(&hdr->b_dva, sizeof (dva_t)); in arc_hdr_realloc_crypt()
3364 hdr->b_birth = 0; in arc_hdr_realloc_crypt()
3365 hdr->b_type = ARC_BUFC_INVALID; in arc_hdr_realloc_crypt()
3366 hdr->b_flags = 0; in arc_hdr_realloc_crypt()
3367 hdr->b_psize = 0; in arc_hdr_realloc_crypt()
3368 hdr->b_lsize = 0; in arc_hdr_realloc_crypt()
3369 hdr->b_spa = 0; in arc_hdr_realloc_crypt()
3370 hdr->b_l2hdr.b_dev = NULL; in arc_hdr_realloc_crypt()
3371 hdr->b_l2hdr.b_daddr = 0; in arc_hdr_realloc_crypt()
3372 hdr->b_l1hdr.b_freeze_cksum = NULL; in arc_hdr_realloc_crypt()
3373 hdr->b_l1hdr.b_buf = NULL; in arc_hdr_realloc_crypt()
3374 hdr->b_l1hdr.b_bufcnt = 0; in arc_hdr_realloc_crypt()
3375 hdr->b_l1hdr.b_byteswap = 0; in arc_hdr_realloc_crypt()
3376 hdr->b_l1hdr.b_state = NULL; in arc_hdr_realloc_crypt()
3377 hdr->b_l1hdr.b_arc_access = 0; in arc_hdr_realloc_crypt()
3378 hdr->b_l1hdr.b_acb = NULL; in arc_hdr_realloc_crypt()
3379 hdr->b_l1hdr.b_pabd = NULL; in arc_hdr_realloc_crypt()
3382 ASSERT(!HDR_HAS_RABD(hdr)); in arc_hdr_realloc_crypt()
3383 hdr->b_crypt_hdr.b_ot = DMU_OT_NONE; in arc_hdr_realloc_crypt()
3384 hdr->b_crypt_hdr.b_ebufcnt = 0; in arc_hdr_realloc_crypt()
3385 hdr->b_crypt_hdr.b_dsobj = 0; in arc_hdr_realloc_crypt()
3386 bzero(hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN); in arc_hdr_realloc_crypt()
3387 bzero(hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN); in arc_hdr_realloc_crypt()
3388 bzero(hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN); in arc_hdr_realloc_crypt()
3391 buf_discard_identity(hdr); in arc_hdr_realloc_crypt()
3392 kmem_cache_free(ocache, hdr); in arc_hdr_realloc_crypt()
3409 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_convert_to_raw() local
3412 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_convert_to_raw()
3413 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); in arc_convert_to_raw()
3416 if (!HDR_PROTECTED(hdr)) in arc_convert_to_raw()
3417 hdr = arc_hdr_realloc_crypt(hdr, B_TRUE); in arc_convert_to_raw()
3418 hdr->b_crypt_hdr.b_dsobj = dsobj; in arc_convert_to_raw()
3419 hdr->b_crypt_hdr.b_ot = ot; in arc_convert_to_raw()
3420 hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ? in arc_convert_to_raw()
3422 if (!arc_hdr_has_uncompressed_buf(hdr)) in arc_convert_to_raw()
3423 arc_cksum_free(hdr); in arc_convert_to_raw()
3426 bcopy(salt, hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN); in arc_convert_to_raw()
3428 bcopy(iv, hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN); in arc_convert_to_raw()
3430 bcopy(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN); in arc_convert_to_raw()
3440 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size, in arc_alloc_buf() local
3444 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, B_FALSE, in arc_alloc_buf()
3463 arc_buf_hdr_t *hdr; in arc_buf_alloc_l2only() local
3466 hdr = kmem_cache_alloc(hdr_l2only_cache, KM_SLEEP); in arc_buf_alloc_l2only()
3467 hdr->b_birth = birth; in arc_buf_alloc_l2only()
3468 hdr->b_type = type; in arc_buf_alloc_l2only()
3469 hdr->b_flags = 0; in arc_buf_alloc_l2only()
3470 arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L2HDR); in arc_buf_alloc_l2only()
3471 HDR_SET_LSIZE(hdr, size); in arc_buf_alloc_l2only()
3472 HDR_SET_PSIZE(hdr, psize); in arc_buf_alloc_l2only()
3473 arc_hdr_set_compress(hdr, compress); in arc_buf_alloc_l2only()
3475 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED); in arc_buf_alloc_l2only()
3477 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); in arc_buf_alloc_l2only()
3478 hdr->b_spa = spa_load_guid(dev->l2ad_vdev->vdev_spa); in arc_buf_alloc_l2only()
3480 hdr->b_dva = dva; in arc_buf_alloc_l2only()
3482 hdr->b_l2hdr.b_dev = dev; in arc_buf_alloc_l2only()
3483 hdr->b_l2hdr.b_daddr = daddr; in arc_buf_alloc_l2only()
3484 hdr->b_l2hdr.b_arcs_state = arcs_state; in arc_buf_alloc_l2only()
3486 return (hdr); in arc_buf_alloc_l2only()
3502 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, in arc_alloc_compressed_buf() local
3506 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, in arc_alloc_compressed_buf()
3509 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); in arc_alloc_compressed_buf()
3518 ASSERT(!abd_is_linear(hdr->b_l1hdr.b_pabd)); in arc_alloc_compressed_buf()
3519 arc_hdr_free_pabd(hdr, B_FALSE); in arc_alloc_compressed_buf()
3520 arc_share_buf(hdr, buf); in arc_alloc_compressed_buf()
3532 arc_buf_hdr_t *hdr; in arc_alloc_raw_buf() local
3542 hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, B_TRUE, in arc_alloc_raw_buf()
3545 hdr->b_crypt_hdr.b_dsobj = dsobj; in arc_alloc_raw_buf()
3546 hdr->b_crypt_hdr.b_ot = ot; in arc_alloc_raw_buf()
3547 hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ? in arc_alloc_raw_buf()
3549 bcopy(salt, hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN); in arc_alloc_raw_buf()
3550 bcopy(iv, hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN); in arc_alloc_raw_buf()
3551 bcopy(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN); in arc_alloc_raw_buf()
3559 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_TRUE, B_TRUE, in arc_alloc_raw_buf()
3562 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); in arc_alloc_raw_buf()
3568 l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr, in l2arc_hdr_arcstats_update() argument
3571 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr; in l2arc_hdr_arcstats_update()
3573 uint64_t lsize = HDR_GET_LSIZE(hdr); in l2arc_hdr_arcstats_update()
3574 uint64_t psize = HDR_GET_PSIZE(hdr); in l2arc_hdr_arcstats_update()
3576 arc_buf_contents_t type = hdr->b_type; in l2arc_hdr_arcstats_update()
3592 if (HDR_PREFETCH(hdr)) { in l2arc_hdr_arcstats_update()
3604 switch (hdr->b_l2hdr.b_arcs_state) { in l2arc_hdr_arcstats_update()
3638 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr) in arc_hdr_l2hdr_destroy() argument
3640 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr; in arc_hdr_l2hdr_destroy()
3642 uint64_t psize = HDR_GET_PSIZE(hdr); in arc_hdr_l2hdr_destroy()
3646 ASSERT(HDR_HAS_L2HDR(hdr)); in arc_hdr_l2hdr_destroy()
3648 list_remove(&dev->l2ad_buflist, hdr); in arc_hdr_l2hdr_destroy()
3650 l2arc_hdr_arcstats_decrement(hdr); in arc_hdr_l2hdr_destroy()
3653 (void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr), in arc_hdr_l2hdr_destroy()
3654 hdr); in arc_hdr_l2hdr_destroy()
3655 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR); in arc_hdr_l2hdr_destroy()
3659 arc_hdr_destroy(arc_buf_hdr_t *hdr) in arc_hdr_destroy() argument
3661 if (HDR_HAS_L1HDR(hdr)) { in arc_hdr_destroy()
3662 ASSERT(hdr->b_l1hdr.b_buf == NULL || in arc_hdr_destroy()
3663 hdr->b_l1hdr.b_bufcnt > 0); in arc_hdr_destroy()
3664 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_hdr_destroy()
3665 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); in arc_hdr_destroy()
3667 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_hdr_destroy()
3668 ASSERT(!HDR_IN_HASH_TABLE(hdr)); in arc_hdr_destroy()
3670 if (HDR_HAS_L2HDR(hdr)) { in arc_hdr_destroy()
3671 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; in arc_hdr_destroy()
3686 if (HDR_HAS_L2HDR(hdr)) in arc_hdr_destroy()
3687 arc_hdr_l2hdr_destroy(hdr); in arc_hdr_destroy()
3699 if (!HDR_EMPTY(hdr)) in arc_hdr_destroy()
3700 buf_discard_identity(hdr); in arc_hdr_destroy()
3702 if (HDR_HAS_L1HDR(hdr)) { in arc_hdr_destroy()
3703 arc_cksum_free(hdr); in arc_hdr_destroy()
3705 while (hdr->b_l1hdr.b_buf != NULL) in arc_hdr_destroy()
3706 arc_buf_destroy_impl(hdr->b_l1hdr.b_buf); in arc_hdr_destroy()
3709 if (hdr->b_l1hdr.b_thawed != NULL) { in arc_hdr_destroy()
3710 kmem_free(hdr->b_l1hdr.b_thawed, 1); in arc_hdr_destroy()
3711 hdr->b_l1hdr.b_thawed = NULL; in arc_hdr_destroy()
3715 if (hdr->b_l1hdr.b_pabd != NULL) in arc_hdr_destroy()
3716 arc_hdr_free_pabd(hdr, B_FALSE); in arc_hdr_destroy()
3718 if (HDR_HAS_RABD(hdr)) in arc_hdr_destroy()
3719 arc_hdr_free_pabd(hdr, B_TRUE); in arc_hdr_destroy()
3722 ASSERT3P(hdr->b_hash_next, ==, NULL); in arc_hdr_destroy()
3723 if (HDR_HAS_L1HDR(hdr)) { in arc_hdr_destroy()
3724 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_hdr_destroy()
3725 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); in arc_hdr_destroy()
3727 if (!HDR_PROTECTED(hdr)) { in arc_hdr_destroy()
3728 kmem_cache_free(hdr_full_cache, hdr); in arc_hdr_destroy()
3730 kmem_cache_free(hdr_full_crypt_cache, hdr); in arc_hdr_destroy()
3733 kmem_cache_free(hdr_l2only_cache, hdr); in arc_hdr_destroy()
3740 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_destroy() local
3742 if (hdr->b_l1hdr.b_state == arc_anon) { in arc_buf_destroy()
3743 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); in arc_buf_destroy()
3744 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_buf_destroy()
3745 VERIFY0(remove_reference(hdr, NULL, tag)); in arc_buf_destroy()
3746 arc_hdr_destroy(hdr); in arc_buf_destroy()
3750 kmutex_t *hash_lock = HDR_LOCK(hdr); in arc_buf_destroy()
3753 ASSERT3P(hdr, ==, buf->b_hdr); in arc_buf_destroy()
3754 ASSERT(hdr->b_l1hdr.b_bufcnt > 0); in arc_buf_destroy()
3755 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in arc_buf_destroy()
3756 ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon); in arc_buf_destroy()
3759 (void) remove_reference(hdr, hash_lock, tag); in arc_buf_destroy()
3777 arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) in arc_evict_hdr() argument
3781 int min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ? in arc_evict_hdr()
3785 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_evict_hdr()
3787 state = hdr->b_l1hdr.b_state; in arc_evict_hdr()
3789 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_evict_hdr()
3790 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_evict_hdr()
3799 if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) { in arc_evict_hdr()
3805 bytes_evicted += HDR_GET_LSIZE(hdr); in arc_evict_hdr()
3807 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr); in arc_evict_hdr()
3809 if (HDR_HAS_L2HDR(hdr)) { in arc_evict_hdr()
3810 ASSERT(hdr->b_l1hdr.b_pabd == NULL); in arc_evict_hdr()
3811 ASSERT(!HDR_HAS_RABD(hdr)); in arc_evict_hdr()
3816 arc_change_state(arc_l2c_only, hdr, hash_lock); in arc_evict_hdr()
3821 hdr = arc_hdr_realloc(hdr, hdr_full_cache, in arc_evict_hdr()
3824 arc_change_state(arc_anon, hdr, hash_lock); in arc_evict_hdr()
3825 arc_hdr_destroy(hdr); in arc_evict_hdr()
3834 if (HDR_IO_IN_PROGRESS(hdr) || in arc_evict_hdr()
3835 ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) && in arc_evict_hdr()
3836 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < min_lifetime * hz)) { in arc_evict_hdr()
3841 ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt)); in arc_evict_hdr()
3842 while (hdr->b_l1hdr.b_buf) { in arc_evict_hdr()
3843 arc_buf_t *buf = hdr->b_l1hdr.b_buf; in arc_evict_hdr()
3849 bytes_evicted += HDR_GET_LSIZE(hdr); in arc_evict_hdr()
3854 if (HDR_HAS_L2HDR(hdr)) { in arc_evict_hdr()
3855 ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr)); in arc_evict_hdr()
3857 if (l2arc_write_eligible(hdr->b_spa, hdr)) { in arc_evict_hdr()
3859 HDR_GET_LSIZE(hdr)); in arc_evict_hdr()
3865 HDR_GET_LSIZE(hdr)); in arc_evict_hdr()
3870 HDR_GET_LSIZE(hdr)); in arc_evict_hdr()
3877 HDR_GET_LSIZE(hdr)); in arc_evict_hdr()
3881 if (hdr->b_l1hdr.b_bufcnt == 0) { in arc_evict_hdr()
3882 arc_cksum_free(hdr); in arc_evict_hdr()
3884 bytes_evicted += arc_hdr_size(hdr); in arc_evict_hdr()
3892 if (hdr->b_l1hdr.b_pabd != NULL) in arc_evict_hdr()
3893 arc_hdr_free_pabd(hdr, B_FALSE); in arc_evict_hdr()
3895 if (HDR_HAS_RABD(hdr)) in arc_evict_hdr()
3896 arc_hdr_free_pabd(hdr, B_TRUE); in arc_evict_hdr()
3898 arc_change_state(evicted_state, hdr, hash_lock); in arc_evict_hdr()
3899 ASSERT(HDR_IN_HASH_TABLE(hdr)); in arc_evict_hdr()
3900 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE); in arc_evict_hdr()
3901 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr); in arc_evict_hdr()
3913 arc_buf_hdr_t *hdr; in arc_evict_state_impl() local
3922 for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL; in arc_evict_state_impl()
3923 hdr = multilist_sublist_prev(mls, marker)) { in arc_evict_state_impl()
3950 if (hdr->b_spa == 0) in arc_evict_state_impl()
3954 if (spa != 0 && hdr->b_spa != spa) { in arc_evict_state_impl()
3959 hash_lock = HDR_LOCK(hdr); in arc_evict_state_impl()
3973 uint64_t evicted = arc_evict_hdr(hdr, hash_lock); in arc_evict_state_impl()
4934 arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, void *tag, in arc_get_data_abd() argument
4937 arc_buf_contents_t type = arc_buf_type(hdr); in arc_get_data_abd()
4939 arc_get_data_impl(hdr, size, tag, do_adapt); in arc_get_data_abd()
4949 arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, void *tag) in arc_get_data_buf() argument
4951 arc_buf_contents_t type = arc_buf_type(hdr); in arc_get_data_buf()
4953 arc_get_data_impl(hdr, size, tag, B_TRUE); in arc_get_data_buf()
4969 arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag, in arc_get_data_impl() argument
4972 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_get_data_impl()
4973 arc_buf_contents_t type = arc_buf_type(hdr); in arc_get_data_impl()
5015 VERIFY3U(hdr->b_type, ==, type); in arc_get_data_impl()
5039 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { in arc_get_data_impl()
5040 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_get_data_impl()
5050 hdr->b_l1hdr.b_state == arc_anon && in arc_get_data_impl()
5058 arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size, void *tag) in arc_free_data_abd() argument
5060 arc_free_data_impl(hdr, size, tag); in arc_free_data_abd()
5065 arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, void *tag) in arc_free_data_buf() argument
5067 arc_buf_contents_t type = arc_buf_type(hdr); in arc_free_data_buf()
5069 arc_free_data_impl(hdr, size, tag); in arc_free_data_buf()
5082 arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag) in arc_free_data_impl() argument
5084 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_free_data_impl()
5085 arc_buf_contents_t type = arc_buf_type(hdr); in arc_free_data_impl()
5088 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { in arc_free_data_impl()
5089 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_free_data_impl()
5097 VERIFY3U(hdr->b_type, ==, type); in arc_free_data_impl()
5111 arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) in arc_access() argument
5116 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_access()
5118 if (hdr->b_l1hdr.b_state == arc_anon) { in arc_access()
5125 ASSERT0(hdr->b_l1hdr.b_arc_access); in arc_access()
5126 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
5127 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); in arc_access()
5128 arc_change_state(arc_mru, hdr, hash_lock); in arc_access()
5130 } else if (hdr->b_l1hdr.b_state == arc_mru) { in arc_access()
5141 if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) { in arc_access()
5142 if (zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { in arc_access()
5145 &hdr->b_l1hdr.b_arc_node)); in arc_access()
5147 if (HDR_HAS_L2HDR(hdr)) in arc_access()
5148 l2arc_hdr_arcstats_decrement_state(hdr); in arc_access()
5149 arc_hdr_clear_flags(hdr, in arc_access()
5153 if (HDR_HAS_L2HDR(hdr)) in arc_access()
5154 l2arc_hdr_arcstats_increment_state(hdr); in arc_access()
5156 hdr->b_l1hdr.b_arc_access = now; in arc_access()
5165 if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) { in arc_access()
5171 hdr->b_l1hdr.b_arc_access = now; in arc_access()
5172 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
5173 arc_change_state(arc_mfu, hdr, hash_lock); in arc_access()
5176 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) { in arc_access()
5183 if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) { in arc_access()
5185 if (zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) > 0) { in arc_access()
5186 if (HDR_HAS_L2HDR(hdr)) in arc_access()
5187 l2arc_hdr_arcstats_decrement_state(hdr); in arc_access()
5188 arc_hdr_clear_flags(hdr, in arc_access()
5191 if (HDR_HAS_L2HDR(hdr)) in arc_access()
5192 l2arc_hdr_arcstats_increment_state(hdr); in arc_access()
5194 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); in arc_access()
5197 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
5200 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
5201 arc_change_state(new_state, hdr, hash_lock); in arc_access()
5204 } else if (hdr->b_l1hdr.b_state == arc_mfu) { in arc_access()
5215 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
5216 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) { in arc_access()
5224 if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) { in arc_access()
5232 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
5233 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
5234 arc_change_state(new_state, hdr, hash_lock); in arc_access()
5237 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) { in arc_access()
5242 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
5243 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
5244 arc_change_state(arc_mfu, hdr, hash_lock); in arc_access()
5258 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_access() local
5265 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) { in arc_buf_access()
5270 kmutex_t *hash_lock = HDR_LOCK(hdr); in arc_buf_access()
5273 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) { in arc_buf_access()
5282 ASSERT(hdr->b_l1hdr.b_state == arc_mru || in arc_buf_access()
5283 hdr->b_l1hdr.b_state == arc_mfu); in arc_buf_access()
5285 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); in arc_buf_access()
5286 arc_access(hdr, hash_lock); in arc_buf_access()
5290 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), in arc_buf_access()
5291 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data, metadata, hits); in arc_buf_access()
5325 arc_hdr_verify(arc_buf_hdr_t *hdr, const blkptr_t *bp) in arc_hdr_verify() argument
5328 ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0); in arc_hdr_verify()
5329 ASSERT3U(arc_hdr_get_compress(hdr), ==, ZIO_COMPRESS_OFF); in arc_hdr_verify()
5331 if (HDR_COMPRESSION_ENABLED(hdr)) { in arc_hdr_verify()
5332 ASSERT3U(arc_hdr_get_compress(hdr), ==, in arc_hdr_verify()
5335 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp)); in arc_hdr_verify()
5336 ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp)); in arc_hdr_verify()
5337 ASSERT3U(!!HDR_PROTECTED(hdr), ==, BP_IS_PROTECTED(bp)); in arc_hdr_verify()
5346 arc_hdr_verify_checksum(spa_t *spa, arc_buf_hdr_t *hdr, const blkptr_t *bp) in arc_hdr_verify_checksum() argument
5348 arc_hdr_verify(hdr, bp); in arc_hdr_verify_checksum()
5354 if (HDR_HAS_RABD(hdr)) { in arc_hdr_verify_checksum()
5355 abd = hdr->b_crypt_hdr.b_rabd; in arc_hdr_verify_checksum()
5357 } else if (HDR_COMPRESSION_ENABLED(hdr)) { in arc_hdr_verify_checksum()
5358 abd = hdr->b_l1hdr.b_pabd; in arc_hdr_verify_checksum()
5366 int psize = HDR_GET_PSIZE(hdr); in arc_hdr_verify_checksum()
5379 err, (void *)hdr, (void *)bp, (void *)abd, buf); in arc_hdr_verify_checksum()
5388 arc_buf_hdr_t *hdr = zio->io_private; in arc_read_done() local
5402 if (HDR_IN_HASH_TABLE(hdr)) { in arc_read_done()
5403 ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp)); in arc_read_done()
5404 ASSERT3U(hdr->b_dva.dva_word[0], ==, in arc_read_done()
5406 ASSERT3U(hdr->b_dva.dva_word[1], ==, in arc_read_done()
5409 arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp, in arc_read_done()
5412 ASSERT((found == hdr && in arc_read_done()
5413 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || in arc_read_done()
5414 (found == hdr && HDR_L2_READING(hdr))); in arc_read_done()
5419 hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp); in arc_read_done()
5420 hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset; in arc_read_done()
5421 zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt, in arc_read_done()
5422 hdr->b_crypt_hdr.b_iv); in arc_read_done()
5430 hdr->b_crypt_hdr.b_mac); in arc_read_done()
5434 zio_crypt_decode_mac_bp(bp, hdr->b_crypt_hdr.b_mac); in arc_read_done()
5442 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64; in arc_read_done()
5444 hdr->b_l1hdr.b_byteswap = in arc_read_done()
5448 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; in arc_read_done()
5452 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED); in arc_read_done()
5454 callback_list = hdr->b_l1hdr.b_acb; in arc_read_done()
5458 hdr->b_l1hdr.b_state == arc_anon) { in arc_read_done()
5465 arc_access(hdr, hash_lock); in arc_read_done()
5484 int error = arc_buf_alloc_impl(hdr, zio->io_spa, in arc_read_done()
5539 hdr->b_l1hdr.b_acb = NULL; in arc_read_done()
5540 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_read_done()
5542 ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr)); in arc_read_done()
5544 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt) || in arc_read_done()
5548 arc_hdr_verify(hdr, zio->io_bp); in arc_read_done()
5550 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); in arc_read_done()
5551 if (hdr->b_l1hdr.b_state != arc_anon) in arc_read_done()
5552 arc_change_state(arc_anon, hdr, hash_lock); in arc_read_done()
5553 if (HDR_IN_HASH_TABLE(hdr)) in arc_read_done()
5554 buf_hash_remove(hdr); in arc_read_done()
5555 freeable = zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt); in arc_read_done()
5563 cv_broadcast(&hdr->b_l1hdr.b_cv); in arc_read_done()
5574 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); in arc_read_done()
5575 freeable = zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt); in arc_read_done()
5605 arc_hdr_destroy(hdr); in arc_read_done()
5631 arc_buf_hdr_t *hdr = NULL; in arc_read() local
5651 hdr = buf_hash_find(guid, bp, &hash_lock); in arc_read()
5661 if (hdr != NULL && HDR_HAS_L1HDR(hdr) && (HDR_HAS_RABD(hdr) || in arc_read()
5662 (hdr->b_l1hdr.b_pabd != NULL && !encrypted_read))) { in arc_read()
5666 if (HDR_IO_IN_PROGRESS(hdr)) { in arc_read()
5667 zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head; in arc_read()
5670 if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) && in arc_read()
5679 arc_buf_hdr_t *, hdr); in arc_read()
5682 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { in arc_read()
5683 arc_hdr_clear_flags(hdr, in arc_read()
5688 cv_wait(&hdr->b_l1hdr.b_cv, hash_lock); in arc_read()
5711 acb->acb_next = hdr->b_l1hdr.b_acb; in arc_read()
5712 hdr->b_l1hdr.b_acb = acb; in arc_read()
5720 ASSERT(hdr->b_l1hdr.b_state == arc_mru || in arc_read()
5721 hdr->b_l1hdr.b_state == arc_mfu); in arc_read()
5724 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { in arc_read()
5732 arc_buf_hdr_t *, hdr); in arc_read()
5735 arc_hdr_clear_flags(hdr, in arc_read()
5739 if (hdr->b_flags & ARC_FLAG_PRESCIENT_PREFETCH) { in arc_read()
5742 arc_hdr_clear_flags(hdr, in arc_read()
5748 arc_hdr_verify_checksum(spa, hdr, bp); in arc_read()
5751 rc = arc_buf_alloc_impl(hdr, spa, zb, private, in arc_read()
5769 (void) remove_reference(hdr, hash_lock, in arc_read()
5778 zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) { in arc_read()
5779 if (HDR_HAS_L2HDR(hdr)) in arc_read()
5780 l2arc_hdr_arcstats_decrement_state(hdr); in arc_read()
5781 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); in arc_read()
5782 if (HDR_HAS_L2HDR(hdr)) in arc_read()
5783 l2arc_hdr_arcstats_increment_state(hdr); in arc_read()
5785 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); in arc_read()
5786 arc_access(hdr, hash_lock); in arc_read()
5788 arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH); in arc_read()
5790 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); in arc_read()
5793 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), in arc_read()
5794 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), in arc_read()
5810 if (hdr == NULL) { in arc_read()
5814 hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, in arc_read()
5819 hdr->b_dva = *BP_IDENTITY(bp); in arc_read()
5820 hdr->b_birth = BP_PHYSICAL_BIRTH(bp); in arc_read()
5821 exists = buf_hash_insert(hdr, &hash_lock); in arc_read()
5826 buf_discard_identity(hdr); in arc_read()
5827 arc_hdr_destroy(hdr); in arc_read()
5837 if (!HDR_HAS_L1HDR(hdr)) { in arc_read()
5838 hdr = arc_hdr_realloc(hdr, hdr_l2only_cache, in arc_read()
5842 if (GHOST_STATE(hdr->b_l1hdr.b_state)) { in arc_read()
5843 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_read()
5844 ASSERT(!HDR_HAS_RABD(hdr)); in arc_read()
5845 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_read()
5847 &hdr->b_l1hdr.b_refcnt)); in arc_read()
5848 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_read()
5849 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); in arc_read()
5850 } else if (HDR_IO_IN_PROGRESS(hdr)) { in arc_read()
5860 cv_wait(&hdr->b_l1hdr.b_cv, hash_lock); in arc_read()
5875 arc_adapt(arc_hdr_size(hdr), hdr->b_l1hdr.b_state); in arc_read()
5876 arc_access(hdr, hash_lock); in arc_read()
5877 arc_hdr_alloc_pabd(hdr, alloc_flags); in arc_read()
5881 ASSERT(HDR_HAS_RABD(hdr)); in arc_read()
5882 size = HDR_GET_PSIZE(hdr); in arc_read()
5883 hdr_abd = hdr->b_crypt_hdr.b_rabd; in arc_read()
5886 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in arc_read()
5887 size = arc_hdr_size(hdr); in arc_read()
5888 hdr_abd = hdr->b_l1hdr.b_pabd; in arc_read()
5890 if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) { in arc_read()
5906 zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) { in arc_read()
5907 if (HDR_HAS_L2HDR(hdr)) in arc_read()
5908 l2arc_hdr_arcstats_decrement_state(hdr); in arc_read()
5909 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); in arc_read()
5910 if (HDR_HAS_L2HDR(hdr)) in arc_read()
5911 l2arc_hdr_arcstats_increment_state(hdr); in arc_read()
5914 arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH); in arc_read()
5917 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); in arc_read()
5919 arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH); in arc_read()
5921 arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT); in arc_read()
5923 arc_hdr_set_flags(hdr, ARC_FLAG_PREDICTIVE_PREFETCH); in arc_read()
5924 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state)); in arc_read()
5934 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); in arc_read()
5935 hdr->b_l1hdr.b_acb = acb; in arc_read()
5936 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_read()
5938 if (HDR_HAS_L2HDR(hdr) && in arc_read()
5939 (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) { in arc_read()
5940 devw = hdr->b_l2hdr.b_dev->l2ad_writing; in arc_read()
5941 addr = hdr->b_l2hdr.b_daddr; in arc_read()
5957 arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); in arc_read()
5959 arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); in arc_read()
5965 ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize); in arc_read()
5967 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp, in arc_read()
5970 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), in arc_read()
5971 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), in arc_read()
5984 if (HDR_HAS_L2HDR(hdr) && in arc_read()
5985 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) && in arc_read()
5986 !(l2arc_noprefetch && HDR_PREFETCH(hdr))) { in arc_read()
5991 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr); in arc_read()
5996 cb->l2rcb_hdr = hdr; in arc_read()
6006 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && in arc_read()
6007 !HDR_COMPRESSION_ENABLED(hdr) && in arc_read()
6008 HDR_GET_PSIZE(hdr) != 0) { in arc_read()
6009 size = HDR_GET_PSIZE(hdr); in arc_read()
6015 HDR_ISTYPE_METADATA(hdr)); in arc_read()
6031 ASSERT3U(arc_hdr_get_compress(hdr), !=, in arc_read()
6049 HDR_GET_PSIZE(hdr)); in arc_read()
6065 arc_buf_hdr_t *, hdr); in arc_read()
6067 if (HDR_L2_WRITING(hdr)) in arc_read()
6076 arc_buf_hdr_t *, hdr); in arc_read()
6082 arc_read_done, hdr, priority, zio_flags, zb); in arc_read()
6103 arc_buf_hdr_t *hdr; in arc_freed() local
6109 hdr = buf_hash_find(guid, bp, &hash_lock); in arc_freed()
6110 if (hdr == NULL) in arc_freed()
6135 if (!HDR_HAS_L1HDR(hdr) || (!HDR_IO_IN_PROGRESS(hdr) && in arc_freed()
6136 zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt))) { in arc_freed()
6137 arc_change_state(arc_anon, hdr, hash_lock); in arc_freed()
6138 arc_hdr_destroy(hdr); in arc_freed()
6163 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_release() local
6165 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_release()
6172 if (hdr->b_l1hdr.b_state == arc_anon) { in arc_release()
6181 ASSERT(!HDR_IN_HASH_TABLE(hdr)); in arc_release()
6182 ASSERT(!HDR_HAS_L2HDR(hdr)); in arc_release()
6183 ASSERT(HDR_EMPTY(hdr)); in arc_release()
6185 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); in arc_release()
6186 ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1); in arc_release()
6187 ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_release()
6189 hdr->b_l1hdr.b_arc_access = 0; in arc_release()
6195 buf_discard_identity(hdr); in arc_release()
6201 kmutex_t *hash_lock = HDR_LOCK(hdr); in arc_release()
6209 while (HDR_IO_IN_PROGRESS(hdr)) { in arc_release()
6210 DTRACE_PROBE1(arc_release__io, arc_buf_hdr_t *, hdr); in arc_release()
6211 cv_wait(&hdr->b_l1hdr.b_cv, hash_lock); in arc_release()
6219 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_release()
6220 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in arc_release()
6224 ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0); in arc_release()
6226 if (HDR_HAS_L2HDR(hdr)) { in arc_release()
6227 mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx); in arc_release()
6237 if (HDR_HAS_L2HDR(hdr)) in arc_release()
6238 arc_hdr_l2hdr_destroy(hdr); in arc_release()
6240 mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx); in arc_release()
6246 if (hdr->b_l1hdr.b_bufcnt > 1) { in arc_release()
6248 uint64_t spa = hdr->b_spa; in arc_release()
6249 uint64_t psize = HDR_GET_PSIZE(hdr); in arc_release()
6250 uint64_t lsize = HDR_GET_LSIZE(hdr); in arc_release()
6251 boolean_t protected = HDR_PROTECTED(hdr); in arc_release()
6252 enum zio_compress compress = arc_hdr_get_compress(hdr); in arc_release()
6253 arc_buf_contents_t type = arc_buf_type(hdr); in arc_release()
6254 VERIFY3U(hdr->b_type, ==, type); in arc_release()
6256 ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL); in arc_release()
6257 (void) remove_reference(hdr, hash_lock, tag); in arc_release()
6260 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf); in arc_release()
6269 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf); in arc_release()
6277 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf); in arc_release()
6284 arc_unshare_buf(hdr, buf); in arc_release()
6292 if (arc_can_share(hdr, lastbuf)) { in arc_release()
6293 arc_share_buf(hdr, lastbuf); in arc_release()
6295 arc_hdr_alloc_pabd(hdr, ARC_HDR_DO_ADAPT); in arc_release()
6296 abd_copy_from_buf(hdr->b_l1hdr.b_pabd, in arc_release()
6300 } else if (HDR_SHARED_DATA(hdr)) { in arc_release()
6310 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF); in arc_release()
6313 ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr)); in arc_release()
6319 if (zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) { in arc_release()
6326 hdr->b_l1hdr.b_bufcnt -= 1; in arc_release()
6328 hdr->b_crypt_hdr.b_ebufcnt -= 1; in arc_release()
6334 if (!arc_hdr_has_uncompressed_buf(hdr)) in arc_release()
6335 arc_cksum_free(hdr); in arc_release()
6344 compress, type, HDR_HAS_RABD(hdr)); in arc_release()
6363 ASSERT(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 1); in arc_release()
6365 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_release()
6366 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_release()
6367 arc_change_state(arc_anon, hdr, hash_lock); in arc_release()
6368 hdr->b_l1hdr.b_arc_access = 0; in arc_release()
6371 buf_discard_identity(hdr); in arc_release()
6406 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_write_ready() local
6410 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_write_ready()
6412 ASSERT(hdr->b_l1hdr.b_bufcnt > 0); in arc_write_ready()
6420 arc_cksum_free(hdr); in arc_write_ready()
6422 if (hdr->b_l1hdr.b_pabd != NULL) { in arc_write_ready()
6424 arc_unshare_buf(hdr, buf); in arc_write_ready()
6426 arc_hdr_free_pabd(hdr, B_FALSE); in arc_write_ready()
6430 if (HDR_HAS_RABD(hdr)) in arc_write_ready()
6431 arc_hdr_free_pabd(hdr, B_TRUE); in arc_write_ready()
6433 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_write_ready()
6434 ASSERT(!HDR_HAS_RABD(hdr)); in arc_write_ready()
6435 ASSERT(!HDR_SHARED_DATA(hdr)); in arc_write_ready()
6440 if (HDR_IO_IN_PROGRESS(hdr)) in arc_write_ready()
6443 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_write_ready()
6445 if (BP_IS_PROTECTED(bp) != !!HDR_PROTECTED(hdr)) in arc_write_ready()
6446 hdr = arc_hdr_realloc_crypt(hdr, BP_IS_PROTECTED(bp)); in arc_write_ready()
6451 ASSERT(HDR_PROTECTED(hdr)); in arc_write_ready()
6455 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64; in arc_write_ready()
6457 hdr->b_l1hdr.b_byteswap = in arc_write_ready()
6461 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; in arc_write_ready()
6464 hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp); in arc_write_ready()
6465 hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset; in arc_write_ready()
6466 zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt, in arc_write_ready()
6467 hdr->b_crypt_hdr.b_iv); in arc_write_ready()
6468 zio_crypt_decode_mac_bp(bp, hdr->b_crypt_hdr.b_mac); in arc_write_ready()
6476 arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH); in arc_write_ready()
6492 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp)); in arc_write_ready()
6495 HDR_SET_PSIZE(hdr, psize); in arc_write_ready()
6496 arc_hdr_set_compress(hdr, compress); in arc_write_ready()
6518 arc_hdr_alloc_pabd(hdr, ARC_HDR_DO_ADAPT|ARC_HDR_ALLOC_RDATA); in arc_write_ready()
6519 abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize); in arc_write_ready()
6520 } else if (zfs_abd_scatter_enabled || !arc_can_share(hdr, buf)) { in arc_write_ready()
6528 arc_hdr_alloc_pabd(hdr, in arc_write_ready()
6530 abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize); in arc_write_ready()
6531 } else if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF && in arc_write_ready()
6534 arc_hdr_alloc_pabd(hdr, ARC_HDR_DO_ADAPT); in arc_write_ready()
6535 abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize); in arc_write_ready()
6537 ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr)); in arc_write_ready()
6538 arc_hdr_alloc_pabd(hdr, ARC_HDR_DO_ADAPT); in arc_write_ready()
6539 abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data, in arc_write_ready()
6545 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); in arc_write_ready()
6546 arc_share_buf(hdr, buf); in arc_write_ready()
6550 arc_hdr_verify(hdr, bp); in arc_write_ready()
6579 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_write_done() local
6581 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); in arc_write_done()
6584 arc_hdr_verify(hdr, zio->io_bp); in arc_write_done()
6587 buf_discard_identity(hdr); in arc_write_done()
6589 hdr->b_dva = *BP_IDENTITY(zio->io_bp); in arc_write_done()
6590 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp); in arc_write_done()
6593 ASSERT(HDR_EMPTY(hdr)); in arc_write_done()
6602 if (!HDR_EMPTY(hdr)) { in arc_write_done()
6610 exists = buf_hash_insert(hdr, &hash_lock); in arc_write_done()
6620 (void *)hdr, (void *)exists); in arc_write_done()
6626 exists = buf_hash_insert(hdr, &hash_lock); in arc_write_done()
6633 (void *)hdr, (void *)exists); in arc_write_done()
6636 ASSERT(hdr->b_l1hdr.b_bufcnt == 1); in arc_write_done()
6637 ASSERT(hdr->b_l1hdr.b_state == arc_anon); in arc_write_done()
6642 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_write_done()
6644 if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon) in arc_write_done()
6645 arc_access(hdr, hash_lock); in arc_write_done()
6648 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_write_done()
6651 ASSERT(!zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_write_done()
6665 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_write() local
6672 ASSERT(!HDR_IO_ERROR(hdr)); in arc_write()
6673 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_write()
6674 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); in arc_write()
6675 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0); in arc_write()
6677 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); in arc_write()
6682 localprop.zp_compress = HDR_GET_COMPRESS(hdr); in arc_write()
6685 (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ? in arc_write()
6687 bcopy(hdr->b_crypt_hdr.b_salt, localprop.zp_salt, in arc_write()
6689 bcopy(hdr->b_crypt_hdr.b_iv, localprop.zp_iv, in arc_write()
6691 bcopy(hdr->b_crypt_hdr.b_mac, localprop.zp_mac, in arc_write()
6700 ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf)); in arc_write()
6701 localprop.zp_compress = HDR_GET_COMPRESS(hdr); in arc_write()
6717 if (hdr->b_l1hdr.b_pabd != NULL) { in arc_write()
6725 arc_unshare_buf(hdr, buf); in arc_write()
6727 arc_hdr_free_pabd(hdr, B_FALSE); in arc_write()
6732 if (HDR_HAS_RABD(hdr)) in arc_write()
6733 arc_hdr_free_pabd(hdr, B_TRUE); in arc_write()
6736 arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF); in arc_write()
6739 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); in arc_write()
6742 abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)), in arc_write()
6743 HDR_GET_LSIZE(hdr), arc_buf_size(buf), &localprop, arc_write_ready, in arc_write()
6940 arc_buf_hdr_t *hdr = obj; in arc_state_multilist_index_func() local
6947 ASSERT(!HDR_EMPTY(hdr)); in arc_state_multilist_index_func()
6961 return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) % in arc_state_multilist_index_func()
7521 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr) in l2arc_write_eligible() argument
7531 if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) || in l2arc_write_eligible()
7532 HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr) || in l2arc_write_eligible()
7533 (l2arc_noprefetch && HDR_PREFETCH(hdr))) in l2arc_write_eligible()
7701 arc_buf_hdr_t *head, *hdr, *hdr_prev; in l2arc_write_done() local
7722 for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) { in l2arc_write_done()
7723 hdr_prev = list_prev(buflist, hdr); in l2arc_write_done()
7725 hash_lock = HDR_LOCK(hdr); in l2arc_write_done()
7746 list_insert_after(buflist, hdr, head); in l2arc_write_done()
7766 ASSERT(HDR_HAS_L1HDR(hdr)); in l2arc_write_done()
7772 list_remove(buflist, hdr); in l2arc_write_done()
7773 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR); in l2arc_write_done()
7775 uint64_t psize = HDR_GET_PSIZE(hdr); in l2arc_write_done()
7776 l2arc_hdr_arcstats_decrement(hdr); in l2arc_write_done()
7781 arc_hdr_size(hdr), hdr); in l2arc_write_done()
7788 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_WRITING); in l2arc_write_done()
7873 arc_buf_hdr_t *hdr = cb->l2rcb_hdr; in l2arc_untransform() local
7885 ASSERT(MUTEX_HELD(HDR_LOCK(hdr))); in l2arc_untransform()
7886 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in l2arc_untransform()
7895 abd_t *eabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, in l2arc_untransform()
7903 salt, iv, mac, HDR_GET_PSIZE(hdr), eabd, in l2arc_untransform()
7904 hdr->b_l1hdr.b_pabd, &no_crypt); in l2arc_untransform()
7906 arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr); in l2arc_untransform()
7916 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, in l2arc_untransform()
7917 arc_hdr_size(hdr), hdr); in l2arc_untransform()
7918 hdr->b_l1hdr.b_pabd = eabd; in l2arc_untransform()
7921 arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr); in l2arc_untransform()
7930 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && in l2arc_untransform()
7931 !HDR_COMPRESSION_ENABLED(hdr)) { in l2arc_untransform()
7932 abd_t *cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, in l2arc_untransform()
7934 void *tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr)); in l2arc_untransform()
7936 ret = zio_decompress_data(HDR_GET_COMPRESS(hdr), in l2arc_untransform()
7937 hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr), in l2arc_untransform()
7938 HDR_GET_LSIZE(hdr)); in l2arc_untransform()
7940 abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr)); in l2arc_untransform()
7941 arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr); in l2arc_untransform()
7945 abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr)); in l2arc_untransform()
7946 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, in l2arc_untransform()
7947 arc_hdr_size(hdr), hdr); in l2arc_untransform()
7948 hdr->b_l1hdr.b_pabd = cabd; in l2arc_untransform()
7950 zio->io_size = HDR_GET_LSIZE(hdr); in l2arc_untransform()
7969 arc_buf_hdr_t *hdr; in l2arc_read_done() local
7981 hdr = cb->l2rcb_hdr; in l2arc_read_done()
7982 ASSERT3P(hdr, !=, NULL); in l2arc_read_done()
7984 hash_lock = HDR_LOCK(hdr); in l2arc_read_done()
7986 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in l2arc_read_done()
7993 ASSERT3U(arc_hdr_size(hdr), <, zio->io_size); in l2arc_read_done()
7996 abd_copy(hdr->b_crypt_hdr.b_rabd, in l2arc_read_done()
7997 cb->l2rcb_abd, arc_hdr_size(hdr)); in l2arc_read_done()
7999 abd_copy(hdr->b_l1hdr.b_pabd, in l2arc_read_done()
8000 cb->l2rcb_abd, arc_hdr_size(hdr)); in l2arc_read_done()
8016 zio->io_size = zio->io_orig_size = arc_hdr_size(hdr); in l2arc_read_done()
8019 ASSERT(HDR_HAS_RABD(hdr)); in l2arc_read_done()
8021 hdr->b_crypt_hdr.b_rabd; in l2arc_read_done()
8023 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); in l2arc_read_done()
8024 zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd; in l2arc_read_done()
8033 ASSERT(zio->io_abd == hdr->b_l1hdr.b_pabd || in l2arc_read_done()
8034 (HDR_HAS_RABD(hdr) && zio->io_abd == hdr->b_crypt_hdr.b_rabd)); in l2arc_read_done()
8038 valid_cksum = arc_cksum_is_equal(hdr, zio); in l2arc_read_done()
8049 !HDR_L2_EVICTED(hdr)) { in l2arc_read_done()
8051 zio->io_private = hdr; in l2arc_read_done()
8074 hdr->b_crypt_hdr.b_rabd : hdr->b_l1hdr.b_pabd; in l2arc_read_done()
8080 hdr, zio->io_priority, cb->l2rcb_flags, in l2arc_read_done()
8088 for (struct arc_callback *acb = hdr->b_l1hdr.b_acb; in l2arc_read_done()
8180 arc_buf_hdr_t *hdr, *hdr_prev; in l2arc_evict() local
8270 for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) { in l2arc_evict()
8271 hdr_prev = list_prev(buflist, hdr); in l2arc_evict()
8273 ASSERT(!HDR_EMPTY(hdr)); in l2arc_evict()
8274 hash_lock = HDR_LOCK(hdr); in l2arc_evict()
8295 ASSERT(HDR_HAS_L2HDR(hdr)); in l2arc_evict()
8298 ASSERT(!HDR_L2_WRITING(hdr)); in l2arc_evict()
8299 ASSERT(!HDR_L2_WRITE_HEAD(hdr)); in l2arc_evict()
8301 if (!all && (hdr->b_l2hdr.b_daddr >= dev->l2ad_evict || in l2arc_evict()
8302 hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) { in l2arc_evict()
8311 if (!HDR_HAS_L1HDR(hdr)) { in l2arc_evict()
8312 ASSERT(!HDR_L2_READING(hdr)); in l2arc_evict()
8318 arc_change_state(arc_anon, hdr, hash_lock); in l2arc_evict()
8319 arc_hdr_destroy(hdr); in l2arc_evict()
8321 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only); in l2arc_evict()
8328 if (HDR_L2_READING(hdr)) { in l2arc_evict()
8330 arc_hdr_set_flags(hdr, ARC_FLAG_L2_EVICTED); in l2arc_evict()
8333 arc_hdr_l2hdr_destroy(hdr); in l2arc_evict()
8366 l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize, in l2arc_apply_transforms() argument
8371 abd_t *cabd = NULL, *eabd = NULL, *to_write = hdr->b_l1hdr.b_pabd; in l2arc_apply_transforms()
8372 enum zio_compress compress = HDR_GET_COMPRESS(hdr); in l2arc_apply_transforms()
8373 uint64_t psize = HDR_GET_PSIZE(hdr); in l2arc_apply_transforms()
8374 uint64_t size = arc_hdr_size(hdr); in l2arc_apply_transforms()
8375 boolean_t ismd = HDR_ISTYPE_METADATA(hdr); in l2arc_apply_transforms()
8376 boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS); in l2arc_apply_transforms()
8381 ASSERT((HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && in l2arc_apply_transforms()
8382 !HDR_COMPRESSION_ENABLED(hdr)) || in l2arc_apply_transforms()
8383 HDR_ENCRYPTED(hdr) || HDR_SHARED_DATA(hdr) || psize != asize); in l2arc_apply_transforms()
8391 if (HDR_HAS_RABD(hdr) && asize != psize) { in l2arc_apply_transforms()
8394 abd_copy(to_write, hdr->b_crypt_hdr.b_rabd, psize); in l2arc_apply_transforms()
8400 if ((compress == ZIO_COMPRESS_OFF || HDR_COMPRESSION_ENABLED(hdr)) && in l2arc_apply_transforms()
8401 !HDR_ENCRYPTED(hdr)) { in l2arc_apply_transforms()
8404 abd_copy(to_write, hdr->b_l1hdr.b_pabd, size); in l2arc_apply_transforms()
8410 if (compress != ZIO_COMPRESS_OFF && !HDR_COMPRESSION_ENABLED(hdr)) { in l2arc_apply_transforms()
8415 ASSERT3U(psize, <=, HDR_GET_PSIZE(hdr)); in l2arc_apply_transforms()
8418 psize = HDR_GET_PSIZE(hdr); in l2arc_apply_transforms()
8423 if (HDR_ENCRYPTED(hdr)) { in l2arc_apply_transforms()
8432 ret = spa_keystore_lookup_key(spa, hdr->b_crypt_hdr.b_dsobj, in l2arc_apply_transforms()
8438 hdr->b_crypt_hdr.b_ot, bswap, hdr->b_crypt_hdr.b_salt, in l2arc_apply_transforms()
8439 hdr->b_crypt_hdr.b_iv, mac, psize, to_write, eabd, in l2arc_apply_transforms()
8451 ASSERT0(bcmp(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN)); in l2arc_apply_transforms()
8461 ASSERT3P(to_write, !=, hdr->b_l1hdr.b_pabd); in l2arc_apply_transforms()
8503 arc_buf_hdr_t *hdr, *hdr_prev, *head; in l2arc_write_buffers() local
8544 hdr = multilist_sublist_head(mls); in l2arc_write_buffers()
8546 hdr = multilist_sublist_tail(mls); in l2arc_write_buffers()
8552 for (; hdr; hdr = hdr_prev) { in l2arc_write_buffers()
8557 hdr_prev = multilist_sublist_next(mls, hdr); in l2arc_write_buffers()
8559 hdr_prev = multilist_sublist_prev(mls, hdr); in l2arc_write_buffers()
8561 hash_lock = HDR_LOCK(hdr); in l2arc_write_buffers()
8569 passed_sz += HDR_GET_LSIZE(hdr); in l2arc_write_buffers()
8578 if (!l2arc_write_eligible(guid, hdr)) { in l2arc_write_buffers()
8583 ASSERT(HDR_HAS_L1HDR(hdr)); in l2arc_write_buffers()
8585 ASSERT3U(HDR_GET_PSIZE(hdr), >, 0); in l2arc_write_buffers()
8586 ASSERT3U(arc_hdr_size(hdr), >, 0); in l2arc_write_buffers()
8587 ASSERT(hdr->b_l1hdr.b_pabd != NULL || in l2arc_write_buffers()
8588 HDR_HAS_RABD(hdr)); in l2arc_write_buffers()
8589 uint64_t psize = HDR_GET_PSIZE(hdr); in l2arc_write_buffers()
8605 arc_hdr_set_flags(hdr, ARC_FLAG_L2_WRITING); in l2arc_write_buffers()
8621 if (HDR_HAS_RABD(hdr) && psize == asize) { in l2arc_write_buffers()
8622 to_write = hdr->b_crypt_hdr.b_rabd; in l2arc_write_buffers()
8623 } else if ((HDR_COMPRESSION_ENABLED(hdr) || in l2arc_write_buffers()
8624 HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) && in l2arc_write_buffers()
8625 !HDR_ENCRYPTED(hdr) && !HDR_SHARED_DATA(hdr) && in l2arc_write_buffers()
8627 to_write = hdr->b_l1hdr.b_pabd; in l2arc_write_buffers()
8630 arc_buf_contents_t type = arc_buf_type(hdr); in l2arc_write_buffers()
8632 ret = l2arc_apply_transforms(spa, hdr, asize, in l2arc_write_buffers()
8635 arc_hdr_clear_flags(hdr, in l2arc_write_buffers()
8669 hdr->b_l2hdr.b_dev = dev; in l2arc_write_buffers()
8670 hdr->b_l2hdr.b_daddr = dev->l2ad_hand; in l2arc_write_buffers()
8671 hdr->b_l2hdr.b_arcs_state = in l2arc_write_buffers()
8672 hdr->b_l1hdr.b_state->arcs_state; in l2arc_write_buffers()
8673 arc_hdr_set_flags(hdr, in l2arc_write_buffers()
8677 list_insert_head(&dev->l2ad_buflist, hdr); in l2arc_write_buffers()
8681 arc_hdr_size(hdr), hdr); in l2arc_write_buffers()
8684 hdr->b_l2hdr.b_daddr, asize, to_write, in l2arc_write_buffers()
8685 ZIO_CHECKSUM_OFF, NULL, hdr, in l2arc_write_buffers()
8689 write_lsize += HDR_GET_LSIZE(hdr); in l2arc_write_buffers()
8696 l2arc_hdr_arcstats_increment(hdr); in l2arc_write_buffers()
8706 if (l2arc_log_blk_insert(dev, hdr)) in l2arc_write_buffers()
9658 arc_buf_hdr_t *hdr, *exists; in l2arc_hdr_restore() local
9668 hdr = arc_buf_alloc_l2only(L2BLK_GET_LSIZE((le)->le_prop), type, in l2arc_hdr_restore()
9682 l2arc_hdr_arcstats_increment(hdr); in l2arc_hdr_restore()
9686 list_insert_tail(&dev->l2ad_buflist, hdr); in l2arc_hdr_restore()
9687 (void) zfs_refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr); in l2arc_hdr_restore()
9690 exists = buf_hash_insert(hdr, &hash_lock); in l2arc_hdr_restore()
9693 arc_hdr_destroy(hdr); in l2arc_hdr_restore()
9980 l2arc_log_blk_insert(l2arc_dev_t *dev, const arc_buf_hdr_t *hdr) in l2arc_log_blk_insert() argument
9991 ASSERT(HDR_HAS_L2HDR(hdr)); in l2arc_log_blk_insert()
9995 le->le_dva = hdr->b_dva; in l2arc_log_blk_insert()
9996 le->le_birth = hdr->b_birth; in l2arc_log_blk_insert()
9997 le->le_daddr = hdr->b_l2hdr.b_daddr; in l2arc_log_blk_insert()
10000 L2BLK_SET_LSIZE((le)->le_prop, HDR_GET_LSIZE(hdr)); in l2arc_log_blk_insert()
10001 L2BLK_SET_PSIZE((le)->le_prop, HDR_GET_PSIZE(hdr)); in l2arc_log_blk_insert()
10002 L2BLK_SET_COMPRESS((le)->le_prop, HDR_GET_COMPRESS(hdr)); in l2arc_log_blk_insert()
10003 L2BLK_SET_TYPE((le)->le_prop, hdr->b_type); in l2arc_log_blk_insert()
10004 L2BLK_SET_PROTECTED((le)->le_prop, !!(HDR_PROTECTED(hdr))); in l2arc_log_blk_insert()
10005 L2BLK_SET_PREFETCH((le)->le_prop, !!(HDR_PREFETCH(hdr))); in l2arc_log_blk_insert()
10006 L2BLK_SET_STATE((le)->le_prop, hdr->b_l1hdr.b_state->arcs_state); in l2arc_log_blk_insert()
10009 HDR_GET_PSIZE(hdr)); in l2arc_log_blk_insert()