15cabbc6Prashanth Sreenivasa/*
25cabbc6Prashanth Sreenivasa * CDDL HEADER START
35cabbc6Prashanth Sreenivasa *
45cabbc6Prashanth Sreenivasa * This file and its contents are supplied under the terms of the
55cabbc6Prashanth Sreenivasa * Common Development and Distribution License ("CDDL"), version 1.0.
65cabbc6Prashanth Sreenivasa * You may only use this file in accordance with the terms of version
75cabbc6Prashanth Sreenivasa * 1.0 of the CDDL.
85cabbc6Prashanth Sreenivasa *
95cabbc6Prashanth Sreenivasa * A full copy of the text of the CDDL should have accompanied this
105cabbc6Prashanth Sreenivasa * source.  A copy of the CDDL is also available via the Internet at
115cabbc6Prashanth Sreenivasa * http://www.illumos.org/license/CDDL.
125cabbc6Prashanth Sreenivasa *
135cabbc6Prashanth Sreenivasa * CDDL HEADER END
145cabbc6Prashanth Sreenivasa */
155cabbc6Prashanth Sreenivasa
165cabbc6Prashanth Sreenivasa/*
1717f1128Serapheim Dimitropoulos * Copyright (c) 2015, 2017 by Delphix. All rights reserved.
185cabbc6Prashanth Sreenivasa */
195cabbc6Prashanth Sreenivasa
205cabbc6Prashanth Sreenivasa#include <sys/dmu_tx.h>
215cabbc6Prashanth Sreenivasa#include <sys/dsl_pool.h>
225cabbc6Prashanth Sreenivasa#include <sys/spa.h>
235cabbc6Prashanth Sreenivasa#include <sys/vdev_impl.h>
245cabbc6Prashanth Sreenivasa#include <sys/vdev_indirect_mapping.h>
255cabbc6Prashanth Sreenivasa#include <sys/zfeature.h>
265cabbc6Prashanth Sreenivasa#include <sys/dmu_objset.h>
275cabbc6Prashanth Sreenivasa
285cabbc6Prashanth Sreenivasastatic boolean_t
295cabbc6Prashanth Sreenivasavdev_indirect_mapping_verify(vdev_indirect_mapping_t *vim)
305cabbc6Prashanth Sreenivasa{
315cabbc6Prashanth Sreenivasa	ASSERT(vim != NULL);
325cabbc6Prashanth Sreenivasa
335cabbc6Prashanth Sreenivasa	ASSERT(vim->vim_object != 0);
345cabbc6Prashanth Sreenivasa	ASSERT(vim->vim_objset != NULL);
355cabbc6Prashanth Sreenivasa	ASSERT(vim->vim_phys != NULL);
365cabbc6Prashanth Sreenivasa	ASSERT(vim->vim_dbuf != NULL);
375cabbc6Prashanth Sreenivasa
385cabbc6Prashanth Sreenivasa	EQUIV(vim->vim_phys->vimp_num_entries > 0,
395cabbc6Prashanth Sreenivasa	    vim->vim_entries != NULL);
405cabbc6Prashanth Sreenivasa	if (vim->vim_phys->vimp_num_entries > 0) {
415cabbc6Prashanth Sreenivasa		vdev_indirect_mapping_entry_phys_t *last_entry =
425cabbc6Prashanth Sreenivasa		    &vim->vim_entries[vim->vim_phys->vimp_num_entries - 1];
435cabbc6Prashanth Sreenivasa		uint64_t offset = DVA_MAPPING_GET_SRC_OFFSET(last_entry);
445cabbc6Prashanth Sreenivasa		uint64_t size = DVA_GET_ASIZE(&last_entry->vimep_dst);
455cabbc6Prashanth Sreenivasa
465cabbc6Prashanth Sreenivasa		ASSERT3U(vim->vim_phys->vimp_max_offset, >=, offset + size);
475cabbc6Prashanth Sreenivasa	}
485cabbc6Prashanth Sreenivasa	if (vim->vim_havecounts) {
495cabbc6Prashanth Sreenivasa		ASSERT(vim->vim_phys->vimp_counts_object != 0);
505cabbc6Prashanth Sreenivasa	}
515cabbc6Prashanth Sreenivasa
525cabbc6Prashanth Sreenivasa	return (B_TRUE);
535cabbc6Prashanth Sreenivasa}
545cabbc6Prashanth Sreenivasa
555cabbc6Prashanth Sreenivasauint64_t
565cabbc6Prashanth Sreenivasavdev_indirect_mapping_num_entries(vdev_indirect_mapping_t *vim)
575cabbc6Prashanth Sreenivasa{
585cabbc6Prashanth Sreenivasa	ASSERT(vdev_indirect_mapping_verify(vim));
595cabbc6Prashanth Sreenivasa
605cabbc6Prashanth Sreenivasa	return (vim->vim_phys->vimp_num_entries);
615cabbc6Prashanth Sreenivasa}
625cabbc6Prashanth Sreenivasa
635cabbc6Prashanth Sreenivasauint64_t
645cabbc6Prashanth Sreenivasavdev_indirect_mapping_max_offset(vdev_indirect_mapping_t *vim)
655cabbc6Prashanth Sreenivasa{
665cabbc6Prashanth Sreenivasa	ASSERT(vdev_indirect_mapping_verify(vim));
675cabbc6Prashanth Sreenivasa
685cabbc6Prashanth Sreenivasa	return (vim->vim_phys->vimp_max_offset);
695cabbc6Prashanth Sreenivasa}
705cabbc6Prashanth Sreenivasa
715cabbc6Prashanth Sreenivasauint64_t
725cabbc6Prashanth Sreenivasavdev_indirect_mapping_object(vdev_indirect_mapping_t *vim)
735cabbc6Prashanth Sreenivasa{
745cabbc6Prashanth Sreenivasa	ASSERT(vdev_indirect_mapping_verify(vim));
755cabbc6Prashanth Sreenivasa
765cabbc6Prashanth Sreenivasa	return (vim->vim_object);
775cabbc6Prashanth Sreenivasa}
785cabbc6Prashanth Sreenivasa
795cabbc6Prashanth Sreenivasauint64_t
805cabbc6Prashanth Sreenivasavdev_indirect_mapping_bytes_mapped(vdev_indirect_mapping_t *vim)
815cabbc6Prashanth Sreenivasa{
825cabbc6Prashanth Sreenivasa	ASSERT(vdev_indirect_mapping_verify(vim));
835cabbc6Prashanth Sreenivasa
845cabbc6Prashanth Sreenivasa	return (vim->vim_phys->vimp_bytes_mapped);
855cabbc6Prashanth Sreenivasa}
865cabbc6Prashanth Sreenivasa
875cabbc6Prashanth Sreenivasa/*
885cabbc6Prashanth Sreenivasa * The length (in bytes) of the mapping object array in memory and
895cabbc6Prashanth Sreenivasa * (logically) on disk.
905cabbc6Prashanth Sreenivasa *
915cabbc6Prashanth Sreenivasa * Note that unlike most of our accessor functions,
925cabbc6Prashanth Sreenivasa * we don't assert that the struct is consistent; therefore it can be
935cabbc6Prashanth Sreenivasa * called while there may be concurrent changes, if we don't care about
945cabbc6Prashanth Sreenivasa * the value being immediately stale (e.g. from spa_removal_get_stats()).
955cabbc6Prashanth Sreenivasa */
965cabbc6Prashanth Sreenivasauint64_t
975cabbc6Prashanth Sreenivasavdev_indirect_mapping_size(vdev_indirect_mapping_t *vim)
985cabbc6Prashanth Sreenivasa{
995cabbc6Prashanth Sreenivasa	return (vim->vim_phys->vimp_num_entries * sizeof (*vim->vim_entries));
1005cabbc6Prashanth Sreenivasa}
1015cabbc6Prashanth Sreenivasa
1025cabbc6Prashanth Sreenivasa/*
1035cabbc6Prashanth Sreenivasa * Compare an offset with an indirect mapping entry; there are three
1045cabbc6Prashanth Sreenivasa * possible scenarios:
1055cabbc6Prashanth Sreenivasa *
1065cabbc6Prashanth Sreenivasa *     1. The offset is "less than" the mapping entry; meaning the
1075cabbc6Prashanth Sreenivasa *        offset is less than the source offset of the mapping entry. In
1085cabbc6Prashanth Sreenivasa *        this case, there is no overlap between the offset and the
1095cabbc6Prashanth Sreenivasa *        mapping entry and -1 will be returned.
1105cabbc6Prashanth Sreenivasa *
1115cabbc6Prashanth Sreenivasa *     2. The offset is "greater than" the mapping entry; meaning the
1125cabbc6Prashanth Sreenivasa *        offset is greater than the mapping entry's source offset plus
1135cabbc6Prashanth Sreenivasa *        the entry's size. In this case, there is no overlap between
1145cabbc6Prashanth Sreenivasa *        the offset and the mapping entry and 1 will be returned.
1155cabbc6Prashanth Sreenivasa *
1165cabbc6Prashanth Sreenivasa *        NOTE: If the offset is actually equal to the entry's offset
1175cabbc6Prashanth Sreenivasa *        plus size, this is considered to be "greater" than the entry,
1185cabbc6Prashanth Sreenivasa *        and this case applies (i.e. 1 will be returned). Thus, the
1195cabbc6Prashanth Sreenivasa *        entry's "range" can be considered to be inclusive at its
1205cabbc6Prashanth Sreenivasa *        start, but exclusive at its end: e.g. [src, src + size).
1215cabbc6Prashanth Sreenivasa *
1225cabbc6Prashanth Sreenivasa *     3. The last case to consider is if the offset actually falls
1235cabbc6Prashanth Sreenivasa *        within the mapping entry's range. If this is the case, the
1245cabbc6Prashanth Sreenivasa *        offset is considered to be "equal to" the mapping entry and
1255cabbc6Prashanth Sreenivasa *        0 will be returned.
1265cabbc6Prashanth Sreenivasa *
1275cabbc6Prashanth Sreenivasa *        NOTE: If the offset is equal to the entry's source offset,
1285cabbc6Prashanth Sreenivasa *        this case applies and 0 will be returned. If the offset is
1295cabbc6Prashanth Sreenivasa *        equal to the entry's source plus its size, this case does
1305cabbc6Prashanth Sreenivasa *        *not* apply (see "NOTE" above for scenario 2), and 1 will be
1315cabbc6Prashanth Sreenivasa *        returned.
1325cabbc6Prashanth Sreenivasa */
1335cabbc6Prashanth Sreenivasastatic int
1345cabbc6Prashanth Sreenivasadva_mapping_overlap_compare(const void *v_key, const void *v_array_elem)
1355cabbc6Prashanth Sreenivasa{
136f02c28eToomas Soome	const uint64_t *key = v_key;
137f02c28eToomas Soome	const vdev_indirect_mapping_entry_phys_t *array_elem =
1385cabbc6Prashanth Sreenivasa	    v_array_elem;
1395cabbc6Prashanth Sreenivasa	uint64_t src_offset = DVA_MAPPING_GET_SRC_OFFSET(array_elem);
1405cabbc6Prashanth Sreenivasa
1415cabbc6Prashanth Sreenivasa	if (*key < src_offset) {
1425cabbc6Prashanth Sreenivasa		return (-1);
1435cabbc6Prashanth Sreenivasa	} else if (*key < src_offset + DVA_GET_ASIZE(&array_elem->vimep_dst)) {
1445cabbc6Prashanth Sreenivasa		return (0);
1455cabbc6Prashanth Sreenivasa	} else {
1465cabbc6Prashanth Sreenivasa		return (1);
1475cabbc6Prashanth Sreenivasa	}
1485cabbc6Prashanth Sreenivasa}
1495cabbc6Prashanth Sreenivasa
1505cabbc6Prashanth Sreenivasa/*
1515cabbc6Prashanth Sreenivasa * Returns the mapping entry for the given offset.
1525cabbc6Prashanth Sreenivasa *
1535cabbc6Prashanth Sreenivasa * It's possible that the given offset will not be in the mapping table
1545cabbc6Prashanth Sreenivasa * (i.e. no mapping entries contain this offset), in which case, the
1555cabbc6Prashanth Sreenivasa * return value value depends on the "next_if_missing" parameter.
1565cabbc6Prashanth Sreenivasa *
1575cabbc6Prashanth Sreenivasa * If the offset is not found in the table and "next_if_missing" is
1585cabbc6Prashanth Sreenivasa * B_FALSE, then NULL will always be returned. The behavior is intended
1595cabbc6Prashanth Sreenivasa * to allow consumers to get the entry corresponding to the offset
1605cabbc6Prashanth Sreenivasa * parameter, iff the offset overlaps with an entry in the table.
1615cabbc6Prashanth Sreenivasa *
1625cabbc6Prashanth Sreenivasa * If the offset is not found in the table and "next_if_missing" is
1635cabbc6Prashanth Sreenivasa * B_TRUE, then the entry nearest to the given offset will be returned,
1645cabbc6Prashanth Sreenivasa * such that the entry's source offset is greater than the offset
1655cabbc6Prashanth Sreenivasa * passed in (i.e. the "next" mapping entry in the table is returned, if
1665cabbc6Prashanth Sreenivasa * the offset is missing from the table). If there are no entries whose
1675cabbc6Prashanth Sreenivasa * source offset is greater than the passed in offset, NULL is returned.
1685cabbc6Prashanth Sreenivasa */
1695cabbc6Prashanth Sreenivasastatic vdev_indirect_mapping_entry_phys_t *
1705cabbc6Prashanth Sreenivasavdev_indirect_mapping_entry_for_offset_impl(vdev_indirect_mapping_t *vim,
1715cabbc6Prashanth Sreenivasa    uint64_t offset, boolean_t next_if_missing)
1725cabbc6Prashanth Sreenivasa{
1735cabbc6Prashanth Sreenivasa	ASSERT(vdev_indirect_mapping_verify(vim));
1745cabbc6Prashanth Sreenivasa	ASSERT(vim->vim_phys->vimp_num_entries > 0);
1755cabbc6Prashanth Sreenivasa
1765cabbc6Prashanth Sreenivasa	vdev_indirect_mapping_entry_phys_t *entry = NULL;
1775cabbc6Prashanth Sreenivasa
1785cabbc6Prashanth Sreenivasa	uint64_t last = vim->vim_phys->vimp_num_entries - 1;
1795cabbc6Prashanth Sreenivasa	uint64_t base = 0;
1805cabbc6Prashanth Sreenivasa
1815cabbc6Prashanth Sreenivasa	/*
1825cabbc6Prashanth Sreenivasa	 * We don't define these inside of the while loop because we use
1835cabbc6Prashanth Sreenivasa	 * their value in the case that offset isn't in the mapping.
1845cabbc6Prashanth Sreenivasa	 */
1855cabbc6Prashanth Sreenivasa	uint64_t mid;
1865cabbc6Prashanth Sreenivasa	int result;
1875cabbc6Prashanth Sreenivasa
1885cabbc6Prashanth Sreenivasa	while (last >= base) {
1895cabbc6Prashanth Sreenivasa		mid = base + ((last - base) >> 1);
1905cabbc6Prashanth Sreenivasa
1915cabbc6Prashanth Sreenivasa		result = dva_mapping_overlap_compare(&offset,
1925cabbc6Prashanth Sreenivasa		    &vim->vim_entries[mid]);
1935cabbc6Prashanth Sreenivasa
1945cabbc6Prashanth Sreenivasa		if (result == 0) {
1955cabbc6Prashanth Sreenivasa			entry = &vim->vim_entries[mid];
1965cabbc6Prashanth Sreenivasa			break;
1975cabbc6Prashanth Sreenivasa		} else if (result < 0) {
1985cabbc6Prashanth Sreenivasa			last = mid - 1;
1995cabbc6Prashanth Sreenivasa		} else {
2005cabbc6Prashanth Sreenivasa			base = mid + 1;
2015cabbc6Prashanth Sreenivasa		}
2025cabbc6Prashanth Sreenivasa	}
2035cabbc6Prashanth Sreenivasa
2045cabbc6Prashanth Sreenivasa	if (entry == NULL && next_if_missing) {
2055cabbc6Prashanth Sreenivasa		ASSERT3U(base, ==, last + 1);
2065cabbc6Prashanth Sreenivasa		ASSERT(mid == base || mid == last);
2075cabbc6Prashanth Sreenivasa		ASSERT3S(result, !=, 0);
2085cabbc6Prashanth Sreenivasa
2095cabbc6Prashanth Sreenivasa		/*
2105cabbc6Prashanth Sreenivasa		 * The offset we're looking for isn't actually contained
2115cabbc6Prashanth Sreenivasa		 * in the mapping table, thus we need to return the
2125cabbc6Prashanth Sreenivasa		 * closest mapping entry that is greater than the
2135cabbc6Prashanth Sreenivasa		 * offset. We reuse the result of the last comparison,
2145cabbc6Prashanth Sreenivasa		 * comparing the mapping entry at index "mid" and the
2155cabbc6Prashanth Sreenivasa		 * offset. The offset is guaranteed to lie between
2165cabbc6Prashanth Sreenivasa		 * indices one less than "mid", and one greater than
2175cabbc6Prashanth Sreenivasa		 * "mid"; we just need to determine if offset is greater
2185cabbc6Prashanth Sreenivasa		 * than, or less than the mapping entry contained at
2195cabbc6Prashanth Sreenivasa		 * index "mid".
2205cabbc6Prashanth Sreenivasa		 */
2215cabbc6Prashanth Sreenivasa
2225cabbc6Prashanth Sreenivasa		uint64_t index;
2235cabbc6Prashanth Sreenivasa		if (result < 0)
2245cabbc6Prashanth Sreenivasa			index = mid;
2255cabbc6Prashanth Sreenivasa		else
2265cabbc6Prashanth Sreenivasa			index = mid + 1;
2275cabbc6Prashanth Sreenivasa
2285cabbc6Prashanth Sreenivasa		ASSERT3U(index, <=, vim->vim_phys->vimp_num_entries);
2295cabbc6Prashanth Sreenivasa
2305cabbc6Prashanth Sreenivasa		if (index == vim->vim_phys->vimp_num_entries) {
2315cabbc6Prashanth Sreenivasa			/*
2325cabbc6Prashanth Sreenivasa			 * If "index" is past the end of the entries
2335cabbc6Prashanth Sreenivasa			 * array, then not only is the offset not in the
2345cabbc6Prashanth Sreenivasa			 * mapping table, but it's actually greater than
2355cabbc6Prashanth Sreenivasa			 * all entries in the table. In this case, we
2365cabbc6Prashanth Sreenivasa			 * can't return a mapping entry greater than the
2375cabbc6Prashanth Sreenivasa			 * offset (since none exist), so we return NULL.
2385cabbc6Prashanth Sreenivasa			 */
2395cabbc6Prashanth Sreenivasa
2405cabbc6Prashanth Sreenivasa			ASSERT3S(dva_mapping_overlap_compare(&offset,
2415cabbc6Prashanth Sreenivasa			    &vim->vim_entries[index - 1]), >, 0);
2425cabbc6Prashanth Sreenivasa
2435cabbc6Prashanth Sreenivasa			return (NULL);
2445cabbc6Prashanth Sreenivasa		} else {
2455cabbc6Prashanth Sreenivasa			/*
2465cabbc6Prashanth Sreenivasa			 * Just to be safe, we verify the offset falls
2475cabbc6Prashanth Sreenivasa			 * in between the mapping entries at index and
2485cabbc6Prashanth Sreenivasa			 * one less than index. Since we know the offset
2495cabbc6Prashanth Sreenivasa			 * doesn't overlap an entry, and we're supposed
2505cabbc6Prashanth Sreenivasa			 * to return the entry just greater than the
2515cabbc6Prashanth Sreenivasa			 * offset, both of the following tests must be
2525cabbc6Prashanth Sreenivasa			 * true.
2535cabbc6Prashanth Sreenivasa			 */
2545cabbc6Prashanth Sreenivasa			ASSERT3S(dva_mapping_overlap_compare(&offset,
2555cabbc6Prashanth Sreenivasa			    &vim->vim_entries[index]), <, 0);
2565cabbc6Prashanth Sreenivasa			IMPLY(index >= 1, dva_mapping_overlap_compare(&offset,
2575cabbc6Prashanth Sreenivasa			    &vim->vim_entries[index - 1]) > 0);
2585cabbc6Prashanth Sreenivasa
2595cabbc6Prashanth Sreenivasa			return (&vim->vim_entries[index]);
2605cabbc6Prashanth Sreenivasa		}
2615cabbc6Prashanth Sreenivasa	} else {
2625cabbc6Prashanth Sreenivasa		return (entry);
2635cabbc6Prashanth Sreenivasa	}
2645cabbc6Prashanth Sreenivasa}
2655cabbc6Prashanth Sreenivasa
2665cabbc6Prashanth Sreenivasavdev_indirect_mapping_entry_phys_t *
2675cabbc6Prashanth Sreenivasavdev_indirect_mapping_entry_for_offset(vdev_indirect_mapping_t *vim,
2685cabbc6Prashanth Sreenivasa    uint64_t offset)
2695cabbc6Prashanth Sreenivasa{
2705cabbc6Prashanth Sreenivasa	return (vdev_indirect_mapping_entry_for_offset_impl(vim, offset,
2715cabbc6Prashanth Sreenivasa	    B_FALSE));
2725cabbc6Prashanth Sreenivasa}
2735cabbc6Prashanth Sreenivasa
2745cabbc6Prashanth Sreenivasavdev_indirect_mapping_entry_phys_t *
2755cabbc6Prashanth Sreenivasavdev_indirect_mapping_entry_for_offset_or_next(vdev_indirect_mapping_t *vim,
2765cabbc6Prashanth Sreenivasa    uint64_t offset)
2775cabbc6Prashanth Sreenivasa{
2785cabbc6Prashanth Sreenivasa	return (vdev_indirect_mapping_entry_for_offset_impl(vim, offset,
2795cabbc6Prashanth Sreenivasa	    B_TRUE));
2805cabbc6Prashanth Sreenivasa}
2815cabbc6Prashanth Sreenivasa
2825cabbc6Prashanth Sreenivasavoid
2835cabbc6Prashanth Sreenivasavdev_indirect_mapping_close(vdev_indirect_mapping_t *vim)
2845cabbc6Prashanth Sreenivasa{
2855cabbc6Prashanth Sreenivasa	ASSERT(vdev_indirect_mapping_verify(vim));
2865cabbc6Prashanth Sreenivasa
2875cabbc6Prashanth Sreenivasa	if (vim->vim_phys->vimp_num_entries > 0) {
2885cabbc6Prashanth Sreenivasa		uint64_t map_size = vdev_indirect_mapping_size(vim);
2895cabbc6Prashanth Sreenivasa		kmem_free(vim->vim_entries, map_size);
2905cabbc6Prashanth Sreenivasa		vim->vim_entries = NULL;
2915cabbc6Prashanth Sreenivasa	}
2925cabbc6Prashanth Sreenivasa
2935cabbc6Prashanth Sreenivasa	dmu_buf_rele(vim->vim_dbuf, vim);
2945cabbc6Prashanth Sreenivasa
2955cabbc6Prashanth Sreenivasa	vim->vim_objset = NULL;
2965cabbc6Prashanth Sreenivasa	vim->vim_object = 0;
2975cabbc6Prashanth Sreenivasa	vim->vim_dbuf = NULL;
2985cabbc6Prashanth Sreenivasa	vim->vim_phys = NULL;
2995cabbc6Prashanth Sreenivasa
3005cabbc6Prashanth Sreenivasa	kmem_free(vim, sizeof (*vim));
3015cabbc6Prashanth Sreenivasa}
3025cabbc6Prashanth Sreenivasa
3035cabbc6Prashanth Sreenivasauint64_t
3045cabbc6Prashanth Sreenivasavdev_indirect_mapping_alloc(objset_t *os, dmu_tx_t *tx)
3055cabbc6Prashanth Sreenivasa{
3065cabbc6Prashanth Sreenivasa	uint64_t object;
3075cabbc6Prashanth Sreenivasa	ASSERT(dmu_tx_is_syncing(tx));
3085cabbc6Prashanth Sreenivasa	uint64_t bonus_size = VDEV_INDIRECT_MAPPING_SIZE_V0;
3095cabbc6Prashanth Sreenivasa
3105cabbc6Prashanth Sreenivasa	if (spa_feature_is_enabled(os->os_spa, SPA_FEATURE_OBSOLETE_COUNTS)) {
3115cabbc6Prashanth Sreenivasa		bonus_size = sizeof (vdev_indirect_mapping_phys_t);
3125cabbc6Prashanth Sreenivasa	}
3135cabbc6Prashanth Sreenivasa
3145cabbc6Prashanth Sreenivasa	object = dmu_object_alloc(os,
3155cabbc6Prashanth Sreenivasa	    DMU_OTN_UINT64_METADATA, SPA_OLD_MAXBLOCKSIZE,
3165cabbc6Prashanth Sreenivasa	    DMU_OTN_UINT64_METADATA, bonus_size,
3175cabbc6Prashanth Sreenivasa	    tx);
3185cabbc6Prashanth Sreenivasa
3195cabbc6Prashanth Sreenivasa	if (spa_feature_is_enabled(os->os_spa, SPA_FEATURE_OBSOLETE_COUNTS)) {
3205cabbc6Prashanth Sreenivasa		dmu_buf_t *dbuf;
3215cabbc6Prashanth Sreenivasa		vdev_indirect_mapping_phys_t *vimp;
3225cabbc6Prashanth Sreenivasa
3235cabbc6Prashanth Sreenivasa		VERIFY0(dmu_bonus_hold(os, object, FTAG, &dbuf));
3245cabbc6Prashanth Sreenivasa		dmu_buf_will_dirty(dbuf, tx);
3255cabbc6Prashanth Sreenivasa		vimp = dbuf->db_data;
3265cabbc6Prashanth Sreenivasa		vimp->vimp_counts_object = dmu_object_alloc(os,
3275cabbc6Prashanth Sreenivasa		    DMU_OTN_UINT32_METADATA, SPA_OLD_MAXBLOCKSIZE,
3285cabbc6Prashanth Sreenivasa		    DMU_OT_NONE, 0, tx);
3295cabbc6Prashanth Sreenivasa		spa_feature_incr(os->os_spa, SPA_FEATURE_OBSOLETE_COUNTS, tx);
3305cabbc6Prashanth Sreenivasa		dmu_buf_rele(dbuf, FTAG);
3315cabbc6Prashanth Sreenivasa	}
3325cabbc6Prashanth Sreenivasa
3335cabbc6Prashanth Sreenivasa	return (object);
3345cabbc6Prashanth Sreenivasa}
3355cabbc6Prashanth Sreenivasa
3365cabbc6Prashanth Sreenivasa
3375cabbc6Prashanth Sreenivasavdev_indirect_mapping_t *
3385cabbc6Prashanth Sreenivasavdev_indirect_mapping_open(objset_t *os, uint64_t mapping_object)
3395cabbc6Prashanth Sreenivasa{
3405cabbc6Prashanth Sreenivasa	vdev_indirect_mapping_t *vim = kmem_zalloc(sizeof (*vim), KM_SLEEP);
3415cabbc6Prashanth Sreenivasa	dmu_object_info_t doi;
3425cabbc6Prashanth Sreenivasa	VERIFY0(dmu_object_info(os, mapping_object, &doi));
3435cabbc6Prashanth Sreenivasa
3445cabbc6Prashanth Sreenivasa	vim->vim_objset = os;
3455cabbc6Prashanth Sreenivasa	vim->vim_object = mapping_object;
3465cabbc6Prashanth Sreenivasa
3475cabbc6Prashanth Sreenivasa	VERIFY0(dmu_bonus_hold(os, vim->vim_object, vim,
3485cabbc6Prashanth Sreenivasa	    &vim->vim_dbuf));
3495cabbc6Prashanth Sreenivasa	vim->vim_phys = vim->vim_dbuf->db_data;
3505cabbc6Prashanth Sreenivasa
3515cabbc6Prashanth Sreenivasa	vim->vim_havecounts =
3525cabbc6Prashanth Sreenivasa	    (doi.doi_bonus_size > VDEV_INDIRECT_MAPPING_SIZE_V0);
3535cabbc6Prashanth Sreenivasa
3545cabbc6Prashanth Sreenivasa	if (vim->vim_phys->vimp_num_entries > 0) {
3555cabbc6Prashanth Sreenivasa		uint64_t map_size = vdev_indirect_mapping_size(vim);
3565cabbc6Prashanth Sreenivasa		vim->vim_entries = kmem_alloc(map_size, KM_SLEEP);
3575cabbc6Prashanth Sreenivasa		VERIFY0(dmu_read(os, vim->vim_object, 0, map_size,
3585cabbc6Prashanth Sreenivasa		    vim->vim_entries, DMU_READ_PREFETCH));
3595cabbc6Prashanth Sreenivasa	}
3605cabbc6Prashanth Sreenivasa
3615cabbc6Prashanth Sreenivasa	ASSERT(vdev_indirect_mapping_verify(vim));
3625cabbc6Prashanth Sreenivasa
3635cabbc6Prashanth Sreenivasa	return (vim);
3645cabbc6Prashanth Sreenivasa}
3655cabbc6Prashanth Sreenivasa
3665cabbc6Prashanth Sreenivasavoid
3675cabbc6Prashanth Sreenivasavdev_indirect_mapping_free(objset_t *os, uint64_t object, dmu_tx_t *tx)
3685cabbc6Prashanth Sreenivasa{
3695cabbc6Prashanth Sreenivasa	vdev_indirect_mapping_t *vim = vdev_indirect_mapping_open(os, object);
3705cabbc6Prashanth Sreenivasa	if (vim->vim_havecounts) {
3715cabbc6Prashanth Sreenivasa		VERIFY0(dmu_object_free(os, vim->vim_phys->vimp_counts_object,
3725cabbc6Prashanth Sreenivasa		    tx));
3735cabbc6Prashanth Sreenivasa		spa_feature_decr(os->os_spa, SPA_FEATURE_OBSOLETE_COUNTS, tx);
3745cabbc6Prashanth Sreenivasa	}
3755cabbc6Prashanth Sreenivasa	vdev_indirect_mapping_close(vim);
3765cabbc6Prashanth Sreenivasa
3775cabbc6Prashanth Sreenivasa	VERIFY0(dmu_object_free(os, object, tx));
3785cabbc6Prashanth Sreenivasa}
3795cabbc6Prashanth Sreenivasa
3805cabbc6Prashanth Sreenivasa/*
3815cabbc6Prashanth Sreenivasa * Append the list of vdev_indirect_mapping_entry_t's to the on-disk
3825cabbc6Prashanth Sreenivasa * mapping object.  Also remove the entries from the list and free them.
3835cabbc6Prashanth Sreenivasa * This also implicitly extends the max_offset of the mapping (to the end
3845cabbc6Prashanth Sreenivasa * of the last entry).
3855cabbc6Prashanth Sreenivasa */
3865cabbc6Prashanth Sreenivasavoid
3875cabbc6Prashanth Sreenivasavdev_indirect_mapping_add_entries(vdev_indirect_mapping_t *vim,
3885cabbc6Prashanth Sreenivasa    list_t *list, dmu_tx_t *tx)
3895cabbc6Prashanth Sreenivasa{
3905cabbc6Prashanth Sreenivasa	vdev_indirect_mapping_entry_phys_t *mapbuf;
3915cabbc6Prashanth Sreenivasa	uint64_t old_size;
3925cabbc6Prashanth Sreenivasa	uint32_t *countbuf = NULL;
3935cabbc6Prashanth Sreenivasa	vdev_indirect_mapping_entry_phys_t *old_entries;
3945cabbc6Prashanth Sreenivasa	uint64_t old_count;
3955cabbc6Prashanth Sreenivasa	uint64_t entries_written = 0;
3965cabbc6Prashanth Sreenivasa
3975cabbc6Prashanth Sreenivasa	ASSERT(vdev_indirect_mapping_verify(vim));
3985cabbc6Prashanth Sreenivasa	ASSERT(dmu_tx_is_syncing(tx));
3995cabbc6Prashanth Sreenivasa	ASSERT(dsl_pool_sync_context(dmu_tx_pool(tx)));
4005cabbc6Prashanth Sreenivasa	ASSERT(!list_is_empty(list));
4015cabbc6Prashanth Sreenivasa
4025cabbc6Prashanth Sreenivasa	old_size = vdev_indirect_mapping_size(vim);
4035cabbc6Prashanth Sreenivasa	old_entries = vim->vim_entries;
4045cabbc6Prashanth Sreenivasa	old_count = vim->vim_phys->vimp_num_entries;
4055cabbc6Prashanth Sreenivasa
4065cabbc6Prashanth Sreenivasa	dmu_buf_will_dirty(vim->vim_dbuf, tx);
4075cabbc6Prashanth Sreenivasa
4085cabbc6Prashanth Sreenivasa	mapbuf = zio_buf_alloc(SPA_OLD_MAXBLOCKSIZE);
4095cabbc6Prashanth Sreenivasa	if (vim->vim_havecounts) {
4105cabbc6Prashanth Sreenivasa		countbuf = zio_buf_alloc(SPA_OLD_MAXBLOCKSIZE);
4115cabbc6Prashanth Sreenivasa		ASSERT(spa_feature_is_active(vim->vim_objset->os_spa,
4125cabbc6Prashanth Sreenivasa		    SPA_FEATURE_OBSOLETE_COUNTS));
4135cabbc6Prashanth Sreenivasa	}
4145cabbc6Prashanth Sreenivasa	while (!list_is_empty(list)) {
4155cabbc6Prashanth Sreenivasa		uint64_t i;
4165cabbc6Prashanth Sreenivasa		/*
4175cabbc6Prashanth Sreenivasa		 * Write entries from the list to the
4185cabbc6Prashanth Sreenivasa		 * vdev_im_object in batches of size SPA_OLD_MAXBLOCKSIZE.
4195cabbc6Prashanth Sreenivasa		 */
4205cabbc6Prashanth Sreenivasa		for (i = 0; i < SPA_OLD_MAXBLOCKSIZE / sizeof (*mapbuf); i++) {
4215cabbc6Prashanth Sreenivasa			vdev_indirect_mapping_entry_t *entry =
4225cabbc6Prashanth Sreenivasa			    list_remove_head(list);
4235cabbc6Prashanth Sreenivasa			if (entry == NULL)
4245cabbc6Prashanth Sreenivasa				break;
4255cabbc6Prashanth Sreenivasa
4265cabbc6Prashanth Sreenivasa			uint64_t size =
4275cabbc6Prashanth Sreenivasa			    DVA_GET_ASIZE(&entry->vime_mapping.vimep_dst);
4285cabbc6Prashanth Sreenivasa			uint64_t src_offset =
4295cabbc6Prashanth Sreenivasa			    DVA_MAPPING_GET_SRC_OFFSET(&entry->vime_mapping);
4305cabbc6Prashanth Sreenivasa
4315cabbc6Prashanth Sreenivasa			/*
4325cabbc6Prashanth Sreenivasa			 * We shouldn't be adding an entry which is fully
4335cabbc6Prashanth Sreenivasa			 * obsolete.
4345cabbc6Prashanth Sreenivasa			 */
4355cabbc6Prashanth Sreenivasa			ASSERT3U(entry->vime_obsolete_count, <, size);
4365cabbc6Prashanth Sreenivasa			IMPLY(entry->vime_obsolete_count != 0,
4375cabbc6Prashanth Sreenivasa			    vim->vim_havecounts);
4385cabbc6Prashanth Sreenivasa
4395cabbc6Prashanth Sreenivasa			mapbuf[i] = entry->vime_mapping;
4405cabbc6Prashanth Sreenivasa			if (vim->vim_havecounts)
4415cabbc6Prashanth Sreenivasa				countbuf[i] = entry->vime_obsolete_count;
4425cabbc6Prashanth Sreenivasa
4435cabbc6Prashanth Sreenivasa			vim->vim_phys->vimp_bytes_mapped += size;
4445cabbc6Prashanth Sreenivasa			ASSERT3U(src_offset, >=,
4455cabbc6Prashanth Sreenivasa			    vim->vim_phys->vimp_max_offset);
4465cabbc6Prashanth Sreenivasa			vim->vim_phys->vimp_max_offset = src_offset + size;
4475cabbc6Prashanth Sreenivasa
4485cabbc6Prashanth Sreenivasa			entries_written++;
4495cabbc6Prashanth Sreenivasa
4505cabbc6Prashanth Sreenivasa			kmem_free(entry, sizeof (*entry));
4515cabbc6Prashanth Sreenivasa		}
4525cabbc6Prashanth Sreenivasa		dmu_write(vim->vim_objset, vim->vim_object,
4535cabbc6Prashanth Sreenivasa		    vim->vim_phys->vimp_num_entries * sizeof (*mapbuf),
4545cabbc6Prashanth Sreenivasa		    i * sizeof (*mapbuf),
4555cabbc6Prashanth Sreenivasa		    mapbuf, tx);
4565cabbc6Prashanth Sreenivasa		if (vim->vim_havecounts) {
4575cabbc6Prashanth Sreenivasa			dmu_write(vim->vim_objset,
4585cabbc6Prashanth Sreenivasa			    vim->vim_phys->vimp_counts_object,
4595cabbc6Prashanth Sreenivasa			    vim->vim_phys->vimp_num_entries *
4605cabbc6Prashanth Sreenivasa			    sizeof (*countbuf),
4615cabbc6Prashanth Sreenivasa			    i * sizeof (*countbuf), countbuf, tx);
4625cabbc6Prashanth Sreenivasa		}
4635cabbc6Prashanth Sreenivasa		vim->vim_phys->vimp_num_entries += i;
4645cabbc6Prashanth Sreenivasa	}
4655cabbc6Prashanth Sreenivasa	zio_buf_free(mapbuf, SPA_OLD_MAXBLOCKSIZE);
4665cabbc6Prashanth Sreenivasa	if (vim->vim_havecounts)
4675cabbc6Prashanth Sreenivasa		zio_buf_free(countbuf, SPA_OLD_MAXBLOCKSIZE);
4685cabbc6Prashanth Sreenivasa
4695cabbc6Prashanth Sreenivasa	/*
4705cabbc6Prashanth Sreenivasa	 * Update the entry array to reflect the new entries. First, copy
4715cabbc6Prashanth Sreenivasa	 * over any old entries then read back the new entries we just wrote.
4725cabbc6Prashanth Sreenivasa	 */
4735cabbc6Prashanth Sreenivasa	uint64_t new_size = vdev_indirect_mapping_size(vim);
4745cabbc6Prashanth Sreenivasa	ASSERT3U(new_size, >, old_size);
4755cabbc6Prashanth Sreenivasa	ASSERT3U(new_size - old_size, ==,
4765cabbc6Prashanth Sreenivasa	    entries_written * sizeof (vdev_indirect_mapping_entry_phys_t));
4775cabbc6Prashanth Sreenivasa	vim->vim_entries = kmem_alloc(new_size, KM_SLEEP);
4785cabbc6Prashanth Sreenivasa	if (old_size > 0) {
4795cabbc6Prashanth Sreenivasa		bcopy(old_entries, vim->vim_entries, old_size);
4805cabbc6Prashanth Sreenivasa		kmem_free(old_entries, old_size);
4815cabbc6Prashanth Sreenivasa	}
4825cabbc6Prashanth Sreenivasa	VERIFY0(dmu_read(vim->vim_objset, vim->vim_object, old_size,
4835cabbc6Prashanth Sreenivasa	    new_size - old_size, &vim->vim_entries[old_count],
4845cabbc6Prashanth Sreenivasa	    DMU_READ_PREFETCH));
4855cabbc6Prashanth Sreenivasa
4865cabbc6Prashanth Sreenivasa	zfs_dbgmsg("txg %llu: wrote %llu entries to "
4875cabbc6Prashanth Sreenivasa	    "indirect mapping obj %llu; max offset=0x%llx",
4885cabbc6Prashanth Sreenivasa	    (u_longlong_t)dmu_tx_get_txg(tx),
4895cabbc6Prashanth Sreenivasa	    (u_longlong_t)entries_written,
4905cabbc6Prashanth Sreenivasa	    (u_longlong_t)vim->vim_object,
4915cabbc6Prashanth Sreenivasa	    (u_longlong_t)vim->vim_phys->vimp_max_offset);
4925cabbc6Prashanth Sreenivasa}
4935cabbc6Prashanth Sreenivasa
4945cabbc6Prashanth Sreenivasa/*
4955cabbc6Prashanth Sreenivasa * Increment the relevant counts for the specified offset and length.
4965cabbc6Prashanth Sreenivasa * The counts array must be obtained from
4975cabbc6Prashanth Sreenivasa * vdev_indirect_mapping_load_obsolete_counts().
4985cabbc6Prashanth Sreenivasa */
4995cabbc6Prashanth Sreenivasavoid
5005cabbc6Prashanth Sreenivasavdev_indirect_mapping_increment_obsolete_count(vdev_indirect_mapping_t *vim,
5015cabbc6Prashanth Sreenivasa    uint64_t offset, uint64_t length, uint32_t *counts)
5025cabbc6Prashanth Sreenivasa{
5035cabbc6Prashanth Sreenivasa	vdev_indirect_mapping_entry_phys_t *mapping;
5045cabbc6Prashanth Sreenivasa	uint64_t index;
5055cabbc6Prashanth Sreenivasa
5065cabbc6Prashanth Sreenivasa	mapping = vdev_indirect_mapping_entry_for_offset(vim,  offset);
5075cabbc6Prashanth Sreenivasa
5085cabbc6Prashanth Sreenivasa	ASSERT(length > 0);
5095cabbc6Prashanth Sreenivasa	ASSERT3P(mapping, !=, NULL);
5105cabbc6Prashanth Sreenivasa
5115cabbc6Prashanth Sreenivasa	index = mapping - vim->vim_entries;
5125cabbc6Prashanth Sreenivasa
5135cabbc6Prashanth Sreenivasa	while (length > 0) {
5145cabbc6Prashanth Sreenivasa		ASSERT3U(index, <, vdev_indirect_mapping_num_entries(vim));
5155cabbc6Prashanth Sreenivasa
5165cabbc6Prashanth Sreenivasa		uint64_t size = DVA_GET_ASIZE(&mapping->vimep_dst);
5175cabbc6Prashanth Sreenivasa		uint64_t inner_offset = offset -
5185cabbc6Prashanth Sreenivasa		    DVA_MAPPING_GET_SRC_OFFSET(mapping);
5195cabbc6Prashanth Sreenivasa		VERIFY3U(inner_offset, <, size);
5205cabbc6Prashanth Sreenivasa		uint64_t inner_size = MIN(length, size - inner_offset);
5215cabbc6Prashanth Sreenivasa
5225cabbc6Prashanth Sreenivasa		VERIFY3U(counts[index] + inner_size, <=, size);
5235cabbc6Prashanth Sreenivasa		counts[index] += inner_size;
5245cabbc6Prashanth Sreenivasa
5255cabbc6Prashanth Sreenivasa		offset += inner_size;
5265cabbc6Prashanth Sreenivasa		length -= inner_size;
5275cabbc6Prashanth Sreenivasa		mapping++;
5285cabbc6Prashanth Sreenivasa		index++;
5295cabbc6Prashanth Sreenivasa	}
5305cabbc6Prashanth Sreenivasa}
5315cabbc6Prashanth Sreenivasa
5325cabbc6Prashanth Sreenivasatypedef struct load_obsolete_space_map_arg {
5335cabbc6Prashanth Sreenivasa	vdev_indirect_mapping_t	*losma_vim;
5345cabbc6Prashanth Sreenivasa	uint32_t		*losma_counts;
5355cabbc6Prashanth Sreenivasa} load_obsolete_space_map_arg_t;
5365cabbc6Prashanth Sreenivasa
5375cabbc6Prashanth Sreenivasastatic int
53817f1128Serapheim Dimitropoulosload_obsolete_sm_callback(space_map_entry_t *sme, void *arg)
5395cabbc6Prashanth Sreenivasa{
5405cabbc6Prashanth Sreenivasa	load_obsolete_space_map_arg_t *losma = arg;
54117f1128Serapheim Dimitropoulos	ASSERT3S(sme->sme_type, ==, SM_ALLOC);
5425cabbc6Prashanth Sreenivasa
5435cabbc6Prashanth Sreenivasa	vdev_indirect_mapping_increment_obsolete_count(losma->losma_vim,
54417f1128Serapheim Dimitropoulos	    sme->sme_offset, sme->sme_run, losma->losma_counts);
5455cabbc6Prashanth Sreenivasa
5465cabbc6Prashanth Sreenivasa	return (0);
5475cabbc6Prashanth Sreenivasa}
5485cabbc6Prashanth Sreenivasa
5495cabbc6Prashanth Sreenivasa/*
5505cabbc6Prashanth Sreenivasa * Modify the counts (increment them) based on the spacemap.
5515cabbc6Prashanth Sreenivasa */
5525cabbc6Prashanth Sreenivasavoid
5535cabbc6Prashanth Sreenivasavdev_indirect_mapping_load_obsolete_spacemap(vdev_indirect_mapping_t *vim,
5545cabbc6Prashanth Sreenivasa    uint32_t *counts, space_map_t *obsolete_space_sm)
5555cabbc6Prashanth Sreenivasa{
5565cabbc6Prashanth Sreenivasa	load_obsolete_space_map_arg_t losma;
5575cabbc6Prashanth Sreenivasa	losma.losma_counts = counts;
5585cabbc6Prashanth Sreenivasa	losma.losma_vim = vim;
5595cabbc6Prashanth Sreenivasa	VERIFY0(space_map_iterate(obsolete_space_sm,
560555d674Serapheim Dimitropoulos	    space_map_length(obsolete_space_sm),
5615cabbc6Prashanth Sreenivasa	    load_obsolete_sm_callback, &losma));
5625cabbc6Prashanth Sreenivasa}
5635cabbc6Prashanth Sreenivasa
5645cabbc6Prashanth Sreenivasa/*
5655cabbc6Prashanth Sreenivasa * Read the obsolete counts from disk, returning them in an array.
5665cabbc6Prashanth Sreenivasa */
5675cabbc6Prashanth Sreenivasauint32_t *
5685cabbc6Prashanth Sreenivasavdev_indirect_mapping_load_obsolete_counts(vdev_indirect_mapping_t *vim)
5695cabbc6Prashanth Sreenivasa{
5705cabbc6Prashanth Sreenivasa	ASSERT(vdev_indirect_mapping_verify(vim));
5715cabbc6Prashanth Sreenivasa
5725cabbc6Prashanth Sreenivasa	uint64_t counts_size =
5735cabbc6Prashanth Sreenivasa	    vim->vim_phys->vimp_num_entries * sizeof (uint32_t);
5745cabbc6Prashanth Sreenivasa	uint32_t *counts = kmem_alloc(counts_size, KM_SLEEP);
5755cabbc6Prashanth Sreenivasa	if (vim->vim_havecounts) {
5765cabbc6Prashanth Sreenivasa		VERIFY0(dmu_read(vim->vim_objset,
5775cabbc6Prashanth Sreenivasa		    vim->vim_phys->vimp_counts_object,
5785cabbc6Prashanth Sreenivasa		    0, counts_size,
5795cabbc6Prashanth Sreenivasa		    counts, DMU_READ_PREFETCH));
5805cabbc6Prashanth Sreenivasa	} else {
5815cabbc6Prashanth Sreenivasa		bzero(counts, counts_size);
5825cabbc6Prashanth Sreenivasa	}
5835cabbc6Prashanth Sreenivasa	return (counts);
5845cabbc6Prashanth Sreenivasa}
5855cabbc6Prashanth Sreenivasa
5865cabbc6Prashanth Sreenivasaextern void
5875cabbc6Prashanth Sreenivasavdev_indirect_mapping_free_obsolete_counts(vdev_indirect_mapping_t *vim,
5885cabbc6Prashanth Sreenivasa    uint32_t *counts)
5895cabbc6Prashanth Sreenivasa{
5905cabbc6Prashanth Sreenivasa	ASSERT(vdev_indirect_mapping_verify(vim));
5915cabbc6Prashanth Sreenivasa
5925cabbc6Prashanth Sreenivasa	kmem_free(counts, vim->vim_phys->vimp_num_entries * sizeof (uint32_t));
5935cabbc6Prashanth Sreenivasa}
594