1b24ab67Jeff Bonwick/*
2b24ab67Jeff Bonwick * CDDL HEADER START
3b24ab67Jeff Bonwick *
4b24ab67Jeff Bonwick * The contents of this file are subject to the terms of the
5b24ab67Jeff Bonwick * Common Development and Distribution License (the "License").
6b24ab67Jeff Bonwick * You may not use this file except in compliance with the License.
7b24ab67Jeff Bonwick *
8b24ab67Jeff Bonwick * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9b24ab67Jeff Bonwick * or http://www.opensolaris.org/os/licensing.
10b24ab67Jeff Bonwick * See the License for the specific language governing permissions
11b24ab67Jeff Bonwick * and limitations under the License.
12b24ab67Jeff Bonwick *
13b24ab67Jeff Bonwick * When distributing Covered Code, include this CDDL HEADER in each
14b24ab67Jeff Bonwick * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15b24ab67Jeff Bonwick * If applicable, add the following below this CDDL HEADER, with the
16b24ab67Jeff Bonwick * fields enclosed by brackets "[]" replaced with your own identifying
17b24ab67Jeff Bonwick * information: Portions Copyright [yyyy] [name of copyright owner]
18b24ab67Jeff Bonwick *
19b24ab67Jeff Bonwick * CDDL HEADER END
20b24ab67Jeff Bonwick */
21b24ab67Jeff Bonwick
22b24ab67Jeff Bonwick/*
233f9d6adLin Ling * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
24770499eDan Kimmel * Copyright (c) 2012, 2016 by Delphix. All rights reserved.
25b24ab67Jeff Bonwick */
26b24ab67Jeff Bonwick
27b24ab67Jeff Bonwick#include <sys/zfs_context.h>
28b24ab67Jeff Bonwick#include <sys/spa.h>
29b24ab67Jeff Bonwick#include <sys/spa_impl.h>
30b24ab67Jeff Bonwick#include <sys/zio.h>
31b24ab67Jeff Bonwick#include <sys/ddt.h>
32b24ab67Jeff Bonwick#include <sys/zap.h>
33b24ab67Jeff Bonwick#include <sys/dmu_tx.h>
34b24ab67Jeff Bonwick#include <sys/arc.h>
35bbfd46cJeff Bonwick#include <sys/dsl_pool.h>
36b24ab67Jeff Bonwick#include <sys/zio_checksum.h>
37b24ab67Jeff Bonwick#include <sys/zio_compress.h>
383f9d6adLin Ling#include <sys/dsl_scan.h>
39770499eDan Kimmel#include <sys/abd.h>
40b24ab67Jeff Bonwick
41837b568George Wilson/*
42837b568George Wilson * Enable/disable prefetching of dedup-ed blocks which are going to be freed.
43837b568George Wilson */
44837b568George Wilsonint zfs_dedup_prefetch = 1;
45837b568George Wilson
46b24ab67Jeff Bonwickstatic const ddt_ops_t *ddt_ops[DDT_TYPES] = {
47b24ab67Jeff Bonwick	&ddt_zap_ops,
48b24ab67Jeff Bonwick};
49b24ab67Jeff Bonwick
50b24ab67Jeff Bonwickstatic const char *ddt_class_name[DDT_CLASSES] = {
51b24ab67Jeff Bonwick	"ditto",
52b24ab67Jeff Bonwick	"duplicate",
53b24ab67Jeff Bonwick	"unique",
54b24ab67Jeff Bonwick};
55b24ab67Jeff Bonwick
56b24ab67Jeff Bonwickstatic void
57b24ab67Jeff Bonwickddt_object_create(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
58b24ab67Jeff Bonwick    dmu_tx_t *tx)
59b24ab67Jeff Bonwick{
60b24ab67Jeff Bonwick	spa_t *spa = ddt->ddt_spa;
61b24ab67Jeff Bonwick	objset_t *os = ddt->ddt_os;
62b24ab67Jeff Bonwick	uint64_t *objectp = &ddt->ddt_object[type][class];
6345818eeMatthew Ahrens	boolean_t prehash = zio_checksum_table[ddt->ddt_checksum].ci_flags &
6445818eeMatthew Ahrens	    ZCHECKSUM_FLAG_DEDUP;
65b24ab67Jeff Bonwick	char name[DDT_NAMELEN];
66b24ab67Jeff Bonwick
67b24ab67Jeff Bonwick	ddt_object_name(ddt, type, class, name);
68b24ab67Jeff Bonwick
69b24ab67Jeff Bonwick	ASSERT(*objectp == 0);
70b24ab67Jeff Bonwick	VERIFY(ddt_ops[type]->ddt_op_create(os, objectp, tx, prehash) == 0);
71b24ab67Jeff Bonwick	ASSERT(*objectp != 0);
72b24ab67Jeff Bonwick
73b24ab67Jeff Bonwick	VERIFY(zap_add(os, DMU_POOL_DIRECTORY_OBJECT, name,
74b24ab67Jeff Bonwick	    sizeof (uint64_t), 1, objectp, tx) == 0);
75b24ab67Jeff Bonwick
76b24ab67Jeff Bonwick	VERIFY(zap_add(os, spa->spa_ddt_stat_object, name,
77b24ab67Jeff Bonwick	    sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t),
78b24ab67Jeff Bonwick	    &ddt->ddt_histogram[type][class], tx) == 0);
79b24ab67Jeff Bonwick}
80b24ab67Jeff Bonwick
81b24ab67Jeff Bonwickstatic void
82b24ab67Jeff Bonwickddt_object_destroy(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
83b24ab67Jeff Bonwick    dmu_tx_t *tx)
84b24ab67Jeff Bonwick{
85b24ab67Jeff Bonwick	spa_t *spa = ddt->ddt_spa;
86b24ab67Jeff Bonwick	objset_t *os = ddt->ddt_os;
87b24ab67Jeff Bonwick	uint64_t *objectp = &ddt->ddt_object[type][class];
88b24ab67Jeff Bonwick	char name[DDT_NAMELEN];
89b24ab67Jeff Bonwick
90b24ab67Jeff Bonwick	ddt_object_name(ddt, type, class, name);
91b24ab67Jeff Bonwick
92b24ab67Jeff Bonwick	ASSERT(*objectp != 0);
93b24ab67Jeff Bonwick	ASSERT(ddt_object_count(ddt, type, class) == 0);
94b24ab67Jeff Bonwick	ASSERT(ddt_histogram_empty(&ddt->ddt_histogram[type][class]));
95b24ab67Jeff Bonwick	VERIFY(zap_remove(os, DMU_POOL_DIRECTORY_OBJECT, name, tx) == 0);
96b24ab67Jeff Bonwick	VERIFY(zap_remove(os, spa->spa_ddt_stat_object, name, tx) == 0);
97b24ab67Jeff Bonwick	VERIFY(ddt_ops[type]->ddt_op_destroy(os, *objectp, tx) == 0);
9830f608aGeorge Wilson	bzero(&ddt->ddt_object_stats[type][class], sizeof (ddt_object_t));
99b24ab67Jeff Bonwick
100b24ab67Jeff Bonwick	*objectp = 0;
101b24ab67Jeff Bonwick}
102b24ab67Jeff Bonwick
103b24ab67Jeff Bonwickstatic int
104b24ab67Jeff Bonwickddt_object_load(ddt_t *ddt, enum ddt_type type, enum ddt_class class)
105b24ab67Jeff Bonwick{
10630f608aGeorge Wilson	ddt_object_t *ddo = &ddt->ddt_object_stats[type][class];
10730f608aGeorge Wilson	dmu_object_info_t doi;
108b24ab67Jeff Bonwick	char name[DDT_NAMELEN];
109b24ab67Jeff Bonwick	int error;
110b24ab67Jeff Bonwick
111b24ab67Jeff Bonwick	ddt_object_name(ddt, type, class, name);
112b24ab67Jeff Bonwick
113b24ab67Jeff Bonwick	error = zap_lookup(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, name,
114b24ab67Jeff Bonwick	    sizeof (uint64_t), 1, &ddt->ddt_object[type][class]);
115b24ab67Jeff Bonwick
11643466aaMax Grossman	if (error != 0)
117b24ab67Jeff Bonwick		return (error);
118b24ab67Jeff Bonwick
11943466aaMax Grossman	VERIFY0(zap_lookup(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name,
120b24ab67Jeff Bonwick	    sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t),
12143466aaMax Grossman	    &ddt->ddt_histogram[type][class]));
122b24ab67Jeff Bonwick
12330f608aGeorge Wilson	/*
12430f608aGeorge Wilson	 * Seed the cached statistics.
12530f608aGeorge Wilson	 */
12630f608aGeorge Wilson	VERIFY(ddt_object_info(ddt, type, class, &doi) == 0);
12730f608aGeorge Wilson
12830f608aGeorge Wilson	ddo->ddo_count = ddt_object_count(ddt, type, class);
12930f608aGeorge Wilson	ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9;
13030f608aGeorge Wilson	ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size;
13130f608aGeorge Wilson
13243466aaMax Grossman	return (0);
133b24ab67Jeff Bonwick}
134b24ab67Jeff Bonwick
135b24ab67Jeff Bonwickstatic void
136b24ab67Jeff Bonwickddt_object_sync(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
137b24ab67Jeff Bonwick    dmu_tx_t *tx)
138b24ab67Jeff Bonwick{
13930f608aGeorge Wilson	ddt_object_t *ddo = &ddt->ddt_object_stats[type][class];
14030f608aGeorge Wilson	dmu_object_info_t doi;
141b24ab67Jeff Bonwick	char name[DDT_NAMELEN];
142b24ab67Jeff Bonwick
143b24ab67Jeff Bonwick	ddt_object_name(ddt, type, class, name);
144b24ab67Jeff Bonwick
145b24ab67Jeff Bonwick	VERIFY(zap_update(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name,
146b24ab67Jeff Bonwick	    sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t),
147b24ab67Jeff Bonwick	    &ddt->ddt_histogram[type][class], tx) == 0);
14830f608aGeorge Wilson
14930f608aGeorge Wilson	/*
15030f608aGeorge Wilson	 * Cache DDT statistics; this is the only time they'll change.
15130f608aGeorge Wilson	 */
15230f608aGeorge Wilson	VERIFY(ddt_object_info(ddt, type, class, &doi) == 0);
15330f608aGeorge Wilson
15430f608aGeorge Wilson	ddo->ddo_count = ddt_object_count(ddt, type, class);
15530f608aGeorge Wilson	ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9;
15630f608aGeorge Wilson	ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size;
157b24ab67Jeff Bonwick}
158b24ab67Jeff Bonwick
159b24ab67Jeff Bonwickstatic int
160b24ab67Jeff Bonwickddt_object_lookup(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
161b24ab67Jeff Bonwick    ddt_entry_t *dde)
162b24ab67Jeff Bonwick{
163b24ab67Jeff Bonwick	if (!ddt_object_exists(ddt, type, class))
164be6fd75Matthew Ahrens		return (SET_ERROR(ENOENT));
165b24ab67Jeff Bonwick
166b24ab67Jeff Bonwick	return (ddt_ops[type]->ddt_op_lookup(ddt->ddt_os,
167b24ab67Jeff Bonwick	    ddt->ddt_object[type][class], dde));
168b24ab67Jeff Bonwick}
169b24ab67Jeff Bonwick
170c7cd242George Wilsonstatic void
171c7cd242George Wilsonddt_object_prefetch(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
172c7cd242George Wilson    ddt_entry_t *dde)
173c7cd242George Wilson{
174c7cd242George Wilson	if (!ddt_object_exists(ddt, type, class))
175c7cd242George Wilson		return;
176c7cd242George Wilson
177c7cd242George Wilson	ddt_ops[type]->ddt_op_prefetch(ddt->ddt_os,
178c7cd242George Wilson	    ddt->ddt_object[type][class], dde);
179c7cd242George Wilson}
180c7cd242George Wilson
1813f9d6adLin Lingint
182b24ab67Jeff Bonwickddt_object_update(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
183b24ab67Jeff Bonwick    ddt_entry_t *dde, dmu_tx_t *tx)
184b24ab67Jeff Bonwick{
185b24ab67Jeff Bonwick	ASSERT(ddt_object_exists(ddt, type, class));
186b24ab67Jeff Bonwick
187b24ab67Jeff Bonwick	return (ddt_ops[type]->ddt_op_update(ddt->ddt_os,
188b24ab67Jeff Bonwick	    ddt->ddt_object[type][class], dde, tx));
189b24ab67Jeff Bonwick}
190b24ab67Jeff Bonwick
191b24ab67Jeff Bonwickstatic int
192b24ab67Jeff Bonwickddt_object_remove(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
193b24ab67Jeff Bonwick    ddt_entry_t *dde, dmu_tx_t *tx)
194b24ab67Jeff Bonwick{
195b24ab67Jeff Bonwick	ASSERT(ddt_object_exists(ddt, type, class));
196b24ab67Jeff Bonwick
197b24ab67Jeff Bonwick	return (ddt_ops[type]->ddt_op_remove(ddt->ddt_os,
198b24ab67Jeff Bonwick	    ddt->ddt_object[type][class], dde, tx));
199b24ab67Jeff Bonwick}
200b24ab67Jeff Bonwick
201b24ab67Jeff Bonwickint
202b24ab67Jeff Bonwickddt_object_walk(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
203bbfd46cJeff Bonwick    uint64_t *walk, ddt_entry_t *dde)
204b24ab67Jeff Bonwick{
205b24ab67Jeff Bonwick	ASSERT(ddt_object_exists(ddt, type, class));
206b24ab67Jeff Bonwick
207b24ab67Jeff Bonwick	return (ddt_ops[type]->ddt_op_walk(ddt->ddt_os,
208b24ab67Jeff Bonwick	    ddt->ddt_object[type][class], dde, walk));
209b24ab67Jeff Bonwick}
210b24ab67Jeff Bonwick
211b24ab67Jeff Bonwickuint64_t
212b24ab67Jeff Bonwickddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class class)
213b24ab67Jeff Bonwick{
214b24ab67Jeff Bonwick	ASSERT(ddt_object_exists(ddt, type, class));
215b24ab67Jeff Bonwick
216b24ab67Jeff Bonwick	return (ddt_ops[type]->ddt_op_count(ddt->ddt_os,
217b24ab67Jeff Bonwick	    ddt->ddt_object[type][class]));
218b24ab67Jeff Bonwick}
219b24ab67Jeff Bonwick
220b24ab67Jeff Bonwickint
221b24ab67Jeff Bonwickddt_object_info(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
222b24ab67Jeff Bonwick    dmu_object_info_t *doi)
223b24ab67Jeff Bonwick{
224b24ab67Jeff Bonwick	if (!ddt_object_exists(ddt, type, class))
225be6fd75Matthew Ahrens		return (SET_ERROR(ENOENT));
226b24ab67Jeff Bonwick
227b24ab67Jeff Bonwick	return (dmu_object_info(ddt->ddt_os, ddt->ddt_object[type][class],
228b24ab67Jeff Bonwick	    doi));
229b24ab67Jeff Bonwick}
230b24ab67Jeff Bonwick
231b24ab67Jeff Bonwickboolean_t
232b24ab67Jeff Bonwickddt_object_exists(ddt_t *ddt, enum ddt_type type, enum ddt_class class)
233b24ab67Jeff Bonwick{
234b24ab67Jeff Bonwick	return (!!ddt->ddt_object[type][class]);
235b24ab67Jeff Bonwick}
236b24ab67Jeff Bonwick
237b24ab67Jeff Bonwickvoid
238b24ab67Jeff Bonwickddt_object_name(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
239b24ab67Jeff Bonwick    char *name)
240b24ab67Jeff Bonwick{
241b24ab67Jeff Bonwick	(void) sprintf(name, DMU_POOL_DDT,
242b24ab67Jeff Bonwick	    zio_checksum_table[ddt->ddt_checksum].ci_name,
243b24ab67Jeff Bonwick	    ddt_ops[type]->ddt_op_name, ddt_class_name[class]);
244b24ab67Jeff Bonwick}
245b24ab67Jeff Bonwick
246b24ab67Jeff Bonwickvoid
247b24ab67Jeff Bonwickddt_bp_fill(const ddt_phys_t *ddp, blkptr_t *bp, uint64_t txg)
248b24ab67Jeff Bonwick{
249b24ab67Jeff Bonwick	ASSERT(txg != 0);
250b24ab67Jeff Bonwick
251b24ab67Jeff Bonwick	for (int d = 0; d < SPA_DVAS_PER_BP; d++)
252b24ab67Jeff Bonwick		bp->blk_dva[d] = ddp->ddp_dva[d];
253b24ab67Jeff Bonwick	BP_SET_BIRTH(bp, txg, ddp->ddp_phys_birth);
254b24ab67Jeff Bonwick}
255b24ab67Jeff Bonwick
256eb63303Tom Caputi/*
257eb63303Tom Caputi * The bp created via this function may be used for repairs and scrub, but it
258eb63303Tom Caputi * will be missing the salt / IV required to do a full decrypting read.
259eb63303Tom Caputi */
260b24ab67Jeff Bonwickvoid
261bbfd46cJeff Bonwickddt_bp_create(enum zio_checksum checksum,
262bbfd46cJeff Bonwick    const ddt_key_t *ddk, const ddt_phys_t *ddp, blkptr_t *bp)
263b24ab67Jeff Bonwick{
264b24ab67Jeff Bonwick	BP_ZERO(bp);
265b24ab67Jeff Bonwick
266b24ab67Jeff Bonwick	if (ddp != NULL)
267b24ab67Jeff Bonwick		ddt_bp_fill(ddp, bp, ddp->ddp_phys_birth);
268b24ab67Jeff Bonwick
269b24ab67Jeff Bonwick	bp->blk_cksum = ddk->ddk_cksum;
270b24ab67Jeff Bonwick
271b24ab67Jeff Bonwick	BP_SET_LSIZE(bp, DDK_GET_LSIZE(ddk));
272b24ab67Jeff Bonwick	BP_SET_PSIZE(bp, DDK_GET_PSIZE(ddk));
273b24ab67Jeff Bonwick	BP_SET_COMPRESS(bp, DDK_GET_COMPRESS(ddk));
274eb63303Tom Caputi	BP_SET_CRYPT(bp, DDK_GET_CRYPT(ddk));
275eb63303Tom Caputi	BP_SET_FILL(bp, 1);
276bbfd46cJeff Bonwick	BP_SET_CHECKSUM(bp, checksum);
2773f9d6adLin Ling	BP_SET_TYPE(bp, DMU_OT_DEDUP);
278b24ab67Jeff Bonwick	BP_SET_LEVEL(bp, 0);
279b24ab67Jeff Bonwick	BP_SET_DEDUP(bp, 0);
280b24ab67Jeff Bonwick	BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER);
281b24ab67Jeff Bonwick}
282b24ab67Jeff Bonwick
283b24ab67Jeff Bonwickvoid
284b24ab67Jeff Bonwickddt_key_fill(ddt_key_t *ddk, const blkptr_t *bp)
285b24ab67Jeff Bonwick{
286b24ab67Jeff Bonwick	ddk->ddk_cksum = bp->blk_cksum;
287b24ab67Jeff Bonwick	ddk->ddk_prop = 0;
288b24ab67Jeff Bonwick
289eb63303Tom Caputi	ASSERT(BP_IS_ENCRYPTED(bp) || !BP_USES_CRYPT(bp));
290eb63303Tom Caputi
291b24ab67Jeff Bonwick	DDK_SET_LSIZE(ddk, BP_GET_LSIZE(bp));
292b24ab67Jeff Bonwick	DDK_SET_PSIZE(ddk, BP_GET_PSIZE(bp));
293b24ab67Jeff Bonwick	DDK_SET_COMPRESS(ddk, BP_GET_COMPRESS(bp));
294eb63303Tom Caputi	DDK_SET_CRYPT(ddk, BP_USES_CRYPT(bp));
295b24ab67Jeff Bonwick}
296b24ab67Jeff Bonwick
297b24ab67Jeff Bonwickvoid
298b24ab67Jeff Bonwickddt_phys_fill(ddt_phys_t *ddp, const blkptr_t *bp)
299b24ab67Jeff Bonwick{
300b24ab67Jeff Bonwick	ASSERT(ddp->ddp_phys_birth == 0);
301b24ab67Jeff Bonwick
302b24ab67Jeff Bonwick	for (int d = 0; d < SPA_DVAS_PER_BP; d++)
303b24ab67Jeff Bonwick		ddp->ddp_dva[d] = bp->blk_dva[d];
304b24ab67Jeff Bonwick	ddp->ddp_phys_birth = BP_PHYSICAL_BIRTH(bp);
305b24ab67Jeff Bonwick}
306b24ab67Jeff Bonwick
307b24ab67Jeff Bonwickvoid
308b24ab67Jeff Bonwickddt_phys_clear(ddt_phys_t *ddp)
309b24ab67Jeff Bonwick{
310b24ab67Jeff Bonwick	bzero(ddp, sizeof (*ddp));
311b24ab67Jeff Bonwick}
312b24ab67Jeff Bonwick
313b24ab67Jeff Bonwickvoid
314b24ab67Jeff Bonwickddt_phys_addref(ddt_phys_t *ddp)
315b24ab67Jeff Bonwick{
316b24ab67Jeff Bonwick	ddp->ddp_refcnt++;
317b24ab67Jeff Bonwick}
318b24ab67Jeff Bonwick
319b24ab67Jeff Bonwickvoid
320b24ab67Jeff Bonwickddt_phys_decref(ddt_phys_t *ddp)
321b24ab67Jeff Bonwick{
322b24ab67Jeff Bonwick	ASSERT((int64_t)ddp->ddp_refcnt > 0);
323b24ab67Jeff Bonwick	ddp->ddp_refcnt--;
324b24ab67Jeff Bonwick}
325b24ab67Jeff Bonwick
326b24ab67Jeff Bonwickvoid
327b24ab67Jeff Bonwickddt_phys_free(ddt_t *ddt, ddt_key_t *ddk, ddt_phys_t *ddp, uint64_t txg)
328b24ab67Jeff Bonwick{
329b24ab67Jeff Bonwick	blkptr_t blk;
330b24ab67Jeff Bonwick
331bbfd46cJeff Bonwick	ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk);
332b24ab67Jeff Bonwick	ddt_phys_clear(ddp);
333b24ab67Jeff Bonwick	zio_free(ddt->ddt_spa, txg, &blk);
334b24ab67Jeff Bonwick}
335b24ab67Jeff Bonwick
336b24ab67Jeff Bonwickddt_phys_t *
337b24ab67Jeff Bonwickddt_phys_select(const ddt_entry_t *dde, const blkptr_t *bp)
338b24ab67Jeff Bonwick{
339b24ab67Jeff Bonwick	ddt_phys_t *ddp = (ddt_phys_t *)dde->dde_phys;
340b24ab67Jeff Bonwick
341b24ab67Jeff Bonwick	for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
342b24ab67Jeff Bonwick		if (DVA_EQUAL(BP_IDENTITY(bp), &ddp->ddp_dva[0]) &&
343b24ab67Jeff Bonwick		    BP_PHYSICAL_BIRTH(bp) == ddp->ddp_phys_birth)
344b24ab67Jeff Bonwick			return (ddp);
345b24ab67Jeff Bonwick	}
346b24ab67Jeff Bonwick	return (NULL);
347b24ab67Jeff Bonwick}
348b24ab67Jeff Bonwick
349b24ab67Jeff Bonwickuint64_t
350b24ab67Jeff Bonwickddt_phys_total_refcnt(const ddt_entry_t *dde)
351b24ab67Jeff Bonwick{
352b24ab67Jeff Bonwick	uint64_t refcnt = 0;
353b24ab67Jeff Bonwick
354b24ab67Jeff Bonwick	for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++)
355b24ab67Jeff Bonwick		refcnt += dde->dde_phys[p].ddp_refcnt;
356b24ab67Jeff Bonwick
357b24ab67Jeff Bonwick	return (refcnt);
358b24ab67Jeff Bonwick}
359b24ab67Jeff Bonwick
360b24ab67Jeff Bonwickstatic void
361b24ab67Jeff Bonwickddt_stat_generate(ddt_t *ddt, ddt_entry_t *dde, ddt_stat_t *dds)
362b24ab67Jeff Bonwick{
363b24ab67Jeff Bonwick	spa_t *spa = ddt->ddt_spa;
364b24ab67Jeff Bonwick	ddt_phys_t *ddp = dde->dde_phys;
365b24ab67Jeff Bonwick	ddt_key_t *ddk = &dde->dde_key;
366b24ab67Jeff Bonwick	uint64_t lsize = DDK_GET_LSIZE(ddk);
367b24ab67Jeff Bonwick	uint64_t psize = DDK_GET_PSIZE(ddk);
368b24ab67Jeff Bonwick
369b24ab67Jeff Bonwick	bzero(dds, sizeof (*dds));
370b24ab67Jeff Bonwick
371b24ab67Jeff Bonwick	for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
372b24ab67Jeff Bonwick		uint64_t dsize = 0;
373b24ab67Jeff Bonwick		uint64_t refcnt = ddp->ddp_refcnt;
374b24ab67Jeff Bonwick
375b24ab67Jeff Bonwick		if (ddp->ddp_phys_birth == 0)
376b24ab67Jeff Bonwick			continue;
377b24ab67Jeff Bonwick
378eb63303Tom Caputi		for (int d = 0; d < DDE_GET_NDVAS(dde); d++)
379b24ab67Jeff Bonwick			dsize += dva_get_dsize_sync(spa, &ddp->ddp_dva[d]);
380b24ab67Jeff Bonwick
381b24ab67Jeff Bonwick		dds->dds_blocks += 1;
382b24ab67Jeff Bonwick		dds->dds_lsize += lsize;
383b24ab67Jeff Bonwick		dds->dds_psize += psize;
384b24ab67Jeff Bonwick		dds->dds_dsize += dsize;
385b24ab67Jeff Bonwick
386b24ab67Jeff Bonwick		dds->dds_ref_blocks += refcnt;
387b24ab67Jeff Bonwick		dds->dds_ref_lsize += lsize * refcnt;
388b24ab67Jeff Bonwick		dds->dds_ref_psize += psize * refcnt;
389b24ab67Jeff Bonwick		dds->dds_ref_dsize += dsize * refcnt;
390b24ab67Jeff Bonwick	}
391b24ab67Jeff Bonwick}
392b24ab67Jeff Bonwick
393b24ab67Jeff Bonwickvoid
394b24ab67Jeff Bonwickddt_stat_add(ddt_stat_t *dst, const ddt_stat_t *src, uint64_t neg)
395b24ab67Jeff Bonwick{
396b24ab67Jeff Bonwick	const uint64_t *s = (const uint64_t *)src;
397b24ab67Jeff Bonwick	uint64_t *d = (uint64_t *)dst;
398b24ab67Jeff Bonwick	uint64_t *d_end = (uint64_t *)(dst + 1);
399b24ab67Jeff Bonwick
400b24ab67Jeff Bonwick	ASSERT(neg == 0 || neg == -1ULL);	/* add or subtract */
401b24ab67Jeff Bonwick
402b24ab67Jeff Bonwick	while (d < d_end)
403b24ab67Jeff Bonwick		*d++ += (*s++ ^ neg) - neg;
404b24ab67Jeff Bonwick}
405b24ab67Jeff Bonwick
406b24ab67Jeff Bonwickstatic void
407b24ab67Jeff Bonwickddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg)
408b24ab67Jeff Bonwick{
409b24ab67Jeff Bonwick	ddt_stat_t dds;
410b24ab67Jeff Bonwick	ddt_histogram_t *ddh;
411b24ab67Jeff Bonwick	int bucket;
412b24ab67Jeff Bonwick
413b24ab67Jeff Bonwick	ddt_stat_generate(ddt, dde, &dds);
414b24ab67Jeff Bonwick
415bf16b11Matthew Ahrens	bucket = highbit64(dds.dds_ref_blocks) - 1;
416b24ab67Jeff Bonwick	ASSERT(bucket >= 0);
417b24ab67Jeff Bonwick
418b24ab67Jeff Bonwick	ddh = &ddt->ddt_histogram[dde->dde_type][dde->dde_class];
419b24ab67Jeff Bonwick
420b24ab67Jeff Bonwick	ddt_stat_add(&ddh->ddh_stat[bucket], &dds, neg);
421b24ab67Jeff Bonwick}
422b24ab67Jeff Bonwick
423b24ab67Jeff Bonwickvoid
424b24ab67Jeff Bonwickddt_histogram_add(ddt_histogram_t *dst, const ddt_histogram_t *src)
425b24ab67Jeff Bonwick{
426b24ab67Jeff Bonwick	for (int h = 0; h < 64; h++)
427b24ab67Jeff Bonwick		ddt_stat_add(&dst->ddh_stat[h], &src->ddh_stat[h], 0);
428b24ab67Jeff Bonwick}
429b24ab67Jeff Bonwick
430b24ab67Jeff Bonwickvoid
431b24ab67Jeff Bonwickddt_histogram_stat(ddt_stat_t *dds, const ddt_histogram_t *ddh)
432b24ab67Jeff Bonwick{
433b24ab67Jeff Bonwick	bzero(dds, sizeof (*dds));
434b24ab67Jeff Bonwick
435b24ab67Jeff Bonwick	for (int h = 0; h < 64; h++)
436b24ab67Jeff Bonwick		ddt_stat_add(dds, &ddh->ddh_stat[h], 0);
437b24ab67Jeff Bonwick}
438b24ab67Jeff Bonwick
439b24ab67Jeff Bonwickboolean_t
440b24ab67Jeff Bonwickddt_histogram_empty(const ddt_histogram_t *ddh)
441b24ab67Jeff Bonwick{
442b24ab67Jeff Bonwick	const uint64_t *s = (const uint64_t *)ddh;
443b24ab67Jeff Bonwick	const uint64_t *s_end = (const uint64_t *)(ddh + 1);
444b24ab67Jeff Bonwick
445b24ab67Jeff Bonwick	while (s < s_end)
446b24ab67Jeff Bonwick		if (*s++ != 0)
447b24ab67Jeff Bonwick			return (B_FALSE);
448b24ab67Jeff Bonwick
449b24ab67Jeff Bonwick	return (B_TRUE);
450b24ab67Jeff Bonwick}
451b24ab67Jeff Bonwick
4529eb19f4George Wilsonvoid
45330f608aGeorge Wilsonddt_get_dedup_object_stats(spa_t *spa, ddt_object_t *ddo_total)
454b24ab67Jeff Bonwick{
45530f608aGeorge Wilson	/* Sum the statistics we cached in ddt_object_sync(). */
456b24ab67Jeff Bonwick	for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
457b24ab67Jeff Bonwick		ddt_t *ddt = spa->spa_ddt[c];
458b24ab67Jeff Bonwick		for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
459b24ab67Jeff Bonwick			for (enum ddt_class class = 0; class < DDT_CLASSES;
460b24ab67Jeff Bonwick			    class++) {
46130f608aGeorge Wilson				ddt_object_t *ddo =
46230f608aGeorge Wilson				    &ddt->ddt_object_stats[type][class];
46330f608aGeorge Wilson				ddo_total->ddo_count += ddo->ddo_count;
46430f608aGeorge Wilson				ddo_total->ddo_dspace += ddo->ddo_dspace;
46530f608aGeorge Wilson				ddo_total->ddo_mspace += ddo->ddo_mspace;
4669eb19f4George Wilson			}
4679eb19f4George Wilson		}
4689eb19f4George Wilson	}
46930f608aGeorge Wilson
47030f608aGeorge Wilson	/* ... and compute the averages. */
47130f608aGeorge Wilson	if (ddo_total->ddo_count != 0) {
47230f608aGeorge Wilson		ddo_total->ddo_dspace /= ddo_total->ddo_count;
47330f608aGeorge Wilson		ddo_total->ddo_mspace /= ddo_total->ddo_count;
47430f608aGeorge Wilson	}
4759eb19f4George Wilson}
4769eb19f4George Wilson
4779eb19f4George Wilsonvoid
4789eb19f4George Wilsonddt_get_dedup_histogram(spa_t *spa, ddt_histogram_t *ddh)
4799eb19f4George Wilson{
4809eb19f4George Wilson	for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
4819eb19f4George Wilson		ddt_t *ddt = spa->spa_ddt[c];
4829eb19f4George Wilson		for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
4839eb19f4George Wilson			for (enum ddt_class class = 0; class < DDT_CLASSES;
4849eb19f4George Wilson			    class++) {
4859eb19f4George Wilson				ddt_histogram_add(ddh,
48630f608aGeorge Wilson				    &ddt->ddt_histogram_cache[type][class]);
487b24ab67Jeff Bonwick			}
488b24ab67Jeff Bonwick		}
489b24ab67Jeff Bonwick	}
4909eb19f4George Wilson}
4919eb19f4George Wilson
4929eb19f4George Wilsonvoid
4939eb19f4George Wilsonddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total)
4949eb19f4George Wilson{
4959eb19f4George Wilson	ddt_histogram_t *ddh_total;
496b24ab67Jeff Bonwick
4979eb19f4George Wilson	ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP);
4989eb19f4George Wilson	ddt_get_dedup_histogram(spa, ddh_total);
4999eb19f4George Wilson	ddt_histogram_stat(dds_total, ddh_total);
5009eb19f4George Wilson	kmem_free(ddh_total, sizeof (ddt_histogram_t));
501485bbbfGeorge Wilson}
502485bbbfGeorge Wilson
503485bbbfGeorge Wilsonuint64_t
504485bbbfGeorge Wilsonddt_get_dedup_dspace(spa_t *spa)
505485bbbfGeorge Wilson{
506485bbbfGeorge Wilson	ddt_stat_t dds_total = { 0 };
507485bbbfGeorge Wilson
508485bbbfGeorge Wilson	ddt_get_dedup_stats(spa, &dds_total);
509485bbbfGeorge Wilson	return (dds_total.dds_ref_dsize - dds_total.dds_dsize);
510485bbbfGeorge Wilson}
511485bbbfGeorge Wilson
512485bbbfGeorge Wilsonuint64_t
513485bbbfGeorge Wilsonddt_get_pool_dedup_ratio(spa_t *spa)
514485bbbfGeorge Wilson{
515485bbbfGeorge Wilson	ddt_stat_t dds_total = { 0 };
516b24ab67Jeff Bonwick
517485bbbfGeorge Wilson	ddt_get_dedup_stats(spa, &dds_total);
518b24ab67Jeff Bonwick	if (dds_total.dds_dsize == 0)
519b24ab67Jeff Bonwick		return (100);
520b24ab67Jeff Bonwick
521b24ab67Jeff Bonwick	return (dds_total.dds_ref_dsize * 100 / dds_total.dds_dsize);
522b24ab67Jeff Bonwick}
523b24ab67Jeff Bonwick
524b24ab67Jeff Bonwickint
525b24ab67Jeff Bonwickddt_ditto_copies_needed(ddt_t *ddt, ddt_entry_t *dde, ddt_phys_t *ddp_willref)
526b24ab67Jeff Bonwick{
527b24ab67Jeff Bonwick	spa_t *spa = ddt->ddt_spa;
528b24ab67Jeff Bonwick	uint64_t total_refcnt = 0;
529b24ab67Jeff Bonwick	uint64_t ditto = spa->spa_dedup_ditto;
530b24ab67Jeff Bonwick	int total_copies = 0;
531b24ab67Jeff Bonwick	int desired_copies = 0;
532eb63303Tom Caputi	int copies_needed = 0;
533b24ab67Jeff Bonwick
534b24ab67Jeff Bonwick	for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) {
535b24ab67Jeff Bonwick		ddt_phys_t *ddp = &dde->dde_phys[p];
536b24ab67Jeff Bonwick		zio_t *zio = dde->dde_lead_zio[p];
537b24ab67Jeff Bonwick		uint64_t refcnt = ddp->ddp_refcnt;	/* committed refs */
538b24ab67Jeff Bonwick		if (zio != NULL)
539b24ab67Jeff Bonwick			refcnt += zio->io_parent_count;	/* pending refs */
540b24ab67Jeff Bonwick		if (ddp == ddp_willref)
541b24ab67Jeff Bonwick			refcnt++;			/* caller's ref */
542b24ab67Jeff Bonwick		if (refcnt != 0) {
543b24ab67Jeff Bonwick			total_refcnt += refcnt;
544b24ab67Jeff Bonwick			total_copies += p;
545b24ab67Jeff Bonwick		}
546b24ab67Jeff Bonwick	}
547b24ab67Jeff Bonwick
548b24ab67Jeff Bonwick	if (ditto == 0 || ditto > UINT32_MAX)
549b24ab67Jeff Bonwick		ditto = UINT32_MAX;
550b24ab67Jeff Bonwick
551b24ab67Jeff Bonwick	if (total_refcnt >= 1)
552b24ab67Jeff Bonwick		desired_copies++;
553b24ab67Jeff Bonwick	if (total_refcnt >= ditto)
554b24ab67Jeff Bonwick		desired_copies++;
555b24ab67Jeff Bonwick	if (total_refcnt >= ditto * ditto)
556b24ab67Jeff Bonwick		desired_copies++;
557b24ab67Jeff Bonwick
558eb63303Tom Caputi	copies_needed = MAX(desired_copies, total_copies) - total_copies;
559eb63303Tom Caputi
560eb63303Tom Caputi	/* encrypted blocks store their IV in DVA[2] */
561eb63303Tom Caputi	if (DDK_GET_CRYPT(&dde->dde_key))
562eb63303Tom Caputi		copies_needed = MIN(copies_needed, SPA_DVAS_PER_BP - 1);
563eb63303Tom Caputi
564eb63303Tom Caputi	return (copies_needed);
565b24ab67Jeff Bonwick}
566b24ab67Jeff Bonwick
567b24ab67Jeff Bonwickint
568b24ab67Jeff Bonwickddt_ditto_copies_present(ddt_entry_t *dde)
569b24ab67Jeff Bonwick{
570b24ab67Jeff Bonwick	ddt_phys_t *ddp = &dde->dde_phys[DDT_PHYS_DITTO];
571b24ab67Jeff Bonwick	dva_t *dva = ddp->ddp_dva;
572b24ab67Jeff Bonwick	int copies = 0 - DVA_GET_GANG(dva);
573b24ab67Jeff Bonwick
574eb63303Tom Caputi	for (int d = 0; d < DDE_GET_NDVAS(dde); d++, dva++)
575b24ab67Jeff Bonwick		if (DVA_IS_VALID(dva))
576b24ab67Jeff Bonwick			copies++;
577b24ab67Jeff Bonwick
578b24ab67Jeff Bonwick	ASSERT(copies >= 0 && copies < SPA_DVAS_PER_BP);
579b24ab67Jeff Bonwick
580b24ab67Jeff Bonwick	return (copies);
581b24ab67Jeff Bonwick}
582b24ab67Jeff Bonwick
583b24ab67Jeff Bonwicksize_t
584b24ab67Jeff Bonwickddt_compress(void *src, uchar_t *dst, size_t s_len, size_t d_len)
585b24ab67Jeff Bonwick{
586b24ab67Jeff Bonwick	uchar_t *version = dst++;
587b24ab67Jeff Bonwick	int cpfunc = ZIO_COMPRESS_ZLE;
588b24ab67Jeff Bonwick	zio_compress_info_t *ci = &zio_compress_table[cpfunc];
589b24ab67Jeff Bonwick	size_t c_len;
590b24ab67Jeff Bonwick
591b24ab67Jeff Bonwick	ASSERT(d_len >= s_len + 1);	/* no compression plus version byte */
592b24ab67Jeff Bonwick
593b24ab67Jeff Bonwick	c_len = ci->ci_compress(src, dst, s_len, d_len - 1, ci->ci_level);
594b24ab67Jeff Bonwick
595b24ab67Jeff Bonwick	if (c_len == s_len) {
596b24ab67Jeff Bonwick		cpfunc = ZIO_COMPRESS_OFF;
597b24ab67Jeff Bonwick		bcopy(src, dst, s_len);
598b24ab67Jeff Bonwick	}
599b24ab67Jeff Bonwick
60043466aaMax Grossman	*version = cpfunc;
60143466aaMax Grossman	/* CONSTCOND */
60243466aaMax Grossman	if (ZFS_HOST_BYTEORDER)
60343466aaMax Grossman		*version |= DDT_COMPRESS_BYTEORDER_MASK;
604b24ab67Jeff Bonwick
605b24ab67Jeff Bonwick	return (c_len + 1);
606b24ab67Jeff Bonwick}
607b24ab67Jeff Bonwick
608b24ab67Jeff Bonwickvoid
609b24ab67Jeff Bonwickddt_decompress(uchar_t *src, void *dst, size_t s_len, size_t d_len)
610b24ab67Jeff Bonwick{
611b24ab67Jeff Bonwick	uchar_t version = *src++;
612b24ab67Jeff Bonwick	int cpfunc = version & DDT_COMPRESS_FUNCTION_MASK;
613b24ab67Jeff Bonwick	zio_compress_info_t *ci = &zio_compress_table[cpfunc];
614b24ab67Jeff Bonwick
615b24ab67Jeff Bonwick	if (ci->ci_decompress != NULL)
616b24ab67Jeff Bonwick		(void) ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level);
617b24ab67Jeff Bonwick	else
618b24ab67Jeff Bonwick		bcopy(src, dst, d_len);
619b24ab67Jeff Bonwick
62043466aaMax Grossman	if (((version & DDT_COMPRESS_BYTEORDER_MASK) != 0) !=
62143466aaMax Grossman	    (ZFS_HOST_BYTEORDER != 0))
622b24ab67Jeff Bonwick		byteswap_uint64_array(dst, d_len);
623b24ab67Jeff Bonwick}
624b24ab67Jeff Bonwick
625b24ab67Jeff Bonwickddt_t *
626b24ab67Jeff Bonwickddt_select_by_checksum(spa_t *spa, enum zio_checksum c)
627b24ab67Jeff Bonwick{
628b24ab67Jeff Bonwick	return (spa->spa_ddt[c]);
629b24ab67Jeff Bonwick}
630b24ab67Jeff Bonwick
631b24ab67Jeff Bonwickddt_t *
632b24ab67Jeff Bonwickddt_select(spa_t *spa, const blkptr_t *bp)
633b24ab67Jeff Bonwick{
634b24ab67Jeff Bonwick	return (spa->spa_ddt[BP_GET_CHECKSUM(bp)]);
635b24ab67Jeff Bonwick}
636b24ab67Jeff Bonwick
637b24ab67Jeff Bonwickvoid
638b24ab67Jeff Bonwickddt_enter(ddt_t *ddt)
639b24ab67Jeff Bonwick{
640b24ab67Jeff Bonwick	mutex_enter(&ddt->ddt_lock);
641b24ab67Jeff Bonwick}
642b24ab67Jeff Bonwick
643b24ab67Jeff Bonwickvoid
644b24ab67Jeff Bonwickddt_exit(ddt_t *ddt)
645b24ab67Jeff Bonwick{
646b24ab67Jeff Bonwick	mutex_exit(&ddt->ddt_lock);
647b24ab67Jeff Bonwick}
648b24ab67Jeff Bonwick
649b24ab67Jeff Bonwickstatic ddt_entry_t *
650b24ab67Jeff Bonwickddt_alloc(const ddt_key_t *ddk)
651b24ab67Jeff Bonwick{
652b24ab67Jeff Bonwick	ddt_entry_t *dde;
653b24ab67Jeff Bonwick
654b24ab67Jeff Bonwick	dde = kmem_zalloc(sizeof (ddt_entry_t), KM_SLEEP);
655b24ab67Jeff Bonwick	cv_init(&dde->dde_cv, NULL, CV_DEFAULT, NULL);
656b24ab67Jeff Bonwick
657b24ab67Jeff Bonwick	dde->dde_key = *ddk;
658b24ab67Jeff Bonwick
659b24ab67Jeff Bonwick	return (dde);
660b24ab67Jeff Bonwick}
661b24ab67Jeff Bonwick
662b24ab67Jeff Bonwickstatic void
663b24ab67Jeff Bonwickddt_free(ddt_entry_t *dde)
664b24ab67Jeff Bonwick{
665b24ab67Jeff Bonwick	ASSERT(!dde->dde_loading);
666b24ab67Jeff Bonwick
667b24ab67Jeff Bonwick	for (int p = 0; p < DDT_PHYS_TYPES; p++)
668b24ab67Jeff Bonwick		ASSERT(dde->dde_lead_zio[p] == NULL);
669b24ab67Jeff Bonwick
670770499eDan Kimmel	if (dde->dde_repair_abd != NULL)
671770499eDan Kimmel		abd_free(dde->dde_repair_abd);
672b24ab67Jeff Bonwick
673b24ab67Jeff Bonwick	cv_destroy(&dde->dde_cv);
674b24ab67Jeff Bonwick	kmem_free(dde, sizeof (*dde));
675b24ab67Jeff Bonwick}
676b24ab67Jeff Bonwick
677b24ab67Jeff Bonwickvoid
678b24ab67Jeff Bonwickddt_remove(ddt_t *ddt, ddt_entry_t *dde)
679b24ab67Jeff Bonwick{
680b24ab67Jeff Bonwick	ASSERT(MUTEX_HELD(&ddt->ddt_lock));
681b24ab67Jeff Bonwick
682b24ab67Jeff Bonwick	avl_remove(&ddt->ddt_tree, dde);
683b24ab67Jeff Bonwick	ddt_free(dde);
684b24ab67Jeff Bonwick}
685b24ab67Jeff Bonwick
686b24ab67Jeff Bonwickddt_entry_t *
687b24ab67Jeff Bonwickddt_lookup(ddt_t *ddt, const blkptr_t *bp, boolean_t add)
688b24ab67Jeff Bonwick{
689b24ab67Jeff Bonwick	ddt_entry_t *dde, dde_search;
690b24ab67Jeff Bonwick	enum ddt_type type;
691b24ab67Jeff Bonwick	enum ddt_class class;
692b24ab67Jeff Bonwick	avl_index_t where;
693b24ab67Jeff Bonwick	int error;
694b24ab67Jeff Bonwick
695b24ab67Jeff Bonwick	ASSERT(MUTEX_HELD(&ddt->ddt_lock));
696b24ab67Jeff Bonwick
697b24ab67Jeff Bonwick	ddt_key_fill(&dde_search.dde_key, bp);
698b24ab67Jeff Bonwick
699b24ab67Jeff Bonwick	dde = avl_find(&ddt->ddt_tree, &dde_search, &where);
700b24ab67Jeff Bonwick	if (dde == NULL) {
701b24ab67Jeff Bonwick		if (!add)
702b24ab67Jeff Bonwick			return (NULL);
703b24ab67Jeff Bonwick		dde = ddt_alloc(&dde_search.dde_key);
704b24ab67Jeff Bonwick		avl_insert(&ddt->ddt_tree, dde, where);
705b24ab67Jeff Bonwick	}
706b24ab67Jeff Bonwick
707b24ab67Jeff Bonwick	while (dde->dde_loading)
708b24ab67Jeff Bonwick		cv_wait(&dde->dde_cv, &ddt->ddt_lock);
709b24ab67Jeff Bonwick
710b24ab67Jeff Bonwick	if (dde->dde_loaded)
711b24ab67Jeff Bonwick		return (dde);
712b24ab67Jeff Bonwick
713b24ab67Jeff Bonwick	dde->dde_loading = B_TRUE;
714b24ab67Jeff Bonwick
715b24ab67Jeff Bonwick	ddt_exit(ddt);
716b24ab67Jeff Bonwick
717b24ab67Jeff Bonwick	error = ENOENT;
718b24ab67Jeff Bonwick
719b24ab67Jeff Bonwick	for (type = 0; type < DDT_TYPES; type++) {
720b24ab67Jeff Bonwick		for (class = 0; class < DDT_CLASSES; class++) {
721b24ab67Jeff Bonwick			error = ddt_object_lookup(ddt, type, class, dde);
7225cabbc6Prashanth Sreenivasa			if (error != ENOENT) {
7235cabbc6Prashanth Sreenivasa				ASSERT0(error);
724b24ab67Jeff Bonwick				break;
7255cabbc6Prashanth Sreenivasa			}
726b24ab67Jeff Bonwick		}
727b24ab67Jeff Bonwick		if (error != ENOENT)
728b24ab67Jeff Bonwick			break;
729b24ab67Jeff Bonwick	}
730b24ab67Jeff Bonwick
731b24ab67Jeff Bonwick	ddt_enter(ddt);
732b24ab67Jeff Bonwick
733b24ab67Jeff Bonwick	ASSERT(dde->dde_loaded == B_FALSE);
734b24ab67Jeff Bonwick	ASSERT(dde->dde_loading == B_TRUE);
735b24ab67Jeff Bonwick
736b24ab67Jeff Bonwick	dde->dde_type = type;	/* will be DDT_TYPES if no entry found */
737b24ab67Jeff Bonwick	dde->dde_class = class;	/* will be DDT_CLASSES if no entry found */
738b24ab67Jeff Bonwick	dde->dde_loaded = B_TRUE;
739b24ab67Jeff Bonwick	dde->dde_loading = B_FALSE;
740b24ab67Jeff Bonwick
741b24ab67Jeff Bonwick	if (error == 0)
742b24ab67Jeff Bonwick		ddt_stat_update(ddt, dde, -1ULL);
743b24ab67Jeff Bonwick
744b24ab67Jeff Bonwick	cv_broadcast(&dde->dde_cv);
745b24ab67Jeff Bonwick
746b24ab67Jeff Bonwick	return (dde);
747b24ab67Jeff Bonwick}
748b24ab67Jeff Bonwick
749c7cd242George Wilsonvoid
750c7cd242George Wilsonddt_prefetch(spa_t *spa, const blkptr_t *bp)
751c7cd242George Wilson{
752c7cd242George Wilson	ddt_t *ddt;
753c7cd242George Wilson	ddt_entry_t dde;
754c7cd242George Wilson
755837b568George Wilson	if (!zfs_dedup_prefetch || bp == NULL || !BP_GET_DEDUP(bp))
756c7cd242George Wilson		return;
757c7cd242George Wilson
758c7cd242George Wilson	/*
759ce053c9George Wilson	 * We only remove the DDT once all tables are empty and only
760ce053c9George Wilson	 * prefetch dedup blocks when there are entries in the DDT.
761ce053c9George Wilson	 * Thus no locking is required as the DDT can't disappear on us.
762c7cd242George Wilson	 */
763c7cd242George Wilson	ddt = ddt_select(spa, bp);
764c7cd242George Wilson	ddt_key_fill(&dde.dde_key, bp);
765c7cd242George Wilson
766c7cd242George Wilson	for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
767c7cd242George Wilson		for (enum ddt_class class = 0; class < DDT_CLASSES; class++) {
768c7cd242George Wilson			ddt_object_prefetch(ddt, type, class, &dde);
769c7cd242George Wilson		}
770c7cd242George Wilson	}
771c7cd242George Wilson}
772c7cd242George Wilson
773c4ab0d3Gvozden Neskovic/*
774c4ab0d3Gvozden Neskovic * Opaque struct used for ddt_key comparison
775c4ab0d3Gvozden Neskovic */
776c4ab0d3Gvozden Neskovic#define	DDT_KEY_CMP_LEN	(sizeof (ddt_key_t) / sizeof (uint16_t))
777c4ab0d3Gvozden Neskovic
778c4ab0d3Gvozden Neskovictypedef struct ddt_key_cmp {
779c4ab0d3Gvozden Neskovic	uint16_t	u16[DDT_KEY_CMP_LEN];
780c4ab0d3Gvozden Neskovic} ddt_key_cmp_t;
781c4ab0d3Gvozden Neskovic
782b24ab67Jeff Bonwickint
783b24ab67Jeff Bonwickddt_entry_compare(const void *x1, const void *x2)
784b24ab67Jeff Bonwick{
785b24ab67Jeff Bonwick	const ddt_entry_t *dde1 = x1;
786b24ab67Jeff Bonwick	const ddt_entry_t *dde2 = x2;
787c4ab0d3Gvozden Neskovic	const ddt_key_cmp_t *k1 = (const ddt_key_cmp_t *)&dde1->dde_key;
788c4ab0d3Gvozden Neskovic	const ddt_key_cmp_t *k2 = (const ddt_key_cmp_t *)&dde2->dde_key;
789c4ab0d3Gvozden Neskovic	int32_t cmp = 0;
790c4ab0d3Gvozden Neskovic
791c4ab0d3Gvozden Neskovic	for (int i = 0; i < DDT_KEY_CMP_LEN; i++) {
792c4ab0d3Gvozden Neskovic		cmp = (int32_t)k1->u16[i] - (int32_t)k2->u16[i];
793c4ab0d3Gvozden Neskovic		if (likely(cmp))
794c4ab0d3Gvozden Neskovic			break;
795b24ab67Jeff Bonwick	}
796b24ab67Jeff Bonwick
797c4ab0d3Gvozden Neskovic	return (AVL_ISIGN(cmp));
798b24ab67Jeff Bonwick}
799b24ab67Jeff Bonwick
800b24ab67Jeff Bonwickstatic ddt_t *
801b24ab67Jeff Bonwickddt_table_alloc(spa_t *spa, enum zio_checksum c)
802b24ab67Jeff Bonwick{
803b24ab67Jeff Bonwick	ddt_t *ddt;
804b24ab67Jeff Bonwick
805b24ab67Jeff Bonwick	ddt = kmem_zalloc(sizeof (*ddt), KM_SLEEP);
806b24ab67Jeff Bonwick
807b24ab67Jeff Bonwick	mutex_init(&ddt->ddt_lock, NULL, MUTEX_DEFAULT, NULL);
808b24ab67Jeff Bonwick	avl_create(&ddt->ddt_tree, ddt_entry_compare,
809b24ab67Jeff Bonwick	    sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node));
810b24ab67Jeff Bonwick	avl_create(&ddt->ddt_repair_tree, ddt_entry_compare,
811b24ab67Jeff Bonwick	    sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node));
812b24ab67Jeff Bonwick	ddt->ddt_checksum = c;
813b24ab67Jeff Bonwick	ddt->ddt_spa = spa;
814b24ab67Jeff Bonwick	ddt->ddt_os = spa->spa_meta_objset;
815b24ab67Jeff Bonwick
816b24ab67Jeff Bonwick	return (ddt);
817b24ab67Jeff Bonwick}
818b24ab67Jeff Bonwick
819b24ab67Jeff Bonwickstatic void
820b24ab67Jeff Bonwickddt_table_free(ddt_t *ddt)
821b24ab67Jeff Bonwick{
822b24ab67Jeff Bonwick	ASSERT(avl_numnodes(&ddt->ddt_tree) == 0);
823b24ab67Jeff Bonwick	ASSERT(avl_numnodes(&ddt->ddt_repair_tree) == 0);
824b24ab67Jeff Bonwick	avl_destroy(&ddt->ddt_tree);
825b24ab67Jeff Bonwick	avl_destroy(&ddt->ddt_repair_tree);
826b24ab67Jeff Bonwick	mutex_destroy(&ddt->ddt_lock);
827b24ab67Jeff Bonwick	kmem_free(ddt, sizeof (*ddt));
828b24ab67Jeff Bonwick}
829b24ab67Jeff Bonwick
830b24ab67Jeff Bonwickvoid
831b24ab67Jeff Bonwickddt_create(spa_t *spa)
832b24ab67Jeff Bonwick{
833b24ab67Jeff Bonwick	spa->spa_dedup_checksum = ZIO_DEDUPCHECKSUM;
834b24ab67Jeff Bonwick
835b24ab67Jeff Bonwick	for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++)
836b24ab67Jeff Bonwick		spa->spa_ddt[c] = ddt_table_alloc(spa, c);
837b24ab67Jeff Bonwick}
838b24ab67Jeff Bonwick
839b24ab67Jeff Bonwickint
840b24ab67Jeff Bonwickddt_load(spa_t *spa)
841b24ab67Jeff Bonwick{
842b24ab67Jeff Bonwick	int error;
843b24ab67Jeff Bonwick
844b24ab67Jeff Bonwick	ddt_create(spa);
845b24ab67Jeff Bonwick
846b24ab67Jeff Bonwick	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
847b24ab67Jeff Bonwick	    DMU_POOL_DDT_STATS, sizeof (uint64_t), 1,
848b24ab67Jeff Bonwick	    &spa->spa_ddt_stat_object);
849b24ab67Jeff Bonwick
850b24ab67Jeff Bonwick	if (error)
851b24ab67Jeff Bonwick		return (error == ENOENT ? 0 : error);
852b24ab67Jeff Bonwick
853b24ab67Jeff Bonwick	for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
85430f608aGeorge Wilson		ddt_t *ddt = spa->spa_ddt[c];
855b24ab67Jeff Bonwick		for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
856b24ab67Jeff Bonwick			for (enum ddt_class class = 0; class < DDT_CLASSES;
857b24ab67Jeff Bonwick			    class++) {
858b24ab67Jeff Bonwick				error = ddt_object_load(ddt, type, class);
859b24ab67Jeff Bonwick				if (error != 0 && error != ENOENT)
860b24ab67Jeff Bonwick					return (error);
861b24ab67Jeff Bonwick			}
862b24ab67Jeff Bonwick		}
86330f608aGeorge Wilson
86430f608aGeorge Wilson		/*
86530f608aGeorge Wilson		 * Seed the cached histograms.
86630f608aGeorge Wilson		 */
86730f608aGeorge Wilson		bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache,
86830f608aGeorge Wilson		    sizeof (ddt->ddt_histogram));
869b24ab67Jeff Bonwick	}
870b24ab67Jeff Bonwick
871b24ab67Jeff Bonwick	return (0);
872b24ab67Jeff Bonwick}
873b24ab67Jeff Bonwick
874b24ab67Jeff Bonwickvoid
875b24ab67Jeff Bonwickddt_unload(spa_t *spa)
876b24ab67Jeff Bonwick{
877b24ab67Jeff Bonwick	for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
878b24ab67Jeff Bonwick		if (spa->spa_ddt[c]) {
879b24ab67Jeff Bonwick			ddt_table_free(spa->spa_ddt[c]);
880b24ab67Jeff Bonwick			spa->spa_ddt[c] = NULL;
881b24ab67Jeff Bonwick		}
882b24ab67Jeff Bonwick	}
883b24ab67Jeff Bonwick}
884b24ab67Jeff Bonwick
885bbfd46cJeff Bonwickboolean_t
886bbfd46cJeff Bonwickddt_class_contains(spa_t *spa, enum ddt_class max_class, const blkptr_t *bp)
887bbfd46cJeff Bonwick{
888bbfd46cJeff Bonwick	ddt_t *ddt;
889bbfd46cJeff Bonwick	ddt_entry_t dde;
890bbfd46cJeff Bonwick
891bbfd46cJeff Bonwick	if (!BP_GET_DEDUP(bp))
892bbfd46cJeff Bonwick		return (B_FALSE);
893bbfd46cJeff Bonwick
894bbfd46cJeff Bonwick	if (max_class == DDT_CLASS_UNIQUE)
895bbfd46cJeff Bonwick		return (B_TRUE);
896bbfd46cJeff Bonwick
897bbfd46cJeff Bonwick	ddt = spa->spa_ddt[BP_GET_CHECKSUM(bp)];
898bbfd46cJeff Bonwick
899bbfd46cJeff Bonwick	ddt_key_fill(&dde.dde_key, bp);
900bbfd46cJeff Bonwick
901bbfd46cJeff Bonwick	for (enum ddt_type type = 0; type < DDT_TYPES; type++)
902bbfd46cJeff Bonwick		for (enum ddt_class class = 0; class <= max_class; class++)
903bbfd46cJeff Bonwick			if (ddt_object_lookup(ddt, type, class, &dde) == 0)
904bbfd46cJeff Bonwick				return (B_TRUE);
905bbfd46cJeff Bonwick
906bbfd46cJeff Bonwick	return (B_FALSE);
907bbfd46cJeff Bonwick}
908bbfd46cJeff Bonwick
909b24ab67Jeff Bonwickddt_entry_t *
910b24ab67Jeff Bonwickddt_repair_start(ddt_t *ddt, const blkptr_t *bp)
911b24ab67Jeff Bonwick{
912b24ab67Jeff Bonwick	ddt_key_t ddk;
913b24ab67Jeff Bonwick	ddt_entry_t *dde;
914b24ab67Jeff Bonwick
915b24ab67Jeff Bonwick	ddt_key_fill(&ddk, bp);
916b24ab67Jeff Bonwick
917b24ab67Jeff Bonwick	dde = ddt_alloc(&ddk);
918b24ab67Jeff Bonwick
919b24ab67Jeff Bonwick	for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
920b24ab67Jeff Bonwick		for (enum ddt_class class = 0; class < DDT_CLASSES; class++) {
921b24ab67Jeff Bonwick			/*
922b24ab67Jeff Bonwick			 * We can only do repair if there are multiple copies
923b24ab67Jeff Bonwick			 * of the block.  For anything in the UNIQUE class,
924b24ab67Jeff Bonwick			 * there's definitely only one copy, so don't even try.
925b24ab67Jeff Bonwick			 */
926b24ab67Jeff Bonwick			if (class != DDT_CLASS_UNIQUE &&
927b24ab67Jeff Bonwick			    ddt_object_lookup(ddt, type, class, dde) == 0)
928b24ab67Jeff Bonwick				return (dde);
929b24ab67Jeff Bonwick		}
930b24ab67Jeff Bonwick	}
931b24ab67Jeff Bonwick
932b24ab67Jeff Bonwick	bzero(dde->dde_phys, sizeof (dde->dde_phys));
933b24ab67Jeff Bonwick
934b24ab67Jeff Bonwick	return (dde);
935b24ab67Jeff Bonwick}
936b24ab67Jeff Bonwick
937b24ab67Jeff Bonwickvoid
938b24ab67Jeff Bonwickddt_repair_done(ddt_t *ddt, ddt_entry_t *dde)
939b24ab67Jeff Bonwick{
940b24ab67Jeff Bonwick	avl_index_t where;
941b24ab67Jeff Bonwick
942b24ab67Jeff Bonwick	ddt_enter(ddt);
943b24ab67Jeff Bonwick
944770499eDan Kimmel	if (dde->dde_repair_abd != NULL && spa_writeable(ddt->ddt_spa) &&
945b24ab67Jeff Bonwick	    avl_find(&ddt->ddt_repair_tree, dde, &where) == NULL)
946b24ab67Jeff Bonwick		avl_insert(&ddt->ddt_repair_tree, dde, where);
947b24ab67Jeff Bonwick	else
948b24ab67Jeff Bonwick		ddt_free(dde);
949b24ab67Jeff Bonwick
950b24ab67Jeff Bonwick	ddt_exit(ddt);
951b24ab67Jeff Bonwick}
952b24ab67Jeff Bonwick
953b24ab67Jeff Bonwickstatic void
954b24ab67Jeff Bonwickddt_repair_entry_done(zio_t *zio)
955b24ab67Jeff Bonwick{
956b24ab67Jeff Bonwick	ddt_entry_t *rdde = zio->io_private;
957b24ab67Jeff Bonwick
958b24ab67Jeff Bonwick	ddt_free(rdde);
959b24ab67Jeff Bonwick}
960b24ab67Jeff Bonwick
961b24ab67Jeff Bonwickstatic void
962b24ab67Jeff Bonwickddt_repair_entry(ddt_t *ddt, ddt_entry_t *dde, ddt_entry_t *rdde, zio_t *rio)
963b24ab67Jeff Bonwick{
964b24ab67Jeff Bonwick	ddt_phys_t *ddp = dde->dde_phys;
965b24ab67Jeff Bonwick	ddt_phys_t *rddp = rdde->dde_phys;
966b24ab67Jeff Bonwick	ddt_key_t *ddk = &dde->dde_key;
967b24ab67Jeff Bonwick	ddt_key_t *rddk = &rdde->dde_key;
968b24ab67Jeff Bonwick	zio_t *zio;
969b24ab67Jeff Bonwick	blkptr_t blk;
970b24ab67Jeff Bonwick
971b24ab67Jeff Bonwick	zio = zio_null(rio, rio->io_spa, NULL,
972b24ab67Jeff Bonwick	    ddt_repair_entry_done, rdde, rio->io_flags);
973b24ab67Jeff Bonwick
974b24ab67Jeff Bonwick	for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++, rddp++) {
975b24ab67Jeff Bonwick		if (ddp->ddp_phys_birth == 0 ||
976b24ab67Jeff Bonwick		    ddp->ddp_phys_birth != rddp->ddp_phys_birth ||
977b24ab67Jeff Bonwick		    bcmp(ddp->ddp_dva, rddp->ddp_dva, sizeof (ddp->ddp_dva)))
978b24ab67Jeff Bonwick			continue;
979bbfd46cJeff Bonwick		ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk);
980b24ab67Jeff Bonwick		zio_nowait(zio_rewrite(zio, zio->io_spa, 0, &blk,
981770499eDan Kimmel		    rdde->dde_repair_abd, DDK_GET_PSIZE(rddk), NULL, NULL,
982b24ab67Jeff Bonwick		    ZIO_PRIORITY_SYNC_WRITE, ZIO_DDT_CHILD_FLAGS(zio), NULL));
983b24ab67Jeff Bonwick	}
984b24ab67Jeff Bonwick
985b24ab67Jeff Bonwick	zio_nowait(zio);
986b24ab67Jeff Bonwick}
987b24ab67Jeff Bonwick
988b24ab67Jeff Bonwickstatic void
989b24ab67Jeff Bonwickddt_repair_table(ddt_t *ddt, zio_t *rio)
990b24ab67Jeff Bonwick{
991b24ab67Jeff Bonwick	spa_t *spa = ddt->ddt_spa;
992b24ab67Jeff Bonwick	ddt_entry_t *dde, *rdde_next, *rdde;
993b24ab67Jeff Bonwick	avl_tree_t *t = &ddt->ddt_repair_tree;
994b24ab67Jeff Bonwick	blkptr_t blk;
995b24ab67Jeff Bonwick
996b24ab67Jeff Bonwick	if (spa_sync_pass(spa) > 1)
997b24ab67Jeff Bonwick		return;
998b24ab67Jeff Bonwick
999b24ab67Jeff Bonwick	ddt_enter(ddt);
1000b24ab67Jeff Bonwick	for (rdde = avl_first(t); rdde != NULL; rdde = rdde_next) {
1001b24ab67Jeff Bonwick		rdde_next = AVL_NEXT(t, rdde);
1002b24ab67Jeff Bonwick		avl_remove(&ddt->ddt_repair_tree, rdde);
1003b24ab67Jeff Bonwick		ddt_exit(ddt);
1004bbfd46cJeff Bonwick		ddt_bp_create(ddt->ddt_checksum, &rdde->dde_key, NULL, &blk);
1005b24ab67Jeff Bonwick		dde = ddt_repair_start(ddt, &blk);
1006b24ab67Jeff Bonwick		ddt_repair_entry(ddt, dde, rdde, rio);
1007b24ab67Jeff Bonwick		ddt_repair_done(ddt, dde);
1008b24ab67Jeff Bonwick		ddt_enter(ddt);
1009b24ab67Jeff Bonwick	}
1010b24ab67Jeff Bonwick	ddt_exit(ddt);
1011b24ab67Jeff Bonwick}
1012b24ab67Jeff Bonwick
1013b24ab67Jeff Bonwickstatic void
1014b24ab67Jeff Bonwickddt_sync_entry(ddt_t *ddt, ddt_entry_t *dde, dmu_tx_t *tx, uint64_t txg)
1015b24ab67Jeff Bonwick{
1016bbfd46cJeff Bonwick	dsl_pool_t *dp = ddt->ddt_spa->spa_dsl_pool;
1017b24ab67Jeff Bonwick	ddt_phys_t *ddp = dde->dde_phys;
1018b24ab67Jeff Bonwick	ddt_key_t *ddk = &dde->dde_key;
1019b24ab67Jeff Bonwick	enum ddt_type otype = dde->dde_type;
1020b24ab67Jeff Bonwick	enum ddt_type ntype = DDT_TYPE_CURRENT;
1021b24ab67Jeff Bonwick	enum ddt_class oclass = dde->dde_class;
1022b24ab67Jeff Bonwick	enum ddt_class nclass;
1023b24ab67Jeff Bonwick	uint64_t total_refcnt = 0;
1024b24ab67Jeff Bonwick
1025b24ab67Jeff Bonwick	ASSERT(dde->dde_loaded);
1026b24ab67Jeff Bonwick	ASSERT(!dde->dde_loading);
1027b24ab67Jeff Bonwick
1028b24ab67Jeff Bonwick	for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
1029b24ab67Jeff Bonwick		ASSERT(dde->dde_lead_zio[p] == NULL);
1030b24ab67Jeff Bonwick		ASSERT((int64_t)ddp->ddp_refcnt >= 0);
1031b24ab67Jeff Bonwick		if (ddp->ddp_phys_birth == 0) {
1032b24ab67Jeff Bonwick			ASSERT(ddp->ddp_refcnt == 0);
1033b24ab67Jeff Bonwick			continue;
1034b24ab67Jeff Bonwick		}
1035b24ab67Jeff Bonwick		if (p == DDT_PHYS_DITTO) {
1036b24ab67Jeff Bonwick			if (ddt_ditto_copies_needed(ddt, dde, NULL) == 0)
1037b24ab67Jeff Bonwick				ddt_phys_free(ddt, ddk, ddp, txg);
1038b24ab67Jeff Bonwick			continue;
1039b24ab67Jeff Bonwick		}
1040b24ab67Jeff Bonwick		if (ddp->ddp_refcnt == 0)
1041b24ab67Jeff Bonwick			ddt_phys_free(ddt, ddk, ddp, txg);
1042b24ab67Jeff Bonwick		total_refcnt += ddp->ddp_refcnt;
1043b24ab67Jeff Bonwick	}
1044b24ab67Jeff Bonwick
1045b24ab67Jeff Bonwick	if (dde->dde_phys[DDT_PHYS_DITTO].ddp_phys_birth != 0)
1046b24ab67Jeff Bonwick		nclass = DDT_CLASS_DITTO;
1047b24ab67Jeff Bonwick	else if (total_refcnt > 1)
1048b24ab67Jeff Bonwick		nclass = DDT_CLASS_DUPLICATE;
1049b24ab67Jeff Bonwick	else
1050b24ab67Jeff Bonwick		nclass = DDT_CLASS_UNIQUE;
1051b24ab67Jeff Bonwick
1052b24ab67Jeff Bonwick	if (otype != DDT_TYPES &&
1053b24ab67Jeff Bonwick	    (otype != ntype || oclass != nclass || total_refcnt == 0)) {
1054b24ab67Jeff Bonwick		VERIFY(ddt_object_remove(ddt, otype, oclass, dde, tx) == 0);
1055b24ab67Jeff Bonwick		ASSERT(ddt_object_lookup(ddt, otype, oclass, dde) == ENOENT);
1056b24ab67Jeff Bonwick	}
1057b24ab67Jeff Bonwick
1058b24ab67Jeff Bonwick	if (total_refcnt != 0) {
1059b24ab67Jeff Bonwick		dde->dde_type = ntype;
1060b24ab67Jeff Bonwick		dde->dde_class = nclass;
1061b24ab67Jeff Bonwick		ddt_stat_update(ddt, dde, 0);
1062b24ab67Jeff Bonwick		if (!ddt_object_exists(ddt, ntype, nclass))
1063b24ab67Jeff Bonwick			ddt_object_create(ddt, ntype, nclass, tx);
1064b24ab67Jeff Bonwick		VERIFY(ddt_object_update(ddt, ntype, nclass, dde, tx) == 0);
1065bbfd46cJeff Bonwick
10663f9d6adLin Ling		/*
10673f9d6adLin Ling		 * If the class changes, the order that we scan this bp
10683f9d6adLin Ling		 * changes.  If it decreases, we could miss it, so
10693f9d6adLin Ling		 * scan it right now.  (This covers both class changing
10703f9d6adLin Ling		 * while we are doing ddt_walk(), and when we are
10713f9d6adLin Ling		 * traversing.)
10723f9d6adLin Ling		 */
10733f9d6adLin Ling		if (nclass < oclass) {
10743f9d6adLin Ling			dsl_scan_ddt_entry(dp->dp_scan,
10753f9d6adLin Ling			    ddt->ddt_checksum, dde, tx);
10763f9d6adLin Ling		}
1077b24ab67Jeff Bonwick	}
1078b24ab67Jeff Bonwick}
1079b24ab67Jeff Bonwick
1080b24ab67Jeff Bonwickstatic void
1081b24ab67Jeff Bonwickddt_sync_table(ddt_t *ddt, dmu_tx_t *tx, uint64_t txg)
1082b24ab67Jeff Bonwick{
1083b24ab67Jeff Bonwick	spa_t *spa = ddt->ddt_spa;
1084b24ab67Jeff Bonwick	ddt_entry_t *dde;
1085b24ab67Jeff Bonwick	void *cookie = NULL;
1086b24ab67Jeff Bonwick
1087b24ab67Jeff Bonwick	if (avl_numnodes(&ddt->ddt_tree) == 0)
1088b24ab67Jeff Bonwick		return;
1089b24ab67Jeff Bonwick
1090b24ab67Jeff Bonwick	ASSERT(spa->spa_uberblock.ub_version >= SPA_VERSION_DEDUP);
1091b24ab67Jeff Bonwick
1092b24ab67Jeff Bonwick	if (spa->spa_ddt_stat_object == 0) {
1093ad135b5Christopher Siden		spa->spa_ddt_stat_object = zap_create_link(ddt->ddt_os,
1094ad135b5Christopher Siden		    DMU_OT_DDT_STATS, DMU_POOL_DIRECTORY_OBJECT,
1095ad135b5Christopher Siden		    DMU_POOL_DDT_STATS, tx);
1096b24ab67Jeff Bonwick	}
1097b24ab67Jeff Bonwick
1098b24ab67Jeff Bonwick	while ((dde = avl_destroy_nodes(&ddt->ddt_tree, &cookie)) != NULL) {
1099b24ab67Jeff Bonwick		ddt_sync_entry(ddt, dde, tx, txg);
1100b24ab67Jeff Bonwick		ddt_free(dde);
1101b24ab67Jeff Bonwick	}
1102b24ab67Jeff Bonwick
1103b24ab67Jeff Bonwick	for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
1104ce053c9George Wilson		uint64_t count = 0;
1105b24ab67Jeff Bonwick		for (enum ddt_class class = 0; class < DDT_CLASSES; class++) {
1106ce053c9George Wilson			if (ddt_object_exists(ddt, type, class)) {
1107ce053c9George Wilson				ddt_object_sync(ddt, type, class, tx);
1108ce053c9George Wilson				count += ddt_object_count(ddt, type, class);
1109ce053c9George Wilson			}
1110ce053c9George Wilson		}
1111ce053c9George Wilson		for (enum ddt_class class = 0; class < DDT_CLASSES; class++) {
1112ce053c9George Wilson			if (count == 0 && ddt_object_exists(ddt, type, class))
1113b24ab67Jeff Bonwick				ddt_object_destroy(ddt, type, class, tx);
1114b24ab67Jeff Bonwick		}
1115b24ab67Jeff Bonwick	}
111630f608aGeorge Wilson
111730f608aGeorge Wilson	bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache,
111830f608aGeorge Wilson	    sizeof (ddt->ddt_histogram));
1119b24ab67Jeff Bonwick}
1120b24ab67Jeff Bonwick
1121b24ab67Jeff Bonwickvoid
1122b24ab67Jeff Bonwickddt_sync(spa_t *spa, uint64_t txg)
1123b24ab67Jeff Bonwick{
1124a3874b8Toomas Soome	dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan;
1125b24ab67Jeff Bonwick	dmu_tx_t *tx;
1126a3874b8Toomas Soome	zio_t *rio;
1127b24ab67Jeff Bonwick
1128b24ab67Jeff Bonwick	ASSERT(spa_syncing_txg(spa) == txg);
1129b24ab67Jeff Bonwick
1130b24ab67Jeff Bonwick	tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
1131b24ab67Jeff Bonwick
1132a3874b8Toomas Soome	rio = zio_root(spa, NULL, NULL,
1133a3874b8Toomas Soome	    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | ZIO_FLAG_SELF_HEAL);
1134