xref: /illumos-gate/usr/src/uts/common/fs/zfs/dsl_pool.c (revision c5832a53)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright (c) 2011, 2019 by Delphix. All rights reserved.
24  * Copyright (c) 2013 Steven Hartland. All rights reserved.
25  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
26  * Copyright (c) 2014 Integros [integros.com]
27  * Copyright 2016 Nexenta Systems, Inc.  All rights reserved.
28  */
29 
30 #include <sys/dsl_pool.h>
31 #include <sys/dsl_dataset.h>
32 #include <sys/dsl_prop.h>
33 #include <sys/dsl_dir.h>
34 #include <sys/dsl_synctask.h>
35 #include <sys/dsl_scan.h>
36 #include <sys/dnode.h>
37 #include <sys/dmu_tx.h>
38 #include <sys/dmu_objset.h>
39 #include <sys/arc.h>
40 #include <sys/zap.h>
41 #include <sys/zio.h>
42 #include <sys/zfs_context.h>
43 #include <sys/fs/zfs.h>
44 #include <sys/zfs_znode.h>
45 #include <sys/spa_impl.h>
46 #include <sys/dsl_deadlist.h>
47 #include <sys/vdev_impl.h>
48 #include <sys/metaslab_impl.h>
49 #include <sys/bptree.h>
50 #include <sys/zfeature.h>
51 #include <sys/zil_impl.h>
52 #include <sys/dsl_userhold.h>
53 #include <sys/mmp.h>
54 
55 /*
56  * ZFS Write Throttle
57  * ------------------
58  *
59  * ZFS must limit the rate of incoming writes to the rate at which it is able
60  * to sync data modifications to the backend storage. Throttling by too much
61  * creates an artificial limit; throttling by too little can only be sustained
62  * for short periods and would lead to highly lumpy performance. On a per-pool
63  * basis, ZFS tracks the amount of modified (dirty) data. As operations change
64  * data, the amount of dirty data increases; as ZFS syncs out data, the amount
65  * of dirty data decreases. When the amount of dirty data exceeds a
66  * predetermined threshold further modifications are blocked until the amount
67  * of dirty data decreases (as data is synced out).
68  *
69  * The limit on dirty data is tunable, and should be adjusted according to
70  * both the IO capacity and available memory of the system. The larger the
71  * window, the more ZFS is able to aggregate and amortize metadata (and data)
72  * changes. However, memory is a limited resource, and allowing for more dirty
73  * data comes at the cost of keeping other useful data in memory (for example
74  * ZFS data cached by the ARC).
75  *
76  * Implementation
77  *
78  * As buffers are modified dsl_pool_willuse_space() increments both the per-
79  * txg (dp_dirty_pertxg[]) and poolwide (dp_dirty_total) accounting of
80  * dirty space used; dsl_pool_dirty_space() decrements those values as data
81  * is synced out from dsl_pool_sync(). While only the poolwide value is
82  * relevant, the per-txg value is useful for debugging. The tunable
83  * zfs_dirty_data_max determines the dirty space limit. Once that value is
84  * exceeded, new writes are halted until space frees up.
85  *
86  * The zfs_dirty_data_sync tunable dictates the threshold at which we
87  * ensure that there is a txg syncing (see the comment in txg.c for a full
88  * description of transaction group stages).
89  *
90  * The IO scheduler uses both the dirty space limit and current amount of
91  * dirty data as inputs. Those values affect the number of concurrent IOs ZFS
92  * issues. See the comment in vdev_queue.c for details of the IO scheduler.
93  *
94  * The delay is also calculated based on the amount of dirty data.  See the
95  * comment above dmu_tx_delay() for details.
96  */
97 
98 /*
99  * zfs_dirty_data_max will be set to zfs_dirty_data_max_percent% of all memory,
100  * capped at zfs_dirty_data_max_max.  It can also be overridden in /etc/system.
101  */
102 uint64_t zfs_dirty_data_max;
103 uint64_t zfs_dirty_data_max_max = 4ULL * 1024 * 1024 * 1024;
104 int zfs_dirty_data_max_percent = 10;
105 
106 /*
107  * If there's at least this much dirty data (as a percentage of
108  * zfs_dirty_data_max), push out a txg.  This should be less than
109  * zfs_vdev_async_write_active_min_dirty_percent.
110  */
111 uint64_t zfs_dirty_data_sync_pct = 20;
112 
113 /*
114  * Once there is this amount of dirty data, the dmu_tx_delay() will kick in
115  * and delay each transaction.
116  * This value should be >= zfs_vdev_async_write_active_max_dirty_percent.
117  */
118 int zfs_delay_min_dirty_percent = 60;
119 
120 /*
121  * This controls how quickly the delay approaches infinity.
122  * Larger values cause it to delay more for a given amount of dirty data.
123  * Therefore larger values will cause there to be less dirty data for a
124  * given throughput.
125  *
126  * For the smoothest delay, this value should be about 1 billion divided
127  * by the maximum number of operations per second.  This will smoothly
128  * handle between 10x and 1/10th this number.
129  *
130  * Note: zfs_delay_scale * zfs_dirty_data_max must be < 2^64, due to the
131  * multiply in dmu_tx_delay().
132  */
133 uint64_t zfs_delay_scale = 1000 * 1000 * 1000 / 2000;
134 
135 /*
136  * This determines the number of threads used by the dp_sync_taskq.
137  */
138 int zfs_sync_taskq_batch_pct = 75;
139 
140 /*
141  * These tunables determine the behavior of how zil_itxg_clean() is
142  * called via zil_clean() in the context of spa_sync(). When an itxg
143  * list needs to be cleaned, TQ_NOSLEEP will be used when dispatching.
144  * If the dispatch fails, the call to zil_itxg_clean() will occur
145  * synchronously in the context of spa_sync(), which can negatively
146  * impact the performance of spa_sync() (e.g. in the case of the itxg
147  * list having a large number of itxs that needs to be cleaned).
148  *
149  * Thus, these tunables can be used to manipulate the behavior of the
150  * taskq used by zil_clean(); they determine the number of taskq entries
151  * that are pre-populated when the taskq is first created (via the
152  * "zfs_zil_clean_taskq_minalloc" tunable) and the maximum number of
153  * taskq entries that are cached after an on-demand allocation (via the
154  * "zfs_zil_clean_taskq_maxalloc").
155  *
156  * The idea being, we want to try reasonably hard to ensure there will
157  * already be a taskq entry pre-allocated by the time that it is needed
158  * by zil_clean(). This way, we can avoid the possibility of an
159  * on-demand allocation of a new taskq entry from failing, which would
160  * result in zil_itxg_clean() being called synchronously from zil_clean()
161  * (which can adversely affect performance of spa_sync()).
162  *
163  * Additionally, the number of threads used by the taskq can be
164  * configured via the "zfs_zil_clean_taskq_nthr_pct" tunable.
165  */
166 int zfs_zil_clean_taskq_nthr_pct = 100;
167 int zfs_zil_clean_taskq_minalloc = 1024;
168 int zfs_zil_clean_taskq_maxalloc = 1024 * 1024;
169 
170 int
dsl_pool_open_special_dir(dsl_pool_t * dp,const char * name,dsl_dir_t ** ddp)171 dsl_pool_open_special_dir(dsl_pool_t *dp, const char *name, dsl_dir_t **ddp)
172 {
173 	uint64_t obj;
174 	int err;
175 
176 	err = zap_lookup(dp->dp_meta_objset,
177 	    dsl_dir_phys(dp->dp_root_dir)->dd_child_dir_zapobj,
178 	    name, sizeof (obj), 1, &obj);
179 	if (err)
180 		return (err);
181 
182 	return (dsl_dir_hold_obj(dp, obj, name, dp, ddp));
183 }
184 
185 static dsl_pool_t *
dsl_pool_open_impl(spa_t * spa,uint64_t txg)186 dsl_pool_open_impl(spa_t *spa, uint64_t txg)
187 {
188 	dsl_pool_t *dp;
189 	blkptr_t *bp = spa_get_rootblkptr(spa);
190 
191 	dp = kmem_zalloc(sizeof (dsl_pool_t), KM_SLEEP);
192 	dp->dp_spa = spa;
193 	dp->dp_meta_rootbp = *bp;
194 	rrw_init(&dp->dp_config_rwlock, B_TRUE);
195 	txg_init(dp, txg);
196 	mmp_init(spa);
197 
198 	txg_list_create(&dp->dp_dirty_datasets, spa,
199 	    offsetof(dsl_dataset_t, ds_dirty_link));
200 	txg_list_create(&dp->dp_dirty_zilogs, spa,
201 	    offsetof(zilog_t, zl_dirty_link));
202 	txg_list_create(&dp->dp_dirty_dirs, spa,
203 	    offsetof(dsl_dir_t, dd_dirty_link));
204 	txg_list_create(&dp->dp_sync_tasks, spa,
205 	    offsetof(dsl_sync_task_t, dst_node));
206 	txg_list_create(&dp->dp_early_sync_tasks, spa,
207 	    offsetof(dsl_sync_task_t, dst_node));
208 
209 	dp->dp_sync_taskq = taskq_create("dp_sync_taskq",
210 	    zfs_sync_taskq_batch_pct, minclsyspri, 1, INT_MAX,
211 	    TASKQ_THREADS_CPU_PCT);
212 
213 	dp->dp_zil_clean_taskq = taskq_create("dp_zil_clean_taskq",
214 	    zfs_zil_clean_taskq_nthr_pct, minclsyspri,
215 	    zfs_zil_clean_taskq_minalloc,
216 	    zfs_zil_clean_taskq_maxalloc,
217 	    TASKQ_PREPOPULATE | TASKQ_THREADS_CPU_PCT);
218 
219 	mutex_init(&dp->dp_lock, NULL, MUTEX_DEFAULT, NULL);
220 	cv_init(&dp->dp_spaceavail_cv, NULL, CV_DEFAULT, NULL);
221 
222 	dp->dp_vnrele_taskq = taskq_create("zfs_vn_rele_taskq", 1, minclsyspri,
223 	    1, 4, 0);
224 	dp->dp_unlinked_drain_taskq = taskq_create("z_unlinked_drain",
225 	    max_ncpus, minclsyspri, max_ncpus, INT_MAX,
226 	    TASKQ_PREPOPULATE | TASKQ_DYNAMIC);
227 
228 	return (dp);
229 }
230 
231 int
dsl_pool_init(spa_t * spa,uint64_t txg,dsl_pool_t ** dpp)232 dsl_pool_init(spa_t *spa, uint64_t txg, dsl_pool_t **dpp)
233 {
234 	int err;
235 	dsl_pool_t *dp = dsl_pool_open_impl(spa, txg);
236 
237 	err = dmu_objset_open_impl(spa, NULL, &dp->dp_meta_rootbp,
238 	    &dp->dp_meta_objset);
239 	if (err != 0)
240 		dsl_pool_close(dp);
241 	else
242 		*dpp = dp;
243 
244 	return (err);
245 }
246 
247 int
dsl_pool_open(dsl_pool_t * dp)248 dsl_pool_open(dsl_pool_t *dp)
249 {
250 	int err;
251 	dsl_dir_t *dd;
252 	dsl_dataset_t *ds;
253 	uint64_t obj;
254 
255 	rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG);
256 	err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
257 	    DMU_POOL_ROOT_DATASET, sizeof (uint64_t), 1,
258 	    &dp->dp_root_dir_obj);
259 	if (err)
260 		goto out;
261 
262 	err = dsl_dir_hold_obj(dp, dp->dp_root_dir_obj,
263 	    NULL, dp, &dp->dp_root_dir);
264 	if (err)
265 		goto out;
266 
267 	err = dsl_pool_open_special_dir(dp, MOS_DIR_NAME, &dp->dp_mos_dir);
268 	if (err)
269 		goto out;
270 
271 	if (spa_version(dp->dp_spa) >= SPA_VERSION_ORIGIN) {
272 		err = dsl_pool_open_special_dir(dp, ORIGIN_DIR_NAME, &dd);
273 		if (err)
274 			goto out;
275 		err = dsl_dataset_hold_obj(dp,
276 		    dsl_dir_phys(dd)->dd_head_dataset_obj, FTAG, &ds);
277 		if (err == 0) {
278 			err = dsl_dataset_hold_obj(dp,
279 			    dsl_dataset_phys(ds)->ds_prev_snap_obj, dp,
280 			    &dp->dp_origin_snap);
281 			dsl_dataset_rele(ds, FTAG);
282 		}
283 		dsl_dir_rele(dd, dp);
284 		if (err)
285 			goto out;
286 	}
287 
288 	if (spa_version(dp->dp_spa) >= SPA_VERSION_DEADLISTS) {
289 		err = dsl_pool_open_special_dir(dp, FREE_DIR_NAME,
290 		    &dp->dp_free_dir);
291 		if (err)
292 			goto out;
293 
294 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
295 		    DMU_POOL_FREE_BPOBJ, sizeof (uint64_t), 1, &obj);
296 		if (err)
297 			goto out;
298 		VERIFY0(bpobj_open(&dp->dp_free_bpobj,
299 		    dp->dp_meta_objset, obj));
300 	}
301 
302 	if (spa_feature_is_active(dp->dp_spa, SPA_FEATURE_OBSOLETE_COUNTS)) {
303 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
304 		    DMU_POOL_OBSOLETE_BPOBJ, sizeof (uint64_t), 1, &obj);
305 		if (err == 0) {
306 			VERIFY0(bpobj_open(&dp->dp_obsolete_bpobj,
307 			    dp->dp_meta_objset, obj));
308 		} else if (err == ENOENT) {
309 			/*
310 			 * We might not have created the remap bpobj yet.
311 			 */
312 			err = 0;
313 		} else {
314 			goto out;
315 		}
316 	}
317 
318 	/*
319 	 * Note: errors ignored, because the these special dirs, used for
320 	 * space accounting, are only created on demand.
321 	 */
322 	(void) dsl_pool_open_special_dir(dp, LEAK_DIR_NAME,
323 	    &dp->dp_leak_dir);
324 
325 	if (spa_feature_is_active(dp->dp_spa, SPA_FEATURE_ASYNC_DESTROY)) {
326 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
327 		    DMU_POOL_BPTREE_OBJ, sizeof (uint64_t), 1,
328 		    &dp->dp_bptree_obj);
329 		if (err != 0)
330 			goto out;
331 	}
332 
333 	if (spa_feature_is_active(dp->dp_spa, SPA_FEATURE_EMPTY_BPOBJ)) {
334 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
335 		    DMU_POOL_EMPTY_BPOBJ, sizeof (uint64_t), 1,
336 		    &dp->dp_empty_bpobj);
337 		if (err != 0)
338 			goto out;
339 	}
340 
341 	err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
342 	    DMU_POOL_TMP_USERREFS, sizeof (uint64_t), 1,
343 	    &dp->dp_tmp_userrefs_obj);
344 	if (err == ENOENT)
345 		err = 0;
346 	if (err)
347 		goto out;
348 
349 	err = dsl_scan_init(dp, dp->dp_tx.tx_open_txg);
350 
351 out:
352 	rrw_exit(&dp->dp_config_rwlock, FTAG);
353 	return (err);
354 }
355 
356 void
dsl_pool_close(dsl_pool_t * dp)357 dsl_pool_close(dsl_pool_t *dp)
358 {
359 	/*
360 	 * Drop our references from dsl_pool_open().
361 	 *
362 	 * Since we held the origin_snap from "syncing" context (which
363 	 * includes pool-opening context), it actually only got a "ref"
364 	 * and not a hold, so just drop that here.
365 	 */
366 	if (dp->dp_origin_snap != NULL)
367 		dsl_dataset_rele(dp->dp_origin_snap, dp);
368 	if (dp->dp_mos_dir != NULL)
369 		dsl_dir_rele(dp->dp_mos_dir, dp);
370 	if (dp->dp_free_dir != NULL)
371 		dsl_dir_rele(dp->dp_free_dir, dp);
372 	if (dp->dp_leak_dir != NULL)
373 		dsl_dir_rele(dp->dp_leak_dir, dp);
374 	if (dp->dp_root_dir != NULL)
375 		dsl_dir_rele(dp->dp_root_dir, dp);
376 
377 	bpobj_close(&dp->dp_free_bpobj);
378 	bpobj_close(&dp->dp_obsolete_bpobj);
379 
380 	/* undo the dmu_objset_open_impl(mos) from dsl_pool_open() */
381 	if (dp->dp_meta_objset != NULL)
382 		dmu_objset_evict(dp->dp_meta_objset);
383 
384 	txg_list_destroy(&dp->dp_dirty_datasets);
385 	txg_list_destroy(&dp->dp_dirty_zilogs);
386 	txg_list_destroy(&dp->dp_sync_tasks);
387 	txg_list_destroy(&dp->dp_early_sync_tasks);
388 	txg_list_destroy(&dp->dp_dirty_dirs);
389 
390 	taskq_destroy(dp->dp_zil_clean_taskq);
391 	taskq_destroy(dp->dp_sync_taskq);
392 
393 	/*
394 	 * We can't set retry to TRUE since we're explicitly specifying
395 	 * a spa to flush. This is good enough; any missed buffers for
396 	 * this spa won't cause trouble, and they'll eventually fall
397 	 * out of the ARC just like any other unused buffer.
398 	 */
399 	arc_flush(dp->dp_spa, FALSE);
400 
401 	mmp_fini(dp->dp_spa);
402 	txg_fini(dp);
403 	dsl_scan_fini(dp);
404 	dmu_buf_user_evict_wait();
405 
406 	rrw_destroy(&dp->dp_config_rwlock);
407 	mutex_destroy(&dp->dp_lock);
408 	taskq_destroy(dp->dp_unlinked_drain_taskq);
409 	taskq_destroy(dp->dp_vnrele_taskq);
410 	if (dp->dp_blkstats != NULL)
411 		kmem_free(dp->dp_blkstats, sizeof (zfs_all_blkstats_t));
412 	kmem_free(dp, sizeof (dsl_pool_t));
413 }
414 
415 void
dsl_pool_create_obsolete_bpobj(dsl_pool_t * dp,dmu_tx_t * tx)416 dsl_pool_create_obsolete_bpobj(dsl_pool_t *dp, dmu_tx_t *tx)
417 {
418 	uint64_t obj;
419 	/*
420 	 * Currently, we only create the obsolete_bpobj where there are
421 	 * indirect vdevs with referenced mappings.
422 	 */
423 	ASSERT(spa_feature_is_active(dp->dp_spa, SPA_FEATURE_DEVICE_REMOVAL));
424 	/* create and open the obsolete_bpobj */
425 	obj = bpobj_alloc(dp->dp_meta_objset, SPA_OLD_MAXBLOCKSIZE, tx);
426 	VERIFY0(bpobj_open(&dp->dp_obsolete_bpobj, dp->dp_meta_objset, obj));
427 	VERIFY0(zap_add(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
428 	    DMU_POOL_OBSOLETE_BPOBJ, sizeof (uint64_t), 1, &obj, tx));
429 	spa_feature_incr(dp->dp_spa, SPA_FEATURE_OBSOLETE_COUNTS, tx);
430 }
431 
432 void
dsl_pool_destroy_obsolete_bpobj(dsl_pool_t * dp,dmu_tx_t * tx)433 dsl_pool_destroy_obsolete_bpobj(dsl_pool_t *dp, dmu_tx_t *tx)
434 {
435 	spa_feature_decr(dp->dp_spa, SPA_FEATURE_OBSOLETE_COUNTS, tx);
436 	VERIFY0(zap_remove(dp->dp_meta_objset,
437 	    DMU_POOL_DIRECTORY_OBJECT,
438 	    DMU_POOL_OBSOLETE_BPOBJ, tx));
439 	bpobj_free(dp->dp_meta_objset,
440 	    dp->dp_obsolete_bpobj.bpo_object, tx);
441 	bpobj_close(&dp->dp_obsolete_bpobj);
442 }
443 
444 dsl_pool_t *
dsl_pool_create(spa_t * spa,nvlist_t * zplprops,dsl_crypto_params_t * dcp,uint64_t txg)445 dsl_pool_create(spa_t *spa, nvlist_t *zplprops, dsl_crypto_params_t *dcp,
446     uint64_t txg)
447 {
448 	int err;
449 	dsl_pool_t *dp = dsl_pool_open_impl(spa, txg);
450 	dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg);
451 	dsl_dataset_t *ds;
452 	uint64_t obj;
453 
454 	rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG);
455 
456 	/* create and open the MOS (meta-objset) */
457 	dp->dp_meta_objset = dmu_objset_create_impl(spa,
458 	    NULL, &dp->dp_meta_rootbp, DMU_OST_META, tx);
459 	spa->spa_meta_objset = dp->dp_meta_objset;
460 
461 	/* create the pool directory */
462 	err = zap_create_claim(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
463 	    DMU_OT_OBJECT_DIRECTORY, DMU_OT_NONE, 0, tx);
464 	ASSERT0(err);
465 
466 	/* Initialize scan structures */
467 	VERIFY0(dsl_scan_init(dp, txg));
468 
469 	/* create and open the root dir */
470 	dp->dp_root_dir_obj = dsl_dir_create_sync(dp, NULL, NULL, tx);
471 	VERIFY0(dsl_dir_hold_obj(dp, dp->dp_root_dir_obj,
472 	    NULL, dp, &dp->dp_root_dir));
473 
474 	/* create and open the meta-objset dir */
475 	(void) dsl_dir_create_sync(dp, dp->dp_root_dir, MOS_DIR_NAME, tx);
476 	VERIFY0(dsl_pool_open_special_dir(dp,
477 	    MOS_DIR_NAME, &dp->dp_mos_dir));
478 
479 	if (spa_version(spa) >= SPA_VERSION_DEADLISTS) {
480 		/* create and open the free dir */
481 		(void) dsl_dir_create_sync(dp, dp->dp_root_dir,
482 		    FREE_DIR_NAME, tx);
483 		VERIFY0(dsl_pool_open_special_dir(dp,
484 		    FREE_DIR_NAME, &dp->dp_free_dir));
485 
486 		/* create and open the free_bplist */
487 		obj = bpobj_alloc(dp->dp_meta_objset, SPA_OLD_MAXBLOCKSIZE, tx);
488 		VERIFY(zap_add(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
489 		    DMU_POOL_FREE_BPOBJ, sizeof (uint64_t), 1, &obj, tx) == 0);
490 		VERIFY0(bpobj_open(&dp->dp_free_bpobj,
491 		    dp->dp_meta_objset, obj));
492 	}
493 
494 	if (spa_version(spa) >= SPA_VERSION_DSL_SCRUB)
495 		dsl_pool_create_origin(dp, tx);
496 
497 	/*
498 	 * Some features may be needed when creating the root dataset, so we
499 	 * create the feature objects here.
500 	 */
501 	if (spa_version(spa) >= SPA_VERSION_FEATURES)
502 		spa_feature_create_zap_objects(spa, tx);
503 
504 	if (dcp != NULL && dcp->cp_crypt != ZIO_CRYPT_OFF &&
505 	    dcp->cp_crypt != ZIO_CRYPT_INHERIT)
506 		spa_feature_enable(spa, SPA_FEATURE_ENCRYPTION, tx);
507 
508 	/* create the root dataset */
509 	obj = dsl_dataset_create_sync_dd(dp->dp_root_dir, NULL, dcp, 0, tx);
510 
511 	/* create the root objset */
512 	VERIFY0(dsl_dataset_hold_obj_flags(dp, obj,
513 	    DS_HOLD_FLAG_DECRYPT, FTAG, &ds));
514 #ifdef _KERNEL
515 	{
516 		objset_t *os;
517 		rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
518 		os = dmu_objset_create_impl(dp->dp_spa, ds,
519 		    dsl_dataset_get_blkptr(ds), DMU_OST_ZFS, tx);
520 		rrw_exit(&ds->ds_bp_rwlock, FTAG);
521 		zfs_create_fs(os, kcred, zplprops, tx);
522 	}
523 #endif
524 	dsl_dataset_rele_flags(ds, DS_HOLD_FLAG_DECRYPT, FTAG);
525 
526 	dmu_tx_commit(tx);
527 
528 	rrw_exit(&dp->dp_config_rwlock, FTAG);
529 
530 	return (dp);
531 }
532 
533 /*
534  * Account for the meta-objset space in its placeholder dsl_dir.
535  */
536 void
dsl_pool_mos_diduse_space(dsl_pool_t * dp,int64_t used,int64_t comp,int64_t uncomp)537 dsl_pool_mos_diduse_space(dsl_pool_t *dp,
538     int64_t used, int64_t comp, int64_t uncomp)
539 {
540 	ASSERT3U(comp, ==, uncomp); /* it's all metadata */
541 	mutex_enter(&dp->dp_lock);
542 	dp->dp_mos_used_delta += used;
543 	dp->dp_mos_compressed_delta += comp;
544 	dp->dp_mos_uncompressed_delta += uncomp;
545 	mutex_exit(&dp->dp_lock);
546 }
547 
548 static void
dsl_pool_sync_mos(dsl_pool_t * dp,dmu_tx_t * tx)549 dsl_pool_sync_mos(dsl_pool_t *dp, dmu_tx_t *tx)
550 {
551 	zio_t *zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
552 	dmu_objset_sync(dp->dp_meta_objset, zio, tx);
553 	VERIFY0(zio_wait(zio));
554 	dprintf_bp(&dp->dp_meta_rootbp, "meta objset rootbp is %s", "");
555 	spa_set_rootblkptr(dp->dp_spa, &dp->dp_meta_rootbp);
556 }
557 
558 static void
dsl_pool_dirty_delta(dsl_pool_t * dp,int64_t delta)559 dsl_pool_dirty_delta(dsl_pool_t *dp, int64_t delta)
560 {
561 	ASSERT(MUTEX_HELD(&dp->dp_lock));
562 
563 	if (delta < 0)
564 		ASSERT3U(-delta, <=, dp->dp_dirty_total);
565 
566 	dp->dp_dirty_total += delta;
567 
568 	/*
569 	 * Note: we signal even when increasing dp_dirty_total.
570 	 * This ensures forward progress -- each thread wakes the next waiter.
571 	 */
572 	if (dp->dp_dirty_total < zfs_dirty_data_max)
573 		cv_signal(&dp->dp_spaceavail_cv);
574 }
575 
576 static boolean_t
dsl_early_sync_task_verify(dsl_pool_t * dp,uint64_t txg)577 dsl_early_sync_task_verify(dsl_pool_t *dp, uint64_t txg)
578 {
579 	spa_t *spa = dp->dp_spa;
580 	vdev_t *rvd = spa->spa_root_vdev;
581 
582 	for (uint64_t c = 0; c < rvd->vdev_children; c++) {
583 		vdev_t *vd = rvd->vdev_child[c];
584 		txg_list_t *tl = &vd->vdev_ms_list;
585 		metaslab_t *ms;
586 
587 		for (ms = txg_list_head(tl, TXG_CLEAN(txg)); ms;
588 		    ms = txg_list_next(tl, ms, TXG_CLEAN(txg))) {
589 			VERIFY(range_tree_is_empty(ms->ms_freeing));
590 			VERIFY(range_tree_is_empty(ms->ms_checkpointing));
591 		}
592 	}
593 
594 	return (B_TRUE);
595 }
596 
597 void
dsl_pool_sync(dsl_pool_t * dp,uint64_t txg)598 dsl_pool_sync(dsl_pool_t *dp, uint64_t txg)
599 {
600 	zio_t *zio;
601 	dmu_tx_t *tx;
602 	dsl_dir_t *dd;
603 	dsl_dataset_t *ds;
604 	objset_t *mos = dp->dp_meta_objset;
605 	list_t synced_datasets;
606 
607 	list_create(&synced_datasets, sizeof (dsl_dataset_t),
608 	    offsetof(dsl_dataset_t, ds_synced_link));
609 
610 	tx = dmu_tx_create_assigned(dp, txg);
611 
612 	/*
613 	 * Run all early sync tasks before writing out any dirty blocks.
614 	 * For more info on early sync tasks see block comment in
615 	 * dsl_early_sync_task().
616 	 */
617 	if (!txg_list_empty(&dp->dp_early_sync_tasks, txg)) {
618 		dsl_sync_task_t *dst;
619 
620 		ASSERT3U(spa_sync_pass(dp->dp_spa), ==, 1);
621 		while ((dst =
622 		    txg_list_remove(&dp->dp_early_sync_tasks, txg)) != NULL) {
623 			ASSERT(dsl_early_sync_task_verify(dp, txg));
624 			dsl_sync_task_sync(dst, tx);
625 		}
626 		ASSERT(dsl_early_sync_task_verify(dp, txg));
627 	}
628 
629 	/*
630 	 * Write out all dirty blocks of dirty datasets.
631 	 */
632 	zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
633 	while ((ds = txg_list_remove(&dp->dp_dirty_datasets, txg)) != NULL) {
634 		/*
635 		 * We must not sync any non-MOS datasets twice, because
636 		 * we may have taken a snapshot of them.  However, we
637 		 * may sync newly-created datasets on pass 2.
638 		 */
639 		ASSERT(!list_link_active(&ds->ds_synced_link));
640 		list_insert_tail(&synced_datasets, ds);
641 		dsl_dataset_sync(ds, zio, tx);
642 	}
643 	VERIFY0(zio_wait(zio));
644 
645 	/*
646 	 * We have written all of the accounted dirty data, so our
647 	 * dp_space_towrite should now be zero.  However, some seldom-used
648 	 * code paths do not adhere to this (e.g. dbuf_undirty(), also
649 	 * rounding error in dbuf_write_physdone).
650 	 * Shore up the accounting of any dirtied space now.
651 	 */
652 	dsl_pool_undirty_space(dp, dp->dp_dirty_pertxg[txg & TXG_MASK], txg);
653 
654 	/*
655 	 * Update the long range free counter after
656 	 * we're done syncing user data
657 	 */
658 	mutex_enter(&dp->dp_lock);
659 	ASSERT(spa_sync_pass(dp->dp_spa) == 1 ||
660 	    dp->dp_long_free_dirty_pertxg[txg & TXG_MASK] == 0);
661 	dp->dp_long_free_dirty_pertxg[txg & TXG_MASK] = 0;
662 	mutex_exit(&dp->dp_lock);
663 
664 	/*
665 	 * After the data blocks have been written (ensured by the zio_wait()
666 	 * above), update the user/group/project space accounting.  This happens
667 	 * in tasks dispatched to dp_sync_taskq, so wait for them before
668 	 * continuing.
669 	 */
670 	for (ds = list_head(&synced_datasets); ds != NULL;
671 	    ds = list_next(&synced_datasets, ds)) {
672 		dmu_objset_do_userquota_updates(ds->ds_objset, tx);
673 	}
674 	taskq_wait(dp->dp_sync_taskq);
675 
676 	/*
677 	 * Sync the datasets again to push out the changes due to
678 	 * userspace updates.  This must be done before we process the
679 	 * sync tasks, so that any snapshots will have the correct
680 	 * user accounting information (and we won't get confused
681 	 * about which blocks are part of the snapshot).
682 	 */
683 	zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
684 	while ((ds = txg_list_remove(&dp->dp_dirty_datasets, txg)) != NULL) {
685 		objset_t *os = ds->ds_objset;
686 
687 		ASSERT(list_link_active(&ds->ds_synced_link));
688 		dmu_buf_rele(ds->ds_dbuf, ds);
689 		dsl_dataset_sync(ds, zio, tx);
690 
691 		/*
692 		 * Release any key mappings created by calls to
693 		 * dsl_dataset_dirty() from the userquota accounting
694 		 * code paths.
695 		 */
696 		if (os->os_encrypted && !os->os_raw_receive &&
697 		    !os->os_next_write_raw[txg & TXG_MASK]) {
698 			ASSERT3P(ds->ds_key_mapping, !=, NULL);
699 			key_mapping_rele(dp->dp_spa, ds->ds_key_mapping, ds);
700 		}
701 	}
702 	VERIFY0(zio_wait(zio));
703 
704 	/*
705 	 * Now that the datasets have been completely synced, we can
706 	 * clean up our in-memory structures accumulated while syncing:
707 	 *
708 	 *  - move dead blocks from the pending deadlist to the on-disk deadlist
709 	 *  - release hold from dsl_dataset_dirty()
710 	 *  - release key mapping hold from dsl_dataset_dirty()
711 	 */
712 	while ((ds = list_remove_head(&synced_datasets)) != NULL) {
713 		objset_t *os = ds->ds_objset;
714 
715 		if (os->os_encrypted && !os->os_raw_receive &&
716 		    !os->os_next_write_raw[txg & TXG_MASK]) {
717 			ASSERT3P(ds->ds_key_mapping, !=, NULL);
718 			key_mapping_rele(dp->dp_spa, ds->ds_key_mapping, ds);
719 		}
720 
721 		dsl_dataset_sync_done(ds, tx);
722 	}
723 	while ((dd = txg_list_remove(&dp->dp_dirty_dirs, txg)) != NULL) {
724 		dsl_dir_sync(dd, tx);
725 	}
726 
727 	/*
728 	 * The MOS's space is accounted for in the pool/$MOS
729 	 * (dp_mos_dir).  We can't modify the mos while we're syncing
730 	 * it, so we remember the deltas and apply them here.
731 	 */
732 	if (dp->dp_mos_used_delta != 0 || dp->dp_mos_compressed_delta != 0 ||
733 	    dp->dp_mos_uncompressed_delta != 0) {
734 		dsl_dir_diduse_space(dp->dp_mos_dir, DD_USED_HEAD,
735 		    dp->dp_mos_used_delta,
736 		    dp->dp_mos_compressed_delta,
737 		    dp->dp_mos_uncompressed_delta, tx);
738 		dp->dp_mos_used_delta = 0;
739 		dp->dp_mos_compressed_delta = 0;
740 		dp->dp_mos_uncompressed_delta = 0;
741 	}
742 
743 	if (dmu_objset_is_dirty(mos, txg)) {
744 		dsl_pool_sync_mos(dp, tx);
745 	}
746 
747 	/*
748 	 * If we modify a dataset in the same txg that we want to destroy it,
749 	 * its dsl_dir's dd_dbuf will be dirty, and thus have a hold on it.
750 	 * dsl_dir_destroy_check() will fail if there are unexpected holds.
751 	 * Therefore, we want to sync the MOS (thus syncing the dd_dbuf
752 	 * and clearing the hold on it) before we process the sync_tasks.
753 	 * The MOS data dirtied by the sync_tasks will be synced on the next
754 	 * pass.
755 	 */
756 	if (!txg_list_empty(&dp->dp_sync_tasks, txg)) {
757 		dsl_sync_task_t *dst;
758 		/*
759 		 * No more sync tasks should have been added while we
760 		 * were syncing.
761 		 */
762 		ASSERT3U(spa_sync_pass(dp->dp_spa), ==, 1);
763 		while ((dst = txg_list_remove(&dp->dp_sync_tasks, txg)) != NULL)
764 			dsl_sync_task_sync(dst, tx);
765 	}
766 
767 	dmu_tx_commit(tx);
768 
769 	DTRACE_PROBE2(dsl_pool_sync__done, dsl_pool_t *dp, dp, uint64_t, txg);
770 }
771 
772 void
dsl_pool_sync_done(dsl_pool_t * dp,uint64_t txg)773 dsl_pool_sync_done(dsl_pool_t *dp, uint64_t txg)
774 {
775 	zilog_t *zilog;
776 
777 	while (zilog = txg_list_head(&dp->dp_dirty_zilogs, txg)) {
778 		dsl_dataset_t *ds = dmu_objset_ds(zilog->zl_os);
779 		/*
780 		 * We don't remove the zilog from the dp_dirty_zilogs
781 		 * list until after we've cleaned it. This ensures that
782 		 * callers of zilog_is_dirty() receive an accurate
783 		 * answer when they are racing with the spa sync thread.
784 		 */
785 		zil_clean(zilog, txg);
786 		(void) txg_list_remove_this(&dp->dp_dirty_zilogs, zilog, txg);
787 		ASSERT(!dmu_objset_is_dirty(zilog->zl_os, txg));
788 		dmu_buf_rele(ds->ds_dbuf, zilog);
789 	}
790 	ASSERT(!dmu_objset_is_dirty(dp->dp_meta_objset, txg));
791 }
792 
793 /*
794  * TRUE if the current thread is the tx_sync_thread or if we
795  * are being called from SPA context during pool initialization.
796  */
797 int
dsl_pool_sync_context(dsl_pool_t * dp)798 dsl_pool_sync_context(dsl_pool_t *dp)
799 {
800 	return (curthread == dp->dp_tx.tx_sync_thread ||
801 	    spa_is_initializing(dp->dp_spa) ||
802 	    taskq_member(dp->dp_sync_taskq, curthread));
803 }
804 
805 /*
806  * This function returns the amount of allocatable space in the pool
807  * minus whatever space is currently reserved by ZFS for specific
808  * purposes. Specifically:
809  *
810  * 1] Any reserved SLOP space
811  * 2] Any space used by the checkpoint
812  * 3] Any space used for deferred frees
813  *
814  * The latter 2 are especially important because they are needed to
815  * rectify the SPA's and DMU's different understanding of how much space
816  * is used. Now the DMU is aware of that extra space tracked by the SPA
817  * without having to maintain a separate special dir (e.g similar to
818  * $MOS, $FREEING, and $LEAKED).
819  *
820  * Note: By deferred frees here, we mean the frees that were deferred
821  * in spa_sync() after sync pass 1 (spa_deferred_bpobj), and not the
822  * segments placed in ms_defer trees during metaslab_sync_done().
823  */
824 uint64_t
dsl_pool_adjustedsize(dsl_pool_t * dp,zfs_space_check_t slop_policy)825 dsl_pool_adjustedsize(dsl_pool_t *dp, zfs_space_check_t slop_policy)
826 {
827 	spa_t *spa = dp->dp_spa;
828 	uint64_t space, resv, adjustedsize;
829 	uint64_t spa_deferred_frees =
830 	    spa->spa_deferred_bpobj.bpo_phys->bpo_bytes;
831 
832 	space = spa_get_dspace(spa)
833 	    - spa_get_checkpoint_space(spa) - spa_deferred_frees;
834 	resv = spa_get_slop_space(spa);
835 
836 	switch (slop_policy) {
837 	case ZFS_SPACE_CHECK_NORMAL:
838 		break;
839 	case ZFS_SPACE_CHECK_RESERVED:
840 		resv >>= 1;
841 		break;
842 	case ZFS_SPACE_CHECK_EXTRA_RESERVED:
843 		resv >>= 2;
844 		break;
845 	case ZFS_SPACE_CHECK_NONE:
846 		resv = 0;
847 		break;
848 	default:
849 		panic("invalid slop policy value: %d", slop_policy);
850 		break;
851 	}
852 	adjustedsize = (space >= resv) ? (space - resv) : 0;
853 
854 	return (adjustedsize);
855 }
856 
857 uint64_t
dsl_pool_unreserved_space(dsl_pool_t * dp,zfs_space_check_t slop_policy)858 dsl_pool_unreserved_space(dsl_pool_t *dp, zfs_space_check_t slop_policy)
859 {
860 	uint64_t poolsize = dsl_pool_adjustedsize(dp, slop_policy);
861 	uint64_t deferred =
862 	    metaslab_class_get_deferred(spa_normal_class(dp->dp_spa));
863 	uint64_t quota = (poolsize >= deferred) ? (poolsize - deferred) : 0;
864 	return (quota);
865 }
866 
867 boolean_t
dsl_pool_need_dirty_delay(dsl_pool_t * dp)868 dsl_pool_need_dirty_delay(dsl_pool_t *dp)
869 {
870 	uint64_t delay_min_bytes =
871 	    zfs_dirty_data_max * zfs_delay_min_dirty_percent / 100;
872 	uint64_t dirty_min_bytes =
873 	    zfs_dirty_data_max * zfs_dirty_data_sync_pct / 100;
874 	boolean_t rv;
875 
876 	mutex_enter(&dp->dp_lock);
877 	if (dp->dp_dirty_total > dirty_min_bytes)
878 		txg_kick(dp);
879 	rv = (dp->dp_dirty_total > delay_min_bytes);
880 	mutex_exit(&dp->dp_lock);
881 	return (rv);
882 }
883 
884 void
dsl_pool_dirty_space(dsl_pool_t * dp,int64_t space,dmu_tx_t * tx)885 dsl_pool_dirty_space(dsl_pool_t *dp, int64_t space, dmu_tx_t *tx)
886 {
887 	if (space > 0) {
888 		mutex_enter(&dp->dp_lock);
889 		dp->dp_dirty_pertxg[tx->tx_txg & TXG_MASK] += space;
890 		dsl_pool_dirty_delta(dp, space);
891 		mutex_exit(&dp->dp_lock);
892 	}
893 }
894 
895 void
dsl_pool_undirty_space(dsl_pool_t * dp,int64_t space,uint64_t txg)896 dsl_pool_undirty_space(dsl_pool_t *dp, int64_t space, uint64_t txg)
897 {
898 	ASSERT3S(space, >=, 0);
899 	if (space == 0)
900 		return;
901 	mutex_enter(&dp->dp_lock);
902 	if (dp->dp_dirty_pertxg[txg & TXG_MASK] < space) {
903 		/* XXX writing something we didn't dirty? */
904 		space = dp->dp_dirty_pertxg[txg & TXG_MASK];
905 	}
906 	ASSERT3U(dp->dp_dirty_pertxg[txg & TXG_MASK], >=, space);
907 	dp->dp_dirty_pertxg[txg & TXG_MASK] -= space;
908 	ASSERT3U(dp->dp_dirty_total, >=, space);
909 	dsl_pool_dirty_delta(dp, -space);
910 	mutex_exit(&dp->dp_lock);
911 }
912 
913 /* ARGSUSED */
914 static int
upgrade_clones_cb(dsl_pool_t * dp,dsl_dataset_t * hds,void * arg)915 upgrade_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg)
916 {
917 	dmu_tx_t *tx = arg;
918 	dsl_dataset_t *ds, *prev = NULL;
919 	int err;
920 
921 	err = dsl_dataset_hold_obj(dp, hds->ds_object, FTAG, &ds);
922 	if (err)
923 		return (err);
924 
925 	while (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
926 		err = dsl_dataset_hold_obj(dp,
927 		    dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev);
928 		if (err) {
929 			dsl_dataset_rele(ds, FTAG);
930 			return (err);
931 		}
932 
933 		if (dsl_dataset_phys(prev)->ds_next_snap_obj != ds->ds_object)
934 			break;
935 		dsl_dataset_rele(ds, FTAG);
936 		ds = prev;
937 		prev = NULL;
938 	}
939 
940 	if (prev == NULL) {
941 		prev = dp->dp_origin_snap;
942 
943 		/*
944 		 * The $ORIGIN can't have any data, or the accounting
945 		 * will be wrong.
946 		 */
947 		rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
948 		ASSERT0(dsl_dataset_phys(prev)->ds_bp.blk_birth);
949 		rrw_exit(&ds->ds_bp_rwlock, FTAG);
950 
951 		/* The origin doesn't get attached to itself */
952 		if (ds->ds_object == prev->ds_object) {
953 			dsl_dataset_rele(ds, FTAG);
954 			return (0);
955 		}
956 
957 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
958 		dsl_dataset_phys(ds)->ds_prev_snap_obj = prev->ds_object;
959 		dsl_dataset_phys(ds)->ds_prev_snap_txg =
960 		    dsl_dataset_phys(prev)->ds_creation_txg;
961 
962 		dmu_buf_will_dirty(ds->ds_dir->dd_dbuf, tx);
963 		dsl_dir_phys(ds->ds_dir)->dd_origin_obj = prev->ds_object;
964 
965 		dmu_buf_will_dirty(prev->ds_dbuf, tx);
966 		dsl_dataset_phys(prev)->ds_num_children++;
967 
968 		if (dsl_dataset_phys(ds)->ds_next_snap_obj == 0) {
969 			ASSERT(ds->ds_prev == NULL);
970 			VERIFY0(dsl_dataset_hold_obj(dp,
971 			    dsl_dataset_phys(ds)->ds_prev_snap_obj,
972 			    ds, &ds->ds_prev));
973 		}
974 	}
975 
976 	ASSERT3U(dsl_dir_phys(ds->ds_dir)->dd_origin_obj, ==, prev->ds_object);
977 	ASSERT3U(dsl_dataset_phys(ds)->ds_prev_snap_obj, ==, prev->ds_object);
978 
979 	if (dsl_dataset_phys(prev)->ds_next_clones_obj == 0) {
980 		dmu_buf_will_dirty(prev->ds_dbuf, tx);
981 		dsl_dataset_phys(prev)->ds_next_clones_obj =
982 		    zap_create(dp->dp_meta_objset,
983 		    DMU_OT_NEXT_CLONES, DMU_OT_NONE, 0, tx);
984 	}
985 	VERIFY0(zap_add_int(dp->dp_meta_objset,
986 	    dsl_dataset_phys(prev)->ds_next_clones_obj, ds->ds_object, tx));
987 
988 	dsl_dataset_rele(ds, FTAG);
989 	if (prev != dp->dp_origin_snap)
990 		dsl_dataset_rele(prev, FTAG);
991 	return (0);
992 }
993 
994 void
dsl_pool_upgrade_clones(dsl_pool_t * dp,dmu_tx_t * tx)995 dsl_pool_upgrade_clones(dsl_pool_t *dp, dmu_tx_t *tx)
996 {
997 	ASSERT(dmu_tx_is_syncing(tx));
998 	ASSERT(dp->dp_origin_snap != NULL);
999 
1000 	VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, upgrade_clones_cb,
1001 	    tx, DS_FIND_CHILDREN | DS_FIND_SERIALIZE));
1002 }
1003 
1004 /* ARGSUSED */
1005 static int
upgrade_dir_clones_cb(dsl_pool_t * dp,dsl_dataset_t * ds,void * arg)1006 upgrade_dir_clones_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
1007 {
1008 	dmu_tx_t *tx = arg;
1009 	objset_t *mos = dp->dp_meta_objset;
1010 
1011 	if (dsl_dir_phys(ds->ds_dir)->dd_origin_obj != 0) {
1012 		dsl_dataset_t *origin;
1013 
1014 		VERIFY0(dsl_dataset_hold_obj(dp,
1015 		    dsl_dir_phys(ds->ds_dir)->dd_origin_obj, FTAG, &origin));
1016 
1017 		if (dsl_dir_phys(origin->ds_dir)->dd_clones == 0) {
1018 			dmu_buf_will_dirty(origin->ds_dir->dd_dbuf, tx);
1019 			dsl_dir_phys(origin->ds_dir)->dd_clones =
1020 			    zap_create(mos, DMU_OT_DSL_CLONES, DMU_OT_NONE,
1021 			    0, tx);
1022 		}
1023 
1024 		VERIFY0(zap_add_int(dp->dp_meta_objset,
1025 		    dsl_dir_phys(origin->ds_dir)->dd_clones,
1026 		    ds->ds_object, tx));
1027 
1028 		dsl_dataset_rele(origin, FTAG);
1029 	}
1030 	return (0);
1031 }
1032 
1033 void
dsl_pool_upgrade_dir_clones(dsl_pool_t * dp,dmu_tx_t * tx)1034 dsl_pool_upgrade_dir_clones(dsl_pool_t *dp, dmu_tx_t *tx)
1035 {
1036 	ASSERT(dmu_tx_is_syncing(tx));
1037 	uint64_t obj;
1038 
1039 	(void) dsl_dir_create_sync(dp, dp->dp_root_dir, FREE_DIR_NAME, tx);
1040 	VERIFY0(dsl_pool_open_special_dir(dp,
1041 	    FREE_DIR_NAME, &dp->dp_free_dir));
1042 
1043 	/*
1044 	 * We can't use bpobj_alloc(), because spa_version() still
1045 	 * returns the old version, and we need a new-version bpobj with
1046 	 * subobj support.  So call dmu_object_alloc() directly.
1047 	 */
1048 	obj = dmu_object_alloc(dp->dp_meta_objset, DMU_OT_BPOBJ,
1049 	    SPA_OLD_MAXBLOCKSIZE, DMU_OT_BPOBJ_HDR, sizeof (bpobj_phys_t), tx);
1050 	VERIFY0(zap_add(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
1051 	    DMU_POOL_FREE_BPOBJ, sizeof (uint64_t), 1, &obj, tx));
1052 	VERIFY0(bpobj_open(&dp->dp_free_bpobj, dp->dp_meta_objset, obj));
1053 
1054 	VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
1055 	    upgrade_dir_clones_cb, tx, DS_FIND_CHILDREN | DS_FIND_SERIALIZE));
1056 }
1057 
1058 void
dsl_pool_create_origin(dsl_pool_t * dp,dmu_tx_t * tx)1059 dsl_pool_create_origin(dsl_pool_t *dp, dmu_tx_t *tx)
1060 {
1061 	uint64_t dsobj;
1062 	dsl_dataset_t *ds;
1063 
1064 	ASSERT(dmu_tx_is_syncing(tx));
1065 	ASSERT(dp->dp_origin_snap == NULL);
1066 	ASSERT(rrw_held(&dp->dp_config_rwlock, RW_WRITER));
1067 
1068 	/* create the origin dir, ds, & snap-ds */
1069 	dsobj = dsl_dataset_create_sync(dp->dp_root_dir, ORIGIN_DIR_NAME,
1070 	    NULL, 0, kcred, NULL, tx);
1071 	VERIFY0(dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds));
1072 	dsl_dataset_snapshot_sync_impl(ds, ORIGIN_DIR_NAME, tx);
1073 	VERIFY0(dsl_dataset_hold_obj(dp, dsl_dataset_phys(ds)->ds_prev_snap_obj,
1074 	    dp, &dp->dp_origin_snap));
1075 	dsl_dataset_rele(ds, FTAG);
1076 }
1077 
1078 taskq_t *
dsl_pool_vnrele_taskq(dsl_pool_t * dp)1079 dsl_pool_vnrele_taskq(dsl_pool_t *dp)
1080 {
1081 	return (dp->dp_vnrele_taskq);
1082 }
1083 
1084 taskq_t *
dsl_pool_unlinked_drain_taskq(dsl_pool_t * dp)1085 dsl_pool_unlinked_drain_taskq(dsl_pool_t *dp)
1086 {
1087 	return (dp->dp_unlinked_drain_taskq);
1088 }
1089 
1090 /*
1091  * Walk through the pool-wide zap object of temporary snapshot user holds
1092  * and release them.
1093  */
1094 void
dsl_pool_clean_tmp_userrefs(dsl_pool_t * dp)1095 dsl_pool_clean_tmp_userrefs(dsl_pool_t *dp)
1096 {
1097 	zap_attribute_t za;
1098 	zap_cursor_t zc;
1099 	objset_t *mos = dp->dp_meta_objset;
1100 	uint64_t zapobj = dp->dp_tmp_userrefs_obj;
1101 	nvlist_t *holds;
1102 
1103 	if (zapobj == 0)
1104 		return;
1105 	ASSERT(spa_version(dp->dp_spa) >= SPA_VERSION_USERREFS);
1106 
1107 	holds = fnvlist_alloc();
1108 
1109 	for (zap_cursor_init(&zc, mos, zapobj);
1110 	    zap_cursor_retrieve(&zc, &za) == 0;
1111 	    zap_cursor_advance(&zc)) {
1112 		char *htag;
1113 		nvlist_t *tags;
1114 
1115 		htag = strchr(za.za_name, '-');
1116 		*htag = '\0';
1117 		++htag;
1118 		if (nvlist_lookup_nvlist(holds, za.za_name, &tags) != 0) {
1119 			tags = fnvlist_alloc();
1120 			fnvlist_add_boolean(tags, htag);
1121 			fnvlist_add_nvlist(holds, za.za_name, tags);
1122 			fnvlist_free(tags);
1123 		} else {
1124 			fnvlist_add_boolean(tags, htag);
1125 		}
1126 	}
1127 	dsl_dataset_user_release_tmp(dp, holds);
1128 	fnvlist_free(holds);
1129 	zap_cursor_fini(&zc);
1130 }
1131 
1132 /*
1133  * Create the pool-wide zap object for storing temporary snapshot holds.
1134  */
1135 void
dsl_pool_user_hold_create_obj(dsl_pool_t * dp,dmu_tx_t * tx)1136 dsl_pool_user_hold_create_obj(dsl_pool_t *dp, dmu_tx_t *tx)
1137 {
1138 	objset_t *mos = dp->dp_meta_objset;
1139 
1140 	ASSERT(dp->dp_tmp_userrefs_obj == 0);
1141 	ASSERT(dmu_tx_is_syncing(tx));
1142 
1143 	dp->dp_tmp_userrefs_obj = zap_create_link(mos, DMU_OT_USERREFS,
1144 	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_TMP_USERREFS, tx);
1145 }
1146 
1147 static int
dsl_pool_user_hold_rele_impl(dsl_pool_t * dp,uint64_t dsobj,const char * tag,uint64_t now,dmu_tx_t * tx,boolean_t holding)1148 dsl_pool_user_hold_rele_impl(dsl_pool_t *dp, uint64_t dsobj,
1149     const char *tag, uint64_t now, dmu_tx_t *tx, boolean_t holding)
1150 {
1151 	objset_t *mos = dp->dp_meta_objset;
1152 	uint64_t zapobj = dp->dp_tmp_userrefs_obj;
1153 	char *name;
1154 	int error;
1155 
1156 	ASSERT(spa_version(dp->dp_spa) >= SPA_VERSION_USERREFS);
1157 	ASSERT(dmu_tx_is_syncing(tx));
1158 
1159 	/*
1160 	 * If the pool was created prior to SPA_VERSION_USERREFS, the
1161 	 * zap object for temporary holds might not exist yet.
1162 	 */
1163 	if (zapobj == 0) {
1164 		if (holding) {
1165 			dsl_pool_user_hold_create_obj(dp, tx);
1166 			zapobj = dp->dp_tmp_userrefs_obj;
1167 		} else {
1168 			return (SET_ERROR(ENOENT));
1169 		}
1170 	}
1171 
1172 	name = kmem_asprintf("%llx-%s", (u_longlong_t)dsobj, tag);
1173 	if (holding)
1174 		error = zap_add(mos, zapobj, name, 8, 1, &now, tx);
1175 	else
1176 		error = zap_remove(mos, zapobj, name, tx);
1177 	strfree(name);
1178 
1179 	return (error);
1180 }
1181 
1182 /*
1183  * Add a temporary hold for the given dataset object and tag.
1184  */
1185 int
dsl_pool_user_hold(dsl_pool_t * dp,uint64_t dsobj,const char * tag,uint64_t now,dmu_tx_t * tx)1186 dsl_pool_user_hold(dsl_pool_t *dp, uint64_t dsobj, const char *tag,
1187     uint64_t now, dmu_tx_t *tx)
1188 {
1189 	return (dsl_pool_user_hold_rele_impl(dp, dsobj, tag, now, tx, B_TRUE));
1190 }
1191 
1192 /*
1193  * Release a temporary hold for the given dataset object and tag.
1194  */
1195 int
dsl_pool_user_release(dsl_pool_t * dp,uint64_t dsobj,const char * tag,dmu_tx_t * tx)1196 dsl_pool_user_release(dsl_pool_t *dp, uint64_t dsobj, const char *tag,
1197     dmu_tx_t *tx)
1198 {
1199 	return (dsl_pool_user_hold_rele_impl(dp, dsobj, tag, 0, tx, B_FALSE));
1200 }
1201 
1202 /*
1203  * DSL Pool Configuration Lock
1204  *
1205  * The dp_config_rwlock protects against changes to DSL state (e.g. dataset
1206  * creation / destruction / rename / property setting).  It must be held for
1207  * read to hold a dataset or dsl_dir.  I.e. you must call
1208  * dsl_pool_config_enter() or dsl_pool_hold() before calling
1209  * dsl_{dataset,dir}_hold{_obj}.  In most circumstances, the dp_config_rwlock
1210  * must be held continuously until all datasets and dsl_dirs are released.
1211  *
1212  * The only exception to this rule is that if a "long hold" is placed on
1213  * a dataset, then the dp_config_rwlock may be dropped while the dataset
1214  * is still held.  The long hold will prevent the dataset from being
1215  * destroyed -- the destroy will fail with EBUSY.  A long hold can be
1216  * obtained by calling dsl_dataset_long_hold(), or by "owning" a dataset
1217  * (by calling dsl_{dataset,objset}_{try}own{_obj}).
1218  *
1219  * Legitimate long-holders (including owners) should be long-running, cancelable
1220  * tasks that should cause "zfs destroy" to fail.  This includes DMU
1221  * consumers (i.e. a ZPL filesystem being mounted or ZVOL being open),
1222  * "zfs send", and "zfs diff".  There are several other long-holders whose
1223  * uses are suboptimal (e.g. "zfs promote", and zil_suspend()).
1224  *
1225  * The usual formula for long-holding would be:
1226  * dsl_pool_hold()
1227  * dsl_dataset_hold()
1228  * ... perform checks ...
1229  * dsl_dataset_long_hold()
1230  * dsl_pool_rele()
1231  * ... perform long-running task ...
1232  * dsl_dataset_long_rele()
1233  * dsl_dataset_rele()
1234  *
1235  * Note that when the long hold is released, the dataset is still held but
1236  * the pool is not held.  The dataset may change arbitrarily during this time
1237  * (e.g. it could be destroyed).  Therefore you shouldn't do anything to the
1238  * dataset except release it.
1239  *
1240  * User-initiated operations (e.g. ioctls, zfs_ioc_*()) are either read-only
1241  * or modifying operations.
1242  *
1243  * Modifying operations should generally use dsl_sync_task().  The synctask
1244  * infrastructure enforces proper locking strategy with respect to the
1245  * dp_config_rwlock.  See the comment above dsl_sync_task() for details.
1246  *
1247  * Read-only operations will manually hold the pool, then the dataset, obtain
1248  * information from the dataset, then release the pool and dataset.
1249  * dmu_objset_{hold,rele}() are convenience routines that also do the pool
1250  * hold/rele.
1251  */
1252 
1253 int
dsl_pool_hold(const char * name,void * tag,dsl_pool_t ** dp)1254 dsl_pool_hold(const char *name, void *tag, dsl_pool_t **dp)
1255 {
1256 	spa_t *spa;
1257 	int error;
1258 
1259 	error = spa_open(name, &spa, tag);
1260 	if (error == 0) {
1261 		*dp = spa_get_dsl(spa);
1262 		dsl_pool_config_enter(*dp, tag);
1263 	}
1264 	return (error);
1265 }
1266 
1267 void
dsl_pool_rele(dsl_pool_t * dp,void * tag)1268 dsl_pool_rele(dsl_pool_t *dp, void *tag)
1269 {
1270 	dsl_pool_config_exit(dp, tag);
1271 	spa_close(dp->dp_spa, tag);
1272 }
1273 
1274 void
dsl_pool_config_enter(dsl_pool_t * dp,void * tag)1275 dsl_pool_config_enter(dsl_pool_t *dp, void *tag)
1276 {
1277 	/*
1278 	 * We use a "reentrant" reader-writer lock, but not reentrantly.
1279 	 *
1280 	 * The rrwlock can (with the track_all flag) track all reading threads,
1281 	 * which is very useful for debugging which code path failed to release
1282 	 * the lock, and for verifying that the *current* thread does hold
1283 	 * the lock.
1284 	 *
1285 	 * (Unlike a rwlock, which knows that N threads hold it for
1286 	 * read, but not *which* threads, so rw_held(RW_READER) returns TRUE
1287 	 * if any thread holds it for read, even if this thread doesn't).
1288 	 */
1289 	ASSERT(!rrw_held(&dp->dp_config_rwlock, RW_READER));
1290 	rrw_enter(&dp->dp_config_rwlock, RW_READER, tag);
1291 }
1292 
1293 void
dsl_pool_config_enter_prio(dsl_pool_t * dp,void * tag)1294 dsl_pool_config_enter_prio(dsl_pool_t *dp, void *tag)
1295 {
1296 	ASSERT(!rrw_held(&dp->dp_config_rwlock, RW_READER));
1297 	rrw_enter_read_prio(&dp->dp_config_rwlock, tag);
1298 }
1299 
1300 void
dsl_pool_config_exit(dsl_pool_t * dp,void * tag)1301 dsl_pool_config_exit(dsl_pool_t *dp, void *tag)
1302 {
1303 	rrw_exit(&dp->dp_config_rwlock, tag);
1304 }
1305 
1306 boolean_t
dsl_pool_config_held(dsl_pool_t * dp)1307 dsl_pool_config_held(dsl_pool_t *dp)
1308 {
1309 	return (RRW_LOCK_HELD(&dp->dp_config_rwlock));
1310 }
1311 
1312 boolean_t
dsl_pool_config_held_writer(dsl_pool_t * dp)1313 dsl_pool_config_held_writer(dsl_pool_t *dp)
1314 {
1315 	return (RRW_WRITE_HELD(&dp->dp_config_rwlock));
1316 }
1317