xref: /illumos-gate/usr/src/uts/common/fs/zfs/spa.c (revision bc9014e6a81272073b9854d9f65dd59e18d18c35)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
25  * Copyright (c) 2013, 2014, Nexenta Systems, Inc.  All rights reserved.
26  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
27  */
28 
29 /*
30  * SPA: Storage Pool Allocator
31  *
32  * This file contains all the routines used when modifying on-disk SPA state.
33  * This includes opening, importing, destroying, exporting a pool, and syncing a
34  * pool.
35  */
36 
37 #include <sys/zfs_context.h>
38 #include <sys/fm/fs/zfs.h>
39 #include <sys/spa_impl.h>
40 #include <sys/zio.h>
41 #include <sys/zio_checksum.h>
42 #include <sys/dmu.h>
43 #include <sys/dmu_tx.h>
44 #include <sys/zap.h>
45 #include <sys/zil.h>
46 #include <sys/ddt.h>
47 #include <sys/vdev_impl.h>
48 #include <sys/metaslab.h>
49 #include <sys/metaslab_impl.h>
50 #include <sys/uberblock_impl.h>
51 #include <sys/txg.h>
52 #include <sys/avl.h>
53 #include <sys/dmu_traverse.h>
54 #include <sys/dmu_objset.h>
55 #include <sys/unique.h>
56 #include <sys/dsl_pool.h>
57 #include <sys/dsl_dataset.h>
58 #include <sys/dsl_dir.h>
59 #include <sys/dsl_prop.h>
60 #include <sys/dsl_synctask.h>
61 #include <sys/fs/zfs.h>
62 #include <sys/arc.h>
63 #include <sys/callb.h>
64 #include <sys/systeminfo.h>
65 #include <sys/spa_boot.h>
66 #include <sys/zfs_ioctl.h>
67 #include <sys/dsl_scan.h>
68 #include <sys/zfeature.h>
69 #include <sys/dsl_destroy.h>
70 
71 #ifdef	_KERNEL
72 #include <sys/bootprops.h>
73 #include <sys/callb.h>
74 #include <sys/cpupart.h>
75 #include <sys/pool.h>
76 #include <sys/sysdc.h>
77 #include <sys/zone.h>
78 #endif	/* _KERNEL */
79 
80 #include "zfs_prop.h"
81 #include "zfs_comutil.h"
82 
83 /*
84  * The interval, in seconds, at which failed configuration cache file writes
85  * should be retried.
86  */
87 static int zfs_ccw_retry_interval = 300;
88 
89 typedef enum zti_modes {
90 	ZTI_MODE_FIXED,			/* value is # of threads (min 1) */
91 	ZTI_MODE_BATCH,			/* cpu-intensive; value is ignored */
92 	ZTI_MODE_NULL,			/* don't create a taskq */
93 	ZTI_NMODES
94 } zti_modes_t;
95 
96 #define	ZTI_P(n, q)	{ ZTI_MODE_FIXED, (n), (q) }
97 #define	ZTI_BATCH	{ ZTI_MODE_BATCH, 0, 1 }
98 #define	ZTI_NULL	{ ZTI_MODE_NULL, 0, 0 }
99 
100 #define	ZTI_N(n)	ZTI_P(n, 1)
101 #define	ZTI_ONE		ZTI_N(1)
102 
103 typedef struct zio_taskq_info {
104 	zti_modes_t zti_mode;
105 	uint_t zti_value;
106 	uint_t zti_count;
107 } zio_taskq_info_t;
108 
109 static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = {
110 	"issue", "issue_high", "intr", "intr_high"
111 };
112 
113 /*
114  * This table defines the taskq settings for each ZFS I/O type. When
115  * initializing a pool, we use this table to create an appropriately sized
116  * taskq. Some operations are low volume and therefore have a small, static
117  * number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE
118  * macros. Other operations process a large amount of data; the ZTI_BATCH
119  * macro causes us to create a taskq oriented for throughput. Some operations
120  * are so high frequency and short-lived that the taskq itself can become a a
121  * point of lock contention. The ZTI_P(#, #) macro indicates that we need an
122  * additional degree of parallelism specified by the number of threads per-
123  * taskq and the number of taskqs; when dispatching an event in this case, the
124  * particular taskq is chosen at random.
125  *
126  * The different taskq priorities are to handle the different contexts (issue
127  * and interrupt) and then to reserve threads for ZIO_PRIORITY_NOW I/Os that
128  * need to be handled with minimum delay.
129  */
130 const zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = {
131 	/* ISSUE	ISSUE_HIGH	INTR		INTR_HIGH */
132 	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* NULL */
133 	{ ZTI_N(8),	ZTI_NULL,	ZTI_P(12, 8),	ZTI_NULL }, /* READ */
134 	{ ZTI_BATCH,	ZTI_N(5),	ZTI_N(8),	ZTI_N(5) }, /* WRITE */
135 	{ ZTI_P(12, 8),	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* FREE */
136 	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* CLAIM */
137 	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* IOCTL */
138 };
139 
140 static void spa_sync_version(void *arg, dmu_tx_t *tx);
141 static void spa_sync_props(void *arg, dmu_tx_t *tx);
142 static boolean_t spa_has_active_shared_spare(spa_t *spa);
143 static int spa_load_impl(spa_t *spa, uint64_t, nvlist_t *config,
144     spa_load_state_t state, spa_import_type_t type, boolean_t mosconfig,
145     char **ereport);
146 static void spa_vdev_resilver_done(spa_t *spa);
147 
148 uint_t		zio_taskq_batch_pct = 75;	/* 1 thread per cpu in pset */
149 id_t		zio_taskq_psrset_bind = PS_NONE;
150 boolean_t	zio_taskq_sysdc = B_TRUE;	/* use SDC scheduling class */
151 uint_t		zio_taskq_basedc = 80;		/* base duty cycle */
152 
153 boolean_t	spa_create_process = B_TRUE;	/* no process ==> no sysdc */
154 extern int	zfs_sync_pass_deferred_free;
155 
156 /*
157  * This (illegal) pool name is used when temporarily importing a spa_t in order
158  * to get the vdev stats associated with the imported devices.
159  */
160 #define	TRYIMPORT_NAME	"$import"
161 
162 /*
163  * ==========================================================================
164  * SPA properties routines
165  * ==========================================================================
166  */
167 
168 /*
169  * Add a (source=src, propname=propval) list to an nvlist.
170  */
171 static void
172 spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, char *strval,
173     uint64_t intval, zprop_source_t src)
174 {
175 	const char *propname = zpool_prop_to_name(prop);
176 	nvlist_t *propval;
177 
178 	VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0);
179 	VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0);
180 
181 	if (strval != NULL)
182 		VERIFY(nvlist_add_string(propval, ZPROP_VALUE, strval) == 0);
183 	else
184 		VERIFY(nvlist_add_uint64(propval, ZPROP_VALUE, intval) == 0);
185 
186 	VERIFY(nvlist_add_nvlist(nvl, propname, propval) == 0);
187 	nvlist_free(propval);
188 }
189 
190 /*
191  * Get property values from the spa configuration.
192  */
193 static void
194 spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
195 {
196 	vdev_t *rvd = spa->spa_root_vdev;
197 	dsl_pool_t *pool = spa->spa_dsl_pool;
198 	uint64_t size, alloc, cap, version;
199 	zprop_source_t src = ZPROP_SRC_NONE;
200 	spa_config_dirent_t *dp;
201 	metaslab_class_t *mc = spa_normal_class(spa);
202 
203 	ASSERT(MUTEX_HELD(&spa->spa_props_lock));
204 
205 	if (rvd != NULL) {
206 		alloc = metaslab_class_get_alloc(spa_normal_class(spa));
207 		size = metaslab_class_get_space(spa_normal_class(spa));
208 		spa_prop_add_list(*nvp, ZPOOL_PROP_NAME, spa_name(spa), 0, src);
209 		spa_prop_add_list(*nvp, ZPOOL_PROP_SIZE, NULL, size, src);
210 		spa_prop_add_list(*nvp, ZPOOL_PROP_ALLOCATED, NULL, alloc, src);
211 		spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL,
212 		    size - alloc, src);
213 
214 		spa_prop_add_list(*nvp, ZPOOL_PROP_FRAGMENTATION, NULL,
215 		    metaslab_class_fragmentation(mc), src);
216 		spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL,
217 		    metaslab_class_expandable_space(mc), src);
218 		spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL,
219 		    (spa_mode(spa) == FREAD), src);
220 
221 		cap = (size == 0) ? 0 : (alloc * 100 / size);
222 		spa_prop_add_list(*nvp, ZPOOL_PROP_CAPACITY, NULL, cap, src);
223 
224 		spa_prop_add_list(*nvp, ZPOOL_PROP_DEDUPRATIO, NULL,
225 		    ddt_get_pool_dedup_ratio(spa), src);
226 
227 		spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL,
228 		    rvd->vdev_state, src);
229 
230 		version = spa_version(spa);
231 		if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION))
232 			src = ZPROP_SRC_DEFAULT;
233 		else
234 			src = ZPROP_SRC_LOCAL;
235 		spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, version, src);
236 	}
237 
238 	if (pool != NULL) {
239 		/*
240 		 * The $FREE directory was introduced in SPA_VERSION_DEADLISTS,
241 		 * when opening pools before this version freedir will be NULL.
242 		 */
243 		if (pool->dp_free_dir != NULL) {
244 			spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL,
245 			    dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes,
246 			    src);
247 		} else {
248 			spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING,
249 			    NULL, 0, src);
250 		}
251 
252 		if (pool->dp_leak_dir != NULL) {
253 			spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL,
254 			    dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes,
255 			    src);
256 		} else {
257 			spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED,
258 			    NULL, 0, src);
259 		}
260 	}
261 
262 	spa_prop_add_list(*nvp, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src);
263 
264 	if (spa->spa_comment != NULL) {
265 		spa_prop_add_list(*nvp, ZPOOL_PROP_COMMENT, spa->spa_comment,
266 		    0, ZPROP_SRC_LOCAL);
267 	}
268 
269 	if (spa->spa_root != NULL)
270 		spa_prop_add_list(*nvp, ZPOOL_PROP_ALTROOT, spa->spa_root,
271 		    0, ZPROP_SRC_LOCAL);
272 
273 	if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) {
274 		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL,
275 		    MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE);
276 	} else {
277 		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL,
278 		    SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE);
279 	}
280 
281 	if ((dp = list_head(&spa->spa_config_list)) != NULL) {
282 		if (dp->scd_path == NULL) {
283 			spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
284 			    "none", 0, ZPROP_SRC_LOCAL);
285 		} else if (strcmp(dp->scd_path, spa_config_path) != 0) {
286 			spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
287 			    dp->scd_path, 0, ZPROP_SRC_LOCAL);
288 		}
289 	}
290 }
291 
292 /*
293  * Get zpool property values.
294  */
295 int
296 spa_prop_get(spa_t *spa, nvlist_t **nvp)
297 {
298 	objset_t *mos = spa->spa_meta_objset;
299 	zap_cursor_t zc;
300 	zap_attribute_t za;
301 	int err;
302 
303 	VERIFY(nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP) == 0);
304 
305 	mutex_enter(&spa->spa_props_lock);
306 
307 	/*
308 	 * Get properties from the spa config.
309 	 */
310 	spa_prop_get_config(spa, nvp);
311 
312 	/* If no pool property object, no more prop to get. */
313 	if (mos == NULL || spa->spa_pool_props_object == 0) {
314 		mutex_exit(&spa->spa_props_lock);
315 		return (0);
316 	}
317 
318 	/*
319 	 * Get properties from the MOS pool property object.
320 	 */
321 	for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object);
322 	    (err = zap_cursor_retrieve(&zc, &za)) == 0;
323 	    zap_cursor_advance(&zc)) {
324 		uint64_t intval = 0;
325 		char *strval = NULL;
326 		zprop_source_t src = ZPROP_SRC_DEFAULT;
327 		zpool_prop_t prop;
328 
329 		if ((prop = zpool_name_to_prop(za.za_name)) == ZPROP_INVAL)
330 			continue;
331 
332 		switch (za.za_integer_length) {
333 		case 8:
334 			/* integer property */
335 			if (za.za_first_integer !=
336 			    zpool_prop_default_numeric(prop))
337 				src = ZPROP_SRC_LOCAL;
338 
339 			if (prop == ZPOOL_PROP_BOOTFS) {
340 				dsl_pool_t *dp;
341 				dsl_dataset_t *ds = NULL;
342 
343 				dp = spa_get_dsl(spa);
344 				dsl_pool_config_enter(dp, FTAG);
345 				if (err = dsl_dataset_hold_obj(dp,
346 				    za.za_first_integer, FTAG, &ds)) {
347 					dsl_pool_config_exit(dp, FTAG);
348 					break;
349 				}
350 
351 				strval = kmem_alloc(
352 				    MAXNAMELEN + strlen(MOS_DIR_NAME) + 1,
353 				    KM_SLEEP);
354 				dsl_dataset_name(ds, strval);
355 				dsl_dataset_rele(ds, FTAG);
356 				dsl_pool_config_exit(dp, FTAG);
357 			} else {
358 				strval = NULL;
359 				intval = za.za_first_integer;
360 			}
361 
362 			spa_prop_add_list(*nvp, prop, strval, intval, src);
363 
364 			if (strval != NULL)
365 				kmem_free(strval,
366 				    MAXNAMELEN + strlen(MOS_DIR_NAME) + 1);
367 
368 			break;
369 
370 		case 1:
371 			/* string property */
372 			strval = kmem_alloc(za.za_num_integers, KM_SLEEP);
373 			err = zap_lookup(mos, spa->spa_pool_props_object,
374 			    za.za_name, 1, za.za_num_integers, strval);
375 			if (err) {
376 				kmem_free(strval, za.za_num_integers);
377 				break;
378 			}
379 			spa_prop_add_list(*nvp, prop, strval, 0, src);
380 			kmem_free(strval, za.za_num_integers);
381 			break;
382 
383 		default:
384 			break;
385 		}
386 	}
387 	zap_cursor_fini(&zc);
388 	mutex_exit(&spa->spa_props_lock);
389 out:
390 	if (err && err != ENOENT) {
391 		nvlist_free(*nvp);
392 		*nvp = NULL;
393 		return (err);
394 	}
395 
396 	return (0);
397 }
398 
399 /*
400  * Validate the given pool properties nvlist and modify the list
401  * for the property values to be set.
402  */
403 static int
404 spa_prop_validate(spa_t *spa, nvlist_t *props)
405 {
406 	nvpair_t *elem;
407 	int error = 0, reset_bootfs = 0;
408 	uint64_t objnum = 0;
409 	boolean_t has_feature = B_FALSE;
410 
411 	elem = NULL;
412 	while ((elem = nvlist_next_nvpair(props, elem)) != NULL) {
413 		uint64_t intval;
414 		char *strval, *slash, *check, *fname;
415 		const char *propname = nvpair_name(elem);
416 		zpool_prop_t prop = zpool_name_to_prop(propname);
417 
418 		switch (prop) {
419 		case ZPROP_INVAL:
420 			if (!zpool_prop_feature(propname)) {
421 				error = SET_ERROR(EINVAL);
422 				break;
423 			}
424 
425 			/*
426 			 * Sanitize the input.
427 			 */
428 			if (nvpair_type(elem) != DATA_TYPE_UINT64) {
429 				error = SET_ERROR(EINVAL);
430 				break;
431 			}
432 
433 			if (nvpair_value_uint64(elem, &intval) != 0) {
434 				error = SET_ERROR(EINVAL);
435 				break;
436 			}
437 
438 			if (intval != 0) {
439 				error = SET_ERROR(EINVAL);
440 				break;
441 			}
442 
443 			fname = strchr(propname, '@') + 1;
444 			if (zfeature_lookup_name(fname, NULL) != 0) {
445 				error = SET_ERROR(EINVAL);
446 				break;
447 			}
448 
449 			has_feature = B_TRUE;
450 			break;
451 
452 		case ZPOOL_PROP_VERSION:
453 			error = nvpair_value_uint64(elem, &intval);
454 			if (!error &&
455 			    (intval < spa_version(spa) ||
456 			    intval > SPA_VERSION_BEFORE_FEATURES ||
457 			    has_feature))
458 				error = SET_ERROR(EINVAL);
459 			break;
460 
461 		case ZPOOL_PROP_DELEGATION:
462 		case ZPOOL_PROP_AUTOREPLACE:
463 		case ZPOOL_PROP_LISTSNAPS:
464 		case ZPOOL_PROP_AUTOEXPAND:
465 			error = nvpair_value_uint64(elem, &intval);
466 			if (!error && intval > 1)
467 				error = SET_ERROR(EINVAL);
468 			break;
469 
470 		case ZPOOL_PROP_BOOTFS:
471 			/*
472 			 * If the pool version is less than SPA_VERSION_BOOTFS,
473 			 * or the pool is still being created (version == 0),
474 			 * the bootfs property cannot be set.
475 			 */
476 			if (spa_version(spa) < SPA_VERSION_BOOTFS) {
477 				error = SET_ERROR(ENOTSUP);
478 				break;
479 			}
480 
481 			/*
482 			 * Make sure the vdev config is bootable
483 			 */
484 			if (!vdev_is_bootable(spa->spa_root_vdev)) {
485 				error = SET_ERROR(ENOTSUP);
486 				break;
487 			}
488 
489 			reset_bootfs = 1;
490 
491 			error = nvpair_value_string(elem, &strval);
492 
493 			if (!error) {
494 				objset_t *os;
495 				uint64_t propval;
496 
497 				if (strval == NULL || strval[0] == '\0') {
498 					objnum = zpool_prop_default_numeric(
499 					    ZPOOL_PROP_BOOTFS);
500 					break;
501 				}
502 
503 				if (error = dmu_objset_hold(strval, FTAG, &os))
504 					break;
505 
506 				/*
507 				 * Must be ZPL, and its property settings
508 				 * must be supported by GRUB (compression
509 				 * is not gzip, and large blocks are not used).
510 				 */
511 
512 				if (dmu_objset_type(os) != DMU_OST_ZFS) {
513 					error = SET_ERROR(ENOTSUP);
514 				} else if ((error =
515 				    dsl_prop_get_int_ds(dmu_objset_ds(os),
516 				    zfs_prop_to_name(ZFS_PROP_COMPRESSION),
517 				    &propval)) == 0 &&
518 				    !BOOTFS_COMPRESS_VALID(propval)) {
519 					error = SET_ERROR(ENOTSUP);
520 				} else if ((error =
521 				    dsl_prop_get_int_ds(dmu_objset_ds(os),
522 				    zfs_prop_to_name(ZFS_PROP_RECORDSIZE),
523 				    &propval)) == 0 &&
524 				    propval > SPA_OLD_MAXBLOCKSIZE) {
525 					error = SET_ERROR(ENOTSUP);
526 				} else {
527 					objnum = dmu_objset_id(os);
528 				}
529 				dmu_objset_rele(os, FTAG);
530 			}
531 			break;
532 
533 		case ZPOOL_PROP_FAILUREMODE:
534 			error = nvpair_value_uint64(elem, &intval);
535 			if (!error && (intval < ZIO_FAILURE_MODE_WAIT ||
536 			    intval > ZIO_FAILURE_MODE_PANIC))
537 				error = SET_ERROR(EINVAL);
538 
539 			/*
540 			 * This is a special case which only occurs when
541 			 * the pool has completely failed. This allows
542 			 * the user to change the in-core failmode property
543 			 * without syncing it out to disk (I/Os might
544 			 * currently be blocked). We do this by returning
545 			 * EIO to the caller (spa_prop_set) to trick it
546 			 * into thinking we encountered a property validation
547 			 * error.
548 			 */
549 			if (!error && spa_suspended(spa)) {
550 				spa->spa_failmode = intval;
551 				error = SET_ERROR(EIO);
552 			}
553 			break;
554 
555 		case ZPOOL_PROP_CACHEFILE:
556 			if ((error = nvpair_value_string(elem, &strval)) != 0)
557 				break;
558 
559 			if (strval[0] == '\0')
560 				break;
561 
562 			if (strcmp(strval, "none") == 0)
563 				break;
564 
565 			if (strval[0] != '/') {
566 				error = SET_ERROR(EINVAL);
567 				break;
568 			}
569 
570 			slash = strrchr(strval, '/');
571 			ASSERT(slash != NULL);
572 
573 			if (slash[1] == '\0' || strcmp(slash, "/.") == 0 ||
574 			    strcmp(slash, "/..") == 0)
575 				error = SET_ERROR(EINVAL);
576 			break;
577 
578 		case ZPOOL_PROP_COMMENT:
579 			if ((error = nvpair_value_string(elem, &strval)) != 0)
580 				break;
581 			for (check = strval; *check != '\0'; check++) {
582 				/*
583 				 * The kernel doesn't have an easy isprint()
584 				 * check.  For this kernel check, we merely
585 				 * check ASCII apart from DEL.  Fix this if
586 				 * there is an easy-to-use kernel isprint().
587 				 */
588 				if (*check >= 0x7f) {
589 					error = SET_ERROR(EINVAL);
590 					break;
591 				}
592 				check++;
593 			}
594 			if (strlen(strval) > ZPROP_MAX_COMMENT)
595 				error = E2BIG;
596 			break;
597 
598 		case ZPOOL_PROP_DEDUPDITTO:
599 			if (spa_version(spa) < SPA_VERSION_DEDUP)
600 				error = SET_ERROR(ENOTSUP);
601 			else
602 				error = nvpair_value_uint64(elem, &intval);
603 			if (error == 0 &&
604 			    intval != 0 && intval < ZIO_DEDUPDITTO_MIN)
605 				error = SET_ERROR(EINVAL);
606 			break;
607 		}
608 
609 		if (error)
610 			break;
611 	}
612 
613 	if (!error && reset_bootfs) {
614 		error = nvlist_remove(props,
615 		    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING);
616 
617 		if (!error) {
618 			error = nvlist_add_uint64(props,
619 			    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum);
620 		}
621 	}
622 
623 	return (error);
624 }
625 
626 void
627 spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync)
628 {
629 	char *cachefile;
630 	spa_config_dirent_t *dp;
631 
632 	if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE),
633 	    &cachefile) != 0)
634 		return;
635 
636 	dp = kmem_alloc(sizeof (spa_config_dirent_t),
637 	    KM_SLEEP);
638 
639 	if (cachefile[0] == '\0')
640 		dp->scd_path = spa_strdup(spa_config_path);
641 	else if (strcmp(cachefile, "none") == 0)
642 		dp->scd_path = NULL;
643 	else
644 		dp->scd_path = spa_strdup(cachefile);
645 
646 	list_insert_head(&spa->spa_config_list, dp);
647 	if (need_sync)
648 		spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
649 }
650 
651 int
652 spa_prop_set(spa_t *spa, nvlist_t *nvp)
653 {
654 	int error;
655 	nvpair_t *elem = NULL;
656 	boolean_t need_sync = B_FALSE;
657 
658 	if ((error = spa_prop_validate(spa, nvp)) != 0)
659 		return (error);
660 
661 	while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) {
662 		zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem));
663 
664 		if (prop == ZPOOL_PROP_CACHEFILE ||
665 		    prop == ZPOOL_PROP_ALTROOT ||
666 		    prop == ZPOOL_PROP_READONLY)
667 			continue;
668 
669 		if (prop == ZPOOL_PROP_VERSION || prop == ZPROP_INVAL) {
670 			uint64_t ver;
671 
672 			if (prop == ZPOOL_PROP_VERSION) {
673 				VERIFY(nvpair_value_uint64(elem, &ver) == 0);
674 			} else {
675 				ASSERT(zpool_prop_feature(nvpair_name(elem)));
676 				ver = SPA_VERSION_FEATURES;
677 				need_sync = B_TRUE;
678 			}
679 
680 			/* Save time if the version is already set. */
681 			if (ver == spa_version(spa))
682 				continue;
683 
684 			/*
685 			 * In addition to the pool directory object, we might
686 			 * create the pool properties object, the features for
687 			 * read object, the features for write object, or the
688 			 * feature descriptions object.
689 			 */
690 			error = dsl_sync_task(spa->spa_name, NULL,
691 			    spa_sync_version, &ver,
692 			    6, ZFS_SPACE_CHECK_RESERVED);
693 			if (error)
694 				return (error);
695 			continue;
696 		}
697 
698 		need_sync = B_TRUE;
699 		break;
700 	}
701 
702 	if (need_sync) {
703 		return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props,
704 		    nvp, 6, ZFS_SPACE_CHECK_RESERVED));
705 	}
706 
707 	return (0);
708 }
709 
710 /*
711  * If the bootfs property value is dsobj, clear it.
712  */
713 void
714 spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx)
715 {
716 	if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) {
717 		VERIFY(zap_remove(spa->spa_meta_objset,
718 		    spa->spa_pool_props_object,
719 		    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0);
720 		spa->spa_bootfs = 0;
721 	}
722 }
723 
724 /*ARGSUSED*/
725 static int
726 spa_change_guid_check(void *arg, dmu_tx_t *tx)
727 {
728 	uint64_t *newguid = arg;
729 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
730 	vdev_t *rvd = spa->spa_root_vdev;
731 	uint64_t vdev_state;
732 
733 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
734 	vdev_state = rvd->vdev_state;
735 	spa_config_exit(spa, SCL_STATE, FTAG);
736 
737 	if (vdev_state != VDEV_STATE_HEALTHY)
738 		return (SET_ERROR(ENXIO));
739 
740 	ASSERT3U(spa_guid(spa), !=, *newguid);
741 
742 	return (0);
743 }
744 
745 static void
746 spa_change_guid_sync(void *arg, dmu_tx_t *tx)
747 {
748 	uint64_t *newguid = arg;
749 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
750 	uint64_t oldguid;
751 	vdev_t *rvd = spa->spa_root_vdev;
752 
753 	oldguid = spa_guid(spa);
754 
755 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
756 	rvd->vdev_guid = *newguid;
757 	rvd->vdev_guid_sum += (*newguid - oldguid);
758 	vdev_config_dirty(rvd);
759 	spa_config_exit(spa, SCL_STATE, FTAG);
760 
761 	spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu",
762 	    oldguid, *newguid);
763 }
764 
765 /*
766  * Change the GUID for the pool.  This is done so that we can later
767  * re-import a pool built from a clone of our own vdevs.  We will modify
768  * the root vdev's guid, our own pool guid, and then mark all of our
769  * vdevs dirty.  Note that we must make sure that all our vdevs are
770  * online when we do this, or else any vdevs that weren't present
771  * would be orphaned from our pool.  We are also going to issue a
772  * sysevent to update any watchers.
773  */
774 int
775 spa_change_guid(spa_t *spa)
776 {
777 	int error;
778 	uint64_t guid;
779 
780 	mutex_enter(&spa->spa_vdev_top_lock);
781 	mutex_enter(&spa_namespace_lock);
782 	guid = spa_generate_guid(NULL);
783 
784 	error = dsl_sync_task(spa->spa_name, spa_change_guid_check,
785 	    spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED);
786 
787 	if (error == 0) {
788 		spa_config_sync(spa, B_FALSE, B_TRUE);
789 		spa_event_notify(spa, NULL, ESC_ZFS_POOL_REGUID);
790 	}
791 
792 	mutex_exit(&spa_namespace_lock);
793 	mutex_exit(&spa->spa_vdev_top_lock);
794 
795 	return (error);
796 }
797 
798 /*
799  * ==========================================================================
800  * SPA state manipulation (open/create/destroy/import/export)
801  * ==========================================================================
802  */
803 
804 static int
805 spa_error_entry_compare(const void *a, const void *b)
806 {
807 	spa_error_entry_t *sa = (spa_error_entry_t *)a;
808 	spa_error_entry_t *sb = (spa_error_entry_t *)b;
809 	int ret;
810 
811 	ret = bcmp(&sa->se_bookmark, &sb->se_bookmark,
812 	    sizeof (zbookmark_phys_t));
813 
814 	if (ret < 0)
815 		return (-1);
816 	else if (ret > 0)
817 		return (1);
818 	else
819 		return (0);
820 }
821 
822 /*
823  * Utility function which retrieves copies of the current logs and
824  * re-initializes them in the process.
825  */
826 void
827 spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub)
828 {
829 	ASSERT(MUTEX_HELD(&spa->spa_errlist_lock));
830 
831 	bcopy(&spa->spa_errlist_last, last, sizeof (avl_tree_t));
832 	bcopy(&spa->spa_errlist_scrub, scrub, sizeof (avl_tree_t));
833 
834 	avl_create(&spa->spa_errlist_scrub,
835 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
836 	    offsetof(spa_error_entry_t, se_avl));
837 	avl_create(&spa->spa_errlist_last,
838 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
839 	    offsetof(spa_error_entry_t, se_avl));
840 }
841 
842 static void
843 spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q)
844 {
845 	const zio_taskq_info_t *ztip = &zio_taskqs[t][q];
846 	enum zti_modes mode = ztip->zti_mode;
847 	uint_t value = ztip->zti_value;
848 	uint_t count = ztip->zti_count;
849 	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
850 	char name[32];
851 	uint_t flags = 0;
852 	boolean_t batch = B_FALSE;
853 
854 	if (mode == ZTI_MODE_NULL) {
855 		tqs->stqs_count = 0;
856 		tqs->stqs_taskq = NULL;
857 		return;
858 	}
859 
860 	ASSERT3U(count, >, 0);
861 
862 	tqs->stqs_count = count;
863 	tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP);
864 
865 	switch (mode) {
866 	case ZTI_MODE_FIXED:
867 		ASSERT3U(value, >=, 1);
868 		value = MAX(value, 1);
869 		break;
870 
871 	case ZTI_MODE_BATCH:
872 		batch = B_TRUE;
873 		flags |= TASKQ_THREADS_CPU_PCT;
874 		value = zio_taskq_batch_pct;
875 		break;
876 
877 	default:
878 		panic("unrecognized mode for %s_%s taskq (%u:%u) in "
879 		    "spa_activate()",
880 		    zio_type_name[t], zio_taskq_types[q], mode, value);
881 		break;
882 	}
883 
884 	for (uint_t i = 0; i < count; i++) {
885 		taskq_t *tq;
886 
887 		if (count > 1) {
888 			(void) snprintf(name, sizeof (name), "%s_%s_%u",
889 			    zio_type_name[t], zio_taskq_types[q], i);
890 		} else {
891 			(void) snprintf(name, sizeof (name), "%s_%s",
892 			    zio_type_name[t], zio_taskq_types[q]);
893 		}
894 
895 		if (zio_taskq_sysdc && spa->spa_proc != &p0) {
896 			if (batch)
897 				flags |= TASKQ_DC_BATCH;
898 
899 			tq = taskq_create_sysdc(name, value, 50, INT_MAX,
900 			    spa->spa_proc, zio_taskq_basedc, flags);
901 		} else {
902 			pri_t pri = maxclsyspri;
903 			/*
904 			 * The write issue taskq can be extremely CPU
905 			 * intensive.  Run it at slightly lower priority
906 			 * than the other taskqs.
907 			 */
908 			if (t == ZIO_TYPE_WRITE && q == ZIO_TASKQ_ISSUE)
909 				pri--;
910 
911 			tq = taskq_create_proc(name, value, pri, 50,
912 			    INT_MAX, spa->spa_proc, flags);
913 		}
914 
915 		tqs->stqs_taskq[i] = tq;
916 	}
917 }
918 
919 static void
920 spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q)
921 {
922 	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
923 
924 	if (tqs->stqs_taskq == NULL) {
925 		ASSERT0(tqs->stqs_count);
926 		return;
927 	}
928 
929 	for (uint_t i = 0; i < tqs->stqs_count; i++) {
930 		ASSERT3P(tqs->stqs_taskq[i], !=, NULL);
931 		taskq_destroy(tqs->stqs_taskq[i]);
932 	}
933 
934 	kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *));
935 	tqs->stqs_taskq = NULL;
936 }
937 
938 /*
939  * Dispatch a task to the appropriate taskq for the ZFS I/O type and priority.
940  * Note that a type may have multiple discrete taskqs to avoid lock contention
941  * on the taskq itself. In that case we choose which taskq at random by using
942  * the low bits of gethrtime().
943  */
944 void
945 spa_taskq_dispatch_ent(spa_t *spa, zio_type_t t, zio_taskq_type_t q,
946     task_func_t *func, void *arg, uint_t flags, taskq_ent_t *ent)
947 {
948 	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
949 	taskq_t *tq;
950 
951 	ASSERT3P(tqs->stqs_taskq, !=, NULL);
952 	ASSERT3U(tqs->stqs_count, !=, 0);
953 
954 	if (tqs->stqs_count == 1) {
955 		tq = tqs->stqs_taskq[0];
956 	} else {
957 		tq = tqs->stqs_taskq[gethrtime() % tqs->stqs_count];
958 	}
959 
960 	taskq_dispatch_ent(tq, func, arg, flags, ent);
961 }
962 
963 static void
964 spa_create_zio_taskqs(spa_t *spa)
965 {
966 	for (int t = 0; t < ZIO_TYPES; t++) {
967 		for (int q = 0; q < ZIO_TASKQ_TYPES; q++) {
968 			spa_taskqs_init(spa, t, q);
969 		}
970 	}
971 }
972 
973 #ifdef _KERNEL
974 static void
975 spa_thread(void *arg)
976 {
977 	callb_cpr_t cprinfo;
978 
979 	spa_t *spa = arg;
980 	user_t *pu = PTOU(curproc);
981 
982 	CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr,
983 	    spa->spa_name);
984 
985 	ASSERT(curproc != &p0);
986 	(void) snprintf(pu->u_psargs, sizeof (pu->u_psargs),
987 	    "zpool-%s", spa->spa_name);
988 	(void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm));
989 
990 	/* bind this thread to the requested psrset */
991 	if (zio_taskq_psrset_bind != PS_NONE) {
992 		pool_lock();
993 		mutex_enter(&cpu_lock);
994 		mutex_enter(&pidlock);
995 		mutex_enter(&curproc->p_lock);
996 
997 		if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind,
998 		    0, NULL, NULL) == 0)  {
999 			curthread->t_bind_pset = zio_taskq_psrset_bind;
1000 		} else {
1001 			cmn_err(CE_WARN,
1002 			    "Couldn't bind process for zfs pool \"%s\" to "
1003 			    "pset %d\n", spa->spa_name, zio_taskq_psrset_bind);
1004 		}
1005 
1006 		mutex_exit(&curproc->p_lock);
1007 		mutex_exit(&pidlock);
1008 		mutex_exit(&cpu_lock);
1009 		pool_unlock();
1010 	}
1011 
1012 	if (zio_taskq_sysdc) {
1013 		sysdc_thread_enter(curthread, 100, 0);
1014 	}
1015 
1016 	spa->spa_proc = curproc;
1017 	spa->spa_did = curthread->t_did;
1018 
1019 	spa_create_zio_taskqs(spa);
1020 
1021 	mutex_enter(&spa->spa_proc_lock);
1022 	ASSERT(spa->spa_proc_state == SPA_PROC_CREATED);
1023 
1024 	spa->spa_proc_state = SPA_PROC_ACTIVE;
1025 	cv_broadcast(&spa->spa_proc_cv);
1026 
1027 	CALLB_CPR_SAFE_BEGIN(&cprinfo);
1028 	while (spa->spa_proc_state == SPA_PROC_ACTIVE)
1029 		cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock);
1030 	CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock);
1031 
1032 	ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE);
1033 	spa->spa_proc_state = SPA_PROC_GONE;
1034 	spa->spa_proc = &p0;
1035 	cv_broadcast(&spa->spa_proc_cv);
1036 	CALLB_CPR_EXIT(&cprinfo);	/* drops spa_proc_lock */
1037 
1038 	mutex_enter(&curproc->p_lock);
1039 	lwp_exit();
1040 }
1041 #endif
1042 
1043 /*
1044  * Activate an uninitialized pool.
1045  */
1046 static void
1047 spa_activate(spa_t *spa, int mode)
1048 {
1049 	ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED);
1050 
1051 	spa->spa_state = POOL_STATE_ACTIVE;
1052 	spa->spa_mode = mode;
1053 
1054 	spa->spa_normal_class = metaslab_class_create(spa, zfs_metaslab_ops);
1055 	spa->spa_log_class = metaslab_class_create(spa, zfs_metaslab_ops);
1056 
1057 	/* Try to create a covering process */
1058 	mutex_enter(&spa->spa_proc_lock);
1059 	ASSERT(spa->spa_proc_state == SPA_PROC_NONE);
1060 	ASSERT(spa->spa_proc == &p0);
1061 	spa->spa_did = 0;
1062 
1063 	/* Only create a process if we're going to be around a while. */
1064 	if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) {
1065 		if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri,
1066 		    NULL, 0) == 0) {
1067 			spa->spa_proc_state = SPA_PROC_CREATED;
1068 			while (spa->spa_proc_state == SPA_PROC_CREATED) {
1069 				cv_wait(&spa->spa_proc_cv,
1070 				    &spa->spa_proc_lock);
1071 			}
1072 			ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE);
1073 			ASSERT(spa->spa_proc != &p0);
1074 			ASSERT(spa->spa_did != 0);
1075 		} else {
1076 #ifdef _KERNEL
1077 			cmn_err(CE_WARN,
1078 			    "Couldn't create process for zfs pool \"%s\"\n",
1079 			    spa->spa_name);
1080 #endif
1081 		}
1082 	}
1083 	mutex_exit(&spa->spa_proc_lock);
1084 
1085 	/* If we didn't create a process, we need to create our taskqs. */
1086 	if (spa->spa_proc == &p0) {
1087 		spa_create_zio_taskqs(spa);
1088 	}
1089 
1090 	list_create(&spa->spa_config_dirty_list, sizeof (vdev_t),
1091 	    offsetof(vdev_t, vdev_config_dirty_node));
1092 	list_create(&spa->spa_evicting_os_list, sizeof (objset_t),
1093 	    offsetof(objset_t, os_evicting_node));
1094 	list_create(&spa->spa_state_dirty_list, sizeof (vdev_t),
1095 	    offsetof(vdev_t, vdev_state_dirty_node));
1096 
1097 	txg_list_create(&spa->spa_vdev_txg_list,
1098 	    offsetof(struct vdev, vdev_txg_node));
1099 
1100 	avl_create(&spa->spa_errlist_scrub,
1101 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1102 	    offsetof(spa_error_entry_t, se_avl));
1103 	avl_create(&spa->spa_errlist_last,
1104 	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1105 	    offsetof(spa_error_entry_t, se_avl));
1106 }
1107 
1108 /*
1109  * Opposite of spa_activate().
1110  */
1111 static void
1112 spa_deactivate(spa_t *spa)
1113 {
1114 	ASSERT(spa->spa_sync_on == B_FALSE);
1115 	ASSERT(spa->spa_dsl_pool == NULL);
1116 	ASSERT(spa->spa_root_vdev == NULL);
1117 	ASSERT(spa->spa_async_zio_root == NULL);
1118 	ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED);
1119 
1120 	spa_evicting_os_wait(spa);
1121 
1122 	txg_list_destroy(&spa->spa_vdev_txg_list);
1123 
1124 	list_destroy(&spa->spa_config_dirty_list);
1125 	list_destroy(&spa->spa_evicting_os_list);
1126 	list_destroy(&spa->spa_state_dirty_list);
1127 
1128 	for (int t = 0; t < ZIO_TYPES; t++) {
1129 		for (int q = 0; q < ZIO_TASKQ_TYPES; q++) {
1130 			spa_taskqs_fini(spa, t, q);
1131 		}
1132 	}
1133 
1134 	metaslab_class_destroy(spa->spa_normal_class);
1135 	spa->spa_normal_class = NULL;
1136 
1137 	metaslab_class_destroy(spa->spa_log_class);
1138 	spa->spa_log_class = NULL;
1139 
1140 	/*
1141 	 * If this was part of an import or the open otherwise failed, we may
1142 	 * still have errors left in the queues.  Empty them just in case.
1143 	 */
1144 	spa_errlog_drain(spa);
1145 
1146 	avl_destroy(&spa->spa_errlist_scrub);
1147 	avl_destroy(&spa->spa_errlist_last);
1148 
1149 	spa->spa_state = POOL_STATE_UNINITIALIZED;
1150 
1151 	mutex_enter(&spa->spa_proc_lock);
1152 	if (spa->spa_proc_state != SPA_PROC_NONE) {
1153 		ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE);
1154 		spa->spa_proc_state = SPA_PROC_DEACTIVATE;
1155 		cv_broadcast(&spa->spa_proc_cv);
1156 		while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) {
1157 			ASSERT(spa->spa_proc != &p0);
1158 			cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock);
1159 		}
1160 		ASSERT(spa->spa_proc_state == SPA_PROC_GONE);
1161 		spa->spa_proc_state = SPA_PROC_NONE;
1162 	}
1163 	ASSERT(spa->spa_proc == &p0);
1164 	mutex_exit(&spa->spa_proc_lock);
1165 
1166 	/*
1167 	 * We want to make sure spa_thread() has actually exited the ZFS
1168 	 * module, so that the module can't be unloaded out from underneath
1169 	 * it.
1170 	 */
1171 	if (spa->spa_did != 0) {
1172 		thread_join(spa->spa_did);
1173 		spa->spa_did = 0;
1174 	}
1175 }
1176 
1177 /*
1178  * Verify a pool configuration, and construct the vdev tree appropriately.  This
1179  * will create all the necessary vdevs in the appropriate layout, with each vdev
1180  * in the CLOSED state.  This will prep the pool before open/creation/import.
1181  * All vdev validation is done by the vdev_alloc() routine.
1182  */
1183 static int
1184 spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent,
1185     uint_t id, int atype)
1186 {
1187 	nvlist_t **child;
1188 	uint_t children;
1189 	int error;
1190 
1191 	if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0)
1192 		return (error);
1193 
1194 	if ((*vdp)->vdev_ops->vdev_op_leaf)
1195 		return (0);
1196 
1197 	error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1198 	    &child, &children);
1199 
1200 	if (error == ENOENT)
1201 		return (0);
1202 
1203 	if (error) {
1204 		vdev_free(*vdp);
1205 		*vdp = NULL;
1206 		return (SET_ERROR(EINVAL));
1207 	}
1208 
1209 	for (int c = 0; c < children; c++) {
1210 		vdev_t *vd;
1211 		if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c,
1212 		    atype)) != 0) {
1213 			vdev_free(*vdp);
1214 			*vdp = NULL;
1215 			return (error);
1216 		}
1217 	}
1218 
1219 	ASSERT(*vdp != NULL);
1220 
1221 	return (0);
1222 }
1223 
1224 /*
1225  * Opposite of spa_load().
1226  */
1227 static void
1228 spa_unload(spa_t *spa)
1229 {
1230 	int i;
1231 
1232 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1233 
1234 	/*
1235 	 * Stop async tasks.
1236 	 */
1237 	spa_async_suspend(spa);
1238 
1239 	/*
1240 	 * Stop syncing.
1241 	 */
1242 	if (spa->spa_sync_on) {
1243 		txg_sync_stop(spa->spa_dsl_pool);
1244 		spa->spa_sync_on = B_FALSE;
1245 	}
1246 
1247 	/*
1248 	 * Wait for any outstanding async I/O to complete.
1249 	 */
1250 	if (spa->spa_async_zio_root != NULL) {
1251 		for (int i = 0; i < max_ncpus; i++)
1252 			(void) zio_wait(spa->spa_async_zio_root[i]);
1253 		kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *));
1254 		spa->spa_async_zio_root = NULL;
1255 	}
1256 
1257 	bpobj_close(&spa->spa_deferred_bpobj);
1258 
1259 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
1260 
1261 	/*
1262 	 * Close all vdevs.
1263 	 */
1264 	if (spa->spa_root_vdev)
1265 		vdev_free(spa->spa_root_vdev);
1266 	ASSERT(spa->spa_root_vdev == NULL);
1267 
1268 	/*
1269 	 * Close the dsl pool.
1270 	 */
1271 	if (spa->spa_dsl_pool) {
1272 		dsl_pool_close(spa->spa_dsl_pool);
1273 		spa->spa_dsl_pool = NULL;
1274 		spa->spa_meta_objset = NULL;
1275 	}
1276 
1277 	ddt_unload(spa);
1278 
1279 
1280 	/*
1281 	 * Drop and purge level 2 cache
1282 	 */
1283 	spa_l2cache_drop(spa);
1284 
1285 	for (i = 0; i < spa->spa_spares.sav_count; i++)
1286 		vdev_free(spa->spa_spares.sav_vdevs[i]);
1287 	if (spa->spa_spares.sav_vdevs) {
1288 		kmem_free(spa->spa_spares.sav_vdevs,
1289 		    spa->spa_spares.sav_count * sizeof (void *));
1290 		spa->spa_spares.sav_vdevs = NULL;
1291 	}
1292 	if (spa->spa_spares.sav_config) {
1293 		nvlist_free(spa->spa_spares.sav_config);
1294 		spa->spa_spares.sav_config = NULL;
1295 	}
1296 	spa->spa_spares.sav_count = 0;
1297 
1298 	for (i = 0; i < spa->spa_l2cache.sav_count; i++) {
1299 		vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]);
1300 		vdev_free(spa->spa_l2cache.sav_vdevs[i]);
1301 	}
1302 	if (spa->spa_l2cache.sav_vdevs) {
1303 		kmem_free(spa->spa_l2cache.sav_vdevs,
1304 		    spa->spa_l2cache.sav_count * sizeof (void *));
1305 		spa->spa_l2cache.sav_vdevs = NULL;
1306 	}
1307 	if (spa->spa_l2cache.sav_config) {
1308 		nvlist_free(spa->spa_l2cache.sav_config);
1309 		spa->spa_l2cache.sav_config = NULL;
1310 	}
1311 	spa->spa_l2cache.sav_count = 0;
1312 
1313 	spa->spa_async_suspended = 0;
1314 
1315 	if (spa->spa_comment != NULL) {
1316 		spa_strfree(spa->spa_comment);
1317 		spa->spa_comment = NULL;
1318 	}
1319 
1320 	spa_config_exit(spa, SCL_ALL, FTAG);
1321 }
1322 
1323 /*
1324  * Load (or re-load) the current list of vdevs describing the active spares for
1325  * this pool.  When this is called, we have some form of basic information in
1326  * 'spa_spares.sav_config'.  We parse this into vdevs, try to open them, and
1327  * then re-generate a more complete list including status information.
1328  */
1329 static void
1330 spa_load_spares(spa_t *spa)
1331 {
1332 	nvlist_t **spares;
1333 	uint_t nspares;
1334 	int i;
1335 	vdev_t *vd, *tvd;
1336 
1337 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
1338 
1339 	/*
1340 	 * First, close and free any existing spare vdevs.
1341 	 */
1342 	for (i = 0; i < spa->spa_spares.sav_count; i++) {
1343 		vd = spa->spa_spares.sav_vdevs[i];
1344 
1345 		/* Undo the call to spa_activate() below */
1346 		if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid,
1347 		    B_FALSE)) != NULL && tvd->vdev_isspare)
1348 			spa_spare_remove(tvd);
1349 		vdev_close(vd);
1350 		vdev_free(vd);
1351 	}
1352 
1353 	if (spa->spa_spares.sav_vdevs)
1354 		kmem_free(spa->spa_spares.sav_vdevs,
1355 		    spa->spa_spares.sav_count * sizeof (void *));
1356 
1357 	if (spa->spa_spares.sav_config == NULL)
1358 		nspares = 0;
1359 	else
1360 		VERIFY(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
1361 		    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0);
1362 
1363 	spa->spa_spares.sav_count = (int)nspares;
1364 	spa->spa_spares.sav_vdevs = NULL;
1365 
1366 	if (nspares == 0)
1367 		return;
1368 
1369 	/*
1370 	 * Construct the array of vdevs, opening them to get status in the
1371 	 * process.   For each spare, there is potentially two different vdev_t
1372 	 * structures associated with it: one in the list of spares (used only
1373 	 * for basic validation purposes) and one in the active vdev
1374 	 * configuration (if it's spared in).  During this phase we open and
1375 	 * validate each vdev on the spare list.  If the vdev also exists in the
1376 	 * active configuration, then we also mark this vdev as an active spare.
1377 	 */
1378 	spa->spa_spares.sav_vdevs = kmem_alloc(nspares * sizeof (void *),
1379 	    KM_SLEEP);
1380 	for (i = 0; i < spa->spa_spares.sav_count; i++) {
1381 		VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0,
1382 		    VDEV_ALLOC_SPARE) == 0);
1383 		ASSERT(vd != NULL);
1384 
1385 		spa->spa_spares.sav_vdevs[i] = vd;
1386 
1387 		if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid,
1388 		    B_FALSE)) != NULL) {
1389 			if (!tvd->vdev_isspare)
1390 				spa_spare_add(tvd);
1391 
1392 			/*
1393 			 * We only mark the spare active if we were successfully
1394 			 * able to load the vdev.  Otherwise, importing a pool
1395 			 * with a bad active spare would result in strange
1396 			 * behavior, because multiple pool would think the spare
1397 			 * is actively in use.
1398 			 *
1399 			 * There is a vulnerability here to an equally bizarre
1400 			 * circumstance, where a dead active spare is later
1401 			 * brought back to life (onlined or otherwise).  Given
1402 			 * the rarity of this scenario, and the extra complexity
1403 			 * it adds, we ignore the possibility.
1404 			 */
1405 			if (!vdev_is_dead(tvd))
1406 				spa_spare_activate(tvd);
1407 		}
1408 
1409 		vd->vdev_top = vd;
1410 		vd->vdev_aux = &spa->spa_spares;
1411 
1412 		if (vdev_open(vd) != 0)
1413 			continue;
1414 
1415 		if (vdev_validate_aux(vd) == 0)
1416 			spa_spare_add(vd);
1417 	}
1418 
1419 	/*
1420 	 * Recompute the stashed list of spares, with status information
1421 	 * this time.
1422 	 */
1423 	VERIFY(nvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES,
1424 	    DATA_TYPE_NVLIST_ARRAY) == 0);
1425 
1426 	spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *),
1427 	    KM_SLEEP);
1428 	for (i = 0; i < spa->spa_spares.sav_count; i++)
1429 		spares[i] = vdev_config_generate(spa,
1430 		    spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE);
1431 	VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config,
1432 	    ZPOOL_CONFIG_SPARES, spares, spa->spa_spares.sav_count) == 0);
1433 	for (i = 0; i < spa->spa_spares.sav_count; i++)
1434 		nvlist_free(spares[i]);
1435 	kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *));
1436 }
1437 
1438 /*
1439  * Load (or re-load) the current list of vdevs describing the active l2cache for
1440  * this pool.  When this is called, we have some form of basic information in
1441  * 'spa_l2cache.sav_config'.  We parse this into vdevs, try to open them, and
1442  * then re-generate a more complete list including status information.
1443  * Devices which are already active have their details maintained, and are
1444  * not re-opened.
1445  */
1446 static void
1447 spa_load_l2cache(spa_t *spa)
1448 {
1449 	nvlist_t **l2cache;
1450 	uint_t nl2cache;
1451 	int i, j, oldnvdevs;
1452 	uint64_t guid;
1453 	vdev_t *vd, **oldvdevs, **newvdevs;
1454 	spa_aux_vdev_t *sav = &spa->spa_l2cache;
1455 
1456 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
1457 
1458 	if (sav->sav_config != NULL) {
1459 		VERIFY(nvlist_lookup_nvlist_array(sav->sav_config,
1460 		    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0);
1461 		newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP);
1462 	} else {
1463 		nl2cache = 0;
1464 		newvdevs = NULL;
1465 	}
1466 
1467 	oldvdevs = sav->sav_vdevs;
1468 	oldnvdevs = sav->sav_count;
1469 	sav->sav_vdevs = NULL;
1470 	sav->sav_count = 0;
1471 
1472 	/*
1473 	 * Process new nvlist of vdevs.
1474 	 */
1475 	for (i = 0; i < nl2cache; i++) {
1476 		VERIFY(nvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID,
1477 		    &guid) == 0);
1478 
1479 		newvdevs[i] = NULL;
1480 		for (j = 0; j < oldnvdevs; j++) {
1481 			vd = oldvdevs[j];
1482 			if (vd != NULL && guid == vd->vdev_guid) {
1483 				/*
1484 				 * Retain previous vdev for add/remove ops.
1485 				 */
1486 				newvdevs[i] = vd;
1487 				oldvdevs[j] = NULL;
1488 				break;
1489 			}
1490 		}
1491 
1492 		if (newvdevs[i] == NULL) {
1493 			/*
1494 			 * Create new vdev
1495 			 */
1496 			VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0,
1497 			    VDEV_ALLOC_L2CACHE) == 0);
1498 			ASSERT(vd != NULL);
1499 			newvdevs[i] = vd;
1500 
1501 			/*
1502 			 * Commit this vdev as an l2cache device,
1503 			 * even if it fails to open.
1504 			 */
1505 			spa_l2cache_add(vd);
1506 
1507 			vd->vdev_top = vd;
1508 			vd->vdev_aux = sav;
1509 
1510 			spa_l2cache_activate(vd);
1511 
1512 			if (vdev_open(vd) != 0)
1513 				continue;
1514 
1515 			(void) vdev_validate_aux(vd);
1516 
1517 			if (!vdev_is_dead(vd))
1518 				l2arc_add_vdev(spa, vd);
1519 		}
1520 	}
1521 
1522 	/*
1523 	 * Purge vdevs that were dropped
1524 	 */
1525 	for (i = 0; i < oldnvdevs; i++) {
1526 		uint64_t pool;
1527 
1528 		vd = oldvdevs[i];
1529 		if (vd != NULL) {
1530 			ASSERT(vd->vdev_isl2cache);
1531 
1532 			if (spa_l2cache_exists(vd->vdev_guid, &pool) &&
1533 			    pool != 0ULL && l2arc_vdev_present(vd))
1534 				l2arc_remove_vdev(vd);
1535 			vdev_clear_stats(vd);
1536 			vdev_free(vd);
1537 		}
1538 	}
1539 
1540 	if (oldvdevs)
1541 		kmem_free(oldvdevs, oldnvdevs * sizeof (void *));
1542 
1543 	if (sav->sav_config == NULL)
1544 		goto out;
1545 
1546 	sav->sav_vdevs = newvdevs;
1547 	sav->sav_count = (int)nl2cache;
1548 
1549 	/*
1550 	 * Recompute the stashed list of l2cache devices, with status
1551 	 * information this time.
1552 	 */
1553 	VERIFY(nvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE,
1554 	    DATA_TYPE_NVLIST_ARRAY) == 0);
1555 
1556 	l2cache = kmem_alloc(sav->sav_count * sizeof (void *), KM_SLEEP);
1557 	for (i = 0; i < sav->sav_count; i++)
1558 		l2cache[i] = vdev_config_generate(spa,
1559 		    sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE);
1560 	VERIFY(nvlist_add_nvlist_array(sav->sav_config,
1561 	    ZPOOL_CONFIG_L2CACHE, l2cache, sav->sav_count) == 0);
1562 out:
1563 	for (i = 0; i < sav->sav_count; i++)
1564 		nvlist_free(l2cache[i]);
1565 	if (sav->sav_count)
1566 		kmem_free(l2cache, sav->sav_count * sizeof (void *));
1567 }
1568 
1569 static int
1570 load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value)
1571 {
1572 	dmu_buf_t *db;
1573 	char *packed = NULL;
1574 	size_t nvsize = 0;
1575 	int error;
1576 	*value = NULL;
1577 
1578 	VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db));
1579 	nvsize = *(uint64_t *)db->db_data;
1580 	dmu_buf_rele(db, FTAG);
1581 
1582 	packed = kmem_alloc(nvsize, KM_SLEEP);
1583 	error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed,
1584 	    DMU_READ_PREFETCH);
1585 	if (error == 0)
1586 		error = nvlist_unpack(packed, nvsize, value, 0);
1587 	kmem_free(packed, nvsize);
1588 
1589 	return (error);
1590 }
1591 
1592 /*
1593  * Checks to see if the given vdev could not be opened, in which case we post a
1594  * sysevent to notify the autoreplace code that the device has been removed.
1595  */
1596 static void
1597 spa_check_removed(vdev_t *vd)
1598 {
1599 	for (int c = 0; c < vd->vdev_children; c++)
1600 		spa_check_removed(vd->vdev_child[c]);
1601 
1602 	if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) &&
1603 	    !vd->vdev_ishole) {
1604 		zfs_post_autoreplace(vd->vdev_spa, vd);
1605 		spa_event_notify(vd->vdev_spa, vd, ESC_ZFS_VDEV_CHECK);
1606 	}
1607 }
1608 
1609 /*
1610  * Validate the current config against the MOS config
1611  */
1612 static boolean_t
1613 spa_config_valid(spa_t *spa, nvlist_t *config)
1614 {
1615 	vdev_t *mrvd, *rvd = spa->spa_root_vdev;
1616 	nvlist_t *nv;
1617 
1618 	VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nv) == 0);
1619 
1620 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
1621 	VERIFY(spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD) == 0);
1622 
1623 	ASSERT3U(rvd->vdev_children, ==, mrvd->vdev_children);
1624 
1625 	/*
1626 	 * If we're doing a normal import, then build up any additional
1627 	 * diagnostic information about missing devices in this config.
1628 	 * We'll pass this up to the user for further processing.
1629 	 */
1630 	if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) {
1631 		nvlist_t **child, *nv;
1632 		uint64_t idx = 0;
1633 
1634 		child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t **),
1635 		    KM_SLEEP);
1636 		VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) == 0);
1637 
1638 		for (int c = 0; c < rvd->vdev_children; c++) {
1639 			vdev_t *tvd = rvd->vdev_child[c];
1640 			vdev_t *mtvd  = mrvd->vdev_child[c];
1641 
1642 			if (tvd->vdev_ops == &vdev_missing_ops &&
1643 			    mtvd->vdev_ops != &vdev_missing_ops &&
1644 			    mtvd->vdev_islog)
1645 				child[idx++] = vdev_config_generate(spa, mtvd,
1646 				    B_FALSE, 0);
1647 		}
1648 
1649 		if (idx) {
1650 			VERIFY(nvlist_add_nvlist_array(nv,
1651 			    ZPOOL_CONFIG_CHILDREN, child, idx) == 0);
1652 			VERIFY(nvlist_add_nvlist(spa->spa_load_info,
1653 			    ZPOOL_CONFIG_MISSING_DEVICES, nv) == 0);
1654 
1655 			for (int i = 0; i < idx; i++)
1656 				nvlist_free(child[i]);
1657 		}
1658 		nvlist_free(nv);
1659 		kmem_free(child, rvd->vdev_children * sizeof (char **));
1660 	}
1661 
1662 	/*
1663 	 * Compare the root vdev tree with the information we have
1664 	 * from the MOS config (mrvd). Check each top-level vdev
1665 	 * with the corresponding MOS config top-level (mtvd).
1666 	 */
1667 	for (int c = 0; c < rvd->vdev_children; c++) {
1668 		vdev_t *tvd = rvd->vdev_child[c];
1669 		vdev_t *mtvd  = mrvd->vdev_child[c];
1670 
1671 		/*
1672 		 * Resolve any "missing" vdevs in the current configuration.
1673 		 * If we find that the MOS config has more accurate information
1674 		 * about the top-level vdev then use that vdev instead.
1675 		 */
1676 		if (tvd->vdev_ops == &vdev_missing_ops &&
1677 		    mtvd->vdev_ops != &vdev_missing_ops) {
1678 
1679 			if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG))
1680 				continue;
1681 
1682 			/*
1683 			 * Device specific actions.
1684 			 */
1685 			if (mtvd->vdev_islog) {
1686 				spa_set_log_state(spa, SPA_LOG_CLEAR);
1687 			} else {
1688 				/*
1689 				 * XXX - once we have 'readonly' pool
1690 				 * support we should be able to handle
1691 				 * missing data devices by transitioning
1692 				 * the pool to readonly.
1693 				 */
1694 				continue;
1695 			}
1696 
1697 			/*
1698 			 * Swap the missing vdev with the data we were
1699 			 * able to obtain from the MOS config.
1700 			 */
1701 			vdev_remove_child(rvd, tvd);
1702 			vdev_remove_child(mrvd, mtvd);
1703 
1704 			vdev_add_child(rvd, mtvd);
1705 			vdev_add_child(mrvd, tvd);
1706 
1707 			spa_config_exit(spa, SCL_ALL, FTAG);
1708 			vdev_load(mtvd);
1709 			spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
1710 
1711 			vdev_reopen(rvd);
1712 		} else if (mtvd->vdev_islog) {
1713 			/*
1714 			 * Load the slog device's state from the MOS config
1715 			 * since it's possible that the label does not
1716 			 * contain the most up-to-date information.
1717 			 */
1718 			vdev_load_log_state(tvd, mtvd);
1719 			vdev_reopen(tvd);
1720 		}
1721 	}
1722 	vdev_free(mrvd);
1723 	spa_config_exit(spa, SCL_ALL, FTAG);
1724 
1725 	/*
1726 	 * Ensure we were able to validate the config.
1727 	 */
1728 	return (rvd->vdev_guid_sum == spa->spa_uberblock.ub_guid_sum);
1729 }
1730 
1731 /*
1732  * Check for missing log devices
1733  */
1734 static boolean_t
1735 spa_check_logs(spa_t *spa)
1736 {
1737 	boolean_t rv = B_FALSE;
1738 
1739 	switch (spa->spa_log_state) {
1740 	case SPA_LOG_MISSING:
1741 		/* need to recheck in case slog has been restored */
1742 	case SPA_LOG_UNKNOWN:
1743 		rv = (dmu_objset_find(spa->spa_name, zil_check_log_chain,
1744 		    NULL, DS_FIND_CHILDREN) != 0);
1745 		if (rv)
1746 			spa_set_log_state(spa, SPA_LOG_MISSING);
1747 		break;
1748 	}
1749 	return (rv);
1750 }
1751 
1752 static boolean_t
1753 spa_passivate_log(spa_t *spa)
1754 {
1755 	vdev_t *rvd = spa->spa_root_vdev;
1756 	boolean_t slog_found = B_FALSE;
1757 
1758 	ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER));
1759 
1760 	if (!spa_has_slogs(spa))
1761 		return (B_FALSE);
1762 
1763 	for (int c = 0; c < rvd->vdev_children; c++) {
1764 		vdev_t *tvd = rvd->vdev_child[c];
1765 		metaslab_group_t *mg = tvd->vdev_mg;
1766 
1767 		if (tvd->vdev_islog) {
1768 			metaslab_group_passivate(mg);
1769 			slog_found = B_TRUE;
1770 		}
1771 	}
1772 
1773 	return (slog_found);
1774 }
1775 
1776 static void
1777 spa_activate_log(spa_t *spa)
1778 {
1779 	vdev_t *rvd = spa->spa_root_vdev;
1780 
1781 	ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER));
1782 
1783 	for (int c = 0; c < rvd->vdev_children; c++) {
1784 		vdev_t *tvd = rvd->vdev_child[c];
1785 		metaslab_group_t *mg = tvd->vdev_mg;
1786 
1787 		if (tvd->vdev_islog)
1788 			metaslab_group_activate(mg);
1789 	}
1790 }
1791 
1792 int
1793 spa_offline_log(spa_t *spa)
1794 {
1795 	int error;
1796 
1797 	error = dmu_objset_find(spa_name(spa), zil_vdev_offline,
1798 	    NULL, DS_FIND_CHILDREN);
1799 	if (error == 0) {
1800 		/*
1801 		 * We successfully offlined the log device, sync out the
1802 		 * current txg so that the "stubby" block can be removed
1803 		 * by zil_sync().
1804 		 */
1805 		txg_wait_synced(spa->spa_dsl_pool, 0);
1806 	}
1807 	return (error);
1808 }
1809 
1810 static void
1811 spa_aux_check_removed(spa_aux_vdev_t *sav)
1812 {
1813 	for (int i = 0; i < sav->sav_count; i++)
1814 		spa_check_removed(sav->sav_vdevs[i]);
1815 }
1816 
1817 void
1818 spa_claim_notify(zio_t *zio)
1819 {
1820 	spa_t *spa = zio->io_spa;
1821 
1822 	if (zio->io_error)
1823 		return;
1824 
1825 	mutex_enter(&spa->spa_props_lock);	/* any mutex will do */
1826 	if (spa->spa_claim_max_txg < zio->io_bp->blk_birth)
1827 		spa->spa_claim_max_txg = zio->io_bp->blk_birth;
1828 	mutex_exit(&spa->spa_props_lock);
1829 }
1830 
1831 typedef struct spa_load_error {
1832 	uint64_t	sle_meta_count;
1833 	uint64_t	sle_data_count;
1834 } spa_load_error_t;
1835 
1836 static void
1837 spa_load_verify_done(zio_t *zio)
1838 {
1839 	blkptr_t *bp = zio->io_bp;
1840 	spa_load_error_t *sle = zio->io_private;
1841 	dmu_object_type_t type = BP_GET_TYPE(bp);
1842 	int error = zio->io_error;
1843 	spa_t *spa = zio->io_spa;
1844 
1845 	if (error) {
1846 		if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) &&
1847 		    type != DMU_OT_INTENT_LOG)
1848 			atomic_inc_64(&sle->sle_meta_count);
1849 		else
1850 			atomic_inc_64(&sle->sle_data_count);
1851 	}
1852 	zio_data_buf_free(zio->io_data, zio->io_size);
1853 
1854 	mutex_enter(&spa->spa_scrub_lock);
1855 	spa->spa_scrub_inflight--;
1856 	cv_broadcast(&spa->spa_scrub_io_cv);
1857 	mutex_exit(&spa->spa_scrub_lock);
1858 }
1859 
1860 /*
1861  * Maximum number of concurrent scrub i/os to create while verifying
1862  * a pool while importing it.
1863  */
1864 int spa_load_verify_maxinflight = 10000;
1865 boolean_t spa_load_verify_metadata = B_TRUE;
1866 boolean_t spa_load_verify_data = B_TRUE;
1867 
1868 /*ARGSUSED*/
1869 static int
1870 spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
1871     const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
1872 {
1873 	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp))
1874 		return (0);
1875 	/*
1876 	 * Note: normally this routine will not be called if
1877 	 * spa_load_verify_metadata is not set.  However, it may be useful
1878 	 * to manually set the flag after the traversal has begun.
1879 	 */
1880 	if (!spa_load_verify_metadata)
1881 		return (0);
1882 	if (BP_GET_BUFC_TYPE(bp) == ARC_BUFC_DATA && !spa_load_verify_data)
1883 		return (0);
1884 
1885 	zio_t *rio = arg;
1886 	size_t size = BP_GET_PSIZE(bp);
1887 	void *data = zio_data_buf_alloc(size);
1888 
1889 	mutex_enter(&spa->spa_scrub_lock);
1890 	while (spa->spa_scrub_inflight >= spa_load_verify_maxinflight)
1891 		cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock);
1892 	spa->spa_scrub_inflight++;
1893 	mutex_exit(&spa->spa_scrub_lock);
1894 
1895 	zio_nowait(zio_read(rio, spa, bp, data, size,
1896 	    spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB,
1897 	    ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL |
1898 	    ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb));
1899 	return (0);
1900 }
1901 
1902 static int
1903 spa_load_verify(spa_t *spa)
1904 {
1905 	zio_t *rio;
1906 	spa_load_error_t sle = { 0 };
1907 	zpool_rewind_policy_t policy;
1908 	boolean_t verify_ok = B_FALSE;
1909 	int error = 0;
1910 
1911 	zpool_get_rewind_policy(spa->spa_config, &policy);
1912 
1913 	if (policy.zrp_request & ZPOOL_NEVER_REWIND)
1914 		return (0);
1915 
1916 	rio = zio_root(spa, NULL, &sle,
1917 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE);
1918 
1919 	if (spa_load_verify_metadata) {
1920 		error = traverse_pool(spa, spa->spa_verify_min_txg,
1921 		    TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA,
1922 		    spa_load_verify_cb, rio);
1923 	}
1924 
1925 	(void) zio_wait(rio);
1926 
1927 	spa->spa_load_meta_errors = sle.sle_meta_count;
1928 	spa->spa_load_data_errors = sle.sle_data_count;
1929 
1930 	if (!error && sle.sle_meta_count <= policy.zrp_maxmeta &&
1931 	    sle.sle_data_count <= policy.zrp_maxdata) {
1932 		int64_t loss = 0;
1933 
1934 		verify_ok = B_TRUE;
1935 		spa->spa_load_txg = spa->spa_uberblock.ub_txg;
1936 		spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp;
1937 
1938 		loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts;
1939 		VERIFY(nvlist_add_uint64(spa->spa_load_info,
1940 		    ZPOOL_CONFIG_LOAD_TIME, spa->spa_load_txg_ts) == 0);
1941 		VERIFY(nvlist_add_int64(spa->spa_load_info,
1942 		    ZPOOL_CONFIG_REWIND_TIME, loss) == 0);
1943 		VERIFY(nvlist_add_uint64(spa->spa_load_info,
1944 		    ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count) == 0);
1945 	} else {
1946 		spa->spa_load_max_txg = spa->spa_uberblock.ub_txg;
1947 	}
1948 
1949 	if (error) {
1950 		if (error != ENXIO && error != EIO)
1951 			error = SET_ERROR(EIO);
1952 		return (error);
1953 	}
1954 
1955 	return (verify_ok ? 0 : EIO);
1956 }
1957 
1958 /*
1959  * Find a value in the pool props object.
1960  */
1961 static void
1962 spa_prop_find(spa_t *spa, zpool_prop_t prop, uint64_t *val)
1963 {
1964 	(void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object,
1965 	    zpool_prop_to_name(prop), sizeof (uint64_t), 1, val);
1966 }
1967 
1968 /*
1969  * Find a value in the pool directory object.
1970  */
1971 static int
1972 spa_dir_prop(spa_t *spa, const char *name, uint64_t *val)
1973 {
1974 	return (zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
1975 	    name, sizeof (uint64_t), 1, val));
1976 }
1977 
1978 static int
1979 spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err)
1980 {
1981 	vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux);
1982 	return (err);
1983 }
1984 
1985 /*
1986  * Fix up config after a partly-completed split.  This is done with the
1987  * ZPOOL_CONFIG_SPLIT nvlist.  Both the splitting pool and the split-off
1988  * pool have that entry in their config, but only the splitting one contains
1989  * a list of all the guids of the vdevs that are being split off.
1990  *
1991  * This function determines what to do with that list: either rejoin
1992  * all the disks to the pool, or complete the splitting process.  To attempt
1993  * the rejoin, each disk that is offlined is marked online again, and
1994  * we do a reopen() call.  If the vdev label for every disk that was
1995  * marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL)
1996  * then we call vdev_split() on each disk, and complete the split.
1997  *
1998  * Otherwise we leave the config alone, with all the vdevs in place in
1999  * the original pool.
2000  */
2001 static void
2002 spa_try_repair(spa_t *spa, nvlist_t *config)
2003 {
2004 	uint_t extracted;
2005 	uint64_t *glist;
2006 	uint_t i, gcount;
2007 	nvlist_t *nvl;
2008 	vdev_t **vd;
2009 	boolean_t attempt_reopen;
2010 
2011 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0)
2012 		return;
2013 
2014 	/* check that the config is complete */
2015 	if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST,
2016 	    &glist, &gcount) != 0)
2017 		return;
2018 
2019 	vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP);
2020 
2021 	/* attempt to online all the vdevs & validate */
2022 	attempt_reopen = B_TRUE;
2023 	for (i = 0; i < gcount; i++) {
2024 		if (glist[i] == 0)	/* vdev is hole */
2025 			continue;
2026 
2027 		vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE);
2028 		if (vd[i] == NULL) {
2029 			/*
2030 			 * Don't bother attempting to reopen the disks;
2031 			 * just do the split.
2032 			 */
2033 			attempt_reopen = B_FALSE;
2034 		} else {
2035 			/* attempt to re-online it */
2036 			vd[i]->vdev_offline = B_FALSE;
2037 		}
2038 	}
2039 
2040 	if (attempt_reopen) {
2041 		vdev_reopen(spa->spa_root_vdev);
2042 
2043 		/* check each device to see what state it's in */
2044 		for (extracted = 0, i = 0; i < gcount; i++) {
2045 			if (vd[i] != NULL &&
2046 			    vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL)
2047 				break;
2048 			++extracted;
2049 		}
2050 	}
2051 
2052 	/*
2053 	 * If every disk has been moved to the new pool, or if we never
2054 	 * even attempted to look at them, then we split them off for
2055 	 * good.
2056 	 */
2057 	if (!attempt_reopen || gcount == extracted) {
2058 		for (i = 0; i < gcount; i++)
2059 			if (vd[i] != NULL)
2060 				vdev_split(vd[i]);
2061 		vdev_reopen(spa->spa_root_vdev);
2062 	}
2063 
2064 	kmem_free(vd, gcount * sizeof (vdev_t *));
2065 }
2066 
2067 static int
2068 spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type,
2069     boolean_t mosconfig)
2070 {
2071 	nvlist_t *config = spa->spa_config;
2072 	char *ereport = FM_EREPORT_ZFS_POOL;
2073 	char *comment;
2074 	int error;
2075 	uint64_t pool_guid;
2076 	nvlist_t *nvl;
2077 
2078 	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid))
2079 		return (SET_ERROR(EINVAL));
2080 
2081 	ASSERT(spa->spa_comment == NULL);
2082 	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
2083 		spa->spa_comment = spa_strdup(comment);
2084 
2085 	/*
2086 	 * Versioning wasn't explicitly added to the label until later, so if
2087 	 * it's not present treat it as the initial version.
2088 	 */
2089 	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
2090 	    &spa->spa_ubsync.ub_version) != 0)
2091 		spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL;
2092 
2093 	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG,
2094 	    &spa->spa_config_txg);
2095 
2096 	if ((state == SPA_LOAD_IMPORT || state == SPA_LOAD_TRYIMPORT) &&
2097 	    spa_guid_exists(pool_guid, 0)) {
2098 		error = SET_ERROR(EEXIST);
2099 	} else {
2100 		spa->spa_config_guid = pool_guid;
2101 
2102 		if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT,
2103 		    &nvl) == 0) {
2104 			VERIFY(nvlist_dup(nvl, &spa->spa_config_splitting,
2105 			    KM_SLEEP) == 0);
2106 		}
2107 
2108 		nvlist_free(spa->spa_load_info);
2109 		spa->spa_load_info = fnvlist_alloc();
2110 
2111 		gethrestime(&spa->spa_loaded_ts);
2112 		error = spa_load_impl(spa, pool_guid, config, state, type,
2113 		    mosconfig, &ereport);
2114 	}
2115 
2116 	/*
2117 	 * Don't count references from objsets that are already closed
2118 	 * and are making their way through the eviction process.
2119 	 */
2120 	spa_evicting_os_wait(spa);
2121 	spa->spa_minref = refcount_count(&spa->spa_refcount);
2122 	if (error) {
2123 		if (error != EEXIST) {
2124 			spa->spa_loaded_ts.tv_sec = 0;
2125 			spa->spa_loaded_ts.tv_nsec = 0;
2126 		}
2127 		if (error != EBADF) {
2128 			zfs_ereport_post(ereport, spa, NULL, NULL, 0, 0);
2129 		}
2130 	}
2131 	spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE;
2132 	spa->spa_ena = 0;
2133 
2134 	return (error);
2135 }
2136 
2137 /*
2138  * Load an existing storage pool, using the pool's builtin spa_config as a
2139  * source of configuration information.
2140  */
2141 static int
2142 spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config,
2143     spa_load_state_t state, spa_import_type_t type, boolean_t mosconfig,
2144     char **ereport)
2145 {
2146 	int error = 0;
2147 	nvlist_t *nvroot = NULL;
2148 	nvlist_t *label;
2149 	vdev_t *rvd;
2150 	uberblock_t *ub = &spa->spa_uberblock;
2151 	uint64_t children, config_cache_txg = spa->spa_config_txg;
2152 	int orig_mode = spa->spa_mode;
2153 	int parse;
2154 	uint64_t obj;
2155 	boolean_t missing_feat_write = B_FALSE;
2156 
2157 	/*
2158 	 * If this is an untrusted config, access the pool in read-only mode.
2159 	 * This prevents things like resilvering recently removed devices.
2160 	 */
2161 	if (!mosconfig)
2162 		spa->spa_mode = FREAD;
2163 
2164 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
2165 
2166 	spa->spa_load_state = state;
2167 
2168 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvroot))
2169 		return (SET_ERROR(EINVAL));
2170 
2171 	parse = (type == SPA_IMPORT_EXISTING ?
2172 	    VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT);
2173 
2174 	/*
2175 	 * Create "The Godfather" zio to hold all async IOs
2176 	 */
2177 	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
2178 	    KM_SLEEP);
2179 	for (int i = 0; i < max_ncpus; i++) {
2180 		spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
2181 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
2182 		    ZIO_FLAG_GODFATHER);
2183 	}
2184 
2185 	/*
2186 	 * Parse the configuration into a vdev tree.  We explicitly set the
2187 	 * value that will be returned by spa_version() since parsing the
2188 	 * configuration requires knowing the version number.
2189 	 */
2190 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
2191 	error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, parse);
2192 	spa_config_exit(spa, SCL_ALL, FTAG);
2193 
2194 	if (error != 0)
2195 		return (error);
2196 
2197 	ASSERT(spa->spa_root_vdev == rvd);
2198 
2199 	if (type != SPA_IMPORT_ASSEMBLE) {
2200 		ASSERT(spa_guid(spa) == pool_guid);
2201 	}
2202 
2203 	/*
2204 	 * Try to open all vdevs, loading each label in the process.
2205 	 */
2206 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
2207 	error = vdev_open(rvd);
2208 	spa_config_exit(spa, SCL_ALL, FTAG);
2209 	if (error != 0)
2210 		return (error);
2211 
2212 	/*
2213 	 * We need to validate the vdev labels against the configuration that
2214 	 * we have in hand, which is dependent on the setting of mosconfig. If
2215 	 * mosconfig is true then we're validating the vdev labels based on
2216 	 * that config.  Otherwise, we're validating against the cached config
2217 	 * (zpool.cache) that was read when we loaded the zfs module, and then
2218 	 * later we will recursively call spa_load() and validate against
2219 	 * the vdev config.
2220 	 *
2221 	 * If we're assembling a new pool that's been split off from an
2222 	 * existing pool, the labels haven't yet been updated so we skip
2223 	 * validation for now.
2224 	 */
2225 	if (type != SPA_IMPORT_ASSEMBLE) {
2226 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
2227 		error = vdev_validate(rvd, mosconfig);
2228 		spa_config_exit(spa, SCL_ALL, FTAG);
2229 
2230 		if (error != 0)
2231 			return (error);
2232 
2233 		if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN)
2234 			return (SET_ERROR(ENXIO));
2235 	}
2236 
2237 	/*
2238 	 * Find the best uberblock.
2239 	 */
2240 	vdev_uberblock_load(rvd, ub, &label);
2241 
2242 	/*
2243 	 * If we weren't able to find a single valid uberblock, return failure.
2244 	 */
2245 	if (ub->ub_txg == 0) {
2246 		nvlist_free(label);
2247 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO));
2248 	}
2249 
2250 	/*
2251 	 * If the pool has an unsupported version we can't open it.
2252 	 */
2253 	if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) {
2254 		nvlist_free(label);
2255 		return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP));
2256 	}
2257 
2258 	if (ub->ub_version >= SPA_VERSION_FEATURES) {
2259 		nvlist_t *features;
2260 
2261 		/*
2262 		 * If we weren't able to find what's necessary for reading the
2263 		 * MOS in the label, return failure.
2264 		 */
2265 		if (label == NULL || nvlist_lookup_nvlist(label,
2266 		    ZPOOL_CONFIG_FEATURES_FOR_READ, &features) != 0) {
2267 			nvlist_free(label);
2268 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
2269 			    ENXIO));
2270 		}
2271 
2272 		/*
2273 		 * Update our in-core representation with the definitive values
2274 		 * from the label.
2275 		 */
2276 		nvlist_free(spa->spa_label_features);
2277 		VERIFY(nvlist_dup(features, &spa->spa_label_features, 0) == 0);
2278 	}
2279 
2280 	nvlist_free(label);
2281 
2282 	/*
2283 	 * Look through entries in the label nvlist's features_for_read. If
2284 	 * there is a feature listed there which we don't understand then we
2285 	 * cannot open a pool.
2286 	 */
2287 	if (ub->ub_version >= SPA_VERSION_FEATURES) {
2288 		nvlist_t *unsup_feat;
2289 
2290 		VERIFY(nvlist_alloc(&unsup_feat, NV_UNIQUE_NAME, KM_SLEEP) ==
2291 		    0);
2292 
2293 		for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features,
2294 		    NULL); nvp != NULL;
2295 		    nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) {
2296 			if (!zfeature_is_supported(nvpair_name(nvp))) {
2297 				VERIFY(nvlist_add_string(unsup_feat,
2298 				    nvpair_name(nvp), "") == 0);
2299 			}
2300 		}
2301 
2302 		if (!nvlist_empty(unsup_feat)) {
2303 			VERIFY(nvlist_add_nvlist(spa->spa_load_info,
2304 			    ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat) == 0);
2305 			nvlist_free(unsup_feat);
2306 			return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT,
2307 			    ENOTSUP));
2308 		}
2309 
2310 		nvlist_free(unsup_feat);
2311 	}
2312 
2313 	/*
2314 	 * If the vdev guid sum doesn't match the uberblock, we have an
2315 	 * incomplete configuration.  We first check to see if the pool
2316 	 * is aware of the complete config (i.e ZPOOL_CONFIG_VDEV_CHILDREN).
2317 	 * If it is, defer the vdev_guid_sum check till later so we
2318 	 * can handle missing vdevs.
2319 	 */
2320 	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VDEV_CHILDREN,
2321 	    &children) != 0 && mosconfig && type != SPA_IMPORT_ASSEMBLE &&
2322 	    rvd->vdev_guid_sum != ub->ub_guid_sum)
2323 		return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO));
2324 
2325 	if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) {
2326 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
2327 		spa_try_repair(spa, config);
2328 		spa_config_exit(spa, SCL_ALL, FTAG);
2329 		nvlist_free(spa->spa_config_splitting);
2330 		spa->spa_config_splitting = NULL;
2331 	}
2332 
2333 	/*
2334 	 * Initialize internal SPA structures.
2335 	 */
2336 	spa->spa_state = POOL_STATE_ACTIVE;
2337 	spa->spa_ubsync = spa->spa_uberblock;
2338 	spa->spa_verify_min_txg = spa->spa_extreme_rewind ?
2339 	    TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1;
2340 	spa->spa_first_txg = spa->spa_last_ubsync_txg ?
2341 	    spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1;
2342 	spa->spa_claim_max_txg = spa->spa_first_txg;
2343 	spa->spa_prev_software_version = ub->ub_software_version;
2344 
2345 	error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool);
2346 	if (error)
2347 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2348 	spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset;
2349 
2350 	if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object) != 0)
2351 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2352 
2353 	if (spa_version(spa) >= SPA_VERSION_FEATURES) {
2354 		boolean_t missing_feat_read = B_FALSE;
2355 		nvlist_t *unsup_feat, *enabled_feat;
2356 
2357 		if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ,
2358 		    &spa->spa_feat_for_read_obj) != 0) {
2359 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2360 		}
2361 
2362 		if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE,
2363 		    &spa->spa_feat_for_write_obj) != 0) {
2364 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2365 		}
2366 
2367 		if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS,
2368 		    &spa->spa_feat_desc_obj) != 0) {
2369 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2370 		}
2371 
2372 		enabled_feat = fnvlist_alloc();
2373 		unsup_feat = fnvlist_alloc();
2374 
2375 		if (!spa_features_check(spa, B_FALSE,
2376 		    unsup_feat, enabled_feat))
2377 			missing_feat_read = B_TRUE;
2378 
2379 		if (spa_writeable(spa) || state == SPA_LOAD_TRYIMPORT) {
2380 			if (!spa_features_check(spa, B_TRUE,
2381 			    unsup_feat, enabled_feat)) {
2382 				missing_feat_write = B_TRUE;
2383 			}
2384 		}
2385 
2386 		fnvlist_add_nvlist(spa->spa_load_info,
2387 		    ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat);
2388 
2389 		if (!nvlist_empty(unsup_feat)) {
2390 			fnvlist_add_nvlist(spa->spa_load_info,
2391 			    ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat);
2392 		}
2393 
2394 		fnvlist_free(enabled_feat);
2395 		fnvlist_free(unsup_feat);
2396 
2397 		if (!missing_feat_read) {
2398 			fnvlist_add_boolean(spa->spa_load_info,
2399 			    ZPOOL_CONFIG_CAN_RDONLY);
2400 		}
2401 
2402 		/*
2403 		 * If the state is SPA_LOAD_TRYIMPORT, our objective is
2404 		 * twofold: to determine whether the pool is available for
2405 		 * import in read-write mode and (if it is not) whether the
2406 		 * pool is available for import in read-only mode. If the pool
2407 		 * is available for import in read-write mode, it is displayed
2408 		 * as available in userland; if it is not available for import
2409 		 * in read-only mode, it is displayed as unavailable in
2410 		 * userland. If the pool is available for import in read-only
2411 		 * mode but not read-write mode, it is displayed as unavailable
2412 		 * in userland with a special note that the pool is actually
2413 		 * available for open in read-only mode.
2414 		 *
2415 		 * As a result, if the state is SPA_LOAD_TRYIMPORT and we are
2416 		 * missing a feature for write, we must first determine whether
2417 		 * the pool can be opened read-only before returning to
2418 		 * userland in order to know whether to display the
2419 		 * abovementioned note.
2420 		 */
2421 		if (missing_feat_read || (missing_feat_write &&
2422 		    spa_writeable(spa))) {
2423 			return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT,
2424 			    ENOTSUP));
2425 		}
2426 
2427 		/*
2428 		 * Load refcounts for ZFS features from disk into an in-memory
2429 		 * cache during SPA initialization.
2430 		 */
2431 		for (spa_feature_t i = 0; i < SPA_FEATURES; i++) {
2432 			uint64_t refcount;
2433 
2434 			error = feature_get_refcount_from_disk(spa,
2435 			    &spa_feature_table[i], &refcount);
2436 			if (error == 0) {
2437 				spa->spa_feat_refcount_cache[i] = refcount;
2438 			} else if (error == ENOTSUP) {
2439 				spa->spa_feat_refcount_cache[i] =
2440 				    SPA_FEATURE_DISABLED;
2441 			} else {
2442 				return (spa_vdev_err(rvd,
2443 				    VDEV_AUX_CORRUPT_DATA, EIO));
2444 			}
2445 		}
2446 	}
2447 
2448 	if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) {
2449 		if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG,
2450 		    &spa->spa_feat_enabled_txg_obj) != 0)
2451 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2452 	}
2453 
2454 	spa->spa_is_initializing = B_TRUE;
2455 	error = dsl_pool_open(spa->spa_dsl_pool);
2456 	spa->spa_is_initializing = B_FALSE;
2457 	if (error != 0)
2458 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2459 
2460 	if (!mosconfig) {
2461 		uint64_t hostid;
2462 		nvlist_t *policy = NULL, *nvconfig;
2463 
2464 		if (load_nvlist(spa, spa->spa_config_object, &nvconfig) != 0)
2465 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2466 
2467 		if (!spa_is_root(spa) && nvlist_lookup_uint64(nvconfig,
2468 		    ZPOOL_CONFIG_HOSTID, &hostid) == 0) {
2469 			char *hostname;
2470 			unsigned long myhostid = 0;
2471 
2472 			VERIFY(nvlist_lookup_string(nvconfig,
2473 			    ZPOOL_CONFIG_HOSTNAME, &hostname) == 0);
2474 
2475 #ifdef	_KERNEL
2476 			myhostid = zone_get_hostid(NULL);
2477 #else	/* _KERNEL */
2478 			/*
2479 			 * We're emulating the system's hostid in userland, so
2480 			 * we can't use zone_get_hostid().
2481 			 */
2482 			(void) ddi_strtoul(hw_serial, NULL, 10, &myhostid);
2483 #endif	/* _KERNEL */
2484 			if (hostid != 0 && myhostid != 0 &&
2485 			    hostid != myhostid) {
2486 				nvlist_free(nvconfig);
2487 				cmn_err(CE_WARN, "pool '%s' could not be "
2488 				    "loaded as it was last accessed by "
2489 				    "another system (host: %s hostid: 0x%lx). "
2490 				    "See: http://illumos.org/msg/ZFS-8000-EY",
2491 				    spa_name(spa), hostname,
2492 				    (unsigned long)hostid);
2493 				return (SET_ERROR(EBADF));
2494 			}
2495 		}
2496 		if (nvlist_lookup_nvlist(spa->spa_config,
2497 		    ZPOOL_REWIND_POLICY, &policy) == 0)
2498 			VERIFY(nvlist_add_nvlist(nvconfig,
2499 			    ZPOOL_REWIND_POLICY, policy) == 0);
2500 
2501 		spa_config_set(spa, nvconfig);
2502 		spa_unload(spa);
2503 		spa_deactivate(spa);
2504 		spa_activate(spa, orig_mode);
2505 
2506 		return (spa_load(spa, state, SPA_IMPORT_EXISTING, B_TRUE));
2507 	}
2508 
2509 	if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj) != 0)
2510 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2511 	error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj);
2512 	if (error != 0)
2513 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2514 
2515 	/*
2516 	 * Load the bit that tells us to use the new accounting function
2517 	 * (raid-z deflation).  If we have an older pool, this will not
2518 	 * be present.
2519 	 */
2520 	error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate);
2521 	if (error != 0 && error != ENOENT)
2522 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2523 
2524 	error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION,
2525 	    &spa->spa_creation_version);
2526 	if (error != 0 && error != ENOENT)
2527 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2528 
2529 	/*
2530 	 * Load the persistent error log.  If we have an older pool, this will
2531 	 * not be present.
2532 	 */
2533 	error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last);
2534 	if (error != 0 && error != ENOENT)
2535 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2536 
2537 	error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB,
2538 	    &spa->spa_errlog_scrub);
2539 	if (error != 0 && error != ENOENT)
2540 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2541 
2542 	/*
2543 	 * Load the history object.  If we have an older pool, this
2544 	 * will not be present.
2545 	 */
2546 	error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history);
2547 	if (error != 0 && error != ENOENT)
2548 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2549 
2550 	/*
2551 	 * If we're assembling the pool from the split-off vdevs of
2552 	 * an existing pool, we don't want to attach the spares & cache
2553 	 * devices.
2554 	 */
2555 
2556 	/*
2557 	 * Load any hot spares for this pool.
2558 	 */
2559 	error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object);
2560 	if (error != 0 && error != ENOENT)
2561 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2562 	if (error == 0 && type != SPA_IMPORT_ASSEMBLE) {
2563 		ASSERT(spa_version(spa) >= SPA_VERSION_SPARES);
2564 		if (load_nvlist(spa, spa->spa_spares.sav_object,
2565 		    &spa->spa_spares.sav_config) != 0)
2566 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2567 
2568 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
2569 		spa_load_spares(spa);
2570 		spa_config_exit(spa, SCL_ALL, FTAG);
2571 	} else if (error == 0) {
2572 		spa->spa_spares.sav_sync = B_TRUE;
2573 	}
2574 
2575 	/*
2576 	 * Load any level 2 ARC devices for this pool.
2577 	 */
2578 	error = spa_dir_prop(spa, DMU_POOL_L2CACHE,
2579 	    &spa->spa_l2cache.sav_object);
2580 	if (error != 0 && error != ENOENT)
2581 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2582 	if (error == 0 && type != SPA_IMPORT_ASSEMBLE) {
2583 		ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE);
2584 		if (load_nvlist(spa, spa->spa_l2cache.sav_object,
2585 		    &spa->spa_l2cache.sav_config) != 0)
2586 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2587 
2588 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
2589 		spa_load_l2cache(spa);
2590 		spa_config_exit(spa, SCL_ALL, FTAG);
2591 	} else if (error == 0) {
2592 		spa->spa_l2cache.sav_sync = B_TRUE;
2593 	}
2594 
2595 	spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);
2596 
2597 	error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object);
2598 	if (error && error != ENOENT)
2599 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2600 
2601 	if (error == 0) {
2602 		uint64_t autoreplace;
2603 
2604 		spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs);
2605 		spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace);
2606 		spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation);
2607 		spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode);
2608 		spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand);
2609 		spa_prop_find(spa, ZPOOL_PROP_DEDUPDITTO,
2610 		    &spa->spa_dedup_ditto);
2611 
2612 		spa->spa_autoreplace = (autoreplace != 0);
2613 	}
2614 
2615 	/*
2616 	 * If the 'autoreplace' property is set, then post a resource notifying
2617 	 * the ZFS DE that it should not issue any faults for unopenable
2618 	 * devices.  We also iterate over the vdevs, and post a sysevent for any
2619 	 * unopenable vdevs so that the normal autoreplace handler can take
2620 	 * over.
2621 	 */
2622 	if (spa->spa_autoreplace && state != SPA_LOAD_TRYIMPORT) {
2623 		spa_check_removed(spa->spa_root_vdev);
2624 		/*
2625 		 * For the import case, this is done in spa_import(), because
2626 		 * at this point we're using the spare definitions from
2627 		 * the MOS config, not necessarily from the userland config.
2628 		 */
2629 		if (state != SPA_LOAD_IMPORT) {
2630 			spa_aux_check_removed(&spa->spa_spares);
2631 			spa_aux_check_removed(&spa->spa_l2cache);
2632 		}
2633 	}
2634 
2635 	/*
2636 	 * Load the vdev state for all toplevel vdevs.
2637 	 */
2638 	vdev_load(rvd);
2639 
2640 	/*
2641 	 * Propagate the leaf DTLs we just loaded all the way up the tree.
2642 	 */
2643 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
2644 	vdev_dtl_reassess(rvd, 0, 0, B_FALSE);
2645 	spa_config_exit(spa, SCL_ALL, FTAG);
2646 
2647 	/*
2648 	 * Load the DDTs (dedup tables).
2649 	 */
2650 	error = ddt_load(spa);
2651 	if (error != 0)
2652 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2653 
2654 	spa_update_dspace(spa);
2655 
2656 	/*
2657 	 * Validate the config, using the MOS config to fill in any
2658 	 * information which might be missing.  If we fail to validate
2659 	 * the config then declare the pool unfit for use. If we're
2660 	 * assembling a pool from a split, the log is not transferred
2661 	 * over.
2662 	 */
2663 	if (type != SPA_IMPORT_ASSEMBLE) {
2664 		nvlist_t *nvconfig;
2665 
2666 		if (load_nvlist(spa, spa->spa_config_object, &nvconfig) != 0)
2667 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2668 
2669 		if (!spa_config_valid(spa, nvconfig)) {
2670 			nvlist_free(nvconfig);
2671 			return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM,
2672 			    ENXIO));
2673 		}
2674 		nvlist_free(nvconfig);
2675 
2676 		/*
2677 		 * Now that we've validated the config, check the state of the
2678 		 * root vdev.  If it can't be opened, it indicates one or
2679 		 * more toplevel vdevs are faulted.
2680 		 */
2681 		if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN)
2682 			return (SET_ERROR(ENXIO));
2683 
2684 		if (spa_check_logs(spa)) {
2685 			*ereport = FM_EREPORT_ZFS_LOG_REPLAY;
2686 			return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG, ENXIO));
2687 		}
2688 	}
2689 
2690 	if (missing_feat_write) {
2691 		ASSERT(state == SPA_LOAD_TRYIMPORT);
2692 
2693 		/*
2694 		 * At this point, we know that we can open the pool in
2695 		 * read-only mode but not read-write mode. We now have enough
2696 		 * information and can return to userland.
2697 		 */
2698 		return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, ENOTSUP));
2699 	}
2700 
2701 	/*
2702 	 * We've successfully opened the pool, verify that we're ready
2703 	 * to start pushing transactions.
2704 	 */
2705 	if (state != SPA_LOAD_TRYIMPORT) {
2706 		if (error = spa_load_verify(spa))
2707 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
2708 			    error));
2709 	}
2710 
2711 	if (spa_writeable(spa) && (state == SPA_LOAD_RECOVER ||
2712 	    spa->spa_load_max_txg == UINT64_MAX)) {
2713 		dmu_tx_t *tx;
2714 		int need_update = B_FALSE;
2715 
2716 		ASSERT(state != SPA_LOAD_TRYIMPORT);
2717 
2718 		/*
2719 		 * Claim log blocks that haven't been committed yet.
2720 		 * This must all happen in a single txg.
2721 		 * Note: spa_claim_max_txg is updated by spa_claim_notify(),
2722 		 * invoked from zil_claim_log_block()'s i/o done callback.
2723 		 * Price of rollback is that we abandon the log.
2724 		 */
2725 		spa->spa_claiming = B_TRUE;
2726 
2727 		tx = dmu_tx_create_assigned(spa_get_dsl(spa),
2728 		    spa_first_txg(spa));
2729 		(void) dmu_objset_find(spa_name(spa),
2730 		    zil_claim, tx, DS_FIND_CHILDREN);
2731 		dmu_tx_commit(tx);
2732 
2733 		spa->spa_claiming = B_FALSE;
2734 
2735 		spa_set_log_state(spa, SPA_LOG_GOOD);
2736 		spa->spa_sync_on = B_TRUE;
2737 		txg_sync_start(spa->spa_dsl_pool);
2738 
2739 		/*
2740 		 * Wait for all claims to sync.  We sync up to the highest
2741 		 * claimed log block birth time so that claimed log blocks
2742 		 * don't appear to be from the future.  spa_claim_max_txg
2743 		 * will have been set for us by either zil_check_log_chain()
2744 		 * (invoked from spa_check_logs()) or zil_claim() above.
2745 		 */
2746 		txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg);
2747 
2748 		/*
2749 		 * If the config cache is stale, or we have uninitialized
2750 		 * metaslabs (see spa_vdev_add()), then update the config.
2751 		 *
2752 		 * If this is a verbatim import, trust the current
2753 		 * in-core spa_config and update the disk labels.
2754 		 */
2755 		if (config_cache_txg != spa->spa_config_txg ||
2756 		    state == SPA_LOAD_IMPORT ||
2757 		    state == SPA_LOAD_RECOVER ||
2758 		    (spa->spa_import_flags & ZFS_IMPORT_VERBATIM))
2759 			need_update = B_TRUE;
2760 
2761 		for (int c = 0; c < rvd->vdev_children; c++)
2762 			if (rvd->vdev_child[c]->vdev_ms_array == 0)
2763 				need_update = B_TRUE;
2764 
2765 		/*
2766 		 * Update the config cache asychronously in case we're the
2767 		 * root pool, in which case the config cache isn't writable yet.
2768 		 */
2769 		if (need_update)
2770 			spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
2771 
2772 		/*
2773 		 * Check all DTLs to see if anything needs resilvering.
2774 		 */
2775 		if (!dsl_scan_resilvering(spa->spa_dsl_pool) &&
2776 		    vdev_resilver_needed(rvd, NULL, NULL))
2777 			spa_async_request(spa, SPA_ASYNC_RESILVER);
2778 
2779 		/*
2780 		 * Log the fact that we booted up (so that we can detect if
2781 		 * we rebooted in the middle of an operation).
2782 		 */
2783 		spa_history_log_version(spa, "open");
2784 
2785 		/*
2786 		 * Delete any inconsistent datasets.
2787 		 */
2788 		(void) dmu_objset_find(spa_name(spa),
2789 		    dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN);
2790 
2791 		/*
2792 		 * Clean up any stale temporary dataset userrefs.
2793 		 */
2794 		dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool);
2795 	}
2796 
2797 	return (0);
2798 }
2799 
2800 static int
2801 spa_load_retry(spa_t *spa, spa_load_state_t state, int mosconfig)
2802 {
2803 	int mode = spa->spa_mode;
2804 
2805 	spa_unload(spa);
2806 	spa_deactivate(spa);
2807 
2808 	spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1;
2809 
2810 	spa_activate(spa, mode);
2811 	spa_async_suspend(spa);
2812 
2813 	return (spa_load(spa, state, SPA_IMPORT_EXISTING, mosconfig));
2814 }
2815 
2816 /*
2817  * If spa_load() fails this function will try loading prior txg's. If
2818  * 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool
2819  * will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this
2820  * function will not rewind the pool and will return the same error as
2821  * spa_load().
2822  */
2823 static int
2824 spa_load_best(spa_t *spa, spa_load_state_t state, int mosconfig,
2825     uint64_t max_request, int rewind_flags)
2826 {
2827 	nvlist_t *loadinfo = NULL;
2828 	nvlist_t *config = NULL;
2829 	int load_error, rewind_error;
2830 	uint64_t safe_rewind_txg;
2831 	uint64_t min_txg;
2832 
2833 	if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) {
2834 		spa->spa_load_max_txg = spa->spa_load_txg;
2835 		spa_set_log_state(spa, SPA_LOG_CLEAR);
2836 	} else {
2837 		spa->spa_load_max_txg = max_request;
2838 		if (max_request != UINT64_MAX)
2839 			spa->spa_extreme_rewind = B_TRUE;
2840 	}
2841 
2842 	load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING,
2843 	    mosconfig);
2844 	if (load_error == 0)
2845 		return (0);
2846 
2847 	if (spa->spa_root_vdev != NULL)
2848 		config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
2849 
2850 	spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg;
2851 	spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp;
2852 
2853 	if (rewind_flags & ZPOOL_NEVER_REWIND) {
2854 		nvlist_free(config);
2855 		return (load_error);
2856 	}
2857 
2858 	if (state == SPA_LOAD_RECOVER) {
2859 		/* Price of rolling back is discarding txgs, including log */
2860 		spa_set_log_state(spa, SPA_LOG_CLEAR);
2861 	} else {
2862 		/*
2863 		 * If we aren't rolling back save the load info from our first
2864 		 * import attempt so that we can restore it after attempting
2865 		 * to rewind.
2866 		 */
2867 		loadinfo = spa->spa_load_info;
2868 		spa->spa_load_info = fnvlist_alloc();
2869 	}
2870 
2871 	spa->spa_load_max_txg = spa->spa_last_ubsync_txg;
2872 	safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE;
2873 	min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ?
2874 	    TXG_INITIAL : safe_rewind_txg;
2875 
2876 	/*
2877 	 * Continue as long as we're finding errors, we're still within
2878 	 * the acceptable rewind range, and we're still finding uberblocks
2879 	 */
2880 	while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg &&
2881 	    spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) {
2882 		if (spa->spa_load_max_txg < safe_rewind_txg)
2883 			spa->spa_extreme_rewind = B_TRUE;
2884 		rewind_error = spa_load_retry(spa, state, mosconfig);
2885 	}
2886 
2887 	spa->spa_extreme_rewind = B_FALSE;
2888 	spa->spa_load_max_txg = UINT64_MAX;
2889 
2890 	if (config && (rewind_error || state != SPA_LOAD_RECOVER))
2891 		spa_config_set(spa, config);
2892 
2893 	if (state == SPA_LOAD_RECOVER) {
2894 		ASSERT3P(loadinfo, ==, NULL);
2895 		return (rewind_error);
2896 	} else {
2897 		/* Store the rewind info as part of the initial load info */
2898 		fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO,
2899 		    spa->spa_load_info);
2900 
2901 		/* Restore the initial load info */
2902 		fnvlist_free(spa->spa_load_info);
2903 		spa->spa_load_info = loadinfo;
2904 
2905 		return (load_error);
2906 	}
2907 }
2908 
2909 /*
2910  * Pool Open/Import
2911  *
2912  * The import case is identical to an open except that the configuration is sent
2913  * down from userland, instead of grabbed from the configuration cache.  For the
2914  * case of an open, the pool configuration will exist in the
2915  * POOL_STATE_UNINITIALIZED state.
2916  *
2917  * The stats information (gen/count/ustats) is used to gather vdev statistics at
2918  * the same time open the pool, without having to keep around the spa_t in some
2919  * ambiguous state.
2920  */
2921 static int
2922 spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t *nvpolicy,
2923     nvlist_t **config)
2924 {
2925 	spa_t *spa;
2926 	spa_load_state_t state = SPA_LOAD_OPEN;
2927 	int error;
2928 	int locked = B_FALSE;
2929 
2930 	*spapp = NULL;
2931 
2932 	/*
2933 	 * As disgusting as this is, we need to support recursive calls to this
2934 	 * function because dsl_dir_open() is called during spa_load(), and ends
2935 	 * up calling spa_open() again.  The real fix is to figure out how to
2936 	 * avoid dsl_dir_open() calling this in the first place.
2937 	 */
2938 	if (mutex_owner(&spa_namespace_lock) != curthread) {
2939 		mutex_enter(&spa_namespace_lock);
2940 		locked = B_TRUE;
2941 	}
2942 
2943 	if ((spa = spa_lookup(pool)) == NULL) {
2944 		if (locked)
2945 			mutex_exit(&spa_namespace_lock);
2946 		return (SET_ERROR(ENOENT));
2947 	}
2948 
2949 	if (spa->spa_state == POOL_STATE_UNINITIALIZED) {
2950 		zpool_rewind_policy_t policy;
2951 
2952 		zpool_get_rewind_policy(nvpolicy ? nvpolicy : spa->spa_config,
2953 		    &policy);
2954 		if (policy.zrp_request & ZPOOL_DO_REWIND)
2955 			state = SPA_LOAD_RECOVER;
2956 
2957 		spa_activate(spa, spa_mode_global);
2958 
2959 		if (state != SPA_LOAD_RECOVER)
2960 			spa->spa_last_ubsync_txg = spa->spa_load_txg = 0;
2961 
2962 		error = spa_load_best(spa, state, B_FALSE, policy.zrp_txg,
2963 		    policy.zrp_request);
2964 
2965 		if (error == EBADF) {
2966 			/*
2967 			 * If vdev_validate() returns failure (indicated by
2968 			 * EBADF), it indicates that one of the vdevs indicates
2969 			 * that the pool has been exported or destroyed.  If
2970 			 * this is the case, the config cache is out of sync and
2971 			 * we should remove the pool from the namespace.
2972 			 */
2973 			spa_unload(spa);
2974 			spa_deactivate(spa);
2975 			spa_config_sync(spa, B_TRUE, B_TRUE);
2976 			spa_remove(spa);
2977 			if (locked)
2978 				mutex_exit(&spa_namespace_lock);
2979 			return (SET_ERROR(ENOENT));
2980 		}
2981 
2982 		if (error) {
2983 			/*
2984 			 * We can't open the pool, but we still have useful
2985 			 * information: the state of each vdev after the
2986 			 * attempted vdev_open().  Return this to the user.
2987 			 */
2988 			if (config != NULL && spa->spa_config) {
2989 				VERIFY(nvlist_dup(spa->spa_config, config,
2990 				    KM_SLEEP) == 0);
2991 				VERIFY(nvlist_add_nvlist(*config,
2992 				    ZPOOL_CONFIG_LOAD_INFO,
2993 				    spa->spa_load_info) == 0);
2994 			}
2995 			spa_unload(spa);
2996 			spa_deactivate(spa);
2997 			spa->spa_last_open_failed = error;
2998 			if (locked)
2999 				mutex_exit(&spa_namespace_lock);
3000 			*spapp = NULL;
3001 			return (error);
3002 		}
3003 	}
3004 
3005 	spa_open_ref(spa, tag);
3006 
3007 	if (config != NULL)
3008 		*config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
3009 
3010 	/*
3011 	 * If we've recovered the pool, pass back any information we
3012 	 * gathered while doing the load.
3013 	 */
3014 	if (state == SPA_LOAD_RECOVER) {
3015 		VERIFY(nvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO,
3016 		    spa->spa_load_info) == 0);
3017 	}
3018 
3019 	if (locked) {
3020 		spa->spa_last_open_failed = 0;
3021 		spa->spa_last_ubsync_txg = 0;
3022 		spa->spa_load_txg = 0;
3023 		mutex_exit(&spa_namespace_lock);
3024 	}
3025 
3026 	*spapp = spa;
3027 
3028 	return (0);
3029 }
3030 
3031 int
3032 spa_open_rewind(const char *name, spa_t **spapp, void *tag, nvlist_t *policy,
3033     nvlist_t **config)
3034 {
3035 	return (spa_open_common(name, spapp, tag, policy, config));
3036 }
3037 
3038 int
3039 spa_open(const char *name, spa_t **spapp, void *tag)
3040 {
3041 	return (spa_open_common(name, spapp, tag, NULL, NULL));
3042 }
3043 
3044 /*
3045  * Lookup the given spa_t, incrementing the inject count in the process,
3046  * preventing it from being exported or destroyed.
3047  */
3048 spa_t *
3049 spa_inject_addref(char *name)
3050 {
3051 	spa_t *spa;
3052 
3053 	mutex_enter(&spa_namespace_lock);
3054 	if ((spa = spa_lookup(name)) == NULL) {
3055 		mutex_exit(&spa_namespace_lock);
3056 		return (NULL);
3057 	}
3058 	spa->spa_inject_ref++;
3059 	mutex_exit(&spa_namespace_lock);
3060 
3061 	return (spa);
3062 }
3063 
3064 void
3065 spa_inject_delref(spa_t *spa)
3066 {
3067 	mutex_enter(&spa_namespace_lock);
3068 	spa->spa_inject_ref--;
3069 	mutex_exit(&spa_namespace_lock);
3070 }
3071 
3072 /*
3073  * Add spares device information to the nvlist.
3074  */
3075 static void
3076 spa_add_spares(spa_t *spa, nvlist_t *config)
3077 {
3078 	nvlist_t **spares;
3079 	uint_t i, nspares;
3080 	nvlist_t *nvroot;
3081 	uint64_t guid;
3082 	vdev_stat_t *vs;
3083 	uint_t vsc;
3084 	uint64_t pool;
3085 
3086 	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
3087 
3088 	if (spa->spa_spares.sav_count == 0)
3089 		return;
3090 
3091 	VERIFY(nvlist_lookup_nvlist(config,
3092 	    ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0);
3093 	VERIFY(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
3094 	    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0);
3095 	if (nspares != 0) {
3096 		VERIFY(nvlist_add_nvlist_array(nvroot,
3097 		    ZPOOL_CONFIG_SPARES, spares, nspares) == 0);
3098 		VERIFY(nvlist_lookup_nvlist_array(nvroot,
3099 		    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0);
3100 
3101 		/*
3102 		 * Go through and find any spares which have since been
3103 		 * repurposed as an active spare.  If this is the case, update
3104 		 * their status appropriately.
3105 		 */
3106 		for (i = 0; i < nspares; i++) {
3107 			VERIFY(nvlist_lookup_uint64(spares[i],
3108 			    ZPOOL_CONFIG_GUID, &guid) == 0);
3109 			if (spa_spare_exists(guid, &pool, NULL) &&
3110 			    pool != 0ULL) {
3111 				VERIFY(nvlist_lookup_uint64_array(
3112 				    spares[i], ZPOOL_CONFIG_VDEV_STATS,
3113 				    (uint64_t **)&vs, &vsc) == 0);
3114 				vs->vs_state = VDEV_STATE_CANT_OPEN;
3115 				vs->vs_aux = VDEV_AUX_SPARED;
3116 			}
3117 		}
3118 	}
3119 }
3120 
3121 /*
3122  * Add l2cache device information to the nvlist, including vdev stats.
3123  */
3124 static void
3125 spa_add_l2cache(spa_t *spa, nvlist_t *config)
3126 {
3127 	nvlist_t **l2cache;
3128 	uint_t i, j, nl2cache;
3129 	nvlist_t *nvroot;
3130 	uint64_t guid;
3131 	vdev_t *vd;
3132 	vdev_stat_t *vs;
3133 	uint_t vsc;
3134 
3135 	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
3136 
3137 	if (spa->spa_l2cache.sav_count == 0)
3138 		return;
3139 
3140 	VERIFY(nvlist_lookup_nvlist(config,
3141 	    ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0);
3142 	VERIFY(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config,
3143 	    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0);
3144 	if (nl2cache != 0) {
3145 		VERIFY(nvlist_add_nvlist_array(nvroot,
3146 		    ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0);
3147 		VERIFY(nvlist_lookup_nvlist_array(nvroot,
3148 		    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0);
3149 
3150 		/*
3151 		 * Update level 2 cache device stats.
3152 		 */
3153 
3154 		for (i = 0; i < nl2cache; i++) {
3155 			VERIFY(nvlist_lookup_uint64(l2cache[i],
3156 			    ZPOOL_CONFIG_GUID, &guid) == 0);
3157 
3158 			vd = NULL;
3159 			for (j = 0; j < spa->spa_l2cache.sav_count; j++) {
3160 				if (guid ==
3161 				    spa->spa_l2cache.sav_vdevs[j]->vdev_guid) {
3162 					vd = spa->spa_l2cache.sav_vdevs[j];
3163 					break;
3164 				}
3165 			}
3166 			ASSERT(vd != NULL);
3167 
3168 			VERIFY(nvlist_lookup_uint64_array(l2cache[i],
3169 			    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc)
3170 			    == 0);
3171 			vdev_get_stats(vd, vs);
3172 		}
3173 	}
3174 }
3175 
3176 static void
3177 spa_add_feature_stats(spa_t *spa, nvlist_t *config)
3178 {
3179 	nvlist_t *features;
3180 	zap_cursor_t zc;
3181 	zap_attribute_t za;
3182 
3183 	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
3184 	VERIFY(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP) == 0);
3185 
3186 	if (spa->spa_feat_for_read_obj != 0) {
3187 		for (zap_cursor_init(&zc, spa->spa_meta_objset,
3188 		    spa->spa_feat_for_read_obj);
3189 		    zap_cursor_retrieve(&zc, &za) == 0;
3190 		    zap_cursor_advance(&zc)) {
3191 			ASSERT(za.za_integer_length == sizeof (uint64_t) &&
3192 			    za.za_num_integers == 1);
3193 			VERIFY3U(0, ==, nvlist_add_uint64(features, za.za_name,
3194 			    za.za_first_integer));
3195 		}
3196 		zap_cursor_fini(&zc);
3197 	}
3198 
3199 	if (spa->spa_feat_for_write_obj != 0) {
3200 		for (zap_cursor_init(&zc, spa->spa_meta_objset,
3201 		    spa->spa_feat_for_write_obj);
3202 		    zap_cursor_retrieve(&zc, &za) == 0;
3203 		    zap_cursor_advance(&zc)) {
3204 			ASSERT(za.za_integer_length == sizeof (uint64_t) &&
3205 			    za.za_num_integers == 1);
3206 			VERIFY3U(0, ==, nvlist_add_uint64(features, za.za_name,
3207 			    za.za_first_integer));
3208 		}
3209 		zap_cursor_fini(&zc);
3210 	}
3211 
3212 	VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS,
3213 	    features) == 0);
3214 	nvlist_free(features);
3215 }
3216 
3217 int
3218 spa_get_stats(const char *name, nvlist_t **config,
3219     char *altroot, size_t buflen)
3220 {
3221 	int error;
3222 	spa_t *spa;
3223 
3224 	*config = NULL;
3225 	error = spa_open_common(name, &spa, FTAG, NULL, config);
3226 
3227 	if (spa != NULL) {
3228 		/*
3229 		 * This still leaves a window of inconsistency where the spares
3230 		 * or l2cache devices could change and the config would be
3231 		 * self-inconsistent.
3232 		 */
3233 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
3234 
3235 		if (*config != NULL) {
3236 			uint64_t loadtimes[2];
3237 
3238 			loadtimes[0] = spa->spa_loaded_ts.tv_sec;
3239 			loadtimes[1] = spa->spa_loaded_ts.tv_nsec;
3240 			VERIFY(nvlist_add_uint64_array(*config,
3241 			    ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2) == 0);
3242 
3243 			VERIFY(nvlist_add_uint64(*config,
3244 			    ZPOOL_CONFIG_ERRCOUNT,
3245 			    spa_get_errlog_size(spa)) == 0);
3246 
3247 			if (spa_suspended(spa))
3248 				VERIFY(nvlist_add_uint64(*config,
3249 				    ZPOOL_CONFIG_SUSPENDED,
3250 				    spa->spa_failmode) == 0);
3251 
3252 			spa_add_spares(spa, *config);
3253 			spa_add_l2cache(spa, *config);
3254 			spa_add_feature_stats(spa, *config);
3255 		}
3256 	}
3257 
3258 	/*
3259 	 * We want to get the alternate root even for faulted pools, so we cheat
3260 	 * and call spa_lookup() directly.
3261 	 */
3262 	if (altroot) {
3263 		if (spa == NULL) {
3264 			mutex_enter(&spa_namespace_lock);
3265 			spa = spa_lookup(name);
3266 			if (spa)
3267 				spa_altroot(spa, altroot, buflen);
3268 			else
3269 				altroot[0] = '\0';
3270 			spa = NULL;
3271 			mutex_exit(&spa_namespace_lock);
3272 		} else {
3273 			spa_altroot(spa, altroot, buflen);
3274 		}
3275 	}
3276 
3277 	if (spa != NULL) {
3278 		spa_config_exit(spa, SCL_CONFIG, FTAG);
3279 		spa_close(spa, FTAG);
3280 	}
3281 
3282 	return (error);
3283 }
3284 
3285 /*
3286  * Validate that the auxiliary device array is well formed.  We must have an
3287  * array of nvlists, each which describes a valid leaf vdev.  If this is an
3288  * import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be
3289  * specified, as long as they are well-formed.
3290  */
3291 static int
3292 spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode,
3293     spa_aux_vdev_t *sav, const char *config, uint64_t version,
3294     vdev_labeltype_t label)
3295 {
3296 	nvlist_t **dev;
3297 	uint_t i, ndev;
3298 	vdev_t *vd;
3299 	int error;
3300 
3301 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
3302 
3303 	/*
3304 	 * It's acceptable to have no devs specified.
3305 	 */
3306 	if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0)
3307 		return (0);
3308 
3309 	if (ndev == 0)
3310 		return (SET_ERROR(EINVAL));
3311 
3312 	/*
3313 	 * Make sure the pool is formatted with a version that supports this
3314 	 * device type.
3315 	 */
3316 	if (spa_version(spa) < version)
3317 		return (SET_ERROR(ENOTSUP));
3318 
3319 	/*
3320 	 * Set the pending device list so we correctly handle device in-use
3321 	 * checking.
3322 	 */
3323 	sav->sav_pending = dev;
3324 	sav->sav_npending = ndev;
3325 
3326 	for (i = 0; i < ndev; i++) {
3327 		if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0,
3328 		    mode)) != 0)
3329 			goto out;
3330 
3331 		if (!vd->vdev_ops->vdev_op_leaf) {
3332 			vdev_free(vd);
3333 			error = SET_ERROR(EINVAL);
3334 			goto out;
3335 		}
3336 
3337 		/*
3338 		 * The L2ARC currently only supports disk devices in
3339 		 * kernel context.  For user-level testing, we allow it.
3340 		 */
3341 #ifdef _KERNEL
3342 		if ((strcmp(config, ZPOOL_CONFIG_L2CACHE) == 0) &&
3343 		    strcmp(vd->vdev_ops->vdev_op_type, VDEV_TYPE_DISK) != 0) {
3344 			error = SET_ERROR(ENOTBLK);
3345 			vdev_free(vd);
3346 			goto out;
3347 		}
3348 #endif
3349 		vd->vdev_top = vd;
3350 
3351 		if ((error = vdev_open(vd)) == 0 &&
3352 		    (error = vdev_label_init(vd, crtxg, label)) == 0) {
3353 			VERIFY(nvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID,
3354 			    vd->vdev_guid) == 0);
3355 		}
3356 
3357 		vdev_free(vd);
3358 
3359 		if (error &&
3360 		    (mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE))
3361 			goto out;
3362 		else
3363 			error = 0;
3364 	}
3365 
3366 out:
3367 	sav->sav_pending = NULL;
3368 	sav->sav_npending = 0;
3369 	return (error);
3370 }
3371 
3372 static int
3373 spa_validate_aux(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode)
3374 {
3375 	int error;
3376 
3377 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
3378 
3379 	if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode,
3380 	    &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES,
3381 	    VDEV_LABEL_SPARE)) != 0) {
3382 		return (error);
3383 	}
3384 
3385 	return (spa_validate_aux_devs(spa, nvroot, crtxg, mode,
3386 	    &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE,
3387 	    VDEV_LABEL_L2CACHE));
3388 }
3389 
3390 static void
3391 spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs,
3392     const char *config)
3393 {
3394 	int i;
3395 
3396 	if (sav->sav_config != NULL) {
3397 		nvlist_t **olddevs;
3398 		uint_t oldndevs;
3399 		nvlist_t **newdevs;
3400 
3401 		/*
3402 		 * Generate new dev list by concatentating with the
3403 		 * current dev list.
3404 		 */
3405 		VERIFY(nvlist_lookup_nvlist_array(sav->sav_config, config,
3406 		    &olddevs, &oldndevs) == 0);
3407 
3408 		newdevs = kmem_alloc(sizeof (void *) *
3409 		    (ndevs + oldndevs), KM_SLEEP);
3410 		for (i = 0; i < oldndevs; i++)
3411 			VERIFY(nvlist_dup(olddevs[i], &newdevs[i],
3412 			    KM_SLEEP) == 0);
3413 		for (i = 0; i < ndevs; i++)
3414 			VERIFY(nvlist_dup(devs[i], &newdevs[i + oldndevs],
3415 			    KM_SLEEP) == 0);
3416 
3417 		VERIFY(nvlist_remove(sav->sav_config, config,
3418 		    DATA_TYPE_NVLIST_ARRAY) == 0);
3419 
3420 		VERIFY(nvlist_add_nvlist_array(sav->sav_config,
3421 		    config, newdevs, ndevs + oldndevs) == 0);
3422 		for (i = 0; i < oldndevs + ndevs; i++)
3423 			nvlist_free(newdevs[i]);
3424 		kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *));
3425 	} else {
3426 		/*
3427 		 * Generate a new dev list.
3428 		 */
3429 		VERIFY(nvlist_alloc(&sav->sav_config, NV_UNIQUE_NAME,
3430 		    KM_SLEEP) == 0);
3431 		VERIFY(nvlist_add_nvlist_array(sav->sav_config, config,
3432 		    devs, ndevs) == 0);
3433 	}
3434 }
3435 
3436 /*
3437  * Stop and drop level 2 ARC devices
3438  */
3439 void
3440 spa_l2cache_drop(spa_t *spa)
3441 {
3442 	vdev_t *vd;
3443 	int i;
3444 	spa_aux_vdev_t *sav = &spa->spa_l2cache;
3445 
3446 	for (i = 0; i < sav->sav_count; i++) {
3447 		uint64_t pool;
3448 
3449 		vd = sav->sav_vdevs[i];
3450 		ASSERT(vd != NULL);
3451 
3452 		if (spa_l2cache_exists(vd->vdev_guid, &pool) &&
3453 		    pool != 0ULL && l2arc_vdev_present(vd))
3454 			l2arc_remove_vdev(vd);
3455 	}
3456 }
3457 
3458 /*
3459  * Pool Creation
3460  */
3461 int
3462 spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,
3463     nvlist_t *zplprops)
3464 {
3465 	spa_t *spa;
3466 	char *altroot = NULL;
3467 	vdev_t *rvd;
3468 	dsl_pool_t *dp;
3469 	dmu_tx_t *tx;
3470 	int error = 0;
3471 	uint64_t txg = TXG_INITIAL;
3472 	nvlist_t **spares, **l2cache;
3473 	uint_t nspares, nl2cache;
3474 	uint64_t version, obj;
3475 	boolean_t has_features;
3476 
3477 	/*
3478 	 * If this pool already exists, return failure.
3479 	 */
3480 	mutex_enter(&spa_namespace_lock);
3481 	if (spa_lookup(pool) != NULL) {
3482 		mutex_exit(&spa_namespace_lock);
3483 		return (SET_ERROR(EEXIST));
3484 	}
3485 
3486 	/*
3487 	 * Allocate a new spa_t structure.
3488 	 */
3489 	(void) nvlist_lookup_string(props,
3490 	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
3491 	spa = spa_add(pool, NULL, altroot);
3492 	spa_activate(spa, spa_mode_global);
3493 
3494 	if (props && (error = spa_prop_validate(spa, props))) {
3495 		spa_deactivate(spa);
3496 		spa_remove(spa);
3497 		mutex_exit(&spa_namespace_lock);
3498 		return (error);
3499 	}
3500 
3501 	has_features = B_FALSE;
3502 	for (nvpair_t *elem = nvlist_next_nvpair(props, NULL);
3503 	    elem != NULL; elem = nvlist_next_nvpair(props, elem)) {
3504 		if (zpool_prop_feature(nvpair_name(elem)))
3505 			has_features = B_TRUE;
3506 	}
3507 
3508 	if (has_features || nvlist_lookup_uint64(props,
3509 	    zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) {
3510 		version = SPA_VERSION;
3511 	}
3512 	ASSERT(SPA_VERSION_IS_SUPPORTED(version));
3513 
3514 	spa->spa_first_txg = txg;
3515 	spa->spa_uberblock.ub_txg = txg - 1;
3516 	spa->spa_uberblock.ub_version = version;
3517 	spa->spa_ubsync = spa->spa_uberblock;
3518 
3519 	/*
3520 	 * Create "The Godfather" zio to hold all async IOs
3521 	 */
3522 	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
3523 	    KM_SLEEP);
3524 	for (int i = 0; i < max_ncpus; i++) {
3525 		spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
3526 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
3527 		    ZIO_FLAG_GODFATHER);
3528 	}
3529 
3530 	/*
3531 	 * Create the root vdev.
3532 	 */
3533 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3534 
3535 	error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD);
3536 
3537 	ASSERT(error != 0 || rvd != NULL);
3538 	ASSERT(error != 0 || spa->spa_root_vdev == rvd);
3539 
3540 	if (error == 0 && !zfs_allocatable_devs(nvroot))
3541 		error = SET_ERROR(EINVAL);
3542 
3543 	if (error == 0 &&
3544 	    (error = vdev_create(rvd, txg, B_FALSE)) == 0 &&
3545 	    (error = spa_validate_aux(spa, nvroot, txg,
3546 	    VDEV_ALLOC_ADD)) == 0) {
3547 		for (int c = 0; c < rvd->vdev_children; c++) {
3548 			vdev_metaslab_set_size(rvd->vdev_child[c]);
3549 			vdev_expand(rvd->vdev_child[c], txg);
3550 		}
3551 	}
3552 
3553 	spa_config_exit(spa, SCL_ALL, FTAG);
3554 
3555 	if (error != 0) {
3556 		spa_unload(spa);
3557 		spa_deactivate(spa);
3558 		spa_remove(spa);
3559 		mutex_exit(&spa_namespace_lock);
3560 		return (error);
3561 	}
3562 
3563 	/*
3564 	 * Get the list of spares, if specified.
3565 	 */
3566 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
3567 	    &spares, &nspares) == 0) {
3568 		VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, NV_UNIQUE_NAME,
3569 		    KM_SLEEP) == 0);
3570 		VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config,
3571 		    ZPOOL_CONFIG_SPARES, spares, nspares) == 0);
3572 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3573 		spa_load_spares(spa);
3574 		spa_config_exit(spa, SCL_ALL, FTAG);
3575 		spa->spa_spares.sav_sync = B_TRUE;
3576 	}
3577 
3578 	/*
3579 	 * Get the list of level 2 cache devices, if specified.
3580 	 */
3581 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
3582 	    &l2cache, &nl2cache) == 0) {
3583 		VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config,
3584 		    NV_UNIQUE_NAME, KM_SLEEP) == 0);
3585 		VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config,
3586 		    ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0);
3587 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3588 		spa_load_l2cache(spa);
3589 		spa_config_exit(spa, SCL_ALL, FTAG);
3590 		spa->spa_l2cache.sav_sync = B_TRUE;
3591 	}
3592 
3593 	spa->spa_is_initializing = B_TRUE;
3594 	spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, txg);
3595 	spa->spa_meta_objset = dp->dp_meta_objset;
3596 	spa->spa_is_initializing = B_FALSE;
3597 
3598 	/*
3599 	 * Create DDTs (dedup tables).
3600 	 */
3601 	ddt_create(spa);
3602 
3603 	spa_update_dspace(spa);
3604 
3605 	tx = dmu_tx_create_assigned(dp, txg);
3606 
3607 	/*
3608 	 * Create the pool config object.
3609 	 */
3610 	spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset,
3611 	    DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE,
3612 	    DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx);
3613 
3614 	if (zap_add(spa->spa_meta_objset,
3615 	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG,
3616 	    sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) {
3617 		cmn_err(CE_PANIC, "failed to add pool config");
3618 	}
3619 
3620 	if (spa_version(spa) >= SPA_VERSION_FEATURES)
3621 		spa_feature_create_zap_objects(spa, tx);
3622 
3623 	if (zap_add(spa->spa_meta_objset,
3624 	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION,
3625 	    sizeof (uint64_t), 1, &version, tx) != 0) {
3626 		cmn_err(CE_PANIC, "failed to add pool version");
3627 	}
3628 
3629 	/* Newly created pools with the right version are always deflated. */
3630 	if (version >= SPA_VERSION_RAIDZ_DEFLATE) {
3631 		spa->spa_deflate = TRUE;
3632 		if (zap_add(spa->spa_meta_objset,
3633 		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE,
3634 		    sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) {
3635 			cmn_err(CE_PANIC, "failed to add deflate");
3636 		}
3637 	}
3638 
3639 	/*
3640 	 * Create the deferred-free bpobj.  Turn off compression
3641 	 * because sync-to-convergence takes longer if the blocksize
3642 	 * keeps changing.
3643 	 */
3644 	obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx);
3645 	dmu_object_set_compress(spa->spa_meta_objset, obj,
3646 	    ZIO_COMPRESS_OFF, tx);
3647 	if (zap_add(spa->spa_meta_objset,
3648 	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ,
3649 	    sizeof (uint64_t), 1, &obj, tx) != 0) {
3650 		cmn_err(CE_PANIC, "failed to add bpobj");
3651 	}
3652 	VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj,
3653 	    spa->spa_meta_objset, obj));
3654 
3655 	/*
3656 	 * Create the pool's history object.
3657 	 */
3658 	if (version >= SPA_VERSION_ZPOOL_HISTORY)
3659 		spa_history_create_obj(spa, tx);
3660 
3661 	/*
3662 	 * Set pool properties.
3663 	 */
3664 	spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS);
3665 	spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);
3666 	spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE);
3667 	spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND);
3668 
3669 	if (props != NULL) {
3670 		spa_configfile_set(spa, props, B_FALSE);
3671 		spa_sync_props(props, tx);
3672 	}
3673 
3674 	dmu_tx_commit(tx);
3675 
3676 	spa->spa_sync_on = B_TRUE;
3677 	txg_sync_start(spa->spa_dsl_pool);
3678 
3679 	/*
3680 	 * We explicitly wait for the first transaction to complete so that our
3681 	 * bean counters are appropriately updated.
3682 	 */
3683 	txg_wait_synced(spa->spa_dsl_pool, txg);
3684 
3685 	spa_config_sync(spa, B_FALSE, B_TRUE);
3686 
3687 	spa_history_log_version(spa, "create");
3688 
3689 	/*
3690 	 * Don't count references from objsets that are already closed
3691 	 * and are making their way through the eviction process.
3692 	 */
3693 	spa_evicting_os_wait(spa);
3694 	spa->spa_minref = refcount_count(&spa->spa_refcount);
3695 
3696 	mutex_exit(&spa_namespace_lock);
3697 
3698 	return (0);
3699 }
3700 
3701 #ifdef _KERNEL
3702 /*
3703  * Get the root pool information from the root disk, then import the root pool
3704  * during the system boot up time.
3705  */
3706 extern int vdev_disk_read_rootlabel(char *, char *, nvlist_t **);
3707 
3708 static nvlist_t *
3709 spa_generate_rootconf(char *devpath, char *devid, uint64_t *guid)
3710 {
3711 	nvlist_t *config;
3712 	nvlist_t *nvtop, *nvroot;
3713 	uint64_t pgid;
3714 
3715 	if (vdev_disk_read_rootlabel(devpath, devid, &config) != 0)
3716 		return (NULL);
3717 
3718 	/*
3719 	 * Add this top-level vdev to the child array.
3720 	 */
3721 	VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
3722 	    &nvtop) == 0);
3723 	VERIFY(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
3724 	    &pgid) == 0);
3725 	VERIFY(nvlist_lookup_uint64(config, ZPOOL_CONFIG_GUID, guid) == 0);
3726 
3727 	/*
3728 	 * Put this pool's top-level vdevs into a root vdev.
3729 	 */
3730 	VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0);
3731 	VERIFY(nvlist_add_string(nvroot, ZPOOL_CONFIG_TYPE,
3732 	    VDEV_TYPE_ROOT) == 0);
3733 	VERIFY(nvlist_add_uint64(nvroot, ZPOOL_CONFIG_ID, 0ULL) == 0);
3734 	VERIFY(nvlist_add_uint64(nvroot, ZPOOL_CONFIG_GUID, pgid) == 0);
3735 	VERIFY(nvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
3736 	    &nvtop, 1) == 0);
3737 
3738 	/*
3739 	 * Replace the existing vdev_tree with the new root vdev in
3740 	 * this pool's configuration (remove the old, add the new).
3741 	 */
3742 	VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, nvroot) == 0);
3743 	nvlist_free(nvroot);
3744 	return (config);
3745 }
3746 
3747 /*
3748  * Walk the vdev tree and see if we can find a device with "better"
3749  * configuration. A configuration is "better" if the label on that
3750  * device has a more recent txg.
3751  */
3752 static void
3753 spa_alt_rootvdev(vdev_t *vd, vdev_t **avd, uint64_t *txg)
3754 {
3755 	for (int c = 0; c < vd->vdev_children; c++)
3756 		spa_alt_rootvdev(vd->vdev_child[c], avd, txg);
3757 
3758 	if (vd->vdev_ops->vdev_op_leaf) {
3759 		nvlist_t *label;
3760 		uint64_t label_txg;
3761 
3762 		if (vdev_disk_read_rootlabel(vd->vdev_physpath, vd->vdev_devid,
3763 		    &label) != 0)
3764 			return;
3765 
3766 		VERIFY(nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_TXG,
3767 		    &label_txg) == 0);
3768 
3769 		/*
3770 		 * Do we have a better boot device?
3771 		 */
3772 		if (label_txg > *txg) {
3773 			*txg = label_txg;
3774 			*avd = vd;
3775 		}
3776 		nvlist_free(label);
3777 	}
3778 }
3779 
3780 /*
3781  * Import a root pool.
3782  *
3783  * For x86. devpath_list will consist of devid and/or physpath name of
3784  * the vdev (e.g. "id1,sd@SSEAGATE..." or "/pci@1f,0/ide@d/disk@0,0:a").
3785  * The GRUB "findroot" command will return the vdev we should boot.
3786  *
3787  * For Sparc, devpath_list consists the physpath name of the booting device
3788  * no matter the rootpool is a single device pool or a mirrored pool.
3789  * e.g.
3790  *	"/pci@1f,0/ide@d/disk@0,0:a"
3791  */
3792 int
3793 spa_import_rootpool(char *devpath, char *devid)
3794 {
3795 	spa_t *spa;
3796 	vdev_t *rvd, *bvd, *avd = NULL;
3797 	nvlist_t *config, *nvtop;
3798 	uint64_t guid, txg;
3799 	char *pname;
3800 	int error;
3801 
3802 	/*
3803 	 * Read the label from the boot device and generate a configuration.
3804 	 */
3805 	config = spa_generate_rootconf(devpath, devid, &guid);
3806 #if defined(_OBP) && defined(_KERNEL)
3807 	if (config == NULL) {
3808 		if (strstr(devpath, "/iscsi/ssd") != NULL) {
3809 			/* iscsi boot */
3810 			get_iscsi_bootpath_phy(devpath);
3811 			config = spa_generate_rootconf(devpath, devid, &guid);
3812 		}
3813 	}
3814 #endif
3815 	if (config == NULL) {
3816 		cmn_err(CE_NOTE, "Cannot read the pool label from '%s'",
3817 		    devpath);
3818 		return (SET_ERROR(EIO));
3819 	}
3820 
3821 	VERIFY(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
3822 	    &pname) == 0);
3823 	VERIFY(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, &txg) == 0);
3824 
3825 	mutex_enter(&spa_namespace_lock);
3826 	if ((spa = spa_lookup(pname)) != NULL) {
3827 		/*
3828 		 * Remove the existing root pool from the namespace so that we
3829 		 * can replace it with the correct config we just read in.
3830 		 */
3831 		spa_remove(spa);
3832 	}
3833 
3834 	spa = spa_add(pname, config, NULL);
3835 	spa->spa_is_root = B_TRUE;
3836 	spa->spa_import_flags = ZFS_IMPORT_VERBATIM;
3837 
3838 	/*
3839 	 * Build up a vdev tree based on the boot device's label config.
3840 	 */
3841 	VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
3842 	    &nvtop) == 0);
3843 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3844 	error = spa_config_parse(spa, &rvd, nvtop, NULL, 0,
3845 	    VDEV_ALLOC_ROOTPOOL);
3846 	spa_config_exit(spa, SCL_ALL, FTAG);
3847 	if (error) {
3848 		mutex_exit(&spa_namespace_lock);
3849 		nvlist_free(config);
3850 		cmn_err(CE_NOTE, "Can not parse the config for pool '%s'",
3851 		    pname);
3852 		return (error);
3853 	}
3854 
3855 	/*
3856 	 * Get the boot vdev.
3857 	 */
3858 	if ((bvd = vdev_lookup_by_guid(rvd, guid)) == NULL) {
3859 		cmn_err(CE_NOTE, "Can not find the boot vdev for guid %llu",
3860 		    (u_longlong_t)guid);
3861 		error = SET_ERROR(ENOENT);
3862 		goto out;
3863 	}
3864 
3865 	/*
3866 	 * Determine if there is a better boot device.
3867 	 */
3868 	avd = bvd;
3869 	spa_alt_rootvdev(rvd, &avd, &txg);
3870 	if (avd != bvd) {
3871 		cmn_err(CE_NOTE, "The boot device is 'degraded'. Please "
3872 		    "try booting from '%s'", avd->vdev_path);
3873 		error = SET_ERROR(EINVAL);
3874 		goto out;
3875 	}
3876 
3877 	/*
3878 	 * If the boot device is part of a spare vdev then ensure that
3879 	 * we're booting off the active spare.
3880 	 */
3881 	if (bvd->vdev_parent->vdev_ops == &vdev_spare_ops &&
3882 	    !bvd->vdev_isspare) {
3883 		cmn_err(CE_NOTE, "The boot device is currently spared. Please "
3884 		    "try booting from '%s'",
3885 		    bvd->vdev_parent->
3886 		    vdev_child[bvd->vdev_parent->vdev_children - 1]->vdev_path);
3887 		error = SET_ERROR(EINVAL);
3888 		goto out;
3889 	}
3890 
3891 	error = 0;
3892 out:
3893 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3894 	vdev_free(rvd);
3895 	spa_config_exit(spa, SCL_ALL, FTAG);
3896 	mutex_exit(&spa_namespace_lock);
3897 
3898 	nvlist_free(config);
3899 	return (error);
3900 }
3901 
3902 #endif
3903 
3904 /*
3905  * Import a non-root pool into the system.
3906  */
3907 int
3908 spa_import(const char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags)
3909 {
3910 	spa_t *spa;
3911 	char *altroot = NULL;
3912 	spa_load_state_t state = SPA_LOAD_IMPORT;
3913 	zpool_rewind_policy_t policy;
3914 	uint64_t mode = spa_mode_global;
3915 	uint64_t readonly = B_FALSE;
3916 	int error;
3917 	nvlist_t *nvroot;
3918 	nvlist_t **spares, **l2cache;
3919 	uint_t nspares, nl2cache;
3920 
3921 	/*
3922 	 * If a pool with this name exists, return failure.
3923 	 */
3924 	mutex_enter(&spa_namespace_lock);
3925 	if (spa_lookup(pool) != NULL) {
3926 		mutex_exit(&spa_namespace_lock);
3927 		return (SET_ERROR(EEXIST));
3928 	}
3929 
3930 	/*
3931 	 * Create and initialize the spa structure.
3932 	 */
3933 	(void) nvlist_lookup_string(props,
3934 	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
3935 	(void) nvlist_lookup_uint64(props,
3936 	    zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly);
3937 	if (readonly)
3938 		mode = FREAD;
3939 	spa = spa_add(pool, config, altroot);
3940 	spa->spa_import_flags = flags;
3941 
3942 	/*
3943 	 * Verbatim import - Take a pool and insert it into the namespace
3944 	 * as if it had been loaded at boot.
3945 	 */
3946 	if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) {
3947 		if (props != NULL)
3948 			spa_configfile_set(spa, props, B_FALSE);
3949 
3950 		spa_config_sync(spa, B_FALSE, B_TRUE);
3951 
3952 		mutex_exit(&spa_namespace_lock);
3953 		return (0);
3954 	}
3955 
3956 	spa_activate(spa, mode);
3957 
3958 	/*
3959 	 * Don't start async tasks until we know everything is healthy.
3960 	 */
3961 	spa_async_suspend(spa);
3962 
3963 	zpool_get_rewind_policy(config, &policy);
3964 	if (policy.zrp_request & ZPOOL_DO_REWIND)
3965 		state = SPA_LOAD_RECOVER;
3966 
3967 	/*
3968 	 * Pass off the heavy lifting to spa_load().  Pass TRUE for mosconfig
3969 	 * because the user-supplied config is actually the one to trust when
3970 	 * doing an import.
3971 	 */
3972 	if (state != SPA_LOAD_RECOVER)
3973 		spa->spa_last_ubsync_txg = spa->spa_load_txg = 0;
3974 
3975 	error = spa_load_best(spa, state, B_TRUE, policy.zrp_txg,
3976 	    policy.zrp_request);
3977 
3978 	/*
3979 	 * Propagate anything learned while loading the pool and pass it
3980 	 * back to caller (i.e. rewind info, missing devices, etc).
3981 	 */
3982 	VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO,
3983 	    spa->spa_load_info) == 0);
3984 
3985 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3986 	/*
3987 	 * Toss any existing sparelist, as it doesn't have any validity
3988 	 * anymore, and conflicts with spa_has_spare().
3989 	 */
3990 	if (spa->spa_spares.sav_config) {
3991 		nvlist_free(spa->spa_spares.sav_config);
3992 		spa->spa_spares.sav_config = NULL;
3993 		spa_load_spares(spa);
3994 	}
3995 	if (spa->spa_l2cache.sav_config) {
3996 		nvlist_free(spa->spa_l2cache.sav_config);
3997 		spa->spa_l2cache.sav_config = NULL;
3998 		spa_load_l2cache(spa);
3999 	}
4000 
4001 	VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
4002 	    &nvroot) == 0);
4003 	if (error == 0)
4004 		error = spa_validate_aux(spa, nvroot, -1ULL,
4005 		    VDEV_ALLOC_SPARE);
4006 	if (error == 0)
4007 		error = spa_validate_aux(spa, nvroot, -1ULL,
4008 		    VDEV_ALLOC_L2CACHE);
4009 	spa_config_exit(spa, SCL_ALL, FTAG);
4010 
4011 	if (props != NULL)
4012 		spa_configfile_set(spa, props, B_FALSE);
4013 
4014 	if (error != 0 || (props && spa_writeable(spa) &&
4015 	    (error = spa_prop_set(spa, props)))) {
4016 		spa_unload(spa);
4017 		spa_deactivate(spa);
4018 		spa_remove(spa);
4019 		mutex_exit(&spa_namespace_lock);
4020 		return (error);
4021 	}
4022 
4023 	spa_async_resume(spa);
4024 
4025 	/*
4026 	 * Override any spares and level 2 cache devices as specified by
4027 	 * the user, as these may have correct device names/devids, etc.
4028 	 */
4029 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
4030 	    &spares, &nspares) == 0) {
4031 		if (spa->spa_spares.sav_config)
4032 			VERIFY(nvlist_remove(spa->spa_spares.sav_config,
4033 			    ZPOOL_CONFIG_SPARES, DATA_TYPE_NVLIST_ARRAY) == 0);
4034 		else
4035 			VERIFY(nvlist_alloc(&spa->spa_spares.sav_config,
4036 			    NV_UNIQUE_NAME, KM_SLEEP) == 0);
4037 		VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config,
4038 		    ZPOOL_CONFIG_SPARES, spares, nspares) == 0);
4039 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4040 		spa_load_spares(spa);
4041 		spa_config_exit(spa, SCL_ALL, FTAG);
4042 		spa->spa_spares.sav_sync = B_TRUE;
4043 	}
4044 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
4045 	    &l2cache, &nl2cache) == 0) {
4046 		if (spa->spa_l2cache.sav_config)
4047 			VERIFY(nvlist_remove(spa->spa_l2cache.sav_config,
4048 			    ZPOOL_CONFIG_L2CACHE, DATA_TYPE_NVLIST_ARRAY) == 0);
4049 		else
4050 			VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config,
4051 			    NV_UNIQUE_NAME, KM_SLEEP) == 0);
4052 		VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config,
4053 		    ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0);
4054 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4055 		spa_load_l2cache(spa);
4056 		spa_config_exit(spa, SCL_ALL, FTAG);
4057 		spa->spa_l2cache.sav_sync = B_TRUE;
4058 	}
4059 
4060 	/*
4061 	 * Check for any removed devices.
4062 	 */
4063 	if (spa->spa_autoreplace) {
4064 		spa_aux_check_removed(&spa->spa_spares);
4065 		spa_aux_check_removed(&spa->spa_l2cache);
4066 	}
4067 
4068 	if (spa_writeable(spa)) {
4069 		/*
4070 		 * Update the config cache to include the newly-imported pool.
4071 		 */
4072 		spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
4073 	}
4074 
4075 	/*
4076 	 * It's possible that the pool was expanded while it was exported.
4077 	 * We kick off an async task to handle this for us.
4078 	 */
4079 	spa_async_request(spa, SPA_ASYNC_AUTOEXPAND);
4080 
4081 	mutex_exit(&spa_namespace_lock);
4082 	spa_history_log_version(spa, "import");
4083 
4084 	return (0);
4085 }
4086 
4087 nvlist_t *
4088 spa_tryimport(nvlist_t *tryconfig)
4089 {
4090 	nvlist_t *config = NULL;
4091 	char *poolname;
4092 	spa_t *spa;
4093 	uint64_t state;
4094 	int error;
4095 
4096 	if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname))
4097 		return (NULL);
4098 
4099 	if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state))
4100 		return (NULL);
4101 
4102 	/*
4103 	 * Create and initialize the spa structure.
4104 	 */
4105 	mutex_enter(&spa_namespace_lock);
4106 	spa = spa_add(TRYIMPORT_NAME, tryconfig, NULL);
4107 	spa_activate(spa, FREAD);
4108 
4109 	/*
4110 	 * Pass off the heavy lifting to spa_load().
4111 	 * Pass TRUE for mosconfig because the user-supplied config
4112 	 * is actually the one to trust when doing an import.
4113 	 */
4114 	error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING, B_TRUE);
4115 
4116 	/*
4117 	 * If 'tryconfig' was at least parsable, return the current config.
4118 	 */
4119 	if (spa->spa_root_vdev != NULL) {
4120 		config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
4121 		VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME,
4122 		    poolname) == 0);
4123 		VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE,
4124 		    state) == 0);
4125 		VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP,
4126 		    spa->spa_uberblock.ub_timestamp) == 0);
4127 		VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO,
4128 		    spa->spa_load_info) == 0);
4129 
4130 		/*
4131 		 * If the bootfs property exists on this pool then we
4132 		 * copy it out so that external consumers can tell which
4133 		 * pools are bootable.
4134 		 */
4135 		if ((!error || error == EEXIST) && spa->spa_bootfs) {
4136 			char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP);
4137 
4138 			/*
4139 			 * We have to play games with the name since the
4140 			 * pool was opened as TRYIMPORT_NAME.
4141 			 */
4142 			if (dsl_dsobj_to_dsname(spa_name(spa),
4143 			    spa->spa_bootfs, tmpname) == 0) {
4144 				char *cp;
4145 				char *dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP);
4146 
4147 				cp = strchr(tmpname, '/');
4148 				if (cp == NULL) {
4149 					(void) strlcpy(dsname, tmpname,
4150 					    MAXPATHLEN);
4151 				} else {
4152 					(void) snprintf(dsname, MAXPATHLEN,
4153 					    "%s/%s", poolname, ++cp);
4154 				}
4155 				VERIFY(nvlist_add_string(config,
4156 				    ZPOOL_CONFIG_BOOTFS, dsname) == 0);
4157 				kmem_free(dsname, MAXPATHLEN);
4158 			}
4159 			kmem_free(tmpname, MAXPATHLEN);
4160 		}
4161 
4162 		/*
4163 		 * Add the list of hot spares and level 2 cache devices.
4164 		 */
4165 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
4166 		spa_add_spares(spa, config);
4167 		spa_add_l2cache(spa, config);
4168 		spa_config_exit(spa, SCL_CONFIG, FTAG);
4169 	}
4170 
4171 	spa_unload(spa);
4172 	spa_deactivate(spa);
4173 	spa_remove(spa);
4174 	mutex_exit(&spa_namespace_lock);
4175 
4176 	return (config);
4177 }
4178 
4179 /*
4180  * Pool export/destroy
4181  *
4182  * The act of destroying or exporting a pool is very simple.  We make sure there
4183  * is no more pending I/O and any references to the pool are gone.  Then, we
4184  * update the pool state and sync all the labels to disk, removing the
4185  * configuration from the cache afterwards. If the 'hardforce' flag is set, then
4186  * we don't sync the labels or remove the configuration cache.
4187  */
4188 static int
4189 spa_export_common(char *pool, int new_state, nvlist_t **oldconfig,
4190     boolean_t force, boolean_t hardforce)
4191 {
4192 	spa_t *spa;
4193 
4194 	if (oldconfig)
4195 		*oldconfig = NULL;
4196 
4197 	if (!(spa_mode_global & FWRITE))
4198 		return (SET_ERROR(EROFS));
4199 
4200 	mutex_enter(&spa_namespace_lock);
4201 	if ((spa = spa_lookup(pool)) == NULL) {
4202 		mutex_exit(&spa_namespace_lock);
4203 		return (SET_ERROR(ENOENT));
4204 	}
4205 
4206 	/*
4207 	 * Put a hold on the pool, drop the namespace lock, stop async tasks,
4208 	 * reacquire the namespace lock, and see if we can export.
4209 	 */
4210 	spa_open_ref(spa, FTAG);
4211 	mutex_exit(&spa_namespace_lock);
4212 	spa_async_suspend(spa);
4213 	mutex_enter(&spa_namespace_lock);
4214 	spa_close(spa, FTAG);
4215 
4216 	/*
4217 	 * The pool will be in core if it's openable,
4218 	 * in which case we can modify its state.
4219 	 */
4220 	if (spa->spa_state != POOL_STATE_UNINITIALIZED && spa->spa_sync_on) {
4221 		/*
4222 		 * Objsets may be open only because they're dirty, so we
4223 		 * have to force it to sync before checking spa_refcnt.
4224 		 */
4225 		txg_wait_synced(spa->spa_dsl_pool, 0);
4226 		spa_evicting_os_wait(spa);
4227 
4228 		/*
4229 		 * A pool cannot be exported or destroyed if there are active
4230 		 * references.  If we are resetting a pool, allow references by
4231 		 * fault injection handlers.
4232 		 */
4233 		if (!spa_refcount_zero(spa) ||
4234 		    (spa->spa_inject_ref != 0 &&
4235 		    new_state != POOL_STATE_UNINITIALIZED)) {
4236 			spa_async_resume(spa);
4237 			mutex_exit(&spa_namespace_lock);
4238 			return (SET_ERROR(EBUSY));
4239 		}
4240 
4241 		/*
4242 		 * A pool cannot be exported if it has an active shared spare.
4243 		 * This is to prevent other pools stealing the active spare
4244 		 * from an exported pool. At user's own will, such pool can
4245 		 * be forcedly exported.
4246 		 */
4247 		if (!force && new_state == POOL_STATE_EXPORTED &&
4248 		    spa_has_active_shared_spare(spa)) {
4249 			spa_async_resume(spa);
4250 			mutex_exit(&spa_namespace_lock);
4251 			return (SET_ERROR(EXDEV));
4252 		}
4253 
4254 		/*
4255 		 * We want this to be reflected on every label,
4256 		 * so mark them all dirty.  spa_unload() will do the
4257 		 * final sync that pushes these changes out.
4258 		 */
4259 		if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) {
4260 			spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4261 			spa->spa_state = new_state;
4262 			spa->spa_final_txg = spa_last_synced_txg(spa) +
4263 			    TXG_DEFER_SIZE + 1;
4264 			vdev_config_dirty(spa->spa_root_vdev);
4265 			spa_config_exit(spa, SCL_ALL, FTAG);
4266 		}
4267 	}
4268 
4269 	spa_event_notify(spa, NULL, ESC_ZFS_POOL_DESTROY);
4270 
4271 	if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
4272 		spa_unload(spa);
4273 		spa_deactivate(spa);
4274 	}
4275 
4276 	if (oldconfig && spa->spa_config)
4277 		VERIFY(nvlist_dup(spa->spa_config, oldconfig, 0) == 0);
4278 
4279 	if (new_state != POOL_STATE_UNINITIALIZED) {
4280 		if (!hardforce)
4281 			spa_config_sync(spa, B_TRUE, B_TRUE);
4282 		spa_remove(spa);
4283 	}
4284 	mutex_exit(&spa_namespace_lock);
4285 
4286 	return (0);
4287 }
4288 
4289 /*
4290  * Destroy a storage pool.
4291  */
4292 int
4293 spa_destroy(char *pool)
4294 {
4295 	return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL,
4296 	    B_FALSE, B_FALSE));
4297 }
4298 
4299 /*
4300  * Export a storage pool.
4301  */
4302 int
4303 spa_export(char *pool, nvlist_t **oldconfig, boolean_t force,
4304     boolean_t hardforce)
4305 {
4306 	return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig,
4307 	    force, hardforce));
4308 }
4309 
4310 /*
4311  * Similar to spa_export(), this unloads the spa_t without actually removing it
4312  * from the namespace in any way.
4313  */
4314 int
4315 spa_reset(char *pool)
4316 {
4317 	return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL,
4318 	    B_FALSE, B_FALSE));
4319 }
4320 
4321 /*
4322  * ==========================================================================
4323  * Device manipulation
4324  * ==========================================================================
4325  */
4326 
4327 /*
4328  * Add a device to a storage pool.
4329  */
4330 int
4331 spa_vdev_add(spa_t *spa, nvlist_t *nvroot)
4332 {
4333 	uint64_t txg, id;
4334 	int error;
4335 	vdev_t *rvd = spa->spa_root_vdev;
4336 	vdev_t *vd, *tvd;
4337 	nvlist_t **spares, **l2cache;
4338 	uint_t nspares, nl2cache;
4339 
4340 	ASSERT(spa_writeable(spa));
4341 
4342 	txg = spa_vdev_enter(spa);
4343 
4344 	if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0,
4345 	    VDEV_ALLOC_ADD)) != 0)
4346 		return (spa_vdev_exit(spa, NULL, txg, error));
4347 
4348 	spa->spa_pending_vdev = vd;	/* spa_vdev_exit() will clear this */
4349 
4350 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares,
4351 	    &nspares) != 0)
4352 		nspares = 0;
4353 
4354 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache,
4355 	    &nl2cache) != 0)
4356 		nl2cache = 0;
4357 
4358 	if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0)
4359 		return (spa_vdev_exit(spa, vd, txg, EINVAL));
4360 
4361 	if (vd->vdev_children != 0 &&
4362 	    (error = vdev_create(vd, txg, B_FALSE)) != 0)
4363 		return (spa_vdev_exit(spa, vd, txg, error));
4364 
4365 	/*
4366 	 * We must validate the spares and l2cache devices after checking the
4367 	 * children.  Otherwise, vdev_inuse() will blindly overwrite the spare.
4368 	 */
4369 	if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0)
4370 		return (spa_vdev_exit(spa, vd, txg, error));
4371 
4372 	/*
4373 	 * Transfer each new top-level vdev from vd to rvd.
4374 	 */
4375 	for (int c = 0; c < vd->vdev_children; c++) {
4376 
4377 		/*
4378 		 * Set the vdev id to the first hole, if one exists.
4379 		 */
4380 		for (id = 0; id < rvd->vdev_children; id++) {
4381 			if (rvd->vdev_child[id]->vdev_ishole) {
4382 				vdev_free(rvd->vdev_child[id]);
4383 				break;
4384 			}
4385 		}
4386 		tvd = vd->vdev_child[c];
4387 		vdev_remove_child(vd, tvd);
4388 		tvd->vdev_id = id;
4389 		vdev_add_child(rvd, tvd);
4390 		vdev_config_dirty(tvd);
4391 	}
4392 
4393 	if (nspares != 0) {
4394 		spa_set_aux_vdevs(&spa->spa_spares, spares, nspares,
4395 		    ZPOOL_CONFIG_SPARES);
4396 		spa_load_spares(spa);
4397 		spa->spa_spares.sav_sync = B_TRUE;
4398 	}
4399 
4400 	if (nl2cache != 0) {
4401 		spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache,
4402 		    ZPOOL_CONFIG_L2CACHE);
4403 		spa_load_l2cache(spa);
4404 		spa->spa_l2cache.sav_sync = B_TRUE;
4405 	}
4406 
4407 	/*
4408 	 * We have to be careful when adding new vdevs to an existing pool.
4409 	 * If other threads start allocating from these vdevs before we
4410 	 * sync the config cache, and we lose power, then upon reboot we may
4411 	 * fail to open the pool because there are DVAs that the config cache
4412 	 * can't translate.  Therefore, we first add the vdevs without
4413 	 * initializing metaslabs; sync the config cache (via spa_vdev_exit());
4414 	 * and then let spa_config_update() initialize the new metaslabs.
4415 	 *
4416 	 * spa_load() checks for added-but-not-initialized vdevs, so that
4417 	 * if we lose power at any point in this sequence, the remaining
4418 	 * steps will be completed the next time we load the pool.
4419 	 */
4420 	(void) spa_vdev_exit(spa, vd, txg, 0);
4421 
4422 	mutex_enter(&spa_namespace_lock);
4423 	spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
4424 	mutex_exit(&spa_namespace_lock);
4425 
4426 	return (0);
4427 }
4428 
4429 /*
4430  * Attach a device to a mirror.  The arguments are the path to any device
4431  * in the mirror, and the nvroot for the new device.  If the path specifies
4432  * a device that is not mirrored, we automatically insert the mirror vdev.
4433  *
4434  * If 'replacing' is specified, the new device is intended to replace the
4435  * existing device; in this case the two devices are made into their own
4436  * mirror using the 'replacing' vdev, which is functionally identical to
4437  * the mirror vdev (it actually reuses all the same ops) but has a few
4438  * extra rules: you can't attach to it after it's been created, and upon
4439  * completion of resilvering, the first disk (the one being replaced)
4440  * is automatically detached.
4441  */
4442 int
4443 spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing)
4444 {
4445 	uint64_t txg, dtl_max_txg;
4446 	vdev_t *rvd = spa->spa_root_vdev;
4447 	vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd;
4448 	vdev_ops_t *pvops;
4449 	char *oldvdpath, *newvdpath;
4450 	int newvd_isspare;
4451 	int error;
4452 
4453 	ASSERT(spa_writeable(spa));
4454 
4455 	txg = spa_vdev_enter(spa);
4456 
4457 	oldvd = spa_lookup_by_guid(spa, guid, B_FALSE);
4458 
4459 	if (oldvd == NULL)
4460 		return (spa_vdev_exit(spa, NULL, txg, ENODEV));
4461 
4462 	if (!oldvd->vdev_ops->vdev_op_leaf)
4463 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
4464 
4465 	pvd = oldvd->vdev_parent;
4466 
4467 	if ((error = spa_config_parse(spa, &newrootvd, nvroot, NULL, 0,
4468 	    VDEV_ALLOC_ATTACH)) != 0)
4469 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
4470 
4471 	if (newrootvd->vdev_children != 1)
4472 		return (spa_vdev_exit(spa, newrootvd, txg, EINVAL));
4473 
4474 	newvd = newrootvd->vdev_child[0];
4475 
4476 	if (!newvd->vdev_ops->vdev_op_leaf)
4477 		return (spa_vdev_exit(spa, newrootvd, txg, EINVAL));
4478 
4479 	if ((error = vdev_create(newrootvd, txg, replacing)) != 0)
4480 		return (spa_vdev_exit(spa, newrootvd, txg, error));
4481 
4482 	/*
4483 	 * Spares can't replace logs
4484 	 */
4485 	if (oldvd->vdev_top->vdev_islog && newvd->vdev_isspare)
4486 		return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
4487 
4488 	if (!replacing) {
4489 		/*
4490 		 * For attach, the only allowable parent is a mirror or the root
4491 		 * vdev.
4492 		 */
4493 		if (pvd->vdev_ops != &vdev_mirror_ops &&
4494 		    pvd->vdev_ops != &vdev_root_ops)
4495 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
4496 
4497 		pvops = &vdev_mirror_ops;
4498 	} else {
4499 		/*
4500 		 * Active hot spares can only be replaced by inactive hot
4501 		 * spares.
4502 		 */
4503 		if (pvd->vdev_ops == &vdev_spare_ops &&
4504 		    oldvd->vdev_isspare &&
4505 		    !spa_has_spare(spa, newvd->vdev_guid))
4506 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
4507 
4508 		/*
4509 		 * If the source is a hot spare, and the parent isn't already a
4510 		 * spare, then we want to create a new hot spare.  Otherwise, we
4511 		 * want to create a replacing vdev.  The user is not allowed to
4512 		 * attach to a spared vdev child unless the 'isspare' state is
4513 		 * the same (spare replaces spare, non-spare replaces
4514 		 * non-spare).
4515 		 */
4516 		if (pvd->vdev_ops == &vdev_replacing_ops &&
4517 		    spa_version(spa) < SPA_VERSION_MULTI_REPLACE) {
4518 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
4519 		} else if (pvd->vdev_ops == &vdev_spare_ops &&
4520 		    newvd->vdev_isspare != oldvd->vdev_isspare) {
4521 			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
4522 		}
4523 
4524 		if (newvd->vdev_isspare)
4525 			pvops = &vdev_spare_ops;
4526 		else
4527 			pvops = &vdev_replacing_ops;
4528 	}
4529 
4530 	/*
4531 	 * Make sure the new device is big enough.
4532 	 */
4533 	if (newvd->vdev_asize < vdev_get_min_asize(oldvd))
4534 		return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW));
4535 
4536 	/*
4537 	 * The new device cannot have a higher alignment requirement
4538 	 * than the top-level vdev.
4539 	 */
4540 	if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift)
4541 		return (spa_vdev_exit(spa, newrootvd, txg, EDOM));
4542 
4543 	/*
4544 	 * If this is an in-place replacement, update oldvd's path and devid
4545 	 * to make it distinguishable from newvd, and unopenable from now on.
4546 	 */
4547 	if (strcmp(oldvd->vdev_path, newvd->vdev_path) == 0) {
4548 		spa_strfree(oldvd->vdev_path);
4549 		oldvd->vdev_path = kmem_alloc(strlen(newvd->vdev_path) + 5,
4550 		    KM_SLEEP);
4551 		(void) sprintf(oldvd->vdev_path, "%s/%s",
4552 		    newvd->vdev_path, "old");
4553 		if (oldvd->vdev_devid != NULL) {
4554 			spa_strfree(oldvd->vdev_devid);
4555 			oldvd->vdev_devid = NULL;
4556 		}
4557 	}
4558 
4559 	/* mark the device being resilvered */
4560 	newvd->vdev_resilver_txg = txg;
4561 
4562 	/*
4563 	 * If the parent is not a mirror, or if we're replacing, insert the new
4564 	 * mirror/replacing/spare vdev above oldvd.
4565 	 */
4566 	if (pvd->vdev_ops != pvops)
4567 		pvd = vdev_add_parent(oldvd, pvops);
4568 
4569 	ASSERT(pvd->vdev_top->vdev_parent == rvd);
4570 	ASSERT(pvd->vdev_ops == pvops);
4571 	ASSERT(oldvd->vdev_parent == pvd);
4572 
4573 	/*
4574 	 * Extract the new device from its root and add it to pvd.
4575 	 */
4576 	vdev_remove_child(newrootvd, newvd);
4577 	newvd->vdev_id = pvd->vdev_children;
4578 	newvd->vdev_crtxg = oldvd->vdev_crtxg;
4579 	vdev_add_child(pvd, newvd);
4580 
4581 	tvd = newvd->vdev_top;
4582 	ASSERT(pvd->vdev_top == tvd);
4583 	ASSERT(tvd->vdev_parent == rvd);
4584 
4585 	vdev_config_dirty(tvd);
4586 
4587 	/*
4588 	 * Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account
4589 	 * for any dmu_sync-ed blocks.  It will propagate upward when
4590 	 * spa_vdev_exit() calls vdev_dtl_reassess().
4591 	 */
4592 	dtl_max_txg = txg + TXG_CONCURRENT_STATES;
4593 
4594 	vdev_dtl_dirty(newvd, DTL_MISSING, TXG_INITIAL,
4595 	    dtl_max_txg - TXG_INITIAL);
4596 
4597 	if (newvd->vdev_isspare) {
4598 		spa_spare_activate(newvd);
4599 		spa_event_notify(spa, newvd, ESC_ZFS_VDEV_SPARE);
4600 	}
4601 
4602 	oldvdpath = spa_strdup(oldvd->vdev_path);
4603 	newvdpath = spa_strdup(newvd->vdev_path);
4604 	newvd_isspare = newvd->vdev_isspare;
4605 
4606 	/*
4607 	 * Mark newvd's DTL dirty in this txg.
4608 	 */
4609 	vdev_dirty(tvd, VDD_DTL, newvd, txg);
4610 
4611 	/*
4612 	 * Schedule the resilver to restart in the future. We do this to
4613 	 * ensure that dmu_sync-ed blocks have been stitched into the
4614 	 * respective datasets.
4615 	 */
4616 	dsl_resilver_restart(spa->spa_dsl_pool, dtl_max_txg);
4617 
4618 	/*
4619 	 * Commit the config
4620 	 */
4621 	(void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0);
4622 
4623 	spa_history_log_internal(spa, "vdev attach", NULL,
4624 	    "%s vdev=%s %s vdev=%s",
4625 	    replacing && newvd_isspare ? "spare in" :
4626 	    replacing ? "replace" : "attach", newvdpath,
4627 	    replacing ? "for" : "to", oldvdpath);
4628 
4629 	spa_strfree(oldvdpath);
4630 	spa_strfree(newvdpath);
4631 
4632 	if (spa->spa_bootfs)
4633 		spa_event_notify(spa, newvd, ESC_ZFS_BOOTFS_VDEV_ATTACH);
4634 
4635 	return (0);
4636 }
4637 
4638 /*
4639  * Detach a device from a mirror or replacing vdev.
4640  *
4641  * If 'replace_done' is specified, only detach if the parent
4642  * is a replacing vdev.
4643  */
4644 int
4645 spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done)
4646 {
4647 	uint64_t txg;
4648 	int error;
4649 	vdev_t *rvd = spa->spa_root_vdev;
4650 	vdev_t *vd, *pvd, *cvd, *tvd;
4651 	boolean_t unspare = B_FALSE;
4652 	uint64_t unspare_guid = 0;
4653 	char *vdpath;
4654 
4655 	ASSERT(spa_writeable(spa));
4656 
4657 	txg = spa_vdev_enter(spa);
4658 
4659 	vd = spa_lookup_by_guid(spa, guid, B_FALSE);
4660 
4661 	if (vd == NULL)
4662 		return (spa_vdev_exit(spa, NULL, txg, ENODEV));
4663 
4664 	if (!vd->vdev_ops->vdev_op_leaf)
4665 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
4666 
4667 	pvd = vd->vdev_parent;
4668 
4669 	/*
4670 	 * If the parent/child relationship is not as expected, don't do it.
4671 	 * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing
4672 	 * vdev that's replacing B with C.  The user's intent in replacing
4673 	 * is to go from M(A,B) to M(A,C).  If the user decides to cancel
4674 	 * the replace by detaching C, the expected behavior is to end up
4675 	 * M(A,B).  But suppose that right after deciding to detach C,
4676 	 * the replacement of B completes.  We would have M(A,C), and then
4677 	 * ask to detach C, which would leave us with just A -- not what
4678 	 * the user wanted.  To prevent this, we make sure that the
4679 	 * parent/child relationship hasn't changed -- in this example,
4680 	 * that C's parent is still the replacing vdev R.
4681 	 */
4682 	if (pvd->vdev_guid != pguid && pguid != 0)
4683 		return (spa_vdev_exit(spa, NULL, txg, EBUSY));
4684 
4685 	/*
4686 	 * Only 'replacing' or 'spare' vdevs can be replaced.
4687 	 */
4688 	if (replace_done && pvd->vdev_ops != &vdev_replacing_ops &&
4689 	    pvd->vdev_ops != &vdev_spare_ops)
4690 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
4691 
4692 	ASSERT(pvd->vdev_ops != &vdev_spare_ops ||
4693 	    spa_version(spa) >= SPA_VERSION_SPARES);
4694 
4695 	/*
4696 	 * Only mirror, replacing, and spare vdevs support detach.
4697 	 */
4698 	if (pvd->vdev_ops != &vdev_replacing_ops &&
4699 	    pvd->vdev_ops != &vdev_mirror_ops &&
4700 	    pvd->vdev_ops != &vdev_spare_ops)
4701 		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
4702 
4703 	/*
4704 	 * If this device has the only valid copy of some data,
4705 	 * we cannot safely detach it.
4706 	 */
4707 	if (vdev_dtl_required(vd))
4708 		return (spa_vdev_exit(spa, NULL, txg, EBUSY));
4709 
4710 	ASSERT(pvd->vdev_children >= 2);
4711 
4712 	/*
4713 	 * If we are detaching the second disk from a replacing vdev, then
4714 	 * check to see if we changed the original vdev's path to have "/old"
4715 	 * at the end in spa_vdev_attach().  If so, undo that change now.
4716 	 */
4717 	if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 &&
4718 	    vd->vdev_path != NULL) {
4719 		size_t len = strlen(vd->vdev_path);
4720 
4721 		for (int c = 0; c < pvd->vdev_children; c++) {
4722 			cvd = pvd->vdev_child[c];
4723 
4724 			if (cvd == vd || cvd->vdev_path == NULL)
4725 				continue;
4726 
4727 			if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 &&
4728 			    strcmp(cvd->vdev_path + len, "/old") == 0) {
4729 				spa_strfree(cvd->vdev_path);
4730 				cvd->vdev_path = spa_strdup(vd->vdev_path);
4731 				break;
4732 			}
4733 		}
4734 	}
4735 
4736 	/*
4737 	 * If we are detaching the original disk from a spare, then it implies
4738 	 * that the spare should become a real disk, and be removed from the
4739 	 * active spare list for the pool.
4740 	 */
4741 	if (pvd->vdev_ops == &vdev_spare_ops &&
4742 	    vd->vdev_id == 0 &&
4743 	    pvd->vdev_child[pvd->vdev_children - 1]->vdev_isspare)
4744 		unspare = B_TRUE;
4745 
4746 	/*
4747 	 * Erase the disk labels so the disk can be used for other things.
4748 	 * This must be done after all other error cases are handled,
4749 	 * but before we disembowel vd (so we can still do I/O to it).
4750 	 * But if we can't do it, don't treat the error as fatal --
4751 	 * it may be that the unwritability of the disk is the reason
4752 	 * it's being detached!
4753 	 */
4754 	error = vdev_label_init(vd, 0, VDEV_LABEL_REMOVE);
4755 
4756 	/*
4757 	 * Remove vd from its parent and compact the parent's children.
4758 	 */
4759 	vdev_remove_child(pvd, vd);
4760 	vdev_compact_children(pvd);
4761 
4762 	/*
4763 	 * Remember one of the remaining children so we can get tvd below.
4764 	 */
4765 	cvd = pvd->vdev_child[pvd->vdev_children - 1];
4766 
4767 	/*
4768 	 * If we need to remove the remaining child from the list of hot spares,
4769 	 * do it now, marking the vdev as no longer a spare in the process.
4770 	 * We must do this before vdev_remove_parent(), because that can
4771 	 * change the GUID if it creates a new toplevel GUID.  For a similar
4772 	 * reason, we must remove the spare now, in the same txg as the detach;
4773 	 * otherwise someone could attach a new sibling, change the GUID, and
4774 	 * the subsequent attempt to spa_vdev_remove(unspare_guid) would fail.
4775 	 */
4776 	if (unspare) {
4777 		ASSERT(cvd->vdev_isspare);
4778 		spa_spare_remove(cvd);
4779 		unspare_guid = cvd->vdev_guid;
4780 		(void) spa_vdev_remove(spa, unspare_guid, B_TRUE);
4781 		cvd->vdev_unspare = B_TRUE;
4782 	}
4783 
4784 	/*
4785 	 * If the parent mirror/replacing vdev only has one child,
4786 	 * the parent is no longer needed.  Remove it from the tree.
4787 	 */
4788 	if (pvd->vdev_children == 1) {
4789 		if (pvd->vdev_ops == &vdev_spare_ops)
4790 			cvd->vdev_unspare = B_FALSE;
4791 		vdev_remove_parent(cvd);
4792 	}
4793 
4794 
4795 	/*
4796 	 * We don't set tvd until now because the parent we just removed
4797 	 * may have been the previous top-level vdev.
4798 	 */
4799 	tvd = cvd->vdev_top;
4800 	ASSERT(tvd->vdev_parent == rvd);
4801 
4802 	/*
4803 	 * Reevaluate the parent vdev state.
4804 	 */
4805 	vdev_propagate_state(cvd);
4806 
4807 	/*
4808 	 * If the 'autoexpand' property is set on the pool then automatically
4809 	 * try to expand the size of the pool. For example if the device we
4810 	 * just detached was smaller than the others, it may be possible to
4811 	 * add metaslabs (i.e. grow the pool). We need to reopen the vdev
4812 	 * first so that we can obtain the updated sizes of the leaf vdevs.
4813 	 */
4814 	if (spa->spa_autoexpand) {
4815 		vdev_reopen(tvd);
4816 		vdev_expand(tvd, txg);
4817 	}
4818 
4819 	vdev_config_dirty(tvd);
4820 
4821 	/*
4822 	 * Mark vd's DTL as dirty in this txg.  vdev_dtl_sync() will see that
4823 	 * vd->vdev_detached is set and free vd's DTL object in syncing context.
4824 	 * But first make sure we're not on any *other* txg's DTL list, to
4825 	 * prevent vd from being accessed after it's freed.
4826 	 */
4827 	vdpath = spa_strdup(vd->vdev_path);
4828 	for (int t = 0; t < TXG_SIZE; t++)
4829 		(void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t);
4830 	vd->vdev_detached = B_TRUE;
4831 	vdev_dirty(tvd, VDD_DTL, vd, txg);
4832 
4833 	spa_event_notify(spa, vd, ESC_ZFS_VDEV_REMOVE);
4834 
4835 	/* hang on to the spa before we release the lock */
4836 	spa_open_ref(spa, FTAG);
4837 
4838 	error = spa_vdev_exit(spa, vd, txg, 0);
4839 
4840 	spa_history_log_internal(spa, "detach", NULL,
4841 	    "vdev=%s", vdpath);
4842 	spa_strfree(vdpath);
4843 
4844 	/*
4845 	 * If this was the removal of the original device in a hot spare vdev,
4846 	 * then we want to go through and remove the device from the hot spare
4847 	 * list of every other pool.
4848 	 */
4849 	if (unspare) {
4850 		spa_t *altspa = NULL;
4851 
4852 		mutex_enter(&spa_namespace_lock);
4853 		while ((altspa = spa_next(altspa)) != NULL) {
4854 			if (altspa->spa_state != POOL_STATE_ACTIVE ||
4855 			    altspa == spa)
4856 				continue;
4857 
4858 			spa_open_ref(altspa, FTAG);
4859 			mutex_exit(&spa_namespace_lock);
4860 			(void) spa_vdev_remove(altspa, unspare_guid, B_TRUE);
4861 			mutex_enter(&spa_namespace_lock);
4862 			spa_close(altspa, FTAG);
4863 		}
4864 		mutex_exit(&spa_namespace_lock);
4865 
4866 		/* search the rest of the vdevs for spares to remove */
4867 		spa_vdev_resilver_done(spa);
4868 	}
4869 
4870 	/* all done with the spa; OK to release */
4871 	mutex_enter(&spa_namespace_lock);
4872 	spa_close(spa, FTAG);
4873 	mutex_exit(&spa_namespace_lock);
4874 
4875 	return (error);
4876 }
4877 
4878 /*
4879  * Split a set of devices from their mirrors, and create a new pool from them.
4880  */
4881 int
4882 spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config,
4883     nvlist_t *props, boolean_t exp)
4884 {
4885 	int error = 0;
4886 	uint64_t txg, *glist;
4887 	spa_t *newspa;
4888 	uint_t c, children, lastlog;
4889 	nvlist_t **child, *nvl, *tmp;
4890 	dmu_tx_t *tx;
4891 	char *altroot = NULL;
4892 	vdev_t *rvd, **vml = NULL;			/* vdev modify list */
4893 	boolean_t activate_slog;
4894 
4895 	ASSERT(spa_writeable(spa));
4896 
4897 	txg = spa_vdev_enter(spa);
4898 
4899 	/* clear the log and flush everything up to now */
4900 	activate_slog = spa_passivate_log(spa);
4901 	(void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG);
4902 	error = spa_offline_log(spa);
4903 	txg = spa_vdev_config_enter(spa);
4904 
4905 	if (activate_slog)
4906 		spa_activate_log(spa);
4907 
4908 	if (error != 0)
4909 		return (spa_vdev_exit(spa, NULL, txg, error));
4910 
4911 	/* check new spa name before going any further */
4912 	if (spa_lookup(newname) != NULL)
4913 		return (spa_vdev_exit(spa, NULL, txg, EEXIST));
4914 
4915 	/*
4916 	 * scan through all the children to ensure they're all mirrors
4917 	 */
4918 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 ||
4919 	    nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child,
4920 	    &children) != 0)
4921 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
4922 
4923 	/* first, check to ensure we've got the right child count */
4924 	rvd = spa->spa_root_vdev;
4925 	lastlog = 0;
4926 	for (c = 0; c < rvd->vdev_children; c++) {
4927 		vdev_t *vd = rvd->vdev_child[c];
4928 
4929 		/* don't count the holes & logs as children */
4930 		if (vd->vdev_islog || vd->vdev_ishole) {
4931 			if (lastlog == 0)
4932 				lastlog = c;
4933 			continue;
4934 		}
4935 
4936 		lastlog = 0;
4937 	}
4938 	if (children != (lastlog != 0 ? lastlog : rvd->vdev_children))
4939 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
4940 
4941 	/* next, ensure no spare or cache devices are part of the split */
4942 	if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 ||
4943 	    nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0)
4944 		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
4945 
4946 	vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP);
4947 	glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP);
4948 
4949 	/* then, loop over each vdev and validate it */
4950 	for (c = 0; c < children; c++) {
4951 		uint64_t is_hole = 0;
4952 
4953 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
4954 		    &is_hole);
4955 
4956 		if (is_hole != 0) {
4957 			if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole ||
4958 			    spa->spa_root_vdev->vdev_child[c]->vdev_islog) {
4959 				continue;
4960 			} else {
4961 				error = SET_ERROR(EINVAL);
4962 				break;
4963 			}
4964 		}
4965 
4966 		/* which disk is going to be split? */
4967 		if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID,
4968 		    &glist[c]) != 0) {
4969 			error = SET_ERROR(EINVAL);
4970 			break;
4971 		}
4972 
4973 		/* look it up in the spa */
4974 		vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE);
4975 		if (vml[c] == NULL) {
4976 			error = SET_ERROR(ENODEV);
4977 			break;
4978 		}
4979 
4980 		/* make sure there's nothing stopping the split */
4981 		if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops ||
4982 		    vml[c]->vdev_islog ||
4983 		    vml[c]->vdev_ishole ||
4984 		    vml[c]->vdev_isspare ||
4985 		    vml[c]->vdev_isl2cache ||
4986 		    !vdev_writeable(vml[c]) ||
4987 		    vml[c]->vdev_children != 0 ||
4988 		    vml[c]->vdev_state != VDEV_STATE_HEALTHY ||
4989 		    c != spa->spa_root_vdev->vdev_child[c]->vdev_id) {
4990 			error = SET_ERROR(EINVAL);
4991 			break;
4992 		}
4993 
4994 		if (vdev_dtl_required(vml[c])) {
4995 			error = SET_ERROR(EBUSY);
4996 			break;
4997 		}
4998 
4999 		/* we need certain info from the top level */
5000 		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY,
5001 		    vml[c]->vdev_top->vdev_ms_array) == 0);
5002 		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT,
5003 		    vml[c]->vdev_top->vdev_ms_shift) == 0);
5004 		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE,
5005 		    vml[c]->vdev_top->vdev_asize) == 0);
5006 		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT,
5007 		    vml[c]->vdev_top->vdev_ashift) == 0);
5008 	}
5009 
5010 	if (error != 0) {
5011 		kmem_free(vml, children * sizeof (vdev_t *));
5012 		kmem_free(glist, children * sizeof (uint64_t));
5013 		return (spa_vdev_exit(spa, NULL, txg, error));
5014 	}
5015 
5016 	/* stop writers from using the disks */
5017 	for (c = 0; c < children; c++) {
5018 		if (vml[c] != NULL)
5019 			vml[c]->vdev_offline = B_TRUE;
5020 	}
5021 	vdev_reopen(spa->spa_root_vdev);
5022 
5023 	/*
5024 	 * Temporarily record the splitting vdevs in the spa config.  This
5025 	 * will disappear once the config is regenerated.
5026 	 */
5027 	VERIFY(nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_SLEEP) == 0);
5028 	VERIFY(nvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST,
5029 	    glist, children) == 0);
5030 	kmem_free(glist, children * sizeof (uint64_t));
5031 
5032 	mutex_enter(&spa->spa_props_lock);
5033 	VERIFY(nvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT,
5034 	    nvl) == 0);
5035 	mutex_exit(&spa->spa_props_lock);
5036 	spa->spa_config_splitting = nvl;
5037 	vdev_config_dirty(spa->spa_root_vdev);
5038 
5039 	/* configure and create the new pool */
5040 	VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname) == 0);
5041 	VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE,
5042 	    exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE) == 0);
5043 	VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_VERSION,
5044 	    spa_version(spa)) == 0);
5045 	VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG,
5046 	    spa->spa_config_txg) == 0);
5047 	VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID,
5048 	    spa_generate_guid(NULL)) == 0);
5049 	(void) nvlist_lookup_string(props,
5050 	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
5051 
5052 	/* add the new pool to the namespace */
5053 	newspa = spa_add(newname, config, altroot);
5054 	newspa->spa_config_txg = spa->spa_config_txg;
5055 	spa_set_log_state(newspa, SPA_LOG_CLEAR);
5056 
5057 	/* release the spa config lock, retaining the namespace lock */
5058 	spa_vdev_config_exit(spa, NULL, txg, 0, FTAG);
5059 
5060 	if (zio_injection_enabled)
5061 		zio_handle_panic_injection(spa, FTAG, 1);
5062 
5063 	spa_activate(newspa, spa_mode_global);
5064 	spa_async_suspend(newspa);
5065 
5066 	/* create the new pool from the disks of the original pool */
5067 	error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE, B_TRUE);
5068 	if (error)
5069 		goto out;
5070 
5071 	/* if that worked, generate a real config for the new pool */
5072 	if (newspa->spa_root_vdev != NULL) {
5073 		VERIFY(nvlist_alloc(&newspa->spa_config_splitting,
5074 		    NV_UNIQUE_NAME, KM_SLEEP) == 0);
5075 		VERIFY(nvlist_add_uint64(newspa->spa_config_splitting,
5076 		    ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa)) == 0);
5077 		spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL,
5078 		    B_TRUE));
5079 	}
5080 
5081 	/* set the props */
5082 	if (props != NULL) {
5083 		spa_configfile_set(newspa, props, B_FALSE);
5084 		error = spa_prop_set(newspa, props);
5085 		if (error)
5086 			goto out;
5087 	}
5088 
5089 	/* flush everything */
5090 	txg = spa_vdev_config_enter(newspa);
5091 	vdev_config_dirty(newspa->spa_root_vdev);
5092 	(void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG);
5093 
5094 	if (zio_injection_enabled)
5095 		zio_handle_panic_injection(spa, FTAG, 2);
5096 
5097 	spa_async_resume(newspa);
5098 
5099 	/* finally, update the original pool's config */
5100 	txg = spa_vdev_config_enter(spa);
5101 	tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
5102 	error = dmu_tx_assign(tx, TXG_WAIT);
5103 	if (error != 0)
5104 		dmu_tx_abort(tx);
5105 	for (c = 0; c < children; c++) {
5106 		if (vml[c] != NULL) {
5107 			vdev_split(vml[c]);
5108 			if (error == 0)
5109 				spa_history_log_internal(spa, "detach", tx,
5110 				    "vdev=%s", vml[c]->vdev_path);
5111 			vdev_free(vml[c]);
5112 		}
5113 	}
5114 	vdev_config_dirty(spa->spa_root_vdev);
5115 	spa->spa_config_splitting = NULL;
5116 	nvlist_free(nvl);
5117 	if (error == 0)
5118 		dmu_tx_commit(tx);
5119 	(void) spa_vdev_exit(spa, NULL, txg, 0);
5120 
5121 	if (zio_injection_enabled)
5122 		zio_handle_panic_injection(spa, FTAG, 3);
5123 
5124 	/* split is complete; log a history record */
5125 	spa_history_log_internal(newspa, "split", NULL,
5126 	    "from pool %s", spa_name(spa));
5127 
5128 	kmem_free(vml, children * sizeof (vdev_t *));
5129 
5130 	/* if we're not going to mount the filesystems in userland, export */
5131 	if (exp)
5132 		error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL,
5133 		    B_FALSE, B_FALSE);
5134 
5135 	return (error);
5136 
5137 out:
5138 	spa_unload(newspa);
5139 	spa_deactivate(newspa);
5140 	spa_remove(newspa);
5141 
5142 	txg = spa_vdev_config_enter(spa);
5143 
5144 	/* re-online all offlined disks */
5145 	for (c = 0; c < children; c++) {
5146 		if (vml[c] != NULL)
5147 			vml[c]->vdev_offline = B_FALSE;
5148 	}
5149 	vdev_reopen(spa->spa_root_vdev);
5150 
5151 	nvlist_free(spa->spa_config_splitting);
5152 	spa->spa_config_splitting = NULL;
5153 	(void) spa_vdev_exit(spa, NULL, txg, error);
5154 
5155 	kmem_free(vml, children * sizeof (vdev_t *));
5156 	return (error);
5157 }
5158 
5159 static nvlist_t *
5160 spa_nvlist_lookup_by_guid(nvlist_t **nvpp, int count, uint64_t target_guid)
5161 {
5162 	for (int i = 0; i < count; i++) {
5163 		uint64_t guid;
5164 
5165 		VERIFY(nvlist_lookup_uint64(nvpp[i], ZPOOL_CONFIG_GUID,
5166 		    &guid) == 0);
5167 
5168 		if (guid == target_guid)
5169 			return (nvpp[i]);
5170 	}
5171 
5172 	return (NULL);
5173 }
5174 
5175 static void
5176 spa_vdev_remove_aux(nvlist_t *config, char *name, nvlist_t **dev, int count,
5177 	nvlist_t *dev_to_remove)
5178 {
5179 	nvlist_t **newdev = NULL;
5180 
5181 	if (count > 1)
5182 		newdev = kmem_alloc((count - 1) * sizeof (void *), KM_SLEEP);
5183 
5184 	for (int i = 0, j = 0; i < count; i++) {
5185 		if (dev[i] == dev_to_remove)
5186 			continue;
5187 		VERIFY(nvlist_dup(dev[i], &newdev[j++], KM_SLEEP) == 0);
5188 	}
5189 
5190 	VERIFY(nvlist_remove(config, name, DATA_TYPE_NVLIST_ARRAY) == 0);
5191 	VERIFY(nvlist_add_nvlist_array(config, name, newdev, count - 1) == 0);
5192 
5193 	for (int i = 0; i < count - 1; i++)
5194 		nvlist_free(newdev[i]);
5195 
5196 	if (count > 1)
5197 		kmem_free(newdev, (count - 1) * sizeof (void *));
5198 }
5199 
5200 /*
5201  * Evacuate the device.
5202  */
5203 static int
5204 spa_vdev_remove_evacuate(spa_t *spa, vdev_t *vd)
5205 {
5206 	uint64_t txg;
5207 	int error = 0;
5208 
5209 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
5210 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
5211 	ASSERT(vd == vd->vdev_top);
5212 
5213 	/*
5214 	 * Evacuate the device.  We don't hold the config lock as writer
5215 	 * since we need to do I/O but we do keep the
5216 	 * spa_namespace_lock held.  Once this completes the device
5217 	 * should no longer have any blocks allocated on it.
5218 	 */
5219 	if (vd->vdev_islog) {
5220 		if (vd->vdev_stat.vs_alloc != 0)
5221 			error = spa_offline_log(spa);
5222 	} else {
5223 		error = SET_ERROR(ENOTSUP);
5224 	}
5225 
5226 	if (error)
5227 		return (error);
5228 
5229 	/*
5230 	 * The evacuation succeeded.  Remove any remaining MOS metadata
5231 	 * associated with this vdev, and wait for these changes to sync.
5232 	 */
5233 	ASSERT0(vd->vdev_stat.vs_alloc);
5234 	txg = spa_vdev_config_enter(spa);
5235 	vd->vdev_removing = B_TRUE;
5236 	vdev_dirty_leaves(vd, VDD_DTL, txg);
5237 	vdev_config_dirty(vd);
5238 	spa_vdev_config_exit(spa, NULL, txg, 0, FTAG);
5239 
5240 	return (0);
5241 }
5242 
5243 /*
5244  * Complete the removal by cleaning up the namespace.
5245  */
5246 static void
5247 spa_vdev_remove_from_namespace(spa_t *spa, vdev_t *vd)
5248 {
5249 	vdev_t *rvd = spa->spa_root_vdev;
5250 	uint64_t id = vd->vdev_id;
5251 	boolean_t last_vdev = (id == (rvd->vdev_children - 1));
5252 
5253 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
5254 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
5255 	ASSERT(vd == vd->vdev_top);
5256 
5257 	/*
5258 	 * Only remove any devices which are empty.
5259 	 */
5260 	if (vd->vdev_stat.vs_alloc != 0)
5261 		return;
5262 
5263 	(void) vdev_label_init(vd, 0, VDEV_LABEL_REMOVE);
5264 
5265 	if (list_link_active(&vd->vdev_state_dirty_node))
5266 		vdev_state_clean(vd);
5267 	if (list_link_active(&vd->vdev_config_dirty_node))
5268 		vdev_config_clean(vd);
5269 
5270 	vdev_free(vd);
5271 
5272 	if (last_vdev) {
5273 		vdev_compact_children(rvd);
5274 	} else {
5275 		vd = vdev_alloc_common(spa, id, 0, &vdev_hole_ops);
5276 		vdev_add_child(rvd, vd);
5277 	}
5278 	vdev_config_dirty(rvd);
5279 
5280 	/*
5281 	 * Reassess the health of our root vdev.
5282 	 */
5283 	vdev_reopen(rvd);
5284 }
5285 
5286 /*
5287  * Remove a device from the pool -
5288  *
5289  * Removing a device from the vdev namespace requires several steps
5290  * and can take a significant amount of time.  As a result we use
5291  * the spa_vdev_config_[enter/exit] functions which allow us to
5292  * grab and release the spa_config_lock while still holding the namespace
5293  * lock.  During each step the configuration is synced out.
5294  *
5295  * Currently, this supports removing only hot spares, slogs, and level 2 ARC
5296  * devices.
5297  */
5298 int
5299 spa_vdev_remove(spa_t *spa, uint64_t guid, boolean_t unspare)
5300 {
5301 	vdev_t *vd;
5302 	metaslab_group_t *mg;
5303 	nvlist_t **spares, **l2cache, *nv;
5304 	uint64_t txg = 0;
5305 	uint_t nspares, nl2cache;
5306 	int error = 0;
5307 	boolean_t locked = MUTEX_HELD(&spa_namespace_lock);
5308 
5309 	ASSERT(spa_writeable(spa));
5310 
5311 	if (!locked)
5312 		txg = spa_vdev_enter(spa);
5313 
5314 	vd = spa_lookup_by_guid(spa, guid, B_FALSE);
5315 
5316 	if (spa->spa_spares.sav_vdevs != NULL &&
5317 	    nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
5318 	    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0 &&
5319 	    (nv = spa_nvlist_lookup_by_guid(spares, nspares, guid)) != NULL) {
5320 		/*
5321 		 * Only remove the hot spare if it's not currently in use
5322 		 * in this pool.
5323 		 */
5324 		if (vd == NULL || unspare) {
5325 			spa_vdev_remove_aux(spa->spa_spares.sav_config,
5326 			    ZPOOL_CONFIG_SPARES, spares, nspares, nv);
5327 			spa_load_spares(spa);
5328 			spa->spa_spares.sav_sync = B_TRUE;
5329 		} else {
5330 			error = SET_ERROR(EBUSY);
5331 		}
5332 	} else if (spa->spa_l2cache.sav_vdevs != NULL &&
5333 	    nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config,
5334 	    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0 &&
5335 	    (nv = spa_nvlist_lookup_by_guid(l2cache, nl2cache, guid)) != NULL) {
5336 		/*
5337 		 * Cache devices can always be removed.
5338 		 */
5339 		spa_vdev_remove_aux(spa->spa_l2cache.sav_config,
5340 		    ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache, nv);
5341 		spa_load_l2cache(spa);
5342 		spa->spa_l2cache.sav_sync = B_TRUE;
5343 	} else if (vd != NULL && vd->vdev_islog) {
5344 		ASSERT(!locked);
5345 		ASSERT(vd == vd->vdev_top);
5346 
5347 		mg = vd->vdev_mg;
5348 
5349 		/*
5350 		 * Stop allocating from this vdev.
5351 		 */
5352 		metaslab_group_passivate(mg);
5353 
5354 		/*
5355 		 * Wait for the youngest allocations and frees to sync,
5356 		 * and then wait for the deferral of those frees to finish.
5357 		 */
5358 		spa_vdev_config_exit(spa, NULL,
5359 		    txg + TXG_CONCURRENT_STATES + TXG_DEFER_SIZE, 0, FTAG);
5360 
5361 		/*
5362 		 * Attempt to evacuate the vdev.
5363 		 */
5364 		error = spa_vdev_remove_evacuate(spa, vd);
5365 
5366 		txg = spa_vdev_config_enter(spa);
5367 
5368 		/*
5369 		 * If we couldn't evacuate the vdev, unwind.
5370 		 */
5371 		if (error) {
5372 			metaslab_group_activate(mg);
5373 			return (spa_vdev_exit(spa, NULL, txg, error));
5374 		}
5375 
5376 		/*
5377 		 * Clean up the vdev namespace.
5378 		 */
5379 		spa_vdev_remove_from_namespace(spa, vd);
5380 
5381 	} else if (vd != NULL) {
5382 		/*
5383 		 * Normal vdevs cannot be removed (yet).
5384 		 */
5385 		error = SET_ERROR(ENOTSUP);
5386 	} else {
5387 		/*
5388 		 * There is no vdev of any kind with the specified guid.
5389 		 */
5390 		error = SET_ERROR(ENOENT);
5391 	}
5392 
5393 	if (!locked)
5394 		return (spa_vdev_exit(spa, NULL, txg, error));
5395 
5396 	return (error);
5397 }
5398 
5399 /*
5400  * Find any device that's done replacing, or a vdev marked 'unspare' that's
5401  * currently spared, so we can detach it.
5402  */
5403 static vdev_t *
5404 spa_vdev_resilver_done_hunt(vdev_t *vd)
5405 {
5406 	vdev_t *newvd, *oldvd;
5407 
5408 	for (int c = 0; c < vd->vdev_children; c++) {
5409 		oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]);
5410 		if (oldvd != NULL)
5411 			return (oldvd);
5412 	}
5413 
5414 	/*
5415 	 * Check for a completed replacement.  We always consider the first
5416 	 * vdev in the list to be the oldest vdev, and the last one to be
5417 	 * the newest (see spa_vdev_attach() for how that works).  In
5418 	 * the case where the newest vdev is faulted, we will not automatically
5419 	 * remove it after a resilver completes.  This is OK as it will require
5420 	 * user intervention to determine which disk the admin wishes to keep.
5421 	 */
5422 	if (vd->vdev_ops == &vdev_replacing_ops) {
5423 		ASSERT(vd->vdev_children > 1);
5424 
5425 		newvd = vd->vdev_child[vd->vdev_children - 1];
5426 		oldvd = vd->vdev_child[0];
5427 
5428 		if (vdev_dtl_empty(newvd, DTL_MISSING) &&
5429 		    vdev_dtl_empty(newvd, DTL_OUTAGE) &&
5430 		    !vdev_dtl_required(oldvd))
5431 			return (oldvd);
5432 	}
5433 
5434 	/*
5435 	 * Check for a completed resilver with the 'unspare' flag set.
5436 	 */
5437 	if (vd->vdev_ops == &vdev_spare_ops) {
5438 		vdev_t *first = vd->vdev_child[0];
5439 		vdev_t *last = vd->vdev_child[vd->vdev_children - 1];
5440 
5441 		if (last->vdev_unspare) {
5442 			oldvd = first;
5443 			newvd = last;
5444 		} else if (first->vdev_unspare) {
5445 			oldvd = last;
5446 			newvd = first;
5447 		} else {
5448 			oldvd = NULL;
5449 		}
5450 
5451 		if (oldvd != NULL &&
5452 		    vdev_dtl_empty(newvd, DTL_MISSING) &&
5453 		    vdev_dtl_empty(newvd, DTL_OUTAGE) &&
5454 		    !vdev_dtl_required(oldvd))
5455 			return (oldvd);
5456 
5457 		/*
5458 		 * If there are more than two spares attached to a disk,
5459 		 * and those spares are not required, then we want to
5460 		 * attempt to free them up now so that they can be used
5461 		 * by other pools.  Once we're back down to a single
5462 		 * disk+spare, we stop removing them.
5463 		 */
5464 		if (vd->vdev_children > 2) {
5465 			newvd = vd->vdev_child[1];
5466 
5467 			if (newvd->vdev_isspare && last->vdev_isspare &&
5468 			    vdev_dtl_empty(last, DTL_MISSING) &&
5469 			    vdev_dtl_empty(last, DTL_OUTAGE) &&
5470 			    !vdev_dtl_required(newvd))
5471 				return (newvd);
5472 		}
5473 	}
5474 
5475 	return (NULL);
5476 }
5477 
5478 static void
5479 spa_vdev_resilver_done(spa_t *spa)
5480 {
5481 	vdev_t *vd, *pvd, *ppvd;
5482 	uint64_t guid, sguid, pguid, ppguid;
5483 
5484 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
5485 
5486 	while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) {
5487 		pvd = vd->vdev_parent;
5488 		ppvd = pvd->vdev_parent;
5489 		guid = vd->vdev_guid;
5490 		pguid = pvd->vdev_guid;
5491 		ppguid = ppvd->vdev_guid;
5492 		sguid = 0;
5493 		/*
5494 		 * If we have just finished replacing a hot spared device, then
5495 		 * we need to detach the parent's first child (the original hot
5496 		 * spare) as well.
5497 		 */
5498 		if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 &&
5499 		    ppvd->vdev_children == 2) {
5500 			ASSERT(pvd->vdev_ops == &vdev_replacing_ops);
5501 			sguid = ppvd->vdev_child[1]->vdev_guid;
5502 		}
5503 		ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd));
5504 
5505 		spa_config_exit(spa, SCL_ALL, FTAG);
5506 		if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0)
5507 			return;
5508 		if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0)
5509 			return;
5510 		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
5511 	}
5512 
5513 	spa_config_exit(spa, SCL_ALL, FTAG);
5514 }
5515 
5516 /*
5517  * Update the stored path or FRU for this vdev.
5518  */
5519 int
5520 spa_vdev_set_common(spa_t *spa, uint64_t guid, const char *value,
5521     boolean_t ispath)
5522 {
5523 	vdev_t *vd;
5524 	boolean_t sync = B_FALSE;
5525 
5526 	ASSERT(spa_writeable(spa));
5527 
5528 	spa_vdev_state_enter(spa, SCL_ALL);
5529 
5530 	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
5531 		return (spa_vdev_state_exit(spa, NULL, ENOENT));
5532 
5533 	if (!vd->vdev_ops->vdev_op_leaf)
5534 		return (spa_vdev_state_exit(spa, NULL, ENOTSUP));
5535 
5536 	if (ispath) {
5537 		if (strcmp(value, vd->vdev_path) != 0) {
5538 			spa_strfree(vd->vdev_path);
5539 			vd->vdev_path = spa_strdup(value);
5540 			sync = B_TRUE;
5541 		}
5542 	} else {
5543 		if (vd->vdev_fru == NULL) {
5544 			vd->vdev_fru = spa_strdup(value);
5545 			sync = B_TRUE;
5546 		} else if (strcmp(value, vd->vdev_fru) != 0) {
5547 			spa_strfree(vd->vdev_fru);
5548 			vd->vdev_fru = spa_strdup(value);
5549 			sync = B_TRUE;
5550 		}
5551 	}
5552 
5553 	return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0));
5554 }
5555 
5556 int
5557 spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath)
5558 {
5559 	return (spa_vdev_set_common(spa, guid, newpath, B_TRUE));
5560 }
5561 
5562 int
5563 spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru)
5564 {
5565 	return (spa_vdev_set_common(spa, guid, newfru, B_FALSE));
5566 }
5567 
5568 /*
5569  * ==========================================================================
5570  * SPA Scanning
5571  * ==========================================================================
5572  */
5573 
5574 int
5575 spa_scan_stop(spa_t *spa)
5576 {
5577 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
5578 	if (dsl_scan_resilvering(spa->spa_dsl_pool))
5579 		return (SET_ERROR(EBUSY));
5580 	return (dsl_scan_cancel(spa->spa_dsl_pool));
5581 }
5582 
5583 int
5584 spa_scan(spa_t *spa, pool_scan_func_t func)
5585 {
5586 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
5587 
5588 	if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE)
5589 		return (SET_ERROR(ENOTSUP));
5590 
5591 	/*
5592 	 * If a resilver was requested, but there is no DTL on a
5593 	 * writeable leaf device, we have nothing to do.
5594 	 */
5595 	if (func == POOL_SCAN_RESILVER &&
5596 	    !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) {
5597 		spa_async_request(spa, SPA_ASYNC_RESILVER_DONE);
5598 		return (0);
5599 	}
5600 
5601 	return (dsl_scan(spa->spa_dsl_pool, func));
5602 }
5603 
5604 /*
5605  * ==========================================================================
5606  * SPA async task processing
5607  * ==========================================================================
5608  */
5609 
5610 static void
5611 spa_async_remove(spa_t *spa, vdev_t *vd)
5612 {
5613 	if (vd->vdev_remove_wanted) {
5614 		vd->vdev_remove_wanted = B_FALSE;
5615 		vd->vdev_delayed_close = B_FALSE;
5616 		vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE);
5617 
5618 		/*
5619 		 * We want to clear the stats, but we don't want to do a full
5620 		 * vdev_clear() as that will cause us to throw away
5621 		 * degraded/faulted state as well as attempt to reopen the
5622 		 * device, all of which is a waste.
5623 		 */
5624 		vd->vdev_stat.vs_read_errors = 0;
5625 		vd->vdev_stat.vs_write_errors = 0;
5626 		vd->vdev_stat.vs_checksum_errors = 0;
5627 
5628 		vdev_state_dirty(vd->vdev_top);
5629 	}
5630 
5631 	for (int c = 0; c < vd->vdev_children; c++)
5632 		spa_async_remove(spa, vd->vdev_child[c]);
5633 }
5634 
5635 static void
5636 spa_async_probe(spa_t *spa, vdev_t *vd)
5637 {
5638 	if (vd->vdev_probe_wanted) {
5639 		vd->vdev_probe_wanted = B_FALSE;
5640 		vdev_reopen(vd);	/* vdev_open() does the actual probe */
5641 	}
5642 
5643 	for (int c = 0; c < vd->vdev_children; c++)
5644 		spa_async_probe(spa, vd->vdev_child[c]);
5645 }
5646 
5647 static void
5648 spa_async_autoexpand(spa_t *spa, vdev_t *vd)
5649 {
5650 	sysevent_id_t eid;
5651 	nvlist_t *attr;
5652 	char *physpath;
5653 
5654 	if (!spa->spa_autoexpand)
5655 		return;
5656 
5657 	for (int c = 0; c < vd->vdev_children; c++) {
5658 		vdev_t *cvd = vd->vdev_child[c];
5659 		spa_async_autoexpand(spa, cvd);
5660 	}
5661 
5662 	if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL)
5663 		return;
5664 
5665 	physpath = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
5666 	(void) snprintf(physpath, MAXPATHLEN, "/devices%s", vd->vdev_physpath);
5667 
5668 	VERIFY(nvlist_alloc(&attr, NV_UNIQUE_NAME, KM_SLEEP) == 0);
5669 	VERIFY(nvlist_add_string(attr, DEV_PHYS_PATH, physpath) == 0);
5670 
5671 	(void) ddi_log_sysevent(zfs_dip, SUNW_VENDOR, EC_DEV_STATUS,
5672 	    ESC_DEV_DLE, attr, &eid, DDI_SLEEP);
5673 
5674 	nvlist_free(attr);
5675 	kmem_free(physpath, MAXPATHLEN);
5676 }
5677 
5678 static void
5679 spa_async_thread(spa_t *spa)
5680 {
5681 	int tasks;
5682 
5683 	ASSERT(spa->spa_sync_on);
5684 
5685 	mutex_enter(&spa->spa_async_lock);
5686 	tasks = spa->spa_async_tasks;
5687 	spa->spa_async_tasks = 0;
5688 	mutex_exit(&spa->spa_async_lock);
5689 
5690 	/*
5691 	 * See if the config needs to be updated.
5692 	 */
5693 	if (tasks & SPA_ASYNC_CONFIG_UPDATE) {
5694 		uint64_t old_space, new_space;
5695 
5696 		mutex_enter(&spa_namespace_lock);
5697 		old_space = metaslab_class_get_space(spa_normal_class(spa));
5698 		spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
5699 		new_space = metaslab_class_get_space(spa_normal_class(spa));
5700 		mutex_exit(&spa_namespace_lock);
5701 
5702 		/*
5703 		 * If the pool grew as a result of the config update,
5704 		 * then log an internal history event.
5705 		 */
5706 		if (new_space != old_space) {
5707 			spa_history_log_internal(spa, "vdev online", NULL,
5708 			    "pool '%s' size: %llu(+%llu)",
5709 			    spa_name(spa), new_space, new_space - old_space);
5710 		}
5711 	}
5712 
5713 	/*
5714 	 * See if any devices need to be marked REMOVED.
5715 	 */
5716 	if (tasks & SPA_ASYNC_REMOVE) {
5717 		spa_vdev_state_enter(spa, SCL_NONE);
5718 		spa_async_remove(spa, spa->spa_root_vdev);
5719 		for (int i = 0; i < spa->spa_l2cache.sav_count; i++)
5720 			spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i]);
5721 		for (int i = 0; i < spa->spa_spares.sav_count; i++)
5722 			spa_async_remove(spa, spa->spa_spares.sav_vdevs[i]);
5723 		(void) spa_vdev_state_exit(spa, NULL, 0);
5724 	}
5725 
5726 	if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) {
5727 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
5728 		spa_async_autoexpand(spa, spa->spa_root_vdev);
5729 		spa_config_exit(spa, SCL_CONFIG, FTAG);
5730 	}
5731 
5732 	/*
5733 	 * See if any devices need to be probed.
5734 	 */
5735 	if (tasks & SPA_ASYNC_PROBE) {
5736 		spa_vdev_state_enter(spa, SCL_NONE);
5737 		spa_async_probe(spa, spa->spa_root_vdev);
5738 		(void) spa_vdev_state_exit(spa, NULL, 0);
5739 	}
5740 
5741 	/*
5742 	 * If any devices are done replacing, detach them.
5743 	 */
5744 	if (tasks & SPA_ASYNC_RESILVER_DONE)
5745 		spa_vdev_resilver_done(spa);
5746 
5747 	/*
5748 	 * Kick off a resilver.
5749 	 */
5750 	if (tasks & SPA_ASYNC_RESILVER)
5751 		dsl_resilver_restart(spa->spa_dsl_pool, 0);
5752 
5753 	/*
5754 	 * Let the world know that we're done.
5755 	 */
5756 	mutex_enter(&spa->spa_async_lock);
5757 	spa->spa_async_thread = NULL;
5758 	cv_broadcast(&spa->spa_async_cv);
5759 	mutex_exit(&spa->spa_async_lock);
5760 	thread_exit();
5761 }
5762 
5763 void
5764 spa_async_suspend(spa_t *spa)
5765 {
5766 	mutex_enter(&spa->spa_async_lock);
5767 	spa->spa_async_suspended++;
5768 	while (spa->spa_async_thread != NULL)
5769 		cv_wait(&spa->spa_async_cv, &spa->spa_async_lock);
5770 	mutex_exit(&spa->spa_async_lock);
5771 }
5772 
5773 void
5774 spa_async_resume(spa_t *spa)
5775 {
5776 	mutex_enter(&spa->spa_async_lock);
5777 	ASSERT(spa->spa_async_suspended != 0);
5778 	spa->spa_async_suspended--;
5779 	mutex_exit(&spa->spa_async_lock);
5780 }
5781 
5782 static boolean_t
5783 spa_async_tasks_pending(spa_t *spa)
5784 {
5785 	uint_t non_config_tasks;
5786 	uint_t config_task;
5787 	boolean_t config_task_suspended;
5788 
5789 	non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE;
5790 	config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE;
5791 	if (spa->spa_ccw_fail_time == 0) {
5792 		config_task_suspended = B_FALSE;
5793 	} else {
5794 		config_task_suspended =
5795 		    (gethrtime() - spa->spa_ccw_fail_time) <
5796 		    (zfs_ccw_retry_interval * NANOSEC);
5797 	}
5798 
5799 	return (non_config_tasks || (config_task && !config_task_suspended));
5800 }
5801 
5802 static void
5803 spa_async_dispatch(spa_t *spa)
5804 {
5805 	mutex_enter(&spa->spa_async_lock);
5806 	if (spa_async_tasks_pending(spa) &&
5807 	    !spa->spa_async_suspended &&
5808 	    spa->spa_async_thread == NULL &&
5809 	    rootdir != NULL)
5810 		spa->spa_async_thread = thread_create(NULL, 0,
5811 		    spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri);
5812 	mutex_exit(&spa->spa_async_lock);
5813 }
5814 
5815 void
5816 spa_async_request(spa_t *spa, int task)
5817 {
5818 	zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task);
5819 	mutex_enter(&spa->spa_async_lock);
5820 	spa->spa_async_tasks |= task;
5821 	mutex_exit(&spa->spa_async_lock);
5822 }
5823 
5824 /*
5825  * ==========================================================================
5826  * SPA syncing routines
5827  * ==========================================================================
5828  */
5829 
5830 static int
5831 bpobj_enqueue_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
5832 {
5833 	bpobj_t *bpo = arg;
5834 	bpobj_enqueue(bpo, bp, tx);
5835 	return (0);
5836 }
5837 
5838 static int
5839 spa_free_sync_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
5840 {
5841 	zio_t *zio = arg;
5842 
5843 	zio_nowait(zio_free_sync(zio, zio->io_spa, dmu_tx_get_txg(tx), bp,
5844 	    zio->io_flags));
5845 	return (0);
5846 }
5847 
5848 /*
5849  * Note: this simple function is not inlined to make it easier to dtrace the
5850  * amount of time spent syncing frees.
5851  */
5852 static void
5853 spa_sync_frees(spa_t *spa, bplist_t *bpl, dmu_tx_t *tx)
5854 {
5855 	zio_t *zio = zio_root(spa, NULL, NULL, 0);
5856 	bplist_iterate(bpl, spa_free_sync_cb, zio, tx);
5857 	VERIFY(zio_wait(zio) == 0);
5858 }
5859 
5860 /*
5861  * Note: this simple function is not inlined to make it easier to dtrace the
5862  * amount of time spent syncing deferred frees.
5863  */
5864 static void
5865 spa_sync_deferred_frees(spa_t *spa, dmu_tx_t *tx)
5866 {
5867 	zio_t *zio = zio_root(spa, NULL, NULL, 0);
5868 	VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj,
5869 	    spa_free_sync_cb, zio, tx), ==, 0);
5870 	VERIFY0(zio_wait(zio));
5871 }
5872 
5873 
5874 static void
5875 spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx)
5876 {
5877 	char *packed = NULL;
5878 	size_t bufsize;
5879 	size_t nvsize = 0;
5880 	dmu_buf_t *db;
5881 
5882 	VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0);
5883 
5884 	/*
5885 	 * Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration
5886 	 * information.  This avoids the dmu_buf_will_dirty() path and
5887 	 * saves us a pre-read to get data we don't actually care about.
5888 	 */
5889 	bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE);
5890 	packed = kmem_alloc(bufsize, KM_SLEEP);
5891 
5892 	VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR,
5893 	    KM_SLEEP) == 0);
5894 	bzero(packed + nvsize, bufsize - nvsize);
5895 
5896 	dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx);
5897 
5898 	kmem_free(packed, bufsize);
5899 
5900 	VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db));
5901 	dmu_buf_will_dirty(db, tx);
5902 	*(uint64_t *)db->db_data = nvsize;
5903 	dmu_buf_rele(db, FTAG);
5904 }
5905 
5906 static void
5907 spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx,
5908     const char *config, const char *entry)
5909 {
5910 	nvlist_t *nvroot;
5911 	nvlist_t **list;
5912 	int i;
5913 
5914 	if (!sav->sav_sync)
5915 		return;
5916 
5917 	/*
5918 	 * Update the MOS nvlist describing the list of available devices.
5919 	 * spa_validate_aux() will have already made sure this nvlist is
5920 	 * valid and the vdevs are labeled appropriately.
5921 	 */
5922 	if (sav->sav_object == 0) {
5923 		sav->sav_object = dmu_object_alloc(spa->spa_meta_objset,
5924 		    DMU_OT_PACKED_NVLIST, 1 << 14, DMU_OT_PACKED_NVLIST_SIZE,
5925 		    sizeof (uint64_t), tx);
5926 		VERIFY(zap_update(spa->spa_meta_objset,
5927 		    DMU_POOL_DIRECTORY_OBJECT, entry, sizeof (uint64_t), 1,
5928 		    &sav->sav_object, tx) == 0);
5929 	}
5930 
5931 	VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0);
5932 	if (sav->sav_count == 0) {
5933 		VERIFY(nvlist_add_nvlist_array(nvroot, config, NULL, 0) == 0);
5934 	} else {
5935 		list = kmem_alloc(sav->sav_count * sizeof (void *), KM_SLEEP);
5936 		for (i = 0; i < sav->sav_count; i++)
5937 			list[i] = vdev_config_generate(spa, sav->sav_vdevs[i],
5938 			    B_FALSE, VDEV_CONFIG_L2CACHE);
5939 		VERIFY(nvlist_add_nvlist_array(nvroot, config, list,
5940 		    sav->sav_count) == 0);
5941 		for (i = 0; i < sav->sav_count; i++)
5942 			nvlist_free(list[i]);
5943 		kmem_free(list, sav->sav_count * sizeof (void *));
5944 	}
5945 
5946 	spa_sync_nvlist(spa, sav->sav_object, nvroot, tx);
5947 	nvlist_free(nvroot);
5948 
5949 	sav->sav_sync = B_FALSE;
5950 }
5951 
5952 static void
5953 spa_sync_config_object(spa_t *spa, dmu_tx_t *tx)
5954 {
5955 	nvlist_t *config;
5956 
5957 	if (list_is_empty(&spa->spa_config_dirty_list))
5958 		return;
5959 
5960 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
5961 
5962 	config = spa_config_generate(spa, spa->spa_root_vdev,
5963 	    dmu_tx_get_txg(tx), B_FALSE);
5964 
5965 	/*
5966 	 * If we're upgrading the spa version then make sure that
5967 	 * the config object gets updated with the correct version.
5968 	 */
5969 	if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version)
5970 		fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION,
5971 		    spa->spa_uberblock.ub_version);
5972 
5973 	spa_config_exit(spa, SCL_STATE, FTAG);
5974 
5975 	if (spa->spa_config_syncing)
5976 		nvlist_free(spa->spa_config_syncing);
5977 	spa->spa_config_syncing = config;
5978 
5979 	spa_sync_nvlist(spa, spa->spa_config_object, config, tx);
5980 }
5981 
5982 static void
5983 spa_sync_version(void *arg, dmu_tx_t *tx)
5984 {
5985 	uint64_t *versionp = arg;
5986 	uint64_t version = *versionp;
5987 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
5988 
5989 	/*
5990 	 * Setting the version is special cased when first creating the pool.
5991 	 */
5992 	ASSERT(tx->tx_txg != TXG_INITIAL);
5993 
5994 	ASSERT(SPA_VERSION_IS_SUPPORTED(version));
5995 	ASSERT(version >= spa_version(spa));
5996 
5997 	spa->spa_uberblock.ub_version = version;
5998 	vdev_config_dirty(spa->spa_root_vdev);
5999 	spa_history_log_internal(spa, "set", tx, "version=%lld", version);
6000 }
6001 
6002 /*
6003  * Set zpool properties.
6004  */
6005 static void
6006 spa_sync_props(void *arg, dmu_tx_t *tx)
6007 {
6008 	nvlist_t *nvp = arg;
6009 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
6010 	objset_t *mos = spa->spa_meta_objset;
6011 	nvpair_t *elem = NULL;
6012 
6013 	mutex_enter(&spa->spa_props_lock);
6014 
6015 	while ((elem = nvlist_next_nvpair(nvp, elem))) {
6016 		uint64_t intval;
6017 		char *strval, *fname;
6018 		zpool_prop_t prop;
6019 		const char *propname;
6020 		zprop_type_t proptype;
6021 		spa_feature_t fid;
6022 
6023 		switch (prop = zpool_name_to_prop(nvpair_name(elem))) {
6024 		case ZPROP_INVAL:
6025 			/*
6026 			 * We checked this earlier in spa_prop_validate().
6027 			 */
6028 			ASSERT(zpool_prop_feature(nvpair_name(elem)));
6029 
6030 			fname = strchr(nvpair_name(elem), '@') + 1;
6031 			VERIFY0(zfeature_lookup_name(fname, &fid));
6032 
6033 			spa_feature_enable(spa, fid, tx);
6034 			spa_history_log_internal(spa, "set", tx,
6035 			    "%s=enabled", nvpair_name(elem));
6036 			break;
6037 
6038 		case ZPOOL_PROP_VERSION:
6039 			intval = fnvpair_value_uint64(elem);
6040 			/*
6041 			 * The version is synced seperatly before other
6042 			 * properties and should be correct by now.
6043 			 */
6044 			ASSERT3U(spa_version(spa), >=, intval);
6045 			break;
6046 
6047 		case ZPOOL_PROP_ALTROOT:
6048 			/*
6049 			 * 'altroot' is a non-persistent property. It should
6050 			 * have been set temporarily at creation or import time.
6051 			 */
6052 			ASSERT(spa->spa_root != NULL);
6053 			break;
6054 
6055 		case ZPOOL_PROP_READONLY:
6056 		case ZPOOL_PROP_CACHEFILE:
6057 			/*
6058 			 * 'readonly' and 'cachefile' are also non-persisitent
6059 			 * properties.
6060 			 */
6061 			break;
6062 		case ZPOOL_PROP_COMMENT:
6063 			strval = fnvpair_value_string(elem);
6064 			if (spa->spa_comment != NULL)
6065 				spa_strfree(spa->spa_comment);
6066 			spa->spa_comment = spa_strdup(strval);
6067 			/*
6068 			 * We need to dirty the configuration on all the vdevs
6069 			 * so that their labels get updated.  It's unnecessary
6070 			 * to do this for pool creation since the vdev's
6071 			 * configuratoin has already been dirtied.
6072 			 */
6073 			if (tx->tx_txg != TXG_INITIAL)
6074 				vdev_config_dirty(spa->spa_root_vdev);
6075 			spa_history_log_internal(spa, "set", tx,
6076 			    "%s=%s", nvpair_name(elem), strval);
6077 			break;
6078 		default:
6079 			/*
6080 			 * Set pool property values in the poolprops mos object.
6081 			 */
6082 			if (spa->spa_pool_props_object == 0) {
6083 				spa->spa_pool_props_object =
6084 				    zap_create_link(mos, DMU_OT_POOL_PROPS,
6085 				    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_PROPS,
6086 				    tx);
6087 			}
6088 
6089 			/* normalize the property name */
6090 			propname = zpool_prop_to_name(prop);
6091 			proptype = zpool_prop_get_type(prop);
6092 
6093 			if (nvpair_type(elem) == DATA_TYPE_STRING) {
6094 				ASSERT(proptype == PROP_TYPE_STRING);
6095 				strval = fnvpair_value_string(elem);
6096 				VERIFY0(zap_update(mos,
6097 				    spa->spa_pool_props_object, propname,
6098 				    1, strlen(strval) + 1, strval, tx));
6099 				spa_history_log_internal(spa, "set", tx,
6100 				    "%s=%s", nvpair_name(elem), strval);
6101 			} else if (nvpair_type(elem) == DATA_TYPE_UINT64) {
6102 				intval = fnvpair_value_uint64(elem);
6103 
6104 				if (proptype == PROP_TYPE_INDEX) {
6105 					const char *unused;
6106 					VERIFY0(zpool_prop_index_to_string(
6107 					    prop, intval, &unused));
6108 				}
6109 				VERIFY0(zap_update(mos,
6110 				    spa->spa_pool_props_object, propname,
6111 				    8, 1, &intval, tx));
6112 				spa_history_log_internal(spa, "set", tx,
6113 				    "%s=%lld", nvpair_name(elem), intval);
6114 			} else {
6115 				ASSERT(0); /* not allowed */
6116 			}
6117 
6118 			switch (prop) {
6119 			case ZPOOL_PROP_DELEGATION:
6120 				spa->spa_delegation = intval;
6121 				break;
6122 			case ZPOOL_PROP_BOOTFS:
6123 				spa->spa_bootfs = intval;
6124 				break;
6125 			case ZPOOL_PROP_FAILUREMODE:
6126 				spa->spa_failmode = intval;
6127 				break;
6128 			case ZPOOL_PROP_AUTOEXPAND:
6129 				spa->spa_autoexpand = intval;
6130 				if (tx->tx_txg != TXG_INITIAL)
6131 					spa_async_request(spa,
6132 					    SPA_ASYNC_AUTOEXPAND);
6133 				break;
6134 			case ZPOOL_PROP_DEDUPDITTO:
6135 				spa->spa_dedup_ditto = intval;
6136 				break;
6137 			default:
6138 				break;
6139 			}
6140 		}
6141 
6142 	}
6143 
6144 	mutex_exit(&spa->spa_props_lock);
6145 }
6146 
6147 /*
6148  * Perform one-time upgrade on-disk changes.  spa_version() does not
6149  * reflect the new version this txg, so there must be no changes this
6150  * txg to anything that the upgrade code depends on after it executes.
6151  * Therefore this must be called after dsl_pool_sync() does the sync
6152  * tasks.
6153  */
6154 static void
6155 spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx)
6156 {
6157 	dsl_pool_t *dp = spa->spa_dsl_pool;
6158 
6159 	ASSERT(spa->spa_sync_pass == 1);
6160 
6161 	rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG);
6162 
6163 	if (spa->spa_ubsync.ub_version < SPA_VERSION_ORIGIN &&
6164 	    spa->spa_uberblock.ub_version >= SPA_VERSION_ORIGIN) {
6165 		dsl_pool_create_origin(dp, tx);
6166 
6167 		/* Keeping the origin open increases spa_minref */
6168 		spa->spa_minref += 3;
6169 	}
6170 
6171 	if (spa->spa_ubsync.ub_version < SPA_VERSION_NEXT_CLONES &&
6172 	    spa->spa_uberblock.ub_version >= SPA_VERSION_NEXT_CLONES) {
6173 		dsl_pool_upgrade_clones(dp, tx);
6174 	}
6175 
6176 	if (spa->spa_ubsync.ub_version < SPA_VERSION_DIR_CLONES &&
6177 	    spa->spa_uberblock.ub_version >= SPA_VERSION_DIR_CLONES) {
6178 		dsl_pool_upgrade_dir_clones(dp, tx);
6179 
6180 		/* Keeping the freedir open increases spa_minref */
6181 		spa->spa_minref += 3;
6182 	}
6183 
6184 	if (spa->spa_ubsync.ub_version < SPA_VERSION_FEATURES &&
6185 	    spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) {
6186 		spa_feature_create_zap_objects(spa, tx);
6187 	}
6188 
6189 	/*
6190 	 * LZ4_COMPRESS feature's behaviour was changed to activate_on_enable
6191 	 * when possibility to use lz4 compression for metadata was added
6192 	 * Old pools that have this feature enabled must be upgraded to have
6193 	 * this feature active
6194 	 */
6195 	if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) {
6196 		boolean_t lz4_en = spa_feature_is_enabled(spa,
6197 		    SPA_FEATURE_LZ4_COMPRESS);
6198 		boolean_t lz4_ac = spa_feature_is_active(spa,
6199 		    SPA_FEATURE_LZ4_COMPRESS);
6200 
6201 		if (lz4_en && !lz4_ac)
6202 			spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx);
6203 	}
6204 	rrw_exit(&dp->dp_config_rwlock, FTAG);
6205 }
6206 
6207 /*
6208  * Sync the specified transaction group.  New blocks may be dirtied as
6209  * part of the process, so we iterate until it converges.
6210  */
6211 void
6212 spa_sync(spa_t *spa, uint64_t txg)
6213 {
6214 	dsl_pool_t *dp = spa->spa_dsl_pool;
6215 	objset_t *mos = spa->spa_meta_objset;
6216 	bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK];
6217 	vdev_t *rvd = spa->spa_root_vdev;
6218 	vdev_t *vd;
6219 	dmu_tx_t *tx;
6220 	int error;
6221 
6222 	VERIFY(spa_writeable(spa));
6223 
6224 	/*
6225 	 * Lock out configuration changes.
6226 	 */
6227 	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
6228 
6229 	spa->spa_syncing_txg = txg;
6230 	spa->spa_sync_pass = 0;
6231 
6232 	/*
6233 	 * If there are any pending vdev state changes, convert them
6234 	 * into config changes that go out with this transaction group.
6235 	 */
6236 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
6237 	while (list_head(&spa->spa_state_dirty_list) != NULL) {
6238 		/*
6239 		 * We need the write lock here because, for aux vdevs,
6240 		 * calling vdev_config_dirty() modifies sav_config.
6241 		 * This is ugly and will become unnecessary when we
6242 		 * eliminate the aux vdev wart by integrating all vdevs
6243 		 * into the root vdev tree.
6244 		 */
6245 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
6246 		spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_WRITER);
6247 		while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) {
6248 			vdev_state_clean(vd);
6249 			vdev_config_dirty(vd);
6250 		}
6251 		spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG);
6252 		spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER);
6253 	}
6254 	spa_config_exit(spa, SCL_STATE, FTAG);
6255 
6256 	tx = dmu_tx_create_assigned(dp, txg);
6257 
6258 	spa->spa_sync_starttime = gethrtime();
6259 	VERIFY(cyclic_reprogram(spa->spa_deadman_cycid,
6260 	    spa->spa_sync_starttime + spa->spa_deadman_synctime));
6261 
6262 	/*
6263 	 * If we are upgrading to SPA_VERSION_RAIDZ_DEFLATE this txg,
6264 	 * set spa_deflate if we have no raid-z vdevs.
6265 	 */
6266 	if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE &&
6267 	    spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) {
6268 		int i;
6269 
6270 		for (i = 0; i < rvd->vdev_children; i++) {
6271 			vd = rvd->vdev_child[i];
6272 			if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE)
6273 				break;
6274 		}
6275 		if (i == rvd->vdev_children) {
6276 			spa->spa_deflate = TRUE;
6277 			VERIFY(0 == zap_add(spa->spa_meta_objset,
6278 			    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE,
6279 			    sizeof (uint64_t), 1, &spa->spa_deflate, tx));
6280 		}
6281 	}
6282 
6283 	/*
6284 	 * Iterate to convergence.
6285 	 */
6286 	do {
6287 		int pass = ++spa->spa_sync_pass;
6288 
6289 		spa_sync_config_object(spa, tx);
6290 		spa_sync_aux_dev(spa, &spa->spa_spares, tx,
6291 		    ZPOOL_CONFIG_SPARES, DMU_POOL_SPARES);
6292 		spa_sync_aux_dev(spa, &spa->spa_l2cache, tx,
6293 		    ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE);
6294 		spa_errlog_sync(spa, txg);
6295 		dsl_pool_sync(dp, txg);
6296 
6297 		if (pass < zfs_sync_pass_deferred_free) {
6298 			spa_sync_frees(spa, free_bpl, tx);
6299 		} else {
6300 			/*
6301 			 * We can not defer frees in pass 1, because
6302 			 * we sync the deferred frees later in pass 1.
6303 			 */
6304 			ASSERT3U(pass, >, 1);
6305 			bplist_iterate(free_bpl, bpobj_enqueue_cb,
6306 			    &spa->spa_deferred_bpobj, tx);
6307 		}
6308 
6309 		ddt_sync(spa, txg);
6310 		dsl_scan_sync(dp, tx);
6311 
6312 		while (vd = txg_list_remove(&spa->spa_vdev_txg_list, txg))
6313 			vdev_sync(vd, txg);
6314 
6315 		if (pass == 1) {
6316 			spa_sync_upgrades(spa, tx);
6317 			ASSERT3U(txg, >=,
6318 			    spa->spa_uberblock.ub_rootbp.blk_birth);
6319 			/*
6320 			 * Note: We need to check if the MOS is dirty
6321 			 * because we could have marked the MOS dirty
6322 			 * without updating the uberblock (e.g. if we
6323 			 * have sync tasks but no dirty user data).  We
6324 			 * need to check the uberblock's rootbp because
6325 			 * it is updated if we have synced out dirty
6326 			 * data (though in this case the MOS will most
6327 			 * likely also be dirty due to second order
6328 			 * effects, we don't want to rely on that here).
6329 			 */
6330 			if (spa->spa_uberblock.ub_rootbp.blk_birth < txg &&
6331 			    !dmu_objset_is_dirty(mos, txg)) {
6332 				/*
6333 				 * Nothing changed on the first pass,
6334 				 * therefore this TXG is a no-op.  Avoid
6335 				 * syncing deferred frees, so that we
6336 				 * can keep this TXG as a no-op.
6337 				 */
6338 				ASSERT(txg_list_empty(&dp->dp_dirty_datasets,
6339 				    txg));
6340 				ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg));
6341 				ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg));
6342 				break;
6343 			}
6344 			spa_sync_deferred_frees(spa, tx);
6345 		}
6346 
6347 	} while (dmu_objset_is_dirty(mos, txg));
6348 
6349 	/*
6350 	 * Rewrite the vdev configuration (which includes the uberblock)
6351 	 * to commit the transaction group.
6352 	 *
6353 	 * If there are no dirty vdevs, we sync the uberblock to a few
6354 	 * random top-level vdevs that are known to be visible in the
6355 	 * config cache (see spa_vdev_add() for a complete description).
6356 	 * If there *are* dirty vdevs, sync the uberblock to all vdevs.
6357 	 */
6358 	for (;;) {
6359 		/*
6360 		 * We hold SCL_STATE to prevent vdev open/close/etc.
6361 		 * while we're attempting to write the vdev labels.
6362 		 */
6363 		spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
6364 
6365 		if (list_is_empty(&spa->spa_config_dirty_list)) {
6366 			vdev_t *svd[SPA_DVAS_PER_BP];
6367 			int svdcount = 0;
6368 			int children = rvd->vdev_children;
6369 			int c0 = spa_get_random(children);
6370 
6371 			for (int c = 0; c < children; c++) {
6372 				vd = rvd->vdev_child[(c0 + c) % children];
6373 				if (vd->vdev_ms_array == 0 || vd->vdev_islog)
6374 					continue;
6375 				svd[svdcount++] = vd;
6376 				if (svdcount == SPA_DVAS_PER_BP)
6377 					break;
6378 			}
6379 			error = vdev_config_sync(svd, svdcount, txg, B_FALSE);
6380 			if (error != 0)
6381 				error = vdev_config_sync(svd, svdcount, txg,
6382 				    B_TRUE);
6383 		} else {
6384 			error = vdev_config_sync(rvd->vdev_child,
6385 			    rvd->vdev_children, txg, B_FALSE);
6386 			if (error != 0)
6387 				error = vdev_config_sync(rvd->vdev_child,
6388 				    rvd->vdev_children, txg, B_TRUE);
6389 		}
6390 
6391 		if (error == 0)
6392 			spa->spa_last_synced_guid = rvd->vdev_guid;
6393 
6394 		spa_config_exit(spa, SCL_STATE, FTAG);
6395 
6396 		if (error == 0)
6397 			break;
6398 		zio_suspend(spa, NULL);
6399 		zio_resume_wait(spa);
6400 	}
6401 	dmu_tx_commit(tx);
6402 
6403 	VERIFY(cyclic_reprogram(spa->spa_deadman_cycid, CY_INFINITY));
6404 
6405 	/*
6406 	 * Clear the dirty config list.
6407 	 */
6408 	while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL)
6409 		vdev_config_clean(vd);
6410 
6411 	/*
6412 	 * Now that the new config has synced transactionally,
6413 	 * let it become visible to the config cache.
6414 	 */
6415 	if (spa->spa_config_syncing != NULL) {
6416 		spa_config_set(spa, spa->spa_config_syncing);
6417 		spa->spa_config_txg = txg;
6418 		spa->spa_config_syncing = NULL;
6419 	}
6420 
6421 	spa->spa_ubsync = spa->spa_uberblock;
6422 
6423 	dsl_pool_sync_done(dp, txg);
6424 
6425 	/*
6426 	 * Update usable space statistics.
6427 	 */
6428 	while (vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg)))
6429 		vdev_sync_done(vd, txg);
6430 
6431 	spa_update_dspace(spa);
6432 
6433 	/*
6434 	 * It had better be the case that we didn't dirty anything
6435 	 * since vdev_config_sync().
6436 	 */
6437 	ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg));
6438 	ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg));
6439 	ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg));
6440 
6441 	spa->spa_sync_pass = 0;
6442 
6443 	spa_config_exit(spa, SCL_CONFIG, FTAG);
6444 
6445 	spa_handle_ignored_writes(spa);
6446 
6447 	/*
6448 	 * If any async tasks have been requested, kick them off.
6449 	 */
6450 	spa_async_dispatch(spa);
6451 }
6452 
6453 /*
6454  * Sync all pools.  We don't want to hold the namespace lock across these
6455  * operations, so we take a reference on the spa_t and drop the lock during the
6456  * sync.
6457  */
6458 void
6459 spa_sync_allpools(void)
6460 {
6461 	spa_t *spa = NULL;
6462 	mutex_enter(&spa_namespace_lock);
6463 	while ((spa = spa_next(spa)) != NULL) {
6464 		if (spa_state(spa) != POOL_STATE_ACTIVE ||
6465 		    !spa_writeable(spa) || spa_suspended(spa))
6466 			continue;
6467 		spa_open_ref(spa, FTAG);
6468 		mutex_exit(&spa_namespace_lock);
6469 		txg_wait_synced(spa_get_dsl(spa), 0);
6470 		mutex_enter(&spa_namespace_lock);
6471 		spa_close(spa, FTAG);
6472 	}
6473 	mutex_exit(&spa_namespace_lock);
6474 }
6475 
6476 /*
6477  * ==========================================================================
6478  * Miscellaneous routines
6479  * ==========================================================================
6480  */
6481 
6482 /*
6483  * Remove all pools in the system.
6484  */
6485 void
6486 spa_evict_all(void)
6487 {
6488 	spa_t *spa;
6489 
6490 	/*
6491 	 * Remove all cached state.  All pools should be closed now,
6492 	 * so every spa in the AVL tree should be unreferenced.
6493 	 */
6494 	mutex_enter(&spa_namespace_lock);
6495 	while ((spa = spa_next(NULL)) != NULL) {
6496 		/*
6497 		 * Stop async tasks.  The async thread may need to detach
6498 		 * a device that's been replaced, which requires grabbing
6499 		 * spa_namespace_lock, so we must drop it here.
6500 		 */
6501 		spa_open_ref(spa, FTAG);
6502 		mutex_exit(&spa_namespace_lock);
6503 		spa_async_suspend(spa);
6504 		mutex_enter(&spa_namespace_lock);
6505 		spa_close(spa, FTAG);
6506 
6507 		if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
6508 			spa_unload(spa);
6509 			spa_deactivate(spa);
6510 		}
6511 		spa_remove(spa);
6512 	}
6513 	mutex_exit(&spa_namespace_lock);
6514 }
6515 
6516 vdev_t *
6517 spa_lookup_by_guid(spa_t *spa, uint64_t guid, boolean_t aux)
6518 {
6519 	vdev_t *vd;
6520 	int i;
6521 
6522 	if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL)
6523 		return (vd);
6524 
6525 	if (aux) {
6526 		for (i = 0; i < spa->spa_l2cache.sav_count; i++) {
6527 			vd = spa->spa_l2cache.sav_vdevs[i];
6528 			if (vd->vdev_guid == guid)
6529 				return (vd);
6530 		}
6531 
6532 		for (i = 0; i < spa->spa_spares.sav_count; i++) {
6533 			vd = spa->spa_spares.sav_vdevs[i];
6534 			if (vd->vdev_guid == guid)
6535 				return (vd);
6536 		}
6537 	}
6538 
6539 	return (NULL);
6540 }
6541 
6542 void
6543 spa_upgrade(spa_t *spa, uint64_t version)
6544 {
6545 	ASSERT(spa_writeable(spa));
6546 
6547 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
6548 
6549 	/*
6550 	 * This should only be called for a non-faulted pool, and since a
6551 	 * future version would result in an unopenable pool, this shouldn't be
6552 	 * possible.
6553 	 */
6554 	ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version));
6555 	ASSERT3U(version, >=, spa->spa_uberblock.ub_version);
6556 
6557 	spa->spa_uberblock.ub_version = version;
6558 	vdev_config_dirty(spa->spa_root_vdev);
6559 
6560 	spa_config_exit(spa, SCL_ALL, FTAG);
6561 
6562 	txg_wait_synced(spa_get_dsl(spa), 0);
6563 }
6564 
6565 boolean_t
6566 spa_has_spare(spa_t *spa, uint64_t guid)
6567 {
6568 	int i;
6569 	uint64_t spareguid;
6570 	spa_aux_vdev_t *sav = &spa->spa_spares;
6571 
6572 	for (i = 0; i < sav->sav_count; i++)
6573 		if (sav->sav_vdevs[i]->vdev_guid == guid)
6574 			return (B_TRUE);
6575 
6576 	for (i = 0; i < sav->sav_npending; i++) {
6577 		if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID,
6578 		    &spareguid) == 0 && spareguid == guid)
6579 			return (B_TRUE);
6580 	}
6581 
6582 	return (B_FALSE);
6583 }
6584 
6585 /*
6586  * Check if a pool has an active shared spare device.
6587  * Note: reference count of an active spare is 2, as a spare and as a replace
6588  */
6589 static boolean_t
6590 spa_has_active_shared_spare(spa_t *spa)
6591 {
6592 	int i, refcnt;
6593 	uint64_t pool;
6594 	spa_aux_vdev_t *sav = &spa->spa_spares;
6595 
6596 	for (i = 0; i < sav->sav_count; i++) {
6597 		if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool,
6598 		    &refcnt) && pool != 0ULL && pool == spa_guid(spa) &&
6599 		    refcnt > 2)
6600 			return (B_TRUE);
6601 	}
6602 
6603 	return (B_FALSE);
6604 }
6605 
6606 /*
6607  * Post a sysevent corresponding to the given event.  The 'name' must be one of
6608  * the event definitions in sys/sysevent/eventdefs.h.  The payload will be
6609  * filled in from the spa and (optionally) the vdev.  This doesn't do anything
6610  * in the userland libzpool, as we don't want consumers to misinterpret ztest
6611  * or zdb as real changes.
6612  */
6613 void
6614 spa_event_notify(spa_t *spa, vdev_t *vd, const char *name)
6615 {
6616 #ifdef _KERNEL
6617 	sysevent_t		*ev;
6618 	sysevent_attr_list_t	*attr = NULL;
6619 	sysevent_value_t	value;
6620 	sysevent_id_t		eid;
6621 
6622 	ev = sysevent_alloc(EC_ZFS, (char *)name, SUNW_KERN_PUB "zfs",
6623 	    SE_SLEEP);
6624 
6625 	value.value_type = SE_DATA_TYPE_STRING;
6626 	value.value.sv_string = spa_name(spa);
6627 	if (sysevent_add_attr(&attr, ZFS_EV_POOL_NAME, &value, SE_SLEEP) != 0)
6628 		goto done;
6629 
6630 	value.value_type = SE_DATA_TYPE_UINT64;
6631 	value.value.sv_uint64 = spa_guid(spa);
6632 	if (sysevent_add_attr(&attr, ZFS_EV_POOL_GUID, &value, SE_SLEEP) != 0)
6633 		goto done;
6634 
6635 	if (vd) {
6636 		value.value_type = SE_DATA_TYPE_UINT64;
6637 		value.value.sv_uint64 = vd->vdev_guid;
6638 		if (sysevent_add_attr(&attr, ZFS_EV_VDEV_GUID, &value,
6639 		    SE_SLEEP) != 0)
6640 			goto done;
6641 
6642 		if (vd->vdev_path) {
6643 			value.value_type = SE_DATA_TYPE_STRING;
6644 			value.value.sv_string = vd->vdev_path;
6645 			if (sysevent_add_attr(&attr, ZFS_EV_VDEV_PATH,
6646 			    &value, SE_SLEEP) != 0)
6647 				goto done;
6648 		}
6649 	}
6650 
6651 	if (sysevent_attach_attributes(ev, attr) != 0)
6652 		goto done;
6653 	attr = NULL;
6654 
6655 	(void) log_sysevent(ev, SE_SLEEP, &eid);
6656 
6657 done:
6658 	if (attr)
6659 		sysevent_free_attr(attr);
6660 	sysevent_free(ev);
6661 #endif
6662 }
6663