1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
25 * Copyright (c) 2015, Nexenta Systems, Inc.  All rights reserved.
26 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
27 * Copyright 2013 Saso Kiselkov. All rights reserved.
28 * Copyright (c) 2014 Integros [integros.com]
29 * Copyright 2016 Toomas Soome <tsoome@me.com>
30 * Copyright 2017 Joyent, Inc.
31 */
32
33/*
34 * SPA: Storage Pool Allocator
35 *
36 * This file contains all the routines used when modifying on-disk SPA state.
37 * This includes opening, importing, destroying, exporting a pool, and syncing a
38 * pool.
39 */
40
41#include <sys/zfs_context.h>
42#include <sys/fm/fs/zfs.h>
43#include <sys/spa_impl.h>
44#include <sys/zio.h>
45#include <sys/zio_checksum.h>
46#include <sys/dmu.h>
47#include <sys/dmu_tx.h>
48#include <sys/zap.h>
49#include <sys/zil.h>
50#include <sys/ddt.h>
51#include <sys/vdev_impl.h>
52#include <sys/metaslab.h>
53#include <sys/metaslab_impl.h>
54#include <sys/uberblock_impl.h>
55#include <sys/txg.h>
56#include <sys/avl.h>
57#include <sys/dmu_traverse.h>
58#include <sys/dmu_objset.h>
59#include <sys/unique.h>
60#include <sys/dsl_pool.h>
61#include <sys/dsl_dataset.h>
62#include <sys/dsl_dir.h>
63#include <sys/dsl_prop.h>
64#include <sys/dsl_synctask.h>
65#include <sys/fs/zfs.h>
66#include <sys/arc.h>
67#include <sys/callb.h>
68#include <sys/systeminfo.h>
69#include <sys/spa_boot.h>
70#include <sys/zfs_ioctl.h>
71#include <sys/dsl_scan.h>
72#include <sys/zfeature.h>
73#include <sys/dsl_destroy.h>
74#include <sys/abd.h>
75
76#ifdef	_KERNEL
77#include <sys/bootprops.h>
78#include <sys/callb.h>
79#include <sys/cpupart.h>
80#include <sys/pool.h>
81#include <sys/sysdc.h>
82#include <sys/zone.h>
83#endif	/* _KERNEL */
84
85#include "zfs_prop.h"
86#include "zfs_comutil.h"
87
88/*
89 * The interval, in seconds, at which failed configuration cache file writes
90 * should be retried.
91 */
92static int zfs_ccw_retry_interval = 300;
93
94typedef enum zti_modes {
95	ZTI_MODE_FIXED,			/* value is # of threads (min 1) */
96	ZTI_MODE_BATCH,			/* cpu-intensive; value is ignored */
97	ZTI_MODE_NULL,			/* don't create a taskq */
98	ZTI_NMODES
99} zti_modes_t;
100
101#define	ZTI_P(n, q)	{ ZTI_MODE_FIXED, (n), (q) }
102#define	ZTI_BATCH	{ ZTI_MODE_BATCH, 0, 1 }
103#define	ZTI_NULL	{ ZTI_MODE_NULL, 0, 0 }
104
105#define	ZTI_N(n)	ZTI_P(n, 1)
106#define	ZTI_ONE		ZTI_N(1)
107
108typedef struct zio_taskq_info {
109	zti_modes_t zti_mode;
110	uint_t zti_value;
111	uint_t zti_count;
112} zio_taskq_info_t;
113
114static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = {
115	"issue", "issue_high", "intr", "intr_high"
116};
117
118/*
119 * This table defines the taskq settings for each ZFS I/O type. When
120 * initializing a pool, we use this table to create an appropriately sized
121 * taskq. Some operations are low volume and therefore have a small, static
122 * number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE
123 * macros. Other operations process a large amount of data; the ZTI_BATCH
124 * macro causes us to create a taskq oriented for throughput. Some operations
125 * are so high frequency and short-lived that the taskq itself can become a a
126 * point of lock contention. The ZTI_P(#, #) macro indicates that we need an
127 * additional degree of parallelism specified by the number of threads per-
128 * taskq and the number of taskqs; when dispatching an event in this case, the
129 * particular taskq is chosen at random.
130 *
131 * The different taskq priorities are to handle the different contexts (issue
132 * and interrupt) and then to reserve threads for ZIO_PRIORITY_NOW I/Os that
133 * need to be handled with minimum delay.
134 */
135const zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = {
136	/* ISSUE	ISSUE_HIGH	INTR		INTR_HIGH */
137	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* NULL */
138	{ ZTI_N(8),	ZTI_NULL,	ZTI_P(12, 8),	ZTI_NULL }, /* READ */
139	{ ZTI_BATCH,	ZTI_N(5),	ZTI_N(8),	ZTI_N(5) }, /* WRITE */
140	{ ZTI_P(12, 8),	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* FREE */
141	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* CLAIM */
142	{ ZTI_ONE,	ZTI_NULL,	ZTI_ONE,	ZTI_NULL }, /* IOCTL */
143};
144
145static sysevent_t *spa_event_create(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl,
146    const char *name);
147static void spa_event_post(sysevent_t *ev);
148static void spa_sync_version(void *arg, dmu_tx_t *tx);
149static void spa_sync_props(void *arg, dmu_tx_t *tx);
150static boolean_t spa_has_active_shared_spare(spa_t *spa);
151static int spa_load_impl(spa_t *spa, uint64_t, nvlist_t *config,
152    spa_load_state_t state, spa_import_type_t type, boolean_t mosconfig,
153    char **ereport);
154static void spa_vdev_resilver_done(spa_t *spa);
155
156uint_t		zio_taskq_batch_pct = 75;	/* 1 thread per cpu in pset */
157id_t		zio_taskq_psrset_bind = PS_NONE;
158boolean_t	zio_taskq_sysdc = B_TRUE;	/* use SDC scheduling class */
159uint_t		zio_taskq_basedc = 80;		/* base duty cycle */
160
161boolean_t	spa_create_process = B_TRUE;	/* no process ==> no sysdc */
162extern int	zfs_sync_pass_deferred_free;
163
164/*
165 * This (illegal) pool name is used when temporarily importing a spa_t in order
166 * to get the vdev stats associated with the imported devices.
167 */
168#define	TRYIMPORT_NAME	"$import"
169
170/*
171 * ==========================================================================
172 * SPA properties routines
173 * ==========================================================================
174 */
175
176/*
177 * Add a (source=src, propname=propval) list to an nvlist.
178 */
179static void
180spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, char *strval,
181    uint64_t intval, zprop_source_t src)
182{
183	const char *propname = zpool_prop_to_name(prop);
184	nvlist_t *propval;
185
186	VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0);
187	VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0);
188
189	if (strval != NULL)
190		VERIFY(nvlist_add_string(propval, ZPROP_VALUE, strval) == 0);
191	else
192		VERIFY(nvlist_add_uint64(propval, ZPROP_VALUE, intval) == 0);
193
194	VERIFY(nvlist_add_nvlist(nvl, propname, propval) == 0);
195	nvlist_free(propval);
196}
197
198/*
199 * Get property values from the spa configuration.
200 */
201static void
202spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
203{
204	vdev_t *rvd = spa->spa_root_vdev;
205	dsl_pool_t *pool = spa->spa_dsl_pool;
206	uint64_t size, alloc, cap, version;
207	zprop_source_t src = ZPROP_SRC_NONE;
208	spa_config_dirent_t *dp;
209	metaslab_class_t *mc = spa_normal_class(spa);
210
211	ASSERT(MUTEX_HELD(&spa->spa_props_lock));
212
213	if (rvd != NULL) {
214		alloc = metaslab_class_get_alloc(spa_normal_class(spa));
215		size = metaslab_class_get_space(spa_normal_class(spa));
216		spa_prop_add_list(*nvp, ZPOOL_PROP_NAME, spa_name(spa), 0, src);
217		spa_prop_add_list(*nvp, ZPOOL_PROP_SIZE, NULL, size, src);
218		spa_prop_add_list(*nvp, ZPOOL_PROP_ALLOCATED, NULL, alloc, src);
219		spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL,
220		    size - alloc, src);
221
222		spa_prop_add_list(*nvp, ZPOOL_PROP_FRAGMENTATION, NULL,
223		    metaslab_class_fragmentation(mc), src);
224		spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL,
225		    metaslab_class_expandable_space(mc), src);
226		spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL,
227		    (spa_mode(spa) == FREAD), src);
228
229		cap = (size == 0) ? 0 : (alloc * 100 / size);
230		spa_prop_add_list(*nvp, ZPOOL_PROP_CAPACITY, NULL, cap, src);
231
232		spa_prop_add_list(*nvp, ZPOOL_PROP_DEDUPRATIO, NULL,
233		    ddt_get_pool_dedup_ratio(spa), src);
234
235		spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL,
236		    rvd->vdev_state, src);
237
238		version = spa_version(spa);
239		if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION))
240			src = ZPROP_SRC_DEFAULT;
241		else
242			src = ZPROP_SRC_LOCAL;
243		spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, version, src);
244	}
245
246	if (pool != NULL) {
247		/*
248		 * The $FREE directory was introduced in SPA_VERSION_DEADLISTS,
249		 * when opening pools before this version freedir will be NULL.
250		 */
251		if (pool->dp_free_dir != NULL) {
252			spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL,
253			    dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes,
254			    src);
255		} else {
256			spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING,
257			    NULL, 0, src);
258		}
259
260		if (pool->dp_leak_dir != NULL) {
261			spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL,
262			    dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes,
263			    src);
264		} else {
265			spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED,
266			    NULL, 0, src);
267		}
268	}
269
270	spa_prop_add_list(*nvp, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src);
271
272	if (spa->spa_comment != NULL) {
273		spa_prop_add_list(*nvp, ZPOOL_PROP_COMMENT, spa->spa_comment,
274		    0, ZPROP_SRC_LOCAL);
275	}
276
277	if (spa->spa_root != NULL)
278		spa_prop_add_list(*nvp, ZPOOL_PROP_ALTROOT, spa->spa_root,
279		    0, ZPROP_SRC_LOCAL);
280
281	if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) {
282		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL,
283		    MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE);
284	} else {
285		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL,
286		    SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE);
287	}
288
289	if ((dp = list_head(&spa->spa_config_list)) != NULL) {
290		if (dp->scd_path == NULL) {
291			spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
292			    "none", 0, ZPROP_SRC_LOCAL);
293		} else if (strcmp(dp->scd_path, spa_config_path) != 0) {
294			spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
295			    dp->scd_path, 0, ZPROP_SRC_LOCAL);
296		}
297	}
298}
299
300/*
301 * Get zpool property values.
302 */
303int
304spa_prop_get(spa_t *spa, nvlist_t **nvp)
305{
306	objset_t *mos = spa->spa_meta_objset;
307	zap_cursor_t zc;
308	zap_attribute_t za;
309	int err;
310
311	VERIFY(nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP) == 0);
312
313	mutex_enter(&spa->spa_props_lock);
314
315	/*
316	 * Get properties from the spa config.
317	 */
318	spa_prop_get_config(spa, nvp);
319
320	/* If no pool property object, no more prop to get. */
321	if (mos == NULL || spa->spa_pool_props_object == 0) {
322		mutex_exit(&spa->spa_props_lock);
323		return (0);
324	}
325
326	/*
327	 * Get properties from the MOS pool property object.
328	 */
329	for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object);
330	    (err = zap_cursor_retrieve(&zc, &za)) == 0;
331	    zap_cursor_advance(&zc)) {
332		uint64_t intval = 0;
333		char *strval = NULL;
334		zprop_source_t src = ZPROP_SRC_DEFAULT;
335		zpool_prop_t prop;
336
337		if ((prop = zpool_name_to_prop(za.za_name)) == ZPROP_INVAL)
338			continue;
339
340		switch (za.za_integer_length) {
341		case 8:
342			/* integer property */
343			if (za.za_first_integer !=
344			    zpool_prop_default_numeric(prop))
345				src = ZPROP_SRC_LOCAL;
346
347			if (prop == ZPOOL_PROP_BOOTFS) {
348				dsl_pool_t *dp;
349				dsl_dataset_t *ds = NULL;
350
351				dp = spa_get_dsl(spa);
352				dsl_pool_config_enter(dp, FTAG);
353				if (err = dsl_dataset_hold_obj(dp,
354				    za.za_first_integer, FTAG, &ds)) {
355					dsl_pool_config_exit(dp, FTAG);
356					break;
357				}
358
359				strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN,
360				    KM_SLEEP);
361				dsl_dataset_name(ds, strval);
362				dsl_dataset_rele(ds, FTAG);
363				dsl_pool_config_exit(dp, FTAG);
364			} else {
365				strval = NULL;
366				intval = za.za_first_integer;
367			}
368
369			spa_prop_add_list(*nvp, prop, strval, intval, src);
370
371			if (strval != NULL)
372				kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN);
373
374			break;
375
376		case 1:
377			/* string property */
378			strval = kmem_alloc(za.za_num_integers, KM_SLEEP);
379			err = zap_lookup(mos, spa->spa_pool_props_object,
380			    za.za_name, 1, za.za_num_integers, strval);
381			if (err) {
382				kmem_free(strval, za.za_num_integers);
383				break;
384			}
385			spa_prop_add_list(*nvp, prop, strval, 0, src);
386			kmem_free(strval, za.za_num_integers);
387			break;
388
389		default:
390			break;
391		}
392	}
393	zap_cursor_fini(&zc);
394	mutex_exit(&spa->spa_props_lock);
395out:
396	if (err && err != ENOENT) {
397		nvlist_free(*nvp);
398		*nvp = NULL;
399		return (err);
400	}
401
402	return (0);
403}
404
405/*
406 * Validate the given pool properties nvlist and modify the list
407 * for the property values to be set.
408 */
409static int
410spa_prop_validate(spa_t *spa, nvlist_t *props)
411{
412	nvpair_t *elem;
413	int error = 0, reset_bootfs = 0;
414	uint64_t objnum = 0;
415	boolean_t has_feature = B_FALSE;
416
417	elem = NULL;
418	while ((elem = nvlist_next_nvpair(props, elem)) != NULL) {
419		uint64_t intval;
420		char *strval, *slash, *check, *fname;
421		const char *propname = nvpair_name(elem);
422		zpool_prop_t prop = zpool_name_to_prop(propname);
423
424		switch (prop) {
425		case ZPROP_INVAL:
426			if (!zpool_prop_feature(propname)) {
427				error = SET_ERROR(EINVAL);
428				break;
429			}
430
431			/*
432			 * Sanitize the input.
433			 */
434			if (nvpair_type(elem) != DATA_TYPE_UINT64) {
435				error = SET_ERROR(EINVAL);
436				break;
437			}
438
439			if (nvpair_value_uint64(elem, &intval) != 0) {
440				error = SET_ERROR(EINVAL);
441				break;
442			}
443
444			if (intval != 0) {
445				error = SET_ERROR(EINVAL);
446				break;
447			}
448
449			fname = strchr(propname, '@') + 1;
450			if (zfeature_lookup_name(fname, NULL) != 0) {
451				error = SET_ERROR(EINVAL);
452				break;
453			}
454
455			has_feature = B_TRUE;
456			break;
457
458		case ZPOOL_PROP_VERSION:
459			error = nvpair_value_uint64(elem, &intval);
460			if (!error &&
461			    (intval < spa_version(spa) ||
462			    intval > SPA_VERSION_BEFORE_FEATURES ||
463			    has_feature))
464				error = SET_ERROR(EINVAL);
465			break;
466
467		case ZPOOL_PROP_DELEGATION:
468		case ZPOOL_PROP_AUTOREPLACE:
469		case ZPOOL_PROP_LISTSNAPS:
470		case ZPOOL_PROP_AUTOEXPAND:
471			error = nvpair_value_uint64(elem, &intval);
472			if (!error && intval > 1)
473				error = SET_ERROR(EINVAL);
474			break;
475
476		case ZPOOL_PROP_BOOTFS:
477			/*
478			 * If the pool version is less than SPA_VERSION_BOOTFS,
479			 * or the pool is still being created (version == 0),
480			 * the bootfs property cannot be set.
481			 */
482			if (spa_version(spa) < SPA_VERSION_BOOTFS) {
483				error = SET_ERROR(ENOTSUP);
484				break;
485			}
486
487			/*
488			 * Make sure the vdev config is bootable
489			 */
490			if (!vdev_is_bootable(spa->spa_root_vdev)) {
491				error = SET_ERROR(ENOTSUP);
492				break;
493			}
494
495			reset_bootfs = 1;
496
497			error = nvpair_value_string(elem, &strval);
498
499			if (!error) {
500				objset_t *os;
501				uint64_t propval;
502
503				if (strval == NULL || strval[0] == '\0') {
504					objnum = zpool_prop_default_numeric(
505					    ZPOOL_PROP_BOOTFS);
506					break;
507				}
508
509				if (error = dmu_objset_hold(strval, FTAG, &os))
510					break;
511
512				/*
513				 * Must be ZPL, and its property settings
514				 * must be supported by GRUB (compression
515				 * is not gzip, and large blocks are not used).
516				 */
517
518				if (dmu_objset_type(os) != DMU_OST_ZFS) {
519					error = SET_ERROR(ENOTSUP);
520				} else if ((error =
521				    dsl_prop_get_int_ds(dmu_objset_ds(os),
522				    zfs_prop_to_name(ZFS_PROP_COMPRESSION),
523				    &propval)) == 0 &&
524				    !BOOTFS_COMPRESS_VALID(propval)) {
525					error = SET_ERROR(ENOTSUP);
526				} else {
527					objnum = dmu_objset_id(os);
528				}
529				dmu_objset_rele(os, FTAG);
530			}
531			break;
532
533		case ZPOOL_PROP_FAILUREMODE:
534			error = nvpair_value_uint64(elem, &intval);
535			if (!error && (intval < ZIO_FAILURE_MODE_WAIT ||
536			    intval > ZIO_FAILURE_MODE_PANIC))
537				error = SET_ERROR(EINVAL);
538
539			/*
540			 * This is a special case which only occurs when
541			 * the pool has completely failed. This allows
542			 * the user to change the in-core failmode property
543			 * without syncing it out to disk (I/Os might
544			 * currently be blocked). We do this by returning
545			 * EIO to the caller (spa_prop_set) to trick it
546			 * into thinking we encountered a property validation
547			 * error.
548			 */
549			if (!error && spa_suspended(spa)) {
550				spa->spa_failmode = intval;
551				error = SET_ERROR(EIO);
552			}
553			break;
554
555		case ZPOOL_PROP_CACHEFILE:
556			if ((error = nvpair_value_string(elem, &strval)) != 0)
557				break;
558
559			if (strval[0] == '\0')
560				break;
561
562			if (strcmp(strval, "none") == 0)
563				break;
564
565			if (strval[0] != '/') {
566				error = SET_ERROR(EINVAL);
567				break;
568			}
569
570			slash = strrchr(strval, '/');
571			ASSERT(slash != NULL);
572
573			if (slash[1] == '\0' || strcmp(slash, "/.") == 0 ||
574			    strcmp(slash, "/..") == 0)
575				error = SET_ERROR(EINVAL);
576			break;
577
578		case ZPOOL_PROP_COMMENT:
579			if ((error = nvpair_value_string(elem, &strval)) != 0)
580				break;
581			for (check = strval; *check != '\0'; check++) {
582				/*
583				 * The kernel doesn't have an easy isprint()
584				 * check.  For this kernel check, we merely
585				 * check ASCII apart from DEL.  Fix this if
586				 * there is an easy-to-use kernel isprint().
587				 */
588				if (*check >= 0x7f) {
589					error = SET_ERROR(EINVAL);
590					break;
591				}
592			}
593			if (strlen(strval) > ZPROP_MAX_COMMENT)
594				error = E2BIG;
595			break;
596
597		case ZPOOL_PROP_DEDUPDITTO:
598			if (spa_version(spa) < SPA_VERSION_DEDUP)
599				error = SET_ERROR(ENOTSUP);
600			else
601				error = nvpair_value_uint64(elem, &intval);
602			if (error == 0 &&
603			    intval != 0 && intval < ZIO_DEDUPDITTO_MIN)
604				error = SET_ERROR(EINVAL);
605			break;
606		}
607
608		if (error)
609			break;
610	}
611
612	if (!error && reset_bootfs) {
613		error = nvlist_remove(props,
614		    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING);
615
616		if (!error) {
617			error = nvlist_add_uint64(props,
618			    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum);
619		}
620	}
621
622	return (error);
623}
624
625void
626spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync)
627{
628	char *cachefile;
629	spa_config_dirent_t *dp;
630
631	if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE),
632	    &cachefile) != 0)
633		return;
634
635	dp = kmem_alloc(sizeof (spa_config_dirent_t),
636	    KM_SLEEP);
637
638	if (cachefile[0] == '\0')
639		dp->scd_path = spa_strdup(spa_config_path);
640	else if (strcmp(cachefile, "none") == 0)
641		dp->scd_path = NULL;
642	else
643		dp->scd_path = spa_strdup(cachefile);
644
645	list_insert_head(&spa->spa_config_list, dp);
646	if (need_sync)
647		spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
648}
649
650int
651spa_prop_set(spa_t *spa, nvlist_t *nvp)
652{
653	int error;
654	nvpair_t *elem = NULL;
655	boolean_t need_sync = B_FALSE;
656
657	if ((error = spa_prop_validate(spa, nvp)) != 0)
658		return (error);
659
660	while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) {
661		zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem));
662
663		if (prop == ZPOOL_PROP_CACHEFILE ||
664		    prop == ZPOOL_PROP_ALTROOT ||
665		    prop == ZPOOL_PROP_READONLY)
666			continue;
667
668		if (prop == ZPOOL_PROP_VERSION || prop == ZPROP_INVAL) {
669			uint64_t ver;
670
671			if (prop == ZPOOL_PROP_VERSION) {
672				VERIFY(nvpair_value_uint64(elem, &ver) == 0);
673			} else {
674				ASSERT(zpool_prop_feature(nvpair_name(elem)));
675				ver = SPA_VERSION_FEATURES;
676				need_sync = B_TRUE;
677			}
678
679			/* Save time if the version is already set. */
680			if (ver == spa_version(spa))
681				continue;
682
683			/*
684			 * In addition to the pool directory object, we might
685			 * create the pool properties object, the features for
686			 * read object, the features for write object, or the
687			 * feature descriptions object.
688			 */
689			error = dsl_sync_task(spa->spa_name, NULL,
690			    spa_sync_version, &ver,
691			    6, ZFS_SPACE_CHECK_RESERVED);
692			if (error)
693				return (error);
694			continue;
695		}
696
697		need_sync = B_TRUE;
698		break;
699	}
700
701	if (need_sync) {
702		return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props,
703		    nvp, 6, ZFS_SPACE_CHECK_RESERVED));
704	}
705
706	return (0);
707}
708
709/*
710 * If the bootfs property value is dsobj, clear it.
711 */
712void
713spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx)
714{
715	if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) {
716		VERIFY(zap_remove(spa->spa_meta_objset,
717		    spa->spa_pool_props_object,
718		    zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0);
719		spa->spa_bootfs = 0;
720	}
721}
722
723/*ARGSUSED*/
724static int
725spa_change_guid_check(void *arg, dmu_tx_t *tx)
726{
727	uint64_t *newguid = arg;
728	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
729	vdev_t *rvd = spa->spa_root_vdev;
730	uint64_t vdev_state;
731
732	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
733	vdev_state = rvd->vdev_state;
734	spa_config_exit(spa, SCL_STATE, FTAG);
735
736	if (vdev_state != VDEV_STATE_HEALTHY)
737		return (SET_ERROR(ENXIO));
738
739	ASSERT3U(spa_guid(spa), !=, *newguid);
740
741	return (0);
742}
743
744static void
745spa_change_guid_sync(void *arg, dmu_tx_t *tx)
746{
747	uint64_t *newguid = arg;
748	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
749	uint64_t oldguid;
750	vdev_t *rvd = spa->spa_root_vdev;
751
752	oldguid = spa_guid(spa);
753
754	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
755	rvd->vdev_guid = *newguid;
756	rvd->vdev_guid_sum += (*newguid - oldguid);
757	vdev_config_dirty(rvd);
758	spa_config_exit(spa, SCL_STATE, FTAG);
759
760	spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu",
761	    oldguid, *newguid);
762}
763
764/*
765 * Change the GUID for the pool.  This is done so that we can later
766 * re-import a pool built from a clone of our own vdevs.  We will modify
767 * the root vdev's guid, our own pool guid, and then mark all of our
768 * vdevs dirty.  Note that we must make sure that all our vdevs are
769 * online when we do this, or else any vdevs that weren't present
770 * would be orphaned from our pool.  We are also going to issue a
771 * sysevent to update any watchers.
772 */
773int
774spa_change_guid(spa_t *spa)
775{
776	int error;
777	uint64_t guid;
778
779	mutex_enter(&spa->spa_vdev_top_lock);
780	mutex_enter(&spa_namespace_lock);
781	guid = spa_generate_guid(NULL);
782
783	error = dsl_sync_task(spa->spa_name, spa_change_guid_check,
784	    spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED);
785
786	if (error == 0) {
787		spa_config_sync(spa, B_FALSE, B_TRUE);
788		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_REGUID);
789	}
790
791	mutex_exit(&spa_namespace_lock);
792	mutex_exit(&spa->spa_vdev_top_lock);
793
794	return (error);
795}
796
797/*
798 * ==========================================================================
799 * SPA state manipulation (open/create/destroy/import/export)
800 * ==========================================================================
801 */
802
803static int
804spa_error_entry_compare(const void *a, const void *b)
805{
806	spa_error_entry_t *sa = (spa_error_entry_t *)a;
807	spa_error_entry_t *sb = (spa_error_entry_t *)b;
808	int ret;
809
810	ret = bcmp(&sa->se_bookmark, &sb->se_bookmark,
811	    sizeof (zbookmark_phys_t));
812
813	if (ret < 0)
814		return (-1);
815	else if (ret > 0)
816		return (1);
817	else
818		return (0);
819}
820
821/*
822 * Utility function which retrieves copies of the current logs and
823 * re-initializes them in the process.
824 */
825void
826spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub)
827{
828	ASSERT(MUTEX_HELD(&spa->spa_errlist_lock));
829
830	bcopy(&spa->spa_errlist_last, last, sizeof (avl_tree_t));
831	bcopy(&spa->spa_errlist_scrub, scrub, sizeof (avl_tree_t));
832
833	avl_create(&spa->spa_errlist_scrub,
834	    spa_error_entry_compare, sizeof (spa_error_entry_t),
835	    offsetof(spa_error_entry_t, se_avl));
836	avl_create(&spa->spa_errlist_last,
837	    spa_error_entry_compare, sizeof (spa_error_entry_t),
838	    offsetof(spa_error_entry_t, se_avl));
839}
840
841static void
842spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q)
843{
844	const zio_taskq_info_t *ztip = &zio_taskqs[t][q];
845	enum zti_modes mode = ztip->zti_mode;
846	uint_t value = ztip->zti_value;
847	uint_t count = ztip->zti_count;
848	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
849	char name[32];
850	uint_t flags = 0;
851	boolean_t batch = B_FALSE;
852
853	if (mode == ZTI_MODE_NULL) {
854		tqs->stqs_count = 0;
855		tqs->stqs_taskq = NULL;
856		return;
857	}
858
859	ASSERT3U(count, >, 0);
860
861	tqs->stqs_count = count;
862	tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP);
863
864	switch (mode) {
865	case ZTI_MODE_FIXED:
866		ASSERT3U(value, >=, 1);
867		value = MAX(value, 1);
868		break;
869
870	case ZTI_MODE_BATCH:
871		batch = B_TRUE;
872		flags |= TASKQ_THREADS_CPU_PCT;
873		value = zio_taskq_batch_pct;
874		break;
875
876	default:
877		panic("unrecognized mode for %s_%s taskq (%u:%u) in "
878		    "spa_activate()",
879		    zio_type_name[t], zio_taskq_types[q], mode, value);
880		break;
881	}
882
883	for (uint_t i = 0; i < count; i++) {
884		taskq_t *tq;
885
886		if (count > 1) {
887			(void) snprintf(name, sizeof (name), "%s_%s_%u",
888			    zio_type_name[t], zio_taskq_types[q], i);
889		} else {
890			(void) snprintf(name, sizeof (name), "%s_%s",
891			    zio_type_name[t], zio_taskq_types[q]);
892		}
893
894		if (zio_taskq_sysdc && spa->spa_proc != &p0) {
895			if (batch)
896				flags |= TASKQ_DC_BATCH;
897
898			tq = taskq_create_sysdc(name, value, 50, INT_MAX,
899			    spa->spa_proc, zio_taskq_basedc, flags);
900		} else {
901			pri_t pri = maxclsyspri;
902			/*
903			 * The write issue taskq can be extremely CPU
904			 * intensive.  Run it at slightly lower priority
905			 * than the other taskqs.
906			 */
907			if (t == ZIO_TYPE_WRITE && q == ZIO_TASKQ_ISSUE)
908				pri--;
909
910			tq = taskq_create_proc(name, value, pri, 50,
911			    INT_MAX, spa->spa_proc, flags);
912		}
913
914		tqs->stqs_taskq[i] = tq;
915	}
916}
917
918static void
919spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q)
920{
921	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
922
923	if (tqs->stqs_taskq == NULL) {
924		ASSERT0(tqs->stqs_count);
925		return;
926	}
927
928	for (uint_t i = 0; i < tqs->stqs_count; i++) {
929		ASSERT3P(tqs->stqs_taskq[i], !=, NULL);
930		taskq_destroy(tqs->stqs_taskq[i]);
931	}
932
933	kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *));
934	tqs->stqs_taskq = NULL;
935}
936
937/*
938 * Dispatch a task to the appropriate taskq for the ZFS I/O type and priority.
939 * Note that a type may have multiple discrete taskqs to avoid lock contention
940 * on the taskq itself. In that case we choose which taskq at random by using
941 * the low bits of gethrtime().
942 */
943void
944spa_taskq_dispatch_ent(spa_t *spa, zio_type_t t, zio_taskq_type_t q,
945    task_func_t *func, void *arg, uint_t flags, taskq_ent_t *ent)
946{
947	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
948	taskq_t *tq;
949
950	ASSERT3P(tqs->stqs_taskq, !=, NULL);
951	ASSERT3U(tqs->stqs_count, !=, 0);
952
953	if (tqs->stqs_count == 1) {
954		tq = tqs->stqs_taskq[0];
955	} else {
956		tq = tqs->stqs_taskq[gethrtime() % tqs->stqs_count];
957	}
958
959	taskq_dispatch_ent(tq, func, arg, flags, ent);
960}
961
962static void
963spa_create_zio_taskqs(spa_t *spa)
964{
965	for (int t = 0; t < ZIO_TYPES; t++) {
966		for (int q = 0; q < ZIO_TASKQ_TYPES; q++) {
967			spa_taskqs_init(spa, t, q);
968		}
969	}
970}
971
972#ifdef _KERNEL
973static void
974spa_thread(void *arg)
975{
976	callb_cpr_t cprinfo;
977
978	spa_t *spa = arg;
979	user_t *pu = PTOU(curproc);
980
981	CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr,
982	    spa->spa_name);
983
984	ASSERT(curproc != &p0);
985	(void) snprintf(pu->u_psargs, sizeof (pu->u_psargs),
986	    "zpool-%s", spa->spa_name);
987	(void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm));
988
989	/* bind this thread to the requested psrset */
990	if (zio_taskq_psrset_bind != PS_NONE) {
991		pool_lock();
992		mutex_enter(&cpu_lock);
993		mutex_enter(&pidlock);
994		mutex_enter(&curproc->p_lock);
995
996		if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind,
997		    0, NULL, NULL) == 0)  {
998			curthread->t_bind_pset = zio_taskq_psrset_bind;
999		} else {
1000			cmn_err(CE_WARN,
1001			    "Couldn't bind process for zfs pool \"%s\" to "
1002			    "pset %d\n", spa->spa_name, zio_taskq_psrset_bind);
1003		}
1004
1005		mutex_exit(&curproc->p_lock);
1006		mutex_exit(&pidlock);
1007		mutex_exit(&cpu_lock);
1008		pool_unlock();
1009	}
1010
1011	if (zio_taskq_sysdc) {
1012		sysdc_thread_enter(curthread, 100, 0);
1013	}
1014
1015	spa->spa_proc = curproc;
1016	spa->spa_did = curthread->t_did;
1017
1018	spa_create_zio_taskqs(spa);
1019
1020	mutex_enter(&spa->spa_proc_lock);
1021	ASSERT(spa->spa_proc_state == SPA_PROC_CREATED);
1022
1023	spa->spa_proc_state = SPA_PROC_ACTIVE;
1024	cv_broadcast(&spa->spa_proc_cv);
1025
1026	CALLB_CPR_SAFE_BEGIN(&cprinfo);
1027	while (spa->spa_proc_state == SPA_PROC_ACTIVE)
1028		cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock);
1029	CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock);
1030
1031	ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE);
1032	spa->spa_proc_state = SPA_PROC_GONE;
1033	spa->spa_proc = &p0;
1034	cv_broadcast(&spa->spa_proc_cv);
1035	CALLB_CPR_EXIT(&cprinfo);	/* drops spa_proc_lock */
1036
1037	mutex_enter(&curproc->p_lock);
1038	lwp_exit();
1039}
1040#endif
1041
1042/*
1043 * Activate an uninitialized pool.
1044 */
1045static void
1046spa_activate(spa_t *spa, int mode)
1047{
1048	ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED);
1049
1050	spa->spa_state = POOL_STATE_ACTIVE;
1051	spa->spa_mode = mode;
1052
1053	spa->spa_normal_class = metaslab_class_create(spa, zfs_metaslab_ops);
1054	spa->spa_log_class = metaslab_class_create(spa, zfs_metaslab_ops);
1055
1056	/* Try to create a covering process */
1057	mutex_enter(&spa->spa_proc_lock);
1058	ASSERT(spa->spa_proc_state == SPA_PROC_NONE);
1059	ASSERT(spa->spa_proc == &p0);
1060	spa->spa_did = 0;
1061
1062	/* Only create a process if we're going to be around a while. */
1063	if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) {
1064		if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri,
1065		    NULL, 0) == 0) {
1066			spa->spa_proc_state = SPA_PROC_CREATED;
1067			while (spa->spa_proc_state == SPA_PROC_CREATED) {
1068				cv_wait(&spa->spa_proc_cv,
1069				    &spa->spa_proc_lock);
1070			}
1071			ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE);
1072			ASSERT(spa->spa_proc != &p0);
1073			ASSERT(spa->spa_did != 0);
1074		} else {
1075#ifdef _KERNEL
1076			cmn_err(CE_WARN,
1077			    "Couldn't create process for zfs pool \"%s\"\n",
1078			    spa->spa_name);
1079#endif
1080		}
1081	}
1082	mutex_exit(&spa->spa_proc_lock);
1083
1084	/* If we didn't create a process, we need to create our taskqs. */
1085	if (spa->spa_proc == &p0) {
1086		spa_create_zio_taskqs(spa);
1087	}
1088
1089	list_create(&spa->spa_config_dirty_list, sizeof (vdev_t),
1090	    offsetof(vdev_t, vdev_config_dirty_node));
1091	list_create(&spa->spa_evicting_os_list, sizeof (objset_t),
1092	    offsetof(objset_t, os_evicting_node));
1093	list_create(&spa->spa_state_dirty_list, sizeof (vdev_t),
1094	    offsetof(vdev_t, vdev_state_dirty_node));
1095
1096	txg_list_create(&spa->spa_vdev_txg_list, spa,
1097	    offsetof(struct vdev, vdev_txg_node));
1098
1099	avl_create(&spa->spa_errlist_scrub,
1100	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1101	    offsetof(spa_error_entry_t, se_avl));
1102	avl_create(&spa->spa_errlist_last,
1103	    spa_error_entry_compare, sizeof (spa_error_entry_t),
1104	    offsetof(spa_error_entry_t, se_avl));
1105}
1106
1107/*
1108 * Opposite of spa_activate().
1109 */
1110static void
1111spa_deactivate(spa_t *spa)
1112{
1113	ASSERT(spa->spa_sync_on == B_FALSE);
1114	ASSERT(spa->spa_dsl_pool == NULL);
1115	ASSERT(spa->spa_root_vdev == NULL);
1116	ASSERT(spa->spa_async_zio_root == NULL);
1117	ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED);
1118
1119	spa_evicting_os_wait(spa);
1120
1121	txg_list_destroy(&spa->spa_vdev_txg_list);
1122
1123	list_destroy(&spa->spa_config_dirty_list);
1124	list_destroy(&spa->spa_evicting_os_list);
1125	list_destroy(&spa->spa_state_dirty_list);
1126
1127	for (int t = 0; t < ZIO_TYPES; t++) {
1128		for (int q = 0; q < ZIO_TASKQ_TYPES; q++) {
1129			spa_taskqs_fini(spa, t, q);
1130		}
1131	}
1132
1133	metaslab_class_destroy(spa->spa_normal_class);
1134	spa->spa_normal_class = NULL;
1135
1136	metaslab_class_destroy(spa->spa_log_class);
1137	spa->spa_log_class = NULL;
1138
1139	/*
1140	 * If this was part of an import or the open otherwise failed, we may
1141	 * still have errors left in the queues.  Empty them just in case.
1142	 */
1143	spa_errlog_drain(spa);
1144
1145	avl_destroy(&spa->spa_errlist_scrub);
1146	avl_destroy(&spa->spa_errlist_last);
1147
1148	spa->spa_state = POOL_STATE_UNINITIALIZED;
1149
1150	mutex_enter(&spa->spa_proc_lock);
1151	if (spa->spa_proc_state != SPA_PROC_NONE) {
1152		ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE);
1153		spa->spa_proc_state = SPA_PROC_DEACTIVATE;
1154		cv_broadcast(&spa->spa_proc_cv);
1155		while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) {
1156			ASSERT(spa->spa_proc != &p0);
1157			cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock);
1158		}
1159		ASSERT(spa->spa_proc_state == SPA_PROC_GONE);
1160		spa->spa_proc_state = SPA_PROC_NONE;
1161	}
1162	ASSERT(spa->spa_proc == &p0);
1163	mutex_exit(&spa->spa_proc_lock);
1164
1165	/*
1166	 * We want to make sure spa_thread() has actually exited the ZFS
1167	 * module, so that the module can't be unloaded out from underneath
1168	 * it.
1169	 */
1170	if (spa->spa_did != 0) {
1171		thread_join(spa->spa_did);
1172		spa->spa_did = 0;
1173	}
1174}
1175
1176/*
1177 * Verify a pool configuration, and construct the vdev tree appropriately.  This
1178 * will create all the necessary vdevs in the appropriate layout, with each vdev
1179 * in the CLOSED state.  This will prep the pool before open/creation/import.
1180 * All vdev validation is done by the vdev_alloc() routine.
1181 */
1182static int
1183spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent,
1184    uint_t id, int atype)
1185{
1186	nvlist_t **child;
1187	uint_t children;
1188	int error;
1189
1190	if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0)
1191		return (error);
1192
1193	if ((*vdp)->vdev_ops->vdev_op_leaf)
1194		return (0);
1195
1196	error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1197	    &child, &children);
1198
1199	if (error == ENOENT)
1200		return (0);
1201
1202	if (error) {
1203		vdev_free(*vdp);
1204		*vdp = NULL;
1205		return (SET_ERROR(EINVAL));
1206	}
1207
1208	for (int c = 0; c < children; c++) {
1209		vdev_t *vd;
1210		if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c,
1211		    atype)) != 0) {
1212			vdev_free(*vdp);
1213			*vdp = NULL;
1214			return (error);
1215		}
1216	}
1217
1218	ASSERT(*vdp != NULL);
1219
1220	return (0);
1221}
1222
1223/*
1224 * Opposite of spa_load().
1225 */
1226static void
1227spa_unload(spa_t *spa)
1228{
1229	int i;
1230
1231	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1232
1233	/*
1234	 * Stop async tasks.
1235	 */
1236	spa_async_suspend(spa);
1237
1238	/*
1239	 * Stop syncing.
1240	 */
1241	if (spa->spa_sync_on) {
1242		txg_sync_stop(spa->spa_dsl_pool);
1243		spa->spa_sync_on = B_FALSE;
1244	}
1245
1246	/*
1247	 * Even though vdev_free() also calls vdev_metaslab_fini, we need
1248	 * to call it earlier, before we wait for async i/o to complete.
1249	 * This ensures that there is no async metaslab prefetching, by
1250	 * calling taskq_wait(mg_taskq).
1251	 */
1252	if (spa->spa_root_vdev != NULL) {
1253		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
1254		for (int c = 0; c < spa->spa_root_vdev->vdev_children; c++)
1255			vdev_metaslab_fini(spa->spa_root_vdev->vdev_child[c]);
1256		spa_config_exit(spa, SCL_ALL, FTAG);
1257	}
1258
1259	/*
1260	 * Wait for any outstanding async I/O to complete.
1261	 */
1262	if (spa->spa_async_zio_root != NULL) {
1263		for (int i = 0; i < max_ncpus; i++)
1264			(void) zio_wait(spa->spa_async_zio_root[i]);
1265		kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *));
1266		spa->spa_async_zio_root = NULL;
1267	}
1268
1269	bpobj_close(&spa->spa_deferred_bpobj);
1270
1271	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
1272
1273	/*
1274	 * Close all vdevs.
1275	 */
1276	if (spa->spa_root_vdev)
1277		vdev_free(spa->spa_root_vdev);
1278	ASSERT(spa->spa_root_vdev == NULL);
1279
1280	/*
1281	 * Close the dsl pool.
1282	 */
1283	if (spa->spa_dsl_pool) {
1284		dsl_pool_close(spa->spa_dsl_pool);
1285		spa->spa_dsl_pool = NULL;
1286		spa->spa_meta_objset = NULL;
1287	}
1288
1289	ddt_unload(spa);
1290
1291	/*
1292	 * Drop and purge level 2 cache
1293	 */
1294	spa_l2cache_drop(spa);
1295
1296	for (i = 0; i < spa->spa_spares.sav_count; i++)
1297		vdev_free(spa->spa_spares.sav_vdevs[i]);
1298	if (spa->spa_spares.sav_vdevs) {
1299		kmem_free(spa->spa_spares.sav_vdevs,
1300		    spa->spa_spares.sav_count * sizeof (void *));
1301		spa->spa_spares.sav_vdevs = NULL;
1302	}
1303	if (spa->spa_spares.sav_config) {
1304		nvlist_free(spa->spa_spares.sav_config);
1305		spa->spa_spares.sav_config = NULL;
1306	}
1307	spa->spa_spares.sav_count = 0;
1308
1309	for (i = 0; i < spa->spa_l2cache.sav_count; i++) {
1310		vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]);
1311		vdev_free(spa->spa_l2cache.sav_vdevs[i]);
1312	}
1313	if (spa->spa_l2cache.sav_vdevs) {
1314		kmem_free(spa->spa_l2cache.sav_vdevs,
1315		    spa->spa_l2cache.sav_count * sizeof (void *));
1316		spa->spa_l2cache.sav_vdevs = NULL;
1317	}
1318	if (spa->spa_l2cache.sav_config) {
1319		nvlist_free(spa->spa_l2cache.sav_config);
1320		spa->spa_l2cache.sav_config = NULL;
1321	}
1322	spa->spa_l2cache.sav_count = 0;
1323
1324	spa->spa_async_suspended = 0;
1325
1326	if (spa->spa_comment != NULL) {
1327		spa_strfree(spa->spa_comment);
1328		spa->spa_comment = NULL;
1329	}
1330
1331	spa_config_exit(spa, SCL_ALL, FTAG);
1332}
1333
1334/*
1335 * Load (or re-load) the current list of vdevs describing the active spares for
1336 * this pool.  When this is called, we have some form of basic information in
1337 * 'spa_spares.sav_config'.  We parse this into vdevs, try to open them, and
1338 * then re-generate a more complete list including status information.
1339 */
1340static void
1341spa_load_spares(spa_t *spa)
1342{
1343	nvlist_t **spares;
1344	uint_t nspares;
1345	int i;
1346	vdev_t *vd, *tvd;
1347
1348	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
1349
1350	/*
1351	 * First, close and free any existing spare vdevs.
1352	 */
1353	for (i = 0; i < spa->spa_spares.sav_count; i++) {
1354		vd = spa->spa_spares.sav_vdevs[i];
1355
1356		/* Undo the call to spa_activate() below */
1357		if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid,
1358		    B_FALSE)) != NULL && tvd->vdev_isspare)
1359			spa_spare_remove(tvd);
1360		vdev_close(vd);
1361		vdev_free(vd);
1362	}
1363
1364	if (spa->spa_spares.sav_vdevs)
1365		kmem_free(spa->spa_spares.sav_vdevs,
1366		    spa->spa_spares.sav_count * sizeof (void *));
1367
1368	if (spa->spa_spares.sav_config == NULL)
1369		nspares = 0;
1370	else
1371		VERIFY(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
1372		    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0);
1373
1374	spa->spa_spares.sav_count = (int)nspares;
1375	spa->spa_spares.sav_vdevs = NULL;
1376
1377	if (nspares == 0)
1378		return;
1379
1380	/*
1381	 * Construct the array of vdevs, opening them to get status in the
1382	 * process.   For each spare, there is potentially two different vdev_t
1383	 * structures associated with it: one in the list of spares (used only
1384	 * for basic validation purposes) and one in the active vdev
1385	 * configuration (if it's spared in).  During this phase we open and
1386	 * validate each vdev on the spare list.  If the vdev also exists in the
1387	 * active configuration, then we also mark this vdev as an active spare.
1388	 */
1389	spa->spa_spares.sav_vdevs = kmem_alloc(nspares * sizeof (void *),
1390	    KM_SLEEP);
1391	for (i = 0; i < spa->spa_spares.sav_count; i++) {
1392		VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0,
1393		    VDEV_ALLOC_SPARE) == 0);
1394		ASSERT(vd != NULL);
1395
1396		spa->spa_spares.sav_vdevs[i] = vd;
1397
1398		if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid,
1399		    B_FALSE)) != NULL) {
1400			if (!tvd->vdev_isspare)
1401				spa_spare_add(tvd);
1402
1403			/*
1404			 * We only mark the spare active if we were successfully
1405			 * able to load the vdev.  Otherwise, importing a pool
1406			 * with a bad active spare would result in strange
1407			 * behavior, because multiple pool would think the spare
1408			 * is actively in use.
1409			 *
1410			 * There is a vulnerability here to an equally bizarre
1411			 * circumstance, where a dead active spare is later
1412			 * brought back to life (onlined or otherwise).  Given
1413			 * the rarity of this scenario, and the extra complexity
1414			 * it adds, we ignore the possibility.
1415			 */
1416			if (!vdev_is_dead(tvd))
1417				spa_spare_activate(tvd);
1418		}
1419
1420		vd->vdev_top = vd;
1421		vd->vdev_aux = &spa->spa_spares;
1422
1423		if (vdev_open(vd) != 0)
1424			continue;
1425
1426		if (vdev_validate_aux(vd) == 0)
1427			spa_spare_add(vd);
1428	}
1429
1430	/*
1431	 * Recompute the stashed list of spares, with status information
1432	 * this time.
1433	 */
1434	VERIFY(nvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES,
1435	    DATA_TYPE_NVLIST_ARRAY) == 0);
1436
1437	spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *),
1438	    KM_SLEEP);
1439	for (i = 0; i < spa->spa_spares.sav_count; i++)
1440		spares[i] = vdev_config_generate(spa,
1441		    spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE);
1442	VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config,
1443	    ZPOOL_CONFIG_SPARES, spares, spa->spa_spares.sav_count) == 0);
1444	for (i = 0; i < spa->spa_spares.sav_count; i++)
1445		nvlist_free(spares[i]);
1446	kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *));
1447}
1448
1449/*
1450 * Load (or re-load) the current list of vdevs describing the active l2cache for
1451 * this pool.  When this is called, we have some form of basic information in
1452 * 'spa_l2cache.sav_config'.  We parse this into vdevs, try to open them, and
1453 * then re-generate a more complete list including status information.
1454 * Devices which are already active have their details maintained, and are
1455 * not re-opened.
1456 */
1457static void
1458spa_load_l2cache(spa_t *spa)
1459{
1460	nvlist_t **l2cache;
1461	uint_t nl2cache;
1462	int i, j, oldnvdevs;
1463	uint64_t guid;
1464	vdev_t *vd, **oldvdevs, **newvdevs;
1465	spa_aux_vdev_t *sav = &spa->spa_l2cache;
1466
1467	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
1468
1469	if (sav->sav_config != NULL) {
1470		VERIFY(nvlist_lookup_nvlist_array(sav->sav_config,
1471		    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0);
1472		newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP);
1473	} else {
1474		nl2cache = 0;
1475		newvdevs = NULL;
1476	}
1477
1478	oldvdevs = sav->sav_vdevs;
1479	oldnvdevs = sav->sav_count;
1480	sav->sav_vdevs = NULL;
1481	sav->sav_count = 0;
1482
1483	/*
1484	 * Process new nvlist of vdevs.
1485	 */
1486	for (i = 0; i < nl2cache; i++) {
1487		VERIFY(nvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID,
1488		    &guid) == 0);
1489
1490		newvdevs[i] = NULL;
1491		for (j = 0; j < oldnvdevs; j++) {
1492			vd = oldvdevs[j];
1493			if (vd != NULL && guid == vd->vdev_guid) {
1494				/*
1495				 * Retain previous vdev for add/remove ops.
1496				 */
1497				newvdevs[i] = vd;
1498				oldvdevs[j] = NULL;
1499				break;
1500			}
1501		}
1502
1503		if (newvdevs[i] == NULL) {
1504			/*
1505			 * Create new vdev
1506			 */
1507			VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0,
1508			    VDEV_ALLOC_L2CACHE) == 0);
1509			ASSERT(vd != NULL);
1510			newvdevs[i] = vd;
1511
1512			/*
1513			 * Commit this vdev as an l2cache device,
1514			 * even if it fails to open.
1515			 */
1516			spa_l2cache_add(vd);
1517
1518			vd->vdev_top = vd;
1519			vd->vdev_aux = sav;
1520
1521			spa_l2cache_activate(vd);
1522
1523			if (vdev_open(vd) != 0)
1524				continue;
1525
1526			(void) vdev_validate_aux(vd);
1527
1528			if (!vdev_is_dead(vd))
1529				l2arc_add_vdev(spa, vd);
1530		}
1531	}
1532
1533	/*
1534	 * Purge vdevs that were dropped
1535	 */
1536	for (i = 0; i < oldnvdevs; i++) {
1537		uint64_t pool;
1538
1539		vd = oldvdevs[i];
1540		if (vd != NULL) {
1541			ASSERT(vd->vdev_isl2cache);
1542
1543			if (spa_l2cache_exists(vd->vdev_guid, &pool) &&
1544			    pool != 0ULL && l2arc_vdev_present(vd))
1545				l2arc_remove_vdev(vd);
1546			vdev_clear_stats(vd);
1547			vdev_free(vd);
1548		}
1549	}
1550
1551	if (oldvdevs)
1552		kmem_free(oldvdevs, oldnvdevs * sizeof (void *));
1553
1554	if (sav->sav_config == NULL)
1555		goto out;
1556
1557	sav->sav_vdevs = newvdevs;
1558	sav->sav_count = (int)nl2cache;
1559
1560	/*
1561	 * Recompute the stashed list of l2cache devices, with status
1562	 * information this time.
1563	 */
1564	VERIFY(nvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE,
1565	    DATA_TYPE_NVLIST_ARRAY) == 0);
1566
1567	l2cache = kmem_alloc(sav->sav_count * sizeof (void *), KM_SLEEP);
1568	for (i = 0; i < sav->sav_count; i++)
1569		l2cache[i] = vdev_config_generate(spa,
1570		    sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE);
1571	VERIFY(nvlist_add_nvlist_array(sav->sav_config,
1572	    ZPOOL_CONFIG_L2CACHE, l2cache, sav->sav_count) == 0);
1573out:
1574	for (i = 0; i < sav->sav_count; i++)
1575		nvlist_free(l2cache[i]);
1576	if (sav->sav_count)
1577		kmem_free(l2cache, sav->sav_count * sizeof (void *));
1578}
1579
1580static int
1581load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value)
1582{
1583	dmu_buf_t *db;
1584	char *packed = NULL;
1585	size_t nvsize = 0;
1586	int error;
1587	*value = NULL;
1588
1589	error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db);
1590	if (error != 0)
1591		return (error);
1592
1593	nvsize = *(uint64_t *)db->db_data;
1594	dmu_buf_rele(db, FTAG);
1595
1596	packed = kmem_alloc(nvsize, KM_SLEEP);
1597	error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed,
1598	    DMU_READ_PREFETCH);
1599	if (error == 0)
1600		error = nvlist_unpack(packed, nvsize, value, 0);
1601	kmem_free(packed, nvsize);
1602
1603	return (error);
1604}
1605
1606/*
1607 * Checks to see if the given vdev could not be opened, in which case we post a
1608 * sysevent to notify the autoreplace code that the device has been removed.
1609 */
1610static void
1611spa_check_removed(vdev_t *vd)
1612{
1613	for (int c = 0; c < vd->vdev_children; c++)
1614		spa_check_removed(vd->vdev_child[c]);
1615
1616	if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) &&
1617	    !vd->vdev_ishole) {
1618		zfs_post_autoreplace(vd->vdev_spa, vd);
1619		spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_CHECK);
1620	}
1621}
1622
1623static void
1624spa_config_valid_zaps(vdev_t *vd, vdev_t *mvd)
1625{
1626	ASSERT3U(vd->vdev_children, ==, mvd->vdev_children);
1627
1628	vd->vdev_top_zap = mvd->vdev_top_zap;
1629	vd->vdev_leaf_zap = mvd->vdev_leaf_zap;
1630
1631	for (uint64_t i = 0; i < vd->vdev_children; i++) {
1632		spa_config_valid_zaps(vd->vdev_child[i], mvd->vdev_child[i]);
1633	}
1634}
1635
1636/*
1637 * Validate the current config against the MOS config
1638 */
1639static boolean_t
1640spa_config_valid(spa_t *spa, nvlist_t *config)
1641{
1642	vdev_t *mrvd, *rvd = spa->spa_root_vdev;
1643	nvlist_t *nv;
1644
1645	VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nv) == 0);
1646
1647	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
1648	VERIFY(spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD) == 0);
1649
1650	ASSERT3U(rvd->vdev_children, ==, mrvd->vdev_children);
1651
1652	/*
1653	 * If we're doing a normal import, then build up any additional
1654	 * diagnostic information about missing devices in this config.
1655	 * We'll pass this up to the user for further processing.
1656	 */
1657	if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) {
1658		nvlist_t **child, *nv;
1659		uint64_t idx = 0;
1660
1661		child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t **),
1662		    KM_SLEEP);
1663		VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) == 0);
1664
1665		for (int c = 0; c < rvd->vdev_children; c++) {
1666			vdev_t *tvd = rvd->vdev_child[c];
1667			vdev_t *mtvd  = mrvd->vdev_child[c];
1668
1669			if (tvd->vdev_ops == &vdev_missing_ops &&
1670			    mtvd->vdev_ops != &vdev_missing_ops &&
1671			    mtvd->vdev_islog)
1672				child[idx++] = vdev_config_generate(spa, mtvd,
1673				    B_FALSE, 0);
1674		}
1675
1676		if (idx) {
1677			VERIFY(nvlist_add_nvlist_array(nv,
1678			    ZPOOL_CONFIG_CHILDREN, child, idx) == 0);
1679			VERIFY(nvlist_add_nvlist(spa->spa_load_info,
1680			    ZPOOL_CONFIG_MISSING_DEVICES, nv) == 0);
1681
1682			for (int i = 0; i < idx; i++)
1683				nvlist_free(child[i]);
1684		}
1685		nvlist_free(nv);
1686		kmem_free(child, rvd->vdev_children * sizeof (char **));
1687	}
1688
1689	/*
1690	 * Compare the root vdev tree with the information we have
1691	 * from the MOS config (mrvd). Check each top-level vdev
1692	 * with the corresponding MOS config top-level (mtvd).
1693	 */
1694	for (int c = 0; c < rvd->vdev_children; c++) {
1695		vdev_t *tvd = rvd->vdev_child[c];
1696		vdev_t *mtvd  = mrvd->vdev_child[c];
1697
1698		/*
1699		 * Resolve any "missing" vdevs in the current configuration.
1700		 * If we find that the MOS config has more accurate information
1701		 * about the top-level vdev then use that vdev instead.
1702		 */
1703		if (tvd->vdev_ops == &vdev_missing_ops &&
1704		    mtvd->vdev_ops != &vdev_missing_ops) {
1705
1706			if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG))
1707				continue;
1708
1709			/*
1710			 * Device specific actions.
1711			 */
1712			if (mtvd->vdev_islog) {
1713				spa_set_log_state(spa, SPA_LOG_CLEAR);
1714			} else {
1715				/*
1716				 * XXX - once we have 'readonly' pool
1717				 * support we should be able to handle
1718				 * missing data devices by transitioning
1719				 * the pool to readonly.
1720				 */
1721				continue;
1722			}
1723
1724			/*
1725			 * Swap the missing vdev with the data we were
1726			 * able to obtain from the MOS config.
1727			 */
1728			vdev_remove_child(rvd, tvd);
1729			vdev_remove_child(mrvd, mtvd);
1730
1731			vdev_add_child(rvd, mtvd);
1732			vdev_add_child(mrvd, tvd);
1733
1734			spa_config_exit(spa, SCL_ALL, FTAG);
1735			vdev_load(mtvd);
1736			spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
1737
1738			vdev_reopen(rvd);
1739		} else {
1740			if (mtvd->vdev_islog) {
1741				/*
1742				 * Load the slog device's state from the MOS
1743				 * config since it's possible that the label
1744				 * does not contain the most up-to-date
1745				 * information.
1746				 */
1747				vdev_load_log_state(tvd, mtvd);
1748				vdev_reopen(tvd);
1749			}
1750
1751			/*
1752			 * Per-vdev ZAP info is stored exclusively in the MOS.
1753			 */
1754			spa_config_valid_zaps(tvd, mtvd);
1755		}
1756	}
1757
1758	vdev_free(mrvd);
1759	spa_config_exit(spa, SCL_ALL, FTAG);
1760
1761	/*
1762	 * Ensure we were able to validate the config.
1763	 */
1764	return (rvd->vdev_guid_sum == spa->spa_uberblock.ub_guid_sum);
1765}
1766
1767/*
1768 * Check for missing log devices
1769 */
1770static boolean_t
1771spa_check_logs(spa_t *spa)
1772{
1773	boolean_t rv = B_FALSE;
1774	dsl_pool_t *dp = spa_get_dsl(spa);
1775
1776	switch (spa->spa_log_state) {
1777	case SPA_LOG_MISSING:
1778		/* need to recheck in case slog has been restored */
1779	case SPA_LOG_UNKNOWN:
1780		rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
1781		    zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0);
1782		if (rv)
1783			spa_set_log_state(spa, SPA_LOG_MISSING);
1784		break;
1785	}
1786	return (rv);
1787}
1788
1789static boolean_t
1790spa_passivate_log(spa_t *spa)
1791{
1792	vdev_t *rvd = spa->spa_root_vdev;
1793	boolean_t slog_found = B_FALSE;
1794
1795	ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER));
1796
1797	if (!spa_has_slogs(spa))
1798		return (B_FALSE);
1799
1800	for (int c = 0; c < rvd->vdev_children; c++) {
1801		vdev_t *tvd = rvd->vdev_child[c];
1802		metaslab_group_t *mg = tvd->vdev_mg;
1803
1804		if (tvd->vdev_islog) {
1805			metaslab_group_passivate(mg);
1806			slog_found = B_TRUE;
1807		}
1808	}
1809
1810	return (slog_found);
1811}
1812
1813static void
1814spa_activate_log(spa_t *spa)
1815{
1816	vdev_t *rvd = spa->spa_root_vdev;
1817
1818	ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER));
1819
1820	for (int c = 0; c < rvd->vdev_children; c++) {
1821		vdev_t *tvd = rvd->vdev_child[c];
1822		metaslab_group_t *mg = tvd->vdev_mg;
1823
1824		if (tvd->vdev_islog)
1825			metaslab_group_activate(mg);
1826	}
1827}
1828
1829int
1830spa_offline_log(spa_t *spa)
1831{
1832	int error;
1833
1834	error = dmu_objset_find(spa_name(spa), zil_vdev_offline,
1835	    NULL, DS_FIND_CHILDREN);
1836	if (error == 0) {
1837		/*
1838		 * We successfully offlined the log device, sync out the
1839		 * current txg so that the "stubby" block can be removed
1840		 * by zil_sync().
1841		 */
1842		txg_wait_synced(spa->spa_dsl_pool, 0);
1843	}
1844	return (error);
1845}
1846
1847static void
1848spa_aux_check_removed(spa_aux_vdev_t *sav)
1849{
1850	for (int i = 0; i < sav->sav_count; i++)
1851		spa_check_removed(sav->sav_vdevs[i]);
1852}
1853
1854void
1855spa_claim_notify(zio_t *zio)
1856{
1857	spa_t *spa = zio->io_spa;
1858
1859	if (zio->io_error)
1860		return;
1861
1862	mutex_enter(&spa->spa_props_lock);	/* any mutex will do */
1863	if (spa->spa_claim_max_txg < zio->io_bp->blk_birth)
1864		spa->spa_claim_max_txg = zio->io_bp->blk_birth;
1865	mutex_exit(&spa->spa_props_lock);
1866}
1867
1868typedef struct spa_load_error {
1869	uint64_t	sle_meta_count;
1870	uint64_t	sle_data_count;
1871} spa_load_error_t;
1872
1873static void
1874spa_load_verify_done(zio_t *zio)
1875{
1876	blkptr_t *bp = zio->io_bp;
1877	spa_load_error_t *sle = zio->io_private;
1878	dmu_object_type_t type = BP_GET_TYPE(bp);
1879	int error = zio->io_error;
1880	spa_t *spa = zio->io_spa;
1881
1882	abd_free(zio->io_abd);
1883	if (error) {
1884		if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) &&
1885		    type != DMU_OT_INTENT_LOG)
1886			atomic_inc_64(&sle->sle_meta_count);
1887		else
1888			atomic_inc_64(&sle->sle_data_count);
1889	}
1890
1891	mutex_enter(&spa->spa_scrub_lock);
1892	spa->spa_scrub_inflight--;
1893	cv_broadcast(&spa->spa_scrub_io_cv);
1894	mutex_exit(&spa->spa_scrub_lock);
1895}
1896
1897/*
1898 * Maximum number of concurrent scrub i/os to create while verifying
1899 * a pool while importing it.
1900 */
1901int spa_load_verify_maxinflight = 10000;
1902boolean_t spa_load_verify_metadata = B_TRUE;
1903boolean_t spa_load_verify_data = B_TRUE;
1904
1905/*ARGSUSED*/
1906static int
1907spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
1908    const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
1909{
1910	if (bp == NULL || BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp))
1911		return (0);
1912	/*
1913	 * Note: normally this routine will not be called if
1914	 * spa_load_verify_metadata is not set.  However, it may be useful
1915	 * to manually set the flag after the traversal has begun.
1916	 */
1917	if (!spa_load_verify_metadata)
1918		return (0);
1919	if (!BP_IS_METADATA(bp) && !spa_load_verify_data)
1920		return (0);
1921
1922	zio_t *rio = arg;
1923	size_t size = BP_GET_PSIZE(bp);
1924
1925	mutex_enter(&spa->spa_scrub_lock);
1926	while (spa->spa_scrub_inflight >= spa_load_verify_maxinflight)
1927		cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock);
1928	spa->spa_scrub_inflight++;
1929	mutex_exit(&spa->spa_scrub_lock);
1930
1931	zio_nowait(zio_read(rio, spa, bp, abd_alloc_for_io(size, B_FALSE), size,
1932	    spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB,
1933	    ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL |
1934	    ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb));
1935	return (0);
1936}
1937
1938/* ARGSUSED */
1939int
1940verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
1941{
1942	if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN)
1943		return (SET_ERROR(ENAMETOOLONG));
1944
1945	return (0);
1946}
1947
1948static int
1949spa_load_verify(spa_t *spa)
1950{
1951	zio_t *rio;
1952	spa_load_error_t sle = { 0 };
1953	zpool_rewind_policy_t policy;
1954	boolean_t verify_ok = B_FALSE;
1955	int error = 0;
1956
1957	zpool_get_rewind_policy(spa->spa_config, &policy);
1958
1959	if (policy.zrp_request & ZPOOL_NEVER_REWIND)
1960		return (0);
1961
1962	dsl_pool_config_enter(spa->spa_dsl_pool, FTAG);
1963	error = dmu_objset_find_dp(spa->spa_dsl_pool,
1964	    spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL,
1965	    DS_FIND_CHILDREN);
1966	dsl_pool_config_exit(spa->spa_dsl_pool, FTAG);
1967	if (error != 0)
1968		return (error);
1969
1970	rio = zio_root(spa, NULL, &sle,
1971	    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE);
1972
1973	if (spa_load_verify_metadata) {
1974		error = traverse_pool(spa, spa->spa_verify_min_txg,
1975		    TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA,
1976		    spa_load_verify_cb, rio);
1977	}
1978
1979	(void) zio_wait(rio);
1980
1981	spa->spa_load_meta_errors = sle.sle_meta_count;
1982	spa->spa_load_data_errors = sle.sle_data_count;
1983
1984	if (!error && sle.sle_meta_count <= policy.zrp_maxmeta &&
1985	    sle.sle_data_count <= policy.zrp_maxdata) {
1986		int64_t loss = 0;
1987
1988		verify_ok = B_TRUE;
1989		spa->spa_load_txg = spa->spa_uberblock.ub_txg;
1990		spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp;
1991
1992		loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts;
1993		VERIFY(nvlist_add_uint64(spa->spa_load_info,
1994		    ZPOOL_CONFIG_LOAD_TIME, spa->spa_load_txg_ts) == 0);
1995		VERIFY(nvlist_add_int64(spa->spa_load_info,
1996		    ZPOOL_CONFIG_REWIND_TIME, loss) == 0);
1997		VERIFY(nvlist_add_uint64(spa->spa_load_info,
1998		    ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count) == 0);
1999	} else {
2000		spa->spa_load_max_txg = spa->spa_uberblock.ub_txg;
2001	}
2002
2003	if (error) {
2004		if (error != ENXIO && error != EIO)
2005			error = SET_ERROR(EIO);
2006		return (error);
2007	}
2008
2009	return (verify_ok ? 0 : EIO);
2010}
2011
2012/*
2013 * Find a value in the pool props object.
2014 */
2015static void
2016spa_prop_find(spa_t *spa, zpool_prop_t prop, uint64_t *val)
2017{
2018	(void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object,
2019	    zpool_prop_to_name(prop), sizeof (uint64_t), 1, val);
2020}
2021
2022/*
2023 * Find a value in the pool directory object.
2024 */
2025static int
2026spa_dir_prop(spa_t *spa, const char *name, uint64_t *val)
2027{
2028	return (zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
2029	    name, sizeof (uint64_t), 1, val));
2030}
2031
2032static int
2033spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err)
2034{
2035	vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux);
2036	return (err);
2037}
2038
2039/*
2040 * Fix up config after a partly-completed split.  This is done with the
2041 * ZPOOL_CONFIG_SPLIT nvlist.  Both the splitting pool and the split-off
2042 * pool have that entry in their config, but only the splitting one contains
2043 * a list of all the guids of the vdevs that are being split off.
2044 *
2045 * This function determines what to do with that list: either rejoin
2046 * all the disks to the pool, or complete the splitting process.  To attempt
2047 * the rejoin, each disk that is offlined is marked online again, and
2048 * we do a reopen() call.  If the vdev label for every disk that was
2049 * marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL)
2050 * then we call vdev_split() on each disk, and complete the split.
2051 *
2052 * Otherwise we leave the config alone, with all the vdevs in place in
2053 * the original pool.
2054 */
2055static void
2056spa_try_repair(spa_t *spa, nvlist_t *config)
2057{
2058	uint_t extracted;
2059	uint64_t *glist;
2060	uint_t i, gcount;
2061	nvlist_t *nvl;
2062	vdev_t **vd;
2063	boolean_t attempt_reopen;
2064
2065	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0)
2066		return;
2067
2068	/* check that the config is complete */
2069	if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST,
2070	    &glist, &gcount) != 0)
2071		return;
2072
2073	vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP);
2074
2075	/* attempt to online all the vdevs & validate */
2076	attempt_reopen = B_TRUE;
2077	for (i = 0; i < gcount; i++) {
2078		if (glist[i] == 0)	/* vdev is hole */
2079			continue;
2080
2081		vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE);
2082		if (vd[i] == NULL) {
2083			/*
2084			 * Don't bother attempting to reopen the disks;
2085			 * just do the split.
2086			 */
2087			attempt_reopen = B_FALSE;
2088		} else {
2089			/* attempt to re-online it */
2090			vd[i]->vdev_offline = B_FALSE;
2091		}
2092	}
2093
2094	if (attempt_reopen) {
2095		vdev_reopen(spa->spa_root_vdev);
2096
2097		/* check each device to see what state it's in */
2098		for (extracted = 0, i = 0; i < gcount; i++) {
2099			if (vd[i] != NULL &&
2100			    vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL)
2101				break;
2102			++extracted;
2103		}
2104	}
2105
2106	/*
2107	 * If every disk has been moved to the new pool, or if we never
2108	 * even attempted to look at them, then we split them off for
2109	 * good.
2110	 */
2111	if (!attempt_reopen || gcount == extracted) {
2112		for (i = 0; i < gcount; i++)
2113			if (vd[i] != NULL)
2114				vdev_split(vd[i]);
2115		vdev_reopen(spa->spa_root_vdev);
2116	}
2117
2118	kmem_free(vd, gcount * sizeof (vdev_t *));
2119}
2120
2121static int
2122spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type,
2123    boolean_t mosconfig)
2124{
2125	nvlist_t *config = spa->spa_config;
2126	char *ereport = FM_EREPORT_ZFS_POOL;
2127	char *comment;
2128	int error;
2129	uint64_t pool_guid;
2130	nvlist_t *nvl;
2131
2132	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid))
2133		return (SET_ERROR(EINVAL));
2134
2135	ASSERT(spa->spa_comment == NULL);
2136	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
2137		spa->spa_comment = spa_strdup(comment);
2138
2139	/*
2140	 * Versioning wasn't explicitly added to the label until later, so if
2141	 * it's not present treat it as the initial version.
2142	 */
2143	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
2144	    &spa->spa_ubsync.ub_version) != 0)
2145		spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL;
2146
2147	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG,
2148	    &spa->spa_config_txg);
2149
2150	if ((state == SPA_LOAD_IMPORT || state == SPA_LOAD_TRYIMPORT) &&
2151	    spa_guid_exists(pool_guid, 0)) {
2152		error = SET_ERROR(EEXIST);
2153	} else {
2154		spa->spa_config_guid = pool_guid;
2155
2156		if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT,
2157		    &nvl) == 0) {
2158			VERIFY(nvlist_dup(nvl, &spa->spa_config_splitting,
2159			    KM_SLEEP) == 0);
2160		}
2161
2162		nvlist_free(spa->spa_load_info);
2163		spa->spa_load_info = fnvlist_alloc();
2164
2165		gethrestime(&spa->spa_loaded_ts);
2166		error = spa_load_impl(spa, pool_guid, config, state, type,
2167		    mosconfig, &ereport);
2168	}
2169
2170	/*
2171	 * Don't count references from objsets that are already closed
2172	 * and are making their way through the eviction process.
2173	 */
2174	spa_evicting_os_wait(spa);
2175	spa->spa_minref = refcount_count(&spa->spa_refcount);
2176	if (error) {
2177		if (error != EEXIST) {
2178			spa->spa_loaded_ts.tv_sec = 0;
2179			spa->spa_loaded_ts.tv_nsec = 0;
2180		}
2181		if (error != EBADF) {
2182			zfs_ereport_post(ereport, spa, NULL, NULL, 0, 0);
2183		}
2184	}
2185	spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE;
2186	spa->spa_ena = 0;
2187
2188	return (error);
2189}
2190
2191/*
2192 * Count the number of per-vdev ZAPs associated with all of the vdevs in the
2193 * vdev tree rooted in the given vd, and ensure that each ZAP is present in the
2194 * spa's per-vdev ZAP list.
2195 */
2196static uint64_t
2197vdev_count_verify_zaps(vdev_t *vd)
2198{
2199	spa_t *spa = vd->vdev_spa;
2200	uint64_t total = 0;
2201	if (vd->vdev_top_zap != 0) {
2202		total++;
2203		ASSERT0(zap_lookup_int(spa->spa_meta_objset,
2204		    spa->spa_all_vdev_zaps, vd->vdev_top_zap));
2205	}
2206	if (vd->vdev_leaf_zap != 0) {
2207		total++;
2208		ASSERT0(zap_lookup_int(spa->spa_meta_objset,
2209		    spa->spa_all_vdev_zaps, vd->vdev_leaf_zap));
2210	}
2211
2212	for (uint64_t i = 0; i < vd->vdev_children; i++) {
2213		total += vdev_count_verify_zaps(vd->vdev_child[i]);
2214	}
2215
2216	return (total);
2217}
2218
2219/*
2220 * Load an existing storage pool, using the pool's builtin spa_config as a
2221 * source of configuration information.
2222 */
2223static int
2224spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config,
2225    spa_load_state_t state, spa_import_type_t type, boolean_t mosconfig,
2226    char **ereport)
2227{
2228	int error = 0;
2229	nvlist_t *nvroot = NULL;
2230	nvlist_t *label;
2231	vdev_t *rvd;
2232	uberblock_t *ub = &spa->spa_uberblock;
2233	uint64_t children, config_cache_txg = spa->spa_config_txg;
2234	int orig_mode = spa->spa_mode;
2235	int parse;
2236	uint64_t obj;
2237	boolean_t missing_feat_write = B_FALSE;
2238
2239	/*
2240	 * If this is an untrusted config, access the pool in read-only mode.
2241	 * This prevents things like resilvering recently removed devices.
2242	 */
2243	if (!mosconfig)
2244		spa->spa_mode = FREAD;
2245
2246	ASSERT(MUTEX_HELD(&spa_namespace_lock));
2247
2248	spa->spa_load_state = state;
2249
2250	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvroot))
2251		return (SET_ERROR(EINVAL));
2252
2253	parse = (type == SPA_IMPORT_EXISTING ?
2254	    VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT);
2255
2256	/*
2257	 * Create "The Godfather" zio to hold all async IOs
2258	 */
2259	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
2260	    KM_SLEEP);
2261	for (int i = 0; i < max_ncpus; i++) {
2262		spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
2263		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
2264		    ZIO_FLAG_GODFATHER);
2265	}
2266
2267	/*
2268	 * Parse the configuration into a vdev tree.  We explicitly set the
2269	 * value that will be returned by spa_version() since parsing the
2270	 * configuration requires knowing the version number.
2271	 */
2272	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
2273	error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, parse);
2274	spa_config_exit(spa, SCL_ALL, FTAG);
2275
2276	if (error != 0)
2277		return (error);
2278
2279	ASSERT(spa->spa_root_vdev == rvd);
2280	ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT);
2281	ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT);
2282
2283	if (type != SPA_IMPORT_ASSEMBLE) {
2284		ASSERT(spa_guid(spa) == pool_guid);
2285	}
2286
2287	/*
2288	 * Try to open all vdevs, loading each label in the process.
2289	 */
2290	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
2291	error = vdev_open(rvd);
2292	spa_config_exit(spa, SCL_ALL, FTAG);
2293	if (error != 0)
2294		return (error);
2295
2296	/*
2297	 * We need to validate the vdev labels against the configuration that
2298	 * we have in hand, which is dependent on the setting of mosconfig. If
2299	 * mosconfig is true then we're validating the vdev labels based on
2300	 * that config.  Otherwise, we're validating against the cached config
2301	 * (zpool.cache) that was read when we loaded the zfs module, and then
2302	 * later we will recursively call spa_load() and validate against
2303	 * the vdev config.
2304	 *
2305	 * If we're assembling a new pool that's been split off from an
2306	 * existing pool, the labels haven't yet been updated so we skip
2307	 * validation for now.
2308	 */
2309	if (type != SPA_IMPORT_ASSEMBLE) {
2310		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
2311		error = vdev_validate(rvd, mosconfig);
2312		spa_config_exit(spa, SCL_ALL, FTAG);
2313
2314		if (error != 0)
2315			return (error);
2316
2317		if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN)
2318			return (SET_ERROR(ENXIO));
2319	}
2320
2321	/*
2322	 * Find the best uberblock.
2323	 */
2324	vdev_uberblock_load(rvd, ub, &label);
2325
2326	/*
2327	 * If we weren't able to find a single valid uberblock, return failure.
2328	 */
2329	if (ub->ub_txg == 0) {
2330		nvlist_free(label);
2331		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO));
2332	}
2333
2334	/*
2335	 * If the pool has an unsupported version we can't open it.
2336	 */
2337	if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) {
2338		nvlist_free(label);
2339		return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP));
2340	}
2341
2342	if (ub->ub_version >= SPA_VERSION_FEATURES) {
2343		nvlist_t *features;
2344
2345		/*
2346		 * If we weren't able to find what's necessary for reading the
2347		 * MOS in the label, return failure.
2348		 */
2349		if (label == NULL || nvlist_lookup_nvlist(label,
2350		    ZPOOL_CONFIG_FEATURES_FOR_READ, &features) != 0) {
2351			nvlist_free(label);
2352			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
2353			    ENXIO));
2354		}
2355
2356		/*
2357		 * Update our in-core representation with the definitive values
2358		 * from the label.
2359		 */
2360		nvlist_free(spa->spa_label_features);
2361		VERIFY(nvlist_dup(features, &spa->spa_label_features, 0) == 0);
2362	}
2363
2364	nvlist_free(label);
2365
2366	/*
2367	 * Look through entries in the label nvlist's features_for_read. If
2368	 * there is a feature listed there which we don't understand then we
2369	 * cannot open a pool.
2370	 */
2371	if (ub->ub_version >= SPA_VERSION_FEATURES) {
2372		nvlist_t *unsup_feat;
2373
2374		VERIFY(nvlist_alloc(&unsup_feat, NV_UNIQUE_NAME, KM_SLEEP) ==
2375		    0);
2376
2377		for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features,
2378		    NULL); nvp != NULL;
2379		    nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) {
2380			if (!zfeature_is_supported(nvpair_name(nvp))) {
2381				VERIFY(nvlist_add_string(unsup_feat,
2382				    nvpair_name(nvp), "") == 0);
2383			}
2384		}
2385
2386		if (!nvlist_empty(unsup_feat)) {
2387			VERIFY(nvlist_add_nvlist(spa->spa_load_info,
2388			    ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat) == 0);
2389			nvlist_free(unsup_feat);
2390			return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT,
2391			    ENOTSUP));
2392		}
2393
2394		nvlist_free(unsup_feat);
2395	}
2396
2397	/*
2398	 * If the vdev guid sum doesn't match the uberblock, we have an
2399	 * incomplete configuration.  We first check to see if the pool
2400	 * is aware of the complete config (i.e ZPOOL_CONFIG_VDEV_CHILDREN).
2401	 * If it is, defer the vdev_guid_sum check till later so we
2402	 * can handle missing vdevs.
2403	 */
2404	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VDEV_CHILDREN,
2405	    &children) != 0 && mosconfig && type != SPA_IMPORT_ASSEMBLE &&
2406	    rvd->vdev_guid_sum != ub->ub_guid_sum)
2407		return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO));
2408
2409	if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) {
2410		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
2411		spa_try_repair(spa, config);
2412		spa_config_exit(spa, SCL_ALL, FTAG);
2413		nvlist_free(spa->spa_config_splitting);
2414		spa->spa_config_splitting = NULL;
2415	}
2416
2417	/*
2418	 * Initialize internal SPA structures.
2419	 */
2420	spa->spa_state = POOL_STATE_ACTIVE;
2421	spa->spa_ubsync = spa->spa_uberblock;
2422	spa->spa_verify_min_txg = spa->spa_extreme_rewind ?
2423	    TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1;
2424	spa->spa_first_txg = spa->spa_last_ubsync_txg ?
2425	    spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1;
2426	spa->spa_claim_max_txg = spa->spa_first_txg;
2427	spa->spa_prev_software_version = ub->ub_software_version;
2428
2429	error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool);
2430	if (error)
2431		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2432	spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset;
2433
2434	if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object) != 0)
2435		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2436
2437	if (spa_version(spa) >= SPA_VERSION_FEATURES) {
2438		boolean_t missing_feat_read = B_FALSE;
2439		nvlist_t *unsup_feat, *enabled_feat;
2440
2441		if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ,
2442		    &spa->spa_feat_for_read_obj) != 0) {
2443			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2444		}
2445
2446		if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE,
2447		    &spa->spa_feat_for_write_obj) != 0) {
2448			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2449		}
2450
2451		if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS,
2452		    &spa->spa_feat_desc_obj) != 0) {
2453			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2454		}
2455
2456		enabled_feat = fnvlist_alloc();
2457		unsup_feat = fnvlist_alloc();
2458
2459		if (!spa_features_check(spa, B_FALSE,
2460		    unsup_feat, enabled_feat))
2461			missing_feat_read = B_TRUE;
2462
2463		if (spa_writeable(spa) || state == SPA_LOAD_TRYIMPORT) {
2464			if (!spa_features_check(spa, B_TRUE,
2465			    unsup_feat, enabled_feat)) {
2466				missing_feat_write = B_TRUE;
2467			}
2468		}
2469
2470		fnvlist_add_nvlist(spa->spa_load_info,
2471		    ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat);
2472
2473		if (!nvlist_empty(unsup_feat)) {
2474			fnvlist_add_nvlist(spa->spa_load_info,
2475			    ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat);
2476		}
2477
2478		fnvlist_free(enabled_feat);
2479		fnvlist_free(unsup_feat);
2480
2481		if (!missing_feat_read) {
2482			fnvlist_add_boolean(spa->spa_load_info,
2483			    ZPOOL_CONFIG_CAN_RDONLY);
2484		}
2485
2486		/*
2487		 * If the state is SPA_LOAD_TRYIMPORT, our objective is
2488		 * twofold: to determine whether the pool is available for
2489		 * import in read-write mode and (if it is not) whether the
2490		 * pool is available for import in read-only mode. If the pool
2491		 * is available for import in read-write mode, it is displayed
2492		 * as available in userland; if it is not available for import
2493		 * in read-only mode, it is displayed as unavailable in
2494		 * userland. If the pool is available for import in read-only
2495		 * mode but not read-write mode, it is displayed as unavailable
2496		 * in userland with a special note that the pool is actually
2497		 * available for open in read-only mode.
2498		 *
2499		 * As a result, if the state is SPA_LOAD_TRYIMPORT and we are
2500		 * missing a feature for write, we must first determine whether
2501		 * the pool can be opened read-only before returning to
2502		 * userland in order to know whether to display the
2503		 * abovementioned note.
2504		 */
2505		if (missing_feat_read || (missing_feat_write &&
2506		    spa_writeable(spa))) {
2507			return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT,
2508			    ENOTSUP));
2509		}
2510
2511		/*
2512		 * Load refcounts for ZFS features from disk into an in-memory
2513		 * cache during SPA initialization.
2514		 */
2515		for (spa_feature_t i = 0; i < SPA_FEATURES; i++) {
2516			uint64_t refcount;
2517
2518			error = feature_get_refcount_from_disk(spa,
2519			    &spa_feature_table[i], &refcount);
2520			if (error == 0) {
2521				spa->spa_feat_refcount_cache[i] = refcount;
2522			} else if (error == ENOTSUP) {
2523				spa->spa_feat_refcount_cache[i] =
2524				    SPA_FEATURE_DISABLED;
2525			} else {
2526				return (spa_vdev_err(rvd,
2527				    VDEV_AUX_CORRUPT_DATA, EIO));
2528			}
2529		}
2530	}
2531
2532	if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) {
2533		if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG,
2534		    &spa->spa_feat_enabled_txg_obj) != 0)
2535			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2536	}
2537
2538	spa->spa_is_initializing = B_TRUE;
2539	error = dsl_pool_open(spa->spa_dsl_pool);
2540	spa->spa_is_initializing = B_FALSE;
2541	if (error != 0)
2542		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2543
2544	if (!mosconfig) {
2545		uint64_t hostid;
2546		nvlist_t *policy = NULL, *nvconfig;
2547
2548		if (load_nvlist(spa, spa->spa_config_object, &nvconfig) != 0)
2549			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2550
2551		if (!spa_is_root(spa) && nvlist_lookup_uint64(nvconfig,
2552		    ZPOOL_CONFIG_HOSTID, &hostid) == 0) {
2553			char *hostname;
2554			unsigned long myhostid = 0;
2555
2556			VERIFY(nvlist_lookup_string(nvconfig,
2557			    ZPOOL_CONFIG_HOSTNAME, &hostname) == 0);
2558
2559#ifdef	_KERNEL
2560			myhostid = zone_get_hostid(NULL);
2561#else	/* _KERNEL */
2562			/*
2563			 * We're emulating the system's hostid in userland, so
2564			 * we can't use zone_get_hostid().
2565			 */
2566			(void) ddi_strtoul(hw_serial, NULL, 10, &myhostid);
2567#endif	/* _KERNEL */
2568			if (hostid != 0 && myhostid != 0 &&
2569			    hostid != myhostid) {
2570				nvlist_free(nvconfig);
2571				cmn_err(CE_WARN, "pool '%s' could not be "
2572				    "loaded as it was last accessed by "
2573				    "another system (host: %s hostid: 0x%lx). "
2574				    "See: http://illumos.org/msg/ZFS-8000-EY",
2575				    spa_name(spa), hostname,
2576				    (unsigned long)hostid);
2577				return (SET_ERROR(EBADF));
2578			}
2579		}
2580		if (nvlist_lookup_nvlist(spa->spa_config,
2581		    ZPOOL_REWIND_POLICY, &policy) == 0)
2582			VERIFY(nvlist_add_nvlist(nvconfig,
2583			    ZPOOL_REWIND_POLICY, policy) == 0);
2584
2585		spa_config_set(spa, nvconfig);
2586		spa_unload(spa);
2587		spa_deactivate(spa);
2588		spa_activate(spa, orig_mode);
2589
2590		return (spa_load(spa, state, SPA_IMPORT_EXISTING, B_TRUE));
2591	}
2592
2593	/* Grab the secret checksum salt from the MOS. */
2594	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
2595	    DMU_POOL_CHECKSUM_SALT, 1,
2596	    sizeof (spa->spa_cksum_salt.zcs_bytes),
2597	    spa->spa_cksum_salt.zcs_bytes);
2598	if (error == ENOENT) {
2599		/* Generate a new salt for subsequent use */
2600		(void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes,
2601		    sizeof (spa->spa_cksum_salt.zcs_bytes));
2602	} else if (error != 0) {
2603		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2604	}
2605
2606	if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj) != 0)
2607		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2608	error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj);
2609	if (error != 0)
2610		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2611
2612	/*
2613	 * Load the bit that tells us to use the new accounting function
2614	 * (raid-z deflation).  If we have an older pool, this will not
2615	 * be present.
2616	 */
2617	error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate);
2618	if (error != 0 && error != ENOENT)
2619		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2620
2621	error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION,
2622	    &spa->spa_creation_version);
2623	if (error != 0 && error != ENOENT)
2624		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2625
2626	/*
2627	 * Load the persistent error log.  If we have an older pool, this will
2628	 * not be present.
2629	 */
2630	error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last);
2631	if (error != 0 && error != ENOENT)
2632		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2633
2634	error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB,
2635	    &spa->spa_errlog_scrub);
2636	if (error != 0 && error != ENOENT)
2637		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2638
2639	/*
2640	 * Load the history object.  If we have an older pool, this
2641	 * will not be present.
2642	 */
2643	error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history);
2644	if (error != 0 && error != ENOENT)
2645		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2646
2647	/*
2648	 * Load the per-vdev ZAP map. If we have an older pool, this will not
2649	 * be present; in this case, defer its creation to a later time to
2650	 * avoid dirtying the MOS this early / out of sync context. See
2651	 * spa_sync_config_object.
2652	 */
2653
2654	/* The sentinel is only available in the MOS config. */
2655	nvlist_t *mos_config;
2656	if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0)
2657		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2658
2659	error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP,
2660	    &spa->spa_all_vdev_zaps);
2661
2662	if (error == ENOENT) {
2663		VERIFY(!nvlist_exists(mos_config,
2664		    ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS));
2665		spa->spa_avz_action = AVZ_ACTION_INITIALIZE;
2666		ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev));
2667	} else if (error != 0) {
2668		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2669	} else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) {
2670		/*
2671		 * An older version of ZFS overwrote the sentinel value, so
2672		 * we have orphaned per-vdev ZAPs in the MOS. Defer their
2673		 * destruction to later; see spa_sync_config_object.
2674		 */
2675		spa->spa_avz_action = AVZ_ACTION_DESTROY;
2676		/*
2677		 * We're assuming that no vdevs have had their ZAPs created
2678		 * before this. Better be sure of it.
2679		 */
2680		ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev));
2681	}
2682	nvlist_free(mos_config);
2683
2684	/*
2685	 * If we're assembling the pool from the split-off vdevs of
2686	 * an existing pool, we don't want to attach the spares & cache
2687	 * devices.
2688	 */
2689
2690	/*
2691	 * Load any hot spares for this pool.
2692	 */
2693	error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object);
2694	if (error != 0 && error != ENOENT)
2695		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2696	if (error == 0 && type != SPA_IMPORT_ASSEMBLE) {
2697		ASSERT(spa_version(spa) >= SPA_VERSION_SPARES);
2698		if (load_nvlist(spa, spa->spa_spares.sav_object,
2699		    &spa->spa_spares.sav_config) != 0)
2700			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2701
2702		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
2703		spa_load_spares(spa);
2704		spa_config_exit(spa, SCL_ALL, FTAG);
2705	} else if (error == 0) {
2706		spa->spa_spares.sav_sync = B_TRUE;
2707	}
2708
2709	/*
2710	 * Load any level 2 ARC devices for this pool.
2711	 */
2712	error = spa_dir_prop(spa, DMU_POOL_L2CACHE,
2713	    &spa->spa_l2cache.sav_object);
2714	if (error != 0 && error != ENOENT)
2715		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2716	if (error == 0 && type != SPA_IMPORT_ASSEMBLE) {
2717		ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE);
2718		if (load_nvlist(spa, spa->spa_l2cache.sav_object,
2719		    &spa->spa_l2cache.sav_config) != 0)
2720			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2721
2722		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
2723		spa_load_l2cache(spa);
2724		spa_config_exit(spa, SCL_ALL, FTAG);
2725	} else if (error == 0) {
2726		spa->spa_l2cache.sav_sync = B_TRUE;
2727	}
2728
2729	spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);
2730
2731	error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object);
2732	if (error && error != ENOENT)
2733		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2734
2735	if (error == 0) {
2736		uint64_t autoreplace;
2737
2738		spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs);
2739		spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace);
2740		spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation);
2741		spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode);
2742		spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand);
2743		spa_prop_find(spa, ZPOOL_PROP_BOOTSIZE, &spa->spa_bootsize);
2744		spa_prop_find(spa, ZPOOL_PROP_DEDUPDITTO,
2745		    &spa->spa_dedup_ditto);
2746
2747		spa->spa_autoreplace = (autoreplace != 0);
2748	}
2749
2750	/*
2751	 * If the 'autoreplace' property is set, then post a resource notifying
2752	 * the ZFS DE that it should not issue any faults for unopenable
2753	 * devices.  We also iterate over the vdevs, and post a sysevent for any
2754	 * unopenable vdevs so that the normal autoreplace handler can take
2755	 * over.
2756	 */
2757	if (spa->spa_autoreplace && state != SPA_LOAD_TRYIMPORT) {
2758		spa_check_removed(spa->spa_root_vdev);
2759		/*
2760		 * For the import case, this is done in spa_import(), because
2761		 * at this point we're using the spare definitions from
2762		 * the MOS config, not necessarily from the userland config.
2763		 */
2764		if (state != SPA_LOAD_IMPORT) {
2765			spa_aux_check_removed(&spa->spa_spares);
2766			spa_aux_check_removed(&spa->spa_l2cache);
2767		}
2768	}
2769
2770	/*
2771	 * Load the vdev state for all toplevel vdevs.
2772	 */
2773	vdev_load(rvd);
2774
2775	/*
2776	 * Propagate the leaf DTLs we just loaded all the way up the tree.
2777	 */
2778	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
2779	vdev_dtl_reassess(rvd, 0, 0, B_FALSE);
2780	spa_config_exit(spa, SCL_ALL, FTAG);
2781
2782	/*
2783	 * Load the DDTs (dedup tables).
2784	 */
2785	error = ddt_load(spa);
2786	if (error != 0)
2787		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2788
2789	spa_update_dspace(spa);
2790
2791	/*
2792	 * Validate the config, using the MOS config to fill in any
2793	 * information which might be missing.  If we fail to validate
2794	 * the config then declare the pool unfit for use. If we're
2795	 * assembling a pool from a split, the log is not transferred
2796	 * over.
2797	 */
2798	if (type != SPA_IMPORT_ASSEMBLE) {
2799		nvlist_t *nvconfig;
2800
2801		if (load_nvlist(spa, spa->spa_config_object, &nvconfig) != 0)
2802			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
2803
2804		if (!spa_config_valid(spa, nvconfig)) {
2805			nvlist_free(nvconfig);
2806			return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM,
2807			    ENXIO));
2808		}
2809		nvlist_free(nvconfig);
2810
2811		/*
2812		 * Now that we've validated the config, check the state of the
2813		 * root vdev.  If it can't be opened, it indicates one or
2814		 * more toplevel vdevs are faulted.
2815		 */
2816		if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN)
2817			return (SET_ERROR(ENXIO));
2818
2819		if (spa_writeable(spa) && spa_check_logs(spa)) {
2820			*ereport = FM_EREPORT_ZFS_LOG_REPLAY;
2821			return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG, ENXIO));
2822		}
2823	}
2824
2825	if (missing_feat_write) {
2826		ASSERT(state == SPA_LOAD_TRYIMPORT);
2827
2828		/*
2829		 * At this point, we know that we can open the pool in
2830		 * read-only mode but not read-write mode. We now have enough
2831		 * information and can return to userland.
2832		 */
2833		return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, ENOTSUP));
2834	}
2835
2836	/*
2837	 * We've successfully opened the pool, verify that we're ready
2838	 * to start pushing transactions.
2839	 */
2840	if (state != SPA_LOAD_TRYIMPORT) {
2841		if (error = spa_load_verify(spa))
2842			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA,
2843			    error));
2844	}
2845
2846	if (spa_writeable(spa) && (state == SPA_LOAD_RECOVER ||
2847	    spa->spa_load_max_txg == UINT64_MAX)) {
2848		dmu_tx_t *tx;
2849		int need_update = B_FALSE;
2850		dsl_pool_t *dp = spa_get_dsl(spa);
2851
2852		ASSERT(state != SPA_LOAD_TRYIMPORT);
2853
2854		/*
2855		 * Claim log blocks that haven't been committed yet.
2856		 * This must all happen in a single txg.
2857		 * Note: spa_claim_max_txg is updated by spa_claim_notify(),
2858		 * invoked from zil_claim_log_block()'s i/o done callback.
2859		 * Price of rollback is that we abandon the log.
2860		 */
2861		spa->spa_claiming = B_TRUE;
2862
2863		tx = dmu_tx_create_assigned(dp, spa_first_txg(spa));
2864		(void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
2865		    zil_claim, tx, DS_FIND_CHILDREN);
2866		dmu_tx_commit(tx);
2867
2868		spa->spa_claiming = B_FALSE;
2869
2870		spa_set_log_state(spa, SPA_LOG_GOOD);
2871		spa->spa_sync_on = B_TRUE;
2872		txg_sync_start(spa->spa_dsl_pool);
2873
2874		/*
2875		 * Wait for all claims to sync.  We sync up to the highest
2876		 * claimed log block birth time so that claimed log blocks
2877		 * don't appear to be from the future.  spa_claim_max_txg
2878		 * will have been set for us by either zil_check_log_chain()
2879		 * (invoked from spa_check_logs()) or zil_claim() above.
2880		 */
2881		txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg);
2882
2883		/*
2884		 * If the config cache is stale, or we have uninitialized
2885		 * metaslabs (see spa_vdev_add()), then update the config.
2886		 *
2887		 * If this is a verbatim import, trust the current
2888		 * in-core spa_config and update the disk labels.
2889		 */
2890		if (config_cache_txg != spa->spa_config_txg ||
2891		    state == SPA_LOAD_IMPORT ||
2892		    state == SPA_LOAD_RECOVER ||
2893		    (spa->spa_import_flags & ZFS_IMPORT_VERBATIM))
2894			need_update = B_TRUE;
2895
2896		for (int c = 0; c < rvd->vdev_children; c++)
2897			if (rvd->vdev_child[c]->vdev_ms_array == 0)
2898				need_update = B_TRUE;
2899
2900		/*
2901		 * Update the config cache asychronously in case we're the
2902		 * root pool, in which case the config cache isn't writable yet.
2903		 */
2904		if (need_update)
2905			spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
2906
2907		/*
2908		 * Check all DTLs to see if anything needs resilvering.
2909		 */
2910		if (!dsl_scan_resilvering(spa->spa_dsl_pool) &&
2911		    vdev_resilver_needed(rvd, NULL, NULL))
2912			spa_async_request(spa, SPA_ASYNC_RESILVER);
2913
2914		/*
2915		 * Log the fact that we booted up (so that we can detect if
2916		 * we rebooted in the middle of an operation).
2917		 */
2918		spa_history_log_version(spa, "open");
2919
2920		/*
2921		 * Delete any inconsistent datasets.
2922		 */
2923		(void) dmu_objset_find(spa_name(spa),
2924		    dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN);
2925
2926		/*
2927		 * Clean up any stale temporary dataset userrefs.
2928		 */
2929		dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool);
2930	}
2931
2932	return (0);
2933}
2934
2935static int
2936spa_load_retry(spa_t *spa, spa_load_state_t state, int mosconfig)
2937{
2938	int mode = spa->spa_mode;
2939
2940	spa_unload(spa);
2941	spa_deactivate(spa);
2942
2943	spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1;
2944
2945	spa_activate(spa, mode);
2946	spa_async_suspend(spa);
2947
2948	return (spa_load(spa, state, SPA_IMPORT_EXISTING, mosconfig));
2949}
2950
2951/*
2952 * If spa_load() fails this function will try loading prior txg's. If
2953 * 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool
2954 * will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this
2955 * function will not rewind the pool and will return the same error as
2956 * spa_load().
2957 */
2958static int
2959spa_load_best(spa_t *spa, spa_load_state_t state, int mosconfig,
2960    uint64_t max_request, int rewind_flags)
2961{
2962	nvlist_t *loadinfo = NULL;
2963	nvlist_t *config = NULL;
2964	int load_error, rewind_error;
2965	uint64_t safe_rewind_txg;
2966	uint64_t min_txg;
2967
2968	if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) {
2969		spa->spa_load_max_txg = spa->spa_load_txg;
2970		spa_set_log_state(spa, SPA_LOG_CLEAR);
2971	} else {
2972		spa->spa_load_max_txg = max_request;
2973		if (max_request != UINT64_MAX)
2974			spa->spa_extreme_rewind = B_TRUE;
2975	}
2976
2977	load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING,
2978	    mosconfig);
2979	if (load_error == 0)
2980		return (0);
2981
2982	if (spa->spa_root_vdev != NULL)
2983		config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
2984
2985	spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg;
2986	spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp;
2987
2988	if (rewind_flags & ZPOOL_NEVER_REWIND) {
2989		nvlist_free(config);
2990		return (load_error);
2991	}
2992
2993	if (state == SPA_LOAD_RECOVER) {
2994		/* Price of rolling back is discarding txgs, including log */
2995		spa_set_log_state(spa, SPA_LOG_CLEAR);
2996	} else {
2997		/*
2998		 * If we aren't rolling back save the load info from our first
2999		 * import attempt so that we can restore it after attempting
3000		 * to rewind.
3001		 */
3002		loadinfo = spa->spa_load_info;
3003		spa->spa_load_info = fnvlist_alloc();
3004	}
3005
3006	spa->spa_load_max_txg = spa->spa_last_ubsync_txg;
3007	safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE;
3008	min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ?
3009	    TXG_INITIAL : safe_rewind_txg;
3010
3011	/*
3012	 * Continue as long as we're finding errors, we're still within
3013	 * the acceptable rewind range, and we're still finding uberblocks
3014	 */
3015	while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg &&
3016	    spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) {
3017		if (spa->spa_load_max_txg < safe_rewind_txg)
3018			spa->spa_extreme_rewind = B_TRUE;
3019		rewind_error = spa_load_retry(spa, state, mosconfig);
3020	}
3021
3022	spa->spa_extreme_rewind = B_FALSE;
3023	spa->spa_load_max_txg = UINT64_MAX;
3024
3025	if (config && (rewind_error || state != SPA_LOAD_RECOVER))
3026		spa_config_set(spa, config);
3027
3028	if (state == SPA_LOAD_RECOVER) {
3029		ASSERT3P(loadinfo, ==, NULL);
3030		return (rewind_error);
3031	} else {
3032		/* Store the rewind info as part of the initial load info */
3033		fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO,
3034		    spa->spa_load_info);
3035
3036		/* Restore the initial load info */
3037		fnvlist_free(spa->spa_load_info);
3038		spa->spa_load_info = loadinfo;
3039
3040		return (load_error);
3041	}
3042}
3043
3044/*
3045 * Pool Open/Import
3046 *
3047 * The import case is identical to an open except that the configuration is sent
3048 * down from userland, instead of grabbed from the configuration cache.  For the
3049 * case of an open, the pool configuration will exist in the
3050 * POOL_STATE_UNINITIALIZED state.
3051 *
3052 * The stats information (gen/count/ustats) is used to gather vdev statistics at
3053 * the same time open the pool, without having to keep around the spa_t in some
3054 * ambiguous state.
3055 */
3056static int
3057spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t *nvpolicy,
3058    nvlist_t **config)
3059{
3060	spa_t *spa;
3061	spa_load_state_t state = SPA_LOAD_OPEN;
3062	int error;
3063	int locked = B_FALSE;
3064
3065	*spapp = NULL;
3066
3067	/*
3068	 * As disgusting as this is, we need to support recursive calls to this
3069	 * function because dsl_dir_open() is called during spa_load(), and ends
3070	 * up calling spa_open() again.  The real fix is to figure out how to
3071	 * avoid dsl_dir_open() calling this in the first place.
3072	 */
3073	if (mutex_owner(&spa_namespace_lock) != curthread) {
3074		mutex_enter(&spa_namespace_lock);
3075		locked = B_TRUE;
3076	}
3077
3078	if ((spa = spa_lookup(pool)) == NULL) {
3079		if (locked)
3080			mutex_exit(&spa_namespace_lock);
3081		return (SET_ERROR(ENOENT));
3082	}
3083
3084	if (spa->spa_state == POOL_STATE_UNINITIALIZED) {
3085		zpool_rewind_policy_t policy;
3086
3087		zpool_get_rewind_policy(nvpolicy ? nvpolicy : spa->spa_config,
3088		    &policy);
3089		if (policy.zrp_request & ZPOOL_DO_REWIND)
3090			state = SPA_LOAD_RECOVER;
3091
3092		spa_activate(spa, spa_mode_global);
3093
3094		if (state != SPA_LOAD_RECOVER)
3095			spa->spa_last_ubsync_txg = spa->spa_load_txg = 0;
3096
3097		error = spa_load_best(spa, state, B_FALSE, policy.zrp_txg,
3098		    policy.zrp_request);
3099
3100		if (error == EBADF) {
3101			/*
3102			 * If vdev_validate() returns failure (indicated by
3103			 * EBADF), it indicates that one of the vdevs indicates
3104			 * that the pool has been exported or destroyed.  If
3105			 * this is the case, the config cache is out of sync and
3106			 * we should remove the pool from the namespace.
3107			 */
3108			spa_unload(spa);
3109			spa_deactivate(spa);
3110			spa_config_sync(spa, B_TRUE, B_TRUE);
3111			spa_remove(spa);
3112			if (locked)
3113				mutex_exit(&spa_namespace_lock);
3114			return (SET_ERROR(ENOENT));
3115		}
3116
3117		if (error) {
3118			/*
3119			 * We can't open the pool, but we still have useful
3120			 * information: the state of each vdev after the
3121			 * attempted vdev_open().  Return this to the user.
3122			 */
3123			if (config != NULL && spa->spa_config) {
3124				VERIFY(nvlist_dup(spa->spa_config, config,
3125				    KM_SLEEP) == 0);
3126				VERIFY(nvlist_add_nvlist(*config,
3127				    ZPOOL_CONFIG_LOAD_INFO,
3128				    spa->spa_load_info) == 0);
3129			}
3130			spa_unload(spa);
3131			spa_deactivate(spa);
3132			spa->spa_last_open_failed = error;
3133			if (locked)
3134				mutex_exit(&spa_namespace_lock);
3135			*spapp = NULL;
3136			return (error);
3137		}
3138	}
3139
3140	spa_open_ref(spa, tag);
3141
3142	if (config != NULL)
3143		*config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
3144
3145	/*
3146	 * If we've recovered the pool, pass back any information we
3147	 * gathered while doing the load.
3148	 */
3149	if (state == SPA_LOAD_RECOVER) {
3150		VERIFY(nvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO,
3151		    spa->spa_load_info) == 0);
3152	}
3153
3154	if (locked) {
3155		spa->spa_last_open_failed = 0;
3156		spa->spa_last_ubsync_txg = 0;
3157		spa->spa_load_txg = 0;
3158		mutex_exit(&spa_namespace_lock);
3159	}
3160
3161	*spapp = spa;
3162
3163	return (0);
3164}
3165
3166int
3167spa_open_rewind(const char *name, spa_t **spapp, void *tag, nvlist_t *policy,
3168    nvlist_t **config)
3169{
3170	return (spa_open_common(name, spapp, tag, policy, config));
3171}
3172
3173int
3174spa_open(const char *name, spa_t **spapp, void *tag)
3175{
3176	return (spa_open_common(name, spapp, tag, NULL, NULL));
3177}
3178
3179/*
3180 * Lookup the given spa_t, incrementing the inject count in the process,
3181 * preventing it from being exported or destroyed.
3182 */
3183spa_t *
3184spa_inject_addref(char *name)
3185{
3186	spa_t *spa;
3187
3188	mutex_enter(&spa_namespace_lock);
3189	if ((spa = spa_lookup(name)) == NULL) {
3190		mutex_exit(&spa_namespace_lock);
3191		return (NULL);
3192	}
3193	spa->spa_inject_ref++;
3194	mutex_exit(&spa_namespace_lock);
3195
3196	return (spa);
3197}
3198
3199void
3200spa_inject_delref(spa_t *spa)
3201{
3202	mutex_enter(&spa_namespace_lock);
3203	spa->spa_inject_ref--;
3204	mutex_exit(&spa_namespace_lock);
3205}
3206
3207/*
3208 * Add spares device information to the nvlist.
3209 */
3210static void
3211spa_add_spares(spa_t *spa, nvlist_t *config)
3212{
3213	nvlist_t **spares;
3214	uint_t i, nspares;
3215	nvlist_t *nvroot;
3216	uint64_t guid;
3217	vdev_stat_t *vs;
3218	uint_t vsc;
3219	uint64_t pool;
3220
3221	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
3222
3223	if (spa->spa_spares.sav_count == 0)
3224		return;
3225
3226	VERIFY(nvlist_lookup_nvlist(config,
3227	    ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0);
3228	VERIFY(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
3229	    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0);
3230	if (nspares != 0) {
3231		VERIFY(nvlist_add_nvlist_array(nvroot,
3232		    ZPOOL_CONFIG_SPARES, spares, nspares) == 0);
3233		VERIFY(nvlist_lookup_nvlist_array(nvroot,
3234		    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0);
3235
3236		/*
3237		 * Go through and find any spares which have since been
3238		 * repurposed as an active spare.  If this is the case, update
3239		 * their status appropriately.
3240		 */
3241		for (i = 0; i < nspares; i++) {
3242			VERIFY(nvlist_lookup_uint64(spares[i],
3243			    ZPOOL_CONFIG_GUID, &guid) == 0);
3244			if (spa_spare_exists(guid, &pool, NULL) &&
3245			    pool != 0ULL) {
3246				VERIFY(nvlist_lookup_uint64_array(
3247				    spares[i], ZPOOL_CONFIG_VDEV_STATS,
3248				    (uint64_t **)&vs, &vsc) == 0);
3249				vs->vs_state = VDEV_STATE_CANT_OPEN;
3250				vs->vs_aux = VDEV_AUX_SPARED;
3251			}
3252		}
3253	}
3254}
3255
3256/*
3257 * Add l2cache device information to the nvlist, including vdev stats.
3258 */
3259static void
3260spa_add_l2cache(spa_t *spa, nvlist_t *config)
3261{
3262	nvlist_t **l2cache;
3263	uint_t i, j, nl2cache;
3264	nvlist_t *nvroot;
3265	uint64_t guid;
3266	vdev_t *vd;
3267	vdev_stat_t *vs;
3268	uint_t vsc;
3269
3270	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
3271
3272	if (spa->spa_l2cache.sav_count == 0)
3273		return;
3274
3275	VERIFY(nvlist_lookup_nvlist(config,
3276	    ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0);
3277	VERIFY(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config,
3278	    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0);
3279	if (nl2cache != 0) {
3280		VERIFY(nvlist_add_nvlist_array(nvroot,
3281		    ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0);
3282		VERIFY(nvlist_lookup_nvlist_array(nvroot,
3283		    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0);
3284
3285		/*
3286		 * Update level 2 cache device stats.
3287		 */
3288
3289		for (i = 0; i < nl2cache; i++) {
3290			VERIFY(nvlist_lookup_uint64(l2cache[i],
3291			    ZPOOL_CONFIG_GUID, &guid) == 0);
3292
3293			vd = NULL;
3294			for (j = 0; j < spa->spa_l2cache.sav_count; j++) {
3295				if (guid ==
3296				    spa->spa_l2cache.sav_vdevs[j]->vdev_guid) {
3297					vd = spa->spa_l2cache.sav_vdevs[j];
3298					break;
3299				}
3300			}
3301			ASSERT(vd != NULL);
3302
3303			VERIFY(nvlist_lookup_uint64_array(l2cache[i],
3304			    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc)
3305			    == 0);
3306			vdev_get_stats(vd, vs);
3307		}
3308	}
3309}
3310
3311static void
3312spa_add_feature_stats(spa_t *spa, nvlist_t *config)
3313{
3314	nvlist_t *features;
3315	zap_cursor_t zc;
3316	zap_attribute_t za;
3317
3318	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
3319	VERIFY(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP) == 0);
3320
3321	if (spa->spa_feat_for_read_obj != 0) {
3322		for (zap_cursor_init(&zc, spa->spa_meta_objset,
3323		    spa->spa_feat_for_read_obj);
3324		    zap_cursor_retrieve(&zc, &za) == 0;
3325		    zap_cursor_advance(&zc)) {
3326			ASSERT(za.za_integer_length == sizeof (uint64_t) &&
3327			    za.za_num_integers == 1);
3328			VERIFY3U(0, ==, nvlist_add_uint64(features, za.za_name,
3329			    za.za_first_integer));
3330		}
3331		zap_cursor_fini(&zc);
3332	}
3333
3334	if (spa->spa_feat_for_write_obj != 0) {
3335		for (zap_cursor_init(&zc, spa->spa_meta_objset,
3336		    spa->spa_feat_for_write_obj);
3337		    zap_cursor_retrieve(&zc, &za) == 0;
3338		    zap_cursor_advance(&zc)) {
3339			ASSERT(za.za_integer_length == sizeof (uint64_t) &&
3340			    za.za_num_integers == 1);
3341			VERIFY3U(0, ==, nvlist_add_uint64(features, za.za_name,
3342			    za.za_first_integer));
3343		}
3344		zap_cursor_fini(&zc);
3345	}
3346
3347	VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS,
3348	    features) == 0);
3349	nvlist_free(features);
3350}
3351
3352int
3353spa_get_stats(const char *name, nvlist_t **config,
3354    char *altroot, size_t buflen)
3355{
3356	int error;
3357	spa_t *spa;
3358
3359	*config = NULL;
3360	error = spa_open_common(name, &spa, FTAG, NULL, config);
3361
3362	if (spa != NULL) {
3363		/*
3364		 * This still leaves a window of inconsistency where the spares
3365		 * or l2cache devices could change and the config would be
3366		 * self-inconsistent.
3367		 */
3368		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
3369
3370		if (*config != NULL) {
3371			uint64_t loadtimes[2];
3372
3373			loadtimes[0] = spa->spa_loaded_ts.tv_sec;
3374			loadtimes[1] = spa->spa_loaded_ts.tv_nsec;
3375			VERIFY(nvlist_add_uint64_array(*config,
3376			    ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2) == 0);
3377
3378			VERIFY(nvlist_add_uint64(*config,
3379			    ZPOOL_CONFIG_ERRCOUNT,
3380			    spa_get_errlog_size(spa)) == 0);
3381
3382			if (spa_suspended(spa))
3383				VERIFY(nvlist_add_uint64(*config,
3384				    ZPOOL_CONFIG_SUSPENDED,
3385				    spa->spa_failmode) == 0);
3386
3387			spa_add_spares(spa, *config);
3388			spa_add_l2cache(spa, *config);
3389			spa_add_feature_stats(spa, *config);
3390		}
3391	}
3392
3393	/*
3394	 * We want to get the alternate root even for faulted pools, so we cheat
3395	 * and call spa_lookup() directly.
3396	 */
3397	if (altroot) {
3398		if (spa == NULL) {
3399			mutex_enter(&spa_namespace_lock);
3400			spa = spa_lookup(name);
3401			if (spa)
3402				spa_altroot(spa, altroot, buflen);
3403			else
3404				altroot[0] = '\0';
3405			spa = NULL;
3406			mutex_exit(&spa_namespace_lock);
3407		} else {
3408			spa_altroot(spa, altroot, buflen);
3409		}
3410	}
3411
3412	if (spa != NULL) {
3413		spa_config_exit(spa, SCL_CONFIG, FTAG);
3414		spa_close(spa, FTAG);
3415	}
3416
3417	return (error);
3418}
3419
3420/*
3421 * Validate that the auxiliary device array is well formed.  We must have an
3422 * array of nvlists, each which describes a valid leaf vdev.  If this is an
3423 * import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be
3424 * specified, as long as they are well-formed.
3425 */
3426static int
3427spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode,
3428    spa_aux_vdev_t *sav, const char *config, uint64_t version,
3429    vdev_labeltype_t label)
3430{
3431	nvlist_t **dev;
3432	uint_t i, ndev;
3433	vdev_t *vd;
3434	int error;
3435
3436	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
3437
3438	/*
3439	 * It's acceptable to have no devs specified.
3440	 */
3441	if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0)
3442		return (0);
3443
3444	if (ndev == 0)
3445		return (SET_ERROR(EINVAL));
3446
3447	/*
3448	 * Make sure the pool is formatted with a version that supports this
3449	 * device type.
3450	 */
3451	if (spa_version(spa) < version)
3452		return (SET_ERROR(ENOTSUP));
3453
3454	/*
3455	 * Set the pending device list so we correctly handle device in-use
3456	 * checking.
3457	 */
3458	sav->sav_pending = dev;
3459	sav->sav_npending = ndev;
3460
3461	for (i = 0; i < ndev; i++) {
3462		if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0,
3463		    mode)) != 0)
3464			goto out;
3465
3466		if (!vd->vdev_ops->vdev_op_leaf) {
3467			vdev_free(vd);
3468			error = SET_ERROR(EINVAL);
3469			goto out;
3470		}
3471
3472		/*
3473		 * The L2ARC currently only supports disk devices in
3474		 * kernel context.  For user-level testing, we allow it.
3475		 */
3476#ifdef _KERNEL
3477		if ((strcmp(config, ZPOOL_CONFIG_L2CACHE) == 0) &&
3478		    strcmp(vd->vdev_ops->vdev_op_type, VDEV_TYPE_DISK) != 0) {
3479			error = SET_ERROR(ENOTBLK);
3480			vdev_free(vd);
3481			goto out;
3482		}
3483#endif
3484		vd->vdev_top = vd;
3485
3486		if ((error = vdev_open(vd)) == 0 &&
3487		    (error = vdev_label_init(vd, crtxg, label)) == 0) {
3488			VERIFY(nvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID,
3489			    vd->vdev_guid) == 0);
3490		}
3491
3492		vdev_free(vd);
3493
3494		if (error &&
3495		    (mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE))
3496			goto out;
3497		else
3498			error = 0;
3499	}
3500
3501out:
3502	sav->sav_pending = NULL;
3503	sav->sav_npending = 0;
3504	return (error);
3505}
3506
3507static int
3508spa_validate_aux(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode)
3509{
3510	int error;
3511
3512	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
3513
3514	if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode,
3515	    &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES,
3516	    VDEV_LABEL_SPARE)) != 0) {
3517		return (error);
3518	}
3519
3520	return (spa_validate_aux_devs(spa, nvroot, crtxg, mode,
3521	    &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE,
3522	    VDEV_LABEL_L2CACHE));
3523}
3524
3525static void
3526spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs,
3527    const char *config)
3528{
3529	int i;
3530
3531	if (sav->sav_config != NULL) {
3532		nvlist_t **olddevs;
3533		uint_t oldndevs;
3534		nvlist_t **newdevs;
3535
3536		/*
3537		 * Generate new dev list by concatentating with the
3538		 * current dev list.
3539		 */
3540		VERIFY(nvlist_lookup_nvlist_array(sav->sav_config, config,
3541		    &olddevs, &oldndevs) == 0);
3542
3543		newdevs = kmem_alloc(sizeof (void *) *
3544		    (ndevs + oldndevs), KM_SLEEP);
3545		for (i = 0; i < oldndevs; i++)
3546			VERIFY(nvlist_dup(olddevs[i], &newdevs[i],
3547			    KM_SLEEP) == 0);
3548		for (i = 0; i < ndevs; i++)
3549			VERIFY(nvlist_dup(devs[i], &newdevs[i + oldndevs],
3550			    KM_SLEEP) == 0);
3551
3552		VERIFY(nvlist_remove(sav->sav_config, config,
3553		    DATA_TYPE_NVLIST_ARRAY) == 0);
3554
3555		VERIFY(nvlist_add_nvlist_array(sav->sav_config,
3556		    config, newdevs, ndevs + oldndevs) == 0);
3557		for (i = 0; i < oldndevs + ndevs; i++)
3558			nvlist_free(newdevs[i]);
3559		kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *));
3560	} else {
3561		/*
3562		 * Generate a new dev list.
3563		 */
3564		VERIFY(nvlist_alloc(&sav->sav_config, NV_UNIQUE_NAME,
3565		    KM_SLEEP) == 0);
3566		VERIFY(nvlist_add_nvlist_array(sav->sav_config, config,
3567		    devs, ndevs) == 0);
3568	}
3569}
3570
3571/*
3572 * Stop and drop level 2 ARC devices
3573 */
3574void
3575spa_l2cache_drop(spa_t *spa)
3576{
3577	vdev_t *vd;
3578	int i;
3579	spa_aux_vdev_t *sav = &spa->spa_l2cache;
3580
3581	for (i = 0; i < sav->sav_count; i++) {
3582		uint64_t pool;
3583
3584		vd = sav->sav_vdevs[i];
3585		ASSERT(vd != NULL);
3586
3587		if (spa_l2cache_exists(vd->vdev_guid, &pool) &&
3588		    pool != 0ULL && l2arc_vdev_present(vd))
3589			l2arc_remove_vdev(vd);
3590	}
3591}
3592
3593/*
3594 * Pool Creation
3595 */
3596int
3597spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,
3598    nvlist_t *zplprops)
3599{
3600	spa_t *spa;
3601	char *altroot = NULL;
3602	vdev_t *rvd;
3603	dsl_pool_t *dp;
3604	dmu_tx_t *tx;
3605	int error = 0;
3606	uint64_t txg = TXG_INITIAL;
3607	nvlist_t **spares, **l2cache;
3608	uint_t nspares, nl2cache;
3609	uint64_t version, obj;
3610	boolean_t has_features;
3611
3612	/*
3613	 * If this pool already exists, return failure.
3614	 */
3615	mutex_enter(&spa_namespace_lock);
3616	if (spa_lookup(pool) != NULL) {
3617		mutex_exit(&spa_namespace_lock);
3618		return (SET_ERROR(EEXIST));
3619	}
3620
3621	/*
3622	 * Allocate a new spa_t structure.
3623	 */
3624	(void) nvlist_lookup_string(props,
3625	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
3626	spa = spa_add(pool, NULL, altroot);
3627	spa_activate(spa, spa_mode_global);
3628
3629	if (props && (error = spa_prop_validate(spa, props))) {
3630		spa_deactivate(spa);
3631		spa_remove(spa);
3632		mutex_exit(&spa_namespace_lock);
3633		return (error);
3634	}
3635
3636	has_features = B_FALSE;
3637	for (nvpair_t *elem = nvlist_next_nvpair(props, NULL);
3638	    elem != NULL; elem = nvlist_next_nvpair(props, elem)) {
3639		if (zpool_prop_feature(nvpair_name(elem)))
3640			has_features = B_TRUE;
3641	}
3642
3643	if (has_features || nvlist_lookup_uint64(props,
3644	    zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) {
3645		version = SPA_VERSION;
3646	}
3647	ASSERT(SPA_VERSION_IS_SUPPORTED(version));
3648
3649	spa->spa_first_txg = txg;
3650	spa->spa_uberblock.ub_txg = txg - 1;
3651	spa->spa_uberblock.ub_version = version;
3652	spa->spa_ubsync = spa->spa_uberblock;
3653	spa->spa_load_state = SPA_LOAD_CREATE;
3654
3655	/*
3656	 * Create "The Godfather" zio to hold all async IOs
3657	 */
3658	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
3659	    KM_SLEEP);
3660	for (int i = 0; i < max_ncpus; i++) {
3661		spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
3662		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
3663		    ZIO_FLAG_GODFATHER);
3664	}
3665
3666	/*
3667	 * Create the root vdev.
3668	 */
3669	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3670
3671	error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD);
3672
3673	ASSERT(error != 0 || rvd != NULL);
3674	ASSERT(error != 0 || spa->spa_root_vdev == rvd);
3675
3676	if (error == 0 && !zfs_allocatable_devs(nvroot))
3677		error = SET_ERROR(EINVAL);
3678
3679	if (error == 0 &&
3680	    (error = vdev_create(rvd, txg, B_FALSE)) == 0 &&
3681	    (error = spa_validate_aux(spa, nvroot, txg,
3682	    VDEV_ALLOC_ADD)) == 0) {
3683		for (int c = 0; c < rvd->vdev_children; c++) {
3684			vdev_metaslab_set_size(rvd->vdev_child[c]);
3685			vdev_expand(rvd->vdev_child[c], txg);
3686		}
3687	}
3688
3689	spa_config_exit(spa, SCL_ALL, FTAG);
3690
3691	if (error != 0) {
3692		spa_unload(spa);
3693		spa_deactivate(spa);
3694		spa_remove(spa);
3695		mutex_exit(&spa_namespace_lock);
3696		return (error);
3697	}
3698
3699	/*
3700	 * Get the list of spares, if specified.
3701	 */
3702	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
3703	    &spares, &nspares) == 0) {
3704		VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, NV_UNIQUE_NAME,
3705		    KM_SLEEP) == 0);
3706		VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config,
3707		    ZPOOL_CONFIG_SPARES, spares, nspares) == 0);
3708		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3709		spa_load_spares(spa);
3710		spa_config_exit(spa, SCL_ALL, FTAG);
3711		spa->spa_spares.sav_sync = B_TRUE;
3712	}
3713
3714	/*
3715	 * Get the list of level 2 cache devices, if specified.
3716	 */
3717	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
3718	    &l2cache, &nl2cache) == 0) {
3719		VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config,
3720		    NV_UNIQUE_NAME, KM_SLEEP) == 0);
3721		VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config,
3722		    ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0);
3723		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3724		spa_load_l2cache(spa);
3725		spa_config_exit(spa, SCL_ALL, FTAG);
3726		spa->spa_l2cache.sav_sync = B_TRUE;
3727	}
3728
3729	spa->spa_is_initializing = B_TRUE;
3730	spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, txg);
3731	spa->spa_meta_objset = dp->dp_meta_objset;
3732	spa->spa_is_initializing = B_FALSE;
3733
3734	/*
3735	 * Create DDTs (dedup tables).
3736	 */
3737	ddt_create(spa);
3738
3739	spa_update_dspace(spa);
3740
3741	tx = dmu_tx_create_assigned(dp, txg);
3742
3743	/*
3744	 * Create the pool config object.
3745	 */
3746	spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset,
3747	    DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE,
3748	    DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx);
3749
3750	if (zap_add(spa->spa_meta_objset,
3751	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG,
3752	    sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) {
3753		cmn_err(CE_PANIC, "failed to add pool config");
3754	}
3755
3756	if (spa_version(spa) >= SPA_VERSION_FEATURES)
3757		spa_feature_create_zap_objects(spa, tx);
3758
3759	if (zap_add(spa->spa_meta_objset,
3760	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION,
3761	    sizeof (uint64_t), 1, &version, tx) != 0) {
3762		cmn_err(CE_PANIC, "failed to add pool version");
3763	}
3764
3765	/* Newly created pools with the right version are always deflated. */
3766	if (version >= SPA_VERSION_RAIDZ_DEFLATE) {
3767		spa->spa_deflate = TRUE;
3768		if (zap_add(spa->spa_meta_objset,
3769		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE,
3770		    sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) {
3771			cmn_err(CE_PANIC, "failed to add deflate");
3772		}
3773	}
3774
3775	/*
3776	 * Create the deferred-free bpobj.  Turn off compression
3777	 * because sync-to-convergence takes longer if the blocksize
3778	 * keeps changing.
3779	 */
3780	obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx);
3781	dmu_object_set_compress(spa->spa_meta_objset, obj,
3782	    ZIO_COMPRESS_OFF, tx);
3783	if (zap_add(spa->spa_meta_objset,
3784	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ,
3785	    sizeof (uint64_t), 1, &obj, tx) != 0) {
3786		cmn_err(CE_PANIC, "failed to add bpobj");
3787	}
3788	VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj,
3789	    spa->spa_meta_objset, obj));
3790
3791	/*
3792	 * Create the pool's history object.
3793	 */
3794	if (version >= SPA_VERSION_ZPOOL_HISTORY)
3795		spa_history_create_obj(spa, tx);
3796
3797	/*
3798	 * Generate some random noise for salted checksums to operate on.
3799	 */
3800	(void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes,
3801	    sizeof (spa->spa_cksum_salt.zcs_bytes));
3802
3803	/*
3804	 * Set pool properties.
3805	 */
3806	spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS);
3807	spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);
3808	spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE);
3809	spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND);
3810
3811	if (props != NULL) {
3812		spa_configfile_set(spa, props, B_FALSE);
3813		spa_sync_props(props, tx);
3814	}
3815
3816	dmu_tx_commit(tx);
3817
3818	spa->spa_sync_on = B_TRUE;
3819	txg_sync_start(spa->spa_dsl_pool);
3820
3821	/*
3822	 * We explicitly wait for the first transaction to complete so that our
3823	 * bean counters are appropriately updated.
3824	 */
3825	txg_wait_synced(spa->spa_dsl_pool, txg);
3826
3827	spa_config_sync(spa, B_FALSE, B_TRUE);
3828	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_CREATE);
3829
3830	spa_history_log_version(spa, "create");
3831
3832	/*
3833	 * Don't count references from objsets that are already closed
3834	 * and are making their way through the eviction process.
3835	 */
3836	spa_evicting_os_wait(spa);
3837	spa->spa_minref = refcount_count(&spa->spa_refcount);
3838	spa->spa_load_state = SPA_LOAD_NONE;
3839
3840	mutex_exit(&spa_namespace_lock);
3841
3842	return (0);
3843}
3844
3845#ifdef _KERNEL
3846/*
3847 * Get the root pool information from the root disk, then import the root pool
3848 * during the system boot up time.
3849 */
3850extern int vdev_disk_read_rootlabel(char *, char *, nvlist_t **);
3851
3852static nvlist_t *
3853spa_generate_rootconf(char *devpath, char *devid, uint64_t *guid)
3854{
3855	nvlist_t *config;
3856	nvlist_t *nvtop, *nvroot;
3857	uint64_t pgid;
3858
3859	if (vdev_disk_read_rootlabel(devpath, devid, &config) != 0)
3860		return (NULL);
3861
3862	/*
3863	 * Add this top-level vdev to the child array.
3864	 */
3865	VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
3866	    &nvtop) == 0);
3867	VERIFY(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
3868	    &pgid) == 0);
3869	VERIFY(nvlist_lookup_uint64(config, ZPOOL_CONFIG_GUID, guid) == 0);
3870
3871	/*
3872	 * Put this pool's top-level vdevs into a root vdev.
3873	 */
3874	VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0);
3875	VERIFY(nvlist_add_string(nvroot, ZPOOL_CONFIG_TYPE,
3876	    VDEV_TYPE_ROOT) == 0);
3877	VERIFY(nvlist_add_uint64(nvroot, ZPOOL_CONFIG_ID, 0ULL) == 0);
3878	VERIFY(nvlist_add_uint64(nvroot, ZPOOL_CONFIG_GUID, pgid) == 0);
3879	VERIFY(nvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
3880	    &nvtop, 1) == 0);
3881
3882	/*
3883	 * Replace the existing vdev_tree with the new root vdev in
3884	 * this pool's configuration (remove the old, add the new).
3885	 */
3886	VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, nvroot) == 0);
3887	nvlist_free(nvroot);
3888	return (config);
3889}
3890
3891/*
3892 * Walk the vdev tree and see if we can find a device with "better"
3893 * configuration. A configuration is "better" if the label on that
3894 * device has a more recent txg.
3895 */
3896static void
3897spa_alt_rootvdev(vdev_t *vd, vdev_t **avd, uint64_t *txg)
3898{
3899	for (int c = 0; c < vd->vdev_children; c++)
3900		spa_alt_rootvdev(vd->vdev_child[c], avd, txg);
3901
3902	if (vd->vdev_ops->vdev_op_leaf) {
3903		nvlist_t *label;
3904		uint64_t label_txg;
3905
3906		if (vdev_disk_read_rootlabel(vd->vdev_physpath, vd->vdev_devid,
3907		    &label) != 0)
3908			return;
3909
3910		VERIFY(nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_TXG,
3911		    &label_txg) == 0);
3912
3913		/*
3914		 * Do we have a better boot device?
3915		 */
3916		if (label_txg > *txg) {
3917			*txg = label_txg;
3918			*avd = vd;
3919		}
3920		nvlist_free(label);
3921	}
3922}
3923
3924/*
3925 * Import a root pool.
3926 *
3927 * For x86. devpath_list will consist of devid and/or physpath name of
3928 * the vdev (e.g. "id1,sd@SSEAGATE..." or "/pci@1f,0/ide@d/disk@0,0:a").
3929 * The GRUB "findroot" command will return the vdev we should boot.
3930 *
3931 * For Sparc, devpath_list consists the physpath name of the booting device
3932 * no matter the rootpool is a single device pool or a mirrored pool.
3933 * e.g.
3934 *	"/pci@1f,0/ide@d/disk@0,0:a"
3935 */
3936int
3937spa_import_rootpool(char *devpath, char *devid)
3938{
3939	spa_t *spa;
3940	vdev_t *rvd, *bvd, *avd = NULL;
3941	nvlist_t *config, *nvtop;
3942	uint64_t guid, txg;
3943	char *pname;
3944	int error;
3945
3946	/*
3947	 * Read the label from the boot device and generate a configuration.
3948	 */
3949	config = spa_generate_rootconf(devpath, devid, &guid);
3950#if defined(_OBP) && defined(_KERNEL)
3951	if (config == NULL) {
3952		if (strstr(devpath, "/iscsi/ssd") != NULL) {
3953			/* iscsi boot */
3954			get_iscsi_bootpath_phy(devpath);
3955			config = spa_generate_rootconf(devpath, devid, &guid);
3956		}
3957	}
3958#endif
3959	if (config == NULL) {
3960		cmn_err(CE_NOTE, "Cannot read the pool label from '%s'",
3961		    devpath);
3962		return (SET_ERROR(EIO));
3963	}
3964
3965	VERIFY(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
3966	    &pname) == 0);
3967	VERIFY(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, &txg) == 0);
3968
3969	mutex_enter(&spa_namespace_lock);
3970	if ((spa = spa_lookup(pname)) != NULL) {
3971		/*
3972		 * Remove the existing root pool from the namespace so that we
3973		 * can replace it with the correct config we just read in.
3974		 */
3975		spa_remove(spa);
3976	}
3977
3978	spa = spa_add(pname, config, NULL);
3979	spa->spa_is_root = B_TRUE;
3980	spa->spa_import_flags = ZFS_IMPORT_VERBATIM;
3981
3982	/*
3983	 * Build up a vdev tree based on the boot device's label config.
3984	 */
3985	VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
3986	    &nvtop) == 0);
3987	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3988	error = spa_config_parse(spa, &rvd, nvtop, NULL, 0,
3989	    VDEV_ALLOC_ROOTPOOL);
3990	spa_config_exit(spa, SCL_ALL, FTAG);
3991	if (error) {
3992		mutex_exit(&spa_namespace_lock);
3993		nvlist_free(config);
3994		cmn_err(CE_NOTE, "Can not parse the config for pool '%s'",
3995		    pname);
3996		return (error);
3997	}
3998
3999	/*
4000	 * Get the boot vdev.
4001	 */
4002	if ((bvd = vdev_lookup_by_guid(rvd, guid)) == NULL) {
4003		cmn_err(CE_NOTE, "Can not find the boot vdev for guid %llu",
4004		    (u_longlong_t)guid);
4005		error = SET_ERROR(ENOENT);
4006		goto out;
4007	}
4008
4009	/*
4010	 * Determine if there is a better boot device.
4011	 */
4012	avd = bvd;
4013	spa_alt_rootvdev(rvd, &avd, &txg);
4014	if (avd != bvd) {
4015		cmn_err(CE_NOTE, "The boot device is 'degraded'. Please "
4016		    "try booting from '%s'", avd->vdev_path);
4017		error = SET_ERROR(EINVAL);
4018		goto out;
4019	}
4020
4021	/*
4022	 * If the boot device is part of a spare vdev then ensure that
4023	 * we're booting off the active spare.
4024	 */
4025	if (bvd->vdev_parent->vdev_ops == &vdev_spare_ops &&
4026	    !bvd->vdev_isspare) {
4027		cmn_err(CE_NOTE, "The boot device is currently spared. Please "
4028		    "try booting from '%s'",
4029		    bvd->vdev_parent->
4030		    vdev_child[bvd->vdev_parent->vdev_children - 1]->vdev_path);
4031		error = SET_ERROR(EINVAL);
4032		goto out;
4033	}
4034
4035	error = 0;
4036out:
4037	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4038	vdev_free(rvd);
4039	spa_config_exit(spa, SCL_ALL, FTAG);
4040	mutex_exit(&spa_namespace_lock);
4041
4042	nvlist_free(config);
4043	return (error);
4044}
4045
4046#endif
4047
4048/*
4049 * Import a non-root pool into the system.
4050 */
4051int
4052spa_import(const char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags)
4053{
4054	spa_t *spa;
4055	char *altroot = NULL;
4056	spa_load_state_t state = SPA_LOAD_IMPORT;
4057	zpool_rewind_policy_t policy;
4058	uint64_t mode = spa_mode_global;
4059	uint64_t readonly = B_FALSE;
4060	int error;
4061	nvlist_t *nvroot;
4062	nvlist_t **spares, **l2cache;
4063	uint_t nspares, nl2cache;
4064
4065	/*
4066	 * If a pool with this name exists, return failure.
4067	 */
4068	mutex_enter(&spa_namespace_lock);
4069	if (spa_lookup(pool) != NULL) {
4070		mutex_exit(&spa_namespace_lock);
4071		return (SET_ERROR(EEXIST));
4072	}
4073
4074	/*
4075	 * Create and initialize the spa structure.
4076	 */
4077	(void) nvlist_lookup_string(props,
4078	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
4079	(void) nvlist_lookup_uint64(props,
4080	    zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly);
4081	if (readonly)
4082		mode = FREAD;
4083	spa = spa_add(pool, config, altroot);
4084	spa->spa_import_flags = flags;
4085
4086	/*
4087	 * Verbatim import - Take a pool and insert it into the namespace
4088	 * as if it had been loaded at boot.
4089	 */
4090	if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) {
4091		if (props != NULL)
4092			spa_configfile_set(spa, props, B_FALSE);
4093
4094		spa_config_sync(spa, B_FALSE, B_TRUE);
4095		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT);
4096
4097		mutex_exit(&spa_namespace_lock);
4098		return (0);
4099	}
4100
4101	spa_activate(spa, mode);
4102
4103	/*
4104	 * Don't start async tasks until we know everything is healthy.
4105	 */
4106	spa_async_suspend(spa);
4107
4108	zpool_get_rewind_policy(config, &policy);
4109	if (policy.zrp_request & ZPOOL_DO_REWIND)
4110		state = SPA_LOAD_RECOVER;
4111
4112	/*
4113	 * Pass off the heavy lifting to spa_load().  Pass TRUE for mosconfig
4114	 * because the user-supplied config is actually the one to trust when
4115	 * doing an import.
4116	 */
4117	if (state != SPA_LOAD_RECOVER)
4118		spa->spa_last_ubsync_txg = spa->spa_load_txg = 0;
4119
4120	error = spa_load_best(spa, state, B_TRUE, policy.zrp_txg,
4121	    policy.zrp_request);
4122
4123	/*
4124	 * Propagate anything learned while loading the pool and pass it
4125	 * back to caller (i.e. rewind info, missing devices, etc).
4126	 */
4127	VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO,
4128	    spa->spa_load_info) == 0);
4129
4130	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4131	/*
4132	 * Toss any existing sparelist, as it doesn't have any validity
4133	 * anymore, and conflicts with spa_has_spare().
4134	 */
4135	if (spa->spa_spares.sav_config) {
4136		nvlist_free(spa->spa_spares.sav_config);
4137		spa->spa_spares.sav_config = NULL;
4138		spa_load_spares(spa);
4139	}
4140	if (spa->spa_l2cache.sav_config) {
4141		nvlist_free(spa->spa_l2cache.sav_config);
4142		spa->spa_l2cache.sav_config = NULL;
4143		spa_load_l2cache(spa);
4144	}
4145
4146	VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
4147	    &nvroot) == 0);
4148	if (error == 0)
4149		error = spa_validate_aux(spa, nvroot, -1ULL,
4150		    VDEV_ALLOC_SPARE);
4151	if (error == 0)
4152		error = spa_validate_aux(spa, nvroot, -1ULL,
4153		    VDEV_ALLOC_L2CACHE);
4154	spa_config_exit(spa, SCL_ALL, FTAG);
4155
4156	if (props != NULL)
4157		spa_configfile_set(spa, props, B_FALSE);
4158
4159	if (error != 0 || (props && spa_writeable(spa) &&
4160	    (error = spa_prop_set(spa, props)))) {
4161		spa_unload(spa);
4162		spa_deactivate(spa);
4163		spa_remove(spa);
4164		mutex_exit(&spa_namespace_lock);
4165		return (error);
4166	}
4167
4168	spa_async_resume(spa);
4169
4170	/*
4171	 * Override any spares and level 2 cache devices as specified by
4172	 * the user, as these may have correct device names/devids, etc.
4173	 */
4174	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
4175	    &spares, &nspares) == 0) {
4176		if (spa->spa_spares.sav_config)
4177			VERIFY(nvlist_remove(spa->spa_spares.sav_config,
4178			    ZPOOL_CONFIG_SPARES, DATA_TYPE_NVLIST_ARRAY) == 0);
4179		else
4180			VERIFY(nvlist_alloc(&spa->spa_spares.sav_config,
4181			    NV_UNIQUE_NAME, KM_SLEEP) == 0);
4182		VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config,
4183		    ZPOOL_CONFIG_SPARES, spares, nspares) == 0);
4184		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4185		spa_load_spares(spa);
4186		spa_config_exit(spa, SCL_ALL, FTAG);
4187		spa->spa_spares.sav_sync = B_TRUE;
4188	}
4189	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
4190	    &l2cache, &nl2cache) == 0) {
4191		if (spa->spa_l2cache.sav_config)
4192			VERIFY(nvlist_remove(spa->spa_l2cache.sav_config,
4193			    ZPOOL_CONFIG_L2CACHE, DATA_TYPE_NVLIST_ARRAY) == 0);
4194		else
4195			VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config,
4196			    NV_UNIQUE_NAME, KM_SLEEP) == 0);
4197		VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config,
4198		    ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0);
4199		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4200		spa_load_l2cache(spa);
4201		spa_config_exit(spa, SCL_ALL, FTAG);
4202		spa->spa_l2cache.sav_sync = B_TRUE;
4203	}
4204
4205	/*
4206	 * Check for any removed devices.
4207	 */
4208	if (spa->spa_autoreplace) {
4209		spa_aux_check_removed(&spa->spa_spares);
4210		spa_aux_check_removed(&spa->spa_l2cache);
4211	}
4212
4213	if (spa_writeable(spa)) {
4214		/*
4215		 * Update the config cache to include the newly-imported pool.
4216		 */
4217		spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
4218	}
4219
4220	/*
4221	 * It's possible that the pool was expanded while it was exported.
4222	 * We kick off an async task to handle this for us.
4223	 */
4224	spa_async_request(spa, SPA_ASYNC_AUTOEXPAND);
4225
4226	spa_history_log_version(spa, "import");
4227
4228	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT);
4229
4230	mutex_exit(&spa_namespace_lock);
4231
4232	return (0);
4233}
4234
4235nvlist_t *
4236spa_tryimport(nvlist_t *tryconfig)
4237{
4238	nvlist_t *config = NULL;
4239	char *poolname;
4240	spa_t *spa;
4241	uint64_t state;
4242	int error;
4243
4244	if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname))
4245		return (NULL);
4246
4247	if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state))
4248		return (NULL);
4249
4250	/*
4251	 * Create and initialize the spa structure.
4252	 */
4253	mutex_enter(&spa_namespace_lock);
4254	spa = spa_add(TRYIMPORT_NAME, tryconfig, NULL);
4255	spa_activate(spa, FREAD);
4256
4257	/*
4258	 * Pass off the heavy lifting to spa_load().
4259	 * Pass TRUE for mosconfig because the user-supplied config
4260	 * is actually the one to trust when doing an import.
4261	 */
4262	error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING, B_TRUE);
4263
4264	/*
4265	 * If 'tryconfig' was at least parsable, return the current config.
4266	 */
4267	if (spa->spa_root_vdev != NULL) {
4268		config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
4269		VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME,
4270		    poolname) == 0);
4271		VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE,
4272		    state) == 0);
4273		VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP,
4274		    spa->spa_uberblock.ub_timestamp) == 0);
4275		VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO,
4276		    spa->spa_load_info) == 0);
4277
4278		/*
4279		 * If the bootfs property exists on this pool then we
4280		 * copy it out so that external consumers can tell which
4281		 * pools are bootable.
4282		 */
4283		if ((!error || error == EEXIST) && spa->spa_bootfs) {
4284			char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP);
4285
4286			/*
4287			 * We have to play games with the name since the
4288			 * pool was opened as TRYIMPORT_NAME.
4289			 */
4290			if (dsl_dsobj_to_dsname(spa_name(spa),
4291			    spa->spa_bootfs, tmpname) == 0) {
4292				char *cp;
4293				char *dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP);
4294
4295				cp = strchr(tmpname, '/');
4296				if (cp == NULL) {
4297					(void) strlcpy(dsname, tmpname,
4298					    MAXPATHLEN);
4299				} else {
4300					(void) snprintf(dsname, MAXPATHLEN,
4301					    "%s/%s", poolname, ++cp);
4302				}
4303				VERIFY(nvlist_add_string(config,
4304				    ZPOOL_CONFIG_BOOTFS, dsname) == 0);
4305				kmem_free(dsname, MAXPATHLEN);
4306			}
4307			kmem_free(tmpname, MAXPATHLEN);
4308		}
4309
4310		/*
4311		 * Add the list of hot spares and level 2 cache devices.
4312		 */
4313		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
4314		spa_add_spares(spa, config);
4315		spa_add_l2cache(spa, config);
4316		spa_config_exit(spa, SCL_CONFIG, FTAG);
4317	}
4318
4319	spa_unload(spa);
4320	spa_deactivate(spa);
4321	spa_remove(spa);
4322	mutex_exit(&spa_namespace_lock);
4323
4324	return (config);
4325}
4326
4327/*
4328 * Pool export/destroy
4329 *
4330 * The act of destroying or exporting a pool is very simple.  We make sure there
4331 * is no more pending I/O and any references to the pool are gone.  Then, we
4332 * update the pool state and sync all the labels to disk, removing the
4333 * configuration from the cache afterwards. If the 'hardforce' flag is set, then
4334 * we don't sync the labels or remove the configuration cache.
4335 */
4336static int
4337spa_export_common(char *pool, int new_state, nvlist_t **oldconfig,
4338    boolean_t force, boolean_t hardforce)
4339{
4340	spa_t *spa;
4341
4342	if (oldconfig)
4343		*oldconfig = NULL;
4344
4345	if (!(spa_mode_global & FWRITE))
4346		return (SET_ERROR(EROFS));
4347
4348	mutex_enter(&spa_namespace_lock);
4349	if ((spa = spa_lookup(pool)) == NULL) {
4350		mutex_exit(&spa_namespace_lock);
4351		return (SET_ERROR(ENOENT));
4352	}
4353
4354	/*
4355	 * Put a hold on the pool, drop the namespace lock, stop async tasks,
4356	 * reacquire the namespace lock, and see if we can export.
4357	 */
4358	spa_open_ref(spa, FTAG);
4359	mutex_exit(&spa_namespace_lock);
4360	spa_async_suspend(spa);
4361	mutex_enter(&spa_namespace_lock);
4362	spa_close(spa, FTAG);
4363
4364	/*
4365	 * The pool will be in core if it's openable,
4366	 * in which case we can modify its state.
4367	 */
4368	if (spa->spa_state != POOL_STATE_UNINITIALIZED && spa->spa_sync_on) {
4369		/*
4370		 * Objsets may be open only because they're dirty, so we
4371		 * have to force it to sync before checking spa_refcnt.
4372		 */
4373		txg_wait_synced(spa->spa_dsl_pool, 0);
4374		spa_evicting_os_wait(spa);
4375
4376		/*
4377		 * A pool cannot be exported or destroyed if there are active
4378		 * references.  If we are resetting a pool, allow references by
4379		 * fault injection handlers.
4380		 */
4381		if (!spa_refcount_zero(spa) ||
4382		    (spa->spa_inject_ref != 0 &&
4383		    new_state != POOL_STATE_UNINITIALIZED)) {
4384			spa_async_resume(spa);
4385			mutex_exit(&spa_namespace_lock);
4386			return (SET_ERROR(EBUSY));
4387		}
4388
4389		/*
4390		 * A pool cannot be exported if it has an active shared spare.
4391		 * This is to prevent other pools stealing the active spare
4392		 * from an exported pool. At user's own will, such pool can
4393		 * be forcedly exported.
4394		 */
4395		if (!force && new_state == POOL_STATE_EXPORTED &&
4396		    spa_has_active_shared_spare(spa)) {
4397			spa_async_resume(spa);
4398			mutex_exit(&spa_namespace_lock);
4399			return (SET_ERROR(EXDEV));
4400		}
4401
4402		/*
4403		 * We want this to be reflected on every label,
4404		 * so mark them all dirty.  spa_unload() will do the
4405		 * final sync that pushes these changes out.
4406		 */
4407		if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) {
4408			spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
4409			spa->spa_state = new_state;
4410			spa->spa_final_txg = spa_last_synced_txg(spa) +
4411			    TXG_DEFER_SIZE + 1;
4412			vdev_config_dirty(spa->spa_root_vdev);
4413			spa_config_exit(spa, SCL_ALL, FTAG);
4414		}
4415	}
4416
4417	spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_DESTROY);
4418
4419	if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
4420		spa_unload(spa);
4421		spa_deactivate(spa);
4422	}
4423
4424	if (oldconfig && spa->spa_config)
4425		VERIFY(nvlist_dup(spa->spa_config, oldconfig, 0) == 0);
4426
4427	if (new_state != POOL_STATE_UNINITIALIZED) {
4428		if (!hardforce)
4429			spa_config_sync(spa, B_TRUE, B_TRUE);
4430		spa_remove(spa);
4431	}
4432	mutex_exit(&spa_namespace_lock);
4433
4434	return (0);
4435}
4436
4437/*
4438 * Destroy a storage pool.
4439 */
4440int
4441spa_destroy(char *pool)
4442{
4443	return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL,
4444	    B_FALSE, B_FALSE));
4445}
4446
4447/*
4448 * Export a storage pool.
4449 */
4450int
4451spa_export(char *pool, nvlist_t **oldconfig, boolean_t force,
4452    boolean_t hardforce)
4453{
4454	return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig,
4455	    force, hardforce));
4456}
4457
4458/*
4459 * Similar to spa_export(), this unloads the spa_t without actually removing it
4460 * from the namespace in any way.
4461 */
4462int
4463spa_reset(char *pool)
4464{
4465	return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL,
4466	    B_FALSE, B_FALSE));
4467}
4468
4469/*
4470 * ==========================================================================
4471 * Device manipulation
4472 * ==========================================================================
4473 */
4474
4475/*
4476 * Add a device to a storage pool.
4477 */
4478int
4479spa_vdev_add(spa_t *spa, nvlist_t *nvroot)
4480{
4481	uint64_t txg, id;
4482	int error;
4483	vdev_t *rvd = spa->spa_root_vdev;
4484	vdev_t *vd, *tvd;
4485	nvlist_t **spares, **l2cache;
4486	uint_t nspares, nl2cache;
4487
4488	ASSERT(spa_writeable(spa));
4489
4490	txg = spa_vdev_enter(spa);
4491
4492	if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0,
4493	    VDEV_ALLOC_ADD)) != 0)
4494		return (spa_vdev_exit(spa, NULL, txg, error));
4495
4496	spa->spa_pending_vdev = vd;	/* spa_vdev_exit() will clear this */
4497
4498	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares,
4499	    &nspares) != 0)
4500		nspares = 0;
4501
4502	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache,
4503	    &nl2cache) != 0)
4504		nl2cache = 0;
4505
4506	if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0)
4507		return (spa_vdev_exit(spa, vd, txg, EINVAL));
4508
4509	if (vd->vdev_children != 0 &&
4510	    (error = vdev_create(vd, txg, B_FALSE)) != 0)
4511		return (spa_vdev_exit(spa, vd, txg, error));
4512
4513	/*
4514	 * We must validate the spares and l2cache devices after checking the
4515	 * children.  Otherwise, vdev_inuse() will blindly overwrite the spare.
4516	 */
4517	if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0)
4518		return (spa_vdev_exit(spa, vd, txg, error));
4519
4520	/*
4521	 * Transfer each new top-level vdev from vd to rvd.
4522	 */
4523	for (int c = 0; c < vd->vdev_children; c++) {
4524
4525		/*
4526		 * Set the vdev id to the first hole, if one exists.
4527		 */
4528		for (id = 0; id < rvd->vdev_children; id++) {
4529			if (rvd->vdev_child[id]->vdev_ishole) {
4530				vdev_free(rvd->vdev_child[id]);
4531				break;
4532			}
4533		}
4534		tvd = vd->vdev_child[c];
4535		vdev_remove_child(vd, tvd);
4536		tvd->vdev_id = id;
4537		vdev_add_child(rvd, tvd);
4538		vdev_config_dirty(tvd);
4539	}
4540
4541	if (nspares != 0) {
4542		spa_set_aux_vdevs(&spa->spa_spares, spares, nspares,
4543		    ZPOOL_CONFIG_SPARES);
4544		spa_load_spares(spa);
4545		spa->spa_spares.sav_sync = B_TRUE;
4546	}
4547
4548	if (nl2cache != 0) {
4549		spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache,
4550		    ZPOOL_CONFIG_L2CACHE);
4551		spa_load_l2cache(spa);
4552		spa->spa_l2cache.sav_sync = B_TRUE;
4553	}
4554
4555	/*
4556	 * We have to be careful when adding new vdevs to an existing pool.
4557	 * If other threads start allocating from these vdevs before we
4558	 * sync the config cache, and we lose power, then upon reboot we may
4559	 * fail to open the pool because there are DVAs that the config cache
4560	 * can't translate.  Therefore, we first add the vdevs without
4561	 * initializing metaslabs; sync the config cache (via spa_vdev_exit());
4562	 * and then let spa_config_update() initialize the new metaslabs.
4563	 *
4564	 * spa_load() checks for added-but-not-initialized vdevs, so that
4565	 * if we lose power at any point in this sequence, the remaining
4566	 * steps will be completed the next time we load the pool.
4567	 */
4568	(void) spa_vdev_exit(spa, vd, txg, 0);
4569
4570	mutex_enter(&spa_namespace_lock);
4571	spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
4572	spa_event_notify(spa, NULL, NULL, ESC_ZFS_VDEV_ADD);
4573	mutex_exit(&spa_namespace_lock);
4574
4575	return (0);
4576}
4577
4578/*
4579 * Attach a device to a mirror.  The arguments are the path to any device
4580 * in the mirror, and the nvroot for the new device.  If the path specifies
4581 * a device that is not mirrored, we automatically insert the mirror vdev.
4582 *
4583 * If 'replacing' is specified, the new device is intended to replace the
4584 * existing device; in this case the two devices are made into their own
4585 * mirror using the 'replacing' vdev, which is functionally identical to
4586 * the mirror vdev (it actually reuses all the same ops) but has a few
4587 * extra rules: you can't attach to it after it's been created, and upon
4588 * completion of resilvering, the first disk (the one being replaced)
4589 * is automatically detached.
4590 */
4591int
4592spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing)
4593{
4594	uint64_t txg, dtl_max_txg;
4595	vdev_t *rvd = spa->spa_root_vdev;
4596	vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd;
4597	vdev_ops_t *pvops;
4598	char *oldvdpath, *newvdpath;
4599	int newvd_isspare;
4600	int error;
4601
4602	ASSERT(spa_writeable(spa));
4603
4604	txg = spa_vdev_enter(spa);
4605
4606	oldvd = spa_lookup_by_guid(spa, guid, B_FALSE);
4607
4608	if (oldvd == NULL)
4609		return (spa_vdev_exit(spa, NULL, txg, ENODEV));
4610
4611	if (!oldvd->vdev_ops->vdev_op_leaf)
4612		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
4613
4614	pvd = oldvd->vdev_parent;
4615
4616	if ((error = spa_config_parse(spa, &newrootvd, nvroot, NULL, 0,
4617	    VDEV_ALLOC_ATTACH)) != 0)
4618		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
4619
4620	if (newrootvd->vdev_children != 1)
4621		return (spa_vdev_exit(spa, newrootvd, txg, EINVAL));
4622
4623	newvd = newrootvd->vdev_child[0];
4624
4625	if (!newvd->vdev_ops->vdev_op_leaf)
4626		return (spa_vdev_exit(spa, newrootvd, txg, EINVAL));
4627
4628	if ((error = vdev_create(newrootvd, txg, replacing)) != 0)
4629		return (spa_vdev_exit(spa, newrootvd, txg, error));
4630
4631	/*
4632	 * Spares can't replace logs
4633	 */
4634	if (oldvd->vdev_top->vdev_islog && newvd->vdev_isspare)
4635		return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
4636
4637	if (!replacing) {
4638		/*
4639		 * For attach, the only allowable parent is a mirror or the root
4640		 * vdev.
4641		 */
4642		if (pvd->vdev_ops != &vdev_mirror_ops &&
4643		    pvd->vdev_ops != &vdev_root_ops)
4644			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
4645
4646		pvops = &vdev_mirror_ops;
4647	} else {
4648		/*
4649		 * Active hot spares can only be replaced by inactive hot
4650		 * spares.
4651		 */
4652		if (pvd->vdev_ops == &vdev_spare_ops &&
4653		    oldvd->vdev_isspare &&
4654		    !spa_has_spare(spa, newvd->vdev_guid))
4655			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
4656
4657		/*
4658		 * If the source is a hot spare, and the parent isn't already a
4659		 * spare, then we want to create a new hot spare.  Otherwise, we
4660		 * want to create a replacing vdev.  The user is not allowed to
4661		 * attach to a spared vdev child unless the 'isspare' state is
4662		 * the same (spare replaces spare, non-spare replaces
4663		 * non-spare).
4664		 */
4665		if (pvd->vdev_ops == &vdev_replacing_ops &&
4666		    spa_version(spa) < SPA_VERSION_MULTI_REPLACE) {
4667			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
4668		} else if (pvd->vdev_ops == &vdev_spare_ops &&
4669		    newvd->vdev_isspare != oldvd->vdev_isspare) {
4670			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
4671		}
4672
4673		if (newvd->vdev_isspare)
4674			pvops = &vdev_spare_ops;
4675		else
4676			pvops = &vdev_replacing_ops;
4677	}
4678
4679	/*
4680	 * Make sure the new device is big enough.
4681	 */
4682	if (newvd->vdev_asize < vdev_get_min_asize(oldvd))
4683		return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW));
4684
4685	/*
4686	 * The new device cannot have a higher alignment requirement
4687	 * than the top-level vdev.
4688	 */
4689	if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift)
4690		return (spa_vdev_exit(spa, newrootvd, txg, EDOM));
4691
4692	/*
4693	 * If this is an in-place replacement, update oldvd's path and devid
4694	 * to make it distinguishable from newvd, and unopenable from now on.
4695	 */
4696	if (strcmp(oldvd->vdev_path, newvd->vdev_path) == 0) {
4697		spa_strfree(oldvd->vdev_path);
4698		oldvd->vdev_path = kmem_alloc(strlen(newvd->vdev_path) + 5,
4699		    KM_SLEEP);
4700		(void) sprintf(oldvd->vdev_path, "%s/%s",
4701		    newvd->vdev_path, "old");
4702		if (oldvd->vdev_devid != NULL) {
4703			spa_strfree(oldvd->vdev_devid);
4704			oldvd->vdev_devid = NULL;
4705		}
4706	}
4707
4708	/* mark the device being resilvered */
4709	newvd->vdev_resilver_txg = txg;
4710
4711	/*
4712	 * If the parent is not a mirror, or if we're replacing, insert the new
4713	 * mirror/replacing/spare vdev above oldvd.
4714	 */
4715	if (pvd->vdev_ops != pvops)
4716		pvd = vdev_add_parent(oldvd, pvops);
4717
4718	ASSERT(pvd->vdev_top->vdev_parent == rvd);
4719	ASSERT(pvd->vdev_ops == pvops);
4720	ASSERT(oldvd->vdev_parent == pvd);
4721
4722	/*
4723	 * Extract the new device from its root and add it to pvd.
4724	 */
4725	vdev_remove_child(newrootvd, newvd);
4726	newvd->vdev_id = pvd->vdev_children;
4727	newvd->vdev_crtxg = oldvd->vdev_crtxg;
4728	vdev_add_child(pvd, newvd);
4729
4730	tvd = newvd->vdev_top;
4731	ASSERT(pvd->vdev_top == tvd);
4732	ASSERT(tvd->vdev_parent == rvd);
4733
4734	vdev_config_dirty(tvd);
4735
4736	/*
4737	 * Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account
4738	 * for any dmu_sync-ed blocks.  It will propagate upward when
4739	 * spa_vdev_exit() calls vdev_dtl_reassess().
4740	 */
4741	dtl_max_txg = txg + TXG_CONCURRENT_STATES;
4742
4743	vdev_dtl_dirty(newvd, DTL_MISSING, TXG_INITIAL,
4744	    dtl_max_txg - TXG_INITIAL);
4745
4746	if (newvd->vdev_isspare) {
4747		spa_spare_activate(newvd);
4748		spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_SPARE);
4749	}
4750
4751	oldvdpath = spa_strdup(oldvd->vdev_path);
4752	newvdpath = spa_strdup(newvd->vdev_path);
4753	newvd_isspare = newvd->vdev_isspare;
4754
4755	/*
4756	 * Mark newvd's DTL dirty in this txg.
4757	 */
4758	vdev_dirty(tvd, VDD_DTL, newvd, txg);
4759
4760	/*
4761	 * Schedule the resilver to restart in the future. We do this to
4762	 * ensure that dmu_sync-ed blocks have been stitched into the
4763	 * respective datasets.
4764	 */
4765	dsl_resilver_restart(spa->spa_dsl_pool, dtl_max_txg);
4766
4767	if (spa->spa_bootfs)
4768		spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH);
4769
4770	spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_ATTACH);
4771
4772	/*
4773	 * Commit the config
4774	 */
4775	(void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0);
4776
4777	spa_history_log_internal(spa, "vdev attach", NULL,
4778	    "%s vdev=%s %s vdev=%s",
4779	    replacing && newvd_isspare ? "spare in" :
4780	    replacing ? "replace" : "attach", newvdpath,
4781	    replacing ? "for" : "to", oldvdpath);
4782
4783	spa_strfree(oldvdpath);
4784	spa_strfree(newvdpath);
4785
4786	return (0);
4787}
4788
4789/*
4790 * Detach a device from a mirror or replacing vdev.
4791 *
4792 * If 'replace_done' is specified, only detach if the parent
4793 * is a replacing vdev.
4794 */
4795int
4796spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done)
4797{
4798	uint64_t txg;
4799	int error;
4800	vdev_t *rvd = spa->spa_root_vdev;
4801	vdev_t *vd, *pvd, *cvd, *tvd;
4802	boolean_t unspare = B_FALSE;
4803	uint64_t unspare_guid = 0;
4804	char *vdpath;
4805
4806	ASSERT(spa_writeable(spa));
4807
4808	txg = spa_vdev_enter(spa);
4809
4810	vd = spa_lookup_by_guid(spa, guid, B_FALSE);
4811
4812	if (vd == NULL)
4813		return (spa_vdev_exit(spa, NULL, txg, ENODEV));
4814
4815	if (!vd->vdev_ops->vdev_op_leaf)
4816		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
4817
4818	pvd = vd->vdev_parent;
4819
4820	/*
4821	 * If the parent/child relationship is not as expected, don't do it.
4822	 * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing
4823	 * vdev that's replacing B with C.  The user's intent in replacing
4824	 * is to go from M(A,B) to M(A,C).  If the user decides to cancel
4825	 * the replace by detaching C, the expected behavior is to end up
4826	 * M(A,B).  But suppose that right after deciding to detach C,
4827	 * the replacement of B completes.  We would have M(A,C), and then
4828	 * ask to detach C, which would leave us with just A -- not what
4829	 * the user wanted.  To prevent this, we make sure that the
4830	 * parent/child relationship hasn't changed -- in this example,
4831	 * that C's parent is still the replacing vdev R.
4832	 */
4833	if (pvd->vdev_guid != pguid && pguid != 0)
4834		return (spa_vdev_exit(spa, NULL, txg, EBUSY));
4835
4836	/*
4837	 * Only 'replacing' or 'spare' vdevs can be replaced.
4838	 */
4839	if (replace_done && pvd->vdev_ops != &vdev_replacing_ops &&
4840	    pvd->vdev_ops != &vdev_spare_ops)
4841		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
4842
4843	ASSERT(pvd->vdev_ops != &vdev_spare_ops ||
4844	    spa_version(spa) >= SPA_VERSION_SPARES);
4845
4846	/*
4847	 * Only mirror, replacing, and spare vdevs support detach.
4848	 */
4849	if (pvd->vdev_ops != &vdev_replacing_ops &&
4850	    pvd->vdev_ops != &vdev_mirror_ops &&
4851	    pvd->vdev_ops != &vdev_spare_ops)
4852		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
4853
4854	/*
4855	 * If this device has the only valid copy of some data,
4856	 * we cannot safely detach it.
4857	 */
4858	if (vdev_dtl_required(vd))
4859		return (spa_vdev_exit(spa, NULL, txg, EBUSY));
4860
4861	ASSERT(pvd->vdev_children >= 2);
4862
4863	/*
4864	 * If we are detaching the second disk from a replacing vdev, then
4865	 * check to see if we changed the original vdev's path to have "/old"
4866	 * at the end in spa_vdev_attach().  If so, undo that change now.
4867	 */
4868	if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 &&
4869	    vd->vdev_path != NULL) {
4870		size_t len = strlen(vd->vdev_path);
4871
4872		for (int c = 0; c < pvd->vdev_children; c++) {
4873			cvd = pvd->vdev_child[c];
4874
4875			if (cvd == vd || cvd->vdev_path == NULL)
4876				continue;
4877
4878			if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 &&
4879			    strcmp(cvd->vdev_path + len, "/old") == 0) {
4880				spa_strfree(cvd->vdev_path);
4881				cvd->vdev_path = spa_strdup(vd->vdev_path);
4882				break;
4883			}
4884		}
4885	}
4886
4887	/*
4888	 * If we are detaching the original disk from a spare, then it implies
4889	 * that the spare should become a real disk, and be removed from the
4890	 * active spare list for the pool.
4891	 */
4892	if (pvd->vdev_ops == &vdev_spare_ops &&
4893	    vd->vdev_id == 0 &&
4894	    pvd->vdev_child[pvd->vdev_children - 1]->vdev_isspare)
4895		unspare = B_TRUE;
4896
4897	/*
4898	 * Erase the disk labels so the disk can be used for other things.
4899	 * This must be done after all other error cases are handled,
4900	 * but before we disembowel vd (so we can still do I/O to it).
4901	 * But if we can't do it, don't treat the error as fatal --
4902	 * it may be that the unwritability of the disk is the reason
4903	 * it's being detached!
4904	 */
4905	error = vdev_label_init(vd, 0, VDEV_LABEL_REMOVE);
4906
4907	/*
4908	 * Remove vd from its parent and compact the parent's children.
4909	 */
4910	vdev_remove_child(pvd, vd);
4911	vdev_compact_children(pvd);
4912
4913	/*
4914	 * Remember one of the remaining children so we can get tvd below.
4915	 */
4916	cvd = pvd->vdev_child[pvd->vdev_children - 1];
4917
4918	/*
4919	 * If we need to remove the remaining child from the list of hot spares,
4920	 * do it now, marking the vdev as no longer a spare in the process.
4921	 * We must do this before vdev_remove_parent(), because that can
4922	 * change the GUID if it creates a new toplevel GUID.  For a similar
4923	 * reason, we must remove the spare now, in the same txg as the detach;
4924	 * otherwise someone could attach a new sibling, change the GUID, and
4925	 * the subsequent attempt to spa_vdev_remove(unspare_guid) would fail.
4926	 */
4927	if (unspare) {
4928		ASSERT(cvd->vdev_isspare);
4929		spa_spare_remove(cvd);
4930		unspare_guid = cvd->vdev_guid;
4931		(void) spa_vdev_remove(spa, unspare_guid, B_TRUE);
4932		cvd->vdev_unspare = B_TRUE;
4933	}
4934
4935	/*
4936	 * If the parent mirror/replacing vdev only has one child,
4937	 * the parent is no longer needed.  Remove it from the tree.
4938	 */
4939	if (pvd->vdev_children == 1) {
4940		if (pvd->vdev_ops == &vdev_spare_ops)
4941			cvd->vdev_unspare = B_FALSE;
4942		vdev_remove_parent(cvd);
4943	}
4944
4945
4946	/*
4947	 * We don't set tvd until now because the parent we just removed
4948	 * may have been the previous top-level vdev.
4949	 */
4950	tvd = cvd->vdev_top;
4951	ASSERT(tvd->vdev_parent == rvd);
4952
4953	/*
4954	 * Reevaluate the parent vdev state.
4955	 */
4956	vdev_propagate_state(cvd);
4957
4958	/*
4959	 * If the 'autoexpand' property is set on the pool then automatically
4960	 * try to expand the size of the pool. For example if the device we
4961	 * just detached was smaller than the others, it may be possible to
4962	 * add metaslabs (i.e. grow the pool). We need to reopen the vdev
4963	 * first so that we can obtain the updated sizes of the leaf vdevs.
4964	 */
4965	if (spa->spa_autoexpand) {
4966		vdev_reopen(tvd);
4967		vdev_expand(tvd, txg);
4968	}
4969
4970	vdev_config_dirty(tvd);
4971
4972	/*
4973	 * Mark vd's DTL as dirty in this txg.  vdev_dtl_sync() will see that
4974	 * vd->vdev_detached is set and free vd's DTL object in syncing context.
4975	 * But first make sure we're not on any *other* txg's DTL list, to
4976	 * prevent vd from being accessed after it's freed.
4977	 */
4978	vdpath = spa_strdup(vd->vdev_path);
4979	for (int t = 0; t < TXG_SIZE; t++)
4980		(void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t);
4981	vd->vdev_detached = B_TRUE;
4982	vdev_dirty(tvd, VDD_DTL, vd, txg);
4983
4984	spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE);
4985
4986	/* hang on to the spa before we release the lock */
4987	spa_open_ref(spa, FTAG);
4988
4989	error = spa_vdev_exit(spa, vd, txg, 0);
4990
4991	spa_history_log_internal(spa, "detach", NULL,
4992	    "vdev=%s", vdpath);
4993	spa_strfree(vdpath);
4994
4995	/*
4996	 * If this was the removal of the original device in a hot spare vdev,
4997	 * then we want to go through and remove the device from the hot spare
4998	 * list of every other pool.
4999	 */
5000	if (unspare) {
5001		spa_t *altspa = NULL;
5002
5003		mutex_enter(&spa_namespace_lock);
5004		while ((altspa = spa_next(altspa)) != NULL) {
5005			if (altspa->spa_state != POOL_STATE_ACTIVE ||
5006			    altspa == spa)
5007				continue;
5008
5009			spa_open_ref(altspa, FTAG);
5010			mutex_exit(&spa_namespace_lock);
5011			(void) spa_vdev_remove(altspa, unspare_guid, B_TRUE);
5012			mutex_enter(&spa_namespace_lock);
5013			spa_close(altspa, FTAG);
5014		}
5015		mutex_exit(&spa_namespace_lock);
5016
5017		/* search the rest of the vdevs for spares to remove */
5018		spa_vdev_resilver_done(spa);
5019	}
5020
5021	/* all done with the spa; OK to release */
5022	mutex_enter(&spa_namespace_lock);
5023	spa_close(spa, FTAG);
5024	mutex_exit(&spa_namespace_lock);
5025
5026	return (error);
5027}
5028
5029/*
5030 * Split a set of devices from their mirrors, and create a new pool from them.
5031 */
5032int
5033spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config,
5034    nvlist_t *props, boolean_t exp)
5035{
5036	int error = 0;
5037	uint64_t txg, *glist;
5038	spa_t *newspa;
5039	uint_t c, children, lastlog;
5040	nvlist_t **child, *nvl, *tmp;
5041	dmu_tx_t *tx;
5042	char *altroot = NULL;
5043	vdev_t *rvd, **vml = NULL;			/* vdev modify list */
5044	boolean_t activate_slog;
5045
5046	ASSERT(spa_writeable(spa));
5047
5048	txg = spa_vdev_enter(spa);
5049
5050	/* clear the log and flush everything up to now */
5051	activate_slog = spa_passivate_log(spa);
5052	(void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG);
5053	error = spa_offline_log(spa);
5054	txg = spa_vdev_config_enter(spa);
5055
5056	if (activate_slog)
5057		spa_activate_log(spa);
5058
5059	if (error != 0)
5060		return (spa_vdev_exit(spa, NULL, txg, error));
5061
5062	/* check new spa name before going any further */
5063	if (spa_lookup(newname) != NULL)
5064		return (spa_vdev_exit(spa, NULL, txg, EEXIST));
5065
5066	/*
5067	 * scan through all the children to ensure they're all mirrors
5068	 */
5069	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 ||
5070	    nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child,
5071	    &children) != 0)
5072		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
5073
5074	/* first, check to ensure we've got the right child count */
5075	rvd = spa->spa_root_vdev;
5076	lastlog = 0;
5077	for (c = 0; c < rvd->vdev_children; c++) {
5078		vdev_t *vd = rvd->vdev_child[c];
5079
5080		/* don't count the holes & logs as children */
5081		if (vd->vdev_islog || vd->vdev_ishole) {
5082			if (lastlog == 0)
5083				lastlog = c;
5084			continue;
5085		}
5086
5087		lastlog = 0;
5088	}
5089	if (children != (lastlog != 0 ? lastlog : rvd->vdev_children))
5090		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
5091
5092	/* next, ensure no spare or cache devices are part of the split */
5093	if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 ||
5094	    nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0)
5095		return (spa_vdev_exit(spa, NULL, txg, EINVAL));
5096
5097	vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP);
5098	glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP);
5099
5100	/* then, loop over each vdev and validate it */
5101	for (c = 0; c < children; c++) {
5102		uint64_t is_hole = 0;
5103
5104		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
5105		    &is_hole);
5106
5107		if (is_hole != 0) {
5108			if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole ||
5109			    spa->spa_root_vdev->vdev_child[c]->vdev_islog) {
5110				continue;
5111			} else {
5112				error = SET_ERROR(EINVAL);
5113				break;
5114			}
5115		}
5116
5117		/* which disk is going to be split? */
5118		if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID,
5119		    &glist[c]) != 0) {
5120			error = SET_ERROR(EINVAL);
5121			break;
5122		}
5123
5124		/* look it up in the spa */
5125		vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE);
5126		if (vml[c] == NULL) {
5127			error = SET_ERROR(ENODEV);
5128			break;
5129		}
5130
5131		/* make sure there's nothing stopping the split */
5132		if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops ||
5133		    vml[c]->vdev_islog ||
5134		    vml[c]->vdev_ishole ||
5135		    vml[c]->vdev_isspare ||
5136		    vml[c]->vdev_isl2cache ||
5137		    !vdev_writeable(vml[c]) ||
5138		    vml[c]->vdev_children != 0 ||
5139		    vml[c]->vdev_state != VDEV_STATE_HEALTHY ||
5140		    c != spa->spa_root_vdev->vdev_child[c]->vdev_id) {
5141			error = SET_ERROR(EINVAL);
5142			break;
5143		}
5144
5145		if (vdev_dtl_required(vml[c])) {
5146			error = SET_ERROR(EBUSY);
5147			break;
5148		}
5149
5150		/* we need certain info from the top level */
5151		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY,
5152		    vml[c]->vdev_top->vdev_ms_array) == 0);
5153		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT,
5154		    vml[c]->vdev_top->vdev_ms_shift) == 0);
5155		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE,
5156		    vml[c]->vdev_top->vdev_asize) == 0);
5157		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT,
5158		    vml[c]->vdev_top->vdev_ashift) == 0);
5159
5160		/* transfer per-vdev ZAPs */
5161		ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0);
5162		VERIFY0(nvlist_add_uint64(child[c],
5163		    ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap));
5164
5165		ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0);
5166		VERIFY0(nvlist_add_uint64(child[c],
5167		    ZPOOL_CONFIG_VDEV_TOP_ZAP,
5168		    vml[c]->vdev_parent->vdev_top_zap));
5169	}
5170
5171	if (error != 0) {
5172		kmem_free(vml, children * sizeof (vdev_t *));
5173		kmem_free(glist, children * sizeof (uint64_t));
5174		return (spa_vdev_exit(spa, NULL, txg, error));
5175	}
5176
5177	/* stop writers from using the disks */
5178	for (c = 0; c < children; c++) {
5179		if (vml[c] != NULL)
5180			vml[c]->vdev_offline = B_TRUE;
5181	}
5182	vdev_reopen(spa->spa_root_vdev);
5183
5184	/*
5185	 * Temporarily record the splitting vdevs in the spa config.  This
5186	 * will disappear once the config is regenerated.
5187	 */
5188	VERIFY(nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_SLEEP) == 0);
5189	VERIFY(nvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST,
5190	    glist, children) == 0);
5191	kmem_free(glist, children * sizeof (uint64_t));
5192
5193	mutex_enter(&spa->spa_props_lock);
5194	VERIFY(nvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT,
5195	    nvl) == 0);
5196	mutex_exit(&spa->spa_props_lock);
5197	spa->spa_config_splitting = nvl;
5198	vdev_config_dirty(spa->spa_root_vdev);
5199
5200	/* configure and create the new pool */
5201	VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname) == 0);
5202	VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE,
5203	    exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE) == 0);
5204	VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_VERSION,
5205	    spa_version(spa)) == 0);
5206	VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG,
5207	    spa->spa_config_txg) == 0);
5208	VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID,
5209	    spa_generate_guid(NULL)) == 0);
5210	VERIFY0(nvlist_add_boolean(config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS));
5211	(void) nvlist_lookup_string(props,
5212	    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot);
5213
5214	/* add the new pool to the namespace */
5215	newspa = spa_add(newname, config, altroot);
5216	newspa->spa_avz_action = AVZ_ACTION_REBUILD;
5217	newspa->spa_config_txg = spa->spa_config_txg;
5218	spa_set_log_state(newspa, SPA_LOG_CLEAR);
5219
5220	/* release the spa config lock, retaining the namespace lock */
5221	spa_vdev_config_exit(spa, NULL, txg, 0, FTAG);
5222
5223	if (zio_injection_enabled)
5224		zio_handle_panic_injection(spa, FTAG, 1);
5225
5226	spa_activate(newspa, spa_mode_global);
5227	spa_async_suspend(newspa);
5228
5229	/* create the new pool from the disks of the original pool */
5230	error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE, B_TRUE);
5231	if (error)
5232		goto out;
5233
5234	/* if that worked, generate a real config for the new pool */
5235	if (newspa->spa_root_vdev != NULL) {
5236		VERIFY(nvlist_alloc(&newspa->spa_config_splitting,
5237		    NV_UNIQUE_NAME, KM_SLEEP) == 0);
5238		VERIFY(nvlist_add_uint64(newspa->spa_config_splitting,
5239		    ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa)) == 0);
5240		spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL,
5241		    B_TRUE));
5242	}
5243
5244	/* set the props */
5245	if (props != NULL) {
5246		spa_configfile_set(newspa, props, B_FALSE);
5247		error = spa_prop_set(newspa, props);
5248		if (error)
5249			goto out;
5250	}
5251
5252	/* flush everything */
5253	txg = spa_vdev_config_enter(newspa);
5254	vdev_config_dirty(newspa->spa_root_vdev);
5255	(void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG);
5256
5257	if (zio_injection_enabled)
5258		zio_handle_panic_injection(spa, FTAG, 2);
5259
5260	spa_async_resume(newspa);
5261
5262	/* finally, update the original pool's config */
5263	txg = spa_vdev_config_enter(spa);
5264	tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
5265	error = dmu_tx_assign(tx, TXG_WAIT);
5266	if (error != 0)
5267		dmu_tx_abort(tx);
5268	for (c = 0; c < children; c++) {
5269		if (vml[c] != NULL) {
5270			vdev_split(vml[c]);
5271			if (error == 0)
5272				spa_history_log_internal(spa, "detach", tx,
5273				    "vdev=%s", vml[c]->vdev_path);
5274
5275			vdev_free(vml[c]);
5276		}
5277	}
5278	spa->spa_avz_action = AVZ_ACTION_REBUILD;
5279	vdev_config_dirty(spa->spa_root_vdev);
5280	spa->spa_config_splitting = NULL;
5281	nvlist_free(nvl);
5282	if (error == 0)
5283		dmu_tx_commit(tx);
5284	(void) spa_vdev_exit(spa, NULL, txg, 0);
5285
5286	if (zio_injection_enabled)
5287		zio_handle_panic_injection(spa, FTAG, 3);
5288
5289	/* split is complete; log a history record */
5290	spa_history_log_internal(newspa, "split", NULL,
5291	    "from pool %s", spa_name(spa));
5292
5293	kmem_free(vml, children * sizeof (vdev_t *));
5294
5295	/* if we're not going to mount the filesystems in userland, export */
5296	if (exp)
5297		error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL,
5298		    B_FALSE, B_FALSE);
5299
5300	return (error);
5301
5302out:
5303	spa_unload(newspa);
5304	spa_deactivate(newspa);
5305	spa_remove(newspa);
5306
5307	txg = spa_vdev_config_enter(spa);
5308
5309	/* re-online all offlined disks */
5310	for (c = 0; c < children; c++) {
5311		if (vml[c] != NULL)
5312			vml[c]->vdev_offline = B_FALSE;
5313	}
5314	vdev_reopen(spa->spa_root_vdev);
5315
5316	nvlist_free(spa->spa_config_splitting);
5317	spa->spa_config_splitting = NULL;
5318	(void) spa_vdev_exit(spa, NULL, txg, error);
5319
5320	kmem_free(vml, children * sizeof (vdev_t *));
5321	return (error);
5322}
5323
5324static nvlist_t *
5325spa_nvlist_lookup_by_guid(nvlist_t **nvpp, int count, uint64_t target_guid)
5326{
5327	for (int i = 0; i < count; i++) {
5328		uint64_t guid;
5329
5330		VERIFY(nvlist_lookup_uint64(nvpp[i], ZPOOL_CONFIG_GUID,
5331		    &guid) == 0);
5332
5333		if (guid == target_guid)
5334			return (nvpp[i]);
5335	}
5336
5337	return (NULL);
5338}
5339
5340static void
5341spa_vdev_remove_aux(nvlist_t *config, char *name, nvlist_t **dev, int count,
5342    nvlist_t *dev_to_remove)
5343{
5344	nvlist_t **newdev = NULL;
5345
5346	if (count > 1)
5347		newdev = kmem_alloc((count - 1) * sizeof (void *), KM_SLEEP);
5348
5349	for (int i = 0, j = 0; i < count; i++) {
5350		if (dev[i] == dev_to_remove)
5351			continue;
5352		VERIFY(nvlist_dup(dev[i], &newdev[j++], KM_SLEEP) == 0);
5353	}
5354
5355	VERIFY(nvlist_remove(config, name, DATA_TYPE_NVLIST_ARRAY) == 0);
5356	VERIFY(nvlist_add_nvlist_array(config, name, newdev, count - 1) == 0);
5357
5358	for (int i = 0; i < count - 1; i++)
5359		nvlist_free(newdev[i]);
5360
5361	if (count > 1)
5362		kmem_free(newdev, (count - 1) * sizeof (void *));
5363}
5364
5365/*
5366 * Evacuate the device.
5367 */
5368static int
5369spa_vdev_remove_evacuate(spa_t *spa, vdev_t *vd)
5370{
5371	uint64_t txg;
5372	int error = 0;
5373
5374	ASSERT(MUTEX_HELD(&spa_namespace_lock));
5375	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
5376	ASSERT(vd == vd->vdev_top);
5377
5378	/*
5379	 * Evacuate the device.  We don't hold the config lock as writer
5380	 * since we need to do I/O but we do keep the
5381	 * spa_namespace_lock held.  Once this completes the device
5382	 * should no longer have any blocks allocated on it.
5383	 */
5384	if (vd->vdev_islog) {
5385		if (vd->vdev_stat.vs_alloc != 0)
5386			error = spa_offline_log(spa);
5387	} else {
5388		error = SET_ERROR(ENOTSUP);
5389	}
5390
5391	if (error)
5392		return (error);
5393
5394	/*
5395	 * The evacuation succeeded.  Remove any remaining MOS metadata
5396	 * associated with this vdev, and wait for these changes to sync.
5397	 */
5398	ASSERT0(vd->vdev_stat.vs_alloc);
5399	txg = spa_vdev_config_enter(spa);
5400	vd->vdev_removing = B_TRUE;
5401	vdev_dirty_leaves(vd, VDD_DTL, txg);
5402	vdev_config_dirty(vd);
5403	spa_vdev_config_exit(spa, NULL, txg, 0, FTAG);
5404
5405	return (0);
5406}
5407
5408/*
5409 * Complete the removal by cleaning up the namespace.
5410 */
5411static void
5412spa_vdev_remove_from_namespace(spa_t *spa, vdev_t *vd)
5413{
5414	vdev_t *rvd = spa->spa_root_vdev;
5415	uint64_t id = vd->vdev_id;
5416	boolean_t last_vdev = (id == (rvd->vdev_children - 1));
5417
5418	ASSERT(MUTEX_HELD(&spa_namespace_lock));
5419	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
5420	ASSERT(vd == vd->vdev_top);
5421
5422	/*
5423	 * Only remove any devices which are empty.
5424	 */
5425	if (vd->vdev_stat.vs_alloc != 0)
5426		return;
5427
5428	(void) vdev_label_init(vd, 0, VDEV_LABEL_REMOVE);
5429
5430	if (list_link_active(&vd->vdev_state_dirty_node))
5431		vdev_state_clean(vd);
5432	if (list_link_active(&vd->vdev_config_dirty_node))
5433		vdev_config_clean(vd);
5434
5435	vdev_free(vd);
5436
5437	if (last_vdev) {
5438		vdev_compact_children(rvd);
5439	} else {
5440		vd = vdev_alloc_common(spa, id, 0, &vdev_hole_ops);
5441		vdev_add_child(rvd, vd);
5442	}
5443	vdev_config_dirty(rvd);
5444
5445	/*
5446	 * Reassess the health of our root vdev.
5447	 */
5448	vdev_reopen(rvd);
5449}
5450
5451/*
5452 * Remove a device from the pool -
5453 *
5454 * Removing a device from the vdev namespace requires several steps
5455 * and can take a significant amount of time.  As a result we use
5456 * the spa_vdev_config_[enter/exit] functions which allow us to
5457 * grab and release the spa_config_lock while still holding the namespace
5458 * lock.  During each step the configuration is synced out.
5459 *
5460 * Currently, this supports removing only hot spares, slogs, and level 2 ARC
5461 * devices.
5462 */
5463int
5464spa_vdev_remove(spa_t *spa, uint64_t guid, boolean_t unspare)
5465{
5466	vdev_t *vd;
5467	sysevent_t *ev = NULL;
5468	metaslab_group_t *mg;
5469	nvlist_t **spares, **l2cache, *nv;
5470	uint64_t txg = 0;
5471	uint_t nspares, nl2cache;
5472	int error = 0;
5473	boolean_t locked = MUTEX_HELD(&spa_namespace_lock);
5474
5475	ASSERT(spa_writeable(spa));
5476
5477	if (!locked)
5478		txg = spa_vdev_enter(spa);
5479
5480	vd = spa_lookup_by_guid(spa, guid, B_FALSE);
5481
5482	if (spa->spa_spares.sav_vdevs != NULL &&
5483	    nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
5484	    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0 &&
5485	    (nv = spa_nvlist_lookup_by_guid(spares, nspares, guid)) != NULL) {
5486		/*
5487		 * Only remove the hot spare if it's not currently in use
5488		 * in this pool.
5489		 */
5490		if (vd == NULL || unspare) {
5491			if (vd == NULL)
5492				vd = spa_lookup_by_guid(spa, guid, B_TRUE);
5493			ev = spa_event_create(spa, vd, NULL,
5494			    ESC_ZFS_VDEV_REMOVE_AUX);
5495			spa_vdev_remove_aux(spa->spa_spares.sav_config,
5496			    ZPOOL_CONFIG_SPARES, spares, nspares, nv);
5497			spa_load_spares(spa);
5498			spa->spa_spares.sav_sync = B_TRUE;
5499		} else {
5500			error = SET_ERROR(EBUSY);
5501		}
5502	} else if (spa->spa_l2cache.sav_vdevs != NULL &&
5503	    nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config,
5504	    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0 &&
5505	    (nv = spa_nvlist_lookup_by_guid(l2cache, nl2cache, guid)) != NULL) {
5506		/*
5507		 * Cache devices can always be removed.
5508		 */
5509		vd = spa_lookup_by_guid(spa, guid, B_TRUE);
5510		ev = spa_event_create(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE_AUX);
5511		spa_vdev_remove_aux(spa->spa_l2cache.sav_config,
5512		    ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache, nv);
5513		spa_load_l2cache(spa);
5514		spa->spa_l2cache.sav_sync = B_TRUE;
5515	} else if (vd != NULL && vd->vdev_islog) {
5516		ASSERT(!locked);
5517		ASSERT(vd == vd->vdev_top);
5518
5519		mg = vd->vdev_mg;
5520
5521		/*
5522		 * Stop allocating from this vdev.
5523		 */
5524		metaslab_group_passivate(mg);
5525
5526		/*
5527		 * Wait for the youngest allocations and frees to sync,
5528		 * and then wait for the deferral of those frees to finish.
5529		 */
5530		spa_vdev_config_exit(spa, NULL,
5531		    txg + TXG_CONCURRENT_STATES + TXG_DEFER_SIZE, 0, FTAG);
5532
5533		/*
5534		 * Attempt to evacuate the vdev.
5535		 */
5536		error = spa_vdev_remove_evacuate(spa, vd);
5537
5538		txg = spa_vdev_config_enter(spa);
5539
5540		/*
5541		 * If we couldn't evacuate the vdev, unwind.
5542		 */
5543		if (error) {
5544			metaslab_group_activate(mg);
5545			return (spa_vdev_exit(spa, NULL, txg, error));
5546		}
5547
5548		/*
5549		 * Clean up the vdev namespace.
5550		 */
5551		ev = spa_event_create(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE_DEV);
5552		spa_vdev_remove_from_namespace(spa, vd);
5553
5554	} else if (vd != NULL) {
5555		/*
5556		 * Normal vdevs cannot be removed (yet).
5557		 */
5558		error = SET_ERROR(ENOTSUP);
5559	} else {
5560		/*
5561		 * There is no vdev of any kind with the specified guid.
5562		 */
5563		error = SET_ERROR(ENOENT);
5564	}
5565
5566	if (!locked)
5567		error = spa_vdev_exit(spa, NULL, txg, error);
5568
5569	if (ev)
5570		spa_event_post(ev);
5571
5572	return (error);
5573}
5574
5575/*
5576 * Find any device that's done replacing, or a vdev marked 'unspare' that's
5577 * currently spared, so we can detach it.
5578 */
5579static vdev_t *
5580spa_vdev_resilver_done_hunt(vdev_t *vd)
5581{
5582	vdev_t *newvd, *oldvd;
5583
5584	for (int c = 0; c < vd->vdev_children; c++) {
5585		oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]);
5586		if (oldvd != NULL)
5587			return (oldvd);
5588	}
5589
5590	/*
5591	 * Check for a completed replacement.  We always consider the first
5592	 * vdev in the list to be the oldest vdev, and the last one to be
5593	 * the newest (see spa_vdev_attach() for how that works).  In
5594	 * the case where the newest vdev is faulted, we will not automatically
5595	 * remove it after a resilver completes.  This is OK as it will require
5596	 * user intervention to determine which disk the admin wishes to keep.
5597	 */
5598	if (vd->vdev_ops == &vdev_replacing_ops) {
5599		ASSERT(vd->vdev_children > 1);
5600
5601		newvd = vd->vdev_child[vd->vdev_children - 1];
5602		oldvd = vd->vdev_child[0];
5603
5604		if (vdev_dtl_empty(newvd, DTL_MISSING) &&
5605		    vdev_dtl_empty(newvd, DTL_OUTAGE) &&
5606		    !vdev_dtl_required(oldvd))
5607			return (oldvd);
5608	}
5609
5610	/*
5611	 * Check for a completed resilver with the 'unspare' flag set.
5612	 */
5613	if (vd->vdev_ops == &vdev_spare_ops) {
5614		vdev_t *first = vd->vdev_child[0];
5615		vdev_t *last = vd->vdev_child[vd->vdev_children - 1];
5616
5617		if (last->vdev_unspare) {
5618			oldvd = first;
5619			newvd = last;
5620		} else if (first->vdev_unspare) {
5621			oldvd = last;
5622			newvd = first;
5623		} else {
5624			oldvd = NULL;
5625		}
5626
5627		if (oldvd != NULL &&
5628		    vdev_dtl_empty(newvd, DTL_MISSING) &&
5629		    vdev_dtl_empty(newvd, DTL_OUTAGE) &&
5630		    !vdev_dtl_required(oldvd))
5631			return (oldvd);
5632
5633		/*
5634		 * If there are more than two spares attached to a disk,
5635		 * and those spares are not required, then we want to
5636		 * attempt to free them up now so that they can be used
5637		 * by other pools.  Once we're back down to a single
5638		 * disk+spare, we stop removing them.
5639		 */
5640		if (vd->vdev_children > 2) {
5641			newvd = vd->vdev_child[1];
5642
5643			if (newvd->vdev_isspare && last->vdev_isspare &&
5644			    vdev_dtl_empty(last, DTL_MISSING) &&
5645			    vdev_dtl_empty(last, DTL_OUTAGE) &&
5646			    !vdev_dtl_required(newvd))
5647				return (newvd);
5648		}
5649	}
5650
5651	return (NULL);
5652}
5653
5654static void
5655spa_vdev_resilver_done(spa_t *spa)
5656{
5657	vdev_t *vd, *pvd, *ppvd;
5658	uint64_t guid, sguid, pguid, ppguid;
5659
5660	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
5661
5662	while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) {
5663		pvd = vd->vdev_parent;
5664		ppvd = pvd->vdev_parent;
5665		guid = vd->vdev_guid;
5666		pguid = pvd->vdev_guid;
5667		ppguid = ppvd->vdev_guid;
5668		sguid = 0;
5669		/*
5670		 * If we have just finished replacing a hot spared device, then
5671		 * we need to detach the parent's first child (the original hot
5672		 * spare) as well.
5673		 */
5674		if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 &&
5675		    ppvd->vdev_children == 2) {
5676			ASSERT(pvd->vdev_ops == &vdev_replacing_ops);
5677			sguid = ppvd->vdev_child[1]->vdev_guid;
5678		}
5679		ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd));
5680
5681		spa_config_exit(spa, SCL_ALL, FTAG);
5682		if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0)
5683			return;
5684		if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0)
5685			return;
5686		spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
5687	}
5688
5689	spa_config_exit(spa, SCL_ALL, FTAG);
5690}
5691
5692/*
5693 * Update the stored path or FRU for this vdev.
5694 */
5695int
5696spa_vdev_set_common(spa_t *spa, uint64_t guid, const char *value,
5697    boolean_t ispath)
5698{
5699	vdev_t *vd;
5700	boolean_t sync = B_FALSE;
5701
5702	ASSERT(spa_writeable(spa));
5703
5704	spa_vdev_state_enter(spa, SCL_ALL);
5705
5706	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
5707		return (spa_vdev_state_exit(spa, NULL, ENOENT));
5708
5709	if (!vd->vdev_ops->vdev_op_leaf)
5710		return (spa_vdev_state_exit(spa, NULL, ENOTSUP));
5711
5712	if (ispath) {
5713		if (strcmp(value, vd->vdev_path) != 0) {
5714			spa_strfree(vd->vdev_path);
5715			vd->vdev_path = spa_strdup(value);
5716			sync = B_TRUE;
5717		}
5718	} else {
5719		if (vd->vdev_fru == NULL) {
5720			vd->vdev_fru = spa_strdup(value);
5721			sync = B_TRUE;
5722		} else if (strcmp(value, vd->vdev_fru) != 0) {
5723			spa_strfree(vd->vdev_fru);
5724			vd->vdev_fru = spa_strdup(value);
5725			sync = B_TRUE;
5726		}
5727	}
5728
5729	return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0));
5730}
5731
5732int
5733spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath)
5734{
5735	return (spa_vdev_set_common(spa, guid, newpath, B_TRUE));
5736}
5737
5738int
5739spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru)
5740{
5741	return (spa_vdev_set_common(spa, guid, newfru, B_FALSE));
5742}
5743
5744/*
5745 * ==========================================================================
5746 * SPA Scanning
5747 * ==========================================================================
5748 */
5749
5750int
5751spa_scan_stop(spa_t *spa)
5752{
5753	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
5754	if (dsl_scan_resilvering(spa->spa_dsl_pool))
5755		return (SET_ERROR(EBUSY));
5756	return (dsl_scan_cancel(spa->spa_dsl_pool));
5757}
5758
5759int
5760spa_scan(spa_t *spa, pool_scan_func_t func)
5761{
5762	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
5763
5764	if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE)
5765		return (SET_ERROR(ENOTSUP));
5766
5767	/*
5768	 * If a resilver was requested, but there is no DTL on a
5769	 * writeable leaf device, we have nothing to do.
5770	 */
5771	if (func == POOL_SCAN_RESILVER &&
5772	    !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) {
5773		spa_async_request(spa, SPA_ASYNC_RESILVER_DONE);
5774		return (0);
5775	}
5776
5777	return (dsl_scan(spa->spa_dsl_pool, func));
5778}
5779
5780/*
5781 * ==========================================================================
5782 * SPA async task processing
5783 * ==========================================================================
5784 */
5785
5786static void
5787spa_async_remove(spa_t *spa, vdev_t *vd)
5788{
5789	if (vd->vdev_remove_wanted) {
5790		vd->vdev_remove_wanted = B_FALSE;
5791		vd->vdev_delayed_close = B_FALSE;
5792		vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE);
5793
5794		/*
5795		 * We want to clear the stats, but we don't want to do a full
5796		 * vdev_clear() as that will cause us to throw away
5797		 * degraded/faulted state as well as attempt to reopen the
5798		 * device, all of which is a waste.
5799		 */
5800		vd->vdev_stat.vs_read_errors = 0;
5801		vd->vdev_stat.vs_write_errors = 0;
5802		vd->vdev_stat.vs_checksum_errors = 0;
5803
5804		vdev_state_dirty(vd->vdev_top);
5805	}
5806
5807	for (int c = 0; c < vd->vdev_children; c++)
5808		spa_async_remove(spa, vd->vdev_child[c]);
5809}
5810
5811static void
5812spa_async_probe(spa_t *spa, vdev_t *vd)
5813{
5814	if (vd->vdev_probe_wanted) {
5815		vd->vdev_probe_wanted = B_FALSE;
5816		vdev_reopen(vd);	/* vdev_open() does the actual probe */
5817	}
5818
5819	for (int c = 0; c < vd->vdev_children; c++)
5820		spa_async_probe(spa, vd->vdev_child[c]);
5821}
5822
5823static void
5824spa_async_autoexpand(spa_t *spa, vdev_t *vd)
5825{
5826	sysevent_id_t eid;
5827	nvlist_t *attr;
5828	char *physpath;
5829
5830	if (!spa->spa_autoexpand)
5831		return;
5832
5833	for (int c = 0; c < vd->vdev_children; c++) {
5834		vdev_t *cvd = vd->vdev_child[c];
5835		spa_async_autoexpand(spa, cvd);
5836	}
5837
5838	if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL)
5839		return;
5840
5841	physpath = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
5842	(void) snprintf(physpath, MAXPATHLEN, "/devices%s", vd->vdev_physpath);
5843
5844	VERIFY(nvlist_alloc(&attr, NV_UNIQUE_NAME, KM_SLEEP) == 0);
5845	VERIFY(nvlist_add_string(attr, DEV_PHYS_PATH, physpath) == 0);
5846
5847	(void) ddi_log_sysevent(zfs_dip, SUNW_VENDOR, EC_DEV_STATUS,
5848	    ESC_DEV_DLE, attr, &eid, DDI_SLEEP);
5849
5850	nvlist_free(attr);
5851	kmem_free(physpath, MAXPATHLEN);
5852}
5853
5854static void
5855spa_async_thread(spa_t *spa)
5856{
5857	int tasks;
5858
5859	ASSERT(spa->spa_sync_on);
5860
5861	mutex_enter(&spa->spa_async_lock);
5862	tasks = spa->spa_async_tasks;
5863	spa->spa_async_tasks = 0;
5864	mutex_exit(&spa->spa_async_lock);
5865
5866	/*
5867	 * See if the config needs to be updated.
5868	 */
5869	if (tasks & SPA_ASYNC_CONFIG_UPDATE) {
5870		uint64_t old_space, new_space;
5871
5872		mutex_enter(&spa_namespace_lock);
5873		old_space = metaslab_class_get_space(spa_normal_class(spa));
5874		spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
5875		new_space = metaslab_class_get_space(spa_normal_class(spa));
5876		mutex_exit(&spa_namespace_lock);
5877
5878		/*
5879		 * If the pool grew as a result of the config update,
5880		 * then log an internal history event.
5881		 */
5882		if (new_space != old_space) {
5883			spa_history_log_internal(spa, "vdev online", NULL,
5884			    "pool '%s' size: %llu(+%llu)",
5885			    spa_name(spa), new_space, new_space - old_space);
5886		}
5887	}
5888
5889	/*
5890	 * See if any devices need to be marked REMOVED.
5891	 */
5892	if (tasks & SPA_ASYNC_REMOVE) {
5893		spa_vdev_state_enter(spa, SCL_NONE);
5894		spa_async_remove(spa, spa->spa_root_vdev);
5895		for (int i = 0; i < spa->spa_l2cache.sav_count; i++)
5896			spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i]);
5897		for (int i = 0; i < spa->spa_spares.sav_count; i++)
5898			spa_async_remove(spa, spa->spa_spares.sav_vdevs[i]);
5899		(void) spa_vdev_state_exit(spa, NULL, 0);
5900	}
5901
5902	if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) {
5903		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
5904		spa_async_autoexpand(spa, spa->spa_root_vdev);
5905		spa_config_exit(spa, SCL_CONFIG, FTAG);
5906	}
5907
5908	/*
5909	 * See if any devices need to be probed.
5910	 */
5911	if (tasks & SPA_ASYNC_PROBE) {
5912		spa_vdev_state_enter(spa, SCL_NONE);
5913		spa_async_probe(spa, spa->spa_root_vdev);
5914		(void) spa_vdev_state_exit(spa, NULL, 0);
5915	}
5916
5917	/*
5918	 * If any devices are done replacing, detach them.
5919	 */
5920	if (tasks & SPA_ASYNC_RESILVER_DONE)
5921		spa_vdev_resilver_done(spa);
5922
5923	/*
5924	 * Kick off a resilver.
5925	 */
5926	if (tasks & SPA_ASYNC_RESILVER)
5927		dsl_resilver_restart(spa->spa_dsl_pool, 0);
5928
5929	/*
5930	 * Let the world know that we're done.
5931	 */
5932	mutex_enter(&spa->spa_async_lock);
5933	spa->spa_async_thread = NULL;
5934	cv_broadcast(&spa->spa_async_cv);
5935	mutex_exit(&spa->spa_async_lock);
5936	thread_exit();
5937}
5938
5939void
5940spa_async_suspend(spa_t *spa)
5941{
5942	mutex_enter(&spa->spa_async_lock);
5943	spa->spa_async_suspended++;
5944	while (spa->spa_async_thread != NULL)
5945		cv_wait(&spa->spa_async_cv, &spa->spa_async_lock);
5946	mutex_exit(&spa->spa_async_lock);
5947}
5948
5949void
5950spa_async_resume(spa_t *spa)
5951{
5952	mutex_enter(&spa->spa_async_lock);
5953	ASSERT(spa->spa_async_suspended != 0);
5954	spa->spa_async_suspended--;
5955	mutex_exit(&spa->spa_async_lock);
5956}
5957
5958static boolean_t
5959spa_async_tasks_pending(spa_t *spa)
5960{
5961	uint_t non_config_tasks;
5962	uint_t config_task;
5963	boolean_t config_task_suspended;
5964
5965	non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE;
5966	config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE;
5967	if (spa->spa_ccw_fail_time == 0) {
5968		config_task_suspended = B_FALSE;
5969	} else {
5970		config_task_suspended =
5971		    (gethrtime() - spa->spa_ccw_fail_time) <
5972		    (zfs_ccw_retry_interval * NANOSEC);
5973	}
5974
5975	return (non_config_tasks || (config_task && !config_task_suspended));
5976}
5977
5978static void
5979spa_async_dispatch(spa_t *spa)
5980{
5981	mutex_enter(&spa->spa_async_lock);
5982	if (spa_async_tasks_pending(spa) &&
5983	    !spa->spa_async_suspended &&
5984	    spa->spa_async_thread == NULL &&
5985	    rootdir != NULL)
5986		spa->spa_async_thread = thread_create(NULL, 0,
5987		    spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri);
5988	mutex_exit(&spa->spa_async_lock);
5989}
5990
5991void
5992spa_async_request(spa_t *spa, int task)
5993{
5994	zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task);
5995	mutex_enter(&spa->spa_async_lock);
5996	spa->spa_async_tasks |= task;
5997	mutex_exit(&spa->spa_async_lock);
5998}
5999
6000/*
6001 * ==========================================================================
6002 * SPA syncing routines
6003 * ==========================================================================
6004 */
6005
6006static int
6007bpobj_enqueue_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
6008{
6009	bpobj_t *bpo = arg;
6010	bpobj_enqueue(bpo, bp, tx);
6011	return (0);
6012}
6013
6014static int
6015spa_free_sync_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
6016{
6017	zio_t *zio = arg;
6018
6019	zio_nowait(zio_free_sync(zio, zio->io_spa, dmu_tx_get_txg(tx), bp,
6020	    zio->io_flags));
6021	return (0);