spa.c revision 416e0cd82df71e380f21199abde8c5f7a18bac51
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#pragma ident	"%Z%%M%	%I%	%E% SMI"
28
29/*
30 * This file contains all the routines used when modifying on-disk SPA state.
31 * This includes opening, importing, destroying, exporting a pool, and syncing a
32 * pool.
33 */
34
35#include <sys/zfs_context.h>
36#include <sys/fm/fs/zfs.h>
37#include <sys/spa_impl.h>
38#include <sys/zio.h>
39#include <sys/zio_checksum.h>
40#include <sys/zio_compress.h>
41#include <sys/dmu.h>
42#include <sys/dmu_tx.h>
43#include <sys/zap.h>
44#include <sys/zil.h>
45#include <sys/vdev_impl.h>
46#include <sys/metaslab.h>
47#include <sys/uberblock_impl.h>
48#include <sys/txg.h>
49#include <sys/avl.h>
50#include <sys/dmu_traverse.h>
51#include <sys/unique.h>
52#include <sys/dsl_pool.h>
53#include <sys/dsl_dir.h>
54#include <sys/dsl_prop.h>
55#include <sys/fs/zfs.h>
56#include <sys/callb.h>
57
58int zio_taskq_threads = 8;
59
60/*
61 * ==========================================================================
62 * SPA state manipulation (open/create/destroy/import/export)
63 * ==========================================================================
64 */
65
66static int
67spa_error_entry_compare(const void *a, const void *b)
68{
69	spa_error_entry_t *sa = (spa_error_entry_t *)a;
70	spa_error_entry_t *sb = (spa_error_entry_t *)b;
71	int ret;
72
73	ret = bcmp(&sa->se_bookmark, &sb->se_bookmark,
74	    sizeof (zbookmark_t));
75
76	if (ret < 0)
77		return (-1);
78	else if (ret > 0)
79		return (1);
80	else
81		return (0);
82}
83
84/*
85 * Utility function which retrieves copies of the current logs and
86 * re-initializes them in the process.
87 */
88void
89spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub)
90{
91	ASSERT(MUTEX_HELD(&spa->spa_errlist_lock));
92
93	bcopy(&spa->spa_errlist_last, last, sizeof (avl_tree_t));
94	bcopy(&spa->spa_errlist_scrub, scrub, sizeof (avl_tree_t));
95
96	avl_create(&spa->spa_errlist_scrub,
97	    spa_error_entry_compare, sizeof (spa_error_entry_t),
98	    offsetof(spa_error_entry_t, se_avl));
99	avl_create(&spa->spa_errlist_last,
100	    spa_error_entry_compare, sizeof (spa_error_entry_t),
101	    offsetof(spa_error_entry_t, se_avl));
102}
103
104/*
105 * Activate an uninitialized pool.
106 */
107static void
108spa_activate(spa_t *spa)
109{
110	int t;
111
112	ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED);
113
114	spa->spa_state = POOL_STATE_ACTIVE;
115
116	spa->spa_normal_class = metaslab_class_create();
117
118	for (t = 0; t < ZIO_TYPES; t++) {
119		spa->spa_zio_issue_taskq[t] = taskq_create("spa_zio_issue",
120		    zio_taskq_threads, maxclsyspri, 50, INT_MAX,
121		    TASKQ_PREPOPULATE);
122		spa->spa_zio_intr_taskq[t] = taskq_create("spa_zio_intr",
123		    zio_taskq_threads, maxclsyspri, 50, INT_MAX,
124		    TASKQ_PREPOPULATE);
125	}
126
127	rw_init(&spa->spa_traverse_lock, NULL, RW_DEFAULT, NULL);
128
129	mutex_init(&spa->spa_async_lock, NULL, MUTEX_DEFAULT, NULL);
130	mutex_init(&spa->spa_config_cache_lock, NULL, MUTEX_DEFAULT, NULL);
131	mutex_init(&spa->spa_scrub_lock, NULL, MUTEX_DEFAULT, NULL);
132	mutex_init(&spa->spa_errlog_lock, NULL, MUTEX_DEFAULT, NULL);
133	mutex_init(&spa->spa_errlist_lock, NULL, MUTEX_DEFAULT, NULL);
134	mutex_init(&spa->spa_config_lock.scl_lock, NULL, MUTEX_DEFAULT, NULL);
135	mutex_init(&spa->spa_sync_bplist.bpl_lock, NULL, MUTEX_DEFAULT, NULL);
136	mutex_init(&spa->spa_history_lock, NULL, MUTEX_DEFAULT, NULL);
137
138	list_create(&spa->spa_dirty_list, sizeof (vdev_t),
139	    offsetof(vdev_t, vdev_dirty_node));
140
141	txg_list_create(&spa->spa_vdev_txg_list,
142	    offsetof(struct vdev, vdev_txg_node));
143
144	avl_create(&spa->spa_errlist_scrub,
145	    spa_error_entry_compare, sizeof (spa_error_entry_t),
146	    offsetof(spa_error_entry_t, se_avl));
147	avl_create(&spa->spa_errlist_last,
148	    spa_error_entry_compare, sizeof (spa_error_entry_t),
149	    offsetof(spa_error_entry_t, se_avl));
150}
151
152/*
153 * Opposite of spa_activate().
154 */
155static void
156spa_deactivate(spa_t *spa)
157{
158	int t;
159
160	ASSERT(spa->spa_sync_on == B_FALSE);
161	ASSERT(spa->spa_dsl_pool == NULL);
162	ASSERT(spa->spa_root_vdev == NULL);
163
164	ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED);
165
166	txg_list_destroy(&spa->spa_vdev_txg_list);
167
168	list_destroy(&spa->spa_dirty_list);
169
170	rw_destroy(&spa->spa_traverse_lock);
171
172	for (t = 0; t < ZIO_TYPES; t++) {
173		taskq_destroy(spa->spa_zio_issue_taskq[t]);
174		taskq_destroy(spa->spa_zio_intr_taskq[t]);
175		spa->spa_zio_issue_taskq[t] = NULL;
176		spa->spa_zio_intr_taskq[t] = NULL;
177	}
178
179	metaslab_class_destroy(spa->spa_normal_class);
180	spa->spa_normal_class = NULL;
181
182	/*
183	 * If this was part of an import or the open otherwise failed, we may
184	 * still have errors left in the queues.  Empty them just in case.
185	 */
186	spa_errlog_drain(spa);
187
188	avl_destroy(&spa->spa_errlist_scrub);
189	avl_destroy(&spa->spa_errlist_last);
190
191	spa->spa_state = POOL_STATE_UNINITIALIZED;
192}
193
194/*
195 * Verify a pool configuration, and construct the vdev tree appropriately.  This
196 * will create all the necessary vdevs in the appropriate layout, with each vdev
197 * in the CLOSED state.  This will prep the pool before open/creation/import.
198 * All vdev validation is done by the vdev_alloc() routine.
199 */
200static int
201spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent,
202    uint_t id, int atype)
203{
204	nvlist_t **child;
205	uint_t c, children;
206	int error;
207
208	if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0)
209		return (error);
210
211	if ((*vdp)->vdev_ops->vdev_op_leaf)
212		return (0);
213
214	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
215	    &child, &children) != 0) {
216		vdev_free(*vdp);
217		*vdp = NULL;
218		return (EINVAL);
219	}
220
221	for (c = 0; c < children; c++) {
222		vdev_t *vd;
223		if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c,
224		    atype)) != 0) {
225			vdev_free(*vdp);
226			*vdp = NULL;
227			return (error);
228		}
229	}
230
231	ASSERT(*vdp != NULL);
232
233	return (0);
234}
235
236/*
237 * Opposite of spa_load().
238 */
239static void
240spa_unload(spa_t *spa)
241{
242	int i;
243
244	/*
245	 * Stop async tasks.
246	 */
247	spa_async_suspend(spa);
248
249	/*
250	 * Stop syncing.
251	 */
252	if (spa->spa_sync_on) {
253		txg_sync_stop(spa->spa_dsl_pool);
254		spa->spa_sync_on = B_FALSE;
255	}
256
257	/*
258	 * Wait for any outstanding prefetch I/O to complete.
259	 */
260	spa_config_enter(spa, RW_WRITER, FTAG);
261	spa_config_exit(spa, FTAG);
262
263	/*
264	 * Close the dsl pool.
265	 */
266	if (spa->spa_dsl_pool) {
267		dsl_pool_close(spa->spa_dsl_pool);
268		spa->spa_dsl_pool = NULL;
269	}
270
271	/*
272	 * Close all vdevs.
273	 */
274	if (spa->spa_root_vdev)
275		vdev_free(spa->spa_root_vdev);
276	ASSERT(spa->spa_root_vdev == NULL);
277
278	for (i = 0; i < spa->spa_nspares; i++)
279		vdev_free(spa->spa_spares[i]);
280	if (spa->spa_spares) {
281		kmem_free(spa->spa_spares, spa->spa_nspares * sizeof (void *));
282		spa->spa_spares = NULL;
283	}
284	if (spa->spa_sparelist) {
285		nvlist_free(spa->spa_sparelist);
286		spa->spa_sparelist = NULL;
287	}
288
289	spa->spa_async_suspended = 0;
290}
291
292/*
293 * Load (or re-load) the current list of vdevs describing the active spares for
294 * this pool.  When this is called, we have some form of basic information in
295 * 'spa_sparelist'.  We parse this into vdevs, try to open them, and then
296 * re-generate a more complete list including status information.
297 */
298static void
299spa_load_spares(spa_t *spa)
300{
301	nvlist_t **spares;
302	uint_t nspares;
303	int i;
304
305	/*
306	 * First, close and free any existing spare vdevs.
307	 */
308	for (i = 0; i < spa->spa_nspares; i++) {
309		vdev_close(spa->spa_spares[i]);
310		vdev_free(spa->spa_spares[i]);
311	}
312	if (spa->spa_spares)
313		kmem_free(spa->spa_spares, spa->spa_nspares * sizeof (void *));
314
315	if (spa->spa_sparelist == NULL)
316		nspares = 0;
317	else
318		VERIFY(nvlist_lookup_nvlist_array(spa->spa_sparelist,
319		    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0);
320
321	spa->spa_nspares = (int)nspares;
322	spa->spa_spares = NULL;
323
324	if (nspares == 0)
325		return;
326
327	/*
328	 * Construct the array of vdevs, opening them to get status in the
329	 * process.
330	 */
331	spa->spa_spares = kmem_alloc(nspares * sizeof (void *), KM_SLEEP);
332	for (i = 0; i < spa->spa_nspares; i++) {
333		vdev_t *vd;
334
335		VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0,
336		    VDEV_ALLOC_SPARE) == 0);
337		ASSERT(vd != NULL);
338
339		spa->spa_spares[i] = vd;
340
341		if (vdev_open(vd) != 0)
342			continue;
343
344		vd->vdev_top = vd;
345		(void) vdev_validate_spare(vd);
346	}
347
348	/*
349	 * Recompute the stashed list of spares, with status information
350	 * this time.
351	 */
352	VERIFY(nvlist_remove(spa->spa_sparelist, ZPOOL_CONFIG_SPARES,
353	    DATA_TYPE_NVLIST_ARRAY) == 0);
354
355	spares = kmem_alloc(spa->spa_nspares * sizeof (void *), KM_SLEEP);
356	for (i = 0; i < spa->spa_nspares; i++)
357		spares[i] = vdev_config_generate(spa, spa->spa_spares[i],
358		    B_TRUE, B_TRUE);
359	VERIFY(nvlist_add_nvlist_array(spa->spa_sparelist, ZPOOL_CONFIG_SPARES,
360	    spares, spa->spa_nspares) == 0);
361	for (i = 0; i < spa->spa_nspares; i++)
362		nvlist_free(spares[i]);
363	kmem_free(spares, spa->spa_nspares * sizeof (void *));
364}
365
366static int
367load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value)
368{
369	dmu_buf_t *db;
370	char *packed = NULL;
371	size_t nvsize = 0;
372	int error;
373	*value = NULL;
374
375	VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db));
376	nvsize = *(uint64_t *)db->db_data;
377	dmu_buf_rele(db, FTAG);
378
379	packed = kmem_alloc(nvsize, KM_SLEEP);
380	error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed);
381	if (error == 0)
382		error = nvlist_unpack(packed, nvsize, value, 0);
383	kmem_free(packed, nvsize);
384
385	return (error);
386}
387
388/*
389 * Load an existing storage pool, using the pool's builtin spa_config as a
390 * source of configuration information.
391 */
392static int
393spa_load(spa_t *spa, nvlist_t *config, spa_load_state_t state, int mosconfig)
394{
395	int error = 0;
396	nvlist_t *nvroot = NULL;
397	vdev_t *rvd;
398	uberblock_t *ub = &spa->spa_uberblock;
399	uint64_t config_cache_txg = spa->spa_config_txg;
400	uint64_t pool_guid;
401	uint64_t version;
402	zio_t *zio;
403
404	spa->spa_load_state = state;
405
406	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvroot) ||
407	    nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid)) {
408		error = EINVAL;
409		goto out;
410	}
411
412	/*
413	 * Versioning wasn't explicitly added to the label until later, so if
414	 * it's not present treat it as the initial version.
415	 */
416	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION, &version) != 0)
417		version = ZFS_VERSION_INITIAL;
418
419	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG,
420	    &spa->spa_config_txg);
421
422	if ((state == SPA_LOAD_IMPORT || state == SPA_LOAD_TRYIMPORT) &&
423	    spa_guid_exists(pool_guid, 0)) {
424		error = EEXIST;
425		goto out;
426	}
427
428	spa->spa_load_guid = pool_guid;
429
430	/*
431	 * Parse the configuration into a vdev tree.  We explicitly set the
432	 * value that will be returned by spa_version() since parsing the
433	 * configuration requires knowing the version number.
434	 */
435	spa_config_enter(spa, RW_WRITER, FTAG);
436	spa->spa_ubsync.ub_version = version;
437	error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_LOAD);
438	spa_config_exit(spa, FTAG);
439
440	if (error != 0)
441		goto out;
442
443	ASSERT(spa->spa_root_vdev == rvd);
444	ASSERT(spa_guid(spa) == pool_guid);
445
446	/*
447	 * Try to open all vdevs, loading each label in the process.
448	 */
449	if (vdev_open(rvd) != 0) {
450		error = ENXIO;
451		goto out;
452	}
453
454	/*
455	 * Validate the labels for all leaf vdevs.  We need to grab the config
456	 * lock because all label I/O is done with the ZIO_FLAG_CONFIG_HELD
457	 * flag.
458	 */
459	spa_config_enter(spa, RW_READER, FTAG);
460	error = vdev_validate(rvd);
461	spa_config_exit(spa, FTAG);
462
463	if (error != 0) {
464		error = EBADF;
465		goto out;
466	}
467
468	if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) {
469		error = ENXIO;
470		goto out;
471	}
472
473	/*
474	 * Find the best uberblock.
475	 */
476	bzero(ub, sizeof (uberblock_t));
477
478	zio = zio_root(spa, NULL, NULL,
479	    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE);
480	vdev_uberblock_load(zio, rvd, ub);
481	error = zio_wait(zio);
482
483	/*
484	 * If we weren't able to find a single valid uberblock, return failure.
485	 */
486	if (ub->ub_txg == 0) {
487		vdev_set_state(rvd, B_TRUE, VDEV_STATE_CANT_OPEN,
488		    VDEV_AUX_CORRUPT_DATA);
489		error = ENXIO;
490		goto out;
491	}
492
493	/*
494	 * If the pool is newer than the code, we can't open it.
495	 */
496	if (ub->ub_version > ZFS_VERSION) {
497		vdev_set_state(rvd, B_TRUE, VDEV_STATE_CANT_OPEN,
498		    VDEV_AUX_VERSION_NEWER);
499		error = ENOTSUP;
500		goto out;
501	}
502
503	/*
504	 * If the vdev guid sum doesn't match the uberblock, we have an
505	 * incomplete configuration.
506	 */
507	if (rvd->vdev_guid_sum != ub->ub_guid_sum && mosconfig) {
508		vdev_set_state(rvd, B_TRUE, VDEV_STATE_CANT_OPEN,
509		    VDEV_AUX_BAD_GUID_SUM);
510		error = ENXIO;
511		goto out;
512	}
513
514	/*
515	 * Initialize internal SPA structures.
516	 */
517	spa->spa_state = POOL_STATE_ACTIVE;
518	spa->spa_ubsync = spa->spa_uberblock;
519	spa->spa_first_txg = spa_last_synced_txg(spa) + 1;
520	error = dsl_pool_open(spa, spa->spa_first_txg, &spa->spa_dsl_pool);
521	if (error) {
522		vdev_set_state(rvd, B_TRUE, VDEV_STATE_CANT_OPEN,
523		    VDEV_AUX_CORRUPT_DATA);
524		goto out;
525	}
526	spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset;
527
528	if (zap_lookup(spa->spa_meta_objset,
529	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG,
530	    sizeof (uint64_t), 1, &spa->spa_config_object) != 0) {
531		vdev_set_state(rvd, B_TRUE, VDEV_STATE_CANT_OPEN,
532		    VDEV_AUX_CORRUPT_DATA);
533		error = EIO;
534		goto out;
535	}
536
537	if (!mosconfig) {
538		nvlist_t *newconfig;
539
540		if (load_nvlist(spa, spa->spa_config_object, &newconfig) != 0) {
541			vdev_set_state(rvd, B_TRUE, VDEV_STATE_CANT_OPEN,
542			    VDEV_AUX_CORRUPT_DATA);
543			error = EIO;
544			goto out;
545		}
546
547		spa_config_set(spa, newconfig);
548		spa_unload(spa);
549		spa_deactivate(spa);
550		spa_activate(spa);
551
552		return (spa_load(spa, newconfig, state, B_TRUE));
553	}
554
555	if (zap_lookup(spa->spa_meta_objset,
556	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPLIST,
557	    sizeof (uint64_t), 1, &spa->spa_sync_bplist_obj) != 0) {
558		vdev_set_state(rvd, B_TRUE, VDEV_STATE_CANT_OPEN,
559		    VDEV_AUX_CORRUPT_DATA);
560		error = EIO;
561		goto out;
562	}
563
564	/*
565	 * Load the bit that tells us to use the new accounting function
566	 * (raid-z deflation).  If we have an older pool, this will not
567	 * be present.
568	 */
569	error = zap_lookup(spa->spa_meta_objset,
570	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE,
571	    sizeof (uint64_t), 1, &spa->spa_deflate);
572	if (error != 0 && error != ENOENT) {
573		vdev_set_state(rvd, B_TRUE, VDEV_STATE_CANT_OPEN,
574		    VDEV_AUX_CORRUPT_DATA);
575		error = EIO;
576		goto out;
577	}
578
579	/*
580	 * Load the persistent error log.  If we have an older pool, this will
581	 * not be present.
582	 */
583	error = zap_lookup(spa->spa_meta_objset,
584	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_ERRLOG_LAST,
585	    sizeof (uint64_t), 1, &spa->spa_errlog_last);
586	if (error != 0 && error != ENOENT) {
587		vdev_set_state(rvd, B_TRUE, VDEV_STATE_CANT_OPEN,
588		    VDEV_AUX_CORRUPT_DATA);
589		error = EIO;
590		goto out;
591	}
592
593	error = zap_lookup(spa->spa_meta_objset,
594	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_ERRLOG_SCRUB,
595	    sizeof (uint64_t), 1, &spa->spa_errlog_scrub);
596	if (error != 0 && error != ENOENT) {
597		vdev_set_state(rvd, B_TRUE, VDEV_STATE_CANT_OPEN,
598		    VDEV_AUX_CORRUPT_DATA);
599		error = EIO;
600		goto out;
601	}
602
603	/*
604	 * Load the history object.  If we have an older pool, this
605	 * will not be present.
606	 */
607	error = zap_lookup(spa->spa_meta_objset,
608	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_HISTORY,
609	    sizeof (uint64_t), 1, &spa->spa_history);
610	if (error != 0 && error != ENOENT) {
611		vdev_set_state(rvd, B_TRUE, VDEV_STATE_CANT_OPEN,
612		    VDEV_AUX_CORRUPT_DATA);
613		error = EIO;
614		goto out;
615	}
616
617	/*
618	 * Load any hot spares for this pool.
619	 */
620	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
621	    DMU_POOL_SPARES, sizeof (uint64_t), 1, &spa->spa_spares_object);
622	if (error != 0 && error != ENOENT) {
623		vdev_set_state(rvd, B_TRUE, VDEV_STATE_CANT_OPEN,
624		    VDEV_AUX_CORRUPT_DATA);
625		error = EIO;
626		goto out;
627	}
628	if (error == 0) {
629		ASSERT(spa_version(spa) >= ZFS_VERSION_SPARES);
630		if (load_nvlist(spa, spa->spa_spares_object,
631		    &spa->spa_sparelist) != 0) {
632			vdev_set_state(rvd, B_TRUE, VDEV_STATE_CANT_OPEN,
633			    VDEV_AUX_CORRUPT_DATA);
634			error = EIO;
635			goto out;
636		}
637
638		spa_config_enter(spa, RW_WRITER, FTAG);
639		spa_load_spares(spa);
640		spa_config_exit(spa, FTAG);
641	}
642
643	/*
644	 * Load the vdev state for all toplevel vdevs.
645	 */
646	vdev_load(rvd);
647
648	/*
649	 * Propagate the leaf DTLs we just loaded all the way up the tree.
650	 */
651	spa_config_enter(spa, RW_WRITER, FTAG);
652	vdev_dtl_reassess(rvd, 0, 0, B_FALSE);
653	spa_config_exit(spa, FTAG);
654
655	/*
656	 * Check the state of the root vdev.  If it can't be opened, it
657	 * indicates one or more toplevel vdevs are faulted.
658	 */
659	if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) {
660		error = ENXIO;
661		goto out;
662	}
663
664	if ((spa_mode & FWRITE) && state != SPA_LOAD_TRYIMPORT) {
665		dmu_tx_t *tx;
666		int need_update = B_FALSE;
667		int c;
668
669		/*
670		 * Claim log blocks that haven't been committed yet.
671		 * This must all happen in a single txg.
672		 */
673		tx = dmu_tx_create_assigned(spa_get_dsl(spa),
674		    spa_first_txg(spa));
675		(void) dmu_objset_find(spa->spa_name,
676		    zil_claim, tx, DS_FIND_CHILDREN);
677		dmu_tx_commit(tx);
678
679		spa->spa_sync_on = B_TRUE;
680		txg_sync_start(spa->spa_dsl_pool);
681
682		/*
683		 * Wait for all claims to sync.
684		 */
685		txg_wait_synced(spa->spa_dsl_pool, 0);
686
687		/*
688		 * If the config cache is stale, or we have uninitialized
689		 * metaslabs (see spa_vdev_add()), then update the config.
690		 */
691		if (config_cache_txg != spa->spa_config_txg ||
692		    state == SPA_LOAD_IMPORT)
693			need_update = B_TRUE;
694
695		for (c = 0; c < rvd->vdev_children; c++)
696			if (rvd->vdev_child[c]->vdev_ms_array == 0)
697				need_update = B_TRUE;
698
699		/*
700		 * Update the config cache asychronously in case we're the
701		 * root pool, in which case the config cache isn't writable yet.
702		 */
703		if (need_update)
704			spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
705	}
706
707	error = 0;
708out:
709	if (error && error != EBADF)
710		zfs_ereport_post(FM_EREPORT_ZFS_POOL, spa, NULL, NULL, 0, 0);
711	spa->spa_load_state = SPA_LOAD_NONE;
712	spa->spa_ena = 0;
713
714	return (error);
715}
716
717/*
718 * Pool Open/Import
719 *
720 * The import case is identical to an open except that the configuration is sent
721 * down from userland, instead of grabbed from the configuration cache.  For the
722 * case of an open, the pool configuration will exist in the
723 * POOL_STATE_UNITIALIZED state.
724 *
725 * The stats information (gen/count/ustats) is used to gather vdev statistics at
726 * the same time open the pool, without having to keep around the spa_t in some
727 * ambiguous state.
728 */
729static int
730spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t **config)
731{
732	spa_t *spa;
733	int error;
734	int loaded = B_FALSE;
735	int locked = B_FALSE;
736
737	*spapp = NULL;
738
739	/*
740	 * As disgusting as this is, we need to support recursive calls to this
741	 * function because dsl_dir_open() is called during spa_load(), and ends
742	 * up calling spa_open() again.  The real fix is to figure out how to
743	 * avoid dsl_dir_open() calling this in the first place.
744	 */
745	if (mutex_owner(&spa_namespace_lock) != curthread) {
746		mutex_enter(&spa_namespace_lock);
747		locked = B_TRUE;
748	}
749
750	if ((spa = spa_lookup(pool)) == NULL) {
751		if (locked)
752			mutex_exit(&spa_namespace_lock);
753		return (ENOENT);
754	}
755	if (spa->spa_state == POOL_STATE_UNINITIALIZED) {
756
757		spa_activate(spa);
758
759		error = spa_load(spa, spa->spa_config, SPA_LOAD_OPEN, B_FALSE);
760
761		if (error == EBADF) {
762			/*
763			 * If vdev_validate() returns failure (indicated by
764			 * EBADF), it indicates that one of the vdevs indicates
765			 * that the pool has been exported or destroyed.  If
766			 * this is the case, the config cache is out of sync and
767			 * we should remove the pool from the namespace.
768			 */
769			zfs_post_ok(spa, NULL);
770			spa_unload(spa);
771			spa_deactivate(spa);
772			spa_remove(spa);
773			spa_config_sync();
774			if (locked)
775				mutex_exit(&spa_namespace_lock);
776			return (ENOENT);
777		}
778
779		if (error) {
780			/*
781			 * We can't open the pool, but we still have useful
782			 * information: the state of each vdev after the
783			 * attempted vdev_open().  Return this to the user.
784			 */
785			if (config != NULL && spa->spa_root_vdev != NULL) {
786				spa_config_enter(spa, RW_READER, FTAG);
787				*config = spa_config_generate(spa, NULL, -1ULL,
788				    B_TRUE);
789				spa_config_exit(spa, FTAG);
790			}
791			spa_unload(spa);
792			spa_deactivate(spa);
793			spa->spa_last_open_failed = B_TRUE;
794			if (locked)
795				mutex_exit(&spa_namespace_lock);
796			*spapp = NULL;
797			return (error);
798		} else {
799			zfs_post_ok(spa, NULL);
800			spa->spa_last_open_failed = B_FALSE;
801		}
802
803		loaded = B_TRUE;
804	}
805
806	spa_open_ref(spa, tag);
807	if (locked)
808		mutex_exit(&spa_namespace_lock);
809
810	*spapp = spa;
811
812	if (config != NULL) {
813		spa_config_enter(spa, RW_READER, FTAG);
814		*config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
815		spa_config_exit(spa, FTAG);
816	}
817
818	/*
819	 * If we just loaded the pool, resilver anything that's out of date.
820	 */
821	if (loaded && (spa_mode & FWRITE))
822		VERIFY(spa_scrub(spa, POOL_SCRUB_RESILVER, B_TRUE) == 0);
823
824	return (0);
825}
826
827int
828spa_open(const char *name, spa_t **spapp, void *tag)
829{
830	return (spa_open_common(name, spapp, tag, NULL));
831}
832
833/*
834 * Lookup the given spa_t, incrementing the inject count in the process,
835 * preventing it from being exported or destroyed.
836 */
837spa_t *
838spa_inject_addref(char *name)
839{
840	spa_t *spa;
841
842	mutex_enter(&spa_namespace_lock);
843	if ((spa = spa_lookup(name)) == NULL) {
844		mutex_exit(&spa_namespace_lock);
845		return (NULL);
846	}
847	spa->spa_inject_ref++;
848	mutex_exit(&spa_namespace_lock);
849
850	return (spa);
851}
852
853void
854spa_inject_delref(spa_t *spa)
855{
856	mutex_enter(&spa_namespace_lock);
857	spa->spa_inject_ref--;
858	mutex_exit(&spa_namespace_lock);
859}
860
861static void
862spa_add_spares(spa_t *spa, nvlist_t *config)
863{
864	nvlist_t **spares;
865	uint_t i, nspares;
866	nvlist_t *nvroot;
867	uint64_t guid;
868	vdev_stat_t *vs;
869	uint_t vsc;
870
871	if (spa->spa_nspares == 0)
872		return;
873
874	VERIFY(nvlist_lookup_nvlist(config,
875	    ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0);
876	VERIFY(nvlist_lookup_nvlist_array(spa->spa_sparelist,
877	    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0);
878	if (nspares != 0) {
879		VERIFY(nvlist_add_nvlist_array(nvroot,
880		    ZPOOL_CONFIG_SPARES, spares, nspares) == 0);
881		VERIFY(nvlist_lookup_nvlist_array(nvroot,
882		    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0);
883
884		/*
885		 * Go through and find any spares which have since been
886		 * repurposed as an active spare.  If this is the case, update
887		 * their status appropriately.
888		 */
889		for (i = 0; i < nspares; i++) {
890			VERIFY(nvlist_lookup_uint64(spares[i],
891			    ZPOOL_CONFIG_GUID, &guid) == 0);
892			if (spa_spare_inuse(guid)) {
893				VERIFY(nvlist_lookup_uint64_array(
894				    spares[i], ZPOOL_CONFIG_STATS,
895				    (uint64_t **)&vs, &vsc) == 0);
896				vs->vs_state = VDEV_STATE_CANT_OPEN;
897				vs->vs_aux = VDEV_AUX_SPARED;
898			}
899		}
900	}
901}
902
903int
904spa_get_stats(const char *name, nvlist_t **config, char *altroot, size_t buflen)
905{
906	int error;
907	spa_t *spa;
908
909	*config = NULL;
910	error = spa_open_common(name, &spa, FTAG, config);
911
912	if (spa && *config != NULL) {
913		VERIFY(nvlist_add_uint64(*config, ZPOOL_CONFIG_ERRCOUNT,
914		    spa_get_errlog_size(spa)) == 0);
915
916		spa_add_spares(spa, *config);
917	}
918
919	/*
920	 * We want to get the alternate root even for faulted pools, so we cheat
921	 * and call spa_lookup() directly.
922	 */
923	if (altroot) {
924		if (spa == NULL) {
925			mutex_enter(&spa_namespace_lock);
926			spa = spa_lookup(name);
927			if (spa)
928				spa_altroot(spa, altroot, buflen);
929			else
930				altroot[0] = '\0';
931			spa = NULL;
932			mutex_exit(&spa_namespace_lock);
933		} else {
934			spa_altroot(spa, altroot, buflen);
935		}
936	}
937
938	if (spa != NULL)
939		spa_close(spa, FTAG);
940
941	return (error);
942}
943
944/*
945 * Validate that the 'spares' array is well formed.  We must have an array of
946 * nvlists, each which describes a valid leaf vdev.
947 */
948static int
949spa_validate_spares(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode)
950{
951	nvlist_t **spares;
952	uint_t i, nspares;
953	vdev_t *vd;
954	int error;
955
956	/*
957	 * It's acceptable to have no spares specified.
958	 */
959	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
960	    &spares, &nspares) != 0)
961		return (0);
962
963	if (nspares == 0)
964		return (EINVAL);
965
966	/*
967	 * Make sure the pool is formatted with a version that supports hot
968	 * spares.
969	 */
970	if (spa_version(spa) < ZFS_VERSION_SPARES)
971		return (ENOTSUP);
972
973	for (i = 0; i < nspares; i++) {
974		if ((error = spa_config_parse(spa, &vd, spares[i], NULL, 0,
975		    mode)) != 0)
976			return (error);
977
978		if (!vd->vdev_ops->vdev_op_leaf) {
979			vdev_free(vd);
980			return (EINVAL);
981		}
982
983		if ((error = vdev_open(vd)) != 0) {
984			vdev_free(vd);
985			return (error);
986		}
987
988		vd->vdev_top = vd;
989		if ((error = vdev_label_spare(vd, crtxg)) != 0) {
990			vdev_free(vd);
991			return (error);
992		}
993
994		VERIFY(nvlist_add_uint64(spares[i], ZPOOL_CONFIG_GUID,
995		    vd->vdev_guid) == 0);
996
997		vdev_free(vd);
998	}
999
1000	return (0);
1001}
1002
1003/*
1004 * Pool Creation
1005 */
1006int
1007spa_create(const char *pool, nvlist_t *nvroot, const char *altroot)
1008{
1009	spa_t *spa;
1010	vdev_t *rvd;
1011	dsl_pool_t *dp;
1012	dmu_tx_t *tx;
1013	int c, error = 0;
1014	uint64_t txg = TXG_INITIAL;
1015	nvlist_t **spares;
1016	uint_t nspares;
1017
1018	/*
1019	 * If this pool already exists, return failure.
1020	 */
1021	mutex_enter(&spa_namespace_lock);
1022	if (spa_lookup(pool) != NULL) {
1023		mutex_exit(&spa_namespace_lock);
1024		return (EEXIST);
1025	}
1026
1027	/*
1028	 * Allocate a new spa_t structure.
1029	 */
1030	spa = spa_add(pool, altroot);
1031	spa_activate(spa);
1032
1033	spa->spa_uberblock.ub_txg = txg - 1;
1034	spa->spa_uberblock.ub_version = ZFS_VERSION;
1035	spa->spa_ubsync = spa->spa_uberblock;
1036
1037	/*
1038	 * Create the root vdev.
1039	 */
1040	spa_config_enter(spa, RW_WRITER, FTAG);
1041
1042	error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD);
1043
1044	ASSERT(error != 0 || rvd != NULL);
1045	ASSERT(error != 0 || spa->spa_root_vdev == rvd);
1046
1047	if (error == 0 && rvd->vdev_children == 0)
1048		error = EINVAL;
1049
1050	if (error == 0 &&
1051	    (error = vdev_create(rvd, txg, B_FALSE)) == 0 &&
1052	    (error = spa_validate_spares(spa, nvroot, txg,
1053	    VDEV_ALLOC_ADD)) == 0) {
1054		for (c = 0; c < rvd->vdev_children; c++)
1055			vdev_init(rvd->vdev_child[c], txg);
1056		vdev_config_dirty(rvd);
1057	}
1058
1059	spa_config_exit(spa, FTAG);
1060
1061	if (error != 0) {
1062		spa_unload(spa);
1063		spa_deactivate(spa);
1064		spa_remove(spa);
1065		mutex_exit(&spa_namespace_lock);
1066		return (error);
1067	}
1068
1069	/*
1070	 * Get the list of spares, if specified.
1071	 */
1072	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
1073	    &spares, &nspares) == 0) {
1074		VERIFY(nvlist_alloc(&spa->spa_sparelist, NV_UNIQUE_NAME,
1075		    KM_SLEEP) == 0);
1076		VERIFY(nvlist_add_nvlist_array(spa->spa_sparelist,
1077		    ZPOOL_CONFIG_SPARES, spares, nspares) == 0);
1078		spa_config_enter(spa, RW_WRITER, FTAG);
1079		spa_load_spares(spa);
1080		spa_config_exit(spa, FTAG);
1081		spa->spa_sync_spares = B_TRUE;
1082	}
1083
1084	spa->spa_dsl_pool = dp = dsl_pool_create(spa, txg);
1085	spa->spa_meta_objset = dp->dp_meta_objset;
1086
1087	tx = dmu_tx_create_assigned(dp, txg);
1088
1089	/*
1090	 * Create the pool config object.
1091	 */
1092	spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset,
1093	    DMU_OT_PACKED_NVLIST, 1 << 14,
1094	    DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx);
1095
1096	if (zap_add(spa->spa_meta_objset,
1097	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG,
1098	    sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) {
1099		cmn_err(CE_PANIC, "failed to add pool config");
1100	}
1101
1102	/* Newly created pools are always deflated. */
1103	spa->spa_deflate = TRUE;
1104	if (zap_add(spa->spa_meta_objset,
1105	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE,
1106	    sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) {
1107		cmn_err(CE_PANIC, "failed to add deflate");
1108	}
1109
1110	/*
1111	 * Create the deferred-free bplist object.  Turn off compression
1112	 * because sync-to-convergence takes longer if the blocksize
1113	 * keeps changing.
1114	 */
1115	spa->spa_sync_bplist_obj = bplist_create(spa->spa_meta_objset,
1116	    1 << 14, tx);
1117	dmu_object_set_compress(spa->spa_meta_objset, spa->spa_sync_bplist_obj,
1118	    ZIO_COMPRESS_OFF, tx);
1119
1120	if (zap_add(spa->spa_meta_objset,
1121	    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPLIST,
1122	    sizeof (uint64_t), 1, &spa->spa_sync_bplist_obj, tx) != 0) {
1123		cmn_err(CE_PANIC, "failed to add bplist");
1124	}
1125
1126	/*
1127	 * Create the pool's history object.
1128	 */
1129	spa_history_create_obj(spa, tx);
1130
1131	dmu_tx_commit(tx);
1132
1133	spa->spa_sync_on = B_TRUE;
1134	txg_sync_start(spa->spa_dsl_pool);
1135
1136	/*
1137	 * We explicitly wait for the first transaction to complete so that our
1138	 * bean counters are appropriately updated.
1139	 */
1140	txg_wait_synced(spa->spa_dsl_pool, txg);
1141
1142	spa_config_sync();
1143
1144	mutex_exit(&spa_namespace_lock);
1145
1146	return (0);
1147}
1148
1149/*
1150 * Import the given pool into the system.  We set up the necessary spa_t and
1151 * then call spa_load() to do the dirty work.
1152 */
1153int
1154spa_import(const char *pool, nvlist_t *config, const char *altroot)
1155{
1156	spa_t *spa;
1157	int error;
1158	nvlist_t *nvroot;
1159	nvlist_t **spares;
1160	uint_t nspares;
1161
1162	if (!(spa_mode & FWRITE))
1163		return (EROFS);
1164
1165	/*
1166	 * If a pool with this name exists, return failure.
1167	 */
1168	mutex_enter(&spa_namespace_lock);
1169	if (spa_lookup(pool) != NULL) {
1170		mutex_exit(&spa_namespace_lock);
1171		return (EEXIST);
1172	}
1173
1174	/*
1175	 * Create and initialize the spa structure.
1176	 */
1177	spa = spa_add(pool, altroot);
1178	spa_activate(spa);
1179
1180	/*
1181	 * Pass off the heavy lifting to spa_load().
1182	 * Pass TRUE for mosconfig because the user-supplied config
1183	 * is actually the one to trust when doing an import.
1184	 */
1185	error = spa_load(spa, config, SPA_LOAD_IMPORT, B_TRUE);
1186
1187	spa_config_enter(spa, RW_WRITER, FTAG);
1188	/*
1189	 * Toss any existing sparelist, as it doesn't have any validity anymore,
1190	 * and conflicts with spa_has_spare().
1191	 */
1192	if (spa->spa_sparelist) {
1193		nvlist_free(spa->spa_sparelist);
1194		spa->spa_sparelist = NULL;
1195		spa_load_spares(spa);
1196	}
1197
1198	VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
1199	    &nvroot) == 0);
1200	if (error == 0)
1201		error = spa_validate_spares(spa, nvroot, -1ULL,
1202		    VDEV_ALLOC_SPARE);
1203	spa_config_exit(spa, FTAG);
1204
1205	if (error != 0) {
1206		spa_unload(spa);
1207		spa_deactivate(spa);
1208		spa_remove(spa);
1209		mutex_exit(&spa_namespace_lock);
1210		return (error);
1211	}
1212
1213	/*
1214	 * Override any spares as specified by the user, as these may have
1215	 * correct device names/devids, etc.
1216	 */
1217	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
1218	    &spares, &nspares) == 0) {
1219		if (spa->spa_sparelist)
1220			VERIFY(nvlist_remove(spa->spa_sparelist,
1221			    ZPOOL_CONFIG_SPARES, DATA_TYPE_NVLIST_ARRAY) == 0);
1222		else
1223			VERIFY(nvlist_alloc(&spa->spa_sparelist,
1224			    NV_UNIQUE_NAME, KM_SLEEP) == 0);
1225		VERIFY(nvlist_add_nvlist_array(spa->spa_sparelist,
1226		    ZPOOL_CONFIG_SPARES, spares, nspares) == 0);
1227		spa_config_enter(spa, RW_WRITER, FTAG);
1228		spa_load_spares(spa);
1229		spa_config_exit(spa, FTAG);
1230		spa->spa_sync_spares = B_TRUE;
1231	}
1232
1233	/*
1234	 * Update the config cache to include the newly-imported pool.
1235	 */
1236	spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
1237
1238	mutex_exit(&spa_namespace_lock);
1239
1240	/*
1241	 * Resilver anything that's out of date.
1242	 */
1243	if (spa_mode & FWRITE)
1244		VERIFY(spa_scrub(spa, POOL_SCRUB_RESILVER, B_TRUE) == 0);
1245
1246	return (0);
1247}
1248
1249/*
1250 * This (illegal) pool name is used when temporarily importing a spa_t in order
1251 * to get the vdev stats associated with the imported devices.
1252 */
1253#define	TRYIMPORT_NAME	"$import"
1254
1255nvlist_t *
1256spa_tryimport(nvlist_t *tryconfig)
1257{
1258	nvlist_t *config = NULL;
1259	char *poolname;
1260	spa_t *spa;
1261	uint64_t state;
1262
1263	if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname))
1264		return (NULL);
1265
1266	if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state))
1267		return (NULL);
1268
1269	/*
1270	 * Create and initialize the spa structure.
1271	 */
1272	mutex_enter(&spa_namespace_lock);
1273	spa = spa_add(TRYIMPORT_NAME, NULL);
1274	spa_activate(spa);
1275
1276	/*
1277	 * Pass off the heavy lifting to spa_load().
1278	 * Pass TRUE for mosconfig because the user-supplied config
1279	 * is actually the one to trust when doing an import.
1280	 */
1281	(void) spa_load(spa, tryconfig, SPA_LOAD_TRYIMPORT, B_TRUE);
1282
1283	/*
1284	 * If 'tryconfig' was at least parsable, return the current config.
1285	 */
1286	if (spa->spa_root_vdev != NULL) {
1287		spa_config_enter(spa, RW_READER, FTAG);
1288		config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
1289		spa_config_exit(spa, FTAG);
1290		VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME,
1291		    poolname) == 0);
1292		VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE,
1293		    state) == 0);
1294
1295		/*
1296		 * Add the list of hot spares.
1297		 */
1298		spa_add_spares(spa, config);
1299	}
1300
1301	spa_unload(spa);
1302	spa_deactivate(spa);
1303	spa_remove(spa);
1304	mutex_exit(&spa_namespace_lock);
1305
1306	return (config);
1307}
1308
1309/*
1310 * Pool export/destroy
1311 *
1312 * The act of destroying or exporting a pool is very simple.  We make sure there
1313 * is no more pending I/O and any references to the pool are gone.  Then, we
1314 * update the pool state and sync all the labels to disk, removing the
1315 * configuration from the cache afterwards.
1316 */
1317static int
1318spa_export_common(char *pool, int new_state, nvlist_t **oldconfig)
1319{
1320	spa_t *spa;
1321
1322	if (oldconfig)
1323		*oldconfig = NULL;
1324
1325	if (!(spa_mode & FWRITE))
1326		return (EROFS);
1327
1328	mutex_enter(&spa_namespace_lock);
1329	if ((spa = spa_lookup(pool)) == NULL) {
1330		mutex_exit(&spa_namespace_lock);
1331		return (ENOENT);
1332	}
1333
1334	/*
1335	 * Put a hold on the pool, drop the namespace lock, stop async tasks,
1336	 * reacquire the namespace lock, and see if we can export.
1337	 */
1338	spa_open_ref(spa, FTAG);
1339	mutex_exit(&spa_namespace_lock);
1340	spa_async_suspend(spa);
1341	mutex_enter(&spa_namespace_lock);
1342	spa_close(spa, FTAG);
1343
1344	/*
1345	 * The pool will be in core if it's openable,
1346	 * in which case we can modify its state.
1347	 */
1348	if (spa->spa_state != POOL_STATE_UNINITIALIZED && spa->spa_sync_on) {
1349		/*
1350		 * Objsets may be open only because they're dirty, so we
1351		 * have to force it to sync before checking spa_refcnt.
1352		 */
1353		spa_scrub_suspend(spa);
1354		txg_wait_synced(spa->spa_dsl_pool, 0);
1355
1356		/*
1357		 * A pool cannot be exported or destroyed if there are active
1358		 * references.  If we are resetting a pool, allow references by
1359		 * fault injection handlers.
1360		 */
1361		if (!spa_refcount_zero(spa) ||
1362		    (spa->spa_inject_ref != 0 &&
1363		    new_state != POOL_STATE_UNINITIALIZED)) {
1364			spa_scrub_resume(spa);
1365			spa_async_resume(spa);
1366			mutex_exit(&spa_namespace_lock);
1367			return (EBUSY);
1368		}
1369
1370		spa_scrub_resume(spa);
1371		VERIFY(spa_scrub(spa, POOL_SCRUB_NONE, B_TRUE) == 0);
1372
1373		/*
1374		 * We want this to be reflected on every label,
1375		 * so mark them all dirty.  spa_unload() will do the
1376		 * final sync that pushes these changes out.
1377		 */
1378		if (new_state != POOL_STATE_UNINITIALIZED) {
1379			spa_config_enter(spa, RW_WRITER, FTAG);
1380			spa->spa_state = new_state;
1381			spa->spa_final_txg = spa_last_synced_txg(spa) + 1;
1382			vdev_config_dirty(spa->spa_root_vdev);
1383			spa_config_exit(spa, FTAG);
1384		}
1385	}
1386
1387	if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
1388		spa_unload(spa);
1389		spa_deactivate(spa);
1390	}
1391
1392	if (oldconfig && spa->spa_config)
1393		VERIFY(nvlist_dup(spa->spa_config, oldconfig, 0) == 0);
1394
1395	if (new_state != POOL_STATE_UNINITIALIZED) {
1396		spa_remove(spa);
1397		spa_config_sync();
1398	}
1399	mutex_exit(&spa_namespace_lock);
1400
1401	return (0);
1402}
1403
1404/*
1405 * Destroy a storage pool.
1406 */
1407int
1408spa_destroy(char *pool)
1409{
1410	return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL));
1411}
1412
1413/*
1414 * Export a storage pool.
1415 */
1416int
1417spa_export(char *pool, nvlist_t **oldconfig)
1418{
1419	return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig));
1420}
1421
1422/*
1423 * Similar to spa_export(), this unloads the spa_t without actually removing it
1424 * from the namespace in any way.
1425 */
1426int
1427spa_reset(char *pool)
1428{
1429	return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL));
1430}
1431
1432
1433/*
1434 * ==========================================================================
1435 * Device manipulation
1436 * ==========================================================================
1437 */
1438
1439/*
1440 * Add capacity to a storage pool.
1441 */
1442int
1443spa_vdev_add(spa_t *spa, nvlist_t *nvroot)
1444{
1445	uint64_t txg;
1446	int c, error;
1447	vdev_t *rvd = spa->spa_root_vdev;
1448	vdev_t *vd, *tvd;
1449	nvlist_t **spares;
1450	uint_t i, nspares;
1451
1452	txg = spa_vdev_enter(spa);
1453
1454	if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0,
1455	    VDEV_ALLOC_ADD)) != 0)
1456		return (spa_vdev_exit(spa, NULL, txg, error));
1457
1458	if ((error = spa_validate_spares(spa, nvroot, txg,
1459	    VDEV_ALLOC_ADD)) != 0)
1460		return (spa_vdev_exit(spa, vd, txg, error));
1461
1462	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
1463	    &spares, &nspares) != 0)
1464		nspares = 0;
1465
1466	if (vd->vdev_children == 0 && nspares == 0)
1467		return (spa_vdev_exit(spa, vd, txg, EINVAL));
1468
1469	if (vd->vdev_children != 0) {
1470		if ((error = vdev_create(vd, txg, B_FALSE)) != 0)
1471			return (spa_vdev_exit(spa, vd, txg, error));
1472
1473		/*
1474		 * Transfer each new top-level vdev from vd to rvd.
1475		 */
1476		for (c = 0; c < vd->vdev_children; c++) {
1477			tvd = vd->vdev_child[c];
1478			vdev_remove_child(vd, tvd);
1479			tvd->vdev_id = rvd->vdev_children;
1480			vdev_add_child(rvd, tvd);
1481			vdev_config_dirty(tvd);
1482		}
1483	}
1484
1485	if (nspares != 0) {
1486		if (spa->spa_sparelist != NULL) {
1487			nvlist_t **oldspares;
1488			uint_t oldnspares;
1489			nvlist_t **newspares;
1490
1491			VERIFY(nvlist_lookup_nvlist_array(spa->spa_sparelist,
1492			    ZPOOL_CONFIG_SPARES, &oldspares, &oldnspares) == 0);
1493
1494			newspares = kmem_alloc(sizeof (void *) *
1495			    (nspares + oldnspares), KM_SLEEP);
1496			for (i = 0; i < oldnspares; i++)
1497				VERIFY(nvlist_dup(oldspares[i],
1498				    &newspares[i], KM_SLEEP) == 0);
1499			for (i = 0; i < nspares; i++)
1500				VERIFY(nvlist_dup(spares[i],
1501				    &newspares[i + oldnspares],
1502				    KM_SLEEP) == 0);
1503
1504			VERIFY(nvlist_remove(spa->spa_sparelist,
1505			    ZPOOL_CONFIG_SPARES, DATA_TYPE_NVLIST_ARRAY) == 0);
1506
1507			VERIFY(nvlist_add_nvlist_array(spa->spa_sparelist,
1508			    ZPOOL_CONFIG_SPARES, newspares,
1509			    nspares + oldnspares) == 0);
1510			for (i = 0; i < oldnspares + nspares; i++)
1511				nvlist_free(newspares[i]);
1512			kmem_free(newspares, (oldnspares + nspares) *
1513			    sizeof (void *));
1514		} else {
1515			VERIFY(nvlist_alloc(&spa->spa_sparelist,
1516			    NV_UNIQUE_NAME, KM_SLEEP) == 0);
1517			VERIFY(nvlist_add_nvlist_array(spa->spa_sparelist,
1518			    ZPOOL_CONFIG_SPARES, spares, nspares) == 0);
1519		}
1520
1521		spa_load_spares(spa);
1522		spa->spa_sync_spares = B_TRUE;
1523	}
1524
1525	/*
1526	 * We have to be careful when adding new vdevs to an existing pool.
1527	 * If other threads start allocating from these vdevs before we
1528	 * sync the config cache, and we lose power, then upon reboot we may
1529	 * fail to open the pool because there are DVAs that the config cache
1530	 * can't translate.  Therefore, we first add the vdevs without
1531	 * initializing metaslabs; sync the config cache (via spa_vdev_exit());
1532	 * and then let spa_config_update() initialize the new metaslabs.
1533	 *
1534	 * spa_load() checks for added-but-not-initialized vdevs, so that
1535	 * if we lose power at any point in this sequence, the remaining
1536	 * steps will be completed the next time we load the pool.
1537	 */
1538	(void) spa_vdev_exit(spa, vd, txg, 0);
1539
1540	mutex_enter(&spa_namespace_lock);
1541	spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
1542	mutex_exit(&spa_namespace_lock);
1543
1544	return (0);
1545}
1546
1547/*
1548 * Attach a device to a mirror.  The arguments are the path to any device
1549 * in the mirror, and the nvroot for the new device.  If the path specifies
1550 * a device that is not mirrored, we automatically insert the mirror vdev.
1551 *
1552 * If 'replacing' is specified, the new device is intended to replace the
1553 * existing device; in this case the two devices are made into their own
1554 * mirror using the 'replacing' vdev, which is functionally idendical to
1555 * the mirror vdev (it actually reuses all the same ops) but has a few
1556 * extra rules: you can't attach to it after it's been created, and upon
1557 * completion of resilvering, the first disk (the one being replaced)
1558 * is automatically detached.
1559 */
1560int
1561spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing)
1562{
1563	uint64_t txg, open_txg;
1564	int error;
1565	vdev_t *rvd = spa->spa_root_vdev;
1566	vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd;
1567	vdev_ops_t *pvops;
1568
1569	txg = spa_vdev_enter(spa);
1570
1571	oldvd = vdev_lookup_by_guid(rvd, guid);
1572
1573	if (oldvd == NULL)
1574		return (spa_vdev_exit(spa, NULL, txg, ENODEV));
1575
1576	if (!oldvd->vdev_ops->vdev_op_leaf)
1577		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
1578
1579	pvd = oldvd->vdev_parent;
1580
1581	if ((error = spa_config_parse(spa, &newrootvd, nvroot, NULL, 0,
1582	    VDEV_ALLOC_ADD)) != 0 || newrootvd->vdev_children != 1)
1583		return (spa_vdev_exit(spa, newrootvd, txg, EINVAL));
1584
1585	newvd = newrootvd->vdev_child[0];
1586
1587	if (!newvd->vdev_ops->vdev_op_leaf)
1588		return (spa_vdev_exit(spa, newrootvd, txg, EINVAL));
1589
1590	if ((error = vdev_create(newrootvd, txg, replacing)) != 0)
1591		return (spa_vdev_exit(spa, newrootvd, txg, error));
1592
1593	if (!replacing) {
1594		/*
1595		 * For attach, the only allowable parent is a mirror or the root
1596		 * vdev.
1597		 */
1598		if (pvd->vdev_ops != &vdev_mirror_ops &&
1599		    pvd->vdev_ops != &vdev_root_ops)
1600			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
1601
1602		pvops = &vdev_mirror_ops;
1603	} else {
1604		/*
1605		 * Active hot spares can only be replaced by inactive hot
1606		 * spares.
1607		 */
1608		if (pvd->vdev_ops == &vdev_spare_ops &&
1609		    pvd->vdev_child[1] == oldvd &&
1610		    !spa_has_spare(spa, newvd->vdev_guid))
1611			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
1612
1613		/*
1614		 * If the source is a hot spare, and the parent isn't already a
1615		 * spare, then we want to create a new hot spare.  Otherwise, we
1616		 * want to create a replacing vdev.
1617		 */
1618		if (pvd->vdev_ops == &vdev_replacing_ops)
1619			return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP));
1620		else if (pvd->vdev_ops != &vdev_spare_ops &&
1621		    newvd->vdev_isspare)
1622			pvops = &vdev_spare_ops;
1623		else
1624			pvops = &vdev_replacing_ops;
1625	}
1626
1627	/*
1628	 * Compare the new device size with the replaceable/attachable
1629	 * device size.
1630	 */
1631	if (newvd->vdev_psize < vdev_get_rsize(oldvd))
1632		return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW));
1633
1634	/*
1635	 * The new device cannot have a higher alignment requirement
1636	 * than the top-level vdev.
1637	 */
1638	if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift)
1639		return (spa_vdev_exit(spa, newrootvd, txg, EDOM));
1640
1641	/*
1642	 * If this is an in-place replacement, update oldvd's path and devid
1643	 * to make it distinguishable from newvd, and unopenable from now on.
1644	 */
1645	if (strcmp(oldvd->vdev_path, newvd->vdev_path) == 0) {
1646		spa_strfree(oldvd->vdev_path);
1647		oldvd->vdev_path = kmem_alloc(strlen(newvd->vdev_path) + 5,
1648		    KM_SLEEP);
1649		(void) sprintf(oldvd->vdev_path, "%s/%s",
1650		    newvd->vdev_path, "old");
1651		if (oldvd->vdev_devid != NULL) {
1652			spa_strfree(oldvd->vdev_devid);
1653			oldvd->vdev_devid = NULL;
1654		}
1655	}
1656
1657	/*
1658	 * If the parent is not a mirror, or if we're replacing, insert the new
1659	 * mirror/replacing/spare vdev above oldvd.
1660	 */
1661	if (pvd->vdev_ops != pvops)
1662		pvd = vdev_add_parent(oldvd, pvops);
1663
1664	ASSERT(pvd->vdev_top->vdev_parent == rvd);
1665	ASSERT(pvd->vdev_ops == pvops);
1666	ASSERT(oldvd->vdev_parent == pvd);
1667
1668	/*
1669	 * Extract the new device from its root and add it to pvd.
1670	 */
1671	vdev_remove_child(newrootvd, newvd);
1672	newvd->vdev_id = pvd->vdev_children;
1673	vdev_add_child(pvd, newvd);
1674
1675	/*
1676	 * If newvd is smaller than oldvd, but larger than its rsize,
1677	 * the addition of newvd may have decreased our parent's asize.
1678	 */
1679	pvd->vdev_asize = MIN(pvd->vdev_asize, newvd->vdev_asize);
1680
1681	tvd = newvd->vdev_top;
1682	ASSERT(pvd->vdev_top == tvd);
1683	ASSERT(tvd->vdev_parent == rvd);
1684
1685	vdev_config_dirty(tvd);
1686
1687	/*
1688	 * Set newvd's DTL to [TXG_INITIAL, open_txg].  It will propagate
1689	 * upward when spa_vdev_exit() calls vdev_dtl_reassess().
1690	 */
1691	open_txg = txg + TXG_CONCURRENT_STATES - 1;
1692
1693	mutex_enter(&newvd->vdev_dtl_lock);
1694	space_map_add(&newvd->vdev_dtl_map, TXG_INITIAL,
1695	    open_txg - TXG_INITIAL + 1);
1696	mutex_exit(&newvd->vdev_dtl_lock);
1697
1698	dprintf("attached %s in txg %llu\n", newvd->vdev_path, txg);
1699
1700	/*
1701	 * Mark newvd's DTL dirty in this txg.
1702	 */
1703	vdev_dirty(tvd, VDD_DTL, newvd, txg);
1704
1705	(void) spa_vdev_exit(spa, newrootvd, open_txg, 0);
1706
1707	/*
1708	 * Kick off a resilver to update newvd.
1709	 */
1710	VERIFY(spa_scrub(spa, POOL_SCRUB_RESILVER, B_TRUE) == 0);
1711
1712	return (0);
1713}
1714
1715/*
1716 * Detach a device from a mirror or replacing vdev.
1717 * If 'replace_done' is specified, only detach if the parent
1718 * is a replacing vdev.
1719 */
1720int
1721spa_vdev_detach(spa_t *spa, uint64_t guid, int replace_done)
1722{
1723	uint64_t txg;
1724	int c, t, error;
1725	vdev_t *rvd = spa->spa_root_vdev;
1726	vdev_t *vd, *pvd, *cvd, *tvd;
1727	boolean_t unspare = B_FALSE;
1728	uint64_t unspare_guid;
1729
1730	txg = spa_vdev_enter(spa);
1731
1732	vd = vdev_lookup_by_guid(rvd, guid);
1733
1734	if (vd == NULL)
1735		return (spa_vdev_exit(spa, NULL, txg, ENODEV));
1736
1737	if (!vd->vdev_ops->vdev_op_leaf)
1738		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
1739
1740	pvd = vd->vdev_parent;
1741
1742	/*
1743	 * If replace_done is specified, only remove this device if it's
1744	 * the first child of a replacing vdev.  For the 'spare' vdev, either
1745	 * disk can be removed.
1746	 */
1747	if (replace_done) {
1748		if (pvd->vdev_ops == &vdev_replacing_ops) {
1749			if (vd->vdev_id != 0)
1750				return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
1751		} else if (pvd->vdev_ops != &vdev_spare_ops) {
1752			return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
1753		}
1754	}
1755
1756	ASSERT(pvd->vdev_ops != &vdev_spare_ops ||
1757	    spa_version(spa) >= ZFS_VERSION_SPARES);
1758
1759	/*
1760	 * Only mirror, replacing, and spare vdevs support detach.
1761	 */
1762	if (pvd->vdev_ops != &vdev_replacing_ops &&
1763	    pvd->vdev_ops != &vdev_mirror_ops &&
1764	    pvd->vdev_ops != &vdev_spare_ops)
1765		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
1766
1767	/*
1768	 * If there's only one replica, you can't detach it.
1769	 */
1770	if (pvd->vdev_children <= 1)
1771		return (spa_vdev_exit(spa, NULL, txg, EBUSY));
1772
1773	/*
1774	 * If all siblings have non-empty DTLs, this device may have the only
1775	 * valid copy of the data, which means we cannot safely detach it.
1776	 *
1777	 * XXX -- as in the vdev_offline() case, we really want a more
1778	 * precise DTL check.
1779	 */
1780	for (c = 0; c < pvd->vdev_children; c++) {
1781		uint64_t dirty;
1782
1783		cvd = pvd->vdev_child[c];
1784		if (cvd == vd)
1785			continue;
1786		if (vdev_is_dead(cvd))
1787			continue;
1788		mutex_enter(&cvd->vdev_dtl_lock);
1789		dirty = cvd->vdev_dtl_map.sm_space |
1790		    cvd->vdev_dtl_scrub.sm_space;
1791		mutex_exit(&cvd->vdev_dtl_lock);
1792		if (!dirty)
1793			break;
1794	}
1795
1796	/*
1797	 * If we are a replacing or spare vdev, then we can always detach the
1798	 * latter child, as that is how one cancels the operation.
1799	 */
1800	if ((pvd->vdev_ops == &vdev_mirror_ops || vd->vdev_id != 1) &&
1801	    c == pvd->vdev_children)
1802		return (spa_vdev_exit(spa, NULL, txg, EBUSY));
1803
1804	/*
1805	 * If we are detaching the original disk from a spare, then it implies
1806	 * that the spare should become a real disk, and be removed from the
1807	 * active spare list for the pool.
1808	 */
1809	if (pvd->vdev_ops == &vdev_spare_ops &&
1810	    vd->vdev_id == 0)
1811		unspare = B_TRUE;
1812
1813	/*
1814	 * Erase the disk labels so the disk can be used for other things.
1815	 * This must be done after all other error cases are handled,
1816	 * but before we disembowel vd (so we can still do I/O to it).
1817	 * But if we can't do it, don't treat the error as fatal --
1818	 * it may be that the unwritability of the disk is the reason
1819	 * it's being detached!
1820	 */
1821	error = vdev_label_init(vd, 0, B_FALSE);
1822	if (error)
1823		dprintf("unable to erase labels on %s\n", vdev_description(vd));
1824
1825	/*
1826	 * Remove vd from its parent and compact the parent's children.
1827	 */
1828	vdev_remove_child(pvd, vd);
1829	vdev_compact_children(pvd);
1830
1831	/*
1832	 * Remember one of the remaining children so we can get tvd below.
1833	 */
1834	cvd = pvd->vdev_child[0];
1835
1836	/*
1837	 * If we need to remove the remaining child from the list of hot spares,
1838	 * do it now, marking the vdev as no longer a spare in the process.  We
1839	 * must do this before vdev_remove_parent(), because that can change the
1840	 * GUID if it creates a new toplevel GUID.
1841	 */
1842	if (unspare) {
1843		ASSERT(cvd->vdev_isspare);
1844		spa_spare_remove(cvd->vdev_guid);
1845		cvd->vdev_isspare = B_FALSE;
1846		unspare_guid = cvd->vdev_guid;
1847	}
1848
1849	/*
1850	 * If the parent mirror/replacing vdev only has one child,
1851	 * the parent is no longer needed.  Remove it from the tree.
1852	 */
1853	if (pvd->vdev_children == 1)
1854		vdev_remove_parent(cvd);
1855
1856	/*
1857	 * We don't set tvd until now because the parent we just removed
1858	 * may have been the previous top-level vdev.
1859	 */
1860	tvd = cvd->vdev_top;
1861	ASSERT(tvd->vdev_parent == rvd);
1862
1863	/*
1864	 * Reopen this top-level vdev to reassess health after detach.
1865	 */
1866	vdev_reopen(tvd);
1867
1868	/*
1869	 * If the device we just detached was smaller than the others,
1870	 * it may be possible to add metaslabs (i.e. grow the pool).
1871	 * vdev_metaslab_init() can't fail because the existing metaslabs
1872	 * are already in core, so there's nothing to read from disk.
1873	 */
1874	VERIFY(vdev_metaslab_init(tvd, txg) == 0);
1875
1876	vdev_config_dirty(tvd);
1877
1878	/*
1879	 * Mark vd's DTL as dirty in this txg.
1880	 * vdev_dtl_sync() will see that vd->vdev_detached is set
1881	 * and free vd's DTL object in syncing context.
1882	 * But first make sure we're not on any *other* txg's DTL list,
1883	 * to prevent vd from being accessed after it's freed.
1884	 */
1885	for (t = 0; t < TXG_SIZE; t++)
1886		(void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t);
1887	vd->vdev_detached = B_TRUE;
1888	vdev_dirty(tvd, VDD_DTL, vd, txg);
1889
1890	dprintf("detached %s in txg %llu\n", vd->vdev_path, txg);
1891
1892	error = spa_vdev_exit(spa, vd, txg, 0);
1893
1894	/*
1895	 * If we are supposed to remove the given vdev from the list of spares,
1896	 * iterate over all pools in the system and replace it if it's present.
1897	 */
1898	if (unspare) {
1899		spa = NULL;
1900		mutex_enter(&spa_namespace_lock);
1901		while ((spa = spa_next(spa)) != NULL) {
1902			if (spa->spa_state != POOL_STATE_ACTIVE)
1903				continue;
1904
1905			(void) spa_vdev_remove(spa, unspare_guid, B_TRUE);
1906		}
1907		mutex_exit(&spa_namespace_lock);
1908	}
1909
1910	return (error);
1911}
1912
1913/*
1914 * Remove a device from the pool.  Currently, this supports removing only hot
1915 * spares.
1916 */
1917int
1918spa_vdev_remove(spa_t *spa, uint64_t guid, boolean_t unspare)
1919{
1920	vdev_t *vd;
1921	nvlist_t **spares, *nv, **newspares;
1922	uint_t i, j, nspares;
1923	int ret = 0;
1924
1925	spa_config_enter(spa, RW_WRITER, FTAG);
1926
1927	vd = spa_lookup_by_guid(spa, guid);
1928
1929	nv = NULL;
1930	if (spa->spa_spares != NULL &&
1931	    nvlist_lookup_nvlist_array(spa->spa_sparelist, ZPOOL_CONFIG_SPARES,
1932	    &spares, &nspares) == 0) {
1933		for (i = 0; i < nspares; i++) {
1934			uint64_t theguid;
1935
1936			VERIFY(nvlist_lookup_uint64(spares[i],
1937			    ZPOOL_CONFIG_GUID, &theguid) == 0);
1938			if (theguid == guid) {
1939				nv = spares[i];
1940				break;
1941			}
1942		}
1943	}
1944
1945	/*
1946	 * We only support removing a hot spare, and only if it's not currently
1947	 * in use in this pool.
1948	 */
1949	if (nv == NULL && vd == NULL) {
1950		ret = ENOENT;
1951		goto out;
1952	}
1953
1954	if (nv == NULL && vd != NULL) {
1955		ret = ENOTSUP;
1956		goto out;
1957	}
1958
1959	if (!unspare && nv != NULL && vd != NULL) {
1960		ret = EBUSY;
1961		goto out;
1962	}
1963
1964	if (nspares == 1) {
1965		newspares = NULL;
1966	} else {
1967		newspares = kmem_alloc((nspares - 1) * sizeof (void *),
1968		    KM_SLEEP);
1969		for (i = 0, j = 0; i < nspares; i++) {
1970			if (spares[i] != nv)
1971				VERIFY(nvlist_dup(spares[i],
1972				    &newspares[j++], KM_SLEEP) == 0);
1973		}
1974	}
1975
1976	VERIFY(nvlist_remove(spa->spa_sparelist, ZPOOL_CONFIG_SPARES,
1977	    DATA_TYPE_NVLIST_ARRAY) == 0);
1978	VERIFY(nvlist_add_nvlist_array(spa->spa_sparelist, ZPOOL_CONFIG_SPARES,
1979	    newspares, nspares - 1) == 0);
1980	for (i = 0; i < nspares - 1; i++)
1981		nvlist_free(newspares[i]);
1982	kmem_free(newspares, (nspares - 1) * sizeof (void *));
1983	spa_load_spares(spa);
1984	spa->spa_sync_spares = B_TRUE;
1985
1986out:
1987	spa_config_exit(spa, FTAG);
1988
1989	return (ret);
1990}
1991
1992/*
1993 * Find any device that's done replacing, so we can detach it.
1994 */
1995static vdev_t *
1996spa_vdev_replace_done_hunt(vdev_t *vd)
1997{
1998	vdev_t *newvd, *oldvd;
1999	int c;
2000
2001	for (c = 0; c < vd->vdev_children; c++) {
2002		oldvd = spa_vdev_replace_done_hunt(vd->vdev_child[c]);
2003		if (oldvd != NULL)
2004			return (oldvd);
2005	}
2006
2007	if (vd->vdev_ops == &vdev_replacing_ops && vd->vdev_children == 2) {
2008		oldvd = vd->vdev_child[0];
2009		newvd = vd->vdev_child[1];
2010
2011		mutex_enter(&newvd->vdev_dtl_lock);
2012		if (newvd->vdev_dtl_map.sm_space == 0 &&
2013		    newvd->vdev_dtl_scrub.sm_space == 0) {
2014			mutex_exit(&newvd->vdev_dtl_lock);
2015			return (oldvd);
2016		}
2017		mutex_exit(&newvd->vdev_dtl_lock);
2018	}
2019
2020	return (NULL);
2021}
2022
2023static void
2024spa_vdev_replace_done(spa_t *spa)
2025{
2026	vdev_t *vd;
2027	vdev_t *pvd;
2028	uint64_t guid;
2029	uint64_t pguid = 0;
2030
2031	spa_config_enter(spa, RW_READER, FTAG);
2032
2033	while ((vd = spa_vdev_replace_done_hunt(spa->spa_root_vdev)) != NULL) {
2034		guid = vd->vdev_guid;
2035		/*
2036		 * If we have just finished replacing a hot spared device, then
2037		 * we need to detach the parent's first child (the original hot
2038		 * spare) as well.
2039		 */
2040		pvd = vd->vdev_parent;
2041		if (pvd->vdev_parent->vdev_ops == &vdev_spare_ops &&
2042		    pvd->vdev_id == 0) {
2043			ASSERT(pvd->vdev_ops == &vdev_replacing_ops);
2044			ASSERT(pvd->vdev_parent->vdev_children == 2);
2045			pguid = pvd->vdev_parent->vdev_child[1]->vdev_guid;
2046		}
2047		spa_config_exit(spa, FTAG);
2048		if (spa_vdev_detach(spa, guid, B_TRUE) != 0)
2049			return;
2050		if (pguid != 0 && spa_vdev_detach(spa, pguid, B_TRUE) != 0)
2051			return;
2052		spa_config_enter(spa, RW_READER, FTAG);
2053	}
2054
2055	spa_config_exit(spa, FTAG);
2056}
2057
2058/*
2059 * Update the stored path for this vdev.  Dirty the vdev configuration, relying
2060 * on spa_vdev_enter/exit() to synchronize the labels and cache.
2061 */
2062int
2063spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath)
2064{
2065	vdev_t *rvd, *vd;
2066	uint64_t txg;
2067
2068	rvd = spa->spa_root_vdev;
2069
2070	txg = spa_vdev_enter(spa);
2071
2072	if ((vd = vdev_lookup_by_guid(rvd, guid)) == NULL) {
2073		/*
2074		 * Determine if this is a reference to a hot spare.  In that
2075		 * case, update the path as stored in the spare list.
2076		 */
2077		nvlist_t **spares;
2078		uint_t i, nspares;
2079		if (spa->spa_sparelist != NULL) {
2080			VERIFY(nvlist_lookup_nvlist_array(spa->spa_sparelist,
2081			    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0);
2082			for (i = 0; i < nspares; i++) {
2083				uint64_t theguid;
2084				VERIFY(nvlist_lookup_uint64(spares[i],
2085				    ZPOOL_CONFIG_GUID, &theguid) == 0);
2086				if (theguid == guid)
2087					break;
2088			}
2089
2090			if (i == nspares)
2091				return (spa_vdev_exit(spa, NULL, txg, ENOENT));
2092
2093			VERIFY(nvlist_add_string(spares[i],
2094			    ZPOOL_CONFIG_PATH, newpath) == 0);
2095			spa_load_spares(spa);
2096			spa->spa_sync_spares = B_TRUE;
2097			return (spa_vdev_exit(spa, NULL, txg, 0));
2098		} else {
2099			return (spa_vdev_exit(spa, NULL, txg, ENOENT));
2100		}
2101	}
2102
2103	if (!vd->vdev_ops->vdev_op_leaf)
2104		return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
2105
2106	spa_strfree(vd->vdev_path);
2107	vd->vdev_path = spa_strdup(newpath);
2108
2109	vdev_config_dirty(vd->vdev_top);
2110
2111	return (spa_vdev_exit(spa, NULL, txg, 0));
2112}
2113
2114/*
2115 * ==========================================================================
2116 * SPA Scrubbing
2117 * ==========================================================================
2118 */
2119
2120void
2121spa_scrub_throttle(spa_t *spa, int direction)
2122{
2123	mutex_enter(&spa->spa_scrub_lock);
2124	spa->spa_scrub_throttled += direction;
2125	ASSERT(spa->spa_scrub_throttled >= 0);
2126	if (spa->spa_scrub_throttled == 0)
2127		cv_broadcast(&spa->spa_scrub_io_cv);
2128	mutex_exit(&spa->spa_scrub_lock);
2129}
2130
2131static void
2132spa_scrub_io_done(zio_t *zio)
2133{
2134	spa_t *spa = zio->io_spa;
2135
2136	zio_buf_free(zio->io_data, zio->io_size);
2137
2138	mutex_enter(&spa->spa_scrub_lock);
2139	if (zio->io_error && !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) {
2140		vdev_t *vd = zio->io_vd ? zio->io_vd : spa->spa_root_vdev;
2141		spa->spa_scrub_errors++;
2142		mutex_enter(&vd->vdev_stat_lock);
2143		vd->vdev_stat.vs_scrub_errors++;
2144		mutex_exit(&vd->vdev_stat_lock);
2145	}
2146	if (--spa->spa_scrub_inflight == 0) {
2147		cv_broadcast(&spa->spa_scrub_io_cv);
2148		ASSERT(spa->spa_scrub_throttled == 0);
2149	}
2150	mutex_exit(&spa->spa_scrub_lock);
2151}
2152
2153static void
2154spa_scrub_io_start(spa_t *spa, blkptr_t *bp, int priority, int flags,
2155    zbookmark_t *zb)
2156{
2157	size_t size = BP_GET_LSIZE(bp);
2158	void *data = zio_buf_alloc(size);
2159
2160	mutex_enter(&spa->spa_scrub_lock);
2161	spa->spa_scrub_inflight++;
2162	mutex_exit(&spa->spa_scrub_lock);
2163
2164	if (zb->zb_level == -1 && BP_GET_TYPE(bp) != DMU_OT_OBJSET)
2165		flags |= ZIO_FLAG_SPECULATIVE;	/* intent log block */
2166
2167	flags |= ZIO_FLAG_SCRUB_THREAD | ZIO_FLAG_CANFAIL;
2168
2169	zio_nowait(zio_read(NULL, spa, bp, data, size,
2170	    spa_scrub_io_done, NULL, priority, flags, zb));
2171}
2172
2173/* ARGSUSED */
2174static int
2175spa_scrub_cb(traverse_blk_cache_t *bc, spa_t *spa, void *a)
2176{
2177	blkptr_t *bp = &bc->bc_blkptr;
2178	vdev_t *vd = spa->spa_root_vdev;
2179	dva_t *dva = bp->blk_dva;
2180	int needs_resilver = B_FALSE;
2181	int d;
2182
2183	if (bc->bc_errno) {
2184		/*
2185		 * We can't scrub this block, but we can continue to scrub
2186		 * the rest of the pool.  Note the error and move along.
2187		 */
2188		mutex_enter(&spa->spa_scrub_lock);
2189		spa->spa_scrub_errors++;
2190		mutex_exit(&spa->spa_scrub_lock);
2191
2192		mutex_enter(&vd->vdev_stat_lock);
2193		vd->vdev_stat.vs_scrub_errors++;
2194		mutex_exit(&vd->vdev_stat_lock);
2195
2196		return (ERESTART);
2197	}
2198
2199	ASSERT(bp->blk_birth < spa->spa_scrub_maxtxg);
2200
2201	for (d = 0; d < BP_GET_NDVAS(bp); d++) {
2202		vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[d]));
2203
2204		ASSERT(vd != NULL);
2205
2206		/*
2207		 * Keep track of how much data we've examined so that
2208		 * zpool(1M) status can make useful progress reports.
2209		 */
2210		mutex_enter(&vd->vdev_stat_lock);
2211		vd->vdev_stat.vs_scrub_examined += DVA_GET_ASIZE(&dva[d]);
2212		mutex_exit(&vd->vdev_stat_lock);
2213
2214		if (spa->spa_scrub_type == POOL_SCRUB_RESILVER) {
2215			if (DVA_GET_GANG(&dva[d])) {
2216				/*
2217				 * Gang members may be spread across multiple
2218				 * vdevs, so the best we can do is look at the
2219				 * pool-wide DTL.
2220				 * XXX -- it would be better to change our
2221				 * allocation policy to ensure that this can't
2222				 * happen.
2223				 */
2224				vd = spa->spa_root_vdev;
2225			}
2226			if (vdev_dtl_contains(&vd->vdev_dtl_map,
2227			    bp->blk_birth, 1))
2228				needs_resilver = B_TRUE;
2229		}
2230	}
2231
2232	if (spa->spa_scrub_type == POOL_SCRUB_EVERYTHING)
2233		spa_scrub_io_start(spa, bp, ZIO_PRIORITY_SCRUB,
2234		    ZIO_FLAG_SCRUB, &bc->bc_bookmark);
2235	else if (needs_resilver)
2236		spa_scrub_io_start(spa, bp, ZIO_PRIORITY_RESILVER,
2237		    ZIO_FLAG_RESILVER, &bc->bc_bookmark);
2238
2239	return (0);
2240}
2241
2242static void
2243spa_scrub_thread(spa_t *spa)
2244{
2245	callb_cpr_t cprinfo;
2246	traverse_handle_t *th = spa->spa_scrub_th;
2247	vdev_t *rvd = spa->spa_root_vdev;
2248	pool_scrub_type_t scrub_type = spa->spa_scrub_type;
2249	int error = 0;
2250	boolean_t complete;
2251
2252	CALLB_CPR_INIT(&cprinfo, &spa->spa_scrub_lock, callb_generic_cpr, FTAG);
2253
2254	/*
2255	 * If we're restarting due to a snapshot create/delete,
2256	 * wait for that to complete.
2257	 */
2258	txg_wait_synced(spa_get_dsl(spa), 0);
2259
2260	dprintf("start %s mintxg=%llu maxtxg=%llu\n",
2261	    scrub_type == POOL_SCRUB_RESILVER ? "resilver" : "scrub",
2262	    spa->spa_scrub_mintxg, spa->spa_scrub_maxtxg);
2263
2264	spa_config_enter(spa, RW_WRITER, FTAG);
2265	vdev_reopen(rvd);		/* purge all vdev caches */
2266	vdev_config_dirty(rvd);		/* rewrite all disk labels */
2267	vdev_scrub_stat_update(rvd, scrub_type, B_FALSE);
2268	spa_config_exit(spa, FTAG);
2269
2270	mutex_enter(&spa->spa_scrub_lock);
2271	spa->spa_scrub_errors = 0;
2272	spa->spa_scrub_active = 1;
2273	ASSERT(spa->spa_scrub_inflight == 0);
2274	ASSERT(spa->spa_scrub_throttled == 0);
2275
2276	while (!spa->spa_scrub_stop) {
2277		CALLB_CPR_SAFE_BEGIN(&cprinfo);
2278		while (spa->spa_scrub_suspended) {
2279			spa->spa_scrub_active = 0;
2280			cv_broadcast(&spa->spa_scrub_cv);
2281			cv_wait(&spa->spa_scrub_cv, &spa->spa_scrub_lock);
2282			spa->spa_scrub_active = 1;
2283		}
2284		CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_scrub_lock);
2285
2286		if (spa->spa_scrub_restart_txg != 0)
2287			break;
2288
2289		mutex_exit(&spa->spa_scrub_lock);
2290		error = traverse_more(th);
2291		mutex_enter(&spa->spa_scrub_lock);
2292		if (error != EAGAIN)
2293			break;
2294
2295		while (spa->spa_scrub_throttled > 0)
2296			cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock);
2297	}
2298
2299	while (spa->spa_scrub_inflight)
2300		cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock);
2301
2302	spa->spa_scrub_active = 0;
2303	cv_broadcast(&spa->spa_scrub_cv);
2304
2305	mutex_exit(&spa->spa_scrub_lock);
2306
2307	spa_config_enter(spa, RW_WRITER, FTAG);
2308
2309	mutex_enter(&spa->spa_scrub_lock);
2310
2311	/*
2312	 * Note: we check spa_scrub_restart_txg under both spa_scrub_lock
2313	 * AND the spa config lock to synchronize with any config changes
2314	 * that revise the DTLs under spa_vdev_enter() / spa_vdev_exit().
2315	 */
2316	if (spa->spa_scrub_restart_txg != 0)
2317		error = ERESTART;
2318
2319	if (spa->spa_scrub_stop)
2320		error = EINTR;
2321
2322	/*
2323	 * Even if there were uncorrectable errors, we consider the scrub
2324	 * completed.  The downside is that if there is a transient error during
2325	 * a resilver, we won't resilver the data properly to the target.  But
2326	 * if the damage is permanent (more likely) we will resilver forever,
2327	 * which isn't really acceptable.  Since there is enough information for
2328	 * the user to know what has failed and why, this seems like a more
2329	 * tractable approach.
2330	 */
2331	complete = (error == 0);
2332
2333	dprintf("end %s to maxtxg=%llu %s, traverse=%d, %llu errors, stop=%u\n",
2334	    scrub_type == POOL_SCRUB_RESILVER ? "resilver" : "scrub",
2335	    spa->spa_scrub_maxtxg, complete ? "done" : "FAILED",
2336	    error, spa->spa_scrub_errors, spa->spa_scrub_stop);
2337
2338	mutex_exit(&spa->spa_scrub_lock);
2339
2340	/*
2341	 * If the scrub/resilver completed, update all DTLs to reflect this.
2342	 * Whether it succeeded or not, vacate all temporary scrub DTLs.
2343	 */
2344	vdev_dtl_reassess(rvd, spa_last_synced_txg(spa) + 1,
2345	    complete ? spa->spa_scrub_maxtxg : 0, B_TRUE);
2346	vdev_scrub_stat_update(rvd, POOL_SCRUB_NONE, complete);
2347	spa_errlog_rotate(spa);
2348
2349	spa_config_exit(spa, FTAG);
2350
2351	mutex_enter(&spa->spa_scrub_lock);
2352
2353	/*
2354	 * We may have finished replacing a device.
2355	 * Let the async thread assess this and handle the detach.
2356	 */
2357	spa_async_request(spa, SPA_ASYNC_REPLACE_DONE);
2358
2359	/*
2360	 * If we were told to restart, our final act is to start a new scrub.
2361	 */
2362	if (error == ERESTART)
2363		spa_async_request(spa, scrub_type == POOL_SCRUB_RESILVER ?
2364		    SPA_ASYNC_RESILVER : SPA_ASYNC_SCRUB);
2365
2366	spa->spa_scrub_type = POOL_SCRUB_NONE;
2367	spa->spa_scrub_active = 0;
2368	spa->spa_scrub_thread = NULL;
2369	cv_broadcast(&spa->spa_scrub_cv);
2370	CALLB_CPR_EXIT(&cprinfo);	/* drops &spa->spa_scrub_lock */
2371	thread_exit();
2372}
2373
2374void
2375spa_scrub_suspend(spa_t *spa)
2376{
2377	mutex_enter(&spa->spa_scrub_lock);
2378	spa->spa_scrub_suspended++;
2379	while (spa->spa_scrub_active) {
2380		cv_broadcast(&spa->spa_scrub_cv);
2381		cv_wait(&spa->spa_scrub_cv, &spa->spa_scrub_lock);
2382	}
2383	while (spa->spa_scrub_inflight)
2384		cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock);
2385	mutex_exit(&spa->spa_scrub_lock);
2386}
2387
2388void
2389spa_scrub_resume(spa_t *spa)
2390{
2391	mutex_enter(&spa->spa_scrub_lock);
2392	ASSERT(spa->spa_scrub_suspended != 0);
2393	if (--spa->spa_scrub_suspended == 0)
2394		cv_broadcast(&spa->spa_scrub_cv);
2395	mutex_exit(&spa->spa_scrub_lock);
2396}
2397
2398void
2399spa_scrub_restart(spa_t *spa, uint64_t txg)
2400{
2401	/*
2402	 * Something happened (e.g. snapshot create/delete) that means
2403	 * we must restart any in-progress scrubs.  The itinerary will
2404	 * fix this properly.
2405	 */
2406	mutex_enter(&spa->spa_scrub_lock);
2407	spa->spa_scrub_restart_txg = txg;
2408	mutex_exit(&spa->spa_scrub_lock);
2409}
2410
2411int
2412spa_scrub(spa_t *spa, pool_scrub_type_t type, boolean_t force)
2413{
2414	space_seg_t *ss;
2415	uint64_t mintxg, maxtxg;
2416	vdev_t *rvd = spa->spa_root_vdev;
2417
2418	if ((uint_t)type >= POOL_SCRUB_TYPES)
2419		return (ENOTSUP);
2420
2421	mutex_enter(&spa->spa_scrub_lock);
2422
2423	/*
2424	 * If there's a scrub or resilver already in progress, stop it.
2425	 */
2426	while (spa->spa_scrub_thread != NULL) {
2427		/*
2428		 * Don't stop a resilver unless forced.
2429		 */
2430		if (spa->spa_scrub_type == POOL_SCRUB_RESILVER && !force) {
2431			mutex_exit(&spa->spa_scrub_lock);
2432			return (EBUSY);
2433		}
2434		spa->spa_scrub_stop = 1;
2435		cv_broadcast(&spa->spa_scrub_cv);
2436		cv_wait(&spa->spa_scrub_cv, &spa->spa_scrub_lock);
2437	}
2438
2439	/*
2440	 * Terminate the previous traverse.
2441	 */
2442	if (spa->spa_scrub_th != NULL) {
2443		traverse_fini(spa->spa_scrub_th);
2444		spa->spa_scrub_th = NULL;
2445	}
2446
2447	if (rvd == NULL) {
2448		ASSERT(spa->spa_scrub_stop == 0);
2449		ASSERT(spa->spa_scrub_type == type);
2450		ASSERT(spa->spa_scrub_restart_txg == 0);
2451		mutex_exit(&spa->spa_scrub_lock);
2452		return (0);
2453	}
2454
2455	mintxg = TXG_INITIAL - 1;
2456	maxtxg = spa_last_synced_txg(spa) + 1;
2457
2458	mutex_enter(&rvd->vdev_dtl_lock);
2459
2460	if (rvd->vdev_dtl_map.sm_space == 0) {
2461		/*
2462		 * The pool-wide DTL is empty.
2463		 * If this is a resilver, there's nothing to do except
2464		 * check whether any in-progress replacements have completed.
2465		 */
2466		if (type == POOL_SCRUB_RESILVER) {
2467			type = POOL_SCRUB_NONE;
2468			spa_async_request(spa, SPA_ASYNC_REPLACE_DONE);
2469		}
2470	} else {
2471		/*
2472		 * The pool-wide DTL is non-empty.
2473		 * If this is a normal scrub, upgrade to a resilver instead.
2474		 */
2475		if (type == POOL_SCRUB_EVERYTHING)
2476			type = POOL_SCRUB_RESILVER;
2477	}
2478
2479	if (type == POOL_SCRUB_RESILVER) {
2480		/*
2481		 * Determine the resilvering boundaries.
2482		 *
2483		 * Note: (mintxg, maxtxg) is an open interval,
2484		 * i.e. mintxg and maxtxg themselves are not included.
2485		 *
2486		 * Note: for maxtxg, we MIN with spa_last_synced_txg(spa) + 1
2487		 * so we don't claim to resilver a txg that's still changing.
2488		 */
2489		ss = avl_first(&rvd->vdev_dtl_map.sm_root);
2490		mintxg = ss->ss_start - 1;
2491		ss = avl_last(&rvd->vdev_dtl_map.sm_root);
2492		maxtxg = MIN(ss->ss_end, maxtxg);
2493	}
2494
2495	mutex_exit(&rvd->vdev_dtl_lock);
2496
2497	spa->spa_scrub_stop = 0;
2498	spa->spa_scrub_type = type;
2499	spa->spa_scrub_restart_txg = 0;
2500
2501	if (type != POOL_SCRUB_NONE) {
2502		spa->spa_scrub_mintxg = mintxg;
2503		spa->spa_scrub_maxtxg = maxtxg;
2504		spa->spa_scrub_th = traverse_init(spa, spa_scrub_cb, NULL,
2505		    ADVANCE_PRE | ADVANCE_PRUNE | ADVANCE_ZIL,
2506		    ZIO_FLAG_CANFAIL);
2507		traverse_add_pool(spa->spa_scrub_th, mintxg, maxtxg);
2508		spa->spa_scrub_thread = thread_create(NULL, 0,
2509		    spa_scrub_thread, spa, 0, &p0, TS_RUN, minclsyspri);
2510	}
2511
2512	mutex_exit(&spa->spa_scrub_lock);
2513
2514	return (0);
2515}
2516
2517/*
2518 * ==========================================================================
2519 * SPA async task processing
2520 * ==========================================================================
2521 */
2522
2523static void
2524spa_async_reopen(spa_t *spa)
2525{
2526	vdev_t *rvd = spa->spa_root_vdev;
2527	vdev_t *tvd;
2528	int c;
2529
2530	spa_config_enter(spa, RW_WRITER, FTAG);
2531
2532	for (c = 0; c < rvd->vdev_children; c++) {
2533		tvd = rvd->vdev_child[c];
2534		if (tvd->vdev_reopen_wanted) {
2535			tvd->vdev_reopen_wanted = 0;
2536			vdev_reopen(tvd);
2537		}
2538	}
2539
2540	spa_config_exit(spa, FTAG);
2541}
2542
2543static void
2544spa_async_thread(spa_t *spa)
2545{
2546	int tasks;
2547
2548	ASSERT(spa->spa_sync_on);
2549
2550	mutex_enter(&spa->spa_async_lock);
2551	tasks = spa->spa_async_tasks;
2552	spa->spa_async_tasks = 0;
2553	mutex_exit(&spa->spa_async_lock);
2554
2555	/*
2556	 * See if the config needs to be updated.
2557	 */
2558	if (tasks & SPA_ASYNC_CONFIG_UPDATE) {
2559		mutex_enter(&spa_namespace_lock);
2560		spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
2561		mutex_exit(&spa_namespace_lock);
2562	}
2563
2564	/*
2565	 * See if any devices need to be reopened.
2566	 */
2567	if (tasks & SPA_ASYNC_REOPEN)
2568		spa_async_reopen(spa);
2569
2570	/*
2571	 * If any devices are done replacing, detach them.
2572	 */
2573	if (tasks & SPA_ASYNC_REPLACE_DONE)
2574		spa_vdev_replace_done(spa);
2575
2576	/*
2577	 * Kick off a scrub.
2578	 */
2579	if (tasks & SPA_ASYNC_SCRUB)
2580		VERIFY(spa_scrub(spa, POOL_SCRUB_EVERYTHING, B_TRUE) == 0);
2581
2582	/*
2583	 * Kick off a resilver.
2584	 */
2585	if (tasks & SPA_ASYNC_RESILVER)
2586		VERIFY(spa_scrub(spa, POOL_SCRUB_RESILVER, B_TRUE) == 0);
2587
2588	/*
2589	 * Let the world know that we're done.
2590	 */
2591	mutex_enter(&spa->spa_async_lock);
2592	spa->spa_async_thread = NULL;
2593	cv_broadcast(&spa->spa_async_cv);
2594	mutex_exit(&spa->spa_async_lock);
2595	thread_exit();
2596}
2597
2598void
2599spa_async_suspend(spa_t *spa)
2600{
2601	mutex_enter(&spa->spa_async_lock);
2602	spa->spa_async_suspended++;
2603	while (spa->spa_async_thread != NULL)
2604		cv_wait(&spa->spa_async_cv, &spa->spa_async_lock);
2605	mutex_exit(&spa->spa_async_lock);
2606}
2607
2608void
2609spa_async_resume(spa_t *spa)
2610{
2611	mutex_enter(&spa->spa_async_lock);
2612	ASSERT(spa->spa_async_suspended != 0);
2613	spa->spa_async_suspended--;
2614	mutex_exit(&spa->spa_async_lock);
2615}
2616
2617static void
2618spa_async_dispatch(spa_t *spa)
2619{
2620	mutex_enter(&spa->spa_async_lock);
2621	if (spa->spa_async_tasks && !spa->spa_async_suspended &&
2622	    spa->spa_async_thread == NULL &&
2623	    rootdir != NULL && !vn_is_readonly(rootdir))
2624		spa->spa_async_thread = thread_create(NULL, 0,
2625		    spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri);
2626	mutex_exit(&spa->spa_async_lock);
2627}
2628
2629void
2630spa_async_request(spa_t *spa, int task)
2631{
2632	mutex_enter(&spa->spa_async_lock);
2633	spa->spa_async_tasks |= task;
2634	mutex_exit(&spa->spa_async_lock);
2635}
2636
2637/*
2638 * ==========================================================================
2639 * SPA syncing routines
2640 * ==========================================================================
2641 */
2642
2643static void
2644spa_sync_deferred_frees(spa_t *spa, uint64_t txg)
2645{
2646	bplist_t *bpl = &spa->spa_sync_bplist;
2647	dmu_tx_t *tx;
2648	blkptr_t blk;
2649	uint64_t itor = 0;
2650	zio_t *zio;
2651	int error;
2652	uint8_t c = 1;
2653
2654	zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CONFIG_HELD);
2655
2656	while (bplist_iterate(bpl, &itor, &blk) == 0)
2657		zio_nowait(zio_free(zio, spa, txg, &blk, NULL, NULL));
2658
2659	error = zio_wait(zio);
2660	ASSERT3U(error, ==, 0);
2661
2662	tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
2663	bplist_vacate(bpl, tx);
2664
2665	/*
2666	 * Pre-dirty the first block so we sync to convergence faster.
2667	 * (Usually only the first block is needed.)
2668	 */
2669	dmu_write(spa->spa_meta_objset, spa->spa_sync_bplist_obj, 0, 1, &c, tx);
2670	dmu_tx_commit(tx);
2671}
2672
2673static void
2674spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx)
2675{
2676	char *packed = NULL;
2677	size_t nvsize = 0;
2678	dmu_buf_t *db;
2679
2680	VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0);
2681
2682	packed = kmem_alloc(nvsize, KM_SLEEP);
2683
2684	VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR,
2685	    KM_SLEEP) == 0);
2686
2687	dmu_write(spa->spa_meta_objset, obj, 0, nvsize, packed, tx);
2688
2689	kmem_free(packed, nvsize);
2690
2691	VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db));
2692	dmu_buf_will_dirty(db, tx);
2693	*(uint64_t *)db->db_data = nvsize;
2694	dmu_buf_rele(db, FTAG);
2695}
2696
2697static void
2698spa_sync_spares(spa_t *spa, dmu_tx_t *tx)
2699{
2700	nvlist_t *nvroot;
2701	nvlist_t **spares;
2702	int i;
2703
2704	if (!spa->spa_sync_spares)
2705		return;
2706
2707	/*
2708	 * Update the MOS nvlist describing the list of available spares.
2709	 * spa_validate_spares() will have already made sure this nvlist is
2710	 * valid and the vdevs are labelled appropriately.
2711	 */
2712	if (spa->spa_spares_object == 0) {
2713		spa->spa_spares_object = dmu_object_alloc(spa->spa_meta_objset,
2714		    DMU_OT_PACKED_NVLIST, 1 << 14,
2715		    DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx);
2716		VERIFY(zap_update(spa->spa_meta_objset,
2717		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SPARES,
2718		    sizeof (uint64_t), 1, &spa->spa_spares_object, tx) == 0);
2719	}
2720
2721	VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0);
2722	if (spa->spa_nspares == 0) {
2723		VERIFY(nvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
2724		    NULL, 0) == 0);
2725	} else {
2726		spares = kmem_alloc(spa->spa_nspares * sizeof (void *),
2727		    KM_SLEEP);
2728		for (i = 0; i < spa->spa_nspares; i++)
2729			spares[i] = vdev_config_generate(spa,
2730			    spa->spa_spares[i], B_FALSE, B_TRUE);
2731		VERIFY(nvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
2732		    spares, spa->spa_nspares) == 0);
2733		for (i = 0; i < spa->spa_nspares; i++)
2734			nvlist_free(spares[i]);
2735		kmem_free(spares, spa->spa_nspares * sizeof (void *));
2736	}
2737
2738	spa_sync_nvlist(spa, spa->spa_spares_object, nvroot, tx);
2739	nvlist_free(nvroot);
2740
2741	spa->spa_sync_spares = B_FALSE;
2742}
2743
2744static void
2745spa_sync_config_object(spa_t *spa, dmu_tx_t *tx)
2746{
2747	nvlist_t *config;
2748
2749	if (list_is_empty(&spa->spa_dirty_list))
2750		return;
2751
2752	config = spa_config_generate(spa, NULL, dmu_tx_get_txg(tx), B_FALSE);
2753
2754	if (spa->spa_config_syncing)
2755		nvlist_free(spa->spa_config_syncing);
2756	spa->spa_config_syncing = config;
2757
2758	spa_sync_nvlist(spa, spa->spa_config_object, config, tx);
2759}
2760
2761/*
2762 * Sync the specified transaction group.  New blocks may be dirtied as
2763 * part of the process, so we iterate until it converges.
2764 */
2765void
2766spa_sync(spa_t *spa, uint64_t txg)
2767{
2768	dsl_pool_t *dp = spa->spa_dsl_pool;
2769	objset_t *mos = spa->spa_meta_objset;
2770	bplist_t *bpl = &spa->spa_sync_bplist;
2771	vdev_t *rvd = spa->spa_root_vdev;
2772	vdev_t *vd;
2773	dmu_tx_t *tx;
2774	int dirty_vdevs;
2775
2776	/*
2777	 * Lock out configuration changes.
2778	 */
2779	spa_config_enter(spa, RW_READER, FTAG);
2780
2781	spa->spa_syncing_txg = txg;
2782	spa->spa_sync_pass = 0;
2783
2784	VERIFY(0 == bplist_open(bpl, mos, spa->spa_sync_bplist_obj));
2785
2786	tx = dmu_tx_create_assigned(dp, txg);
2787
2788	/*
2789	 * If we are upgrading to ZFS_VERSION_RAIDZ_DEFLATE this txg,
2790	 * set spa_deflate if we have no raid-z vdevs.
2791	 */
2792	if (spa->spa_ubsync.ub_version < ZFS_VERSION_RAIDZ_DEFLATE &&
2793	    spa->spa_uberblock.ub_version >= ZFS_VERSION_RAIDZ_DEFLATE) {
2794		int i;
2795
2796		for (i = 0; i < rvd->vdev_children; i++) {
2797			vd = rvd->vdev_child[i];
2798			if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE)
2799				break;
2800		}
2801		if (i == rvd->vdev_children) {
2802			spa->spa_deflate = TRUE;
2803			VERIFY(0 == zap_add(spa->spa_meta_objset,
2804			    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE,
2805			    sizeof (uint64_t), 1, &spa->spa_deflate, tx));
2806		}
2807	}
2808
2809	/*
2810	 * If anything has changed in this txg, push the deferred frees
2811	 * from the previous txg.  If not, leave them alone so that we
2812	 * don't generate work on an otherwise idle system.
2813	 */
2814	if (!txg_list_empty(&dp->dp_dirty_datasets, txg) ||
2815	    !txg_list_empty(&dp->dp_dirty_dirs, txg) ||
2816	    !txg_list_empty(&dp->dp_sync_tasks, txg))
2817		spa_sync_deferred_frees(spa, txg);
2818
2819	/*
2820	 * Iterate to convergence.
2821	 */
2822	do {
2823		spa->spa_sync_pass++;
2824
2825		spa_sync_config_object(spa, tx);
2826		spa_sync_spares(spa, tx);
2827		spa_errlog_sync(spa, txg);
2828		dsl_pool_sync(dp, txg);
2829
2830		dirty_vdevs = 0;
2831		while (vd = txg_list_remove(&spa->spa_vdev_txg_list, txg)) {
2832			vdev_sync(vd, txg);
2833			dirty_vdevs++;
2834		}
2835
2836		bplist_sync(bpl, tx);
2837	} while (dirty_vdevs);
2838
2839	bplist_close(bpl);
2840
2841	dprintf("txg %llu passes %d\n", txg, spa->spa_sync_pass);
2842
2843	/*
2844	 * Rewrite the vdev configuration (which includes the uberblock)
2845	 * to commit the transaction group.
2846	 *
2847	 * If there are any dirty vdevs, sync the uberblock to all vdevs.
2848	 * Otherwise, pick a random top-level vdev that's known to be
2849	 * visible in the config cache (see spa_vdev_add() for details).
2850	 * If the write fails, try the next vdev until we're tried them all.
2851	 */
2852	if (!list_is_empty(&spa->spa_dirty_list)) {
2853		VERIFY(vdev_config_sync(rvd, txg) == 0);
2854	} else {
2855		int children = rvd->vdev_children;
2856		int c0 = spa_get_random(children);
2857		int c;
2858
2859		for (c = 0; c < children; c++) {
2860			vd = rvd->vdev_child[(c0 + c) % children];
2861			if (vd->vdev_ms_array == 0)
2862				continue;
2863			if (vdev_config_sync(vd, txg) == 0)
2864				break;
2865		}
2866		if (c == children)
2867			VERIFY(vdev_config_sync(rvd, txg) == 0);
2868	}
2869
2870	dmu_tx_commit(tx);
2871
2872	/*
2873	 * Clear the dirty config list.
2874	 */
2875	while ((vd = list_head(&spa->spa_dirty_list)) != NULL)
2876		vdev_config_clean(vd);
2877
2878	/*
2879	 * Now that the new config has synced transactionally,
2880	 * let it become visible to the config cache.
2881	 */
2882	if (spa->spa_config_syncing != NULL) {
2883		spa_config_set(spa, spa->spa_config_syncing);
2884		spa->spa_config_txg = txg;
2885		spa->spa_config_syncing = NULL;
2886	}
2887
2888	/*
2889	 * Make a stable copy of the fully synced uberblock.
2890	 * We use this as the root for pool traversals.
2891	 */
2892	spa->spa_traverse_wanted = 1;	/* tells traverse_more() to stop */
2893
2894	spa_scrub_suspend(spa);		/* stop scrubbing and finish I/Os */
2895
2896	rw_enter(&spa->spa_traverse_lock, RW_WRITER);
2897	spa->spa_traverse_wanted = 0;
2898	spa->spa_ubsync = spa->spa_uberblock;
2899	rw_exit(&spa->spa_traverse_lock);
2900
2901	spa_scrub_resume(spa);		/* resume scrub with new ubsync */
2902
2903	/*
2904	 * Clean up the ZIL records for the synced txg.
2905	 */
2906	dsl_pool_zil_clean(dp);
2907
2908	/*
2909	 * Update usable space statistics.
2910	 */
2911	while (vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg)))
2912		vdev_sync_done(vd, txg);
2913
2914	/*
2915	 * It had better be the case that we didn't dirty anything
2916	 * since vdev_config_sync().
2917	 */
2918	ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg));
2919	ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg));
2920	ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg));
2921	ASSERT(bpl->bpl_queue == NULL);
2922
2923	spa_config_exit(spa, FTAG);
2924
2925	/*
2926	 * If any async tasks have been requested, kick them off.
2927	 */
2928	spa_async_dispatch(spa);
2929}
2930
2931/*
2932 * Sync all pools.  We don't want to hold the namespace lock across these
2933 * operations, so we take a reference on the spa_t and drop the lock during the
2934 * sync.
2935 */
2936void
2937spa_sync_allpools(void)
2938{
2939	spa_t *spa = NULL;
2940	mutex_enter(&spa_namespace_lock);
2941	while ((spa = spa_next(spa)) != NULL) {
2942		if (spa_state(spa) != POOL_STATE_ACTIVE)
2943			continue;
2944		spa_open_ref(spa, FTAG);
2945		mutex_exit(&spa_namespace_lock);
2946		txg_wait_synced(spa_get_dsl(spa), 0);
2947		mutex_enter(&spa_namespace_lock);
2948		spa_close(spa, FTAG);
2949	}
2950	mutex_exit(&spa_namespace_lock);
2951}
2952
2953/*
2954 * ==========================================================================
2955 * Miscellaneous routines
2956 * ==========================================================================
2957 */
2958
2959/*
2960 * Remove all pools in the system.
2961 */
2962void
2963spa_evict_all(void)
2964{
2965	spa_t *spa;
2966
2967	/*
2968	 * Remove all cached state.  All pools should be closed now,
2969	 * so every spa in the AVL tree should be unreferenced.
2970	 */
2971	mutex_enter(&spa_namespace_lock);
2972	while ((spa = spa_next(NULL)) != NULL) {
2973		/*
2974		 * Stop async tasks.  The async thread may need to detach
2975		 * a device that's been replaced, which requires grabbing
2976		 * spa_namespace_lock, so we must drop it here.
2977		 */
2978		spa_open_ref(spa, FTAG);
2979		mutex_exit(&spa_namespace_lock);
2980		spa_async_suspend(spa);
2981		VERIFY(spa_scrub(spa, POOL_SCRUB_NONE, B_TRUE) == 0);
2982		mutex_enter(&spa_namespace_lock);
2983		spa_close(spa, FTAG);
2984
2985		if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
2986			spa_unload(spa);
2987			spa_deactivate(spa);
2988		}
2989		spa_remove(spa);
2990	}
2991	mutex_exit(&spa_namespace_lock);
2992}
2993
2994vdev_t *
2995spa_lookup_by_guid(spa_t *spa, uint64_t guid)
2996{
2997	return (vdev_lookup_by_guid(spa->spa_root_vdev, guid));
2998}
2999
3000void
3001spa_upgrade(spa_t *spa)
3002{
3003	spa_config_enter(spa, RW_WRITER, FTAG);
3004
3005	/*
3006	 * This should only be called for a non-faulted pool, and since a
3007	 * future version would result in an unopenable pool, this shouldn't be
3008	 * possible.
3009	 */
3010	ASSERT(spa->spa_uberblock.ub_version <= ZFS_VERSION);
3011
3012	spa->spa_uberblock.ub_version = ZFS_VERSION;
3013	vdev_config_dirty(spa->spa_root_vdev);
3014
3015	spa_config_exit(spa, FTAG);
3016
3017	txg_wait_synced(spa_get_dsl(spa), 0);
3018}
3019
3020boolean_t
3021spa_has_spare(spa_t *spa, uint64_t guid)
3022{
3023	int i;
3024
3025	for (i = 0; i < spa->spa_nspares; i++)
3026		if (spa->spa_spares[i]->vdev_guid == guid)
3027			return (B_TRUE);
3028
3029	return (B_FALSE);
3030}
3031