1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
25 * Copyright 2017 Nexenta Systems, Inc.
26 * Copyright (c) 2014 Integros [integros.com]
27 * Copyright 2016 Toomas Soome <tsoome@me.com>
28 * Copyright 2017 Joyent, Inc.
29 */
30
31#include <sys/zfs_context.h>
32#include <sys/fm/fs/zfs.h>
33#include <sys/spa.h>
34#include <sys/spa_impl.h>
35#include <sys/bpobj.h>
36#include <sys/dmu.h>
37#include <sys/dmu_tx.h>
38#include <sys/dsl_dir.h>
39#include <sys/vdev_impl.h>
40#include <sys/uberblock_impl.h>
41#include <sys/metaslab.h>
42#include <sys/metaslab_impl.h>
43#include <sys/space_map.h>
44#include <sys/space_reftree.h>
45#include <sys/zio.h>
46#include <sys/zap.h>
47#include <sys/fs/zfs.h>
48#include <sys/arc.h>
49#include <sys/zil.h>
50#include <sys/dsl_scan.h>
51#include <sys/abd.h>
52#include <sys/vdev_initialize.h>
53
54/*
55 * Virtual device management.
56 */
57
58static vdev_ops_t *vdev_ops_table[] = {
59	&vdev_root_ops,
60	&vdev_raidz_ops,
61	&vdev_mirror_ops,
62	&vdev_replacing_ops,
63	&vdev_spare_ops,
64	&vdev_disk_ops,
65	&vdev_file_ops,
66	&vdev_missing_ops,
67	&vdev_hole_ops,
68	&vdev_indirect_ops,
69	NULL
70};
71
72/* maximum scrub/resilver I/O queue per leaf vdev */
73int zfs_scrub_limit = 10;
74
75/* maximum number of metaslabs per top-level vdev */
76int vdev_max_ms_count = 200;
77
78/* minimum amount of metaslabs per top-level vdev */
79int vdev_min_ms_count = 16;
80
81/* see comment in vdev_metaslab_set_size() */
82int vdev_default_ms_shift = 29;
83
84boolean_t vdev_validate_skip = B_FALSE;
85
86/*
87 * Since the DTL space map of a vdev is not expected to have a lot of
88 * entries, we default its block size to 4K.
89 */
90int vdev_dtl_sm_blksz = (1 << 12);
91
92/*
93 * vdev-wide space maps that have lots of entries written to them at
94 * the end of each transaction can benefit from a higher I/O bandwidth
95 * (e.g. vdev_obsolete_sm), thus we default their block size to 128K.
96 */
97int vdev_standard_sm_blksz = (1 << 17);
98
99/*PRINTFLIKE2*/
100void
101vdev_dbgmsg(vdev_t *vd, const char *fmt, ...)
102{
103	va_list adx;
104	char buf[256];
105
106	va_start(adx, fmt);
107	(void) vsnprintf(buf, sizeof (buf), fmt, adx);
108	va_end(adx);
109
110	if (vd->vdev_path != NULL) {
111		zfs_dbgmsg("%s vdev '%s': %s", vd->vdev_ops->vdev_op_type,
112		    vd->vdev_path, buf);
113	} else {
114		zfs_dbgmsg("%s-%llu vdev (guid %llu): %s",
115		    vd->vdev_ops->vdev_op_type,
116		    (u_longlong_t)vd->vdev_id,
117		    (u_longlong_t)vd->vdev_guid, buf);
118	}
119}
120
121void
122vdev_dbgmsg_print_tree(vdev_t *vd, int indent)
123{
124	char state[20];
125
126	if (vd->vdev_ishole || vd->vdev_ops == &vdev_missing_ops) {
127		zfs_dbgmsg("%*svdev %u: %s", indent, "", vd->vdev_id,
128		    vd->vdev_ops->vdev_op_type);
129		return;
130	}
131
132	switch (vd->vdev_state) {
133	case VDEV_STATE_UNKNOWN:
134		(void) snprintf(state, sizeof (state), "unknown");
135		break;
136	case VDEV_STATE_CLOSED:
137		(void) snprintf(state, sizeof (state), "closed");
138		break;
139	case VDEV_STATE_OFFLINE:
140		(void) snprintf(state, sizeof (state), "offline");
141		break;
142	case VDEV_STATE_REMOVED:
143		(void) snprintf(state, sizeof (state), "removed");
144		break;
145	case VDEV_STATE_CANT_OPEN:
146		(void) snprintf(state, sizeof (state), "can't open");
147		break;
148	case VDEV_STATE_FAULTED:
149		(void) snprintf(state, sizeof (state), "faulted");
150		break;
151	case VDEV_STATE_DEGRADED:
152		(void) snprintf(state, sizeof (state), "degraded");
153		break;
154	case VDEV_STATE_HEALTHY:
155		(void) snprintf(state, sizeof (state), "healthy");
156		break;
157	default:
158		(void) snprintf(state, sizeof (state), "<state %u>",
159		    (uint_t)vd->vdev_state);
160	}
161
162	zfs_dbgmsg("%*svdev %u: %s%s, guid: %llu, path: %s, %s", indent,
163	    "", vd->vdev_id, vd->vdev_ops->vdev_op_type,
164	    vd->vdev_islog ? " (log)" : "",
165	    (u_longlong_t)vd->vdev_guid,
166	    vd->vdev_path ? vd->vdev_path : "N/A", state);
167
168	for (uint64_t i = 0; i < vd->vdev_children; i++)
169		vdev_dbgmsg_print_tree(vd->vdev_child[i], indent + 2);
170}
171
172/*
173 * Given a vdev type, return the appropriate ops vector.
174 */
175static vdev_ops_t *
176vdev_getops(const char *type)
177{
178	vdev_ops_t *ops, **opspp;
179
180	for (opspp = vdev_ops_table; (ops = *opspp) != NULL; opspp++)
181		if (strcmp(ops->vdev_op_type, type) == 0)
182			break;
183
184	return (ops);
185}
186
187/* ARGSUSED */
188void
189vdev_default_xlate(vdev_t *vd, const range_seg_t *in, range_seg_t *res)
190{
191	res->rs_start = in->rs_start;
192	res->rs_end = in->rs_end;
193}
194
195/*
196 * Default asize function: return the MAX of psize with the asize of
197 * all children.  This is what's used by anything other than RAID-Z.
198 */
199uint64_t
200vdev_default_asize(vdev_t *vd, uint64_t psize)
201{
202	uint64_t asize = P2ROUNDUP(psize, 1ULL << vd->vdev_top->vdev_ashift);
203	uint64_t csize;
204
205	for (int c = 0; c < vd->vdev_children; c++) {
206		csize = vdev_psize_to_asize(vd->vdev_child[c], psize);
207		asize = MAX(asize, csize);
208	}
209
210	return (asize);
211}
212
213/*
214 * Get the minimum allocatable size. We define the allocatable size as
215 * the vdev's asize rounded to the nearest metaslab. This allows us to
216 * replace or attach devices which don't have the same physical size but
217 * can still satisfy the same number of allocations.
218 */
219uint64_t
220vdev_get_min_asize(vdev_t *vd)
221{
222	vdev_t *pvd = vd->vdev_parent;
223
224	/*
225	 * If our parent is NULL (inactive spare or cache) or is the root,
226	 * just return our own asize.
227	 */
228	if (pvd == NULL)
229		return (vd->vdev_asize);
230
231	/*
232	 * The top-level vdev just returns the allocatable size rounded
233	 * to the nearest metaslab.
234	 */
235	if (vd == vd->vdev_top)
236		return (P2ALIGN(vd->vdev_asize, 1ULL << vd->vdev_ms_shift));
237
238	/*
239	 * The allocatable space for a raidz vdev is N * sizeof(smallest child),
240	 * so each child must provide at least 1/Nth of its asize.
241	 */
242	if (pvd->vdev_ops == &vdev_raidz_ops)
243		return ((pvd->vdev_min_asize + pvd->vdev_children - 1) /
244		    pvd->vdev_children);
245
246	return (pvd->vdev_min_asize);
247}
248
249void
250vdev_set_min_asize(vdev_t *vd)
251{
252	vd->vdev_min_asize = vdev_get_min_asize(vd);
253
254	for (int c = 0; c < vd->vdev_children; c++)
255		vdev_set_min_asize(vd->vdev_child[c]);
256}
257
258vdev_t *
259vdev_lookup_top(spa_t *spa, uint64_t vdev)
260{
261	vdev_t *rvd = spa->spa_root_vdev;
262
263	ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0);
264
265	if (vdev < rvd->vdev_children) {
266		ASSERT(rvd->vdev_child[vdev] != NULL);
267		return (rvd->vdev_child[vdev]);
268	}
269
270	return (NULL);
271}
272
273vdev_t *
274vdev_lookup_by_guid(vdev_t *vd, uint64_t guid)
275{
276	vdev_t *mvd;
277
278	if (vd->vdev_guid == guid)
279		return (vd);
280
281	for (int c = 0; c < vd->vdev_children; c++)
282		if ((mvd = vdev_lookup_by_guid(vd->vdev_child[c], guid)) !=
283		    NULL)
284			return (mvd);
285
286	return (NULL);
287}
288
289static int
290vdev_count_leaves_impl(vdev_t *vd)
291{
292	int n = 0;
293
294	if (vd->vdev_ops->vdev_op_leaf)
295		return (1);
296
297	for (int c = 0; c < vd->vdev_children; c++)
298		n += vdev_count_leaves_impl(vd->vdev_child[c]);
299
300	return (n);
301}
302
303int
304vdev_count_leaves(spa_t *spa)
305{
306	return (vdev_count_leaves_impl(spa->spa_root_vdev));
307}
308
309void
310vdev_add_child(vdev_t *pvd, vdev_t *cvd)
311{
312	size_t oldsize, newsize;
313	uint64_t id = cvd->vdev_id;
314	vdev_t **newchild;
315	spa_t *spa = cvd->vdev_spa;
316
317	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
318	ASSERT(cvd->vdev_parent == NULL);
319
320	cvd->vdev_parent = pvd;
321
322	if (pvd == NULL)
323		return;
324
325	ASSERT(id >= pvd->vdev_children || pvd->vdev_child[id] == NULL);
326
327	oldsize = pvd->vdev_children * sizeof (vdev_t *);
328	pvd->vdev_children = MAX(pvd->vdev_children, id + 1);
329	newsize = pvd->vdev_children * sizeof (vdev_t *);
330
331	newchild = kmem_zalloc(newsize, KM_SLEEP);
332	if (pvd->vdev_child != NULL) {
333		bcopy(pvd->vdev_child, newchild, oldsize);
334		kmem_free(pvd->vdev_child, oldsize);
335	}
336
337	pvd->vdev_child = newchild;
338	pvd->vdev_child[id] = cvd;
339
340	cvd->vdev_top = (pvd->vdev_top ? pvd->vdev_top: cvd);
341	ASSERT(cvd->vdev_top->vdev_parent->vdev_parent == NULL);
342
343	/*
344	 * Walk up all ancestors to update guid sum.
345	 */
346	for (; pvd != NULL; pvd = pvd->vdev_parent)
347		pvd->vdev_guid_sum += cvd->vdev_guid_sum;
348}
349
350void
351vdev_remove_child(vdev_t *pvd, vdev_t *cvd)
352{
353	int c;
354	uint_t id = cvd->vdev_id;
355
356	ASSERT(cvd->vdev_parent == pvd);
357
358	if (pvd == NULL)
359		return;
360
361	ASSERT(id < pvd->vdev_children);
362	ASSERT(pvd->vdev_child[id] == cvd);
363
364	pvd->vdev_child[id] = NULL;
365	cvd->vdev_parent = NULL;
366
367	for (c = 0; c < pvd->vdev_children; c++)
368		if (pvd->vdev_child[c])
369			break;
370
371	if (c == pvd->vdev_children) {
372		kmem_free(pvd->vdev_child, c * sizeof (vdev_t *));
373		pvd->vdev_child = NULL;
374		pvd->vdev_children = 0;
375	}
376
377	/*
378	 * Walk up all ancestors to update guid sum.
379	 */
380	for (; pvd != NULL; pvd = pvd->vdev_parent)
381		pvd->vdev_guid_sum -= cvd->vdev_guid_sum;
382}
383
384/*
385 * Remove any holes in the child array.
386 */
387void
388vdev_compact_children(vdev_t *pvd)
389{
390	vdev_t **newchild, *cvd;
391	int oldc = pvd->vdev_children;
392	int newc;
393
394	ASSERT(spa_config_held(pvd->vdev_spa, SCL_ALL, RW_WRITER) == SCL_ALL);
395
396	for (int c = newc = 0; c < oldc; c++)
397		if (pvd->vdev_child[c])
398			newc++;
399
400	newchild = kmem_alloc(newc * sizeof (vdev_t *), KM_SLEEP);
401
402	for (int c = newc = 0; c < oldc; c++) {
403		if ((cvd = pvd->vdev_child[c]) != NULL) {
404			newchild[newc] = cvd;
405			cvd->vdev_id = newc++;
406		}
407	}
408
409	kmem_free(pvd->vdev_child, oldc * sizeof (vdev_t *));
410	pvd->vdev_child = newchild;
411	pvd->vdev_children = newc;
412}
413
414/*
415 * Allocate and minimally initialize a vdev_t.
416 */
417vdev_t *
418vdev_alloc_common(spa_t *spa, uint_t id, uint64_t guid, vdev_ops_t *ops)
419{
420	vdev_t *vd;
421	vdev_indirect_config_t *vic;
422
423	vd = kmem_zalloc(sizeof (vdev_t), KM_SLEEP);
424	vic = &vd->vdev_indirect_config;
425
426	if (spa->spa_root_vdev == NULL) {
427		ASSERT(ops == &vdev_root_ops);
428		spa->spa_root_vdev = vd;
429		spa->spa_load_guid = spa_generate_guid(NULL);
430	}
431
432	if (guid == 0 && ops != &vdev_hole_ops) {
433		if (spa->spa_root_vdev == vd) {
434			/*
435			 * The root vdev's guid will also be the pool guid,
436			 * which must be unique among all pools.
437			 */
438			guid = spa_generate_guid(NULL);
439		} else {
440			/*
441			 * Any other vdev's guid must be unique within the pool.
442			 */
443			guid = spa_generate_guid(spa);
444		}
445		ASSERT(!spa_guid_exists(spa_guid(spa), guid));
446	}
447
448	vd->vdev_spa = spa;
449	vd->vdev_id = id;
450	vd->vdev_guid = guid;
451	vd->vdev_guid_sum = guid;
452	vd->vdev_ops = ops;
453	vd->vdev_state = VDEV_STATE_CLOSED;
454	vd->vdev_ishole = (ops == &vdev_hole_ops);
455	vic->vic_prev_indirect_vdev = UINT64_MAX;
456
457	rw_init(&vd->vdev_indirect_rwlock, NULL, RW_DEFAULT, NULL);
458	mutex_init(&vd->vdev_obsolete_lock, NULL, MUTEX_DEFAULT, NULL);
459	vd->vdev_obsolete_segments = range_tree_create(NULL, NULL);
460
461	mutex_init(&vd->vdev_dtl_lock, NULL, MUTEX_DEFAULT, NULL);
462	mutex_init(&vd->vdev_stat_lock, NULL, MUTEX_DEFAULT, NULL);
463	mutex_init(&vd->vdev_probe_lock, NULL, MUTEX_DEFAULT, NULL);
464	mutex_init(&vd->vdev_queue_lock, NULL, MUTEX_DEFAULT, NULL);
465	mutex_init(&vd->vdev_initialize_lock, NULL, MUTEX_DEFAULT, NULL);
466	mutex_init(&vd->vdev_initialize_io_lock, NULL, MUTEX_DEFAULT, NULL);
467	cv_init(&vd->vdev_initialize_cv, NULL, CV_DEFAULT, NULL);
468	cv_init(&vd->vdev_initialize_io_cv, NULL, CV_DEFAULT, NULL);
469
470	for (int t = 0; t < DTL_TYPES; t++) {
471		vd->vdev_dtl[t] = range_tree_create(NULL, NULL);
472	}
473	txg_list_create(&vd->vdev_ms_list, spa,
474	    offsetof(struct metaslab, ms_txg_node));
475	txg_list_create(&vd->vdev_dtl_list, spa,
476	    offsetof(struct vdev, vdev_dtl_node));
477	vd->vdev_stat.vs_timestamp = gethrtime();
478	vdev_queue_init(vd);
479	vdev_cache_init(vd);
480
481	return (vd);
482}
483
484/*
485 * Allocate a new vdev.  The 'alloctype' is used to control whether we are
486 * creating a new vdev or loading an existing one - the behavior is slightly
487 * different for each case.
488 */
489int
490vdev_alloc(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, uint_t id,
491    int alloctype)
492{
493	vdev_ops_t *ops;
494	char *type;
495	uint64_t guid = 0, islog, nparity;
496	vdev_t *vd;
497	vdev_indirect_config_t *vic;
498
499	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
500
501	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) != 0)
502		return (SET_ERROR(EINVAL));
503
504	if ((ops = vdev_getops(type)) == NULL)
505		return (SET_ERROR(EINVAL));
506
507	/*
508	 * If this is a load, get the vdev guid from the nvlist.
509	 * Otherwise, vdev_alloc_common() will generate one for us.
510	 */
511	if (alloctype == VDEV_ALLOC_LOAD) {
512		uint64_t label_id;
513
514		if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_ID, &label_id) ||
515		    label_id != id)
516			return (SET_ERROR(EINVAL));
517
518		if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
519			return (SET_ERROR(EINVAL));
520	} else if (alloctype == VDEV_ALLOC_SPARE) {
521		if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
522			return (SET_ERROR(EINVAL));
523	} else if (alloctype == VDEV_ALLOC_L2CACHE) {
524		if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
525			return (SET_ERROR(EINVAL));
526	} else if (alloctype == VDEV_ALLOC_ROOTPOOL) {
527		if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
528			return (SET_ERROR(EINVAL));
529	}
530
531	/*
532	 * The first allocated vdev must be of type 'root'.
533	 */
534	if (ops != &vdev_root_ops && spa->spa_root_vdev == NULL)
535		return (SET_ERROR(EINVAL));
536
537	/*
538	 * Determine whether we're a log vdev.
539	 */
540	islog = 0;
541	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_IS_LOG, &islog);
542	if (islog && spa_version(spa) < SPA_VERSION_SLOGS)
543		return (SET_ERROR(ENOTSUP));
544
545	if (ops == &vdev_hole_ops && spa_version(spa) < SPA_VERSION_HOLES)
546		return (SET_ERROR(ENOTSUP));
547
548	/*
549	 * Set the nparity property for RAID-Z vdevs.
550	 */
551	nparity = -1ULL;
552	if (ops == &vdev_raidz_ops) {
553		if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NPARITY,
554		    &nparity) == 0) {
555			if (nparity == 0 || nparity > VDEV_RAIDZ_MAXPARITY)
556				return (SET_ERROR(EINVAL));
557			/*
558			 * Previous versions could only support 1 or 2 parity
559			 * device.
560			 */
561			if (nparity > 1 &&
562			    spa_version(spa) < SPA_VERSION_RAIDZ2)
563				return (SET_ERROR(ENOTSUP));
564			if (nparity > 2 &&
565			    spa_version(spa) < SPA_VERSION_RAIDZ3)
566				return (SET_ERROR(ENOTSUP));
567		} else {
568			/*
569			 * We require the parity to be specified for SPAs that
570			 * support multiple parity levels.
571			 */
572			if (spa_version(spa) >= SPA_VERSION_RAIDZ2)
573				return (SET_ERROR(EINVAL));
574			/*
575			 * Otherwise, we default to 1 parity device for RAID-Z.
576			 */
577			nparity = 1;
578		}
579	} else {
580		nparity = 0;
581	}
582	ASSERT(nparity != -1ULL);
583
584	vd = vdev_alloc_common(spa, id, guid, ops);
585	vic = &vd->vdev_indirect_config;
586
587	vd->vdev_islog = islog;
588	vd->vdev_nparity = nparity;
589
590	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &vd->vdev_path) == 0)
591		vd->vdev_path = spa_strdup(vd->vdev_path);
592	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_DEVID, &vd->vdev_devid) == 0)
593		vd->vdev_devid = spa_strdup(vd->vdev_devid);
594	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PHYS_PATH,
595	    &vd->vdev_physpath) == 0)
596		vd->vdev_physpath = spa_strdup(vd->vdev_physpath);
597	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_FRU, &vd->vdev_fru) == 0)
598		vd->vdev_fru = spa_strdup(vd->vdev_fru);
599
600	/*
601	 * Set the whole_disk property.  If it's not specified, leave the value
602	 * as -1.
603	 */
604	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_WHOLE_DISK,
605	    &vd->vdev_wholedisk) != 0)
606		vd->vdev_wholedisk = -1ULL;
607
608	ASSERT0(vic->vic_mapping_object);
609	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_INDIRECT_OBJECT,
610	    &vic->vic_mapping_object);
611	ASSERT0(vic->vic_births_object);
612	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_INDIRECT_BIRTHS,
613	    &vic->vic_births_object);
614	ASSERT3U(vic->vic_prev_indirect_vdev, ==, UINT64_MAX);
615	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_PREV_INDIRECT_VDEV,
616	    &vic->vic_prev_indirect_vdev);
617
618	/*
619	 * Look for the 'not present' flag.  This will only be set if the device
620	 * was not present at the time of import.
621	 */
622	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
623	    &vd->vdev_not_present);
624
625	/*
626	 * Get the alignment requirement.
627	 */
628	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_ASHIFT, &vd->vdev_ashift);
629
630	/*
631	 * Retrieve the vdev creation time.
632	 */
633	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_CREATE_TXG,
634	    &vd->vdev_crtxg);
635
636	/*
637	 * If we're a top-level vdev, try to load the allocation parameters.
638	 */
639	if (parent && !parent->vdev_parent &&
640	    (alloctype == VDEV_ALLOC_LOAD || alloctype == VDEV_ALLOC_SPLIT)) {
641		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_METASLAB_ARRAY,
642		    &vd->vdev_ms_array);
643		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_METASLAB_SHIFT,
644		    &vd->vdev_ms_shift);
645		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_ASIZE,
646		    &vd->vdev_asize);
647		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_REMOVING,
648		    &vd->vdev_removing);
649		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_VDEV_TOP_ZAP,
650		    &vd->vdev_top_zap);
651	} else {
652		ASSERT0(vd->vdev_top_zap);
653	}
654
655	if (parent && !parent->vdev_parent && alloctype != VDEV_ALLOC_ATTACH) {
656		ASSERT(alloctype == VDEV_ALLOC_LOAD ||
657		    alloctype == VDEV_ALLOC_ADD ||
658		    alloctype == VDEV_ALLOC_SPLIT ||
659		    alloctype == VDEV_ALLOC_ROOTPOOL);
660		vd->vdev_mg = metaslab_group_create(islog ?
661		    spa_log_class(spa) : spa_normal_class(spa), vd,
662		    spa->spa_alloc_count);
663	}
664
665	if (vd->vdev_ops->vdev_op_leaf &&
666	    (alloctype == VDEV_ALLOC_LOAD || alloctype == VDEV_ALLOC_SPLIT)) {
667		(void) nvlist_lookup_uint64(nv,
668		    ZPOOL_CONFIG_VDEV_LEAF_ZAP, &vd->vdev_leaf_zap);
669	} else {
670		ASSERT0(vd->vdev_leaf_zap);
671	}
672
673	/*
674	 * If we're a leaf vdev, try to load the DTL object and other state.
675	 */
676
677	if (vd->vdev_ops->vdev_op_leaf &&
678	    (alloctype == VDEV_ALLOC_LOAD || alloctype == VDEV_ALLOC_L2CACHE ||
679	    alloctype == VDEV_ALLOC_ROOTPOOL)) {
680		if (alloctype == VDEV_ALLOC_LOAD) {
681			(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_DTL,
682			    &vd->vdev_dtl_object);
683			(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_UNSPARE,
684			    &vd->vdev_unspare);
685		}
686
687		if (alloctype == VDEV_ALLOC_ROOTPOOL) {
688			uint64_t spare = 0;
689
690			if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_IS_SPARE,
691			    &spare) == 0 && spare)
692				spa_spare_add(vd);
693		}
694
695		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_OFFLINE,
696		    &vd->vdev_offline);
697
698		(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_RESILVER_TXG,
699		    &vd->vdev_resilver_txg);
700
701		/*
702		 * When importing a pool, we want to ignore the persistent fault
703		 * state, as the diagnosis made on another system may not be
704		 * valid in the current context.  Local vdevs will
705		 * remain in the faulted state.
706		 */
707		if (spa_load_state(spa) == SPA_LOAD_OPEN) {
708			(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_FAULTED,
709			    &vd->vdev_faulted);
710			(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_DEGRADED,
711			    &vd->vdev_degraded);
712			(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_REMOVED,
713			    &vd->vdev_removed);
714
715			if (vd->vdev_faulted || vd->vdev_degraded) {
716				char *aux;
717
718				vd->vdev_label_aux =
719				    VDEV_AUX_ERR_EXCEEDED;
720				if (nvlist_lookup_string(nv,
721				    ZPOOL_CONFIG_AUX_STATE, &aux) == 0 &&
722				    strcmp(aux, "external") == 0)
723					vd->vdev_label_aux = VDEV_AUX_EXTERNAL;
724			}
725		}
726	}
727
728	/*
729	 * Add ourselves to the parent's list of children.
730	 */
731	vdev_add_child(parent, vd);
732
733	*vdp = vd;
734
735	return (0);
736}
737
738void
739vdev_free(vdev_t *vd)
740{
741	spa_t *spa = vd->vdev_spa;
742	ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
743
744	/*
745	 * vdev_free() implies closing the vdev first.  This is simpler than
746	 * trying to ensure complicated semantics for all callers.
747	 */
748	vdev_close(vd);
749
750	ASSERT(!list_link_active(&vd->vdev_config_dirty_node));
751	ASSERT(!list_link_active(&vd->vdev_state_dirty_node));
752
753	/*
754	 * Free all children.
755	 */
756	for (int c = 0; c < vd->vdev_children; c++)
757		vdev_free(vd->vdev_child[c]);
758
759	ASSERT(vd->vdev_child == NULL);
760	ASSERT(vd->vdev_guid_sum == vd->vdev_guid);
761	ASSERT(vd->vdev_initialize_thread == NULL);
762
763	/*
764	 * Discard allocation state.
765	 */
766	if (vd->vdev_mg != NULL) {
767		vdev_metaslab_fini(vd);
768		metaslab_group_destroy(vd->vdev_mg);
769	}
770
771	ASSERT0(vd->vdev_stat.vs_space);
772	ASSERT0(vd->vdev_stat.vs_dspace);
773	ASSERT0(vd->vdev_stat.vs_alloc);
774
775	/*
776	 * Remove this vdev from its parent's child list.
777	 */
778	vdev_remove_child(vd->vdev_parent, vd);
779
780	ASSERT(vd->vdev_parent == NULL);
781
782	/*
783	 * Clean up vdev structure.
784	 */
785	vdev_queue_fini(vd);
786	vdev_cache_fini(vd);
787
788	if (vd->vdev_path)
789		spa_strfree(vd->vdev_path);
790	if (vd->vdev_devid)
791		spa_strfree(vd->vdev_devid);
792	if (vd->vdev_physpath)
793		spa_strfree(vd->vdev_physpath);
794	if (vd->vdev_fru)
795		spa_strfree(vd->vdev_fru);
796
797	if (vd->vdev_isspare)
798		spa_spare_remove(vd);
799	if (vd->vdev_isl2cache)
800		spa_l2cache_remove(vd);
801
802	txg_list_destroy(&vd->vdev_ms_list);
803	txg_list_destroy(&vd->vdev_dtl_list);
804
805	mutex_enter(&vd->vdev_dtl_lock);
806	space_map_close(vd->vdev_dtl_sm);
807	for (int t = 0; t < DTL_TYPES; t++) {
808		range_tree_vacate(vd->vdev_dtl[t], NULL, NULL);
809		range_tree_destroy(vd->vdev_dtl[t]);
810	}
811	mutex_exit(&vd->vdev_dtl_lock);
812
813	EQUIV(vd->vdev_indirect_births != NULL,
814	    vd->vdev_indirect_mapping != NULL);
815	if (vd->vdev_indirect_births != NULL) {
816		vdev_indirect_mapping_close(vd->vdev_indirect_mapping);
817		vdev_indirect_births_close(vd->vdev_indirect_births);
818	}
819
820	if (vd->vdev_obsolete_sm != NULL) {
821		ASSERT(vd->vdev_removing ||
822		    vd->vdev_ops == &vdev_indirect_ops);
823		space_map_close(vd->vdev_obsolete_sm);
824		vd->vdev_obsolete_sm = NULL;
825	}
826	range_tree_destroy(vd->vdev_obsolete_segments);
827	rw_destroy(&vd->vdev_indirect_rwlock);
828	mutex_destroy(&vd->vdev_obsolete_lock);
829
830	mutex_destroy(&vd->vdev_queue_lock);
831	mutex_destroy(&vd->vdev_dtl_lock);
832	mutex_destroy(&vd->vdev_stat_lock);
833	mutex_destroy(&vd->vdev_probe_lock);
834	mutex_destroy(&vd->vdev_initialize_lock);
835	mutex_destroy(&vd->vdev_initialize_io_lock);
836	cv_destroy(&vd->vdev_initialize_io_cv);
837	cv_destroy(&vd->vdev_initialize_cv);
838
839	if (vd == spa->spa_root_vdev)
840		spa->spa_root_vdev = NULL;
841
842	kmem_free(vd, sizeof (vdev_t));
843}
844
845/*
846 * Transfer top-level vdev state from svd to tvd.
847 */
848static void
849vdev_top_transfer(vdev_t *svd, vdev_t *tvd)
850{
851	spa_t *spa = svd->vdev_spa;
852	metaslab_t *msp;
853	vdev_t *vd;
854	int t;
855
856	ASSERT(tvd == tvd->vdev_top);
857
858	tvd->vdev_ms_array = svd->vdev_ms_array;
859	tvd->vdev_ms_shift = svd->vdev_ms_shift;
860	tvd->vdev_ms_count = svd->vdev_ms_count;
861	tvd->vdev_top_zap = svd->vdev_top_zap;
862
863	svd->vdev_ms_array = 0;
864	svd->vdev_ms_shift = 0;
865	svd->vdev_ms_count = 0;
866	svd->vdev_top_zap = 0;
867
868	if (tvd->vdev_mg)
869		ASSERT3P(tvd->vdev_mg, ==, svd->vdev_mg);
870	tvd->vdev_mg = svd->vdev_mg;
871	tvd->vdev_ms = svd->vdev_ms;
872
873	svd->vdev_mg = NULL;
874	svd->vdev_ms = NULL;
875
876	if (tvd->vdev_mg != NULL)
877		tvd->vdev_mg->mg_vd = tvd;
878
879	tvd->vdev_checkpoint_sm = svd->vdev_checkpoint_sm;
880	svd->vdev_checkpoint_sm = NULL;
881
882	tvd->vdev_stat.vs_alloc = svd->vdev_stat.vs_alloc;
883	tvd->vdev_stat.vs_space = svd->vdev_stat.vs_space;
884	tvd->vdev_stat.vs_dspace = svd->vdev_stat.vs_dspace;
885
886	svd->vdev_stat.vs_alloc = 0;
887	svd->vdev_stat.vs_space = 0;
888	svd->vdev_stat.vs_dspace = 0;
889
890	/*
891	 * State which may be set on a top-level vdev that's in the
892	 * process of being removed.
893	 */
894	ASSERT0(tvd->vdev_indirect_config.vic_births_object);
895	ASSERT0(tvd->vdev_indirect_config.vic_mapping_object);
896	ASSERT3U(tvd->vdev_indirect_config.vic_prev_indirect_vdev, ==, -1ULL);
897	ASSERT3P(tvd->vdev_indirect_mapping, ==, NULL);
898	ASSERT3P(tvd->vdev_indirect_births, ==, NULL);
899	ASSERT3P(tvd->vdev_obsolete_sm, ==, NULL);
900	ASSERT0(tvd->vdev_removing);
901	tvd->vdev_removing = svd->vdev_removing;
902	tvd->vdev_indirect_config = svd->vdev_indirect_config;
903	tvd->vdev_indirect_mapping = svd->vdev_indirect_mapping;
904	tvd->vdev_indirect_births = svd->vdev_indirect_births;
905	range_tree_swap(&svd->vdev_obsolete_segments,
906	    &tvd->vdev_obsolete_segments);
907	tvd->vdev_obsolete_sm = svd->vdev_obsolete_sm;
908	svd->vdev_indirect_config.vic_mapping_object = 0;
909	svd->vdev_indirect_config.vic_births_object = 0;
910	svd->vdev_indirect_config.vic_prev_indirect_vdev = -1ULL;
911	svd->vdev_indirect_mapping = NULL;
912	svd->vdev_indirect_births = NULL;
913	svd->vdev_obsolete_sm = NULL;
914	svd->vdev_removing = 0;
915
916	for (t = 0; t < TXG_SIZE; t++) {
917		while ((msp = txg_list_remove(&svd->vdev_ms_list, t)) != NULL)
918			(void) txg_list_add(&tvd->vdev_ms_list, msp, t);
919		while ((vd = txg_list_remove(&svd->vdev_dtl_list, t)) != NULL)
920			(void) txg_list_add(&tvd->vdev_dtl_list, vd, t);
921		if (txg_list_remove_this(&spa->spa_vdev_txg_list, svd, t))
922			(void) txg_list_add(&spa->spa_vdev_txg_list, tvd, t);
923	}
924
925	if (list_link_active(&svd->vdev_config_dirty_node)) {
926		vdev_config_clean(svd);
927		vdev_config_dirty(tvd);
928	}
929
930	if (list_link_active(&svd->vdev_state_dirty_node)) {
931		vdev_state_clean(svd);
932		vdev_state_dirty(tvd);
933	}
934
935	tvd->vdev_deflate_ratio = svd->vdev_deflate_ratio;
936	svd->vdev_deflate_ratio = 0;
937
938	tvd->vdev_islog = svd->vdev_islog;
939	svd->vdev_islog = 0;
940}
941
942static void
943vdev_top_update(vdev_t *tvd, vdev_t *vd)
944{
945	if (vd == NULL)
946		return;
947
948	vd->vdev_top = tvd;
949
950	for (int c = 0; c < vd->vdev_children; c++)
951		vdev_top_update(tvd, vd->vdev_child[c]);
952}
953
954/*
955 * Add a mirror/replacing vdev above an existing vdev.
956 */
957vdev_t *
958vdev_add_parent(vdev_t *cvd, vdev_ops_t *ops)
959{
960	spa_t *spa = cvd->vdev_spa;
961	vdev_t *pvd = cvd->vdev_parent;
962	vdev_t *mvd;
963
964	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
965
966	mvd = vdev_alloc_common(spa, cvd->vdev_id, 0, ops);
967
968	mvd->vdev_asize = cvd->vdev_asize;
969	mvd->vdev_min_asize = cvd->vdev_min_asize;
970	mvd->vdev_max_asize = cvd->vdev_max_asize;
971	mvd->vdev_psize = cvd->vdev_psize;
972	mvd->vdev_ashift = cvd->vdev_ashift;
973	mvd->vdev_state = cvd->vdev_state;
974	mvd->vdev_crtxg = cvd->vdev_crtxg;
975
976	vdev_remove_child(pvd, cvd);
977	vdev_add_child(pvd, mvd);
978	cvd->vdev_id = mvd->vdev_children;
979	vdev_add_child(mvd, cvd);
980	vdev_top_update(cvd->vdev_top, cvd->vdev_top);
981
982	if (mvd == mvd->vdev_top)
983		vdev_top_transfer(cvd, mvd);
984
985	return (mvd);
986}
987
988/*
989 * Remove a 1-way mirror/replacing vdev from the tree.
990 */
991void
992vdev_remove_parent(vdev_t *cvd)
993{
994	vdev_t *mvd = cvd->vdev_parent;
995	vdev_t *pvd = mvd->vdev_parent;
996
997	ASSERT(spa_config_held(cvd->vdev_spa, SCL_ALL, RW_WRITER) == SCL_ALL);
998
999	ASSERT(mvd->vdev_children == 1);
1000	ASSERT(mvd->vdev_ops == &vdev_mirror_ops ||
1001	    mvd->vdev_ops == &vdev_replacing_ops ||
1002	    mvd->vdev_ops == &vdev_spare_ops);
1003	cvd->vdev_ashift = mvd->vdev_ashift;
1004
1005	vdev_remove_child(mvd, cvd);
1006	vdev_remove_child(pvd, mvd);
1007
1008	/*
1009	 * If cvd will replace mvd as a top-level vdev, preserve mvd's guid.
1010	 * Otherwise, we could have detached an offline device, and when we
1011	 * go to import the pool we'll think we have two top-level vdevs,
1012	 * instead of a different version of the same top-level vdev.
1013	 */
1014	if (mvd->vdev_top == mvd) {
1015		uint64_t guid_delta = mvd->vdev_guid - cvd->vdev_guid;
1016		cvd->vdev_orig_guid = cvd->vdev_guid;
1017		cvd->vdev_guid += guid_delta;
1018		cvd->vdev_guid_sum += guid_delta;
1019	}
1020	cvd->vdev_id = mvd->vdev_id;
1021	vdev_add_child(pvd, cvd);
1022	vdev_top_update(cvd->vdev_top, cvd->vdev_top);
1023
1024	if (cvd == cvd->vdev_top)
1025		vdev_top_transfer(mvd, cvd);
1026
1027	ASSERT(mvd->vdev_children == 0);
1028	vdev_free(mvd);
1029}
1030
1031int
1032vdev_metaslab_init(vdev_t *vd, uint64_t txg)
1033{
1034	spa_t *spa = vd->vdev_spa;
1035	objset_t *mos = spa->spa_meta_objset;
1036	uint64_t m;
1037	uint64_t oldc = vd->vdev_ms_count;
1038	uint64_t newc = vd->vdev_asize >> vd->vdev_ms_shift;
1039	metaslab_t **mspp;
1040	int error;
1041
1042	ASSERT(txg == 0 || spa_config_held(spa, SCL_ALLOC, RW_WRITER));
1043
1044	/*
1045	 * This vdev is not being allocated from yet or is a hole.
1046	 */
1047	if (vd->vdev_ms_shift == 0)
1048		return (0);
1049
1050	ASSERT(!vd->vdev_ishole);
1051
1052	ASSERT(oldc <= newc);
1053
1054	mspp = kmem_zalloc(newc * sizeof (*mspp), KM_SLEEP);
1055
1056	if (oldc != 0) {
1057		bcopy(vd->vdev_ms, mspp, oldc * sizeof (*mspp));
1058		kmem_free(vd->vdev_ms, oldc * sizeof (*mspp));
1059	}
1060
1061	vd->vdev_ms = mspp;
1062	vd->vdev_ms_count = newc;
1063	for (m = oldc; m < newc; m++) {
1064		uint64_t object = 0;
1065
1066		/*
1067		 * vdev_ms_array may be 0 if we are creating the "fake"
1068		 * metaslabs for an indirect vdev for zdb's leak detection.
1069		 * See zdb_leak_init().
1070		 */
1071		if (txg == 0 && vd->vdev_ms_array != 0) {
1072			error = dmu_read(mos, vd->vdev_ms_array,
1073			    m * sizeof (uint64_t), sizeof (uint64_t), &object,
1074			    DMU_READ_PREFETCH);
1075			if (error != 0) {
1076				vdev_dbgmsg(vd, "unable to read the metaslab "
1077				    "array [error=%d]", error);
1078				return (error);
1079			}
1080		}
1081
1082		error = metaslab_init(vd->vdev_mg, m, object, txg,
1083		    &(vd->vdev_ms[m]));
1084		if (error != 0) {
1085			vdev_dbgmsg(vd, "metaslab_init failed [error=%d]",
1086			    error);
1087			return (error);
1088		}
1089	}
1090
1091	if (txg == 0)
1092		spa_config_enter(spa, SCL_ALLOC, FTAG, RW_WRITER);
1093
1094	/*
1095	 * If the vdev is being removed we don't activate
1096	 * the metaslabs since we want to ensure that no new
1097	 * allocations are performed on this device.
1098	 */
1099	if (oldc == 0 && !vd->vdev_removing)
1100		metaslab_group_activate(vd->vdev_mg);
1101
1102	if (txg == 0)
1103		spa_config_exit(spa, SCL_ALLOC, FTAG);
1104
1105	return (0);
1106}
1107
1108void
1109vdev_metaslab_fini(vdev_t *vd)
1110{
1111	if (vd->vdev_checkpoint_sm != NULL) {
1112		ASSERT(spa_feature_is_active(vd->vdev_spa,
1113		    SPA_FEATURE_POOL_CHECKPOINT));
1114		space_map_close(vd->vdev_checkpoint_sm);
1115		/*
1116		 * Even though we close the space map, we need to set its
1117		 * pointer to NULL. The reason is that vdev_metaslab_fini()
1118		 * may be called multiple times for certain operations
1119		 * (i.e. when destroying a pool) so we need to ensure that
1120		 * this clause never executes twice. This logic is similar
1121		 * to the one used for the vdev_ms clause below.
1122		 */
1123		vd->vdev_checkpoint_sm = NULL;
1124	}
1125
1126	if (vd->vdev_ms != NULL) {
1127		uint64_t count = vd->vdev_ms_count;
1128
1129		metaslab_group_passivate(vd->vdev_mg);
1130		for (uint64_t m = 0; m < count; m++) {
1131			metaslab_t *msp = vd->vdev_ms[m];
1132
1133			if (msp != NULL)
1134				metaslab_fini(msp);
1135		}
1136		kmem_free(vd->vdev_ms, count * sizeof (metaslab_t *));
1137		vd->vdev_ms = NULL;
1138
1139		vd->vdev_ms_count = 0;
1140	}
1141	ASSERT0(vd->vdev_ms_count);
1142}
1143
1144typedef struct vdev_probe_stats {
1145	boolean_t	vps_readable;
1146	boolean_t	vps_writeable;
1147	int		vps_flags;
1148} vdev_probe_stats_t;
1149
1150static void
1151vdev_probe_done(zio_t *zio)
1152{
1153	spa_t *spa = zio->io_spa;
1154	vdev_t *vd = zio->io_vd;
1155	vdev_probe_stats_t *vps = zio->io_private;
1156
1157	ASSERT(vd->vdev_probe_zio != NULL);
1158
1159	if (zio->io_type == ZIO_TYPE_READ) {
1160		if (zio->io_error == 0)
1161			vps->vps_readable = 1;
1162		if (zio->io_error == 0 && spa_writeable(spa)) {
1163			zio_nowait(zio_write_phys(vd->vdev_probe_zio, vd,
1164			    zio->io_offset, zio->io_size, zio->io_abd,
1165			    ZIO_CHECKSUM_OFF, vdev_probe_done, vps,
1166			    ZIO_PRIORITY_SYNC_WRITE, vps->vps_flags, B_TRUE));
1167		} else {
1168			abd_free(zio->io_abd);
1169		}
1170	} else if (zio->io_type == ZIO_TYPE_WRITE) {
1171		if (zio->io_error == 0)
1172			vps->vps_writeable = 1;
1173		abd_free(zio->io_abd);
1174	} else if (zio->io_type == ZIO_TYPE_NULL) {
1175		zio_t *pio;
1176
1177		vd->vdev_cant_read |= !vps->vps_readable;
1178		vd->vdev_cant_write |= !vps->vps_writeable;
1179
1180		if (vdev_readable(vd) &&
1181		    (vdev_writeable(vd) || !spa_writeable(spa))) {
1182			zio->io_error = 0;
1183		} else {
1184			ASSERT(zio->io_error != 0);
1185			vdev_dbgmsg(vd, "failed probe");
1186			zfs_ereport_post(FM_EREPORT_ZFS_PROBE_FAILURE,
1187			    spa, vd, NULL, 0, 0);
1188			zio->io_error = SET_ERROR(ENXIO);
1189		}
1190
1191		mutex_enter(&vd->vdev_probe_lock);
1192		ASSERT(vd->vdev_probe_zio == zio);
1193		vd->vdev_probe_zio = NULL;
1194		mutex_exit(&vd->vdev_probe_lock);
1195
1196		zio_link_t *zl = NULL;
1197		while ((pio = zio_walk_parents(zio, &zl)) != NULL)
1198			if (!vdev_accessible(vd, pio))
1199				pio->io_error = SET_ERROR(ENXIO);
1200
1201		kmem_free(vps, sizeof (*vps));
1202	}
1203}
1204
1205/*
1206 * Determine whether this device is accessible.
1207 *
1208 * Read and write to several known locations: the pad regions of each
1209 * vdev label but the first, which we leave alone in case it contains
1210 * a VTOC.
1211 */
1212zio_t *
1213vdev_probe(vdev_t *vd, zio_t *zio)
1214{
1215	spa_t *spa = vd->vdev_spa;
1216	vdev_probe_stats_t *vps = NULL;
1217	zio_t *pio;
1218
1219	ASSERT(vd->vdev_ops->vdev_op_leaf);
1220
1221	/*
1222	 * Don't probe the probe.
1223	 */
1224	if (zio && (zio->io_flags & ZIO_FLAG_PROBE))
1225		return (NULL);
1226
1227	/*
1228	 * To prevent 'probe storms' when a device fails, we create
1229	 * just one probe i/o at a time.  All zios that want to probe
1230	 * this vdev will become parents of the probe io.
1231	 */
1232	mutex_enter(&vd->vdev_probe_lock);
1233
1234	if ((pio = vd->vdev_probe_zio) == NULL) {
1235		vps = kmem_zalloc(sizeof (*vps), KM_SLEEP);
1236
1237		vps->vps_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_PROBE |
1238		    ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_AGGREGATE |
1239		    ZIO_FLAG_TRYHARD;
1240
1241		if (spa_config_held(spa, SCL_ZIO, RW_WRITER)) {
1242			/*
1243			 * vdev_cant_read and vdev_cant_write can only
1244			 * transition from TRUE to FALSE when we have the
1245			 * SCL_ZIO lock as writer; otherwise they can only
1246			 * transition from FALSE to TRUE.  This ensures that
1247			 * any zio looking at these values can assume that
1248			 * failures persist for the life of the I/O.  That's
1249			 * important because when a device has intermittent
1250			 * connectivity problems, we want to ensure that
1251			 * they're ascribed to the device (ENXIO) and not
1252			 * the zio (EIO).
1253			 *
1254			 * Since we hold SCL_ZIO as writer here, clear both
1255			 * values so the probe can reevaluate from first
1256			 * principles.
1257			 */
1258			vps->vps_flags |= ZIO_FLAG_CONFIG_WRITER;
1259			vd->vdev_cant_read = B_FALSE;
1260			vd->vdev_cant_write = B_FALSE;
1261		}
1262
1263		vd->vdev_probe_zio = pio = zio_null(NULL, spa, vd,
1264		    vdev_probe_done, vps,
1265		    vps->vps_flags | ZIO_FLAG_DONT_PROPAGATE);
1266
1267		/*
1268		 * We can't change the vdev state in this context, so we
1269		 * kick off an async task to do it on our behalf.
1270		 */
1271		if (zio != NULL) {
1272			vd->vdev_probe_wanted = B_TRUE;
1273			spa_async_request(spa, SPA_ASYNC_PROBE);
1274		}
1275	}
1276
1277	if (zio != NULL)
1278		zio_add_child(zio, pio);
1279
1280	mutex_exit(&vd->vdev_probe_lock);
1281
1282	if (vps == NULL) {
1283		ASSERT(zio != NULL);
1284		return (NULL);
1285	}
1286
1287	for (int l = 1; l < VDEV_LABELS; l++) {
1288		zio_nowait(zio_read_phys(pio, vd,
1289		    vdev_label_offset(vd->vdev_psize, l,
1290		    offsetof(vdev_label_t, vl_pad2)), VDEV_PAD_SIZE,
1291		    abd_alloc_for_io(VDEV_PAD_SIZE, B_TRUE),
1292		    ZIO_CHECKSUM_OFF, vdev_probe_done, vps,
1293		    ZIO_PRIORITY_SYNC_READ, vps->vps_flags, B_TRUE));
1294	}
1295
1296	if (zio == NULL)
1297		return (pio);
1298
1299	zio_nowait(pio);
1300	return (NULL);
1301}
1302
1303static void
1304vdev_open_child(void *arg)
1305{
1306	vdev_t *vd = arg;
1307
1308	vd->vdev_open_thread = curthread;
1309	vd->vdev_open_error = vdev_open(vd);
1310	vd->vdev_open_thread = NULL;
1311}
1312
1313boolean_t
1314vdev_uses_zvols(vdev_t *vd)
1315{
1316	if (vd->vdev_path && strncmp(vd->vdev_path, ZVOL_DIR,
1317	    strlen(ZVOL_DIR)) == 0)
1318		return (B_TRUE);
1319	for (int c = 0; c < vd->vdev_children; c++)
1320		if (vdev_uses_zvols(vd->vdev_child[c]))
1321			return (B_TRUE);
1322	return (B_FALSE);
1323}
1324
1325void
1326vdev_open_children(vdev_t *vd)
1327{
1328	taskq_t *tq;
1329	int children = vd->vdev_children;
1330
1331	/*
1332	 * in order to handle pools on top of zvols, do the opens
1333	 * in a single thread so that the same thread holds the
1334	 * spa_namespace_lock
1335	 */
1336	if (vdev_uses_zvols(vd)) {
1337		for (int c = 0; c < children; c++)
1338			vd->vdev_child[c]->vdev_open_error =
1339			    vdev_open(vd->vdev_child[c]);
1340		return;
1341	}
1342	tq = taskq_create("vdev_open", children, minclsyspri,
1343	    children, children, TASKQ_PREPOPULATE);
1344
1345	for (int c = 0; c < children; c++)
1346		VERIFY(taskq_dispatch(tq, vdev_open_child, vd->vdev_child[c],
1347		    TQ_SLEEP) != NULL);
1348
1349	taskq_destroy(tq);
1350}
1351
1352/*
1353 * Compute the raidz-deflation ratio.  Note, we hard-code
1354 * in 128k (1 << 17) because it is the "typical" blocksize.
1355 * Even though SPA_MAXBLOCKSIZE changed, this algorithm can not change,
1356 * otherwise it would inconsistently account for existing bp's.
1357 */
1358static void
1359vdev_set_deflate_ratio(vdev_t *vd)
1360{
1361	if (vd == vd->vdev_top && !vd->vdev_ishole && vd->vdev_ashift != 0) {
1362		vd->vdev_deflate_ratio = (1 << 17) /
1363		    (vdev_psize_to_asize(vd, 1 << 17) >> SPA_MINBLOCKSHIFT);
1364	}
1365}
1366
1367/*
1368 * Prepare a virtual device for access.
1369 */
1370int
1371vdev_open(vdev_t *vd)
1372{
1373	spa_t *spa = vd->vdev_spa;
1374	int error;
1375	uint64_t osize = 0;
1376	uint64_t max_osize = 0;
1377	uint64_t asize, max_asize, psize;
1378	uint64_t ashift = 0;
1379
1380	ASSERT(vd->vdev_open_thread == curthread ||
1381	    spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);
1382	ASSERT(vd->vdev_state == VDEV_STATE_CLOSED ||
1383	    vd->vdev_state == VDEV_STATE_CANT_OPEN ||
1384	    vd->vdev_state == VDEV_STATE_OFFLINE);
1385
1386	vd->vdev_stat.vs_aux = VDEV_AUX_NONE;
1387	vd->vdev_cant_read = B_FALSE;
1388	vd->vdev_cant_write = B_FALSE;
1389	vd->vdev_min_asize = vdev_get_min_asize(vd);
1390
1391	/*
1392	 * If this vdev is not removed, check its fault status.  If it's
1393	 * faulted, bail out of the open.
1394	 */
1395	if (!vd->vdev_removed && vd->vdev_faulted) {
1396		ASSERT(vd->vdev_children == 0);
1397		ASSERT(vd->vdev_label_aux == VDEV_AUX_ERR_EXCEEDED ||
1398		    vd->vdev_label_aux == VDEV_AUX_EXTERNAL);
1399		vdev_set_state(vd, B_TRUE, VDEV_STATE_FAULTED,
1400		    vd->vdev_label_aux);
1401		return (SET_ERROR(ENXIO));
1402	} else if (vd->vdev_offline) {
1403		ASSERT(vd->vdev_children == 0);
1404		vdev_set_state(vd, B_TRUE, VDEV_STATE_OFFLINE, VDEV_AUX_NONE);
1405		return (SET_ERROR(ENXIO));
1406	}
1407
1408	error = vd->vdev_ops->vdev_op_open(vd, &osize, &max_osize, &ashift);
1409
1410	/*
1411	 * Reset the vdev_reopening flag so that we actually close
1412	 * the vdev on error.
1413	 */
1414	vd->vdev_reopening = B_FALSE;
1415	if (zio_injection_enabled && error == 0)
1416		error = zio_handle_device_injection(vd, NULL, ENXIO);
1417
1418	if (error) {
1419		if (vd->vdev_removed &&
1420		    vd->vdev_stat.vs_aux != VDEV_AUX_OPEN_FAILED)
1421			vd->vdev_removed = B_FALSE;
1422
1423		if (vd->vdev_stat.vs_aux == VDEV_AUX_CHILDREN_OFFLINE) {
1424			vdev_set_state(vd, B_TRUE, VDEV_STATE_OFFLINE,
1425			    vd->vdev_stat.vs_aux);
1426		} else {
1427			vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
1428			    vd->vdev_stat.vs_aux);
1429		}
1430		return (error);
1431	}
1432
1433	vd->vdev_removed = B_FALSE;
1434
1435	/*
1436	 * Recheck the faulted flag now that we have confirmed that
1437	 * the vdev is accessible.  If we're faulted, bail.
1438	 */
1439	if (vd->vdev_faulted) {
1440		ASSERT(vd->vdev_children == 0);
1441		ASSERT(vd->vdev_label_aux == VDEV_AUX_ERR_EXCEEDED ||
1442		    vd->vdev_label_aux == VDEV_AUX_EXTERNAL);
1443		vdev_set_state(vd, B_TRUE, VDEV_STATE_FAULTED,
1444		    vd->vdev_label_aux);
1445		return (SET_ERROR(ENXIO));
1446	}
1447
1448	if (vd->vdev_degraded) {
1449		ASSERT(vd->vdev_children == 0);
1450		vdev_set_state(vd, B_TRUE, VDEV_STATE_DEGRADED,
1451		    VDEV_AUX_ERR_EXCEEDED);
1452	} else {
1453		vdev_set_state(vd, B_TRUE, VDEV_STATE_HEALTHY, 0);
1454	}
1455
1456	/*
1457	 * For hole or missing vdevs we just return success.
1458	 */
1459	if (vd->vdev_ishole || vd->vdev_ops == &vdev_missing_ops)
1460		return (0);
1461
1462	for (int c = 0; c < vd->vdev_children; c++) {
1463		if (vd->vdev_child[c]->vdev_state != VDEV_STATE_HEALTHY) {
1464			vdev_set_state(vd, B_TRUE, VDEV_STATE_DEGRADED,
1465			    VDEV_AUX_NONE);
1466			break;
1467		}
1468	}
1469
1470	osize = P2ALIGN(osize, (uint64_t)sizeof (vdev_label_t));
1471	max_osize = P2ALIGN(max_osize, (uint64_t)sizeof (vdev_label_t));
1472
1473	if (vd->vdev_children == 0) {
1474		if (osize < SPA_MINDEVSIZE) {
1475			vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
1476			    VDEV_AUX_TOO_SMALL);
1477			return (SET_ERROR(EOVERFLOW));
1478		}
1479		psize = osize;
1480		asize = osize - (VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE);
1481		max_asize = max_osize - (VDEV_LABEL_START_SIZE +
1482		    VDEV_LABEL_END_SIZE);
1483	} else {
1484		if (vd->vdev_parent != NULL && osize < SPA_MINDEVSIZE -
1485		    (VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE)) {
1486			vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
1487			    VDEV_AUX_TOO_SMALL);
1488			return (SET_ERROR(EOVERFLOW));
1489		}
1490		psize = 0;
1491		asize = osize;
1492		max_asize = max_osize;
1493	}
1494
1495	vd->vdev_psize = psize;
1496
1497	/*
1498	 * Make sure the allocatable size hasn't shrunk too much.
1499	 */
1500	if (asize < vd->vdev_min_asize) {
1501		vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
1502		    VDEV_AUX_BAD_LABEL);
1503		return (SET_ERROR(EINVAL));
1504	}
1505
1506	if (vd->vdev_asize == 0) {
1507		/*
1508		 * This is the first-ever open, so use the computed values.
1509		 * For testing purposes, a higher ashift can be requested.
1510		 */
1511		vd->vdev_asize = asize;
1512		vd->vdev_max_asize = max_asize;
1513		vd->vdev_ashift = MAX(ashift, vd->vdev_ashift);
1514	} else {
1515		/*
1516		 * Detect if the alignment requirement has increased.
1517		 * We don't want to make the pool unavailable, just
1518		 * issue a warning instead.
1519		 */
1520		if (ashift > vd->vdev_top->vdev_ashift &&
1521		    vd->vdev_ops->vdev_op_leaf) {
1522			cmn_err(CE_WARN,
1523			    "Disk, '%s', has a block alignment that is "
1524			    "larger than the pool's alignment\n",
1525			    vd->vdev_path);
1526		}
1527		vd->vdev_max_asize = max_asize;
1528	}
1529
1530	/*
1531	 * If all children are healthy we update asize if either:
1532	 * The asize has increased, due to a device expansion caused by dynamic
1533	 * LUN growth or vdev replacement, and automatic expansion is enabled;
1534	 * making the additional space available.
1535	 *
1536	 * The asize has decreased, due to a device shrink usually caused by a
1537	 * vdev replace with a smaller device. This ensures that calculations
1538	 * based of max_asize and asize e.g. esize are always valid. It's safe
1539	 * to do this as we've already validated that asize is greater than
1540	 * vdev_min_asize.
1541	 */
1542	if (vd->vdev_state == VDEV_STATE_HEALTHY &&
1543	    ((asize > vd->vdev_asize &&
1544	    (vd->vdev_expanding || spa->spa_autoexpand)) ||
1545	    (asize < vd->vdev_asize)))
1546		vd->vdev_asize = asize;
1547
1548	vdev_set_min_asize(vd);
1549
1550	/*
1551	 * Ensure we can issue some IO before declaring the
1552	 * vdev open for business.
1553	 */
1554	if (vd->vdev_ops->vdev_op_leaf &&
1555	    (error = zio_wait(vdev_probe(vd, NULL))) != 0) {
1556		vdev_set_state(vd, B_TRUE, VDEV_STATE_FAULTED,
1557		    VDEV_AUX_ERR_EXCEEDED);
1558		return (error);
1559	}
1560
1561	/*
1562	 * Track the min and max ashift values for normal data devices.
1563	 */
1564	if (vd->vdev_top == vd && vd->vdev_ashift != 0 &&
1565	    !vd->vdev_islog && vd->vdev_aux == NULL) {
1566		if (vd->vdev_ashift > spa->spa_max_ashift)
1567			spa->spa_max_ashift = vd->vdev_ashift;
1568		if (vd->vdev_ashift < spa->spa_min_ashift)
1569			spa->spa_min_ashift = vd->vdev_ashift;
1570	}
1571
1572	/*
1573	 * If a leaf vdev has a DTL, and seems healthy, then kick off a
1574	 * resilver.  But don't do this if we are doing a reopen for a scrub,
1575	 * since this would just restart the scrub we are already doing.
1576	 */
1577	if (vd->vdev_ops->vdev_op_leaf && !spa->spa_scrub_reopen &&
1578	    vdev_resilver_needed(vd, NULL, NULL))
1579		spa_async_request(spa, SPA_ASYNC_RESILVER);
1580
1581	return (0);
1582}
1583
1584/*
1585 * Called once the vdevs are all opened, this routine validates the label
1586 * contents. This needs to be done before vdev_load() so that we don't
1587 * inadvertently do repair I/Os to the wrong device.
1588 *
1589 * This function will only return failure if one of the vdevs indicates that it
1590 * has since been destroyed or exported.  This is only possible if
1591 * /etc/zfs/zpool.cache was readonly at the time.  Otherwise, the vdev state
1592 * will be updated but the function will return 0.
1593 */
1594int
1595vdev_validate(vdev_t *vd)
1596{
1597	spa_t *spa = vd->vdev_spa;
1598	nvlist_t *label;
1599	uint64_t guid = 0, aux_guid = 0, top_guid;
1600	uint64_t state;
1601	nvlist_t *nvl;
1602	uint64_t txg;
1603
1604	if (vdev_validate_skip)
1605		return (0);
1606
1607	for (uint64_t c = 0; c < vd->vdev_children; c++)
1608		if (vdev_validate(vd->vdev_child[c]) != 0)
1609			return (SET_ERROR(EBADF));
1610
1611	/*
1612	 * If the device has already failed, or was marked offline, don't do
1613	 * any further validation.  Otherwise, label I/O will fail and we will
1614	 * overwrite the previous state.
1615	 */
1616	if (!vd->vdev_ops->vdev_op_leaf || !vdev_readable(vd))
1617		return (0);
1618
1619	/*
1620	 * If we are performing an extreme rewind, we allow for a label that
1621	 * was modified at a point after the current txg.
1622	 * If config lock is not held do not check for the txg. spa_sync could
1623	 * be updating the vdev's label before updating spa_last_synced_txg.
1624	 */
1625	if (spa->spa_extreme_rewind || spa_last_synced_txg(spa) == 0 ||
1626	    spa_config_held(spa, SCL_CONFIG, RW_WRITER) != SCL_CONFIG)
1627		txg = UINT64_MAX;
1628	else
1629		txg = spa_last_synced_txg(spa);
1630
1631	if ((label = vdev_label_read_config(vd, txg)) == NULL) {
1632		vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
1633		    VDEV_AUX_BAD_LABEL);
1634		vdev_dbgmsg(vd, "vdev_validate: failed reading config for "
1635		    "txg %llu", (u_longlong_t)txg);
1636		return (0);
1637	}
1638
1639	/*
1640	 * Determine if this vdev has been split off into another
1641	 * pool.  If so, then refuse to open it.
1642	 */
1643	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_SPLIT_GUID,
1644	    &aux_guid) == 0 && aux_guid == spa_guid(spa)) {
1645		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
1646		    VDEV_AUX_SPLIT_POOL);
1647		nvlist_free(label);
1648		vdev_dbgmsg(vd, "vdev_validate: vdev split into other pool");
1649		return (0);
1650	}
1651
1652	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_GUID, &guid) != 0) {
1653		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
1654		    VDEV_AUX_CORRUPT_DATA);
1655		nvlist_free(label);
1656		vdev_dbgmsg(vd, "vdev_validate: '%s' missing from label",
1657		    ZPOOL_CONFIG_POOL_GUID);
1658		return (0);
1659	}
1660
1661	/*
1662	 * If config is not trusted then ignore the spa guid check. This is
1663	 * necessary because if the machine crashed during a re-guid the new
1664	 * guid might have been written to all of the vdev labels, but not the
1665	 * cached config. The check will be performed again once we have the
1666	 * trusted config from the MOS.
1667	 */
1668	if (spa->spa_trust_config && guid != spa_guid(spa)) {
1669		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
1670		    VDEV_AUX_CORRUPT_DATA);
1671		nvlist_free(label);
1672		vdev_dbgmsg(vd, "vdev_validate: vdev label pool_guid doesn't "
1673		    "match config (%llu != %llu)", (u_longlong_t)guid,
1674		    (u_longlong_t)spa_guid(spa));
1675		return (0);
1676	}
1677
1678	if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_VDEV_TREE, &nvl)
1679	    != 0 || nvlist_lookup_uint64(nvl, ZPOOL_CONFIG_ORIG_GUID,
1680	    &aux_guid) != 0)
1681		aux_guid = 0;
1682
1683	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID, &guid) != 0) {
1684		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
1685		    VDEV_AUX_CORRUPT_DATA);
1686		nvlist_free(label);
1687		vdev_dbgmsg(vd, "vdev_validate: '%s' missing from label",
1688		    ZPOOL_CONFIG_GUID);
1689		return (0);
1690	}
1691
1692	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_TOP_GUID, &top_guid)
1693	    != 0) {
1694		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
1695		    VDEV_AUX_CORRUPT_DATA);
1696		nvlist_free(label);
1697		vdev_dbgmsg(vd, "vdev_validate: '%s' missing from label",
1698		    ZPOOL_CONFIG_TOP_GUID);
1699		return (0);
1700	}
1701
1702	/*
1703	 * If this vdev just became a top-level vdev because its sibling was
1704	 * detached, it will have adopted the parent's vdev guid -- but the
1705	 * label may or may not be on disk yet. Fortunately, either version
1706	 * of the label will have the same top guid, so if we're a top-level
1707	 * vdev, we can safely compare to that instead.
1708	 * However, if the config comes from a cachefile that failed to update
1709	 * after the detach, a top-level vdev will appear as a non top-level
1710	 * vdev in the config. Also relax the constraints if we perform an
1711	 * extreme rewind.
1712	 *
1713	 * If we split this vdev off instead, then we also check the
1714	 * original pool's guid. We don't want to consider the vdev
1715	 * corrupt if it is partway through a split operation.
1716	 */
1717	if (vd->vdev_guid != guid && vd->vdev_guid != aux_guid) {
1718		boolean_t mismatch = B_FALSE;
1719		if (spa->spa_trust_config && !spa->spa_extreme_rewind) {
1720			if (vd != vd->vdev_top || vd->vdev_guid != top_guid)
1721				mismatch = B_TRUE;
1722		} else {
1723			if (vd->vdev_guid != top_guid &&
1724			    vd->vdev_top->vdev_guid != guid)
1725				mismatch = B_TRUE;
1726		}
1727
1728		if (mismatch) {
1729			vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
1730			    VDEV_AUX_CORRUPT_DATA);
1731			nvlist_free(label);
1732			vdev_dbgmsg(vd, "vdev_validate: config guid "
1733			    "doesn't match label guid");
1734			vdev_dbgmsg(vd, "CONFIG: guid %llu, top_guid %llu",
1735			    (u_longlong_t)vd->vdev_guid,
1736			    (u_longlong_t)vd->vdev_top->vdev_guid);
1737			vdev_dbgmsg(vd, "LABEL: guid %llu, top_guid %llu, "
1738			    "aux_guid %llu", (u_longlong_t)guid,
1739			    (u_longlong_t)top_guid, (u_longlong_t)aux_guid);
1740			return (0);
1741		}
1742	}
1743
1744	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_STATE,
1745	    &state) != 0) {
1746		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
1747		    VDEV_AUX_CORRUPT_DATA);
1748		nvlist_free(label);
1749		vdev_dbgmsg(vd, "vdev_validate: '%s' missing from label",
1750		    ZPOOL_CONFIG_POOL_STATE);
1751		return (0);
1752	}
1753
1754	nvlist_free(label);
1755
1756	/*
1757	 * If this is a verbatim import, no need to check the
1758	 * state of the pool.
1759	 */
1760	if (!(spa->spa_import_flags & ZFS_IMPORT_VERBATIM) &&
1761	    spa_load_state(spa) == SPA_LOAD_OPEN &&
1762	    state != POOL_STATE_ACTIVE) {
1763		vdev_dbgmsg(vd, "vdev_validate: invalid pool state (%llu) "
1764		    "for spa %s", (u_longlong_t)state, spa->spa_name);
1765		return (SET_ERROR(EBADF));
1766	}
1767
1768	/*
1769	 * If we were able to open and validate a vdev that was
1770	 * previously marked permanently unavailable, clear that state
1771	 * now.
1772	 */
1773	if (vd->vdev_not_present)
1774		vd->vdev_not_present = 0;
1775
1776	return (0);
1777}
1778
1779static void
1780vdev_copy_path_impl(vdev_t *svd, vdev_t *dvd)
1781{
1782	if (svd->vdev_path != NULL && dvd->vdev_path != NULL) {
1783		if (strcmp(svd->vdev_path, dvd->vdev_path) != 0) {
1784			zfs_dbgmsg("vdev_copy_path: vdev %llu: path changed "
1785			    "from '%s' to '%s'", (u_longlong_t)dvd->vdev_guid,
1786			    dvd->vdev_path, svd->vdev_path);
1787			spa_strfree(dvd->vdev_path);
1788			dvd->vdev_path = spa_strdup(svd->vdev_path);
1789		}
1790	} else if (svd->vdev_path != NULL) {
1791		dvd->vdev_path = spa_strdup(svd->vdev_path);
1792		zfs_dbgmsg("vdev_copy_path: vdev %llu: path set to '%s'",
1793		    (u_longlong_t)dvd->vdev_guid, dvd->vdev_path);
1794	}
1795}
1796
1797/*
1798 * Recursively copy vdev paths from one vdev to another. Source and destination
1799 * vdev trees must have same geometry otherwise return error. Intended to copy
1800 * paths from userland config into MOS config.
1801 */
1802int
1803vdev_copy_path_strict(vdev_t *svd, vdev_t *dvd)
1804{
1805	if ((svd->vdev_ops == &vdev_missing_ops) ||
1806	    (svd->vdev_ishole && dvd->vdev_ishole) ||
1807	    (dvd->vdev_ops == &vdev_indirect_ops))
1808		return (0);
1809
1810	if (svd->vdev_ops != dvd->vdev_ops) {
1811		vdev_dbgmsg(svd, "vdev_copy_path: vdev type mismatch: %s != %s",
1812		    svd->vdev_ops->vdev_op_type, dvd->vdev_ops->vdev_op_type);
1813		return (SET_ERROR(EINVAL));
1814	}
1815
1816	if (svd->vdev_guid != dvd->vdev_guid) {
1817		vdev_dbgmsg(svd, "vdev_copy_path: guids mismatch (%llu != "
1818		    "%llu)", (u_longlong_t)svd->vdev_guid,
1819		    (u_longlong_t)dvd->vdev_guid);
1820		return (SET_ERROR(EINVAL));
1821	}
1822
1823	if (svd->vdev_children != dvd->vdev_children) {
1824		vdev_dbgmsg(svd, "vdev_copy_path: children count mismatch: "
1825		    "%llu != %llu", (u_longlong_t)svd->vdev_children,
1826		    (u_longlong_t)dvd->vdev_children);
1827		return (SET_ERROR(EINVAL));
1828	}
1829
1830	for (uint64_t i = 0; i < svd->vdev_children; i++) {
1831		int error = vdev_copy_path_strict(svd->vdev_child[i],
1832		    dvd->vdev_child[i]);
1833		if (error != 0)
1834			return (error);
1835	}
1836
1837	if (svd->vdev_ops->vdev_op_leaf)
1838		vdev_copy_path_impl(svd, dvd);
1839
1840	return (0);
1841}
1842
1843static void
1844vdev_copy_path_search(vdev_t *stvd, vdev_t *dvd)
1845{
1846	ASSERT(stvd->vdev_top == stvd);
1847	ASSERT3U(stvd->vdev_id, ==, dvd->vdev_top->vdev_id);
1848
1849	for (uint64_t i = 0; i < dvd->vdev_children; i++) {
1850		vdev_copy_path_search(stvd, dvd->vdev_child[i]);
1851	}
1852
1853	if (!dvd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(dvd))
1854		return;
1855
1856	/*
1857	 * The idea here is that while a vdev can shift positions within
1858	 * a top vdev (when replacing, attaching mirror, etc.) it cannot
1859	 * step outside of it.
1860	 */
1861	vdev_t *vd = vdev_lookup_by_guid(stvd, dvd->vdev_guid);
1862
1863	if (vd == NULL || vd->vdev_ops != dvd->vdev_ops)
1864		return;
1865
1866	ASSERT(vd->vdev_ops->vdev_op_leaf);
1867
1868	vdev_copy_path_impl(vd, dvd);
1869}
1870
1871/*
1872 * Recursively copy vdev paths from one root vdev to another. Source and
1873 * destination vdev trees may differ in geometry. For each destination leaf
1874 * vdev, search a vdev with the same guid and top vdev id in the source.
1875 * Intended to copy paths from userland config into MOS config.
1876 */
1877void
1878vdev_copy_path_relaxed(vdev_t *srvd, vdev_t *drvd)
1879{
1880	uint64_t children = MIN(srvd->vdev_children, drvd->vdev_children);
1881	ASSERT(srvd->vdev_ops == &vdev_root_ops);
1882	ASSERT(drvd->vdev_ops == &vdev_root_ops);
1883
1884	for (uint64_t i = 0; i < children; i++) {
1885		vdev_copy_path_search(srvd->vdev_child[i],
1886		    drvd->vdev_child[i]);
1887	}
1888}
1889
1890/*
1891 * Close a virtual device.
1892 */
1893void
1894vdev_close(vdev_t *vd)
1895{
1896	spa_t *spa = vd->vdev_spa;
1897	vdev_t *pvd = vd->vdev_parent;
1898
1899	ASSERT(spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);
1900
1901	/*
1902	 * If our parent is reopening, then we are as well, unless we are
1903	 * going offline.
1904	 */
1905	if (pvd != NULL && pvd->vdev_reopening)
1906		vd->vdev_reopening = (pvd->vdev_reopening && !vd->vdev_offline);
1907
1908	vd->vdev_ops->vdev_op_close(vd);
1909
1910	vdev_cache_purge(vd);
1911
1912	/*
1913	 * We record the previous state before we close it, so that if we are
1914	 * doing a reopen(), we don't generate FMA ereports if we notice that
1915	 * it's still faulted.
1916	 */
1917	vd->vdev_prevstate = vd->vdev_state;
1918
1919	if (vd->vdev_offline)
1920		vd->vdev_state = VDEV_STATE_OFFLINE;
1921	else
1922		vd->vdev_state = VDEV_STATE_CLOSED;
1923	vd->vdev_stat.vs_aux = VDEV_AUX_NONE;
1924}
1925
1926void
1927vdev_hold(vdev_t *vd)
1928{
1929	spa_t *spa = vd->vdev_spa;
1930
1931	ASSERT(spa_is_root(spa));
1932	if (spa->spa_state == POOL_STATE_UNINITIALIZED)
1933		return;
1934
1935	for (int c = 0; c < vd->vdev_children; c++)
1936		vdev_hold(vd->vdev_child[c]);
1937
1938	if (vd->vdev_ops->vdev_op_leaf)
1939		vd->vdev_ops->vdev_op_hold(vd);
1940}
1941
1942void
1943vdev_rele(vdev_t *vd)
1944{
1945	spa_t *spa = vd->vdev_spa;
1946
1947	ASSERT(spa_is_root(spa));
1948	for (int c = 0; c < vd->vdev_children; c++)
1949		vdev_rele(vd->vdev_child[c]);
1950
1951	if (vd->vdev_ops->vdev_op_leaf)
1952		vd->vdev_ops->vdev_op_rele(vd);
1953}
1954
1955/*
1956 * Reopen all interior vdevs and any unopened leaves.  We don't actually
1957 * reopen leaf vdevs which had previously been opened as they might deadlock
1958 * on the spa_config_lock.  Instead we only obtain the leaf's physical size.
1959 * If the leaf has never been opened then open it, as usual.
1960 */
1961void
1962vdev_reopen(vdev_t *vd)
1963{
1964	spa_t *spa = vd->vdev_spa;
1965
1966	ASSERT(spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);
1967
1968	/* set the reopening flag unless we're taking the vdev offline */
1969	vd->vdev_reopening = !vd->vdev_offline;
1970	vdev_close(vd);
1971	(void) vdev_open(vd);
1972
1973	/*
1974	 * Call vdev_validate() here to make sure we have the same device.
1975	 * Otherwise, a device with an invalid label could be successfully
1976	 * opened in response to vdev_reopen().
1977	 */
1978	if (vd->vdev_aux) {
1979		(void) vdev_validate_aux(vd);
1980		if (vdev_readable(vd) && vdev_writeable(vd) &&
1981		    vd->vdev_aux == &spa->spa_l2cache &&
1982		    !l2arc_vdev_present(vd))
1983			l2arc_add_vdev(spa, vd);
1984	} else {
1985		(void) vdev_validate(vd);
1986	}
1987
1988	/*
1989	 * Reassess parent vdev's health.
1990	 */
1991	vdev_propagate_state(vd);
1992}
1993
1994int
1995vdev_create(vdev_t *vd, uint64_t txg, boolean_t isreplacing)
1996{
1997	int error;
1998
1999	/*
2000	 * Normally, partial opens (e.g. of a mirror) are allowed.
2001	 * For a create, however, we want to fail the request if
2002	 * there are any components we can't open.
2003	 */
2004	error = vdev_open(vd);
2005
2006	if (error || vd->vdev_state != VDEV_STATE_HEALTHY) {
2007		vdev_close(vd);
2008		return (error ? error : ENXIO);
2009	}
2010
2011	/*
2012	 * Recursively load DTLs and initialize all labels.
2013	 */
2014	if ((error = vdev_dtl_load(vd)) != 0 ||
2015	    (error = vdev_label_init(vd, txg, isreplacing ?
2016	    VDEV_LABEL_REPLACE : VDEV_LABEL_CREATE)) != 0) {
2017		vdev_close(vd);
2018		return (error);
2019	}
2020
2021	return (0);
2022}
2023
2024void
2025vdev_metaslab_set_size(vdev_t *vd)
2026{
2027	uint64_t asize = vd->vdev_asize;
2028	uint64_t ms_shift = 0;
2029
2030	/*
2031	 * For vdevs that are bigger than 8G the metaslab size varies in
2032	 * a way that the number of metaslabs increases in powers of two,
2033	 * linearly in terms of vdev_asize, starting from 16 metaslabs.
2034	 * So for vdev_asize of 8G we get 16 metaslabs, for 16G, we get 32,
2035	 * and so on, until we hit the maximum metaslab count limit
2036	 * [vdev_max_ms_count] from which point the metaslab count stays
2037	 * the same.
2038	 */
2039	ms_shift = vdev_default_ms_shift;
2040
2041	if ((asize >> ms_shift) < vdev_min_ms_count) {
2042		/*
2043		 * For devices that are less than 8G we want to have
2044		 * exactly 16 metaslabs. We don't want less as integer
2045		 * division rounds down, so less metaslabs mean more
2046		 * wasted space. We don't want more as these vdevs are
2047		 * small and in the likely event that we are running
2048		 * out of space, the SPA will have a hard time finding
2049		 * space due to fragmentation.
2050		 */
2051		ms_shift = highbit64(asize / vdev_min_ms_count);
2052		ms_shift = MAX(ms_shift, SPA_MAXBLOCKSHIFT);
2053
2054	} else if ((asize >> ms_shift) > vdev_max_ms_count) {
2055		ms_shift = highbit64(asize / vdev_max_ms_count);
2056	}
2057
2058	vd->vdev_ms_shift = ms_shift;
2059	ASSERT3U(vd->vdev_ms_shift, >=, SPA_MAXBLOCKSHIFT);
2060}
2061
2062void
2063vdev_dirty(vdev_t *vd, int flags, void *arg, uint64_t txg)
2064{
2065	ASSERT(vd == vd->vdev_top);
2066	/* indirect vdevs don't have metaslabs or dtls */
2067	ASSERT(vdev_is_concrete(vd) || flags == 0);
2068	ASSERT(ISP2(flags));
2069	ASSERT(spa_writeable(vd->vdev_spa));
2070
2071	if (flags & VDD_METASLAB)
2072		(void) txg_list_add(&vd->vdev_ms_list, arg, txg);
2073
2074	if (flags & VDD_DTL)
2075		(void) txg_list_add(&vd->vdev_dtl_list, arg, txg);
2076
2077	(void) txg_list_add(&vd->vdev_spa->spa_vdev_txg_list, vd, txg);
2078}
2079
2080void
2081vdev_dirty_leaves(vdev_t *vd, int flags, uint64_t txg)
2082{
2083	for (int c = 0; c < vd->vdev_children; c++)
2084		vdev_dirty_leaves(vd->vdev_child[c], flags, txg);
2085
2086	if (vd->vdev_ops->vdev_op_leaf)
2087		vdev_dirty(vd->vdev_top, flags, vd, txg);
2088}
2089
2090/*
2091 * DTLs.
2092 *
2093 * A vdev's DTL (dirty time log) is the set of transaction groups for which
2094 * the vdev has less than perfect replication.  There are four kinds of DTL:
2095 *
2096 * DTL_MISSING: txgs for which the vdev has no valid copies of the data
2097 *
2098 * DTL_PARTIAL: txgs for which data is available, but not fully replicated
2099 *
2100 * DTL_SCRUB: the txgs that could not be repaired by the last scrub; upon
2101 *	scrub completion, DTL_SCRUB replaces DTL_MISSING in the range of
2102 *	txgs that was scrubbed.
2103 *
2104 * DTL_OUTAGE: txgs which cannot currently be read, whether due to
2105 *	persistent errors or just some device being offline.
2106 *	Unlike the other three, the DTL_OUTAGE map is not generally
2107 *	maintained; it's only computed when needed, typically to
2108 *	determine whether a device can be detached.
2109 *
2110 * For leaf vdevs, DTL_MISSING and DTL_PARTIAL are identical: the device
2111 * either has the data or it doesn't.
2112 *
2113 * For interior vdevs such as mirror and RAID-Z the picture is more complex.
2114 * A vdev's DTL_PARTIAL is the union of its children's DTL_PARTIALs, because
2115 * if any child is less than fully replicated, then so is its parent.
2116 * A vdev's DTL_MISSING is a modified union of its children's DTL_MISSINGs,
2117 * comprising only those txgs which appear in 'maxfaults' or more children;
2118 * those are the txgs we don't have enough replication to read.  For example,
2119 * double-parity RAID-Z can tolerate up to two missing devices (maxfaults == 2);
2120 * thus, its DTL_MISSING consists of the set of txgs that appear in more than
2121 * two child DTL_MISSING maps.
2122 *
2123 * It should be clear from the above that to compute the DTLs and outage maps
2124 * for all vdevs, it suffices to know just the leaf vdevs' DTL_MISSING maps.
2125 * Therefore, that is all we keep on disk.  When loading the pool, or after
2126 * a configuration change, we generate all other DTLs from first principles.
2127 */
2128void
2129vdev_dtl_dirty(vdev_t *vd, vdev_dtl_type_t t, uint64_t txg, uint64_t size)
2130{
2131	range_tree_t *rt = vd->vdev_dtl[t];
2132
2133	ASSERT(t < DTL_TYPES);
2134	ASSERT(vd != vd->vdev_spa->spa_root_vdev);
2135	ASSERT(spa_writeable(vd->vdev_spa));
2136
2137	mutex_enter(&vd->vdev_dtl_lock);
2138	if (!range_tree_contains(rt, txg, size))
2139		range_tree_add(rt, txg, size);
2140	mutex_exit(&vd->vdev_dtl_lock);
2141}
2142
2143boolean_t
2144vdev_dtl_contains(vdev_t *vd, vdev_dtl_type_t t, uint64_t txg, uint64_t size)
2145{
2146	range_tree_t *rt = vd->vdev_dtl[t];
2147	boolean_t dirty = B_FALSE;
2148
2149	ASSERT(t < DTL_TYPES);
2150	ASSERT(vd != vd->vdev_spa->spa_root_vdev);
2151
2152	/*
2153	 * While we are loading the pool, the DTLs have not been loaded yet.
2154	 * Ignore the DTLs and try all devices.  This avoids a recursive
2155	 * mutex enter on the vdev_dtl_lock, and also makes us try hard
2156	 * when loading the pool (relying on the checksum to ensure that
2157	 * we get the right data -- note that we while loading, we are
2158	 * only reading the MOS, which is always checksummed).
2159	 */
2160	if (vd->vdev_spa->spa_load_state != SPA_LOAD_NONE)
2161		return (B_FALSE);
2162
2163	mutex_enter(&vd->vdev_dtl_lock);
2164	if (!range_tree_is_empty(rt))
2165		dirty = range_tree_contains(rt, txg, size);
2166	mutex_exit(&vd->vdev_dtl_lock);
2167
2168	return (dirty);
2169}
2170
2171boolean_t
2172vdev_dtl_empty(vdev_t *vd, vdev_dtl_type_t t)
2173{
2174	range_tree_t *rt = vd->vdev_dtl[t];
2175	boolean_t empty;
2176
2177	mutex_enter(&vd->vdev_dtl_lock);
2178	empty = range_tree_is_empty(rt);
2179	mutex_exit(&vd->vdev_dtl_lock);
2180
2181	return (empty);
2182}
2183
2184/*
2185 * Returns the lowest txg in the DTL range.
2186 */
2187static uint64_t
2188vdev_dtl_min(vdev_t *vd)
2189{
2190	range_seg_t *rs;
2191
2192	ASSERT(MUTEX_HELD(&vd->vdev_dtl_lock));
2193	ASSERT3U(range_tree_space(vd->vdev_dtl[DTL_MISSING]), !=, 0);
2194	ASSERT0(vd->vdev_children);
2195
2196	rs = avl_first(&vd->vdev_dtl[DTL_MISSING]->rt_root);
2197	return (rs->rs_start - 1);
2198}
2199
2200/*
2201 * Returns the highest txg in the DTL.
2202 */
2203static uint64_t
2204vdev_dtl_max(vdev_t *vd)
2205{
2206	range_seg_t *rs;
2207
2208	ASSERT(MUTEX_HELD(&vd->vdev_dtl_lock));
2209	ASSERT3U(range_tree_space(vd->vdev_dtl[DTL_MISSING]), !=, 0);
2210	ASSERT0(vd->vdev_children);
2211
2212	rs = avl_last(&vd->vdev_dtl[DTL_MISSING]->rt_root);
2213	return (rs->rs_end);
2214}
2215
2216/*
2217 * Determine if a resilvering vdev should remove any DTL entries from
2218 * its range. If the vdev was resilvering for the entire duration of the
2219 * scan then it should excise that range from its DTLs. Otherwise, this
2220 * vdev is considered partially resilvered and should leave its DTL
2221 * entries intact. The comment in vdev_dtl_reassess() describes how we
2222 * excise the DTLs.
2223 */
2224static boolean_t
2225vdev_dtl_should_excise(vdev_t *vd)
2226{
2227	spa_t *spa = vd->vdev_spa;
2228	dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan;
2229
2230	ASSERT0(scn->scn_phys.scn_errors);
2231	ASSERT0(vd->vdev_children);
2232
2233	if (vd->vdev_state < VDEV_STATE_DEGRADED)
2234		return (B_FALSE);
2235
2236	if (vd->vdev_resilver_txg == 0 ||
2237	    range_tree_is_empty(vd->vdev_dtl[DTL_MISSING]))
2238		return (B_TRUE);
2239
2240	/*
2241	 * When a resilver is initiated the scan will assign the scn_max_txg
2242	 * value to the highest txg value that exists in all DTLs. If this
2243	 * device's max DTL is not part of this scan (i.e. it is not in
2244	 * the range (scn_min_txg, scn_max_txg] then it is not eligible
2245	 * for excision.
2246	 */
2247	if (vdev_dtl_max(vd) <= scn->scn_phys.scn_max_txg) {
2248		ASSERT3U(scn->scn_phys.scn_min_txg, <=, vdev_dtl_min(vd));
2249		ASSERT3U(scn->scn_phys.scn_min_txg, <, vd->vdev_resilver_txg);
2250		ASSERT3U(vd->vdev_resilver_txg, <=, scn->scn_phys.scn_max_txg);
2251		return (B_TRUE);
2252	}
2253	return (B_FALSE);
2254}
2255
2256/*
2257 * Reassess DTLs after a config change or scrub completion.
2258 */
2259void
2260vdev_dtl_reassess(vdev_t *vd, uint64_t txg, uint64_t scrub_txg, int scrub_done)
2261{
2262	spa_t *spa = vd->vdev_spa;
2263	avl_tree_t reftree;
2264	int minref;
2265
2266	ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0);
2267
2268	for (int c = 0; c < vd->vdev_children; c++)
2269		vdev_dtl_reassess(vd->vdev_child[c], txg,
2270		    scrub_txg, scrub_done);
2271
2272	if (vd == spa->spa_root_vdev || !vdev_is_concrete(vd) || vd->vdev_aux)
2273		return;
2274
2275	if (vd->vdev_ops->vdev_op_leaf) {
2276		dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan;
2277
2278		mutex_enter(&vd->vdev_dtl_lock);
2279
2280		/*
2281		 * If we've completed a scan cleanly then determine
2282		 * if this vdev should remove any DTLs. We only want to
2283		 * excise regions on vdevs that were available during
2284		 * the entire duration of this scan.
2285		 */
2286		if (scrub_txg != 0 &&
2287		    (spa->spa_scrub_started ||
2288		    (scn != NULL && scn->scn_phys.scn_errors == 0)) &&
2289		    vdev_dtl_should_excise(vd)) {
2290			/*
2291			 * We completed a scrub up to scrub_txg.  If we
2292			 * did it without rebooting, then the scrub dtl
2293			 * will be valid, so excise the old region and
2294			 * fold in the scrub dtl.  Otherwise, leave the
2295			 * dtl as-is if there was an error.
2296			 *
2297			 * There's little trick here: to excise the beginning
2298			 * of the DTL_MISSING map, we put it into a reference
2299			 * tree and then add a segment with refcnt -1 that
2300			 * covers the range [0, scrub_txg).  This means
2301			 * that each txg in that range has refcnt -1 or 0.
2302			 * We then add DTL_SCRUB with a refcnt of 2, so that
2303			 * entries in the range [0, scrub_txg) will have a
2304			 * positive refcnt -- either 1 or 2.  We then convert
2305			 * the reference tree into the new DTL_MISSING map.
2306			 */
2307			space_reftree_create(&reftree);
2308			space_reftree_add_map(&reftree,
2309			    vd->vdev_dtl[DTL_MISSING], 1);
2310			space_reftree_add_seg(&reftree, 0, scrub_txg, -1);
2311			space_reftree_add_map(&reftree,
2312			    vd->vdev_dtl[DTL_SCRUB], 2);
2313			space_reftree_generate_map(&reftree,
2314			    vd->vdev_dtl[DTL_MISSING], 1);
2315			space_reftree_destroy(&reftree);
2316		}
2317		range_tree_vacate(vd->vdev_dtl[DTL_PARTIAL], NULL, NULL);
2318		range_tree_walk(vd->vdev_dtl[DTL_MISSING],
2319		    range_tree_add, vd->vdev_dtl[DTL_PARTIAL]);
2320		if (scrub_done)
2321			range_tree_vacate(vd->vdev_dtl[DTL_SCRUB], NULL, NULL);
2322		range_tree_vacate(vd->vdev_dtl[DTL_OUTAGE], NULL, NULL);
2323		if (!vdev_readable(vd))
2324			range_tree_add(vd->vdev_dtl[DTL_OUTAGE], 0, -1ULL);
2325		else
2326			range_tree_walk(vd->vdev_dtl[DTL_MISSING],
2327			    range_tree_add, vd->vdev_dtl[DTL_OUTAGE]);
2328
2329		/*
2330		 * If the vdev was resilvering and no longer has any
2331		 * DTLs then reset its resilvering flag.
2332		 */
2333		if (vd->vdev_resilver_txg != 0 &&
2334		    range_tree_is_empty(vd->vdev_dtl[DTL_MISSING]) &&
2335		    range_tree_is_empty(vd->vdev_dtl[DTL_OUTAGE]))
2336			vd->vdev_resilver_txg = 0;
2337
2338		mutex_exit(&vd->vdev_dtl_lock);
2339
2340		if (txg != 0)
2341			vdev_dirty(vd->vdev_top, VDD_DTL, vd, txg);
2342		return;
2343	}
2344
2345	mutex_enter(&vd->vdev_dtl_lock);
2346	for (int t = 0; t < DTL_TYPES; t++) {
2347		/* account for child's outage in parent's missing map */
2348		int s = (t == DTL_MISSING) ? DTL_OUTAGE: t;
2349		if (t == DTL_SCRUB)
2350			continue;			/* leaf vdevs only */
2351		if (t == DTL_PARTIAL)
2352			minref = 1;			/* i.e. non-zero */
2353		else if (vd->vdev_nparity != 0)
2354			minref = vd->vdev_nparity + 1;	/* RAID-Z */
2355		else
2356			minref = vd->vdev_children;	/* any kind of mirror */
2357		space_reftree_create(&reftree);
2358		for (int c = 0; c < vd->vdev_children; c++) {
2359			vdev_t *cvd = vd->vdev_child[c];
2360			mutex_enter(&cvd->vdev_dtl_lock);
2361			space_reftree_add_map(&reftree, cvd->vdev_dtl[s], 1);
2362			mutex_exit(&cvd->vdev_dtl_lock);
2363		}
2364		space_reftree_generate_map(&reftree, vd->vdev_dtl[t], minref);
2365		space_reftree_destroy(&reftree);
2366	}
2367	mutex_exit(&vd->vdev_dtl_lock);
2368}
2369
2370int
2371vdev_dtl_load(vdev_t *vd)
2372{
2373	spa_t *spa = vd->vdev_spa;
2374	objset_t *mos = spa->spa_meta_objset;
2375	int error = 0;
2376
2377	if (vd->vdev_ops->vdev_op_leaf && vd->vdev_dtl_object != 0) {
2378		ASSERT(vdev_is_concrete(vd));
2379
2380		error = space_map_open(&vd->vdev_dtl_sm, mos,
2381		    vd->vdev_dtl_object, 0, -1ULL, 0);
2382		if (error)
2383			return (error);
2384		ASSERT(vd->vdev_dtl_sm != NULL);
2385
2386		mutex_enter(&vd->vdev_dtl_lock);
2387
2388		/*
2389		 * Now that we've opened the space_map we need to update
2390		 * the in-core DTL.
2391		 */
2392		space_map_update(vd->vdev_dtl_sm);
2393
2394		error = space_map_load(vd->vdev_dtl_sm,
2395		    vd->vdev_dtl[DTL_MISSING], SM_ALLOC);
2396		mutex_exit(&vd->vdev_dtl_lock);
2397
2398		return (error);
2399	}
2400
2401	for (int c = 0; c < vd->vdev_children; c++) {
2402		error = vdev_dtl_load(vd->vdev_child[c]);
2403		if (error != 0)
2404			break;
2405	}
2406
2407	return (error);
2408}
2409
2410void
2411vdev_destroy_unlink_zap(vdev_t *vd, uint64_t zapobj, dmu_tx_t *tx)
2412{
2413	spa_t *spa = vd->vdev_spa;
2414
2415	VERIFY0(zap_destroy(spa->spa_meta_objset, zapobj, tx));
2416	VERIFY0(zap_remove_int(spa->spa_meta_objset, spa->spa_all_vdev_zaps,
2417	    zapobj, tx));
2418}
2419
2420uint64_t
2421vdev_create_link_zap(vdev_t *vd, dmu_tx_t *tx)
2422{
2423	spa_t *spa = vd->vdev_spa;
2424	uint64_t zap = zap_create(spa->spa_meta_objset, DMU_OTN_ZAP_METADATA,
2425	    DMU_OT_NONE, 0, tx);
2426
2427	ASSERT(zap != 0);
2428	VERIFY0(zap_add_int(spa->spa_meta_objset, spa->spa_all_vdev_zaps,
2429	    zap, tx));
2430
2431	return (zap);
2432}
2433
2434void
2435vdev_construct_zaps(vdev_t *vd, dmu_tx_t *tx)
2436{
2437	if (vd->vdev_ops != &vdev_hole_ops &&
2438	    vd->vdev_ops != &vdev_missing_ops &&
2439	    vd->vdev_ops != &vdev_root_ops &&
2440	    !vd->vdev_top->vdev_removing) {
2441		if (vd->vdev_ops->vdev_op_leaf && vd->vdev_leaf_zap == 0) {
2442			vd->vdev_leaf_zap = vdev_create_link_zap(vd, tx);
2443		}
2444		if (vd == vd->vdev_top && vd->vdev_top_zap == 0) {
2445			vd->vdev_top_zap = vdev_create_link_zap(vd, tx);
2446		}
2447	}
2448	for (uint64_t i = 0; i < vd->vdev_children; i++) {
2449		vdev_construct_zaps(vd->vdev_child[i], tx);
2450	}
2451}
2452
2453void
2454vdev_dtl_sync(vdev_t *vd, uint64_t txg)
2455{
2456	spa_t *spa = vd->vdev_spa;
2457	range_tree_t *rt = vd->vdev_dtl[DTL_MISSING];
2458	objset_t *mos = spa->spa_meta_objset;
2459	range_tree_t *rtsync;
2460	dmu_tx_t *tx;
2461	uint64_t object = space_map_object(vd->vdev_dtl_sm);
2462
2463	ASSERT(vdev_is_concrete(vd));
2464	ASSERT(vd->vdev_ops->vdev_op_leaf);
2465
2466	tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
2467
2468	if (vd->vdev_detached || vd->vdev_top->vdev_removing) {
2469		mutex_enter(&vd->vdev_dtl_lock);
2470		space_map_free(vd->vdev_dtl_sm, tx);
2471		space_map_close(vd->vdev_dtl_sm);
2472		vd->vdev_dtl_sm = NULL;
2473		mutex_exit(&vd->vdev_dtl_lock);
2474
2475		/*
2476		 * We only destroy the leaf ZAP for detached leaves or for
2477		 * removed log devices. Removed data devices handle leaf ZAP
2478		 * cleanup later, once cancellation is no longer possible.
2479		 */
2480		if (vd->vdev_leaf_zap != 0 && (vd->vdev_detached ||
2481		    vd->vdev_top->vdev_islog)) {
2482			vdev_destroy_unlink_zap(vd, vd->vdev_leaf_zap, tx);
2483			vd->vdev_leaf_zap = 0;
2484		}
2485
2486		dmu_tx_commit(tx);
2487		return;
2488	}
2489
2490	if (vd->vdev_dtl_sm == NULL) {
2491		uint64_t new_object;
2492
2493		new_object = space_map_alloc(mos, vdev_dtl_sm_blksz, tx);
2494		VERIFY3U(new_object, !=, 0);
2495
2496		VERIFY0(space_map_open(&vd->vdev_dtl_sm, mos, new_object,
2497		    0, -1ULL, 0));
2498		ASSERT(vd->vdev_dtl_sm != NULL);
2499	}
2500
2501	rtsync = range_tree_create(NULL, NULL);
2502
2503	mutex_enter(&vd->vdev_dtl_lock);
2504	range_tree_walk(rt, range_tree_add, rtsync);
2505	mutex_exit(&vd->vdev_dtl_lock);
2506
2507	space_map_truncate(vd->vdev_dtl_sm, vdev_dtl_sm_blksz, tx);
2508	space_map_write(vd->vdev_dtl_sm, rtsync, SM_ALLOC, SM_NO_VDEVID, tx);
2509	range_tree_vacate(rtsync, NULL, NULL);
2510
2511	range_tree_destroy(rtsync);
2512
2513	/*
2514	 * If the object for the space map has changed then dirty
2515	 * the top level so that we update the config.
2516	 */
2517	if (object != space_map_object(vd->vdev_dtl_sm)) {
2518		vdev_dbgmsg(vd, "txg %llu, spa %s, DTL old object %llu, "
2519		    "new object %llu", (u_longlong_t)txg, spa_name(spa),
2520		    (u_longlong_t)object,
2521		    (u_longlong_t)space_map_object(vd->vdev_dtl_sm));
2522		vdev_config_dirty(vd->vdev_top);
2523	}
2524
2525	dmu_tx_commit(tx);
2526
2527	mutex_enter(&vd->vdev_dtl_lock);
2528	space_map_update(vd->vdev_dtl_sm);
2529	mutex_exit(&vd->vdev_dtl_lock);
2530}
2531
2532/*
2533 * Determine whether the specified vdev can be offlined/detached/removed
2534 * without losing data.
2535 */
2536boolean_t
2537vdev_dtl_required(vdev_t *vd)
2538{
2539	spa_t *spa = vd->vdev_spa;
2540	vdev_t *tvd = vd->vdev_top;
2541	uint8_t cant_read = vd->vdev_cant_read;
2542	boolean_t required;
2543
2544	ASSERT(spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);
2545
2546	if (vd == spa->spa_root_vdev || vd == tvd)
2547		return (B_TRUE);
2548
2549	/*
2550	 * Temporarily mark the device as unreadable, and then determine
2551	 * whether this results in any DTL outages in the top-level vdev.
2552	 * If not, we can safely offline/detach/remove the device.
2553	 */
2554	vd->vdev_cant_read = B_TRUE;
2555	vdev_dtl_reassess(tvd, 0, 0, B_FALSE);
2556	required = !vdev_dtl_empty(tvd, DTL_OUTAGE);
2557	vd->vdev_cant_read = cant_read;
2558	vdev_dtl_reassess(tvd, 0, 0, B_FALSE);
2559
2560	if (!required && zio_injection_enabled)
2561		required = !!zio_handle_device_injection(vd, NULL, ECHILD);
2562
2563	return (required);
2564}
2565
2566/*
2567 * Determine if resilver is needed, and if so the txg range.
2568 */
2569boolean_t
2570vdev_resilver_needed(vdev_t *vd, uint64_t *minp, uint64_t *maxp)
2571{
2572	boolean_t needed = B_FALSE;
2573	uint64_t thismin = UINT64_MAX;
2574	uint64_t thismax = 0;
2575
2576	if (vd->vdev_children == 0) {
2577		mutex_enter(&vd->vdev_dtl_lock);
2578		if (!range_tree_is_empty(vd->vdev_dtl[DTL_MISSING]) &&
2579		    vdev_writeable(vd)) {
2580
2581			thismin = vdev_dtl_min(vd);
2582			thismax = vdev_dtl_max(vd);
2583			needed = B_TRUE;
2584		}
2585		mutex_exit(&vd->vdev_dtl_lock);
2586	} else {
2587		for (int c = 0; c < vd->vdev_children; c++) {
2588			vdev_t *cvd = vd->vdev_child[c];
2589			uint64_t cmin, cmax;
2590
2591			if (vdev_resilver_needed(cvd, &cmin, &cmax)) {
2592				thismin = MIN(thismin, cmin);
2593				thismax = MAX(thismax, cmax);
2594				needed = B_TRUE;
2595			}
2596		}
2597	}
2598
2599	if (needed && minp) {
2600		*minp = thismin;
2601		*maxp = thismax;
2602	}
2603	return (needed);
2604}
2605
2606/*
2607 * Gets the checkpoint space map object from the vdev's ZAP.
2608 * Returns the spacemap object, or 0 if it wasn't in the ZAP
2609 * or the ZAP doesn't exist yet.
2610 */
2611int
2612vdev_checkpoint_sm_object(vdev_t *vd)
2613{
2614	ASSERT0(spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
2615	if (vd->vdev_top_zap == 0) {
2616		return (0);
2617	}
2618
2619	uint64_t sm_obj = 0;
2620	int err = zap_lookup(spa_meta_objset(vd->vdev_spa), vd->vdev_top_zap,
2621	    VDEV_TOP_ZAP_POOL_CHECKPOINT_SM, sizeof (uint64_t), 1, &sm_obj);
2622
2623	ASSERT(err == 0 || err == ENOENT);
2624
2625	return (sm_obj);
2626}
2627
2628int
2629vdev_load(vdev_t *vd)
2630{
2631	int error = 0;
2632	/*
2633	 * Recursively load all children.
2634	 */
2635	for (int c = 0; c < vd->vdev_children; c++) {
2636		error = vdev_load(vd->vdev_child[c]);
2637		if (error != 0) {
2638			return (error);
2639		}
2640	}
2641
2642	vdev_set_deflate_ratio(vd);
2643
2644	/*
2645	 * If this is a top-level vdev, initialize its metaslabs.
2646	 */
2647	if (vd == vd->vdev_top && vdev_is_concrete(vd)) {
2648		if (vd->vdev_ashift == 0 || vd->vdev_asize == 0) {
2649			vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
2650			    VDEV_AUX_CORRUPT_DATA);
2651			vdev_dbgmsg(vd, "vdev_load: invalid size. ashift=%llu, "
2652			    "asize=%llu", (u_longlong_t)vd->vdev_ashift,
2653			    (u_longlong_t)vd->vdev_asize);
2654			return (SET_ERROR(ENXIO));
2655		} else if ((error = vdev_metaslab_init(vd, 0)) != 0) {
2656			vdev_dbgmsg(vd, "vdev_load: metaslab_init failed "
2657			    "[error=%d]", error);
2658			vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
2659			    VDEV_AUX_CORRUPT_DATA);
2660			return (error);
2661		}
2662
2663		uint64_t checkpoint_sm_obj = vdev_checkpoint_sm_object(vd);
2664		if (checkpoint_sm_obj != 0) {
2665			objset_t *mos = spa_meta_objset(vd->vdev_spa);
2666			ASSERT(vd->vdev_asize != 0);
2667			ASSERT3P(vd->vdev_checkpoint_sm, ==, NULL);
2668
2669			if ((error = space_map_open(&vd->vdev_checkpoint_sm,
2670			    mos, checkpoint_sm_obj, 0, vd->vdev_asize,
2671			    vd->vdev_ashift))) {
2672				vdev_dbgmsg(vd, "vdev_load: space_map_open "
2673				    "failed for checkpoint spacemap (obj %llu) "
2674				    "[error=%d]",
2675				    (u_longlong_t)checkpoint_sm_obj, error);
2676				return (error);
2677			}
2678			ASSERT3P(vd->vdev_checkpoint_sm, !=, NULL);
2679			space_map_update(vd->vdev_checkpoint_sm);
2680
2681			/*
2682			 * Since the checkpoint_sm contains free entries
2683			 * exclusively we can use sm_alloc to indicate the
2684			 * culmulative checkpointed space that has been freed.
2685			 */
2686			vd->vdev_stat.vs_checkpoint_space =
2687			    -vd->vdev_checkpoint_sm->sm_alloc;
2688			vd->vdev_spa->spa_checkpoint_info.sci_dspace +=
2689			    vd->vdev_stat.vs_checkpoint_space;
2690		}
2691	}
2692
2693	/*
2694	 * If this is a leaf vdev, load its DTL.
2695	 */
2696	if (vd->vdev_ops->vdev_op_leaf && (error = vdev_dtl_load(vd)) != 0) {
2697		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
2698		    VDEV_AUX_CORRUPT_DATA);
2699		vdev_dbgmsg(vd, "vdev_load: vdev_dtl_load failed "
2700		    "[error=%d]", error);
2701		return (error);
2702	}
2703
2704	uint64_t obsolete_sm_object = vdev_obsolete_sm_object(vd);
2705	if (obsolete_sm_object != 0) {
2706		objset_t *mos = vd->vdev_spa->spa_meta_objset;
2707		ASSERT(vd->vdev_asize != 0);
2708		ASSERT3P(vd->vdev_obsolete_sm, ==, NULL);
2709
2710		if ((error = space_map_open(&vd->vdev_obsolete_sm, mos,
2711		    obsolete_sm_object, 0, vd->vdev_asize, 0))) {
2712			vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
2713			    VDEV_AUX_CORRUPT_DATA);
2714			vdev_dbgmsg(vd, "vdev_load: space_map_open failed for "
2715			    "obsolete spacemap (obj %llu) [error=%d]",
2716			    (u_longlong_t)obsolete_sm_object, error);
2717			return (error);
2718		}
2719		space_map_update(vd->vdev_obsolete_sm);
2720	}
2721
2722	return (0);
2723}
2724
2725/*
2726 * The special vdev case is used for hot spares and l2cache devices.  Its
2727 * sole purpose it to set the vdev state for the associated vdev.  To do this,
2728 * we make sure that we can open the underlying device, then try to read the
2729 * label, and make sure that the label is sane and that it hasn't been
2730 * repurposed to another pool.
2731 */
2732int
2733vdev_validate_aux(vdev_t *vd)
2734{
2735	nvlist_t *label;
2736	uint64_t guid, version;
2737	uint64_t state;
2738
2739	if (!vdev_readable(vd))
2740		return (0);
2741
2742	if ((label = vdev_label_read_config(vd, -1ULL)) == NULL) {
2743		vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
2744		    VDEV_AUX_CORRUPT_DATA);
2745		return (-1);
2746	}
2747
2748	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_VERSION, &version) != 0 ||
2749	    !SPA_VERSION_IS_SUPPORTED(version) ||
2750	    nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID, &guid) != 0 ||
2751	    guid != vd->vdev_guid ||
2752	    nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_STATE, &state) != 0) {
2753		vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
2754		    VDEV_AUX_CORRUPT_DATA);
2755		nvlist_free(label);
2756		return (-1);
2757	}
2758
2759	/*
2760	 * We don't actually check the pool state here.  If it's in fact in
2761	 * use by another pool, we update this fact on the fly when requested.
2762	 */
2763	nvlist_free(label);
2764	return (0);
2765}
2766
2767/*
2768 * Free the objects used to store this vdev's spacemaps, and the array
2769 * that points to them.
2770 */
2771void
2772vdev_destroy_spacemaps(vdev_t *vd, dmu_tx_t *tx)
2773{
2774	if (vd->vdev_ms_array == 0)
2775		return;
2776
2777	objset_t *mos = vd->vdev_spa->spa_meta_objset;
2778	uint64_t array_count = vd->vdev_asize >> vd->vdev_ms_shift;
2779	size_t array_bytes = array_count * sizeof (uint64_t);
2780	uint64_t *smobj_array = kmem_alloc(array_bytes, KM_SLEEP);
2781	VERIFY0(dmu_read(mos, vd->vdev_ms_array, 0,
2782	    array_bytes, smobj_array, 0));
2783
2784	for (uint64_t i = 0; i < array_count; i++) {
2785		uint64_t smobj = smobj_array[i];
2786		if (smobj == 0)
2787			continue;
2788
2789		space_map_free_obj(mos, smobj, tx);
2790	}
2791
2792	kmem_free(smobj_array, array_bytes);
2793	VERIFY0(dmu_object_free(mos, vd->vdev_ms_array, tx));
2794	vd->vdev_ms_array = 0;
2795}
2796
2797static void
2798vdev_remove_empty(vdev_t *vd, uint64_t txg)
2799{
2800	spa_t *spa = vd->vdev_spa;
2801	dmu_tx_t *tx;
2802
2803	ASSERT(vd == vd->vdev_top);
2804	ASSERT3U(txg, ==, spa_syncing_txg(spa));
2805
2806	if (vd->vdev_ms != NULL) {
2807		metaslab_group_t *mg = vd->vdev_mg;
2808
2809		metaslab_group_histogram_verify(mg);
2810		metaslab_class_histogram_verify(mg->mg_class);
2811
2812		for (int m = 0; m < vd->vdev_ms_count; m++) {
2813			metaslab_t *msp = vd->vdev_ms[m];
2814
2815			if (msp == NULL || msp->ms_sm == NULL)
2816				continue;
2817
2818			mutex_enter(&msp->ms_lock);
2819			/*
2820			 * If the metaslab was not loaded when the vdev
2821			 * was removed then the histogram accounting may
2822			 * not be accurate. Update the histogram information
2823			 * here so that we ensure that the metaslab group
2824			 * and metaslab class are up-to-date.
2825			 */
2826			metaslab_group_histogram_remove(mg, msp);
2827
2828			VERIFY0(space_map_allocated(msp->ms_sm));
2829			space_map_close(msp->ms_sm);
2830			msp->ms_sm = NULL;
2831			mutex_exit(&msp->ms_lock);
2832		}
2833
2834		if (vd->vdev_checkpoint_sm != NULL) {
2835			ASSERT(spa_has_checkpoint(spa));
2836			space_map_close(vd->vdev_checkpoint_sm);
2837			vd->vdev_checkpoint_sm = NULL;
2838		}
2839
2840		metaslab_group_histogram_verify(mg);
2841		metaslab_class_histogram_verify(mg->mg_class);
2842		for (int i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++)
2843			ASSERT0(mg->mg_histogram[i]);
2844	}
2845
2846	tx = dmu_tx_create_assigned(spa_get_dsl(spa), txg);
2847	vdev_destroy_spacemaps(vd, tx);
2848
2849	if (vd->vdev_islog && vd->vdev_top_zap != 0) {
2850		vdev_destroy_unlink_zap(vd, vd->vdev_top_zap, tx);
2851		vd->vdev_top_zap = 0;
2852	}
2853	dmu_tx_commit(tx);
2854}
2855
2856void
2857vdev_sync_done(vdev_t *vd, uint64_t txg)
2858{
2859	metaslab_t *msp;
2860	boolean_t reassess = !txg_list_empty(&vd->vdev_ms_list, TXG_CLEAN(txg));
2861
2862	ASSERT(vdev_is_concrete(vd));
2863
2864	while ((msp = txg_list_remove(&vd->vdev_ms_list, TXG_CLEAN(txg)))
2865	    != NULL)
2866		metaslab_sync_done(msp, txg);
2867
2868	if (reassess)
2869		metaslab_sync_reassess(vd->vdev_mg);
2870}
2871
2872void
2873vdev_sync(vdev_t *vd, uint64_t txg)
2874{
2875	spa_t *spa = vd->vdev_spa;
2876	vdev_t *lvd;
2877	metaslab_t *msp;
2878	dmu_tx_t *tx;
2879
2880	if (range_tree_space(vd->vdev_obsolete_segments) > 0) {
2881		dmu_tx_t *tx;
2882
2883		ASSERT(vd->vdev_removing ||
2884		    vd->vdev_ops == &vdev_indirect_ops);
2885
2886		tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
2887		vdev_indirect_sync_obsolete(vd, tx);
2888		dmu_tx_commit(tx);
2889
2890		/*
2891		 * If the vdev is indirect, it can't have dirty
2892		 * metaslabs or DTLs.
2893		 */
2894		if (vd->vdev_ops == &vdev_indirect_ops) {
2895			ASSERT(txg_list_empty(&vd->vdev_ms_list, txg));
2896			ASSERT(txg_list_empty(&vd->vdev_dtl_list, txg));
2897			return;
2898		}
2899	}
2900
2901	ASSERT(vdev_is_concrete(vd));
2902
2903	if (vd->vdev_ms_array == 0 && vd->vdev_ms_shift != 0 &&
2904	    !vd->vdev_removing) {
2905		ASSERT(vd == vd->vdev_top);
2906		ASSERT0(vd->vdev_indirect_config.vic_mapping_object);
2907		tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
2908		vd->vdev_ms_array = dmu_object_alloc(spa->spa_meta_objset,
2909		    DMU_OT_OBJECT_ARRAY, 0, DMU_OT_NONE, 0, tx);
2910		ASSERT(vd->vdev_ms_array != 0);
2911		vdev_config_dirty(vd);
2912		dmu_tx_commit(tx);
2913	}
2914
2915	while ((msp = txg_list_remove(&vd->vdev_ms_list, txg)) != NULL) {
2916		metaslab_sync(msp, txg);
2917		(void) txg_list_add(&vd->vdev_ms_list, msp, TXG_CLEAN(txg));
2918	}
2919
2920	while ((lvd = txg_list_remove(&vd->vdev_dtl_list, txg)) != NULL)
2921		vdev_dtl_sync(lvd, txg);
2922
2923	/*
2924	 * Remove the metadata associated with this vdev once it's empty.
2925	 * Note that this is typically used for log/cache device removal;
2926	 * we don't empty toplevel vdevs when removing them.  But if
2927	 * a toplevel happens to be emptied, this is not harmful.
2928	 */
2929	if (vd->vdev_stat.vs_alloc == 0 && vd->vdev_removing) {
2930		vdev_remove_empty(vd, txg);
2931	}
2932
2933	(void) txg_list_add(&spa->spa_vdev_txg_list, vd, TXG_CLEAN(txg));
2934}
2935
2936uint64_t
2937vdev_psize_to_asize(vdev_t *vd, uint64_t psize)
2938{
2939	return (vd->vdev_ops->vdev_op_asize(vd, psize));
2940}
2941
2942/*
2943 * Mark the given vdev faulted.  A faulted vdev behaves as if the device could
2944 * not be opened, and no I/O is attempted.
2945 */
2946int
2947vdev_fault(spa_t *spa, uint64_t guid, vdev_aux_t aux)
2948{
2949	vdev_t *vd, *tvd;
2950
2951	spa_vdev_state_enter(spa, SCL_NONE);
2952
2953	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
2954		return (spa_vdev_state_exit(spa, NULL, ENODEV));
2955
2956	if (!vd->vdev_ops->vdev_op_leaf)
2957		return (spa_vdev_state_exit(spa, NULL, ENOTSUP));
2958
2959	tvd = vd->vdev_top;
2960
2961	/*
2962	 * We don't directly use the aux state here, but if we do a
2963	 * vdev_reopen(), we need this value to be present to remember why we
2964	 * were faulted.
2965	 */
2966	vd->vdev_label_aux = aux;
2967
2968	/*
2969	 * Faulted state takes precedence over degraded.
2970	 */
2971	vd->vdev_delayed_close = B_FALSE;
2972	vd->vdev_faulted = 1ULL;
2973	vd->vdev_degraded = 0ULL;
2974	vdev_set_state(vd, B_FALSE, VDEV_STATE_FAULTED, aux);
2975
2976	/*
2977	 * If this device has the only valid copy of the data, then
2978	 * back off and simply mark the vdev as degraded instead.
2979	 */
2980	if (!tvd->vdev_islog && vd->vdev_aux == NULL && vdev_dtl_required(vd)) {
2981		vd->vdev_degraded = 1ULL;
2982		vd->vdev_faulted = 0ULL;
2983
2984		/*
2985		 * If we reopen the device and it's not dead, only then do we
2986		 * mark it degraded.
2987		 */
2988		vdev_reopen(tvd);
2989
2990		if (vdev_readable(vd))
2991			vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, aux);
2992	}
2993
2994	return (spa_vdev_state_exit(spa, vd, 0));
2995}
2996
2997/*
2998 * Mark the given vdev degraded.  A degraded vdev is purely an indication to the
2999 * user that something is wrong.  The vdev continues to operate as normal as far
3000 * as I/O is concerned.
3001 */
3002int
3003vdev_degrade(spa_t *spa, uint64_t guid, vdev_aux_t aux)
3004{
3005	vdev_t *vd;
3006
3007	spa_vdev_state_enter(spa, SCL_NONE);
3008
3009	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
3010		return (spa_vdev_state_exit(spa, NULL, ENODEV));
3011
3012	if (!vd->vdev_ops->vdev_op_leaf)
3013		return (spa_vdev_state_exit(spa, NULL, ENOTSUP));
3014
3015	/*
3016	 * If the vdev is already faulted, then don't do anything.
3017	 */
3018	if (vd->vdev_faulted || vd->vdev_degraded)
3019		return (spa_vdev_state_exit(spa, NULL, 0));
3020
3021	vd->vdev_degraded = 1ULL;
3022	if (!vdev_is_dead(vd))
3023		vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED,
3024		    aux);
3025
3026	return (spa_vdev_state_exit(spa, vd, 0));
3027}
3028
3029/*
3030 * Online the given vdev.
3031 *
3032 * If 'ZFS_ONLINE_UNSPARE' is set, it implies two things.  First, any attached
3033 * spare device should be detached when the device finishes resilvering.
3034 * Second, the online should be treated like a 'test' online case, so no FMA
3035 * events are generated if the device fails to open.
3036 */
3037int
3038vdev_online(spa_t *spa, uint64_t guid, uint64_t flags, vdev_state_t *newstate)
3039{
3040	vdev_t *vd, *tvd, *pvd, *rvd = spa->spa_root_vdev;
3041	boolean_t wasoffline;
3042	vdev_state_t oldstate;
3043
3044	spa_vdev_state_enter(spa, SCL_NONE);
3045
3046	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
3047		return (spa_vdev_state_exit(spa, NULL, ENODEV));
3048
3049	if (!vd->vdev_ops->vdev_op_leaf)
3050		return (spa_vdev_state_exit(spa, NULL, ENOTSUP));
3051
3052	wasoffline = (vd->vdev_offline || vd->vdev_tmpoffline);
3053	oldstate = vd->vdev_state;
3054
3055	tvd = vd->vdev_top;
3056	vd->vdev_offline = B_FALSE;
3057	vd->vdev_tmpoffline = B_FALSE;
3058	vd->vdev_checkremove = !!(flags & ZFS_ONLINE_CHECKREMOVE);
3059	vd->vdev_forcefault = !!(flags & ZFS_ONLINE_FORCEFAULT);
3060
3061	/* XXX - L2ARC 1.0 does not support expansion */
3062	if (!vd->vdev_aux) {
3063		for (pvd = vd; pvd != rvd; pvd = pvd->vdev_parent)
3064			pvd->vdev_expanding = !!(flags & ZFS_ONLINE_EXPAND);
3065	}
3066
3067	vdev_reopen(tvd);
3068	vd->vdev_checkremove = vd->vdev_forcefault = B_FALSE;
3069
3070	if (!vd->vdev_aux) {
3071		for (pvd = vd; pvd != rvd; pvd = pvd->vdev_parent)
3072			pvd->vdev_expanding = B_FALSE;
3073	}
3074
3075	if (newstate)
3076		*newstate = vd->vdev_state;
3077	if ((flags & ZFS_ONLINE_UNSPARE) &&
3078	    !vdev_is_dead(vd) && vd->vdev_parent &&
3079	    vd->vdev_parent->vdev_ops == &vdev_spare_ops &&
3080	    vd->vdev_parent->vdev_child[0] == vd)
3081		vd->vdev_unspare = B_TRUE;
3082
3083	if ((flags & ZFS_ONLINE_EXPAND) || spa->spa_autoexpand) {
3084
3085		/* XXX - L2ARC 1.0 does not support expansion */
3086		if (vd->vdev_aux)
3087			return (spa_vdev_state_exit(spa, vd, ENOTSUP));
3088		spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
3089	}
3090
3091	/* Restart initializing if necessary */
3092	mutex_enter(&vd->vdev_initialize_lock);
3093	if (vdev_writeable(vd) &&
3094	    vd->vdev_initialize_thread == NULL &&
3095	    vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) {
3096		(void) vdev_initialize(vd);
3097	}
3098	mutex_exit(&vd->vdev_initialize_lock);
3099
3100	if (wasoffline ||
3101	    (oldstate < VDEV_STATE_DEGRADED &&
3102	    vd->vdev_state >= VDEV_STATE_DEGRADED))
3103		spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_ONLINE);
3104
3105	return (spa_vdev_state_exit(spa, vd, 0));
3106}
3107
3108static int
3109vdev_offline_locked(spa_t *spa, uint64_t guid, uint64_t flags)
3110{
3111	vdev_t *vd, *tvd;
3112	int error = 0;
3113	uint64_t generation;
3114	metaslab_group_t *mg;
3115
3116top:
3117	spa_vdev_state_enter(spa, SCL_ALLOC);
3118
3119	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
3120		return (spa_vdev_state_exit(spa, NULL, ENODEV));
3121
3122	if (!vd->vdev_ops->vdev_op_leaf)
3123		return (spa_vdev_state_exit(spa, NULL, ENOTSUP));
3124
3125	tvd = vd->vdev_top;
3126	mg = tvd->vdev_mg;
3127	generation = spa->spa_config_generation + 1;
3128
3129	/*
3130	 * If the device isn't already offline, try to offline it.
3131	 */
3132	if (!vd->vdev_offline) {
3133		/*
3134		 * If this device has the only valid copy of some data,
3135		 * don't allow it to be offlined. Log devices are always
3136		 * expendable.
3137		 */
3138		if (!tvd->vdev_islog && vd->vdev_aux == NULL &&
3139		    vdev_dtl_required(vd))
3140			return (spa_vdev_state_exit(spa, NULL, EBUSY));
3141
3142		/*
3143		 * If the top-level is a slog and it has had allocations
3144		 * then proceed.  We check that the vdev's metaslab group
3145		 * is not NULL since it's possible that we may have just
3146		 * added this vdev but not yet initialized its metaslabs.
3147		 */
3148		if (tvd->vdev_islog && mg != NULL) {
3149			/*
3150			 * Prevent any future allocations.
3151			 */
3152			metaslab_group_passivate(mg);
3153			(void) spa_vdev_state_exit(spa, vd, 0);
3154
3155			error = spa_reset_logs(spa);
3156
3157			/*
3158			 * If the log device was successfully reset but has
3159			 * checkpointed data, do not offline it.
3160			 */
3161			if (error == 0 &&
3162			    tvd->vdev_checkpoint_sm != NULL) {
3163				ASSERT3U(tvd->vdev_checkpoint_sm->sm_alloc,
3164				    !=, 0);
3165				error = ZFS_ERR_CHECKPOINT_EXISTS;
3166			}
3167
3168			spa_vdev_state_enter(spa, SCL_ALLOC);
3169
3170			/*
3171			 * Check to see if the config has changed.
3172			 */
3173			if (error || generation != spa->spa_config_generation) {
3174				metaslab_group_activate(mg);
3175				if (error)
3176					return (spa_vdev_state_exit(spa,
3177					    vd, error));
3178				(void) spa_vdev_state_exit(spa, vd, 0);
3179				goto top;
3180			}
3181			ASSERT0(tvd->vdev_stat.vs_alloc);
3182		}
3183
3184		/*
3185		 * Offline this device and reopen its top-level vdev.
3186		 * If the top-level vdev is a log device then just offline
3187		 * it. Otherwise, if this action results in the top-level
3188		 * vdev becoming unusable, undo it and fail the request.
3189		 */
3190		vd->vdev_offline = B_TRUE;
3191		vdev_reopen(tvd);
3192
3193		if (!tvd->vdev_islog && vd->vdev_aux == NULL &&
3194		    vdev_is_dead(tvd)) {
3195			vd->vdev_offline = B_FALSE;
3196			vdev_reopen(tvd);
3197			return (spa_vdev_state_exit(spa, NULL, EBUSY));
3198		}
3199
3200		/*
3201		 * Add the device back into the metaslab rotor so that
3202		 * once we online the device it's open for business.
3203		 */
3204		if (tvd->vdev_islog && mg != NULL)
3205			metaslab_group_activate(mg);
3206	}
3207
3208	vd->vdev_tmpoffline = !!(flags & ZFS_OFFLINE_TEMPORARY);
3209
3210	return (spa_vdev_state_exit(spa, vd, 0));
3211}
3212
3213int
3214vdev_offline(spa_t *spa, uint64_t guid, uint64_t flags)
3215{
3216	int error;
3217
3218	mutex_enter(&spa->spa_vdev_top_lock);
3219	error = vdev_offline_locked(spa, guid, flags);
3220	mutex_exit(&spa->spa_vdev_top_lock);
3221
3222	return (error);
3223}
3224
3225/*
3226 * Clear the error counts associated with this vdev.  Unlike vdev_online() and
3227 * vdev_offline(), we assume the spa config is locked.  We also clear all
3228 * children.  If 'vd' is NULL, then the user wants to clear all vdevs.
3229 */
3230void
3231vdev_clear(spa_t *spa, vdev_t *vd)
3232{
3233	vdev_t *rvd = spa->spa_root_vdev;
3234
3235	ASSERT(spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);
3236
3237	if (vd == NULL)
3238		vd = rvd;
3239
3240	vd->vdev_stat.vs_read_errors = 0;
3241	vd->vdev_stat.vs_write_errors = 0;
3242	vd->vdev_stat.vs_checksum_errors = 0;
3243
3244	for (int c = 0; c < vd->vdev_children; c++)
3245		vdev_clear(spa, vd->vdev_child[c]);
3246
3247	/*
3248	 * It makes no sense to "clear" an indirect vdev.
3249	 */
3250	if (!vdev_is_concrete(vd))
3251		return;
3252
3253	/*
3254	 * If we're in the FAULTED state or have experienced failed I/O, then
3255	 * clear the persistent state and attempt to reopen the device.  We
3256	 * also mark the vdev config dirty, so that the new faulted state is
3257	 * written out to disk.
3258	 */
3259	if (vd->vdev_faulted || vd->vdev_degraded ||
3260	    !vdev_readable(vd) || !vdev_writeable(vd)) {
3261
3262		/*
3263		 * When reopening in reponse to a clear event, it may be due to
3264		 * a fmadm repair request.  In this case, if the device is
3265		 * still broken, we want to still post the ereport again.
3266		 */
3267		vd->vdev_forcefault = B_TRUE;
3268
3269		vd->vdev_faulted = vd->vdev_degraded = 0ULL;
3270		vd->vdev_cant_read = B_FALSE;
3271		vd->vdev_cant_write = B_FALSE;
3272
3273		vdev_reopen(vd == rvd ? rvd : vd->vdev_top);
3274
3275		vd->vdev_forcefault = B_FALSE;
3276
3277		if (vd != rvd && vdev_writeable(vd->vdev_top))
3278			vdev_state_dirty(vd->vdev_top);
3279
3280		if (vd->vdev_aux == NULL && !vdev_is_dead(vd))
3281			spa_async_request(spa, SPA_ASYNC_RESILVER);
3282
3283		spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_CLEAR);
3284	}
3285
3286	/*
3287	 * When clearing a FMA-diagnosed fault, we always want to
3288	 * unspare the device, as we assume that the original spare was
3289	 * done in response to the FMA fault.
3290	 */
3291	if (!vdev_is_dead(vd) && vd->vdev_parent != NULL &&
3292	    vd->vdev_parent->vdev_ops == &vdev_spare_ops &&
3293	    vd->vdev_parent->vdev_child[0] == vd)
3294		vd->vdev_unspare = B_TRUE;
3295}
3296
3297boolean_t
3298vdev_is_dead(vdev_t *vd)
3299{
3300	/*
3301	 * Holes and missing devices are always considered "dead".
3302	 * This simplifies the code since we don't have to check for
3303	 * these types of devices in the various code paths.
3304	 * Instead we rely on the fact that we skip over dead devices
3305	 * before issuing I/O to them.
3306	 */
3307	return (vd->vdev_state < VDEV_STATE_DEGRADED ||
3308	    vd->vdev_ops == &vdev_hole_ops ||
3309	    vd->vdev_ops == &vdev_missing_ops);
3310}
3311
3312boolean_t
3313vdev_readable(vdev_t *vd)
3314{
3315	return (!vdev_is_dead(vd) && !vd->vdev_cant_read);
3316}
3317
3318boolean_t
3319vdev_writeable(vdev_t *vd)
3320{
3321	return (!vdev_is_dead(vd) && !vd->vdev_cant_write &&
3322	    vdev_is_concrete(vd));
3323}
3324
3325boolean_t
3326vdev_allocatable(vdev_t *vd)
3327{
3328	uint64_t state = vd->vdev_state;
3329
3330	/*
3331	 * We currently allow allocations from vdevs which may be in the
3332	 * process of reopening (i.e. VDEV_STATE_CLOSED). If the device
3333	 * fails to reopen then we'll catch it later when we're holding
3334	 * the proper locks.  Note that we have to get the vdev state
3335	 * in a local variable because although it changes atomically,
3336	 * we're asking two separate questions about it.
3337	 */
3338	return (!(state < VDEV_STATE_DEGRADED && state != VDEV_STATE_CLOSED) &&
3339	    !vd->vdev_cant_write && vdev_is_concrete(vd) &&
3340	    vd->vdev_mg->mg_initialized);
3341}
3342
3343boolean_t
3344vdev_accessible(vdev_t *vd, zio_t *zio)
3345{
3346	ASSERT(zio->io_vd == vd);
3347
3348	if (vdev_is_dead(vd) || vd->vdev_remove_wanted)
3349		return (B_FALSE);
3350
3351	if (zio->io_type == ZIO_TYPE_READ)
3352		return (!vd->vdev_cant_read);
3353
3354	if (zio->io_type == ZIO_TYPE_WRITE)
3355		return (!vd->vdev_cant_write);
3356
3357	return (B_TRUE);
3358}
3359
3360boolean_t
3361vdev_is_spacemap_addressable(vdev_t *vd)
3362{
3363	/*
3364	 * Assuming 47 bits of the space map entry dedicated for the entry's
3365	 * offset (see description in space_map.h), we calculate the maximum
3366	 * address that can be described by a space map entry for the given
3367	 * device.
3368	 */
3369	uint64_t shift = vd->vdev_ashift + 47;
3370
3371	if (shift >= 63) /* detect potential overflow */
3372		return (B_TRUE);
3373
3374	return (vd->vdev_asize < (1ULL << shift));
3375}
3376
3377/*
3378 * Get statistics for the given vdev.
3379 */
3380void
3381vdev_get_stats(vdev_t *vd, vdev_stat_t *vs)
3382{
3383	spa_t *spa = vd->vdev_spa;
3384	vdev_t *rvd = spa->spa_root_vdev;
3385	vdev_t *tvd = vd->vdev_top;
3386
3387	ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0);
3388
3389	mutex_enter(&vd->vdev_stat_lock);
3390	bcopy(&vd->vdev_stat, vs, sizeof (*vs));
3391	vs->vs_timestamp = gethrtime() - vs->vs_timestamp;
3392	vs->vs_state = vd->vdev_state;
3393	vs->vs_rsize = vdev_get_min_asize(vd);
3394	if (vd->vdev_ops->vdev_op_leaf) {
3395		vs->vs_rsize += VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE;
3396		/*
3397		 * Report intializing progress. Since we don't have the
3398		 * initializing locks held, this is only an estimate (although a
3399		 * fairly accurate one).
3400		 */
3401		vs->vs_initialize_bytes_done = vd->vdev_initialize_bytes_done;
3402		vs->vs_initialize_bytes_est = vd->vdev_initialize_bytes_est;
3403		vs->vs_initialize_state = vd->vdev_initialize_state;
3404		vs->vs_initialize_action_time = vd->vdev_initialize_action_time;
3405	}
3406	/*
3407	 * Report expandable space on top-level, non-auxillary devices only.
3408	 * The expandable space is reported in terms of metaslab sized units
3409	 * since that determines how much space the pool can expand.
3410	 */
3411	if (vd->vdev_aux == NULL && tvd != NULL) {
3412		vs->vs_esize = P2ALIGN(vd->vdev_max_asize - vd->vdev_asize -
3413		    spa->spa_bootsize, 1ULL << tvd->vdev_ms_shift);
3414	}
3415	if (vd->vdev_aux == NULL && vd == vd->vdev_top &&
3416	    vdev_is_concrete(vd)) {
3417		vs->vs_fragmentation = vd->vdev_mg->mg_fragmentation;
3418	}
3419
3420	/*
3421	 * If we're getting stats on the root vdev, aggregate the I/O counts
3422	 * over all top-level vdevs (i.e. the direct children of the root).
3423	 */
3424	if (vd == rvd) {
3425		for (int c = 0; c < rvd->vdev_children; c++) {
3426			vdev_t *cvd = rvd->vdev_child[c];
3427			vdev_stat_t *cvs = &cvd->vdev_stat;
3428
3429			for (int t = 0; t < ZIO_TYPES; t++) {
3430				vs->vs_ops[t] += cvs->vs_ops[t];
3431				vs->vs_bytes[t] += cvs->vs_bytes[t];
3432			}
3433			cvs->vs_scan_removing = cvd->vdev_removing;
3434		}
3435	}
3436	mutex_exit(&vd->vdev_stat_lock);
3437}
3438
3439void
3440vdev_clear_stats(vdev_t *vd)
3441{
3442	mutex_enter(&vd->vdev_stat_lock);
3443	vd->vdev_stat.vs_space = 0;
3444	vd->vdev_stat.vs_dspace = 0;
3445	vd->vdev_stat.vs_alloc = 0;
3446	mutex_exit(&vd->vdev_stat_lock);
3447}
3448
3449void
3450vdev_scan_stat_init(vdev_t *vd)
3451{
3452	vdev_stat_t *vs = &vd->vdev_stat;
3453
3454	for (int c = 0; c < vd->vdev_children; c++)
3455		vdev_scan_stat_init(vd->vdev_child[c]);
3456
3457	mutex_enter(&vd->vdev_stat_lock);
3458	vs->vs_scan_processed = 0;
3459	mutex_exit(&vd->vdev_stat_lock);
3460}
3461
3462void
3463vdev_stat_update(zio_t *zio, uint64_t psize)
3464{
3465	spa_t *spa = zio->io_spa;
3466	vdev_t *rvd = spa->spa_root_vdev;
3467	vdev_t *vd = zio->io_vd ? zio->io_vd : rvd;
3468	vdev_t *pvd;
3469	uint64_t txg = zio->io_txg;
3470	vdev_stat_t *vs = &vd->vdev_stat;
3471	zio_type_t type = zio->io_type;
3472	int flags = zio->io_flags;
3473
3474	/*
3475	 * If this i/o is a gang leader, it didn't do any actual work.
3476	 */
3477	if (zio->io_gang_tree)
3478		return;
3479
3480	if (zio->io_error == 0) {
3481		/*
3482		 * If this is a root i/o, don't count it -- we've already
3483		 * counted the top-level vdevs, and vdev_get_stats() will
3484		 * aggregate them when asked.  This reduces contention on
3485		 * the root vdev_stat_lock and implicitly handles blocks
3486		 * that compress away to holes, for which there is no i/o.
3487		 * (Holes never create vdev children, so all the counters
3488		 * remain zero, which is what we want.)
3489		 *
3490		 * Note: this only applies to successful i/o (io_error == 0)
3491		 * because unlike i/o counts, errors are not additive.
3492		 * When reading a ditto block, for example, failure of
3493		 * one top-level vdev does not imply a root-level error.
3494		 */
3495		if (vd == rvd)
3496			return;
3497
3498		ASSERT(vd == zio->io_vd);
3499
3500		if (flags & ZIO_FLAG_IO_BYPASS)
3501			return;
3502
3503		mutex_enter(&vd->vdev_stat_lock);
3504
3505		if (flags & ZIO_FLAG_IO_REPAIR) {
3506			if (flags & ZIO_FLAG_SCAN_THREAD) {
3507				dsl_scan_phys_t *scn_phys =
3508				    &spa->spa_dsl_pool->dp_scan->scn_phys;
3509				uint64_t *processed = &scn_phys->scn_processed;
3510
3511				/* XXX cleanup? */
3512				if (vd->vdev_ops->vdev_op_leaf)
3513					atomic_add_64(processed, psize);
3514				vs->vs_scan_processed += psize;
3515			}
3516
3517			if (flags & ZIO_FLAG_SELF_HEAL)
3518				vs->vs_self_healed += psize;
3519		}
3520
3521		vs->vs_ops[type]++;
3522		vs->vs_bytes[type] += psize;
3523
3524		mutex_exit(&vd->vdev_stat_lock);
3525		return;
3526	}
3527
3528	if (flags & ZIO_FLAG_SPECULATIVE)
3529		return;
3530
3531	/*
3532	 * If this is an I/O error that is going to be retried, then ignore the
3533	 * error.  Otherwise, the user may interpret B_FAILFAST I/O errors as
3534	 * hard errors, when in reality they can happen for any number of
3535	 * innocuous reasons (bus resets, MPxIO link failure, etc).
3536	 */
3537	if (zio->io_error == EIO &&
3538	    !(zio->io_flags & ZIO_FLAG_IO_RETRY))
3539		return;
3540
3541	/*
3542	 * Intent logs writes won't propagate their error to the root
3543	 * I/O so don't mark these types of failures as pool-level
3544	 * errors.
3545	 */
3546	if (zio->io_vd == NULL && (zio->io_flags & ZIO_FLAG_DONT_PROPAGATE))
3547		return;
3548
3549	mutex_enter(&vd->vdev_stat_lock);
3550	if (type == ZIO_TYPE_READ && !vdev_is_dead(vd)) {
3551		if (zio->io_error == ECKSUM)
3552			vs->vs_checksum_errors++;
3553		else
3554			vs->vs_read_errors++;
3555	}
3556	if (type == ZIO_TYPE_WRITE && !vdev_is_dead(vd))
3557		vs->vs_write_errors++;
3558	mutex_exit(&vd->vdev_stat_lock);
3559
3560	if (spa->spa_load_state == SPA_LOAD_NONE &&
3561	    type == ZIO_TYPE_WRITE && txg != 0 &&
3562	    (!(flags & ZIO_FLAG_IO_REPAIR) ||
3563	    (flags & ZIO_FLAG_SCAN_THREAD) ||
3564	    spa->spa_claiming)) {
3565		/*
3566		 * This is either a normal write (not a repair), or it's
3567		 * a repair induced by the scrub thread, or it's a repair
3568		 * made by zil_claim() during spa_load() in the first txg.
3569		 * In the normal case, we commit the DTL change in the same
3570		 * txg as the block was born.  In the scrub-induced repair
3571		 * case, we know that scrubs run in first-pass syncing context,
3572		 * so we commit the DTL change in spa_syncing_txg(spa).
3573		 * In the zil_claim() case, we commit in spa_first_txg(spa).
3574		 *
3575		 * We currently do not make DTL entries for failed spontaneous
3576		 * self-healing writes triggered by normal (non-scrubbing)
3577		 * reads, because we have no transactional context in which to
3578		 * do so -- and it's not clear that it'd be desirable anyway.
3579		 */
3580		if (vd->vdev_ops->vdev_op_leaf) {
3581			uint64_t commit_txg = txg;
3582			if (flags & ZIO_FLAG_SCAN_THREAD) {
3583				ASSERT(flags & ZIO_FLAG_IO_REPAIR);
3584				ASSERT(spa_sync_pass(spa) == 1);
3585				vdev_dtl_dirty(vd, DTL_SCRUB, txg, 1);
3586				commit_txg = spa_syncing_txg(spa);
3587			} else if (spa->spa_claiming) {
3588				ASSERT(flags & ZIO_FLAG_IO_REPAIR);
3589				commit_txg = spa_first_txg(spa);
3590			}
3591			ASSERT(commit_txg >= spa_syncing_txg(spa));
3592			if (vdev_dtl_contains(vd, DTL_MISSING, txg, 1))
3593				return;
3594			for (pvd = vd; pvd != rvd; pvd = pvd->vdev_parent)
3595				vdev_dtl_dirty(pvd, DTL_PARTIAL, txg, 1);
3596			vdev_dirty(vd->vdev_top, VDD_DTL, vd, commit_txg);
3597		}
3598		if (vd != rvd)
3599			vdev_dtl_dirty(vd, DTL_MISSING, txg, 1);
3600	}
3601}
3602
3603/*
3604 * Update the in-core space usage stats for this vdev, its metaslab class,
3605 * and the root vdev.
3606 */
3607void
3608vdev_space_update(vdev_t *vd, int64_t alloc_delta, int64_t defer_delta,
3609    int64_t space_delta)
3610{
3611	int64_t dspace_delta = space_delta;
3612	spa_t *spa = vd->vdev_spa;
3613	vdev_t *rvd = spa->spa_root_vdev;
3614	metaslab_group_t *mg = vd->vdev_mg;
3615	metaslab_class_t *mc = mg ? mg->mg_class : NULL;
3616
3617	ASSERT(vd == vd->vdev_top);
3618
3619	/*
3620	 * Apply the inverse of the psize-to-asize (ie. RAID-Z) space-expansion
3621	 * factor.  We must calculate this here and not at the root vdev
3622	 * because the root vdev's psize-to-asize is simply the max of its
3623	 * childrens', thus not accurate enough for us.
3624	 */
3625	ASSERT((dspace_delta & (SPA_MINBLOCKSIZE-1)) == 0);
3626	ASSERT(vd->vdev_deflate_ratio != 0 || vd->vdev_isl2cache);
3627	dspace_delta = (dspace_delta >> SPA_MINBLOCKSHIFT) *
3628	    vd->vdev_deflate_ratio;
3629
3630	mutex_enter(&vd->vdev_stat_lock);
3631	vd->vdev_stat.vs_alloc += alloc_delta;
3632	vd->vdev_stat.vs_space += space_delta;
3633	vd->vdev_stat.vs_dspace += dspace_delta;
3634	mutex_exit(&vd->vdev_stat_lock);
3635
3636	if (mc == spa_normal_class(spa)) {
3637		mutex_enter(&rvd->vdev_stat_lock);
3638		rvd->vdev_stat.vs_alloc += alloc_delta;
3639		rvd->vdev_stat.vs_space += space_delta;
3640		rvd->vdev_stat.vs_dspace += dspace_delta;
3641		mutex_exit(&rvd->vdev_stat_lock);
3642	}
3643
3644	if (mc != NULL) {
3645		ASSERT(rvd == vd->vdev_parent);
3646		ASSERT(vd->vdev_ms_count != 0);
3647
3648		metaslab_class_space_update(mc,
3649		    alloc_delta, defer_delta, space_delta, dspace_delta);
3650	}
3651}
3652
3653/*
3654 * Mark a top-level vdev's config as dirty, placing it on the dirty list
3655 * so that it will be written out next time the vdev configuration is synced.
3656 * If the root vdev is specified (vdev_top == NULL), dirty all top-level vdevs.
3657 */
3658void
3659vdev_config_dirty(vdev_t *vd)
3660{
3661	spa_t *spa = vd->vdev_spa;
3662	vdev_t *rvd = spa->spa_root_vdev;
3663	int c;
3664
3665	ASSERT(spa_writeable(spa));
3666
3667	/*
3668	 * If this is an aux vdev (as with l2cache and spare devices), then we
3669	 * update the vdev config manually and set the sync flag.
3670	 */
3671	if (vd->vdev_aux != NULL) {
3672		spa_aux_vdev_t *sav = vd->vdev_aux;
3673		nvlist_t **aux;
3674		uint_t naux;
3675
3676		for (c = 0; c < sav->sav_count; c++) {
3677			if (sav->sav_vdevs[c] == vd)
3678				break;
3679		}
3680
3681		if (c == sav->sav_count) {
3682			/*
3683			 * We're being removed.  There's nothing more to do.
3684			 */
3685			ASSERT(sav->sav_sync == B_TRUE);
3686			return;
3687		}
3688
3689		sav->sav_sync = B_TRUE;
3690
3691		if (nvlist_lookup_nvlist_array(sav->sav_config,
3692		    ZPOOL_CONFIG_L2CACHE, &aux, &naux) != 0) {
3693			VERIFY(nvlist_lookup_nvlist_array(sav->sav_config,
3694			    ZPOOL_CONFIG_SPARES, &aux, &naux) == 0);
3695		}
3696
3697		ASSERT(c < naux);
3698
3699		/*
3700		 * Setting the nvlist in the middle if the array is a little
3701		 * sketchy, but it will work.
3702		 */
3703		nvlist_free(aux[c]);
3704		aux[c] = vdev_config_generate(spa, vd, B_TRUE, 0);
3705
3706		return;
3707	}
3708
3709	/*
3710	 * The dirty list is protected by the SCL_CONFIG lock.  The caller
3711	 * must either hold SCL_CONFIG as writer, or must be the sync thread
3712	 * (which holds SCL_CONFIG as reader).  There's only one sync thread,
3713	 * so this is sufficient to ensure mutual exclusion.
3714	 */
3715	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_WRITER) ||
3716	    (dsl_pool_sync_context(spa_get_dsl(spa)) &&
3717	    spa_config_held(spa, SCL_CONFIG, RW_READER)));
3718
3719	if (vd == rvd) {
3720		for (c = 0; c < rvd->vdev_children; c++)
3721			vdev_config_dirty(rvd->vdev_child[c]);
3722	} else {
3723		ASSERT(vd == vd->vdev_top);
3724
3725		if (!list_link_active(&vd->vdev_config_dirty_node) &&
3726		    vdev_is_concrete(vd)) {
3727			list_insert_head(&spa->spa_config_dirty_list, vd);
3728		}
3729	}
3730}
3731
3732void
3733vdev_config_clean(vdev_t *vd)
3734{
3735	spa_t *spa = vd->vdev_spa;
3736
3737	ASSERT(spa_config_held(spa, SCL_CONFIG, RW_WRITER) ||
3738	    (dsl_pool_sync_context(spa_get_dsl(spa)) &&
3739	    spa_config_held(spa, SCL_CONFIG, RW_READER)));
3740
3741	ASSERT(list_link_active(&vd->vdev_config_dirty_node));
3742	list_remove(&spa->spa_config_dirty_list, vd);
3743}
3744
3745/*
3746 * Mark a top-level vdev's state as dirty, so that the next pass of
3747 * spa_sync() can convert this into vdev_config_dirty().  We distinguish
3748 * the state changes from larger config changes because they require
3749 * much less locking, and are often needed for administrative actions.
3750 */
3751void
3752vdev_state_dirty(vdev_t *vd)
3753{
3754	spa_t *spa = vd->vdev_spa;
3755
3756	ASSERT(spa_writeable(spa));
3757	ASSERT(vd == vd->vdev_top);
3758
3759	/*
3760	 * The state list is protected by the SCL_STATE lock.  The caller
3761	 * must either hold SCL_STATE as writer, or must be the sync thread
3762	 * (which holds SCL_STATE as reader).  There's only one sync thread,
3763	 * so this is sufficient to ensure mutual exclusion.
3764	 */
3765	ASSERT(spa_config_held(spa, SCL_STATE, RW_WRITER) ||
3766	    (dsl_pool_sync_context(spa_get_dsl(spa)) &&
3767	    spa_config_held(spa, SCL_STATE, RW_READER)));
3768
3769	if (!list_link_active(&vd->vdev_state_dirty_node) &&
3770	    vdev_is_concrete(vd))
3771		list_insert_head(&spa->spa_state_dirty_list, vd);
3772}
3773
3774void
3775vdev_state_clean(vdev_t *vd)
3776{
3777	spa_t *spa = vd->vdev_spa;
3778
3779	ASSERT(spa_config_held(spa, SCL_STATE, RW_WRITER) ||
3780	    (dsl_pool_sync_context(spa_get_dsl(spa)) &&
3781	    spa_config_held(spa, SCL_STATE, RW_READER)));
3782
3783	ASSERT(list_link_active(&vd->vdev_state_dirty_node));
3784	list_remove(&spa->spa_state_dirty_list, vd);
3785}
3786
3787/*
3788 * Propagate vdev state up from children to parent.
3789 */
3790void
3791vdev_propagate_state(vdev_t *vd)
3792{
3793	spa_t *spa = vd->vdev_spa;
3794	vdev_t *rvd = spa->spa_root_vdev;
3795	int degraded = 0, faulted = 0;
3796	int corrupted = 0;
3797	vdev_t *child;
3798
3799	if (vd->vdev_children > 0) {
3800		for (int c = 0; c < vd->vdev_children; c++) {
3801			child = vd->vdev_child[c];
3802
3803			/*
3804			 * Don't factor holes or indirect vdevs into the
3805			 * decision.
3806			 */
3807			if (!vdev_is_concrete(child))
3808				continue;
3809
3810			if (!vdev_readable(child) ||
3811			    (!vdev_writeable(child) && spa_writeable(spa))) {
3812				/*
3813				 * Root special: if there is a top-level log
3814				 * device, treat the root vdev as if it were
3815				 * degraded.
3816				 */
3817				if (child->vdev_islog && vd == rvd)
3818					degraded++;
3819				else
3820					faulted++;
3821			} else if (child->vdev_state <= VDEV_STATE_DEGRADED) {
3822				degraded++;
3823			}
3824
3825			if (child->vdev_stat.vs_aux == VDEV_AUX_CORRUPT_DATA)
3826				corrupted++;
3827		}
3828
3829		vd->vdev_ops->vdev_op_state_change(vd, faulted, degraded);
3830
3831		/*
3832		 * Root special: if there is a top-level vdev that cannot be
3833		 * opened due to corrupted metadata, then propagate the root
3834		 * vdev's aux state as 'corrupt' rather than 'insufficient
3835		 * replicas'.
3836		 */
3837		if (corrupted && vd == rvd &&
3838		    rvd->vdev_state == VDEV_STATE_CANT_OPEN)
3839			vdev_set_state(rvd, B_FALSE, VDEV_STATE_CANT_OPEN,
3840			    VDEV_AUX_CORRUPT_DATA);
3841	}
3842
3843	if (vd->vdev_parent)
3844		vdev_propagate_state(vd->vdev_parent);
3845}
3846
3847/*
3848 * Set a vdev's state.  If this is during an open, we don't update the parent
3849 * state, because we're in the process of opening children depth-first.
3850 * Otherwise, we propagate the change to the parent.
3851 *
3852 * If this routine places a device in a faulted state, an appropriate ereport is
3853 * generated.
3854 */
3855void
3856vdev_set_state(vdev_t *vd, boolean_t isopen, vdev_state_t state, vdev_aux_t aux)
3857{
3858	uint64_t save_state;
3859	spa_t *spa = vd->vdev_spa;
3860
3861	if (state == vd->vdev_state) {
3862		vd->vdev_stat.vs_aux = aux;
3863		return;
3864	}
3865
3866	save_state = vd->vdev_state;
3867
3868	vd->vdev_state = state;
3869	vd->vdev_stat.vs_aux = aux;
3870
3871	/*
3872	 * If we are setting the vdev state to anything but an open state, then
3873	 * always close the underlying device unless the device has requested
3874	 * a delayed close (i.e. we're about to remove or fault the device).
3875	 * Otherwise, we keep accessible but invalid devices open forever.
3876	 * We don't call vdev_close() itself, because that implies some extra
3877	 * checks (offline, etc) that we don't want here.  This is limited to
3878	 * leaf devices, because otherwise closing the device will affect other
3879	 * children.
3880	 */
3881	if (!vd->vdev_delayed_close && vdev_is_dead(vd) &&
3882	    vd->vdev_ops->vdev_op_leaf)
3883		vd->vdev_ops->vdev_op_close(vd);
3884
3885	/*
3886	 * If we have brought this vdev back into service, we need
3887	 * to notify fmd so that it can gracefully repair any outstanding
3888	 * cases due to a missing device.  We do this in all cases, even those
3889	 * that probably don't correlate to a repaired fault.  This is sure to
3890	 * catch all cases, and we let the zfs-retire agent sort it out.  If
3891	 * this is a transient state it's OK, as the retire agent will
3892	 * double-check the state of the vdev before repairing it.
3893	 */
3894	if (state == VDEV_STATE_HEALTHY && vd->vdev_ops->vdev_op_leaf &&
3895	    vd->vdev_prevstate != state)
3896		zfs_post_state_change(spa, vd);
3897
3898	if (vd->vdev_removed &&
3899	    state == VDEV_STATE_CANT_OPEN &&
3900	    (aux == VDEV_AUX_OPEN_FAILED || vd->vdev_checkremove)) {
3901		/*
3902		 * If the previous state is set to VDEV_STATE_REMOVED, then this
3903		 * device was previously marked removed and someone attempted to
3904		 * reopen it.  If this failed due to a nonexistent device, then
3905		 * keep the device in the REMOVED state.  We also let this be if
3906		 * it is one of our special test online cases, which is only
3907		 * attempting to online the device and shouldn't generate an FMA
3908		 * fault.
3909		 */
3910		vd->vdev_state = VDEV_STATE_REMOVED;
3911		vd->vdev_stat.vs_aux = VDEV_AUX_NONE;
3912	} else if (state == VDEV_STATE_REMOVED) {
3913		vd->vdev_removed = B_TRUE;
3914	} else if (state == VDEV_STATE_CANT_OPEN) {
3915		/*
3916		 * If we fail to open a vdev during an import or recovery, we
3917		 * mark it as "not available", which signifies that it was
3918		 * never there to begin with.  Failure to open such a device
3919		 * is not considered an error.
3920		 */
3921		if ((spa_load_state(spa) == SPA_LOAD_IMPORT ||
3922		    spa_load_state(spa) == SPA_LOAD_RECOVER) &&
3923		    vd->vdev_ops->vdev_op_leaf)
3924			vd->vdev_not_present = 1;
3925
3926		/*
3927		 * Post the appropriate ereport.  If the 'prevstate' field is
3928		 * set to something other than VDEV_STATE_UNKNOWN, it indicates
3929		 * that this is part of a vdev_reopen().  In this case, we don't
3930		 * want to post the ereport if the device was already in the
3931		 * CANT_OPEN state beforehand.
3932		 *
3933		 * If the 'checkremove' flag is set, then this is an attempt to
3934		 * online the device in response to an insertion event.  If we
3935		 * hit this case, then we have detected an insertion event for a
3936		 * faulted or offline device that wasn't in the removed state.
3937		 * In this scenario, we don't post an ereport because we are
3938		 * about to replace the device, or attempt an online with
3939		 * vdev_forcefault, which will generate the fault for us.
3940		 */
3941		if ((vd->vdev_prevstate != state || vd->vdev_forcefault) &&
3942		    !vd->vdev_not_present && !vd->vdev_checkremove &&
3943		    vd != spa->spa_root_vdev) {
3944			const char *class;
3945
3946			switch (aux) {
3947			case VDEV_AUX_OPEN_FAILED:
3948				class = FM_EREPORT_ZFS_DEVICE_OPEN_FAILED;
3949				break;
3950			case VDEV_AUX_CORRUPT_DATA:
3951				class = FM_EREPORT_ZFS_DEVICE_CORRUPT_DATA;
3952				break;
3953			case VDEV_AUX_NO_REPLICAS:
3954				class = FM_EREPORT_ZFS_DEVICE_NO_REPLICAS;
3955				break;
3956			case VDEV_AUX_BAD_GUID_SUM:
3957				class = FM_EREPORT_ZFS_DEVICE_BAD_GUID_SUM;
3958				break;
3959			case VDEV_AUX_TOO_SMALL:
3960				class = FM_EREPORT_ZFS_DEVICE_TOO_SMALL;
3961				break;
3962			case VDEV_AUX_BAD_LABEL:
3963				class = FM_EREPORT_ZFS_DEVICE_BAD_LABEL;
3964				break;
3965			default:
3966				class = FM_EREPORT_ZFS_DEVICE_UNKNOWN;
3967			}
3968
3969			zfs_ereport_post(class, spa, vd, NULL, save_state, 0);
3970		}
3971
3972		/* Erase any notion of persistent removed state */
3973		vd->vdev_removed = B_FALSE;
3974	} else {
3975		vd->vdev_removed = B_FALSE;
3976	}
3977
3978	if (!isopen && vd->vdev_parent)
3979		vdev_propagate_state(vd->vdev_parent);
3980}
3981
3982boolean_t
3983vdev_children_are_offline(vdev_t *vd)
3984{
3985	ASSERT(!vd->vdev_ops->vdev_op_leaf);
3986
3987	for (uint64_t i = 0; i < vd->vdev_children; i++) {
3988		if (vd->vdev_child[i]->vdev_state != VDEV_STATE_OFFLINE)
3989			return (B_FALSE);
3990	}
3991
3992	return (B_TRUE);
3993}
3994
3995/*
3996 * Check the vdev configuration to ensure that it's capable of supporting
3997 * a root pool. We do not support partial configuration.
3998 * In addition, only a single top-level vdev is allowed.
3999 */
4000boolean_t
4001vdev_is_bootable(vdev_t *vd)
4002{
4003	if (!vd->vdev_ops->vdev_op_leaf) {
4004		char *vdev_type = vd->vdev_ops->vdev_op_type;
4005
4006		if (strcmp(vdev_type, VDEV_TYPE_ROOT) == 0 &&
4007		    vd->vdev_children > 1) {
4008			return (B_FALSE);
4009		} else if (strcmp(vdev_type, VDEV_TYPE_MISSING) == 0 ||
4010		    strcmp(vdev_type, VDEV_TYPE_INDIRECT) == 0) {
4011			return (B_FALSE);
4012		}
4013	}
4014
4015	for (int c = 0; c < vd->vdev_children; c++) {
4016		if (!vdev_is_bootable(vd->vdev_child[c]))
4017			return (B_FALSE);
4018	}
4019	return (B_TRUE);
4020}
4021
4022boolean_t
4023vdev_is_concrete(vdev_t *vd)
4024{
4025	vdev_ops_t *ops = vd->vdev_ops;
4026	if (ops == &vdev_indirect_ops || ops == &vdev_hole_ops ||
4027	    ops == &vdev_missing_ops || ops == &vdev_root_ops) {
4028		return (B_FALSE);
4029	} else {
4030		return (B_TRUE);
4031	}
4032}
4033
4034/*
4035 * Determine if a log device has valid content.  If the vdev was
4036 * removed or faulted in the MOS config then we know that
4037 * the content on the log device has already been written to the pool.
4038 */
4039boolean_t
4040vdev_log_state_valid(vdev_t *vd)
4041{
4042	if (vd->vdev_ops->vdev_op_leaf && !vd->vdev_faulted &&
4043	    !vd->vdev_removed)
4044		return (B_TRUE);
4045
4046	for (int c = 0; c < vd->vdev_children; c++)
4047		if (vdev_log_state_valid(vd->vdev_child[c]))
4048			return (B_TRUE);
4049
4050	return (B_FALSE);
4051}
4052
4053/*
4054 * Expand a vdev if possible.
4055 */
4056void
4057vdev_expand(vdev_t *vd, uint64_t txg)
4058{
4059	ASSERT(vd->vdev_top == vd);
4060	ASSERT(spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER) == SCL_ALL);
4061
4062	vdev_set_deflate_ratio(vd);
4063
4064	if ((vd->vdev_asize >> vd->vdev_ms_shift) > vd->vdev_ms_count &&
4065	    vdev_is_concrete(vd)) {
4066		VERIFY(vdev_metaslab_init(vd, txg) == 0);
4067		vdev_config_dirty(vd);
4068	}
4069}
4070
4071/*
4072 * Split a vdev.
4073 */
4074void
4075vdev_split(vdev_t *vd)
4076{
4077	vdev_t *cvd, *pvd = vd->vdev_parent;
4078
4079	vdev_remove_child(pvd, vd);
4080	vdev_compact_children(pvd);
4081
4082	cvd = pvd->vdev_child[0];
4083	if (pvd->vdev_children == 1) {
4084		vdev_remove_parent(cvd);
4085		cvd->vdev_splitting = B_TRUE;
4086	}
4087	vdev_propagate_state(cvd);
4088}
4089
4090void
4091vdev_deadman(vdev_t *vd)
4092{
4093	for (int c = 0; c < vd->vdev_children; c++) {
4094		vdev_t *cvd = vd->vdev_child[c];
4095
4096		vdev_deadman(cvd);
4097	}
4098
4099	if (vd->vdev_ops->vdev_op_leaf) {
4100		vdev_queue_t *vq = &vd->vdev_queue;
4101
4102		mutex_enter(&vq->vq_lock);
4103		if (avl_numnodes(&vq->vq_active_tree) > 0) {
4104			spa_t *spa = vd->vdev_spa;
4105			zio_t *fio;
4106			uint64_t delta;
4107
4108			/*
4109			 * Look at the head of all the pending queues,
4110			 * if any I/O has been outstanding for longer than
4111			 * the spa_deadman_synctime we panic the system.
4112			 */
4113			fio = avl_first(&vq->vq_active_tree);
4114			delta = gethrtime() - fio->io_timestamp;
4115			if (delta > spa_deadman_synctime(spa)) {
4116				vdev_dbgmsg(vd, "SLOW IO: zio timestamp "
4117				    "%lluns, delta %lluns, last io %lluns",
4118				    fio->io_timestamp, (u_longlong_t)delta,
4119				    vq->vq_io_complete_ts);
4120				fm_panic("I/O to pool '%s' appears to be "
4121				    "hung.", spa_name(spa));
4122			}
4123		}
4124		mutex_exit(&vq->vq_lock);
4125	}
4126}
4127