1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
28  * Copyright 2019 Joyent, Inc.
29  */
30 
31 #include <sys/zfs_context.h>
32 #include <sys/spa.h>
33 #include <sys/spa_impl.h>
34 #include <sys/dsl_pool.h>
35 #include <sys/dsl_scan.h>
36 #include <sys/vdev_impl.h>
37 #include <sys/zio.h>
38 #include <sys/abd.h>
39 #include <sys/fs/zfs.h>
40 
41 /*
42  * Vdev mirror kstats
43  */
44 static kstat_t *mirror_ksp = NULL;
45 
46 typedef struct mirror_stats {
47 	kstat_named_t vdev_mirror_stat_rotating_linear;
48 	kstat_named_t vdev_mirror_stat_rotating_offset;
49 	kstat_named_t vdev_mirror_stat_rotating_seek;
50 	kstat_named_t vdev_mirror_stat_non_rotating_linear;
51 	kstat_named_t vdev_mirror_stat_non_rotating_seek;
52 
53 	kstat_named_t vdev_mirror_stat_preferred_found;
54 	kstat_named_t vdev_mirror_stat_preferred_not_found;
55 } mirror_stats_t;
56 
57 static mirror_stats_t mirror_stats = {
58 	/* New I/O follows directly the last I/O */
59 	{ "rotating_linear",			KSTAT_DATA_UINT64 },
60 	/* New I/O is within zfs_vdev_mirror_rotating_seek_offset of the last */
61 	{ "rotating_offset",			KSTAT_DATA_UINT64 },
62 	/* New I/O requires random seek */
63 	{ "rotating_seek",			KSTAT_DATA_UINT64 },
64 	/* New I/O follows directly the last I/O  (nonrot) */
65 	{ "non_rotating_linear",		KSTAT_DATA_UINT64 },
66 	/* New I/O requires random seek (nonrot) */
67 	{ "non_rotating_seek",			KSTAT_DATA_UINT64 },
68 	/* Preferred child vdev found */
69 	{ "preferred_found",			KSTAT_DATA_UINT64 },
70 	/* Preferred child vdev not found or equal load  */
71 	{ "preferred_not_found",		KSTAT_DATA_UINT64 },
72 
73 };
74 
75 #define	MIRROR_STAT(stat)		(mirror_stats.stat.value.ui64)
76 #define	MIRROR_INCR(stat, val)		atomic_add_64(&MIRROR_STAT(stat), val)
77 #define	MIRROR_BUMP(stat)		MIRROR_INCR(stat, 1)
78 
79 void
vdev_mirror_stat_init(void)80 vdev_mirror_stat_init(void)
81 {
82 	mirror_ksp = kstat_create("zfs", 0, "vdev_mirror_stats",
83 	    "misc", KSTAT_TYPE_NAMED,
84 	    sizeof (mirror_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
85 	if (mirror_ksp != NULL) {
86 		mirror_ksp->ks_data = &mirror_stats;
87 		kstat_install(mirror_ksp);
88 	}
89 }
90 
91 void
vdev_mirror_stat_fini(void)92 vdev_mirror_stat_fini(void)
93 {
94 	if (mirror_ksp != NULL) {
95 		kstat_delete(mirror_ksp);
96 		mirror_ksp = NULL;
97 	}
98 }
99 
100 /*
101  * Virtual device vector for mirroring.
102  */
103 
104 typedef struct mirror_child {
105 	vdev_t		*mc_vd;
106 	uint64_t	mc_offset;
107 	int		mc_error;
108 	int		mc_load;
109 	uint8_t		mc_tried;
110 	uint8_t		mc_skipped;
111 	uint8_t		mc_speculative;
112 } mirror_child_t;
113 
114 typedef struct mirror_map {
115 	int		*mm_preferred;
116 	int		mm_preferred_cnt;
117 	int		mm_children;
118 	int		mm_resilvering;
119 	int		mm_root;
120 	mirror_child_t	mm_child[];
121 } mirror_map_t;
122 
123 int vdev_mirror_shift = 21;
124 
125 /*
126  * The load configuration settings below are tuned by default for
127  * the case where all devices are of the same rotational type.
128  *
129  * If there is a mixture of rotating and non-rotating media, setting
130  * zfs_vdev_mirror_non_rotating_seek_inc to 0 may well provide better results
131  * as it will direct more reads to the non-rotating vdevs which are more likely
132  * to have a higher performance.
133  */
134 
135 /* Rotating media load calculation configuration. */
136 static int zfs_vdev_mirror_rotating_inc = 0;
137 static int zfs_vdev_mirror_rotating_seek_inc = 5;
138 static int zfs_vdev_mirror_rotating_seek_offset = 1 * 1024 * 1024;
139 
140 /* Non-rotating media load calculation configuration. */
141 static int zfs_vdev_mirror_non_rotating_inc = 0;
142 static int zfs_vdev_mirror_non_rotating_seek_inc = 1;
143 
144 static inline size_t
vdev_mirror_map_size(int children)145 vdev_mirror_map_size(int children)
146 {
147 	return (offsetof(mirror_map_t, mm_child[children]) +
148 	    sizeof (int) * children);
149 }
150 
151 static inline mirror_map_t *
vdev_mirror_map_alloc(int children,boolean_t resilvering,boolean_t root)152 vdev_mirror_map_alloc(int children, boolean_t resilvering, boolean_t root)
153 {
154 	mirror_map_t *mm;
155 
156 	mm = kmem_zalloc(vdev_mirror_map_size(children), KM_SLEEP);
157 	mm->mm_children = children;
158 	mm->mm_resilvering = resilvering;
159 	mm->mm_root = root;
160 	mm->mm_preferred = (int *)((uintptr_t)mm +
161 	    offsetof(mirror_map_t, mm_child[children]));
162 
163 	return (mm);
164 }
165 
166 static void
vdev_mirror_map_free(zio_t * zio)167 vdev_mirror_map_free(zio_t *zio)
168 {
169 	mirror_map_t *mm = zio->io_vsd;
170 
171 	kmem_free(mm, vdev_mirror_map_size(mm->mm_children));
172 }
173 
174 static const zio_vsd_ops_t vdev_mirror_vsd_ops = {
175 	.vsd_free = vdev_mirror_map_free,
176 	.vsd_cksum_report = zio_vsd_default_cksum_report
177 };
178 
179 static int
vdev_mirror_load(mirror_map_t * mm,vdev_t * vd,uint64_t zio_offset)180 vdev_mirror_load(mirror_map_t *mm, vdev_t *vd, uint64_t zio_offset)
181 {
182 	uint64_t last_offset;
183 	int64_t offset_diff;
184 	int load;
185 
186 	/* All DVAs have equal weight at the root. */
187 	if (mm->mm_root)
188 		return (INT_MAX);
189 
190 	/*
191 	 * We don't return INT_MAX if the device is resilvering i.e.
192 	 * vdev_resilver_txg != 0 as when tested performance was slightly
193 	 * worse overall when resilvering with compared to without.
194 	 */
195 
196 	/* Fix zio_offset for leaf vdevs */
197 	if (vd->vdev_ops->vdev_op_leaf)
198 		zio_offset += VDEV_LABEL_START_SIZE;
199 
200 	/* Standard load based on pending queue length. */
201 	load = vdev_queue_length(vd);
202 	last_offset = vdev_queue_last_offset(vd);
203 
204 	if (vd->vdev_nonrot) {
205 		/* Non-rotating media. */
206 		if (last_offset == zio_offset) {
207 			MIRROR_BUMP(vdev_mirror_stat_non_rotating_linear);
208 			return (load + zfs_vdev_mirror_non_rotating_inc);
209 		}
210 
211 		/*
212 		 * Apply a seek penalty even for non-rotating devices as
213 		 * sequential I/O's can be aggregated into fewer operations on
214 		 * the device, thus avoiding unnecessary per-command overhead
215 		 * and boosting performance.
216 		 */
217 		MIRROR_BUMP(vdev_mirror_stat_non_rotating_seek);
218 		return (load + zfs_vdev_mirror_non_rotating_seek_inc);
219 	}
220 
221 	/* Rotating media I/O's which directly follow the last I/O. */
222 	if (last_offset == zio_offset) {
223 		MIRROR_BUMP(vdev_mirror_stat_rotating_linear);
224 		return (load + zfs_vdev_mirror_rotating_inc);
225 	}
226 
227 	/*
228 	 * Apply half the seek increment to I/O's within seek offset
229 	 * of the last I/O issued to this vdev as they should incur less
230 	 * of a seek increment.
231 	 */
232 	offset_diff = (int64_t)(last_offset - zio_offset);
233 	if (ABS(offset_diff) < zfs_vdev_mirror_rotating_seek_offset) {
234 		MIRROR_BUMP(vdev_mirror_stat_rotating_offset);
235 		return (load + (zfs_vdev_mirror_rotating_seek_inc / 2));
236 	}
237 
238 	/* Apply the full seek increment to all other I/O's. */
239 	MIRROR_BUMP(vdev_mirror_stat_rotating_seek);
240 	return (load + zfs_vdev_mirror_rotating_seek_inc);
241 }
242 
243 static mirror_map_t *
vdev_mirror_map_init(zio_t * zio)244 vdev_mirror_map_init(zio_t *zio)
245 {
246 	mirror_map_t *mm = NULL;
247 	mirror_child_t *mc;
248 	vdev_t *vd = zio->io_vd;
249 	int c;
250 
251 	if (vd == NULL) {
252 		dva_t *dva = zio->io_bp->blk_dva;
253 		spa_t *spa = zio->io_spa;
254 		dsl_scan_t *scn = NULL;
255 		dva_t dva_copy[SPA_DVAS_PER_BP];
256 
257 		if (spa->spa_dsl_pool != NULL) {
258 			scn = spa->spa_dsl_pool->dp_scan;
259 		}
260 		/*
261 		 * The sequential scrub code sorts and issues all DVAs
262 		 * of a bp separately. Each of these IOs includes all
263 		 * original DVA copies so that repairs can be performed
264 		 * in the event of an error, but we only actually want
265 		 * to check the first DVA since the others will be
266 		 * checked by their respective sorted IOs. Only if we
267 		 * hit an error will we try all DVAs upon retrying.
268 		 *
269 		 * Note: This check is safe even if the user switches
270 		 * from a legacy scrub to a sequential one in the middle
271 		 * of processing, since scn_is_sorted isn't updated until
272 		 * all outstanding IOs from the previous scrub pass
273 		 * complete.
274 		 */
275 		if ((zio->io_flags & ZIO_FLAG_SCRUB) &&
276 		    !(zio->io_flags & ZIO_FLAG_IO_RETRY) &&
277 		    scn != NULL &&
278 		    scn->scn_is_sorted &&
279 		    dsl_scan_scrubbing(spa->spa_dsl_pool)) {
280 			c = 1;
281 		} else {
282 			c = BP_GET_NDVAS(zio->io_bp);
283 		}
284 
285 		/*
286 		 * If we do not trust the pool config, some DVAs might be
287 		 * invalid or point to vdevs that do not exist. We skip them.
288 		 */
289 		if (!spa_trust_config(spa)) {
290 			ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ);
291 			int j = 0;
292 			for (int i = 0; i < c; i++) {
293 				if (zfs_dva_valid(spa, &dva[i], zio->io_bp))
294 					dva_copy[j++] = dva[i];
295 			}
296 			if (j == 0) {
297 				zio->io_vsd = NULL;
298 				zio->io_error = ENXIO;
299 				return (NULL);
300 			}
301 			if (j < c) {
302 				dva = dva_copy;
303 				c = j;
304 			}
305 		}
306 
307 		mm = vdev_mirror_map_alloc(c, B_FALSE, B_TRUE);
308 		for (c = 0; c < mm->mm_children; c++) {
309 			mc = &mm->mm_child[c];
310 
311 			mc->mc_vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[c]));
312 			mc->mc_offset = DVA_GET_OFFSET(&dva[c]);
313 		}
314 	} else {
315 		/*
316 		 * If we are resilvering, then we should handle scrub reads
317 		 * differently; we shouldn't issue them to the resilvering
318 		 * device because it might not have those blocks.
319 		 *
320 		 * We are resilvering iff:
321 		 * 1) We are a replacing vdev (ie our name is "replacing-1" or
322 		 *    "spare-1" or something like that), and
323 		 * 2) The pool is currently being resilvered.
324 		 *
325 		 * We cannot simply check vd->vdev_resilver_txg, because it's
326 		 * not set in this path.
327 		 *
328 		 * Nor can we just check our vdev_ops; there are cases (such as
329 		 * when a user types "zpool replace pool odev spare_dev" and
330 		 * spare_dev is in the spare list, or when a spare device is
331 		 * automatically used to replace a DEGRADED device) when
332 		 * resilvering is complete but both the original vdev and the
333 		 * spare vdev remain in the pool.  That behavior is intentional.
334 		 * It helps implement the policy that a spare should be
335 		 * automatically removed from the pool after the user replaces
336 		 * the device that originally failed.
337 		 */
338 		boolean_t replacing = (vd->vdev_ops == &vdev_replacing_ops ||
339 		    vd->vdev_ops == &vdev_spare_ops) &&
340 		    spa_load_state(vd->vdev_spa) == SPA_LOAD_NONE &&
341 		    dsl_scan_resilvering(vd->vdev_spa->spa_dsl_pool);
342 		mm = vdev_mirror_map_alloc(vd->vdev_children, replacing,
343 		    B_FALSE);
344 		for (c = 0; c < mm->mm_children; c++) {
345 			mc = &mm->mm_child[c];
346 			mc->mc_vd = vd->vdev_child[c];
347 			mc->mc_offset = zio->io_offset;
348 		}
349 	}
350 
351 	zio->io_vsd = mm;
352 	zio->io_vsd_ops = &vdev_mirror_vsd_ops;
353 	return (mm);
354 }
355 
356 static int
vdev_mirror_open(vdev_t * vd,uint64_t * asize,uint64_t * max_asize,uint64_t * ashift)357 vdev_mirror_open(vdev_t *vd, uint64_t *asize, uint64_t *max_asize,
358     uint64_t *ashift)
359 {
360 	int numerrors = 0;
361 	int lasterror = 0;
362 
363 	if (vd->vdev_children == 0) {
364 		vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
365 		return (SET_ERROR(EINVAL));
366 	}
367 
368 	vdev_open_children(vd);
369 
370 	for (int c = 0; c < vd->vdev_children; c++) {
371 		vdev_t *cvd = vd->vdev_child[c];
372 
373 		if (cvd->vdev_open_error) {
374 			lasterror = cvd->vdev_open_error;
375 			numerrors++;
376 			continue;
377 		}
378 
379 		*asize = MIN(*asize - 1, cvd->vdev_asize - 1) + 1;
380 		*max_asize = MIN(*max_asize - 1, cvd->vdev_max_asize - 1) + 1;
381 		*ashift = MAX(*ashift, cvd->vdev_ashift);
382 	}
383 
384 	if (numerrors == vd->vdev_children) {
385 		if (vdev_children_are_offline(vd))
386 			vd->vdev_stat.vs_aux = VDEV_AUX_CHILDREN_OFFLINE;
387 		else
388 			vd->vdev_stat.vs_aux = VDEV_AUX_NO_REPLICAS;
389 		return (lasterror);
390 	}
391 
392 	return (0);
393 }
394 
395 static void
vdev_mirror_close(vdev_t * vd)396 vdev_mirror_close(vdev_t *vd)
397 {
398 	for (int c = 0; c < vd->vdev_children; c++)
399 		vdev_close(vd->vdev_child[c]);
400 }
401 
402 static void
vdev_mirror_child_done(zio_t * zio)403 vdev_mirror_child_done(zio_t *zio)
404 {
405 	mirror_child_t *mc = zio->io_private;
406 
407 	mc->mc_error = zio->io_error;
408 	mc->mc_tried = 1;
409 	mc->mc_skipped = 0;
410 }
411 
412 static void
vdev_mirror_scrub_done(zio_t * zio)413 vdev_mirror_scrub_done(zio_t *zio)
414 {
415 	mirror_child_t *mc = zio->io_private;
416 
417 	if (zio->io_error == 0) {
418 		zio_t *pio;
419 		zio_link_t *zl = NULL;
420 
421 		mutex_enter(&zio->io_lock);
422 		while ((pio = zio_walk_parents(zio, &zl)) != NULL) {
423 			mutex_enter(&pio->io_lock);
424 			ASSERT3U(zio->io_size, >=, pio->io_size);
425 			abd_copy(pio->io_abd, zio->io_abd, pio->io_size);
426 			mutex_exit(&pio->io_lock);
427 		}
428 		mutex_exit(&zio->io_lock);
429 	}
430 
431 	abd_free(zio->io_abd);
432 
433 	mc->mc_error = zio->io_error;
434 	mc->mc_tried = 1;
435 	mc->mc_skipped = 0;
436 }
437 
438 /*
439  * Check the other, lower-index DVAs to see if they're on the same
440  * vdev as the child we picked.  If they are, use them since they
441  * are likely to have been allocated from the primary metaslab in
442  * use at the time, and hence are more likely to have locality with
443  * single-copy data.
444  */
445 static int
vdev_mirror_dva_select(zio_t * zio,int p)446 vdev_mirror_dva_select(zio_t *zio, int p)
447 {
448 	dva_t *dva = zio->io_bp->blk_dva;
449 	mirror_map_t *mm = zio->io_vsd;
450 	int preferred;
451 	int c;
452 
453 	preferred = mm->mm_preferred[p];
454 	for (p--; p >= 0; p--) {
455 		c = mm->mm_preferred[p];
456 		if (DVA_GET_VDEV(&dva[c]) == DVA_GET_VDEV(&dva[preferred]))
457 			preferred = c;
458 	}
459 	return (preferred);
460 }
461 
462 static int
vdev_mirror_preferred_child_randomize(zio_t * zio)463 vdev_mirror_preferred_child_randomize(zio_t *zio)
464 {
465 	mirror_map_t *mm = zio->io_vsd;
466 	int p;
467 
468 	if (mm->mm_root) {
469 		p = spa_get_random(mm->mm_preferred_cnt);
470 		return (vdev_mirror_dva_select(zio, p));
471 	}
472 
473 	/*
474 	 * To ensure we don't always favour the first matching vdev,
475 	 * which could lead to wear leveling issues on SSD's, we
476 	 * use the I/O offset as a pseudo random seed into the vdevs
477 	 * which have the lowest load.
478 	 */
479 	p = (zio->io_offset >> vdev_mirror_shift) % mm->mm_preferred_cnt;
480 	return (mm->mm_preferred[p]);
481 }
482 
483 /*
484  * Try to find a vdev whose DTL doesn't contain the block we want to read
485  * prefering vdevs based on determined load.
486  *
487  * Try to find a child whose DTL doesn't contain the block we want to read.
488  * If we can't, try the read on any vdev we haven't already tried.
489  */
490 static int
vdev_mirror_child_select(zio_t * zio)491 vdev_mirror_child_select(zio_t *zio)
492 {
493 	mirror_map_t *mm = zio->io_vsd;
494 	uint64_t txg = zio->io_txg;
495 	int c, lowest_load;
496 
497 	ASSERT(zio->io_bp == NULL || BP_PHYSICAL_BIRTH(zio->io_bp) == txg);
498 
499 	lowest_load = INT_MAX;
500 	mm->mm_preferred_cnt = 0;
501 	for (c = 0; c < mm->mm_children; c++) {
502 		mirror_child_t *mc;
503 
504 		mc = &mm->mm_child[c];
505 		if (mc->mc_tried || mc->mc_skipped)
506 			continue;
507 
508 		if (mc->mc_vd == NULL || !vdev_readable(mc->mc_vd)) {
509 			mc->mc_error = SET_ERROR(ENXIO);
510 			mc->mc_tried = 1;	/* don't even try */
511 			mc->mc_skipped = 1;
512 			continue;
513 		}
514 
515 		if (vdev_dtl_contains(mc->mc_vd, DTL_MISSING, txg, 1)) {
516 			mc->mc_error = SET_ERROR(ESTALE);
517 			mc->mc_skipped = 1;
518 			mc->mc_speculative = 1;
519 			continue;
520 		}
521 
522 		mc->mc_load = vdev_mirror_load(mm, mc->mc_vd, mc->mc_offset);
523 		if (mc->mc_load > lowest_load)
524 			continue;
525 
526 		if (mc->mc_load < lowest_load) {
527 			lowest_load = mc->mc_load;
528 			mm->mm_preferred_cnt = 0;
529 		}
530 		mm->mm_preferred[mm->mm_preferred_cnt] = c;
531 		mm->mm_preferred_cnt++;
532 	}
533 
534 	if (mm->mm_preferred_cnt == 1) {
535 		MIRROR_BUMP(vdev_mirror_stat_preferred_found);
536 		return (mm->mm_preferred[0]);
537 	}
538 
539 	if (mm->mm_preferred_cnt > 1) {
540 		MIRROR_BUMP(vdev_mirror_stat_preferred_not_found);
541 		return (vdev_mirror_preferred_child_randomize(zio));
542 	}
543 
544 	/*
545 	 * Every device is either missing or has this txg in its DTL.
546 	 * Look for any child we haven't already tried before giving up.
547 	 */
548 	for (c = 0; c < mm->mm_children; c++) {
549 		if (!mm->mm_child[c].mc_tried)
550 			return (c);
551 	}
552 
553 	/*
554 	 * Every child failed.  There's no place left to look.
555 	 */
556 	return (-1);
557 }
558 
559 static void
vdev_mirror_io_start(zio_t * zio)560 vdev_mirror_io_start(zio_t *zio)
561 {
562 	mirror_map_t *mm;
563 	mirror_child_t *mc;
564 	int c, children;
565 
566 	mm = vdev_mirror_map_init(zio);
567 
568 	if (mm == NULL) {
569 		ASSERT(!spa_trust_config(zio->io_spa));
570 		ASSERT(zio->io_type == ZIO_TYPE_READ);
571 		zio_execute(zio);
572 		return;
573 	}
574 
575 	if (zio->io_type == ZIO_TYPE_READ) {
576 		if (zio->io_bp != NULL &&
577 		    (zio->io_flags & ZIO_FLAG_SCRUB) && !mm->mm_resilvering) {
578 			/*
579 			 * For scrubbing reads (if we can verify the
580 			 * checksum here, as indicated by io_bp being
581 			 * non-NULL) we need to allocate a read buffer for
582 			 * each child and issue reads to all children.  If
583 			 * any child succeeds, it will copy its data into
584 			 * zio->io_data in vdev_mirror_scrub_done.
585 			 */
586 			for (c = 0; c < mm->mm_children; c++) {
587 				mc = &mm->mm_child[c];
588 				zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
589 				    mc->mc_vd, mc->mc_offset,
590 				    abd_alloc_sametype(zio->io_abd,
591 				    zio->io_size), zio->io_size,
592 				    zio->io_type, zio->io_priority, 0,
593 				    vdev_mirror_scrub_done, mc));
594 			}
595 			zio_execute(zio);
596 			return;
597 		}
598 		/*
599 		 * For normal reads just pick one child.
600 		 */
601 		c = vdev_mirror_child_select(zio);
602 		children = (c >= 0);
603 	} else {
604 		ASSERT(zio->io_type == ZIO_TYPE_WRITE);
605 
606 		/*
607 		 * Writes go to all children.
608 		 */
609 		c = 0;
610 		children = mm->mm_children;
611 	}
612 
613 	while (children--) {
614 		mc = &mm->mm_child[c];
615 		zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
616 		    mc->mc_vd, mc->mc_offset, zio->io_abd, zio->io_size,
617 		    zio->io_type, zio->io_priority, 0,
618 		    vdev_mirror_child_done, mc));
619 		c++;
620 	}
621 
622 	zio_execute(zio);
623 }
624 
625 static int
vdev_mirror_worst_error(mirror_map_t * mm)626 vdev_mirror_worst_error(mirror_map_t *mm)
627 {
628 	int error[2] = { 0, 0 };
629 
630 	for (int c = 0; c < mm->mm_children; c++) {
631 		mirror_child_t *mc = &mm->mm_child[c];
632 		int s = mc->mc_speculative;
633 		error[s] = zio_worst_error(error[s], mc->mc_error);
634 	}
635 
636 	return (error[0] ? error[0] : error[1]);
637 }
638 
639 static void
vdev_mirror_io_done(zio_t * zio)640 vdev_mirror_io_done(zio_t *zio)
641 {
642 	mirror_map_t *mm = zio->io_vsd;
643 	mirror_child_t *mc;
644 	int c;
645 	int good_copies = 0;
646 	int unexpected_errors = 0;
647 
648 	if (mm == NULL)
649 		return;
650 
651 	for (c = 0; c < mm->mm_children; c++) {
652 		mc = &mm->mm_child[c];
653 
654 		if (mc->mc_error) {
655 			if (!mc->mc_skipped)
656 				unexpected_errors++;
657 		} else if (mc->mc_tried) {
658 			good_copies++;
659 		}
660 	}
661 
662 	if (zio->io_type == ZIO_TYPE_WRITE) {
663 		/*
664 		 * XXX -- for now, treat partial writes as success.
665 		 *
666 		 * Now that we support write reallocation, it would be better
667 		 * to treat partial failure as real failure unless there are
668 		 * no non-degraded top-level vdevs left, and not update DTLs
669 		 * if we intend to reallocate.
670 		 */
671 		/* XXPOLICY */
672 		if (good_copies != mm->mm_children) {
673 			/*
674 			 * Always require at least one good copy.
675 			 *
676 			 * For ditto blocks (io_vd == NULL), require
677 			 * all copies to be good.
678 			 *
679 			 * XXX -- for replacing vdevs, there's no great answer.
680 			 * If the old device is really dead, we may not even
681 			 * be able to access it -- so we only want to
682 			 * require good writes to the new device.  But if
683 			 * the new device turns out to be flaky, we want
684 			 * to be able to detach it -- which requires all
685 			 * writes to the old device to have succeeded.
686 			 */
687 			if (good_copies == 0 || zio->io_vd == NULL)
688 				zio->io_error = vdev_mirror_worst_error(mm);
689 		}
690 		return;
691 	}
692 
693 	ASSERT(zio->io_type == ZIO_TYPE_READ);
694 
695 	/*
696 	 * If we don't have a good copy yet, keep trying other children.
697 	 */
698 	/* XXPOLICY */
699 	if (good_copies == 0 && (c = vdev_mirror_child_select(zio)) != -1) {
700 		ASSERT(c >= 0 && c < mm->mm_children);
701 		mc = &mm->mm_child[c];
702 		zio_vdev_io_redone(zio);
703 		zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
704 		    mc->mc_vd, mc->mc_offset, zio->io_abd, zio->io_size,
705 		    ZIO_TYPE_READ, zio->io_priority, 0,
706 		    vdev_mirror_child_done, mc));
707 		return;
708 	}
709 
710 	/* XXPOLICY */
711 	if (good_copies == 0) {
712 		zio->io_error = vdev_mirror_worst_error(mm);
713 		ASSERT(zio->io_error != 0);
714 	}
715 
716 	if (good_copies && spa_writeable(zio->io_spa) &&
717 	    (unexpected_errors ||
718 	    (zio->io_flags & ZIO_FLAG_RESILVER) ||
719 	    ((zio->io_flags & ZIO_FLAG_SCRUB) && mm->mm_resilvering))) {
720 		/*
721 		 * Use the good data we have in hand to repair damaged children.
722 		 */
723 		for (c = 0; c < mm->mm_children; c++) {
724 			/*
725 			 * Don't rewrite known good children.
726 			 * Not only is it unnecessary, it could
727 			 * actually be harmful: if the system lost
728 			 * power while rewriting the only good copy,
729 			 * there would be no good copies left!
730 			 */
731 			mc = &mm->mm_child[c];
732 
733 			if (mc->mc_error == 0) {
734 				if (mc->mc_tried)
735 					continue;
736 				/*
737 				 * We didn't try this child.  We need to
738 				 * repair it if:
739 				 * 1. it's a scrub (in which case we have
740 				 * tried everything that was healthy)
741 				 *  - or -
742 				 * 2. it's an indirect vdev (in which case
743 				 * it could point to any other vdev, which
744 				 * might have a bad DTL)
745 				 *  - or -
746 				 * 3. the DTL indicates that this data is
747 				 * missing from this vdev
748 				 */
749 				if (!(zio->io_flags & ZIO_FLAG_SCRUB) &&
750 				    mc->mc_vd->vdev_ops != &vdev_indirect_ops &&
751 				    !vdev_dtl_contains(mc->mc_vd, DTL_PARTIAL,
752 				    zio->io_txg, 1))
753 					continue;
754 				mc->mc_error = SET_ERROR(ESTALE);
755 			}
756 
757 			zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
758 			    mc->mc_vd, mc->mc_offset,
759 			    zio->io_abd, zio->io_size,
760 			    ZIO_TYPE_WRITE, ZIO_PRIORITY_ASYNC_WRITE,
761 			    ZIO_FLAG_IO_REPAIR | (unexpected_errors ?
762 			    ZIO_FLAG_SELF_HEAL : 0), NULL, NULL));
763 		}
764 	}
765 }
766 
767 static void
vdev_mirror_state_change(vdev_t * vd,int faulted,int degraded)768 vdev_mirror_state_change(vdev_t *vd, int faulted, int degraded)
769 {
770 	if (faulted == vd->vdev_children) {
771 		if (vdev_children_are_offline(vd)) {
772 			vdev_set_state(vd, B_FALSE, VDEV_STATE_OFFLINE,
773 			    VDEV_AUX_CHILDREN_OFFLINE);
774 		} else {
775 			vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
776 			    VDEV_AUX_NO_REPLICAS);
777 		}
778 	} else if (degraded + faulted != 0) {
779 		vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, VDEV_AUX_NONE);
780 	} else {
781 		vdev_set_state(vd, B_FALSE, VDEV_STATE_HEALTHY, VDEV_AUX_NONE);
782 	}
783 }
784 
785 static int
vdev_mirror_dumpio(vdev_t * vd,caddr_t data,size_t size,uint64_t offset,uint64_t origoffset,boolean_t doread,boolean_t isdump)786 vdev_mirror_dumpio(vdev_t *vd, caddr_t data, size_t size,
787     uint64_t offset, uint64_t origoffset, boolean_t doread, boolean_t isdump)
788 {
789 	uint64_t numerrors;
790 	int err = EIO;
791 
792 	for (uint64_t c = 0; c < vd->vdev_children; c++) {
793 		vdev_t *cvd = vd->vdev_child[c];
794 
795 		if (cvd->vdev_ops->vdev_op_dumpio == NULL) {
796 			err = EINVAL;
797 		} else {
798 			err = cvd->vdev_ops->vdev_op_dumpio(cvd, data, size,
799 			    offset, origoffset, doread, isdump);
800 		}
801 		if (err != 0) {
802 			numerrors++;
803 		} else if (doread) {
804 			break;
805 		}
806 	}
807 	if (err != 0) {
808 		return (SET_ERROR(err));
809 	}
810 
811 	return (0);
812 }
813 
814 vdev_ops_t vdev_mirror_ops = {
815 	.vdev_op_open = vdev_mirror_open,
816 	.vdev_op_close = vdev_mirror_close,
817 	.vdev_op_asize = vdev_default_asize,
818 	.vdev_op_io_start = vdev_mirror_io_start,
819 	.vdev_op_io_done = vdev_mirror_io_done,
820 	.vdev_op_state_change = vdev_mirror_state_change,
821 	.vdev_op_need_resilver = NULL,
822 	.vdev_op_hold = NULL,
823 	.vdev_op_rele = NULL,
824 	.vdev_op_remap = NULL,
825 	.vdev_op_xlate = vdev_default_xlate,
826 	.vdev_op_dumpio = vdev_mirror_dumpio,
827 	.vdev_op_type = VDEV_TYPE_MIRROR,	/* name of this vdev type */
828 	.vdev_op_leaf = B_FALSE			/* not a leaf vdev */
829 };
830 
831 vdev_ops_t vdev_replacing_ops = {
832 	.vdev_op_open = vdev_mirror_open,
833 	.vdev_op_close = vdev_mirror_close,
834 	.vdev_op_asize = vdev_default_asize,
835 	.vdev_op_io_start = vdev_mirror_io_start,
836 	.vdev_op_io_done = vdev_mirror_io_done,
837 	.vdev_op_state_change = vdev_mirror_state_change,
838 	.vdev_op_need_resilver = NULL,
839 	.vdev_op_hold = NULL,
840 	.vdev_op_rele = NULL,
841 	.vdev_op_remap = NULL,
842 	.vdev_op_xlate = vdev_default_xlate,
843 	.vdev_op_dumpio = vdev_mirror_dumpio,
844 	.vdev_op_type = VDEV_TYPE_REPLACING,	/* name of this vdev type */
845 	.vdev_op_leaf = B_FALSE			/* not a leaf vdev */
846 };
847 
848 vdev_ops_t vdev_spare_ops = {
849 	.vdev_op_open = vdev_mirror_open,
850 	.vdev_op_close = vdev_mirror_close,
851 	.vdev_op_asize = vdev_default_asize,
852 	.vdev_op_io_start = vdev_mirror_io_start,
853 	.vdev_op_io_done = vdev_mirror_io_done,
854 	.vdev_op_state_change = vdev_mirror_state_change,
855 	.vdev_op_need_resilver = NULL,
856 	.vdev_op_hold = NULL,
857 	.vdev_op_rele = NULL,
858 	.vdev_op_remap = NULL,
859 	.vdev_op_xlate = vdev_default_xlate,
860 	.vdev_op_dumpio = vdev_mirror_dumpio,
861 	.vdev_op_type = VDEV_TYPE_SPARE,	/* name of this vdev type */
862 	.vdev_op_leaf = B_FALSE			/* not a leaf vdev */
863 };
864