1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
24  * Copyright 2020 Joyent, Inc.
25  */
26 
27 #include <sys/zfs_context.h>
28 #include <sys/time.h>
29 #include <sys/wait.h>
30 #include <sys/zio.h>
31 #include <umem.h>
32 #include <sys/vdev_raidz.h>
33 #include <sys/vdev_raidz_impl.h>
34 #include <assert.h>
35 #include <stdio.h>
36 #include <strings.h>
37 #include <unistd.h>
38 #include "raidz_test.h"
39 
40 static int *rand_data;
41 raidz_test_opts_t rto_opts;
42 
43 static char gdb[256];
44 static const char gdb_tmpl[] = "gdb -ex \"set pagination 0\" -p %d";
45 
46 #define	boot_ncpus	(sysconf(_SC_NPROCESSORS_ONLN))
47 
48 static void print_opts(raidz_test_opts_t *opts, boolean_t force)
49 {
50 	char *verbose;
51 	switch (opts->rto_v) {
52 		case 0:
53 			verbose = "no";
54 			break;
55 		case 1:
56 			verbose = "info";
57 			break;
58 		default:
59 			verbose = "debug";
60 			break;
61 	}
62 
63 	if (force || opts->rto_v >= D_INFO) {
64 		(void) fprintf(stdout, DBLSEP "Running with options:\n"
65 		    "  (-a) zio ashift                   : %zu\n"
66 		    "  (-o) zio offset                   : 1 << %zu\n"
67 		    "  (-d) number of raidz data columns : %zu\n"
68 		    "  (-s) size of DATA                 : 1 << %zu\n"
69 		    "  (-S) sweep parameters             : %s \n"
70 		    "  (-v) verbose                      : %s \n\n",
71 		    opts->rto_ashift,			/* -a */
72 		    ilog2(opts->rto_offset),		/* -o */
73 		    opts->rto_dcols,			/* -d */
74 		    ilog2(opts->rto_dsize),		/* -s */
75 		    opts->rto_sweep ? "yes" : "no",	/* -S */
76 		    verbose);				/* -v */
77 	}
78 }
79 
80 static void usage(boolean_t requested)
81 {
82 	const raidz_test_opts_t *o = &rto_opts_defaults;
83 
84 	FILE *fp = requested ? stdout : stderr;
85 
86 	(void) fprintf(fp, "Usage:\n"
87 	    "\t[-a zio ashift (default: %zu)]\n"
88 	    "\t[-o zio offset, exponent radix 2 (default: %zu)]\n"
89 	    "\t[-d number of raidz data columns (default: %zu)]\n"
90 	    "\t[-s zio size, exponent radix 2 (default: %zu)]\n"
91 	    "\t[-S parameter sweep (default: %s)]\n"
92 	    "\t[-t timeout for parameter sweep test]\n"
93 	    "\t[-B benchmark all raidz implementations]\n"
94 	    "\t[-v increase verbosity (default: %zu)]\n"
95 	    "\t[-h (print help)]\n"
96 	    "\t[-T test the test, see if failure would be detected]\n"
97 	    "\t[-D debug (attach gdb on SIGSEGV)]\n"
98 	    "",
99 	    o->rto_ashift,				/* -a */
100 	    ilog2(o->rto_offset),			/* -o */
101 	    o->rto_dcols,				/* -d */
102 	    ilog2(o->rto_dsize),			/* -s */
103 	    rto_opts.rto_sweep ? "yes" : "no",		/* -S */
104 	    o->rto_v);					/* -d */
105 
106 	exit(requested ? 0 : 1);
107 }
108 
109 static void process_options(int argc, char **argv)
110 {
111 	size_t value;
112 	int opt;
113 
114 	raidz_test_opts_t *o = &rto_opts;
115 
116 	bcopy(&rto_opts_defaults, o, sizeof (*o));
117 
118 	while ((opt = getopt(argc, argv, "TDBSvha:o:d:s:t:")) != -1) {
119 		value = 0;
120 
121 		switch (opt) {
122 		case 'a':
123 			value = strtoull(optarg, NULL, 0);
124 			o->rto_ashift = MIN(13, MAX(9, value));
125 			break;
126 		case 'o':
127 			value = strtoull(optarg, NULL, 0);
128 			o->rto_offset = ((1ULL << MIN(12, value)) >> 9) << 9;
129 			break;
130 		case 'd':
131 			value = strtoull(optarg, NULL, 0);
132 			o->rto_dcols = MIN(255, MAX(1, value));
133 			break;
134 		case 's':
135 			value = strtoull(optarg, NULL, 0);
136 			o->rto_dsize = 1ULL <<  MIN(SPA_MAXBLOCKSHIFT,
137 			    MAX(SPA_MINBLOCKSHIFT, value));
138 			break;
139 		case 't':
140 			value = strtoull(optarg, NULL, 0);
141 			o->rto_sweep_timeout = value;
142 			break;
143 		case 'v':
144 			o->rto_v++;
145 			break;
146 		case 'S':
147 			o->rto_sweep = 1;
148 			break;
149 		case 'B':
150 			o->rto_benchmark = 1;
151 			break;
152 		case 'D':
153 			o->rto_gdb = 1;
154 			break;
155 		case 'T':
156 			o->rto_sanity = 1;
157 			break;
158 		case 'h':
159 			usage(B_TRUE);
160 			break;
161 		case '?':
162 		default:
163 			usage(B_FALSE);
164 			break;
165 		}
166 	}
167 }
168 
169 #define	DATA_COL(rm, i) ((rm)->rm_col[raidz_parity(rm) + (i)].rc_abd)
170 #define	DATA_COL_SIZE(rm, i) ((rm)->rm_col[raidz_parity(rm) + (i)].rc_size)
171 
172 #define	CODE_COL(rm, i) ((rm)->rm_col[(i)].rc_abd)
173 #define	CODE_COL_SIZE(rm, i) ((rm)->rm_col[(i)].rc_size)
174 
175 static int
176 cmp_code(raidz_test_opts_t *opts, const raidz_map_t *rm, const int parity)
177 {
178 	int i, ret = 0;
179 
180 	VERIFY(parity >= 1 && parity <= 3);
181 
182 	for (i = 0; i < parity; i++) {
183 		if (abd_cmp(CODE_COL(rm, i), CODE_COL(opts->rm_golden, i),
184 		    CODE_COL(rm, i)->abd_size) != 0) {
185 			ret++;
186 			LOG_OPT(D_DEBUG, opts,
187 			    "\nParity block [%d] different!\n", i);
188 		}
189 	}
190 	return (ret);
191 }
192 
193 static int
194 cmp_data(raidz_test_opts_t *opts, raidz_map_t *rm)
195 {
196 	int i, ret = 0;
197 	int dcols = opts->rm_golden->rm_cols - raidz_parity(opts->rm_golden);
198 
199 	for (i = 0; i < dcols; i++) {
200 		if (abd_cmp(DATA_COL(opts->rm_golden, i), DATA_COL(rm, i),
201 		    DATA_COL(opts->rm_golden, i)->abd_size) != 0) {
202 			ret++;
203 
204 			LOG_OPT(D_DEBUG, opts,
205 			    "\nData block [%d] different!\n", i);
206 		}
207 	}
208 	return (ret);
209 }
210 
211 static int
212 init_rand(void *data, size_t size, void *private)
213 {
214 	int i;
215 	int *dst = (int *)data;
216 
217 	for (i = 0; i < size / sizeof (int); i++)
218 		dst[i] = rand_data[i];
219 
220 	return (0);
221 }
222 
223 static void
224 corrupt_colums(raidz_map_t *rm, const int *tgts, const int cnt)
225 {
226 	int i;
227 	raidz_col_t *col;
228 
229 	for (i = 0; i < cnt; i++) {
230 		col = &rm->rm_col[tgts[i]];
231 		(void) abd_iterate_func(col->rc_abd, 0, col->rc_size,
232 		    init_rand, NULL);
233 	}
234 }
235 
236 void
237 init_zio_abd(zio_t *zio)
238 {
239 	(void) abd_iterate_func(zio->io_abd, 0, zio->io_size, init_rand, NULL);
240 }
241 
242 static void
243 fini_raidz_map(zio_t **zio, raidz_map_t **rm)
244 {
245 	vdev_raidz_map_free(*rm);
246 	raidz_free((*zio)->io_abd, (*zio)->io_size);
247 	umem_free(*zio, sizeof (zio_t));
248 
249 	*zio = NULL;
250 	*rm = NULL;
251 }
252 
253 static int
254 init_raidz_golden_map(raidz_test_opts_t *opts, const int parity)
255 {
256 	int err = 0;
257 	zio_t *zio_test;
258 	raidz_map_t *rm_test;
259 	const size_t total_ncols = opts->rto_dcols + parity;
260 
261 	if (opts->rm_golden) {
262 		fini_raidz_map(&opts->zio_golden, &opts->rm_golden);
263 	}
264 
265 	opts->zio_golden = umem_zalloc(sizeof (zio_t), UMEM_NOFAIL);
266 	zio_test = umem_zalloc(sizeof (zio_t), UMEM_NOFAIL);
267 
268 	opts->zio_golden->io_offset = zio_test->io_offset = opts->rto_offset;
269 	opts->zio_golden->io_size = zio_test->io_size = opts->rto_dsize;
270 
271 	opts->zio_golden->io_abd = raidz_alloc(opts->rto_dsize);
272 	zio_test->io_abd = raidz_alloc(opts->rto_dsize);
273 
274 	init_zio_abd(opts->zio_golden);
275 	init_zio_abd(zio_test);
276 
277 	VERIFY0(vdev_raidz_impl_set("original"));
278 
279 	opts->rm_golden = vdev_raidz_map_alloc(opts->zio_golden,
280 	    opts->rto_ashift, total_ncols, parity);
281 	rm_test = vdev_raidz_map_alloc(zio_test,
282 	    opts->rto_ashift, total_ncols, parity);
283 
284 	VERIFY(opts->zio_golden);
285 	VERIFY(opts->rm_golden);
286 
287 	vdev_raidz_generate_parity(opts->rm_golden);
288 	vdev_raidz_generate_parity(rm_test);
289 
290 	/* sanity check */
291 	err |= cmp_data(opts, rm_test);
292 	err |= cmp_code(opts, rm_test, parity);
293 
294 	if (err)
295 		ERRMSG("initializing the golden copy ... [FAIL]!\n");
296 
297 	/* tear down raidz_map of test zio */
298 	fini_raidz_map(&zio_test, &rm_test);
299 
300 	return (err);
301 }
302 
303 static raidz_map_t *
304 init_raidz_map(raidz_test_opts_t *opts, zio_t **zio, const int parity)
305 {
306 	raidz_map_t *rm = NULL;
307 	const size_t alloc_dsize = opts->rto_dsize;
308 	const size_t total_ncols = opts->rto_dcols + parity;
309 	const int ccols[] = { 0, 1, 2 };
310 
311 	VERIFY(zio);
312 	VERIFY(parity <= 3 && parity >= 1);
313 
314 	*zio = umem_zalloc(sizeof (zio_t), UMEM_NOFAIL);
315 
316 	(*zio)->io_offset = 0;
317 	(*zio)->io_size = alloc_dsize;
318 	(*zio)->io_abd = raidz_alloc(alloc_dsize);
319 	init_zio_abd(*zio);
320 
321 	rm = vdev_raidz_map_alloc(*zio, opts->rto_ashift,
322 	    total_ncols, parity);
323 	VERIFY(rm);
324 
325 	/* Make sure code columns are destroyed */
326 	corrupt_colums(rm, ccols, parity);
327 
328 	return (rm);
329 }
330 
331 static int
332 run_gen_check(raidz_test_opts_t *opts)
333 {
334 	char **impl_name;
335 	int fn, err = 0;
336 	zio_t *zio_test;
337 	raidz_map_t *rm_test;
338 
339 	err = init_raidz_golden_map(opts, PARITY_PQR);
340 	if (0 != err)
341 		return (err);
342 
343 	LOG(D_INFO, DBLSEP);
344 	LOG(D_INFO, "Testing parity generation...\n");
345 
346 	for (impl_name = (char **)raidz_impl_names+1; *impl_name != NULL;
347 	    impl_name++) {
348 
349 		LOG(D_INFO, SEP);
350 		LOG(D_INFO, "\tTesting [%s] implementation...", *impl_name);
351 
352 		if (0 != vdev_raidz_impl_set(*impl_name)) {
353 			LOG(D_INFO, "[SKIP]\n");
354 			continue;
355 		} else {
356 			LOG(D_INFO, "[SUPPORTED]\n");
357 		}
358 
359 		for (fn = 0; fn < RAIDZ_GEN_NUM; fn++) {
360 
361 			/* Check if should stop */
362 			if (rto_opts.rto_should_stop)
363 				return (err);
364 
365 			/* create suitable raidz_map */
366 			rm_test = init_raidz_map(opts, &zio_test, fn+1);
367 			VERIFY(rm_test);
368 
369 			LOG(D_INFO, "\t\tTesting method [%s] ...",
370 			    raidz_gen_name[fn]);
371 
372 			if (!opts->rto_sanity)
373 				vdev_raidz_generate_parity(rm_test);
374 
375 			if (cmp_code(opts, rm_test, fn+1) != 0) {
376 				LOG(D_INFO, "[FAIL]\n");
377 				err++;
378 			} else
379 				LOG(D_INFO, "[PASS]\n");
380 
381 			fini_raidz_map(&zio_test, &rm_test);
382 		}
383 	}
384 
385 	fini_raidz_map(&opts->zio_golden, &opts->rm_golden);
386 
387 	return (err);
388 }
389 
390 static int
391 run_rec_check_impl(raidz_test_opts_t *opts, raidz_map_t *rm, const int fn)
392 {
393 	int x0, x1, x2;
394 	int tgtidx[3];
395 	int err = 0;
396 	static const int rec_tgts[7][3] = {
397 		{1, 2, 3},	/* rec_p:   bad QR & D[0]	*/
398 		{0, 2, 3},	/* rec_q:   bad PR & D[0]	*/
399 		{0, 1, 3},	/* rec_r:   bad PQ & D[0]	*/
400 		{2, 3, 4},	/* rec_pq:  bad R  & D[0][1]	*/
401 		{1, 3, 4},	/* rec_pr:  bad Q  & D[0][1]	*/
402 		{0, 3, 4},	/* rec_qr:  bad P  & D[0][1]	*/
403 		{3, 4, 5}	/* rec_pqr: bad    & D[0][1][2] */
404 	};
405 
406 	memcpy(tgtidx, rec_tgts[fn], sizeof (tgtidx));
407 
408 	if (fn < RAIDZ_REC_PQ) {
409 		/* can reconstruct 1 failed data disk */
410 		for (x0 = 0; x0 < opts->rto_dcols; x0++) {
411 			if (x0 >= rm->rm_cols - raidz_parity(rm))
412 				continue;
413 
414 			/* Check if should stop */
415 			if (rto_opts.rto_should_stop)
416 				return (err);
417 
418 			LOG(D_DEBUG, "[%d] ", x0);
419 
420 			tgtidx[2] = x0 + raidz_parity(rm);
421 
422 			corrupt_colums(rm, tgtidx+2, 1);
423 
424 			if (!opts->rto_sanity)
425 				(void) vdev_raidz_reconstruct(rm, tgtidx, 3);
426 
427 			if (cmp_data(opts, rm) != 0) {
428 				err++;
429 				LOG(D_DEBUG, "\nREC D[%d]... [FAIL]\n", x0);
430 			}
431 		}
432 
433 	} else if (fn < RAIDZ_REC_PQR) {
434 		/* can reconstruct 2 failed data disk */
435 		for (x0 = 0; x0 < opts->rto_dcols; x0++) {
436 			if (x0 >= rm->rm_cols - raidz_parity(rm))
437 				continue;
438 			for (x1 = x0 + 1; x1 < opts->rto_dcols; x1++) {
439 				if (x1 >= rm->rm_cols - raidz_parity(rm))
440 					continue;
441 
442 				/* Check if should stop */
443 				if (rto_opts.rto_should_stop)
444 					return (err);
445 
446 				LOG(D_DEBUG, "[%d %d] ", x0, x1);
447 
448 				tgtidx[1] = x0 + raidz_parity(rm);
449 				tgtidx[2] = x1 + raidz_parity(rm);
450 
451 				corrupt_colums(rm, tgtidx+1, 2);
452 
453 				if (!opts->rto_sanity)
454 					(void) vdev_raidz_reconstruct(rm,
455 					    tgtidx, 3);
456 
457 				if (cmp_data(opts, rm) != 0) {
458 					err++;
459 					LOG(D_DEBUG, "\nREC D[%d %d]... "
460 					    "[FAIL]\n", x0, x1);
461 				}
462 			}
463 		}
464 	} else {
465 		/* can reconstruct 3 failed data disk */
466 		for (x0 = 0; x0 < opts->rto_dcols; x0++) {
467 			if (x0 >= rm->rm_cols - raidz_parity(rm))
468 				continue;
469 			for (x1 = x0 + 1; x1 < opts->rto_dcols; x1++) {
470 				if (x1 >= rm->rm_cols - raidz_parity(rm))
471 					continue;
472 				for (x2 = x1 + 1; x2 < opts->rto_dcols; x2++) {
473 					if (x2 >=
474 					    rm->rm_cols - raidz_parity(rm))
475 						continue;
476 
477 					/* Check if should stop */
478 					if (rto_opts.rto_should_stop)
479 						return (err);
480 
481 					LOG(D_DEBUG, "[%d %d %d]", x0, x1, x2);
482 
483 					tgtidx[0] = x0 + raidz_parity(rm);
484 					tgtidx[1] = x1 + raidz_parity(rm);
485 					tgtidx[2] = x2 + raidz_parity(rm);
486 
487 					corrupt_colums(rm, tgtidx, 3);
488 
489 					if (!opts->rto_sanity)
490 						(void) vdev_raidz_reconstruct(
491 						    rm, tgtidx, 3);
492 
493 					if (cmp_data(opts, rm) != 0) {
494 						err++;
495 						LOG(D_DEBUG,
496 						    "\nREC D[%d %d %d]... "
497 						    "[FAIL]\n", x0, x1, x2);
498 					}
499 				}
500 			}
501 		}
502 	}
503 	return (err);
504 }
505 
506 static int
507 run_rec_check(raidz_test_opts_t *opts)
508 {
509 	char **impl_name;
510 	unsigned fn, err = 0;
511 	zio_t *zio_test;
512 	raidz_map_t *rm_test;
513 
514 	err = init_raidz_golden_map(opts, PARITY_PQR);
515 	if (0 != err)
516 		return (err);
517 
518 	LOG(D_INFO, DBLSEP);
519 	LOG(D_INFO, "Testing data reconstruction...\n");
520 
521 	for (impl_name = (char **)raidz_impl_names+1; *impl_name != NULL;
522 	    impl_name++) {
523 
524 		LOG(D_INFO, SEP);
525 		LOG(D_INFO, "\tTesting [%s] implementation...", *impl_name);
526 
527 		if (vdev_raidz_impl_set(*impl_name) != 0) {
528 			LOG(D_INFO, "[SKIP]\n");
529 			continue;
530 		} else
531 			LOG(D_INFO, "[SUPPORTED]\n");
532 
533 
534 		/* create suitable raidz_map */
535 		rm_test = init_raidz_map(opts, &zio_test, PARITY_PQR);
536 		/* generate parity */
537 		vdev_raidz_generate_parity(rm_test);
538 
539 		for (fn = 0; fn < RAIDZ_REC_NUM; fn++) {
540 
541 			LOG(D_INFO, "\t\tTesting method [%s] ...",
542 			    raidz_rec_name[fn]);
543 
544 			if (run_rec_check_impl(opts, rm_test, fn) != 0) {
545 				LOG(D_INFO, "[FAIL]\n");
546 				err++;
547 
548 			} else
549 				LOG(D_INFO, "[PASS]\n");
550 
551 		}
552 		/* tear down test raidz_map */
553 		fini_raidz_map(&zio_test, &rm_test);
554 	}
555 
556 	fini_raidz_map(&opts->zio_golden, &opts->rm_golden);
557 
558 	return (err);
559 }
560 
561 static int
562 run_test(raidz_test_opts_t *opts)
563 {
564 	int err = 0;
565 
566 	if (opts == NULL)
567 		opts = &rto_opts;
568 
569 	print_opts(opts, B_FALSE);
570 
571 	err |= run_gen_check(opts);
572 	err |= run_rec_check(opts);
573 
574 	return (err);
575 }
576 
577 #define	SWEEP_RUNNING	0
578 #define	SWEEP_FINISHED	1
579 #define	SWEEP_ERROR	2
580 #define	SWEEP_TIMEOUT	3
581 
582 static int sweep_state = 0;
583 static raidz_test_opts_t failed_opts;
584 
585 static kmutex_t sem_mtx;
586 static kcondvar_t sem_cv;
587 static int max_free_slots;
588 static int free_slots;
589 
590 static void
591 sweep_thread(void *arg)
592 {
593 	int err = 0;
594 	raidz_test_opts_t *opts = (raidz_test_opts_t *)arg;
595 	VERIFY(opts != NULL);
596 
597 	err = run_test(opts);
598 
599 	if (rto_opts.rto_sanity) {
600 		/* 25% chance that a sweep test fails */
601 		if (rand() < (RAND_MAX/4))
602 			err = 1;
603 	}
604 
605 	if (0 != err) {
606 		mutex_enter(&sem_mtx);
607 		memcpy(&failed_opts, opts, sizeof (raidz_test_opts_t));
608 		sweep_state = SWEEP_ERROR;
609 		mutex_exit(&sem_mtx);
610 	}
611 
612 	umem_free(opts, sizeof (raidz_test_opts_t));
613 
614 	/* signal the next thread */
615 	mutex_enter(&sem_mtx);
616 	free_slots++;
617 	cv_signal(&sem_cv);
618 	mutex_exit(&sem_mtx);
619 
620 	thread_exit();
621 }
622 
623 static int
624 run_sweep(void)
625 {
626 	static const size_t dcols_v[] = { 1, 2, 3, 4, 5, 6, 7, 8, 12, 15, 16 };
627 	static const size_t ashift_v[] = { 9, 12, 14 };
628 	static const size_t size_v[] = { 1 << 9, 21 * (1 << 9), 13 * (1 << 12),
629 		1 << 17, (1 << 20) - (1 << 12), SPA_MAXBLOCKSIZE };
630 
631 	(void) setvbuf(stdout, NULL, _IONBF, 0);
632 
633 	ulong_t total_comb = ARRAY_SIZE(size_v) * ARRAY_SIZE(ashift_v) *
634 	    ARRAY_SIZE(dcols_v);
635 	ulong_t tried_comb = 0;
636 	hrtime_t time_diff, start_time = gethrtime();
637 	raidz_test_opts_t *opts;
638 	int a, d, s;
639 
640 	max_free_slots = free_slots = MAX(2, boot_ncpus);
641 
642 	mutex_init(&sem_mtx, NULL, MUTEX_DEFAULT, NULL);
643 	cv_init(&sem_cv, NULL, CV_DEFAULT, NULL);
644 
645 	for (s = 0; s < ARRAY_SIZE(size_v); s++)
646 	for (a = 0; a < ARRAY_SIZE(ashift_v); a++)
647 	for (d = 0; d < ARRAY_SIZE(dcols_v); d++) {
648 
649 		if (size_v[s] < (1 << ashift_v[a])) {
650 			total_comb--;
651 			continue;
652 		}
653 
654 		if (++tried_comb % 20 == 0)
655 			LOG(D_ALL, "%lu/%lu... ", tried_comb, total_comb);
656 
657 		/* wait for signal to start new thread */
658 		mutex_enter(&sem_mtx);
659 		while (cv_timedwait_sig(&sem_cv, &sem_mtx,
660 		    ddi_get_lbolt() + hz)) {
661 
662 			/* check if should stop the test (timeout) */
663 			time_diff = (gethrtime() - start_time) / NANOSEC;
664 			if (rto_opts.rto_sweep_timeout > 0 &&
665 			    time_diff >= rto_opts.rto_sweep_timeout) {
666 				sweep_state = SWEEP_TIMEOUT;
667 				rto_opts.rto_should_stop = B_TRUE;
668 				mutex_exit(&sem_mtx);
669 				goto exit;
670 			}
671 
672 			/* check if should stop the test (error) */
673 			if (sweep_state != SWEEP_RUNNING) {
674 				mutex_exit(&sem_mtx);
675 				goto exit;
676 			}
677 
678 			/* exit loop if a slot is available */
679 			if (free_slots > 0) {
680 				break;
681 			}
682 		}
683 
684 		free_slots--;
685 		mutex_exit(&sem_mtx);
686 
687 		opts = umem_zalloc(sizeof (raidz_test_opts_t), UMEM_NOFAIL);
688 		opts->rto_ashift = ashift_v[a];
689 		opts->rto_dcols = dcols_v[d];
690 		opts->rto_offset = (1 << ashift_v[a]) * rand();
691 		opts->rto_dsize = size_v[s];
692 		opts->rto_v = 0; /* be quiet */
693 
694 		VERIFY3P(thread_create(NULL, 0, sweep_thread, (void *) opts,
695 		    0, NULL, TS_RUN, maxclsyspri), !=, NULL);
696 	}
697 
698 exit:
699 	LOG(D_ALL, "\nWaiting for test threads to finish...\n");
700 	mutex_enter(&sem_mtx);
701 	VERIFY(free_slots <= max_free_slots);
702 	while (free_slots < max_free_slots) {
703 		(void) cv_wait(&sem_cv, &sem_mtx);
704 	}
705 	mutex_exit(&sem_mtx);
706 
707 	if (sweep_state == SWEEP_ERROR) {
708 		ERRMSG("Sweep test failed! Failed option: \n");
709 		print_opts(&failed_opts, B_TRUE);
710 	} else {
711 		if (sweep_state == SWEEP_TIMEOUT)
712 			LOG(D_ALL, "Test timeout (%lus). Stopping...\n",
713 			    (ulong_t)rto_opts.rto_sweep_timeout);
714 
715 		LOG(D_ALL, "Sweep test succeeded on %lu raidz maps!\n",
716 		    (ulong_t)tried_comb);
717 	}
718 
719 	mutex_destroy(&sem_mtx);
720 
721 	return (sweep_state == SWEEP_ERROR ? SWEEP_ERROR : 0);
722 }
723 
724 int
725 main(int argc, char **argv)
726 {
727 	size_t i;
728 	int err = 0;
729 
730 	/* init gdb string early */
731 	(void) sprintf(gdb, gdb_tmpl, getpid());
732 
733 	(void) setvbuf(stdout, NULL, _IOLBF, 0);
734 
735 	dprintf_setup(&argc, argv);
736 
737 	process_options(argc, argv);
738 
739 	kernel_init(FREAD);
740 
741 	/* setup random data because rand() is not reentrant */
742 	rand_data = (int *)umem_alloc(SPA_MAXBLOCKSIZE, UMEM_NOFAIL);
743 	srand((unsigned)time(NULL) * getpid());
744 	for (i = 0; i < SPA_MAXBLOCKSIZE / sizeof (int); i++)
745 		rand_data[i] = rand();
746 
747 	mprotect((void *)rand_data, SPA_MAXBLOCKSIZE, PROT_READ);
748 
749 	if (rto_opts.rto_benchmark) {
750 		run_raidz_benchmark();
751 	} else if (rto_opts.rto_sweep) {
752 		err = run_sweep();
753 	} else {
754 		err = run_test(NULL);
755 	}
756 
757 	umem_free(rand_data, SPA_MAXBLOCKSIZE);
758 	kernel_fini();
759 
760 	return (err);
761 }
762