xref: /illumos-gate/usr/src/cmd/ztest/ztest.c (revision eb633035c80613ec93d62f90482837adaaf21a0a)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
24  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
25  * Copyright (c) 2013 Steven Hartland. All rights reserved.
26  * Copyright (c) 2014 Integros [integros.com]
27  * Copyright 2017 Joyent, Inc.
28  * Copyright (c) 2017, Intel Corporation.
29  * Copyright 2017 RackTop Systems.
30  */
31 
32 /*
33  * The objective of this program is to provide a DMU/ZAP/SPA stress test
34  * that runs entirely in userland, is easy to use, and easy to extend.
35  *
36  * The overall design of the ztest program is as follows:
37  *
38  * (1) For each major functional area (e.g. adding vdevs to a pool,
39  *     creating and destroying datasets, reading and writing objects, etc)
40  *     we have a simple routine to test that functionality.  These
41  *     individual routines do not have to do anything "stressful".
42  *
43  * (2) We turn these simple functionality tests into a stress test by
44  *     running them all in parallel, with as many threads as desired,
45  *     and spread across as many datasets, objects, and vdevs as desired.
46  *
47  * (3) While all this is happening, we inject faults into the pool to
48  *     verify that self-healing data really works.
49  *
50  * (4) Every time we open a dataset, we change its checksum and compression
51  *     functions.  Thus even individual objects vary from block to block
52  *     in which checksum they use and whether they're compressed.
53  *
54  * (5) To verify that we never lose on-disk consistency after a crash,
55  *     we run the entire test in a child of the main process.
56  *     At random times, the child self-immolates with a SIGKILL.
57  *     This is the software equivalent of pulling the power cord.
58  *     The parent then runs the test again, using the existing
59  *     storage pool, as many times as desired. If backwards compatibility
60  *     testing is enabled ztest will sometimes run the "older" version
61  *     of ztest after a SIGKILL.
62  *
63  * (6) To verify that we don't have future leaks or temporal incursions,
64  *     many of the functional tests record the transaction group number
65  *     as part of their data.  When reading old data, they verify that
66  *     the transaction group number is less than the current, open txg.
67  *     If you add a new test, please do this if applicable.
68  *
69  * When run with no arguments, ztest runs for about five minutes and
70  * produces no output if successful.  To get a little bit of information,
71  * specify -V.  To get more information, specify -VV, and so on.
72  *
73  * To turn this into an overnight stress test, use -T to specify run time.
74  *
75  * You can ask more more vdevs [-v], datasets [-d], or threads [-t]
76  * to increase the pool capacity, fanout, and overall stress level.
77  *
78  * Use the -k option to set the desired frequency of kills.
79  *
80  * When ztest invokes itself it passes all relevant information through a
81  * temporary file which is mmap-ed in the child process. This allows shared
82  * memory to survive the exec syscall. The ztest_shared_hdr_t struct is always
83  * stored at offset 0 of this file and contains information on the size and
84  * number of shared structures in the file. The information stored in this file
85  * must remain backwards compatible with older versions of ztest so that
86  * ztest can invoke them during backwards compatibility testing (-B).
87  */
88 
89 #include <sys/zfs_context.h>
90 #include <sys/spa.h>
91 #include <sys/dmu.h>
92 #include <sys/txg.h>
93 #include <sys/dbuf.h>
94 #include <sys/zap.h>
95 #include <sys/dmu_objset.h>
96 #include <sys/poll.h>
97 #include <sys/stat.h>
98 #include <sys/time.h>
99 #include <sys/wait.h>
100 #include <sys/mman.h>
101 #include <sys/resource.h>
102 #include <sys/zio.h>
103 #include <sys/zil.h>
104 #include <sys/zil_impl.h>
105 #include <sys/vdev_impl.h>
106 #include <sys/vdev_file.h>
107 #include <sys/vdev_initialize.h>
108 #include <sys/spa_impl.h>
109 #include <sys/metaslab_impl.h>
110 #include <sys/dsl_prop.h>
111 #include <sys/dsl_dataset.h>
112 #include <sys/dsl_destroy.h>
113 #include <sys/dsl_scan.h>
114 #include <sys/zio_checksum.h>
115 #include <sys/refcount.h>
116 #include <sys/zfeature.h>
117 #include <sys/dsl_userhold.h>
118 #include <sys/abd.h>
119 #include <stdio.h>
120 #include <stdio_ext.h>
121 #include <stdlib.h>
122 #include <unistd.h>
123 #include <signal.h>
124 #include <umem.h>
125 #include <dlfcn.h>
126 #include <ctype.h>
127 #include <math.h>
128 #include <sys/fs/zfs.h>
129 #include <libnvpair.h>
130 #include <libzfs.h>
131 #include <libcmdutils.h>
132 
133 static int ztest_fd_data = -1;
134 static int ztest_fd_rand = -1;
135 
136 typedef struct ztest_shared_hdr {
137 	uint64_t	zh_hdr_size;
138 	uint64_t	zh_opts_size;
139 	uint64_t	zh_size;
140 	uint64_t	zh_stats_size;
141 	uint64_t	zh_stats_count;
142 	uint64_t	zh_ds_size;
143 	uint64_t	zh_ds_count;
144 } ztest_shared_hdr_t;
145 
146 static ztest_shared_hdr_t *ztest_shared_hdr;
147 
148 enum ztest_class_state {
149 	ZTEST_VDEV_CLASS_OFF,
150 	ZTEST_VDEV_CLASS_ON,
151 	ZTEST_VDEV_CLASS_RND
152 };
153 
154 typedef struct ztest_shared_opts {
155 	char zo_pool[ZFS_MAX_DATASET_NAME_LEN];
156 	char zo_dir[ZFS_MAX_DATASET_NAME_LEN];
157 	char zo_alt_ztest[MAXNAMELEN];
158 	char zo_alt_libpath[MAXNAMELEN];
159 	uint64_t zo_vdevs;
160 	uint64_t zo_vdevtime;
161 	size_t zo_vdev_size;
162 	int zo_ashift;
163 	int zo_mirrors;
164 	int zo_raidz;
165 	int zo_raidz_parity;
166 	int zo_datasets;
167 	int zo_threads;
168 	uint64_t zo_passtime;
169 	uint64_t zo_killrate;
170 	int zo_verbose;
171 	int zo_init;
172 	uint64_t zo_time;
173 	uint64_t zo_maxloops;
174 	uint64_t zo_metaslab_force_ganging;
175 	int zo_mmp_test;
176 	int zo_special_vdevs;
177 } ztest_shared_opts_t;
178 
179 static const ztest_shared_opts_t ztest_opts_defaults = {
180 	.zo_pool = { 'z', 't', 'e', 's', 't', '\0' },
181 	.zo_dir = { '/', 't', 'm', 'p', '\0' },
182 	.zo_alt_ztest = { '\0' },
183 	.zo_alt_libpath = { '\0' },
184 	.zo_vdevs = 5,
185 	.zo_ashift = SPA_MINBLOCKSHIFT,
186 	.zo_mirrors = 2,
187 	.zo_raidz = 4,
188 	.zo_raidz_parity = 1,
189 	.zo_vdev_size = SPA_MINDEVSIZE * 4,	/* 256m default size */
190 	.zo_datasets = 7,
191 	.zo_threads = 23,
192 	.zo_passtime = 60,		/* 60 seconds */
193 	.zo_killrate = 70,		/* 70% kill rate */
194 	.zo_verbose = 0,
195 	.zo_mmp_test = 0,
196 	.zo_init = 1,
197 	.zo_time = 300,			/* 5 minutes */
198 	.zo_maxloops = 50,		/* max loops during spa_freeze() */
199 	.zo_metaslab_force_ganging = 32 << 10,
200 	.zo_special_vdevs = ZTEST_VDEV_CLASS_RND,
201 };
202 
203 extern uint64_t metaslab_force_ganging;
204 extern uint64_t metaslab_df_alloc_threshold;
205 extern uint64_t zfs_deadman_synctime_ms;
206 extern int metaslab_preload_limit;
207 extern boolean_t zfs_compressed_arc_enabled;
208 extern boolean_t zfs_abd_scatter_enabled;
209 extern int dmu_object_alloc_chunk_shift;
210 extern boolean_t zfs_force_some_double_word_sm_entries;
211 extern unsigned long zfs_reconstruct_indirect_damage_fraction;
212 
213 static ztest_shared_opts_t *ztest_shared_opts;
214 static ztest_shared_opts_t ztest_opts;
215 static char *ztest_wkeydata = "abcdefghijklmnopqrstuvwxyz012345";
216 
217 typedef struct ztest_shared_ds {
218 	uint64_t	zd_seq;
219 } ztest_shared_ds_t;
220 
221 static ztest_shared_ds_t *ztest_shared_ds;
222 #define	ZTEST_GET_SHARED_DS(d) (&ztest_shared_ds[d])
223 
224 #define	BT_MAGIC	0x123456789abcdefULL
225 #define	MAXFAULTS() \
226 	(MAX(zs->zs_mirrors, 1) * (ztest_opts.zo_raidz_parity + 1) - 1)
227 
228 enum ztest_io_type {
229 	ZTEST_IO_WRITE_TAG,
230 	ZTEST_IO_WRITE_PATTERN,
231 	ZTEST_IO_WRITE_ZEROES,
232 	ZTEST_IO_TRUNCATE,
233 	ZTEST_IO_SETATTR,
234 	ZTEST_IO_REWRITE,
235 	ZTEST_IO_TYPES
236 };
237 
238 typedef struct ztest_block_tag {
239 	uint64_t	bt_magic;
240 	uint64_t	bt_objset;
241 	uint64_t	bt_object;
242 	uint64_t	bt_dnodesize;
243 	uint64_t	bt_offset;
244 	uint64_t	bt_gen;
245 	uint64_t	bt_txg;
246 	uint64_t	bt_crtxg;
247 } ztest_block_tag_t;
248 
249 typedef struct bufwad {
250 	uint64_t	bw_index;
251 	uint64_t	bw_txg;
252 	uint64_t	bw_data;
253 } bufwad_t;
254 
255 /*
256  * It would be better to use a rangelock_t per object.  Unfortunately
257  * the rangelock_t is not a drop-in replacement for rl_t, because we
258  * still need to map from object ID to rangelock_t.
259  */
260 typedef enum {
261 	RL_READER,
262 	RL_WRITER,
263 	RL_APPEND
264 } rl_type_t;
265 
266 typedef struct rll {
267 	void		*rll_writer;
268 	int		rll_readers;
269 	kmutex_t	rll_lock;
270 	kcondvar_t	rll_cv;
271 } rll_t;
272 
273 typedef struct rl {
274 	uint64_t	rl_object;
275 	uint64_t	rl_offset;
276 	uint64_t	rl_size;
277 	rll_t		*rl_lock;
278 } rl_t;
279 
280 #define	ZTEST_RANGE_LOCKS	64
281 #define	ZTEST_OBJECT_LOCKS	64
282 
283 /*
284  * Object descriptor.  Used as a template for object lookup/create/remove.
285  */
286 typedef struct ztest_od {
287 	uint64_t	od_dir;
288 	uint64_t	od_object;
289 	dmu_object_type_t od_type;
290 	dmu_object_type_t od_crtype;
291 	uint64_t	od_blocksize;
292 	uint64_t	od_crblocksize;
293 	uint64_t	od_crdnodesize;
294 	uint64_t	od_gen;
295 	uint64_t	od_crgen;
296 	char		od_name[ZFS_MAX_DATASET_NAME_LEN];
297 } ztest_od_t;
298 
299 /*
300  * Per-dataset state.
301  */
302 typedef struct ztest_ds {
303 	ztest_shared_ds_t *zd_shared;
304 	objset_t	*zd_os;
305 	krwlock_t	zd_zilog_lock;
306 	zilog_t		*zd_zilog;
307 	ztest_od_t	*zd_od;		/* debugging aid */
308 	char		zd_name[ZFS_MAX_DATASET_NAME_LEN];
309 	kmutex_t	zd_dirobj_lock;
310 	rll_t		zd_object_lock[ZTEST_OBJECT_LOCKS];
311 	rll_t		zd_range_lock[ZTEST_RANGE_LOCKS];
312 } ztest_ds_t;
313 
314 /*
315  * Per-iteration state.
316  */
317 typedef void ztest_func_t(ztest_ds_t *zd, uint64_t id);
318 
319 typedef struct ztest_info {
320 	ztest_func_t	*zi_func;	/* test function */
321 	uint64_t	zi_iters;	/* iterations per execution */
322 	uint64_t	*zi_interval;	/* execute every <interval> seconds */
323 } ztest_info_t;
324 
325 typedef struct ztest_shared_callstate {
326 	uint64_t	zc_count;	/* per-pass count */
327 	uint64_t	zc_time;	/* per-pass time */
328 	uint64_t	zc_next;	/* next time to call this function */
329 } ztest_shared_callstate_t;
330 
331 static ztest_shared_callstate_t *ztest_shared_callstate;
332 #define	ZTEST_GET_SHARED_CALLSTATE(c) (&ztest_shared_callstate[c])
333 
334 /*
335  * Note: these aren't static because we want dladdr() to work.
336  */
337 ztest_func_t ztest_dmu_read_write;
338 ztest_func_t ztest_dmu_write_parallel;
339 ztest_func_t ztest_dmu_object_alloc_free;
340 ztest_func_t ztest_dmu_object_next_chunk;
341 ztest_func_t ztest_dmu_commit_callbacks;
342 ztest_func_t ztest_zap;
343 ztest_func_t ztest_zap_parallel;
344 ztest_func_t ztest_zil_commit;
345 ztest_func_t ztest_zil_remount;
346 ztest_func_t ztest_dmu_read_write_zcopy;
347 ztest_func_t ztest_dmu_objset_create_destroy;
348 ztest_func_t ztest_dmu_prealloc;
349 ztest_func_t ztest_fzap;
350 ztest_func_t ztest_dmu_snapshot_create_destroy;
351 ztest_func_t ztest_dsl_prop_get_set;
352 ztest_func_t ztest_spa_prop_get_set;
353 ztest_func_t ztest_spa_create_destroy;
354 ztest_func_t ztest_fault_inject;
355 ztest_func_t ztest_ddt_repair;
356 ztest_func_t ztest_dmu_snapshot_hold;
357 ztest_func_t ztest_mmp_enable_disable;
358 ztest_func_t ztest_scrub;
359 ztest_func_t ztest_dsl_dataset_promote_busy;
360 ztest_func_t ztest_vdev_attach_detach;
361 ztest_func_t ztest_vdev_LUN_growth;
362 ztest_func_t ztest_vdev_add_remove;
363 ztest_func_t ztest_vdev_class_add;
364 ztest_func_t ztest_vdev_aux_add_remove;
365 ztest_func_t ztest_split_pool;
366 ztest_func_t ztest_reguid;
367 ztest_func_t ztest_spa_upgrade;
368 ztest_func_t ztest_device_removal;
369 ztest_func_t ztest_remap_blocks;
370 ztest_func_t ztest_spa_checkpoint_create_discard;
371 ztest_func_t ztest_initialize;
372 ztest_func_t ztest_verify_dnode_bt;
373 
374 uint64_t zopt_always = 0ULL * NANOSEC;		/* all the time */
375 uint64_t zopt_incessant = 1ULL * NANOSEC / 10;	/* every 1/10 second */
376 uint64_t zopt_often = 1ULL * NANOSEC;		/* every second */
377 uint64_t zopt_sometimes = 10ULL * NANOSEC;	/* every 10 seconds */
378 uint64_t zopt_rarely = 60ULL * NANOSEC;		/* every 60 seconds */
379 
380 ztest_info_t ztest_info[] = {
381 	{ ztest_dmu_read_write,			1,	&zopt_always	},
382 	{ ztest_dmu_write_parallel,		10,	&zopt_always	},
383 	{ ztest_dmu_object_alloc_free,		1,	&zopt_always	},
384 	{ ztest_dmu_object_next_chunk,		1,	&zopt_sometimes	},
385 	{ ztest_dmu_commit_callbacks,		1,	&zopt_always	},
386 	{ ztest_zap,				30,	&zopt_always	},
387 	{ ztest_zap_parallel,			100,	&zopt_always	},
388 	{ ztest_split_pool,			1,	&zopt_always	},
389 	{ ztest_zil_commit,			1,	&zopt_incessant	},
390 	{ ztest_zil_remount,			1,	&zopt_sometimes	},
391 	{ ztest_dmu_read_write_zcopy,		1,	&zopt_often	},
392 	{ ztest_dmu_objset_create_destroy,	1,	&zopt_often	},
393 	{ ztest_dsl_prop_get_set,		1,	&zopt_often	},
394 	{ ztest_spa_prop_get_set,		1,	&zopt_sometimes	},
395 #if 0
396 	{ ztest_dmu_prealloc,			1,	&zopt_sometimes	},
397 #endif
398 	{ ztest_fzap,				1,	&zopt_sometimes	},
399 	{ ztest_dmu_snapshot_create_destroy,	1,	&zopt_sometimes	},
400 	{ ztest_spa_create_destroy,		1,	&zopt_sometimes	},
401 	{ ztest_fault_inject,			1,	&zopt_incessant	},
402 	{ ztest_ddt_repair,			1,	&zopt_sometimes	},
403 	{ ztest_dmu_snapshot_hold,		1,	&zopt_sometimes	},
404 	{ ztest_mmp_enable_disable,		1,	&zopt_sometimes	},
405 	{ ztest_reguid,				1,	&zopt_rarely	},
406 	{ ztest_scrub,				1,	&zopt_often	},
407 	{ ztest_spa_upgrade,			1,	&zopt_rarely	},
408 	{ ztest_dsl_dataset_promote_busy,	1,	&zopt_rarely	},
409 	{ ztest_vdev_attach_detach,		1,	&zopt_incessant	},
410 	{ ztest_vdev_LUN_growth,		1,	&zopt_rarely	},
411 	{ ztest_vdev_add_remove,		1,
412 	    &ztest_opts.zo_vdevtime				},
413 	{ ztest_vdev_class_add,			1,
414 	    &ztest_opts.zo_vdevtime				},
415 	{ ztest_vdev_aux_add_remove,		1,
416 	    &ztest_opts.zo_vdevtime				},
417 	{ ztest_device_removal,			1,	&zopt_sometimes	},
418 	{ ztest_remap_blocks,			1,	&zopt_sometimes },
419 	{ ztest_spa_checkpoint_create_discard,	1,	&zopt_rarely	},
420 	{ ztest_initialize,			1,	&zopt_sometimes },
421 	{ ztest_verify_dnode_bt,		1,	&zopt_sometimes }
422 };
423 
424 #define	ZTEST_FUNCS	(sizeof (ztest_info) / sizeof (ztest_info_t))
425 
426 /*
427  * The following struct is used to hold a list of uncalled commit callbacks.
428  * The callbacks are ordered by txg number.
429  */
430 typedef struct ztest_cb_list {
431 	kmutex_t zcl_callbacks_lock;
432 	list_t	zcl_callbacks;
433 } ztest_cb_list_t;
434 
435 /*
436  * Stuff we need to share writably between parent and child.
437  */
438 typedef struct ztest_shared {
439 	boolean_t	zs_do_init;
440 	hrtime_t	zs_proc_start;
441 	hrtime_t	zs_proc_stop;
442 	hrtime_t	zs_thread_start;
443 	hrtime_t	zs_thread_stop;
444 	hrtime_t	zs_thread_kill;
445 	uint64_t	zs_enospc_count;
446 	uint64_t	zs_vdev_next_leaf;
447 	uint64_t	zs_vdev_aux;
448 	uint64_t	zs_alloc;
449 	uint64_t	zs_space;
450 	uint64_t	zs_splits;
451 	uint64_t	zs_mirrors;
452 	uint64_t	zs_metaslab_sz;
453 	uint64_t	zs_metaslab_df_alloc_threshold;
454 	uint64_t	zs_guid;
455 } ztest_shared_t;
456 
457 #define	ID_PARALLEL	-1ULL
458 
459 static char ztest_dev_template[] = "%s/%s.%llua";
460 static char ztest_aux_template[] = "%s/%s.%s.%llu";
461 ztest_shared_t *ztest_shared;
462 
463 static spa_t *ztest_spa = NULL;
464 static ztest_ds_t *ztest_ds;
465 
466 static kmutex_t ztest_vdev_lock;
467 static boolean_t ztest_device_removal_active = B_FALSE;
468 static kmutex_t ztest_checkpoint_lock;
469 
470 /*
471  * The ztest_name_lock protects the pool and dataset namespace used by
472  * the individual tests. To modify the namespace, consumers must grab
473  * this lock as writer. Grabbing the lock as reader will ensure that the
474  * namespace does not change while the lock is held.
475  */
476 static krwlock_t ztest_name_lock;
477 
478 static boolean_t ztest_dump_core = B_TRUE;
479 static boolean_t ztest_exiting;
480 
481 /* Global commit callback list */
482 static ztest_cb_list_t zcl;
483 
484 enum ztest_object {
485 	ZTEST_META_DNODE = 0,
486 	ZTEST_DIROBJ,
487 	ZTEST_OBJECTS
488 };
489 
490 static void usage(boolean_t) __NORETURN;
491 
492 /*
493  * These libumem hooks provide a reasonable set of defaults for the allocator's
494  * debugging facilities.
495  */
496 const char *
497 _umem_debug_init()
498 {
499 	return ("default,verbose"); /* $UMEM_DEBUG setting */
500 }
501 
502 const char *
503 _umem_logging_init(void)
504 {
505 	return ("fail,contents"); /* $UMEM_LOGGING setting */
506 }
507 
508 #define	FATAL_MSG_SZ	1024
509 
510 char *fatal_msg;
511 
512 static void
513 fatal(int do_perror, char *message, ...)
514 {
515 	va_list args;
516 	int save_errno = errno;
517 	char buf[FATAL_MSG_SZ];
518 
519 	(void) fflush(stdout);
520 
521 	va_start(args, message);
522 	(void) sprintf(buf, "ztest: ");
523 	/* LINTED */
524 	(void) vsprintf(buf + strlen(buf), message, args);
525 	va_end(args);
526 	if (do_perror) {
527 		(void) snprintf(buf + strlen(buf), FATAL_MSG_SZ - strlen(buf),
528 		    ": %s", strerror(save_errno));
529 	}
530 	(void) fprintf(stderr, "%s\n", buf);
531 	fatal_msg = buf;			/* to ease debugging */
532 	if (ztest_dump_core)
533 		abort();
534 	exit(3);
535 }
536 
537 static int
538 str2shift(const char *buf)
539 {
540 	const char *ends = "BKMGTPEZ";
541 	int i;
542 
543 	if (buf[0] == '\0')
544 		return (0);
545 	for (i = 0; i < strlen(ends); i++) {
546 		if (toupper(buf[0]) == ends[i])
547 			break;
548 	}
549 	if (i == strlen(ends)) {
550 		(void) fprintf(stderr, "ztest: invalid bytes suffix: %s\n",
551 		    buf);
552 		usage(B_FALSE);
553 	}
554 	if (buf[1] == '\0' || (toupper(buf[1]) == 'B' && buf[2] == '\0')) {
555 		return (10*i);
556 	}
557 	(void) fprintf(stderr, "ztest: invalid bytes suffix: %s\n", buf);
558 	usage(B_FALSE);
559 	/* NOTREACHED */
560 }
561 
562 static uint64_t
563 nicenumtoull(const char *buf)
564 {
565 	char *end;
566 	uint64_t val;
567 
568 	val = strtoull(buf, &end, 0);
569 	if (end == buf) {
570 		(void) fprintf(stderr, "ztest: bad numeric value: %s\n", buf);
571 		usage(B_FALSE);
572 	} else if (end[0] == '.') {
573 		double fval = strtod(buf, &end);
574 		fval *= pow(2, str2shift(end));
575 		if (fval > UINT64_MAX) {
576 			(void) fprintf(stderr, "ztest: value too large: %s\n",
577 			    buf);
578 			usage(B_FALSE);
579 		}
580 		val = (uint64_t)fval;
581 	} else {
582 		int shift = str2shift(end);
583 		if (shift >= 64 || (val << shift) >> shift != val) {
584 			(void) fprintf(stderr, "ztest: value too large: %s\n",
585 			    buf);
586 			usage(B_FALSE);
587 		}
588 		val <<= shift;
589 	}
590 	return (val);
591 }
592 
593 static void
594 usage(boolean_t requested)
595 {
596 	const ztest_shared_opts_t *zo = &ztest_opts_defaults;
597 
598 	char nice_vdev_size[NN_NUMBUF_SZ];
599 	char nice_force_ganging[NN_NUMBUF_SZ];
600 	FILE *fp = requested ? stdout : stderr;
601 
602 	nicenum(zo->zo_vdev_size, nice_vdev_size, sizeof (nice_vdev_size));
603 	nicenum(zo->zo_metaslab_force_ganging, nice_force_ganging,
604 	    sizeof (nice_force_ganging));
605 
606 	(void) fprintf(fp, "Usage: %s\n"
607 	    "\t[-v vdevs (default: %llu)]\n"
608 	    "\t[-s size_of_each_vdev (default: %s)]\n"
609 	    "\t[-a alignment_shift (default: %d)] use 0 for random\n"
610 	    "\t[-m mirror_copies (default: %d)]\n"
611 	    "\t[-r raidz_disks (default: %d)]\n"
612 	    "\t[-R raidz_parity (default: %d)]\n"
613 	    "\t[-d datasets (default: %d)]\n"
614 	    "\t[-t threads (default: %d)]\n"
615 	    "\t[-g gang_block_threshold (default: %s)]\n"
616 	    "\t[-i init_count (default: %d)] initialize pool i times\n"
617 	    "\t[-k kill_percentage (default: %llu%%)]\n"
618 	    "\t[-p pool_name (default: %s)]\n"
619 	    "\t[-f dir (default: %s)] file directory for vdev files\n"
620 	    "\t[-M] Multi-host simulate pool imported on remote host\n"
621 	    "\t[-V] verbose (use multiple times for ever more blather)\n"
622 	    "\t[-E] use existing pool instead of creating new one\n"
623 	    "\t[-T time (default: %llu sec)] total run time\n"
624 	    "\t[-F freezeloops (default: %llu)] max loops in spa_freeze()\n"
625 	    "\t[-P passtime (default: %llu sec)] time per pass\n"
626 	    "\t[-B alt_ztest (default: <none>)] alternate ztest path\n"
627 	    "\t[-C vdev class state (default: random)] special=on|off|random\n"
628 	    "\t[-o variable=value] ... set global variable to an unsigned\n"
629 	    "\t    32-bit integer value\n"
630 	    "\t[-h] (print help)\n"
631 	    "",
632 	    zo->zo_pool,
633 	    (u_longlong_t)zo->zo_vdevs,			/* -v */
634 	    nice_vdev_size,				/* -s */
635 	    zo->zo_ashift,				/* -a */
636 	    zo->zo_mirrors,				/* -m */
637 	    zo->zo_raidz,				/* -r */
638 	    zo->zo_raidz_parity,			/* -R */
639 	    zo->zo_datasets,				/* -d */
640 	    zo->zo_threads,				/* -t */
641 	    nice_force_ganging,				/* -g */
642 	    zo->zo_init,				/* -i */
643 	    (u_longlong_t)zo->zo_killrate,		/* -k */
644 	    zo->zo_pool,				/* -p */
645 	    zo->zo_dir,					/* -f */
646 	    (u_longlong_t)zo->zo_time,			/* -T */
647 	    (u_longlong_t)zo->zo_maxloops,		/* -F */
648 	    (u_longlong_t)zo->zo_passtime);
649 	exit(requested ? 0 : 1);
650 }
651 
652 
653 static void
654 ztest_parse_name_value(const char *input, ztest_shared_opts_t *zo)
655 {
656 	char name[32];
657 	char *value;
658 	int state = ZTEST_VDEV_CLASS_RND;
659 
660 	(void) strlcpy(name, input, sizeof (name));
661 
662 	value = strchr(name, '=');
663 	if (value == NULL) {
664 		(void) fprintf(stderr, "missing value in property=value "
665 		    "'-C' argument (%s)\n", input);
666 		usage(B_FALSE);
667 	}
668 	*(value) = '\0';
669 	value++;
670 
671 	if (strcmp(value, "on") == 0) {
672 		state = ZTEST_VDEV_CLASS_ON;
673 	} else if (strcmp(value, "off") == 0) {
674 		state = ZTEST_VDEV_CLASS_OFF;
675 	} else if (strcmp(value, "random") == 0) {
676 		state = ZTEST_VDEV_CLASS_RND;
677 	} else {
678 		(void) fprintf(stderr, "invalid property value '%s'\n", value);
679 		usage(B_FALSE);
680 	}
681 
682 	if (strcmp(name, "special") == 0) {
683 		zo->zo_special_vdevs = state;
684 	} else {
685 		(void) fprintf(stderr, "invalid property name '%s'\n", name);
686 		usage(B_FALSE);
687 	}
688 	if (zo->zo_verbose >= 3)
689 		(void) printf("%s vdev state is '%s'\n", name, value);
690 }
691 
692 static void
693 process_options(int argc, char **argv)
694 {
695 	char *path;
696 	ztest_shared_opts_t *zo = &ztest_opts;
697 
698 	int opt;
699 	uint64_t value;
700 	char altdir[MAXNAMELEN] = { 0 };
701 
702 	bcopy(&ztest_opts_defaults, zo, sizeof (*zo));
703 
704 	while ((opt = getopt(argc, argv,
705 	    "v:s:a:m:r:R:d:t:g:i:k:p:f:MVET:P:hF:B:C:o:")) != EOF) {
706 		value = 0;
707 		switch (opt) {
708 		case 'v':
709 		case 's':
710 		case 'a':
711 		case 'm':
712 		case 'r':
713 		case 'R':
714 		case 'd':
715 		case 't':
716 		case 'g':
717 		case 'i':
718 		case 'k':
719 		case 'T':
720 		case 'P':
721 		case 'F':
722 			value = nicenumtoull(optarg);
723 		}
724 		switch (opt) {
725 		case 'v':
726 			zo->zo_vdevs = value;
727 			break;
728 		case 's':
729 			zo->zo_vdev_size = MAX(SPA_MINDEVSIZE, value);
730 			break;
731 		case 'a':
732 			zo->zo_ashift = value;
733 			break;
734 		case 'm':
735 			zo->zo_mirrors = value;
736 			break;
737 		case 'r':
738 			zo->zo_raidz = MAX(1, value);
739 			break;
740 		case 'R':
741 			zo->zo_raidz_parity = MIN(MAX(value, 1), 3);
742 			break;
743 		case 'd':
744 			zo->zo_datasets = MAX(1, value);
745 			break;
746 		case 't':
747 			zo->zo_threads = MAX(1, value);
748 			break;
749 		case 'g':
750 			zo->zo_metaslab_force_ganging =
751 			    MAX(SPA_MINBLOCKSIZE << 1, value);
752 			break;
753 		case 'i':
754 			zo->zo_init = value;
755 			break;
756 		case 'k':
757 			zo->zo_killrate = value;
758 			break;
759 		case 'p':
760 			(void) strlcpy(zo->zo_pool, optarg,
761 			    sizeof (zo->zo_pool));
762 			break;
763 		case 'f':
764 			path = realpath(optarg, NULL);
765 			if (path == NULL) {
766 				(void) fprintf(stderr, "error: %s: %s\n",
767 				    optarg, strerror(errno));
768 				usage(B_FALSE);
769 			} else {
770 				(void) strlcpy(zo->zo_dir, path,
771 				    sizeof (zo->zo_dir));
772 			}
773 			break;
774 		case 'M':
775 			zo->zo_mmp_test = 1;
776 			break;
777 		case 'V':
778 			zo->zo_verbose++;
779 			break;
780 		case 'E':
781 			zo->zo_init = 0;
782 			break;
783 		case 'T':
784 			zo->zo_time = value;
785 			break;
786 		case 'P':
787 			zo->zo_passtime = MAX(1, value);
788 			break;
789 		case 'F':
790 			zo->zo_maxloops = MAX(1, value);
791 			break;
792 		case 'B':
793 			(void) strlcpy(altdir, optarg, sizeof (altdir));
794 			break;
795 		case 'C':
796 			ztest_parse_name_value(optarg, zo);
797 			break;
798 		case 'o':
799 			if (set_global_var(optarg) != 0)
800 				usage(B_FALSE);
801 			break;
802 		case 'h':
803 			usage(B_TRUE);
804 			break;
805 		case '?':
806 		default:
807 			usage(B_FALSE);
808 			break;
809 		}
810 	}
811 
812 	zo->zo_raidz_parity = MIN(zo->zo_raidz_parity, zo->zo_raidz - 1);
813 
814 	zo->zo_vdevtime =
815 	    (zo->zo_vdevs > 0 ? zo->zo_time * NANOSEC / zo->zo_vdevs :
816 	    UINT64_MAX >> 2);
817 
818 	if (strlen(altdir) > 0) {
819 		char *cmd;
820 		char *realaltdir;
821 		char *bin;
822 		char *ztest;
823 		char *isa;
824 		int isalen;
825 
826 		cmd = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
827 		realaltdir = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
828 
829 		VERIFY(NULL != realpath(getexecname(), cmd));
830 		if (0 != access(altdir, F_OK)) {
831 			ztest_dump_core = B_FALSE;
832 			fatal(B_TRUE, "invalid alternate ztest path: %s",
833 			    altdir);
834 		}
835 		VERIFY(NULL != realpath(altdir, realaltdir));
836 
837 		/*
838 		 * 'cmd' should be of the form "<anything>/usr/bin/<isa>/ztest".
839 		 * We want to extract <isa> to determine if we should use
840 		 * 32 or 64 bit binaries.
841 		 */
842 		bin = strstr(cmd, "/usr/bin/");
843 		ztest = strstr(bin, "/ztest");
844 		isa = bin + 9;
845 		isalen = ztest - isa;
846 		(void) snprintf(zo->zo_alt_ztest, sizeof (zo->zo_alt_ztest),
847 		    "%s/usr/bin/%.*s/ztest", realaltdir, isalen, isa);
848 		(void) snprintf(zo->zo_alt_libpath, sizeof (zo->zo_alt_libpath),
849 		    "%s/usr/lib/%.*s", realaltdir, isalen, isa);
850 
851 		if (0 != access(zo->zo_alt_ztest, X_OK)) {
852 			ztest_dump_core = B_FALSE;
853 			fatal(B_TRUE, "invalid alternate ztest: %s",
854 			    zo->zo_alt_ztest);
855 		} else if (0 != access(zo->zo_alt_libpath, X_OK)) {
856 			ztest_dump_core = B_FALSE;
857 			fatal(B_TRUE, "invalid alternate lib directory %s",
858 			    zo->zo_alt_libpath);
859 		}
860 
861 		umem_free(cmd, MAXPATHLEN);
862 		umem_free(realaltdir, MAXPATHLEN);
863 	}
864 }
865 
866 static void
867 ztest_kill(ztest_shared_t *zs)
868 {
869 	zs->zs_alloc = metaslab_class_get_alloc(spa_normal_class(ztest_spa));
870 	zs->zs_space = metaslab_class_get_space(spa_normal_class(ztest_spa));
871 
872 	/*
873 	 * Before we kill off ztest, make sure that the config is updated.
874 	 * See comment above spa_write_cachefile().
875 	 */
876 	mutex_enter(&spa_namespace_lock);
877 	spa_write_cachefile(ztest_spa, B_FALSE, B_FALSE);
878 	mutex_exit(&spa_namespace_lock);
879 
880 	zfs_dbgmsg_print(FTAG);
881 	(void) kill(getpid(), SIGKILL);
882 }
883 
884 static uint64_t
885 ztest_random(uint64_t range)
886 {
887 	uint64_t r;
888 
889 	ASSERT3S(ztest_fd_rand, >=, 0);
890 
891 	if (range == 0)
892 		return (0);
893 
894 	if (read(ztest_fd_rand, &r, sizeof (r)) != sizeof (r))
895 		fatal(1, "short read from /dev/urandom");
896 
897 	return (r % range);
898 }
899 
900 /* ARGSUSED */
901 static void
902 ztest_record_enospc(const char *s)
903 {
904 	ztest_shared->zs_enospc_count++;
905 }
906 
907 static uint64_t
908 ztest_get_ashift(void)
909 {
910 	if (ztest_opts.zo_ashift == 0)
911 		return (SPA_MINBLOCKSHIFT + ztest_random(5));
912 	return (ztest_opts.zo_ashift);
913 }
914 
915 static nvlist_t *
916 make_vdev_file(char *path, char *aux, char *pool, size_t size, uint64_t ashift)
917 {
918 	char pathbuf[MAXPATHLEN];
919 	uint64_t vdev;
920 	nvlist_t *file;
921 
922 	if (ashift == 0)
923 		ashift = ztest_get_ashift();
924 
925 	if (path == NULL) {
926 		path = pathbuf;
927 
928 		if (aux != NULL) {
929 			vdev = ztest_shared->zs_vdev_aux;
930 			(void) snprintf(path, sizeof (pathbuf),
931 			    ztest_aux_template, ztest_opts.zo_dir,
932 			    pool == NULL ? ztest_opts.zo_pool : pool,
933 			    aux, vdev);
934 		} else {
935 			vdev = ztest_shared->zs_vdev_next_leaf++;
936 			(void) snprintf(path, sizeof (pathbuf),
937 			    ztest_dev_template, ztest_opts.zo_dir,
938 			    pool == NULL ? ztest_opts.zo_pool : pool, vdev);
939 		}
940 	}
941 
942 	if (size != 0) {
943 		int fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666);
944 		if (fd == -1)
945 			fatal(1, "can't open %s", path);
946 		if (ftruncate(fd, size) != 0)
947 			fatal(1, "can't ftruncate %s", path);
948 		(void) close(fd);
949 	}
950 
951 	VERIFY(nvlist_alloc(&file, NV_UNIQUE_NAME, 0) == 0);
952 	VERIFY(nvlist_add_string(file, ZPOOL_CONFIG_TYPE, VDEV_TYPE_FILE) == 0);
953 	VERIFY(nvlist_add_string(file, ZPOOL_CONFIG_PATH, path) == 0);
954 	VERIFY(nvlist_add_uint64(file, ZPOOL_CONFIG_ASHIFT, ashift) == 0);
955 
956 	return (file);
957 }
958 
959 static nvlist_t *
960 make_vdev_raidz(char *path, char *aux, char *pool, size_t size,
961     uint64_t ashift, int r)
962 {
963 	nvlist_t *raidz, **child;
964 	int c;
965 
966 	if (r < 2)
967 		return (make_vdev_file(path, aux, pool, size, ashift));
968 	child = umem_alloc(r * sizeof (nvlist_t *), UMEM_NOFAIL);
969 
970 	for (c = 0; c < r; c++)
971 		child[c] = make_vdev_file(path, aux, pool, size, ashift);
972 
973 	VERIFY(nvlist_alloc(&raidz, NV_UNIQUE_NAME, 0) == 0);
974 	VERIFY(nvlist_add_string(raidz, ZPOOL_CONFIG_TYPE,
975 	    VDEV_TYPE_RAIDZ) == 0);
976 	VERIFY(nvlist_add_uint64(raidz, ZPOOL_CONFIG_NPARITY,
977 	    ztest_opts.zo_raidz_parity) == 0);
978 	VERIFY(nvlist_add_nvlist_array(raidz, ZPOOL_CONFIG_CHILDREN,
979 	    child, r) == 0);
980 
981 	for (c = 0; c < r; c++)
982 		nvlist_free(child[c]);
983 
984 	umem_free(child, r * sizeof (nvlist_t *));
985 
986 	return (raidz);
987 }
988 
989 static nvlist_t *
990 make_vdev_mirror(char *path, char *aux, char *pool, size_t size,
991     uint64_t ashift, int r, int m)
992 {
993 	nvlist_t *mirror, **child;
994 	int c;
995 
996 	if (m < 1)
997 		return (make_vdev_raidz(path, aux, pool, size, ashift, r));
998 
999 	child = umem_alloc(m * sizeof (nvlist_t *), UMEM_NOFAIL);
1000 
1001 	for (c = 0; c < m; c++)
1002 		child[c] = make_vdev_raidz(path, aux, pool, size, ashift, r);
1003 
1004 	VERIFY(nvlist_alloc(&mirror, NV_UNIQUE_NAME, 0) == 0);
1005 	VERIFY(nvlist_add_string(mirror, ZPOOL_CONFIG_TYPE,
1006 	    VDEV_TYPE_MIRROR) == 0);
1007 	VERIFY(nvlist_add_nvlist_array(mirror, ZPOOL_CONFIG_CHILDREN,
1008 	    child, m) == 0);
1009 
1010 	for (c = 0; c < m; c++)
1011 		nvlist_free(child[c]);
1012 
1013 	umem_free(child, m * sizeof (nvlist_t *));
1014 
1015 	return (mirror);
1016 }
1017 
1018 static nvlist_t *
1019 make_vdev_root(char *path, char *aux, char *pool, size_t size, uint64_t ashift,
1020     const char *class, int r, int m, int t)
1021 {
1022 	nvlist_t *root, **child;
1023 	int c;
1024 	boolean_t log;
1025 
1026 	ASSERT(t > 0);
1027 
1028 	log = (class != NULL && strcmp(class, "log") == 0);
1029 
1030 	child = umem_alloc(t * sizeof (nvlist_t *), UMEM_NOFAIL);
1031 
1032 	for (c = 0; c < t; c++) {
1033 		child[c] = make_vdev_mirror(path, aux, pool, size, ashift,
1034 		    r, m);
1035 		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
1036 		    log) == 0);
1037 
1038 		if (class != NULL && class[0] != '\0') {
1039 			ASSERT(m > 1 || log);   /* expecting a mirror */
1040 			VERIFY(nvlist_add_string(child[c],
1041 			    ZPOOL_CONFIG_ALLOCATION_BIAS, class) == 0);
1042 		}
1043 	}
1044 
1045 	VERIFY(nvlist_alloc(&root, NV_UNIQUE_NAME, 0) == 0);
1046 	VERIFY(nvlist_add_string(root, ZPOOL_CONFIG_TYPE, VDEV_TYPE_ROOT) == 0);
1047 	VERIFY(nvlist_add_nvlist_array(root, aux ? aux : ZPOOL_CONFIG_CHILDREN,
1048 	    child, t) == 0);
1049 
1050 	for (c = 0; c < t; c++)
1051 		nvlist_free(child[c]);
1052 
1053 	umem_free(child, t * sizeof (nvlist_t *));
1054 
1055 	return (root);
1056 }
1057 
1058 /*
1059  * Find a random spa version. Returns back a random spa version in the
1060  * range [initial_version, SPA_VERSION_FEATURES].
1061  */
1062 static uint64_t
1063 ztest_random_spa_version(uint64_t initial_version)
1064 {
1065 	uint64_t version = initial_version;
1066 
1067 	if (version <= SPA_VERSION_BEFORE_FEATURES) {
1068 		version = version +
1069 		    ztest_random(SPA_VERSION_BEFORE_FEATURES - version + 1);
1070 	}
1071 
1072 	if (version > SPA_VERSION_BEFORE_FEATURES)
1073 		version = SPA_VERSION_FEATURES;
1074 
1075 	ASSERT(SPA_VERSION_IS_SUPPORTED(version));
1076 	return (version);
1077 }
1078 
1079 static int
1080 ztest_random_blocksize(void)
1081 {
1082 	uint64_t block_shift;
1083 
1084 	ASSERT(ztest_spa->spa_max_ashift != 0);
1085 
1086 	/*
1087 	 * Choose a block size >= the ashift.
1088 	 * If the SPA supports new MAXBLOCKSIZE, test up to 1MB blocks.
1089 	 */
1090 	int maxbs = SPA_OLD_MAXBLOCKSHIFT;
1091 	if (spa_maxblocksize(ztest_spa) == SPA_MAXBLOCKSIZE)
1092 		maxbs = 20;
1093 	block_shift = ztest_random(maxbs - ztest_spa->spa_max_ashift + 1);
1094 	return (1 << (SPA_MINBLOCKSHIFT + block_shift));
1095 }
1096 
1097 static int
1098 ztest_random_dnodesize(void)
1099 {
1100 	int slots;
1101 	int max_slots = spa_maxdnodesize(ztest_spa) >> DNODE_SHIFT;
1102 
1103 	if (max_slots == DNODE_MIN_SLOTS)
1104 		return (DNODE_MIN_SIZE);
1105 
1106 	/*
1107 	 * Weight the random distribution more heavily toward smaller
1108 	 * dnode sizes since that is more likely to reflect real-world
1109 	 * usage.
1110 	 */
1111 	ASSERT3U(max_slots, >, 4);
1112 	switch (ztest_random(10)) {
1113 	case 0:
1114 		slots = 5 + ztest_random(max_slots - 4);
1115 		break;
1116 	case 1 ... 4:
1117 		slots = 2 + ztest_random(3);
1118 		break;
1119 	default:
1120 		slots = 1;
1121 		break;
1122 	}
1123 
1124 	return (slots << DNODE_SHIFT);
1125 }
1126 
1127 static int
1128 ztest_random_ibshift(void)
1129 {
1130 	return (DN_MIN_INDBLKSHIFT +
1131 	    ztest_random(DN_MAX_INDBLKSHIFT - DN_MIN_INDBLKSHIFT + 1));
1132 }
1133 
1134 static uint64_t
1135 ztest_random_vdev_top(spa_t *spa, boolean_t log_ok)
1136 {
1137 	uint64_t top;
1138 	vdev_t *rvd = spa->spa_root_vdev;
1139 	vdev_t *tvd;
1140 
1141 	ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0);
1142 
1143 	do {
1144 		top = ztest_random(rvd->vdev_children);
1145 		tvd = rvd->vdev_child[top];
1146 	} while (!vdev_is_concrete(tvd) || (tvd->vdev_islog && !log_ok) ||
1147 	    tvd->vdev_mg == NULL || tvd->vdev_mg->mg_class == NULL);
1148 
1149 	return (top);
1150 }
1151 
1152 static uint64_t
1153 ztest_random_dsl_prop(zfs_prop_t prop)
1154 {
1155 	uint64_t value;
1156 
1157 	do {
1158 		value = zfs_prop_random_value(prop, ztest_random(-1ULL));
1159 	} while (prop == ZFS_PROP_CHECKSUM && value == ZIO_CHECKSUM_OFF);
1160 
1161 	return (value);
1162 }
1163 
1164 static int
1165 ztest_dsl_prop_set_uint64(char *osname, zfs_prop_t prop, uint64_t value,
1166     boolean_t inherit)
1167 {
1168 	const char *propname = zfs_prop_to_name(prop);
1169 	const char *valname;
1170 	char setpoint[MAXPATHLEN];
1171 	uint64_t curval;
1172 	int error;
1173 
1174 	error = dsl_prop_set_int(osname, propname,
1175 	    (inherit ? ZPROP_SRC_NONE : ZPROP_SRC_LOCAL), value);
1176 
1177 	if (error == ENOSPC) {
1178 		ztest_record_enospc(FTAG);
1179 		return (error);
1180 	}
1181 	ASSERT0(error);
1182 
1183 	VERIFY0(dsl_prop_get_integer(osname, propname, &curval, setpoint));
1184 
1185 	if (ztest_opts.zo_verbose >= 6) {
1186 		VERIFY(zfs_prop_index_to_string(prop, curval, &valname) == 0);
1187 		(void) printf("%s %s = %s at '%s'\n",
1188 		    osname, propname, valname, setpoint);
1189 	}
1190 
1191 	return (error);
1192 }
1193 
1194 static int
1195 ztest_spa_prop_set_uint64(zpool_prop_t prop, uint64_t value)
1196 {
1197 	spa_t *spa = ztest_spa;
1198 	nvlist_t *props = NULL;
1199 	int error;
1200 
1201 	VERIFY(nvlist_alloc(&props, NV_UNIQUE_NAME, 0) == 0);
1202 	VERIFY(nvlist_add_uint64(props, zpool_prop_to_name(prop), value) == 0);
1203 
1204 	error = spa_prop_set(spa, props);
1205 
1206 	nvlist_free(props);
1207 
1208 	if (error == ENOSPC) {
1209 		ztest_record_enospc(FTAG);
1210 		return (error);
1211 	}
1212 	ASSERT0(error);
1213 
1214 	return (error);
1215 }
1216 
1217 static int
1218 ztest_dmu_objset_own(const char *name, dmu_objset_type_t type,
1219     boolean_t readonly, boolean_t decrypt, void *tag, objset_t **osp)
1220 {
1221 	int err;
1222 
1223 	err = dmu_objset_own(name, type, readonly, decrypt, tag, osp);
1224 	if (decrypt && err == EACCES) {
1225 		char ddname[ZFS_MAX_DATASET_NAME_LEN];
1226 		dsl_crypto_params_t *dcp;
1227 		nvlist_t *crypto_args = fnvlist_alloc();
1228 		char *cp = NULL;
1229 
1230 		/* spa_keystore_load_wkey() expects a dsl dir name */
1231 		(void) strcpy(ddname, name);
1232 		cp = strchr(ddname, '@');
1233 		if (cp != NULL)
1234 			*cp = '\0';
1235 
1236 		fnvlist_add_uint8_array(crypto_args, "wkeydata",
1237 		    (uint8_t *)ztest_wkeydata, WRAPPING_KEY_LEN);
1238 		VERIFY0(dsl_crypto_params_create_nvlist(DCP_CMD_NONE, NULL,
1239 		    crypto_args, &dcp));
1240 		err = spa_keystore_load_wkey(ddname, dcp, B_FALSE);
1241 		dsl_crypto_params_free(dcp, B_FALSE);
1242 		fnvlist_free(crypto_args);
1243 
1244 		if (err != 0)
1245 			return (err);
1246 
1247 		err = dmu_objset_own(name, type, readonly, decrypt, tag, osp);
1248 	}
1249 
1250 	return (err);
1251 }
1252 
1253 static void
1254 ztest_rll_init(rll_t *rll)
1255 {
1256 	rll->rll_writer = NULL;
1257 	rll->rll_readers = 0;
1258 	mutex_init(&rll->rll_lock, NULL, USYNC_THREAD, NULL);
1259 	cv_init(&rll->rll_cv, NULL, USYNC_THREAD, NULL);
1260 }
1261 
1262 static void
1263 ztest_rll_destroy(rll_t *rll)
1264 {
1265 	ASSERT(rll->rll_writer == NULL);
1266 	ASSERT(rll->rll_readers == 0);
1267 	mutex_destroy(&rll->rll_lock);
1268 	cv_destroy(&rll->rll_cv);
1269 }
1270 
1271 static void
1272 ztest_rll_lock(rll_t *rll, rl_type_t type)
1273 {
1274 	mutex_enter(&rll->rll_lock);
1275 
1276 	if (type == RL_READER) {
1277 		while (rll->rll_writer != NULL)
1278 			cv_wait(&rll->rll_cv, &rll->rll_lock);
1279 		rll->rll_readers++;
1280 	} else {
1281 		while (rll->rll_writer != NULL || rll->rll_readers)
1282 			cv_wait(&rll->rll_cv, &rll->rll_lock);
1283 		rll->rll_writer = curthread;
1284 	}
1285 
1286 	mutex_exit(&rll->rll_lock);
1287 }
1288 
1289 static void
1290 ztest_rll_unlock(rll_t *rll)
1291 {
1292 	mutex_enter(&rll->rll_lock);
1293 
1294 	if (rll->rll_writer) {
1295 		ASSERT(rll->rll_readers == 0);
1296 		rll->rll_writer = NULL;
1297 	} else {
1298 		ASSERT(rll->rll_readers != 0);
1299 		ASSERT(rll->rll_writer == NULL);
1300 		rll->rll_readers--;
1301 	}
1302 
1303 	if (rll->rll_writer == NULL && rll->rll_readers == 0)
1304 		cv_broadcast(&rll->rll_cv);
1305 
1306 	mutex_exit(&rll->rll_lock);
1307 }
1308 
1309 static void
1310 ztest_object_lock(ztest_ds_t *zd, uint64_t object, rl_type_t type)
1311 {
1312 	rll_t *rll = &zd->zd_object_lock[object & (ZTEST_OBJECT_LOCKS - 1)];
1313 
1314 	ztest_rll_lock(rll, type);
1315 }
1316 
1317 static void
1318 ztest_object_unlock(ztest_ds_t *zd, uint64_t object)
1319 {
1320 	rll_t *rll = &zd->zd_object_lock[object & (ZTEST_OBJECT_LOCKS - 1)];
1321 
1322 	ztest_rll_unlock(rll);
1323 }
1324 
1325 static rl_t *
1326 ztest_range_lock(ztest_ds_t *zd, uint64_t object, uint64_t offset,
1327     uint64_t size, rl_type_t type)
1328 {
1329 	uint64_t hash = object ^ (offset % (ZTEST_RANGE_LOCKS + 1));
1330 	rll_t *rll = &zd->zd_range_lock[hash & (ZTEST_RANGE_LOCKS - 1)];
1331 	rl_t *rl;
1332 
1333 	rl = umem_alloc(sizeof (*rl), UMEM_NOFAIL);
1334 	rl->rl_object = object;
1335 	rl->rl_offset = offset;
1336 	rl->rl_size = size;
1337 	rl->rl_lock = rll;
1338 
1339 	ztest_rll_lock(rll, type);
1340 
1341 	return (rl);
1342 }
1343 
1344 static void
1345 ztest_range_unlock(rl_t *rl)
1346 {
1347 	rll_t *rll = rl->rl_lock;
1348 
1349 	ztest_rll_unlock(rll);
1350 
1351 	umem_free(rl, sizeof (*rl));
1352 }
1353 
1354 static void
1355 ztest_zd_init(ztest_ds_t *zd, ztest_shared_ds_t *szd, objset_t *os)
1356 {
1357 	zd->zd_os = os;
1358 	zd->zd_zilog = dmu_objset_zil(os);
1359 	zd->zd_shared = szd;
1360 	dmu_objset_name(os, zd->zd_name);
1361 
1362 	if (zd->zd_shared != NULL)
1363 		zd->zd_shared->zd_seq = 0;
1364 
1365 	rw_init(&zd->zd_zilog_lock, NULL, USYNC_THREAD, NULL);
1366 	mutex_init(&zd->zd_dirobj_lock, NULL, USYNC_THREAD, NULL);
1367 
1368 	for (int l = 0; l < ZTEST_OBJECT_LOCKS; l++)
1369 		ztest_rll_init(&zd->zd_object_lock[l]);
1370 
1371 	for (int l = 0; l < ZTEST_RANGE_LOCKS; l++)
1372 		ztest_rll_init(&zd->zd_range_lock[l]);
1373 }
1374 
1375 static void
1376 ztest_zd_fini(ztest_ds_t *zd)
1377 {
1378 	mutex_destroy(&zd->zd_dirobj_lock);
1379 
1380 	for (int l = 0; l < ZTEST_OBJECT_LOCKS; l++)
1381 		ztest_rll_destroy(&zd->zd_object_lock[l]);
1382 
1383 	for (int l = 0; l < ZTEST_RANGE_LOCKS; l++)
1384 		ztest_rll_destroy(&zd->zd_range_lock[l]);
1385 }
1386 
1387 #define	TXG_MIGHTWAIT	(ztest_random(10) == 0 ? TXG_NOWAIT : TXG_WAIT)
1388 
1389 static uint64_t
1390 ztest_tx_assign(dmu_tx_t *tx, uint64_t txg_how, const char *tag)
1391 {
1392 	uint64_t txg;
1393 	int error;
1394 
1395 	/*
1396 	 * Attempt to assign tx to some transaction group.
1397 	 */
1398 	error = dmu_tx_assign(tx, txg_how);
1399 	if (error) {
1400 		if (error == ERESTART) {
1401 			ASSERT(txg_how == TXG_NOWAIT);
1402 			dmu_tx_wait(tx);
1403 		} else {
1404 			ASSERT3U(error, ==, ENOSPC);
1405 			ztest_record_enospc(tag);
1406 		}
1407 		dmu_tx_abort(tx);
1408 		return (0);
1409 	}
1410 	txg = dmu_tx_get_txg(tx);
1411 	ASSERT(txg != 0);
1412 	return (txg);
1413 }
1414 
1415 static void
1416 ztest_pattern_set(void *buf, uint64_t size, uint64_t value)
1417 {
1418 	uint64_t *ip = buf;
1419 	uint64_t *ip_end = (uint64_t *)((uintptr_t)buf + (uintptr_t)size);
1420 
1421 	while (ip < ip_end)
1422 		*ip++ = value;
1423 }
1424 
1425 static boolean_t
1426 ztest_pattern_match(void *buf, uint64_t size, uint64_t value)
1427 {
1428 	uint64_t *ip = buf;
1429 	uint64_t *ip_end = (uint64_t *)((uintptr_t)buf + (uintptr_t)size);
1430 	uint64_t diff = 0;
1431 
1432 	while (ip < ip_end)
1433 		diff |= (value - *ip++);
1434 
1435 	return (diff == 0);
1436 }
1437 
1438 static void
1439 ztest_bt_generate(ztest_block_tag_t *bt, objset_t *os, uint64_t object,
1440     uint64_t dnodesize, uint64_t offset, uint64_t gen, uint64_t txg,
1441     uint64_t crtxg)
1442 {
1443 	bt->bt_magic = BT_MAGIC;
1444 	bt->bt_objset = dmu_objset_id(os);
1445 	bt->bt_object = object;
1446 	bt->bt_dnodesize = dnodesize;
1447 	bt->bt_offset = offset;
1448 	bt->bt_gen = gen;
1449 	bt->bt_txg = txg;
1450 	bt->bt_crtxg = crtxg;
1451 }
1452 
1453 static void
1454 ztest_bt_verify(ztest_block_tag_t *bt, objset_t *os, uint64_t object,
1455     uint64_t dnodesize, uint64_t offset, uint64_t gen, uint64_t txg,
1456     uint64_t crtxg)
1457 {
1458 	ASSERT3U(bt->bt_magic, ==, BT_MAGIC);
1459 	ASSERT3U(bt->bt_objset, ==, dmu_objset_id(os));
1460 	ASSERT3U(bt->bt_object, ==, object);
1461 	ASSERT3U(bt->bt_dnodesize, ==, dnodesize);
1462 	ASSERT3U(bt->bt_offset, ==, offset);
1463 	ASSERT3U(bt->bt_gen, <=, gen);
1464 	ASSERT3U(bt->bt_txg, <=, txg);
1465 	ASSERT3U(bt->bt_crtxg, ==, crtxg);
1466 }
1467 
1468 static ztest_block_tag_t *
1469 ztest_bt_bonus(dmu_buf_t *db)
1470 {
1471 	dmu_object_info_t doi;
1472 	ztest_block_tag_t *bt;
1473 
1474 	dmu_object_info_from_db(db, &doi);
1475 	ASSERT3U(doi.doi_bonus_size, <=, db->db_size);
1476 	ASSERT3U(doi.doi_bonus_size, >=, sizeof (*bt));
1477 	bt = (void *)((char *)db->db_data + doi.doi_bonus_size - sizeof (*bt));
1478 
1479 	return (bt);
1480 }
1481 
1482 /*
1483  * Generate a token to fill up unused bonus buffer space.  Try to make
1484  * it unique to the object, generation, and offset to verify that data
1485  * is not getting overwritten by data from other dnodes.
1486  */
1487 #define	ZTEST_BONUS_FILL_TOKEN(obj, ds, gen, offset)	\
1488 	(((ds) << 48) | ((gen) << 32) | ((obj) << 8) | (offset))
1489 
1490 /*
1491  * Fill up the unused bonus buffer region before the block tag with a
1492  * verifiable pattern. Filling the whole bonus area with non-zero data
1493  * helps ensure that all dnode traversal code properly skips the
1494  * interior regions of large dnodes.
1495  */
1496 void
1497 ztest_fill_unused_bonus(dmu_buf_t *db, void *end, uint64_t obj,
1498     objset_t *os, uint64_t gen)
1499 {
1500 	uint64_t *bonusp;
1501 
1502 	ASSERT(IS_P2ALIGNED((char *)end - (char *)db->db_data, 8));
1503 
1504 	for (bonusp = db->db_data; bonusp < (uint64_t *)end; bonusp++) {
1505 		uint64_t token = ZTEST_BONUS_FILL_TOKEN(obj, dmu_objset_id(os),
1506 		    gen, bonusp - (uint64_t *)db->db_data);
1507 		*bonusp = token;
1508 	}
1509 }
1510 
1511 /*
1512  * Verify that the unused area of a bonus buffer is filled with the
1513  * expected tokens.
1514  */
1515 void
1516 ztest_verify_unused_bonus(dmu_buf_t *db, void *end, uint64_t obj,
1517     objset_t *os, uint64_t gen)
1518 {
1519 	uint64_t *bonusp;
1520 
1521 	for (bonusp = db->db_data; bonusp < (uint64_t *)end; bonusp++) {
1522 		uint64_t token = ZTEST_BONUS_FILL_TOKEN(obj, dmu_objset_id(os),
1523 		    gen, bonusp - (uint64_t *)db->db_data);
1524 		VERIFY3U(*bonusp, ==, token);
1525 	}
1526 }
1527 
1528 /*
1529  * ZIL logging ops
1530  */
1531 
1532 #define	lrz_type	lr_mode
1533 #define	lrz_blocksize	lr_uid
1534 #define	lrz_ibshift	lr_gid
1535 #define	lrz_bonustype	lr_rdev
1536 #define	lrz_dnodesize	lr_crtime[1]
1537 
1538 static void
1539 ztest_log_create(ztest_ds_t *zd, dmu_tx_t *tx, lr_create_t *lr)
1540 {
1541 	char *name = (void *)(lr + 1);		/* name follows lr */
1542 	size_t namesize = strlen(name) + 1;
1543 	itx_t *itx;
1544 
1545 	if (zil_replaying(zd->zd_zilog, tx))
1546 		return;
1547 
1548 	itx = zil_itx_create(TX_CREATE, sizeof (*lr) + namesize);
1549 	bcopy(&lr->lr_common + 1, &itx->itx_lr + 1,
1550 	    sizeof (*lr) + namesize - sizeof (lr_t));
1551 
1552 	zil_itx_assign(zd->zd_zilog, itx, tx);
1553 }
1554 
1555 static void
1556 ztest_log_remove(ztest_ds_t *zd, dmu_tx_t *tx, lr_remove_t *lr, uint64_t object)
1557 {
1558 	char *name = (void *)(lr + 1);		/* name follows lr */
1559 	size_t namesize = strlen(name) + 1;
1560 	itx_t *itx;
1561 
1562 	if (zil_replaying(zd->zd_zilog, tx))
1563 		return;
1564 
1565 	itx = zil_itx_create(TX_REMOVE, sizeof (*lr) + namesize);
1566 	bcopy(&lr->lr_common + 1, &itx->itx_lr + 1,
1567 	    sizeof (*lr) + namesize - sizeof (lr_t));
1568 
1569 	itx->itx_oid = object;
1570 	zil_itx_assign(zd->zd_zilog, itx, tx);
1571 }
1572 
1573 static void
1574 ztest_log_write(ztest_ds_t *zd, dmu_tx_t *tx, lr_write_t *lr)
1575 {
1576 	itx_t *itx;
1577 	itx_wr_state_t write_state = ztest_random(WR_NUM_STATES);
1578 
1579 	if (zil_replaying(zd->zd_zilog, tx))
1580 		return;
1581 
1582 	if (lr->lr_length > ZIL_MAX_LOG_DATA)
1583 		write_state = WR_INDIRECT;
1584 
1585 	itx = zil_itx_create(TX_WRITE,
1586 	    sizeof (*lr) + (write_state == WR_COPIED ? lr->lr_length : 0));
1587 
1588 	if (write_state == WR_COPIED &&
1589 	    dmu_read(zd->zd_os, lr->lr_foid, lr->lr_offset, lr->lr_length,
1590 	    ((lr_write_t *)&itx->itx_lr) + 1, DMU_READ_NO_PREFETCH) != 0) {
1591 		zil_itx_destroy(itx);
1592 		itx = zil_itx_create(TX_WRITE, sizeof (*lr));
1593 		write_state = WR_NEED_COPY;
1594 	}
1595 	itx->itx_private = zd;
1596 	itx->itx_wr_state = write_state;
1597 	itx->itx_sync = (ztest_random(8) == 0);
1598 
1599 	bcopy(&lr->lr_common + 1, &itx->itx_lr + 1,
1600 	    sizeof (*lr) - sizeof (lr_t));
1601 
1602 	zil_itx_assign(zd->zd_zilog, itx, tx);
1603 }
1604 
1605 static void
1606 ztest_log_truncate(ztest_ds_t *zd, dmu_tx_t *tx, lr_truncate_t *lr)
1607 {
1608 	itx_t *itx;
1609 
1610 	if (zil_replaying(zd->zd_zilog, tx))
1611 		return;
1612 
1613 	itx = zil_itx_create(TX_TRUNCATE, sizeof (*lr));
1614 	bcopy(&lr->lr_common + 1, &itx->itx_lr + 1,
1615 	    sizeof (*lr) - sizeof (lr_t));
1616 
1617 	itx->itx_sync = B_FALSE;
1618 	zil_itx_assign(zd->zd_zilog, itx, tx);
1619 }
1620 
1621 static void
1622 ztest_log_setattr(ztest_ds_t *zd, dmu_tx_t *tx, lr_setattr_t *lr)
1623 {
1624 	itx_t *itx;
1625 
1626 	if (zil_replaying(zd->zd_zilog, tx))
1627 		return;
1628 
1629 	itx = zil_itx_create(TX_SETATTR, sizeof (*lr));
1630 	bcopy(&lr->lr_common + 1, &itx->itx_lr + 1,
1631 	    sizeof (*lr) - sizeof (lr_t));
1632 
1633 	itx->itx_sync = B_FALSE;
1634 	zil_itx_assign(zd->zd_zilog, itx, tx);
1635 }
1636 
1637 /*
1638  * ZIL replay ops
1639  */
1640 static int
1641 ztest_replay_create(void *arg1, void *arg2, boolean_t byteswap)
1642 {
1643 	ztest_ds_t *zd = arg1;
1644 	lr_create_t *lr = arg2;
1645 	char *name = (void *)(lr + 1);		/* name follows lr */
1646 	objset_t *os = zd->zd_os;
1647 	ztest_block_tag_t *bbt;
1648 	dmu_buf_t *db;
1649 	dmu_tx_t *tx;
1650 	uint64_t txg;
1651 	int error = 0;
1652 	int bonuslen;
1653 
1654 	if (byteswap)
1655 		byteswap_uint64_array(lr, sizeof (*lr));
1656 
1657 	ASSERT(lr->lr_doid == ZTEST_DIROBJ);
1658 	ASSERT(name[0] != '\0');
1659 
1660 	tx = dmu_tx_create(os);
1661 
1662 	dmu_tx_hold_zap(tx, lr->lr_doid, B_TRUE, name);
1663 
1664 	if (lr->lrz_type == DMU_OT_ZAP_OTHER) {
1665 		dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, B_TRUE, NULL);
1666 	} else {
1667 		dmu_tx_hold_bonus(tx, DMU_NEW_OBJECT);
1668 	}
1669 
1670 	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
1671 	if (txg == 0)
1672 		return (ENOSPC);
1673 
1674 	ASSERT(dmu_objset_zil(os)->zl_replay == !!lr->lr_foid);
1675 	bonuslen = DN_BONUS_SIZE(lr->lrz_dnodesize);
1676 
1677 	if (lr->lrz_type == DMU_OT_ZAP_OTHER) {
1678 		if (lr->lr_foid == 0) {
1679 			lr->lr_foid = zap_create_dnsize(os,
1680 			    lr->lrz_type, lr->lrz_bonustype,
1681 			    bonuslen, lr->lrz_dnodesize, tx);
1682 		} else {
1683 			error = zap_create_claim_dnsize(os, lr->lr_foid,
1684 			    lr->lrz_type, lr->lrz_bonustype,
1685 			    bonuslen, lr->lrz_dnodesize, tx);
1686 		}
1687 	} else {
1688 		if (lr->lr_foid == 0) {
1689 			lr->lr_foid = dmu_object_alloc_dnsize(os,
1690 			    lr->lrz_type, 0, lr->lrz_bonustype,
1691 			    bonuslen, lr->lrz_dnodesize, tx);
1692 		} else {
1693 			error = dmu_object_claim_dnsize(os, lr->lr_foid,
1694 			    lr->lrz_type, 0, lr->lrz_bonustype,
1695 			    bonuslen, lr->lrz_dnodesize, tx);
1696 		}
1697 	}
1698 
1699 	if (error) {
1700 		ASSERT3U(error, ==, EEXIST);
1701 		ASSERT(zd->zd_zilog->zl_replay);
1702 		dmu_tx_commit(tx);
1703 		return (error);
1704 	}
1705 
1706 	ASSERT(lr->lr_foid != 0);
1707 
1708 	if (lr->lrz_type != DMU_OT_ZAP_OTHER)
1709 		VERIFY3U(0, ==, dmu_object_set_blocksize(os, lr->lr_foid,
1710 		    lr->lrz_blocksize, lr->lrz_ibshift, tx));
1711 
1712 	VERIFY3U(0, ==, dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));
1713 	bbt = ztest_bt_bonus(db);
1714 	dmu_buf_will_dirty(db, tx);
1715 	ztest_bt_generate(bbt, os, lr->lr_foid, lr->lrz_dnodesize, -1ULL,
1716 	    lr->lr_gen, txg, txg);
1717 	ztest_fill_unused_bonus(db, bbt, lr->lr_foid, os, lr->lr_gen);
1718 	dmu_buf_rele(db, FTAG);
1719 
1720 	VERIFY3U(0, ==, zap_add(os, lr->lr_doid, name, sizeof (uint64_t), 1,
1721 	    &lr->lr_foid, tx));
1722 
1723 	(void) ztest_log_create(zd, tx, lr);
1724 
1725 	dmu_tx_commit(tx);
1726 
1727 	return (0);
1728 }
1729 
1730 static int
1731 ztest_replay_remove(void *arg1, void *arg2, boolean_t byteswap)
1732 {
1733 	ztest_ds_t *zd = arg1;
1734 	lr_remove_t *lr = arg2;
1735 	char *name = (void *)(lr + 1);		/* name follows lr */
1736 	objset_t *os = zd->zd_os;
1737 	dmu_object_info_t doi;
1738 	dmu_tx_t *tx;
1739 	uint64_t object, txg;
1740 
1741 	if (byteswap)
1742 		byteswap_uint64_array(lr, sizeof (*lr));
1743 
1744 	ASSERT(lr->lr_doid == ZTEST_DIROBJ);
1745 	ASSERT(name[0] != '\0');
1746 
1747 	VERIFY3U(0, ==,
1748 	    zap_lookup(os, lr->lr_doid, name, sizeof (object), 1, &object));
1749 	ASSERT(object != 0);
1750 
1751 	ztest_object_lock(zd, object, RL_WRITER);
1752 
1753 	VERIFY3U(0, ==, dmu_object_info(os, object, &doi));
1754 
1755 	tx = dmu_tx_create(os);
1756 
1757 	dmu_tx_hold_zap(tx, lr->lr_doid, B_FALSE, name);
1758 	dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END);
1759 
1760 	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
1761 	if (txg == 0) {
1762 		ztest_object_unlock(zd, object);
1763 		return (ENOSPC);
1764 	}
1765 
1766 	if (doi.doi_type == DMU_OT_ZAP_OTHER) {
1767 		VERIFY3U(0, ==, zap_destroy(os, object, tx));
1768 	} else {
1769 		VERIFY3U(0, ==, dmu_object_free(os, object, tx));
1770 	}
1771 
1772 	VERIFY3U(0, ==, zap_remove(os, lr->lr_doid, name, tx));
1773 
1774 	(void) ztest_log_remove(zd, tx, lr, object);
1775 
1776 	dmu_tx_commit(tx);
1777 
1778 	ztest_object_unlock(zd, object);
1779 
1780 	return (0);
1781 }
1782 
1783 static int
1784 ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
1785 {
1786 	ztest_ds_t *zd = arg1;
1787 	lr_write_t *lr = arg2;
1788 	objset_t *os = zd->zd_os;
1789 	void *data = lr + 1;			/* data follows lr */
1790 	uint64_t offset, length;
1791 	ztest_block_tag_t *bt = data;
1792 	ztest_block_tag_t *bbt;
1793 	uint64_t gen, txg, lrtxg, crtxg;
1794 	dmu_object_info_t doi;
1795 	dmu_tx_t *tx;
1796 	dmu_buf_t *db;
1797 	arc_buf_t *abuf = NULL;
1798 	rl_t *rl;
1799 
1800 	if (byteswap)
1801 		byteswap_uint64_array(lr, sizeof (*lr));
1802 
1803 	offset = lr->lr_offset;
1804 	length = lr->lr_length;
1805 
1806 	/* If it's a dmu_sync() block, write the whole block */
1807 	if (lr->lr_common.lrc_reclen == sizeof (lr_write_t)) {
1808 		uint64_t blocksize = BP_GET_LSIZE(&lr->lr_blkptr);
1809 		if (length < blocksize) {
1810 			offset -= offset % blocksize;
1811 			length = blocksize;
1812 		}
1813 	}
1814 
1815 	if (bt->bt_magic == BSWAP_64(BT_MAGIC))
1816 		byteswap_uint64_array(bt, sizeof (*bt));
1817 
1818 	if (bt->bt_magic != BT_MAGIC)
1819 		bt = NULL;
1820 
1821 	ztest_object_lock(zd, lr->lr_foid, RL_READER);
1822 	rl = ztest_range_lock(zd, lr->lr_foid, offset, length, RL_WRITER);
1823 
1824 	VERIFY3U(0, ==, dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));
1825 
1826 	dmu_object_info_from_db(db, &doi);
1827 
1828 	bbt = ztest_bt_bonus(db);
1829 	ASSERT3U(bbt->bt_magic, ==, BT_MAGIC);
1830 	gen = bbt->bt_gen;
1831 	crtxg = bbt->bt_crtxg;
1832 	lrtxg = lr->lr_common.lrc_txg;
1833 
1834 	tx = dmu_tx_create(os);
1835 
1836 	dmu_tx_hold_write(tx, lr->lr_foid, offset, length);
1837 
1838 	if (ztest_random(8) == 0 && length == doi.doi_data_block_size &&
1839 	    P2PHASE(offset, length) == 0)
1840 		abuf = dmu_request_arcbuf(db, length);
1841 
1842 	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
1843 	if (txg == 0) {
1844 		if (abuf != NULL)
1845 			dmu_return_arcbuf(abuf);
1846 		dmu_buf_rele(db, FTAG);
1847 		ztest_range_unlock(rl);
1848 		ztest_object_unlock(zd, lr->lr_foid);
1849 		return (ENOSPC);
1850 	}
1851 
1852 	if (bt != NULL) {
1853 		/*
1854 		 * Usually, verify the old data before writing new data --
1855 		 * but not always, because we also want to verify correct
1856 		 * behavior when the data was not recently read into cache.
1857 		 */
1858 		ASSERT(offset % doi.doi_data_block_size == 0);
1859 		if (ztest_random(4) != 0) {
1860 			int prefetch = ztest_random(2) ?
1861 			    DMU_READ_PREFETCH : DMU_READ_NO_PREFETCH;
1862 			ztest_block_tag_t rbt;
1863 
1864 			VERIFY(dmu_read(os, lr->lr_foid, offset,
1865 			    sizeof (rbt), &rbt, prefetch) == 0);
1866 			if (rbt.bt_magic == BT_MAGIC) {
1867 				ztest_bt_verify(&rbt, os, lr->lr_foid, 0,
1868 				    offset, gen, txg, crtxg);
1869 			}
1870 		}
1871 
1872 		/*
1873 		 * Writes can appear to be newer than the bonus buffer because
1874 		 * the ztest_get_data() callback does a dmu_read() of the
1875 		 * open-context data, which may be different than the data
1876 		 * as it was when the write was generated.
1877 		 */
1878 		if (zd->zd_zilog->zl_replay) {
1879 			ztest_bt_verify(bt, os, lr->lr_foid, 0, offset,
1880 			    MAX(gen, bt->bt_gen), MAX(txg, lrtxg),
1881 			    bt->bt_crtxg);
1882 		}
1883 
1884 		/*
1885 		 * Set the bt's gen/txg to the bonus buffer's gen/txg
1886 		 * so that all of the usual ASSERTs will work.
1887 		 */
1888 		ztest_bt_generate(bt, os, lr->lr_foid, 0, offset, gen, txg,
1889 		    crtxg);
1890 	}
1891 
1892 	if (abuf == NULL) {
1893 		dmu_write(os, lr->lr_foid, offset, length, data, tx);
1894 	} else {
1895 		bcopy(data, abuf->b_data, length);
1896 		dmu_assign_arcbuf_by_dbuf(db, offset, abuf, tx);
1897 	}
1898 
1899 	(void) ztest_log_write(zd, tx, lr);
1900 
1901 	dmu_buf_rele(db, FTAG);
1902 
1903 	dmu_tx_commit(tx);
1904 
1905 	ztest_range_unlock(rl);
1906 	ztest_object_unlock(zd, lr->lr_foid);
1907 
1908 	return (0);
1909 }
1910 
1911 static int
1912 ztest_replay_truncate(void *arg1, void *arg2, boolean_t byteswap)
1913 {
1914 	ztest_ds_t *zd = arg1;
1915 	lr_truncate_t *lr = arg2;
1916 	objset_t *os = zd->zd_os;
1917 	dmu_tx_t *tx;
1918 	uint64_t txg;
1919 	rl_t *rl;
1920 
1921 	if (byteswap)
1922 		byteswap_uint64_array(lr, sizeof (*lr));
1923 
1924 	ztest_object_lock(zd, lr->lr_foid, RL_READER);
1925 	rl = ztest_range_lock(zd, lr->lr_foid, lr->lr_offset, lr->lr_length,
1926 	    RL_WRITER);
1927 
1928 	tx = dmu_tx_create(os);
1929 
1930 	dmu_tx_hold_free(tx, lr->lr_foid, lr->lr_offset, lr->lr_length);
1931 
1932 	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
1933 	if (txg == 0) {
1934 		ztest_range_unlock(rl);
1935 		ztest_object_unlock(zd, lr->lr_foid);
1936 		return (ENOSPC);
1937 	}
1938 
1939 	VERIFY(dmu_free_range(os, lr->lr_foid, lr->lr_offset,
1940 	    lr->lr_length, tx) == 0);
1941 
1942 	(void) ztest_log_truncate(zd, tx, lr);
1943 
1944 	dmu_tx_commit(tx);
1945 
1946 	ztest_range_unlock(rl);
1947 	ztest_object_unlock(zd, lr->lr_foid);
1948 
1949 	return (0);
1950 }
1951 
1952 static int
1953 ztest_replay_setattr(void *arg1, void *arg2, boolean_t byteswap)
1954 {
1955 	ztest_ds_t *zd = arg1;
1956 	lr_setattr_t *lr = arg2;
1957 	objset_t *os = zd->zd_os;
1958 	dmu_tx_t *tx;
1959 	dmu_buf_t *db;
1960 	ztest_block_tag_t *bbt;
1961 	uint64_t txg, lrtxg, crtxg, dnodesize;
1962 
1963 	if (byteswap)
1964 		byteswap_uint64_array(lr, sizeof (*lr));
1965 
1966 	ztest_object_lock(zd, lr->lr_foid, RL_WRITER);
1967 
1968 	VERIFY3U(0, ==, dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));
1969 
1970 	tx = dmu_tx_create(os);
1971 	dmu_tx_hold_bonus(tx, lr->lr_foid);
1972 
1973 	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
1974 	if (txg == 0) {
1975 		dmu_buf_rele(db, FTAG);
1976 		ztest_object_unlock(zd, lr->lr_foid);
1977 		return (ENOSPC);
1978 	}
1979 
1980 	bbt = ztest_bt_bonus(db);
1981 	ASSERT3U(bbt->bt_magic, ==, BT_MAGIC);
1982 	crtxg = bbt->bt_crtxg;
1983 	lrtxg = lr->lr_common.lrc_txg;
1984 	dnodesize = bbt->bt_dnodesize;
1985 
1986 	if (zd->zd_zilog->zl_replay) {
1987 		ASSERT(lr->lr_size != 0);
1988 		ASSERT(lr->lr_mode != 0);
1989 		ASSERT(lrtxg != 0);
1990 	} else {
1991 		/*
1992 		 * Randomly change the size and increment the generation.
1993 		 */
1994 		lr->lr_size = (ztest_random(db->db_size / sizeof (*bbt)) + 1) *
1995 		    sizeof (*bbt);
1996 		lr->lr_mode = bbt->bt_gen + 1;
1997 		ASSERT(lrtxg == 0);
1998 	}
1999 
2000 	/*
2001 	 * Verify that the current bonus buffer is not newer than our txg.
2002 	 */
2003 	ztest_bt_verify(bbt, os, lr->lr_foid, dnodesize, -1ULL, lr->lr_mode,
2004 	    MAX(txg, lrtxg), crtxg);
2005 
2006 	dmu_buf_will_dirty(db, tx);
2007 
2008 	ASSERT3U(lr->lr_size, >=, sizeof (*bbt));
2009 	ASSERT3U(lr->lr_size, <=, db->db_size);
2010 	VERIFY0(dmu_set_bonus(db, lr->lr_size, tx));
2011 	bbt = ztest_bt_bonus(db);
2012 
2013 	ztest_bt_generate(bbt, os, lr->lr_foid, dnodesize, -1ULL, lr->lr_mode,
2014 	    txg, crtxg);
2015 	ztest_fill_unused_bonus(db, bbt, lr->lr_foid, os, bbt->bt_gen);
2016 
2017 	dmu_buf_rele(db, FTAG);
2018 
2019 	(void) ztest_log_setattr(zd, tx, lr);
2020 
2021 	dmu_tx_commit(tx);
2022 
2023 	ztest_object_unlock(zd, lr->lr_foid);
2024 
2025 	return (0);
2026 }
2027 
2028 zil_replay_func_t *ztest_replay_vector[TX_MAX_TYPE] = {
2029 	NULL,			/* 0 no such transaction type */
2030 	ztest_replay_create,	/* TX_CREATE */
2031 	NULL,			/* TX_MKDIR */
2032 	NULL,			/* TX_MKXATTR */
2033 	NULL,			/* TX_SYMLINK */
2034 	ztest_replay_remove,	/* TX_REMOVE */
2035 	NULL,			/* TX_RMDIR */
2036 	NULL,			/* TX_LINK */
2037 	NULL,			/* TX_RENAME */
2038 	ztest_replay_write,	/* TX_WRITE */
2039 	ztest_replay_truncate,	/* TX_TRUNCATE */
2040 	ztest_replay_setattr,	/* TX_SETATTR */
2041 	NULL,			/* TX_ACL */
2042 	NULL,			/* TX_CREATE_ACL */
2043 	NULL,			/* TX_CREATE_ATTR */
2044 	NULL,			/* TX_CREATE_ACL_ATTR */
2045 	NULL,			/* TX_MKDIR_ACL */
2046 	NULL,			/* TX_MKDIR_ATTR */
2047 	NULL,			/* TX_MKDIR_ACL_ATTR */
2048 	NULL,			/* TX_WRITE2 */
2049 };
2050 
2051 /*
2052  * ZIL get_data callbacks
2053  */
2054 
2055 /* ARGSUSED */
2056 static void
2057 ztest_get_done(zgd_t *zgd, int error)
2058 {
2059 	ztest_ds_t *zd = zgd->zgd_private;
2060 	uint64_t object = ((rl_t *)zgd->zgd_lr)->rl_object;
2061 
2062 	if (zgd->zgd_db)
2063 		dmu_buf_rele(zgd->zgd_db, zgd);
2064 
2065 	ztest_range_unlock((rl_t *)zgd->zgd_lr);
2066 	ztest_object_unlock(zd, object);
2067 
2068 	umem_free(zgd, sizeof (*zgd));
2069 }
2070 
2071 static int
2072 ztest_get_data(void *arg, lr_write_t *lr, char *buf, struct lwb *lwb,
2073     zio_t *zio)
2074 {
2075 	ztest_ds_t *zd = arg;
2076 	objset_t *os = zd->zd_os;
2077 	uint64_t object = lr->lr_foid;
2078 	uint64_t offset = lr->lr_offset;
2079 	uint64_t size = lr->lr_length;
2080 	uint64_t txg = lr->lr_common.lrc_txg;
2081 	uint64_t crtxg;
2082 	dmu_object_info_t doi;
2083 	dmu_buf_t *db;
2084 	zgd_t *zgd;
2085 	int error;
2086 
2087 	ASSERT3P(lwb, !=, NULL);
2088 	ASSERT3P(zio, !=, NULL);
2089 	ASSERT3U(size, !=, 0);
2090 
2091 	ztest_object_lock(zd, object, RL_READER);
2092 	error = dmu_bonus_hold(os, object, FTAG, &db);
2093 	if (error) {
2094 		ztest_object_unlock(zd, object);
2095 		return (error);
2096 	}
2097 
2098 	crtxg = ztest_bt_bonus(db)->bt_crtxg;
2099 
2100 	if (crtxg == 0 || crtxg > txg) {
2101 		dmu_buf_rele(db, FTAG);
2102 		ztest_object_unlock(zd, object);
2103 		return (ENOENT);
2104 	}
2105 
2106 	dmu_object_info_from_db(db, &doi);
2107 	dmu_buf_rele(db, FTAG);
2108 	db = NULL;
2109 
2110 	zgd = umem_zalloc(sizeof (*zgd), UMEM_NOFAIL);
2111 	zgd->zgd_lwb = lwb;
2112 	zgd->zgd_private = zd;
2113 
2114 	if (buf != NULL) {	/* immediate write */
2115 		zgd->zgd_lr = (struct locked_range *)ztest_range_lock(zd,
2116 		    object, offset, size, RL_READER);
2117 
2118 		error = dmu_read(os, object, offset, size, buf,
2119 		    DMU_READ_NO_PREFETCH);
2120 		ASSERT(error == 0);
2121 	} else {
2122 		size = doi.doi_data_block_size;
2123 		if (ISP2(size)) {
2124 			offset = P2ALIGN(offset, size);
2125 		} else {
2126 			ASSERT(offset < size);
2127 			offset = 0;
2128 		}
2129 
2130 		zgd->zgd_lr = (struct locked_range *)ztest_range_lock(zd,
2131 		    object, offset, size, RL_READER);
2132 
2133 		error = dmu_buf_hold(os, object, offset, zgd, &db,
2134 		    DMU_READ_NO_PREFETCH);
2135 
2136 		if (error == 0) {
2137 			blkptr_t *bp = &lr->lr_blkptr;
2138 
2139 			zgd->zgd_db = db;
2140 			zgd->zgd_bp = bp;
2141 
2142 			ASSERT(db->db_offset == offset);
2143 			ASSERT(db->db_size == size);
2144 
2145 			error = dmu_sync(zio, lr->lr_common.lrc_txg,
2146 			    ztest_get_done, zgd);
2147 
2148 			if (error == 0)
2149 				return (0);
2150 		}
2151 	}
2152 
2153 	ztest_get_done(zgd, error);
2154 
2155 	return (error);
2156 }
2157 
2158 static void *
2159 ztest_lr_alloc(size_t lrsize, char *name)
2160 {
2161 	char *lr;
2162 	size_t namesize = name ? strlen(name) + 1 : 0;
2163 
2164 	lr = umem_zalloc(lrsize + namesize, UMEM_NOFAIL);
2165 
2166 	if (name)
2167 		bcopy(name, lr + lrsize, namesize);
2168 
2169 	return (lr);
2170 }
2171 
2172 void
2173 ztest_lr_free(void *lr, size_t lrsize, char *name)
2174 {
2175 	size_t namesize = name ? strlen(name) + 1 : 0;
2176 
2177 	umem_free(lr, lrsize + namesize);
2178 }
2179 
2180 /*
2181  * Lookup a bunch of objects.  Returns the number of objects not found.
2182  */
2183 static int
2184 ztest_lookup(ztest_ds_t *zd, ztest_od_t *od, int count)
2185 {
2186 	int missing = 0;
2187 	int error;
2188 
2189 	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
2190 
2191 	for (int i = 0; i < count; i++, od++) {
2192 		od->od_object = 0;
2193 		error = zap_lookup(zd->zd_os, od->od_dir, od->od_name,
2194 		    sizeof (uint64_t), 1, &od->od_object);
2195 		if (error) {
2196 			ASSERT(error == ENOENT);
2197 			ASSERT(od->od_object == 0);
2198 			missing++;
2199 		} else {
2200 			dmu_buf_t *db;
2201 			ztest_block_tag_t *bbt;
2202 			dmu_object_info_t doi;
2203 
2204 			ASSERT(od->od_object != 0);
2205 			ASSERT(missing == 0);	/* there should be no gaps */
2206 
2207 			ztest_object_lock(zd, od->od_object, RL_READER);
2208 			VERIFY3U(0, ==, dmu_bonus_hold(zd->zd_os,
2209 			    od->od_object, FTAG, &db));
2210 			dmu_object_info_from_db(db, &doi);
2211 			bbt = ztest_bt_bonus(db);
2212 			ASSERT3U(bbt->bt_magic, ==, BT_MAGIC);
2213 			od->od_type = doi.doi_type;
2214 			od->od_blocksize = doi.doi_data_block_size;
2215 			od->od_gen = bbt->bt_gen;
2216 			dmu_buf_rele(db, FTAG);
2217 			ztest_object_unlock(zd, od->od_object);
2218 		}
2219 	}
2220 
2221 	return (missing);
2222 }
2223 
2224 static int
2225 ztest_create(ztest_ds_t *zd, ztest_od_t *od, int count)
2226 {
2227 	int missing = 0;
2228 
2229 	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
2230 
2231 	for (int i = 0; i < count; i++, od++) {
2232 		if (missing) {
2233 			od->od_object = 0;
2234 			missing++;
2235 			continue;
2236 		}
2237 
2238 		lr_create_t *lr = ztest_lr_alloc(sizeof (*lr), od->od_name);
2239 
2240 		lr->lr_doid = od->od_dir;
2241 		lr->lr_foid = 0;	/* 0 to allocate, > 0 to claim */
2242 		lr->lrz_type = od->od_crtype;
2243 		lr->lrz_blocksize = od->od_crblocksize;
2244 		lr->lrz_ibshift = ztest_random_ibshift();
2245 		lr->lrz_bonustype = DMU_OT_UINT64_OTHER;
2246 		lr->lrz_dnodesize = od->od_crdnodesize;
2247 		lr->lr_gen = od->od_crgen;
2248 		lr->lr_crtime[0] = time(NULL);
2249 
2250 		if (ztest_replay_create(zd, lr, B_FALSE) != 0) {
2251 			ASSERT(missing == 0);
2252 			od->od_object = 0;
2253 			missing++;
2254 		} else {
2255 			od->od_object = lr->lr_foid;
2256 			od->od_type = od->od_crtype;
2257 			od->od_blocksize = od->od_crblocksize;
2258 			od->od_gen = od->od_crgen;
2259 			ASSERT(od->od_object != 0);
2260 		}
2261 
2262 		ztest_lr_free(lr, sizeof (*lr), od->od_name);
2263 	}
2264 
2265 	return (missing);
2266 }
2267 
2268 static int
2269 ztest_remove(ztest_ds_t *zd, ztest_od_t *od, int count)
2270 {
2271 	int missing = 0;
2272 	int error;
2273 
2274 	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
2275 
2276 	od += count - 1;
2277 
2278 	for (int i = count - 1; i >= 0; i--, od--) {
2279 		if (missing) {
2280 			missing++;
2281 			continue;
2282 		}
2283 
2284 		/*
2285 		 * No object was found.
2286 		 */
2287 		if (od->od_object == 0)
2288 			continue;
2289 
2290 		lr_remove_t *lr = ztest_lr_alloc(sizeof (*lr), od->od_name);
2291 
2292 		lr->lr_doid = od->od_dir;
2293 
2294 		if ((error = ztest_replay_remove(zd, lr, B_FALSE)) != 0) {
2295 			ASSERT3U(error, ==, ENOSPC);
2296 			missing++;
2297 		} else {
2298 			od->od_object = 0;
2299 		}
2300 		ztest_lr_free(lr, sizeof (*lr), od->od_name);
2301 	}
2302 
2303 	return (missing);
2304 }
2305 
2306 static int
2307 ztest_write(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size,
2308     void *data)
2309 {
2310 	lr_write_t *lr;
2311 	int error;
2312 
2313 	lr = ztest_lr_alloc(sizeof (*lr) + size, NULL);
2314 
2315 	lr->lr_foid = object;
2316 	lr->lr_offset = offset;
2317 	lr->lr_length = size;
2318 	lr->lr_blkoff = 0;
2319 	BP_ZERO(&lr->lr_blkptr);
2320 
2321 	bcopy(data, lr + 1, size);
2322 
2323 	error = ztest_replay_write(zd, lr, B_FALSE);
2324 
2325 	ztest_lr_free(lr, sizeof (*lr) + size, NULL);
2326 
2327 	return (error);
2328 }
2329 
2330 static int
2331 ztest_truncate(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size)
2332 {
2333 	lr_truncate_t *lr;
2334 	int error;
2335 
2336 	lr = ztest_lr_alloc(sizeof (*lr), NULL);
2337 
2338 	lr->lr_foid = object;
2339 	lr->lr_offset = offset;
2340 	lr->lr_length = size;
2341 
2342 	error = ztest_replay_truncate(zd, lr, B_FALSE);
2343 
2344 	ztest_lr_free(lr, sizeof (*lr), NULL);
2345 
2346 	return (error);
2347 }
2348 
2349 static int
2350 ztest_setattr(ztest_ds_t *zd, uint64_t object)
2351 {
2352 	lr_setattr_t *lr;
2353 	int error;
2354 
2355 	lr = ztest_lr_alloc(sizeof (*lr), NULL);
2356 
2357 	lr->lr_foid = object;
2358 	lr->lr_size = 0;
2359 	lr->lr_mode = 0;
2360 
2361 	error = ztest_replay_setattr(zd, lr, B_FALSE);
2362 
2363 	ztest_lr_free(lr, sizeof (*lr), NULL);
2364 
2365 	return (error);
2366 }
2367 
2368 static void
2369 ztest_prealloc(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size)
2370 {
2371 	objset_t *os = zd->zd_os;
2372 	dmu_tx_t *tx;
2373 	uint64_t txg;
2374 	rl_t *rl;
2375 
2376 	txg_wait_synced(dmu_objset_pool(os), 0);
2377 
2378 	ztest_object_lock(zd, object, RL_READER);
2379 	rl = ztest_range_lock(zd, object, offset, size, RL_WRITER);
2380 
2381 	tx = dmu_tx_create(os);
2382 
2383 	dmu_tx_hold_write(tx, object, offset, size);
2384 
2385 	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
2386 
2387 	if (txg != 0) {
2388 		dmu_prealloc(os, object, offset, size, tx);
2389 		dmu_tx_commit(tx);
2390 		txg_wait_synced(dmu_objset_pool(os), txg);
2391 	} else {
2392 		(void) dmu_free_long_range(os, object, offset, size);
2393 	}
2394 
2395 	ztest_range_unlock(rl);
2396 	ztest_object_unlock(zd, object);
2397 }
2398 
2399 static void
2400 ztest_io(ztest_ds_t *zd, uint64_t object, uint64_t offset)
2401 {
2402 	int err;
2403 	ztest_block_tag_t wbt;
2404 	dmu_object_info_t doi;
2405 	enum ztest_io_type io_type;
2406 	uint64_t blocksize;
2407 	void *data;
2408 
2409 	VERIFY(dmu_object_info(zd->zd_os, object, &doi) == 0);
2410 	blocksize = doi.doi_data_block_size;
2411 	data = umem_alloc(blocksize, UMEM_NOFAIL);
2412 
2413 	/*
2414 	 * Pick an i/o type at random, biased toward writing block tags.
2415 	 */
2416 	io_type = ztest_random(ZTEST_IO_TYPES);
2417 	if (ztest_random(2) == 0)
2418 		io_type = ZTEST_IO_WRITE_TAG;
2419 
2420 	rw_enter(&zd->zd_zilog_lock, RW_READER);
2421 
2422 	switch (io_type) {
2423 
2424 	case ZTEST_IO_WRITE_TAG:
2425 		ztest_bt_generate(&wbt, zd->zd_os, object, doi.doi_dnodesize,
2426 		    offset, 0, 0, 0);
2427 		(void) ztest_write(zd, object, offset, sizeof (wbt), &wbt);
2428 		break;
2429 
2430 	case ZTEST_IO_WRITE_PATTERN:
2431 		(void) memset(data, 'a' + (object + offset) % 5, blocksize);
2432 		if (ztest_random(2) == 0) {
2433 			/*
2434 			 * Induce fletcher2 collisions to ensure that
2435 			 * zio_ddt_collision() detects and resolves them
2436 			 * when using fletcher2-verify for deduplication.
2437 			 */
2438 			((uint64_t *)data)[0] ^= 1ULL << 63;
2439 			((uint64_t *)data)[4] ^= 1ULL << 63;
2440 		}
2441 		(void) ztest_write(zd, object, offset, blocksize, data);
2442 		break;
2443 
2444 	case ZTEST_IO_WRITE_ZEROES:
2445 		bzero(data, blocksize);
2446 		(void) ztest_write(zd, object, offset, blocksize, data);
2447 		break;
2448 
2449 	case ZTEST_IO_TRUNCATE:
2450 		(void) ztest_truncate(zd, object, offset, blocksize);
2451 		break;
2452 
2453 	case ZTEST_IO_SETATTR:
2454 		(void) ztest_setattr(zd, object);
2455 		break;
2456 
2457 	case ZTEST_IO_REWRITE:
2458 		rw_enter(&ztest_name_lock, RW_READER);
2459 		err = ztest_dsl_prop_set_uint64(zd->zd_name,
2460 		    ZFS_PROP_CHECKSUM, spa_dedup_checksum(ztest_spa),
2461 		    B_FALSE);
2462 		VERIFY(err == 0 || err == ENOSPC);
2463 		err = ztest_dsl_prop_set_uint64(zd->zd_name,
2464 		    ZFS_PROP_COMPRESSION,
2465 		    ztest_random_dsl_prop(ZFS_PROP_COMPRESSION),
2466 		    B_FALSE);
2467 		VERIFY(err == 0 || err == ENOSPC);
2468 		rw_exit(&ztest_name_lock);
2469 
2470 		VERIFY0(dmu_read(zd->zd_os, object, offset, blocksize, data,
2471 		    DMU_READ_NO_PREFETCH));
2472 
2473 		(void) ztest_write(zd, object, offset, blocksize, data);
2474 		break;
2475 	}
2476 
2477 	rw_exit(&zd->zd_zilog_lock);
2478 
2479 	umem_free(data, blocksize);
2480 }
2481 
2482 /*
2483  * Initialize an object description template.
2484  */
2485 static void
2486 ztest_od_init(ztest_od_t *od, uint64_t id, char *tag, uint64_t index,
2487     dmu_object_type_t type, uint64_t blocksize, uint64_t dnodesize,
2488     uint64_t gen)
2489 {
2490 	od->od_dir = ZTEST_DIROBJ;
2491 	od->od_object = 0;
2492 
2493 	od->od_crtype = type;
2494 	od->od_crblocksize = blocksize ? blocksize : ztest_random_blocksize();
2495 	od->od_crdnodesize = dnodesize ? dnodesize : ztest_random_dnodesize();
2496 	od->od_crgen = gen;
2497 
2498 	od->od_type = DMU_OT_NONE;
2499 	od->od_blocksize = 0;
2500 	od->od_gen = 0;
2501 
2502 	(void) snprintf(od->od_name, sizeof (od->od_name), "%s(%lld)[%llu]",
2503 	    tag, (int64_t)id, index);
2504 }
2505 
2506 /*
2507  * Lookup or create the objects for a test using the od template.
2508  * If the objects do not all exist, or if 'remove' is specified,
2509  * remove any existing objects and create new ones.  Otherwise,
2510  * use the existing objects.
2511  */
2512 static int
2513 ztest_object_init(ztest_ds_t *zd, ztest_od_t *od, size_t size, boolean_t remove)
2514 {
2515 	int count = size / sizeof (*od);
2516 	int rv = 0;
2517 
2518 	mutex_enter(&zd->zd_dirobj_lock);
2519 	if ((ztest_lookup(zd, od, count) != 0 || remove) &&
2520 	    (ztest_remove(zd, od, count) != 0 ||
2521 	    ztest_create(zd, od, count) != 0))
2522 		rv = -1;
2523 	zd->zd_od = od;
2524 	mutex_exit(&zd->zd_dirobj_lock);
2525 
2526 	return (rv);
2527 }
2528 
2529 /* ARGSUSED */
2530 void
2531 ztest_zil_commit(ztest_ds_t *zd, uint64_t id)
2532 {
2533 	zilog_t *zilog = zd->zd_zilog;
2534 
2535 	rw_enter(&zd->zd_zilog_lock, RW_READER);
2536 
2537 	zil_commit(zilog, ztest_random(ZTEST_OBJECTS));
2538 
2539 	/*
2540 	 * Remember the committed values in zd, which is in parent/child
2541 	 * shared memory.  If we die, the next iteration of ztest_run()
2542 	 * will verify that the log really does contain this record.
2543 	 */
2544 	mutex_enter(&zilog->zl_lock);
2545 	ASSERT(zd->zd_shared != NULL);
2546 	ASSERT3U(zd->zd_shared->zd_seq, <=, zilog->zl_commit_lr_seq);
2547 	zd->zd_shared->zd_seq = zilog->zl_commit_lr_seq;
2548 	mutex_exit(&zilog->zl_lock);
2549 
2550 	rw_exit(&zd->zd_zilog_lock);
2551 }
2552 
2553 /*
2554  * This function is designed to simulate the operations that occur during a
2555  * mount/unmount operation.  We hold the dataset across these operations in an
2556  * attempt to expose any implicit assumptions about ZIL management.
2557  */
2558 /* ARGSUSED */
2559 void
2560 ztest_zil_remount(ztest_ds_t *zd, uint64_t id)
2561 {
2562 	objset_t *os = zd->zd_os;
2563 
2564 	/*
2565 	 * We grab the zd_dirobj_lock to ensure that no other thread is
2566 	 * updating the zil (i.e. adding in-memory log records) and the
2567 	 * zd_zilog_lock to block any I/O.
2568 	 */
2569 	mutex_enter(&zd->zd_dirobj_lock);
2570 	rw_enter(&zd->zd_zilog_lock, RW_WRITER);
2571 
2572 	/* zfsvfs_teardown() */
2573 	zil_close(zd->zd_zilog);
2574 
2575 	/* zfsvfs_setup() */
2576 	VERIFY(zil_open(os, ztest_get_data) == zd->zd_zilog);
2577 	zil_replay(os, zd, ztest_replay_vector);
2578 
2579 	rw_exit(&zd->zd_zilog_lock);
2580 	mutex_exit(&zd->zd_dirobj_lock);
2581 }
2582 
2583 /*
2584  * Verify that we can't destroy an active pool, create an existing pool,
2585  * or create a pool with a bad vdev spec.
2586  */
2587 /* ARGSUSED */
2588 void
2589 ztest_spa_create_destroy(ztest_ds_t *zd, uint64_t id)
2590 {
2591 	ztest_shared_opts_t *zo = &ztest_opts;
2592 	spa_t *spa;
2593 	nvlist_t *nvroot;
2594 
2595 	if (zo->zo_mmp_test)
2596 		return;
2597 
2598 	/*
2599 	 * Attempt to create using a bad file.
2600 	 */
2601 	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, NULL, 0, 0, 1);
2602 	VERIFY3U(ENOENT, ==,
2603 	    spa_create("ztest_bad_file", nvroot, NULL, NULL, NULL));
2604 	nvlist_free(nvroot);
2605 
2606 	/*
2607 	 * Attempt to create using a bad mirror.
2608 	 */
2609 	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, NULL, 0, 2, 1);
2610 	VERIFY3U(ENOENT, ==,
2611 	    spa_create("ztest_bad_mirror", nvroot, NULL, NULL, NULL));
2612 	nvlist_free(nvroot);
2613 
2614 	/*
2615 	 * Attempt to create an existing pool.  It shouldn't matter
2616 	 * what's in the nvroot; we should fail with EEXIST.
2617 	 */
2618 	rw_enter(&ztest_name_lock, RW_READER);
2619 	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, NULL, 0, 0, 1);
2620 	VERIFY3U(EEXIST, ==,
2621 	    spa_create(zo->zo_pool, nvroot, NULL, NULL, NULL));
2622 	nvlist_free(nvroot);
2623 	VERIFY3U(0, ==, spa_open(zo->zo_pool, &spa, FTAG));
2624 	VERIFY3U(EBUSY, ==, spa_destroy(zo->zo_pool));
2625 	spa_close(spa, FTAG);
2626 
2627 	rw_exit(&ztest_name_lock);
2628 }
2629 
2630 /*
2631  * Start and then stop the MMP threads to ensure the startup and shutdown code
2632  * works properly.  Actual protection and property-related code tested via ZTS.
2633  */
2634 /* ARGSUSED */
2635 void
2636 ztest_mmp_enable_disable(ztest_ds_t *zd, uint64_t id)
2637 {
2638 	ztest_shared_opts_t *zo = &ztest_opts;
2639 	spa_t *spa = ztest_spa;
2640 
2641 	if (zo->zo_mmp_test)
2642 		return;
2643 
2644 	/*
2645 	 * Since enabling MMP involves setting a property, it could not be done
2646 	 * while the pool is suspended.
2647 	 */
2648 	if (spa_suspended(spa))
2649 		return;
2650 
2651 	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
2652 	mutex_enter(&spa->spa_props_lock);
2653 
2654 	zfs_multihost_fail_intervals = 0;
2655 
2656 	if (!spa_multihost(spa)) {
2657 		spa->spa_multihost = B_TRUE;
2658 		mmp_thread_start(spa);
2659 	}
2660 
2661 	mutex_exit(&spa->spa_props_lock);
2662 	spa_config_exit(spa, SCL_CONFIG, FTAG);
2663 
2664 	txg_wait_synced(spa_get_dsl(spa), 0);
2665 	mmp_signal_all_threads();
2666 	txg_wait_synced(spa_get_dsl(spa), 0);
2667 
2668 	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
2669 	mutex_enter(&spa->spa_props_lock);
2670 
2671 	if (spa_multihost(spa)) {
2672 		mmp_thread_stop(spa);
2673 		spa->spa_multihost = B_FALSE;
2674 	}
2675 
2676 	mutex_exit(&spa->spa_props_lock);
2677 	spa_config_exit(spa, SCL_CONFIG, FTAG);
2678 }
2679 
2680 /* ARGSUSED */
2681 void
2682 ztest_spa_upgrade(ztest_ds_t *zd, uint64_t id)
2683 {
2684 	spa_t *spa;
2685 	uint64_t initial_version = SPA_VERSION_INITIAL;
2686 	uint64_t version, newversion;
2687 	nvlist_t *nvroot, *props;
2688 	char *name;
2689 
2690 	if (ztest_opts.zo_mmp_test)
2691 		return;
2692 
2693 	mutex_enter(&ztest_vdev_lock);
2694 	name = kmem_asprintf("%s_upgrade", ztest_opts.zo_pool);
2695 
2696 	/*
2697 	 * Clean up from previous runs.
2698 	 */
2699 	(void) spa_destroy(name);
2700 
2701 	nvroot = make_vdev_root(NULL, NULL, name, ztest_opts.zo_vdev_size, 0,
2702 	    NULL, ztest_opts.zo_raidz, ztest_opts.zo_mirrors, 1);
2703 
2704 	/*
2705 	 * If we're configuring a RAIDZ device then make sure that the
2706 	 * the initial version is capable of supporting that feature.
2707 	 */
2708 	switch (ztest_opts.zo_raidz_parity) {
2709 	case 0:
2710 	case 1:
2711 		initial_version = SPA_VERSION_INITIAL;
2712 		break;
2713 	case 2:
2714 		initial_version = SPA_VERSION_RAIDZ2;
2715 		break;
2716 	case 3:
2717 		initial_version = SPA_VERSION_RAIDZ3;
2718 		break;
2719 	}
2720 
2721 	/*
2722 	 * Create a pool with a spa version that can be upgraded. Pick
2723 	 * a value between initial_version and SPA_VERSION_BEFORE_FEATURES.
2724 	 */
2725 	do {
2726 		version = ztest_random_spa_version(initial_version);
2727 	} while (version > SPA_VERSION_BEFORE_FEATURES);
2728 
2729 	props = fnvlist_alloc();
2730 	fnvlist_add_uint64(props,
2731 	    zpool_prop_to_name(ZPOOL_PROP_VERSION), version);
2732 	VERIFY0(spa_create(name, nvroot, props, NULL, NULL));
2733 	fnvlist_free(nvroot);
2734 	fnvlist_free(props);
2735 
2736 	VERIFY0(spa_open(name, &spa, FTAG));
2737 	VERIFY3U(spa_version(spa), ==, version);
2738 	newversion = ztest_random_spa_version(version + 1);
2739 
2740 	if (ztest_opts.zo_verbose >= 4) {
2741 		(void) printf("upgrading spa version from %llu to %llu\n",
2742 		    (u_longlong_t)version, (u_longlong_t)newversion);
2743 	}
2744 
2745 	spa_upgrade(spa, newversion);
2746 	VERIFY3U(spa_version(spa), >, version);
2747 	VERIFY3U(spa_version(spa), ==, fnvlist_lookup_uint64(spa->spa_config,
2748 	    zpool_prop_to_name(ZPOOL_PROP_VERSION)));
2749 	spa_close(spa, FTAG);
2750 
2751 	strfree(name);
2752 	mutex_exit(&ztest_vdev_lock);
2753 }
2754 
2755 static void
2756 ztest_spa_checkpoint(spa_t *spa)
2757 {
2758 	ASSERT(MUTEX_HELD(&ztest_checkpoint_lock));
2759 
2760 	int error = spa_checkpoint(spa->spa_name);
2761 
2762 	switch (error) {
2763 	case 0:
2764 	case ZFS_ERR_DEVRM_IN_PROGRESS:
2765 	case ZFS_ERR_DISCARDING_CHECKPOINT:
2766 	case ZFS_ERR_CHECKPOINT_EXISTS:
2767 		break;
2768 	case ENOSPC:
2769 		ztest_record_enospc(FTAG);
2770 		break;
2771 	default:
2772 		fatal(0, "spa_checkpoint(%s) = %d", spa->spa_name, error);
2773 	}
2774 }
2775 
2776 static void
2777 ztest_spa_discard_checkpoint(spa_t *spa)
2778 {
2779 	ASSERT(MUTEX_HELD(&ztest_checkpoint_lock));
2780 
2781 	int error = spa_checkpoint_discard(spa->spa_name);
2782 
2783 	switch (error) {
2784 	case 0:
2785 	case ZFS_ERR_DISCARDING_CHECKPOINT:
2786 	case ZFS_ERR_NO_CHECKPOINT:
2787 		break;
2788 	default:
2789 		fatal(0, "spa_discard_checkpoint(%s) = %d",
2790 		    spa->spa_name, error);
2791 	}
2792 
2793 }
2794 
2795 /* ARGSUSED */
2796 void
2797 ztest_spa_checkpoint_create_discard(ztest_ds_t *zd, uint64_t id)
2798 {
2799 	spa_t *spa = ztest_spa;
2800 
2801 	mutex_enter(&ztest_checkpoint_lock);
2802 	if (ztest_random(2) == 0) {
2803 		ztest_spa_checkpoint(spa);
2804 	} else {
2805 		ztest_spa_discard_checkpoint(spa);
2806 	}
2807 	mutex_exit(&ztest_checkpoint_lock);
2808 }
2809 
2810 
2811 static vdev_t *
2812 vdev_lookup_by_path(vdev_t *vd, const char *path)
2813 {
2814 	vdev_t *mvd;
2815 
2816 	if (vd->vdev_path != NULL && strcmp(path, vd->vdev_path) == 0)
2817 		return (vd);
2818 
2819 	for (int c = 0; c < vd->vdev_children; c++)
2820 		if ((mvd = vdev_lookup_by_path(vd->vdev_child[c], path)) !=
2821 		    NULL)
2822 			return (mvd);
2823 
2824 	return (NULL);
2825 }
2826 
2827 /*
2828  * Find the first available hole which can be used as a top-level.
2829  */
2830 int
2831 find_vdev_hole(spa_t *spa)
2832 {
2833 	vdev_t *rvd = spa->spa_root_vdev;
2834 	int c;
2835 
2836 	ASSERT(spa_config_held(spa, SCL_VDEV, RW_READER) == SCL_VDEV);
2837 
2838 	for (c = 0; c < rvd->vdev_children; c++) {
2839 		vdev_t *cvd = rvd->vdev_child[c];
2840 
2841 		if (cvd->vdev_ishole)
2842 			break;
2843 	}
2844 	return (c);
2845 }
2846 
2847 /*
2848  * Verify that vdev_add() works as expected.
2849  */
2850 /* ARGSUSED */
2851 void
2852 ztest_vdev_add_remove(ztest_ds_t *zd, uint64_t id)
2853 {
2854 	ztest_shared_t *zs = ztest_shared;
2855 	spa_t *spa = ztest_spa;
2856 	uint64_t leaves;
2857 	uint64_t guid;
2858 	nvlist_t *nvroot;
2859 	int error;
2860 
2861 	if (ztest_opts.zo_mmp_test)
2862 		return;
2863 
2864 	mutex_enter(&ztest_vdev_lock);
2865 	leaves = MAX(zs->zs_mirrors + zs->zs_splits, 1) * ztest_opts.zo_raidz;
2866 
2867 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
2868 
2869 	ztest_shared->zs_vdev_next_leaf = find_vdev_hole(spa) * leaves;
2870 
2871 	/*
2872 	 * If we have slogs then remove them 1/4 of the time.
2873 	 */
2874 	if (spa_has_slogs(spa) && ztest_random(4) == 0) {
2875 		metaslab_group_t *mg;
2876 
2877 		/*
2878 		 * find the first real slog in log allocation class
2879 		 */
2880 		mg =  spa_log_class(spa)->mc_rotor;
2881 		while (!mg->mg_vd->vdev_islog)
2882 			mg = mg->mg_next;
2883 
2884 		guid = mg->mg_vd->vdev_guid;
2885 
2886 		spa_config_exit(spa, SCL_VDEV, FTAG);
2887 
2888 		/*
2889 		 * We have to grab the zs_name_lock as writer to
2890 		 * prevent a race between removing a slog (dmu_objset_find)
2891 		 * and destroying a dataset. Removing the slog will
2892 		 * grab a reference on the dataset which may cause
2893 		 * dmu_objset_destroy() to fail with EBUSY thus
2894 		 * leaving the dataset in an inconsistent state.
2895 		 */
2896 		rw_enter(&ztest_name_lock, RW_WRITER);
2897 		error = spa_vdev_remove(spa, guid, B_FALSE);
2898 		rw_exit(&ztest_name_lock);
2899 
2900 		switch (error) {
2901 		case 0:
2902 		case EEXIST:
2903 		case ZFS_ERR_CHECKPOINT_EXISTS:
2904 		case ZFS_ERR_DISCARDING_CHECKPOINT:
2905 			break;
2906 		default:
2907 			fatal(0, "spa_vdev_remove() = %d", error);
2908 		}
2909 	} else {
2910 		spa_config_exit(spa, SCL_VDEV, FTAG);
2911 
2912 		/*
2913 		 * Make 1/4 of the devices be log devices
2914 		 */
2915 		nvroot = make_vdev_root(NULL, NULL, NULL,
2916 		    ztest_opts.zo_vdev_size, 0, (ztest_random(4) == 0) ?
2917 		    "log" : NULL, ztest_opts.zo_raidz, zs->zs_mirrors, 1);
2918 
2919 		error = spa_vdev_add(spa, nvroot);
2920 		nvlist_free(nvroot);
2921 
2922 		switch (error) {
2923 		case 0:
2924 			break;
2925 		case ENOSPC:
2926 			ztest_record_enospc("spa_vdev_add");
2927 			break;
2928 		default:
2929 			fatal(0, "spa_vdev_add() = %d", error);
2930 		}
2931 	}
2932 
2933 	mutex_exit(&ztest_vdev_lock);
2934 }
2935 
2936 /* ARGSUSED */
2937 void
2938 ztest_vdev_class_add(ztest_ds_t *zd, uint64_t id)
2939 {
2940 	ztest_shared_t *zs = ztest_shared;
2941 	spa_t *spa = ztest_spa;
2942 	uint64_t leaves;
2943 	nvlist_t *nvroot;
2944 	const char *class = (ztest_random(2) == 0) ?
2945 	    VDEV_ALLOC_BIAS_SPECIAL : VDEV_ALLOC_BIAS_DEDUP;
2946 	int error;
2947 
2948 	/*
2949 	 * By default add a special vdev 50% of the time
2950 	 */
2951 	if ((ztest_opts.zo_special_vdevs == ZTEST_VDEV_CLASS_OFF) ||
2952 	    (ztest_opts.zo_special_vdevs == ZTEST_VDEV_CLASS_RND &&
2953 	    ztest_random(2) == 0)) {
2954 		return;
2955 	}
2956 
2957 	mutex_enter(&ztest_vdev_lock);
2958 
2959 	/* Only test with mirrors */
2960 	if (zs->zs_mirrors < 2) {
2961 		mutex_exit(&ztest_vdev_lock);
2962 		return;
2963 	}
2964 
2965 	/* requires feature@allocation_classes */
2966 	if (!spa_feature_is_enabled(spa, SPA_FEATURE_ALLOCATION_CLASSES)) {
2967 		mutex_exit(&ztest_vdev_lock);
2968 		return;
2969 	}
2970 
2971 	leaves = MAX(zs->zs_mirrors + zs->zs_splits, 1) * ztest_opts.zo_raidz;
2972 
2973 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
2974 	ztest_shared->zs_vdev_next_leaf = find_vdev_hole(spa) * leaves;
2975 	spa_config_exit(spa, SCL_VDEV, FTAG);
2976 
2977 	nvroot = make_vdev_root(NULL, NULL, NULL, ztest_opts.zo_vdev_size, 0,
2978 	    class, ztest_opts.zo_raidz, zs->zs_mirrors, 1);
2979 
2980 	error = spa_vdev_add(spa, nvroot);
2981 	nvlist_free(nvroot);
2982 
2983 	if (error == ENOSPC)
2984 		ztest_record_enospc("spa_vdev_add");
2985 	else if (error != 0)
2986 		fatal(0, "spa_vdev_add() = %d", error);
2987 
2988 	/*
2989 	 * 50% of the time allow small blocks in the special class
2990 	 */
2991 	if (error == 0 &&
2992 	    spa_special_class(spa)->mc_groups == 1 && ztest_random(2) == 0) {
2993 		if (ztest_opts.zo_verbose >= 3)
2994 			(void) printf("Enabling special VDEV small blocks\n");
2995 		(void) ztest_dsl_prop_set_uint64(zd->zd_name,
2996 		    ZFS_PROP_SPECIAL_SMALL_BLOCKS, 32768, B_FALSE);
2997 	}
2998 
2999 	mutex_exit(&ztest_vdev_lock);
3000 
3001 	if (ztest_opts.zo_verbose >= 3) {
3002 		metaslab_class_t *mc;
3003 
3004 		if (strcmp(class, VDEV_ALLOC_BIAS_SPECIAL) == 0)
3005 			mc = spa_special_class(spa);
3006 		else
3007 			mc = spa_dedup_class(spa);
3008 		(void) printf("Added a %s mirrored vdev (of %d)\n",
3009 		    class, (int)mc->mc_groups);
3010 	}
3011 }
3012 
3013 /*
3014  * Verify that adding/removing aux devices (l2arc, hot spare) works as expected.
3015  */
3016 /* ARGSUSED */
3017 void
3018 ztest_vdev_aux_add_remove(ztest_ds_t *zd, uint64_t id)
3019 {
3020 	ztest_shared_t *zs = ztest_shared;
3021 	spa_t *spa = ztest_spa;
3022 	vdev_t *rvd = spa->spa_root_vdev;
3023 	spa_aux_vdev_t *sav;
3024 	char *aux;
3025 	uint64_t guid = 0;
3026 	int error;
3027 
3028 	if (ztest_opts.zo_mmp_test)
3029 		return;
3030 
3031 	if (ztest_random(2) == 0) {
3032 		sav = &spa->spa_spares;
3033 		aux = ZPOOL_CONFIG_SPARES;
3034 	} else {
3035 		sav = &spa->spa_l2cache;
3036 		aux = ZPOOL_CONFIG_L2CACHE;
3037 	}
3038 
3039 	mutex_enter(&ztest_vdev_lock);
3040 
3041 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
3042 
3043 	if (sav->sav_count != 0 && ztest_random(4) == 0) {
3044 		/*
3045 		 * Pick a random device to remove.
3046 		 */
3047 		guid = sav->sav_vdevs[ztest_random(sav->sav_count)]->vdev_guid;
3048 	} else {
3049 		/*
3050 		 * Find an unused device we can add.
3051 		 */
3052 		zs->zs_vdev_aux = 0;
3053 		for (;;) {
3054 			char path[MAXPATHLEN];
3055 			int c;
3056 			(void) snprintf(path, sizeof (path), ztest_aux_template,
3057 			    ztest_opts.zo_dir, ztest_opts.zo_pool, aux,
3058 			    zs->zs_vdev_aux);
3059 			for (c = 0; c < sav->sav_count; c++)
3060 				if (strcmp(sav->sav_vdevs[c]->vdev_path,
3061 				    path) == 0)
3062 					break;
3063 			if (c == sav->sav_count &&
3064 			    vdev_lookup_by_path(rvd, path) == NULL)
3065 				break;
3066 			zs->zs_vdev_aux++;
3067 		}
3068 	}
3069 
3070 	spa_config_exit(spa, SCL_VDEV, FTAG);
3071 
3072 	if (guid == 0) {
3073 		/*
3074 		 * Add a new device.
3075 		 */
3076 		nvlist_t *nvroot = make_vdev_root(NULL, aux, NULL,
3077 		    (ztest_opts.zo_vdev_size * 5) / 4, 0, NULL, 0, 0, 1);
3078 		error = spa_vdev_add(spa, nvroot);
3079 
3080 		switch (error) {
3081 		case 0:
3082 			break;
3083 		default:
3084 			fatal(0, "spa_vdev_add(%p) = %d", nvroot, error);
3085 		}
3086 		nvlist_free(nvroot);
3087 	} else {
3088 		/*
3089 		 * Remove an existing device.  Sometimes, dirty its
3090 		 * vdev state first to make sure we handle removal
3091 		 * of devices that have pending state changes.
3092 		 */
3093 		if (ztest_random(2) == 0)
3094 			(void) vdev_online(spa, guid, 0, NULL);
3095 
3096 		error = spa_vdev_remove(spa, guid, B_FALSE);
3097 
3098 		switch (error) {
3099 		case 0:
3100 		case EBUSY:
3101 		case ZFS_ERR_CHECKPOINT_EXISTS:
3102 		case ZFS_ERR_DISCARDING_CHECKPOINT:
3103 			break;
3104 		default:
3105 			fatal(0, "spa_vdev_remove(%llu) = %d", guid, error);
3106 		}
3107 	}
3108 
3109 	mutex_exit(&ztest_vdev_lock);
3110 }
3111 
3112 /*
3113  * split a pool if it has mirror tlvdevs
3114  */
3115 /* ARGSUSED */
3116 void
3117 ztest_split_pool(ztest_ds_t *zd, uint64_t id)
3118 {
3119 	ztest_shared_t *zs = ztest_shared;
3120 	spa_t *spa = ztest_spa;
3121 	vdev_t *rvd = spa->spa_root_vdev;
3122 	nvlist_t *tree, **child, *config, *split, **schild;
3123 	uint_t c, children, schildren = 0, lastlogid = 0;
3124 	int error = 0;
3125 
3126 	if (ztest_opts.zo_mmp_test)
3127 		return;
3128 
3129 	mutex_enter(&ztest_vdev_lock);
3130 
3131 	/* ensure we have a useable config; mirrors of raidz aren't supported */
3132 	if (zs->zs_mirrors < 3 || ztest_opts.zo_raidz > 1) {
3133 		mutex_exit(&ztest_vdev_lock);
3134 		return;
3135 	}
3136 
3137 	/* clean up the old pool, if any */
3138 	(void) spa_destroy("splitp");
3139 
3140 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
3141 
3142 	/* generate a config from the existing config */
3143 	mutex_enter(&spa->spa_props_lock);
3144 	VERIFY(nvlist_lookup_nvlist(spa->spa_config, ZPOOL_CONFIG_VDEV_TREE,
3145 	    &tree) == 0);
3146 	mutex_exit(&spa->spa_props_lock);
3147 
3148 	VERIFY(nvlist_lookup_nvlist_array(tree, ZPOOL_CONFIG_CHILDREN, &child,
3149 	    &children) == 0);
3150 
3151 	schild = malloc(rvd->vdev_children * sizeof (nvlist_t *));
3152 	for (c = 0; c < children; c++) {
3153 		vdev_t *tvd = rvd->vdev_child[c];
3154 		nvlist_t **mchild;
3155 		uint_t mchildren;
3156 
3157 		if (tvd->vdev_islog || tvd->vdev_ops == &vdev_hole_ops) {
3158 			VERIFY(nvlist_alloc(&schild[schildren], NV_UNIQUE_NAME,
3159 			    0) == 0);
3160 			VERIFY(nvlist_add_string(schild[schildren],
3161 			    ZPOOL_CONFIG_TYPE, VDEV_TYPE_HOLE) == 0);
3162 			VERIFY(nvlist_add_uint64(schild[schildren],
3163 			    ZPOOL_CONFIG_IS_HOLE, 1) == 0);
3164 			if (lastlogid == 0)
3165 				lastlogid = schildren;
3166 			++schildren;
3167 			continue;
3168 		}
3169 		lastlogid = 0;
3170 		VERIFY(nvlist_lookup_nvlist_array(child[c],
3171 		    ZPOOL_CONFIG_CHILDREN, &mchild, &mchildren) == 0);
3172 		VERIFY(nvlist_dup(mchild[0], &schild[schildren++], 0) == 0);
3173 	}
3174 
3175 	/* OK, create a config that can be used to split */
3176 	VERIFY(nvlist_alloc(&split, NV_UNIQUE_NAME, 0) == 0);
3177 	VERIFY(nvlist_add_string(split, ZPOOL_CONFIG_TYPE,
3178 	    VDEV_TYPE_ROOT) == 0);
3179 	VERIFY(nvlist_add_nvlist_array(split, ZPOOL_CONFIG_CHILDREN, schild,
3180 	    lastlogid != 0 ? lastlogid : schildren) == 0);
3181 
3182 	VERIFY(nvlist_alloc(&config, NV_UNIQUE_NAME, 0) == 0);
3183 	VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, split) == 0);
3184 
3185 	for (c = 0; c < schildren; c++)
3186 		nvlist_free(schild[c]);
3187 	free(schild);
3188 	nvlist_free(split);
3189 
3190 	spa_config_exit(spa, SCL_VDEV, FTAG);
3191 
3192 	rw_enter(&ztest_name_lock, RW_WRITER);
3193 	error = spa_vdev_split_mirror(spa, "splitp", config, NULL, B_FALSE);
3194 	rw_exit(&ztest_name_lock);
3195 
3196 	nvlist_free(config);
3197 
3198 	if (error == 0) {
3199 		(void) printf("successful split - results:\n");
3200 		mutex_enter(&spa_namespace_lock);
3201 		show_pool_stats(spa);
3202 		show_pool_stats(spa_lookup("splitp"));
3203 		mutex_exit(&spa_namespace_lock);
3204 		++zs->zs_splits;
3205 		--zs->zs_mirrors;
3206 	}
3207 	mutex_exit(&ztest_vdev_lock);
3208 }
3209 
3210 /*
3211  * Verify that we can attach and detach devices.
3212  */
3213 /* ARGSUSED */
3214 void
3215 ztest_vdev_attach_detach(ztest_ds_t *zd, uint64_t id)
3216 {
3217 	ztest_shared_t *zs = ztest_shared;
3218 	spa_t *spa = ztest_spa;
3219 	spa_aux_vdev_t *sav = &spa->spa_spares;
3220 	vdev_t *rvd = spa->spa_root_vdev;
3221 	vdev_t *oldvd, *newvd, *pvd;
3222 	nvlist_t *root;
3223 	uint64_t leaves;
3224 	uint64_t leaf, top;
3225 	uint64_t ashift = ztest_get_ashift();
3226 	uint64_t oldguid, pguid;
3227 	uint64_t oldsize, newsize;
3228 	char oldpath[MAXPATHLEN], newpath[MAXPATHLEN];
3229 	int replacing;
3230 	int oldvd_has_siblings = B_FALSE;
3231 	int newvd_is_spare = B_FALSE;
3232 	int oldvd_is_log;
3233 	int error, expected_error;
3234 
3235 	if (ztest_opts.zo_mmp_test)
3236 		return;
3237 
3238 	mutex_enter(&ztest_vdev_lock);
3239 	leaves = MAX(zs->zs_mirrors, 1) * ztest_opts.zo_raidz;
3240 
3241 	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3242 
3243 	/*
3244 	 * If a vdev is in the process of being removed, its removal may
3245 	 * finish while we are in progress, leading to an unexpected error
3246 	 * value.  Don't bother trying to attach while we are in the middle
3247 	 * of removal.
3248 	 */
3249 	if (ztest_device_removal_active) {
3250 		spa_config_exit(spa, SCL_ALL, FTAG);
3251 		mutex_exit(&ztest_vdev_lock);
3252 		return;
3253 	}
3254 
3255 	/*
3256 	 * Decide whether to do an attach or a replace.
3257 	 */
3258 	replacing = ztest_random(2);
3259 
3260 	/*
3261 	 * Pick a random top-level vdev.
3262 	 */
3263 	top = ztest_random_vdev_top(spa, B_TRUE);
3264 
3265 	/*
3266 	 * Pick a random leaf within it.
3267 	 */
3268 	leaf = ztest_random(leaves);
3269 
3270 	/*
3271 	 * Locate this vdev.
3272 	 */
3273 	oldvd = rvd->vdev_child[top];
3274 
3275 	/* pick a child from the mirror */
3276 	if (zs->zs_mirrors >= 1) {
3277 		ASSERT(oldvd->vdev_ops == &vdev_mirror_ops);
3278 		ASSERT(oldvd->vdev_children >= zs->zs_mirrors);
3279 		oldvd = oldvd->vdev_child[leaf / ztest_opts.zo_raidz];
3280 	}
3281 
3282 	/* pick a child out of the raidz group */
3283 	if (ztest_opts.zo_raidz > 1) {
3284 		ASSERT(oldvd->vdev_ops == &vdev_raidz_ops);
3285 		ASSERT(oldvd->vdev_children == ztest_opts.zo_raidz);
3286 		oldvd = oldvd->vdev_child[leaf % ztest_opts.zo_raidz];
3287 	}
3288 
3289 	/*
3290 	 * If we're already doing an attach or replace, oldvd may be a
3291 	 * mirror vdev -- in which case, pick a random child.
3292 	 */
3293 	while (oldvd->vdev_children != 0) {
3294 		oldvd_has_siblings = B_TRUE;
3295 		ASSERT(oldvd->vdev_children >= 2);
3296 		oldvd = oldvd->vdev_child[ztest_random(oldvd->vdev_children)];
3297 	}
3298 
3299 	oldguid = oldvd->vdev_guid;
3300 	oldsize = vdev_get_min_asize(oldvd);
3301 	oldvd_is_log = oldvd->vdev_top->vdev_islog;
3302 	(void) strcpy(oldpath, oldvd->vdev_path);
3303 	pvd = oldvd->vdev_parent;
3304 	pguid = pvd->vdev_guid;
3305 
3306 	/*
3307 	 * If oldvd has siblings, then half of the time, detach it.
3308 	 */
3309 	if (oldvd_has_siblings && ztest_random(2) == 0) {
3310 		spa_config_exit(spa, SCL_ALL, FTAG);
3311 		error = spa_vdev_detach(spa, oldguid, pguid, B_FALSE);
3312 		if (error != 0 && error != ENODEV && error != EBUSY &&
3313 		    error != ENOTSUP && error != ZFS_ERR_CHECKPOINT_EXISTS &&
3314 		    error != ZFS_ERR_DISCARDING_CHECKPOINT)
3315 			fatal(0, "detach (%s) returned %d", oldpath, error);
3316 		mutex_exit(&ztest_vdev_lock);
3317 		return;
3318 	}
3319 
3320 	/*
3321 	 * For the new vdev, choose with equal probability between the two
3322 	 * standard paths (ending in either 'a' or 'b') or a random hot spare.
3323 	 */
3324 	if (sav->sav_count != 0 && ztest_random(3) == 0) {
3325 		newvd = sav->sav_vdevs[ztest_random(sav->sav_count)];
3326 		newvd_is_spare = B_TRUE;
3327 		(void) strcpy(newpath, newvd->vdev_path);
3328 	} else {
3329 		(void) snprintf(newpath, sizeof (newpath), ztest_dev_template,
3330 		    ztest_opts.zo_dir, ztest_opts.zo_pool,
3331 		    top * leaves + leaf);
3332 		if (ztest_random(2) == 0)
3333 			newpath[strlen(newpath) - 1] = 'b';
3334 		newvd = vdev_lookup_by_path(rvd, newpath);
3335 	}
3336 
3337 	if (newvd) {
3338 		/*
3339 		 * Reopen to ensure the vdev's asize field isn't stale.
3340 		 */
3341 		vdev_reopen(newvd);
3342 		newsize = vdev_get_min_asize(newvd);
3343 	} else {
3344 		/*
3345 		 * Make newsize a little bigger or smaller than oldsize.
3346 		 * If it's smaller, the attach should fail.
3347 		 * If it's larger, and we're doing a replace,
3348 		 * we should get dynamic LUN growth when we're done.
3349 		 */
3350 		newsize = 10 * oldsize / (9 + ztest_random(3));
3351 	}
3352 
3353 	/*
3354 	 * If pvd is not a mirror or root, the attach should fail with ENOTSUP,
3355 	 * unless it's a replace; in that case any non-replacing parent is OK.
3356 	 *
3357 	 * If newvd is already part of the pool, it should fail with EBUSY.
3358 	 *
3359 	 * If newvd is too small, it should fail with EOVERFLOW.
3360 	 */
3361 	if (pvd->vdev_ops != &vdev_mirror_ops &&
3362 	    pvd->vdev_ops != &vdev_root_ops && (!replacing ||
3363 	    pvd->vdev_ops == &vdev_replacing_ops ||
3364 	    pvd->vdev_ops == &vdev_spare_ops))
3365 		expected_error = ENOTSUP;
3366 	else if (newvd_is_spare && (!replacing || oldvd_is_log))
3367 		expected_error = ENOTSUP;
3368 	else if (newvd == oldvd)
3369 		expected_error = replacing ? 0 : EBUSY;
3370 	else if (vdev_lookup_by_path(rvd, newpath) != NULL)
3371 		expected_error = EBUSY;
3372 	else if (newsize < oldsize)
3373 		expected_error = EOVERFLOW;
3374 	else if (ashift > oldvd->vdev_top->vdev_ashift)
3375 		expected_error = EDOM;
3376 	else
3377 		expected_error = 0;
3378 
3379 	spa_config_exit(spa, SCL_ALL, FTAG);
3380 
3381 	/*
3382 	 * Build the nvlist describing newpath.
3383 	 */
3384 	root = make_vdev_root(newpath, NULL, NULL, newvd == NULL ? newsize : 0,
3385 	    ashift, NULL, 0, 0, 1);
3386 
3387 	error = spa_vdev_attach(spa, oldguid, root, replacing);
3388 
3389 	nvlist_free(root);
3390 
3391 	/*
3392 	 * If our parent was the replacing vdev, but the replace completed,
3393 	 * then instead of failing with ENOTSUP we may either succeed,
3394 	 * fail with ENODEV, or fail with EOVERFLOW.
3395 	 */
3396 	if (expected_error == ENOTSUP &&
3397 	    (error == 0 || error == ENODEV || error == EOVERFLOW))
3398 		expected_error = error;
3399 
3400 	/*
3401 	 * If someone grew the LUN, the replacement may be too small.
3402 	 */
3403 	if (error == EOVERFLOW || error == EBUSY)
3404 		expected_error = error;
3405 
3406 	if (error == ZFS_ERR_CHECKPOINT_EXISTS ||
3407 	    error == ZFS_ERR_DISCARDING_CHECKPOINT)
3408 		expected_error = error;
3409 
3410 	/* XXX workaround 6690467 */
3411 	if (error != expected_error && expected_error != EBUSY) {
3412 		fatal(0, "attach (%s %llu, %s %llu, %d) "
3413 		    "returned %d, expected %d",
3414 		    oldpath, oldsize, newpath,
3415 		    newsize, replacing, error, expected_error);
3416 	}
3417 
3418 	mutex_exit(&ztest_vdev_lock);
3419 }
3420 
3421 /* ARGSUSED */
3422 void
3423 ztest_device_removal(ztest_ds_t *zd, uint64_t id)
3424 {
3425 	spa_t *spa = ztest_spa;
3426 	vdev_t *vd;
3427 	uint64_t guid;
3428 	int error;
3429 
3430 	mutex_enter(&ztest_vdev_lock);
3431 
3432 	if (ztest_device_removal_active) {
3433 		mutex_exit(&ztest_vdev_lock);
3434 		return;
3435 	}
3436 
3437 	/*
3438 	 * Remove a random top-level vdev and wait for removal to finish.
3439 	 */
3440 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
3441 	vd = vdev_lookup_top(spa, ztest_random_vdev_top(spa, B_FALSE));
3442 	guid = vd->vdev_guid;
3443 	spa_config_exit(spa, SCL_VDEV, FTAG);
3444 
3445 	error = spa_vdev_remove(spa, guid, B_FALSE);
3446 	if (error == 0) {
3447 		ztest_device_removal_active = B_TRUE;
3448 		mutex_exit(&ztest_vdev_lock);
3449 
3450 		while (spa->spa_vdev_removal != NULL)
3451 			txg_wait_synced(spa_get_dsl(spa), 0);
3452 	} else {
3453 		mutex_exit(&ztest_vdev_lock);
3454 		return;
3455 	}
3456 
3457 	/*
3458 	 * The pool needs to be scrubbed after completing device removal.
3459 	 * Failure to do so may result in checksum errors due to the
3460 	 * strategy employed by ztest_fault_inject() when selecting which
3461 	 * offset are redundant and can be damaged.
3462 	 */
3463 	error = spa_scan(spa, POOL_SCAN_SCRUB);
3464 	if (error == 0) {
3465 		while (dsl_scan_scrubbing(spa_get_dsl(spa)))
3466 			txg_wait_synced(spa_get_dsl(spa), 0);
3467 	}
3468 
3469 	mutex_enter(&ztest_vdev_lock);
3470 	ztest_device_removal_active = B_FALSE;
3471 	mutex_exit(&ztest_vdev_lock);
3472 }
3473 
3474 /*
3475  * Callback function which expands the physical size of the vdev.
3476  */
3477 vdev_t *
3478 grow_vdev(vdev_t *vd, void *arg)
3479 {
3480 	spa_t *spa = vd->vdev_spa;
3481 	size_t *newsize = arg;
3482 	size_t fsize;
3483 	int fd;
3484 
3485 	ASSERT(spa_config_held(spa, SCL_STATE, RW_READER) == SCL_STATE);
3486 	ASSERT(vd->vdev_ops->vdev_op_leaf);
3487 
3488 	if ((fd = open(vd->vdev_path, O_RDWR)) == -1)
3489 		return (vd);
3490 
3491 	fsize = lseek(fd, 0, SEEK_END);
3492 	(void) ftruncate(fd, *newsize);
3493 
3494 	if (ztest_opts.zo_verbose >= 6) {
3495 		(void) printf("%s grew from %lu to %lu bytes\n",
3496 		    vd->vdev_path, (ulong_t)fsize, (ulong_t)*newsize);
3497 	}
3498 	(void) close(fd);
3499 	return (NULL);
3500 }
3501 
3502 /*
3503  * Callback function which expands a given vdev by calling vdev_online().
3504  */
3505 /* ARGSUSED */
3506 vdev_t *
3507 online_vdev(vdev_t *vd, void *arg)
3508 {
3509 	spa_t *spa = vd->vdev_spa;
3510 	vdev_t *tvd = vd->vdev_top;
3511 	uint64_t guid = vd->vdev_guid;
3512 	uint64_t generation = spa->spa_config_generation + 1;
3513 	vdev_state_t newstate = VDEV_STATE_UNKNOWN;
3514 	int error;
3515 
3516 	ASSERT(spa_config_held(spa, SCL_STATE, RW_READER) == SCL_STATE);
3517 	ASSERT(vd->vdev_ops->vdev_op_leaf);
3518 
3519 	/* Calling vdev_online will initialize the new metaslabs */
3520 	spa_config_exit(spa, SCL_STATE, spa);
3521 	error = vdev_online(spa, guid, ZFS_ONLINE_EXPAND, &newstate);
3522 	spa_config_enter(spa, SCL_STATE, spa, RW_READER);
3523 
3524 	/*
3525 	 * If vdev_online returned an error or the underlying vdev_open
3526 	 * failed then we abort the expand. The only way to know that
3527 	 * vdev_open fails is by checking the returned newstate.
3528 	 */
3529 	if (error || newstate != VDEV_STATE_HEALTHY) {
3530 		if (ztest_opts.zo_verbose >= 5) {
3531 			(void) printf("Unable to expand vdev, state %llu, "
3532 			    "error %d\n", (u_longlong_t)newstate, error);
3533 		}
3534 		return (vd);
3535 	}
3536 	ASSERT3U(newstate, ==, VDEV_STATE_HEALTHY);
3537 
3538 	/*
3539 	 * Since we dropped the lock we need to ensure that we're
3540 	 * still talking to the original vdev. It's possible this
3541 	 * vdev may have been detached/replaced while we were
3542 	 * trying to online it.
3543 	 */
3544 	if (generation != spa->spa_config_generation) {
3545 		if (ztest_opts.zo_verbose >= 5) {
3546 			(void) printf("vdev configuration has changed, "
3547 			    "guid %llu, state %llu, expected gen %llu, "
3548 			    "got gen %llu\n",
3549 			    (u_longlong_t)guid,
3550 			    (u_longlong_t)tvd->vdev_state,
3551 			    (u_longlong_t)generation,
3552 			    (u_longlong_t)spa->spa_config_generation);
3553 		}
3554 		return (vd);
3555 	}
3556 	return (NULL);
3557 }
3558 
3559 /*
3560  * Traverse the vdev tree calling the supplied function.
3561  * We continue to walk the tree until we either have walked all
3562  * children or we receive a non-NULL return from the callback.
3563  * If a NULL callback is passed, then we just return back the first
3564  * leaf vdev we encounter.
3565  */
3566 vdev_t *
3567 vdev_walk_tree(vdev_t *vd, vdev_t *(*func)(vdev_t *, void *), void *arg)
3568 {
3569 	if (vd->vdev_ops->vdev_op_leaf) {
3570 		if (func == NULL)
3571 			return (vd);
3572 		else
3573 			return (func(vd, arg));
3574 	}
3575 
3576 	for (uint_t c = 0; c < vd->vdev_children; c++) {
3577 		vdev_t *cvd = vd->vdev_child[c];
3578 		if ((cvd = vdev_walk_tree(cvd, func, arg)) != NULL)
3579 			return (cvd);
3580 	}
3581 	return (NULL);
3582 }
3583 
3584 /*
3585  * Verify that dynamic LUN growth works as expected.
3586  */
3587 /* ARGSUSED */
3588 void
3589 ztest_vdev_LUN_growth(ztest_ds_t *zd, uint64_t id)
3590 {
3591 	spa_t *spa = ztest_spa;
3592 	vdev_t *vd, *tvd;
3593 	metaslab_class_t *mc;
3594 	metaslab_group_t *mg;
3595 	size_t psize, newsize;
3596 	uint64_t top;
3597 	uint64_t old_class_space, new_class_space, old_ms_count, new_ms_count;
3598 
3599 	mutex_enter(&ztest_checkpoint_lock);
3600 	mutex_enter(&ztest_vdev_lock);
3601 	spa_config_enter(spa, SCL_STATE, spa, RW_READER);
3602 
3603 	/*
3604 	 * If there is a vdev removal in progress, it could complete while
3605 	 * we are running, in which case we would not be able to verify
3606 	 * that the metaslab_class space increased (because it decreases
3607 	 * when the device removal completes).
3608 	 */
3609 	if (ztest_device_removal_active) {
3610 		spa_config_exit(spa, SCL_STATE, spa);
3611 		mutex_exit(&ztest_vdev_lock);
3612 		mutex_exit(&ztest_checkpoint_lock);
3613 		return;
3614 	}
3615 
3616 	top = ztest_random_vdev_top(spa, B_TRUE);
3617 
3618 	tvd = spa->spa_root_vdev->vdev_child[top];
3619 	mg = tvd->vdev_mg;
3620 	mc = mg->mg_class;
3621 	old_ms_count = tvd->vdev_ms_count;
3622 	old_class_space = metaslab_class_get_space(mc);
3623 
3624 	/*
3625 	 * Determine the size of the first leaf vdev associated with
3626 	 * our top-level device.
3627 	 */
3628 	vd = vdev_walk_tree(tvd, NULL, NULL);
3629 	ASSERT3P(vd, !=, NULL);
3630 	ASSERT(vd->vdev_ops->vdev_op_leaf);
3631 
3632 	psize = vd->vdev_psize;
3633 
3634 	/*
3635 	 * We only try to expand the vdev if it's healthy, less than 4x its
3636 	 * original size, and it has a valid psize.
3637 	 */
3638 	if (tvd->vdev_state != VDEV_STATE_HEALTHY ||
3639 	    psize == 0 || psize >= 4 * ztest_opts.zo_vdev_size) {
3640 		spa_config_exit(spa, SCL_STATE, spa);
3641 		mutex_exit(&ztest_vdev_lock);
3642 		mutex_exit(&ztest_checkpoint_lock);
3643 		return;
3644 	}
3645 	ASSERT(psize > 0);
3646 	newsize = psize + MAX(psize / 8, SPA_MAXBLOCKSIZE);
3647 	ASSERT3U(newsize, >, psize);
3648 
3649 	if (ztest_opts.zo_verbose >= 6) {
3650 		(void) printf("Expanding LUN %s from %lu to %lu\n",
3651 		    vd->vdev_path, (ulong_t)psize, (ulong_t)newsize);
3652 	}
3653 
3654 	/*
3655 	 * Growing the vdev is a two step process:
3656 	 *	1). expand the physical size (i.e. relabel)
3657 	 *	2). online the vdev to create the new metaslabs
3658 	 */
3659 	if (vdev_walk_tree(tvd, grow_vdev, &newsize) != NULL ||
3660 	    vdev_walk_tree(tvd, online_vdev, NULL) != NULL ||
3661 	    tvd->vdev_state != VDEV_STATE_HEALTHY) {
3662 		if (ztest_opts.zo_verbose >= 5) {
3663 			(void) printf("Could not expand LUN because "
3664 			    "the vdev configuration changed.\n");
3665 		}
3666 		spa_config_exit(spa, SCL_STATE, spa);
3667 		mutex_exit(&ztest_vdev_lock);
3668 		mutex_exit(&ztest_checkpoint_lock);
3669 		return;
3670 	}
3671 
3672 	spa_config_exit(spa, SCL_STATE, spa);
3673 
3674 	/*
3675 	 * Expanding the LUN will update the config asynchronously,
3676 	 * thus we must wait for the async thread to complete any
3677 	 * pending tasks before proceeding.
3678 	 */
3679 	for (;;) {
3680 		boolean_t done;
3681 		mutex_enter(&spa->spa_async_lock);
3682 		done = (spa->spa_async_thread == NULL && !spa->spa_async_tasks);
3683 		mutex_exit(&spa->spa_async_lock);
3684 		if (done)
3685 			break;
3686 		txg_wait_synced(spa_get_dsl(spa), 0);
3687 		(void) poll(NULL, 0, 100);
3688 	}
3689 
3690 	spa_config_enter(spa, SCL_STATE, spa, RW_READER);
3691 
3692 	tvd = spa->spa_root_vdev->vdev_child[top];
3693 	new_ms_count = tvd->vdev_ms_count;
3694 	new_class_space = metaslab_class_get_space(mc);
3695 
3696 	if (tvd->vdev_mg != mg || mg->mg_class != mc) {
3697 		if (ztest_opts.zo_verbose >= 5) {
3698 			(void) printf("Could not verify LUN expansion due to "
3699 			    "intervening vdev offline or remove.\n");
3700 		}
3701 		spa_config_exit(spa, SCL_STATE, spa);
3702 		mutex_exit(&ztest_vdev_lock);
3703 		mutex_exit(&ztest_checkpoint_lock);
3704 		return;
3705 	}
3706 
3707 	/*
3708 	 * Make sure we were able to grow the vdev.
3709 	 */
3710 	if (new_ms_count <= old_ms_count) {
3711 		fatal(0, "LUN expansion failed: ms_count %llu < %llu\n",
3712 		    old_ms_count, new_ms_count);
3713 	}
3714 
3715 	/*
3716 	 * Make sure we were able to grow the pool.
3717 	 */
3718 	if (new_class_space <= old_class_space) {
3719 		fatal(0, "LUN expansion failed: class_space %llu < %llu\n",
3720 		    old_class_space, new_class_space);
3721 	}
3722 
3723 	if (ztest_opts.zo_verbose >= 5) {
3724 		char oldnumbuf[NN_NUMBUF_SZ], newnumbuf[NN_NUMBUF_SZ];
3725 
3726 		nicenum(old_class_space, oldnumbuf, sizeof (oldnumbuf));
3727 		nicenum(new_class_space, newnumbuf, sizeof (newnumbuf));
3728 		(void) printf("%s grew from %s to %s\n",
3729 		    spa->spa_name, oldnumbuf, newnumbuf);
3730 	}
3731 
3732 	spa_config_exit(spa, SCL_STATE, spa);
3733 	mutex_exit(&ztest_vdev_lock);
3734 	mutex_exit(&ztest_checkpoint_lock);
3735 }
3736 
3737 /*
3738  * Verify that dmu_objset_{create,destroy,open,close} work as expected.
3739  */
3740 /* ARGSUSED */
3741 static void
3742 ztest_objset_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx)
3743 {
3744 	/*
3745 	 * Create the objects common to all ztest datasets.
3746 	 */
3747 	VERIFY(zap_create_claim(os, ZTEST_DIROBJ,
3748 	    DMU_OT_ZAP_OTHER, DMU_OT_NONE, 0, tx) == 0);
3749 }
3750 
3751 static int
3752 ztest_dataset_create(char *dsname)
3753 {
3754 	int err;
3755 	uint64_t rand;
3756 	dsl_crypto_params_t *dcp = NULL;
3757 
3758 	/*
3759 	 * 50% of the time, we create encrypted datasets
3760 	 * using a random cipher suite and a hard-coded
3761 	 * wrapping key.
3762 	 */
3763 #ifdef WITHCRYPTO
3764 	/*
3765 	 * Until the crypto framework is compiled in userland, the ztest using
3766 	 * crypto will not work.
3767 	 */
3768 	rand = ztest_random(2);
3769 #else
3770 	rand = 0;
3771 #endif
3772 	if (rand != 0) {
3773 		nvlist_t *crypto_args = fnvlist_alloc();
3774 		nvlist_t *props = fnvlist_alloc();
3775 
3776 		/* slight bias towards the default cipher suite */
3777 		rand = ztest_random(ZIO_CRYPT_FUNCTIONS);
3778 		if (rand < ZIO_CRYPT_AES_128_CCM)
3779 			rand = ZIO_CRYPT_ON;
3780 
3781 		fnvlist_add_uint64(props,
3782 		    zfs_prop_to_name(ZFS_PROP_ENCRYPTION), rand);
3783 		fnvlist_add_uint8_array(crypto_args, "wkeydata",
3784 		    (uint8_t *)ztest_wkeydata, WRAPPING_KEY_LEN);
3785 
3786 		/*
3787 		 * These parameters aren't really used by the kernel. They
3788 		 * are simply stored so that userspace knows how to load
3789 		 * the wrapping key.
3790 		 */
3791 		fnvlist_add_uint64(props,
3792 		    zfs_prop_to_name(ZFS_PROP_KEYFORMAT), ZFS_KEYFORMAT_RAW);
3793 		fnvlist_add_string(props,
3794 		    zfs_prop_to_name(ZFS_PROP_KEYLOCATION), "prompt");
3795 		fnvlist_add_uint64(props,
3796 		    zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 0ULL);
3797 		fnvlist_add_uint64(props,
3798 		    zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 0ULL);
3799 
3800 		VERIFY0(dsl_crypto_params_create_nvlist(DCP_CMD_NONE, props,
3801 		    crypto_args, &dcp));
3802 
3803 		fnvlist_free(crypto_args);
3804 		fnvlist_free(props);
3805 	}
3806 
3807 	err = dmu_objset_create(dsname, DMU_OST_OTHER, 0, dcp,
3808 	    ztest_objset_create_cb, NULL);
3809 	dsl_crypto_params_free(dcp, !!err);
3810 
3811 	rand = ztest_random(100);
3812 	if (err || rand < 80)
3813 		return (err);
3814 
3815 	if (ztest_opts.zo_verbose >= 6)
3816 		(void) printf("Setting dataset %s to sync always\n", dsname);
3817 	return (ztest_dsl_prop_set_uint64(dsname, ZFS_PROP_SYNC,
3818 	    ZFS_SYNC_ALWAYS, B_FALSE));
3819 }
3820 
3821 /* ARGSUSED */
3822 static int
3823 ztest_objset_destroy_cb(const char *name, void *arg)
3824 {
3825 	objset_t *os;
3826 	dmu_object_info_t doi;
3827 	int error;
3828 
3829 	/*
3830 	 * Verify that the dataset contains a directory object.
3831 	 */
3832 	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_TRUE,
3833 	    B_TRUE, FTAG, &os));
3834 	error = dmu_object_info(os, ZTEST_DIROBJ, &doi);
3835 	if (error != ENOENT) {
3836 		/* We could have crashed in the middle of destroying it */
3837 		ASSERT0(error);
3838 		ASSERT3U(doi.doi_type, ==, DMU_OT_ZAP_OTHER);
3839 		ASSERT3S(doi.doi_physical_blocks_512, >=, 0);
3840 	}
3841 	dmu_objset_disown(os, B_TRUE, FTAG);
3842 
3843 	/*
3844 	 * Destroy the dataset.
3845 	 */
3846 	if (strchr(name, '@') != NULL) {
3847 		VERIFY0(dsl_destroy_snapshot(name, B_TRUE));
3848 	} else {
3849 		error = dsl_destroy_head(name);
3850 		/* There could be a hold on this dataset */
3851 		if (error != EBUSY)
3852 			ASSERT0(error);
3853 	}
3854 	return (0);
3855 }
3856 
3857 static boolean_t
3858 ztest_snapshot_create(char *osname, uint64_t id)
3859 {
3860 	char snapname[ZFS_MAX_DATASET_NAME_LEN];
3861 	int error;
3862 
3863 	(void) snprintf(snapname, sizeof (snapname), "%llu", (u_longlong_t)id);
3864 
3865 	error = dmu_objset_snapshot_one(osname, snapname);
3866 	if (error == ENOSPC) {
3867 		ztest_record_enospc(FTAG);
3868 		return (B_FALSE);
3869 	}
3870 	if (error != 0 && error != EEXIST) {
3871 		fatal(0, "ztest_snapshot_create(%s@%s) = %d", osname,
3872 		    snapname, error);
3873 	}
3874 	return (B_TRUE);
3875 }
3876 
3877 static boolean_t
3878 ztest_snapshot_destroy(char *osname, uint64_t id)
3879 {
3880 	char snapname[ZFS_MAX_DATASET_NAME_LEN];
3881 	int error;
3882 
3883 	(void) snprintf(snapname, sizeof (snapname), "%s@%llu", osname,
3884 	    (u_longlong_t)id);
3885 
3886 	error = dsl_destroy_snapshot(snapname, B_FALSE);
3887 	if (error != 0 && error != ENOENT)
3888 		fatal(0, "ztest_snapshot_destroy(%s) = %d", snapname, error);
3889 	return (B_TRUE);
3890 }
3891 
3892 /* ARGSUSED */
3893 void
3894 ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id)
3895 {
3896 	ztest_ds_t zdtmp;
3897 	int iters;
3898 	int error;
3899 	objset_t *os, *os2;
3900 	char name[ZFS_MAX_DATASET_NAME_LEN];
3901 	zilog_t *zilog;
3902 
3903 	rw_enter(&ztest_name_lock, RW_READER);
3904 
3905 	(void) snprintf(name, sizeof (name), "%s/temp_%llu",
3906 	    ztest_opts.zo_pool, (u_longlong_t)id);
3907 
3908 	/*
3909 	 * If this dataset exists from a previous run, process its replay log
3910 	 * half of the time.  If we don't replay it, then dmu_objset_destroy()
3911 	 * (invoked from ztest_objset_destroy_cb()) should just throw it away.
3912 	 */
3913 	if (ztest_random(2) == 0 &&
3914 	    ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
3915 	    B_TRUE, FTAG, &os) == 0) {
3916 		ztest_zd_init(&zdtmp, NULL, os);
3917 		zil_replay(os, &zdtmp, ztest_replay_vector);
3918 		ztest_zd_fini(&zdtmp);
3919 		dmu_objset_disown(os, B_TRUE, FTAG);
3920 	}
3921 
3922 	/*
3923 	 * There may be an old instance of the dataset we're about to
3924 	 * create lying around from a previous run.  If so, destroy it
3925 	 * and all of its snapshots.
3926 	 */
3927 	(void) dmu_objset_find(name, ztest_objset_destroy_cb, NULL,
3928 	    DS_FIND_CHILDREN | DS_FIND_SNAPSHOTS);
3929 
3930 	/*
3931 	 * Verify that the destroyed dataset is no longer in the namespace.
3932 	 */
3933 	VERIFY3U(ENOENT, ==, ztest_dmu_objset_own(name, DMU_OST_OTHER, B_TRUE,
3934 	    B_TRUE, FTAG, &os));
3935 
3936 	/*
3937 	 * Verify that we can create a new dataset.
3938 	 */
3939 	error = ztest_dataset_create(name);
3940 	if (error) {
3941 		if (error == ENOSPC) {
3942 			ztest_record_enospc(FTAG);
3943 			rw_exit(&ztest_name_lock);
3944 			return;
3945 		}
3946 		fatal(0, "dmu_objset_create(%s) = %d", name, error);
3947 	}
3948 
3949 	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE, B_TRUE,
3950 	    FTAG, &os));
3951 
3952 	ztest_zd_init(&zdtmp, NULL, os);
3953 
3954 	/*
3955 	 * Open the intent log for it.
3956 	 */
3957 	zilog = zil_open(os, ztest_get_data);
3958 
3959 	/*
3960 	 * Put some objects in there, do a little I/O to them,
3961 	 * and randomly take a couple of snapshots along the way.
3962 	 */
3963 	iters = ztest_random(5);
3964 	for (int i = 0; i < iters; i++) {
3965 		ztest_dmu_object_alloc_free(&zdtmp, id);
3966 		if (ztest_random(iters) == 0)
3967 			(void) ztest_snapshot_create(name, i);
3968 	}
3969 
3970 	/*
3971 	 * Verify that we cannot create an existing dataset.
3972 	 */
3973 	VERIFY3U(EEXIST, ==,
3974 	    dmu_objset_create(name, DMU_OST_OTHER, 0, NULL, NULL, NULL));
3975 
3976 	/*
3977 	 * Verify that we can hold an objset that is also owned.
3978 	 */
3979 	VERIFY3U(0, ==, dmu_objset_hold(name, FTAG, &os2));
3980 	dmu_objset_rele(os2, FTAG);
3981 
3982 	/*
3983 	 * Verify that we cannot own an objset that is already owned.
3984 	 */
3985 	VERIFY3U(EBUSY, ==, ztest_dmu_objset_own(name, DMU_OST_OTHER,
3986 	    B_FALSE, B_TRUE, FTAG, &os2));
3987 
3988 	zil_close(zilog);
3989 	dmu_objset_disown(os, B_TRUE, FTAG);
3990 	ztest_zd_fini(&zdtmp);
3991 
3992 	rw_exit(&ztest_name_lock);
3993 }
3994 
3995 /*
3996  * Verify that dmu_snapshot_{create,destroy,open,close} work as expected.
3997  */
3998 void
3999 ztest_dmu_snapshot_create_destroy(ztest_ds_t *zd, uint64_t id)
4000 {
4001 	rw_enter(&ztest_name_lock, RW_READER);
4002 	(void) ztest_snapshot_destroy(zd->zd_name, id);
4003 	(void) ztest_snapshot_create(zd->zd_name, id);
4004 	rw_exit(&ztest_name_lock);
4005 }
4006 
4007 /*
4008  * Cleanup non-standard snapshots and clones.
4009  */
4010 void
4011 ztest_dsl_dataset_cleanup(char *osname, uint64_t id)
4012 {
4013 	char snap1name[ZFS_MAX_DATASET_NAME_LEN];
4014 	char clone1name[ZFS_MAX_DATASET_NAME_LEN];
4015 	char snap2name[ZFS_MAX_DATASET_NAME_LEN];
4016 	char clone2name[ZFS_MAX_DATASET_NAME_LEN];
4017 	char snap3name[ZFS_MAX_DATASET_NAME_LEN];
4018 	int error;
4019 
4020 	(void) snprintf(snap1name, sizeof (snap1name),
4021 	    "%s@s1_%llu", osname, id);
4022 	(void) snprintf(clone1name, sizeof (clone1name),
4023 	    "%s/c1_%llu", osname, id);
4024 	(void) snprintf(snap2name, sizeof (snap2name),
4025 	    "%s@s2_%llu", clone1name, id);
4026 	(void) snprintf(clone2name, sizeof (clone2name),
4027 	    "%s/c2_%llu", osname, id);
4028 	(void) snprintf(snap3name, sizeof (snap3name),
4029 	    "%s@s3_%llu", clone1name, id);
4030 
4031 	error = dsl_destroy_head(clone2name);
4032 	if (error && error != ENOENT)
4033 		fatal(0, "dsl_destroy_head(%s) = %d", clone2name, error);
4034 	error = dsl_destroy_snapshot(snap3name, B_FALSE);
4035 	if (error && error != ENOENT)
4036 		fatal(0, "dsl_destroy_snapshot(%s) = %d", snap3name, error);
4037 	error = dsl_destroy_snapshot(snap2name, B_FALSE);
4038 	if (error && error != ENOENT)
4039 		fatal(0, "dsl_destroy_snapshot(%s) = %d", snap2name, error);
4040 	error = dsl_destroy_head(clone1name);
4041 	if (error && error != ENOENT)
4042 		fatal(0, "dsl_destroy_head(%s) = %d", clone1name, error);
4043 	error = dsl_destroy_snapshot(snap1name, B_FALSE);
4044 	if (error && error != ENOENT)
4045 		fatal(0, "dsl_destroy_snapshot(%s) = %d", snap1name, error);
4046 }
4047 
4048 /*
4049  * Verify dsl_dataset_promote handles EBUSY
4050  */
4051 void
4052 ztest_dsl_dataset_promote_busy(ztest_ds_t *zd, uint64_t id)
4053 {
4054 	objset_t *os;
4055 	char snap1name[ZFS_MAX_DATASET_NAME_LEN];
4056 	char clone1name[ZFS_MAX_DATASET_NAME_LEN];
4057 	char snap2name[ZFS_MAX_DATASET_NAME_LEN];
4058 	char clone2name[ZFS_MAX_DATASET_NAME_LEN];
4059 	char snap3name[ZFS_MAX_DATASET_NAME_LEN];
4060 	char *osname = zd->zd_name;
4061 	int error;
4062 
4063 	rw_enter(&ztest_name_lock, RW_READER);
4064 
4065 	ztest_dsl_dataset_cleanup(osname, id);
4066 
4067 	(void) snprintf(snap1name, sizeof (snap1name),
4068 	    "%s@s1_%llu", osname, id);
4069 	(void) snprintf(clone1name, sizeof (clone1name),
4070 	    "%s/c1_%llu", osname, id);
4071 	(void) snprintf(snap2name, sizeof (snap2name),
4072 	    "%s@s2_%llu", clone1name, id);
4073 	(void) snprintf(clone2name, sizeof (clone2name),
4074 	    "%s/c2_%llu", osname, id);
4075 	(void) snprintf(snap3name, sizeof (snap3name),
4076 	    "%s@s3_%llu", clone1name, id);
4077 
4078 	error = dmu_objset_snapshot_one(osname, strchr(snap1name, '@') + 1);
4079 	if (error && error != EEXIST) {
4080 		if (error == ENOSPC) {
4081 			ztest_record_enospc(FTAG);
4082 			goto out;
4083 		}
4084 		fatal(0, "dmu_take_snapshot(%s) = %d", snap1name, error);
4085 	}
4086 
4087 	error = dmu_objset_clone(clone1name, snap1name);
4088 	if (error) {
4089 		if (error == ENOSPC) {
4090 			ztest_record_enospc(FTAG);
4091 			goto out;
4092 		}
4093 		fatal(0, "dmu_objset_create(%s) = %d", clone1name, error);
4094 	}
4095 
4096 	error = dmu_objset_snapshot_one(clone1name, strchr(snap2name, '@') + 1);
4097 	if (error && error != EEXIST) {
4098 		if (error == ENOSPC) {
4099 			ztest_record_enospc(FTAG);
4100 			goto out;
4101 		}
4102 		fatal(0, "dmu_open_snapshot(%s) = %d", snap2name, error);
4103 	}
4104 
4105 	error = dmu_objset_snapshot_one(clone1name, strchr(snap3name, '@') + 1);
4106 	if (error && error != EEXIST) {
4107 		if (error == ENOSPC) {
4108 			ztest_record_enospc(FTAG);
4109 			goto out;
4110 		}
4111 		fatal(0, "dmu_open_snapshot(%s) = %d", snap3name, error);
4112 	}
4113 
4114 	error = dmu_objset_clone(clone2name, snap3name);
4115 	if (error) {
4116 		if (error == ENOSPC) {
4117 			ztest_record_enospc(FTAG);
4118 			goto out;
4119 		}
4120 		fatal(0, "dmu_objset_create(%s) = %d", clone2name, error);
4121 	}
4122 
4123 	error = ztest_dmu_objset_own(snap2name, DMU_OST_ANY, B_TRUE, B_TRUE,
4124 	    FTAG, &os);
4125 	if (error)
4126 		fatal(0, "dmu_objset_own(%s) = %d", snap2name, error);
4127 	error = dsl_dataset_promote(clone2name, NULL);
4128 	if (error == ENOSPC) {
4129 		dmu_objset_disown(os, B_TRUE, FTAG);
4130 		ztest_record_enospc(FTAG);
4131 		goto out;
4132 	}
4133 	if (error != EBUSY)
4134 		fatal(0, "dsl_dataset_promote(%s), %d, not EBUSY", clone2name,
4135 		    error);
4136 	dmu_objset_disown(os, B_TRUE, FTAG);
4137 
4138 out:
4139 	ztest_dsl_dataset_cleanup(osname, id);
4140 
4141 	rw_exit(&ztest_name_lock);
4142 }
4143 
4144 /*
4145  * Verify that dmu_object_{alloc,free} work as expected.
4146  */
4147 void
4148 ztest_dmu_object_alloc_free(ztest_ds_t *zd, uint64_t id)
4149 {
4150 	ztest_od_t od[4];
4151 	int batchsize = sizeof (od) / sizeof (od[0]);
4152 
4153 	for (int b = 0; b < batchsize; b++) {
4154 		ztest_od_init(&od[b], id, FTAG, b, DMU_OT_UINT64_OTHER,
4155 		    0, 0, 0);
4156 	}
4157 
4158 	/*
4159 	 * Destroy the previous batch of objects, create a new batch,
4160 	 * and do some I/O on the new objects.
4161 	 */
4162 	if (ztest_object_init(zd, od, sizeof (od), B_TRUE) != 0)
4163 		return;
4164 
4165 	while (ztest_random(4 * batchsize) != 0)
4166 		ztest_io(zd, od[ztest_random(batchsize)].od_object,
4167 		    ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);
4168 }
4169 
4170 /*
4171  * Rewind the global allocator to verify object allocation backfilling.
4172  */
4173 void
4174 ztest_dmu_object_next_chunk(ztest_ds_t *zd, uint64_t id)
4175 {
4176 	objset_t *os = zd->zd_os;
4177 	int dnodes_per_chunk = 1 << dmu_object_alloc_chunk_shift;
4178 	uint64_t object;
4179 
4180 	/*
4181 	 * Rewind the global allocator randomly back to a lower object number
4182 	 * to force backfilling and reclamation of recently freed dnodes.
4183 	 */
4184 	mutex_enter(&os->os_obj_lock);
4185 	object = ztest_random(os->os_obj_next_chunk);
4186 	os->os_obj_next_chunk = P2ALIGN(object, dnodes_per_chunk);
4187 	mutex_exit(&os->os_obj_lock);
4188 }
4189 
4190 /*
4191  * Verify that dmu_{read,write} work as expected.
4192  */
4193 void
4194 ztest_dmu_read_write(ztest_ds_t *zd, uint64_t id)
4195 {
4196 	objset_t *os = zd->zd_os;
4197 	ztest_od_t od[2];
4198 	dmu_tx_t *tx;
4199 	int i, freeit, error;
4200 	uint64_t n, s, txg;
4201 	bufwad_t *packbuf, *bigbuf, *pack, *bigH, *bigT;
4202 	uint64_t packobj, packoff, packsize, bigobj, bigoff, bigsize;
4203 	uint64_t chunksize = (1000 + ztest_random(1000)) * sizeof (uint64_t);
4204 	uint64_t regions = 997;
4205 	uint64_t stride = 123456789ULL;
4206 	uint64_t width = 40;
4207 	int free_percent = 5;
4208 
4209 	/*
4210 	 * This test uses two objects, packobj and bigobj, that are always
4211 	 * updated together (i.e. in the same tx) so that their contents are
4212 	 * in sync and can be compared.  Their contents relate to each other
4213 	 * in a simple way: packobj is a dense array of 'bufwad' structures,
4214 	 * while bigobj is a sparse array of the same bufwads.  Specifically,
4215 	 * for any index n, there are three bufwads that should be identical:
4216 	 *
4217 	 *	packobj, at offset n * sizeof (bufwad_t)
4218 	 *	bigobj, at the head of the nth chunk
4219 	 *	bigobj, at the tail of the nth chunk
4220 	 *
4221 	 * The chunk size is arbitrary. It doesn't have to be a power of two,
4222 	 * and it doesn't have any relation to the object blocksize.
4223 	 * The only requirement is that it can hold at least two bufwads.
4224 	 *
4225 	 * Normally, we write the bufwad to each of these locations.
4226 	 * However, free_percent of the time we instead write zeroes to
4227 	 * packobj and perform a dmu_free_range() on bigobj.  By comparing
4228 	 * bigobj to packobj, we can verify that the DMU is correctly
4229 	 * tracking which parts of an object are allocated and free,
4230 	 * and that the contents of the allocated blocks are correct.
4231 	 */
4232 
4233 	/*
4234 	 * Read the directory info.  If it's the first time, set things up.
4235 	 */
4236 	ztest_od_init(&od[0], id, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0,
4237 	    chunksize);
4238 	ztest_od_init(&od[1], id, FTAG, 1, DMU_OT_UINT64_OTHER, 0, 0,
4239 	    chunksize);
4240 
4241 	if (ztest_object_init(zd, od, sizeof (od), B_FALSE) != 0)
4242 		return;
4243 
4244 	bigobj = od[0].od_object;
4245 	packobj = od[1].od_object;
4246 	chunksize = od[0].od_gen;
4247 	ASSERT(chunksize == od[1].od_gen);
4248 
4249 	/*
4250 	 * Prefetch a random chunk of the big object.
4251 	 * Our aim here is to get some async reads in flight
4252 	 * for blocks that we may free below; the DMU should
4253 	 * handle this race correctly.
4254 	 */
4255 	n = ztest_random(regions) * stride + ztest_random(width);
4256 	s = 1 + ztest_random(2 * width - 1);
4257 	dmu_prefetch(os, bigobj, 0, n * chunksize, s * chunksize,
4258 	    ZIO_PRIORITY_SYNC_READ);
4259 
4260 	/*
4261 	 * Pick a random index and compute the offsets into packobj and bigobj.
4262 	 */
4263 	n = ztest_random(regions) * stride + ztest_random(width);
4264 	s = 1 + ztest_random(width - 1);
4265 
4266 	packoff = n * sizeof (bufwad_t);
4267 	packsize = s * sizeof (bufwad_t);
4268 
4269 	bigoff = n * chunksize;
4270 	bigsize = s * chunksize;
4271 
4272 	packbuf = umem_alloc(packsize, UMEM_NOFAIL);
4273 	bigbuf = umem_alloc(bigsize, UMEM_NOFAIL);
4274 
4275 	/*
4276 	 * free_percent of the time, free a range of bigobj rather than
4277 	 * overwriting it.
4278 	 */
4279 	freeit = (ztest_random(100) < free_percent);
4280 
4281 	/*
4282 	 * Read the current contents of our objects.
4283 	 */
4284 	error = dmu_read(os, packobj, packoff, packsize, packbuf,
4285 	    DMU_READ_PREFETCH);
4286 	ASSERT0(error);
4287 	error = dmu_read(os, bigobj, bigoff, bigsize, bigbuf,
4288 	    DMU_READ_PREFETCH);
4289 	ASSERT0(error);
4290 
4291 	/*
4292 	 * Get a tx for the mods to both packobj and bigobj.
4293 	 */
4294 	tx = dmu_tx_create(os);
4295 
4296 	dmu_tx_hold_write(tx, packobj, packoff, packsize);
4297 
4298 	if (freeit)
4299 		dmu_tx_hold_free(tx, bigobj, bigoff, bigsize);
4300 	else
4301 		dmu_tx_hold_write(tx, bigobj, bigoff, bigsize);
4302 
4303 	/* This accounts for setting the checksum/compression. */
4304 	dmu_tx_hold_bonus(tx, bigobj);
4305 
4306 	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
4307 	if (txg == 0) {
4308 		umem_free(packbuf, packsize);
4309 		umem_free(bigbuf, bigsize);
4310 		return;
4311 	}
4312 
4313 	enum zio_checksum cksum;
4314 	do {
4315 		cksum = (enum zio_checksum)
4316 		    ztest_random_dsl_prop(ZFS_PROP_CHECKSUM);
4317 	} while (cksum >= ZIO_CHECKSUM_LEGACY_FUNCTIONS);
4318 	dmu_object_set_checksum(os, bigobj, cksum, tx);
4319 
4320 	enum zio_compress comp;
4321 	do {
4322 		comp = (enum zio_compress)
4323 		    ztest_random_dsl_prop(ZFS_PROP_COMPRESSION);
4324 	} while (comp >= ZIO_COMPRESS_LEGACY_FUNCTIONS);
4325 	dmu_object_set_compress(os, bigobj, comp, tx);
4326 
4327 	/*
4328 	 * For each index from n to n + s, verify that the existing bufwad
4329 	 * in packobj matches the bufwads at the head and tail of the
4330 	 * corresponding chunk in bigobj.  Then update all three bufwads
4331 	 * with the new values we want to write out.
4332 	 */
4333 	for (i = 0; i < s; i++) {
4334 		/* LINTED */
4335 		pack = (bufwad_t *)((char *)packbuf + i * sizeof (bufwad_t));
4336 		/* LINTED */
4337 		bigH = (bufwad_t *)((char *)bigbuf + i * chunksize);
4338 		/* LINTED */
4339 		bigT = (bufwad_t *)((char *)bigH + chunksize) - 1;
4340 
4341 		ASSERT((uintptr_t)bigH - (uintptr_t)bigbuf < bigsize);
4342 		ASSERT((uintptr_t)bigT - (uintptr_t)bigbuf < bigsize);
4343 
4344 		if (pack->bw_txg > txg)
4345 			fatal(0, "future leak: got %llx, open txg is %llx",
4346 			    pack->bw_txg, txg);
4347 
4348 		if (pack->bw_data != 0 && pack->bw_index != n + i)
4349 			fatal(0, "wrong index: got %llx, wanted %llx+%llx",
4350 			    pack->bw_index, n, i);
4351 
4352 		if (bcmp(pack, bigH, sizeof (bufwad_t)) != 0)
4353 			fatal(0, "pack/bigH mismatch in %p/%p", pack, bigH);
4354 
4355 		if (bcmp(pack, bigT, sizeof (bufwad_t)) != 0)
4356 			fatal(0, "pack/bigT mismatch in %p/%p", pack, bigT);
4357 
4358 		if (freeit) {
4359 			bzero(pack, sizeof (bufwad_t));
4360 		} else {
4361 			pack->bw_index = n + i;
4362 			pack->bw_txg = txg;
4363 			pack->bw_data = 1 + ztest_random(-2ULL);
4364 		}
4365 		*bigH = *pack;
4366 		*bigT = *pack;
4367 	}
4368 
4369 	/*
4370 	 * We've verified all the old bufwads, and made new ones.
4371 	 * Now write them out.
4372 	 */
4373 	dmu_write(os, packobj, packoff, packsize, packbuf, tx);
4374 
4375 	if (freeit) {
4376 		if (ztest_opts.zo_verbose >= 7) {
4377 			(void) printf("freeing offset %llx size %llx"
4378 			    " txg %llx\n",
4379 			    (u_longlong_t)bigoff,
4380 			    (u_longlong_t)bigsize,
4381 			    (u_longlong_t)txg);
4382 		}
4383 		VERIFY(0 == dmu_free_range(os, bigobj, bigoff, bigsize, tx));
4384 	} else {
4385 		if (ztest_opts.zo_verbose >= 7) {
4386 			(void) printf("writing offset %llx size %llx"
4387 			    " txg %llx\n",
4388 			    (u_longlong_t)bigoff,
4389 			    (u_longlong_t)bigsize,
4390 			    (u_longlong_t)txg);
4391 		}
4392 		dmu_write(os, bigobj, bigoff, bigsize, bigbuf, tx);
4393 	}
4394 
4395 	dmu_tx_commit(tx);
4396 
4397 	/*
4398 	 * Sanity check the stuff we just wrote.
4399 	 */
4400 	{
4401 		void *packcheck = umem_alloc(packsize, UMEM_NOFAIL);
4402 		void *bigcheck = umem_alloc(bigsize, UMEM_NOFAIL);
4403 
4404 		VERIFY(0 == dmu_read(os, packobj, packoff,
4405 		    packsize, packcheck, DMU_READ_PREFETCH));
4406 		VERIFY(0 == dmu_read(os, bigobj, bigoff,
4407 		    bigsize, bigcheck, DMU_READ_PREFETCH));
4408 
4409 		ASSERT(bcmp(packbuf, packcheck, packsize) == 0);
4410 		ASSERT(bcmp(bigbuf, bigcheck, bigsize) == 0);
4411 
4412 		umem_free(packcheck, packsize);
4413 		umem_free(bigcheck, bigsize);
4414 	}
4415 
4416 	umem_free(packbuf, packsize);
4417 	umem_free(bigbuf, bigsize);
4418 }
4419 
4420 void
4421 compare_and_update_pbbufs(uint64_t s, bufwad_t *packbuf, bufwad_t *bigbuf,
4422     uint64_t bigsize, uint64_t n, uint64_t chunksize, uint64_t txg)
4423 {
4424 	uint64_t i;
4425 	bufwad_t *pack;
4426 	bufwad_t *bigH;
4427 	bufwad_t *bigT;
4428 
4429 	/*
4430 	 * For each index from n to n + s, verify that the existing bufwad
4431 	 * in packobj matches the bufwads at the head and tail of the
4432 	 * corresponding chunk in bigobj.  Then update all three bufwads
4433 	 * with the new values we want to write out.
4434 	 */
4435 	for (i = 0; i < s; i++) {
4436 		/* LINTED */
4437 		pack = (bufwad_t *)((char *)packbuf + i * sizeof (bufwad_t));
4438 		/* LINTED */
4439 		bigH = (bufwad_t *)((char *)bigbuf + i * chunksize);
4440 		/* LINTED */
4441 		bigT = (bufwad_t *)((char *)bigH + chunksize) - 1;
4442 
4443 		ASSERT((uintptr_t)bigH - (uintptr_t)bigbuf < bigsize);
4444 		ASSERT((uintptr_t)bigT - (uintptr_t)bigbuf < bigsize);
4445 
4446 		if (pack->bw_txg > txg)
4447 			fatal(0, "future leak: got %llx, open txg is %llx",
4448 			    pack->bw_txg, txg);
4449 
4450 		if (pack->bw_data != 0 && pack->bw_index != n + i)
4451 			fatal(0, "wrong index: got %llx, wanted %llx+%llx",
4452 			    pack->bw_index, n, i);
4453 
4454 		if (bcmp(pack, bigH, sizeof (bufwad_t)) != 0)
4455 			fatal(0, "pack/bigH mismatch in %p/%p", pack, bigH);
4456 
4457 		if (bcmp(pack, bigT, sizeof (bufwad_t)) != 0)
4458 			fatal(0, "pack/bigT mismatch in %p/%p", pack, bigT);
4459 
4460 		pack->bw_index = n + i;
4461 		pack->bw_txg = txg;
4462 		pack->bw_data = 1 + ztest_random(-2ULL);
4463 
4464 		*bigH = *pack;
4465 		*bigT = *pack;
4466 	}
4467 }
4468 
4469 void
4470 ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
4471 {
4472 	objset_t *os = zd->zd_os;
4473 	ztest_od_t od[2];
4474 	dmu_tx_t *tx;
4475 	uint64_t i;
4476 	int error;
4477 	uint64_t n, s, txg;
4478 	bufwad_t *packbuf, *bigbuf;
4479 	uint64_t packobj, packoff, packsize, bigobj, bigoff, bigsize;
4480 	uint64_t blocksize = ztest_random_blocksize();
4481 	uint64_t chunksize = blocksize;
4482 	uint64_t regions = 997;
4483 	uint64_t stride = 123456789ULL;
4484 	uint64_t width = 9;
4485 	dmu_buf_t *bonus_db;
4486 	arc_buf_t **bigbuf_arcbufs;
4487 	dmu_object_info_t doi;
4488 
4489 	/*
4490 	 * This test uses two objects, packobj and bigobj, that are always
4491 	 * updated together (i.e. in the same tx) so that their contents are
4492 	 * in sync and can be compared.  Their contents relate to each other
4493 	 * in a simple way: packobj is a dense array of 'bufwad' structures,
4494 	 * while bigobj is a sparse array of the same bufwads.  Specifically,
4495 	 * for any index n, there are three bufwads that should be identical:
4496 	 *
4497 	 *	packobj, at offset n * sizeof (bufwad_t)
4498 	 *	bigobj, at the head of the nth chunk
4499 	 *	bigobj, at the tail of the nth chunk
4500 	 *
4501 	 * The chunk size is set equal to bigobj block size so that
4502 	 * dmu_assign_arcbuf_by_dbuf() can be tested for object updates.
4503 	 */
4504 
4505 	/*
4506 	 * Read the directory info.  If it's the first time, set things up.
4507 	 */
4508 	ztest_od_init(&od[0], id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize,
4509 	    0, 0);
4510 	ztest_od_init(&od[1], id, FTAG, 1, DMU_OT_UINT64_OTHER, 0, 0,
4511 	    chunksize);
4512 
4513 	if (ztest_object_init(zd, od, sizeof (od), B_FALSE) != 0)
4514 		return;
4515 
4516 	bigobj = od[0].od_object;
4517 	packobj = od[1].od_object;
4518 	blocksize = od[0].od_blocksize;
4519 	chunksize = blocksize;
4520 	ASSERT(chunksize == od[1].od_gen);
4521 
4522 	VERIFY(dmu_object_info(os, bigobj, &doi) == 0);
4523 	VERIFY(ISP2(doi.doi_data_block_size));
4524 	VERIFY(chunksize == doi.doi_data_block_size);
4525 	VERIFY(chunksize >= 2 * sizeof (bufwad_t));
4526 
4527 	/*
4528 	 * Pick a random index and compute the offsets into packobj and bigobj.
4529 	 */
4530 	n = ztest_random(regions) * stride + ztest_random(width);
4531 	s = 1 + ztest_random(width - 1);
4532 
4533 	packoff = n * sizeof (bufwad_t);
4534 	packsize = s * sizeof (bufwad_t);
4535 
4536 	bigoff = n * chunksize;
4537 	bigsize = s * chunksize;
4538 
4539 	packbuf = umem_zalloc(packsize, UMEM_NOFAIL);
4540 	bigbuf = umem_zalloc(bigsize, UMEM_NOFAIL);
4541 
4542 	VERIFY3U(0, ==, dmu_bonus_hold(os, bigobj, FTAG, &bonus_db));
4543 
4544 	bigbuf_arcbufs = umem_zalloc(2 * s * sizeof (arc_buf_t *), UMEM_NOFAIL);
4545 
4546 	/*
4547 	 * Iteration 0 test zcopy for DB_UNCACHED dbufs.
4548 	 * Iteration 1 test zcopy to already referenced dbufs.
4549 	 * Iteration 2 test zcopy to dirty dbuf in the same txg.
4550 	 * Iteration 3 test zcopy to dbuf dirty in previous txg.
4551 	 * Iteration 4 test zcopy when dbuf is no longer dirty.
4552 	 * Iteration 5 test zcopy when it can't be done.
4553 	 * Iteration 6 one more zcopy write.
4554 	 */
4555 	for (i = 0; i < 7; i++) {
4556 		uint64_t j;
4557 		uint64_t off;
4558 
4559 		/*
4560 		 * In iteration 5 (i == 5) use arcbufs
4561 		 * that don't match bigobj blksz to test
4562 		 * dmu_assign_arcbuf_by_dbuf() when it can't directly
4563 		 * assign an arcbuf to a dbuf.
4564 		 */
4565 		for (j = 0; j < s; j++) {
4566 			if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
4567 				bigbuf_arcbufs[j] =
4568 				    dmu_request_arcbuf(bonus_db, chunksize);
4569 			} else {
4570 				bigbuf_arcbufs[2 * j] =
4571 				    dmu_request_arcbuf(bonus_db, chunksize / 2);
4572 				bigbuf_arcbufs[2 * j + 1] =
4573 				    dmu_request_arcbuf(bonus_db, chunksize / 2);
4574 			}
4575 		}
4576 
4577 		/*
4578 		 * Get a tx for the mods to both packobj and bigobj.
4579 		 */
4580 		tx = dmu_tx_create(os);
4581 
4582 		dmu_tx_hold_write(tx, packobj, packoff, packsize);
4583 		dmu_tx_hold_write(tx, bigobj, bigoff, bigsize);
4584 
4585 		txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
4586 		if (txg == 0) {
4587 			umem_free(packbuf, packsize);
4588 			umem_free(bigbuf, bigsize);
4589 			for (j = 0; j < s; j++) {
4590 				if (i != 5 ||
4591 				    chunksize < (SPA_MINBLOCKSIZE * 2)) {
4592 					dmu_return_arcbuf(bigbuf_arcbufs[j]);
4593 				} else {
4594 					dmu_return_arcbuf(
4595 					    bigbuf_arcbufs[2 * j]);
4596 					dmu_return_arcbuf(
4597 					    bigbuf_arcbufs[2 * j + 1]);
4598 				}
4599 			}
4600 			umem_free(bigbuf_arcbufs, 2 * s * sizeof (arc_buf_t *));
4601 			dmu_buf_rele(bonus_db, FTAG);
4602 			return;
4603 		}
4604 
4605 		/*
4606 		 * 50% of the time don't read objects in the 1st iteration to
4607 		 * test dmu_assign_arcbuf_by_dbuf() for the case when there are
4608 		 * no existing dbufs for the specified offsets.
4609 		 */
4610 		if (i != 0 || ztest_random(2) != 0) {
4611 			error = dmu_read(os, packobj, packoff,
4612 			    packsize, packbuf, DMU_READ_PREFETCH);
4613 			ASSERT0(error);
4614 			error = dmu_read(os, bigobj, bigoff, bigsize,
4615 			    bigbuf, DMU_READ_PREFETCH);
4616 			ASSERT0(error);
4617 		}
4618 		compare_and_update_pbbufs(s, packbuf, bigbuf, bigsize,
4619 		    n, chunksize, txg);
4620 
4621 		/*
4622 		 * We've verified all the old bufwads, and made new ones.
4623 		 * Now write them out.
4624 		 */
4625 		dmu_write(os, packobj, packoff, packsize, packbuf, tx);
4626 		if (ztest_opts.zo_verbose >= 7) {
4627 			(void) printf("writing offset %llx size %llx"
4628 			    " txg %llx\n",
4629 			    (u_longlong_t)bigoff,
4630 			    (u_longlong_t)bigsize,
4631 			    (u_longlong_t)txg);
4632 		}
4633 		for (off = bigoff, j = 0; j < s; j++, off += chunksize) {
4634 			dmu_buf_t *dbt;
4635 			if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
4636 				bcopy((caddr_t)bigbuf + (off - bigoff),
4637 				    bigbuf_arcbufs[j]->b_data, chunksize);
4638 			} else {
4639 				bcopy((caddr_t)bigbuf + (off - bigoff),
4640 				    bigbuf_arcbufs[2 * j]->b_data,
4641 				    chunksize / 2);
4642 				bcopy((caddr_t)bigbuf + (off - bigoff) +
4643 				    chunksize / 2,
4644 				    bigbuf_arcbufs[2 * j + 1]->b_data,
4645 				    chunksize / 2);
4646 			}
4647 
4648 			if (i == 1) {
4649 				VERIFY(dmu_buf_hold(os, bigobj, off,
4650 				    FTAG, &dbt, DMU_READ_NO_PREFETCH) == 0);
4651 			}
4652 			if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
4653 				dmu_assign_arcbuf_by_dbuf(bonus_db, off,
4654 				    bigbuf_arcbufs[j], tx);
4655 			} else {
4656 				dmu_assign_arcbuf_by_dbuf(bonus_db, off,
4657 				    bigbuf_arcbufs[2 * j], tx);
4658 				dmu_assign_arcbuf_by_dbuf(bonus_db,
4659 				    off + chunksize / 2,
4660 				    bigbuf_arcbufs[2 * j + 1], tx);
4661 			}
4662 			if (i == 1) {
4663 				dmu_buf_rele(dbt, FTAG);
4664 			}
4665 		}
4666 		dmu_tx_commit(tx);
4667 
4668 		/*
4669 		 * Sanity check the stuff we just wrote.
4670 		 */
4671 		{
4672 			void *packcheck = umem_alloc(packsize, UMEM_NOFAIL);
4673 			void *bigcheck = umem_alloc(bigsize, UMEM_NOFAIL);
4674 
4675 			VERIFY(0 == dmu_read(os, packobj, packoff,
4676 			    packsize, packcheck, DMU_READ_PREFETCH));
4677 			VERIFY(0 == dmu_read(os, bigobj, bigoff,
4678 			    bigsize, bigcheck, DMU_READ_PREFETCH));
4679 
4680 			ASSERT(bcmp(packbuf, packcheck, packsize) == 0);
4681 			ASSERT(bcmp(bigbuf, bigcheck, bigsize) == 0);
4682 
4683 			umem_free(packcheck, packsize);
4684 			umem_free(bigcheck, bigsize);
4685 		}
4686 		if (i == 2) {
4687 			txg_wait_open(dmu_objset_pool(os), 0);
4688 		} else if (i == 3) {
4689 			txg_wait_synced(dmu_objset_pool(os), 0);
4690 		}
4691 	}
4692 
4693 	dmu_buf_rele(bonus_db, FTAG);
4694 	umem_free(packbuf, packsize);
4695 	umem_free(bigbuf, bigsize);
4696 	umem_free(bigbuf_arcbufs, 2 * s * sizeof (arc_buf_t *));
4697 }
4698 
4699 /* ARGSUSED */
4700 void
4701 ztest_dmu_write_parallel(ztest_ds_t *zd, uint64_t id)
4702 {
4703 	ztest_od_t od[1];
4704 	uint64_t offset = (1ULL << (ztest_random(20) + 43)) +
4705 	    (ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);
4706 
4707 	/*
4708 	 * Have multiple threads write to large offsets in an object
4709 	 * to verify that parallel writes to an object -- even to the
4710 	 * same blocks within the object -- doesn't cause any trouble.
4711 	 */
4712 	ztest_od_init(&od[0], ID_PARALLEL, FTAG, 0, DMU_OT_UINT64_OTHER,
4713 	    0, 0, 0);
4714 
4715 	if (ztest_object_init(zd, od, sizeof (od), B_FALSE) != 0)
4716 		return;
4717 
4718 	while (ztest_random(10) != 0)
4719 		ztest_io(zd, od[0].od_object, offset);
4720 }
4721 
4722 void
4723 ztest_dmu_prealloc(ztest_ds_t *zd, uint64_t id)
4724 {
4725 	ztest_od_t od[1];
4726 	uint64_t offset = (1ULL << (ztest_random(4) + SPA_MAXBLOCKSHIFT)) +
4727 	    (ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);
4728 	uint64_t count = ztest_random(20) + 1;
4729 	uint64_t blocksize = ztest_random_blocksize();
4730 	void *data;
4731 
4732 	ztest_od_init(&od[0], id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize,
4733 	    0, 0);
4734 
4735 	if (ztest_object_init(zd, od, sizeof (od), !ztest_random(2)) != 0)
4736 		return;
4737 
4738 	if (ztest_truncate(zd, od[0].od_object, offset, count * blocksize) != 0)
4739 		return;
4740 
4741 	ztest_prealloc(zd, od[0].od_object, offset, count * blocksize);
4742 
4743 	data = umem_zalloc(blocksize, UMEM_NOFAIL);
4744 
4745 	while (ztest_random(count) != 0) {
4746 		uint64_t randoff = offset + (ztest_random(count) * blocksize);
4747 		if (ztest_write(zd, od[0].od_object, randoff, blocksize,
4748 		    data) != 0)
4749 			break;
4750 		while (ztest_random(4) != 0)
4751 			ztest_io(zd, od[0].od_object, randoff);
4752 	}
4753 
4754 	umem_free(data, blocksize);
4755 }
4756 
4757 /*
4758  * Verify that zap_{create,destroy,add,remove,update} work as expected.
4759  */
4760 #define	ZTEST_ZAP_MIN_INTS	1
4761 #define	ZTEST_ZAP_MAX_INTS	4
4762 #define	ZTEST_ZAP_MAX_PROPS	1000
4763 
4764 void
4765 ztest_zap(ztest_ds_t *zd, uint64_t id)
4766 {
4767 	objset_t *os = zd->zd_os;
4768 	ztest_od_t od[1];
4769 	uint64_t object;
4770 	uint64_t txg, last_txg;
4771 	uint64_t value[ZTEST_ZAP_MAX_INTS];
4772 	uint64_t zl_ints, zl_intsize, prop;
4773 	int i, ints;
4774 	dmu_tx_t *tx;
4775 	char propname[100], txgname[100];
4776 	int error;
4777 	char *hc[2] = { "s.acl.h", ".s.open.h.hyLZlg" };
4778 
4779 	ztest_od_init(&od[0], id, FTAG, 0, DMU_OT_ZAP_OTHER, 0, 0, 0);
4780 
4781 	if (ztest_object_init(zd, od, sizeof (od), !ztest_random(2)) != 0)
4782 		return;
4783 
4784 	object = od[0].od_object;
4785 
4786 	/*
4787 	 * Generate a known hash collision, and verify that
4788 	 * we can lookup and remove both entries.
4789 	 */
4790 	tx = dmu_tx_create(os);
4791 	dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
4792 	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
4793 	if (txg == 0)
4794 		return;
4795 	for (i = 0; i < 2; i++) {
4796 		value[i] = i;
4797 		VERIFY3U(0, ==, zap_add(os, object, hc[i], sizeof (uint64_t),
4798 		    1, &value[i], tx));
4799 	}
4800 	for (i = 0; i < 2; i++) {
4801 		VERIFY3U(EEXIST, ==, zap_add(os, object, hc[i],
4802 		    sizeof (uint64_t), 1, &value[i], tx));
4803 		VERIFY3U(0, ==,
4804 		    zap_length(os, object, hc[i], &zl_intsize, &zl_ints));
4805 		ASSERT3U(zl_intsize, ==, sizeof (uint64_t));
4806 		ASSERT3U(zl_ints, ==, 1);
4807 	}
4808 	for (i = 0; i < 2; i++) {
4809 		VERIFY3U(0, ==, zap_remove(os, object, hc[i], tx));
4810 	}
4811 	dmu_tx_commit(tx);
4812 
4813 	/*
4814 	 * Generate a buch of random entries.
4815 	 */
4816 	ints = MAX(ZTEST_ZAP_MIN_INTS, object % ZTEST_ZAP_MAX_INTS);
4817 
4818 	prop = ztest_random(ZTEST_ZAP_MAX_PROPS);
4819 	(void) sprintf(propname, "prop_%llu", (u_longlong_t)prop);
4820 	(void) sprintf(txgname, "txg_%llu", (u_longlong_t)prop);
4821 	bzero(value, sizeof (value));
4822 	last_txg = 0;
4823 
4824 	/*
4825 	 * If these zap entries already exist, validate their contents.
4826 	 */
4827 	error = zap_length(os, object, txgname, &zl_intsize, &zl_ints);
4828 	if (error == 0) {
4829 		ASSERT3U(zl_intsize, ==, sizeof (uint64_t));
4830 		ASSERT3U(zl_ints, ==, 1);
4831 
4832 		VERIFY(zap_lookup(os, object, txgname, zl_intsize,
4833 		    zl_ints, &last_txg) == 0);
4834 
4835 		VERIFY(zap_length(os, object, propname, &zl_intsize,
4836 		    &zl_ints) == 0);
4837 
4838 		ASSERT3U(zl_intsize, ==, sizeof (uint64_t));
4839 		ASSERT3U(zl_ints, ==, ints);
4840 
4841 		VERIFY(zap_lookup(os, object, propname, zl_intsize,
4842 		    zl_ints, value) == 0);
4843 
4844 		for (i = 0; i < ints; i++) {
4845 			ASSERT3U(value[i], ==, last_txg + object + i);
4846 		}
4847 	} else {
4848 		ASSERT3U(error, ==, ENOENT);
4849 	}
4850 
4851 	/*
4852 	 * Atomically update two entries in our zap object.
4853 	 * The first is named txg_%llu, and contains the txg
4854 	 * in which the property was last updated.  The second
4855 	 * is named prop_%llu, and the nth element of its value
4856 	 * should be txg + object + n.
4857 	 */
4858 	tx = dmu_tx_create(os);
4859 	dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
4860 	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
4861 	if (txg == 0)
4862 		return;
4863 
4864 	if (last_txg > txg)
4865 		fatal(0, "zap future leak: old %llu new %llu", last_txg, txg);
4866 
4867 	for (i = 0; i < ints; i++)
4868 		value[i] = txg + object + i;
4869 
4870 	VERIFY3U(0, ==, zap_update(os, object, txgname, sizeof (uint64_t),
4871 	    1, &txg, tx));
4872 	VERIFY3U(0, ==, zap_update(os, object, propname, sizeof (uint64_t),
4873 	    ints, value, tx));
4874 
4875 	dmu_tx_commit(tx);
4876 
4877 	/*
4878 	 * Remove a random pair of entries.
4879 	 */
4880 	prop = ztest_random(ZTEST_ZAP_MAX_PROPS);
4881 	(void) sprintf(propname, "prop_%llu", (u_longlong_t)prop);
4882 	(void) sprintf(txgname, "txg_%llu", (u_longlong_t)prop);
4883 
4884 	error = zap_length(os, object, txgname, &zl_intsize, &zl_ints);
4885 
4886 	if (error == ENOENT)
4887 		return;
4888 
4889 	ASSERT0(error);
4890 
4891 	tx = dmu_tx_create(os);
4892 	dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
4893 	txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
4894 	if (txg == 0)
4895 		return;
4896 	VERIFY3U(0, ==, zap_remove(os, object, txgname, tx));
4897 	VERIFY3U(0, ==, zap_remove(os, object, propname, tx));
4898 	dmu_tx_commit(tx);
4899 }
4900 
4901 /*
4902  * Testcase to test the upgrading of a microzap to fatzap.
4903  */
4904 void
4905 ztest_fzap(ztest_ds_t *zd, uint64_t id)
4906 {
4907 	objset_t *os = zd->zd_os;
4908 	ztest_od_t od[1];
4909 	uint64_t object, txg;
4910 
4911 	ztest_od_init(&od[0], id, FTAG, 0, DMU_OT_ZAP_OTHER, 0, 0, 0);
4912 
4913 	if (ztest_object_init(zd, od, sizeof (od), !ztest_random(2)) != 0)
4914 		return;
4915 
4916 	object = od[0].od_object;
4917 
4918 	/*
4919 	 * Add entries to this ZAP and make sure it spills over
4920 	 * and gets upgraded to a fatzap. Also, since we are adding
4921 	 * 2050 entries we should see ptrtbl growth and leaf-block split.
4922 	 */
4923 	for (int i = 0; i < 2050; i++) {
4924 		char name[ZFS_MAX_DATASET_NAME_LEN];
4925 		uint64_t value = i;
4926 		dmu_tx_t *tx;
4927 		int error;
4928 
4929 		(void) snprintf(name, sizeof (name), "fzap-%llu-%llu",
4930 		    id, value);
4931 
4932 		tx = dmu_tx_create(os);
4933 		dmu_tx_hold_zap(tx, object, B_TRUE, name);
4934 		txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
4935 		if (txg == 0)
4936 			return;
4937 		error = zap_add(os, object, name, sizeof (uint64_t), 1,
4938 		    &value, tx);
4939 		ASSERT(error == 0 || error == EEXIST);
4940 		dmu_tx_commit(tx);
4941 	}
4942 }
4943 
4944 /* ARGSUSED */
4945 void
4946 ztest_zap_parallel(ztest_ds_t *zd, uint64_t id)
4947 {
4948 	objset_t *os = zd->zd_os;
4949 	ztest_od_t od[1];
4950 	uint64_t txg, object, count, wsize, wc, zl_wsize, zl_wc;
4951 	dmu_tx_t *tx;
4952 	int i, namelen, error;
4953 	int micro = ztest_random(2);
4954 	char name[20], string_value[20];
4955 	void *data;
4956 
4957 	ztest_od_init(&od[0], ID_PARALLEL, FTAG, micro, DMU_OT_ZAP_OTHER,
4958 	    0, 0, 0);
4959 
4960 	if (ztest_object_init(zd, od, sizeof (od), B_FALSE) != 0)
4961 		return;
4962 
4963 	object = od[0].od_object;
4964 
4965 	/*
4966 	 * Generate a random name of the form 'xxx.....' where each
4967 	 * x is a random printable character and the dots are dots.
4968 	 * There are 94 such characters, and the name length goes from
4969 	 * 6 to 20, so there are 94^3 * 15 = 12,458,760 possible names.
4970 	 */
4971 	namelen = ztest_random(sizeof (name) - 5) + 5 + 1;
4972 
4973 	for (i = 0; i < 3; i++)
4974 		name[i] = '!' + ztest_random('~' - '!' + 1);
4975 	for (; i < namelen - 1; i++)
4976 		name[i] = '.';
4977 	name[i] = '\0';
4978 
4979 	if ((namelen & 1) || micro) {
4980 		wsize = sizeof (txg);
4981 		wc = 1;
4982 		data = &txg;
4983 	} else {
4984 		wsize = 1;
4985 		wc = namelen;
4986 		data = string_value;
4987 	}
4988 
4989 	count = -1ULL;
4990 	VERIFY0(zap_count(os, object, &count));
4991 	ASSERT(count != -1ULL);
4992 
4993 	/*
4994 	 * Select an operation: length, lookup, add, update, remove.
4995 	 */
4996 	i = ztest_random(5);
4997 
4998 	if (i >= 2) {
4999 		tx = dmu_tx_create(os);
5000 		dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
5001 		txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
5002 		if (txg == 0)
5003 			return;
5004 		bcopy(name, string_value, namelen);
5005 	} else {
5006 		tx = NULL;
5007 		txg = 0;
5008 		bzero(string_value, namelen);
5009 	}
5010 
5011 	switch (i) {
5012 
5013 	case 0:
5014 		error = zap_length(os, object, name, &zl_wsize, &zl_wc);
5015 		if (error == 0) {
5016 			ASSERT3U(wsize, ==, zl_wsize);
5017 			ASSERT3U(wc, ==, zl_wc);
5018 		} else {
5019 			ASSERT3U(error, ==, ENOENT);
5020 		}
5021 		break;
5022 
5023 	case 1:
5024 		error = zap_lookup(os, object, name, wsize, wc, data);
5025 		if (error == 0) {
5026 			if (data == string_value &&
5027 			    bcmp(name, data, namelen) != 0)
5028 				fatal(0, "name '%s' != val '%s' len %d",
5029 				    name, data, namelen);
5030 		} else {
5031 			ASSERT3U(error, ==, ENOENT);
5032 		}
5033 		break;
5034 
5035 	case 2:
5036 		error = zap_add(os, object, name, wsize, wc, data, tx);
5037 		ASSERT(error == 0 || error == EEXIST);
5038 		break;
5039 
5040 	case 3:
5041 		VERIFY(zap_update(os, object, name, wsize, wc, data, tx) == 0);
5042 		break;
5043 
5044 	case 4:
5045 		error = zap_remove(os, object, name, tx);
5046 		ASSERT(error == 0 || error == ENOENT);
5047 		break;
5048 	}
5049 
5050 	if (tx != NULL)
5051 		dmu_tx_commit(tx);
5052 }
5053 
5054 /*
5055  * Commit callback data.
5056  */
5057 typedef struct ztest_cb_data {
5058 	list_node_t		zcd_node;
5059 	uint64_t		zcd_txg;
5060 	int			zcd_expected_err;
5061 	boolean_t		zcd_added;
5062 	boolean_t		zcd_called;
5063 	spa_t			*zcd_spa;
5064 } ztest_cb_data_t;
5065 
5066 /* This is the actual commit callback function */
5067 static void
5068 ztest_commit_callback(void *arg, int error)
5069 {
5070 	ztest_cb_data_t *data = arg;
5071 	uint64_t synced_txg;
5072 
5073 	VERIFY(data != NULL);
5074 	VERIFY3S(data->zcd_expected_err, ==, error);
5075 	VERIFY(!data->zcd_called);
5076 
5077 	synced_txg = spa_last_synced_txg(data->zcd_spa);
5078 	if (data->zcd_txg > synced_txg)
5079 		fatal(0, "commit callback of txg %" PRIu64 " called prematurely"
5080 		    ", last synced txg = %" PRIu64 "\n", data->zcd_txg,
5081 		    synced_txg);
5082 
5083 	data->zcd_called = B_TRUE;
5084 
5085 	if (error == ECANCELED) {
5086 		ASSERT0(data->zcd_txg);
5087 		ASSERT(!data->zcd_added);
5088 
5089 		/*
5090 		 * The private callback data should be destroyed here, but
5091 		 * since we are going to check the zcd_called field after
5092 		 * dmu_tx_abort(), we will destroy it there.
5093 		 */
5094 		return;
5095 	}
5096 
5097 	/* Was this callback added to the global callback list? */
5098 	if (!data->zcd_added)
5099 		goto out;
5100 
5101 	ASSERT3U(data->zcd_txg, !=, 0);
5102 
5103 	/* Remove our callback from the list */
5104 	mutex_enter(&zcl.zcl_callbacks_lock);
5105 	list_remove(&zcl.zcl_callbacks, data);
5106 	mutex_exit(&zcl.zcl_callbacks_lock);
5107 
5108 out:
5109 	umem_free(data, sizeof (ztest_cb_data_t));
5110 }
5111 
5112 /* Allocate and initialize callback data structure */
5113 static ztest_cb_data_t *
5114 ztest_create_cb_data(objset_t *os, uint64_t txg)
5115 {
5116 	ztest_cb_data_t *cb_data;
5117 
5118 	cb_data = umem_zalloc(sizeof (ztest_cb_data_t), UMEM_NOFAIL);
5119 
5120 	cb_data->zcd_txg = txg;
5121 	cb_data->zcd_spa = dmu_objset_spa(os);
5122 
5123 	return (cb_data);
5124 }
5125 
5126 /*
5127  * If a number of txgs equal to this threshold have been created after a commit
5128  * callback has been registered but not called, then we assume there is an
5129  * implementation bug.
5130  */
5131 #define	ZTEST_COMMIT_CALLBACK_THRESH	(TXG_CONCURRENT_STATES + 2)
5132 
5133 /*
5134  * Commit callback test.
5135  */
5136 void
5137 ztest_dmu_commit_callbacks(ztest_ds_t *zd, uint64_t id)
5138 {
5139 	objset_t *os = zd->zd_os;
5140 	ztest_od_t od[1];
5141 	dmu_tx_t *tx;
5142 	ztest_cb_data_t *cb_data[3], *tmp_cb;
5143 	uint64_t old_txg, txg;
5144 	int i, error;
5145 
5146 	ztest_od_init(&od[0], id, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0);
5147 
5148 	if (ztest_object_init(zd, od, sizeof (od), B_FALSE) != 0)
5149 		return;
5150 
5151 	tx = dmu_tx_create(os);
5152 
5153 	cb_data[0] = ztest_create_cb_data(os, 0);
5154 	dmu_tx_callback_register(tx, ztest_commit_callback, cb_data[0]);
5155 
5156 	dmu_tx_hold_write(tx, od[0].od_object, 0, sizeof (uint64_t));
5157 
5158 	/* Every once in a while, abort the transaction on purpose */
5159 	if (ztest_random(100) == 0)
5160 		error = -1;
5161 
5162 	if (!error)
5163 		error = dmu_tx_assign(tx, TXG_NOWAIT);
5164 
5165 	txg = error ? 0 : dmu_tx_get_txg(tx);
5166 
5167 	cb_data[0]->zcd_txg = txg;
5168 	cb_data[1] = ztest_create_cb_data(os, txg);
5169 	dmu_tx_callback_register(tx, ztest_commit_callback, cb_data[1]);
5170 
5171 	if (error) {
5172 		/*
5173 		 * It's not a strict requirement to call the registered
5174 		 * callbacks from inside dmu_tx_abort(), but that's what
5175 		 * it's supposed to happen in the current implementation
5176 		 * so we will check for that.
5177 		 */
5178 		for (i = 0; i < 2; i++) {
5179 			cb_data[i]->zcd_expected_err = ECANCELED;
5180 			VERIFY(!cb_data[i]->zcd_called);
5181 		}
5182 
5183 		dmu_tx_abort(tx);
5184 
5185 		for (i = 0; i < 2; i++) {
5186 			VERIFY(cb_data[i]->zcd_called);
5187 			umem_free(cb_data[i], sizeof (ztest_cb_data_t));
5188 		}
5189 
5190 		return;
5191 	}
5192 
5193 	cb_data[2] = ztest_create_cb_data(os, txg);
5194 	dmu_tx_callback_register(tx, ztest_commit_callback, cb_data[2]);
5195 
5196 	/*
5197 	 * Read existing data to make sure there isn't a future leak.
5198 	 */
5199 	VERIFY(0 == dmu_read(os, od[0].od_object, 0, sizeof (uint64_t),
5200 	    &old_txg, DMU_READ_PREFETCH));
5201 
5202 	if (old_txg > txg)
5203 		fatal(0, "future leak: got %" PRIu64 ", open txg is %" PRIu64,
5204 		    old_txg, txg);
5205 
5206 	dmu_write(os, od[0].od_object, 0, sizeof (uint64_t), &txg, tx);
5207 
5208 	mutex_enter(&zcl.zcl_callbacks_lock);
5209 
5210 	/*
5211 	 * Since commit callbacks don't have any ordering requirement and since
5212 	 * it is theoretically possible for a commit callback to be called
5213 	 * after an arbitrary amount of time has elapsed since its txg has been
5214 	 * synced, it is difficult to reliably determine whether a commit
5215 	 * callback hasn't been called due to high load or due to a flawed
5216 	 * implementation.
5217 	 *
5218 	 * In practice, we will assume that if after a certain number of txgs a
5219 	 * commit callback hasn't been called, then most likely there's an
5220 	 * implementation bug..
5221 	 */
5222 	tmp_cb = list_head(&zcl.zcl_callbacks);
5223 	if (tmp_cb != NULL &&
5224 	    (txg - ZTEST_COMMIT_CALLBACK_THRESH) > tmp_cb->zcd_txg) {
5225 		fatal(0, "Commit callback threshold exceeded, oldest txg: %"
5226 		    PRIu64 ", open txg: %" PRIu64 "\n", tmp_cb->zcd_txg, txg);
5227 	}
5228 
5229 	/*
5230 	 * Let's find the place to insert our callbacks.
5231 	 *
5232 	 * Even though the list is ordered by txg, it is possible for the
5233 	 * insertion point to not be the end because our txg may already be
5234 	 * quiescing at this point and other callbacks in the open txg
5235 	 * (from other objsets) may have sneaked in.
5236 	 */
5237 	tmp_cb = list_tail(&zcl.zcl_callbacks);
5238 	while (tmp_cb != NULL && tmp_cb->zcd_txg > txg)
5239 		tmp_cb = list_prev(&zcl.zcl_callbacks, tmp_cb);
5240 
5241 	/* Add the 3 callbacks to the list */
5242 	for (i = 0; i < 3; i++) {
5243 		if (tmp_cb == NULL)
5244 			list_insert_head(&zcl.zcl_callbacks, cb_data[i]);
5245 		else
5246 			list_insert_after(&zcl.zcl_callbacks, tmp_cb,
5247 			    cb_data[i]);
5248 
5249 		cb_data[i]->zcd_added = B_TRUE;
5250 		VERIFY(!cb_data[i]->zcd_called);
5251 
5252 		tmp_cb = cb_data[i];
5253 	}
5254 
5255 	mutex_exit(&zcl.zcl_callbacks_lock);
5256 
5257 	dmu_tx_commit(tx);
5258 }
5259 
5260 /*
5261  * Visit each object in the dataset. Verify that its properties
5262  * are consistent what was stored in the block tag when it was created,
5263  * and that its unused bonus buffer space has not been overwritten.
5264  */
5265 void
5266 ztest_verify_dnode_bt(ztest_ds_t *zd, uint64_t id)
5267 {
5268 	objset_t *os = zd->zd_os;
5269 	uint64_t obj;
5270 	int err = 0;
5271 
5272 	for (obj = 0; err == 0; err = dmu_object_next(os, &obj, FALSE, 0)) {
5273 		ztest_block_tag_t *bt = NULL;
5274 		dmu_object_info_t doi;
5275 		dmu_buf_t *db;
5276 
5277 		if (dmu_bonus_hold(os, obj, FTAG, &db) != 0)
5278 			continue;
5279 
5280 		dmu_object_info_from_db(db, &doi);
5281 		if (doi.doi_bonus_size >= sizeof (*bt))
5282 			bt = ztest_bt_bonus(db);
5283 
5284 		if (bt && bt->bt_magic == BT_MAGIC) {
5285 			ztest_bt_verify(bt, os, obj, doi.doi_dnodesize,
5286 			    bt->bt_offset, bt->bt_gen, bt->bt_txg,
5287 			    bt->bt_crtxg);
5288 			ztest_verify_unused_bonus(db, bt, obj, os, bt->bt_gen);
5289 		}
5290 
5291 		dmu_buf_rele(db, FTAG);
5292 	}
5293 }
5294 
5295 /* ARGSUSED */
5296 void
5297 ztest_dsl_prop_get_set(ztest_ds_t *zd, uint64_t id)
5298 {
5299 	zfs_prop_t proplist[] = {
5300 		ZFS_PROP_CHECKSUM,
5301 		ZFS_PROP_COMPRESSION,
5302 		ZFS_PROP_COPIES,
5303 		ZFS_PROP_DEDUP
5304 	};
5305 
5306 	rw_enter(&ztest_name_lock, RW_READER);
5307 
5308 	for (int p = 0; p < sizeof (proplist) / sizeof (proplist[0]); p++)
5309 		(void) ztest_dsl_prop_set_uint64(zd->zd_name, proplist[p],
5310 		    ztest_random_dsl_prop(proplist[p]), (int)ztest_random(2));
5311 
5312 	rw_exit(&ztest_name_lock);
5313 }
5314 
5315 /* ARGSUSED */
5316 void
5317 ztest_remap_blocks(ztest_ds_t *zd, uint64_t id)
5318 {
5319 	rw_enter(&ztest_name_lock, RW_READER);
5320 
5321 	int error = dmu_objset_remap_indirects(zd->zd_name);
5322 	if (error == ENOSPC)
5323 		error = 0;
5324 	ASSERT0(error);
5325 
5326 	rw_exit(&ztest_name_lock);
5327 }
5328 
5329 /* ARGSUSED */
5330 void
5331 ztest_spa_prop_get_set(ztest_ds_t *zd, uint64_t id)
5332 {
5333 	nvlist_t *props = NULL;
5334 
5335 	rw_enter(&ztest_name_lock, RW_READER);
5336 
5337 	(void) ztest_spa_prop_set_uint64(ZPOOL_PROP_DEDUPDITTO,
5338 	    ZIO_DEDUPDITTO_MIN + ztest_random(ZIO_DEDUPDITTO_MIN));
5339 
5340 	VERIFY0(spa_prop_get(ztest_spa, &props));
5341 
5342 	if (ztest_opts.zo_verbose >= 6)
5343 		dump_nvlist(props, 4);
5344 
5345 	nvlist_free(props);
5346 
5347 	rw_exit(&ztest_name_lock);
5348 }
5349 
5350 static int
5351 user_release_one(const char *snapname, const char *holdname)
5352 {
5353 	nvlist_t *snaps, *holds;
5354 	int error;
5355 
5356 	snaps = fnvlist_alloc();
5357 	holds = fnvlist_alloc();
5358 	fnvlist_add_boolean(holds, holdname);
5359 	fnvlist_add_nvlist(snaps, snapname, holds);
5360 	fnvlist_free(holds);
5361 	error = dsl_dataset_user_release(snaps, NULL);
5362 	fnvlist_free(snaps);
5363 	return (error);
5364 }
5365 
5366 /*
5367  * Test snapshot hold/release and deferred destroy.
5368  */
5369 void
5370 ztest_dmu_snapshot_hold(ztest_ds_t *zd, uint64_t id)
5371 {
5372 	int error;
5373 	objset_t *os = zd->zd_os;
5374 	objset_t *origin;
5375 	char snapname[100];
5376 	char fullname[100];
5377 	char clonename[100];
5378 	char tag[100];
5379 	char osname[ZFS_MAX_DATASET_NAME_LEN];
5380 	nvlist_t *holds;
5381 
5382 	rw_enter(&ztest_name_lock, RW_READER);
5383 
5384 	dmu_objset_name(os, osname);
5385 
5386 	(void) snprintf(snapname, sizeof (snapname), "sh1_%llu", id);
5387 	(void) snprintf(fullname, sizeof (fullname), "%s@%s", osname, snapname);
5388 	(void) snprintf(clonename, sizeof (clonename),
5389 	    "%s/ch1_%llu", osname, id);
5390 	(void) snprintf(tag, sizeof (tag), "tag_%llu", id);
5391 
5392 	/*
5393 	 * Clean up from any previous run.
5394 	 */
5395 	error = dsl_destroy_head(clonename);
5396 	if (error != ENOENT)
5397 		ASSERT0(error);
5398 	error = user_release_one(fullname, tag);
5399 	if (error != ESRCH && error != ENOENT)
5400 		ASSERT0(error);
5401 	error = dsl_destroy_snapshot(fullname, B_FALSE);
5402 	if (error != ENOENT)
5403 		ASSERT0(error);
5404 
5405 	/*
5406 	 * Create snapshot, clone it, mark snap for deferred destroy,
5407 	 * destroy clone, verify snap was also destroyed.
5408 	 */
5409 	error = dmu_objset_snapshot_one(osname, snapname);
5410 	if (error) {
5411 		if (error == ENOSPC) {
5412 			ztest_record_enospc("dmu_objset_snapshot");
5413 			goto out;
5414 		}
5415 		fatal(0, "dmu_objset_snapshot(%s) = %d", fullname, error);
5416 	}
5417 
5418 	error = dmu_objset_clone(clonename, fullname);
5419 	if (error) {
5420 		if (error == ENOSPC) {
5421 			ztest_record_enospc("dmu_objset_clone");
5422 			goto out;
5423 		}
5424 		fatal(0, "dmu_objset_clone(%s) = %d", clonename, error);
5425 	}
5426 
5427 	error = dsl_destroy_snapshot(fullname, B_TRUE);
5428 	if (error) {
5429 		fatal(0, "dsl_destroy_snapshot(%s, B_TRUE) = %d",
5430 		    fullname, error);
5431 	}
5432 
5433 	error = dsl_destroy_head(clonename);
5434 	if (error)
5435 		fatal(0, "dsl_destroy_head(%s) = %d", clonename, error);
5436 
5437 	error = dmu_objset_hold(fullname, FTAG, &origin);
5438 	if (error != ENOENT)
5439 		fatal(0, "dmu_objset_hold(%s) = %d", fullname, error);
5440 
5441 	/*
5442 	 * Create snapshot, add temporary hold, verify that we can't
5443 	 * destroy a held snapshot, mark for deferred destroy,
5444 	 * release hold, verify snapshot was destroyed.
5445 	 */
5446 	error = dmu_objset_snapshot_one(osname, snapname);
5447 	if (error) {
5448 		if (error == ENOSPC) {
5449 			ztest_record_enospc("dmu_objset_snapshot");
5450 			goto out;
5451 		}
5452 		fatal(0, "dmu_objset_snapshot(%s) = %d", fullname, error);
5453 	}
5454 
5455 	holds = fnvlist_alloc();
5456 	fnvlist_add_string(holds, fullname, tag);
5457 	error = dsl_dataset_user_hold(holds, 0, NULL);
5458 	fnvlist_free(holds);
5459 
5460 	if (error == ENOSPC) {
5461 		ztest_record_enospc("dsl_dataset_user_hold");
5462 		goto out;
5463 	} else if (error) {
5464 		fatal(0, "dsl_dataset_user_hold(%s, %s) = %u",
5465 		    fullname, tag, error);
5466 	}
5467 
5468 	error = dsl_destroy_snapshot(fullname, B_FALSE);
5469 	if (error != EBUSY) {
5470 		fatal(0, "dsl_destroy_snapshot(%s, B_FALSE) = %d",
5471 		    fullname, error);
5472 	}
5473 
5474 	error = dsl_destroy_snapshot(fullname, B_TRUE);
5475 	if (error) {
5476 		fatal(0, "dsl_destroy_snapshot(%s, B_TRUE) = %d",
5477 		    fullname, error);
5478 	}
5479 
5480 	error = user_release_one(fullname, tag);
5481 	if (error)
5482 		fatal(0, "user_release_one(%s, %s) = %d", fullname, tag, error);
5483 
5484 	VERIFY3U(dmu_objset_hold(fullname, FTAG, &origin), ==, ENOENT);
5485 
5486 out:
5487 	rw_exit(&ztest_name_lock);
5488 }
5489 
5490 /*
5491  * Inject random faults into the on-disk data.
5492  */
5493 /* ARGSUSED */
5494 void
5495 ztest_fault_inject(ztest_ds_t *zd, uint64_t id)
5496 {
5497 	ztest_shared_t *zs = ztest_shared;
5498 	spa_t *spa = ztest_spa;
5499 	int fd;
5500 	uint64_t offset;
5501 	uint64_t leaves;
5502 	uint64_t bad = 0x1990c0ffeedecade;
5503 	uint64_t top, leaf;
5504 	char path0[MAXPATHLEN];
5505 	char pathrand[MAXPATHLEN];
5506 	size_t fsize;
5507 	int bshift = SPA_MAXBLOCKSHIFT + 2;
5508 	int iters = 1000;
5509 	int maxfaults;
5510 	int mirror_save;
5511 	vdev_t *vd0 = NULL;
5512 	uint64_t guid0 = 0;
5513 	boolean_t islog = B_FALSE;
5514 
5515 	mutex_enter(&ztest_vdev_lock);
5516 
5517 	/*
5518 	 * Device removal is in progress, fault injection must be disabled
5519 	 * until it completes and the pool is scrubbed.  The fault injection
5520 	 * strategy for damaging blocks does not take in to account evacuated
5521 	 * blocks which may have already been damaged.
5522 	 */
5523 	if (ztest_device_removal_active) {
5524 		mutex_exit(&ztest_vdev_lock);
5525 		return;
5526 	}
5527 
5528 	maxfaults = MAXFAULTS();
5529 	leaves = MAX(zs->zs_mirrors, 1) * ztest_opts.zo_raidz;
5530 	mirror_save = zs->zs_mirrors;
5531 	mutex_exit(&ztest_vdev_lock);
5532 
5533 	ASSERT(leaves >= 1);
5534 
5535 	/*
5536 	 * Grab the name lock as reader. There are some operations
5537 	 * which don't like to have their vdevs changed while
5538 	 * they are in progress (i.e. spa_change_guid). Those
5539 	 * operations will have grabbed the name lock as writer.
5540 	 */
5541 	rw_enter(&ztest_name_lock, RW_READER);
5542 
5543 	/*
5544 	 * We need SCL_STATE here because we're going to look at vd0->vdev_tsd.
5545 	 */
5546 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
5547 
5548 	if (ztest_random(2) == 0) {
5549 		/*
5550 		 * Inject errors on a normal data device or slog device.
5551 		 */
5552 		top = ztest_random_vdev_top(spa, B_TRUE);
5553 		leaf = ztest_random(leaves) + zs->zs_splits;
5554 
5555 		/*
5556 		 * Generate paths to the first leaf in this top-level vdev,
5557 		 * and to the random leaf we selected.  We'll induce transient
5558 		 * write failures and random online/offline activity on leaf 0,
5559 		 * and we'll write random garbage to the randomly chosen leaf.
5560 		 */
5561 		(void) snprintf(path0, sizeof (path0), ztest_dev_template,
5562 		    ztest_opts.zo_dir, ztest_opts.zo_pool,
5563 		    top * leaves + zs->zs_splits);
5564 		(void) snprintf(pathrand, sizeof (pathrand), ztest_dev_template,
5565 		    ztest_opts.zo_dir, ztest_opts.zo_pool,
5566 		    top * leaves + leaf);
5567 
5568 		vd0 = vdev_lookup_by_path(spa->spa_root_vdev, path0);
5569 		if (vd0 != NULL && vd0->vdev_top->vdev_islog)
5570 			islog = B_TRUE;
5571 
5572 		/*
5573 		 * If the top-level vdev needs to be resilvered
5574 		 * then we only allow faults on the device that is
5575 		 * resilvering.
5576 		 */
5577 		if (vd0 != NULL && maxfaults != 1 &&
5578 		    (!vdev_resilver_needed(vd0->vdev_top, NULL, NULL) ||
5579 		    vd0->vdev_resilver_txg != 0)) {
5580 			/*
5581 			 * Make vd0 explicitly claim to be unreadable,
5582 			 * or unwriteable, or reach behind its back
5583 			 * and close the underlying fd.  We can do this if
5584 			 * maxfaults == 0 because we'll fail and reexecute,
5585 			 * and we can do it if maxfaults >= 2 because we'll
5586 			 * have enough redundancy.  If maxfaults == 1, the
5587 			 * combination of this with injection of random data
5588 			 * corruption below exceeds the pool's fault tolerance.
5589 			 */
5590 			vdev_file_t *vf = vd0->vdev_tsd;
5591 
5592 			zfs_dbgmsg("injecting fault to vdev %llu; maxfaults=%d",
5593 			    (long long)vd0->vdev_id, (int)maxfaults);
5594 
5595 			if (vf != NULL && ztest_random(3) == 0) {
5596 				(void) close(vf->vf_vnode->v_fd);
5597 				vf->vf_vnode->v_fd = -1;
5598 			} else if (ztest_random(2) == 0) {
5599 				vd0->vdev_cant_read = B_TRUE;
5600 			} else {
5601 				vd0->vdev_cant_write = B_TRUE;
5602 			}
5603 			guid0 = vd0->vdev_guid;
5604 		}
5605 	} else {
5606 		/*
5607 		 * Inject errors on an l2cache device.
5608 		 */
5609 		spa_aux_vdev_t *sav = &spa->spa_l2cache;
5610 
5611 		if (sav->sav_count == 0) {
5612 			spa_config_exit(spa, SCL_STATE, FTAG);
5613 			rw_exit(&ztest_name_lock);
5614 			return;
5615 		}
5616 		vd0 = sav->sav_vdevs[ztest_random(sav->sav_count)];
5617 		guid0 = vd0->vdev_guid;
5618 		(void) strcpy(path0, vd0->vdev_path);
5619 		(void) strcpy(pathrand, vd0->vdev_path);
5620 
5621 		leaf = 0;
5622 		leaves = 1;
5623 		maxfaults = INT_MAX;	/* no limit on cache devices */
5624 	}
5625 
5626 	spa_config_exit(spa, SCL_STATE, FTAG);
5627 	rw_exit(&ztest_name_lock);
5628 
5629 	/*
5630 	 * If we can tolerate two or more faults, or we're dealing
5631 	 * with a slog, randomly online/offline vd0.
5632 	 */
5633 	if ((maxfaults >= 2 || islog) && guid0 != 0) {
5634 		if (ztest_random(10) < 6) {
5635 			int flags = (ztest_random(2) == 0 ?
5636 			    ZFS_OFFLINE_TEMPORARY : 0);
5637 
5638 			/*
5639 			 * We have to grab the zs_name_lock as writer to
5640 			 * prevent a race between offlining a slog and
5641 			 * destroying a dataset. Offlining the slog will
5642 			 * grab a reference on the dataset which may cause
5643 			 * dmu_objset_destroy() to fail with EBUSY thus
5644 			 * leaving the dataset in an inconsistent state.
5645 			 */
5646 			if (islog)
5647 				rw_enter(&ztest_name_lock, RW_WRITER);
5648 
5649 			VERIFY(vdev_offline(spa, guid0, flags) != EBUSY);
5650 
5651 			if (islog)
5652 				rw_exit(&ztest_name_lock);
5653 		} else {
5654 			/*
5655 			 * Ideally we would like to be able to randomly
5656 			 * call vdev_[on|off]line without holding locks
5657 			 * to force unpredictable failures but the side
5658 			 * effects of vdev_[on|off]line prevent us from
5659 			 * doing so. We grab the ztest_vdev_lock here to
5660 			 * prevent a race between injection testing and
5661 			 * aux_vdev removal.
5662 			 */
5663 			mutex_enter(&ztest_vdev_lock);
5664 			(void) vdev_online(spa, guid0, 0, NULL);
5665 			mutex_exit(&ztest_vdev_lock);
5666 		}
5667 	}
5668 
5669 	if (maxfaults == 0)
5670 		return;
5671 
5672 	/*
5673 	 * We have at least single-fault tolerance, so inject data corruption.
5674 	 */
5675 	fd = open(pathrand, O_RDWR);
5676 
5677 	if (fd == -1) /* we hit a gap in the device namespace */
5678 		return;
5679 
5680 	fsize = lseek(fd, 0, SEEK_END);
5681 
5682 	while (--iters != 0) {
5683 		/*
5684 		 * The offset must be chosen carefully to ensure that
5685 		 * we do not inject a given logical block with errors
5686 		 * on two different leaf devices, because ZFS can not
5687 		 * tolerate that (if maxfaults==1).
5688 		 *
5689 		 * We divide each leaf into chunks of size
5690 		 * (# leaves * SPA_MAXBLOCKSIZE * 4).  Within each chunk
5691 		 * there is a series of ranges to which we can inject errors.
5692 		 * Each range can accept errors on only a single leaf vdev.
5693 		 * The error injection ranges are separated by ranges
5694 		 * which we will not inject errors on any device (DMZs).
5695 		 * Each DMZ must be large enough such that a single block
5696 		 * can not straddle it, so that a single block can not be
5697 		 * a target in two different injection ranges (on different
5698 		 * leaf vdevs).
5699 		 *
5700 		 * For example, with 3 leaves, each chunk looks like:
5701 		 *    0 to  32M: injection range for leaf 0
5702 		 *  32M to  64M: DMZ - no injection allowed
5703 		 *  64M to  96M: injection range for leaf 1
5704 		 *  96M to 128M: DMZ - no injection allowed
5705 		 * 128M to 160M: injection range for leaf 2
5706 		 * 160M to 192M: DMZ - no injection allowed
5707 		 */
5708 		offset = ztest_random(fsize / (leaves << bshift)) *
5709 		    (leaves << bshift) + (leaf << bshift) +
5710 		    (ztest_random(1ULL << (bshift - 1)) & -8ULL);
5711 
5712 		/*
5713 		 * Only allow damage to the labels at one end of the vdev.
5714 		 *
5715 		 * If all labels are damaged, the device will be totally
5716 		 * inaccessible, which will result in loss of data,
5717 		 * because we also damage (parts of) the other side of
5718 		 * the mirror/raidz.
5719 		 *
5720 		 * Additionally, we will always have both an even and an
5721 		 * odd label, so that we can handle crashes in the
5722 		 * middle of vdev_config_sync().
5723 		 */
5724 		if ((leaf & 1) == 0 && offset < VDEV_LABEL_START_SIZE)
5725 			continue;
5726 
5727 		/*
5728 		 * The two end labels are stored at the "end" of the disk, but
5729 		 * the end of the disk (vdev_psize) is aligned to
5730 		 * sizeof (vdev_label_t).
5731 		 */
5732 		uint64_t psize = P2ALIGN(fsize, sizeof (vdev_label_t));
5733 		if ((leaf & 1) == 1 &&
5734 		    offset + sizeof (bad) > psize - VDEV_LABEL_END_SIZE)
5735 			continue;
5736 
5737 		mutex_enter(&ztest_vdev_lock);
5738 		if (mirror_save != zs->zs_mirrors) {
5739 			mutex_exit(&ztest_vdev_lock);
5740 			(void) close(fd);
5741 			return;
5742 		}
5743 
5744 		if (pwrite(fd, &bad, sizeof (bad), offset) != sizeof (bad))
5745 			fatal(1, "can't inject bad word at 0x%llx in %s",
5746 			    offset, pathrand);
5747 
5748 		mutex_exit(&ztest_vdev_lock);
5749 
5750 		if (ztest_opts.zo_verbose >= 7)
5751 			(void) printf("injected bad word into %s,"
5752 			    " offset 0x%llx\n", pathrand, (u_longlong_t)offset);
5753 	}
5754 
5755 	(void) close(fd);
5756 }
5757 
5758 /*
5759  * Verify that DDT repair works as expected.
5760  */
5761 void
5762 ztest_ddt_repair(ztest_ds_t *zd, uint64_t id)
5763 {
5764 	ztest_shared_t *zs = ztest_shared;
5765 	spa_t *spa = ztest_spa;
5766 	objset_t *os = zd->zd_os;
5767 	ztest_od_t od[1];
5768 	uint64_t object, blocksize, txg, pattern, psize;
5769 	enum zio_checksum checksum = spa_dedup_checksum(spa);
5770 	dmu_buf_t *db;
5771 	dmu_tx_t *tx;
5772 	abd_t *abd;
5773 	blkptr_t blk;
5774 	int copies = 2 * ZIO_DEDUPDITTO_MIN;
5775 
5776 	blocksize = ztest_random_blocksize();
5777 	blocksize = MIN(blocksize, 2048);	/* because we write so many */
5778 
5779 	ztest_od_init(&od[0], id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize,
5780 	    0, 0);
5781 
5782 	if (ztest_object_init(zd, od, sizeof (od), B_FALSE) != 0)
5783 		return;
5784 
5785 	/*
5786 	 * Take the name lock as writer to prevent anyone else from changing
5787 	 * the pool and dataset properies we need to maintain during this test.
5788 	 */
5789 	rw_enter(&ztest_name_lock, RW_WRITER);
5790 
5791 	if (ztest_dsl_prop_set_uint64(zd->zd_name, ZFS_PROP_DEDUP, checksum,
5792 	    B_FALSE) != 0 ||
5793 	    ztest_dsl_prop_set_uint64(zd->zd_name, ZFS_PROP_COPIES, 1,
5794 	    B_FALSE) != 0) {
5795 		rw_exit(&ztest_name_lock);
5796 		return;
5797 	}
5798 
5799 	dmu_objset_stats_t dds;
5800 	dsl_pool_config_enter(dmu_objset_pool(os), FTAG);
5801 	dmu_objset_fast_stat(os, &dds);
5802 	dsl_pool_config_exit(dmu_objset_pool(os), FTAG);
5803 
5804 	object = od[0].od_object;
5805 	blocksize = od[0].od_blocksize;
5806 	pattern = zs->zs_guid ^ dds.dds_guid;
5807 
5808 	ASSERT(object != 0);
5809 
5810 	tx = dmu_tx_create(os);
5811 	dmu_tx_hold_write(tx, object, 0, copies * blocksize);
5812 	txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
5813 	if (txg == 0) {
5814 		rw_exit(&ztest_name_lock);
5815 		return;
5816 	}
5817 
5818 	/*
5819 	 * Write all the copies of our block.
5820 	 */
5821 	for (int i = 0; i < copies; i++) {
5822 		uint64_t offset = i * blocksize;
5823 		int error = dmu_buf_hold(os, object, offset, FTAG, &db,
5824 		    DMU_READ_NO_PREFETCH);
5825 		if (error != 0) {
5826 			fatal(B_FALSE, "dmu_buf_hold(%p, %llu, %llu) = %u",
5827 			    os, (long long)object, (long long) offset, error);
5828 		}
5829 		ASSERT(db->db_offset == offset);
5830 		ASSERT(db->db_size == blocksize);
5831 		ASSERT(ztest_pattern_match(db->db_data, db->db_size, pattern) ||
5832 		    ztest_pattern_match(db->db_data, db->db_size, 0ULL));
5833 		dmu_buf_will_fill(db, tx);
5834 		ztest_pattern_set(db->db_data, db->db_size, pattern);
5835 		dmu_buf_rele(db, FTAG);
5836 	}
5837 
5838 	dmu_tx_commit(tx);
5839 	txg_wait_synced(spa_get_dsl(spa), txg);
5840 
5841 	/*
5842 	 * Find out what block we got.
5843 	 */
5844 	VERIFY0(dmu_buf_hold(os, object, 0, FTAG, &db,
5845 	    DMU_READ_NO_PREFETCH));
5846 	blk = *((dmu_buf_impl_t *)db)->db_blkptr;
5847 	dmu_buf_rele(db, FTAG);
5848 
5849 	/*
5850 	 * Damage the block.  Dedup-ditto will save us when we read it later.
5851 	 */
5852 	psize = BP_GET_PSIZE(&blk);
5853 	abd = abd_alloc_linear(psize, B_TRUE);
5854 	ztest_pattern_set(abd_to_buf(abd), psize, ~pattern);
5855 
5856 	(void) zio_wait(zio_rewrite(NULL, spa, 0, &blk,
5857 	    abd, psize, NULL, NULL, ZIO_PRIORITY_SYNC_WRITE,
5858 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_INDUCE_DAMAGE, NULL));
5859 
5860 	abd_free(abd);
5861 
5862 	rw_exit(&ztest_name_lock);
5863 }
5864 
5865 /*
5866  * Scrub the pool.
5867  */
5868 /* ARGSUSED */
5869 void
5870 ztest_scrub(ztest_ds_t *zd, uint64_t id)
5871 {
5872 	spa_t *spa = ztest_spa;
5873 
5874 	/*
5875 	 * Scrub in progress by device removal.
5876 	 */
5877 	if (ztest_device_removal_active)
5878 		return;
5879 
5880 	(void) spa_scan(spa, POOL_SCAN_SCRUB);
5881 	(void) poll(NULL, 0, 100); /* wait a moment, then force a restart */
5882 	(void) spa_scan(spa, POOL_SCAN_SCRUB);
5883 }
5884 
5885 /*
5886  * Change the guid for the pool.
5887  */
5888 /* ARGSUSED */
5889 void
5890 ztest_reguid(ztest_ds_t *zd, uint64_t id)
5891 {
5892 	spa_t *spa = ztest_spa;
5893 	uint64_t orig, load;
5894 	int error;
5895 
5896 	if (ztest_opts.zo_mmp_test)
5897 		return;
5898 
5899 	orig = spa_guid(spa);
5900 	load = spa_load_guid(spa);
5901 
5902 	rw_enter(&ztest_name_lock, RW_WRITER);
5903 	error = spa_change_guid(spa);
5904 	rw_exit(&ztest_name_lock);
5905 
5906 	if (error != 0)
5907 		return;
5908 
5909 	if (ztest_opts.zo_verbose >= 4) {
5910 		(void) printf("Changed guid old %llu -> %llu\n",
5911 		    (u_longlong_t)orig, (u_longlong_t)spa_guid(spa));
5912 	}
5913 
5914 	VERIFY3U(orig, !=, spa_guid(spa));
5915 	VERIFY3U(load, ==, spa_load_guid(spa));
5916 }
5917 
5918 static vdev_t *
5919 ztest_random_concrete_vdev_leaf(vdev_t *vd)
5920 {
5921 	if (vd == NULL)
5922 		return (NULL);
5923 
5924 	if (vd->vdev_children == 0)
5925 		return (vd);
5926 
5927 	vdev_t *eligible[vd->vdev_children];
5928 	int eligible_idx = 0, i;
5929 	for (i = 0; i < vd->vdev_children; i++) {
5930 		vdev_t *cvd = vd->vdev_child[i];
5931 		if (cvd->vdev_top->vdev_removing)
5932 			continue;
5933 		if (cvd->vdev_children > 0 ||
5934 		    (vdev_is_concrete(cvd) && !cvd->vdev_detached)) {
5935 			eligible[eligible_idx++] = cvd;
5936 		}
5937 	}
5938 	VERIFY(eligible_idx > 0);
5939 
5940 	uint64_t child_no = ztest_random(eligible_idx);
5941 	return (ztest_random_concrete_vdev_leaf(eligible[child_no]));
5942 }
5943 
5944 /* ARGSUSED */
5945 void
5946 ztest_initialize(ztest_ds_t *zd, uint64_t id)
5947 {
5948 	spa_t *spa = ztest_spa;
5949 	int error = 0;
5950 
5951 	mutex_enter(&ztest_vdev_lock);
5952 
5953 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
5954 
5955 	/* Random leaf vdev */
5956 	vdev_t *rand_vd = ztest_random_concrete_vdev_leaf(spa->spa_root_vdev);
5957 	if (rand_vd == NULL) {
5958 		spa_config_exit(spa, SCL_VDEV, FTAG);
5959 		mutex_exit(&ztest_vdev_lock);
5960 		return;
5961 	}
5962 
5963 	/*
5964 	 * The random vdev we've selected may change as soon as we
5965 	 * drop the spa_config_lock. We create local copies of things
5966 	 * we're interested in.
5967 	 */
5968 	uint64_t guid = rand_vd->vdev_guid;
5969 	char *path = strdup(rand_vd->vdev_path);
5970 	boolean_t active = rand_vd->vdev_initialize_thread != NULL;
5971 
5972 	zfs_dbgmsg("vd %p, guid %llu", rand_vd, guid);
5973 	spa_config_exit(spa, SCL_VDEV, FTAG);
5974 
5975 	uint64_t cmd = ztest_random(POOL_INITIALIZE_FUNCS);
5976 	error = spa_vdev_initialize(spa, guid, cmd);
5977 	switch (cmd) {
5978 	case POOL_INITIALIZE_CANCEL:
5979 		if (ztest_opts.zo_verbose >= 4) {
5980 			(void) printf("Cancel initialize %s", path);
5981 			if (!active)
5982 				(void) printf(" failed (no initialize active)");
5983 			(void) printf("\n");
5984 		}
5985 		break;
5986 	case POOL_INITIALIZE_DO:
5987 		if (ztest_opts.zo_verbose >= 4) {
5988 			(void) printf("Start initialize %s", path);
5989 			if (active && error == 0)
5990 				(void) printf(" failed (already active)");
5991 			else if (error != 0)
5992 				(void) printf(" failed (error %d)", error);
5993 			(void) printf("\n");
5994 		}
5995 		break;
5996 	case POOL_INITIALIZE_SUSPEND:
5997 		if (ztest_opts.zo_verbose >= 4) {
5998 			(void) printf("Suspend initialize %s", path);
5999 			if (!active)
6000 				(void) printf(" failed (no initialize active)");
6001 			(void) printf("\n");
6002 		}
6003 		break;
6004 	}
6005 	free(path);
6006 	mutex_exit(&ztest_vdev_lock);
6007 }
6008 
6009 /*
6010  * Verify pool integrity by running zdb.
6011  */
6012 static void
6013 ztest_run_zdb(char *pool)
6014 {
6015 	int status;
6016 	char zdb[MAXPATHLEN + MAXNAMELEN + 20];
6017 	char zbuf[1024];
6018 	char *bin;
6019 	char *ztest;
6020 	char *isa;
6021 	int isalen;
6022 	FILE *fp;
6023 
6024 	(void) realpath(getexecname(), zdb);
6025 
6026 	/* zdb lives in /usr/sbin, while ztest lives in /usr/bin */
6027 	bin = strstr(zdb, "/usr/bin/");
6028 	ztest = strstr(bin, "/ztest");
6029 	isa = bin + 8;
6030 	isalen = ztest - isa;
6031 	isa = strdup(isa);
6032 	/* LINTED */
6033 	(void) sprintf(bin,
6034 	    "/usr/sbin%.*s/zdb -bcc%s%s -G -d -U %s "
6035 	    "-o zfs_reconstruct_indirect_combinations_max=65536 %s",
6036 	    isalen,
6037 	    isa,
6038 	    ztest_opts.zo_verbose >= 3 ? "s" : "",
6039 	    ztest_opts.zo_verbose >= 4 ? "v" : "",
6040 	    spa_config_path,
6041 	    pool);
6042 	free(isa);
6043 
6044 	if (ztest_opts.zo_verbose >= 5)
6045 		(void) printf("Executing %s\n", strstr(zdb, "zdb "));
6046 
6047 	fp = popen(zdb, "r");
6048 
6049 	while (fgets(zbuf, sizeof (zbuf), fp) != NULL)
6050 		if (ztest_opts.zo_verbose >= 3)
6051 			(void) printf("%s", zbuf);
6052 
6053 	status = pclose(fp);
6054 
6055 	if (status == 0)
6056 		return;
6057 
6058 	ztest_dump_core = 0;
6059 	if (WIFEXITED(status))
6060 		fatal(0, "'%s' exit code %d", zdb, WEXITSTATUS(status));
6061 	else
6062 		fatal(0, "'%s' died with signal %d", zdb, WTERMSIG(status));
6063 }
6064 
6065 static void
6066 ztest_walk_pool_directory(char *header)
6067 {
6068 	spa_t *spa = NULL;
6069 
6070 	if (ztest_opts.zo_verbose >= 6)
6071 		(void) printf("%s\n", header);
6072 
6073 	mutex_enter(&spa_namespace_lock);
6074 	while ((spa = spa_next(spa)) != NULL)
6075 		if (ztest_opts.zo_verbose >= 6)
6076 			(void) printf("\t%s\n", spa_name(spa));
6077 	mutex_exit(&spa_namespace_lock);
6078 }
6079 
6080 static void
6081 ztest_spa_import_export(char *oldname, char *newname)
6082 {
6083 	nvlist_t *config, *newconfig;
6084 	uint64_t pool_guid;
6085 	spa_t *spa;
6086 	int error;
6087 
6088 	if (ztest_opts.zo_verbose >= 4) {
6089 		(void) printf("import/export: old = %s, new = %s\n",
6090 		    oldname, newname);
6091 	}
6092 
6093 	/*
6094 	 * Clean up from previous runs.
6095 	 */
6096 	(void) spa_destroy(newname);
6097 
6098 	/*
6099 	 * Get the pool's configuration and guid.
6100 	 */
6101 	VERIFY3U(0, ==, spa_open(oldname, &spa, FTAG));
6102 
6103 	/*
6104 	 * Kick off a scrub to tickle scrub/export races.
6105 	 */
6106 	if (ztest_random(2) == 0)
6107 		(void) spa_scan(spa, POOL_SCAN_SCRUB);
6108 
6109 	pool_guid = spa_guid(spa);
6110 	spa_close(spa, FTAG);
6111 
6112 	ztest_walk_pool_directory("pools before export");
6113 
6114 	/*
6115 	 * Export it.
6116 	 */
6117 	VERIFY3U(0, ==, spa_export(oldname, &config, B_FALSE, B_FALSE));
6118 
6119 	ztest_walk_pool_directory("pools after export");
6120 
6121 	/*
6122 	 * Try to import it.
6123 	 */
6124 	newconfig = spa_tryimport(config);
6125 	ASSERT(newconfig != NULL);
6126 	nvlist_free(newconfig);
6127 
6128 	/*
6129 	 * Import it under the new name.
6130 	 */
6131 	error = spa_import(newname, config, NULL, 0);
6132 	if (error != 0) {
6133 		dump_nvlist(config, 0);
6134 		fatal(B_FALSE, "couldn't import pool %s as %s: error %u",
6135 		    oldname, newname, error);
6136 	}
6137 
6138 	ztest_walk_pool_directory("pools after import");
6139 
6140 	/*
6141 	 * Try to import it again -- should fail with EEXIST.
6142 	 */
6143 	VERIFY3U(EEXIST, ==, spa_import(newname, config, NULL, 0));
6144 
6145 	/*
6146 	 * Try to import it under a different name -- should fail with EEXIST.
6147 	 */
6148 	VERIFY3U(EEXIST, ==, spa_import(oldname, config, NULL, 0));
6149 
6150 	/*
6151 	 * Verify that the pool is no longer visible under the old name.
6152 	 */
6153 	VERIFY3U(ENOENT, ==, spa_open(oldname, &spa, FTAG));
6154 
6155 	/*
6156 	 * Verify that we can open and close the pool using the new name.
6157 	 */
6158 	VERIFY3U(0, ==, spa_open(newname, &spa, FTAG));
6159 	ASSERT(pool_guid == spa_guid(spa));
6160 	spa_close(spa, FTAG);
6161 
6162 	nvlist_free(config);
6163 }
6164 
6165 static void
6166 ztest_resume(spa_t *spa)
6167 {
6168 	if (spa_suspended(spa) && ztest_opts.zo_verbose >= 6)
6169 		(void) printf("resuming from suspended state\n");
6170 	spa_vdev_state_enter(spa, SCL_NONE);
6171 	vdev_clear(spa, NULL);
6172 	(void) spa_vdev_state_exit(spa, NULL, 0);
6173 	(void) zio_resume(spa);
6174 }
6175 
6176 static void *
6177 ztest_resume_thread(void *arg)
6178 {
6179 	spa_t *spa = arg;
6180 
6181 	while (!ztest_exiting) {
6182 		if (spa_suspended(spa))
6183 			ztest_resume(spa);
6184 		(void) poll(NULL, 0, 100);
6185 
6186 		/*
6187 		 * Periodically change the zfs_compressed_arc_enabled setting.
6188 		 */
6189 		if (ztest_random(10) == 0)
6190 			zfs_compressed_arc_enabled = ztest_random(2);
6191 
6192 		/*
6193 		 * Periodically change the zfs_abd_scatter_enabled setting.
6194 		 */
6195 		if (ztest_random(10) == 0)
6196 			zfs_abd_scatter_enabled = ztest_random(2);
6197 	}
6198 	return (NULL);
6199 }
6200 
6201 static void *
6202 ztest_deadman_thread(void *arg)
6203 {
6204 	ztest_shared_t *zs = arg;
6205 	spa_t *spa = ztest_spa;
6206 	hrtime_t delta, total = 0;
6207 
6208 	for (;;) {
6209 		delta = zs->zs_thread_stop - zs->zs_thread_start +
6210 		    MSEC2NSEC(zfs_deadman_synctime_ms);
6211 
6212 		(void) poll(NULL, 0, (int)NSEC2MSEC(delta));
6213 
6214 		/*
6215 		 * If the pool is suspended then fail immediately. Otherwise,
6216 		 * check to see if the pool is making any progress. If
6217 		 * vdev_deadman() discovers that there hasn't been any recent
6218 		 * I/Os then it will end up aborting the tests.
6219 		 */
6220 		if (spa_suspended(spa) || spa->spa_root_vdev == NULL) {
6221 			fatal(0, "aborting test after %llu seconds because "
6222 			    "pool has transitioned to a suspended state.",
6223 			    zfs_deadman_synctime_ms / 1000);
6224 			return (NULL);
6225 		}
6226 		vdev_deadman(spa->spa_root_vdev);
6227 
6228 		total += zfs_deadman_synctime_ms/1000;
6229 		(void) printf("ztest has been running for %lld seconds\n",
6230 		    total);
6231 	}
6232 }
6233 
6234 static void
6235 ztest_execute(int test, ztest_info_t *zi, uint64_t id)
6236 {
6237 	ztest_ds_t *zd = &ztest_ds[id % ztest_opts.zo_datasets];
6238 	ztest_shared_callstate_t *zc = ZTEST_GET_SHARED_CALLSTATE(test);
6239 	hrtime_t functime = gethrtime();
6240 
6241 	for (int i = 0; i < zi->zi_iters; i++)
6242 		zi->zi_func(zd, id);
6243 
6244 	functime = gethrtime() - functime;
6245 
6246 	atomic_add_64(&zc->zc_count, 1);
6247 	atomic_add_64(&zc->zc_time, functime);
6248 
6249 	if (ztest_opts.zo_verbose >= 4) {
6250 		Dl_info dli;
6251 		(void) dladdr((void *)zi->zi_func, &dli);
6252 		(void) printf("%6.2f sec in %s\n",
6253 		    (double)functime / NANOSEC, dli.dli_sname);
6254 	}
6255 }
6256 
6257 static void *
6258 ztest_thread(void *arg)
6259 {
6260 	int rand;
6261 	uint64_t id = (uintptr_t)arg;
6262 	ztest_shared_t *zs = ztest_shared;
6263 	uint64_t call_next;
6264 	hrtime_t now;
6265 	ztest_info_t *zi;
6266 	ztest_shared_callstate_t *zc;
6267 
6268 	while ((now = gethrtime()) < zs->zs_thread_stop) {
6269 		/*
6270 		 * See if it's time to force a crash.
6271 		 */
6272 		if (now > zs->zs_thread_kill)
6273 			ztest_kill(zs);
6274 
6275 		/*
6276 		 * If we're getting ENOSPC with some regularity, stop.
6277 		 */
6278 		if (zs->zs_enospc_count > 10)
6279 			break;
6280 
6281 		/*
6282 		 * Pick a random function to execute.
6283 		 */
6284 		rand = ztest_random(ZTEST_FUNCS);
6285 		zi = &ztest_info[rand];
6286 		zc = ZTEST_GET_SHARED_CALLSTATE(rand);
6287 		call_next = zc->zc_next;
6288 
6289 		if (now >= call_next &&
6290 		    atomic_cas_64(&zc->zc_next, call_next, call_next +
6291 		    ztest_random(2 * zi->zi_interval[0] + 1)) == call_next) {
6292 			ztest_execute(rand, zi, id);
6293 		}
6294 	}
6295 
6296 	return (NULL);
6297 }
6298 
6299 static void
6300 ztest_dataset_name(char *dsname, char *pool, int d)
6301 {
6302 	(void) snprintf(dsname, ZFS_MAX_DATASET_NAME_LEN, "%s/ds_%d", pool, d);
6303 }
6304 
6305 static void
6306 ztest_dataset_destroy(int d)
6307 {
6308 	char name[ZFS_MAX_DATASET_NAME_LEN];
6309 
6310 	ztest_dataset_name(name, ztest_opts.zo_pool, d);
6311 
6312 	if (ztest_opts.zo_verbose >= 3)
6313 		(void) printf("Destroying %s to free up space\n", name);
6314 
6315 	/*
6316 	 * Cleanup any non-standard clones and snapshots.  In general,
6317 	 * ztest thread t operates on dataset (t % zopt_datasets),
6318 	 * so there may be more than one thing to clean up.
6319 	 */
6320 	for (int t = d; t < ztest_opts.zo_threads;
6321 	    t += ztest_opts.zo_datasets) {
6322 		ztest_dsl_dataset_cleanup(name, t);
6323 	}
6324 
6325 	(void) dmu_objset_find(name, ztest_objset_destroy_cb, NULL,
6326 	    DS_FIND_SNAPSHOTS | DS_FIND_CHILDREN);
6327 }
6328 
6329 static void
6330 ztest_dataset_dirobj_verify(ztest_ds_t *zd)
6331 {
6332 	uint64_t usedobjs, dirobjs, scratch;
6333 
6334 	/*
6335 	 * ZTEST_DIROBJ is the object directory for the entire dataset.
6336 	 * Therefore, the number of objects in use should equal the
6337 	 * number of ZTEST_DIROBJ entries, +1 for ZTEST_DIROBJ itself.
6338 	 * If not, we have an object leak.
6339 	 *
6340 	 * Note that we can only check this in ztest_dataset_open(),
6341 	 * when the open-context and syncing-context values agree.
6342 	 * That's because zap_count() returns the open-context value,
6343 	 * while dmu_objset_space() returns the rootbp fill count.
6344 	 */
6345 	VERIFY3U(0, ==, zap_count(zd->zd_os, ZTEST_DIROBJ, &dirobjs));
6346 	dmu_objset_space(zd->zd_os, &scratch, &scratch, &usedobjs, &scratch);
6347 	ASSERT3U(dirobjs + 1, ==, usedobjs);
6348 }
6349 
6350 static int
6351 ztest_dataset_open(int d)
6352 {
6353 	ztest_ds_t *zd = &ztest_ds[d];
6354 	uint64_t committed_seq = ZTEST_GET_SHARED_DS(d)->zd_seq;
6355 	objset_t *os;
6356 	zilog_t *zilog;
6357 	char name[ZFS_MAX_DATASET_NAME_LEN];
6358 	int error;
6359 
6360 	ztest_dataset_name(name, ztest_opts.zo_pool, d);
6361 
6362 	rw_enter(&ztest_name_lock, RW_READER);
6363 
6364 	error = ztest_dataset_create(name);
6365 	if (error == ENOSPC) {
6366 		rw_exit(&ztest_name_lock);
6367 		ztest_record_enospc(FTAG);
6368 		return (error);
6369 	}
6370 	ASSERT(error == 0 || error == EEXIST);
6371 
6372 	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
6373 	    B_TRUE, zd, &os));
6374 	rw_exit(&ztest_name_lock);
6375 
6376 	ztest_zd_init(zd, ZTEST_GET_SHARED_DS(d), os);
6377 
6378 	zilog = zd->zd_zilog;
6379 
6380 	if (zilog->zl_header->zh_claim_lr_seq != 0 &&
6381 	    zilog->zl_header->zh_claim_lr_seq < committed_seq)
6382 		fatal(0, "missing log records: claimed %llu < committed %llu",
6383 		    zilog->zl_header->zh_claim_lr_seq, committed_seq);
6384 
6385 	ztest_dataset_dirobj_verify(zd);
6386 
6387 	zil_replay(os, zd, ztest_replay_vector);
6388 
6389 	ztest_dataset_dirobj_verify(zd);
6390 
6391 	if (ztest_opts.zo_verbose >= 6)
6392 		(void) printf("%s replay %llu blocks, %llu records, seq %llu\n",
6393 		    zd->zd_name,
6394 		    (u_longlong_t)zilog->zl_parse_blk_count,
6395 		    (u_longlong_t)zilog->zl_parse_lr_count,
6396 		    (u_longlong_t)zilog->zl_replaying_seq);
6397 
6398 	zilog = zil_open(os, ztest_get_data);
6399 
6400 	if (zilog->zl_replaying_seq != 0 &&
6401 	    zilog->zl_replaying_seq < committed_seq)
6402 		fatal(0, "missing log records: replayed %llu < committed %llu",
6403 		    zilog->zl_replaying_seq, committed_seq);
6404 
6405 	return (0);
6406 }
6407 
6408 static void
6409 ztest_dataset_close(int d)
6410 {
6411 	ztest_ds_t *zd = &ztest_ds[d];
6412 
6413 	zil_close(zd->zd_zilog);
6414 	dmu_objset_disown(zd->zd_os, B_TRUE, zd);
6415 
6416 	ztest_zd_fini(zd);
6417 }
6418 
6419 /*
6420  * Kick off threads to run tests on all datasets in parallel.
6421  */
6422 static void
6423 ztest_run(ztest_shared_t *zs)
6424 {
6425 	thread_t *tid;
6426 	spa_t *spa;
6427 	objset_t *os;
6428 	thread_t resume_tid;
6429 	int error;
6430 
6431 	ztest_exiting = B_FALSE;
6432 
6433 	/*
6434 	 * Initialize parent/child shared state.
6435 	 */
6436 	mutex_init(&ztest_checkpoint_lock, NULL, USYNC_THREAD, NULL);
6437 	mutex_init(&ztest_vdev_lock, NULL, USYNC_THREAD, NULL);
6438 	rw_init(&ztest_name_lock, NULL, USYNC_THREAD, NULL);
6439 
6440 	zs->zs_thread_start = gethrtime();
6441 	zs->zs_thread_stop =
6442 	    zs->zs_thread_start + ztest_opts.zo_passtime * NANOSEC;
6443 	zs->zs_thread_stop = MIN(zs->zs_thread_stop, zs->zs_proc_stop);
6444 	zs->zs_thread_kill = zs->zs_thread_stop;
6445 	if (ztest_random(100) < ztest_opts.zo_killrate) {
6446 		zs->zs_thread_kill -=
6447 		    ztest_random(ztest_opts.zo_passtime * NANOSEC);
6448 	}
6449 
6450 	mutex_init(&zcl.zcl_callbacks_lock, NULL, USYNC_THREAD, NULL);
6451 
6452 	list_create(&zcl.zcl_callbacks, sizeof (ztest_cb_data_t),
6453 	    offsetof(ztest_cb_data_t, zcd_node));
6454 
6455 	/*
6456 	 * Open our pool.
6457 	 */
6458 	kernel_init(FREAD | FWRITE);
6459 	VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
6460 	metaslab_preload_limit = ztest_random(20) + 1;
6461 	ztest_spa = spa;
6462 
6463 	dmu_objset_stats_t dds;
6464 	VERIFY0(ztest_dmu_objset_own(ztest_opts.zo_pool,
6465 	    DMU_OST_ANY, B_TRUE, B_TRUE, FTAG, &os));
6466 	dsl_pool_config_enter(dmu_objset_pool(os), FTAG);
6467 	dmu_objset_fast_stat(os, &dds);
6468 	dsl_pool_config_exit(dmu_objset_pool(os), FTAG);
6469 	zs->zs_guid = dds.dds_guid;
6470 	dmu_objset_disown(os, B_TRUE, FTAG);
6471 
6472 	spa->spa_dedup_ditto = 2 * ZIO_DEDUPDITTO_MIN;
6473 
6474 	/*
6475 	 * We don't expect the pool to suspend unless maxfaults == 0,
6476 	 * in which case ztest_fault_inject() temporarily takes away
6477 	 * the only valid replica.
6478 	 */
6479 	if (MAXFAULTS() == 0)
6480 		spa->spa_failmode = ZIO_FAILURE_MODE_WAIT;
6481 	else
6482 		spa->spa_failmode = ZIO_FAILURE_MODE_PANIC;
6483 
6484 	/*
6485 	 * Create a thread to periodically resume suspended I/O.
6486 	 */
6487 	VERIFY(thr_create(0, 0, ztest_resume_thread, spa, THR_BOUND,
6488 	    &resume_tid) == 0);
6489 
6490 	/*
6491 	 * Create a deadman thread to abort() if we hang.
6492 	 */
6493 	VERIFY(thr_create(0, 0, ztest_deadman_thread, zs, THR_BOUND,
6494 	    NULL) == 0);
6495 
6496 	/*
6497 	 * Verify that we can safely inquire about any object,
6498 	 * whether it's allocated or not.  To make it interesting,
6499 	 * we probe a 5-wide window around each power of two.
6500 	 * This hits all edge cases, including zero and the max.
6501 	 */
6502 	for (int t = 0; t < 64; t++) {
6503 		for (int d = -5; d <= 5; d++) {
6504 			error = dmu_object_info(spa->spa_meta_objset,
6505 			    (1ULL << t) + d, NULL);
6506 			ASSERT(error == 0 || error == ENOENT ||
6507 			    error == EINVAL);
6508 		}
6509 	}
6510 
6511 	/*
6512 	 * If we got any ENOSPC errors on the previous run, destroy something.
6513 	 */
6514 	if (zs->zs_enospc_count != 0) {
6515 		int d = ztest_random(ztest_opts.zo_datasets);
6516 		ztest_dataset_destroy(d);
6517 	}
6518 	zs->zs_enospc_count = 0;
6519 
6520 	tid = umem_zalloc(ztest_opts.zo_threads * sizeof (thread_t),
6521 	    UMEM_NOFAIL);
6522 
6523 	if (ztest_opts.zo_verbose >= 4)
6524 		(void) printf("starting main threads...\n");
6525 
6526 	/*
6527 	 * Kick off all the tests that run in parallel.
6528 	 */
6529 	for (int t = 0; t < ztest_opts.zo_threads; t++) {
6530 		if (t < ztest_opts.zo_datasets &&
6531 		    ztest_dataset_open(t) != 0)
6532 			return;
6533 		VERIFY(thr_create(0, 0, ztest_thread, (void *)(uintptr_t)t,
6534 		    THR_BOUND, &tid[t]) == 0);
6535 	}
6536 
6537 	/*
6538 	 * Wait for all of the tests to complete.  We go in reverse order
6539 	 * so we don't close datasets while threads are still using them.
6540 	 */
6541 	for (int t = ztest_opts.zo_threads - 1; t >= 0; t--) {
6542 		VERIFY(thr_join(tid[t], NULL, NULL) == 0);
6543 		if (t < ztest_opts.zo_datasets)
6544 			ztest_dataset_close(t);
6545 	}
6546 
6547 	txg_wait_synced(spa_get_dsl(spa), 0);
6548 
6549 	zs->zs_alloc = metaslab_class_get_alloc(spa_normal_class(spa));
6550 	zs->zs_space = metaslab_class_get_space(spa_normal_class(spa));
6551 	zfs_dbgmsg_print(FTAG);
6552 
6553 	umem_free(tid, ztest_opts.zo_threads * sizeof (thread_t));
6554 
6555 	/* Kill the resume thread */
6556 	ztest_exiting = B_TRUE;
6557 	VERIFY(thr_join(resume_tid, NULL, NULL) == 0);
6558 	ztest_resume(spa);
6559 
6560 	/*
6561 	 * Right before closing the pool, kick off a bunch of async I/O;
6562 	 * spa_close() should wait for it to complete.
6563 	 */
6564 	for (uint64_t object = 1; object < 50; object++) {
6565 		dmu_prefetch(spa->spa_meta_objset, object, 0, 0, 1ULL << 20,
6566 		    ZIO_PRIORITY_SYNC_READ);
6567 	}
6568 
6569 	spa_close(spa, FTAG);
6570 
6571 	/*
6572 	 * Verify that we can loop over all pools.
6573 	 */
6574 	mutex_enter(&spa_namespace_lock);
6575 	for (spa = spa_next(NULL); spa != NULL; spa = spa_next(spa))
6576 		if (ztest_opts.zo_verbose > 3)
6577 			(void) printf("spa_next: found %s\n", spa_name(spa));
6578 	mutex_exit(&spa_namespace_lock);
6579 
6580 	/*
6581 	 * Verify that we can export the pool and reimport it under a
6582 	 * different name.
6583 	 */
6584 	if ((ztest_random(2) == 0) && !ztest_opts.zo_mmp_test) {
6585 		char name[ZFS_MAX_DATASET_NAME_LEN];
6586 		(void) snprintf(name, sizeof (name), "%s_import",
6587 		    ztest_opts.zo_pool);
6588 		ztest_spa_import_export(ztest_opts.zo_pool, name);
6589 		ztest_spa_import_export(name, ztest_opts.zo_pool);
6590 	}
6591 
6592 	kernel_fini();
6593 
6594 	list_destroy(&zcl.zcl_callbacks);
6595 
6596 	mutex_destroy(&zcl.zcl_callbacks_lock);
6597 
6598 	rw_destroy(&ztest_name_lock);
6599 	mutex_destroy(&ztest_vdev_lock);
6600 	mutex_destroy(&ztest_checkpoint_lock);
6601 }
6602 
6603 static void
6604 ztest_freeze(void)
6605 {
6606 	ztest_ds_t *zd = &ztest_ds[0];
6607 	spa_t *spa;
6608 	int numloops = 0;
6609 
6610 	if (ztest_opts.zo_verbose >= 3)
6611 		(void) printf("testing spa_freeze()...\n");
6612 
6613 	kernel_init(FREAD | FWRITE);
6614 	VERIFY3U(0, ==, spa_open(ztest_opts.zo_pool, &spa, FTAG));
6615 	VERIFY3U(0, ==, ztest_dataset_open(0));
6616 	ztest_spa = spa;
6617 
6618 	/*
6619 	 * Force the first log block to be transactionally allocated.
6620 	 * We have to do this before we freeze the pool -- otherwise
6621 	 * the log chain won't be anchored.
6622 	 */
6623 	while (BP_IS_HOLE(&zd->zd_zilog->zl_header->zh_log)) {
6624 		ztest_dmu_object_alloc_free(zd, 0);
6625 		zil_commit(zd->zd_zilog, 0);
6626 	}
6627 
6628 	txg_wait_synced(spa_get_dsl(spa), 0);
6629 
6630 	/*
6631 	 * Freeze the pool.  This stops spa_sync() from doing anything,
6632 	 * so that the only way to record changes from now on is the ZIL.
6633 	 */
6634 	spa_freeze(spa);
6635 
6636 	/*
6637 	 * Because it is hard to predict how much space a write will actually
6638 	 * require beforehand, we leave ourselves some fudge space to write over
6639 	 * capacity.
6640 	 */
6641 	uint64_t capacity = metaslab_class_get_space(spa_normal_class(spa)) / 2;
6642 
6643 	/*
6644 	 * Run tests that generate log records but don't alter the pool config
6645 	 * or depend on DSL sync tasks (snapshots, objset create/destroy, etc).
6646 	 * We do a txg_wait_synced() after each iteration to force the txg
6647 	 * to increase well beyond the last synced value in the uberblock.
6648 	 * The ZIL should be OK with that.
6649 	 *
6650 	 * Run a random number of times less than zo_maxloops and ensure we do
6651 	 * not run out of space on the pool.
6652 	 */
6653 	while (ztest_random(10) != 0 &&
6654 	    numloops++ < ztest_opts.zo_maxloops &&
6655 	    metaslab_class_get_alloc(spa_normal_class(spa)) < capacity) {
6656 		ztest_od_t od;
6657 		ztest_od_init(&od, 0, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0);
6658 		VERIFY0(ztest_object_init(zd, &od, sizeof (od), B_FALSE));
6659 		ztest_io(zd, od.od_object,
6660 		    ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);
6661 		txg_wait_synced(spa_get_dsl(spa), 0);
6662 	}
6663 
6664 	/*
6665 	 * Commit all of the changes we just generated.
6666 	 */
6667 	zil_commit(zd->zd_zilog, 0);
6668 	txg_wait_synced(spa_get_dsl(spa), 0);
6669 
6670 	/*
6671 	 * Close our dataset and close the pool.
6672 	 */
6673 	ztest_dataset_close(0);
6674 	spa_close(spa, FTAG);
6675 	kernel_fini();
6676 
6677 	/*
6678 	 * Open and close the pool and dataset to induce log replay.
6679 	 */
6680 	kernel_init(FREAD | FWRITE);
6681 	VERIFY3U(0, ==, spa_open(ztest_opts.zo_pool, &spa, FTAG));
6682 	ASSERT(spa_freeze_txg(spa) == UINT64_MAX);
6683 	VERIFY3U(0, ==, ztest_dataset_open(0));
6684 	ztest_spa = spa;
6685 	txg_wait_synced(spa_get_dsl(spa), 0);
6686 	ztest_dataset_close(0);
6687 	ztest_reguid(NULL, 0);
6688 
6689 	spa_close(spa, FTAG);
6690 	kernel_fini();
6691 }
6692 
6693 void
6694 print_time(hrtime_t t, char *timebuf)
6695 {
6696 	hrtime_t s = t / NANOSEC;
6697 	hrtime_t m = s / 60;
6698 	hrtime_t h = m / 60;
6699 	hrtime_t d = h / 24;
6700 
6701 	s -= m * 60;
6702 	m -= h * 60;
6703 	h -= d * 24;
6704 
6705 	timebuf[0] = '\0';
6706 
6707 	if (d)
6708 		(void) sprintf(timebuf,
6709 		    "%llud%02lluh%02llum%02llus", d, h, m, s);
6710 	else if (h)
6711 		(void) sprintf(timebuf, "%lluh%02llum%02llus", h, m, s);
6712 	else if (m)
6713 		(void) sprintf(timebuf, "%llum%02llus", m, s);
6714 	else
6715 		(void) sprintf(timebuf, "%llus", s);
6716 }
6717 
6718 static nvlist_t *
6719 make_random_props()
6720 {
6721 	nvlist_t *props;
6722 
6723 	VERIFY(nvlist_alloc(&props, NV_UNIQUE_NAME, 0) == 0);
6724 
6725 	if (ztest_random(2) == 0)
6726 		return (props);
6727 	VERIFY(nvlist_add_uint64(props, "autoreplace", 1) == 0);
6728 
6729 	return (props);
6730 }
6731 
6732 /*
6733  * Import a storage pool with the given name.
6734  */
6735 static void
6736 ztest_import(ztest_shared_t *zs)
6737 {
6738 	libzfs_handle_t *hdl;
6739 	importargs_t args = { 0 };
6740 	spa_t *spa;
6741 	nvlist_t *cfg = NULL;
6742 	int nsearch = 1;
6743 	char *searchdirs[nsearch];
6744 	char *name = ztest_opts.zo_pool;
6745 	int flags = ZFS_IMPORT_MISSING_LOG;
6746 	int error;
6747 
6748 	mutex_init(&ztest_vdev_lock, NULL, MUTEX_DEFAULT, NULL);
6749 	rw_init(&ztest_name_lock, NULL, USYNC_THREAD, NULL);
6750 
6751 	kernel_init(FREAD | FWRITE);
6752 	hdl = libzfs_init();
6753 
6754 	searchdirs[0] = ztest_opts.zo_dir;
6755 	args.paths = nsearch;
6756 	args.path = searchdirs;
6757 	args.can_be_active = B_FALSE;
6758 
6759 	error = zpool_tryimport(hdl, name, &cfg, &args);
6760 	if (error)
6761 		(void) fatal(0, "No pools found\n");
6762 
6763 	VERIFY0(spa_import(name, cfg, NULL, flags));
6764 	VERIFY0(spa_open(name, &spa, FTAG));
6765 	zs->zs_metaslab_sz =
6766 	    1ULL << spa->spa_root_vdev->vdev_child[0]->vdev_ms_shift;
6767 	spa_close(spa, FTAG);
6768 
6769 	libzfs_fini(hdl);
6770 	kernel_fini();
6771 
6772 	if (!ztest_opts.zo_mmp_test) {
6773 		ztest_run_zdb(ztest_opts.zo_pool);
6774 		ztest_freeze();
6775 		ztest_run_zdb(ztest_opts.zo_pool);
6776 	}
6777 
6778 	rw_destroy(&ztest_name_lock);
6779 	mutex_destroy(&ztest_vdev_lock);
6780 }
6781 
6782 /*
6783  * Create a storage pool with the given name and initial vdev size.
6784  * Then test spa_freeze() functionality.
6785  */
6786 static void
6787 ztest_init(ztest_shared_t *zs)
6788 {
6789 	spa_t *spa;
6790 	nvlist_t *nvroot, *props;
6791 
6792 	mutex_init(&ztest_vdev_lock, NULL, USYNC_THREAD, NULL);
6793 	mutex_init(&ztest_checkpoint_lock, NULL, USYNC_THREAD, NULL);
6794 	rw_init(&ztest_name_lock, NULL, USYNC_THREAD, NULL);
6795 
6796 	kernel_init(FREAD | FWRITE);
6797 
6798 	/*
6799 	 * Create the storage pool.
6800 	 */
6801 	(void) spa_destroy(ztest_opts.zo_pool);
6802 	ztest_shared->zs_vdev_next_leaf = 0;
6803 	zs->zs_splits = 0;
6804 	zs->zs_mirrors = ztest_opts.zo_mirrors;
6805 	nvroot = make_vdev_root(NULL, NULL, NULL, ztest_opts.zo_vdev_size, 0,
6806 	    NULL, ztest_opts.zo_raidz, zs->zs_mirrors, 1);
6807 	props = make_random_props();
6808 	for (int i = 0; i < SPA_FEATURES; i++) {
6809 		char buf[1024];
6810 		(void) snprintf(buf, sizeof (buf), "feature@%s",
6811 		    spa_feature_table[i].fi_uname);
6812 		VERIFY3U(0, ==, nvlist_add_uint64(props, buf, 0));
6813 	}
6814 	VERIFY3U(0, ==,
6815 	    spa_create(ztest_opts.zo_pool, nvroot, props, NULL, NULL));
6816 	nvlist_free(nvroot);
6817 	nvlist_free(props);
6818 
6819 	VERIFY3U(0, ==, spa_open(ztest_opts.zo_pool, &spa, FTAG));
6820 	zs->zs_metaslab_sz =
6821 	    1ULL << spa->spa_root_vdev->vdev_child[0]->vdev_ms_shift;
6822 
6823 	spa_close(spa, FTAG);
6824 
6825 	kernel_fini();
6826 
6827 	if (!ztest_opts.zo_mmp_test) {
6828 		ztest_run_zdb(ztest_opts.zo_pool);
6829 		ztest_freeze();
6830 		ztest_run_zdb(ztest_opts.zo_pool);
6831 	}
6832 
6833 	rw_destroy(&ztest_name_lock);
6834 	mutex_destroy(&ztest_vdev_lock);
6835 	mutex_destroy(&ztest_checkpoint_lock);
6836 }
6837 
6838 static void
6839 setup_data_fd(void)
6840 {
6841 	static char ztest_name_data[] = "/tmp/ztest.data.XXXXXX";
6842 
6843 	ztest_fd_data = mkstemp(ztest_name_data);
6844 	ASSERT3S(ztest_fd_data, >=, 0);
6845 	(void) unlink(ztest_name_data);
6846 }
6847 
6848 static int
6849 shared_data_size(ztest_shared_hdr_t *hdr)
6850 {
6851 	int size;
6852 
6853 	size = hdr->zh_hdr_size;
6854 	size += hdr->zh_opts_size;
6855 	size += hdr->zh_size;
6856 	size += hdr->zh_stats_size * hdr->zh_stats_count;
6857 	size += hdr->zh_ds_size * hdr->zh_ds_count;
6858 
6859 	return (size);
6860 }
6861 
6862 static void
6863 setup_hdr(void)
6864 {
6865 	int size;
6866 	ztest_shared_hdr_t *hdr;
6867 
6868 	hdr = (void *)mmap(0, P2ROUNDUP(sizeof (*hdr), getpagesize()),
6869 	    PROT_READ | PROT_WRITE, MAP_SHARED, ztest_fd_data, 0);
6870 	ASSERT(hdr != MAP_FAILED);
6871 
6872 	VERIFY3U(0, ==, ftruncate(ztest_fd_data, sizeof (ztest_shared_hdr_t)));
6873 
6874 	hdr->zh_hdr_size = sizeof (ztest_shared_hdr_t);
6875 	hdr->zh_opts_size = sizeof (ztest_shared_opts_t);
6876 	hdr->zh_size = sizeof (ztest_shared_t);
6877 	hdr->zh_stats_size = sizeof (ztest_shared_callstate_t);
6878 	hdr->zh_stats_count = ZTEST_FUNCS;
6879 	hdr->zh_ds_size = sizeof (ztest_shared_ds_t);
6880 	hdr->zh_ds_count = ztest_opts.zo_datasets;
6881 
6882 	size = shared_data_size(hdr);
6883 	VERIFY3U(0, ==, ftruncate(ztest_fd_data, size));
6884 
6885 	(void) munmap((caddr_t)hdr, P2ROUNDUP(sizeof (*hdr), getpagesize()));
6886 }
6887 
6888 static void
6889 setup_data(void)
6890 {
6891 	int size, offset;
6892 	ztest_shared_hdr_t *hdr;
6893 	uint8_t *buf;
6894 
6895 	hdr = (void *)mmap(0, P2ROUNDUP(sizeof (*hdr), getpagesize()),
6896 	    PROT_READ, MAP_SHARED, ztest_fd_data, 0);
6897 	ASSERT(hdr != MAP_FAILED);
6898 
6899 	size = shared_data_size(hdr);
6900 
6901 	(void) munmap((caddr_t)hdr, P2ROUNDUP(sizeof (*hdr), getpagesize()));
6902 	hdr = ztest_shared_hdr = (void *)mmap(0, P2ROUNDUP(size, getpagesize()),
6903 	    PROT_READ | PROT_WRITE, MAP_SHARED, ztest_fd_data, 0);
6904 	ASSERT(hdr != MAP_FAILED);
6905 	buf = (uint8_t *)hdr;
6906 
6907 	offset = hdr->zh_hdr_size;
6908 	ztest_shared_opts = (void *)&buf[offset];
6909 	offset += hdr->zh_opts_size;
6910 	ztest_shared = (void *)&buf[offset];
6911 	offset += hdr->zh_size;
6912 	ztest_shared_callstate = (void *)&buf[offset];
6913 	offset += hdr->zh_stats_size * hdr->zh_stats_count;
6914 	ztest_shared_ds = (void *)&buf[offset];
6915 }
6916 
6917 static boolean_t
6918 exec_child(char *cmd, char *libpath, boolean_t ignorekill, int *statusp)
6919 {
6920 	pid_t pid;
6921 	int status;
6922 	char *cmdbuf = NULL;
6923 
6924 	pid = fork();
6925 
6926 	if (cmd == NULL) {
6927 		cmdbuf = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
6928 		(void) strlcpy(cmdbuf, getexecname(), MAXPATHLEN);
6929 		cmd = cmdbuf;
6930 	}
6931 
6932 	if (pid == -1)
6933 		fatal(1, "fork failed");
6934 
6935 	if (pid == 0) {	/* child */
6936 		char *emptyargv[2] = { cmd, NULL };
6937 		char fd_data_str[12];
6938 
6939 		struct rlimit rl = { 1024, 1024 };
6940 		(void) setrlimit(RLIMIT_NOFILE, &rl);
6941 
6942 		(void) close(ztest_fd_rand);
6943 		VERIFY3U(11, >=,
6944 		    snprintf(fd_data_str, 12, "%d", ztest_fd_data));
6945 		VERIFY0(setenv("ZTEST_FD_DATA", fd_data_str, 1));
6946 
6947 		(void) enable_extended_FILE_stdio(-1, -1);
6948 		if (libpath != NULL)
6949 			VERIFY(0 == setenv("LD_LIBRARY_PATH", libpath, 1));
6950 		(void) execv(cmd, emptyargv);
6951 		ztest_dump_core = B_FALSE;
6952 		fatal(B_TRUE, "exec failed: %s", cmd);
6953 	}
6954 
6955 	if (cmdbuf != NULL) {
6956 		umem_free(cmdbuf, MAXPATHLEN);
6957 		cmd = NULL;
6958 	}
6959 
6960 	while (waitpid(pid, &status, 0) != pid)
6961 		continue;
6962 	if (statusp != NULL)
6963 		*statusp = status;
6964 
6965 	if (WIFEXITED(status)) {
6966 		if (WEXITSTATUS(status) != 0) {
6967 			(void) fprintf(stderr, "child exited with code %d\n",
6968 			    WEXITSTATUS(status));
6969 			exit(2);
6970 		}
6971 		return (B_FALSE);
6972 	} else if (WIFSIGNALED(status)) {
6973 		if (!ignorekill || WTERMSIG(status) != SIGKILL) {
6974 			(void) fprintf(stderr, "child died with signal %d\n",
6975 			    WTERMSIG(status));
6976 			exit(3);
6977 		}
6978 		return (B_TRUE);
6979 	} else {
6980 		(void) fprintf(stderr, "something strange happened to child\n");
6981 		exit(4);
6982 		/* NOTREACHED */
6983 	}
6984 }
6985 
6986 static void
6987 ztest_run_init(void)
6988 {
6989 	ztest_shared_t *zs = ztest_shared;
6990 
6991 	/*
6992 	 * Blow away any existing copy of zpool.cache
6993 	 */
6994 	(void) remove(spa_config_path);
6995 
6996 	if (ztest_opts.zo_init == 0) {
6997 		if (ztest_opts.zo_verbose >= 1)
6998 			(void) printf("Importing pool %s\n",
6999 			    ztest_opts.zo_pool);
7000 		ztest_import(zs);
7001 		return;
7002 	}
7003 
7004 	/*
7005 	 * Create and initialize our storage pool.
7006 	 */
7007 	for (int i = 1; i <= ztest_opts.zo_init; i++) {
7008 		bzero(zs, sizeof (ztest_shared_t));
7009 		if (ztest_opts.zo_verbose >= 3 &&
7010 		    ztest_opts.zo_init != 1) {
7011 			(void) printf("ztest_init(), pass %d\n", i);
7012 		}
7013 		ztest_init(zs);
7014 	}
7015 }
7016 
7017 int
7018 main(int argc, char **argv)
7019 {
7020 	int kills = 0;
7021 	int iters = 0;
7022 	int older = 0;
7023 	int newer = 0;
7024 	ztest_shared_t *zs;
7025 	ztest_info_t *zi;
7026 	ztest_shared_callstate_t *zc;
7027 	char timebuf[100];
7028 	char numbuf[NN_NUMBUF_SZ];
7029 	char *cmd;
7030 	boolean_t hasalt;
7031 	char *fd_data_str = getenv("ZTEST_FD_DATA");
7032 
7033 	(void) setvbuf(stdout, NULL, _IOLBF, 0);
7034 
7035 	dprintf_setup(&argc, argv);
7036 	zfs_deadman_synctime_ms = 300000;
7037 	/*
7038 	 * As two-word space map entries may not come up often (especially
7039 	 * if pool and vdev sizes are small) we want to force at least some
7040 	 * of them so the feature get tested.
7041 	 */
7042 	zfs_force_some_double_word_sm_entries = B_TRUE;
7043 
7044 	/*
7045 	 * Verify that even extensively damaged split blocks with many
7046 	 * segments can be reconstructed in a reasonable amount of time
7047 	 * when reconstruction is known to be possible.
7048 	 */
7049 	zfs_reconstruct_indirect_damage_fraction = 4;
7050 
7051 	ztest_fd_rand = open("/dev/urandom", O_RDONLY);
7052 	ASSERT3S(ztest_fd_rand, >=, 0);
7053 
7054 	if (!fd_data_str) {
7055 		process_options(argc, argv);
7056 
7057 		setup_data_fd();
7058 		setup_hdr();
7059 		setup_data();
7060 		bcopy(&ztest_opts, ztest_shared_opts,
7061 		    sizeof (*ztest_shared_opts));
7062 	} else {
7063 		ztest_fd_data = atoi(fd_data_str);
7064 		setup_data();
7065 		bcopy(ztest_shared_opts, &ztest_opts, sizeof (ztest_opts));
7066 	}
7067 	ASSERT3U(ztest_opts.zo_datasets, ==, ztest_shared_hdr->zh_ds_count);
7068 
7069 	/* Override location of zpool.cache */
7070 	VERIFY3U(asprintf((char **)&spa_config_path, "%s/zpool.cache",
7071 	    ztest_opts.zo_dir), !=, -1);
7072 
7073 	ztest_ds = umem_alloc(ztest_opts.zo_datasets * sizeof (ztest_ds_t),
7074 	    UMEM_NOFAIL);
7075 	zs = ztest_shared;
7076 
7077 	if (fd_data_str) {
7078 		metaslab_force_ganging = ztest_opts.zo_metaslab_force_ganging;
7079 		metaslab_df_alloc_threshold =
7080 		    zs->zs_metaslab_df_alloc_threshold;
7081 
7082 		if (zs->zs_do_init)
7083 			ztest_run_init();
7084 		else
7085 			ztest_run(zs);
7086 		exit(0);
7087 	}
7088 
7089 	hasalt = (strlen(ztest_opts.zo_alt_ztest) != 0);
7090 
7091 	if (ztest_opts.zo_verbose >= 1) {
7092 		(void) printf("%llu vdevs, %d datasets, %d threads,"
7093 		    " %llu seconds...\n",
7094 		    (u_longlong_t)ztest_opts.zo_vdevs,
7095 		    ztest_opts.zo_datasets,
7096 		    ztest_opts.zo_threads,
7097 		    (u_longlong_t)ztest_opts.zo_time);
7098 	}
7099 
7100 	cmd = umem_alloc(MAXNAMELEN, UMEM_NOFAIL);
7101 	(void) strlcpy(cmd, getexecname(), MAXNAMELEN);
7102 
7103 	zs->zs_do_init = B_TRUE;
7104 	if (strlen(ztest_opts.zo_alt_ztest) != 0) {
7105 		if (ztest_opts.zo_verbose >= 1) {
7106 			(void) printf("Executing older ztest for "
7107 			    "initialization: %s\n", ztest_opts.zo_alt_ztest);
7108 		}
7109 		VERIFY(!exec_child(ztest_opts.zo_alt_ztest,
7110 		    ztest_opts.zo_alt_libpath, B_FALSE, NULL));
7111 	} else {
7112 		VERIFY(!exec_child(NULL, NULL, B_FALSE, NULL));
7113 	}
7114 	zs->zs_do_init = B_FALSE;
7115 
7116 	zs->zs_proc_start = gethrtime();
7117 	zs->zs_proc_stop = zs->zs_proc_start + ztest_opts.zo_time * NANOSEC;
7118 
7119 	for (int f = 0; f < ZTEST_FUNCS; f++) {
7120 		zi = &ztest_info[f];
7121 		zc = ZTEST_GET_SHARED_CALLSTATE(f);
7122 		if (zs->zs_proc_start + zi->zi_interval[0] > zs->zs_proc_stop)
7123 			zc->zc_next = UINT64_MAX;
7124 		else
7125 			zc->zc_next = zs->zs_proc_start +
7126 			    ztest_random(2 * zi->zi_interval[0] + 1);
7127 	}
7128 
7129 	/*
7130 	 * Run the tests in a loop.  These tests include fault injection
7131 	 * to verify that self-healing data works, and forced crashes
7132 	 * to verify that we never lose on-disk consistency.
7133 	 */
7134 	while (gethrtime() < zs->zs_proc_stop) {
7135 		int status;
7136 		boolean_t killed;
7137 
7138 		/*
7139 		 * Initialize the workload counters for each function.
7140 		 */
7141 		for (int f = 0; f < ZTEST_FUNCS; f++) {
7142 			zc = ZTEST_GET_SHARED_CALLSTATE(f);
7143 			zc->zc_count = 0;
7144 			zc->zc_time = 0;
7145 		}
7146 
7147 		/* Set the allocation switch size */
7148 		zs->zs_metaslab_df_alloc_threshold =
7149 		    ztest_random(zs->zs_metaslab_sz / 4) + 1;
7150 
7151 		if (!hasalt || ztest_random(2) == 0) {
7152 			if (hasalt && ztest_opts.zo_verbose >= 1) {
7153 				(void) printf("Executing newer ztest: %s\n",
7154 				    cmd);
7155 			}
7156 			newer++;
7157 			killed = exec_child(cmd, NULL, B_TRUE, &status);
7158 		} else {
7159 			if (hasalt && ztest_opts.zo_verbose >= 1) {
7160 				(void) printf("Executing older ztest: %s\n",
7161 				    ztest_opts.zo_alt_ztest);
7162 			}
7163 			older++;
7164 			killed = exec_child(ztest_opts.zo_alt_ztest,
7165 			    ztest_opts.zo_alt_libpath, B_TRUE, &status);
7166 		}
7167 
7168 		if (killed)
7169 			kills++;
7170 		iters++;
7171 
7172 		if (ztest_opts.zo_verbose >= 1) {
7173 			hrtime_t now = gethrtime();
7174 
7175 			now = MIN(now, zs->zs_proc_stop);
7176 			print_time(zs->zs_proc_stop - now, timebuf);
7177 			nicenum(zs->zs_space, numbuf, sizeof (numbuf));
7178 
7179 			(void) printf("Pass %3d, %8s, %3llu ENOSPC, "
7180 			    "%4.1f%% of %5s used, %3.0f%% done, %8s to go\n",
7181 			    iters,
7182 			    WIFEXITED(status) ? "Complete" : "SIGKILL",
7183 			    (u_longlong_t)zs->zs_enospc_count,
7184 			    100.0 * zs->zs_alloc / zs->zs_space,
7185 			    numbuf,
7186 			    100.0 * (now - zs->zs_proc_start) /
7187 			    (ztest_opts.zo_time * NANOSEC), timebuf);
7188 		}
7189 
7190 		if (ztest_opts.zo_verbose >= 2) {
7191 			(void) printf("\nWorkload summary:\n\n");
7192 			(void) printf("%7s %9s   %s\n",
7193 			    "Calls", "Time", "Function");
7194 			(void) printf("%7s %9s   %s\n",
7195 			    "-----", "----", "--------");
7196 			for (int f = 0; f < ZTEST_FUNCS; f++) {
7197 				Dl_info dli;
7198 
7199 				zi = &ztest_info[f];
7200 				zc = ZTEST_GET_SHARED_CALLSTATE(f);
7201 				print_time(zc->zc_time, timebuf);
7202 				(void) dladdr((void *)zi->zi_func, &dli);
7203 				(void) printf("%7llu %9s   %s\n",
7204 				    (u_longlong_t)zc->zc_count, timebuf,
7205 				    dli.dli_sname);
7206 			}
7207 			(void) printf("\n");
7208 		}
7209 
7210 		if (!ztest_opts.zo_mmp_test)
7211 			ztest_run_zdb(ztest_opts.zo_pool);
7212 	}
7213 
7214 	if (ztest_opts.zo_verbose >= 1) {
7215 		if (hasalt) {
7216 			(void) printf("%d runs of older ztest: %s\n", older,
7217 			    ztest_opts.zo_alt_ztest);
7218 			(void) printf("%d runs of newer ztest: %s\n", newer,
7219 			    cmd);
7220 		}
7221 		(void) printf("%d killed, %d completed, %.0f%% kill rate\n",
7222 		    kills, iters - kills, (100.0 * kills) / MAX(1, iters));
7223 	}
7224 
7225 	umem_free(cmd, MAXNAMELEN);
7226 
7227 	return (0);
7228 }
7229