13b005d3pjd/*
23b005d3pjd * CDDL HEADER START
33b005d3pjd *
43b005d3pjd * The contents of this file are subject to the terms of the
53b005d3pjd * Common Development and Distribution License (the "License").
63b005d3pjd * You may not use this file except in compliance with the License.
73b005d3pjd *
83b005d3pjd * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
93b005d3pjd * or http://www.opensolaris.org/os/licensing.
103b005d3pjd * See the License for the specific language governing permissions
113b005d3pjd * and limitations under the License.
123b005d3pjd *
133b005d3pjd * When distributing Covered Code, include this CDDL HEADER in each
143b005d3pjd * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
153b005d3pjd * If applicable, add the following below this CDDL HEADER, with the
163b005d3pjd * fields enclosed by brackets "[]" replaced with your own identifying
173b005d3pjd * information: Portions Copyright [yyyy] [name of copyright owner]
183b005d3pjd *
193b005d3pjd * CDDL HEADER END
203b005d3pjd */
213b005d3pjd/*
221b03c5bpjd * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
238f9d694avg * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
2473cab71mm * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
256a061d7mm * Copyright (c) 2012 Martin Matuska <mm@FreeBSD.org>.  All rights reserved.
26ab3dbcbdelphij * Copyright (c) 2013 Steven Hartland. All rights reserved.
27eb06d29mav * Copyright (c) 2014 Integros [integros.com]
287c825baavg * Copyright 2017 Joyent, Inc.
298f9d694avg * Copyright (c) 2017, Intel Corporation.
300014145mav * Copyright 2017 RackTop Systems.
313b005d3pjd */
323b005d3pjd
333b005d3pjd/*
343b005d3pjd * The objective of this program is to provide a DMU/ZAP/SPA stress test
353b005d3pjd * that runs entirely in userland, is easy to use, and easy to extend.
363b005d3pjd *
373b005d3pjd * The overall design of the ztest program is as follows:
383b005d3pjd *
393b005d3pjd * (1) For each major functional area (e.g. adding vdevs to a pool,
403b005d3pjd *     creating and destroying datasets, reading and writing objects, etc)
413b005d3pjd *     we have a simple routine to test that functionality.  These
423b005d3pjd *     individual routines do not have to do anything "stressful".
433b005d3pjd *
443b005d3pjd * (2) We turn these simple functionality tests into a stress test by
453b005d3pjd *     running them all in parallel, with as many threads as desired,
463b005d3pjd *     and spread across as many datasets, objects, and vdevs as desired.
473b005d3pjd *
483b005d3pjd * (3) While all this is happening, we inject faults into the pool to
493b005d3pjd *     verify that self-healing data really works.
503b005d3pjd *
513b005d3pjd * (4) Every time we open a dataset, we change its checksum and compression
523b005d3pjd *     functions.  Thus even individual objects vary from block to block
533b005d3pjd *     in which checksum they use and whether they're compressed.
543b005d3pjd *
553b005d3pjd * (5) To verify that we never lose on-disk consistency after a crash,
563b005d3pjd *     we run the entire test in a child of the main process.
573b005d3pjd *     At random times, the child self-immolates with a SIGKILL.
583b005d3pjd *     This is the software equivalent of pulling the power cord.
593b005d3pjd *     The parent then runs the test again, using the existing
60f95fd16delphij *     storage pool, as many times as desired. If backwards compatibility
616a061d7mm *     testing is enabled ztest will sometimes run the "older" version
626a061d7mm *     of ztest after a SIGKILL.
633b005d3pjd *
643b005d3pjd * (6) To verify that we don't have future leaks or temporal incursions,
653b005d3pjd *     many of the functional tests record the transaction group number
663b005d3pjd *     as part of their data.  When reading old data, they verify that
673b005d3pjd *     the transaction group number is less than the current, open txg.
683b005d3pjd *     If you add a new test, please do this if applicable.
693b005d3pjd *
703b005d3pjd * When run with no arguments, ztest runs for about five minutes and
713b005d3pjd * produces no output if successful.  To get a little bit of information,
723b005d3pjd * specify -V.  To get more information, specify -VV, and so on.
733b005d3pjd *
743b005d3pjd * To turn this into an overnight stress test, use -T to specify run time.
753b005d3pjd *
763b005d3pjd * You can ask more more vdevs [-v], datasets [-d], or threads [-t]
773b005d3pjd * to increase the pool capacity, fanout, and overall stress level.
783b005d3pjd *
796a061d7mm * Use the -k option to set the desired frequency of kills.
806a061d7mm *
816a061d7mm * When ztest invokes itself it passes all relevant information through a
826a061d7mm * temporary file which is mmap-ed in the child process. This allows shared
836a061d7mm * memory to survive the exec syscall. The ztest_shared_hdr_t struct is always
846a061d7mm * stored at offset 0 of this file and contains information on the size and
856a061d7mm * number of shared structures in the file. The information stored in this file
866a061d7mm * must remain backwards compatible with older versions of ztest so that
876a061d7mm * ztest can invoke them during backwards compatibility testing (-B).
883b005d3pjd */
893b005d3pjd
903b005d3pjd#include <sys/zfs_context.h>
913b005d3pjd#include <sys/spa.h>
923b005d3pjd#include <sys/dmu.h>
933b005d3pjd#include <sys/txg.h>
94b2946e8mm#include <sys/dbuf.h>
953b005d3pjd#include <sys/zap.h>
963b005d3pjd#include <sys/dmu_objset.h>
973b005d3pjd#include <sys/poll.h>
983b005d3pjd#include <sys/stat.h>
993b005d3pjd#include <sys/time.h>
1003b005d3pjd#include <sys/wait.h>
1013b005d3pjd#include <sys/mman.h>
1023b005d3pjd#include <sys/resource.h>
1033b005d3pjd#include <sys/zio.h>
1043b005d3pjd#include <sys/zil.h>
1051b03c5bpjd#include <sys/zil_impl.h>
1063b005d3pjd#include <sys/vdev_impl.h>
107bbe899bpjd#include <sys/vdev_file.h>
1087bffd76mav#include <sys/vdev_initialize.h>
1093b005d3pjd#include <sys/spa_impl.h>
1101b03c5bpjd#include <sys/metaslab_impl.h>
1113b005d3pjd#include <sys/dsl_prop.h>
112b04969bmm#include <sys/dsl_dataset.h>
1137c87858mm#include <sys/dsl_destroy.h>
1141b03c5bpjd#include <sys/dsl_scan.h>
1151b03c5bpjd#include <sys/zio_checksum.h>
1163b005d3pjd#include <sys/refcount.h>
117cc61ab2mm#include <sys/zfeature.h>
1187c87858mm#include <sys/dsl_userhold.h>
1196abbaacavg#include <sys/abd.h>
1203b005d3pjd#include <stdio.h>
1213b005d3pjd#include <stdio_ext.h>
1223b005d3pjd#include <stdlib.h>
1233b005d3pjd#include <unistd.h>
1243b005d3pjd#include <signal.h>
1253b005d3pjd#include <umem.h>
1263b005d3pjd#include <dlfcn.h>
1273b005d3pjd#include <ctype.h>
1283b005d3pjd#include <math.h>
1293b005d3pjd#include <errno.h>
1303b005d3pjd#include <sys/fs/zfs.h>
1311b03c5bpjd#include <libnvpair.h>
132f3b212bavg#include <libzfs.h>
1337c825baavg#include <libcmdutils.h>
1343b005d3pjd
1358e1fdecdelphijstatic int ztest_fd_data = -1;
1368e1fdecdelphijstatic int ztest_fd_rand = -1;
1376a061d7mm
1386a061d7mmtypedef struct ztest_shared_hdr {
1396a061d7mm	uint64_t	zh_hdr_size;
1406a061d7mm	uint64_t	zh_opts_size;
1416a061d7mm	uint64_t	zh_size;
1426a061d7mm	uint64_t	zh_stats_size;
1436a061d7mm	uint64_t	zh_stats_count;
1446a061d7mm	uint64_t	zh_ds_size;
1456a061d7mm	uint64_t	zh_ds_count;
1466a061d7mm} ztest_shared_hdr_t;
1476a061d7mm
1486a061d7mmstatic ztest_shared_hdr_t *ztest_shared_hdr;
1496a061d7mm
1508f9d694avgenum ztest_class_state {
1518f9d694avg	ZTEST_VDEV_CLASS_OFF,
1528f9d694avg	ZTEST_VDEV_CLASS_ON,
1538f9d694avg	ZTEST_VDEV_CLASS_RND
1548f9d694avg};
1558f9d694avg
1566a061d7mmtypedef struct ztest_shared_opts {
157ae8d156mav	char zo_pool[ZFS_MAX_DATASET_NAME_LEN];
158ae8d156mav	char zo_dir[ZFS_MAX_DATASET_NAME_LEN];
1596a061d7mm	char zo_alt_ztest[MAXNAMELEN];
1606a061d7mm	char zo_alt_libpath[MAXNAMELEN];
1616a061d7mm	uint64_t zo_vdevs;
1626a061d7mm	uint64_t zo_vdevtime;
1636a061d7mm	size_t zo_vdev_size;
1646a061d7mm	int zo_ashift;
1656a061d7mm	int zo_mirrors;
1666a061d7mm	int zo_raidz;
1676a061d7mm	int zo_raidz_parity;
1686a061d7mm	int zo_datasets;
1696a061d7mm	int zo_threads;
1706a061d7mm	uint64_t zo_passtime;
1716a061d7mm	uint64_t zo_killrate;
1726a061d7mm	int zo_verbose;
1736a061d7mm	int zo_init;
1746a061d7mm	uint64_t zo_time;
1756a061d7mm	uint64_t zo_maxloops;
176d31db32mav	uint64_t zo_metaslab_force_ganging;
177f3b212bavg	int zo_mmp_test;
1788f9d694avg	int zo_special_vdevs;
1796a061d7mm} ztest_shared_opts_t;
1806a061d7mm
1816a061d7mmstatic const ztest_shared_opts_t ztest_opts_defaults = {
1826a061d7mm	.zo_pool = { 'z', 't', 'e', 's', 't', '\0' },
1836a061d7mm	.zo_dir = { '/', 't', 'm', 'p', '\0' },
1846a061d7mm	.zo_alt_ztest = { '\0' },
1856a061d7mm	.zo_alt_libpath = { '\0' },
1866a061d7mm	.zo_vdevs = 5,
1876a061d7mm	.zo_ashift = SPA_MINBLOCKSHIFT,
1886a061d7mm	.zo_mirrors = 2,
1896a061d7mm	.zo_raidz = 4,
1906a061d7mm	.zo_raidz_parity = 1,
191ee44db6mav	.zo_vdev_size = SPA_MINDEVSIZE * 4,	/* 256m default size */
1926a061d7mm	.zo_datasets = 7,
1936a061d7mm	.zo_threads = 23,
1946a061d7mm	.zo_passtime = 60,		/* 60 seconds */
1956a061d7mm	.zo_killrate = 70,		/* 70% kill rate */
1966a061d7mm	.zo_verbose = 0,
197f3b212bavg	.zo_mmp_test = 0,
1986a061d7mm	.zo_init = 1,
1996a061d7mm	.zo_time = 300,			/* 5 minutes */
2006a061d7mm	.zo_maxloops = 50,		/* max loops during spa_freeze() */
2018f9d694avg	.zo_metaslab_force_ganging = 32 << 10,
2028f9d694avg	.zo_special_vdevs = ZTEST_VDEV_CLASS_RND,
2036a061d7mm};
2046a061d7mm
205d31db32mavextern uint64_t metaslab_force_ganging;
2066a061d7mmextern uint64_t metaslab_df_alloc_threshold;
20751e896cavgextern uint64_t zfs_deadman_synctime_ms;
2080b2372bdelphijextern int metaslab_preload_limit;
209e7f8bc5mavextern boolean_t zfs_compressed_arc_enabled;
2106abbaacavgextern boolean_t zfs_abd_scatter_enabled;
211bd8e9d1avgextern int dmu_object_alloc_chunk_shift;
2128e1f6camavextern boolean_t zfs_force_some_double_word_sm_entries;
213935d5eeavgextern unsigned long zfs_reconstruct_indirect_damage_fraction;
2146a061d7mm
2156a061d7mmstatic ztest_shared_opts_t *ztest_shared_opts;
2166a061d7mmstatic ztest_shared_opts_t ztest_opts;
2176a061d7mm
2186a061d7mmtypedef struct ztest_shared_ds {
2196a061d7mm	uint64_t	zd_seq;
2206a061d7mm} ztest_shared_ds_t;
2216a061d7mm
2226a061d7mmstatic ztest_shared_ds_t *ztest_shared_ds;
2236a061d7mm#define	ZTEST_GET_SHARED_DS(d) (&ztest_shared_ds[d])
2241b03c5bpjd
2251b03c5bpjd#define	BT_MAGIC	0x123456789abcdefULL
2266a061d7mm#define	MAXFAULTS() \
2276a061d7mm	(MAX(zs->zs_mirrors, 1) * (ztest_opts.zo_raidz_parity + 1) - 1)
2281b03c5bpjd
2291b03c5bpjdenum ztest_io_type {
2301b03c5bpjd	ZTEST_IO_WRITE_TAG,
2311b03c5bpjd	ZTEST_IO_WRITE_PATTERN,
2321b03c5bpjd	ZTEST_IO_WRITE_ZEROES,
2331b03c5bpjd	ZTEST_IO_TRUNCATE,
2341b03c5bpjd	ZTEST_IO_SETATTR,
2353a0bfecmm	ZTEST_IO_REWRITE,
2361b03c5bpjd	ZTEST_IO_TYPES
2371b03c5bpjd};
2383b005d3pjd
239bbe899bpjdtypedef struct ztest_block_tag {
2401b03c5bpjd	uint64_t	bt_magic;
241bbe899bpjd	uint64_t	bt_objset;
242bbe899bpjd	uint64_t	bt_object;
24346cdbe4mmacy	uint64_t	bt_dnodesize;
244bbe899bpjd	uint64_t	bt_offset;
2451b03c5bpjd	uint64_t	bt_gen;
246bbe899bpjd	uint64_t	bt_txg;
2471b03c5bpjd	uint64_t	bt_crtxg;
248bbe899bpjd} ztest_block_tag_t;
249bbe899bpjd
2501b03c5bpjdtypedef struct bufwad {
2511b03c5bpjd	uint64_t	bw_index;
2521b03c5bpjd	uint64_t	bw_txg;
2531b03c5bpjd	uint64_t	bw_data;
2541b03c5bpjd} bufwad_t;
2551b03c5bpjd
2561b03c5bpjd/*
2574cc8429avg * It would be better to use a rangelock_t per object.  Unfortunately
2584cc8429avg * the rangelock_t is not a drop-in replacement for rl_t, because we
2594cc8429avg * still need to map from object ID to rangelock_t.
2601b03c5bpjd */
2611b03c5bpjdtypedef enum {
2621b03c5bpjd	RL_READER,
2631b03c5bpjd	RL_WRITER,
2641b03c5bpjd	RL_APPEND
2651b03c5bpjd} rl_type_t;
2661b03c5bpjd
2671b03c5bpjdtypedef struct rll {
2681b03c5bpjd	void		*rll_writer;
2691b03c5bpjd	int		rll_readers;
2700014145mav	kmutex_t	rll_lock;
2710014145mav	kcondvar_t	rll_cv;
2721b03c5bpjd} rll_t;
2731b03c5bpjd
2741b03c5bpjdtypedef struct rl {
2751b03c5bpjd	uint64_t	rl_object;
2761b03c5bpjd	uint64_t	rl_offset;
2771b03c5bpjd	uint64_t	rl_size;
2781b03c5bpjd	rll_t		*rl_lock;
2791b03c5bpjd} rl_t;
2801b03c5bpjd
2811b03c5bpjd#define	ZTEST_RANGE_LOCKS	64
2821b03c5bpjd#define	ZTEST_OBJECT_LOCKS	64
2831b03c5bpjd
2841b03c5bpjd/*
2851b03c5bpjd * Object descriptor.  Used as a template for object lookup/create/remove.
2861b03c5bpjd */
2871b03c5bpjdtypedef struct ztest_od {
2881b03c5bpjd	uint64_t	od_dir;
2891b03c5bpjd	uint64_t	od_object;
2901b03c5bpjd	dmu_object_type_t od_type;
2911b03c5bpjd	dmu_object_type_t od_crtype;
2921b03c5bpjd	uint64_t	od_blocksize;
2931b03c5bpjd	uint64_t	od_crblocksize;
29446cdbe4mmacy	uint64_t	od_crdnodesize;
2951b03c5bpjd	uint64_t	od_gen;
2961b03c5bpjd	uint64_t	od_crgen;
297ae8d156mav	char		od_name[ZFS_MAX_DATASET_NAME_LEN];
2981b03c5bpjd} ztest_od_t;
2991b03c5bpjd
3001b03c5bpjd/*
3011b03c5bpjd * Per-dataset state.
3021b03c5bpjd */
3031b03c5bpjdtypedef struct ztest_ds {
3046a061d7mm	ztest_shared_ds_t *zd_shared;
3051b03c5bpjd	objset_t	*zd_os;
3060014145mav	krwlock_t	zd_zilog_lock;
3071b03c5bpjd	zilog_t		*zd_zilog;
3081b03c5bpjd	ztest_od_t	*zd_od;		/* debugging aid */
309ae8d156mav	char		zd_name[ZFS_MAX_DATASET_NAME_LEN];
3100014145mav	kmutex_t	zd_dirobj_lock;
3111b03c5bpjd	rll_t		zd_object_lock[ZTEST_OBJECT_LOCKS];
3121b03c5bpjd	rll_t		zd_range_lock[ZTEST_RANGE_LOCKS];
3131b03c5bpjd} ztest_ds_t;
3141b03c5bpjd
3151b03c5bpjd/*
3161b03c5bpjd * Per-iteration state.
3171b03c5bpjd */
3181b03c5bpjdtypedef void ztest_func_t(ztest_ds_t *zd, uint64_t id);
3191b03c5bpjd
3201b03c5bpjdtypedef struct ztest_info {
3211b03c5bpjd	ztest_func_t	*zi_func;	/* test function */
3221b03c5bpjd	uint64_t	zi_iters;	/* iterations per execution */
3231b03c5bpjd	uint64_t	*zi_interval;	/* execute every <interval> seconds */
3241b03c5bpjd} ztest_info_t;
3253b005d3pjd
3266a061d7mmtypedef struct ztest_shared_callstate {
3276a061d7mm	uint64_t	zc_count;	/* per-pass count */
3286a061d7mm	uint64_t	zc_time;	/* per-pass time */
3296a061d7mm	uint64_t	zc_next;	/* next time to call this function */
3306a061d7mm} ztest_shared_callstate_t;
3316a061d7mm
3326a061d7mmstatic ztest_shared_callstate_t *ztest_shared_callstate;
3336a061d7mm#define	ZTEST_GET_SHARED_CALLSTATE(c) (&ztest_shared_callstate[c])
3346a061d7mm
3353b005d3pjd/*
3363b005d3pjd * Note: these aren't static because we want dladdr() to work.
3373b005d3pjd */
3383b005d3pjdztest_func_t ztest_dmu_read_write;
3393b005d3pjdztest_func_t ztest_dmu_write_parallel;
3403b005d3pjdztest_func_t ztest_dmu_object_alloc_free;
341bd8e9d1avgztest_func_t ztest_dmu_object_next_chunk;
3421b03c5bpjdztest_func_t ztest_dmu_commit_callbacks;
3433b005d3pjdztest_func_t ztest_zap;
3443b005d3pjdztest_func_t ztest_zap_parallel;
3451b03c5bpjdztest_func_t ztest_zil_commit;
346f2d210fmmztest_func_t ztest_zil_remount;
3471b03c5bpjdztest_func_t ztest_dmu_read_write_zcopy;
3483b005d3pjdztest_func_t ztest_dmu_objset_create_destroy;
3491b03c5bpjdztest_func_t ztest_dmu_prealloc;
3501b03c5bpjdztest_func_t ztest_fzap;
3513b005d3pjdztest_func_t ztest_dmu_snapshot_create_destroy;
3521b03c5bpjdztest_func_t ztest_dsl_prop_get_set;
3531b03c5bpjdztest_func_t ztest_spa_prop_get_set;
3543b005d3pjdztest_func_t ztest_spa_create_destroy;
3553b005d3pjdztest_func_t ztest_fault_inject;
3561b03c5bpjdztest_func_t ztest_ddt_repair;
3571b03c5bpjdztest_func_t ztest_dmu_snapshot_hold;
358f3b212bavgztest_func_t ztest_mmp_enable_disable;
3591b03c5bpjdztest_func_t ztest_scrub;
3601b03c5bpjdztest_func_t ztest_dsl_dataset_promote_busy;
3613b005d3pjdztest_func_t ztest_vdev_attach_detach;
3623b005d3pjdztest_func_t ztest_vdev_LUN_growth;
3633b005d3pjdztest_func_t ztest_vdev_add_remove;
3648f9d694avgztest_func_t ztest_vdev_class_add;
365bbe899bpjdztest_func_t ztest_vdev_aux_add_remove;
3661b03c5bpjdztest_func_t ztest_split_pool;
36773cab71mmztest_func_t ztest_reguid;
36882387c4mmztest_func_t ztest_spa_upgrade;
3696da4389mavztest_func_t ztest_device_removal;
3706da4389mavztest_func_t ztest_remap_blocks;
371b1ec8f2mavztest_func_t ztest_spa_checkpoint_create_discard;
3727bffd76mavztest_func_t ztest_initialize;
37346cdbe4mmacyztest_func_t ztest_verify_dnode_bt;
3743b005d3pjd
3751b03c5bpjduint64_t zopt_always = 0ULL * NANOSEC;		/* all the time */
3761b03c5bpjduint64_t zopt_incessant = 1ULL * NANOSEC / 10;	/* every 1/10 second */
3771b03c5bpjduint64_t zopt_often = 1ULL * NANOSEC;		/* every second */
3781b03c5bpjduint64_t zopt_sometimes = 10ULL * NANOSEC;	/* every 10 seconds */
3791b03c5bpjduint64_t zopt_rarely = 60ULL * NANOSEC;		/* every 60 seconds */
3803b005d3pjd
3813b005d3pjdztest_info_t ztest_info[] = {
382bbe899bpjd	{ ztest_dmu_read_write,			1,	&zopt_always	},
3831b03c5bpjd	{ ztest_dmu_write_parallel,		10,	&zopt_always	},
384bbe899bpjd	{ ztest_dmu_object_alloc_free,		1,	&zopt_always	},
385bd8e9d1avg	{ ztest_dmu_object_next_chunk,		1,	&zopt_sometimes	},
3861b03c5bpjd	{ ztest_dmu_commit_callbacks,		1,	&zopt_always	},
387bbe899bpjd	{ ztest_zap,				30,	&zopt_always	},
388bbe899bpjd	{ ztest_zap_parallel,			100,	&zopt_always	},
3891b03c5bpjd	{ ztest_split_pool,			1,	&zopt_always	},
3901b03c5bpjd	{ ztest_zil_commit,			1,	&zopt_incessant	},
391f2d210fmm	{ ztest_zil_remount,			1,	&zopt_sometimes	},
3921b03c5bpjd	{ ztest_dmu_read_write_zcopy,		1,	&zopt_often	},
3931b03c5bpjd	{ ztest_dmu_objset_create_destroy,	1,	&zopt_often	},
3941b03c5bpjd	{ ztest_dsl_prop_get_set,		1,	&zopt_often	},
3951b03c5bpjd	{ ztest_spa_prop_get_set,		1,	&zopt_sometimes	},
3961b03c5bpjd#if 0
3971b03c5bpjd	{ ztest_dmu_prealloc,			1,	&zopt_sometimes	},
3981b03c5bpjd#endif
3991b03c5bpjd	{ ztest_fzap,				1,	&zopt_sometimes	},
4001b03c5bpjd	{ ztest_dmu_snapshot_create_destroy,	1,	&zopt_sometimes	},
4011b03c5bpjd	{ ztest_spa_create_destroy,		1,	&zopt_sometimes	},
402a73aecdsef	{ ztest_fault_inject,			1,	&zopt_incessant	},
4031b03c5bpjd	{ ztest_ddt_repair,			1,	&zopt_sometimes	},
4041b03c5bpjd	{ ztest_dmu_snapshot_hold,		1,	&zopt_sometimes	},
405f3b212bavg	{ ztest_mmp_enable_disable,		1,	&zopt_sometimes	},
4060171695delphij	{ ztest_reguid,				1,	&zopt_rarely	},
407a73aecdsef	{ ztest_scrub,				1,	&zopt_often	},
40882387c4mm	{ ztest_spa_upgrade,			1,	&zopt_rarely	},
409b2946e8mm	{ ztest_dsl_dataset_promote_busy,	1,	&zopt_rarely	},
410a73aecdsef	{ ztest_vdev_attach_detach,		1,	&zopt_incessant	},
4111b03c5bpjd	{ ztest_vdev_LUN_growth,		1,	&zopt_rarely	},
4126a061d7mm	{ ztest_vdev_add_remove,		1,
4136a061d7mm	    &ztest_opts.zo_vdevtime				},
4148f9d694avg	{ ztest_vdev_class_add,			1,
4158f9d694avg	    &ztest_opts.zo_vdevtime				},
4166a061d7mm	{ ztest_vdev_aux_add_remove,		1,
4176a061d7mm	    &ztest_opts.zo_vdevtime				},
4186da4389mav	{ ztest_device_removal,			1,	&zopt_sometimes	},
419b1ec8f2mav	{ ztest_remap_blocks,			1,	&zopt_sometimes },
4207bffd76mav	{ ztest_spa_checkpoint_create_discard,	1,	&zopt_rarely	},
42146cdbe4mmacy	{ ztest_initialize,			1,	&zopt_sometimes },
42246cdbe4mmacy	{ ztest_verify_dnode_bt,		1,	&zopt_sometimes }
4233b005d3pjd};
4243b005d3pjd
4253b005d3pjd#define	ZTEST_FUNCS	(sizeof (ztest_info) / sizeof (ztest_info_t))
4263b005d3pjd
4271b03c5bpjd/*
4281b03c5bpjd * The following struct is used to hold a list of uncalled commit callbacks.
4291b03c5bpjd * The callbacks are ordered by txg number.
4301b03c5bpjd */
4311b03c5bpjdtypedef struct ztest_cb_list {
4320014145mav	kmutex_t zcl_callbacks_lock;
4331b03c5bpjd	list_t	zcl_callbacks;
4341b03c5bpjd} ztest_cb_list_t;
4353b005d3pjd
4363b005d3pjd/*
437