xref: /illumos-gate/usr/src/uts/common/fs/zfs/zio_inject.c (revision b853d39a)
1ea8dc4b6Seschrock /*
2ea8dc4b6Seschrock  * CDDL HEADER START
3ea8dc4b6Seschrock  *
4ea8dc4b6Seschrock  * The contents of this file are subject to the terms of the
5ea8dc4b6Seschrock  * Common Development and Distribution License (the "License").
6ea8dc4b6Seschrock  * You may not use this file except in compliance with the License.
7ea8dc4b6Seschrock  *
8ea8dc4b6Seschrock  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9ea8dc4b6Seschrock  * or http://www.opensolaris.org/os/licensing.
10ea8dc4b6Seschrock  * See the License for the specific language governing permissions
11ea8dc4b6Seschrock  * and limitations under the License.
12ea8dc4b6Seschrock  *
13ea8dc4b6Seschrock  * When distributing Covered Code, include this CDDL HEADER in each
14ea8dc4b6Seschrock  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15ea8dc4b6Seschrock  * If applicable, add the following below this CDDL HEADER, with the
16ea8dc4b6Seschrock  * fields enclosed by brackets "[]" replaced with your own identifying
17ea8dc4b6Seschrock  * information: Portions Copyright [yyyy] [name of copyright owner]
18ea8dc4b6Seschrock  *
19ea8dc4b6Seschrock  * CDDL HEADER END
20ea8dc4b6Seschrock  */
21ea8dc4b6Seschrock /*
2298d1cbfeSGeorge Wilson  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
2397e81309SPrakash Surya  * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
24*b853d39aSDon Brady  * Copyright (c) 2017, Intel Corporation.
25ea8dc4b6Seschrock  */
26ea8dc4b6Seschrock 
27ea8dc4b6Seschrock /*
28ea8dc4b6Seschrock  * ZFS fault injection
29ea8dc4b6Seschrock  *
30ea8dc4b6Seschrock  * To handle fault injection, we keep track of a series of zinject_record_t
31ea8dc4b6Seschrock  * structures which describe which logical block(s) should be injected with a
32ea8dc4b6Seschrock  * fault.  These are kept in a global list.  Each record corresponds to a given
33ea8dc4b6Seschrock  * spa_t and maintains a special hold on the spa_t so that it cannot be deleted
34ea8dc4b6Seschrock  * or exported while the injection record exists.
35ea8dc4b6Seschrock  *
36ea8dc4b6Seschrock  * Device level injection is done using the 'zi_guid' field.  If this is set, it
37ea8dc4b6Seschrock  * means that the error is destined for a particular device, not a piece of
38ea8dc4b6Seschrock  * data.
39ea8dc4b6Seschrock  *
40ea8dc4b6Seschrock  * This is a rather poor data structure and algorithm, but we don't expect more
41ea8dc4b6Seschrock  * than a few faults at any one time, so it should be sufficient for our needs.
42ea8dc4b6Seschrock  */
43ea8dc4b6Seschrock 
44ea8dc4b6Seschrock #include <sys/arc.h>
45ea8dc4b6Seschrock #include <sys/zio_impl.h>
46ea8dc4b6Seschrock #include <sys/zfs_ioctl.h>
47ea8dc4b6Seschrock #include <sys/vdev_impl.h>
48b24ab676SJeff Bonwick #include <sys/dmu_objset.h>
49d8ab6e12SDon Brady #include <sys/dsl_dataset.h>
5021bf64a7Sgw #include <sys/fs/zfs.h>
51ea8dc4b6Seschrock 
52d8ab6e12SDon Brady uint32_t zio_injection_enabled = 0;
53ea8dc4b6Seschrock 
5497e81309SPrakash Surya /*
5597e81309SPrakash Surya  * Data describing each zinject handler registered on the system, and
5697e81309SPrakash Surya  * contains the list node linking the handler in the global zinject
5797e81309SPrakash Surya  * handler list.
5897e81309SPrakash Surya  */
59ea8dc4b6Seschrock typedef struct inject_handler {
60ea8dc4b6Seschrock 	int			zi_id;
61ea8dc4b6Seschrock 	spa_t			*zi_spa;
62ea8dc4b6Seschrock 	zinject_record_t	zi_record;
6397e81309SPrakash Surya 	uint64_t		*zi_lanes;
6497e81309SPrakash Surya 	int			zi_next_lane;
65ea8dc4b6Seschrock 	list_node_t		zi_link;
66ea8dc4b6Seschrock } inject_handler_t;
67ea8dc4b6Seschrock 
6897e81309SPrakash Surya /*
6997e81309SPrakash Surya  * List of all zinject handlers registered on the system, protected by
7097e81309SPrakash Surya  * the inject_lock defined below.
7197e81309SPrakash Surya  */
72ea8dc4b6Seschrock static list_t inject_handlers;
7397e81309SPrakash Surya 
7497e81309SPrakash Surya /*
7597e81309SPrakash Surya  * This protects insertion into, and traversal of, the inject handler
7697e81309SPrakash Surya  * list defined above; as well as the inject_delay_count. Any time a
7797e81309SPrakash Surya  * handler is inserted or removed from the list, this lock should be
7897e81309SPrakash Surya  * taken as a RW_WRITER; and any time traversal is done over the list
7997e81309SPrakash Surya  * (without modification to it) this lock should be taken as a RW_READER.
8097e81309SPrakash Surya  */
81ea8dc4b6Seschrock static krwlock_t inject_lock;
8297e81309SPrakash Surya 
8397e81309SPrakash Surya /*
8497e81309SPrakash Surya  * This holds the number of zinject delay handlers that have been
8597e81309SPrakash Surya  * registered on the system. It is protected by the inject_lock defined
8697e81309SPrakash Surya  * above. Thus modifications to this count must be a RW_WRITER of the
8797e81309SPrakash Surya  * inject_lock, and reads of this count must be (at least) a RW_READER
8897e81309SPrakash Surya  * of the lock.
8997e81309SPrakash Surya  */
9097e81309SPrakash Surya static int inject_delay_count = 0;
9197e81309SPrakash Surya 
9297e81309SPrakash Surya /*
9397e81309SPrakash Surya  * This lock is used only in zio_handle_io_delay(), refer to the comment
9497e81309SPrakash Surya  * in that function for more details.
9597e81309SPrakash Surya  */
9697e81309SPrakash Surya static kmutex_t inject_delay_mtx;
9797e81309SPrakash Surya 
9897e81309SPrakash Surya /*
9997e81309SPrakash Surya  * Used to assign unique identifying numbers to each new zinject handler.
10097e81309SPrakash Surya  */
101ea8dc4b6Seschrock static int inject_next_id = 1;
102ea8dc4b6Seschrock 
103*b853d39aSDon Brady /*
104*b853d39aSDon Brady  * Test if the requested frequency was triggered
105*b853d39aSDon Brady  */
106*b853d39aSDon Brady static boolean_t
freq_triggered(uint32_t frequency)107*b853d39aSDon Brady freq_triggered(uint32_t frequency)
108*b853d39aSDon Brady {
109*b853d39aSDon Brady 	/*
110*b853d39aSDon Brady 	 * zero implies always (100%)
111*b853d39aSDon Brady 	 */
112*b853d39aSDon Brady 	if (frequency == 0)
113*b853d39aSDon Brady 		return (B_TRUE);
114*b853d39aSDon Brady 
115*b853d39aSDon Brady 	/*
116*b853d39aSDon Brady 	 * Note: we still handle legacy (unscaled) frequecy values
117*b853d39aSDon Brady 	 */
118*b853d39aSDon Brady 	uint32_t maximum = (frequency <= 100) ? 100 : ZI_PERCENTAGE_MAX;
119*b853d39aSDon Brady 
120*b853d39aSDon Brady 	return (spa_get_random(maximum) < frequency);
121*b853d39aSDon Brady }
122*b853d39aSDon Brady 
123ea8dc4b6Seschrock /*
124ea8dc4b6Seschrock  * Returns true if the given record matches the I/O in progress.
125ea8dc4b6Seschrock  */
126ea8dc4b6Seschrock static boolean_t
zio_match_handler(zbookmark_phys_t * zb,uint64_t type,int dva,zinject_record_t * record,int error)12712a8814cSTom Caputi zio_match_handler(zbookmark_phys_t *zb, uint64_t type, int dva,
128ea8dc4b6Seschrock     zinject_record_t *record, int error)
129ea8dc4b6Seschrock {
130ea8dc4b6Seschrock 	/*
131ea8dc4b6Seschrock 	 * Check for a match against the MOS, which is based on type
132ea8dc4b6Seschrock 	 */
133b24ab676SJeff Bonwick 	if (zb->zb_objset == DMU_META_OBJSET &&
134b24ab676SJeff Bonwick 	    record->zi_objset == DMU_META_OBJSET &&
135b24ab676SJeff Bonwick 	    record->zi_object == DMU_META_DNODE_OBJECT) {
136ea8dc4b6Seschrock 		if (record->zi_type == DMU_OT_NONE ||
137ea8dc4b6Seschrock 		    type == record->zi_type)
138*b853d39aSDon Brady 			return (freq_triggered(record->zi_freq));
139ea8dc4b6Seschrock 		else
140ea8dc4b6Seschrock 			return (B_FALSE);
141ea8dc4b6Seschrock 	}
142ea8dc4b6Seschrock 
143ea8dc4b6Seschrock 	/*
144ea8dc4b6Seschrock 	 * Check for an exact match.
145ea8dc4b6Seschrock 	 */
146ea8dc4b6Seschrock 	if (zb->zb_objset == record->zi_objset &&
147ea8dc4b6Seschrock 	    zb->zb_object == record->zi_object &&
148ea8dc4b6Seschrock 	    zb->zb_level == record->zi_level &&
149ea8dc4b6Seschrock 	    zb->zb_blkid >= record->zi_start &&
150ea8dc4b6Seschrock 	    zb->zb_blkid <= record->zi_end &&
15112a8814cSTom Caputi 	    (record->zi_dvas == 0 || (record->zi_dvas & (1ULL << dva))) &&
15212a8814cSTom Caputi 	    error == record->zi_error) {
153*b853d39aSDon Brady 		return (freq_triggered(record->zi_freq));
15412a8814cSTom Caputi 	}
155ea8dc4b6Seschrock 
156ea8dc4b6Seschrock 	return (B_FALSE);
157ea8dc4b6Seschrock }
158ea8dc4b6Seschrock 
15988ecc943SGeorge Wilson /*
16088ecc943SGeorge Wilson  * Panic the system when a config change happens in the function
16188ecc943SGeorge Wilson  * specified by tag.
16288ecc943SGeorge Wilson  */
16388ecc943SGeorge Wilson void
zio_handle_panic_injection(spa_t * spa,char * tag,uint64_t type)1641195e687SMark J Musante zio_handle_panic_injection(spa_t *spa, char *tag, uint64_t type)
16588ecc943SGeorge Wilson {
16688ecc943SGeorge Wilson 	inject_handler_t *handler;
16788ecc943SGeorge Wilson 
16888ecc943SGeorge Wilson 	rw_enter(&inject_lock, RW_READER);
16988ecc943SGeorge Wilson 
17088ecc943SGeorge Wilson 	for (handler = list_head(&inject_handlers); handler != NULL;
17188ecc943SGeorge Wilson 	    handler = list_next(&inject_handlers, handler)) {
17288ecc943SGeorge Wilson 
17388ecc943SGeorge Wilson 		if (spa != handler->zi_spa)
17488ecc943SGeorge Wilson 			continue;
17588ecc943SGeorge Wilson 
1761195e687SMark J Musante 		if (handler->zi_record.zi_type == type &&
1771195e687SMark J Musante 		    strcmp(tag, handler->zi_record.zi_func) == 0)
17888ecc943SGeorge Wilson 			panic("Panic requested in function %s\n", tag);
17988ecc943SGeorge Wilson 	}
18088ecc943SGeorge Wilson 
18188ecc943SGeorge Wilson 	rw_exit(&inject_lock);
18288ecc943SGeorge Wilson }
18388ecc943SGeorge Wilson 
18412a8814cSTom Caputi 
18512a8814cSTom Caputi /*
18612a8814cSTom Caputi  * If this is a physical I/O for a vdev child determine which DVA it is
18712a8814cSTom Caputi  * for. We iterate backwards through the DVAs matching on the offset so
18812a8814cSTom Caputi  * that we end up with ZI_NO_DVA (-1) if we don't find a match.
18912a8814cSTom Caputi  */
19012a8814cSTom Caputi static int
zio_match_dva(zio_t * zio)19112a8814cSTom Caputi zio_match_dva(zio_t *zio)
19212a8814cSTom Caputi {
19312a8814cSTom Caputi 	int i = ZI_NO_DVA;
19412a8814cSTom Caputi 
19512a8814cSTom Caputi 	if (zio->io_bp != NULL && zio->io_vd != NULL &&
19612a8814cSTom Caputi 	    zio->io_child_type == ZIO_CHILD_VDEV) {
19712a8814cSTom Caputi 		for (i = BP_GET_NDVAS(zio->io_bp) - 1; i >= 0; i--) {
19812a8814cSTom Caputi 			dva_t *dva = &zio->io_bp->blk_dva[i];
19912a8814cSTom Caputi 			uint64_t off = DVA_GET_OFFSET(dva);
20012a8814cSTom Caputi 			vdev_t *vd = vdev_lookup_top(zio->io_spa,
20112a8814cSTom Caputi 			    DVA_GET_VDEV(dva));
20212a8814cSTom Caputi 
20312a8814cSTom Caputi 			/* Compensate for vdev label added to leaves */
20412a8814cSTom Caputi 			if (zio->io_vd->vdev_ops->vdev_op_leaf)
20512a8814cSTom Caputi 				off += VDEV_LABEL_START_SIZE;
20612a8814cSTom Caputi 
20712a8814cSTom Caputi 			if (zio->io_vd == vd && zio->io_offset == off)
20812a8814cSTom Caputi 				break;
20912a8814cSTom Caputi 		}
21012a8814cSTom Caputi 	}
21112a8814cSTom Caputi 
21212a8814cSTom Caputi 	return (i);
21312a8814cSTom Caputi }
21412a8814cSTom Caputi 
21512a8814cSTom Caputi 
216eb633035STom Caputi /*
217eb633035STom Caputi  * Inject a decryption failure. Decryption failures can occur in
218eb633035STom Caputi  * both the ARC and the ZIO layers.
219eb633035STom Caputi  */
220eb633035STom Caputi int
zio_handle_decrypt_injection(spa_t * spa,const zbookmark_phys_t * zb,uint64_t type,int error)221eb633035STom Caputi zio_handle_decrypt_injection(spa_t *spa, const zbookmark_phys_t *zb,
222eb633035STom Caputi     uint64_t type, int error)
223eb633035STom Caputi {
224eb633035STom Caputi 	int ret = 0;
225eb633035STom Caputi 	inject_handler_t *handler;
226eb633035STom Caputi 
227eb633035STom Caputi 	rw_enter(&inject_lock, RW_READER);
228eb633035STom Caputi 
229eb633035STom Caputi 	for (handler = list_head(&inject_handlers); handler != NULL;
230eb633035STom Caputi 	    handler = list_next(&inject_handlers, handler)) {
231eb633035STom Caputi 
232eb633035STom Caputi 		if (spa != handler->zi_spa ||
233eb633035STom Caputi 		    handler->zi_record.zi_cmd != ZINJECT_DECRYPT_FAULT)
234eb633035STom Caputi 			continue;
235eb633035STom Caputi 
236eb633035STom Caputi 		if (zio_match_handler((zbookmark_phys_t *)zb, type, ZI_NO_DVA,
237eb633035STom Caputi 		    &handler->zi_record, error)) {
238eb633035STom Caputi 			ret = error;
239eb633035STom Caputi 			break;
240eb633035STom Caputi 		}
241eb633035STom Caputi 	}
242eb633035STom Caputi 
243eb633035STom Caputi 	rw_exit(&inject_lock);
244eb633035STom Caputi 	return (ret);
245eb633035STom Caputi }
246eb633035STom Caputi 
247ea8dc4b6Seschrock /*
248ea8dc4b6Seschrock  * Determine if the I/O in question should return failure.  Returns the errno
249ea8dc4b6Seschrock  * to be returned to the caller.
250ea8dc4b6Seschrock  */
251ea8dc4b6Seschrock int
zio_handle_fault_injection(zio_t * zio,int error)252ea8dc4b6Seschrock zio_handle_fault_injection(zio_t *zio, int error)
253ea8dc4b6Seschrock {
254ea8dc4b6Seschrock 	int ret = 0;
255ea8dc4b6Seschrock 	inject_handler_t *handler;
256ea8dc4b6Seschrock 
257ea8dc4b6Seschrock 	/*
258ea8dc4b6Seschrock 	 * Ignore I/O not associated with any logical data.
259ea8dc4b6Seschrock 	 */
260ea8dc4b6Seschrock 	if (zio->io_logical == NULL)
261ea8dc4b6Seschrock 		return (0);
262ea8dc4b6Seschrock 
263ea8dc4b6Seschrock 	/*
264ea8dc4b6Seschrock 	 * Currently, we only support fault injection on reads.
265ea8dc4b6Seschrock 	 */
266ea8dc4b6Seschrock 	if (zio->io_type != ZIO_TYPE_READ)
267ea8dc4b6Seschrock 		return (0);
268ea8dc4b6Seschrock 
269ea8dc4b6Seschrock 	rw_enter(&inject_lock, RW_READER);
270ea8dc4b6Seschrock 
271ea8dc4b6Seschrock 	for (handler = list_head(&inject_handlers); handler != NULL;
272ea8dc4b6Seschrock 	    handler = list_next(&inject_handlers, handler)) {
273ea8dc4b6Seschrock 
274283b8460SGeorge.Wilson 		if (zio->io_spa != handler->zi_spa ||
275283b8460SGeorge.Wilson 		    handler->zi_record.zi_cmd != ZINJECT_DATA_FAULT)
276ea8dc4b6Seschrock 			continue;
277ea8dc4b6Seschrock 
27812a8814cSTom Caputi 		/* If this handler matches, return the specified error */
279ea8dc4b6Seschrock 		if (zio_match_handler(&zio->io_logical->io_bookmark,
280ea8dc4b6Seschrock 		    zio->io_bp ? BP_GET_TYPE(zio->io_bp) : DMU_OT_NONE,
28112a8814cSTom Caputi 		    zio_match_dva(zio), &handler->zi_record, error)) {
282ea8dc4b6Seschrock 			ret = error;
283ea8dc4b6Seschrock 			break;
284ea8dc4b6Seschrock 		}
285ea8dc4b6Seschrock 	}
286ea8dc4b6Seschrock 
287ea8dc4b6Seschrock 	rw_exit(&inject_lock);
288ea8dc4b6Seschrock 
289ea8dc4b6Seschrock 	return (ret);
290ea8dc4b6Seschrock }
291ea8dc4b6Seschrock 
29221bf64a7Sgw /*
29321bf64a7Sgw  * Determine if the zio is part of a label update and has an injection
29421bf64a7Sgw  * handler associated with that portion of the label. Currently, we
29521bf64a7Sgw  * allow error injection in either the nvlist or the uberblock region of
29621bf64a7Sgw  * of the vdev label.
29721bf64a7Sgw  */
29821bf64a7Sgw int
zio_handle_label_injection(zio_t * zio,int error)29921bf64a7Sgw zio_handle_label_injection(zio_t *zio, int error)
30021bf64a7Sgw {
30121bf64a7Sgw 	inject_handler_t *handler;
30221bf64a7Sgw 	vdev_t *vd = zio->io_vd;
30321bf64a7Sgw 	uint64_t offset = zio->io_offset;
30421bf64a7Sgw 	int label;
30521bf64a7Sgw 	int ret = 0;
30621bf64a7Sgw 
3078f18d1faSGeorge Wilson 	if (offset >= VDEV_LABEL_START_SIZE &&
30821bf64a7Sgw 	    offset < vd->vdev_psize - VDEV_LABEL_END_SIZE)
30921bf64a7Sgw 		return (0);
31021bf64a7Sgw 
31121bf64a7Sgw 	rw_enter(&inject_lock, RW_READER);
31221bf64a7Sgw 
31321bf64a7Sgw 	for (handler = list_head(&inject_handlers); handler != NULL;
31421bf64a7Sgw 	    handler = list_next(&inject_handlers, handler)) {
31521bf64a7Sgw 		uint64_t start = handler->zi_record.zi_start;
31621bf64a7Sgw 		uint64_t end = handler->zi_record.zi_end;
31721bf64a7Sgw 
318283b8460SGeorge.Wilson 		if (handler->zi_record.zi_cmd != ZINJECT_LABEL_FAULT)
31921bf64a7Sgw 			continue;
32021bf64a7Sgw 
32121bf64a7Sgw 		/*
32221bf64a7Sgw 		 * The injection region is the relative offsets within a
32321bf64a7Sgw 		 * vdev label. We must determine the label which is being
32421bf64a7Sgw 		 * updated and adjust our region accordingly.
32521bf64a7Sgw 		 */
32621bf64a7Sgw 		label = vdev_label_number(vd->vdev_psize, offset);
32721bf64a7Sgw 		start = vdev_label_offset(vd->vdev_psize, label, start);
32821bf64a7Sgw 		end = vdev_label_offset(vd->vdev_psize, label, end);
32921bf64a7Sgw 
33021bf64a7Sgw 		if (zio->io_vd->vdev_guid == handler->zi_record.zi_guid &&
33121bf64a7Sgw 		    (offset >= start && offset <= end)) {
33221bf64a7Sgw 			ret = error;
33321bf64a7Sgw 			break;
33421bf64a7Sgw 		}
33521bf64a7Sgw 	}
33621bf64a7Sgw 	rw_exit(&inject_lock);
33721bf64a7Sgw 	return (ret);
33821bf64a7Sgw }
33921bf64a7Sgw 
34021bf64a7Sgw 
341ea8dc4b6Seschrock int
zio_handle_device_injection(vdev_t * vd,zio_t * zio,int error)3428956713aSEric Schrock zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error)
343ea8dc4b6Seschrock {
344ea8dc4b6Seschrock 	inject_handler_t *handler;
345ea8dc4b6Seschrock 	int ret = 0;
346ea8dc4b6Seschrock 
3478f18d1faSGeorge Wilson 	/*
3488f18d1faSGeorge Wilson 	 * We skip over faults in the labels unless it's during
3498f18d1faSGeorge Wilson 	 * device open (i.e. zio == NULL).
3508f18d1faSGeorge Wilson 	 */
3518f18d1faSGeorge Wilson 	if (zio != NULL) {
3528f18d1faSGeorge Wilson 		uint64_t offset = zio->io_offset;
3538f18d1faSGeorge Wilson 
3548f18d1faSGeorge Wilson 		if (offset < VDEV_LABEL_START_SIZE ||
3558f18d1faSGeorge Wilson 		    offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE)
3563c708518SMark J Musante 			return (0);
3578f18d1faSGeorge Wilson 	}
3588f18d1faSGeorge Wilson 
359ea8dc4b6Seschrock 	rw_enter(&inject_lock, RW_READER);
360ea8dc4b6Seschrock 
361ea8dc4b6Seschrock 	for (handler = list_head(&inject_handlers); handler != NULL;
362ea8dc4b6Seschrock 	    handler = list_next(&inject_handlers, handler)) {
363ea8dc4b6Seschrock 
364283b8460SGeorge.Wilson 		if (handler->zi_record.zi_cmd != ZINJECT_DEVICE_FAULT)
36521bf64a7Sgw 			continue;
36621bf64a7Sgw 
367ea8dc4b6Seschrock 		if (vd->vdev_guid == handler->zi_record.zi_guid) {
3688956713aSEric Schrock 			if (handler->zi_record.zi_failfast &&
3698956713aSEric Schrock 			    (zio == NULL || (zio->io_flags &
3708956713aSEric Schrock 			    (ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD)))) {
3718956713aSEric Schrock 				continue;
3728956713aSEric Schrock 			}
3738956713aSEric Schrock 
3748f18d1faSGeorge Wilson 			/* Handle type specific I/O failures */
3758f18d1faSGeorge Wilson 			if (zio != NULL &&
3768f18d1faSGeorge Wilson 			    handler->zi_record.zi_iotype != ZIO_TYPES &&
3778f18d1faSGeorge Wilson 			    handler->zi_record.zi_iotype != zio->io_type)
3788f18d1faSGeorge Wilson 				continue;
3798f18d1faSGeorge Wilson 
380ea8dc4b6Seschrock 			if (handler->zi_record.zi_error == error) {
381*b853d39aSDon Brady 				/*
382*b853d39aSDon Brady 				 * limit error injection if requested
383*b853d39aSDon Brady 				 */
384*b853d39aSDon Brady 				if (!freq_triggered(handler->zi_record.zi_freq))
385*b853d39aSDon Brady 					continue;
386*b853d39aSDon Brady 
387ea8dc4b6Seschrock 				/*
388ea8dc4b6Seschrock 				 * For a failed open, pretend like the device
389ea8dc4b6Seschrock 				 * has gone away.
390ea8dc4b6Seschrock 				 */
391ea8dc4b6Seschrock 				if (error == ENXIO)
392ea8dc4b6Seschrock 					vd->vdev_stat.vs_aux =
393ea8dc4b6Seschrock 					    VDEV_AUX_OPEN_FAILED;
39498d1cbfeSGeorge Wilson 
39598d1cbfeSGeorge Wilson 				/*
39698d1cbfeSGeorge Wilson 				 * Treat these errors as if they had been
39798d1cbfeSGeorge Wilson 				 * retried so that all the appropriate stats
39898d1cbfeSGeorge Wilson 				 * and FMA events are generated.
39998d1cbfeSGeorge Wilson 				 */
40098d1cbfeSGeorge Wilson 				if (!handler->zi_record.zi_failfast &&
40198d1cbfeSGeorge Wilson 				    zio != NULL)
40298d1cbfeSGeorge Wilson 					zio->io_flags |= ZIO_FLAG_IO_RETRY;
40398d1cbfeSGeorge Wilson 
404ea8dc4b6Seschrock 				ret = error;
405ea8dc4b6Seschrock 				break;
406ea8dc4b6Seschrock 			}
407ea8dc4b6Seschrock 			if (handler->zi_record.zi_error == ENXIO) {
408be6fd75aSMatthew Ahrens 				ret = SET_ERROR(EIO);
409ea8dc4b6Seschrock 				break;
410ea8dc4b6Seschrock 			}
411ea8dc4b6Seschrock 		}
412ea8dc4b6Seschrock 	}
413ea8dc4b6Seschrock 
414ea8dc4b6Seschrock 	rw_exit(&inject_lock);
415ea8dc4b6Seschrock 
416ea8dc4b6Seschrock 	return (ret);
417ea8dc4b6Seschrock }
418ea8dc4b6Seschrock 
419468c413aSTim Haley /*
420468c413aSTim Haley  * Simulate hardware that ignores cache flushes.  For requested number
421468c413aSTim Haley  * of seconds nix the actual writing to disk.
422468c413aSTim Haley  */
423468c413aSTim Haley void
zio_handle_ignored_writes(zio_t * zio)424468c413aSTim Haley zio_handle_ignored_writes(zio_t *zio)
425468c413aSTim Haley {
426468c413aSTim Haley 	inject_handler_t *handler;
427468c413aSTim Haley 
428468c413aSTim Haley 	rw_enter(&inject_lock, RW_READER);
429468c413aSTim Haley 
430468c413aSTim Haley 	for (handler = list_head(&inject_handlers); handler != NULL;
431468c413aSTim Haley 	    handler = list_next(&inject_handlers, handler)) {
432468c413aSTim Haley 
433468c413aSTim Haley 		/* Ignore errors not destined for this pool */
434283b8460SGeorge.Wilson 		if (zio->io_spa != handler->zi_spa ||
435283b8460SGeorge.Wilson 		    handler->zi_record.zi_cmd != ZINJECT_IGNORED_WRITES)
436468c413aSTim Haley 			continue;
437468c413aSTim Haley 
438468c413aSTim Haley 		/*
439468c413aSTim Haley 		 * Positive duration implies # of seconds, negative
440468c413aSTim Haley 		 * a number of txgs
441468c413aSTim Haley 		 */
442468c413aSTim Haley 		if (handler->zi_record.zi_timer == 0) {
443468c413aSTim Haley 			if (handler->zi_record.zi_duration > 0)
444d3d50737SRafael Vanoni 				handler->zi_record.zi_timer = ddi_get_lbolt64();
445468c413aSTim Haley 			else
446468c413aSTim Haley 				handler->zi_record.zi_timer = zio->io_txg;
447468c413aSTim Haley 		}
448a33cae98STim Haley 
449a33cae98STim Haley 		/* Have a "problem" writing 60% of the time */
450a33cae98STim Haley 		if (spa_get_random(100) < 60)
451a33cae98STim Haley 			zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES;
452468c413aSTim Haley 		break;
453468c413aSTim Haley 	}
454468c413aSTim Haley 
455468c413aSTim Haley 	rw_exit(&inject_lock);
456468c413aSTim Haley }
457468c413aSTim Haley 
458468c413aSTim Haley void
spa_handle_ignored_writes(spa_t * spa)459468c413aSTim Haley spa_handle_ignored_writes(spa_t *spa)
460468c413aSTim Haley {
461468c413aSTim Haley 	inject_handler_t *handler;
462468c413aSTim Haley 
463468c413aSTim Haley 	if (zio_injection_enabled == 0)
464468c413aSTim Haley 		return;
465468c413aSTim Haley 
466468c413aSTim Haley 	rw_enter(&inject_lock, RW_READER);
467468c413aSTim Haley 
468468c413aSTim Haley 	for (handler = list_head(&inject_handlers); handler != NULL;
469468c413aSTim Haley 	    handler = list_next(&inject_handlers, handler)) {
470468c413aSTim Haley 
471283b8460SGeorge.Wilson 		if (spa != handler->zi_spa ||
472283b8460SGeorge.Wilson 		    handler->zi_record.zi_cmd != ZINJECT_IGNORED_WRITES)
473468c413aSTim Haley 			continue;
474468c413aSTim Haley 
475468c413aSTim Haley 		if (handler->zi_record.zi_duration > 0) {
476468c413aSTim Haley 			VERIFY(handler->zi_record.zi_timer == 0 ||
477468c413aSTim Haley 			    handler->zi_record.zi_timer +
478d3d50737SRafael Vanoni 			    handler->zi_record.zi_duration * hz >
479d3d50737SRafael Vanoni 			    ddi_get_lbolt64());
480468c413aSTim Haley 		} else {
481468c413aSTim Haley 			/* duration is negative so the subtraction here adds */
482468c413aSTim Haley 			VERIFY(handler->zi_record.zi_timer == 0 ||
483468c413aSTim Haley 			    handler->zi_record.zi_timer -
484468c413aSTim Haley 			    handler->zi_record.zi_duration >=
485b24ab676SJeff Bonwick 			    spa_syncing_txg(spa));
486468c413aSTim Haley 		}
487468c413aSTim Haley 	}
488468c413aSTim Haley 
489468c413aSTim Haley 	rw_exit(&inject_lock);
490468c413aSTim Haley }
491468c413aSTim Haley 
49297e81309SPrakash Surya hrtime_t
zio_handle_io_delay(zio_t * zio)493283b8460SGeorge.Wilson zio_handle_io_delay(zio_t *zio)
494283b8460SGeorge.Wilson {
495283b8460SGeorge.Wilson 	vdev_t *vd = zio->io_vd;
49697e81309SPrakash Surya 	inject_handler_t *min_handler = NULL;
49797e81309SPrakash Surya 	hrtime_t min_target = 0;
498283b8460SGeorge.Wilson 
499283b8460SGeorge.Wilson 	rw_enter(&inject_lock, RW_READER);
500283b8460SGeorge.Wilson 
50197e81309SPrakash Surya 	/*
50297e81309SPrakash Surya 	 * inject_delay_count is a subset of zio_injection_enabled that
50397e81309SPrakash Surya 	 * is only incremented for delay handlers. These checks are
50497e81309SPrakash Surya 	 * mainly added to remind the reader why we're not explicitly
50597e81309SPrakash Surya 	 * checking zio_injection_enabled like the other functions.
50697e81309SPrakash Surya 	 */
50797e81309SPrakash Surya 	IMPLY(inject_delay_count > 0, zio_injection_enabled > 0);
50897e81309SPrakash Surya 	IMPLY(zio_injection_enabled == 0, inject_delay_count == 0);
50997e81309SPrakash Surya 
51097e81309SPrakash Surya 	/*
51197e81309SPrakash Surya 	 * If there aren't any inject delay handlers registered, then we
51297e81309SPrakash Surya 	 * can short circuit and simply return 0 here. A value of zero
51397e81309SPrakash Surya 	 * informs zio_delay_interrupt() that this request should not be
51497e81309SPrakash Surya 	 * delayed. This short circuit keeps us from acquiring the
51597e81309SPrakash Surya 	 * inject_delay_mutex unnecessarily.
51697e81309SPrakash Surya 	 */
51797e81309SPrakash Surya 	if (inject_delay_count == 0) {
51897e81309SPrakash Surya 		rw_exit(&inject_lock);
51997e81309SPrakash Surya 		return (0);
52097e81309SPrakash Surya 	}
52197e81309SPrakash Surya 
52297e81309SPrakash Surya 	/*
52397e81309SPrakash Surya 	 * Each inject handler has a number of "lanes" associated with
52497e81309SPrakash Surya 	 * it. Each lane is able to handle requests independently of one
52597e81309SPrakash Surya 	 * another, and at a latency defined by the inject handler
52697e81309SPrakash Surya 	 * record's zi_timer field. Thus if a handler in configured with
52797e81309SPrakash Surya 	 * a single lane with a 10ms latency, it will delay requests
52897e81309SPrakash Surya 	 * such that only a single request is completed every 10ms. So,
52997e81309SPrakash Surya 	 * if more than one request is attempted per each 10ms interval,
53097e81309SPrakash Surya 	 * the average latency of the requests will be greater than
53197e81309SPrakash Surya 	 * 10ms; but if only a single request is submitted each 10ms
53297e81309SPrakash Surya 	 * interval the average latency will be 10ms.
53397e81309SPrakash Surya 	 *
53497e81309SPrakash Surya 	 * We need to acquire this mutex to prevent multiple concurrent
53597e81309SPrakash Surya 	 * threads being assigned to the same lane of a given inject
53697e81309SPrakash Surya 	 * handler. The mutex allows us to perform the following two
53797e81309SPrakash Surya 	 * operations atomically:
53897e81309SPrakash Surya 	 *
53997e81309SPrakash Surya 	 *	1. determine the minimum handler and minimum target
54097e81309SPrakash Surya 	 *	   value of all the possible handlers
54197e81309SPrakash Surya 	 *	2. update that minimum handler's lane array
54297e81309SPrakash Surya 	 *
54397e81309SPrakash Surya 	 * Without atomicity, two (or more) threads could pick the same
54497e81309SPrakash Surya 	 * lane in step (1), and then conflict with each other in step
54597e81309SPrakash Surya 	 * (2). This could allow a single lane handler to process
54697e81309SPrakash Surya 	 * multiple requests simultaneously, which shouldn't be possible.
54797e81309SPrakash Surya 	 */
54897e81309SPrakash Surya 	mutex_enter(&inject_delay_mtx);
549283b8460SGeorge.Wilson 
55097e81309SPrakash Surya 	for (inject_handler_t *handler = list_head(&inject_handlers);
55197e81309SPrakash Surya 	    handler != NULL; handler = list_next(&inject_handlers, handler)) {
552283b8460SGeorge.Wilson 		if (handler->zi_record.zi_cmd != ZINJECT_DELAY_IO)
553283b8460SGeorge.Wilson 			continue;
554283b8460SGeorge.Wilson 
555*b853d39aSDon Brady 		if (!freq_triggered(handler->zi_record.zi_freq))
556*b853d39aSDon Brady 			continue;
557*b853d39aSDon Brady 
55897e81309SPrakash Surya 		if (vd->vdev_guid != handler->zi_record.zi_guid)
55997e81309SPrakash Surya 			continue;
56097e81309SPrakash Surya 
56197e81309SPrakash Surya 		/*
56297e81309SPrakash Surya 		 * Defensive; should never happen as the array allocation
56397e81309SPrakash Surya 		 * occurs prior to inserting this handler on the list.
56497e81309SPrakash Surya 		 */
56597e81309SPrakash Surya 		ASSERT3P(handler->zi_lanes, !=, NULL);
56697e81309SPrakash Surya 
56797e81309SPrakash Surya 		/*
56897e81309SPrakash Surya 		 * This should never happen, the zinject command should
56997e81309SPrakash Surya 		 * prevent a user from setting an IO delay with zero lanes.
57097e81309SPrakash Surya 		 */
57197e81309SPrakash Surya 		ASSERT3U(handler->zi_record.zi_nlanes, !=, 0);
57297e81309SPrakash Surya 
57397e81309SPrakash Surya 		ASSERT3U(handler->zi_record.zi_nlanes, >,
57497e81309SPrakash Surya 		    handler->zi_next_lane);
57597e81309SPrakash Surya 
57697e81309SPrakash Surya 		/*
57797e81309SPrakash Surya 		 * We want to issue this IO to the lane that will become
57897e81309SPrakash Surya 		 * idle the soonest, so we compare the soonest this
57997e81309SPrakash Surya 		 * specific handler can complete the IO with all other
58097e81309SPrakash Surya 		 * handlers, to find the lowest value of all possible
58197e81309SPrakash Surya 		 * lanes. We then use this lane to submit the request.
58297e81309SPrakash Surya 		 *
58397e81309SPrakash Surya 		 * Since each handler has a constant value for its
58497e81309SPrakash Surya 		 * delay, we can just use the "next" lane for that
58597e81309SPrakash Surya 		 * handler; as it will always be the lane with the
58697e81309SPrakash Surya 		 * lowest value for that particular handler (i.e. the
58797e81309SPrakash Surya 		 * lane that will become idle the soonest). This saves a
58897e81309SPrakash Surya 		 * scan of each handler's lanes array.
58997e81309SPrakash Surya 		 *
59097e81309SPrakash Surya 		 * There's two cases to consider when determining when
59197e81309SPrakash Surya 		 * this specific IO request should complete. If this
59297e81309SPrakash Surya 		 * lane is idle, we want to "submit" the request now so
59397e81309SPrakash Surya 		 * it will complete after zi_timer milliseconds. Thus,
59497e81309SPrakash Surya 		 * we set the target to now + zi_timer.
59597e81309SPrakash Surya 		 *
59697e81309SPrakash Surya 		 * If the lane is busy, we want this request to complete
59797e81309SPrakash Surya 		 * zi_timer milliseconds after the lane becomes idle.
59897e81309SPrakash Surya 		 * Since the 'zi_lanes' array holds the time at which
59997e81309SPrakash Surya 		 * each lane will become idle, we use that value to
60097e81309SPrakash Surya 		 * determine when this request should complete.
60197e81309SPrakash Surya 		 */
60297e81309SPrakash Surya 		hrtime_t idle = handler->zi_record.zi_timer + gethrtime();
60397e81309SPrakash Surya 		hrtime_t busy = handler->zi_record.zi_timer +
60497e81309SPrakash Surya 		    handler->zi_lanes[handler->zi_next_lane];
60597e81309SPrakash Surya 		hrtime_t target = MAX(idle, busy);
60697e81309SPrakash Surya 
60797e81309SPrakash Surya 		if (min_handler == NULL) {
60897e81309SPrakash Surya 			min_handler = handler;
60997e81309SPrakash Surya 			min_target = target;
61097e81309SPrakash Surya 			continue;
611283b8460SGeorge.Wilson 		}
612283b8460SGeorge.Wilson 
61397e81309SPrakash Surya 		ASSERT3P(min_handler, !=, NULL);
61497e81309SPrakash Surya 		ASSERT3U(min_target, !=, 0);
61597e81309SPrakash Surya 
61697e81309SPrakash Surya 		/*
61797e81309SPrakash Surya 		 * We don't yet increment the "next lane" variable since
61897e81309SPrakash Surya 		 * we still might find a lower value lane in another
61997e81309SPrakash Surya 		 * handler during any remaining iterations. Once we're
62097e81309SPrakash Surya 		 * sure we've selected the absolute minimum, we'll claim
62197e81309SPrakash Surya 		 * the lane and increment the handler's "next lane"
62297e81309SPrakash Surya 		 * field below.
62397e81309SPrakash Surya 		 */
62497e81309SPrakash Surya 
62597e81309SPrakash Surya 		if (target < min_target) {
62697e81309SPrakash Surya 			min_handler = handler;
62797e81309SPrakash Surya 			min_target = target;
62897e81309SPrakash Surya 		}
629283b8460SGeorge.Wilson 	}
63097e81309SPrakash Surya 
63197e81309SPrakash Surya 	/*
63297e81309SPrakash Surya 	 * 'min_handler' will be NULL if no IO delays are registered for
63397e81309SPrakash Surya 	 * this vdev, otherwise it will point to the handler containing
63497e81309SPrakash Surya 	 * the lane that will become idle the soonest.
63597e81309SPrakash Surya 	 */
63697e81309SPrakash Surya 	if (min_handler != NULL) {
63797e81309SPrakash Surya 		ASSERT3U(min_target, !=, 0);
63897e81309SPrakash Surya 		min_handler->zi_lanes[min_handler->zi_next_lane] = min_target;
63997e81309SPrakash Surya 
64097e81309SPrakash Surya 		/*
64197e81309SPrakash Surya 		 * If we've used all possible lanes for this handler,
64297e81309SPrakash Surya 		 * loop back and start using the first lane again;
64397e81309SPrakash Surya 		 * otherwise, just increment the lane index.
64497e81309SPrakash Surya 		 */
64597e81309SPrakash Surya 		min_handler->zi_next_lane = (min_handler->zi_next_lane + 1) %
64697e81309SPrakash Surya 		    min_handler->zi_record.zi_nlanes;
64797e81309SPrakash Surya 	}
64897e81309SPrakash Surya 
64997e81309SPrakash Surya 	mutex_exit(&inject_delay_mtx);
650283b8460SGeorge.Wilson 	rw_exit(&inject_lock);
65197e81309SPrakash Surya 
65297e81309SPrakash Surya 	return (min_target);
653283b8460SGeorge.Wilson }
654283b8460SGeorge.Wilson 
655d8ab6e12SDon Brady static int
zio_calculate_range(const char * pool,zinject_record_t * record)656d8ab6e12SDon Brady zio_calculate_range(const char *pool, zinject_record_t *record)
657d8ab6e12SDon Brady {
658d8ab6e12SDon Brady 	dsl_pool_t *dp;
659d8ab6e12SDon Brady 	dsl_dataset_t *ds;
660d8ab6e12SDon Brady 	objset_t *os = NULL;
661d8ab6e12SDon Brady 	dnode_t *dn = NULL;
662d8ab6e12SDon Brady 	int error;
663d8ab6e12SDon Brady 
664d8ab6e12SDon Brady 	/*
665d8ab6e12SDon Brady 	 * Obtain the dnode for object using pool, objset, and object
666d8ab6e12SDon Brady 	 */
667d8ab6e12SDon Brady 	error = dsl_pool_hold(pool, FTAG, &dp);
668d8ab6e12SDon Brady 	if (error)
669d8ab6e12SDon Brady 		return (error);
670d8ab6e12SDon Brady 
671d8ab6e12SDon Brady 	error = dsl_dataset_hold_obj(dp, record->zi_objset, FTAG, &ds);
672d8ab6e12SDon Brady 	dsl_pool_rele(dp, FTAG);
673d8ab6e12SDon Brady 	if (error)
674d8ab6e12SDon Brady 		return (error);
675d8ab6e12SDon Brady 
676d8ab6e12SDon Brady 	error = dmu_objset_from_ds(ds, &os);
677d8ab6e12SDon Brady 	dsl_dataset_rele(ds, FTAG);
678d8ab6e12SDon Brady 	if (error)
679d8ab6e12SDon Brady 		return (error);
680d8ab6e12SDon Brady 
681d8ab6e12SDon Brady 	error = dnode_hold(os, record->zi_object, FTAG, &dn);
682d8ab6e12SDon Brady 	if (error)
683d8ab6e12SDon Brady 		return (error);
684d8ab6e12SDon Brady 
685d8ab6e12SDon Brady 	/*
686d8ab6e12SDon Brady 	 * Translate the range into block IDs
687d8ab6e12SDon Brady 	 */
688d8ab6e12SDon Brady 	if (record->zi_start != 0 || record->zi_end != -1ULL) {
689d8ab6e12SDon Brady 		record->zi_start >>= dn->dn_datablkshift;
690d8ab6e12SDon Brady 		record->zi_end >>= dn->dn_datablkshift;
691d8ab6e12SDon Brady 	}
692d8ab6e12SDon Brady 	if (record->zi_level > 0) {
693d8ab6e12SDon Brady 		if (record->zi_level >= dn->dn_nlevels) {
694d8ab6e12SDon Brady 			dnode_rele(dn, FTAG);
695d8ab6e12SDon Brady 			return (SET_ERROR(EDOM));
696d8ab6e12SDon Brady 		}
697d8ab6e12SDon Brady 
698d8ab6e12SDon Brady 		if (record->zi_start != 0 || record->zi_end != 0) {
699d8ab6e12SDon Brady 			int shift = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
700d8ab6e12SDon Brady 
701d8ab6e12SDon Brady 			for (int level = record->zi_level; level > 0; level--) {
702d8ab6e12SDon Brady 				record->zi_start >>= shift;
703d8ab6e12SDon Brady 				record->zi_end >>= shift;
704d8ab6e12SDon Brady 			}
705d8ab6e12SDon Brady 		}
706d8ab6e12SDon Brady 	}
707d8ab6e12SDon Brady 
708d8ab6e12SDon Brady 	dnode_rele(dn, FTAG);
709d8ab6e12SDon Brady 	return (0);
710d8ab6e12SDon Brady }
711d8ab6e12SDon Brady 
712ea8dc4b6Seschrock /*
713ea8dc4b6Seschrock  * Create a new handler for the given record.  We add it to the list, adding
714ea8dc4b6Seschrock  * a reference to the spa_t in the process.  We increment zio_injection_enabled,
715ea8dc4b6Seschrock  * which is the switch to trigger all fault injection.
716ea8dc4b6Seschrock  */
717ea8dc4b6Seschrock int
zio_inject_fault(char * name,int flags,int * id,zinject_record_t * record)718ea8dc4b6Seschrock zio_inject_fault(char *name, int flags, int *id, zinject_record_t *record)
719ea8dc4b6Seschrock {
720ea8dc4b6Seschrock 	inject_handler_t *handler;
721ea8dc4b6Seschrock 	int error;
722ea8dc4b6Seschrock 	spa_t *spa;
723ea8dc4b6Seschrock 
724ea8dc4b6Seschrock 	/*
725ea8dc4b6Seschrock 	 * If this is pool-wide metadata, make sure we unload the corresponding
726ea8dc4b6Seschrock 	 * spa_t, so that the next attempt to load it will trigger the fault.
727ea8dc4b6Seschrock 	 * We call spa_reset() to unload the pool appropriately.
728ea8dc4b6Seschrock 	 */
729ea8dc4b6Seschrock 	if (flags & ZINJECT_UNLOAD_SPA)
730ea8dc4b6Seschrock 		if ((error = spa_reset(name)) != 0)
731ea8dc4b6Seschrock 			return (error);
732ea8dc4b6Seschrock 
73397e81309SPrakash Surya 	if (record->zi_cmd == ZINJECT_DELAY_IO) {
73497e81309SPrakash Surya 		/*
73597e81309SPrakash Surya 		 * A value of zero for the number of lanes or for the
73697e81309SPrakash Surya 		 * delay time doesn't make sense.
73797e81309SPrakash Surya 		 */
73897e81309SPrakash Surya 		if (record->zi_timer == 0 || record->zi_nlanes == 0)
73997e81309SPrakash Surya 			return (SET_ERROR(EINVAL));
74097e81309SPrakash Surya 
74197e81309SPrakash Surya 		/*
74297e81309SPrakash Surya 		 * The number of lanes is directly mapped to the size of
74397e81309SPrakash Surya 		 * an array used by the handler. Thus, to ensure the
74497e81309SPrakash Surya 		 * user doesn't trigger an allocation that's "too large"
74597e81309SPrakash Surya 		 * we cap the number of lanes here.
74697e81309SPrakash Surya 		 */
74797e81309SPrakash Surya 		if (record->zi_nlanes >= UINT16_MAX)
74897e81309SPrakash Surya 			return (SET_ERROR(EINVAL));
74997e81309SPrakash Surya 	}
75097e81309SPrakash Surya 
751d8ab6e12SDon Brady 	/*
752d8ab6e12SDon Brady 	 * If the supplied range was in bytes -- calculate the actual blkid
753d8ab6e12SDon Brady 	 */
754d8ab6e12SDon Brady 	if (flags & ZINJECT_CALC_RANGE) {
755d8ab6e12SDon Brady 		error = zio_calculate_range(name, record);
756d8ab6e12SDon Brady 		if (error != 0)
757d8ab6e12SDon Brady 			return (error);
758d8ab6e12SDon Brady 	}
759d8ab6e12SDon Brady 
760ea8dc4b6Seschrock 	if (!(flags & ZINJECT_NULL)) {
761ea8dc4b6Seschrock 		/*
762ea8dc4b6Seschrock 		 * spa_inject_ref() will add an injection reference, which will
763ea8dc4b6Seschrock 		 * prevent the pool from being removed from the namespace while
764ea8dc4b6Seschrock 		 * still allowing it to be unloaded.
765ea8dc4b6Seschrock 		 */
766ea8dc4b6Seschrock 		if ((spa = spa_inject_addref(name)) == NULL)
767be6fd75aSMatthew Ahrens 			return (SET_ERROR(ENOENT));
768ea8dc4b6Seschrock 
769ea8dc4b6Seschrock 		handler = kmem_alloc(sizeof (inject_handler_t), KM_SLEEP);
770ea8dc4b6Seschrock 
77197e81309SPrakash Surya 		handler->zi_spa = spa;
77297e81309SPrakash Surya 		handler->zi_record = *record;
77397e81309SPrakash Surya 
77497e81309SPrakash Surya 		if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
77597e81309SPrakash Surya 			handler->zi_lanes = kmem_zalloc(
77697e81309SPrakash Surya 			    sizeof (*handler->zi_lanes) *
77797e81309SPrakash Surya 			    handler->zi_record.zi_nlanes, KM_SLEEP);
77897e81309SPrakash Surya 			handler->zi_next_lane = 0;
77997e81309SPrakash Surya 		} else {
78097e81309SPrakash Surya 			handler->zi_lanes = NULL;
78197e81309SPrakash Surya 			handler->zi_next_lane = 0;
78297e81309SPrakash Surya 		}
78397e81309SPrakash Surya 
784ea8dc4b6Seschrock 		rw_enter(&inject_lock, RW_WRITER);
785ea8dc4b6Seschrock 
78697e81309SPrakash Surya 		/*
78797e81309SPrakash Surya 		 * We can't move this increment into the conditional
78897e81309SPrakash Surya 		 * above because we need to hold the RW_WRITER lock of
78997e81309SPrakash Surya 		 * inject_lock, and we don't want to hold that while
79097e81309SPrakash Surya 		 * allocating the handler's zi_lanes array.
79197e81309SPrakash Surya 		 */
79297e81309SPrakash Surya 		if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
79397e81309SPrakash Surya 			ASSERT3S(inject_delay_count, >=, 0);
79497e81309SPrakash Surya 			inject_delay_count++;
79597e81309SPrakash Surya 			ASSERT3S(inject_delay_count, >, 0);
79697e81309SPrakash Surya 		}
79797e81309SPrakash Surya 
798ea8dc4b6Seschrock 		*id = handler->zi_id = inject_next_id++;
799ea8dc4b6Seschrock 		list_insert_tail(&inject_handlers, handler);
8001a5e258fSJosef 'Jeff' Sipek 		atomic_inc_32(&zio_injection_enabled);
801ea8dc4b6Seschrock 
802ea8dc4b6Seschrock 		rw_exit(&inject_lock);
803ea8dc4b6Seschrock 	}
804ea8dc4b6Seschrock 
805ea8dc4b6Seschrock 	/*
806ea8dc4b6Seschrock 	 * Flush the ARC, so that any attempts to read this data will end up
807ea8dc4b6Seschrock 	 * going to the ZIO layer.  Note that this is a little overkill, but
808ea8dc4b6Seschrock 	 * we don't have the necessary ARC interfaces to do anything else, and
809ea8dc4b6Seschrock 	 * fault injection isn't a performance critical path.
810ea8dc4b6Seschrock 	 */
811ea8dc4b6Seschrock 	if (flags & ZINJECT_FLUSH_ARC)
812244781f1SPrakash Surya 		/*
813244781f1SPrakash Surya 		 * We must use FALSE to ensure arc_flush returns, since
814244781f1SPrakash Surya 		 * we're not preventing concurrent ARC insertions.
815244781f1SPrakash Surya 		 */
816244781f1SPrakash Surya 		arc_flush(NULL, FALSE);
817ea8dc4b6Seschrock 
818ea8dc4b6Seschrock 	return (0);
819ea8dc4b6Seschrock }
820ea8dc4b6Seschrock 
821ea8dc4b6Seschrock /*
822ea8dc4b6Seschrock  * Returns the next record with an ID greater than that supplied to the
823ea8dc4b6Seschrock  * function.  Used to iterate over all handlers in the system.
824ea8dc4b6Seschrock  */
825ea8dc4b6Seschrock int
zio_inject_list_next(int * id,char * name,size_t buflen,zinject_record_t * record)826ea8dc4b6Seschrock zio_inject_list_next(int *id, char *name, size_t buflen,
827ea8dc4b6Seschrock     zinject_record_t *record)
828ea8dc4b6Seschrock {
829ea8dc4b6Seschrock 	inject_handler_t *handler;
830ea8dc4b6Seschrock 	int ret;
831ea8dc4b6Seschrock 
832ea8dc4b6Seschrock 	mutex_enter(&spa_namespace_lock);
833ea8dc4b6Seschrock 	rw_enter(&inject_lock, RW_READER);
834ea8dc4b6Seschrock 
835ea8dc4b6Seschrock 	for (handler = list_head(&inject_handlers); handler != NULL;
836ea8dc4b6Seschrock 	    handler = list_next(&inject_handlers, handler))
837ea8dc4b6Seschrock 		if (handler->zi_id > *id)
838ea8dc4b6Seschrock 			break;
839ea8dc4b6Seschrock 
840ea8dc4b6Seschrock 	if (handler) {
841ea8dc4b6Seschrock 		*record = handler->zi_record;
842ea8dc4b6Seschrock 		*id = handler->zi_id;
843ea8dc4b6Seschrock 		(void) strncpy(name, spa_name(handler->zi_spa), buflen);
844ea8dc4b6Seschrock 		ret = 0;
845ea8dc4b6Seschrock 	} else {
846be6fd75aSMatthew Ahrens 		ret = SET_ERROR(ENOENT);
847ea8dc4b6Seschrock 	}
848ea8dc4b6Seschrock 
849ea8dc4b6Seschrock 	rw_exit(&inject_lock);
850ea8dc4b6Seschrock 	mutex_exit(&spa_namespace_lock);
851ea8dc4b6Seschrock 
852ea8dc4b6Seschrock 	return (ret);
853ea8dc4b6Seschrock }
854ea8dc4b6Seschrock 
855ea8dc4b6Seschrock /*
856ea8dc4b6Seschrock  * Clear the fault handler with the given identifier, or return ENOENT if none
857ea8dc4b6Seschrock  * exists.
858ea8dc4b6Seschrock  */
859ea8dc4b6Seschrock int
zio_clear_fault(int id)860ea8dc4b6Seschrock zio_clear_fault(int id)
861ea8dc4b6Seschrock {
862ea8dc4b6Seschrock 	inject_handler_t *handler;
863ea8dc4b6Seschrock 
864ea8dc4b6Seschrock 	rw_enter(&inject_lock, RW_WRITER);
865ea8dc4b6Seschrock 
866ea8dc4b6Seschrock 	for (handler = list_head(&inject_handlers); handler != NULL;
867ea8dc4b6Seschrock 	    handler = list_next(&inject_handlers, handler))
868ea8dc4b6Seschrock 		if (handler->zi_id == id)
869ea8dc4b6Seschrock 			break;
870ea8dc4b6Seschrock 
871ea8dc4b6Seschrock 	if (handler == NULL) {
872679b018dSMark J Musante 		rw_exit(&inject_lock);
873be6fd75aSMatthew Ahrens 		return (SET_ERROR(ENOENT));
874ea8dc4b6Seschrock 	}
875ea8dc4b6Seschrock 
87697e81309SPrakash Surya 	if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
87797e81309SPrakash Surya 		ASSERT3S(inject_delay_count, >, 0);
87897e81309SPrakash Surya 		inject_delay_count--;
87997e81309SPrakash Surya 		ASSERT3S(inject_delay_count, >=, 0);
88097e81309SPrakash Surya 	}
88197e81309SPrakash Surya 
882679b018dSMark J Musante 	list_remove(&inject_handlers, handler);
883ea8dc4b6Seschrock 	rw_exit(&inject_lock);
884ea8dc4b6Seschrock 
88597e81309SPrakash Surya 	if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
88697e81309SPrakash Surya 		ASSERT3P(handler->zi_lanes, !=, NULL);
88797e81309SPrakash Surya 		kmem_free(handler->zi_lanes, sizeof (*handler->zi_lanes) *
88897e81309SPrakash Surya 		    handler->zi_record.zi_nlanes);
88997e81309SPrakash Surya 	} else {
89097e81309SPrakash Surya 		ASSERT3P(handler->zi_lanes, ==, NULL);
89197e81309SPrakash Surya 	}
89297e81309SPrakash Surya 
893679b018dSMark J Musante 	spa_inject_delref(handler->zi_spa);
894679b018dSMark J Musante 	kmem_free(handler, sizeof (inject_handler_t));
8951a5e258fSJosef 'Jeff' Sipek 	atomic_dec_32(&zio_injection_enabled);
896679b018dSMark J Musante 
897679b018dSMark J Musante 	return (0);
898ea8dc4b6Seschrock }
899ea8dc4b6Seschrock 
900ea8dc4b6Seschrock void
zio_inject_init(void)901ea8dc4b6Seschrock zio_inject_init(void)
902ea8dc4b6Seschrock {
9039b3f6b42SEric Kustarz 	rw_init(&inject_lock, NULL, RW_DEFAULT, NULL);
90497e81309SPrakash Surya 	mutex_init(&inject_delay_mtx, NULL, MUTEX_DEFAULT, NULL);
905ea8dc4b6Seschrock 	list_create(&inject_handlers, sizeof (inject_handler_t),
906ea8dc4b6Seschrock 	    offsetof(inject_handler_t, zi_link));
907ea8dc4b6Seschrock }
908ea8dc4b6Seschrock 
909ea8dc4b6Seschrock void
zio_inject_fini(void)910ea8dc4b6Seschrock zio_inject_fini(void)
911ea8dc4b6Seschrock {
912ea8dc4b6Seschrock 	list_destroy(&inject_handlers);
91397e81309SPrakash Surya 	mutex_destroy(&inject_delay_mtx);
9149b3f6b42SEric Kustarz 	rw_destroy(&inject_lock);
915ea8dc4b6Seschrock }
916