1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * ZFS fault injection 30 * 31 * To handle fault injection, we keep track of a series of zinject_record_t 32 * structures which describe which logical block(s) should be injected with a 33 * fault. These are kept in a global list. Each record corresponds to a given 34 * spa_t and maintains a special hold on the spa_t so that it cannot be deleted 35 * or exported while the injection record exists. 36 * 37 * Device level injection is done using the 'zi_guid' field. If this is set, it 38 * means that the error is destined for a particular device, not a piece of 39 * data. 40 * 41 * This is a rather poor data structure and algorithm, but we don't expect more 42 * than a few faults at any one time, so it should be sufficient for our needs. 43 */ 44 45 #include <sys/arc.h> 46 #include <sys/zio_impl.h> 47 #include <sys/zfs_ioctl.h> 48 #include <sys/spa_impl.h> 49 #include <sys/vdev_impl.h> 50 51 uint32_t zio_injection_enabled; 52 53 typedef struct inject_handler { 54 int zi_id; 55 spa_t *zi_spa; 56 zinject_record_t zi_record; 57 list_node_t zi_link; 58 } inject_handler_t; 59 60 static list_t inject_handlers; 61 static krwlock_t inject_lock; 62 static int inject_next_id = 1; 63 64 /* 65 * Returns true if the given record matches the I/O in progress. 66 */ 67 static boolean_t 68 zio_match_handler(zbookmark_t *zb, uint64_t type, 69 zinject_record_t *record, int error) 70 { 71 /* 72 * Check for a match against the MOS, which is based on type 73 */ 74 if (zb->zb_objset == 0 && record->zi_objset == 0 && 75 record->zi_object == 0) { 76 if (record->zi_type == DMU_OT_NONE || 77 type == record->zi_type) 78 return (record->zi_freq == 0 || 79 spa_get_random(100) < record->zi_freq); 80 else 81 return (B_FALSE); 82 } 83 84 /* 85 * Check for an exact match. 86 */ 87 if (zb->zb_objset == record->zi_objset && 88 zb->zb_object == record->zi_object && 89 zb->zb_level == record->zi_level && 90 zb->zb_blkid >= record->zi_start && 91 zb->zb_blkid <= record->zi_end && 92 error == record->zi_error) 93 return (record->zi_freq == 0 || 94 spa_get_random(100) < record->zi_freq); 95 96 return (B_FALSE); 97 } 98 99 /* 100 * Determine if the I/O in question should return failure. Returns the errno 101 * to be returned to the caller. 102 */ 103 int 104 zio_handle_fault_injection(zio_t *zio, int error) 105 { 106 int ret = 0; 107 inject_handler_t *handler; 108 109 /* 110 * Ignore I/O not associated with any logical data. 111 */ 112 if (zio->io_logical == NULL) 113 return (0); 114 115 /* 116 * Currently, we only support fault injection on reads. 117 */ 118 if (zio->io_type != ZIO_TYPE_READ) 119 return (0); 120 121 rw_enter(&inject_lock, RW_READER); 122 123 for (handler = list_head(&inject_handlers); handler != NULL; 124 handler = list_next(&inject_handlers, handler)) { 125 126 /* Ignore errors not destined for this pool */ 127 if (zio->io_spa != handler->zi_spa) 128 continue; 129 130 /* Ignore device errors */ 131 if (handler->zi_record.zi_guid != 0) 132 continue; 133 134 /* If this handler matches, return EIO */ 135 if (zio_match_handler(&zio->io_logical->io_bookmark, 136 zio->io_bp ? BP_GET_TYPE(zio->io_bp) : DMU_OT_NONE, 137 &handler->zi_record, error)) { 138 ret = error; 139 break; 140 } 141 } 142 143 rw_exit(&inject_lock); 144 145 return (ret); 146 } 147 148 int 149 zio_handle_device_injection(vdev_t *vd, int error) 150 { 151 inject_handler_t *handler; 152 int ret = 0; 153 154 rw_enter(&inject_lock, RW_READER); 155 156 for (handler = list_head(&inject_handlers); handler != NULL; 157 handler = list_next(&inject_handlers, handler)) { 158 159 if (vd->vdev_guid == handler->zi_record.zi_guid) { 160 if (handler->zi_record.zi_error == error) { 161 /* 162 * For a failed open, pretend like the device 163 * has gone away. 164 */ 165 if (error == ENXIO) 166 vd->vdev_stat.vs_aux = 167 VDEV_AUX_OPEN_FAILED; 168 ret = error; 169 break; 170 } 171 if (handler->zi_record.zi_error == ENXIO) { 172 ret = EIO; 173 break; 174 } 175 } 176 } 177 178 rw_exit(&inject_lock); 179 180 return (ret); 181 } 182 183 /* 184 * Create a new handler for the given record. We add it to the list, adding 185 * a reference to the spa_t in the process. We increment zio_injection_enabled, 186 * which is the switch to trigger all fault injection. 187 */ 188 int 189 zio_inject_fault(char *name, int flags, int *id, zinject_record_t *record) 190 { 191 inject_handler_t *handler; 192 int error; 193 spa_t *spa; 194 195 /* 196 * If this is pool-wide metadata, make sure we unload the corresponding 197 * spa_t, so that the next attempt to load it will trigger the fault. 198 * We call spa_reset() to unload the pool appropriately. 199 */ 200 if (flags & ZINJECT_UNLOAD_SPA) 201 if ((error = spa_reset(name)) != 0) 202 return (error); 203 204 if (!(flags & ZINJECT_NULL)) { 205 /* 206 * spa_inject_ref() will add an injection reference, which will 207 * prevent the pool from being removed from the namespace while 208 * still allowing it to be unloaded. 209 */ 210 if ((spa = spa_inject_addref(name)) == NULL) 211 return (ENOENT); 212 213 handler = kmem_alloc(sizeof (inject_handler_t), KM_SLEEP); 214 215 rw_enter(&inject_lock, RW_WRITER); 216 217 *id = handler->zi_id = inject_next_id++; 218 handler->zi_spa = spa; 219 handler->zi_record = *record; 220 list_insert_tail(&inject_handlers, handler); 221 atomic_add_32(&zio_injection_enabled, 1); 222 223 rw_exit(&inject_lock); 224 } 225 226 /* 227 * Flush the ARC, so that any attempts to read this data will end up 228 * going to the ZIO layer. Note that this is a little overkill, but 229 * we don't have the necessary ARC interfaces to do anything else, and 230 * fault injection isn't a performance critical path. 231 */ 232 if (flags & ZINJECT_FLUSH_ARC) 233 arc_flush(NULL); 234 235 return (0); 236 } 237 238 /* 239 * Returns the next record with an ID greater than that supplied to the 240 * function. Used to iterate over all handlers in the system. 241 */ 242 int 243 zio_inject_list_next(int *id, char *name, size_t buflen, 244 zinject_record_t *record) 245 { 246 inject_handler_t *handler; 247 int ret; 248 249 mutex_enter(&spa_namespace_lock); 250 rw_enter(&inject_lock, RW_READER); 251 252 for (handler = list_head(&inject_handlers); handler != NULL; 253 handler = list_next(&inject_handlers, handler)) 254 if (handler->zi_id > *id) 255 break; 256 257 if (handler) { 258 *record = handler->zi_record; 259 *id = handler->zi_id; 260 (void) strncpy(name, spa_name(handler->zi_spa), buflen); 261 ret = 0; 262 } else { 263 ret = ENOENT; 264 } 265 266 rw_exit(&inject_lock); 267 mutex_exit(&spa_namespace_lock); 268 269 return (ret); 270 } 271 272 /* 273 * Clear the fault handler with the given identifier, or return ENOENT if none 274 * exists. 275 */ 276 int 277 zio_clear_fault(int id) 278 { 279 inject_handler_t *handler; 280 int ret; 281 282 rw_enter(&inject_lock, RW_WRITER); 283 284 for (handler = list_head(&inject_handlers); handler != NULL; 285 handler = list_next(&inject_handlers, handler)) 286 if (handler->zi_id == id) 287 break; 288 289 if (handler == NULL) { 290 ret = ENOENT; 291 } else { 292 list_remove(&inject_handlers, handler); 293 spa_inject_delref(handler->zi_spa); 294 kmem_free(handler, sizeof (inject_handler_t)); 295 atomic_add_32(&zio_injection_enabled, -1); 296 ret = 0; 297 } 298 299 rw_exit(&inject_lock); 300 301 return (ret); 302 } 303 304 void 305 zio_inject_init(void) 306 { 307 list_create(&inject_handlers, sizeof (inject_handler_t), 308 offsetof(inject_handler_t, zi_link)); 309 } 310 311 void 312 zio_inject_fini(void) 313 { 314 list_destroy(&inject_handlers); 315 } 316