1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5ea8dc4b6Seschrock * Common Development and Distribution License (the "License"). 6ea8dc4b6Seschrock * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 22*1ab7f2deSmaybee * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23fa9e4066Sahrens * Use is subject to license terms. 24fa9e4066Sahrens */ 25fa9e4066Sahrens 26fa9e4066Sahrens #pragma ident "%Z%%M% %I% %E% SMI" 27fa9e4066Sahrens 28fa9e4066Sahrens #include <sys/zfs_context.h> 29ea8dc4b6Seschrock #include <sys/fm/fs/zfs.h> 30fa9e4066Sahrens #include <sys/spa.h> 31fa9e4066Sahrens #include <sys/txg.h> 32fa9e4066Sahrens #include <sys/spa_impl.h> 33fa9e4066Sahrens #include <sys/vdev_impl.h> 34fa9e4066Sahrens #include <sys/zio_impl.h> 35fa9e4066Sahrens #include <sys/zio_compress.h> 36fa9e4066Sahrens #include <sys/zio_checksum.h> 37fa9e4066Sahrens 38fa9e4066Sahrens /* 39fa9e4066Sahrens * ========================================================================== 40fa9e4066Sahrens * I/O priority table 41fa9e4066Sahrens * ========================================================================== 42fa9e4066Sahrens */ 43fa9e4066Sahrens uint8_t zio_priority_table[ZIO_PRIORITY_TABLE_SIZE] = { 44fa9e4066Sahrens 0, /* ZIO_PRIORITY_NOW */ 45fa9e4066Sahrens 0, /* ZIO_PRIORITY_SYNC_READ */ 46fa9e4066Sahrens 0, /* ZIO_PRIORITY_SYNC_WRITE */ 47fa9e4066Sahrens 6, /* ZIO_PRIORITY_ASYNC_READ */ 48fa9e4066Sahrens 4, /* ZIO_PRIORITY_ASYNC_WRITE */ 49fa9e4066Sahrens 4, /* ZIO_PRIORITY_FREE */ 50fa9e4066Sahrens 0, /* ZIO_PRIORITY_CACHE_FILL */ 51fa9e4066Sahrens 0, /* ZIO_PRIORITY_LOG_WRITE */ 52fa9e4066Sahrens 10, /* ZIO_PRIORITY_RESILVER */ 53fa9e4066Sahrens 20, /* ZIO_PRIORITY_SCRUB */ 54fa9e4066Sahrens }; 55fa9e4066Sahrens 56fa9e4066Sahrens /* 57fa9e4066Sahrens * ========================================================================== 58fa9e4066Sahrens * I/O type descriptions 59fa9e4066Sahrens * ========================================================================== 60fa9e4066Sahrens */ 61fa9e4066Sahrens char *zio_type_name[ZIO_TYPES] = { 62fa9e4066Sahrens "null", "read", "write", "free", "claim", "ioctl" }; 63fa9e4066Sahrens 64d63d470bSgw /* Force an allocation failure when non-zero */ 65d63d470bSgw uint16_t zio_zil_fail_shift = 0; 660a4e9518Sgw uint16_t zio_io_fail_shift = 0; 670a4e9518Sgw 680a4e9518Sgw /* Enable/disable the write-retry logic */ 690a4e9518Sgw int zio_write_retry = 1; 700a4e9518Sgw 710a4e9518Sgw /* Taskq to handle reissuing of I/Os */ 720a4e9518Sgw taskq_t *zio_taskq; 730a4e9518Sgw int zio_resume_threads = 4; 74d63d470bSgw 75fa9e4066Sahrens typedef struct zio_sync_pass { 76fa9e4066Sahrens int zp_defer_free; /* defer frees after this pass */ 77fa9e4066Sahrens int zp_dontcompress; /* don't compress after this pass */ 78fa9e4066Sahrens int zp_rewrite; /* rewrite new bps after this pass */ 79fa9e4066Sahrens } zio_sync_pass_t; 80fa9e4066Sahrens 81fa9e4066Sahrens zio_sync_pass_t zio_sync_pass = { 82fa9e4066Sahrens 1, /* zp_defer_free */ 83fa9e4066Sahrens 4, /* zp_dontcompress */ 84fa9e4066Sahrens 1, /* zp_rewrite */ 85fa9e4066Sahrens }; 86fa9e4066Sahrens 870a4e9518Sgw static boolean_t zio_io_should_fail(uint16_t); 880a4e9518Sgw 89fa9e4066Sahrens /* 90fa9e4066Sahrens * ========================================================================== 91fa9e4066Sahrens * I/O kmem caches 92fa9e4066Sahrens * ========================================================================== 93fa9e4066Sahrens */ 94ccae0b50Seschrock kmem_cache_t *zio_cache; 95fa9e4066Sahrens kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 96ad23a2dbSjohansen kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 97ad23a2dbSjohansen 98ad23a2dbSjohansen #ifdef _KERNEL 99ad23a2dbSjohansen extern vmem_t *zio_alloc_arena; 100ad23a2dbSjohansen #endif 101fa9e4066Sahrens 1020a4e9518Sgw /* 1030a4e9518Sgw * Determine if we are allowed to issue the IO based on the 1040a4e9518Sgw * pool state. If we must wait then block until we are told 1050a4e9518Sgw * that we may continue. 1060a4e9518Sgw */ 1070a4e9518Sgw #define ZIO_ENTER(spa) { \ 1080a4e9518Sgw if (spa->spa_state == POOL_STATE_IO_FAILURE) { \ 1090a4e9518Sgw mutex_enter(&spa->spa_zio_lock); \ 1100a4e9518Sgw while (spa->spa_state == POOL_STATE_IO_FAILURE) \ 1110a4e9518Sgw cv_wait(&spa->spa_zio_cv, &spa->spa_zio_lock); \ 1120a4e9518Sgw mutex_exit(&spa->spa_zio_lock); \ 1130a4e9518Sgw } \ 1140a4e9518Sgw } 1150a4e9518Sgw 1160a4e9518Sgw /* 1170a4e9518Sgw * An allocation zio is one that either currently has the DVA allocate 1180a4e9518Sgw * stage set or will have it later in it's lifetime. 1190a4e9518Sgw */ 1200a4e9518Sgw #define IO_IS_ALLOCATING(zio) \ 12117f17c2dSbonwick ((zio)->io_orig_pipeline & (1U << ZIO_STAGE_DVA_ALLOCATE)) 1220a4e9518Sgw 123fa9e4066Sahrens void 124fa9e4066Sahrens zio_init(void) 125fa9e4066Sahrens { 126fa9e4066Sahrens size_t c; 127ad23a2dbSjohansen vmem_t *data_alloc_arena = NULL; 128ad23a2dbSjohansen 129ad23a2dbSjohansen #ifdef _KERNEL 130ad23a2dbSjohansen data_alloc_arena = zio_alloc_arena; 131ad23a2dbSjohansen #endif 132fa9e4066Sahrens 133ccae0b50Seschrock zio_cache = kmem_cache_create("zio_cache", sizeof (zio_t), 0, 134ccae0b50Seschrock NULL, NULL, NULL, NULL, NULL, 0); 135ccae0b50Seschrock 136fa9e4066Sahrens /* 137fa9e4066Sahrens * For small buffers, we want a cache for each multiple of 138fa9e4066Sahrens * SPA_MINBLOCKSIZE. For medium-size buffers, we want a cache 139fa9e4066Sahrens * for each quarter-power of 2. For large buffers, we want 140fa9e4066Sahrens * a cache for each multiple of PAGESIZE. 141fa9e4066Sahrens */ 142fa9e4066Sahrens for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 143fa9e4066Sahrens size_t size = (c + 1) << SPA_MINBLOCKSHIFT; 144fa9e4066Sahrens size_t p2 = size; 145fa9e4066Sahrens size_t align = 0; 146fa9e4066Sahrens 147fa9e4066Sahrens while (p2 & (p2 - 1)) 148fa9e4066Sahrens p2 &= p2 - 1; 149fa9e4066Sahrens 150fa9e4066Sahrens if (size <= 4 * SPA_MINBLOCKSIZE) { 151fa9e4066Sahrens align = SPA_MINBLOCKSIZE; 152fa9e4066Sahrens } else if (P2PHASE(size, PAGESIZE) == 0) { 153fa9e4066Sahrens align = PAGESIZE; 154fa9e4066Sahrens } else if (P2PHASE(size, p2 >> 2) == 0) { 155fa9e4066Sahrens align = p2 >> 2; 156fa9e4066Sahrens } 157fa9e4066Sahrens 158fa9e4066Sahrens if (align != 0) { 159ad23a2dbSjohansen char name[36]; 1605ad82045Snd (void) sprintf(name, "zio_buf_%lu", (ulong_t)size); 161fa9e4066Sahrens zio_buf_cache[c] = kmem_cache_create(name, size, 162a0965f35Sbonwick align, NULL, NULL, NULL, NULL, NULL, KMC_NODEBUG); 163ad23a2dbSjohansen 164ad23a2dbSjohansen (void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size); 165ad23a2dbSjohansen zio_data_buf_cache[c] = kmem_cache_create(name, size, 166ad23a2dbSjohansen align, NULL, NULL, NULL, NULL, data_alloc_arena, 167ad23a2dbSjohansen KMC_NODEBUG); 168ad23a2dbSjohansen 169fa9e4066Sahrens } 170fa9e4066Sahrens } 171fa9e4066Sahrens 172fa9e4066Sahrens while (--c != 0) { 173fa9e4066Sahrens ASSERT(zio_buf_cache[c] != NULL); 174fa9e4066Sahrens if (zio_buf_cache[c - 1] == NULL) 175fa9e4066Sahrens zio_buf_cache[c - 1] = zio_buf_cache[c]; 176ad23a2dbSjohansen 177ad23a2dbSjohansen ASSERT(zio_data_buf_cache[c] != NULL); 178ad23a2dbSjohansen if (zio_data_buf_cache[c - 1] == NULL) 179ad23a2dbSjohansen zio_data_buf_cache[c - 1] = zio_data_buf_cache[c]; 180fa9e4066Sahrens } 181ea8dc4b6Seschrock 1820a4e9518Sgw zio_taskq = taskq_create("zio_taskq", zio_resume_threads, 1830a4e9518Sgw maxclsyspri, 50, INT_MAX, TASKQ_PREPOPULATE); 1840a4e9518Sgw 185ea8dc4b6Seschrock zio_inject_init(); 186fa9e4066Sahrens } 187fa9e4066Sahrens 188fa9e4066Sahrens void 189fa9e4066Sahrens zio_fini(void) 190fa9e4066Sahrens { 191fa9e4066Sahrens size_t c; 192fa9e4066Sahrens kmem_cache_t *last_cache = NULL; 193ad23a2dbSjohansen kmem_cache_t *last_data_cache = NULL; 194fa9e4066Sahrens 195fa9e4066Sahrens for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 196fa9e4066Sahrens if (zio_buf_cache[c] != last_cache) { 197fa9e4066Sahrens last_cache = zio_buf_cache[c]; 198fa9e4066Sahrens kmem_cache_destroy(zio_buf_cache[c]); 199fa9e4066Sahrens } 200fa9e4066Sahrens zio_buf_cache[c] = NULL; 201ad23a2dbSjohansen 202ad23a2dbSjohansen if (zio_data_buf_cache[c] != last_data_cache) { 203ad23a2dbSjohansen last_data_cache = zio_data_buf_cache[c]; 204ad23a2dbSjohansen kmem_cache_destroy(zio_data_buf_cache[c]); 205ad23a2dbSjohansen } 206ad23a2dbSjohansen zio_data_buf_cache[c] = NULL; 207fa9e4066Sahrens } 208ea8dc4b6Seschrock 2090a4e9518Sgw taskq_destroy(zio_taskq); 2100a4e9518Sgw 211ccae0b50Seschrock kmem_cache_destroy(zio_cache); 212ccae0b50Seschrock 213ea8dc4b6Seschrock zio_inject_fini(); 214fa9e4066Sahrens } 215fa9e4066Sahrens 216fa9e4066Sahrens /* 217fa9e4066Sahrens * ========================================================================== 218fa9e4066Sahrens * Allocate and free I/O buffers 219fa9e4066Sahrens * ========================================================================== 220fa9e4066Sahrens */ 221ad23a2dbSjohansen 222ad23a2dbSjohansen /* 223ad23a2dbSjohansen * Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a 224ad23a2dbSjohansen * crashdump if the kernel panics, so use it judiciously. Obviously, it's 225ad23a2dbSjohansen * useful to inspect ZFS metadata, but if possible, we should avoid keeping 226ad23a2dbSjohansen * excess / transient data in-core during a crashdump. 227ad23a2dbSjohansen */ 228fa9e4066Sahrens void * 229fa9e4066Sahrens zio_buf_alloc(size_t size) 230fa9e4066Sahrens { 231fa9e4066Sahrens size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 232fa9e4066Sahrens 233fa9e4066Sahrens ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 234fa9e4066Sahrens 235*1ab7f2deSmaybee return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE)); 236fa9e4066Sahrens } 237fa9e4066Sahrens 238ad23a2dbSjohansen /* 239ad23a2dbSjohansen * Use zio_data_buf_alloc to allocate data. The data will not appear in a 240ad23a2dbSjohansen * crashdump if the kernel panics. This exists so that we will limit the amount 241ad23a2dbSjohansen * of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount 242ad23a2dbSjohansen * of kernel heap dumped to disk when the kernel panics) 243ad23a2dbSjohansen */ 244ad23a2dbSjohansen void * 245ad23a2dbSjohansen zio_data_buf_alloc(size_t size) 246ad23a2dbSjohansen { 247ad23a2dbSjohansen size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 248ad23a2dbSjohansen 249ad23a2dbSjohansen ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 250ad23a2dbSjohansen 251*1ab7f2deSmaybee return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE)); 252ad23a2dbSjohansen } 253ad23a2dbSjohansen 254fa9e4066Sahrens void 255fa9e4066Sahrens zio_buf_free(void *buf, size_t size) 256fa9e4066Sahrens { 257fa9e4066Sahrens size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 258fa9e4066Sahrens 259fa9e4066Sahrens ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 260fa9e4066Sahrens 261fa9e4066Sahrens kmem_cache_free(zio_buf_cache[c], buf); 262fa9e4066Sahrens } 263fa9e4066Sahrens 264ad23a2dbSjohansen void 265ad23a2dbSjohansen zio_data_buf_free(void *buf, size_t size) 266ad23a2dbSjohansen { 267ad23a2dbSjohansen size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 268ad23a2dbSjohansen 269ad23a2dbSjohansen ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 270ad23a2dbSjohansen 271ad23a2dbSjohansen kmem_cache_free(zio_data_buf_cache[c], buf); 272ad23a2dbSjohansen } 273b3995adbSahrens 274fa9e4066Sahrens /* 275fa9e4066Sahrens * ========================================================================== 276fa9e4066Sahrens * Push and pop I/O transform buffers 277fa9e4066Sahrens * ========================================================================== 278fa9e4066Sahrens */ 279fa9e4066Sahrens static void 280fa9e4066Sahrens zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize) 281fa9e4066Sahrens { 282fa9e4066Sahrens zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); 283fa9e4066Sahrens 284fa9e4066Sahrens zt->zt_data = data; 285fa9e4066Sahrens zt->zt_size = size; 286fa9e4066Sahrens zt->zt_bufsize = bufsize; 287fa9e4066Sahrens 288fa9e4066Sahrens zt->zt_next = zio->io_transform_stack; 289fa9e4066Sahrens zio->io_transform_stack = zt; 290fa9e4066Sahrens 291fa9e4066Sahrens zio->io_data = data; 292fa9e4066Sahrens zio->io_size = size; 293fa9e4066Sahrens } 294fa9e4066Sahrens 295fa9e4066Sahrens static void 296fa9e4066Sahrens zio_pop_transform(zio_t *zio, void **data, uint64_t *size, uint64_t *bufsize) 297fa9e4066Sahrens { 298fa9e4066Sahrens zio_transform_t *zt = zio->io_transform_stack; 299fa9e4066Sahrens 300fa9e4066Sahrens *data = zt->zt_data; 301fa9e4066Sahrens *size = zt->zt_size; 302fa9e4066Sahrens *bufsize = zt->zt_bufsize; 303fa9e4066Sahrens 304fa9e4066Sahrens zio->io_transform_stack = zt->zt_next; 305fa9e4066Sahrens kmem_free(zt, sizeof (zio_transform_t)); 306fa9e4066Sahrens 307fa9e4066Sahrens if ((zt = zio->io_transform_stack) != NULL) { 308fa9e4066Sahrens zio->io_data = zt->zt_data; 309fa9e4066Sahrens zio->io_size = zt->zt_size; 310fa9e4066Sahrens } 311fa9e4066Sahrens } 312fa9e4066Sahrens 313fa9e4066Sahrens static void 314fa9e4066Sahrens zio_clear_transform_stack(zio_t *zio) 315fa9e4066Sahrens { 316fa9e4066Sahrens void *data; 317fa9e4066Sahrens uint64_t size, bufsize; 318fa9e4066Sahrens 319fa9e4066Sahrens ASSERT(zio->io_transform_stack != NULL); 320fa9e4066Sahrens 321fa9e4066Sahrens zio_pop_transform(zio, &data, &size, &bufsize); 322fa9e4066Sahrens while (zio->io_transform_stack != NULL) { 323fa9e4066Sahrens zio_buf_free(data, bufsize); 324fa9e4066Sahrens zio_pop_transform(zio, &data, &size, &bufsize); 325fa9e4066Sahrens } 326fa9e4066Sahrens } 327fa9e4066Sahrens 328fa9e4066Sahrens /* 329fa9e4066Sahrens * ========================================================================== 330fa9e4066Sahrens * Create the various types of I/O (read, write, free) 331fa9e4066Sahrens * ========================================================================== 332fa9e4066Sahrens */ 333fa9e4066Sahrens static zio_t * 334fa9e4066Sahrens zio_create(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 335fa9e4066Sahrens void *data, uint64_t size, zio_done_func_t *done, void *private, 336fa9e4066Sahrens zio_type_t type, int priority, int flags, uint8_t stage, uint32_t pipeline) 337fa9e4066Sahrens { 338fa9e4066Sahrens zio_t *zio; 339fa9e4066Sahrens 340fa9e4066Sahrens ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); 341fa9e4066Sahrens ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 342fa9e4066Sahrens 343ccae0b50Seschrock zio = kmem_cache_alloc(zio_cache, KM_SLEEP); 344ccae0b50Seschrock bzero(zio, sizeof (zio_t)); 345fa9e4066Sahrens zio->io_parent = pio; 346fa9e4066Sahrens zio->io_spa = spa; 347fa9e4066Sahrens zio->io_txg = txg; 348fdb2e906Sek zio->io_flags = flags; 349fa9e4066Sahrens if (bp != NULL) { 350fa9e4066Sahrens zio->io_bp = bp; 351fa9e4066Sahrens zio->io_bp_copy = *bp; 352fa9e4066Sahrens zio->io_bp_orig = *bp; 353fa9e4066Sahrens } 354fa9e4066Sahrens zio->io_done = done; 355fa9e4066Sahrens zio->io_private = private; 356fa9e4066Sahrens zio->io_type = type; 357fa9e4066Sahrens zio->io_priority = priority; 358fa9e4066Sahrens zio->io_stage = stage; 359fa9e4066Sahrens zio->io_pipeline = pipeline; 360fa9e4066Sahrens zio->io_timestamp = lbolt64; 3615ad82045Snd mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL); 362c25056deSgw cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL); 363fa9e4066Sahrens zio_push_transform(zio, data, size, size); 364fa9e4066Sahrens 365b3995adbSahrens /* 366b3995adbSahrens * Note on config lock: 367b3995adbSahrens * 368b3995adbSahrens * If CONFIG_HELD is set, then the caller already has the config 369b3995adbSahrens * lock, so we don't need it for this io. 370b3995adbSahrens * 371b3995adbSahrens * We set CONFIG_GRABBED to indicate that we have grabbed the 372b3995adbSahrens * config lock on behalf of this io, so it should be released 373b3995adbSahrens * in zio_done. 374b3995adbSahrens * 375b3995adbSahrens * Unless CONFIG_HELD is set, we will grab the config lock for 376b3995adbSahrens * any top-level (parent-less) io, *except* NULL top-level ios. 377b3995adbSahrens * The NULL top-level ios rarely have any children, so we delay 378b3995adbSahrens * grabbing the lock until the first child is added (but it is 379b3995adbSahrens * still grabbed on behalf of the top-level i/o, so additional 380b3995adbSahrens * children don't need to also grab it). This greatly reduces 381b3995adbSahrens * contention on the config lock. 382b3995adbSahrens */ 383fa9e4066Sahrens if (pio == NULL) { 384b3995adbSahrens if (type != ZIO_TYPE_NULL && 385b3995adbSahrens !(flags & ZIO_FLAG_CONFIG_HELD)) { 386e05725b1Sbonwick spa_config_enter(spa, RW_READER, zio); 387b3995adbSahrens zio->io_flags |= ZIO_FLAG_CONFIG_GRABBED; 388b3995adbSahrens } 389fa9e4066Sahrens zio->io_root = zio; 390fa9e4066Sahrens } else { 391fa9e4066Sahrens zio->io_root = pio->io_root; 392ea8dc4b6Seschrock if (!(flags & ZIO_FLAG_NOBOOKMARK)) 393ea8dc4b6Seschrock zio->io_logical = pio->io_logical; 394fa9e4066Sahrens mutex_enter(&pio->io_lock); 395b3995adbSahrens if (pio->io_parent == NULL && 396b3995adbSahrens pio->io_type == ZIO_TYPE_NULL && 397b3995adbSahrens !(pio->io_flags & ZIO_FLAG_CONFIG_GRABBED) && 398b3995adbSahrens !(pio->io_flags & ZIO_FLAG_CONFIG_HELD)) { 399b3995adbSahrens pio->io_flags |= ZIO_FLAG_CONFIG_GRABBED; 400e05725b1Sbonwick spa_config_enter(spa, RW_READER, pio); 401b3995adbSahrens } 402fa9e4066Sahrens if (stage < ZIO_STAGE_READY) 403fa9e4066Sahrens pio->io_children_notready++; 404fa9e4066Sahrens pio->io_children_notdone++; 405fa9e4066Sahrens zio->io_sibling_next = pio->io_child; 406fa9e4066Sahrens zio->io_sibling_prev = NULL; 407fa9e4066Sahrens if (pio->io_child != NULL) 408fa9e4066Sahrens pio->io_child->io_sibling_prev = zio; 409fa9e4066Sahrens pio->io_child = zio; 41044cd46caSbillm zio->io_ndvas = pio->io_ndvas; 411fa9e4066Sahrens mutex_exit(&pio->io_lock); 412fa9e4066Sahrens } 413fa9e4066Sahrens 4140a4e9518Sgw /* 4150a4e9518Sgw * Save off the original state incase we need to retry later. 4160a4e9518Sgw */ 4170a4e9518Sgw zio->io_orig_stage = zio->io_stage; 4180a4e9518Sgw zio->io_orig_pipeline = zio->io_pipeline; 4190a4e9518Sgw zio->io_orig_flags = zio->io_flags; 4200a4e9518Sgw 421fa9e4066Sahrens return (zio); 422fa9e4066Sahrens } 423fa9e4066Sahrens 4240a4e9518Sgw static void 4250a4e9518Sgw zio_reset(zio_t *zio) 4260a4e9518Sgw { 4270a4e9518Sgw zio_clear_transform_stack(zio); 4280a4e9518Sgw 4290a4e9518Sgw zio->io_flags = zio->io_orig_flags; 4300a4e9518Sgw zio->io_stage = zio->io_orig_stage; 4310a4e9518Sgw zio->io_pipeline = zio->io_orig_pipeline; 4320a4e9518Sgw zio_push_transform(zio, zio->io_data, zio->io_size, zio->io_size); 4330a4e9518Sgw } 4340a4e9518Sgw 435fa9e4066Sahrens zio_t * 436fa9e4066Sahrens zio_null(zio_t *pio, spa_t *spa, zio_done_func_t *done, void *private, 437fa9e4066Sahrens int flags) 438fa9e4066Sahrens { 439fa9e4066Sahrens zio_t *zio; 440fa9e4066Sahrens 441fa9e4066Sahrens zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 442fa9e4066Sahrens ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, ZIO_STAGE_OPEN, 443fa9e4066Sahrens ZIO_WAIT_FOR_CHILDREN_PIPELINE); 444fa9e4066Sahrens 445fa9e4066Sahrens return (zio); 446fa9e4066Sahrens } 447fa9e4066Sahrens 448fa9e4066Sahrens zio_t * 449fa9e4066Sahrens zio_root(spa_t *spa, zio_done_func_t *done, void *private, int flags) 450fa9e4066Sahrens { 451fa9e4066Sahrens return (zio_null(NULL, spa, done, private, flags)); 452fa9e4066Sahrens } 453fa9e4066Sahrens 454fa9e4066Sahrens zio_t * 455fa9e4066Sahrens zio_read(zio_t *pio, spa_t *spa, blkptr_t *bp, void *data, 456fa9e4066Sahrens uint64_t size, zio_done_func_t *done, void *private, 457ea8dc4b6Seschrock int priority, int flags, zbookmark_t *zb) 458fa9e4066Sahrens { 459fa9e4066Sahrens zio_t *zio; 460fa9e4066Sahrens 461fa9e4066Sahrens ASSERT3U(size, ==, BP_GET_LSIZE(bp)); 462fa9e4066Sahrens 4630a4e9518Sgw /* 4640a4e9518Sgw * If the user has specified that we allow I/Os to continue 4650a4e9518Sgw * then attempt to satisfy the read. 4660a4e9518Sgw */ 4670a4e9518Sgw if (spa_get_failmode(spa) != ZIO_FAILURE_MODE_CONTINUE) 4680a4e9518Sgw ZIO_ENTER(spa); 4690a4e9518Sgw 470fa9e4066Sahrens zio = zio_create(pio, spa, bp->blk_birth, bp, data, size, done, private, 471faafa6e3Sahrens ZIO_TYPE_READ, priority, flags | ZIO_FLAG_USER, 472faafa6e3Sahrens ZIO_STAGE_OPEN, ZIO_READ_PIPELINE); 473ea8dc4b6Seschrock zio->io_bookmark = *zb; 474ea8dc4b6Seschrock 475ea8dc4b6Seschrock zio->io_logical = zio; 476fa9e4066Sahrens 477fa9e4066Sahrens /* 478fa9e4066Sahrens * Work off our copy of the bp so the caller can free it. 479fa9e4066Sahrens */ 480fa9e4066Sahrens zio->io_bp = &zio->io_bp_copy; 481fa9e4066Sahrens 482fa9e4066Sahrens return (zio); 483fa9e4066Sahrens } 484fa9e4066Sahrens 485fa9e4066Sahrens zio_t * 48644cd46caSbillm zio_write(zio_t *pio, spa_t *spa, int checksum, int compress, int ncopies, 487fa9e4066Sahrens uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 488c717a561Smaybee zio_done_func_t *ready, zio_done_func_t *done, void *private, int priority, 489c717a561Smaybee int flags, zbookmark_t *zb) 490fa9e4066Sahrens { 491fa9e4066Sahrens zio_t *zio; 492fa9e4066Sahrens 493fa9e4066Sahrens ASSERT(checksum >= ZIO_CHECKSUM_OFF && 494fa9e4066Sahrens checksum < ZIO_CHECKSUM_FUNCTIONS); 495fa9e4066Sahrens 496fa9e4066Sahrens ASSERT(compress >= ZIO_COMPRESS_OFF && 497fa9e4066Sahrens compress < ZIO_COMPRESS_FUNCTIONS); 498fa9e4066Sahrens 4990a4e9518Sgw ZIO_ENTER(spa); 5000a4e9518Sgw 501fa9e4066Sahrens zio = zio_create(pio, spa, txg, bp, data, size, done, private, 502faafa6e3Sahrens ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_USER, 503fa9e4066Sahrens ZIO_STAGE_OPEN, ZIO_WRITE_PIPELINE); 504fa9e4066Sahrens 505c717a561Smaybee zio->io_ready = ready; 506c717a561Smaybee 507ea8dc4b6Seschrock zio->io_bookmark = *zb; 508ea8dc4b6Seschrock 509ea8dc4b6Seschrock zio->io_logical = zio; 510ea8dc4b6Seschrock 511fa9e4066Sahrens zio->io_checksum = checksum; 512fa9e4066Sahrens zio->io_compress = compress; 51344cd46caSbillm zio->io_ndvas = ncopies; 514fa9e4066Sahrens 515fa9e4066Sahrens if (bp->blk_birth != txg) { 516fa9e4066Sahrens /* XXX the bp usually (always?) gets re-zeroed later */ 517fa9e4066Sahrens BP_ZERO(bp); 518fa9e4066Sahrens BP_SET_LSIZE(bp, size); 519fa9e4066Sahrens BP_SET_PSIZE(bp, size); 52044cd46caSbillm } else { 52144cd46caSbillm /* Make sure someone doesn't change their mind on overwrites */ 52244cd46caSbillm ASSERT(MIN(zio->io_ndvas + BP_IS_GANG(bp), 52344cd46caSbillm spa_max_replication(spa)) == BP_GET_NDVAS(bp)); 524fa9e4066Sahrens } 525fa9e4066Sahrens 526fa9e4066Sahrens return (zio); 527fa9e4066Sahrens } 528fa9e4066Sahrens 529fa9e4066Sahrens zio_t * 530fa9e4066Sahrens zio_rewrite(zio_t *pio, spa_t *spa, int checksum, 531fa9e4066Sahrens uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 532ea8dc4b6Seschrock zio_done_func_t *done, void *private, int priority, int flags, 533ea8dc4b6Seschrock zbookmark_t *zb) 534fa9e4066Sahrens { 535fa9e4066Sahrens zio_t *zio; 536fa9e4066Sahrens 537fa9e4066Sahrens zio = zio_create(pio, spa, txg, bp, data, size, done, private, 538faafa6e3Sahrens ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_USER, 539e05725b1Sbonwick ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE(bp)); 540fa9e4066Sahrens 541ea8dc4b6Seschrock zio->io_bookmark = *zb; 542fa9e4066Sahrens zio->io_checksum = checksum; 543fa9e4066Sahrens zio->io_compress = ZIO_COMPRESS_OFF; 544fa9e4066Sahrens 54544cd46caSbillm if (pio != NULL) 54644cd46caSbillm ASSERT3U(zio->io_ndvas, <=, BP_GET_NDVAS(bp)); 54744cd46caSbillm 548fa9e4066Sahrens return (zio); 549fa9e4066Sahrens } 550fa9e4066Sahrens 5510a4e9518Sgw static void 5520a4e9518Sgw zio_write_allocate_ready(zio_t *zio) 5530a4e9518Sgw { 5540a4e9518Sgw /* Free up the previous block */ 5550a4e9518Sgw if (!BP_IS_HOLE(&zio->io_bp_orig)) { 5560a4e9518Sgw zio_nowait(zio_free(zio, zio->io_spa, zio->io_txg, 5570a4e9518Sgw &zio->io_bp_orig, NULL, NULL)); 5580a4e9518Sgw } 5590a4e9518Sgw } 5600a4e9518Sgw 561fa9e4066Sahrens static zio_t * 562fa9e4066Sahrens zio_write_allocate(zio_t *pio, spa_t *spa, int checksum, 563fa9e4066Sahrens uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 564fa9e4066Sahrens zio_done_func_t *done, void *private, int priority, int flags) 565fa9e4066Sahrens { 566fa9e4066Sahrens zio_t *zio; 567fa9e4066Sahrens 568fa9e4066Sahrens BP_ZERO(bp); 569fa9e4066Sahrens BP_SET_LSIZE(bp, size); 570fa9e4066Sahrens BP_SET_PSIZE(bp, size); 571fa9e4066Sahrens BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); 572fa9e4066Sahrens 573fa9e4066Sahrens zio = zio_create(pio, spa, txg, bp, data, size, done, private, 574fa9e4066Sahrens ZIO_TYPE_WRITE, priority, flags, 575fa9e4066Sahrens ZIO_STAGE_OPEN, ZIO_WRITE_ALLOCATE_PIPELINE); 576fa9e4066Sahrens 577fa9e4066Sahrens zio->io_checksum = checksum; 578fa9e4066Sahrens zio->io_compress = ZIO_COMPRESS_OFF; 5790a4e9518Sgw zio->io_ready = zio_write_allocate_ready; 580fa9e4066Sahrens 581fa9e4066Sahrens return (zio); 582fa9e4066Sahrens } 583fa9e4066Sahrens 584fa9e4066Sahrens zio_t * 585fa9e4066Sahrens zio_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 586fa9e4066Sahrens zio_done_func_t *done, void *private) 587fa9e4066Sahrens { 588fa9e4066Sahrens zio_t *zio; 589fa9e4066Sahrens 590fa9e4066Sahrens ASSERT(!BP_IS_HOLE(bp)); 591fa9e4066Sahrens 592fa9e4066Sahrens if (txg == spa->spa_syncing_txg && 593fa9e4066Sahrens spa->spa_sync_pass > zio_sync_pass.zp_defer_free) { 594fa9e4066Sahrens bplist_enqueue_deferred(&spa->spa_sync_bplist, bp); 595fa9e4066Sahrens return (zio_null(pio, spa, NULL, NULL, 0)); 596fa9e4066Sahrens } 597fa9e4066Sahrens 598fa9e4066Sahrens zio = zio_create(pio, spa, txg, bp, NULL, 0, done, private, 599faafa6e3Sahrens ZIO_TYPE_FREE, ZIO_PRIORITY_FREE, ZIO_FLAG_USER, 600e05725b1Sbonwick ZIO_STAGE_OPEN, ZIO_FREE_PIPELINE(bp)); 601fa9e4066Sahrens 602fa9e4066Sahrens zio->io_bp = &zio->io_bp_copy; 603fa9e4066Sahrens 604fa9e4066Sahrens return (zio); 605fa9e4066Sahrens } 606fa9e4066Sahrens 607fa9e4066Sahrens zio_t * 608fa9e4066Sahrens zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 609fa9e4066Sahrens zio_done_func_t *done, void *private) 610fa9e4066Sahrens { 611fa9e4066Sahrens zio_t *zio; 612fa9e4066Sahrens 613fa9e4066Sahrens /* 614fa9e4066Sahrens * A claim is an allocation of a specific block. Claims are needed 615fa9e4066Sahrens * to support immediate writes in the intent log. The issue is that 616fa9e4066Sahrens * immediate writes contain committed data, but in a txg that was 617fa9e4066Sahrens * *not* committed. Upon opening the pool after an unclean shutdown, 618fa9e4066Sahrens * the intent log claims all blocks that contain immediate write data 619fa9e4066Sahrens * so that the SPA knows they're in use. 620fa9e4066Sahrens * 621fa9e4066Sahrens * All claims *must* be resolved in the first txg -- before the SPA 622fa9e4066Sahrens * starts allocating blocks -- so that nothing is allocated twice. 623fa9e4066Sahrens */ 624fa9e4066Sahrens ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa)); 625fa9e4066Sahrens ASSERT3U(spa_first_txg(spa), <=, txg); 626fa9e4066Sahrens 627fa9e4066Sahrens zio = zio_create(pio, spa, txg, bp, NULL, 0, done, private, 628fa9e4066Sahrens ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, 0, 629e05725b1Sbonwick ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE(bp)); 630fa9e4066Sahrens 631fa9e4066Sahrens zio->io_bp = &zio->io_bp_copy; 632fa9e4066Sahrens 633fa9e4066Sahrens return (zio); 634fa9e4066Sahrens } 635fa9e4066Sahrens 636fa9e4066Sahrens zio_t * 637fa9e4066Sahrens zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, 638fa9e4066Sahrens zio_done_func_t *done, void *private, int priority, int flags) 639fa9e4066Sahrens { 640fa9e4066Sahrens zio_t *zio; 641fa9e4066Sahrens int c; 642fa9e4066Sahrens 643fa9e4066Sahrens if (vd->vdev_children == 0) { 644fa9e4066Sahrens zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 645fa9e4066Sahrens ZIO_TYPE_IOCTL, priority, flags, 646fa9e4066Sahrens ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); 647fa9e4066Sahrens 648fa9e4066Sahrens zio->io_vd = vd; 649fa9e4066Sahrens zio->io_cmd = cmd; 650fa9e4066Sahrens } else { 651fa9e4066Sahrens zio = zio_null(pio, spa, NULL, NULL, flags); 652fa9e4066Sahrens 653fa9e4066Sahrens for (c = 0; c < vd->vdev_children; c++) 654fa9e4066Sahrens zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, 655fa9e4066Sahrens done, private, priority, flags)); 656fa9e4066Sahrens } 657fa9e4066Sahrens 658fa9e4066Sahrens return (zio); 659fa9e4066Sahrens } 660fa9e4066Sahrens 661fa9e4066Sahrens static void 662fa9e4066Sahrens zio_phys_bp_init(vdev_t *vd, blkptr_t *bp, uint64_t offset, uint64_t size, 663fa94a07fSbrendan int checksum, boolean_t labels) 664fa9e4066Sahrens { 665fa9e4066Sahrens ASSERT(vd->vdev_children == 0); 666fa9e4066Sahrens 667fa9e4066Sahrens ASSERT(size <= SPA_MAXBLOCKSIZE); 668fa9e4066Sahrens ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 669fa9e4066Sahrens ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); 670fa9e4066Sahrens 671fa94a07fSbrendan #ifdef ZFS_DEBUG 672fa94a07fSbrendan if (labels) { 673fa94a07fSbrendan ASSERT(offset + size <= VDEV_LABEL_START_SIZE || 674fa94a07fSbrendan offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 675fa94a07fSbrendan } 676fa94a07fSbrendan #endif 677fa9e4066Sahrens ASSERT3U(offset + size, <=, vd->vdev_psize); 678fa9e4066Sahrens 679fa9e4066Sahrens BP_ZERO(bp); 680fa9e4066Sahrens 681fa9e4066Sahrens BP_SET_LSIZE(bp, size); 682fa9e4066Sahrens BP_SET_PSIZE(bp, size); 683fa9e4066Sahrens 684fa9e4066Sahrens BP_SET_CHECKSUM(bp, checksum); 685fa9e4066Sahrens BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); 686fa9e4066Sahrens BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 687fa9e4066Sahrens 688fa9e4066Sahrens if (checksum != ZIO_CHECKSUM_OFF) 689fa9e4066Sahrens ZIO_SET_CHECKSUM(&bp->blk_cksum, offset, 0, 0, 0); 690fa9e4066Sahrens } 691fa9e4066Sahrens 692fa9e4066Sahrens zio_t * 693fa9e4066Sahrens zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 694fa9e4066Sahrens void *data, int checksum, zio_done_func_t *done, void *private, 695fa94a07fSbrendan int priority, int flags, boolean_t labels) 696fa9e4066Sahrens { 697fa9e4066Sahrens zio_t *zio; 698fa9e4066Sahrens blkptr_t blk; 699fa9e4066Sahrens 7000a4e9518Sgw ZIO_ENTER(vd->vdev_spa); 7010a4e9518Sgw 702fa94a07fSbrendan zio_phys_bp_init(vd, &blk, offset, size, checksum, labels); 703fa9e4066Sahrens 704fa9e4066Sahrens zio = zio_create(pio, vd->vdev_spa, 0, &blk, data, size, done, private, 705fa9e4066Sahrens ZIO_TYPE_READ, priority, flags | ZIO_FLAG_PHYSICAL, 706fa9e4066Sahrens ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); 707fa9e4066Sahrens 708fa9e4066Sahrens zio->io_vd = vd; 709fa9e4066Sahrens zio->io_offset = offset; 710fa9e4066Sahrens 711fa9e4066Sahrens /* 712fa9e4066Sahrens * Work off our copy of the bp so the caller can free it. 713fa9e4066Sahrens */ 714fa9e4066Sahrens zio->io_bp = &zio->io_bp_copy; 715fa9e4066Sahrens 716fa9e4066Sahrens return (zio); 717fa9e4066Sahrens } 718fa9e4066Sahrens 719fa9e4066Sahrens zio_t * 720fa9e4066Sahrens zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 721fa9e4066Sahrens void *data, int checksum, zio_done_func_t *done, void *private, 722fa94a07fSbrendan int priority, int flags, boolean_t labels) 723fa9e4066Sahrens { 724fa9e4066Sahrens zio_block_tail_t *zbt; 725fa9e4066Sahrens void *wbuf; 726fa9e4066Sahrens zio_t *zio; 727fa9e4066Sahrens blkptr_t blk; 728fa9e4066Sahrens 7290a4e9518Sgw ZIO_ENTER(vd->vdev_spa); 7300a4e9518Sgw 731fa94a07fSbrendan zio_phys_bp_init(vd, &blk, offset, size, checksum, labels); 732fa9e4066Sahrens 733fa9e4066Sahrens zio = zio_create(pio, vd->vdev_spa, 0, &blk, data, size, done, private, 734fa9e4066Sahrens ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_PHYSICAL, 735fa9e4066Sahrens ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); 736fa9e4066Sahrens 737fa9e4066Sahrens zio->io_vd = vd; 738fa9e4066Sahrens zio->io_offset = offset; 739fa9e4066Sahrens 740fa9e4066Sahrens zio->io_bp = &zio->io_bp_copy; 741fa9e4066Sahrens zio->io_checksum = checksum; 742fa9e4066Sahrens 743fa9e4066Sahrens if (zio_checksum_table[checksum].ci_zbt) { 744fa9e4066Sahrens /* 745fa9e4066Sahrens * zbt checksums are necessarily destructive -- they modify 746fa9e4066Sahrens * one word of the write buffer to hold the verifier/checksum. 747fa9e4066Sahrens * Therefore, we must make a local copy in case the data is 748fa9e4066Sahrens * being written to multiple places. 749fa9e4066Sahrens */ 750fa9e4066Sahrens wbuf = zio_buf_alloc(size); 751fa9e4066Sahrens bcopy(data, wbuf, size); 752fa9e4066Sahrens zio_push_transform(zio, wbuf, size, size); 753fa9e4066Sahrens 754fa9e4066Sahrens zbt = (zio_block_tail_t *)((char *)wbuf + size) - 1; 755fa9e4066Sahrens zbt->zbt_cksum = blk.blk_cksum; 756fa9e4066Sahrens } 757fa9e4066Sahrens 758fa9e4066Sahrens return (zio); 759fa9e4066Sahrens } 760fa9e4066Sahrens 761fa9e4066Sahrens /* 762fa9e4066Sahrens * Create a child I/O to do some work for us. It has no associated bp. 763fa9e4066Sahrens */ 764fa9e4066Sahrens zio_t * 765fa9e4066Sahrens zio_vdev_child_io(zio_t *zio, blkptr_t *bp, vdev_t *vd, uint64_t offset, 766fa9e4066Sahrens void *data, uint64_t size, int type, int priority, int flags, 767fa9e4066Sahrens zio_done_func_t *done, void *private) 768fa9e4066Sahrens { 769fa9e4066Sahrens uint32_t pipeline = ZIO_VDEV_CHILD_PIPELINE; 770fa9e4066Sahrens zio_t *cio; 771fa9e4066Sahrens 772fa9e4066Sahrens if (type == ZIO_TYPE_READ && bp != NULL) { 773fa9e4066Sahrens /* 774fa9e4066Sahrens * If we have the bp, then the child should perform the 775fa9e4066Sahrens * checksum and the parent need not. This pushes error 776fa9e4066Sahrens * detection as close to the leaves as possible and 777fa9e4066Sahrens * eliminates redundant checksums in the interior nodes. 778fa9e4066Sahrens */ 779fa9e4066Sahrens pipeline |= 1U << ZIO_STAGE_CHECKSUM_VERIFY; 780fa9e4066Sahrens zio->io_pipeline &= ~(1U << ZIO_STAGE_CHECKSUM_VERIFY); 781fa9e4066Sahrens } 782fa9e4066Sahrens 783fa9e4066Sahrens cio = zio_create(zio, zio->io_spa, zio->io_txg, bp, data, size, 784fa9e4066Sahrens done, private, type, priority, 785fa9e4066Sahrens (zio->io_flags & ZIO_FLAG_VDEV_INHERIT) | ZIO_FLAG_CANFAIL | flags, 78644cd46caSbillm ZIO_STAGE_VDEV_IO_START - 1, pipeline); 787fa9e4066Sahrens 788fa9e4066Sahrens cio->io_vd = vd; 789fa9e4066Sahrens cio->io_offset = offset; 790fa9e4066Sahrens 791fa9e4066Sahrens return (cio); 792fa9e4066Sahrens } 793fa9e4066Sahrens 794fa9e4066Sahrens /* 795fa9e4066Sahrens * ========================================================================== 796fa9e4066Sahrens * Initiate I/O, either sync or async 797fa9e4066Sahrens * ========================================================================== 798fa9e4066Sahrens */ 799fa9e4066Sahrens int 800fa9e4066Sahrens zio_wait(zio_t *zio) 801fa9e4066Sahrens { 802fa9e4066Sahrens int error; 803fa9e4066Sahrens 804fa9e4066Sahrens ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 805fa9e4066Sahrens 806fa9e4066Sahrens zio->io_waiter = curthread; 807fa9e4066Sahrens 808e05725b1Sbonwick zio_execute(zio); 809fa9e4066Sahrens 810fa9e4066Sahrens mutex_enter(&zio->io_lock); 811fa9e4066Sahrens while (zio->io_stalled != ZIO_STAGE_DONE) 812fa9e4066Sahrens cv_wait(&zio->io_cv, &zio->io_lock); 813fa9e4066Sahrens mutex_exit(&zio->io_lock); 814fa9e4066Sahrens 815fa9e4066Sahrens error = zio->io_error; 8165ad82045Snd mutex_destroy(&zio->io_lock); 817c25056deSgw cv_destroy(&zio->io_cv); 818ccae0b50Seschrock kmem_cache_free(zio_cache, zio); 819fa9e4066Sahrens 820fa9e4066Sahrens return (error); 821fa9e4066Sahrens } 822fa9e4066Sahrens 823fa9e4066Sahrens void 824fa9e4066Sahrens zio_nowait(zio_t *zio) 825fa9e4066Sahrens { 826e05725b1Sbonwick zio_execute(zio); 827e05725b1Sbonwick } 828e05725b1Sbonwick 829e05725b1Sbonwick void 830e05725b1Sbonwick zio_interrupt(zio_t *zio) 831e05725b1Sbonwick { 832e05725b1Sbonwick (void) taskq_dispatch(zio->io_spa->spa_zio_intr_taskq[zio->io_type], 833e05725b1Sbonwick (task_func_t *)zio_execute, zio, TQ_SLEEP); 834e05725b1Sbonwick } 835e05725b1Sbonwick 836e05725b1Sbonwick static int 837e05725b1Sbonwick zio_issue_async(zio_t *zio) 838e05725b1Sbonwick { 839e05725b1Sbonwick (void) taskq_dispatch(zio->io_spa->spa_zio_issue_taskq[zio->io_type], 840e05725b1Sbonwick (task_func_t *)zio_execute, zio, TQ_SLEEP); 841e05725b1Sbonwick 842e05725b1Sbonwick return (ZIO_PIPELINE_STOP); 843fa9e4066Sahrens } 844fa9e4066Sahrens 845fa9e4066Sahrens /* 846fa9e4066Sahrens * ========================================================================== 847fa9e4066Sahrens * I/O pipeline interlocks: parent/child dependency scoreboarding 848fa9e4066Sahrens * ========================================================================== 849fa9e4066Sahrens */ 850e05725b1Sbonwick static int 851fa9e4066Sahrens zio_wait_for_children(zio_t *zio, uint32_t stage, uint64_t *countp) 852fa9e4066Sahrens { 853e05725b1Sbonwick int rv = ZIO_PIPELINE_CONTINUE; 854e05725b1Sbonwick 855fa9e4066Sahrens mutex_enter(&zio->io_lock); 856e05725b1Sbonwick ASSERT(zio->io_stalled == 0); 857e05725b1Sbonwick if (*countp != 0) { 858fa9e4066Sahrens zio->io_stalled = stage; 859e05725b1Sbonwick rv = ZIO_PIPELINE_STOP; 860fa9e4066Sahrens } 861e05725b1Sbonwick mutex_exit(&zio->io_lock); 862e05725b1Sbonwick 863e05725b1Sbonwick return (rv); 864fa9e4066Sahrens } 865fa9e4066Sahrens 866fa9e4066Sahrens static void 867fa9e4066Sahrens zio_notify_parent(zio_t *zio, uint32_t stage, uint64_t *countp) 868fa9e4066Sahrens { 869fa9e4066Sahrens zio_t *pio = zio->io_parent; 870fa9e4066Sahrens 871fa9e4066Sahrens mutex_enter(&pio->io_lock); 872fa9e4066Sahrens if (pio->io_error == 0 && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) 873fa9e4066Sahrens pio->io_error = zio->io_error; 8740a4e9518Sgw ASSERT3U(*countp, >, 0); 875fa9e4066Sahrens if (--*countp == 0 && pio->io_stalled == stage) { 876fa9e4066Sahrens pio->io_stalled = 0; 877fa9e4066Sahrens mutex_exit(&pio->io_lock); 878e05725b1Sbonwick zio_execute(pio); 879fa9e4066Sahrens } else { 880fa9e4066Sahrens mutex_exit(&pio->io_lock); 881fa9e4066Sahrens } 882fa9e4066Sahrens } 883fa9e4066Sahrens 884e05725b1Sbonwick int 885e05725b1Sbonwick zio_wait_for_children_ready(zio_t *zio) 886fa9e4066Sahrens { 887e05725b1Sbonwick return (zio_wait_for_children(zio, ZIO_STAGE_WAIT_FOR_CHILDREN_READY, 888e05725b1Sbonwick &zio->io_children_notready)); 889fa9e4066Sahrens } 890fa9e4066Sahrens 891e05725b1Sbonwick int 892e05725b1Sbonwick zio_wait_for_children_done(zio_t *zio) 893fa9e4066Sahrens { 894e05725b1Sbonwick return (zio_wait_for_children(zio, ZIO_STAGE_WAIT_FOR_CHILDREN_DONE, 895e05725b1Sbonwick &zio->io_children_notdone)); 896fa9e4066Sahrens } 897fa9e4066Sahrens 898e05725b1Sbonwick static int 8990a4e9518Sgw zio_read_init(zio_t *zio) 9000a4e9518Sgw { 901e05725b1Sbonwick blkptr_t *bp = zio->io_bp; 902e05725b1Sbonwick 903e05725b1Sbonwick if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) { 904e05725b1Sbonwick uint64_t csize = BP_GET_PSIZE(bp); 9050a4e9518Sgw void *cbuf = zio_buf_alloc(csize); 9060a4e9518Sgw 9070a4e9518Sgw zio_push_transform(zio, cbuf, csize, csize); 9080a4e9518Sgw zio->io_pipeline |= 1U << ZIO_STAGE_READ_DECOMPRESS; 9090a4e9518Sgw } 9100a4e9518Sgw 911e05725b1Sbonwick if (BP_IS_GANG(bp)) { 9120a4e9518Sgw uint64_t gsize = SPA_GANGBLOCKSIZE; 9130a4e9518Sgw void *gbuf = zio_buf_alloc(gsize); 9140a4e9518Sgw 9150a4e9518Sgw zio_push_transform(zio, gbuf, gsize, gsize); 9160a4e9518Sgw zio->io_pipeline |= 1U << ZIO_STAGE_READ_GANG_MEMBERS; 9170a4e9518Sgw } 918e05725b1Sbonwick 919e05725b1Sbonwick if (!dmu_ot[BP_GET_TYPE(bp)].ot_metadata && BP_GET_LEVEL(bp) == 0) 920e05725b1Sbonwick zio->io_flags |= ZIO_FLAG_DONT_CACHE; 921e05725b1Sbonwick 922e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 9230a4e9518Sgw } 9240a4e9518Sgw 925e05725b1Sbonwick static int 926fa9e4066Sahrens zio_ready(zio_t *zio) 927fa9e4066Sahrens { 928fa9e4066Sahrens zio_t *pio = zio->io_parent; 929fa9e4066Sahrens 930c717a561Smaybee if (zio->io_ready) 931c717a561Smaybee zio->io_ready(zio); 932c717a561Smaybee 933fa9e4066Sahrens if (pio != NULL) 934e05725b1Sbonwick zio_notify_parent(zio, ZIO_STAGE_WAIT_FOR_CHILDREN_READY, 935fa9e4066Sahrens &pio->io_children_notready); 936fa9e4066Sahrens 937fa9e4066Sahrens if (zio->io_bp) 938fa9e4066Sahrens zio->io_bp_copy = *zio->io_bp; 939fa9e4066Sahrens 940e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 941fa9e4066Sahrens } 942fa9e4066Sahrens 943e05725b1Sbonwick static int 9440a4e9518Sgw zio_vdev_retry_io(zio_t *zio) 945fa9e4066Sahrens { 946fa9e4066Sahrens zio_t *pio = zio->io_parent; 9470a4e9518Sgw 9480a4e9518Sgw /* 9490a4e9518Sgw * Preserve the failed bp so that the io_ready() callback can 9500a4e9518Sgw * update the accounting accordingly. The callback will also be 9510a4e9518Sgw * responsible for freeing the previously allocated block, if one 9520a4e9518Sgw * exists. 9530a4e9518Sgw */ 9540a4e9518Sgw zio->io_bp_orig = *zio->io_bp; 9550a4e9518Sgw 9560a4e9518Sgw /* 9570a4e9518Sgw * We must zero out the old DVA and blk_birth before reallocating 958e9ed0a81Sgw * the bp. 9590a4e9518Sgw */ 960e9ed0a81Sgw BP_ZERO_DVAS(zio->io_bp); 9610a4e9518Sgw zio_reset(zio); 9620a4e9518Sgw 9630a4e9518Sgw if (pio) { 9640a4e9518Sgw /* 9650a4e9518Sgw * Let the parent know that we will 9660a4e9518Sgw * re-alloc the write (=> new bp info). 9670a4e9518Sgw */ 9680a4e9518Sgw mutex_enter(&pio->io_lock); 9690a4e9518Sgw pio->io_children_notready++; 9700a4e9518Sgw 9710a4e9518Sgw /* 9720a4e9518Sgw * If the parent I/O is still in the open stage, then 9730a4e9518Sgw * don't bother telling it to retry since it hasn't 9740a4e9518Sgw * progressed far enough for it to care. 9750a4e9518Sgw */ 9760a4e9518Sgw if (pio->io_stage > ZIO_STAGE_OPEN && IO_IS_ALLOCATING(pio)) 9770a4e9518Sgw pio->io_flags |= ZIO_FLAG_WRITE_RETRY; 9780a4e9518Sgw 979e05725b1Sbonwick ASSERT(pio->io_stage <= ZIO_STAGE_WAIT_FOR_CHILDREN_DONE); 9800a4e9518Sgw mutex_exit(&pio->io_lock); 9810a4e9518Sgw } 9820a4e9518Sgw 9830a4e9518Sgw /* 9840a4e9518Sgw * We are getting ready to process the retry request so clear 9850a4e9518Sgw * the flag and the zio's current error status. 9860a4e9518Sgw */ 9870a4e9518Sgw zio->io_flags &= ~ZIO_FLAG_WRITE_RETRY; 9880a4e9518Sgw zio->io_error = 0; 989e05725b1Sbonwick 990e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 9910a4e9518Sgw } 9920a4e9518Sgw 9930a4e9518Sgw int 9940a4e9518Sgw zio_vdev_resume_io(spa_t *spa) 9950a4e9518Sgw { 9960a4e9518Sgw zio_t *zio; 9970a4e9518Sgw 9980a4e9518Sgw mutex_enter(&spa->spa_zio_lock); 9990a4e9518Sgw 10000a4e9518Sgw /* 10010a4e9518Sgw * Probe all of vdevs that have experienced an I/O error. 10020a4e9518Sgw * If we are still unable to verify the integrity of the vdev 10030a4e9518Sgw * then we prevent the resume from proceeeding. 10040a4e9518Sgw */ 10050a4e9518Sgw for (zio = list_head(&spa->spa_zio_list); zio != NULL; 10060a4e9518Sgw zio = list_next(&spa->spa_zio_list, zio)) { 10070a4e9518Sgw int error = 0; 10080a4e9518Sgw 10090a4e9518Sgw /* We only care about I/Os that must succeed */ 10100a4e9518Sgw if (zio->io_vd == NULL || zio->io_flags & ZIO_FLAG_CANFAIL) 10110a4e9518Sgw continue; 10120a4e9518Sgw error = vdev_probe(zio->io_vd); 10130a4e9518Sgw if (error) { 10140a4e9518Sgw mutex_exit(&spa->spa_zio_lock); 10150a4e9518Sgw return (error); 10160a4e9518Sgw } 10170a4e9518Sgw } 10180a4e9518Sgw 10190a4e9518Sgw /* 10200a4e9518Sgw * Clear the vdev stats so that I/O can flow. 10210a4e9518Sgw */ 10220a4e9518Sgw vdev_clear(spa, NULL, B_FALSE); 10230a4e9518Sgw 10240a4e9518Sgw spa->spa_state = POOL_STATE_ACTIVE; 10250a4e9518Sgw while ((zio = list_head(&spa->spa_zio_list)) != NULL) { 10260a4e9518Sgw list_remove(&spa->spa_zio_list, zio); 10270a4e9518Sgw zio->io_error = 0; 10280a4e9518Sgw 10290a4e9518Sgw /* 10300a4e9518Sgw * If we are resuming an allocating I/O then we force it 10310a4e9518Sgw * to retry and let it resume operation where it left off. 10320a4e9518Sgw * Otherwise, go back to the ready stage and pick up from 10330a4e9518Sgw * there. 10340a4e9518Sgw */ 10350a4e9518Sgw if (zio_write_retry && IO_IS_ALLOCATING(zio)) { 10360a4e9518Sgw zio->io_flags |= ZIO_FLAG_WRITE_RETRY; 10370a4e9518Sgw zio->io_stage--; 10380a4e9518Sgw } else { 10390a4e9518Sgw zio->io_stage = ZIO_STAGE_READY; 10400a4e9518Sgw } 10410a4e9518Sgw 1042e05725b1Sbonwick (void) taskq_dispatch(zio_taskq, (task_func_t *)zio_execute, 10430a4e9518Sgw zio, TQ_SLEEP); 10440a4e9518Sgw } 10450a4e9518Sgw mutex_exit(&spa->spa_zio_lock); 10460a4e9518Sgw 10470a4e9518Sgw /* 10480a4e9518Sgw * Wait for the taskqs to finish and recheck the pool state since 10490a4e9518Sgw * it's possible that a resumed I/O has failed again. 10500a4e9518Sgw */ 10510a4e9518Sgw taskq_wait(zio_taskq); 10520a4e9518Sgw if (spa_state(spa) == POOL_STATE_IO_FAILURE) 10530a4e9518Sgw return (EIO); 10540a4e9518Sgw 10550a4e9518Sgw mutex_enter(&spa->spa_zio_lock); 10560a4e9518Sgw cv_broadcast(&spa->spa_zio_cv); 10570a4e9518Sgw mutex_exit(&spa->spa_zio_lock); 10580a4e9518Sgw 10590a4e9518Sgw return (0); 10600a4e9518Sgw } 10610a4e9518Sgw 1062e05725b1Sbonwick static int 10630a4e9518Sgw zio_vdev_suspend_io(zio_t *zio) 10640a4e9518Sgw { 10650a4e9518Sgw spa_t *spa = zio->io_spa; 10660a4e9518Sgw 10670a4e9518Sgw /* 10680a4e9518Sgw * We've experienced an unrecoverable failure so 10690a4e9518Sgw * set the pool state accordingly and queue all 10700a4e9518Sgw * failed IOs. 10710a4e9518Sgw */ 10720a4e9518Sgw spa->spa_state = POOL_STATE_IO_FAILURE; 10730a4e9518Sgw 10740a4e9518Sgw mutex_enter(&spa->spa_zio_lock); 10750a4e9518Sgw list_insert_tail(&spa->spa_zio_list, zio); 10760a4e9518Sgw 10770a4e9518Sgw #ifndef _KERNEL 10780a4e9518Sgw /* Used to notify ztest that the pool has suspended */ 10790a4e9518Sgw cv_broadcast(&spa->spa_zio_cv); 10800a4e9518Sgw #endif 10810a4e9518Sgw mutex_exit(&spa->spa_zio_lock); 1082e05725b1Sbonwick 1083e05725b1Sbonwick return (ZIO_PIPELINE_STOP); 10840a4e9518Sgw } 10850a4e9518Sgw 1086e05725b1Sbonwick static int 10870a4e9518Sgw zio_assess(zio_t *zio) 10880a4e9518Sgw { 1089fa9e4066Sahrens spa_t *spa = zio->io_spa; 1090fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 1091fa9e4066Sahrens vdev_t *vd = zio->io_vd; 1092fa9e4066Sahrens 1093fa9e4066Sahrens ASSERT(zio->io_children_notready == 0); 1094fa9e4066Sahrens ASSERT(zio->io_children_notdone == 0); 1095fa9e4066Sahrens 1096fa9e4066Sahrens if (bp != NULL) { 1097fa9e4066Sahrens ASSERT(bp->blk_pad[0] == 0); 1098fa9e4066Sahrens ASSERT(bp->blk_pad[1] == 0); 1099fa9e4066Sahrens ASSERT(bp->blk_pad[2] == 0); 1100fa9e4066Sahrens ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0); 1101fa9e4066Sahrens if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) && 110244cd46caSbillm !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) { 1103fa9e4066Sahrens ASSERT(!BP_SHOULD_BYTESWAP(bp)); 110444cd46caSbillm if (zio->io_ndvas != 0) 110544cd46caSbillm ASSERT3U(zio->io_ndvas, <=, BP_GET_NDVAS(bp)); 110644cd46caSbillm ASSERT(BP_COUNT_GANG(bp) == 0 || 110744cd46caSbillm (BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp))); 110844cd46caSbillm } 1109fa9e4066Sahrens } 1110fa9e4066Sahrens 11110a4e9518Sgw /* 11120a4e9518Sgw * Some child I/O has indicated that a retry is necessary, so 11130a4e9518Sgw * we set an error on the I/O and let the logic below do the 11140a4e9518Sgw * rest. 11150a4e9518Sgw */ 11160a4e9518Sgw if (zio->io_flags & ZIO_FLAG_WRITE_RETRY) 11170a4e9518Sgw zio->io_error = ERESTART; 11180a4e9518Sgw 1119fa9e4066Sahrens if (vd != NULL) 1120fa9e4066Sahrens vdev_stat_update(zio); 1121fa9e4066Sahrens 1122fa9e4066Sahrens if (zio->io_error) { 1123ea8dc4b6Seschrock /* 1124ea8dc4b6Seschrock * If this I/O is attached to a particular vdev, 1125ea8dc4b6Seschrock * generate an error message describing the I/O failure 1126ea8dc4b6Seschrock * at the block level. We ignore these errors if the 1127ea8dc4b6Seschrock * device is currently unavailable. 1128ea8dc4b6Seschrock */ 1129ecc2d604Sbonwick if (zio->io_error != ECKSUM && vd != NULL && !vdev_is_dead(vd)) 11300a4e9518Sgw zfs_ereport_post(FM_EREPORT_ZFS_IO, spa, vd, zio, 0, 0); 1131ea8dc4b6Seschrock 1132ea8dc4b6Seschrock if ((zio->io_error == EIO || 1133ea8dc4b6Seschrock !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) && 1134ea8dc4b6Seschrock zio->io_logical == zio) { 1135ea8dc4b6Seschrock /* 1136ea8dc4b6Seschrock * For root I/O requests, tell the SPA to log the error 1137ea8dc4b6Seschrock * appropriately. Also, generate a logical data 1138ea8dc4b6Seschrock * ereport. 1139ea8dc4b6Seschrock */ 11400a4e9518Sgw spa_log_error(spa, zio); 1141ea8dc4b6Seschrock 11420a4e9518Sgw zfs_ereport_post(FM_EREPORT_ZFS_DATA, spa, NULL, zio, 11430a4e9518Sgw 0, 0); 1144ea8dc4b6Seschrock } 1145fa9e4066Sahrens 1146ea8dc4b6Seschrock /* 1147e9ed0a81Sgw * If we are an allocating I/O then we attempt to reissue 1148e9ed0a81Sgw * the I/O on another vdev unless the pool is out of space. 1149e9ed0a81Sgw * We handle this condition based on the spa's failmode 1150e9ed0a81Sgw * property. 11510a4e9518Sgw */ 11520a4e9518Sgw if (zio_write_retry && zio->io_error != ENOSPC && 1153e05725b1Sbonwick IO_IS_ALLOCATING(zio)) 1154e05725b1Sbonwick return (zio_vdev_retry_io(zio)); 1155e05725b1Sbonwick 11560a4e9518Sgw ASSERT(!(zio->io_flags & ZIO_FLAG_WRITE_RETRY)); 11570a4e9518Sgw 11580a4e9518Sgw /* 11590a4e9518Sgw * For I/O requests that cannot fail, we carry out 11600a4e9518Sgw * the requested behavior based on the failmode pool 11610a4e9518Sgw * property. 11620a4e9518Sgw * 11630a4e9518Sgw * XXX - Need to differentiate between an ENOSPC as 11640a4e9518Sgw * a result of vdev failures vs. a full pool. 1165ea8dc4b6Seschrock */ 1166ea8dc4b6Seschrock if (!(zio->io_flags & ZIO_FLAG_CANFAIL)) { 1167d58459f4Sek char *blkbuf; 1168d58459f4Sek 11690a4e9518Sgw #ifdef ZFS_DEBUG 1170d58459f4Sek blkbuf = kmem_alloc(BP_SPRINTF_LEN, KM_NOSLEEP); 1171d58459f4Sek if (blkbuf) { 1172d58459f4Sek sprintf_blkptr(blkbuf, BP_SPRINTF_LEN, 1173d58459f4Sek bp ? bp : &zio->io_bp_copy); 1174d58459f4Sek } 11750a4e9518Sgw cmn_err(CE_WARN, "ZFS: %s (%s on %s off %llx: zio %p " 11760a4e9518Sgw "%s): error %d", zio->io_error == ECKSUM ? 1177ea8dc4b6Seschrock "bad checksum" : "I/O failure", 1178ea8dc4b6Seschrock zio_type_name[zio->io_type], 1179ea8dc4b6Seschrock vdev_description(vd), 1180ea8dc4b6Seschrock (u_longlong_t)zio->io_offset, 11810a4e9518Sgw (void *)zio, blkbuf ? blkbuf : "", zio->io_error); 11820a4e9518Sgw #endif 11830a4e9518Sgw 11840a4e9518Sgw if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC) { 11850a4e9518Sgw fm_panic("Pool '%s' has encountered an " 11860a4e9518Sgw "uncorrectable I/O failure and the " 11870a4e9518Sgw "failure mode property for this pool " 11880a4e9518Sgw "is set to panic.", spa_name(spa)); 11890a4e9518Sgw } 1190e05725b1Sbonwick cmn_err(CE_WARN, "Pool '%s' has encountered " 1191e05725b1Sbonwick "an uncorrectable I/O error. " 1192e05725b1Sbonwick "Manual intervention is required.", spa_name(spa)); 1193e05725b1Sbonwick return (zio_vdev_suspend_io(zio)); 1194ea8dc4b6Seschrock } 1195fa9e4066Sahrens } 11960a4e9518Sgw ASSERT(!(zio->io_flags & ZIO_FLAG_WRITE_RETRY)); 11970a4e9518Sgw ASSERT(zio->io_children_notready == 0); 1198e05725b1Sbonwick 1199e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 12000a4e9518Sgw } 12010a4e9518Sgw 1202e05725b1Sbonwick static int 12030a4e9518Sgw zio_done(zio_t *zio) 12040a4e9518Sgw { 12050a4e9518Sgw zio_t *pio = zio->io_parent; 12060a4e9518Sgw spa_t *spa = zio->io_spa; 12070a4e9518Sgw 12080a4e9518Sgw ASSERT(zio->io_children_notready == 0); 12090a4e9518Sgw ASSERT(zio->io_children_notdone == 0); 12100a4e9518Sgw 1211fa9e4066Sahrens zio_clear_transform_stack(zio); 1212fa9e4066Sahrens 1213fa9e4066Sahrens if (zio->io_done) 1214fa9e4066Sahrens zio->io_done(zio); 1215fa9e4066Sahrens 1216fa9e4066Sahrens ASSERT(zio->io_delegate_list == NULL); 1217fa9e4066Sahrens ASSERT(zio->io_delegate_next == NULL); 1218fa9e4066Sahrens 1219fa9e4066Sahrens if (pio != NULL) { 1220fa9e4066Sahrens zio_t *next, *prev; 1221fa9e4066Sahrens 1222fa9e4066Sahrens mutex_enter(&pio->io_lock); 1223fa9e4066Sahrens next = zio->io_sibling_next; 1224fa9e4066Sahrens prev = zio->io_sibling_prev; 1225fa9e4066Sahrens if (next != NULL) 1226fa9e4066Sahrens next->io_sibling_prev = prev; 1227fa9e4066Sahrens if (prev != NULL) 1228fa9e4066Sahrens prev->io_sibling_next = next; 1229fa9e4066Sahrens if (pio->io_child == zio) 1230fa9e4066Sahrens pio->io_child = next; 1231fa9e4066Sahrens mutex_exit(&pio->io_lock); 1232fa9e4066Sahrens 1233e05725b1Sbonwick zio_notify_parent(zio, ZIO_STAGE_WAIT_FOR_CHILDREN_DONE, 1234fa9e4066Sahrens &pio->io_children_notdone); 1235fa9e4066Sahrens } 1236fa9e4066Sahrens 1237b3995adbSahrens /* 1238ccae0b50Seschrock * Note: this I/O is now done, and will shortly be freed, so there is no 1239ccae0b50Seschrock * need to clear this (or any other) flag. 1240b3995adbSahrens */ 1241b3995adbSahrens if (zio->io_flags & ZIO_FLAG_CONFIG_GRABBED) 1242ea8dc4b6Seschrock spa_config_exit(spa, zio); 1243fa9e4066Sahrens 1244fa9e4066Sahrens if (zio->io_waiter != NULL) { 1245fa9e4066Sahrens mutex_enter(&zio->io_lock); 1246fa9e4066Sahrens ASSERT(zio->io_stage == ZIO_STAGE_DONE); 1247fa9e4066Sahrens zio->io_stalled = zio->io_stage; 1248fa9e4066Sahrens cv_broadcast(&zio->io_cv); 1249fa9e4066Sahrens mutex_exit(&zio->io_lock); 1250fa9e4066Sahrens } else { 1251c25056deSgw mutex_destroy(&zio->io_lock); 1252c25056deSgw cv_destroy(&zio->io_cv); 1253ccae0b50Seschrock kmem_cache_free(zio_cache, zio); 1254fa9e4066Sahrens } 1255e05725b1Sbonwick 1256e05725b1Sbonwick return (ZIO_PIPELINE_STOP); 1257fa9e4066Sahrens } 1258fa9e4066Sahrens 1259fa9e4066Sahrens /* 1260fa9e4066Sahrens * ========================================================================== 1261fa9e4066Sahrens * Compression support 1262fa9e4066Sahrens * ========================================================================== 1263fa9e4066Sahrens */ 1264e05725b1Sbonwick static int 1265fa9e4066Sahrens zio_write_compress(zio_t *zio) 1266fa9e4066Sahrens { 1267fa9e4066Sahrens int compress = zio->io_compress; 1268fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 1269fa9e4066Sahrens void *cbuf; 1270fa9e4066Sahrens uint64_t lsize = zio->io_size; 1271fa9e4066Sahrens uint64_t csize = lsize; 1272fa9e4066Sahrens uint64_t cbufsize = 0; 1273fa9e4066Sahrens int pass; 1274fa9e4066Sahrens 1275fa9e4066Sahrens if (bp->blk_birth == zio->io_txg) { 1276fa9e4066Sahrens /* 1277fa9e4066Sahrens * We're rewriting an existing block, which means we're 1278fa9e4066Sahrens * working on behalf of spa_sync(). For spa_sync() to 1279fa9e4066Sahrens * converge, it must eventually be the case that we don't 1280fa9e4066Sahrens * have to allocate new blocks. But compression changes 1281fa9e4066Sahrens * the blocksize, which forces a reallocate, and makes 1282fa9e4066Sahrens * convergence take longer. Therefore, after the first 1283fa9e4066Sahrens * few passes, stop compressing to ensure convergence. 1284fa9e4066Sahrens */ 1285fa9e4066Sahrens pass = spa_sync_pass(zio->io_spa); 1286fa9e4066Sahrens if (pass > zio_sync_pass.zp_dontcompress) 1287fa9e4066Sahrens compress = ZIO_COMPRESS_OFF; 1288fa9e4066Sahrens } else { 1289fa9e4066Sahrens ASSERT(BP_IS_HOLE(bp)); 1290fa9e4066Sahrens pass = 1; 1291fa9e4066Sahrens } 1292fa9e4066Sahrens 1293fa9e4066Sahrens if (compress != ZIO_COMPRESS_OFF) 1294fa9e4066Sahrens if (!zio_compress_data(compress, zio->io_data, zio->io_size, 1295fa9e4066Sahrens &cbuf, &csize, &cbufsize)) 1296fa9e4066Sahrens compress = ZIO_COMPRESS_OFF; 1297fa9e4066Sahrens 1298fa9e4066Sahrens if (compress != ZIO_COMPRESS_OFF && csize != 0) 1299fa9e4066Sahrens zio_push_transform(zio, cbuf, csize, cbufsize); 1300fa9e4066Sahrens 1301fa9e4066Sahrens /* 1302fa9e4066Sahrens * The final pass of spa_sync() must be all rewrites, but the first 1303fa9e4066Sahrens * few passes offer a trade-off: allocating blocks defers convergence, 1304fa9e4066Sahrens * but newly allocated blocks are sequential, so they can be written 1305fa9e4066Sahrens * to disk faster. Therefore, we allow the first few passes of 1306fa9e4066Sahrens * spa_sync() to reallocate new blocks, but force rewrites after that. 1307fa9e4066Sahrens * There should only be a handful of blocks after pass 1 in any case. 1308fa9e4066Sahrens */ 1309fa9e4066Sahrens if (bp->blk_birth == zio->io_txg && BP_GET_PSIZE(bp) == csize && 1310fa9e4066Sahrens pass > zio_sync_pass.zp_rewrite) { 1311fa9e4066Sahrens ASSERT(csize != 0); 1312a2eea2e1Sahrens BP_SET_LSIZE(bp, lsize); 1313a2eea2e1Sahrens BP_SET_COMPRESS(bp, compress); 1314e05725b1Sbonwick zio->io_pipeline = ZIO_REWRITE_PIPELINE(bp); 1315fa9e4066Sahrens } else { 131687bd5c1eSahrens if (bp->blk_birth == zio->io_txg) 131787bd5c1eSahrens BP_ZERO(bp); 1318fa9e4066Sahrens if (csize == 0) { 1319fa9e4066Sahrens BP_ZERO(bp); 1320fa9e4066Sahrens zio->io_pipeline = ZIO_WAIT_FOR_CHILDREN_PIPELINE; 1321fa9e4066Sahrens } else { 132244cd46caSbillm ASSERT3U(BP_GET_NDVAS(bp), ==, 0); 1323fa9e4066Sahrens BP_SET_LSIZE(bp, lsize); 1324fa9e4066Sahrens BP_SET_PSIZE(bp, csize); 1325fa9e4066Sahrens BP_SET_COMPRESS(bp, compress); 1326fa9e4066Sahrens } 1327fa9e4066Sahrens } 1328fa9e4066Sahrens 1329e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 1330fa9e4066Sahrens } 1331fa9e4066Sahrens 1332e05725b1Sbonwick static int 1333fa9e4066Sahrens zio_read_decompress(zio_t *zio) 1334fa9e4066Sahrens { 1335fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 1336fa9e4066Sahrens void *data; 1337fa9e4066Sahrens uint64_t size; 1338fa9e4066Sahrens uint64_t bufsize; 1339fa9e4066Sahrens int compress = BP_GET_COMPRESS(bp); 1340fa9e4066Sahrens 1341fa9e4066Sahrens ASSERT(compress != ZIO_COMPRESS_OFF); 1342fa9e4066Sahrens 1343fa9e4066Sahrens zio_pop_transform(zio, &data, &size, &bufsize); 1344fa9e4066Sahrens 1345fa9e4066Sahrens if (zio_decompress_data(compress, data, size, 1346fa9e4066Sahrens zio->io_data, zio->io_size)) 1347fa9e4066Sahrens zio->io_error = EIO; 1348fa9e4066Sahrens 1349fa9e4066Sahrens zio_buf_free(data, bufsize); 1350fa9e4066Sahrens 1351e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 1352fa9e4066Sahrens } 1353fa9e4066Sahrens 1354fa9e4066Sahrens /* 1355fa9e4066Sahrens * ========================================================================== 1356fa9e4066Sahrens * Gang block support 1357fa9e4066Sahrens * ========================================================================== 1358fa9e4066Sahrens */ 1359fa9e4066Sahrens static void 1360fa9e4066Sahrens zio_gang_byteswap(zio_t *zio) 1361fa9e4066Sahrens { 1362fa9e4066Sahrens ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); 1363fa9e4066Sahrens 1364fa9e4066Sahrens if (BP_SHOULD_BYTESWAP(zio->io_bp)) 1365fa9e4066Sahrens byteswap_uint64_array(zio->io_data, zio->io_size); 1366fa9e4066Sahrens } 1367fa9e4066Sahrens 1368e05725b1Sbonwick static int 1369fa9e4066Sahrens zio_get_gang_header(zio_t *zio) 1370fa9e4066Sahrens { 1371fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 1372fa9e4066Sahrens uint64_t gsize = SPA_GANGBLOCKSIZE; 1373fa9e4066Sahrens void *gbuf = zio_buf_alloc(gsize); 1374fa9e4066Sahrens 137544cd46caSbillm ASSERT(BP_IS_GANG(bp)); 1376fa9e4066Sahrens 1377fa9e4066Sahrens zio_push_transform(zio, gbuf, gsize, gsize); 1378fa9e4066Sahrens 1379fa9e4066Sahrens zio_nowait(zio_create(zio, zio->io_spa, bp->blk_birth, bp, gbuf, gsize, 1380fa9e4066Sahrens NULL, NULL, ZIO_TYPE_READ, zio->io_priority, 1381fa9e4066Sahrens zio->io_flags & ZIO_FLAG_GANG_INHERIT, 13820a4e9518Sgw ZIO_STAGE_OPEN, ZIO_READ_GANG_PIPELINE)); 1383fa9e4066Sahrens 1384e05725b1Sbonwick return (zio_wait_for_children_done(zio)); 1385fa9e4066Sahrens } 1386fa9e4066Sahrens 1387e05725b1Sbonwick static int 1388fa9e4066Sahrens zio_read_gang_members(zio_t *zio) 1389fa9e4066Sahrens { 1390fa9e4066Sahrens zio_gbh_phys_t *gbh; 1391fa9e4066Sahrens uint64_t gsize, gbufsize, loff, lsize; 1392fa9e4066Sahrens int i; 1393fa9e4066Sahrens 139444cd46caSbillm ASSERT(BP_IS_GANG(zio->io_bp)); 1395fa9e4066Sahrens 1396fa9e4066Sahrens zio_gang_byteswap(zio); 1397fa9e4066Sahrens zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1398fa9e4066Sahrens 1399fa9e4066Sahrens for (loff = 0, i = 0; loff != zio->io_size; loff += lsize, i++) { 1400fa9e4066Sahrens blkptr_t *gbp = &gbh->zg_blkptr[i]; 1401fa9e4066Sahrens lsize = BP_GET_PSIZE(gbp); 1402fa9e4066Sahrens 1403fa9e4066Sahrens ASSERT(BP_GET_COMPRESS(gbp) == ZIO_COMPRESS_OFF); 1404fa9e4066Sahrens ASSERT3U(lsize, ==, BP_GET_LSIZE(gbp)); 1405fa9e4066Sahrens ASSERT3U(loff + lsize, <=, zio->io_size); 1406fa9e4066Sahrens ASSERT(i < SPA_GBH_NBLKPTRS); 1407fa9e4066Sahrens ASSERT(!BP_IS_HOLE(gbp)); 1408fa9e4066Sahrens 1409fa9e4066Sahrens zio_nowait(zio_read(zio, zio->io_spa, gbp, 1410e05725b1Sbonwick (char *)zio->io_data + loff, lsize, 1411e05725b1Sbonwick NULL, NULL, zio->io_priority, 1412e05725b1Sbonwick zio->io_flags & ZIO_FLAG_GANG_INHERIT, &zio->io_bookmark)); 1413fa9e4066Sahrens } 1414fa9e4066Sahrens 1415fa9e4066Sahrens zio_buf_free(gbh, gbufsize); 1416e05725b1Sbonwick 1417e05725b1Sbonwick return (zio_wait_for_children_done(zio)); 1418fa9e4066Sahrens } 1419fa9e4066Sahrens 1420e05725b1Sbonwick static int 1421fa9e4066Sahrens zio_rewrite_gang_members(zio_t *zio) 1422fa9e4066Sahrens { 1423fa9e4066Sahrens zio_gbh_phys_t *gbh; 1424fa9e4066Sahrens uint64_t gsize, gbufsize, loff, lsize; 1425fa9e4066Sahrens int i; 1426fa9e4066Sahrens 142744cd46caSbillm ASSERT(BP_IS_GANG(zio->io_bp)); 1428fa9e4066Sahrens ASSERT3U(zio->io_size, ==, SPA_GANGBLOCKSIZE); 1429fa9e4066Sahrens 1430fa9e4066Sahrens zio_gang_byteswap(zio); 1431fa9e4066Sahrens zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1432fa9e4066Sahrens 1433fa9e4066Sahrens ASSERT(gsize == gbufsize); 1434fa9e4066Sahrens 1435fa9e4066Sahrens for (loff = 0, i = 0; loff != zio->io_size; loff += lsize, i++) { 1436fa9e4066Sahrens blkptr_t *gbp = &gbh->zg_blkptr[i]; 1437fa9e4066Sahrens lsize = BP_GET_PSIZE(gbp); 1438fa9e4066Sahrens 1439fa9e4066Sahrens ASSERT(BP_GET_COMPRESS(gbp) == ZIO_COMPRESS_OFF); 1440fa9e4066Sahrens ASSERT3U(lsize, ==, BP_GET_LSIZE(gbp)); 1441fa9e4066Sahrens ASSERT3U(loff + lsize, <=, zio->io_size); 1442fa9e4066Sahrens ASSERT(i < SPA_GBH_NBLKPTRS); 1443fa9e4066Sahrens ASSERT(!BP_IS_HOLE(gbp)); 1444fa9e4066Sahrens 1445fa9e4066Sahrens zio_nowait(zio_rewrite(zio, zio->io_spa, zio->io_checksum, 1446fa9e4066Sahrens zio->io_txg, gbp, (char *)zio->io_data + loff, lsize, 1447e05725b1Sbonwick NULL, NULL, zio->io_priority, 1448e05725b1Sbonwick zio->io_flags & ZIO_FLAG_GANG_INHERIT, &zio->io_bookmark)); 1449fa9e4066Sahrens } 1450fa9e4066Sahrens 1451fa9e4066Sahrens zio_push_transform(zio, gbh, gsize, gbufsize); 1452e05725b1Sbonwick 1453e05725b1Sbonwick return (zio_wait_for_children_ready(zio)); 1454fa9e4066Sahrens } 1455fa9e4066Sahrens 1456e05725b1Sbonwick static int 1457fa9e4066Sahrens zio_free_gang_members(zio_t *zio) 1458fa9e4066Sahrens { 1459fa9e4066Sahrens zio_gbh_phys_t *gbh; 1460fa9e4066Sahrens uint64_t gsize, gbufsize; 1461fa9e4066Sahrens int i; 1462fa9e4066Sahrens 146344cd46caSbillm ASSERT(BP_IS_GANG(zio->io_bp)); 1464fa9e4066Sahrens 1465fa9e4066Sahrens zio_gang_byteswap(zio); 1466fa9e4066Sahrens zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1467fa9e4066Sahrens 1468fa9e4066Sahrens for (i = 0; i < SPA_GBH_NBLKPTRS; i++) { 1469fa9e4066Sahrens blkptr_t *gbp = &gbh->zg_blkptr[i]; 1470fa9e4066Sahrens 1471fa9e4066Sahrens if (BP_IS_HOLE(gbp)) 1472fa9e4066Sahrens continue; 1473fa9e4066Sahrens zio_nowait(zio_free(zio, zio->io_spa, zio->io_txg, 1474fa9e4066Sahrens gbp, NULL, NULL)); 1475fa9e4066Sahrens } 1476fa9e4066Sahrens 1477fa9e4066Sahrens zio_buf_free(gbh, gbufsize); 1478e05725b1Sbonwick 1479e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 1480fa9e4066Sahrens } 1481fa9e4066Sahrens 1482e05725b1Sbonwick static int 1483fa9e4066Sahrens zio_claim_gang_members(zio_t *zio) 1484fa9e4066Sahrens { 1485fa9e4066Sahrens zio_gbh_phys_t *gbh; 1486fa9e4066Sahrens uint64_t gsize, gbufsize; 1487fa9e4066Sahrens int i; 1488fa9e4066Sahrens 148944cd46caSbillm ASSERT(BP_IS_GANG(zio->io_bp)); 1490fa9e4066Sahrens 1491fa9e4066Sahrens zio_gang_byteswap(zio); 1492fa9e4066Sahrens zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1493fa9e4066Sahrens 1494fa9e4066Sahrens for (i = 0; i < SPA_GBH_NBLKPTRS; i++) { 1495fa9e4066Sahrens blkptr_t *gbp = &gbh->zg_blkptr[i]; 1496fa9e4066Sahrens if (BP_IS_HOLE(gbp)) 1497fa9e4066Sahrens continue; 1498fa9e4066Sahrens zio_nowait(zio_claim(zio, zio->io_spa, zio->io_txg, 1499fa9e4066Sahrens gbp, NULL, NULL)); 1500fa9e4066Sahrens } 1501fa9e4066Sahrens 1502fa9e4066Sahrens zio_buf_free(gbh, gbufsize); 1503e05725b1Sbonwick 1504e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 1505fa9e4066Sahrens } 1506fa9e4066Sahrens 1507fa9e4066Sahrens static void 1508fa9e4066Sahrens zio_write_allocate_gang_member_done(zio_t *zio) 1509fa9e4066Sahrens { 1510fa9e4066Sahrens zio_t *pio = zio->io_parent; 151144cd46caSbillm dva_t *cdva = zio->io_bp->blk_dva; 151244cd46caSbillm dva_t *pdva = pio->io_bp->blk_dva; 1513fa9e4066Sahrens uint64_t asize; 151444cd46caSbillm int d; 1515fa9e4066Sahrens 151644cd46caSbillm ASSERT3U(pio->io_ndvas, ==, zio->io_ndvas); 151744cd46caSbillm ASSERT3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp)); 151844cd46caSbillm ASSERT3U(zio->io_ndvas, <=, BP_GET_NDVAS(zio->io_bp)); 151944cd46caSbillm ASSERT3U(pio->io_ndvas, <=, BP_GET_NDVAS(pio->io_bp)); 1520fa9e4066Sahrens 1521fa9e4066Sahrens mutex_enter(&pio->io_lock); 152244cd46caSbillm for (d = 0; d < BP_GET_NDVAS(pio->io_bp); d++) { 152344cd46caSbillm ASSERT(DVA_GET_GANG(&pdva[d])); 152444cd46caSbillm asize = DVA_GET_ASIZE(&pdva[d]); 152544cd46caSbillm asize += DVA_GET_ASIZE(&cdva[d]); 152644cd46caSbillm DVA_SET_ASIZE(&pdva[d], asize); 152744cd46caSbillm } 1528fa9e4066Sahrens mutex_exit(&pio->io_lock); 1529fa9e4066Sahrens } 1530fa9e4066Sahrens 15310a4e9518Sgw static int 15328654d025Sperrin zio_write_allocate_gang_members(zio_t *zio, metaslab_class_t *mc) 1533fa9e4066Sahrens { 1534fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 153544cd46caSbillm dva_t *dva = bp->blk_dva; 153644cd46caSbillm spa_t *spa = zio->io_spa; 1537fa9e4066Sahrens zio_gbh_phys_t *gbh; 153844cd46caSbillm uint64_t txg = zio->io_txg; 1539fa9e4066Sahrens uint64_t resid = zio->io_size; 1540fa9e4066Sahrens uint64_t maxalloc = P2ROUNDUP(zio->io_size >> 1, SPA_MINBLOCKSIZE); 1541fa9e4066Sahrens uint64_t gsize, loff, lsize; 1542fa9e4066Sahrens uint32_t gbps_left; 154344cd46caSbillm int ndvas = zio->io_ndvas; 154444cd46caSbillm int gbh_ndvas = MIN(ndvas + 1, spa_max_replication(spa)); 1545fa9e4066Sahrens int error; 154644cd46caSbillm int i, d; 1547fa9e4066Sahrens 1548fa9e4066Sahrens gsize = SPA_GANGBLOCKSIZE; 1549fa9e4066Sahrens gbps_left = SPA_GBH_NBLKPTRS; 1550fa9e4066Sahrens 15518654d025Sperrin error = metaslab_alloc(spa, mc, gsize, bp, gbh_ndvas, txg, NULL, 15528654d025Sperrin B_FALSE); 1553e05725b1Sbonwick if (error) { 1554e05725b1Sbonwick zio->io_error = error; 1555e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 1556e05725b1Sbonwick } 1557fa9e4066Sahrens 155844cd46caSbillm for (d = 0; d < gbh_ndvas; d++) 155944cd46caSbillm DVA_SET_GANG(&dva[d], 1); 1560fa9e4066Sahrens 156144cd46caSbillm bp->blk_birth = txg; 1562fa9e4066Sahrens 1563fa9e4066Sahrens gbh = zio_buf_alloc(gsize); 1564fa9e4066Sahrens bzero(gbh, gsize); 1565fa9e4066Sahrens 1566fa9e4066Sahrens for (loff = 0, i = 0; loff != zio->io_size; 1567fa9e4066Sahrens loff += lsize, resid -= lsize, gbps_left--, i++) { 1568fa9e4066Sahrens blkptr_t *gbp = &gbh->zg_blkptr[i]; 156944cd46caSbillm dva = gbp->blk_dva; 1570fa9e4066Sahrens 1571fa9e4066Sahrens ASSERT(gbps_left != 0); 1572fa9e4066Sahrens maxalloc = MIN(maxalloc, resid); 1573fa9e4066Sahrens 1574fa9e4066Sahrens while (resid <= maxalloc * gbps_left) { 15758654d025Sperrin error = metaslab_alloc(spa, mc, maxalloc, gbp, ndvas, 157667bd71c6Sperrin txg, bp, B_FALSE); 1577fa9e4066Sahrens if (error == 0) 1578fa9e4066Sahrens break; 1579fa9e4066Sahrens ASSERT3U(error, ==, ENOSPC); 15800a4e9518Sgw /* XXX - free up previous allocations? */ 1581e05725b1Sbonwick if (maxalloc == SPA_MINBLOCKSIZE) { 1582e05725b1Sbonwick zio->io_error = error; 1583e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 1584e05725b1Sbonwick } 1585fa9e4066Sahrens maxalloc = P2ROUNDUP(maxalloc >> 1, SPA_MINBLOCKSIZE); 1586fa9e4066Sahrens } 1587fa9e4066Sahrens 1588fa9e4066Sahrens if (resid <= maxalloc * gbps_left) { 1589fa9e4066Sahrens lsize = maxalloc; 1590fa9e4066Sahrens BP_SET_LSIZE(gbp, lsize); 1591fa9e4066Sahrens BP_SET_PSIZE(gbp, lsize); 1592fa9e4066Sahrens BP_SET_COMPRESS(gbp, ZIO_COMPRESS_OFF); 159344cd46caSbillm gbp->blk_birth = txg; 159444cd46caSbillm zio_nowait(zio_rewrite(zio, spa, 159544cd46caSbillm zio->io_checksum, txg, gbp, 1596fa9e4066Sahrens (char *)zio->io_data + loff, lsize, 1597fa9e4066Sahrens zio_write_allocate_gang_member_done, NULL, 1598e9ed0a81Sgw zio->io_priority, 1599e9ed0a81Sgw zio->io_flags & ZIO_FLAG_GANG_INHERIT, 1600ea8dc4b6Seschrock &zio->io_bookmark)); 1601fa9e4066Sahrens } else { 1602fa9e4066Sahrens lsize = P2ROUNDUP(resid / gbps_left, SPA_MINBLOCKSIZE); 1603fa9e4066Sahrens ASSERT(lsize != SPA_MINBLOCKSIZE); 160444cd46caSbillm zio_nowait(zio_write_allocate(zio, spa, 160544cd46caSbillm zio->io_checksum, txg, gbp, 1606fa9e4066Sahrens (char *)zio->io_data + loff, lsize, 1607fa9e4066Sahrens zio_write_allocate_gang_member_done, NULL, 1608e9ed0a81Sgw zio->io_priority, 1609e9ed0a81Sgw zio->io_flags & ZIO_FLAG_GANG_INHERIT)); 1610fa9e4066Sahrens } 1611fa9e4066Sahrens } 1612fa9e4066Sahrens 1613fa9e4066Sahrens ASSERT(resid == 0 && loff == zio->io_size); 1614fa9e4066Sahrens 1615fa9e4066Sahrens zio->io_pipeline |= 1U << ZIO_STAGE_GANG_CHECKSUM_GENERATE; 1616fa9e4066Sahrens 1617fa9e4066Sahrens zio_push_transform(zio, gbh, gsize, gsize); 1618e05725b1Sbonwick 161944cd46caSbillm /* 1620e05725b1Sbonwick * As much as we'd like this to be 'ready' instead of 'done', 162144cd46caSbillm * updating our ASIZE doesn't happen until the io_done callback, 162244cd46caSbillm * so we have to wait for that to finish in order for our BP 162344cd46caSbillm * to be stable. 162444cd46caSbillm */ 1625e05725b1Sbonwick return (zio_wait_for_children_done(zio)); 1626fa9e4066Sahrens } 1627fa9e4066Sahrens 1628fa9e4066Sahrens /* 1629fa9e4066Sahrens * ========================================================================== 1630fa9e4066Sahrens * Allocate and free blocks 1631fa9e4066Sahrens * ========================================================================== 1632fa9e4066Sahrens */ 1633e05725b1Sbonwick static int 1634fa9e4066Sahrens zio_dva_allocate(zio_t *zio) 1635fa9e4066Sahrens { 16368654d025Sperrin spa_t *spa = zio->io_spa; 16378654d025Sperrin metaslab_class_t *mc = spa->spa_normal_class; 1638fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 1639fa9e4066Sahrens int error; 1640fa9e4066Sahrens 1641fa9e4066Sahrens ASSERT(BP_IS_HOLE(bp)); 164244cd46caSbillm ASSERT3U(BP_GET_NDVAS(bp), ==, 0); 164344cd46caSbillm ASSERT3U(zio->io_ndvas, >, 0); 16448654d025Sperrin ASSERT3U(zio->io_ndvas, <=, spa_max_replication(spa)); 1645fa9e4066Sahrens 16460a4e9518Sgw /* 16470a4e9518Sgw * For testing purposes, we force I/Os to retry. We don't allow 16480a4e9518Sgw * retries beyond the first pass since those I/Os are non-allocating 1649e9ed0a81Sgw * writes. 16500a4e9518Sgw */ 16510a4e9518Sgw if (zio_io_fail_shift && 16520a4e9518Sgw spa_sync_pass(zio->io_spa) <= zio_sync_pass.zp_rewrite && 16530a4e9518Sgw zio_io_should_fail(zio_io_fail_shift)) 16540a4e9518Sgw zio->io_flags |= ZIO_FLAG_WRITE_RETRY; 16550a4e9518Sgw 1656fa9e4066Sahrens ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 1657fa9e4066Sahrens 16588654d025Sperrin error = metaslab_alloc(spa, mc, zio->io_size, bp, zio->io_ndvas, 165967bd71c6Sperrin zio->io_txg, NULL, B_FALSE); 1660fa9e4066Sahrens 1661fa9e4066Sahrens if (error == 0) { 1662fa9e4066Sahrens bp->blk_birth = zio->io_txg; 16630a4e9518Sgw } else if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE) { 1664e05725b1Sbonwick return (zio_write_allocate_gang_members(zio, mc)); 1665fa9e4066Sahrens } else { 1666fa9e4066Sahrens zio->io_error = error; 1667fa9e4066Sahrens } 1668e05725b1Sbonwick 1669e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 1670fa9e4066Sahrens } 1671fa9e4066Sahrens 1672e05725b1Sbonwick static int 1673fa9e4066Sahrens zio_dva_free(zio_t *zio) 1674fa9e4066Sahrens { 1675fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 1676fa9e4066Sahrens 1677d80c45e0Sbonwick metaslab_free(zio->io_spa, bp, zio->io_txg, B_FALSE); 1678fa9e4066Sahrens 1679fa9e4066Sahrens BP_ZERO(bp); 1680fa9e4066Sahrens 1681e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 1682fa9e4066Sahrens } 1683fa9e4066Sahrens 1684e05725b1Sbonwick static int 1685fa9e4066Sahrens zio_dva_claim(zio_t *zio) 1686fa9e4066Sahrens { 1687d80c45e0Sbonwick zio->io_error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg); 1688fa9e4066Sahrens 1689e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 1690fa9e4066Sahrens } 1691fa9e4066Sahrens 1692fa9e4066Sahrens /* 1693fa9e4066Sahrens * ========================================================================== 1694fa9e4066Sahrens * Read and write to physical devices 1695fa9e4066Sahrens * ========================================================================== 1696fa9e4066Sahrens */ 1697fa9e4066Sahrens 1698e05725b1Sbonwick static int 169944cd46caSbillm zio_vdev_io_start(zio_t *zio) 1700fa9e4066Sahrens { 1701fa9e4066Sahrens vdev_t *vd = zio->io_vd; 170244cd46caSbillm vdev_t *tvd = vd ? vd->vdev_top : NULL; 170344cd46caSbillm blkptr_t *bp = zio->io_bp; 170444cd46caSbillm uint64_t align; 17050a4e9518Sgw spa_t *spa = zio->io_spa; 17060a4e9518Sgw 17070a4e9518Sgw /* 17080a4e9518Sgw * If the pool is already in a failure state then just suspend 17090a4e9518Sgw * this IO until the problem is resolved. We will reissue them 17100a4e9518Sgw * at that time. 17110a4e9518Sgw */ 17120a4e9518Sgw if (spa_state(spa) == POOL_STATE_IO_FAILURE && 1713e05725b1Sbonwick zio->io_type == ZIO_TYPE_WRITE) 1714e05725b1Sbonwick return (zio_vdev_suspend_io(zio)); 171544cd46caSbillm 1716e05725b1Sbonwick /* 1717e05725b1Sbonwick * The mirror_ops handle multiple DVAs in a single BP 1718e05725b1Sbonwick */ 1719e05725b1Sbonwick if (vd == NULL) 1720e05725b1Sbonwick return (vdev_mirror_ops.vdev_op_io_start(zio)); 172144cd46caSbillm 172244cd46caSbillm align = 1ULL << tvd->vdev_ashift; 1723fa9e4066Sahrens 1724ecc2d604Sbonwick if (zio->io_retries == 0 && vd == tvd) 1725fa9e4066Sahrens zio->io_flags |= ZIO_FLAG_FAILFAST; 1726fa9e4066Sahrens 1727e05725b1Sbonwick if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) && vd->vdev_children == 0) { 1728fa9e4066Sahrens zio->io_flags |= ZIO_FLAG_PHYSICAL; 1729fa9e4066Sahrens zio->io_offset += VDEV_LABEL_START_SIZE; 1730fa9e4066Sahrens } 1731fa9e4066Sahrens 1732ecc2d604Sbonwick if (P2PHASE(zio->io_size, align) != 0) { 1733ecc2d604Sbonwick uint64_t asize = P2ROUNDUP(zio->io_size, align); 1734ecc2d604Sbonwick char *abuf = zio_buf_alloc(asize); 1735ecc2d604Sbonwick ASSERT(vd == tvd); 1736ecc2d604Sbonwick if (zio->io_type == ZIO_TYPE_WRITE) { 1737ecc2d604Sbonwick bcopy(zio->io_data, abuf, zio->io_size); 1738ecc2d604Sbonwick bzero(abuf + zio->io_size, asize - zio->io_size); 1739ecc2d604Sbonwick } 1740ecc2d604Sbonwick zio_push_transform(zio, abuf, asize, asize); 1741ecc2d604Sbonwick ASSERT(!(zio->io_flags & ZIO_FLAG_SUBBLOCK)); 1742ecc2d604Sbonwick zio->io_flags |= ZIO_FLAG_SUBBLOCK; 1743ecc2d604Sbonwick } 1744ecc2d604Sbonwick 1745ecc2d604Sbonwick ASSERT(P2PHASE(zio->io_offset, align) == 0); 1746ecc2d604Sbonwick ASSERT(P2PHASE(zio->io_size, align) == 0); 1747ecc2d604Sbonwick ASSERT(bp == NULL || 1748ecc2d604Sbonwick P2ROUNDUP(ZIO_GET_IOSIZE(zio), align) == zio->io_size); 1749fa9e4066Sahrens ASSERT(zio->io_type != ZIO_TYPE_WRITE || (spa_mode & FWRITE)); 1750fa9e4066Sahrens 1751e05725b1Sbonwick return (vd->vdev_ops->vdev_op_io_start(zio)); 1752fa9e4066Sahrens } 1753fa9e4066Sahrens 1754e05725b1Sbonwick static int 1755fa9e4066Sahrens zio_vdev_io_done(zio_t *zio) 1756fa9e4066Sahrens { 175744cd46caSbillm if (zio->io_vd == NULL) 1758e05725b1Sbonwick return (vdev_mirror_ops.vdev_op_io_done(zio)); 1759e05725b1Sbonwick 1760e05725b1Sbonwick return (zio->io_vd->vdev_ops->vdev_op_io_done(zio)); 1761fa9e4066Sahrens } 1762fa9e4066Sahrens 1763fa9e4066Sahrens /* XXPOLICY */ 1764ea8dc4b6Seschrock boolean_t 1765fa9e4066Sahrens zio_should_retry(zio_t *zio) 1766fa9e4066Sahrens { 1767fa9e4066Sahrens vdev_t *vd = zio->io_vd; 1768fa9e4066Sahrens 1769fa9e4066Sahrens if (zio->io_error == 0) 1770fa9e4066Sahrens return (B_FALSE); 1771fa9e4066Sahrens if (zio->io_delegate_list != NULL) 1772fa9e4066Sahrens return (B_FALSE); 177344cd46caSbillm if (vd && vd != vd->vdev_top) 1774fa9e4066Sahrens return (B_FALSE); 1775fa9e4066Sahrens if (zio->io_flags & ZIO_FLAG_DONT_RETRY) 1776fa9e4066Sahrens return (B_FALSE); 1777ea8dc4b6Seschrock if (zio->io_retries > 0) 1778fa9e4066Sahrens return (B_FALSE); 1779fa9e4066Sahrens 1780fa9e4066Sahrens return (B_TRUE); 1781fa9e4066Sahrens } 1782fa9e4066Sahrens 1783e05725b1Sbonwick static int 1784fa9e4066Sahrens zio_vdev_io_assess(zio_t *zio) 1785fa9e4066Sahrens { 1786fa9e4066Sahrens vdev_t *vd = zio->io_vd; 178744cd46caSbillm vdev_t *tvd = vd ? vd->vdev_top : NULL; 1788fa9e4066Sahrens 1789fa9e4066Sahrens ASSERT(zio->io_vsd == NULL); 1790fa9e4066Sahrens 1791ecc2d604Sbonwick if (zio->io_flags & ZIO_FLAG_SUBBLOCK) { 1792ecc2d604Sbonwick void *abuf; 1793ecc2d604Sbonwick uint64_t asize; 1794ecc2d604Sbonwick ASSERT(vd == tvd); 1795ecc2d604Sbonwick zio_pop_transform(zio, &abuf, &asize, &asize); 1796ecc2d604Sbonwick if (zio->io_type == ZIO_TYPE_READ) 1797ecc2d604Sbonwick bcopy(abuf, zio->io_data, zio->io_size); 1798ecc2d604Sbonwick zio_buf_free(abuf, asize); 1799ecc2d604Sbonwick zio->io_flags &= ~ZIO_FLAG_SUBBLOCK; 1800ecc2d604Sbonwick } 1801ecc2d604Sbonwick 1802ea8dc4b6Seschrock if (zio_injection_enabled && !zio->io_error) 1803ea8dc4b6Seschrock zio->io_error = zio_handle_fault_injection(zio, EIO); 1804ea8dc4b6Seschrock 1805fa9e4066Sahrens /* 1806fa9e4066Sahrens * If the I/O failed, determine whether we should attempt to retry it. 1807fa9e4066Sahrens */ 1808fa9e4066Sahrens /* XXPOLICY */ 1809fa9e4066Sahrens if (zio_should_retry(zio)) { 1810fa9e4066Sahrens ASSERT(tvd == vd); 1811fa9e4066Sahrens 1812fa9e4066Sahrens zio->io_retries++; 1813fa9e4066Sahrens zio->io_error = 0; 181417f17c2dSbonwick zio->io_flags &= ZIO_FLAG_RETRY_INHERIT; 1815fa9e4066Sahrens /* XXPOLICY */ 1816fa9e4066Sahrens zio->io_flags &= ~ZIO_FLAG_FAILFAST; 1817fa9e4066Sahrens zio->io_flags |= ZIO_FLAG_DONT_CACHE; 181844cd46caSbillm zio->io_stage = ZIO_STAGE_VDEV_IO_START - 1; 1819fa9e4066Sahrens 1820e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 1821ea8dc4b6Seschrock } 1822fa9e4066Sahrens 1823e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 1824fa9e4066Sahrens } 1825fa9e4066Sahrens 1826fa9e4066Sahrens void 1827fa9e4066Sahrens zio_vdev_io_reissue(zio_t *zio) 1828fa9e4066Sahrens { 1829fa9e4066Sahrens ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 1830fa9e4066Sahrens ASSERT(zio->io_error == 0); 1831fa9e4066Sahrens 1832fa9e4066Sahrens zio->io_stage--; 1833fa9e4066Sahrens } 1834fa9e4066Sahrens 1835fa9e4066Sahrens void 1836fa9e4066Sahrens zio_vdev_io_redone(zio_t *zio) 1837fa9e4066Sahrens { 1838fa9e4066Sahrens ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); 1839fa9e4066Sahrens 1840fa9e4066Sahrens zio->io_stage--; 1841fa9e4066Sahrens } 1842fa9e4066Sahrens 1843fa9e4066Sahrens void 1844fa9e4066Sahrens zio_vdev_io_bypass(zio_t *zio) 1845fa9e4066Sahrens { 1846fa9e4066Sahrens ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 1847fa9e4066Sahrens ASSERT(zio->io_error == 0); 1848fa9e4066Sahrens 1849fa9e4066Sahrens zio->io_flags |= ZIO_FLAG_IO_BYPASS; 1850fa9e4066Sahrens zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS - 1; 1851fa9e4066Sahrens } 1852fa9e4066Sahrens 1853fa9e4066Sahrens /* 1854fa9e4066Sahrens * ========================================================================== 1855fa9e4066Sahrens * Generate and verify checksums 1856fa9e4066Sahrens * ========================================================================== 1857fa9e4066Sahrens */ 1858e05725b1Sbonwick static int 1859fa9e4066Sahrens zio_checksum_generate(zio_t *zio) 1860fa9e4066Sahrens { 1861fa9e4066Sahrens int checksum = zio->io_checksum; 1862fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 1863fa9e4066Sahrens 1864fa9e4066Sahrens ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 1865fa9e4066Sahrens 1866fa9e4066Sahrens BP_SET_CHECKSUM(bp, checksum); 1867fa9e4066Sahrens BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1868fa9e4066Sahrens 1869fa9e4066Sahrens zio_checksum(checksum, &bp->blk_cksum, zio->io_data, zio->io_size); 1870fa9e4066Sahrens 1871e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 1872fa9e4066Sahrens } 1873fa9e4066Sahrens 1874e05725b1Sbonwick static int 1875fa9e4066Sahrens zio_gang_checksum_generate(zio_t *zio) 1876fa9e4066Sahrens { 1877fa9e4066Sahrens zio_cksum_t zc; 1878fa9e4066Sahrens zio_gbh_phys_t *gbh = zio->io_data; 1879fa9e4066Sahrens 188044cd46caSbillm ASSERT(BP_IS_GANG(zio->io_bp)); 1881fa9e4066Sahrens ASSERT3U(zio->io_size, ==, SPA_GANGBLOCKSIZE); 1882fa9e4066Sahrens 1883fa9e4066Sahrens zio_set_gang_verifier(zio, &gbh->zg_tail.zbt_cksum); 1884fa9e4066Sahrens 1885fa9e4066Sahrens zio_checksum(ZIO_CHECKSUM_GANG_HEADER, &zc, zio->io_data, zio->io_size); 1886fa9e4066Sahrens 1887e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 1888fa9e4066Sahrens } 1889fa9e4066Sahrens 1890e05725b1Sbonwick static int 1891fa9e4066Sahrens zio_checksum_verify(zio_t *zio) 1892fa9e4066Sahrens { 1893fa9e4066Sahrens if (zio->io_bp != NULL) { 1894fa9e4066Sahrens zio->io_error = zio_checksum_error(zio); 1895ea8dc4b6Seschrock if (zio->io_error && !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) 1896ea8dc4b6Seschrock zfs_ereport_post(FM_EREPORT_ZFS_CHECKSUM, 1897ea8dc4b6Seschrock zio->io_spa, zio->io_vd, zio, 0, 0); 1898fa9e4066Sahrens } 1899fa9e4066Sahrens 1900e05725b1Sbonwick return (ZIO_PIPELINE_CONTINUE); 1901fa9e4066Sahrens } 1902fa9e4066Sahrens 1903fa9e4066Sahrens /* 1904fa9e4066Sahrens * Called by RAID-Z to ensure we don't compute the checksum twice. 1905fa9e4066Sahrens */ 1906fa9e4066Sahrens void 1907fa9e4066Sahrens zio_checksum_verified(zio_t *zio) 1908fa9e4066Sahrens { 1909fa9e4066Sahrens zio->io_pipeline &= ~(1U << ZIO_STAGE_CHECKSUM_VERIFY); 1910fa9e4066Sahrens } 1911fa9e4066Sahrens 1912fa9e4066Sahrens /* 1913fa9e4066Sahrens * Set the external verifier for a gang block based on stuff in the bp 1914fa9e4066Sahrens */ 1915fa9e4066Sahrens void 1916fa9e4066Sahrens zio_set_gang_verifier(zio_t *zio, zio_cksum_t *zcp) 1917fa9e4066Sahrens { 191844cd46caSbillm blkptr_t *bp = zio->io_bp; 191944cd46caSbillm 192044cd46caSbillm zcp->zc_word[0] = DVA_GET_VDEV(BP_IDENTITY(bp)); 192144cd46caSbillm zcp->zc_word[1] = DVA_GET_OFFSET(BP_IDENTITY(bp)); 192244cd46caSbillm zcp->zc_word[2] = bp->blk_birth; 1923fa9e4066Sahrens zcp->zc_word[3] = 0; 1924fa9e4066Sahrens } 1925fa9e4066Sahrens 1926fa9e4066Sahrens /* 1927fa9e4066Sahrens * ========================================================================== 1928fa9e4066Sahrens * Define the pipeline 1929fa9e4066Sahrens * ========================================================================== 1930fa9e4066Sahrens */ 1931e05725b1Sbonwick typedef int zio_pipe_stage_t(zio_t *zio); 1932fa9e4066Sahrens 1933fa9e4066Sahrens zio_pipe_stage_t *zio_pipeline[ZIO_STAGE_DONE + 2] = { 1934e05725b1Sbonwick NULL, 1935e05725b1Sbonwick zio_wait_for_children_ready, 1936e05725b1Sbonwick zio_read_init, 1937e05725b1Sbonwick zio_issue_async, 1938fa9e4066Sahrens zio_write_compress, 1939fa9e4066Sahrens zio_checksum_generate, 1940fa9e4066Sahrens zio_get_gang_header, 1941fa9e4066Sahrens zio_rewrite_gang_members, 1942fa9e4066Sahrens zio_free_gang_members, 1943fa9e4066Sahrens zio_claim_gang_members, 1944fa9e4066Sahrens zio_dva_allocate, 1945fa9e4066Sahrens zio_dva_free, 1946fa9e4066Sahrens zio_dva_claim, 1947fa9e4066Sahrens zio_gang_checksum_generate, 1948fa9e4066Sahrens zio_ready, 1949fa9e4066Sahrens zio_vdev_io_start, 1950fa9e4066Sahrens zio_vdev_io_done, 1951fa9e4066Sahrens zio_vdev_io_assess, 1952e05725b1Sbonwick zio_wait_for_children_done, 1953fa9e4066Sahrens zio_checksum_verify, 1954fa9e4066Sahrens zio_read_gang_members, 1955fa9e4066Sahrens zio_read_decompress, 19560a4e9518Sgw zio_assess, 1957fa9e4066Sahrens zio_done, 1958e05725b1Sbonwick NULL 1959fa9e4066Sahrens }; 1960fa9e4066Sahrens 1961fa9e4066Sahrens /* 1962e05725b1Sbonwick * Execute the I/O pipeline until one of the following occurs: 1963e05725b1Sbonwick * (1) the I/O completes; (2) the pipeline stalls waiting for 1964e05725b1Sbonwick * dependent child I/Os; (3) the I/O issues, so we're waiting 1965e05725b1Sbonwick * for an I/O completion interrupt; (4) the I/O is delegated by 1966e05725b1Sbonwick * vdev-level caching or aggregation; (5) the I/O is deferred 1967e05725b1Sbonwick * due to vdev-level queueing; (6) the I/O is handed off to 1968e05725b1Sbonwick * another thread. In all cases, the pipeline stops whenever 1969e05725b1Sbonwick * there's no CPU work; it never burns a thread in cv_wait(). 1970e05725b1Sbonwick * 1971e05725b1Sbonwick * There's no locking on io_stage because there's no legitimate way 1972e05725b1Sbonwick * for multiple threads to be attempting to process the same I/O. 1973fa9e4066Sahrens */ 1974fa9e4066Sahrens void 1975e05725b1Sbonwick zio_execute(zio_t *zio) 1976fa9e4066Sahrens { 1977e05725b1Sbonwick while (zio->io_stage < ZIO_STAGE_DONE) { 1978e05725b1Sbonwick uint32_t pipeline = zio->io_pipeline; 1979e05725b1Sbonwick int rv; 1980fa9e4066Sahrens 1981e05725b1Sbonwick ASSERT(!MUTEX_HELD(&zio->io_lock)); 1982fa9e4066Sahrens 1983e05725b1Sbonwick /* 1984e05725b1Sbonwick * If an error occurred outside the vdev stack, 1985e05725b1Sbonwick * just execute the interlock stages to clean up. 1986e05725b1Sbonwick */ 1987e05725b1Sbonwick if (zio->io_error && 1988e05725b1Sbonwick ((1U << zio->io_stage) & ZIO_VDEV_IO_STAGES) == 0) 1989fa9e4066Sahrens pipeline &= ZIO_ERROR_PIPELINE_MASK; 1990fa9e4066Sahrens 1991e05725b1Sbonwick while (((1U << ++zio->io_stage) & pipeline) == 0) 1992e05725b1Sbonwick continue; 1993fa9e4066Sahrens 1994e05725b1Sbonwick ASSERT(zio->io_stage <= ZIO_STAGE_DONE); 1995e05725b1Sbonwick ASSERT(zio->io_stalled == 0); 1996fa9e4066Sahrens 1997e05725b1Sbonwick rv = zio_pipeline[zio->io_stage](zio); 1998fa9e4066Sahrens 1999e05725b1Sbonwick if (rv == ZIO_PIPELINE_STOP) 2000e05725b1Sbonwick return; 2001fa9e4066Sahrens 2002e05725b1Sbonwick ASSERT(rv == ZIO_PIPELINE_CONTINUE); 2003fa9e4066Sahrens } 2004fa9e4066Sahrens } 2005fa9e4066Sahrens 2006d63d470bSgw static boolean_t 20070a4e9518Sgw zio_io_should_fail(uint16_t range) 2008d63d470bSgw { 2009d63d470bSgw static uint16_t allocs = 0; 2010d63d470bSgw 20110a4e9518Sgw return (P2PHASE(allocs++, 1U<<range) == 0); 2012d63d470bSgw } 2013d63d470bSgw 2014fa9e4066Sahrens /* 2015fa9e4066Sahrens * Try to allocate an intent log block. Return 0 on success, errno on failure. 2016fa9e4066Sahrens */ 2017fa9e4066Sahrens int 201867bd71c6Sperrin zio_alloc_blk(spa_t *spa, uint64_t size, blkptr_t *new_bp, blkptr_t *old_bp, 201967bd71c6Sperrin uint64_t txg) 2020fa9e4066Sahrens { 2021fa9e4066Sahrens int error; 2022fa9e4066Sahrens 2023ea8dc4b6Seschrock spa_config_enter(spa, RW_READER, FTAG); 2024fa9e4066Sahrens 20250a4e9518Sgw if (zio_zil_fail_shift && zio_io_should_fail(zio_zil_fail_shift)) { 2026d63d470bSgw spa_config_exit(spa, FTAG); 2027d63d470bSgw return (ENOSPC); 2028d63d470bSgw } 2029d63d470bSgw 203067bd71c6Sperrin /* 20318654d025Sperrin * We were passed the previous log block's DVA in bp->blk_dva[0]. 20328654d025Sperrin * We use that as a hint for which vdev to allocate from next. 203367bd71c6Sperrin */ 20348654d025Sperrin error = metaslab_alloc(spa, spa->spa_log_class, size, 20358654d025Sperrin new_bp, 1, txg, old_bp, B_TRUE); 20368654d025Sperrin 20378654d025Sperrin if (error) 20388654d025Sperrin error = metaslab_alloc(spa, spa->spa_normal_class, size, 20398654d025Sperrin new_bp, 1, txg, old_bp, B_TRUE); 2040fa9e4066Sahrens 2041fa9e4066Sahrens if (error == 0) { 204267bd71c6Sperrin BP_SET_LSIZE(new_bp, size); 204367bd71c6Sperrin BP_SET_PSIZE(new_bp, size); 204467bd71c6Sperrin BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF); 204567bd71c6Sperrin BP_SET_CHECKSUM(new_bp, ZIO_CHECKSUM_ZILOG); 204667bd71c6Sperrin BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); 204767bd71c6Sperrin BP_SET_LEVEL(new_bp, 0); 204867bd71c6Sperrin BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER); 204967bd71c6Sperrin new_bp->blk_birth = txg; 2050fa9e4066Sahrens } 2051fa9e4066Sahrens 2052ea8dc4b6Seschrock spa_config_exit(spa, FTAG); 2053fa9e4066Sahrens 2054fa9e4066Sahrens return (error); 2055fa9e4066Sahrens } 2056fa9e4066Sahrens 2057fa9e4066Sahrens /* 2058fa9e4066Sahrens * Free an intent log block. We know it can't be a gang block, so there's 2059fa9e4066Sahrens * nothing to do except metaslab_free() it. 2060fa9e4066Sahrens */ 2061fa9e4066Sahrens void 2062fa9e4066Sahrens zio_free_blk(spa_t *spa, blkptr_t *bp, uint64_t txg) 2063fa9e4066Sahrens { 206444cd46caSbillm ASSERT(!BP_IS_GANG(bp)); 2065fa9e4066Sahrens 2066ea8dc4b6Seschrock spa_config_enter(spa, RW_READER, FTAG); 2067fa9e4066Sahrens 2068d80c45e0Sbonwick metaslab_free(spa, bp, txg, B_FALSE); 2069fa9e4066Sahrens 2070ea8dc4b6Seschrock spa_config_exit(spa, FTAG); 2071fa9e4066Sahrens } 207246341222Sperrin 207346341222Sperrin /* 207446341222Sperrin * start an async flush of the write cache for this vdev 207546341222Sperrin */ 207646341222Sperrin void 207717f17c2dSbonwick zio_flush(zio_t *zio, vdev_t *vd) 207846341222Sperrin { 207917f17c2dSbonwick zio_nowait(zio_ioctl(zio, zio->io_spa, vd, DKIOCFLUSHWRITECACHE, 208046341222Sperrin NULL, NULL, ZIO_PRIORITY_NOW, 208146341222Sperrin ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_RETRY)); 208246341222Sperrin } 2083