1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5ea8dc4b6Seschrock * Common Development and Distribution License (the "License"). 6ea8dc4b6Seschrock * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 22d58459f4Sek * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23fa9e4066Sahrens * Use is subject to license terms. 24fa9e4066Sahrens */ 25fa9e4066Sahrens 26fa9e4066Sahrens #pragma ident "%Z%%M% %I% %E% SMI" 27fa9e4066Sahrens 28fa9e4066Sahrens #include <sys/zfs_context.h> 29ea8dc4b6Seschrock #include <sys/fm/fs/zfs.h> 30fa9e4066Sahrens #include <sys/spa.h> 31fa9e4066Sahrens #include <sys/txg.h> 32fa9e4066Sahrens #include <sys/spa_impl.h> 33fa9e4066Sahrens #include <sys/vdev_impl.h> 34fa9e4066Sahrens #include <sys/zio_impl.h> 35fa9e4066Sahrens #include <sys/zio_compress.h> 36fa9e4066Sahrens #include <sys/zio_checksum.h> 37fa9e4066Sahrens 38fa9e4066Sahrens /* 39fa9e4066Sahrens * ========================================================================== 40fa9e4066Sahrens * I/O priority table 41fa9e4066Sahrens * ========================================================================== 42fa9e4066Sahrens */ 43fa9e4066Sahrens uint8_t zio_priority_table[ZIO_PRIORITY_TABLE_SIZE] = { 44fa9e4066Sahrens 0, /* ZIO_PRIORITY_NOW */ 45fa9e4066Sahrens 0, /* ZIO_PRIORITY_SYNC_READ */ 46fa9e4066Sahrens 0, /* ZIO_PRIORITY_SYNC_WRITE */ 47fa9e4066Sahrens 6, /* ZIO_PRIORITY_ASYNC_READ */ 48fa9e4066Sahrens 4, /* ZIO_PRIORITY_ASYNC_WRITE */ 49fa9e4066Sahrens 4, /* ZIO_PRIORITY_FREE */ 50fa9e4066Sahrens 0, /* ZIO_PRIORITY_CACHE_FILL */ 51fa9e4066Sahrens 0, /* ZIO_PRIORITY_LOG_WRITE */ 52fa9e4066Sahrens 10, /* ZIO_PRIORITY_RESILVER */ 53fa9e4066Sahrens 20, /* ZIO_PRIORITY_SCRUB */ 54fa9e4066Sahrens }; 55fa9e4066Sahrens 56fa9e4066Sahrens /* 57fa9e4066Sahrens * ========================================================================== 58fa9e4066Sahrens * I/O type descriptions 59fa9e4066Sahrens * ========================================================================== 60fa9e4066Sahrens */ 61fa9e4066Sahrens char *zio_type_name[ZIO_TYPES] = { 62fa9e4066Sahrens "null", "read", "write", "free", "claim", "ioctl" }; 63fa9e4066Sahrens 64fa9e4066Sahrens /* At or above this size, force gang blocking - for testing */ 65fa9e4066Sahrens uint64_t zio_gang_bang = SPA_MAXBLOCKSIZE + 1; 66fa9e4066Sahrens 67d63d470bSgw /* Force an allocation failure when non-zero */ 68d63d470bSgw uint16_t zio_zil_fail_shift = 0; 69d63d470bSgw 70fa9e4066Sahrens typedef struct zio_sync_pass { 71fa9e4066Sahrens int zp_defer_free; /* defer frees after this pass */ 72fa9e4066Sahrens int zp_dontcompress; /* don't compress after this pass */ 73fa9e4066Sahrens int zp_rewrite; /* rewrite new bps after this pass */ 74fa9e4066Sahrens } zio_sync_pass_t; 75fa9e4066Sahrens 76fa9e4066Sahrens zio_sync_pass_t zio_sync_pass = { 77fa9e4066Sahrens 1, /* zp_defer_free */ 78fa9e4066Sahrens 4, /* zp_dontcompress */ 79fa9e4066Sahrens 1, /* zp_rewrite */ 80fa9e4066Sahrens }; 81fa9e4066Sahrens 82fa9e4066Sahrens /* 83fa9e4066Sahrens * ========================================================================== 84fa9e4066Sahrens * I/O kmem caches 85fa9e4066Sahrens * ========================================================================== 86fa9e4066Sahrens */ 87*ccae0b50Seschrock kmem_cache_t *zio_cache; 88fa9e4066Sahrens kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 89ad23a2dbSjohansen kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 90ad23a2dbSjohansen 91ad23a2dbSjohansen #ifdef _KERNEL 92ad23a2dbSjohansen extern vmem_t *zio_alloc_arena; 93ad23a2dbSjohansen #endif 94fa9e4066Sahrens 95fa9e4066Sahrens void 96fa9e4066Sahrens zio_init(void) 97fa9e4066Sahrens { 98fa9e4066Sahrens size_t c; 99ad23a2dbSjohansen vmem_t *data_alloc_arena = NULL; 100ad23a2dbSjohansen 101ad23a2dbSjohansen #ifdef _KERNEL 102ad23a2dbSjohansen data_alloc_arena = zio_alloc_arena; 103ad23a2dbSjohansen #endif 104fa9e4066Sahrens 105*ccae0b50Seschrock zio_cache = kmem_cache_create("zio_cache", sizeof (zio_t), 0, 106*ccae0b50Seschrock NULL, NULL, NULL, NULL, NULL, 0); 107*ccae0b50Seschrock 108fa9e4066Sahrens /* 109fa9e4066Sahrens * For small buffers, we want a cache for each multiple of 110fa9e4066Sahrens * SPA_MINBLOCKSIZE. For medium-size buffers, we want a cache 111fa9e4066Sahrens * for each quarter-power of 2. For large buffers, we want 112fa9e4066Sahrens * a cache for each multiple of PAGESIZE. 113fa9e4066Sahrens */ 114fa9e4066Sahrens for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 115fa9e4066Sahrens size_t size = (c + 1) << SPA_MINBLOCKSHIFT; 116fa9e4066Sahrens size_t p2 = size; 117fa9e4066Sahrens size_t align = 0; 118fa9e4066Sahrens 119fa9e4066Sahrens while (p2 & (p2 - 1)) 120fa9e4066Sahrens p2 &= p2 - 1; 121fa9e4066Sahrens 122fa9e4066Sahrens if (size <= 4 * SPA_MINBLOCKSIZE) { 123fa9e4066Sahrens align = SPA_MINBLOCKSIZE; 124fa9e4066Sahrens } else if (P2PHASE(size, PAGESIZE) == 0) { 125fa9e4066Sahrens align = PAGESIZE; 126fa9e4066Sahrens } else if (P2PHASE(size, p2 >> 2) == 0) { 127fa9e4066Sahrens align = p2 >> 2; 128fa9e4066Sahrens } 129fa9e4066Sahrens 130fa9e4066Sahrens if (align != 0) { 131ad23a2dbSjohansen char name[36]; 1325ad82045Snd (void) sprintf(name, "zio_buf_%lu", (ulong_t)size); 133fa9e4066Sahrens zio_buf_cache[c] = kmem_cache_create(name, size, 134a0965f35Sbonwick align, NULL, NULL, NULL, NULL, NULL, KMC_NODEBUG); 135ad23a2dbSjohansen 136ad23a2dbSjohansen (void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size); 137ad23a2dbSjohansen zio_data_buf_cache[c] = kmem_cache_create(name, size, 138ad23a2dbSjohansen align, NULL, NULL, NULL, NULL, data_alloc_arena, 139ad23a2dbSjohansen KMC_NODEBUG); 140ad23a2dbSjohansen 141fa9e4066Sahrens dprintf("creating cache for size %5lx align %5lx\n", 142fa9e4066Sahrens size, align); 143fa9e4066Sahrens } 144fa9e4066Sahrens } 145fa9e4066Sahrens 146fa9e4066Sahrens while (--c != 0) { 147fa9e4066Sahrens ASSERT(zio_buf_cache[c] != NULL); 148fa9e4066Sahrens if (zio_buf_cache[c - 1] == NULL) 149fa9e4066Sahrens zio_buf_cache[c - 1] = zio_buf_cache[c]; 150ad23a2dbSjohansen 151ad23a2dbSjohansen ASSERT(zio_data_buf_cache[c] != NULL); 152ad23a2dbSjohansen if (zio_data_buf_cache[c - 1] == NULL) 153ad23a2dbSjohansen zio_data_buf_cache[c - 1] = zio_data_buf_cache[c]; 154fa9e4066Sahrens } 155ea8dc4b6Seschrock 156ea8dc4b6Seschrock zio_inject_init(); 157fa9e4066Sahrens } 158fa9e4066Sahrens 159fa9e4066Sahrens void 160fa9e4066Sahrens zio_fini(void) 161fa9e4066Sahrens { 162fa9e4066Sahrens size_t c; 163fa9e4066Sahrens kmem_cache_t *last_cache = NULL; 164ad23a2dbSjohansen kmem_cache_t *last_data_cache = NULL; 165fa9e4066Sahrens 166fa9e4066Sahrens for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 167fa9e4066Sahrens if (zio_buf_cache[c] != last_cache) { 168fa9e4066Sahrens last_cache = zio_buf_cache[c]; 169fa9e4066Sahrens kmem_cache_destroy(zio_buf_cache[c]); 170fa9e4066Sahrens } 171fa9e4066Sahrens zio_buf_cache[c] = NULL; 172ad23a2dbSjohansen 173ad23a2dbSjohansen if (zio_data_buf_cache[c] != last_data_cache) { 174ad23a2dbSjohansen last_data_cache = zio_data_buf_cache[c]; 175ad23a2dbSjohansen kmem_cache_destroy(zio_data_buf_cache[c]); 176ad23a2dbSjohansen } 177ad23a2dbSjohansen zio_data_buf_cache[c] = NULL; 178fa9e4066Sahrens } 179ea8dc4b6Seschrock 180*ccae0b50Seschrock kmem_cache_destroy(zio_cache); 181*ccae0b50Seschrock 182ea8dc4b6Seschrock zio_inject_fini(); 183fa9e4066Sahrens } 184fa9e4066Sahrens 185fa9e4066Sahrens /* 186fa9e4066Sahrens * ========================================================================== 187fa9e4066Sahrens * Allocate and free I/O buffers 188fa9e4066Sahrens * ========================================================================== 189fa9e4066Sahrens */ 190ad23a2dbSjohansen 191ad23a2dbSjohansen /* 192ad23a2dbSjohansen * Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a 193ad23a2dbSjohansen * crashdump if the kernel panics, so use it judiciously. Obviously, it's 194ad23a2dbSjohansen * useful to inspect ZFS metadata, but if possible, we should avoid keeping 195ad23a2dbSjohansen * excess / transient data in-core during a crashdump. 196ad23a2dbSjohansen */ 197fa9e4066Sahrens void * 198fa9e4066Sahrens zio_buf_alloc(size_t size) 199fa9e4066Sahrens { 200fa9e4066Sahrens size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 201fa9e4066Sahrens 202fa9e4066Sahrens ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 203fa9e4066Sahrens 204fa9e4066Sahrens return (kmem_cache_alloc(zio_buf_cache[c], KM_SLEEP)); 205fa9e4066Sahrens } 206fa9e4066Sahrens 207ad23a2dbSjohansen /* 208ad23a2dbSjohansen * Use zio_data_buf_alloc to allocate data. The data will not appear in a 209ad23a2dbSjohansen * crashdump if the kernel panics. This exists so that we will limit the amount 210ad23a2dbSjohansen * of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount 211ad23a2dbSjohansen * of kernel heap dumped to disk when the kernel panics) 212ad23a2dbSjohansen */ 213ad23a2dbSjohansen void * 214ad23a2dbSjohansen zio_data_buf_alloc(size_t size) 215ad23a2dbSjohansen { 216ad23a2dbSjohansen size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 217ad23a2dbSjohansen 218ad23a2dbSjohansen ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 219ad23a2dbSjohansen 220ad23a2dbSjohansen return (kmem_cache_alloc(zio_data_buf_cache[c], KM_SLEEP)); 221ad23a2dbSjohansen } 222ad23a2dbSjohansen 223fa9e4066Sahrens void 224fa9e4066Sahrens zio_buf_free(void *buf, size_t size) 225fa9e4066Sahrens { 226fa9e4066Sahrens size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 227fa9e4066Sahrens 228fa9e4066Sahrens ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 229fa9e4066Sahrens 230fa9e4066Sahrens kmem_cache_free(zio_buf_cache[c], buf); 231fa9e4066Sahrens } 232fa9e4066Sahrens 233ad23a2dbSjohansen void 234ad23a2dbSjohansen zio_data_buf_free(void *buf, size_t size) 235ad23a2dbSjohansen { 236ad23a2dbSjohansen size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 237ad23a2dbSjohansen 238ad23a2dbSjohansen ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 239ad23a2dbSjohansen 240ad23a2dbSjohansen kmem_cache_free(zio_data_buf_cache[c], buf); 241ad23a2dbSjohansen } 242b3995adbSahrens 243fa9e4066Sahrens /* 244fa9e4066Sahrens * ========================================================================== 245fa9e4066Sahrens * Push and pop I/O transform buffers 246fa9e4066Sahrens * ========================================================================== 247fa9e4066Sahrens */ 248fa9e4066Sahrens static void 249fa9e4066Sahrens zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize) 250fa9e4066Sahrens { 251fa9e4066Sahrens zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); 252fa9e4066Sahrens 253fa9e4066Sahrens zt->zt_data = data; 254fa9e4066Sahrens zt->zt_size = size; 255fa9e4066Sahrens zt->zt_bufsize = bufsize; 256fa9e4066Sahrens 257fa9e4066Sahrens zt->zt_next = zio->io_transform_stack; 258fa9e4066Sahrens zio->io_transform_stack = zt; 259fa9e4066Sahrens 260fa9e4066Sahrens zio->io_data = data; 261fa9e4066Sahrens zio->io_size = size; 262fa9e4066Sahrens } 263fa9e4066Sahrens 264fa9e4066Sahrens static void 265fa9e4066Sahrens zio_pop_transform(zio_t *zio, void **data, uint64_t *size, uint64_t *bufsize) 266fa9e4066Sahrens { 267fa9e4066Sahrens zio_transform_t *zt = zio->io_transform_stack; 268fa9e4066Sahrens 269fa9e4066Sahrens *data = zt->zt_data; 270fa9e4066Sahrens *size = zt->zt_size; 271fa9e4066Sahrens *bufsize = zt->zt_bufsize; 272fa9e4066Sahrens 273fa9e4066Sahrens zio->io_transform_stack = zt->zt_next; 274fa9e4066Sahrens kmem_free(zt, sizeof (zio_transform_t)); 275fa9e4066Sahrens 276fa9e4066Sahrens if ((zt = zio->io_transform_stack) != NULL) { 277fa9e4066Sahrens zio->io_data = zt->zt_data; 278fa9e4066Sahrens zio->io_size = zt->zt_size; 279fa9e4066Sahrens } 280fa9e4066Sahrens } 281fa9e4066Sahrens 282fa9e4066Sahrens static void 283fa9e4066Sahrens zio_clear_transform_stack(zio_t *zio) 284fa9e4066Sahrens { 285fa9e4066Sahrens void *data; 286fa9e4066Sahrens uint64_t size, bufsize; 287fa9e4066Sahrens 288fa9e4066Sahrens ASSERT(zio->io_transform_stack != NULL); 289fa9e4066Sahrens 290fa9e4066Sahrens zio_pop_transform(zio, &data, &size, &bufsize); 291fa9e4066Sahrens while (zio->io_transform_stack != NULL) { 292fa9e4066Sahrens zio_buf_free(data, bufsize); 293fa9e4066Sahrens zio_pop_transform(zio, &data, &size, &bufsize); 294fa9e4066Sahrens } 295fa9e4066Sahrens } 296fa9e4066Sahrens 297fa9e4066Sahrens /* 298fa9e4066Sahrens * ========================================================================== 299fa9e4066Sahrens * Create the various types of I/O (read, write, free) 300fa9e4066Sahrens * ========================================================================== 301fa9e4066Sahrens */ 302fa9e4066Sahrens static zio_t * 303fa9e4066Sahrens zio_create(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 304fa9e4066Sahrens void *data, uint64_t size, zio_done_func_t *done, void *private, 305fa9e4066Sahrens zio_type_t type, int priority, int flags, uint8_t stage, uint32_t pipeline) 306fa9e4066Sahrens { 307fa9e4066Sahrens zio_t *zio; 308fa9e4066Sahrens 309fa9e4066Sahrens ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); 310fa9e4066Sahrens ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 311fa9e4066Sahrens 312*ccae0b50Seschrock zio = kmem_cache_alloc(zio_cache, KM_SLEEP); 313*ccae0b50Seschrock bzero(zio, sizeof (zio_t)); 314fa9e4066Sahrens zio->io_parent = pio; 315fa9e4066Sahrens zio->io_spa = spa; 316fa9e4066Sahrens zio->io_txg = txg; 317fa9e4066Sahrens if (bp != NULL) { 318fa9e4066Sahrens zio->io_bp = bp; 319fa9e4066Sahrens zio->io_bp_copy = *bp; 320fa9e4066Sahrens zio->io_bp_orig = *bp; 321fa9e4066Sahrens } 322fa9e4066Sahrens zio->io_done = done; 323fa9e4066Sahrens zio->io_private = private; 324fa9e4066Sahrens zio->io_type = type; 325fa9e4066Sahrens zio->io_priority = priority; 326fa9e4066Sahrens zio->io_stage = stage; 327fa9e4066Sahrens zio->io_pipeline = pipeline; 328fa9e4066Sahrens zio->io_async_stages = ZIO_ASYNC_PIPELINE_STAGES; 329fa9e4066Sahrens zio->io_timestamp = lbolt64; 330fa9e4066Sahrens zio->io_flags = flags; 3315ad82045Snd mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL); 332fa9e4066Sahrens zio_push_transform(zio, data, size, size); 333fa9e4066Sahrens 334b3995adbSahrens /* 335b3995adbSahrens * Note on config lock: 336b3995adbSahrens * 337b3995adbSahrens * If CONFIG_HELD is set, then the caller already has the config 338b3995adbSahrens * lock, so we don't need it for this io. 339b3995adbSahrens * 340b3995adbSahrens * We set CONFIG_GRABBED to indicate that we have grabbed the 341b3995adbSahrens * config lock on behalf of this io, so it should be released 342b3995adbSahrens * in zio_done. 343b3995adbSahrens * 344b3995adbSahrens * Unless CONFIG_HELD is set, we will grab the config lock for 345b3995adbSahrens * any top-level (parent-less) io, *except* NULL top-level ios. 346b3995adbSahrens * The NULL top-level ios rarely have any children, so we delay 347b3995adbSahrens * grabbing the lock until the first child is added (but it is 348b3995adbSahrens * still grabbed on behalf of the top-level i/o, so additional 349b3995adbSahrens * children don't need to also grab it). This greatly reduces 350b3995adbSahrens * contention on the config lock. 351b3995adbSahrens */ 352fa9e4066Sahrens if (pio == NULL) { 353b3995adbSahrens if (type != ZIO_TYPE_NULL && 354b3995adbSahrens !(flags & ZIO_FLAG_CONFIG_HELD)) { 355ea8dc4b6Seschrock spa_config_enter(zio->io_spa, RW_READER, zio); 356b3995adbSahrens zio->io_flags |= ZIO_FLAG_CONFIG_GRABBED; 357b3995adbSahrens } 358fa9e4066Sahrens zio->io_root = zio; 359fa9e4066Sahrens } else { 360fa9e4066Sahrens zio->io_root = pio->io_root; 361ea8dc4b6Seschrock if (!(flags & ZIO_FLAG_NOBOOKMARK)) 362ea8dc4b6Seschrock zio->io_logical = pio->io_logical; 363fa9e4066Sahrens mutex_enter(&pio->io_lock); 364b3995adbSahrens if (pio->io_parent == NULL && 365b3995adbSahrens pio->io_type == ZIO_TYPE_NULL && 366b3995adbSahrens !(pio->io_flags & ZIO_FLAG_CONFIG_GRABBED) && 367b3995adbSahrens !(pio->io_flags & ZIO_FLAG_CONFIG_HELD)) { 368b3995adbSahrens pio->io_flags |= ZIO_FLAG_CONFIG_GRABBED; 369b3995adbSahrens spa_config_enter(zio->io_spa, RW_READER, pio); 370b3995adbSahrens } 371fa9e4066Sahrens if (stage < ZIO_STAGE_READY) 372fa9e4066Sahrens pio->io_children_notready++; 373fa9e4066Sahrens pio->io_children_notdone++; 374fa9e4066Sahrens zio->io_sibling_next = pio->io_child; 375fa9e4066Sahrens zio->io_sibling_prev = NULL; 376fa9e4066Sahrens if (pio->io_child != NULL) 377fa9e4066Sahrens pio->io_child->io_sibling_prev = zio; 378fa9e4066Sahrens pio->io_child = zio; 37944cd46caSbillm zio->io_ndvas = pio->io_ndvas; 380fa9e4066Sahrens mutex_exit(&pio->io_lock); 381fa9e4066Sahrens } 382fa9e4066Sahrens 383fa9e4066Sahrens return (zio); 384fa9e4066Sahrens } 385fa9e4066Sahrens 386fa9e4066Sahrens zio_t * 387fa9e4066Sahrens zio_null(zio_t *pio, spa_t *spa, zio_done_func_t *done, void *private, 388fa9e4066Sahrens int flags) 389fa9e4066Sahrens { 390fa9e4066Sahrens zio_t *zio; 391fa9e4066Sahrens 392fa9e4066Sahrens zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 393fa9e4066Sahrens ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, ZIO_STAGE_OPEN, 394fa9e4066Sahrens ZIO_WAIT_FOR_CHILDREN_PIPELINE); 395fa9e4066Sahrens 396fa9e4066Sahrens return (zio); 397fa9e4066Sahrens } 398fa9e4066Sahrens 399fa9e4066Sahrens zio_t * 400fa9e4066Sahrens zio_root(spa_t *spa, zio_done_func_t *done, void *private, int flags) 401fa9e4066Sahrens { 402fa9e4066Sahrens return (zio_null(NULL, spa, done, private, flags)); 403fa9e4066Sahrens } 404fa9e4066Sahrens 405fa9e4066Sahrens zio_t * 406fa9e4066Sahrens zio_read(zio_t *pio, spa_t *spa, blkptr_t *bp, void *data, 407fa9e4066Sahrens uint64_t size, zio_done_func_t *done, void *private, 408ea8dc4b6Seschrock int priority, int flags, zbookmark_t *zb) 409fa9e4066Sahrens { 410fa9e4066Sahrens zio_t *zio; 411fa9e4066Sahrens 412fa9e4066Sahrens ASSERT3U(size, ==, BP_GET_LSIZE(bp)); 413fa9e4066Sahrens 414fa9e4066Sahrens zio = zio_create(pio, spa, bp->blk_birth, bp, data, size, done, private, 415faafa6e3Sahrens ZIO_TYPE_READ, priority, flags | ZIO_FLAG_USER, 416faafa6e3Sahrens ZIO_STAGE_OPEN, ZIO_READ_PIPELINE); 417ea8dc4b6Seschrock zio->io_bookmark = *zb; 418ea8dc4b6Seschrock 419ea8dc4b6Seschrock zio->io_logical = zio; 420fa9e4066Sahrens 421fa9e4066Sahrens /* 422fa9e4066Sahrens * Work off our copy of the bp so the caller can free it. 423fa9e4066Sahrens */ 424fa9e4066Sahrens zio->io_bp = &zio->io_bp_copy; 425fa9e4066Sahrens 426fa9e4066Sahrens if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) { 427fa9e4066Sahrens uint64_t csize = BP_GET_PSIZE(bp); 428fa9e4066Sahrens void *cbuf = zio_buf_alloc(csize); 429fa9e4066Sahrens 430fa9e4066Sahrens zio_push_transform(zio, cbuf, csize, csize); 431fa9e4066Sahrens zio->io_pipeline |= 1U << ZIO_STAGE_READ_DECOMPRESS; 432fa9e4066Sahrens } 433fa9e4066Sahrens 43444cd46caSbillm if (BP_IS_GANG(bp)) { 435fa9e4066Sahrens uint64_t gsize = SPA_GANGBLOCKSIZE; 436fa9e4066Sahrens void *gbuf = zio_buf_alloc(gsize); 437fa9e4066Sahrens 438fa9e4066Sahrens zio_push_transform(zio, gbuf, gsize, gsize); 439fa9e4066Sahrens zio->io_pipeline |= 1U << ZIO_STAGE_READ_GANG_MEMBERS; 440fa9e4066Sahrens } 441fa9e4066Sahrens 442fa9e4066Sahrens return (zio); 443fa9e4066Sahrens } 444fa9e4066Sahrens 445fa9e4066Sahrens zio_t * 44644cd46caSbillm zio_write(zio_t *pio, spa_t *spa, int checksum, int compress, int ncopies, 447fa9e4066Sahrens uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 448c717a561Smaybee zio_done_func_t *ready, zio_done_func_t *done, void *private, int priority, 449c717a561Smaybee int flags, zbookmark_t *zb) 450fa9e4066Sahrens { 451fa9e4066Sahrens zio_t *zio; 452fa9e4066Sahrens 453fa9e4066Sahrens ASSERT(checksum >= ZIO_CHECKSUM_OFF && 454fa9e4066Sahrens checksum < ZIO_CHECKSUM_FUNCTIONS); 455fa9e4066Sahrens 456fa9e4066Sahrens ASSERT(compress >= ZIO_COMPRESS_OFF && 457fa9e4066Sahrens compress < ZIO_COMPRESS_FUNCTIONS); 458fa9e4066Sahrens 459fa9e4066Sahrens zio = zio_create(pio, spa, txg, bp, data, size, done, private, 460faafa6e3Sahrens ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_USER, 461fa9e4066Sahrens ZIO_STAGE_OPEN, ZIO_WRITE_PIPELINE); 462fa9e4066Sahrens 463c717a561Smaybee zio->io_ready = ready; 464c717a561Smaybee 465ea8dc4b6Seschrock zio->io_bookmark = *zb; 466ea8dc4b6Seschrock 467ea8dc4b6Seschrock zio->io_logical = zio; 468ea8dc4b6Seschrock 469fa9e4066Sahrens zio->io_checksum = checksum; 470fa9e4066Sahrens zio->io_compress = compress; 47144cd46caSbillm zio->io_ndvas = ncopies; 472fa9e4066Sahrens 473fa9e4066Sahrens if (compress != ZIO_COMPRESS_OFF) 474fa9e4066Sahrens zio->io_async_stages |= 1U << ZIO_STAGE_WRITE_COMPRESS; 475fa9e4066Sahrens 476fa9e4066Sahrens if (bp->blk_birth != txg) { 477fa9e4066Sahrens /* XXX the bp usually (always?) gets re-zeroed later */ 478fa9e4066Sahrens BP_ZERO(bp); 479fa9e4066Sahrens BP_SET_LSIZE(bp, size); 480fa9e4066Sahrens BP_SET_PSIZE(bp, size); 48144cd46caSbillm } else { 48244cd46caSbillm /* Make sure someone doesn't change their mind on overwrites */ 48344cd46caSbillm ASSERT(MIN(zio->io_ndvas + BP_IS_GANG(bp), 48444cd46caSbillm spa_max_replication(spa)) == BP_GET_NDVAS(bp)); 485fa9e4066Sahrens } 486fa9e4066Sahrens 487fa9e4066Sahrens return (zio); 488fa9e4066Sahrens } 489fa9e4066Sahrens 490fa9e4066Sahrens zio_t * 491fa9e4066Sahrens zio_rewrite(zio_t *pio, spa_t *spa, int checksum, 492fa9e4066Sahrens uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 493ea8dc4b6Seschrock zio_done_func_t *done, void *private, int priority, int flags, 494ea8dc4b6Seschrock zbookmark_t *zb) 495fa9e4066Sahrens { 496fa9e4066Sahrens zio_t *zio; 497fa9e4066Sahrens 498fa9e4066Sahrens zio = zio_create(pio, spa, txg, bp, data, size, done, private, 499faafa6e3Sahrens ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_USER, 500fa9e4066Sahrens ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE); 501fa9e4066Sahrens 502ea8dc4b6Seschrock zio->io_bookmark = *zb; 503fa9e4066Sahrens zio->io_checksum = checksum; 504fa9e4066Sahrens zio->io_compress = ZIO_COMPRESS_OFF; 505fa9e4066Sahrens 50644cd46caSbillm if (pio != NULL) 50744cd46caSbillm ASSERT3U(zio->io_ndvas, <=, BP_GET_NDVAS(bp)); 50844cd46caSbillm 509fa9e4066Sahrens return (zio); 510fa9e4066Sahrens } 511fa9e4066Sahrens 512fa9e4066Sahrens static zio_t * 513fa9e4066Sahrens zio_write_allocate(zio_t *pio, spa_t *spa, int checksum, 514fa9e4066Sahrens uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 515fa9e4066Sahrens zio_done_func_t *done, void *private, int priority, int flags) 516fa9e4066Sahrens { 517fa9e4066Sahrens zio_t *zio; 518fa9e4066Sahrens 519fa9e4066Sahrens BP_ZERO(bp); 520fa9e4066Sahrens BP_SET_LSIZE(bp, size); 521fa9e4066Sahrens BP_SET_PSIZE(bp, size); 522fa9e4066Sahrens BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); 523fa9e4066Sahrens 524fa9e4066Sahrens zio = zio_create(pio, spa, txg, bp, data, size, done, private, 525fa9e4066Sahrens ZIO_TYPE_WRITE, priority, flags, 526fa9e4066Sahrens ZIO_STAGE_OPEN, ZIO_WRITE_ALLOCATE_PIPELINE); 527fa9e4066Sahrens 528fa9e4066Sahrens zio->io_checksum = checksum; 529fa9e4066Sahrens zio->io_compress = ZIO_COMPRESS_OFF; 530fa9e4066Sahrens 531fa9e4066Sahrens return (zio); 532fa9e4066Sahrens } 533fa9e4066Sahrens 534fa9e4066Sahrens zio_t * 535fa9e4066Sahrens zio_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 536fa9e4066Sahrens zio_done_func_t *done, void *private) 537fa9e4066Sahrens { 538fa9e4066Sahrens zio_t *zio; 539fa9e4066Sahrens 540fa9e4066Sahrens ASSERT(!BP_IS_HOLE(bp)); 541fa9e4066Sahrens 542fa9e4066Sahrens if (txg == spa->spa_syncing_txg && 543fa9e4066Sahrens spa->spa_sync_pass > zio_sync_pass.zp_defer_free) { 544fa9e4066Sahrens bplist_enqueue_deferred(&spa->spa_sync_bplist, bp); 545fa9e4066Sahrens return (zio_null(pio, spa, NULL, NULL, 0)); 546fa9e4066Sahrens } 547fa9e4066Sahrens 548fa9e4066Sahrens zio = zio_create(pio, spa, txg, bp, NULL, 0, done, private, 549faafa6e3Sahrens ZIO_TYPE_FREE, ZIO_PRIORITY_FREE, ZIO_FLAG_USER, 550fa9e4066Sahrens ZIO_STAGE_OPEN, ZIO_FREE_PIPELINE); 551fa9e4066Sahrens 552fa9e4066Sahrens zio->io_bp = &zio->io_bp_copy; 553fa9e4066Sahrens 554fa9e4066Sahrens return (zio); 555fa9e4066Sahrens } 556fa9e4066Sahrens 557fa9e4066Sahrens zio_t * 558fa9e4066Sahrens zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 559fa9e4066Sahrens zio_done_func_t *done, void *private) 560fa9e4066Sahrens { 561fa9e4066Sahrens zio_t *zio; 562fa9e4066Sahrens 563fa9e4066Sahrens /* 564fa9e4066Sahrens * A claim is an allocation of a specific block. Claims are needed 565fa9e4066Sahrens * to support immediate writes in the intent log. The issue is that 566fa9e4066Sahrens * immediate writes contain committed data, but in a txg that was 567fa9e4066Sahrens * *not* committed. Upon opening the pool after an unclean shutdown, 568fa9e4066Sahrens * the intent log claims all blocks that contain immediate write data 569fa9e4066Sahrens * so that the SPA knows they're in use. 570fa9e4066Sahrens * 571fa9e4066Sahrens * All claims *must* be resolved in the first txg -- before the SPA 572fa9e4066Sahrens * starts allocating blocks -- so that nothing is allocated twice. 573fa9e4066Sahrens */ 574fa9e4066Sahrens ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa)); 575fa9e4066Sahrens ASSERT3U(spa_first_txg(spa), <=, txg); 576fa9e4066Sahrens 577fa9e4066Sahrens zio = zio_create(pio, spa, txg, bp, NULL, 0, done, private, 578fa9e4066Sahrens ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, 0, 579fa9e4066Sahrens ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE); 580fa9e4066Sahrens 581fa9e4066Sahrens zio->io_bp = &zio->io_bp_copy; 582fa9e4066Sahrens 583fa9e4066Sahrens return (zio); 584fa9e4066Sahrens } 585fa9e4066Sahrens 586fa9e4066Sahrens zio_t * 587fa9e4066Sahrens zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, 588fa9e4066Sahrens zio_done_func_t *done, void *private, int priority, int flags) 589fa9e4066Sahrens { 590fa9e4066Sahrens zio_t *zio; 591fa9e4066Sahrens int c; 592fa9e4066Sahrens 593fa9e4066Sahrens if (vd->vdev_children == 0) { 594fa9e4066Sahrens zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 595fa9e4066Sahrens ZIO_TYPE_IOCTL, priority, flags, 596fa9e4066Sahrens ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); 597fa9e4066Sahrens 598fa9e4066Sahrens zio->io_vd = vd; 599fa9e4066Sahrens zio->io_cmd = cmd; 600fa9e4066Sahrens } else { 601fa9e4066Sahrens zio = zio_null(pio, spa, NULL, NULL, flags); 602fa9e4066Sahrens 603fa9e4066Sahrens for (c = 0; c < vd->vdev_children; c++) 604fa9e4066Sahrens zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, 605fa9e4066Sahrens done, private, priority, flags)); 606fa9e4066Sahrens } 607fa9e4066Sahrens 608fa9e4066Sahrens return (zio); 609fa9e4066Sahrens } 610fa9e4066Sahrens 611fa9e4066Sahrens static void 612fa9e4066Sahrens zio_phys_bp_init(vdev_t *vd, blkptr_t *bp, uint64_t offset, uint64_t size, 613fa9e4066Sahrens int checksum) 614fa9e4066Sahrens { 615fa9e4066Sahrens ASSERT(vd->vdev_children == 0); 616fa9e4066Sahrens 617fa9e4066Sahrens ASSERT(size <= SPA_MAXBLOCKSIZE); 618fa9e4066Sahrens ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 619fa9e4066Sahrens ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); 620fa9e4066Sahrens 621fa9e4066Sahrens ASSERT(offset + size <= VDEV_LABEL_START_SIZE || 622fa9e4066Sahrens offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 623fa9e4066Sahrens ASSERT3U(offset + size, <=, vd->vdev_psize); 624fa9e4066Sahrens 625fa9e4066Sahrens BP_ZERO(bp); 626fa9e4066Sahrens 627fa9e4066Sahrens BP_SET_LSIZE(bp, size); 628fa9e4066Sahrens BP_SET_PSIZE(bp, size); 629fa9e4066Sahrens 630fa9e4066Sahrens BP_SET_CHECKSUM(bp, checksum); 631fa9e4066Sahrens BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); 632fa9e4066Sahrens BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 633fa9e4066Sahrens 634fa9e4066Sahrens if (checksum != ZIO_CHECKSUM_OFF) 635fa9e4066Sahrens ZIO_SET_CHECKSUM(&bp->blk_cksum, offset, 0, 0, 0); 636fa9e4066Sahrens } 637fa9e4066Sahrens 638fa9e4066Sahrens zio_t * 639fa9e4066Sahrens zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 640fa9e4066Sahrens void *data, int checksum, zio_done_func_t *done, void *private, 641fa9e4066Sahrens int priority, int flags) 642fa9e4066Sahrens { 643fa9e4066Sahrens zio_t *zio; 644fa9e4066Sahrens blkptr_t blk; 645fa9e4066Sahrens 646fa9e4066Sahrens zio_phys_bp_init(vd, &blk, offset, size, checksum); 647fa9e4066Sahrens 648fa9e4066Sahrens zio = zio_create(pio, vd->vdev_spa, 0, &blk, data, size, done, private, 649fa9e4066Sahrens ZIO_TYPE_READ, priority, flags | ZIO_FLAG_PHYSICAL, 650fa9e4066Sahrens ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); 651fa9e4066Sahrens 652fa9e4066Sahrens zio->io_vd = vd; 653fa9e4066Sahrens zio->io_offset = offset; 654fa9e4066Sahrens 655fa9e4066Sahrens /* 656fa9e4066Sahrens * Work off our copy of the bp so the caller can free it. 657fa9e4066Sahrens */ 658fa9e4066Sahrens zio->io_bp = &zio->io_bp_copy; 659fa9e4066Sahrens 660fa9e4066Sahrens return (zio); 661fa9e4066Sahrens } 662fa9e4066Sahrens 663fa9e4066Sahrens zio_t * 664fa9e4066Sahrens zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 665fa9e4066Sahrens void *data, int checksum, zio_done_func_t *done, void *private, 666fa9e4066Sahrens int priority, int flags) 667fa9e4066Sahrens { 668fa9e4066Sahrens zio_block_tail_t *zbt; 669fa9e4066Sahrens void *wbuf; 670fa9e4066Sahrens zio_t *zio; 671fa9e4066Sahrens blkptr_t blk; 672fa9e4066Sahrens 673fa9e4066Sahrens zio_phys_bp_init(vd, &blk, offset, size, checksum); 674fa9e4066Sahrens 675fa9e4066Sahrens zio = zio_create(pio, vd->vdev_spa, 0, &blk, data, size, done, private, 676fa9e4066Sahrens ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_PHYSICAL, 677fa9e4066Sahrens ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); 678fa9e4066Sahrens 679fa9e4066Sahrens zio->io_vd = vd; 680fa9e4066Sahrens zio->io_offset = offset; 681fa9e4066Sahrens 682fa9e4066Sahrens zio->io_bp = &zio->io_bp_copy; 683fa9e4066Sahrens zio->io_checksum = checksum; 684fa9e4066Sahrens 685fa9e4066Sahrens if (zio_checksum_table[checksum].ci_zbt) { 686fa9e4066Sahrens /* 687fa9e4066Sahrens * zbt checksums are necessarily destructive -- they modify 688fa9e4066Sahrens * one word of the write buffer to hold the verifier/checksum. 689fa9e4066Sahrens * Therefore, we must make a local copy in case the data is 690fa9e4066Sahrens * being written to multiple places. 691fa9e4066Sahrens */ 692fa9e4066Sahrens wbuf = zio_buf_alloc(size); 693fa9e4066Sahrens bcopy(data, wbuf, size); 694fa9e4066Sahrens zio_push_transform(zio, wbuf, size, size); 695fa9e4066Sahrens 696fa9e4066Sahrens zbt = (zio_block_tail_t *)((char *)wbuf + size) - 1; 697fa9e4066Sahrens zbt->zbt_cksum = blk.blk_cksum; 698fa9e4066Sahrens } 699fa9e4066Sahrens 700fa9e4066Sahrens return (zio); 701fa9e4066Sahrens } 702fa9e4066Sahrens 703fa9e4066Sahrens /* 704fa9e4066Sahrens * Create a child I/O to do some work for us. It has no associated bp. 705fa9e4066Sahrens */ 706fa9e4066Sahrens zio_t * 707fa9e4066Sahrens zio_vdev_child_io(zio_t *zio, blkptr_t *bp, vdev_t *vd, uint64_t offset, 708fa9e4066Sahrens void *data, uint64_t size, int type, int priority, int flags, 709fa9e4066Sahrens zio_done_func_t *done, void *private) 710fa9e4066Sahrens { 711fa9e4066Sahrens uint32_t pipeline = ZIO_VDEV_CHILD_PIPELINE; 712fa9e4066Sahrens zio_t *cio; 713fa9e4066Sahrens 714fa9e4066Sahrens if (type == ZIO_TYPE_READ && bp != NULL) { 715fa9e4066Sahrens /* 716fa9e4066Sahrens * If we have the bp, then the child should perform the 717fa9e4066Sahrens * checksum and the parent need not. This pushes error 718fa9e4066Sahrens * detection as close to the leaves as possible and 719fa9e4066Sahrens * eliminates redundant checksums in the interior nodes. 720fa9e4066Sahrens */ 721fa9e4066Sahrens pipeline |= 1U << ZIO_STAGE_CHECKSUM_VERIFY; 722fa9e4066Sahrens zio->io_pipeline &= ~(1U << ZIO_STAGE_CHECKSUM_VERIFY); 723fa9e4066Sahrens } 724fa9e4066Sahrens 725fa9e4066Sahrens cio = zio_create(zio, zio->io_spa, zio->io_txg, bp, data, size, 726fa9e4066Sahrens done, private, type, priority, 727fa9e4066Sahrens (zio->io_flags & ZIO_FLAG_VDEV_INHERIT) | ZIO_FLAG_CANFAIL | flags, 72844cd46caSbillm ZIO_STAGE_VDEV_IO_START - 1, pipeline); 729fa9e4066Sahrens 730fa9e4066Sahrens cio->io_vd = vd; 731fa9e4066Sahrens cio->io_offset = offset; 732fa9e4066Sahrens 733fa9e4066Sahrens return (cio); 734fa9e4066Sahrens } 735fa9e4066Sahrens 736fa9e4066Sahrens /* 737fa9e4066Sahrens * ========================================================================== 738fa9e4066Sahrens * Initiate I/O, either sync or async 739fa9e4066Sahrens * ========================================================================== 740fa9e4066Sahrens */ 741fa9e4066Sahrens int 742fa9e4066Sahrens zio_wait(zio_t *zio) 743fa9e4066Sahrens { 744fa9e4066Sahrens int error; 745fa9e4066Sahrens 746fa9e4066Sahrens ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 747fa9e4066Sahrens 748fa9e4066Sahrens zio->io_waiter = curthread; 749fa9e4066Sahrens 750fa9e4066Sahrens zio_next_stage_async(zio); 751fa9e4066Sahrens 752fa9e4066Sahrens mutex_enter(&zio->io_lock); 753fa9e4066Sahrens while (zio->io_stalled != ZIO_STAGE_DONE) 754fa9e4066Sahrens cv_wait(&zio->io_cv, &zio->io_lock); 755fa9e4066Sahrens mutex_exit(&zio->io_lock); 756fa9e4066Sahrens 757fa9e4066Sahrens error = zio->io_error; 7585ad82045Snd mutex_destroy(&zio->io_lock); 759*ccae0b50Seschrock kmem_cache_free(zio_cache, zio); 760fa9e4066Sahrens 761fa9e4066Sahrens return (error); 762fa9e4066Sahrens } 763fa9e4066Sahrens 764fa9e4066Sahrens void 765fa9e4066Sahrens zio_nowait(zio_t *zio) 766fa9e4066Sahrens { 767fa9e4066Sahrens zio_next_stage_async(zio); 768fa9e4066Sahrens } 769fa9e4066Sahrens 770fa9e4066Sahrens /* 771fa9e4066Sahrens * ========================================================================== 772fa9e4066Sahrens * I/O pipeline interlocks: parent/child dependency scoreboarding 773fa9e4066Sahrens * ========================================================================== 774fa9e4066Sahrens */ 775fa9e4066Sahrens static void 776fa9e4066Sahrens zio_wait_for_children(zio_t *zio, uint32_t stage, uint64_t *countp) 777fa9e4066Sahrens { 778fa9e4066Sahrens mutex_enter(&zio->io_lock); 779fa9e4066Sahrens if (*countp == 0) { 780fa9e4066Sahrens ASSERT(zio->io_stalled == 0); 781fa9e4066Sahrens mutex_exit(&zio->io_lock); 782fa9e4066Sahrens zio_next_stage(zio); 783fa9e4066Sahrens } else { 784fa9e4066Sahrens zio->io_stalled = stage; 785fa9e4066Sahrens mutex_exit(&zio->io_lock); 786fa9e4066Sahrens } 787fa9e4066Sahrens } 788fa9e4066Sahrens 789fa9e4066Sahrens static void 790fa9e4066Sahrens zio_notify_parent(zio_t *zio, uint32_t stage, uint64_t *countp) 791fa9e4066Sahrens { 792fa9e4066Sahrens zio_t *pio = zio->io_parent; 793fa9e4066Sahrens 794fa9e4066Sahrens mutex_enter(&pio->io_lock); 795fa9e4066Sahrens if (pio->io_error == 0 && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) 796fa9e4066Sahrens pio->io_error = zio->io_error; 797fa9e4066Sahrens if (--*countp == 0 && pio->io_stalled == stage) { 798fa9e4066Sahrens pio->io_stalled = 0; 799fa9e4066Sahrens mutex_exit(&pio->io_lock); 800fa9e4066Sahrens zio_next_stage_async(pio); 801fa9e4066Sahrens } else { 802fa9e4066Sahrens mutex_exit(&pio->io_lock); 803fa9e4066Sahrens } 804fa9e4066Sahrens } 805fa9e4066Sahrens 806fa9e4066Sahrens static void 807fa9e4066Sahrens zio_wait_children_ready(zio_t *zio) 808fa9e4066Sahrens { 809fa9e4066Sahrens zio_wait_for_children(zio, ZIO_STAGE_WAIT_CHILDREN_READY, 810fa9e4066Sahrens &zio->io_children_notready); 811fa9e4066Sahrens } 812fa9e4066Sahrens 813fa9e4066Sahrens void 814fa9e4066Sahrens zio_wait_children_done(zio_t *zio) 815fa9e4066Sahrens { 816fa9e4066Sahrens zio_wait_for_children(zio, ZIO_STAGE_WAIT_CHILDREN_DONE, 817fa9e4066Sahrens &zio->io_children_notdone); 818fa9e4066Sahrens } 819fa9e4066Sahrens 820fa9e4066Sahrens static void 821fa9e4066Sahrens zio_ready(zio_t *zio) 822fa9e4066Sahrens { 823fa9e4066Sahrens zio_t *pio = zio->io_parent; 824fa9e4066Sahrens 825c717a561Smaybee if (zio->io_ready) 826c717a561Smaybee zio->io_ready(zio); 827c717a561Smaybee 828fa9e4066Sahrens if (pio != NULL) 829fa9e4066Sahrens zio_notify_parent(zio, ZIO_STAGE_WAIT_CHILDREN_READY, 830fa9e4066Sahrens &pio->io_children_notready); 831fa9e4066Sahrens 832fa9e4066Sahrens if (zio->io_bp) 833fa9e4066Sahrens zio->io_bp_copy = *zio->io_bp; 834fa9e4066Sahrens 835fa9e4066Sahrens zio_next_stage(zio); 836fa9e4066Sahrens } 837fa9e4066Sahrens 838fa9e4066Sahrens static void 839fa9e4066Sahrens zio_done(zio_t *zio) 840fa9e4066Sahrens { 841fa9e4066Sahrens zio_t *pio = zio->io_parent; 842fa9e4066Sahrens spa_t *spa = zio->io_spa; 843fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 844fa9e4066Sahrens vdev_t *vd = zio->io_vd; 845fa9e4066Sahrens 846fa9e4066Sahrens ASSERT(zio->io_children_notready == 0); 847fa9e4066Sahrens ASSERT(zio->io_children_notdone == 0); 848fa9e4066Sahrens 849fa9e4066Sahrens if (bp != NULL) { 850fa9e4066Sahrens ASSERT(bp->blk_pad[0] == 0); 851fa9e4066Sahrens ASSERT(bp->blk_pad[1] == 0); 852fa9e4066Sahrens ASSERT(bp->blk_pad[2] == 0); 853fa9e4066Sahrens ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0); 854fa9e4066Sahrens if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) && 85544cd46caSbillm !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) { 856fa9e4066Sahrens ASSERT(!BP_SHOULD_BYTESWAP(bp)); 85744cd46caSbillm if (zio->io_ndvas != 0) 85844cd46caSbillm ASSERT3U(zio->io_ndvas, <=, BP_GET_NDVAS(bp)); 85944cd46caSbillm ASSERT(BP_COUNT_GANG(bp) == 0 || 86044cd46caSbillm (BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp))); 86144cd46caSbillm } 862fa9e4066Sahrens } 863fa9e4066Sahrens 864fa9e4066Sahrens if (vd != NULL) 865fa9e4066Sahrens vdev_stat_update(zio); 866fa9e4066Sahrens 867fa9e4066Sahrens if (zio->io_error) { 868ea8dc4b6Seschrock /* 869ea8dc4b6Seschrock * If this I/O is attached to a particular vdev, 870ea8dc4b6Seschrock * generate an error message describing the I/O failure 871ea8dc4b6Seschrock * at the block level. We ignore these errors if the 872ea8dc4b6Seschrock * device is currently unavailable. 873ea8dc4b6Seschrock */ 874ecc2d604Sbonwick if (zio->io_error != ECKSUM && vd != NULL && !vdev_is_dead(vd)) 875ea8dc4b6Seschrock zfs_ereport_post(FM_EREPORT_ZFS_IO, 876ecc2d604Sbonwick zio->io_spa, vd, zio, 0, 0); 877ea8dc4b6Seschrock 878ea8dc4b6Seschrock if ((zio->io_error == EIO || 879ea8dc4b6Seschrock !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) && 880ea8dc4b6Seschrock zio->io_logical == zio) { 881ea8dc4b6Seschrock /* 882ea8dc4b6Seschrock * For root I/O requests, tell the SPA to log the error 883ea8dc4b6Seschrock * appropriately. Also, generate a logical data 884ea8dc4b6Seschrock * ereport. 885ea8dc4b6Seschrock */ 886ea8dc4b6Seschrock spa_log_error(zio->io_spa, zio); 887ea8dc4b6Seschrock 888ea8dc4b6Seschrock zfs_ereport_post(FM_EREPORT_ZFS_DATA, 889ea8dc4b6Seschrock zio->io_spa, NULL, zio, 0, 0); 890ea8dc4b6Seschrock } 891fa9e4066Sahrens 892ea8dc4b6Seschrock /* 893ea8dc4b6Seschrock * For I/O requests that cannot fail, panic appropriately. 894ea8dc4b6Seschrock */ 895ea8dc4b6Seschrock if (!(zio->io_flags & ZIO_FLAG_CANFAIL)) { 896d58459f4Sek char *blkbuf; 897d58459f4Sek 898d58459f4Sek blkbuf = kmem_alloc(BP_SPRINTF_LEN, KM_NOSLEEP); 899d58459f4Sek if (blkbuf) { 900d58459f4Sek sprintf_blkptr(blkbuf, BP_SPRINTF_LEN, 901d58459f4Sek bp ? bp : &zio->io_bp_copy); 902d58459f4Sek } 903ea8dc4b6Seschrock panic("ZFS: %s (%s on %s off %llx: zio %p %s): error " 904ea8dc4b6Seschrock "%d", zio->io_error == ECKSUM ? 905ea8dc4b6Seschrock "bad checksum" : "I/O failure", 906ea8dc4b6Seschrock zio_type_name[zio->io_type], 907ea8dc4b6Seschrock vdev_description(vd), 908ea8dc4b6Seschrock (u_longlong_t)zio->io_offset, 909d58459f4Sek zio, blkbuf ? blkbuf : "", zio->io_error); 910ea8dc4b6Seschrock } 911fa9e4066Sahrens } 912fa9e4066Sahrens zio_clear_transform_stack(zio); 913fa9e4066Sahrens 914fa9e4066Sahrens if (zio->io_done) 915fa9e4066Sahrens zio->io_done(zio); 916fa9e4066Sahrens 917fa9e4066Sahrens ASSERT(zio->io_delegate_list == NULL); 918fa9e4066Sahrens ASSERT(zio->io_delegate_next == NULL); 919fa9e4066Sahrens 920fa9e4066Sahrens if (pio != NULL) { 921fa9e4066Sahrens zio_t *next, *prev; 922fa9e4066Sahrens 923fa9e4066Sahrens mutex_enter(&pio->io_lock); 924fa9e4066Sahrens next = zio->io_sibling_next; 925fa9e4066Sahrens prev = zio->io_sibling_prev; 926fa9e4066Sahrens if (next != NULL) 927fa9e4066Sahrens next->io_sibling_prev = prev; 928fa9e4066Sahrens if (prev != NULL) 929fa9e4066Sahrens prev->io_sibling_next = next; 930fa9e4066Sahrens if (pio->io_child == zio) 931fa9e4066Sahrens pio->io_child = next; 932fa9e4066Sahrens mutex_exit(&pio->io_lock); 933fa9e4066Sahrens 934fa9e4066Sahrens zio_notify_parent(zio, ZIO_STAGE_WAIT_CHILDREN_DONE, 935fa9e4066Sahrens &pio->io_children_notdone); 936fa9e4066Sahrens } 937fa9e4066Sahrens 938b3995adbSahrens /* 939*ccae0b50Seschrock * Note: this I/O is now done, and will shortly be freed, so there is no 940*ccae0b50Seschrock * need to clear this (or any other) flag. 941b3995adbSahrens */ 942b3995adbSahrens if (zio->io_flags & ZIO_FLAG_CONFIG_GRABBED) 943ea8dc4b6Seschrock spa_config_exit(spa, zio); 944fa9e4066Sahrens 945fa9e4066Sahrens if (zio->io_waiter != NULL) { 946fa9e4066Sahrens mutex_enter(&zio->io_lock); 947fa9e4066Sahrens ASSERT(zio->io_stage == ZIO_STAGE_DONE); 948fa9e4066Sahrens zio->io_stalled = zio->io_stage; 949fa9e4066Sahrens cv_broadcast(&zio->io_cv); 950fa9e4066Sahrens mutex_exit(&zio->io_lock); 951fa9e4066Sahrens } else { 952*ccae0b50Seschrock kmem_cache_free(zio_cache, zio); 953fa9e4066Sahrens } 954fa9e4066Sahrens } 955fa9e4066Sahrens 956fa9e4066Sahrens /* 957fa9e4066Sahrens * ========================================================================== 958fa9e4066Sahrens * Compression support 959fa9e4066Sahrens * ========================================================================== 960fa9e4066Sahrens */ 961fa9e4066Sahrens static void 962fa9e4066Sahrens zio_write_compress(zio_t *zio) 963fa9e4066Sahrens { 964fa9e4066Sahrens int compress = zio->io_compress; 965fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 966fa9e4066Sahrens void *cbuf; 967fa9e4066Sahrens uint64_t lsize = zio->io_size; 968fa9e4066Sahrens uint64_t csize = lsize; 969fa9e4066Sahrens uint64_t cbufsize = 0; 970fa9e4066Sahrens int pass; 971fa9e4066Sahrens 972fa9e4066Sahrens if (bp->blk_birth == zio->io_txg) { 973fa9e4066Sahrens /* 974fa9e4066Sahrens * We're rewriting an existing block, which means we're 975fa9e4066Sahrens * working on behalf of spa_sync(). For spa_sync() to 976fa9e4066Sahrens * converge, it must eventually be the case that we don't 977fa9e4066Sahrens * have to allocate new blocks. But compression changes 978fa9e4066Sahrens * the blocksize, which forces a reallocate, and makes 979fa9e4066Sahrens * convergence take longer. Therefore, after the first 980fa9e4066Sahrens * few passes, stop compressing to ensure convergence. 981fa9e4066Sahrens */ 982fa9e4066Sahrens pass = spa_sync_pass(zio->io_spa); 983fa9e4066Sahrens if (pass > zio_sync_pass.zp_dontcompress) 984fa9e4066Sahrens compress = ZIO_COMPRESS_OFF; 985fa9e4066Sahrens } else { 986fa9e4066Sahrens ASSERT(BP_IS_HOLE(bp)); 987fa9e4066Sahrens pass = 1; 988fa9e4066Sahrens } 989fa9e4066Sahrens 990fa9e4066Sahrens if (compress != ZIO_COMPRESS_OFF) 991fa9e4066Sahrens if (!zio_compress_data(compress, zio->io_data, zio->io_size, 992fa9e4066Sahrens &cbuf, &csize, &cbufsize)) 993fa9e4066Sahrens compress = ZIO_COMPRESS_OFF; 994fa9e4066Sahrens 995fa9e4066Sahrens if (compress != ZIO_COMPRESS_OFF && csize != 0) 996fa9e4066Sahrens zio_push_transform(zio, cbuf, csize, cbufsize); 997fa9e4066Sahrens 998fa9e4066Sahrens /* 999fa9e4066Sahrens * The final pass of spa_sync() must be all rewrites, but the first 1000fa9e4066Sahrens * few passes offer a trade-off: allocating blocks defers convergence, 1001fa9e4066Sahrens * but newly allocated blocks are sequential, so they can be written 1002fa9e4066Sahrens * to disk faster. Therefore, we allow the first few passes of 1003fa9e4066Sahrens * spa_sync() to reallocate new blocks, but force rewrites after that. 1004fa9e4066Sahrens * There should only be a handful of blocks after pass 1 in any case. 1005fa9e4066Sahrens */ 1006fa9e4066Sahrens if (bp->blk_birth == zio->io_txg && BP_GET_PSIZE(bp) == csize && 1007fa9e4066Sahrens pass > zio_sync_pass.zp_rewrite) { 1008fa9e4066Sahrens ASSERT(csize != 0); 1009a2eea2e1Sahrens BP_SET_LSIZE(bp, lsize); 1010a2eea2e1Sahrens BP_SET_COMPRESS(bp, compress); 1011fa9e4066Sahrens zio->io_pipeline = ZIO_REWRITE_PIPELINE; 1012fa9e4066Sahrens } else { 101387bd5c1eSahrens if (bp->blk_birth == zio->io_txg) 101487bd5c1eSahrens BP_ZERO(bp); 1015fa9e4066Sahrens if (csize == 0) { 1016fa9e4066Sahrens BP_ZERO(bp); 1017fa9e4066Sahrens zio->io_pipeline = ZIO_WAIT_FOR_CHILDREN_PIPELINE; 1018fa9e4066Sahrens } else { 101944cd46caSbillm ASSERT3U(BP_GET_NDVAS(bp), ==, 0); 1020fa9e4066Sahrens BP_SET_LSIZE(bp, lsize); 1021fa9e4066Sahrens BP_SET_PSIZE(bp, csize); 1022fa9e4066Sahrens BP_SET_COMPRESS(bp, compress); 1023fa9e4066Sahrens zio->io_pipeline = ZIO_WRITE_ALLOCATE_PIPELINE; 1024fa9e4066Sahrens } 1025fa9e4066Sahrens } 1026fa9e4066Sahrens 1027fa9e4066Sahrens zio_next_stage(zio); 1028fa9e4066Sahrens } 1029fa9e4066Sahrens 1030fa9e4066Sahrens static void 1031fa9e4066Sahrens zio_read_decompress(zio_t *zio) 1032fa9e4066Sahrens { 1033fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 1034fa9e4066Sahrens void *data; 1035fa9e4066Sahrens uint64_t size; 1036fa9e4066Sahrens uint64_t bufsize; 1037fa9e4066Sahrens int compress = BP_GET_COMPRESS(bp); 1038fa9e4066Sahrens 1039fa9e4066Sahrens ASSERT(compress != ZIO_COMPRESS_OFF); 1040fa9e4066Sahrens 1041fa9e4066Sahrens zio_pop_transform(zio, &data, &size, &bufsize); 1042fa9e4066Sahrens 1043fa9e4066Sahrens if (zio_decompress_data(compress, data, size, 1044fa9e4066Sahrens zio->io_data, zio->io_size)) 1045fa9e4066Sahrens zio->io_error = EIO; 1046fa9e4066Sahrens 1047fa9e4066Sahrens zio_buf_free(data, bufsize); 1048fa9e4066Sahrens 1049fa9e4066Sahrens zio_next_stage(zio); 1050fa9e4066Sahrens } 1051fa9e4066Sahrens 1052fa9e4066Sahrens /* 1053fa9e4066Sahrens * ========================================================================== 1054fa9e4066Sahrens * Gang block support 1055fa9e4066Sahrens * ========================================================================== 1056fa9e4066Sahrens */ 1057fa9e4066Sahrens static void 1058fa9e4066Sahrens zio_gang_pipeline(zio_t *zio) 1059fa9e4066Sahrens { 1060fa9e4066Sahrens /* 1061fa9e4066Sahrens * By default, the pipeline assumes that we're dealing with a gang 1062fa9e4066Sahrens * block. If we're not, strip out any gang-specific stages. 1063fa9e4066Sahrens */ 106444cd46caSbillm if (!BP_IS_GANG(zio->io_bp)) 1065fa9e4066Sahrens zio->io_pipeline &= ~ZIO_GANG_STAGES; 1066fa9e4066Sahrens 1067fa9e4066Sahrens zio_next_stage(zio); 1068fa9e4066Sahrens } 1069fa9e4066Sahrens 1070fa9e4066Sahrens static void 1071fa9e4066Sahrens zio_gang_byteswap(zio_t *zio) 1072fa9e4066Sahrens { 1073fa9e4066Sahrens ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); 1074fa9e4066Sahrens 1075fa9e4066Sahrens if (BP_SHOULD_BYTESWAP(zio->io_bp)) 1076fa9e4066Sahrens byteswap_uint64_array(zio->io_data, zio->io_size); 1077fa9e4066Sahrens } 1078fa9e4066Sahrens 1079fa9e4066Sahrens static void 1080fa9e4066Sahrens zio_get_gang_header(zio_t *zio) 1081fa9e4066Sahrens { 1082fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 1083fa9e4066Sahrens uint64_t gsize = SPA_GANGBLOCKSIZE; 1084fa9e4066Sahrens void *gbuf = zio_buf_alloc(gsize); 1085fa9e4066Sahrens 108644cd46caSbillm ASSERT(BP_IS_GANG(bp)); 1087fa9e4066Sahrens 1088fa9e4066Sahrens zio_push_transform(zio, gbuf, gsize, gsize); 1089fa9e4066Sahrens 1090fa9e4066Sahrens zio_nowait(zio_create(zio, zio->io_spa, bp->blk_birth, bp, gbuf, gsize, 1091fa9e4066Sahrens NULL, NULL, ZIO_TYPE_READ, zio->io_priority, 1092fa9e4066Sahrens zio->io_flags & ZIO_FLAG_GANG_INHERIT, 1093fa9e4066Sahrens ZIO_STAGE_OPEN, ZIO_READ_PIPELINE)); 1094fa9e4066Sahrens 1095fa9e4066Sahrens zio_wait_children_done(zio); 1096fa9e4066Sahrens } 1097fa9e4066Sahrens 1098fa9e4066Sahrens static void 1099fa9e4066Sahrens zio_read_gang_members(zio_t *zio) 1100fa9e4066Sahrens { 1101fa9e4066Sahrens zio_gbh_phys_t *gbh; 1102fa9e4066Sahrens uint64_t gsize, gbufsize, loff, lsize; 1103fa9e4066Sahrens int i; 1104fa9e4066Sahrens 110544cd46caSbillm ASSERT(BP_IS_GANG(zio->io_bp)); 1106fa9e4066Sahrens 1107fa9e4066Sahrens zio_gang_byteswap(zio); 1108fa9e4066Sahrens zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1109fa9e4066Sahrens 1110fa9e4066Sahrens for (loff = 0, i = 0; loff != zio->io_size; loff += lsize, i++) { 1111fa9e4066Sahrens blkptr_t *gbp = &gbh->zg_blkptr[i]; 1112fa9e4066Sahrens lsize = BP_GET_PSIZE(gbp); 1113fa9e4066Sahrens 1114fa9e4066Sahrens ASSERT(BP_GET_COMPRESS(gbp) == ZIO_COMPRESS_OFF); 1115fa9e4066Sahrens ASSERT3U(lsize, ==, BP_GET_LSIZE(gbp)); 1116fa9e4066Sahrens ASSERT3U(loff + lsize, <=, zio->io_size); 1117fa9e4066Sahrens ASSERT(i < SPA_GBH_NBLKPTRS); 1118fa9e4066Sahrens ASSERT(!BP_IS_HOLE(gbp)); 1119fa9e4066Sahrens 1120fa9e4066Sahrens zio_nowait(zio_read(zio, zio->io_spa, gbp, 1121fa9e4066Sahrens (char *)zio->io_data + loff, lsize, NULL, NULL, 1122ea8dc4b6Seschrock zio->io_priority, zio->io_flags & ZIO_FLAG_GANG_INHERIT, 1123ea8dc4b6Seschrock &zio->io_bookmark)); 1124fa9e4066Sahrens } 1125fa9e4066Sahrens 1126fa9e4066Sahrens zio_buf_free(gbh, gbufsize); 1127fa9e4066Sahrens zio_wait_children_done(zio); 1128fa9e4066Sahrens } 1129fa9e4066Sahrens 1130fa9e4066Sahrens static void 1131fa9e4066Sahrens zio_rewrite_gang_members(zio_t *zio) 1132fa9e4066Sahrens { 1133fa9e4066Sahrens zio_gbh_phys_t *gbh; 1134fa9e4066Sahrens uint64_t gsize, gbufsize, loff, lsize; 1135fa9e4066Sahrens int i; 1136fa9e4066Sahrens 113744cd46caSbillm ASSERT(BP_IS_GANG(zio->io_bp)); 1138fa9e4066Sahrens ASSERT3U(zio->io_size, ==, SPA_GANGBLOCKSIZE); 1139fa9e4066Sahrens 1140fa9e4066Sahrens zio_gang_byteswap(zio); 1141fa9e4066Sahrens zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1142fa9e4066Sahrens 1143fa9e4066Sahrens ASSERT(gsize == gbufsize); 1144fa9e4066Sahrens 1145fa9e4066Sahrens for (loff = 0, i = 0; loff != zio->io_size; loff += lsize, i++) { 1146fa9e4066Sahrens blkptr_t *gbp = &gbh->zg_blkptr[i]; 1147fa9e4066Sahrens lsize = BP_GET_PSIZE(gbp); 1148fa9e4066Sahrens 1149fa9e4066Sahrens ASSERT(BP_GET_COMPRESS(gbp) == ZIO_COMPRESS_OFF); 1150fa9e4066Sahrens ASSERT3U(lsize, ==, BP_GET_LSIZE(gbp)); 1151fa9e4066Sahrens ASSERT3U(loff + lsize, <=, zio->io_size); 1152fa9e4066Sahrens ASSERT(i < SPA_GBH_NBLKPTRS); 1153fa9e4066Sahrens ASSERT(!BP_IS_HOLE(gbp)); 1154fa9e4066Sahrens 1155fa9e4066Sahrens zio_nowait(zio_rewrite(zio, zio->io_spa, zio->io_checksum, 1156fa9e4066Sahrens zio->io_txg, gbp, (char *)zio->io_data + loff, lsize, 1157ea8dc4b6Seschrock NULL, NULL, zio->io_priority, zio->io_flags, 1158ea8dc4b6Seschrock &zio->io_bookmark)); 1159fa9e4066Sahrens } 1160fa9e4066Sahrens 1161fa9e4066Sahrens zio_push_transform(zio, gbh, gsize, gbufsize); 1162fa9e4066Sahrens zio_wait_children_ready(zio); 1163fa9e4066Sahrens } 1164fa9e4066Sahrens 1165fa9e4066Sahrens static void 1166fa9e4066Sahrens zio_free_gang_members(zio_t *zio) 1167fa9e4066Sahrens { 1168fa9e4066Sahrens zio_gbh_phys_t *gbh; 1169fa9e4066Sahrens uint64_t gsize, gbufsize; 1170fa9e4066Sahrens int i; 1171fa9e4066Sahrens 117244cd46caSbillm ASSERT(BP_IS_GANG(zio->io_bp)); 1173fa9e4066Sahrens 1174fa9e4066Sahrens zio_gang_byteswap(zio); 1175fa9e4066Sahrens zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1176fa9e4066Sahrens 1177fa9e4066Sahrens for (i = 0; i < SPA_GBH_NBLKPTRS; i++) { 1178fa9e4066Sahrens blkptr_t *gbp = &gbh->zg_blkptr[i]; 1179fa9e4066Sahrens 1180fa9e4066Sahrens if (BP_IS_HOLE(gbp)) 1181fa9e4066Sahrens continue; 1182fa9e4066Sahrens zio_nowait(zio_free(zio, zio->io_spa, zio->io_txg, 1183fa9e4066Sahrens gbp, NULL, NULL)); 1184fa9e4066Sahrens } 1185fa9e4066Sahrens 1186fa9e4066Sahrens zio_buf_free(gbh, gbufsize); 1187fa9e4066Sahrens zio_next_stage(zio); 1188fa9e4066Sahrens } 1189fa9e4066Sahrens 1190fa9e4066Sahrens static void 1191fa9e4066Sahrens zio_claim_gang_members(zio_t *zio) 1192fa9e4066Sahrens { 1193fa9e4066Sahrens zio_gbh_phys_t *gbh; 1194fa9e4066Sahrens uint64_t gsize, gbufsize; 1195fa9e4066Sahrens int i; 1196fa9e4066Sahrens 119744cd46caSbillm ASSERT(BP_IS_GANG(zio->io_bp)); 1198fa9e4066Sahrens 1199fa9e4066Sahrens zio_gang_byteswap(zio); 1200fa9e4066Sahrens zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1201fa9e4066Sahrens 1202fa9e4066Sahrens for (i = 0; i < SPA_GBH_NBLKPTRS; i++) { 1203fa9e4066Sahrens blkptr_t *gbp = &gbh->zg_blkptr[i]; 1204fa9e4066Sahrens if (BP_IS_HOLE(gbp)) 1205fa9e4066Sahrens continue; 1206fa9e4066Sahrens zio_nowait(zio_claim(zio, zio->io_spa, zio->io_txg, 1207fa9e4066Sahrens gbp, NULL, NULL)); 1208fa9e4066Sahrens } 1209fa9e4066Sahrens 1210fa9e4066Sahrens zio_buf_free(gbh, gbufsize); 1211fa9e4066Sahrens zio_next_stage(zio); 1212fa9e4066Sahrens } 1213fa9e4066Sahrens 1214fa9e4066Sahrens static void 1215fa9e4066Sahrens zio_write_allocate_gang_member_done(zio_t *zio) 1216fa9e4066Sahrens { 1217fa9e4066Sahrens zio_t *pio = zio->io_parent; 121844cd46caSbillm dva_t *cdva = zio->io_bp->blk_dva; 121944cd46caSbillm dva_t *pdva = pio->io_bp->blk_dva; 1220fa9e4066Sahrens uint64_t asize; 122144cd46caSbillm int d; 1222fa9e4066Sahrens 122344cd46caSbillm ASSERT3U(pio->io_ndvas, ==, zio->io_ndvas); 122444cd46caSbillm ASSERT3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp)); 122544cd46caSbillm ASSERT3U(zio->io_ndvas, <=, BP_GET_NDVAS(zio->io_bp)); 122644cd46caSbillm ASSERT3U(pio->io_ndvas, <=, BP_GET_NDVAS(pio->io_bp)); 1227fa9e4066Sahrens 1228fa9e4066Sahrens mutex_enter(&pio->io_lock); 122944cd46caSbillm for (d = 0; d < BP_GET_NDVAS(pio->io_bp); d++) { 123044cd46caSbillm ASSERT(DVA_GET_GANG(&pdva[d])); 123144cd46caSbillm asize = DVA_GET_ASIZE(&pdva[d]); 123244cd46caSbillm asize += DVA_GET_ASIZE(&cdva[d]); 123344cd46caSbillm DVA_SET_ASIZE(&pdva[d], asize); 123444cd46caSbillm } 1235fa9e4066Sahrens mutex_exit(&pio->io_lock); 1236fa9e4066Sahrens } 1237fa9e4066Sahrens 1238fa9e4066Sahrens static void 1239fa9e4066Sahrens zio_write_allocate_gang_members(zio_t *zio) 1240fa9e4066Sahrens { 1241fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 124244cd46caSbillm dva_t *dva = bp->blk_dva; 124344cd46caSbillm spa_t *spa = zio->io_spa; 1244fa9e4066Sahrens zio_gbh_phys_t *gbh; 124544cd46caSbillm uint64_t txg = zio->io_txg; 1246fa9e4066Sahrens uint64_t resid = zio->io_size; 1247fa9e4066Sahrens uint64_t maxalloc = P2ROUNDUP(zio->io_size >> 1, SPA_MINBLOCKSIZE); 1248fa9e4066Sahrens uint64_t gsize, loff, lsize; 1249fa9e4066Sahrens uint32_t gbps_left; 125044cd46caSbillm int ndvas = zio->io_ndvas; 125144cd46caSbillm int gbh_ndvas = MIN(ndvas + 1, spa_max_replication(spa)); 1252fa9e4066Sahrens int error; 125344cd46caSbillm int i, d; 1254fa9e4066Sahrens 1255fa9e4066Sahrens gsize = SPA_GANGBLOCKSIZE; 1256fa9e4066Sahrens gbps_left = SPA_GBH_NBLKPTRS; 1257fa9e4066Sahrens 125867bd71c6Sperrin error = metaslab_alloc(spa, gsize, bp, gbh_ndvas, txg, NULL, B_FALSE); 1259fa9e4066Sahrens if (error == ENOSPC) 1260fa9e4066Sahrens panic("can't allocate gang block header"); 1261fa9e4066Sahrens ASSERT(error == 0); 1262fa9e4066Sahrens 126344cd46caSbillm for (d = 0; d < gbh_ndvas; d++) 126444cd46caSbillm DVA_SET_GANG(&dva[d], 1); 1265fa9e4066Sahrens 126644cd46caSbillm bp->blk_birth = txg; 1267fa9e4066Sahrens 1268fa9e4066Sahrens gbh = zio_buf_alloc(gsize); 1269fa9e4066Sahrens bzero(gbh, gsize); 1270fa9e4066Sahrens 127144cd46caSbillm /* We need to test multi-level gang blocks */ 127244cd46caSbillm if (maxalloc >= zio_gang_bang && (lbolt & 0x1) == 0) 127344cd46caSbillm maxalloc = MAX(maxalloc >> 2, SPA_MINBLOCKSIZE); 127444cd46caSbillm 1275fa9e4066Sahrens for (loff = 0, i = 0; loff != zio->io_size; 1276fa9e4066Sahrens loff += lsize, resid -= lsize, gbps_left--, i++) { 1277fa9e4066Sahrens blkptr_t *gbp = &gbh->zg_blkptr[i]; 127844cd46caSbillm dva = gbp->blk_dva; 1279fa9e4066Sahrens 1280fa9e4066Sahrens ASSERT(gbps_left != 0); 1281fa9e4066Sahrens maxalloc = MIN(maxalloc, resid); 1282fa9e4066Sahrens 1283fa9e4066Sahrens while (resid <= maxalloc * gbps_left) { 128444cd46caSbillm error = metaslab_alloc(spa, maxalloc, gbp, ndvas, 128567bd71c6Sperrin txg, bp, B_FALSE); 1286fa9e4066Sahrens if (error == 0) 1287fa9e4066Sahrens break; 1288fa9e4066Sahrens ASSERT3U(error, ==, ENOSPC); 1289fa9e4066Sahrens if (maxalloc == SPA_MINBLOCKSIZE) 1290fa9e4066Sahrens panic("really out of space"); 1291fa9e4066Sahrens maxalloc = P2ROUNDUP(maxalloc >> 1, SPA_MINBLOCKSIZE); 1292fa9e4066Sahrens } 1293fa9e4066Sahrens 1294fa9e4066Sahrens if (resid <= maxalloc * gbps_left) { 1295fa9e4066Sahrens lsize = maxalloc; 1296fa9e4066Sahrens BP_SET_LSIZE(gbp, lsize); 1297fa9e4066Sahrens BP_SET_PSIZE(gbp, lsize); 1298fa9e4066Sahrens BP_SET_COMPRESS(gbp, ZIO_COMPRESS_OFF); 129944cd46caSbillm gbp->blk_birth = txg; 130044cd46caSbillm zio_nowait(zio_rewrite(zio, spa, 130144cd46caSbillm zio->io_checksum, txg, gbp, 1302fa9e4066Sahrens (char *)zio->io_data + loff, lsize, 1303fa9e4066Sahrens zio_write_allocate_gang_member_done, NULL, 1304ea8dc4b6Seschrock zio->io_priority, zio->io_flags, 1305ea8dc4b6Seschrock &zio->io_bookmark)); 1306fa9e4066Sahrens } else { 1307fa9e4066Sahrens lsize = P2ROUNDUP(resid / gbps_left, SPA_MINBLOCKSIZE); 1308fa9e4066Sahrens ASSERT(lsize != SPA_MINBLOCKSIZE); 130944cd46caSbillm zio_nowait(zio_write_allocate(zio, spa, 131044cd46caSbillm zio->io_checksum, txg, gbp, 1311fa9e4066Sahrens (char *)zio->io_data + loff, lsize, 1312fa9e4066Sahrens zio_write_allocate_gang_member_done, NULL, 1313fa9e4066Sahrens zio->io_priority, zio->io_flags)); 1314fa9e4066Sahrens } 1315fa9e4066Sahrens } 1316fa9e4066Sahrens 1317fa9e4066Sahrens ASSERT(resid == 0 && loff == zio->io_size); 1318fa9e4066Sahrens 1319fa9e4066Sahrens zio->io_pipeline |= 1U << ZIO_STAGE_GANG_CHECKSUM_GENERATE; 1320fa9e4066Sahrens 1321fa9e4066Sahrens zio_push_transform(zio, gbh, gsize, gsize); 132244cd46caSbillm /* 132344cd46caSbillm * As much as we'd like this to be zio_wait_children_ready(), 132444cd46caSbillm * updating our ASIZE doesn't happen until the io_done callback, 132544cd46caSbillm * so we have to wait for that to finish in order for our BP 132644cd46caSbillm * to be stable. 132744cd46caSbillm */ 1328fa9e4066Sahrens zio_wait_children_done(zio); 1329fa9e4066Sahrens } 1330fa9e4066Sahrens 1331fa9e4066Sahrens /* 1332fa9e4066Sahrens * ========================================================================== 1333fa9e4066Sahrens * Allocate and free blocks 1334fa9e4066Sahrens * ========================================================================== 1335fa9e4066Sahrens */ 1336fa9e4066Sahrens static void 1337fa9e4066Sahrens zio_dva_allocate(zio_t *zio) 1338fa9e4066Sahrens { 1339fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 1340fa9e4066Sahrens int error; 1341fa9e4066Sahrens 1342fa9e4066Sahrens ASSERT(BP_IS_HOLE(bp)); 134344cd46caSbillm ASSERT3U(BP_GET_NDVAS(bp), ==, 0); 134444cd46caSbillm ASSERT3U(zio->io_ndvas, >, 0); 134544cd46caSbillm ASSERT3U(zio->io_ndvas, <=, spa_max_replication(zio->io_spa)); 1346fa9e4066Sahrens 1347fa9e4066Sahrens /* For testing, make some blocks above a certain size be gang blocks */ 1348fa9e4066Sahrens if (zio->io_size >= zio_gang_bang && (lbolt & 0x3) == 0) { 1349fa9e4066Sahrens zio_write_allocate_gang_members(zio); 1350fa9e4066Sahrens return; 1351fa9e4066Sahrens } 1352fa9e4066Sahrens 1353fa9e4066Sahrens ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 1354fa9e4066Sahrens 135544cd46caSbillm error = metaslab_alloc(zio->io_spa, zio->io_size, bp, zio->io_ndvas, 135667bd71c6Sperrin zio->io_txg, NULL, B_FALSE); 1357fa9e4066Sahrens 1358fa9e4066Sahrens if (error == 0) { 1359fa9e4066Sahrens bp->blk_birth = zio->io_txg; 1360fa9e4066Sahrens } else if (error == ENOSPC) { 1361fa9e4066Sahrens if (zio->io_size == SPA_MINBLOCKSIZE) 1362fa9e4066Sahrens panic("really, truly out of space"); 1363fa9e4066Sahrens zio_write_allocate_gang_members(zio); 1364fa9e4066Sahrens return; 1365fa9e4066Sahrens } else { 1366fa9e4066Sahrens zio->io_error = error; 1367fa9e4066Sahrens } 1368fa9e4066Sahrens zio_next_stage(zio); 1369fa9e4066Sahrens } 1370fa9e4066Sahrens 1371fa9e4066Sahrens static void 1372fa9e4066Sahrens zio_dva_free(zio_t *zio) 1373fa9e4066Sahrens { 1374fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 1375fa9e4066Sahrens 1376d80c45e0Sbonwick metaslab_free(zio->io_spa, bp, zio->io_txg, B_FALSE); 1377fa9e4066Sahrens 1378fa9e4066Sahrens BP_ZERO(bp); 1379fa9e4066Sahrens 1380fa9e4066Sahrens zio_next_stage(zio); 1381fa9e4066Sahrens } 1382fa9e4066Sahrens 1383fa9e4066Sahrens static void 1384fa9e4066Sahrens zio_dva_claim(zio_t *zio) 1385fa9e4066Sahrens { 1386d80c45e0Sbonwick zio->io_error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg); 1387fa9e4066Sahrens 1388fa9e4066Sahrens zio_next_stage(zio); 1389fa9e4066Sahrens } 1390fa9e4066Sahrens 1391fa9e4066Sahrens /* 1392fa9e4066Sahrens * ========================================================================== 1393fa9e4066Sahrens * Read and write to physical devices 1394fa9e4066Sahrens * ========================================================================== 1395fa9e4066Sahrens */ 1396fa9e4066Sahrens 1397fa9e4066Sahrens static void 139844cd46caSbillm zio_vdev_io_start(zio_t *zio) 1399fa9e4066Sahrens { 1400fa9e4066Sahrens vdev_t *vd = zio->io_vd; 140144cd46caSbillm vdev_t *tvd = vd ? vd->vdev_top : NULL; 140244cd46caSbillm blkptr_t *bp = zio->io_bp; 140344cd46caSbillm uint64_t align; 140444cd46caSbillm 140544cd46caSbillm if (vd == NULL) { 140644cd46caSbillm /* The mirror_ops handle multiple DVAs in a single BP */ 140744cd46caSbillm vdev_mirror_ops.vdev_op_io_start(zio); 140844cd46caSbillm return; 140944cd46caSbillm } 141044cd46caSbillm 141144cd46caSbillm align = 1ULL << tvd->vdev_ashift; 1412fa9e4066Sahrens 1413ecc2d604Sbonwick if (zio->io_retries == 0 && vd == tvd) 1414fa9e4066Sahrens zio->io_flags |= ZIO_FLAG_FAILFAST; 1415fa9e4066Sahrens 141644cd46caSbillm if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) && 141744cd46caSbillm vd->vdev_children == 0) { 1418fa9e4066Sahrens zio->io_flags |= ZIO_FLAG_PHYSICAL; 1419fa9e4066Sahrens zio->io_offset += VDEV_LABEL_START_SIZE; 1420fa9e4066Sahrens } 1421fa9e4066Sahrens 1422ecc2d604Sbonwick if (P2PHASE(zio->io_size, align) != 0) { 1423ecc2d604Sbonwick uint64_t asize = P2ROUNDUP(zio->io_size, align); 1424ecc2d604Sbonwick char *abuf = zio_buf_alloc(asize); 1425ecc2d604Sbonwick ASSERT(vd == tvd); 1426ecc2d604Sbonwick if (zio->io_type == ZIO_TYPE_WRITE) { 1427ecc2d604Sbonwick bcopy(zio->io_data, abuf, zio->io_size); 1428ecc2d604Sbonwick bzero(abuf + zio->io_size, asize - zio->io_size); 1429ecc2d604Sbonwick } 1430ecc2d604Sbonwick zio_push_transform(zio, abuf, asize, asize); 1431ecc2d604Sbonwick ASSERT(!(zio->io_flags & ZIO_FLAG_SUBBLOCK)); 1432ecc2d604Sbonwick zio->io_flags |= ZIO_FLAG_SUBBLOCK; 1433ecc2d604Sbonwick } 1434ecc2d604Sbonwick 1435ecc2d604Sbonwick ASSERT(P2PHASE(zio->io_offset, align) == 0); 1436ecc2d604Sbonwick ASSERT(P2PHASE(zio->io_size, align) == 0); 1437ecc2d604Sbonwick ASSERT(bp == NULL || 1438ecc2d604Sbonwick P2ROUNDUP(ZIO_GET_IOSIZE(zio), align) == zio->io_size); 1439fa9e4066Sahrens ASSERT(zio->io_type != ZIO_TYPE_WRITE || (spa_mode & FWRITE)); 1440fa9e4066Sahrens 1441fa9e4066Sahrens vdev_io_start(zio); 1442fa9e4066Sahrens 1443fa9e4066Sahrens /* zio_next_stage_async() gets called from io completion interrupt */ 1444fa9e4066Sahrens } 1445fa9e4066Sahrens 1446fa9e4066Sahrens static void 1447fa9e4066Sahrens zio_vdev_io_done(zio_t *zio) 1448fa9e4066Sahrens { 144944cd46caSbillm if (zio->io_vd == NULL) 145044cd46caSbillm /* The mirror_ops handle multiple DVAs in a single BP */ 145144cd46caSbillm vdev_mirror_ops.vdev_op_io_done(zio); 145244cd46caSbillm else 145344cd46caSbillm vdev_io_done(zio); 1454fa9e4066Sahrens } 1455fa9e4066Sahrens 1456fa9e4066Sahrens /* XXPOLICY */ 1457ea8dc4b6Seschrock boolean_t 1458fa9e4066Sahrens zio_should_retry(zio_t *zio) 1459fa9e4066Sahrens { 1460fa9e4066Sahrens vdev_t *vd = zio->io_vd; 1461fa9e4066Sahrens 1462fa9e4066Sahrens if (zio->io_error == 0) 1463fa9e4066Sahrens return (B_FALSE); 1464fa9e4066Sahrens if (zio->io_delegate_list != NULL) 1465fa9e4066Sahrens return (B_FALSE); 146644cd46caSbillm if (vd && vd != vd->vdev_top) 1467fa9e4066Sahrens return (B_FALSE); 1468fa9e4066Sahrens if (zio->io_flags & ZIO_FLAG_DONT_RETRY) 1469fa9e4066Sahrens return (B_FALSE); 1470ea8dc4b6Seschrock if (zio->io_retries > 0) 1471fa9e4066Sahrens return (B_FALSE); 1472fa9e4066Sahrens 1473fa9e4066Sahrens return (B_TRUE); 1474fa9e4066Sahrens } 1475fa9e4066Sahrens 1476fa9e4066Sahrens static void 1477fa9e4066Sahrens zio_vdev_io_assess(zio_t *zio) 1478fa9e4066Sahrens { 1479fa9e4066Sahrens vdev_t *vd = zio->io_vd; 148044cd46caSbillm vdev_t *tvd = vd ? vd->vdev_top : NULL; 1481fa9e4066Sahrens 1482fa9e4066Sahrens ASSERT(zio->io_vsd == NULL); 1483fa9e4066Sahrens 1484ecc2d604Sbonwick if (zio->io_flags & ZIO_FLAG_SUBBLOCK) { 1485ecc2d604Sbonwick void *abuf; 1486ecc2d604Sbonwick uint64_t asize; 1487ecc2d604Sbonwick ASSERT(vd == tvd); 1488ecc2d604Sbonwick zio_pop_transform(zio, &abuf, &asize, &asize); 1489ecc2d604Sbonwick if (zio->io_type == ZIO_TYPE_READ) 1490ecc2d604Sbonwick bcopy(abuf, zio->io_data, zio->io_size); 1491ecc2d604Sbonwick zio_buf_free(abuf, asize); 1492ecc2d604Sbonwick zio->io_flags &= ~ZIO_FLAG_SUBBLOCK; 1493ecc2d604Sbonwick } 1494ecc2d604Sbonwick 1495ea8dc4b6Seschrock if (zio_injection_enabled && !zio->io_error) 1496ea8dc4b6Seschrock zio->io_error = zio_handle_fault_injection(zio, EIO); 1497ea8dc4b6Seschrock 1498fa9e4066Sahrens /* 1499fa9e4066Sahrens * If the I/O failed, determine whether we should attempt to retry it. 1500fa9e4066Sahrens */ 1501fa9e4066Sahrens /* XXPOLICY */ 1502fa9e4066Sahrens if (zio_should_retry(zio)) { 1503fa9e4066Sahrens ASSERT(tvd == vd); 1504fa9e4066Sahrens 1505fa9e4066Sahrens zio->io_retries++; 1506fa9e4066Sahrens zio->io_error = 0; 1507b3995adbSahrens zio->io_flags &= ZIO_FLAG_VDEV_INHERIT | 1508b3995adbSahrens ZIO_FLAG_CONFIG_GRABBED; 1509fa9e4066Sahrens /* XXPOLICY */ 1510fa9e4066Sahrens zio->io_flags &= ~ZIO_FLAG_FAILFAST; 1511fa9e4066Sahrens zio->io_flags |= ZIO_FLAG_DONT_CACHE; 151244cd46caSbillm zio->io_stage = ZIO_STAGE_VDEV_IO_START - 1; 1513fa9e4066Sahrens 1514fa9e4066Sahrens dprintf("retry #%d for %s to %s offset %llx\n", 1515fa9e4066Sahrens zio->io_retries, zio_type_name[zio->io_type], 1516fa9e4066Sahrens vdev_description(vd), zio->io_offset); 1517fa9e4066Sahrens 1518ea8dc4b6Seschrock zio_next_stage_async(zio); 1519ea8dc4b6Seschrock return; 1520ea8dc4b6Seschrock } 1521fa9e4066Sahrens 152244cd46caSbillm if (zio->io_error != 0 && zio->io_error != ECKSUM && 152344cd46caSbillm !(zio->io_flags & ZIO_FLAG_SPECULATIVE) && vd) { 1524fa9e4066Sahrens /* 1525ea8dc4b6Seschrock * Poor man's hotplug support. Even if we're done retrying this 1526ea8dc4b6Seschrock * I/O, try to reopen the vdev to see if it's still attached. 1527ea8dc4b6Seschrock * To avoid excessive thrashing, we only try it once a minute. 1528ea8dc4b6Seschrock * This also has the effect of detecting when missing devices 1529ea8dc4b6Seschrock * have come back, by polling the device once a minute. 1530ea8dc4b6Seschrock * 1531ea8dc4b6Seschrock * We need to do this asynchronously because we can't grab 1532ea8dc4b6Seschrock * all the necessary locks way down here. 1533fa9e4066Sahrens */ 1534ea8dc4b6Seschrock if (gethrtime() - vd->vdev_last_try > 60ULL * NANOSEC) { 1535ea8dc4b6Seschrock vd->vdev_last_try = gethrtime(); 1536ea8dc4b6Seschrock tvd->vdev_reopen_wanted = 1; 1537ea8dc4b6Seschrock spa_async_request(vd->vdev_spa, SPA_ASYNC_REOPEN); 1538ea8dc4b6Seschrock } 1539fa9e4066Sahrens } 1540fa9e4066Sahrens 1541fa9e4066Sahrens zio_next_stage(zio); 1542fa9e4066Sahrens } 1543fa9e4066Sahrens 1544fa9e4066Sahrens void 1545fa9e4066Sahrens zio_vdev_io_reissue(zio_t *zio) 1546fa9e4066Sahrens { 1547fa9e4066Sahrens ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 1548fa9e4066Sahrens ASSERT(zio->io_error == 0); 1549fa9e4066Sahrens 1550fa9e4066Sahrens zio->io_stage--; 1551fa9e4066Sahrens } 1552fa9e4066Sahrens 1553fa9e4066Sahrens void 1554fa9e4066Sahrens zio_vdev_io_redone(zio_t *zio) 1555fa9e4066Sahrens { 1556fa9e4066Sahrens ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); 1557fa9e4066Sahrens 1558fa9e4066Sahrens zio->io_stage--; 1559fa9e4066Sahrens } 1560fa9e4066Sahrens 1561fa9e4066Sahrens void 1562fa9e4066Sahrens zio_vdev_io_bypass(zio_t *zio) 1563fa9e4066Sahrens { 1564fa9e4066Sahrens ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 1565fa9e4066Sahrens ASSERT(zio->io_error == 0); 1566fa9e4066Sahrens 1567fa9e4066Sahrens zio->io_flags |= ZIO_FLAG_IO_BYPASS; 1568fa9e4066Sahrens zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS - 1; 1569fa9e4066Sahrens } 1570fa9e4066Sahrens 1571fa9e4066Sahrens /* 1572fa9e4066Sahrens * ========================================================================== 1573fa9e4066Sahrens * Generate and verify checksums 1574fa9e4066Sahrens * ========================================================================== 1575fa9e4066Sahrens */ 1576fa9e4066Sahrens static void 1577fa9e4066Sahrens zio_checksum_generate(zio_t *zio) 1578fa9e4066Sahrens { 1579fa9e4066Sahrens int checksum = zio->io_checksum; 1580fa9e4066Sahrens blkptr_t *bp = zio->io_bp; 1581fa9e4066Sahrens 1582fa9e4066Sahrens ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 1583fa9e4066Sahrens 1584fa9e4066Sahrens BP_SET_CHECKSUM(bp, checksum); 1585fa9e4066Sahrens BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1586fa9e4066Sahrens 1587fa9e4066Sahrens zio_checksum(checksum, &bp->blk_cksum, zio->io_data, zio->io_size); 1588fa9e4066Sahrens 1589fa9e4066Sahrens zio_next_stage(zio); 1590fa9e4066Sahrens } 1591fa9e4066Sahrens 1592fa9e4066Sahrens static void 1593fa9e4066Sahrens zio_gang_checksum_generate(zio_t *zio) 1594fa9e4066Sahrens { 1595fa9e4066Sahrens zio_cksum_t zc; 1596fa9e4066Sahrens zio_gbh_phys_t *gbh = zio->io_data; 1597fa9e4066Sahrens 159844cd46caSbillm ASSERT(BP_IS_GANG(zio->io_bp)); 1599fa9e4066Sahrens ASSERT3U(zio->io_size, ==, SPA_GANGBLOCKSIZE); 1600fa9e4066Sahrens 1601fa9e4066Sahrens zio_set_gang_verifier(zio, &gbh->zg_tail.zbt_cksum); 1602fa9e4066Sahrens 1603fa9e4066Sahrens zio_checksum(ZIO_CHECKSUM_GANG_HEADER, &zc, zio->io_data, zio->io_size); 1604fa9e4066Sahrens 1605fa9e4066Sahrens zio_next_stage(zio); 1606fa9e4066Sahrens } 1607fa9e4066Sahrens 1608fa9e4066Sahrens static void 1609fa9e4066Sahrens zio_checksum_verify(zio_t *zio) 1610fa9e4066Sahrens { 1611fa9e4066Sahrens if (zio->io_bp != NULL) { 1612fa9e4066Sahrens zio->io_error = zio_checksum_error(zio); 1613ea8dc4b6Seschrock if (zio->io_error && !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) 1614ea8dc4b6Seschrock zfs_ereport_post(FM_EREPORT_ZFS_CHECKSUM, 1615ea8dc4b6Seschrock zio->io_spa, zio->io_vd, zio, 0, 0); 1616fa9e4066Sahrens } 1617fa9e4066Sahrens 1618fa9e4066Sahrens zio_next_stage(zio); 1619fa9e4066Sahrens } 1620fa9e4066Sahrens 1621fa9e4066Sahrens /* 1622fa9e4066Sahrens * Called by RAID-Z to ensure we don't compute the checksum twice. 1623fa9e4066Sahrens */ 1624fa9e4066Sahrens void 1625fa9e4066Sahrens zio_checksum_verified(zio_t *zio) 1626fa9e4066Sahrens { 1627fa9e4066Sahrens zio->io_pipeline &= ~(1U << ZIO_STAGE_CHECKSUM_VERIFY); 1628fa9e4066Sahrens } 1629fa9e4066Sahrens 1630fa9e4066Sahrens /* 1631fa9e4066Sahrens * Set the external verifier for a gang block based on stuff in the bp 1632fa9e4066Sahrens */ 1633fa9e4066Sahrens void 1634fa9e4066Sahrens zio_set_gang_verifier(zio_t *zio, zio_cksum_t *zcp) 1635fa9e4066Sahrens { 163644cd46caSbillm blkptr_t *bp = zio->io_bp; 163744cd46caSbillm 163844cd46caSbillm zcp->zc_word[0] = DVA_GET_VDEV(BP_IDENTITY(bp)); 163944cd46caSbillm zcp->zc_word[1] = DVA_GET_OFFSET(BP_IDENTITY(bp)); 164044cd46caSbillm zcp->zc_word[2] = bp->blk_birth; 1641fa9e4066Sahrens zcp->zc_word[3] = 0; 1642fa9e4066Sahrens } 1643fa9e4066Sahrens 1644fa9e4066Sahrens /* 1645fa9e4066Sahrens * ========================================================================== 1646fa9e4066Sahrens * Define the pipeline 1647fa9e4066Sahrens * ========================================================================== 1648fa9e4066Sahrens */ 1649fa9e4066Sahrens typedef void zio_pipe_stage_t(zio_t *zio); 1650fa9e4066Sahrens 1651fa9e4066Sahrens static void 1652fa9e4066Sahrens zio_badop(zio_t *zio) 1653fa9e4066Sahrens { 1654fa9e4066Sahrens panic("Invalid I/O pipeline stage %u for zio %p", zio->io_stage, zio); 1655fa9e4066Sahrens } 1656fa9e4066Sahrens 1657fa9e4066Sahrens zio_pipe_stage_t *zio_pipeline[ZIO_STAGE_DONE + 2] = { 1658fa9e4066Sahrens zio_badop, 1659fa9e4066Sahrens zio_wait_children_ready, 1660fa9e4066Sahrens zio_write_compress, 1661fa9e4066Sahrens zio_checksum_generate, 1662fa9e4066Sahrens zio_gang_pipeline, 1663fa9e4066Sahrens zio_get_gang_header, 1664fa9e4066Sahrens zio_rewrite_gang_members, 1665fa9e4066Sahrens zio_free_gang_members, 1666fa9e4066Sahrens zio_claim_gang_members, 1667fa9e4066Sahrens zio_dva_allocate, 1668fa9e4066Sahrens zio_dva_free, 1669fa9e4066Sahrens zio_dva_claim, 1670fa9e4066Sahrens zio_gang_checksum_generate, 1671fa9e4066Sahrens zio_ready, 1672fa9e4066Sahrens zio_vdev_io_start, 1673fa9e4066Sahrens zio_vdev_io_done, 1674fa9e4066Sahrens zio_vdev_io_assess, 1675fa9e4066Sahrens zio_wait_children_done, 1676fa9e4066Sahrens zio_checksum_verify, 1677fa9e4066Sahrens zio_read_gang_members, 1678fa9e4066Sahrens zio_read_decompress, 1679fa9e4066Sahrens zio_done, 1680fa9e4066Sahrens zio_badop 1681fa9e4066Sahrens }; 1682fa9e4066Sahrens 1683fa9e4066Sahrens /* 1684fa9e4066Sahrens * Move an I/O to the next stage of the pipeline and execute that stage. 1685fa9e4066Sahrens * There's no locking on io_stage because there's no legitimate way for 1686fa9e4066Sahrens * multiple threads to be attempting to process the same I/O. 1687fa9e4066Sahrens */ 1688fa9e4066Sahrens void 1689fa9e4066Sahrens zio_next_stage(zio_t *zio) 1690fa9e4066Sahrens { 1691fa9e4066Sahrens uint32_t pipeline = zio->io_pipeline; 1692fa9e4066Sahrens 1693fa9e4066Sahrens ASSERT(!MUTEX_HELD(&zio->io_lock)); 1694fa9e4066Sahrens 1695fa9e4066Sahrens if (zio->io_error) { 1696fa9e4066Sahrens dprintf("zio %p vdev %s offset %llx stage %d error %d\n", 1697fa9e4066Sahrens zio, vdev_description(zio->io_vd), 1698fa9e4066Sahrens zio->io_offset, zio->io_stage, zio->io_error); 1699fa9e4066Sahrens if (((1U << zio->io_stage) & ZIO_VDEV_IO_PIPELINE) == 0) 1700fa9e4066Sahrens pipeline &= ZIO_ERROR_PIPELINE_MASK; 1701fa9e4066Sahrens } 1702fa9e4066Sahrens 1703fa9e4066Sahrens while (((1U << ++zio->io_stage) & pipeline) == 0) 1704fa9e4066Sahrens continue; 1705fa9e4066Sahrens 1706fa9e4066Sahrens ASSERT(zio->io_stage <= ZIO_STAGE_DONE); 1707fa9e4066Sahrens ASSERT(zio->io_stalled == 0); 1708fa9e4066Sahrens 17099bc11082Sek /* 17109bc11082Sek * See the comment in zio_next_stage_async() about per-CPU taskqs. 17119bc11082Sek */ 17129bc11082Sek if (((1U << zio->io_stage) & zio->io_async_stages) && 17139bc11082Sek (zio->io_stage == ZIO_STAGE_WRITE_COMPRESS) && 17149bc11082Sek !(zio->io_flags & ZIO_FLAG_METADATA)) { 17159bc11082Sek taskq_t *tq = zio->io_spa->spa_zio_issue_taskq[zio->io_type]; 17169bc11082Sek (void) taskq_dispatch(tq, 17179bc11082Sek (task_func_t *)zio_pipeline[zio->io_stage], zio, TQ_SLEEP); 17189bc11082Sek } else { 17199bc11082Sek zio_pipeline[zio->io_stage](zio); 17209bc11082Sek } 1721fa9e4066Sahrens } 1722fa9e4066Sahrens 1723fa9e4066Sahrens void 1724fa9e4066Sahrens zio_next_stage_async(zio_t *zio) 1725fa9e4066Sahrens { 1726fa9e4066Sahrens taskq_t *tq; 1727fa9e4066Sahrens uint32_t pipeline = zio->io_pipeline; 1728fa9e4066Sahrens 1729fa9e4066Sahrens ASSERT(!MUTEX_HELD(&zio->io_lock)); 1730fa9e4066Sahrens 1731fa9e4066Sahrens if (zio->io_error) { 1732fa9e4066Sahrens dprintf("zio %p vdev %s offset %llx stage %d error %d\n", 1733fa9e4066Sahrens zio, vdev_description(zio->io_vd), 1734fa9e4066Sahrens zio->io_offset, zio->io_stage, zio->io_error); 1735fa9e4066Sahrens if (((1U << zio->io_stage) & ZIO_VDEV_IO_PIPELINE) == 0) 1736fa9e4066Sahrens pipeline &= ZIO_ERROR_PIPELINE_MASK; 1737fa9e4066Sahrens } 1738fa9e4066Sahrens 1739fa9e4066Sahrens while (((1U << ++zio->io_stage) & pipeline) == 0) 1740fa9e4066Sahrens continue; 1741fa9e4066Sahrens 1742fa9e4066Sahrens ASSERT(zio->io_stage <= ZIO_STAGE_DONE); 1743fa9e4066Sahrens ASSERT(zio->io_stalled == 0); 1744fa9e4066Sahrens 1745fa9e4066Sahrens /* 1746fa9e4066Sahrens * For performance, we'll probably want two sets of task queues: 1747fa9e4066Sahrens * per-CPU issue taskqs and per-CPU completion taskqs. The per-CPU 1748fa9e4066Sahrens * part is for read performance: since we have to make a pass over 1749fa9e4066Sahrens * the data to checksum it anyway, we want to do this on the same CPU 1750fa9e4066Sahrens * that issued the read, because (assuming CPU scheduling affinity) 1751fa9e4066Sahrens * that thread is probably still there. Getting this optimization 1752fa9e4066Sahrens * right avoids performance-hostile cache-to-cache transfers. 1753fa9e4066Sahrens * 1754fa9e4066Sahrens * Note that having two sets of task queues is also necessary for 1755fa9e4066Sahrens * correctness: if all of the issue threads get bogged down waiting 1756fa9e4066Sahrens * for dependent reads (e.g. metaslab freelist) to complete, then 1757fa9e4066Sahrens * there won't be any threads available to service I/O completion 1758fa9e4066Sahrens * interrupts. 1759fa9e4066Sahrens */ 1760fa9e4066Sahrens if ((1U << zio->io_stage) & zio->io_async_stages) { 1761fa9e4066Sahrens if (zio->io_stage < ZIO_STAGE_VDEV_IO_DONE) 1762fa9e4066Sahrens tq = zio->io_spa->spa_zio_issue_taskq[zio->io_type]; 1763fa9e4066Sahrens else 1764fa9e4066Sahrens tq = zio->io_spa->spa_zio_intr_taskq[zio->io_type]; 1765fa9e4066Sahrens (void) taskq_dispatch(tq, 1766fa9e4066Sahrens (task_func_t *)zio_pipeline[zio->io_stage], zio, TQ_SLEEP); 1767fa9e4066Sahrens } else { 1768fa9e4066Sahrens zio_pipeline[zio->io_stage](zio); 1769fa9e4066Sahrens } 1770fa9e4066Sahrens } 1771fa9e4066Sahrens 1772d63d470bSgw static boolean_t 1773d63d470bSgw zio_alloc_should_fail(void) 1774d63d470bSgw { 1775d63d470bSgw static uint16_t allocs = 0; 1776d63d470bSgw 1777d63d470bSgw return (P2PHASE(allocs++, 1U<<zio_zil_fail_shift) == 0); 1778d63d470bSgw } 1779d63d470bSgw 1780fa9e4066Sahrens /* 1781fa9e4066Sahrens * Try to allocate an intent log block. Return 0 on success, errno on failure. 1782fa9e4066Sahrens */ 1783fa9e4066Sahrens int 178467bd71c6Sperrin zio_alloc_blk(spa_t *spa, uint64_t size, blkptr_t *new_bp, blkptr_t *old_bp, 178567bd71c6Sperrin uint64_t txg) 1786fa9e4066Sahrens { 1787fa9e4066Sahrens int error; 1788fa9e4066Sahrens 1789ea8dc4b6Seschrock spa_config_enter(spa, RW_READER, FTAG); 1790fa9e4066Sahrens 1791d63d470bSgw if (zio_zil_fail_shift && zio_alloc_should_fail()) { 1792d63d470bSgw spa_config_exit(spa, FTAG); 1793d63d470bSgw return (ENOSPC); 1794d63d470bSgw } 1795d63d470bSgw 179667bd71c6Sperrin /* 179767bd71c6Sperrin * We were passed the previous log blocks dva_t in bp->blk_dva[0]. 179867bd71c6Sperrin */ 179967bd71c6Sperrin error = metaslab_alloc(spa, size, new_bp, 1, txg, old_bp, B_TRUE); 1800fa9e4066Sahrens 1801fa9e4066Sahrens if (error == 0) { 180267bd71c6Sperrin BP_SET_LSIZE(new_bp, size); 180367bd71c6Sperrin BP_SET_PSIZE(new_bp, size); 180467bd71c6Sperrin BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF); 180567bd71c6Sperrin BP_SET_CHECKSUM(new_bp, ZIO_CHECKSUM_ZILOG); 180667bd71c6Sperrin BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); 180767bd71c6Sperrin BP_SET_LEVEL(new_bp, 0); 180867bd71c6Sperrin BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER); 180967bd71c6Sperrin new_bp->blk_birth = txg; 1810fa9e4066Sahrens } 1811fa9e4066Sahrens 1812ea8dc4b6Seschrock spa_config_exit(spa, FTAG); 1813fa9e4066Sahrens 1814fa9e4066Sahrens return (error); 1815fa9e4066Sahrens } 1816fa9e4066Sahrens 1817fa9e4066Sahrens /* 1818fa9e4066Sahrens * Free an intent log block. We know it can't be a gang block, so there's 1819fa9e4066Sahrens * nothing to do except metaslab_free() it. 1820fa9e4066Sahrens */ 1821fa9e4066Sahrens void 1822fa9e4066Sahrens zio_free_blk(spa_t *spa, blkptr_t *bp, uint64_t txg) 1823fa9e4066Sahrens { 182444cd46caSbillm ASSERT(!BP_IS_GANG(bp)); 1825fa9e4066Sahrens 1826ea8dc4b6Seschrock spa_config_enter(spa, RW_READER, FTAG); 1827fa9e4066Sahrens 1828d80c45e0Sbonwick metaslab_free(spa, bp, txg, B_FALSE); 1829fa9e4066Sahrens 1830ea8dc4b6Seschrock spa_config_exit(spa, FTAG); 1831fa9e4066Sahrens } 1832