1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5fe9cf88cSperrin * Common Development and Distribution License (the "License"). 6fe9cf88cSperrin * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 2255da60b9SMark J Musante * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23ce636f8bSMatthew Ahrens * Copyright (c) 2012 by Delphix. All rights reserved. 24fa9e4066Sahrens */ 25fa9e4066Sahrens 2655da60b9SMark J Musante /* Portions Copyright 2010 Robert Milkowski */ 2755da60b9SMark J Musante 28fa9e4066Sahrens #include <sys/zfs_context.h> 29fa9e4066Sahrens #include <sys/spa.h> 30fa9e4066Sahrens #include <sys/dmu.h> 31fa9e4066Sahrens #include <sys/zap.h> 32fa9e4066Sahrens #include <sys/arc.h> 33fa9e4066Sahrens #include <sys/stat.h> 34fa9e4066Sahrens #include <sys/resource.h> 35fa9e4066Sahrens #include <sys/zil.h> 36fa9e4066Sahrens #include <sys/zil_impl.h> 37fa9e4066Sahrens #include <sys/dsl_dataset.h> 384b964adaSGeorge Wilson #include <sys/vdev_impl.h> 39d63d470bSgw #include <sys/dmu_tx.h> 403f9d6ad7SLin Ling #include <sys/dsl_pool.h> 41fa9e4066Sahrens 42fa9e4066Sahrens /* 43fa9e4066Sahrens * The zfs intent log (ZIL) saves transaction records of system calls 44fa9e4066Sahrens * that change the file system in memory with enough information 45fa9e4066Sahrens * to be able to replay them. These are stored in memory until 46fa9e4066Sahrens * either the DMU transaction group (txg) commits them to the stable pool 47fa9e4066Sahrens * and they can be discarded, or they are flushed to the stable log 48fa9e4066Sahrens * (also in the pool) due to a fsync, O_DSYNC or other synchronous 49fa9e4066Sahrens * requirement. In the event of a panic or power fail then those log 50fa9e4066Sahrens * records (transactions) are replayed. 51fa9e4066Sahrens * 52fa9e4066Sahrens * There is one ZIL per file system. Its on-disk (pool) format consists 53fa9e4066Sahrens * of 3 parts: 54fa9e4066Sahrens * 55fa9e4066Sahrens * - ZIL header 56fa9e4066Sahrens * - ZIL blocks 57fa9e4066Sahrens * - ZIL records 58fa9e4066Sahrens * 59fa9e4066Sahrens * A log record holds a system call transaction. Log blocks can 60fa9e4066Sahrens * hold many log records and the blocks are chained together. 61fa9e4066Sahrens * Each ZIL block contains a block pointer (blkptr_t) to the next 62fa9e4066Sahrens * ZIL block in the chain. The ZIL header points to the first 63fa9e4066Sahrens * block in the chain. Note there is not a fixed place in the pool 64fa9e4066Sahrens * to hold blocks. They are dynamically allocated and freed as 65fa9e4066Sahrens * needed from the blocks available. Figure X shows the ZIL structure: 66fa9e4066Sahrens */ 67fa9e4066Sahrens 68fa9e4066Sahrens /* 69416e0cd8Sek * This global ZIL switch affects all pools 70fa9e4066Sahrens */ 7155da60b9SMark J Musante int zil_replay_disable = 0; /* disable intent logging replay */ 72416e0cd8Sek 73416e0cd8Sek /* 74416e0cd8Sek * Tunable parameter for debugging or performance analysis. Setting 75416e0cd8Sek * zfs_nocacheflush will cause corruption on power loss if a volatile 76416e0cd8Sek * out-of-order write cache is enabled. 77416e0cd8Sek */ 78416e0cd8Sek boolean_t zfs_nocacheflush = B_FALSE; 79fa9e4066Sahrens 80fa9e4066Sahrens static kmem_cache_t *zil_lwb_cache; 81fa9e4066Sahrens 8291de656bSNeil Perrin static void zil_async_to_sync(zilog_t *zilog, uint64_t foid); 838f18d1faSGeorge Wilson 846e1f5caaSNeil Perrin #define LWB_EMPTY(lwb) ((BP_GET_LSIZE(&lwb->lwb_blk) - \ 856e1f5caaSNeil Perrin sizeof (zil_chain_t)) == (lwb->lwb_sz - lwb->lwb_nused)) 866e1f5caaSNeil Perrin 876e1f5caaSNeil Perrin 885002558fSNeil Perrin /* 895002558fSNeil Perrin * ziltest is by and large an ugly hack, but very useful in 905002558fSNeil Perrin * checking replay without tedious work. 915002558fSNeil Perrin * When running ziltest we want to keep all itx's and so maintain 925002558fSNeil Perrin * a single list in the zl_itxg[] that uses a high txg: ZILTEST_TXG 935002558fSNeil Perrin * We subtract TXG_CONCURRENT_STATES to allow for common code. 945002558fSNeil Perrin */ 955002558fSNeil Perrin #define ZILTEST_TXG (UINT64_MAX - TXG_CONCURRENT_STATES) 965002558fSNeil Perrin 97fa9e4066Sahrens static int 98b24ab676SJeff Bonwick zil_bp_compare(const void *x1, const void *x2) 99fa9e4066Sahrens { 100b24ab676SJeff Bonwick const dva_t *dva1 = &((zil_bp_node_t *)x1)->zn_dva; 101b24ab676SJeff Bonwick const dva_t *dva2 = &((zil_bp_node_t *)x2)->zn_dva; 102fa9e4066Sahrens 103fa9e4066Sahrens if (DVA_GET_VDEV(dva1) < DVA_GET_VDEV(dva2)) 104fa9e4066Sahrens return (-1); 105fa9e4066Sahrens if (DVA_GET_VDEV(dva1) > DVA_GET_VDEV(dva2)) 106fa9e4066Sahrens return (1); 107fa9e4066Sahrens 108fa9e4066Sahrens if (DVA_GET_OFFSET(dva1) < DVA_GET_OFFSET(dva2)) 109fa9e4066Sahrens return (-1); 110fa9e4066Sahrens if (DVA_GET_OFFSET(dva1) > DVA_GET_OFFSET(dva2)) 111fa9e4066Sahrens return (1); 112fa9e4066Sahrens 113fa9e4066Sahrens return (0); 114fa9e4066Sahrens } 115fa9e4066Sahrens 116fa9e4066Sahrens static void 117b24ab676SJeff Bonwick zil_bp_tree_init(zilog_t *zilog) 118fa9e4066Sahrens { 119b24ab676SJeff Bonwick avl_create(&zilog->zl_bp_tree, zil_bp_compare, 120b24ab676SJeff Bonwick sizeof (zil_bp_node_t), offsetof(zil_bp_node_t, zn_node)); 121fa9e4066Sahrens } 122fa9e4066Sahrens 123fa9e4066Sahrens static void 124b24ab676SJeff Bonwick zil_bp_tree_fini(zilog_t *zilog) 125fa9e4066Sahrens { 126b24ab676SJeff Bonwick avl_tree_t *t = &zilog->zl_bp_tree; 127b24ab676SJeff Bonwick zil_bp_node_t *zn; 128fa9e4066Sahrens void *cookie = NULL; 129fa9e4066Sahrens 130fa9e4066Sahrens while ((zn = avl_destroy_nodes(t, &cookie)) != NULL) 131b24ab676SJeff Bonwick kmem_free(zn, sizeof (zil_bp_node_t)); 132fa9e4066Sahrens 133fa9e4066Sahrens avl_destroy(t); 134fa9e4066Sahrens } 135fa9e4066Sahrens 136b24ab676SJeff Bonwick int 137b24ab676SJeff Bonwick zil_bp_tree_add(zilog_t *zilog, const blkptr_t *bp) 138fa9e4066Sahrens { 139b24ab676SJeff Bonwick avl_tree_t *t = &zilog->zl_bp_tree; 140b24ab676SJeff Bonwick const dva_t *dva = BP_IDENTITY(bp); 141b24ab676SJeff Bonwick zil_bp_node_t *zn; 142fa9e4066Sahrens avl_index_t where; 143fa9e4066Sahrens 144fa9e4066Sahrens if (avl_find(t, dva, &where) != NULL) 145fa9e4066Sahrens return (EEXIST); 146fa9e4066Sahrens 147b24ab676SJeff Bonwick zn = kmem_alloc(sizeof (zil_bp_node_t), KM_SLEEP); 148fa9e4066Sahrens zn->zn_dva = *dva; 149fa9e4066Sahrens avl_insert(t, zn, where); 150fa9e4066Sahrens 151fa9e4066Sahrens return (0); 152fa9e4066Sahrens } 153fa9e4066Sahrens 154d80c45e0Sbonwick static zil_header_t * 155d80c45e0Sbonwick zil_header_in_syncing_context(zilog_t *zilog) 156d80c45e0Sbonwick { 157d80c45e0Sbonwick return ((zil_header_t *)zilog->zl_header); 158d80c45e0Sbonwick } 159d80c45e0Sbonwick 160d80c45e0Sbonwick static void 161d80c45e0Sbonwick zil_init_log_chain(zilog_t *zilog, blkptr_t *bp) 162d80c45e0Sbonwick { 163d80c45e0Sbonwick zio_cksum_t *zc = &bp->blk_cksum; 164d80c45e0Sbonwick 165d80c45e0Sbonwick zc->zc_word[ZIL_ZC_GUID_0] = spa_get_random(-1ULL); 166d80c45e0Sbonwick zc->zc_word[ZIL_ZC_GUID_1] = spa_get_random(-1ULL); 167d80c45e0Sbonwick zc->zc_word[ZIL_ZC_OBJSET] = dmu_objset_id(zilog->zl_os); 168d80c45e0Sbonwick zc->zc_word[ZIL_ZC_SEQ] = 1ULL; 169d80c45e0Sbonwick } 170d80c45e0Sbonwick 171fa9e4066Sahrens /* 172b24ab676SJeff Bonwick * Read a log block and make sure it's valid. 173fa9e4066Sahrens */ 174fa9e4066Sahrens static int 1756e1f5caaSNeil Perrin zil_read_log_block(zilog_t *zilog, const blkptr_t *bp, blkptr_t *nbp, void *dst, 1766e1f5caaSNeil Perrin char **end) 177fa9e4066Sahrens { 178b24ab676SJeff Bonwick enum zio_flag zio_flags = ZIO_FLAG_CANFAIL; 17913506d1eSmaybee uint32_t aflags = ARC_WAIT; 180b24ab676SJeff Bonwick arc_buf_t *abuf = NULL; 181b24ab676SJeff Bonwick zbookmark_t zb; 182fa9e4066Sahrens int error; 183fa9e4066Sahrens 184b24ab676SJeff Bonwick if (zilog->zl_header->zh_claim_txg == 0) 185b24ab676SJeff Bonwick zio_flags |= ZIO_FLAG_SPECULATIVE | ZIO_FLAG_SCRUB; 186ea8dc4b6Seschrock 187b24ab676SJeff Bonwick if (!(zilog->zl_header->zh_flags & ZIL_CLAIM_LR_SEQ_VALID)) 188b24ab676SJeff Bonwick zio_flags |= ZIO_FLAG_SPECULATIVE; 189fa9e4066Sahrens 190b24ab676SJeff Bonwick SET_BOOKMARK(&zb, bp->blk_cksum.zc_word[ZIL_ZC_OBJSET], 191b24ab676SJeff Bonwick ZB_ZIL_OBJECT, ZB_ZIL_LEVEL, bp->blk_cksum.zc_word[ZIL_ZC_SEQ]); 192b24ab676SJeff Bonwick 1931b912ec7SGeorge Wilson error = arc_read(NULL, zilog->zl_spa, bp, arc_getbuf_func, &abuf, 194b24ab676SJeff Bonwick ZIO_PRIORITY_SYNC_READ, zio_flags, &aflags, &zb); 195fa9e4066Sahrens 196d80c45e0Sbonwick if (error == 0) { 197d80c45e0Sbonwick zio_cksum_t cksum = bp->blk_cksum; 198fa9e4066Sahrens 199d80c45e0Sbonwick /* 200f5e6e722SNeil Perrin * Validate the checksummed log block. 201f5e6e722SNeil Perrin * 202d80c45e0Sbonwick * Sequence numbers should be... sequential. The checksum 203d80c45e0Sbonwick * verifier for the next block should be bp's checksum plus 1. 204f5e6e722SNeil Perrin * 205f5e6e722SNeil Perrin * Also check the log chain linkage and size used. 206d80c45e0Sbonwick */ 207d80c45e0Sbonwick cksum.zc_word[ZIL_ZC_SEQ]++; 208d80c45e0Sbonwick 2096e1f5caaSNeil Perrin if (BP_GET_CHECKSUM(bp) == ZIO_CHECKSUM_ZILOG2) { 2106e1f5caaSNeil Perrin zil_chain_t *zilc = abuf->b_data; 2116e1f5caaSNeil Perrin char *lr = (char *)(zilc + 1); 2126e1f5caaSNeil Perrin uint64_t len = zilc->zc_nused - sizeof (zil_chain_t); 2136e1f5caaSNeil Perrin 2146e1f5caaSNeil Perrin if (bcmp(&cksum, &zilc->zc_next_blk.blk_cksum, 2156e1f5caaSNeil Perrin sizeof (cksum)) || BP_IS_HOLE(&zilc->zc_next_blk)) { 2166e1f5caaSNeil Perrin error = ECKSUM; 2176e1f5caaSNeil Perrin } else { 2186e1f5caaSNeil Perrin bcopy(lr, dst, len); 2196e1f5caaSNeil Perrin *end = (char *)dst + len; 2206e1f5caaSNeil Perrin *nbp = zilc->zc_next_blk; 2216e1f5caaSNeil Perrin } 2226e1f5caaSNeil Perrin } else { 2236e1f5caaSNeil Perrin char *lr = abuf->b_data; 2246e1f5caaSNeil Perrin uint64_t size = BP_GET_LSIZE(bp); 2256e1f5caaSNeil Perrin zil_chain_t *zilc = (zil_chain_t *)(lr + size) - 1; 2266e1f5caaSNeil Perrin 2276e1f5caaSNeil Perrin if (bcmp(&cksum, &zilc->zc_next_blk.blk_cksum, 2286e1f5caaSNeil Perrin sizeof (cksum)) || BP_IS_HOLE(&zilc->zc_next_blk) || 2296e1f5caaSNeil Perrin (zilc->zc_nused > (size - sizeof (*zilc)))) { 2306e1f5caaSNeil Perrin error = ECKSUM; 2316e1f5caaSNeil Perrin } else { 2326e1f5caaSNeil Perrin bcopy(lr, dst, zilc->zc_nused); 2336e1f5caaSNeil Perrin *end = (char *)dst + zilc->zc_nused; 2346e1f5caaSNeil Perrin *nbp = zilc->zc_next_blk; 2356e1f5caaSNeil Perrin } 2366e1f5caaSNeil Perrin } 237fa9e4066Sahrens 238*3b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(abuf, &abuf)); 239fa9e4066Sahrens } 240fa9e4066Sahrens 241b24ab676SJeff Bonwick return (error); 242b24ab676SJeff Bonwick } 243b24ab676SJeff Bonwick 244b24ab676SJeff Bonwick /* 245b24ab676SJeff Bonwick * Read a TX_WRITE log data block. 246b24ab676SJeff Bonwick */ 247b24ab676SJeff Bonwick static int 248b24ab676SJeff Bonwick zil_read_log_data(zilog_t *zilog, const lr_write_t *lr, void *wbuf) 249b24ab676SJeff Bonwick { 250b24ab676SJeff Bonwick enum zio_flag zio_flags = ZIO_FLAG_CANFAIL; 251b24ab676SJeff Bonwick const blkptr_t *bp = &lr->lr_blkptr; 252b24ab676SJeff Bonwick uint32_t aflags = ARC_WAIT; 253b24ab676SJeff Bonwick arc_buf_t *abuf = NULL; 254b24ab676SJeff Bonwick zbookmark_t zb; 255b24ab676SJeff Bonwick int error; 256b24ab676SJeff Bonwick 257b24ab676SJeff Bonwick if (BP_IS_HOLE(bp)) { 258b24ab676SJeff Bonwick if (wbuf != NULL) 259b24ab676SJeff Bonwick bzero(wbuf, MAX(BP_GET_LSIZE(bp), lr->lr_length)); 260b24ab676SJeff Bonwick return (0); 261b24ab676SJeff Bonwick } 262b24ab676SJeff Bonwick 263b24ab676SJeff Bonwick if (zilog->zl_header->zh_claim_txg == 0) 264b24ab676SJeff Bonwick zio_flags |= ZIO_FLAG_SPECULATIVE | ZIO_FLAG_SCRUB; 265b24ab676SJeff Bonwick 266b24ab676SJeff Bonwick SET_BOOKMARK(&zb, dmu_objset_id(zilog->zl_os), lr->lr_foid, 267b24ab676SJeff Bonwick ZB_ZIL_LEVEL, lr->lr_offset / BP_GET_LSIZE(bp)); 268b24ab676SJeff Bonwick 2691b912ec7SGeorge Wilson error = arc_read(NULL, zilog->zl_spa, bp, arc_getbuf_func, &abuf, 270b24ab676SJeff Bonwick ZIO_PRIORITY_SYNC_READ, zio_flags, &aflags, &zb); 271b24ab676SJeff Bonwick 272b24ab676SJeff Bonwick if (error == 0) { 273b24ab676SJeff Bonwick if (wbuf != NULL) 274b24ab676SJeff Bonwick bcopy(abuf->b_data, wbuf, arc_buf_size(abuf)); 275b24ab676SJeff Bonwick (void) arc_buf_remove_ref(abuf, &abuf); 276b24ab676SJeff Bonwick } 277fa9e4066Sahrens 278d80c45e0Sbonwick return (error); 279fa9e4066Sahrens } 280fa9e4066Sahrens 281fa9e4066Sahrens /* 282fa9e4066Sahrens * Parse the intent log, and call parse_func for each valid record within. 283fa9e4066Sahrens */ 284b24ab676SJeff Bonwick int 285fa9e4066Sahrens zil_parse(zilog_t *zilog, zil_parse_blk_func_t *parse_blk_func, 286fa9e4066Sahrens zil_parse_lr_func_t *parse_lr_func, void *arg, uint64_t txg) 287fa9e4066Sahrens { 288d80c45e0Sbonwick const zil_header_t *zh = zilog->zl_header; 289b24ab676SJeff Bonwick boolean_t claimed = !!zh->zh_claim_txg; 290b24ab676SJeff Bonwick uint64_t claim_blk_seq = claimed ? zh->zh_claim_blk_seq : UINT64_MAX; 291b24ab676SJeff Bonwick uint64_t claim_lr_seq = claimed ? zh->zh_claim_lr_seq : UINT64_MAX; 292b24ab676SJeff Bonwick uint64_t max_blk_seq = 0; 293b24ab676SJeff Bonwick uint64_t max_lr_seq = 0; 294b24ab676SJeff Bonwick uint64_t blk_count = 0; 295b24ab676SJeff Bonwick uint64_t lr_count = 0; 296b24ab676SJeff Bonwick blkptr_t blk, next_blk; 297fa9e4066Sahrens char *lrbuf, *lrp; 298b24ab676SJeff Bonwick int error = 0; 299fa9e4066Sahrens 300b24ab676SJeff Bonwick /* 301b24ab676SJeff Bonwick * Old logs didn't record the maximum zh_claim_lr_seq. 302b24ab676SJeff Bonwick */ 303b24ab676SJeff Bonwick if (!(zh->zh_flags & ZIL_CLAIM_LR_SEQ_VALID)) 304b24ab676SJeff Bonwick claim_lr_seq = UINT64_MAX; 305fa9e4066Sahrens 306fa9e4066Sahrens /* 307fa9e4066Sahrens * Starting at the block pointed to by zh_log we read the log chain. 308fa9e4066Sahrens * For each block in the chain we strongly check that block to 309fa9e4066Sahrens * ensure its validity. We stop when an invalid block is found. 310fa9e4066Sahrens * For each block pointer in the chain we call parse_blk_func(). 311fa9e4066Sahrens * For each record in each valid block we call parse_lr_func(). 312d80c45e0Sbonwick * If the log has been claimed, stop if we encounter a sequence 313d80c45e0Sbonwick * number greater than the highest claimed sequence number. 314fa9e4066Sahrens */ 315b24ab676SJeff Bonwick lrbuf = zio_buf_alloc(SPA_MAXBLOCKSIZE); 316b24ab676SJeff Bonwick zil_bp_tree_init(zilog); 317d80c45e0Sbonwick 318b24ab676SJeff Bonwick for (blk = zh->zh_log; !BP_IS_HOLE(&blk); blk = next_blk) { 319b24ab676SJeff Bonwick uint64_t blk_seq = blk.blk_cksum.zc_word[ZIL_ZC_SEQ]; 320b24ab676SJeff Bonwick int reclen; 3216e1f5caaSNeil Perrin char *end; 322d80c45e0Sbonwick 323b24ab676SJeff Bonwick if (blk_seq > claim_blk_seq) 324b24ab676SJeff Bonwick break; 325b24ab676SJeff Bonwick if ((error = parse_blk_func(zilog, &blk, arg, txg)) != 0) 326b24ab676SJeff Bonwick break; 3276e1f5caaSNeil Perrin ASSERT3U(max_blk_seq, <, blk_seq); 328b24ab676SJeff Bonwick max_blk_seq = blk_seq; 329b24ab676SJeff Bonwick blk_count++; 330fa9e4066Sahrens 331b24ab676SJeff Bonwick if (max_lr_seq == claim_lr_seq && max_blk_seq == claim_blk_seq) 332b24ab676SJeff Bonwick break; 333fa9e4066Sahrens 3346e1f5caaSNeil Perrin error = zil_read_log_block(zilog, &blk, &next_blk, lrbuf, &end); 335*3b2aab18SMatthew Ahrens if (error != 0) 336fa9e4066Sahrens break; 337fa9e4066Sahrens 3386e1f5caaSNeil Perrin for (lrp = lrbuf; lrp < end; lrp += reclen) { 339fa9e4066Sahrens lr_t *lr = (lr_t *)lrp; 340fa9e4066Sahrens reclen = lr->lrc_reclen; 341fa9e4066Sahrens ASSERT3U(reclen, >=, sizeof (lr_t)); 342b24ab676SJeff Bonwick if (lr->lrc_seq > claim_lr_seq) 343b24ab676SJeff Bonwick goto done; 344b24ab676SJeff Bonwick if ((error = parse_lr_func(zilog, lr, arg, txg)) != 0) 345b24ab676SJeff Bonwick goto done; 3466e1f5caaSNeil Perrin ASSERT3U(max_lr_seq, <, lr->lrc_seq); 347b24ab676SJeff Bonwick max_lr_seq = lr->lrc_seq; 348b24ab676SJeff Bonwick lr_count++; 349fa9e4066Sahrens } 350fa9e4066Sahrens } 351b24ab676SJeff Bonwick done: 352b24ab676SJeff Bonwick zilog->zl_parse_error = error; 353b24ab676SJeff Bonwick zilog->zl_parse_blk_seq = max_blk_seq; 354b24ab676SJeff Bonwick zilog->zl_parse_lr_seq = max_lr_seq; 355b24ab676SJeff Bonwick zilog->zl_parse_blk_count = blk_count; 356b24ab676SJeff Bonwick zilog->zl_parse_lr_count = lr_count; 357b24ab676SJeff Bonwick 358b24ab676SJeff Bonwick ASSERT(!claimed || !(zh->zh_flags & ZIL_CLAIM_LR_SEQ_VALID) || 359b24ab676SJeff Bonwick (max_blk_seq == claim_blk_seq && max_lr_seq == claim_lr_seq)); 360d80c45e0Sbonwick 361b24ab676SJeff Bonwick zil_bp_tree_fini(zilog); 362b24ab676SJeff Bonwick zio_buf_free(lrbuf, SPA_MAXBLOCKSIZE); 363b24ab676SJeff Bonwick 364b24ab676SJeff Bonwick return (error); 365fa9e4066Sahrens } 366fa9e4066Sahrens 367b24ab676SJeff Bonwick static int 368fa9e4066Sahrens zil_claim_log_block(zilog_t *zilog, blkptr_t *bp, void *tx, uint64_t first_txg) 369fa9e4066Sahrens { 370fa9e4066Sahrens /* 371fa9e4066Sahrens * Claim log block if not already committed and not already claimed. 372b24ab676SJeff Bonwick * If tx == NULL, just verify that the block is claimable. 373fa9e4066Sahrens */ 374b24ab676SJeff Bonwick if (bp->blk_birth < first_txg || zil_bp_tree_add(zilog, bp) != 0) 375b24ab676SJeff Bonwick return (0); 376b24ab676SJeff Bonwick 377b24ab676SJeff Bonwick return (zio_wait(zio_claim(NULL, zilog->zl_spa, 378b24ab676SJeff Bonwick tx == NULL ? 0 : first_txg, bp, spa_claim_notify, NULL, 379b24ab676SJeff Bonwick ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | ZIO_FLAG_SCRUB))); 380fa9e4066Sahrens } 381fa9e4066Sahrens 382b24ab676SJeff Bonwick static int 383fa9e4066Sahrens zil_claim_log_record(zilog_t *zilog, lr_t *lrc, void *tx, uint64_t first_txg) 384fa9e4066Sahrens { 385b24ab676SJeff Bonwick lr_write_t *lr = (lr_write_t *)lrc; 386b24ab676SJeff Bonwick int error; 387b24ab676SJeff Bonwick 388b24ab676SJeff Bonwick if (lrc->lrc_txtype != TX_WRITE) 389b24ab676SJeff Bonwick return (0); 390b24ab676SJeff Bonwick 391b24ab676SJeff Bonwick /* 392b24ab676SJeff Bonwick * If the block is not readable, don't claim it. This can happen 393b24ab676SJeff Bonwick * in normal operation when a log block is written to disk before 394b24ab676SJeff Bonwick * some of the dmu_sync() blocks it points to. In this case, the 395b24ab676SJeff Bonwick * transaction cannot have been committed to anyone (we would have 396b24ab676SJeff Bonwick * waited for all writes to be stable first), so it is semantically 397b24ab676SJeff Bonwick * correct to declare this the end of the log. 398b24ab676SJeff Bonwick */ 399b24ab676SJeff Bonwick if (lr->lr_blkptr.blk_birth >= first_txg && 400b24ab676SJeff Bonwick (error = zil_read_log_data(zilog, lr, NULL)) != 0) 401b24ab676SJeff Bonwick return (error); 402b24ab676SJeff Bonwick return (zil_claim_log_block(zilog, &lr->lr_blkptr, tx, first_txg)); 403fa9e4066Sahrens } 404fa9e4066Sahrens 405fa9e4066Sahrens /* ARGSUSED */ 406b24ab676SJeff Bonwick static int 407fa9e4066Sahrens zil_free_log_block(zilog_t *zilog, blkptr_t *bp, void *tx, uint64_t claim_txg) 408fa9e4066Sahrens { 409b24ab676SJeff Bonwick zio_free_zil(zilog->zl_spa, dmu_tx_get_txg(tx), bp); 410b24ab676SJeff Bonwick 411b24ab676SJeff Bonwick return (0); 412fa9e4066Sahrens } 413fa9e4066Sahrens 414b24ab676SJeff Bonwick static int 415fa9e4066Sahrens zil_free_log_record(zilog_t *zilog, lr_t *lrc, void *tx, uint64_t claim_txg) 416fa9e4066Sahrens { 417b24ab676SJeff Bonwick lr_write_t *lr = (lr_write_t *)lrc; 418b24ab676SJeff Bonwick blkptr_t *bp = &lr->lr_blkptr; 419b24ab676SJeff Bonwick 420fa9e4066Sahrens /* 421fa9e4066Sahrens * If we previously claimed it, we need to free it. 422fa9e4066Sahrens */ 423b24ab676SJeff Bonwick if (claim_txg != 0 && lrc->lrc_txtype == TX_WRITE && 424b24ab676SJeff Bonwick bp->blk_birth >= claim_txg && zil_bp_tree_add(zilog, bp) == 0) 425b24ab676SJeff Bonwick zio_free(zilog->zl_spa, dmu_tx_get_txg(tx), bp); 426b24ab676SJeff Bonwick 427b24ab676SJeff Bonwick return (0); 428fa9e4066Sahrens } 429fa9e4066Sahrens 4306e1f5caaSNeil Perrin static lwb_t * 4316e1f5caaSNeil Perrin zil_alloc_lwb(zilog_t *zilog, blkptr_t *bp, uint64_t txg) 4326e1f5caaSNeil Perrin { 4336e1f5caaSNeil Perrin lwb_t *lwb; 4346e1f5caaSNeil Perrin 4356e1f5caaSNeil Perrin lwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP); 4366e1f5caaSNeil Perrin lwb->lwb_zilog = zilog; 4376e1f5caaSNeil Perrin lwb->lwb_blk = *bp; 4386e1f5caaSNeil Perrin lwb->lwb_buf = zio_buf_alloc(BP_GET_LSIZE(bp)); 4396e1f5caaSNeil Perrin lwb->lwb_max_txg = txg; 4406e1f5caaSNeil Perrin lwb->lwb_zio = NULL; 4416e1f5caaSNeil Perrin lwb->lwb_tx = NULL; 4426e1f5caaSNeil Perrin if (BP_GET_CHECKSUM(bp) == ZIO_CHECKSUM_ZILOG2) { 4436e1f5caaSNeil Perrin lwb->lwb_nused = sizeof (zil_chain_t); 4446e1f5caaSNeil Perrin lwb->lwb_sz = BP_GET_LSIZE(bp); 4456e1f5caaSNeil Perrin } else { 4466e1f5caaSNeil Perrin lwb->lwb_nused = 0; 4476e1f5caaSNeil Perrin lwb->lwb_sz = BP_GET_LSIZE(bp) - sizeof (zil_chain_t); 4486e1f5caaSNeil Perrin } 4496e1f5caaSNeil Perrin 4506e1f5caaSNeil Perrin mutex_enter(&zilog->zl_lock); 4516e1f5caaSNeil Perrin list_insert_tail(&zilog->zl_lwb_list, lwb); 4526e1f5caaSNeil Perrin mutex_exit(&zilog->zl_lock); 4536e1f5caaSNeil Perrin 4546e1f5caaSNeil Perrin return (lwb); 4556e1f5caaSNeil Perrin } 4566e1f5caaSNeil Perrin 457ce636f8bSMatthew Ahrens /* 458ce636f8bSMatthew Ahrens * Called when we create in-memory log transactions so that we know 459ce636f8bSMatthew Ahrens * to cleanup the itxs at the end of spa_sync(). 460ce636f8bSMatthew Ahrens */ 461ce636f8bSMatthew Ahrens void 462ce636f8bSMatthew Ahrens zilog_dirty(zilog_t *zilog, uint64_t txg) 463ce636f8bSMatthew Ahrens { 464ce636f8bSMatthew Ahrens dsl_pool_t *dp = zilog->zl_dmu_pool; 465ce636f8bSMatthew Ahrens dsl_dataset_t *ds = dmu_objset_ds(zilog->zl_os); 466ce636f8bSMatthew Ahrens 467ce636f8bSMatthew Ahrens if (dsl_dataset_is_snapshot(ds)) 468ce636f8bSMatthew Ahrens panic("dirtying snapshot!"); 469ce636f8bSMatthew Ahrens 470*3b2aab18SMatthew Ahrens if (txg_list_add(&dp->dp_dirty_zilogs, zilog, txg)) { 471ce636f8bSMatthew Ahrens /* up the hold count until we can be written out */ 472ce636f8bSMatthew Ahrens dmu_buf_add_ref(ds->ds_dbuf, zilog); 473ce636f8bSMatthew Ahrens } 474ce636f8bSMatthew Ahrens } 475ce636f8bSMatthew Ahrens 476ce636f8bSMatthew Ahrens boolean_t 477ce636f8bSMatthew Ahrens zilog_is_dirty(zilog_t *zilog) 478ce636f8bSMatthew Ahrens { 479ce636f8bSMatthew Ahrens dsl_pool_t *dp = zilog->zl_dmu_pool; 480ce636f8bSMatthew Ahrens 481ce636f8bSMatthew Ahrens for (int t = 0; t < TXG_SIZE; t++) { 482ce636f8bSMatthew Ahrens if (txg_list_member(&dp->dp_dirty_zilogs, zilog, t)) 483ce636f8bSMatthew Ahrens return (B_TRUE); 484ce636f8bSMatthew Ahrens } 485ce636f8bSMatthew Ahrens return (B_FALSE); 486ce636f8bSMatthew Ahrens } 487ce636f8bSMatthew Ahrens 488fa9e4066Sahrens /* 489fa9e4066Sahrens * Create an on-disk intent log. 490fa9e4066Sahrens */ 4916e1f5caaSNeil Perrin static lwb_t * 492fa9e4066Sahrens zil_create(zilog_t *zilog) 493fa9e4066Sahrens { 494d80c45e0Sbonwick const zil_header_t *zh = zilog->zl_header; 4956e1f5caaSNeil Perrin lwb_t *lwb = NULL; 496d80c45e0Sbonwick uint64_t txg = 0; 497d80c45e0Sbonwick dmu_tx_t *tx = NULL; 498fa9e4066Sahrens blkptr_t blk; 499d80c45e0Sbonwick int error = 0; 500fa9e4066Sahrens 501fa9e4066Sahrens /* 502d80c45e0Sbonwick * Wait for any previous destroy to complete. 503fa9e4066Sahrens */ 504d80c45e0Sbonwick txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 505d80c45e0Sbonwick 506d80c45e0Sbonwick ASSERT(zh->zh_claim_txg == 0); 507d80c45e0Sbonwick ASSERT(zh->zh_replay_seq == 0); 508d80c45e0Sbonwick 509d80c45e0Sbonwick blk = zh->zh_log; 510fa9e4066Sahrens 511fa9e4066Sahrens /* 5126e1f5caaSNeil Perrin * Allocate an initial log block if: 5136e1f5caaSNeil Perrin * - there isn't one already 5146e1f5caaSNeil Perrin * - the existing block is the wrong endianess 515fa9e4066Sahrens */ 516899217ddSNeil Perrin if (BP_IS_HOLE(&blk) || BP_SHOULD_BYTESWAP(&blk)) { 517d80c45e0Sbonwick tx = dmu_tx_create(zilog->zl_os); 518b24ab676SJeff Bonwick VERIFY(dmu_tx_assign(tx, TXG_WAIT) == 0); 519d80c45e0Sbonwick dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 520d80c45e0Sbonwick txg = dmu_tx_get_txg(tx); 521d80c45e0Sbonwick 522899217ddSNeil Perrin if (!BP_IS_HOLE(&blk)) { 523b24ab676SJeff Bonwick zio_free_zil(zilog->zl_spa, txg, &blk); 524899217ddSNeil Perrin BP_ZERO(&blk); 525899217ddSNeil Perrin } 526899217ddSNeil Perrin 527b24ab676SJeff Bonwick error = zio_alloc_zil(zilog->zl_spa, txg, &blk, NULL, 528b24ab676SJeff Bonwick ZIL_MIN_BLKSZ, zilog->zl_logbias == ZFS_LOGBIAS_LATENCY); 529d80c45e0Sbonwick 530d80c45e0Sbonwick if (error == 0) 531d80c45e0Sbonwick zil_init_log_chain(zilog, &blk); 53213f5297eSperrin } 533fa9e4066Sahrens 534d80c45e0Sbonwick /* 535d80c45e0Sbonwick * Allocate a log write buffer (lwb) for the first log block. 536d80c45e0Sbonwick */ 5376e1f5caaSNeil Perrin if (error == 0) 5386e1f5caaSNeil Perrin lwb = zil_alloc_lwb(zilog, &blk, txg); 539fa9e4066Sahrens 540d80c45e0Sbonwick /* 541d80c45e0Sbonwick * If we just allocated the first log block, commit our transaction 542d80c45e0Sbonwick * and wait for zil_sync() to stuff the block poiner into zh_log. 543d80c45e0Sbonwick * (zh is part of the MOS, so we cannot modify it in open context.) 544d80c45e0Sbonwick */ 545d80c45e0Sbonwick if (tx != NULL) { 546d80c45e0Sbonwick dmu_tx_commit(tx); 54713f5297eSperrin txg_wait_synced(zilog->zl_dmu_pool, txg); 548d80c45e0Sbonwick } 549d80c45e0Sbonwick 550d80c45e0Sbonwick ASSERT(bcmp(&blk, &zh->zh_log, sizeof (blk)) == 0); 5516e1f5caaSNeil Perrin 5526e1f5caaSNeil Perrin return (lwb); 553fa9e4066Sahrens } 554fa9e4066Sahrens 555fa9e4066Sahrens /* 556fa9e4066Sahrens * In one tx, free all log blocks and clear the log header. 557d80c45e0Sbonwick * If keep_first is set, then we're replaying a log with no content. 558d80c45e0Sbonwick * We want to keep the first block, however, so that the first 559d80c45e0Sbonwick * synchronous transaction doesn't require a txg_wait_synced() 560d80c45e0Sbonwick * in zil_create(). We don't need to txg_wait_synced() here either 561d80c45e0Sbonwick * when keep_first is set, because both zil_create() and zil_destroy() 562d80c45e0Sbonwick * will wait for any in-progress destroys to complete. 563fa9e4066Sahrens */ 564fa9e4066Sahrens void 565d80c45e0Sbonwick zil_destroy(zilog_t *zilog, boolean_t keep_first) 566fa9e4066Sahrens { 567d80c45e0Sbonwick const zil_header_t *zh = zilog->zl_header; 568d80c45e0Sbonwick lwb_t *lwb; 569fa9e4066Sahrens dmu_tx_t *tx; 570fa9e4066Sahrens uint64_t txg; 571fa9e4066Sahrens 572d80c45e0Sbonwick /* 573d80c45e0Sbonwick * Wait for any previous destroy to complete. 574d80c45e0Sbonwick */ 575d80c45e0Sbonwick txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 576fa9e4066Sahrens 577b24ab676SJeff Bonwick zilog->zl_old_header = *zh; /* debugging aid */ 578b24ab676SJeff Bonwick 579d80c45e0Sbonwick if (BP_IS_HOLE(&zh->zh_log)) 580fa9e4066Sahrens return; 581fa9e4066Sahrens 582fa9e4066Sahrens tx = dmu_tx_create(zilog->zl_os); 583b24ab676SJeff Bonwick VERIFY(dmu_tx_assign(tx, TXG_WAIT) == 0); 584fa9e4066Sahrens dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 585fa9e4066Sahrens txg = dmu_tx_get_txg(tx); 586fa9e4066Sahrens 587d80c45e0Sbonwick mutex_enter(&zilog->zl_lock); 588d80c45e0Sbonwick 589d80c45e0Sbonwick ASSERT3U(zilog->zl_destroy_txg, <, txg); 590fa9e4066Sahrens zilog->zl_destroy_txg = txg; 591b24ab676SJeff Bonwick zilog->zl_keep_first = keep_first; 592d80c45e0Sbonwick 593d80c45e0Sbonwick if (!list_is_empty(&zilog->zl_lwb_list)) { 594d80c45e0Sbonwick ASSERT(zh->zh_claim_txg == 0); 595c9ba2a43SEric Schrock VERIFY(!keep_first); 596d80c45e0Sbonwick while ((lwb = list_head(&zilog->zl_lwb_list)) != NULL) { 597d80c45e0Sbonwick list_remove(&zilog->zl_lwb_list, lwb); 598d80c45e0Sbonwick if (lwb->lwb_buf != NULL) 599d80c45e0Sbonwick zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 600b24ab676SJeff Bonwick zio_free_zil(zilog->zl_spa, txg, &lwb->lwb_blk); 601d80c45e0Sbonwick kmem_cache_free(zil_lwb_cache, lwb); 602d80c45e0Sbonwick } 603b24ab676SJeff Bonwick } else if (!keep_first) { 604ce636f8bSMatthew Ahrens zil_destroy_sync(zilog, tx); 605d80c45e0Sbonwick } 606b19a79ecSperrin mutex_exit(&zilog->zl_lock); 607fa9e4066Sahrens 608fa9e4066Sahrens dmu_tx_commit(tx); 609fa9e4066Sahrens } 610fa9e4066Sahrens 611ce636f8bSMatthew Ahrens void 612ce636f8bSMatthew Ahrens zil_destroy_sync(zilog_t *zilog, dmu_tx_t *tx) 613ce636f8bSMatthew Ahrens { 614ce636f8bSMatthew Ahrens ASSERT(list_is_empty(&zilog->zl_lwb_list)); 615ce636f8bSMatthew Ahrens (void) zil_parse(zilog, zil_free_log_block, 616ce636f8bSMatthew Ahrens zil_free_log_record, tx, zilog->zl_header->zh_claim_txg); 617ce636f8bSMatthew Ahrens } 618ce636f8bSMatthew Ahrens 6191d452cf5Sahrens int 620fd136879SMatthew Ahrens zil_claim(const char *osname, void *txarg) 621fa9e4066Sahrens { 622fa9e4066Sahrens dmu_tx_t *tx = txarg; 623fa9e4066Sahrens uint64_t first_txg = dmu_tx_get_txg(tx); 624fa9e4066Sahrens zilog_t *zilog; 625fa9e4066Sahrens zil_header_t *zh; 626fa9e4066Sahrens objset_t *os; 627fa9e4066Sahrens int error; 628fa9e4066Sahrens 629*3b2aab18SMatthew Ahrens error = dmu_objset_own(osname, DMU_OST_ANY, B_FALSE, FTAG, &os); 630*3b2aab18SMatthew Ahrens if (error != 0) { 631b87f3af3Sperrin cmn_err(CE_WARN, "can't open objset for %s", osname); 6321d452cf5Sahrens return (0); 633fa9e4066Sahrens } 634fa9e4066Sahrens 635fa9e4066Sahrens zilog = dmu_objset_zil(os); 636d80c45e0Sbonwick zh = zil_header_in_syncing_context(zilog); 637fa9e4066Sahrens 638b24ab676SJeff Bonwick if (spa_get_log_state(zilog->zl_spa) == SPA_LOG_CLEAR) { 639e6ca193dSGeorge Wilson if (!BP_IS_HOLE(&zh->zh_log)) 640b24ab676SJeff Bonwick zio_free_zil(zilog->zl_spa, first_txg, &zh->zh_log); 641e6ca193dSGeorge Wilson BP_ZERO(&zh->zh_log); 642e6ca193dSGeorge Wilson dsl_dataset_dirty(dmu_objset_ds(os), tx); 643*3b2aab18SMatthew Ahrens dmu_objset_disown(os, FTAG); 644468c413aSTim Haley return (0); 645e6ca193dSGeorge Wilson } 646e6ca193dSGeorge Wilson 647fa9e4066Sahrens /* 648d80c45e0Sbonwick * Claim all log blocks if we haven't already done so, and remember 649d80c45e0Sbonwick * the highest claimed sequence number. This ensures that if we can 650d80c45e0Sbonwick * read only part of the log now (e.g. due to a missing device), 651d80c45e0Sbonwick * but we can read the entire log later, we will not try to replay 652d80c45e0Sbonwick * or destroy beyond the last block we successfully claimed. 653fa9e4066Sahrens */ 654fa9e4066Sahrens ASSERT3U(zh->zh_claim_txg, <=, first_txg); 655fa9e4066Sahrens if (zh->zh_claim_txg == 0 && !BP_IS_HOLE(&zh->zh_log)) { 656b24ab676SJeff Bonwick (void) zil_parse(zilog, zil_claim_log_block, 657d80c45e0Sbonwick zil_claim_log_record, tx, first_txg); 658b24ab676SJeff Bonwick zh->zh_claim_txg = first_txg; 659b24ab676SJeff Bonwick zh->zh_claim_blk_seq = zilog->zl_parse_blk_seq; 660b24ab676SJeff Bonwick zh->zh_claim_lr_seq = zilog->zl_parse_lr_seq; 661b24ab676SJeff Bonwick if (zilog->zl_parse_lr_count || zilog->zl_parse_blk_count > 1) 662b24ab676SJeff Bonwick zh->zh_flags |= ZIL_REPLAY_NEEDED; 663b24ab676SJeff Bonwick zh->zh_flags |= ZIL_CLAIM_LR_SEQ_VALID; 664fa9e4066Sahrens dsl_dataset_dirty(dmu_objset_ds(os), tx); 665fa9e4066Sahrens } 666d80c45e0Sbonwick 667fa9e4066Sahrens ASSERT3U(first_txg, ==, (spa_last_synced_txg(zilog->zl_spa) + 1)); 668*3b2aab18SMatthew Ahrens dmu_objset_disown(os, FTAG); 6691d452cf5Sahrens return (0); 670b87f3af3Sperrin } 671b87f3af3Sperrin 672b87f3af3Sperrin /* 673b87f3af3Sperrin * Check the log by walking the log chain. 674b87f3af3Sperrin * Checksum errors are ok as they indicate the end of the chain. 675b87f3af3Sperrin * Any other error (no device or read failure) returns an error. 676b87f3af3Sperrin */ 677b87f3af3Sperrin int 678fd136879SMatthew Ahrens zil_check_log_chain(const char *osname, void *tx) 679b87f3af3Sperrin { 680b87f3af3Sperrin zilog_t *zilog; 681b87f3af3Sperrin objset_t *os; 6824b964adaSGeorge Wilson blkptr_t *bp; 683b87f3af3Sperrin int error; 684b87f3af3Sperrin 685b24ab676SJeff Bonwick ASSERT(tx == NULL); 686b24ab676SJeff Bonwick 687503ad85cSMatthew Ahrens error = dmu_objset_hold(osname, FTAG, &os); 688*3b2aab18SMatthew Ahrens if (error != 0) { 689b87f3af3Sperrin cmn_err(CE_WARN, "can't open objset for %s", osname); 690b87f3af3Sperrin return (0); 691b87f3af3Sperrin } 692b87f3af3Sperrin 693b87f3af3Sperrin zilog = dmu_objset_zil(os); 6944b964adaSGeorge Wilson bp = (blkptr_t *)&zilog->zl_header->zh_log; 6954b964adaSGeorge Wilson 6964b964adaSGeorge Wilson /* 6974b964adaSGeorge Wilson * Check the first block and determine if it's on a log device 6984b964adaSGeorge Wilson * which may have been removed or faulted prior to loading this 6994b964adaSGeorge Wilson * pool. If so, there's no point in checking the rest of the log 7004b964adaSGeorge Wilson * as its content should have already been synced to the pool. 7014b964adaSGeorge Wilson */ 7024b964adaSGeorge Wilson if (!BP_IS_HOLE(bp)) { 7034b964adaSGeorge Wilson vdev_t *vd; 7044b964adaSGeorge Wilson boolean_t valid = B_TRUE; 7054b964adaSGeorge Wilson 7064b964adaSGeorge Wilson spa_config_enter(os->os_spa, SCL_STATE, FTAG, RW_READER); 7074b964adaSGeorge Wilson vd = vdev_lookup_top(os->os_spa, DVA_GET_VDEV(&bp->blk_dva[0])); 7084b964adaSGeorge Wilson if (vd->vdev_islog && vdev_is_dead(vd)) 7094b964adaSGeorge Wilson valid = vdev_log_state_valid(vd); 7104b964adaSGeorge Wilson spa_config_exit(os->os_spa, SCL_STATE, FTAG); 7114b964adaSGeorge Wilson 7124b964adaSGeorge Wilson if (!valid) { 7134b964adaSGeorge Wilson dmu_objset_rele(os, FTAG); 7144b964adaSGeorge Wilson return (0); 7154b964adaSGeorge Wilson } 7164b964adaSGeorge Wilson } 717b87f3af3Sperrin 718b24ab676SJeff Bonwick /* 719b24ab676SJeff Bonwick * Because tx == NULL, zil_claim_log_block() will not actually claim 720b24ab676SJeff Bonwick * any blocks, but just determine whether it is possible to do so. 721b24ab676SJeff Bonwick * In addition to checking the log chain, zil_claim_log_block() 722b24ab676SJeff Bonwick * will invoke zio_claim() with a done func of spa_claim_notify(), 723b24ab676SJeff Bonwick * which will update spa_max_claim_txg. See spa_load() for details. 724b24ab676SJeff Bonwick */ 725b24ab676SJeff Bonwick error = zil_parse(zilog, zil_claim_log_block, zil_claim_log_record, tx, 726b24ab676SJeff Bonwick zilog->zl_header->zh_claim_txg ? -1ULL : spa_first_txg(os->os_spa)); 727b24ab676SJeff Bonwick 728503ad85cSMatthew Ahrens dmu_objset_rele(os, FTAG); 729b24ab676SJeff Bonwick 730b24ab676SJeff Bonwick return ((error == ECKSUM || error == ENOENT) ? 0 : error); 731b87f3af3Sperrin } 732b87f3af3Sperrin 73317f17c2dSbonwick static int 73417f17c2dSbonwick zil_vdev_compare(const void *x1, const void *x2) 73517f17c2dSbonwick { 7365002558fSNeil Perrin const uint64_t v1 = ((zil_vdev_node_t *)x1)->zv_vdev; 7375002558fSNeil Perrin const uint64_t v2 = ((zil_vdev_node_t *)x2)->zv_vdev; 73817f17c2dSbonwick 73917f17c2dSbonwick if (v1 < v2) 74017f17c2dSbonwick return (-1); 74117f17c2dSbonwick if (v1 > v2) 74217f17c2dSbonwick return (1); 74317f17c2dSbonwick 74417f17c2dSbonwick return (0); 74517f17c2dSbonwick } 74617f17c2dSbonwick 747fa9e4066Sahrens void 748b24ab676SJeff Bonwick zil_add_block(zilog_t *zilog, const blkptr_t *bp) 749fa9e4066Sahrens { 75017f17c2dSbonwick avl_tree_t *t = &zilog->zl_vdev_tree; 75117f17c2dSbonwick avl_index_t where; 75217f17c2dSbonwick zil_vdev_node_t *zv, zvsearch; 75317f17c2dSbonwick int ndvas = BP_GET_NDVAS(bp); 75417f17c2dSbonwick int i; 755fa9e4066Sahrens 756416e0cd8Sek if (zfs_nocacheflush) 757fa9e4066Sahrens return; 758fa9e4066Sahrens 75917f17c2dSbonwick ASSERT(zilog->zl_writer); 76017f17c2dSbonwick 76117f17c2dSbonwick /* 76217f17c2dSbonwick * Even though we're zl_writer, we still need a lock because the 76317f17c2dSbonwick * zl_get_data() callbacks may have dmu_sync() done callbacks 76417f17c2dSbonwick * that will run concurrently. 76517f17c2dSbonwick */ 76617f17c2dSbonwick mutex_enter(&zilog->zl_vdev_lock); 76717f17c2dSbonwick for (i = 0; i < ndvas; i++) { 76817f17c2dSbonwick zvsearch.zv_vdev = DVA_GET_VDEV(&bp->blk_dva[i]); 76917f17c2dSbonwick if (avl_find(t, &zvsearch, &where) == NULL) { 77017f17c2dSbonwick zv = kmem_alloc(sizeof (*zv), KM_SLEEP); 77117f17c2dSbonwick zv->zv_vdev = zvsearch.zv_vdev; 77217f17c2dSbonwick avl_insert(t, zv, where); 77367bd71c6Sperrin } 77467bd71c6Sperrin } 77517f17c2dSbonwick mutex_exit(&zilog->zl_vdev_lock); 776fa9e4066Sahrens } 777fa9e4066Sahrens 77891de656bSNeil Perrin static void 77967bd71c6Sperrin zil_flush_vdevs(zilog_t *zilog) 78067bd71c6Sperrin { 78167bd71c6Sperrin spa_t *spa = zilog->zl_spa; 78217f17c2dSbonwick avl_tree_t *t = &zilog->zl_vdev_tree; 78317f17c2dSbonwick void *cookie = NULL; 78417f17c2dSbonwick zil_vdev_node_t *zv; 78517f17c2dSbonwick zio_t *zio; 786fa9e4066Sahrens 78767bd71c6Sperrin ASSERT(zilog->zl_writer); 78867bd71c6Sperrin 78917f17c2dSbonwick /* 79017f17c2dSbonwick * We don't need zl_vdev_lock here because we're the zl_writer, 79117f17c2dSbonwick * and all zl_get_data() callbacks are done. 79217f17c2dSbonwick */ 79317f17c2dSbonwick if (avl_numnodes(t) == 0) 79417f17c2dSbonwick return; 79517f17c2dSbonwick 796e14bb325SJeff Bonwick spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 79717f17c2dSbonwick 798e14bb325SJeff Bonwick zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 799fa9e4066Sahrens 80017f17c2dSbonwick while ((zv = avl_destroy_nodes(t, &cookie)) != NULL) { 80117f17c2dSbonwick vdev_t *vd = vdev_lookup_top(spa, zv->zv_vdev); 80217f17c2dSbonwick if (vd != NULL) 80317f17c2dSbonwick zio_flush(zio, vd); 80417f17c2dSbonwick kmem_free(zv, sizeof (*zv)); 80567bd71c6Sperrin } 80617f17c2dSbonwick 807fa9e4066Sahrens /* 808fa9e4066Sahrens * Wait for all the flushes to complete. Not all devices actually 809fa9e4066Sahrens * support the DKIOCFLUSHWRITECACHE ioctl, so it's OK if it fails. 810fa9e4066Sahrens */ 81117f17c2dSbonwick (void) zio_wait(zio); 81217f17c2dSbonwick 813e14bb325SJeff Bonwick spa_config_exit(spa, SCL_STATE, FTAG); 814fa9e4066Sahrens } 815fa9e4066Sahrens 816fa9e4066Sahrens /* 817fa9e4066Sahrens * Function called when a log block write completes 818fa9e4066Sahrens */ 819fa9e4066Sahrens static void 820fa9e4066Sahrens zil_lwb_write_done(zio_t *zio) 821fa9e4066Sahrens { 822fa9e4066Sahrens lwb_t *lwb = zio->io_private; 823fa9e4066Sahrens zilog_t *zilog = lwb->lwb_zilog; 824b24ab676SJeff Bonwick dmu_tx_t *tx = lwb->lwb_tx; 825fa9e4066Sahrens 826e14bb325SJeff Bonwick ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF); 827e14bb325SJeff Bonwick ASSERT(BP_GET_TYPE(zio->io_bp) == DMU_OT_INTENT_LOG); 828e14bb325SJeff Bonwick ASSERT(BP_GET_LEVEL(zio->io_bp) == 0); 829e14bb325SJeff Bonwick ASSERT(BP_GET_BYTEORDER(zio->io_bp) == ZFS_HOST_BYTEORDER); 830e14bb325SJeff Bonwick ASSERT(!BP_IS_GANG(zio->io_bp)); 831e14bb325SJeff Bonwick ASSERT(!BP_IS_HOLE(zio->io_bp)); 832e14bb325SJeff Bonwick ASSERT(zio->io_bp->blk_fill == 0); 833e14bb325SJeff Bonwick 834fa9e4066Sahrens /* 835ef0d8e11SNeil Perrin * Ensure the lwb buffer pointer is cleared before releasing 836ef0d8e11SNeil Perrin * the txg. If we have had an allocation failure and 837ef0d8e11SNeil Perrin * the txg is waiting to sync then we want want zil_sync() 838ef0d8e11SNeil Perrin * to remove the lwb so that it's not picked up as the next new 839ef0d8e11SNeil Perrin * one in zil_commit_writer(). zil_sync() will only remove 840ef0d8e11SNeil Perrin * the lwb if lwb_buf is null. 841fa9e4066Sahrens */ 842fa9e4066Sahrens zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 843fa9e4066Sahrens mutex_enter(&zilog->zl_lock); 844fa9e4066Sahrens lwb->lwb_buf = NULL; 845b24ab676SJeff Bonwick lwb->lwb_tx = NULL; 846b24ab676SJeff Bonwick mutex_exit(&zilog->zl_lock); 847ef0d8e11SNeil Perrin 848ef0d8e11SNeil Perrin /* 849ef0d8e11SNeil Perrin * Now that we've written this log block, we have a stable pointer 850ef0d8e11SNeil Perrin * to the next block in the chain, so it's OK to let the txg in 851b24ab676SJeff Bonwick * which we allocated the next block sync. 852ef0d8e11SNeil Perrin */ 853b24ab676SJeff Bonwick dmu_tx_commit(tx); 854fa9e4066Sahrens } 855fa9e4066Sahrens 856c5c6ffa0Smaybee /* 857c5c6ffa0Smaybee * Initialize the io for a log block. 858c5c6ffa0Smaybee */ 859c5c6ffa0Smaybee static void 860c5c6ffa0Smaybee zil_lwb_write_init(zilog_t *zilog, lwb_t *lwb) 861c5c6ffa0Smaybee { 862c5c6ffa0Smaybee zbookmark_t zb; 863c5c6ffa0Smaybee 864b24ab676SJeff Bonwick SET_BOOKMARK(&zb, lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_OBJSET], 865b24ab676SJeff Bonwick ZB_ZIL_OBJECT, ZB_ZIL_LEVEL, 866b24ab676SJeff Bonwick lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_SEQ]); 867c5c6ffa0Smaybee 868b19a79ecSperrin if (zilog->zl_root_zio == NULL) { 869b19a79ecSperrin zilog->zl_root_zio = zio_root(zilog->zl_spa, NULL, NULL, 870b19a79ecSperrin ZIO_FLAG_CANFAIL); 871b19a79ecSperrin } 87267bd71c6Sperrin if (lwb->lwb_zio == NULL) { 87367bd71c6Sperrin lwb->lwb_zio = zio_rewrite(zilog->zl_root_zio, zilog->zl_spa, 8746e1f5caaSNeil Perrin 0, &lwb->lwb_blk, lwb->lwb_buf, BP_GET_LSIZE(&lwb->lwb_blk), 875e6ca193dSGeorge Wilson zil_lwb_write_done, lwb, ZIO_PRIORITY_LOG_WRITE, 8768f18d1faSGeorge Wilson ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE, &zb); 87767bd71c6Sperrin } 878c5c6ffa0Smaybee } 879c5c6ffa0Smaybee 8806e1f5caaSNeil Perrin /* 8816e1f5caaSNeil Perrin * Define a limited set of intent log block sizes. 8826e1f5caaSNeil Perrin * These must be a multiple of 4KB. Note only the amount used (again 8836e1f5caaSNeil Perrin * aligned to 4KB) actually gets written. However, we can't always just 8846e1f5caaSNeil Perrin * allocate SPA_MAXBLOCKSIZE as the slog space could be exhausted. 8856e1f5caaSNeil Perrin */ 8866e1f5caaSNeil Perrin uint64_t zil_block_buckets[] = { 8876e1f5caaSNeil Perrin 4096, /* non TX_WRITE */ 8886e1f5caaSNeil Perrin 8192+4096, /* data base */ 8896e1f5caaSNeil Perrin 32*1024 + 4096, /* NFS writes */ 8906e1f5caaSNeil Perrin UINT64_MAX 8916e1f5caaSNeil Perrin }; 8926e1f5caaSNeil Perrin 893d48e086fSNeil Perrin /* 894d48e086fSNeil Perrin * Use the slog as long as the logbias is 'latency' and the current commit size 895d48e086fSNeil Perrin * is less than the limit or the total list size is less than 2X the limit. 896d48e086fSNeil Perrin * Limit checking is disabled by setting zil_slog_limit to UINT64_MAX. 897d48e086fSNeil Perrin */ 898d48e086fSNeil Perrin uint64_t zil_slog_limit = 1024 * 1024; 899d48e086fSNeil Perrin #define USE_SLOG(zilog) (((zilog)->zl_logbias == ZFS_LOGBIAS_LATENCY) && \ 900d48e086fSNeil Perrin (((zilog)->zl_cur_used < zil_slog_limit) || \ 901d48e086fSNeil Perrin ((zilog)->zl_itx_list_sz < (zil_slog_limit << 1)))) 902d48e086fSNeil Perrin 903fa9e4066Sahrens /* 904fa9e4066Sahrens * Start a log block write and advance to the next log block. 905fa9e4066Sahrens * Calls are serialized. 906fa9e4066Sahrens */ 907fa9e4066Sahrens static lwb_t * 908fa9e4066Sahrens zil_lwb_write_start(zilog_t *zilog, lwb_t *lwb) 909fa9e4066Sahrens { 9106e1f5caaSNeil Perrin lwb_t *nlwb = NULL; 9116e1f5caaSNeil Perrin zil_chain_t *zilc; 912d80c45e0Sbonwick spa_t *spa = zilog->zl_spa; 9136e1f5caaSNeil Perrin blkptr_t *bp; 914b24ab676SJeff Bonwick dmu_tx_t *tx; 915fa9e4066Sahrens uint64_t txg; 916ada693c4SNeil Perrin uint64_t zil_blksz, wsz; 9176e1f5caaSNeil Perrin int i, error; 9186e1f5caaSNeil Perrin 9196e1f5caaSNeil Perrin if (BP_GET_CHECKSUM(&lwb->lwb_blk) == ZIO_CHECKSUM_ZILOG2) { 9206e1f5caaSNeil Perrin zilc = (zil_chain_t *)lwb->lwb_buf; 9216e1f5caaSNeil Perrin bp = &zilc->zc_next_blk; 9226e1f5caaSNeil Perrin } else { 9236e1f5caaSNeil Perrin zilc = (zil_chain_t *)(lwb->lwb_buf + lwb->lwb_sz); 9246e1f5caaSNeil Perrin bp = &zilc->zc_next_blk; 9256e1f5caaSNeil Perrin } 926fa9e4066Sahrens 9276e1f5caaSNeil Perrin ASSERT(lwb->lwb_nused <= lwb->lwb_sz); 928fa9e4066Sahrens 929fa9e4066Sahrens /* 930fa9e4066Sahrens * Allocate the next block and save its address in this block 931fa9e4066Sahrens * before writing it in order to establish the log chain. 932fa9e4066Sahrens * Note that if the allocation of nlwb synced before we wrote 933fa9e4066Sahrens * the block that points at it (lwb), we'd leak it if we crashed. 934b24ab676SJeff Bonwick * Therefore, we don't do dmu_tx_commit() until zil_lwb_write_done(). 935b24ab676SJeff Bonwick * We dirty the dataset to ensure that zil_sync() will be called 936b24ab676SJeff Bonwick * to clean up in the event of allocation failure or I/O failure. 937fa9e4066Sahrens */ 938b24ab676SJeff Bonwick tx = dmu_tx_create(zilog->zl_os); 939b24ab676SJeff Bonwick VERIFY(dmu_tx_assign(tx, TXG_WAIT) == 0); 940b24ab676SJeff Bonwick dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 941b24ab676SJeff Bonwick txg = dmu_tx_get_txg(tx); 942b24ab676SJeff Bonwick 943b24ab676SJeff Bonwick lwb->lwb_tx = tx; 944fa9e4066Sahrens 945fa9e4066Sahrens /* 9466e1f5caaSNeil Perrin * Log blocks are pre-allocated. Here we select the size of the next 9476e1f5caaSNeil Perrin * block, based on size used in the last block. 9486e1f5caaSNeil Perrin * - first find the smallest bucket that will fit the block from a 9496e1f5caaSNeil Perrin * limited set of block sizes. This is because it's faster to write 9506e1f5caaSNeil Perrin * blocks allocated from the same metaslab as they are adjacent or 9516e1f5caaSNeil Perrin * close. 9526e1f5caaSNeil Perrin * - next find the maximum from the new suggested size and an array of 9536e1f5caaSNeil Perrin * previous sizes. This lessens a picket fence effect of wrongly 9546e1f5caaSNeil Perrin * guesssing the size if we have a stream of say 2k, 64k, 2k, 64k 9556e1f5caaSNeil Perrin * requests. 9566e1f5caaSNeil Perrin * 9576e1f5caaSNeil Perrin * Note we only write what is used, but we can't just allocate 9586e1f5caaSNeil Perrin * the maximum block size because we can exhaust the available 9596e1f5caaSNeil Perrin * pool log space. 960fa9e4066Sahrens */ 9616e1f5caaSNeil Perrin zil_blksz = zilog->zl_cur_used + sizeof (zil_chain_t); 9626e1f5caaSNeil Perrin for (i = 0; zil_blksz > zil_block_buckets[i]; i++) 9636e1f5caaSNeil Perrin continue; 9646e1f5caaSNeil Perrin zil_blksz = zil_block_buckets[i]; 9656e1f5caaSNeil Perrin if (zil_blksz == UINT64_MAX) 9666e1f5caaSNeil Perrin zil_blksz = SPA_MAXBLOCKSIZE; 9676e1f5caaSNeil Perrin zilog->zl_prev_blks[zilog->zl_prev_rotor] = zil_blksz; 9686e1f5caaSNeil Perrin for (i = 0; i < ZIL_PREV_BLKS; i++) 9696e1f5caaSNeil Perrin zil_blksz = MAX(zil_blksz, zilog->zl_prev_blks[i]); 9706e1f5caaSNeil Perrin zilog->zl_prev_rotor = (zilog->zl_prev_rotor + 1) & (ZIL_PREV_BLKS - 1); 971fa9e4066Sahrens 97267bd71c6Sperrin BP_ZERO(bp); 97367bd71c6Sperrin /* pass the old blkptr in order to spread log blocks across devs */ 974b24ab676SJeff Bonwick error = zio_alloc_zil(spa, txg, bp, &lwb->lwb_blk, zil_blksz, 975d48e086fSNeil Perrin USE_SLOG(zilog)); 976*3b2aab18SMatthew Ahrens if (error == 0) { 9776e1f5caaSNeil Perrin ASSERT3U(bp->blk_birth, ==, txg); 9786e1f5caaSNeil Perrin bp->blk_cksum = lwb->lwb_blk.blk_cksum; 9796e1f5caaSNeil Perrin bp->blk_cksum.zc_word[ZIL_ZC_SEQ]++; 980d63d470bSgw 981ea8dc4b6Seschrock /* 9826e1f5caaSNeil Perrin * Allocate a new log write buffer (lwb). 983ea8dc4b6Seschrock */ 9846e1f5caaSNeil Perrin nlwb = zil_alloc_lwb(zilog, bp, txg); 9856e1f5caaSNeil Perrin 9866e1f5caaSNeil Perrin /* Record the block for later vdev flushing */ 9876e1f5caaSNeil Perrin zil_add_block(zilog, &lwb->lwb_blk); 988fa9e4066Sahrens } 989fa9e4066Sahrens 9906e1f5caaSNeil Perrin if (BP_GET_CHECKSUM(&lwb->lwb_blk) == ZIO_CHECKSUM_ZILOG2) { 9916e1f5caaSNeil Perrin /* For Slim ZIL only write what is used. */ 992ada693c4SNeil Perrin wsz = P2ROUNDUP_TYPED(lwb->lwb_nused, ZIL_MIN_BLKSZ, uint64_t); 993ada693c4SNeil Perrin ASSERT3U(wsz, <=, lwb->lwb_sz); 994ada693c4SNeil Perrin zio_shrink(lwb->lwb_zio, wsz); 995fa9e4066Sahrens 996ada693c4SNeil Perrin } else { 997ada693c4SNeil Perrin wsz = lwb->lwb_sz; 9986e1f5caaSNeil Perrin } 999ada693c4SNeil Perrin 10006e1f5caaSNeil Perrin zilc->zc_pad = 0; 10016e1f5caaSNeil Perrin zilc->zc_nused = lwb->lwb_nused; 10026e1f5caaSNeil Perrin zilc->zc_eck.zec_cksum = lwb->lwb_blk.blk_cksum; 1003fa9e4066Sahrens 1004ada693c4SNeil Perrin /* 1005ada693c4SNeil Perrin * clear unused data for security 1006ada693c4SNeil Perrin */ 1007ada693c4SNeil Perrin bzero(lwb->lwb_buf + lwb->lwb_nused, wsz - lwb->lwb_nused); 1008ada693c4SNeil Perrin 10096e1f5caaSNeil Perrin zio_nowait(lwb->lwb_zio); /* Kick off the write for the old log block */ 101067bd71c6Sperrin 1011fa9e4066Sahrens /* 10126e1f5caaSNeil Perrin * If there was an allocation failure then nlwb will be null which 10136e1f5caaSNeil Perrin * forces a txg_wait_synced(). 1014fa9e4066Sahrens */ 1015fa9e4066Sahrens return (nlwb); 1016fa9e4066Sahrens } 1017fa9e4066Sahrens 1018fa9e4066Sahrens static lwb_t * 1019fa9e4066Sahrens zil_lwb_commit(zilog_t *zilog, itx_t *itx, lwb_t *lwb) 1020fa9e4066Sahrens { 1021fa9e4066Sahrens lr_t *lrc = &itx->itx_lr; /* common log record */ 1022b24ab676SJeff Bonwick lr_write_t *lrw = (lr_write_t *)lrc; 1023b24ab676SJeff Bonwick char *lr_buf; 1024fa9e4066Sahrens uint64_t txg = lrc->lrc_txg; 1025fa9e4066Sahrens uint64_t reclen = lrc->lrc_reclen; 1026b24ab676SJeff Bonwick uint64_t dlen = 0; 1027fa9e4066Sahrens 1028fa9e4066Sahrens if (lwb == NULL) 1029fa9e4066Sahrens return (NULL); 1030b24ab676SJeff Bonwick 1031fa9e4066Sahrens ASSERT(lwb->lwb_buf != NULL); 1032ce636f8bSMatthew Ahrens ASSERT(zilog_is_dirty(zilog) || 1033ce636f8bSMatthew Ahrens spa_freeze_txg(zilog->zl_spa) != UINT64_MAX); 1034fa9e4066Sahrens 1035c5c6ffa0Smaybee if (lrc->lrc_txtype == TX_WRITE && itx->itx_wr_state == WR_NEED_COPY) 1036c5c6ffa0Smaybee dlen = P2ROUNDUP_TYPED( 1037b24ab676SJeff Bonwick lrw->lr_length, sizeof (uint64_t), uint64_t); 1038fa9e4066Sahrens 1039104e2ed7Sperrin zilog->zl_cur_used += (reclen + dlen); 104022ac5be4Sperrin 104167bd71c6Sperrin zil_lwb_write_init(zilog, lwb); 104267bd71c6Sperrin 1043fa9e4066Sahrens /* 1044fa9e4066Sahrens * If this record won't fit in the current log block, start a new one. 1045fa9e4066Sahrens */ 10466e1f5caaSNeil Perrin if (lwb->lwb_nused + reclen + dlen > lwb->lwb_sz) { 1047fa9e4066Sahrens lwb = zil_lwb_write_start(zilog, lwb); 1048c5c6ffa0Smaybee if (lwb == NULL) 1049fa9e4066Sahrens return (NULL); 105067bd71c6Sperrin zil_lwb_write_init(zilog, lwb); 10516e1f5caaSNeil Perrin ASSERT(LWB_EMPTY(lwb)); 10526e1f5caaSNeil Perrin if (lwb->lwb_nused + reclen + dlen > lwb->lwb_sz) { 1053fa9e4066Sahrens txg_wait_synced(zilog->zl_dmu_pool, txg); 1054fa9e4066Sahrens return (lwb); 1055fa9e4066Sahrens } 1056fa9e4066Sahrens } 1057fa9e4066Sahrens 1058b24ab676SJeff Bonwick lr_buf = lwb->lwb_buf + lwb->lwb_nused; 1059b24ab676SJeff Bonwick bcopy(lrc, lr_buf, reclen); 1060b24ab676SJeff Bonwick lrc = (lr_t *)lr_buf; 1061b24ab676SJeff Bonwick lrw = (lr_write_t *)lrc; 1062c5c6ffa0Smaybee 1063c5c6ffa0Smaybee /* 1064c5c6ffa0Smaybee * If it's a write, fetch the data or get its blkptr as appropriate. 1065c5c6ffa0Smaybee */ 1066c5c6ffa0Smaybee if (lrc->lrc_txtype == TX_WRITE) { 1067c5c6ffa0Smaybee if (txg > spa_freeze_txg(zilog->zl_spa)) 1068c5c6ffa0Smaybee txg_wait_synced(zilog->zl_dmu_pool, txg); 1069c5c6ffa0Smaybee if (itx->itx_wr_state != WR_COPIED) { 1070c5c6ffa0Smaybee char *dbuf; 1071c5c6ffa0Smaybee int error; 1072c5c6ffa0Smaybee 1073c5c6ffa0Smaybee if (dlen) { 1074c5c6ffa0Smaybee ASSERT(itx->itx_wr_state == WR_NEED_COPY); 1075b24ab676SJeff Bonwick dbuf = lr_buf + reclen; 1076b24ab676SJeff Bonwick lrw->lr_common.lrc_reclen += dlen; 1077c5c6ffa0Smaybee } else { 1078c5c6ffa0Smaybee ASSERT(itx->itx_wr_state == WR_INDIRECT); 1079c5c6ffa0Smaybee dbuf = NULL; 1080c5c6ffa0Smaybee } 1081c5c6ffa0Smaybee error = zilog->zl_get_data( 1082b24ab676SJeff Bonwick itx->itx_private, lrw, dbuf, lwb->lwb_zio); 1083c87b8fc5SMark J Musante if (error == EIO) { 1084c87b8fc5SMark J Musante txg_wait_synced(zilog->zl_dmu_pool, txg); 1085c87b8fc5SMark J Musante return (lwb); 1086c87b8fc5SMark J Musante } 1087*3b2aab18SMatthew Ahrens if (error != 0) { 1088c5c6ffa0Smaybee ASSERT(error == ENOENT || error == EEXIST || 1089c5c6ffa0Smaybee error == EALREADY); 1090c5c6ffa0Smaybee return (lwb); 1091c5c6ffa0Smaybee } 1092c5c6ffa0Smaybee } 1093104e2ed7Sperrin } 1094c5c6ffa0Smaybee 1095b24ab676SJeff Bonwick /* 1096b24ab676SJeff Bonwick * We're actually making an entry, so update lrc_seq to be the 1097b24ab676SJeff Bonwick * log record sequence number. Note that this is generally not 1098b24ab676SJeff Bonwick * equal to the itx sequence number because not all transactions 1099b24ab676SJeff Bonwick * are synchronous, and sometimes spa_sync() gets there first. 1100b24ab676SJeff Bonwick */ 1101b24ab676SJeff Bonwick lrc->lrc_seq = ++zilog->zl_lr_seq; /* we are single threaded */ 1102c5c6ffa0Smaybee lwb->lwb_nused += reclen + dlen; 1103fa9e4066Sahrens lwb->lwb_max_txg = MAX(lwb->lwb_max_txg, txg); 11046e1f5caaSNeil Perrin ASSERT3U(lwb->lwb_nused, <=, lwb->lwb_sz); 1105fb09f5aaSMadhav Suresh ASSERT0(P2PHASE(lwb->lwb_nused, sizeof (uint64_t))); 1106fa9e4066Sahrens 1107fa9e4066Sahrens return (lwb); 1108fa9e4066Sahrens } 1109fa9e4066Sahrens 1110fa9e4066Sahrens itx_t * 1111da6c28aaSamw zil_itx_create(uint64_t txtype, size_t lrsize) 1112fa9e4066Sahrens { 1113fa9e4066Sahrens itx_t *itx; 1114fa9e4066Sahrens 1115b4d654b0Sperrin lrsize = P2ROUNDUP_TYPED(lrsize, sizeof (uint64_t), size_t); 1116fa9e4066Sahrens 1117fa9e4066Sahrens itx = kmem_alloc(offsetof(itx_t, itx_lr) + lrsize, KM_SLEEP); 1118fa9e4066Sahrens itx->itx_lr.lrc_txtype = txtype; 1119fa9e4066Sahrens itx->itx_lr.lrc_reclen = lrsize; 1120abf76b6eSperrin itx->itx_sod = lrsize; /* if write & WR_NEED_COPY will be increased */ 1121fa9e4066Sahrens itx->itx_lr.lrc_seq = 0; /* defensive */ 11225002558fSNeil Perrin itx->itx_sync = B_TRUE; /* default is synchronous */ 1123fa9e4066Sahrens 1124fa9e4066Sahrens return (itx); 1125fa9e4066Sahrens } 1126fa9e4066Sahrens 1127b24ab676SJeff Bonwick void 1128b24ab676SJeff Bonwick zil_itx_destroy(itx_t *itx) 1129b24ab676SJeff Bonwick { 1130b24ab676SJeff Bonwick kmem_free(itx, offsetof(itx_t, itx_lr) + itx->itx_lr.lrc_reclen); 1131b24ab676SJeff Bonwick } 1132b24ab676SJeff Bonwick 11335002558fSNeil Perrin /* 11345002558fSNeil Perrin * Free up the sync and async itxs. The itxs_t has already been detached 11355002558fSNeil Perrin * so no locks are needed. 11365002558fSNeil Perrin */ 11375002558fSNeil Perrin static void 11385002558fSNeil Perrin zil_itxg_clean(itxs_t *itxs) 1139fa9e4066Sahrens { 11405002558fSNeil Perrin itx_t *itx; 11415002558fSNeil Perrin list_t *list; 11425002558fSNeil Perrin avl_tree_t *t; 11435002558fSNeil Perrin void *cookie; 11445002558fSNeil Perrin itx_async_node_t *ian; 11455002558fSNeil Perrin 11465002558fSNeil Perrin list = &itxs->i_sync_list; 11475002558fSNeil Perrin while ((itx = list_head(list)) != NULL) { 11485002558fSNeil Perrin list_remove(list, itx); 11495002558fSNeil Perrin kmem_free(itx, offsetof(itx_t, itx_lr) + 11505002558fSNeil Perrin itx->itx_lr.lrc_reclen); 11515002558fSNeil Perrin } 1152fa9e4066Sahrens 11535002558fSNeil Perrin cookie = NULL; 11545002558fSNeil Perrin t = &itxs->i_async_tree; 11555002558fSNeil Perrin while ((ian = avl_destroy_nodes(t, &cookie)) != NULL) { 11565002558fSNeil Perrin list = &ian->ia_list; 11575002558fSNeil Perrin while ((itx = list_head(list)) != NULL) { 11585002558fSNeil Perrin list_remove(list, itx); 11595002558fSNeil Perrin kmem_free(itx, offsetof(itx_t, itx_lr) + 11605002558fSNeil Perrin itx->itx_lr.lrc_reclen); 11615002558fSNeil Perrin } 11625002558fSNeil Perrin list_destroy(list); 11635002558fSNeil Perrin kmem_free(ian, sizeof (itx_async_node_t)); 11645002558fSNeil Perrin } 11655002558fSNeil Perrin avl_destroy(t); 1166fa9e4066Sahrens 11675002558fSNeil Perrin kmem_free(itxs, sizeof (itxs_t)); 11685002558fSNeil Perrin } 11695002558fSNeil Perrin 11705002558fSNeil Perrin static int 11715002558fSNeil Perrin zil_aitx_compare(const void *x1, const void *x2) 11725002558fSNeil Perrin { 11735002558fSNeil Perrin const uint64_t o1 = ((itx_async_node_t *)x1)->ia_foid; 11745002558fSNeil Perrin const uint64_t o2 = ((itx_async_node_t *)x2)->ia_foid; 1175fa9e4066Sahrens 11765002558fSNeil Perrin if (o1 < o2) 11775002558fSNeil Perrin return (-1); 11785002558fSNeil Perrin if (o1 > o2) 11795002558fSNeil Perrin return (1); 11805002558fSNeil Perrin 11815002558fSNeil Perrin return (0); 1182fa9e4066Sahrens } 1183fa9e4066Sahrens 1184fa9e4066Sahrens /* 11855002558fSNeil Perrin * Remove all async itx with the given oid. 1186fa9e4066Sahrens */ 118791de656bSNeil Perrin static void 11885002558fSNeil Perrin zil_remove_async(zilog_t *zilog, uint64_t oid) 1189fa9e4066Sahrens { 11905002558fSNeil Perrin uint64_t otxg, txg; 11915002558fSNeil Perrin itx_async_node_t *ian; 11925002558fSNeil Perrin avl_tree_t *t; 11935002558fSNeil Perrin avl_index_t where; 1194a584ef65Sjohansen list_t clean_list; 1195fa9e4066Sahrens itx_t *itx; 1196fa9e4066Sahrens 11975002558fSNeil Perrin ASSERT(oid != 0); 1198a584ef65Sjohansen list_create(&clean_list, sizeof (itx_t), offsetof(itx_t, itx_node)); 1199a584ef65Sjohansen 12005002558fSNeil Perrin if (spa_freeze_txg(zilog->zl_spa) != UINT64_MAX) /* ziltest support */ 12015002558fSNeil Perrin otxg = ZILTEST_TXG; 12025002558fSNeil Perrin else 12035002558fSNeil Perrin otxg = spa_last_synced_txg(zilog->zl_spa) + 1; 1204a584ef65Sjohansen 12055002558fSNeil Perrin for (txg = otxg; txg < (otxg + TXG_CONCURRENT_STATES); txg++) { 12065002558fSNeil Perrin itxg_t *itxg = &zilog->zl_itxg[txg & TXG_MASK]; 12075002558fSNeil Perrin 12085002558fSNeil Perrin mutex_enter(&itxg->itxg_lock); 12095002558fSNeil Perrin if (itxg->itxg_txg != txg) { 12105002558fSNeil Perrin mutex_exit(&itxg->itxg_lock); 12115002558fSNeil Perrin continue; 12125002558fSNeil Perrin } 1213a584ef65Sjohansen 12145002558fSNeil Perrin /* 12155002558fSNeil Perrin * Locate the object node and append its list. 12165002558fSNeil Perrin */ 12175002558fSNeil Perrin t = &itxg->itxg_itxs->i_async_tree; 12185002558fSNeil Perrin ian = avl_find(t, &oid, &where); 12195002558fSNeil Perrin if (ian != NULL) 12205002558fSNeil Perrin list_move_tail(&clean_list, &ian->ia_list); 12215002558fSNeil Perrin mutex_exit(&itxg->itxg_lock); 12225002558fSNeil Perrin } 1223a584ef65Sjohansen while ((itx = list_head(&clean_list)) != NULL) { 1224a584ef65Sjohansen list_remove(&clean_list, itx); 12255002558fSNeil Perrin kmem_free(itx, offsetof(itx_t, itx_lr) + 12265002558fSNeil Perrin itx->itx_lr.lrc_reclen); 1227a584ef65Sjohansen } 1228a584ef65Sjohansen list_destroy(&clean_list); 1229fa9e4066Sahrens } 1230fa9e4066Sahrens 12315002558fSNeil Perrin void 12325002558fSNeil Perrin zil_itx_assign(zilog_t *zilog, itx_t *itx, dmu_tx_t *tx) 12335002558fSNeil Perrin { 12345002558fSNeil Perrin uint64_t txg; 12355002558fSNeil Perrin itxg_t *itxg; 12365002558fSNeil Perrin itxs_t *itxs, *clean = NULL; 12375002558fSNeil Perrin 12385002558fSNeil Perrin /* 123991de656bSNeil Perrin * Object ids can be re-instantiated in the next txg so 12405002558fSNeil Perrin * remove any async transactions to avoid future leaks. 12415002558fSNeil Perrin * This can happen if a fsync occurs on the re-instantiated 12425002558fSNeil Perrin * object for a WR_INDIRECT or WR_NEED_COPY write, which gets 12435002558fSNeil Perrin * the new file data and flushes a write record for the old object. 12445002558fSNeil Perrin */ 12455002558fSNeil Perrin if ((itx->itx_lr.lrc_txtype & ~TX_CI) == TX_REMOVE) 124651bd2f97SNeil Perrin zil_remove_async(zilog, itx->itx_oid); 12475002558fSNeil Perrin 124891de656bSNeil Perrin /* 124991de656bSNeil Perrin * Ensure the data of a renamed file is committed before the rename. 125091de656bSNeil Perrin */ 125191de656bSNeil Perrin if ((itx->itx_lr.lrc_txtype & ~TX_CI) == TX_RENAME) 125291de656bSNeil Perrin zil_async_to_sync(zilog, itx->itx_oid); 125391de656bSNeil Perrin 1254ce636f8bSMatthew Ahrens if (spa_freeze_txg(zilog->zl_spa) != UINT64_MAX) 12555002558fSNeil Perrin txg = ZILTEST_TXG; 12565002558fSNeil Perrin else 12575002558fSNeil Perrin txg = dmu_tx_get_txg(tx); 12585002558fSNeil Perrin 12595002558fSNeil Perrin itxg = &zilog->zl_itxg[txg & TXG_MASK]; 12605002558fSNeil Perrin mutex_enter(&itxg->itxg_lock); 12615002558fSNeil Perrin itxs = itxg->itxg_itxs; 12625002558fSNeil Perrin if (itxg->itxg_txg != txg) { 12635002558fSNeil Perrin if (itxs != NULL) { 12645002558fSNeil Perrin /* 12655002558fSNeil Perrin * The zil_clean callback hasn't got around to cleaning 12665002558fSNeil Perrin * this itxg. Save the itxs for release below. 12675002558fSNeil Perrin * This should be rare. 12685002558fSNeil Perrin */ 12695002558fSNeil Perrin atomic_add_64(&zilog->zl_itx_list_sz, -itxg->itxg_sod); 12705002558fSNeil Perrin itxg->itxg_sod = 0; 12715002558fSNeil Perrin clean = itxg->itxg_itxs; 12725002558fSNeil Perrin } 12735002558fSNeil Perrin ASSERT(itxg->itxg_sod == 0); 12745002558fSNeil Perrin itxg->itxg_txg = txg; 12755002558fSNeil Perrin itxs = itxg->itxg_itxs = kmem_zalloc(sizeof (itxs_t), KM_SLEEP); 12765002558fSNeil Perrin 12775002558fSNeil Perrin list_create(&itxs->i_sync_list, sizeof (itx_t), 12785002558fSNeil Perrin offsetof(itx_t, itx_node)); 12795002558fSNeil Perrin avl_create(&itxs->i_async_tree, zil_aitx_compare, 12805002558fSNeil Perrin sizeof (itx_async_node_t), 12815002558fSNeil Perrin offsetof(itx_async_node_t, ia_node)); 12825002558fSNeil Perrin } 12835002558fSNeil Perrin if (itx->itx_sync) { 12845002558fSNeil Perrin list_insert_tail(&itxs->i_sync_list, itx); 12855002558fSNeil Perrin atomic_add_64(&zilog->zl_itx_list_sz, itx->itx_sod); 12865002558fSNeil Perrin itxg->itxg_sod += itx->itx_sod; 12875002558fSNeil Perrin } else { 12885002558fSNeil Perrin avl_tree_t *t = &itxs->i_async_tree; 12895002558fSNeil Perrin uint64_t foid = ((lr_ooo_t *)&itx->itx_lr)->lr_foid; 12905002558fSNeil Perrin itx_async_node_t *ian; 12915002558fSNeil Perrin avl_index_t where; 12925002558fSNeil Perrin 12935002558fSNeil Perrin ian = avl_find(t, &foid, &where); 12945002558fSNeil Perrin if (ian == NULL) { 12955002558fSNeil Perrin ian = kmem_alloc(sizeof (itx_async_node_t), KM_SLEEP); 12965002558fSNeil Perrin list_create(&ian->ia_list, sizeof (itx_t), 12975002558fSNeil Perrin offsetof(itx_t, itx_node)); 12985002558fSNeil Perrin ian->ia_foid = foid; 12995002558fSNeil Perrin avl_insert(t, ian, where); 13005002558fSNeil Perrin } 13015002558fSNeil Perrin list_insert_tail(&ian->ia_list, itx); 13025002558fSNeil Perrin } 13035002558fSNeil Perrin 13045002558fSNeil Perrin itx->itx_lr.lrc_txg = dmu_tx_get_txg(tx); 1305ce636f8bSMatthew Ahrens zilog_dirty(zilog, txg); 13065002558fSNeil Perrin mutex_exit(&itxg->itxg_lock); 13075002558fSNeil Perrin 13085002558fSNeil Perrin /* Release the old itxs now we've dropped the lock */ 13095002558fSNeil Perrin if (clean != NULL) 13105002558fSNeil Perrin zil_itxg_clean(clean); 13115002558fSNeil Perrin } 13125002558fSNeil Perrin 1313b19a79ecSperrin /* 131467bd71c6Sperrin * If there are any in-memory intent log transactions which have now been 1315ce636f8bSMatthew Ahrens * synced then start up a taskq to free them. We should only do this after we 1316ce636f8bSMatthew Ahrens * have written out the uberblocks (i.e. txg has been comitted) so that 1317ce636f8bSMatthew Ahrens * don't inadvertently clean out in-memory log records that would be required 1318ce636f8bSMatthew Ahrens * by zil_commit(). 1319b19a79ecSperrin */ 1320fa9e4066Sahrens void 13215002558fSNeil Perrin zil_clean(zilog_t *zilog, uint64_t synced_txg) 1322fa9e4066Sahrens { 13235002558fSNeil Perrin itxg_t *itxg = &zilog->zl_itxg[synced_txg & TXG_MASK]; 13245002558fSNeil Perrin itxs_t *clean_me; 132567bd71c6Sperrin 13265002558fSNeil Perrin mutex_enter(&itxg->itxg_lock); 13275002558fSNeil Perrin if (itxg->itxg_itxs == NULL || itxg->itxg_txg == ZILTEST_TXG) { 13285002558fSNeil Perrin mutex_exit(&itxg->itxg_lock); 13295002558fSNeil Perrin return; 13305002558fSNeil Perrin } 13315002558fSNeil Perrin ASSERT3U(itxg->itxg_txg, <=, synced_txg); 13325002558fSNeil Perrin ASSERT(itxg->itxg_txg != 0); 13335002558fSNeil Perrin ASSERT(zilog->zl_clean_taskq != NULL); 13345002558fSNeil Perrin atomic_add_64(&zilog->zl_itx_list_sz, -itxg->itxg_sod); 13355002558fSNeil Perrin itxg->itxg_sod = 0; 13365002558fSNeil Perrin clean_me = itxg->itxg_itxs; 13375002558fSNeil Perrin itxg->itxg_itxs = NULL; 13385002558fSNeil Perrin itxg->itxg_txg = 0; 13395002558fSNeil Perrin mutex_exit(&itxg->itxg_lock); 13405002558fSNeil Perrin /* 13415002558fSNeil Perrin * Preferably start a task queue to free up the old itxs but 13425002558fSNeil Perrin * if taskq_dispatch can't allocate resources to do that then 13435002558fSNeil Perrin * free it in-line. This should be rare. Note, using TQ_SLEEP 13445002558fSNeil Perrin * created a bad performance problem. 13455002558fSNeil Perrin */ 13465002558fSNeil Perrin if (taskq_dispatch(zilog->zl_clean_taskq, 13475002558fSNeil Perrin (void (*)(void *))zil_itxg_clean, clean_me, TQ_NOSLEEP) == NULL) 13485002558fSNeil Perrin zil_itxg_clean(clean_me); 13495002558fSNeil Perrin } 13505002558fSNeil Perrin 13515002558fSNeil Perrin /* 13525002558fSNeil Perrin * Get the list of itxs to commit into zl_itx_commit_list. 13535002558fSNeil Perrin */ 135491de656bSNeil Perrin static void 13555002558fSNeil Perrin zil_get_commit_list(zilog_t *zilog) 13565002558fSNeil Perrin { 13575002558fSNeil Perrin uint64_t otxg, txg; 13585002558fSNeil Perrin list_t *commit_list = &zilog->zl_itx_commit_list; 13595002558fSNeil Perrin uint64_t push_sod = 0; 13605002558fSNeil Perrin 13615002558fSNeil Perrin if (spa_freeze_txg(zilog->zl_spa) != UINT64_MAX) /* ziltest support */ 13625002558fSNeil Perrin otxg = ZILTEST_TXG; 13635002558fSNeil Perrin else 13645002558fSNeil Perrin otxg = spa_last_synced_txg(zilog->zl_spa) + 1; 13655002558fSNeil Perrin 13665002558fSNeil Perrin for (txg = otxg; txg < (otxg + TXG_CONCURRENT_STATES); txg++) { 13675002558fSNeil Perrin itxg_t *itxg = &zilog->zl_itxg[txg & TXG_MASK]; 13685002558fSNeil Perrin 13695002558fSNeil Perrin mutex_enter(&itxg->itxg_lock); 13705002558fSNeil Perrin if (itxg->itxg_txg != txg) { 13715002558fSNeil Perrin mutex_exit(&itxg->itxg_lock); 13725002558fSNeil Perrin continue; 13735002558fSNeil Perrin } 13745002558fSNeil Perrin 13755002558fSNeil Perrin list_move_tail(commit_list, &itxg->itxg_itxs->i_sync_list); 13765002558fSNeil Perrin push_sod += itxg->itxg_sod; 13775002558fSNeil Perrin itxg->itxg_sod = 0; 13785002558fSNeil Perrin 13795002558fSNeil Perrin mutex_exit(&itxg->itxg_lock); 13805002558fSNeil Perrin } 13815002558fSNeil Perrin atomic_add_64(&zilog->zl_itx_list_sz, -push_sod); 13825002558fSNeil Perrin } 13835002558fSNeil Perrin 13845002558fSNeil Perrin /* 13855002558fSNeil Perrin * Move the async itxs for a specified object to commit into sync lists. 13865002558fSNeil Perrin */ 138791de656bSNeil Perrin static void 13885002558fSNeil Perrin zil_async_to_sync(zilog_t *zilog, uint64_t foid) 13895002558fSNeil Perrin { 13905002558fSNeil Perrin uint64_t otxg, txg; 13915002558fSNeil Perrin itx_async_node_t *ian; 13925002558fSNeil Perrin avl_tree_t *t; 13935002558fSNeil Perrin avl_index_t where; 13945002558fSNeil Perrin 13955002558fSNeil Perrin if (spa_freeze_txg(zilog->zl_spa) != UINT64_MAX) /* ziltest support */ 13965002558fSNeil Perrin otxg = ZILTEST_TXG; 13975002558fSNeil Perrin else 13985002558fSNeil Perrin otxg = spa_last_synced_txg(zilog->zl_spa) + 1; 13995002558fSNeil Perrin 14005002558fSNeil Perrin for (txg = otxg; txg < (otxg + TXG_CONCURRENT_STATES); txg++) { 14015002558fSNeil Perrin itxg_t *itxg = &zilog->zl_itxg[txg & TXG_MASK]; 14025002558fSNeil Perrin 14035002558fSNeil Perrin mutex_enter(&itxg->itxg_lock); 14045002558fSNeil Perrin if (itxg->itxg_txg != txg) { 14055002558fSNeil Perrin mutex_exit(&itxg->itxg_lock); 14065002558fSNeil Perrin continue; 14075002558fSNeil Perrin } 14085002558fSNeil Perrin 14095002558fSNeil Perrin /* 14105002558fSNeil Perrin * If a foid is specified then find that node and append its 14115002558fSNeil Perrin * list. Otherwise walk the tree appending all the lists 14125002558fSNeil Perrin * to the sync list. We add to the end rather than the 14135002558fSNeil Perrin * beginning to ensure the create has happened. 14145002558fSNeil Perrin */ 14155002558fSNeil Perrin t = &itxg->itxg_itxs->i_async_tree; 14165002558fSNeil Perrin if (foid != 0) { 14175002558fSNeil Perrin ian = avl_find(t, &foid, &where); 14185002558fSNeil Perrin if (ian != NULL) { 14195002558fSNeil Perrin list_move_tail(&itxg->itxg_itxs->i_sync_list, 14205002558fSNeil Perrin &ian->ia_list); 14215002558fSNeil Perrin } 14225002558fSNeil Perrin } else { 14235002558fSNeil Perrin void *cookie = NULL; 14245002558fSNeil Perrin 14255002558fSNeil Perrin while ((ian = avl_destroy_nodes(t, &cookie)) != NULL) { 14265002558fSNeil Perrin list_move_tail(&itxg->itxg_itxs->i_sync_list, 14275002558fSNeil Perrin &ian->ia_list); 14285002558fSNeil Perrin list_destroy(&ian->ia_list); 14295002558fSNeil Perrin kmem_free(ian, sizeof (itx_async_node_t)); 14305002558fSNeil Perrin } 14315002558fSNeil Perrin } 14325002558fSNeil Perrin mutex_exit(&itxg->itxg_lock); 143367bd71c6Sperrin } 1434fa9e4066Sahrens } 1435fa9e4066Sahrens 1436e14bb325SJeff Bonwick static void 14375002558fSNeil Perrin zil_commit_writer(zilog_t *zilog) 1438fa9e4066Sahrens { 1439fa9e4066Sahrens uint64_t txg; 14405002558fSNeil Perrin itx_t *itx; 1441fa9e4066Sahrens lwb_t *lwb; 14425002558fSNeil Perrin spa_t *spa = zilog->zl_spa; 1443b24ab676SJeff Bonwick int error = 0; 1444fa9e4066Sahrens 1445e14bb325SJeff Bonwick ASSERT(zilog->zl_root_zio == NULL); 14465002558fSNeil Perrin 14475002558fSNeil Perrin mutex_exit(&zilog->zl_lock); 14485002558fSNeil Perrin 14495002558fSNeil Perrin zil_get_commit_list(zilog); 14505002558fSNeil Perrin 14515002558fSNeil Perrin /* 14525002558fSNeil Perrin * Return if there's nothing to commit before we dirty the fs by 14535002558fSNeil Perrin * calling zil_create(). 14545002558fSNeil Perrin */ 14555002558fSNeil Perrin if (list_head(&zilog->zl_itx_commit_list) == NULL) { 14565002558fSNeil Perrin mutex_enter(&zilog->zl_lock); 14575002558fSNeil Perrin return; 14585002558fSNeil Perrin } 1459fa9e4066Sahrens 1460fa9e4066Sahrens if (zilog->zl_suspend) { 1461fa9e4066Sahrens lwb = NULL; 1462fa9e4066Sahrens } else { 1463fa9e4066Sahrens lwb = list_tail(&zilog->zl_lwb_list); 14645002558fSNeil Perrin if (lwb == NULL) 14656e1f5caaSNeil Perrin lwb = zil_create(zilog); 1466fa9e4066Sahrens } 1467fa9e4066Sahrens 1468b19a79ecSperrin DTRACE_PROBE1(zil__cw1, zilog_t *, zilog); 14695002558fSNeil Perrin while (itx = list_head(&zilog->zl_itx_commit_list)) { 1470fa9e4066Sahrens txg = itx->itx_lr.lrc_txg; 1471fa9e4066Sahrens ASSERT(txg); 1472fa9e4066Sahrens 14735002558fSNeil Perrin if (txg > spa_last_synced_txg(spa) || txg > spa_freeze_txg(spa)) 1474fa9e4066Sahrens lwb = zil_lwb_commit(zilog, itx, lwb); 14755002558fSNeil Perrin list_remove(&zilog->zl_itx_commit_list, itx); 14765002558fSNeil Perrin kmem_free(itx, offsetof(itx_t, itx_lr) 14775002558fSNeil Perrin + itx->itx_lr.lrc_reclen); 1478fa9e4066Sahrens } 1479b19a79ecSperrin DTRACE_PROBE1(zil__cw2, zilog_t *, zilog); 1480fa9e4066Sahrens 1481fa9e4066Sahrens /* write the last block out */ 148267bd71c6Sperrin if (lwb != NULL && lwb->lwb_zio != NULL) 1483fa9e4066Sahrens lwb = zil_lwb_write_start(zilog, lwb); 1484fa9e4066Sahrens 148522ac5be4Sperrin zilog->zl_cur_used = 0; 1486fa9e4066Sahrens 1487fa9e4066Sahrens /* 1488b19a79ecSperrin * Wait if necessary for the log blocks to be on stable storage. 1489fa9e4066Sahrens */ 1490b19a79ecSperrin if (zilog->zl_root_zio) { 1491b24ab676SJeff Bonwick error = zio_wait(zilog->zl_root_zio); 1492e14bb325SJeff Bonwick zilog->zl_root_zio = NULL; 149317f17c2dSbonwick zil_flush_vdevs(zilog); 1494fa9e4066Sahrens } 149522ac5be4Sperrin 1496b24ab676SJeff Bonwick if (error || lwb == NULL) 1497fa9e4066Sahrens txg_wait_synced(zilog->zl_dmu_pool, 0); 149867bd71c6Sperrin 149967bd71c6Sperrin mutex_enter(&zilog->zl_lock); 1500b24ab676SJeff Bonwick 1501b24ab676SJeff Bonwick /* 1502b24ab676SJeff Bonwick * Remember the highest committed log sequence number for ztest. 1503b24ab676SJeff Bonwick * We only update this value when all the log writes succeeded, 1504b24ab676SJeff Bonwick * because ztest wants to ASSERT that it got the whole log chain. 1505b24ab676SJeff Bonwick */ 1506b24ab676SJeff Bonwick if (error == 0 && lwb != NULL) 1507b24ab676SJeff Bonwick zilog->zl_commit_lr_seq = zilog->zl_lr_seq; 1508b19a79ecSperrin } 1509b19a79ecSperrin 1510b19a79ecSperrin /* 15115002558fSNeil Perrin * Commit zfs transactions to stable storage. 1512b19a79ecSperrin * If foid is 0 push out all transactions, otherwise push only those 15135002558fSNeil Perrin * for that object or might reference that object. 15145002558fSNeil Perrin * 15155002558fSNeil Perrin * itxs are committed in batches. In a heavily stressed zil there will be 15165002558fSNeil Perrin * a commit writer thread who is writing out a bunch of itxs to the log 15175002558fSNeil Perrin * for a set of committing threads (cthreads) in the same batch as the writer. 15185002558fSNeil Perrin * Those cthreads are all waiting on the same cv for that batch. 15195002558fSNeil Perrin * 15205002558fSNeil Perrin * There will also be a different and growing batch of threads that are 15215002558fSNeil Perrin * waiting to commit (qthreads). When the committing batch completes 15225002558fSNeil Perrin * a transition occurs such that the cthreads exit and the qthreads become 15235002558fSNeil Perrin * cthreads. One of the new cthreads becomes the writer thread for the 15245002558fSNeil Perrin * batch. Any new threads arriving become new qthreads. 15255002558fSNeil Perrin * 15265002558fSNeil Perrin * Only 2 condition variables are needed and there's no transition 15275002558fSNeil Perrin * between the two cvs needed. They just flip-flop between qthreads 15285002558fSNeil Perrin * and cthreads. 15295002558fSNeil Perrin * 15305002558fSNeil Perrin * Using this scheme we can efficiently wakeup up only those threads 15315002558fSNeil Perrin * that have been committed. 1532b19a79ecSperrin */ 1533b19a79ecSperrin void 15345002558fSNeil Perrin zil_commit(zilog_t *zilog, uint64_t foid) 1535b19a79ecSperrin { 15365002558fSNeil Perrin uint64_t mybatch; 1537b19a79ecSperrin 15385002558fSNeil Perrin if (zilog->zl_sync == ZFS_SYNC_DISABLED) 15395002558fSNeil Perrin return; 1540b19a79ecSperrin 15415002558fSNeil Perrin /* move the async itxs for the foid to the sync queues */ 15425002558fSNeil Perrin zil_async_to_sync(zilog, foid); 1543b19a79ecSperrin 15445002558fSNeil Perrin mutex_enter(&zilog->zl_lock); 15455002558fSNeil Perrin mybatch = zilog->zl_next_batch; 154667bd71c6Sperrin while (zilog->zl_writer) { 15475002558fSNeil Perrin cv_wait(&zilog->zl_cv_batch[mybatch & 1], &zilog->zl_lock); 15485002558fSNeil Perrin if (mybatch <= zilog->zl_com_batch) { 154967bd71c6Sperrin mutex_exit(&zilog->zl_lock); 155067bd71c6Sperrin return; 155167bd71c6Sperrin } 155267bd71c6Sperrin } 1553b24ab676SJeff Bonwick 15545002558fSNeil Perrin zilog->zl_next_batch++; 15555002558fSNeil Perrin zilog->zl_writer = B_TRUE; 15565002558fSNeil Perrin zil_commit_writer(zilog); 15575002558fSNeil Perrin zilog->zl_com_batch = mybatch; 15585002558fSNeil Perrin zilog->zl_writer = B_FALSE; 15595002558fSNeil Perrin mutex_exit(&zilog->zl_lock); 1560b24ab676SJeff Bonwick 15615002558fSNeil Perrin /* wake up one thread to become the next writer */ 15625002558fSNeil Perrin cv_signal(&zilog->zl_cv_batch[(mybatch+1) & 1]); 1563b24ab676SJeff Bonwick 15645002558fSNeil Perrin /* wake up all threads waiting for this batch to be committed */ 15655002558fSNeil Perrin cv_broadcast(&zilog->zl_cv_batch[mybatch & 1]); 1566b24ab676SJeff Bonwick } 1567b24ab676SJeff Bonwick 1568fa9e4066Sahrens /* 1569fa9e4066Sahrens * Called in syncing context to free committed log blocks and update log header. 1570fa9e4066Sahrens */ 1571fa9e4066Sahrens void 1572fa9e4066Sahrens zil_sync(zilog_t *zilog, dmu_tx_t *tx) 1573fa9e4066Sahrens { 1574d80c45e0Sbonwick zil_header_t *zh = zil_header_in_syncing_context(zilog); 1575fa9e4066Sahrens uint64_t txg = dmu_tx_get_txg(tx); 1576fa9e4066Sahrens spa_t *spa = zilog->zl_spa; 1577b24ab676SJeff Bonwick uint64_t *replayed_seq = &zilog->zl_replayed_seq[txg & TXG_MASK]; 1578fa9e4066Sahrens lwb_t *lwb; 1579fa9e4066Sahrens 158014843421SMatthew Ahrens /* 158114843421SMatthew Ahrens * We don't zero out zl_destroy_txg, so make sure we don't try 158214843421SMatthew Ahrens * to destroy it twice. 158314843421SMatthew Ahrens */ 158414843421SMatthew Ahrens if (spa_sync_pass(spa) != 1) 158514843421SMatthew Ahrens return; 158614843421SMatthew Ahrens 1587d80c45e0Sbonwick mutex_enter(&zilog->zl_lock); 1588d80c45e0Sbonwick 1589fa9e4066Sahrens ASSERT(zilog->zl_stop_sync == 0); 1590fa9e4066Sahrens 1591b24ab676SJeff Bonwick if (*replayed_seq != 0) { 1592b24ab676SJeff Bonwick ASSERT(zh->zh_replay_seq < *replayed_seq); 1593b24ab676SJeff Bonwick zh->zh_replay_seq = *replayed_seq; 1594b24ab676SJeff Bonwick *replayed_seq = 0; 1595b24ab676SJeff Bonwick } 1596fa9e4066Sahrens 1597fa9e4066Sahrens if (zilog->zl_destroy_txg == txg) { 1598d80c45e0Sbonwick blkptr_t blk = zh->zh_log; 1599d80c45e0Sbonwick 1600d80c45e0Sbonwick ASSERT(list_head(&zilog->zl_lwb_list) == NULL); 1601d80c45e0Sbonwick 1602d80c45e0Sbonwick bzero(zh, sizeof (zil_header_t)); 16031209a471SNeil Perrin bzero(zilog->zl_replayed_seq, sizeof (zilog->zl_replayed_seq)); 1604d80c45e0Sbonwick 1605d80c45e0Sbonwick if (zilog->zl_keep_first) { 1606d80c45e0Sbonwick /* 1607d80c45e0Sbonwick * If this block was part of log chain that couldn't 1608d80c45e0Sbonwick * be claimed because a device was missing during 1609d80c45e0Sbonwick * zil_claim(), but that device later returns, 1610d80c45e0Sbonwick * then this block could erroneously appear valid. 1611d80c45e0Sbonwick * To guard against this, assign a new GUID to the new 1612d80c45e0Sbonwick * log chain so it doesn't matter what blk points to. 1613d80c45e0Sbonwick */ 1614d80c45e0Sbonwick zil_init_log_chain(zilog, &blk); 1615d80c45e0Sbonwick zh->zh_log = blk; 1616d80c45e0Sbonwick } 1617fa9e4066Sahrens } 1618fa9e4066Sahrens 1619e6ca193dSGeorge Wilson while ((lwb = list_head(&zilog->zl_lwb_list)) != NULL) { 1620b19a79ecSperrin zh->zh_log = lwb->lwb_blk; 1621fa9e4066Sahrens if (lwb->lwb_buf != NULL || lwb->lwb_max_txg > txg) 1622fa9e4066Sahrens break; 1623fa9e4066Sahrens list_remove(&zilog->zl_lwb_list, lwb); 1624b24ab676SJeff Bonwick zio_free_zil(spa, txg, &lwb->lwb_blk); 1625fa9e4066Sahrens kmem_cache_free(zil_lwb_cache, lwb); 1626d63d470bSgw 1627d63d470bSgw /* 1628d63d470bSgw * If we don't have anything left in the lwb list then 1629d63d470bSgw * we've had an allocation failure and we need to zero 1630d63d470bSgw * out the zil_header blkptr so that we don't end 1631d63d470bSgw * up freeing the same block twice. 1632d63d470bSgw */ 1633d63d470bSgw if (list_head(&zilog->zl_lwb_list) == NULL) 1634d63d470bSgw BP_ZERO(&zh->zh_log); 1635fa9e4066Sahrens } 1636fa9e4066Sahrens mutex_exit(&zilog->zl_lock); 1637fa9e4066Sahrens } 1638fa9e4066Sahrens 1639fa9e4066Sahrens void 1640fa9e4066Sahrens zil_init(void) 1641fa9e4066Sahrens { 1642fa9e4066Sahrens zil_lwb_cache = kmem_cache_create("zil_lwb_cache", 16435ad82045Snd sizeof (struct lwb), 0, NULL, NULL, NULL, NULL, NULL, 0); 1644fa9e4066Sahrens } 1645fa9e4066Sahrens 1646fa9e4066Sahrens void 1647fa9e4066Sahrens zil_fini(void) 1648fa9e4066Sahrens { 1649fa9e4066Sahrens kmem_cache_destroy(zil_lwb_cache); 1650fa9e4066Sahrens } 1651fa9e4066Sahrens 165255da60b9SMark J Musante void 165355da60b9SMark J Musante zil_set_sync(zilog_t *zilog, uint64_t sync) 165455da60b9SMark J Musante { 165555da60b9SMark J Musante zilog->zl_sync = sync; 165655da60b9SMark J Musante } 165755da60b9SMark J Musante 1658e09fa4daSNeil Perrin void 1659e09fa4daSNeil Perrin zil_set_logbias(zilog_t *zilog, uint64_t logbias) 1660e09fa4daSNeil Perrin { 1661e09fa4daSNeil Perrin zilog->zl_logbias = logbias; 1662e09fa4daSNeil Perrin } 1663e09fa4daSNeil Perrin 1664fa9e4066Sahrens zilog_t * 1665fa9e4066Sahrens zil_alloc(objset_t *os, zil_header_t *zh_phys) 1666fa9e4066Sahrens { 1667fa9e4066Sahrens zilog_t *zilog; 1668fa9e4066Sahrens 1669fa9e4066Sahrens zilog = kmem_zalloc(sizeof (zilog_t), KM_SLEEP); 1670fa9e4066Sahrens 1671fa9e4066Sahrens zilog->zl_header = zh_phys; 1672fa9e4066Sahrens zilog->zl_os = os; 1673fa9e4066Sahrens zilog->zl_spa = dmu_objset_spa(os); 1674fa9e4066Sahrens zilog->zl_dmu_pool = dmu_objset_pool(os); 1675d80c45e0Sbonwick zilog->zl_destroy_txg = TXG_INITIAL - 1; 1676e09fa4daSNeil Perrin zilog->zl_logbias = dmu_objset_logbias(os); 167755da60b9SMark J Musante zilog->zl_sync = dmu_objset_syncprop(os); 16785002558fSNeil Perrin zilog->zl_next_batch = 1; 1679fa9e4066Sahrens 16805ad82045Snd mutex_init(&zilog->zl_lock, NULL, MUTEX_DEFAULT, NULL); 16815ad82045Snd 16825002558fSNeil Perrin for (int i = 0; i < TXG_SIZE; i++) { 16835002558fSNeil Perrin mutex_init(&zilog->zl_itxg[i].itxg_lock, NULL, 16845002558fSNeil Perrin MUTEX_DEFAULT, NULL); 16855002558fSNeil Perrin } 1686fa9e4066Sahrens 1687fa9e4066Sahrens list_create(&zilog->zl_lwb_list, sizeof (lwb_t), 1688fa9e4066Sahrens offsetof(lwb_t, lwb_node)); 1689fa9e4066Sahrens 16905002558fSNeil Perrin list_create(&zilog->zl_itx_commit_list, sizeof (itx_t), 16915002558fSNeil Perrin offsetof(itx_t, itx_node)); 16925002558fSNeil Perrin 169317f17c2dSbonwick mutex_init(&zilog->zl_vdev_lock, NULL, MUTEX_DEFAULT, NULL); 169417f17c2dSbonwick 169517f17c2dSbonwick avl_create(&zilog->zl_vdev_tree, zil_vdev_compare, 169617f17c2dSbonwick sizeof (zil_vdev_node_t), offsetof(zil_vdev_node_t, zv_node)); 1697fa9e4066Sahrens 1698b7b97454Sperrin cv_init(&zilog->zl_cv_writer, NULL, CV_DEFAULT, NULL); 1699b7b97454Sperrin cv_init(&zilog->zl_cv_suspend, NULL, CV_DEFAULT, NULL); 17005002558fSNeil Perrin cv_init(&zilog->zl_cv_batch[0], NULL, CV_DEFAULT, NULL); 17015002558fSNeil Perrin cv_init(&zilog->zl_cv_batch[1], NULL, CV_DEFAULT, NULL); 1702b7b97454Sperrin 1703fa9e4066Sahrens return (zilog); 1704fa9e4066Sahrens } 1705fa9e4066Sahrens 1706fa9e4066Sahrens void 1707fa9e4066Sahrens zil_free(zilog_t *zilog) 1708fa9e4066Sahrens { 1709fa9e4066Sahrens zilog->zl_stop_sync = 1; 1710fa9e4066Sahrens 1711*3b2aab18SMatthew Ahrens ASSERT0(zilog->zl_suspend); 1712*3b2aab18SMatthew Ahrens ASSERT0(zilog->zl_suspending); 1713*3b2aab18SMatthew Ahrens 1714c9ba2a43SEric Schrock ASSERT(list_is_empty(&zilog->zl_lwb_list)); 1715fa9e4066Sahrens list_destroy(&zilog->zl_lwb_list); 1716fa9e4066Sahrens 171717f17c2dSbonwick avl_destroy(&zilog->zl_vdev_tree); 171817f17c2dSbonwick mutex_destroy(&zilog->zl_vdev_lock); 1719fa9e4066Sahrens 17205002558fSNeil Perrin ASSERT(list_is_empty(&zilog->zl_itx_commit_list)); 17215002558fSNeil Perrin list_destroy(&zilog->zl_itx_commit_list); 17225002558fSNeil Perrin 17235002558fSNeil Perrin for (int i = 0; i < TXG_SIZE; i++) { 17245002558fSNeil Perrin /* 17255002558fSNeil Perrin * It's possible for an itx to be generated that doesn't dirty 17265002558fSNeil Perrin * a txg (e.g. ztest TX_TRUNCATE). So there's no zil_clean() 17275002558fSNeil Perrin * callback to remove the entry. We remove those here. 17285002558fSNeil Perrin * 17295002558fSNeil Perrin * Also free up the ziltest itxs. 17305002558fSNeil Perrin */ 17315002558fSNeil Perrin if (zilog->zl_itxg[i].itxg_itxs) 17325002558fSNeil Perrin zil_itxg_clean(zilog->zl_itxg[i].itxg_itxs); 17335002558fSNeil Perrin mutex_destroy(&zilog->zl_itxg[i].itxg_lock); 17345002558fSNeil Perrin } 17355002558fSNeil Perrin 17365ad82045Snd mutex_destroy(&zilog->zl_lock); 1737fa9e4066Sahrens 1738b7b97454Sperrin cv_destroy(&zilog->zl_cv_writer); 1739b7b97454Sperrin cv_destroy(&zilog->zl_cv_suspend); 17405002558fSNeil Perrin cv_destroy(&zilog->zl_cv_batch[0]); 17415002558fSNeil Perrin cv_destroy(&zilog->zl_cv_batch[1]); 1742b7b97454Sperrin 1743fa9e4066Sahrens kmem_free(zilog, sizeof (zilog_t)); 1744fa9e4066Sahrens } 1745fa9e4066Sahrens 1746fa9e4066Sahrens /* 1747fa9e4066Sahrens * Open an intent log. 1748fa9e4066Sahrens */ 1749fa9e4066Sahrens zilog_t * 1750fa9e4066Sahrens zil_open(objset_t *os, zil_get_data_t *get_data) 1751fa9e4066Sahrens { 1752fa9e4066Sahrens zilog_t *zilog = dmu_objset_zil(os); 1753fa9e4066Sahrens 1754c9ba2a43SEric Schrock ASSERT(zilog->zl_clean_taskq == NULL); 1755c9ba2a43SEric Schrock ASSERT(zilog->zl_get_data == NULL); 1756c9ba2a43SEric Schrock ASSERT(list_is_empty(&zilog->zl_lwb_list)); 1757c9ba2a43SEric Schrock 1758fa9e4066Sahrens zilog->zl_get_data = get_data; 1759fa9e4066Sahrens zilog->zl_clean_taskq = taskq_create("zil_clean", 1, minclsyspri, 1760fa9e4066Sahrens 2, 2, TASKQ_PREPOPULATE); 1761fa9e4066Sahrens 1762fa9e4066Sahrens return (zilog); 1763fa9e4066Sahrens } 1764fa9e4066Sahrens 1765fa9e4066Sahrens /* 1766fa9e4066Sahrens * Close an intent log. 1767fa9e4066Sahrens */ 1768fa9e4066Sahrens void 1769fa9e4066Sahrens zil_close(zilog_t *zilog) 1770fa9e4066Sahrens { 1771c9ba2a43SEric Schrock lwb_t *lwb; 17725002558fSNeil Perrin uint64_t txg = 0; 17735002558fSNeil Perrin 17745002558fSNeil Perrin zil_commit(zilog, 0); /* commit all itx */ 17755002558fSNeil Perrin 1776d80c45e0Sbonwick /* 17775002558fSNeil Perrin * The lwb_max_txg for the stubby lwb will reflect the last activity 17785002558fSNeil Perrin * for the zil. After a txg_wait_synced() on the txg we know all the 17795002558fSNeil Perrin * callbacks have occurred that may clean the zil. Only then can we 17805002558fSNeil Perrin * destroy the zl_clean_taskq. 1781d80c45e0Sbonwick */ 17825002558fSNeil Perrin mutex_enter(&zilog->zl_lock); 1783c9ba2a43SEric Schrock lwb = list_tail(&zilog->zl_lwb_list); 1784c9ba2a43SEric Schrock if (lwb != NULL) 1785c9ba2a43SEric Schrock txg = lwb->lwb_max_txg; 17865002558fSNeil Perrin mutex_exit(&zilog->zl_lock); 17875002558fSNeil Perrin if (txg) 1788d80c45e0Sbonwick txg_wait_synced(zilog->zl_dmu_pool, txg); 1789ce636f8bSMatthew Ahrens ASSERT(!zilog_is_dirty(zilog)); 1790d80c45e0Sbonwick 1791fa9e4066Sahrens taskq_destroy(zilog->zl_clean_taskq); 1792fa9e4066Sahrens zilog->zl_clean_taskq = NULL; 1793fa9e4066Sahrens zilog->zl_get_data = NULL; 1794c9ba2a43SEric Schrock 1795c9ba2a43SEric Schrock /* 1796c9ba2a43SEric Schrock * We should have only one LWB left on the list; remove it now. 1797c9ba2a43SEric Schrock */ 1798c9ba2a43SEric Schrock mutex_enter(&zilog->zl_lock); 1799c9ba2a43SEric Schrock lwb = list_head(&zilog->zl_lwb_list); 1800c9ba2a43SEric Schrock if (lwb != NULL) { 1801c9ba2a43SEric Schrock ASSERT(lwb == list_tail(&zilog->zl_lwb_list)); 1802c9ba2a43SEric Schrock list_remove(&zilog->zl_lwb_list, lwb); 1803c9ba2a43SEric Schrock zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 1804c9ba2a43SEric Schrock kmem_cache_free(zil_lwb_cache, lwb); 1805c9ba2a43SEric Schrock } 1806c9ba2a43SEric Schrock mutex_exit(&zilog->zl_lock); 1807fa9e4066Sahrens } 1808fa9e4066Sahrens 1809*3b2aab18SMatthew Ahrens static char *suspend_tag = "zil suspending"; 1810*3b2aab18SMatthew Ahrens 1811fa9e4066Sahrens /* 1812fa9e4066Sahrens * Suspend an intent log. While in suspended mode, we still honor 1813fa9e4066Sahrens * synchronous semantics, but we rely on txg_wait_synced() to do it. 1814*3b2aab18SMatthew Ahrens * On old version pools, we suspend the log briefly when taking a 1815*3b2aab18SMatthew Ahrens * snapshot so that it will have an empty intent log. 1816*3b2aab18SMatthew Ahrens * 1817*3b2aab18SMatthew Ahrens * Long holds are not really intended to be used the way we do here -- 1818*3b2aab18SMatthew Ahrens * held for such a short time. A concurrent caller of dsl_dataset_long_held() 1819*3b2aab18SMatthew Ahrens * could fail. Therefore we take pains to only put a long hold if it is 1820*3b2aab18SMatthew Ahrens * actually necessary. Fortunately, it will only be necessary if the 1821*3b2aab18SMatthew Ahrens * objset is currently mounted (or the ZVOL equivalent). In that case it 1822*3b2aab18SMatthew Ahrens * will already have a long hold, so we are not really making things any worse. 1823*3b2aab18SMatthew Ahrens * 1824*3b2aab18SMatthew Ahrens * Ideally, we would locate the existing long-holder (i.e. the zfsvfs_t or 1825*3b2aab18SMatthew Ahrens * zvol_state_t), and use their mechanism to prevent their hold from being 1826*3b2aab18SMatthew Ahrens * dropped (e.g. VFS_HOLD()). However, that would be even more pain for 1827*3b2aab18SMatthew Ahrens * very little gain. 1828*3b2aab18SMatthew Ahrens * 1829*3b2aab18SMatthew Ahrens * if cookiep == NULL, this does both the suspend & resume. 1830*3b2aab18SMatthew Ahrens * Otherwise, it returns with the dataset "long held", and the cookie 1831*3b2aab18SMatthew Ahrens * should be passed into zil_resume(). 1832fa9e4066Sahrens */ 1833fa9e4066Sahrens int 1834*3b2aab18SMatthew Ahrens zil_suspend(const char *osname, void **cookiep) 1835fa9e4066Sahrens { 1836*3b2aab18SMatthew Ahrens objset_t *os; 1837*3b2aab18SMatthew Ahrens zilog_t *zilog; 1838*3b2aab18SMatthew Ahrens const zil_header_t *zh; 1839*3b2aab18SMatthew Ahrens int error; 1840*3b2aab18SMatthew Ahrens 1841*3b2aab18SMatthew Ahrens error = dmu_objset_hold(osname, suspend_tag, &os); 1842*3b2aab18SMatthew Ahrens if (error != 0) 1843*3b2aab18SMatthew Ahrens return (error); 1844*3b2aab18SMatthew Ahrens zilog = dmu_objset_zil(os); 1845fa9e4066Sahrens 1846fa9e4066Sahrens mutex_enter(&zilog->zl_lock); 1847*3b2aab18SMatthew Ahrens zh = zilog->zl_header; 1848*3b2aab18SMatthew Ahrens 18493589c4f0SNeil Perrin if (zh->zh_flags & ZIL_REPLAY_NEEDED) { /* unplayed log */ 1850fa9e4066Sahrens mutex_exit(&zilog->zl_lock); 1851*3b2aab18SMatthew Ahrens dmu_objset_rele(os, suspend_tag); 1852fa9e4066Sahrens return (EBUSY); 1853fa9e4066Sahrens } 1854*3b2aab18SMatthew Ahrens 1855*3b2aab18SMatthew Ahrens /* 1856*3b2aab18SMatthew Ahrens * Don't put a long hold in the cases where we can avoid it. This 1857*3b2aab18SMatthew Ahrens * is when there is no cookie so we are doing a suspend & resume 1858*3b2aab18SMatthew Ahrens * (i.e. called from zil_vdev_offline()), and there's nothing to do 1859*3b2aab18SMatthew Ahrens * for the suspend because it's already suspended, or there's no ZIL. 1860*3b2aab18SMatthew Ahrens */ 1861*3b2aab18SMatthew Ahrens if (cookiep == NULL && !zilog->zl_suspending && 1862*3b2aab18SMatthew Ahrens (zilog->zl_suspend > 0 || BP_IS_HOLE(&zh->zh_log))) { 1863*3b2aab18SMatthew Ahrens mutex_exit(&zilog->zl_lock); 1864*3b2aab18SMatthew Ahrens dmu_objset_rele(os, suspend_tag); 1865*3b2aab18SMatthew Ahrens return (0); 1866*3b2aab18SMatthew Ahrens } 1867*3b2aab18SMatthew Ahrens 1868*3b2aab18SMatthew Ahrens dsl_dataset_long_hold(dmu_objset_ds(os), suspend_tag); 1869*3b2aab18SMatthew Ahrens dsl_pool_rele(dmu_objset_pool(os), suspend_tag); 1870*3b2aab18SMatthew Ahrens 1871*3b2aab18SMatthew Ahrens zilog->zl_suspend++; 1872*3b2aab18SMatthew Ahrens 1873*3b2aab18SMatthew Ahrens if (zilog->zl_suspend > 1) { 1874d80c45e0Sbonwick /* 1875*3b2aab18SMatthew Ahrens * Someone else is already suspending it. 1876d80c45e0Sbonwick * Just wait for them to finish. 1877d80c45e0Sbonwick */ 1878*3b2aab18SMatthew Ahrens 1879d80c45e0Sbonwick while (zilog->zl_suspending) 1880d80c45e0Sbonwick cv_wait(&zilog->zl_cv_suspend, &zilog->zl_lock); 1881d80c45e0Sbonwick mutex_exit(&zilog->zl_lock); 1882*3b2aab18SMatthew Ahrens 1883*3b2aab18SMatthew Ahrens if (cookiep == NULL) 1884*3b2aab18SMatthew Ahrens zil_resume(os); 1885*3b2aab18SMatthew Ahrens else 1886*3b2aab18SMatthew Ahrens *cookiep = os; 1887*3b2aab18SMatthew Ahrens return (0); 1888*3b2aab18SMatthew Ahrens } 1889*3b2aab18SMatthew Ahrens 1890*3b2aab18SMatthew Ahrens /* 1891*3b2aab18SMatthew Ahrens * If there is no pointer to an on-disk block, this ZIL must not 1892*3b2aab18SMatthew Ahrens * be active (e.g. filesystem not mounted), so there's nothing 1893*3b2aab18SMatthew Ahrens * to clean up. 1894*3b2aab18SMatthew Ahrens */ 1895*3b2aab18SMatthew Ahrens if (BP_IS_HOLE(&zh->zh_log)) { 1896*3b2aab18SMatthew Ahrens ASSERT(cookiep != NULL); /* fast path already handled */ 1897*3b2aab18SMatthew Ahrens 1898*3b2aab18SMatthew Ahrens *cookiep = os; 1899*3b2aab18SMatthew Ahrens mutex_exit(&zilog->zl_lock); 1900d80c45e0Sbonwick return (0); 1901d80c45e0Sbonwick } 1902*3b2aab18SMatthew Ahrens 1903d80c45e0Sbonwick zilog->zl_suspending = B_TRUE; 1904fa9e4066Sahrens mutex_exit(&zilog->zl_lock); 1905fa9e4066Sahrens 19065002558fSNeil Perrin zil_commit(zilog, 0); 1907fa9e4066Sahrens 1908d80c45e0Sbonwick zil_destroy(zilog, B_FALSE); 1909d80c45e0Sbonwick 1910d80c45e0Sbonwick mutex_enter(&zilog->zl_lock); 1911d80c45e0Sbonwick zilog->zl_suspending = B_FALSE; 1912d80c45e0Sbonwick cv_broadcast(&zilog->zl_cv_suspend); 1913d80c45e0Sbonwick mutex_exit(&zilog->zl_lock); 1914fa9e4066Sahrens 1915*3b2aab18SMatthew Ahrens if (cookiep == NULL) 1916*3b2aab18SMatthew Ahrens zil_resume(os); 1917*3b2aab18SMatthew Ahrens else 1918*3b2aab18SMatthew Ahrens *cookiep = os; 1919fa9e4066Sahrens return (0); 1920fa9e4066Sahrens } 1921fa9e4066Sahrens 1922fa9e4066Sahrens void 1923*3b2aab18SMatthew Ahrens zil_resume(void *cookie) 1924fa9e4066Sahrens { 1925*3b2aab18SMatthew Ahrens objset_t *os = cookie; 1926*3b2aab18SMatthew Ahrens zilog_t *zilog = dmu_objset_zil(os); 1927*3b2aab18SMatthew Ahrens 1928fa9e4066Sahrens mutex_enter(&zilog->zl_lock); 1929fa9e4066Sahrens ASSERT(zilog->zl_suspend != 0); 1930fa9e4066Sahrens zilog->zl_suspend--; 1931fa9e4066Sahrens mutex_exit(&zilog->zl_lock); 1932*3b2aab18SMatthew Ahrens dsl_dataset_long_rele(dmu_objset_ds(os), suspend_tag); 1933*3b2aab18SMatthew Ahrens dsl_dataset_rele(dmu_objset_ds(os), suspend_tag); 1934fa9e4066Sahrens } 1935fa9e4066Sahrens 1936fa9e4066Sahrens typedef struct zil_replay_arg { 1937fa9e4066Sahrens zil_replay_func_t **zr_replay; 1938fa9e4066Sahrens void *zr_arg; 1939fa9e4066Sahrens boolean_t zr_byteswap; 1940b24ab676SJeff Bonwick char *zr_lr; 1941fa9e4066Sahrens } zil_replay_arg_t; 1942fa9e4066Sahrens 1943b24ab676SJeff Bonwick static int 1944b24ab676SJeff Bonwick zil_replay_error(zilog_t *zilog, lr_t *lr, int error) 1945b24ab676SJeff Bonwick { 1946b24ab676SJeff Bonwick char name[MAXNAMELEN]; 1947b24ab676SJeff Bonwick 1948b24ab676SJeff Bonwick zilog->zl_replaying_seq--; /* didn't actually replay this one */ 1949b24ab676SJeff Bonwick 1950b24ab676SJeff Bonwick dmu_objset_name(zilog->zl_os, name); 1951b24ab676SJeff Bonwick 1952b24ab676SJeff Bonwick cmn_err(CE_WARN, "ZFS replay transaction error %d, " 1953b24ab676SJeff Bonwick "dataset %s, seq 0x%llx, txtype %llu %s\n", error, name, 1954b24ab676SJeff Bonwick (u_longlong_t)lr->lrc_seq, 1955b24ab676SJeff Bonwick (u_longlong_t)(lr->lrc_txtype & ~TX_CI), 1956b24ab676SJeff Bonwick (lr->lrc_txtype & TX_CI) ? "CI" : ""); 1957b24ab676SJeff Bonwick 1958b24ab676SJeff Bonwick return (error); 1959b24ab676SJeff Bonwick } 1960b24ab676SJeff Bonwick 1961b24ab676SJeff Bonwick static int 1962fa9e4066Sahrens zil_replay_log_record(zilog_t *zilog, lr_t *lr, void *zra, uint64_t claim_txg) 1963fa9e4066Sahrens { 1964fa9e4066Sahrens zil_replay_arg_t *zr = zra; 1965d80c45e0Sbonwick const zil_header_t *zh = zilog->zl_header; 1966fa9e4066Sahrens uint64_t reclen = lr->lrc_reclen; 1967fa9e4066Sahrens uint64_t txtype = lr->lrc_txtype; 1968b24ab676SJeff Bonwick int error = 0; 1969fa9e4066Sahrens 1970b24ab676SJeff Bonwick zilog->zl_replaying_seq = lr->lrc_seq; 1971fa9e4066Sahrens 1972fa9e4066Sahrens if (lr->lrc_seq <= zh->zh_replay_seq) /* already replayed */ 1973b24ab676SJeff Bonwick return (0); 1974b24ab676SJeff Bonwick 1975b24ab676SJeff Bonwick if (lr->lrc_txg < claim_txg) /* already committed */ 1976b24ab676SJeff Bonwick return (0); 1977fa9e4066Sahrens 1978da6c28aaSamw /* Strip case-insensitive bit, still present in log record */ 1979da6c28aaSamw txtype &= ~TX_CI; 1980da6c28aaSamw 1981b24ab676SJeff Bonwick if (txtype == 0 || txtype >= TX_MAX_TYPE) 1982b24ab676SJeff Bonwick return (zil_replay_error(zilog, lr, EINVAL)); 1983b24ab676SJeff Bonwick 1984b24ab676SJeff Bonwick /* 1985b24ab676SJeff Bonwick * If this record type can be logged out of order, the object 1986b24ab676SJeff Bonwick * (lr_foid) may no longer exist. That's legitimate, not an error. 1987b24ab676SJeff Bonwick */ 1988b24ab676SJeff Bonwick if (TX_OOO(txtype)) { 1989b24ab676SJeff Bonwick error = dmu_object_info(zilog->zl_os, 1990b24ab676SJeff Bonwick ((lr_ooo_t *)lr)->lr_foid, NULL); 1991b24ab676SJeff Bonwick if (error == ENOENT || error == EEXIST) 1992b24ab676SJeff Bonwick return (0); 19931209a471SNeil Perrin } 19941209a471SNeil Perrin 1995fa9e4066Sahrens /* 1996fa9e4066Sahrens * Make a copy of the data so we can revise and extend it. 1997fa9e4066Sahrens */ 1998b24ab676SJeff Bonwick bcopy(lr, zr->zr_lr, reclen); 1999b24ab676SJeff Bonwick 2000b24ab676SJeff Bonwick /* 2001b24ab676SJeff Bonwick * If this is a TX_WRITE with a blkptr, suck in the data. 2002b24ab676SJeff Bonwick */ 2003b24ab676SJeff Bonwick if (txtype == TX_WRITE && reclen == sizeof (lr_write_t)) { 2004b24ab676SJeff Bonwick error = zil_read_log_data(zilog, (lr_write_t *)lr, 2005b24ab676SJeff Bonwick zr->zr_lr + reclen); 2006*3b2aab18SMatthew Ahrens if (error != 0) 2007b24ab676SJeff Bonwick return (zil_replay_error(zilog, lr, error)); 2008b24ab676SJeff Bonwick } 2009fa9e4066Sahrens 2010fa9e4066Sahrens /* 2011fa9e4066Sahrens * The log block containing this lr may have been byteswapped 2012fa9e4066Sahrens * so that we can easily examine common fields like lrc_txtype. 2013b24ab676SJeff Bonwick * However, the log is a mix of different record types, and only the 2014fa9e4066Sahrens * replay vectors know how to byteswap their records. Therefore, if 2015fa9e4066Sahrens * the lr was byteswapped, undo it before invoking the replay vector. 2016fa9e4066Sahrens */ 2017fa9e4066Sahrens if (zr->zr_byteswap) 2018b24ab676SJeff Bonwick byteswap_uint64_array(zr->zr_lr, reclen); 2019fa9e4066Sahrens 2020fa9e4066Sahrens /* 2021fa9e4066Sahrens * We must now do two things atomically: replay this log record, 20221209a471SNeil Perrin * and update the log header sequence number to reflect the fact that 20231209a471SNeil Perrin * we did so. At the end of each replay function the sequence number 20241209a471SNeil Perrin * is updated if we are in replay mode. 2025fa9e4066Sahrens */ 2026b24ab676SJeff Bonwick error = zr->zr_replay[txtype](zr->zr_arg, zr->zr_lr, zr->zr_byteswap); 2027*3b2aab18SMatthew Ahrens if (error != 0) { 202867bd71c6Sperrin /* 202967bd71c6Sperrin * The DMU's dnode layer doesn't see removes until the txg 203067bd71c6Sperrin * commits, so a subsequent claim can spuriously fail with 20311209a471SNeil Perrin * EEXIST. So if we receive any error we try syncing out 2032b24ab676SJeff Bonwick * any removes then retry the transaction. Note that we 2033b24ab676SJeff Bonwick * specify B_FALSE for byteswap now, so we don't do it twice. 203467bd71c6Sperrin */ 2035b24ab676SJeff Bonwick txg_wait_synced(spa_get_dsl(zilog->zl_spa), 0); 2036b24ab676SJeff Bonwick error = zr->zr_replay[txtype](zr->zr_arg, zr->zr_lr, B_FALSE); 2037*3b2aab18SMatthew Ahrens if (error != 0) 2038b24ab676SJeff Bonwick return (zil_replay_error(zilog, lr, error)); 2039fa9e4066Sahrens } 2040b24ab676SJeff Bonwick return (0); 204167bd71c6Sperrin } 2042fa9e4066Sahrens 204367bd71c6Sperrin /* ARGSUSED */ 2044b24ab676SJeff Bonwick static int 204567bd71c6Sperrin zil_incr_blks(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg) 204667bd71c6Sperrin { 204767bd71c6Sperrin zilog->zl_replay_blks++; 2048b24ab676SJeff Bonwick 2049b24ab676SJeff Bonwick return (0); 2050fa9e4066Sahrens } 2051fa9e4066Sahrens 2052fa9e4066Sahrens /* 205313f5297eSperrin * If this dataset has a non-empty intent log, replay it and destroy it. 2054fa9e4066Sahrens */ 2055fa9e4066Sahrens void 20561209a471SNeil Perrin zil_replay(objset_t *os, void *arg, zil_replay_func_t *replay_func[TX_MAX_TYPE]) 2057fa9e4066Sahrens { 2058fa9e4066Sahrens zilog_t *zilog = dmu_objset_zil(os); 2059d80c45e0Sbonwick const zil_header_t *zh = zilog->zl_header; 2060d80c45e0Sbonwick zil_replay_arg_t zr; 206113f5297eSperrin 20623589c4f0SNeil Perrin if ((zh->zh_flags & ZIL_REPLAY_NEEDED) == 0) { 2063d80c45e0Sbonwick zil_destroy(zilog, B_TRUE); 206413f5297eSperrin return; 206513f5297eSperrin } 2066fa9e4066Sahrens 2067fa9e4066Sahrens zr.zr_replay = replay_func; 2068fa9e4066Sahrens zr.zr_arg = arg; 2069d80c45e0Sbonwick zr.zr_byteswap = BP_SHOULD_BYTESWAP(&zh->zh_log); 2070b24ab676SJeff Bonwick zr.zr_lr = kmem_alloc(2 * SPA_MAXBLOCKSIZE, KM_SLEEP); 2071fa9e4066Sahrens 2072fa9e4066Sahrens /* 2073fa9e4066Sahrens * Wait for in-progress removes to sync before starting replay. 2074fa9e4066Sahrens */ 2075fa9e4066Sahrens txg_wait_synced(zilog->zl_dmu_pool, 0); 2076fa9e4066Sahrens 20771209a471SNeil Perrin zilog->zl_replay = B_TRUE; 2078d3d50737SRafael Vanoni zilog->zl_replay_time = ddi_get_lbolt(); 207967bd71c6Sperrin ASSERT(zilog->zl_replay_blks == 0); 208067bd71c6Sperrin (void) zil_parse(zilog, zil_incr_blks, zil_replay_log_record, &zr, 2081d80c45e0Sbonwick zh->zh_claim_txg); 2082b24ab676SJeff Bonwick kmem_free(zr.zr_lr, 2 * SPA_MAXBLOCKSIZE); 2083fa9e4066Sahrens 2084d80c45e0Sbonwick zil_destroy(zilog, B_FALSE); 2085a4611edeSahrens txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 20861209a471SNeil Perrin zilog->zl_replay = B_FALSE; 2087fa9e4066Sahrens } 2088436b2950Sperrin 2089b24ab676SJeff Bonwick boolean_t 2090b24ab676SJeff Bonwick zil_replaying(zilog_t *zilog, dmu_tx_t *tx) 2091436b2950Sperrin { 209255da60b9SMark J Musante if (zilog->zl_sync == ZFS_SYNC_DISABLED) 2093b24ab676SJeff Bonwick return (B_TRUE); 2094436b2950Sperrin 2095b24ab676SJeff Bonwick if (zilog->zl_replay) { 2096b24ab676SJeff Bonwick dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 2097b24ab676SJeff Bonwick zilog->zl_replayed_seq[dmu_tx_get_txg(tx) & TXG_MASK] = 2098b24ab676SJeff Bonwick zilog->zl_replaying_seq; 2099b24ab676SJeff Bonwick return (B_TRUE); 2100b19a79ecSperrin } 2101b19a79ecSperrin 2102b24ab676SJeff Bonwick return (B_FALSE); 2103436b2950Sperrin } 2104e6ca193dSGeorge Wilson 2105e6ca193dSGeorge Wilson /* ARGSUSED */ 2106e6ca193dSGeorge Wilson int 2107fd136879SMatthew Ahrens zil_vdev_offline(const char *osname, void *arg) 2108e6ca193dSGeorge Wilson { 2109e6ca193dSGeorge Wilson int error; 2110e6ca193dSGeorge Wilson 2111*3b2aab18SMatthew Ahrens error = zil_suspend(osname, NULL); 2112*3b2aab18SMatthew Ahrens if (error != 0) 2113*3b2aab18SMatthew Ahrens return (EEXIST); 2114*3b2aab18SMatthew Ahrens return (0); 2115e6ca193dSGeorge Wilson } 2116