1*fa9e4066Sahrens /* 2*fa9e4066Sahrens * CDDL HEADER START 3*fa9e4066Sahrens * 4*fa9e4066Sahrens * The contents of this file are subject to the terms of the 5*fa9e4066Sahrens * Common Development and Distribution License, Version 1.0 only 6*fa9e4066Sahrens * (the "License"). You may not use this file except in compliance 7*fa9e4066Sahrens * with the License. 8*fa9e4066Sahrens * 9*fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 11*fa9e4066Sahrens * See the License for the specific language governing permissions 12*fa9e4066Sahrens * and limitations under the License. 13*fa9e4066Sahrens * 14*fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 15*fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 17*fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 18*fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 19*fa9e4066Sahrens * 20*fa9e4066Sahrens * CDDL HEADER END 21*fa9e4066Sahrens */ 22*fa9e4066Sahrens /* 23*fa9e4066Sahrens * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*fa9e4066Sahrens * Use is subject to license terms. 25*fa9e4066Sahrens */ 26*fa9e4066Sahrens 27*fa9e4066Sahrens #pragma ident "%Z%%M% %I% %E% SMI" 28*fa9e4066Sahrens 29*fa9e4066Sahrens #include <sys/zfs_context.h> 30*fa9e4066Sahrens #include <sys/dmu_objset.h> 31*fa9e4066Sahrens #include <sys/dmu_traverse.h> 32*fa9e4066Sahrens #include <sys/dsl_dataset.h> 33*fa9e4066Sahrens #include <sys/dsl_dir.h> 34*fa9e4066Sahrens #include <sys/dsl_pool.h> 35*fa9e4066Sahrens #include <sys/dnode.h> 36*fa9e4066Sahrens #include <sys/spa.h> 37*fa9e4066Sahrens #include <sys/zio.h> 38*fa9e4066Sahrens #include <sys/dmu_impl.h> 39*fa9e4066Sahrens 40*fa9e4066Sahrens #define BP_SPAN_SHIFT(level, width) ((level) * (width)) 41*fa9e4066Sahrens 42*fa9e4066Sahrens #define BP_EQUAL(b1, b2) \ 43*fa9e4066Sahrens (DVA_EQUAL(BP_IDENTITY(b1), BP_IDENTITY(b2)) && \ 44*fa9e4066Sahrens (b1)->blk_birth == (b2)->blk_birth) 45*fa9e4066Sahrens 46*fa9e4066Sahrens /* 47*fa9e4066Sahrens * Compare two bookmarks. 48*fa9e4066Sahrens * 49*fa9e4066Sahrens * For ADVANCE_PRE, the visitation order is: 50*fa9e4066Sahrens * 51*fa9e4066Sahrens * objset 0, 1, 2, ..., ZB_MAXOBJSET. 52*fa9e4066Sahrens * object 0, 1, 2, ..., ZB_MAXOBJECT. 53*fa9e4066Sahrens * blkoff 0, 1, 2, ... 54*fa9e4066Sahrens * level ZB_MAXLEVEL, ..., 2, 1, 0. 55*fa9e4066Sahrens * 56*fa9e4066Sahrens * where blkoff = blkid << BP_SPAN_SHIFT(level, width), and thus a valid 57*fa9e4066Sahrens * ordering vector is: 58*fa9e4066Sahrens * 59*fa9e4066Sahrens * < objset, object, blkoff, -level > 60*fa9e4066Sahrens * 61*fa9e4066Sahrens * For ADVANCE_POST, the starting offsets aren't sequential but ending 62*fa9e4066Sahrens * offsets [blkoff = (blkid + 1) << BP_SPAN_SHIFT(level, width)] are. 63*fa9e4066Sahrens * The visitation order is: 64*fa9e4066Sahrens * 65*fa9e4066Sahrens * objset 1, 2, ..., ZB_MAXOBJSET, 0. 66*fa9e4066Sahrens * object 1, 2, ..., ZB_MAXOBJECT, 0. 67*fa9e4066Sahrens * blkoff 1, 2, ... 68*fa9e4066Sahrens * level 0, 1, 2, ..., ZB_MAXLEVEL. 69*fa9e4066Sahrens * 70*fa9e4066Sahrens * and thus a valid ordering vector is: 71*fa9e4066Sahrens * 72*fa9e4066Sahrens * < objset - 1, object - 1, blkoff, level > 73*fa9e4066Sahrens * 74*fa9e4066Sahrens * Both orderings can be expressed as: 75*fa9e4066Sahrens * 76*fa9e4066Sahrens * < objset + bias, object + bias, blkoff, level ^ bias > 77*fa9e4066Sahrens * 78*fa9e4066Sahrens * where 'bias' is either 0 or -1 (for ADVANCE_PRE or ADVANCE_POST) 79*fa9e4066Sahrens * and 'blkoff' is (blkid - bias) << BP_SPAN_SHIFT(level, wshift). 80*fa9e4066Sahrens * 81*fa9e4066Sahrens * Special case: an objset's osphys is represented as level -1 of object 0. 82*fa9e4066Sahrens * It is always either the very first or very last block we visit in an objset. 83*fa9e4066Sahrens * Therefore, if either bookmark's level is -1, level alone determines order. 84*fa9e4066Sahrens */ 85*fa9e4066Sahrens static int 86*fa9e4066Sahrens compare_bookmark(zbookmark_t *szb, zbookmark_t *ezb, dnode_phys_t *dnp, 87*fa9e4066Sahrens int advance) 88*fa9e4066Sahrens { 89*fa9e4066Sahrens int bias = (advance & ADVANCE_PRE) ? 0 : -1; 90*fa9e4066Sahrens uint64_t sblkoff, eblkoff; 91*fa9e4066Sahrens int slevel, elevel, wshift; 92*fa9e4066Sahrens 93*fa9e4066Sahrens if (szb->zb_objset + bias < ezb->zb_objset + bias) 94*fa9e4066Sahrens return (-1); 95*fa9e4066Sahrens 96*fa9e4066Sahrens if (szb->zb_objset + bias > ezb->zb_objset + bias) 97*fa9e4066Sahrens return (1); 98*fa9e4066Sahrens 99*fa9e4066Sahrens slevel = szb->zb_level; 100*fa9e4066Sahrens elevel = ezb->zb_level; 101*fa9e4066Sahrens 102*fa9e4066Sahrens if ((slevel | elevel) < 0) 103*fa9e4066Sahrens return ((slevel ^ bias) - (elevel ^ bias)); 104*fa9e4066Sahrens 105*fa9e4066Sahrens if (szb->zb_object + bias < ezb->zb_object + bias) 106*fa9e4066Sahrens return (-1); 107*fa9e4066Sahrens 108*fa9e4066Sahrens if (szb->zb_object + bias > ezb->zb_object + bias) 109*fa9e4066Sahrens return (1); 110*fa9e4066Sahrens 111*fa9e4066Sahrens if (dnp == NULL) 112*fa9e4066Sahrens return (0); 113*fa9e4066Sahrens 114*fa9e4066Sahrens wshift = dnp->dn_indblkshift - SPA_BLKPTRSHIFT; 115*fa9e4066Sahrens 116*fa9e4066Sahrens sblkoff = (szb->zb_blkid - bias) << BP_SPAN_SHIFT(slevel, wshift); 117*fa9e4066Sahrens eblkoff = (ezb->zb_blkid - bias) << BP_SPAN_SHIFT(elevel, wshift); 118*fa9e4066Sahrens 119*fa9e4066Sahrens if (sblkoff < eblkoff) 120*fa9e4066Sahrens return (-1); 121*fa9e4066Sahrens 122*fa9e4066Sahrens if (sblkoff > eblkoff) 123*fa9e4066Sahrens return (1); 124*fa9e4066Sahrens 125*fa9e4066Sahrens return ((elevel ^ bias) - (slevel ^ bias)); 126*fa9e4066Sahrens } 127*fa9e4066Sahrens 128*fa9e4066Sahrens #define SET_BOOKMARK(zb, objset, object, level, blkid) \ 129*fa9e4066Sahrens { \ 130*fa9e4066Sahrens (zb)->zb_objset = objset; \ 131*fa9e4066Sahrens (zb)->zb_object = object; \ 132*fa9e4066Sahrens (zb)->zb_level = level; \ 133*fa9e4066Sahrens (zb)->zb_blkid = blkid; \ 134*fa9e4066Sahrens } 135*fa9e4066Sahrens 136*fa9e4066Sahrens #define SET_BOOKMARK_LB(zb, level, blkid) \ 137*fa9e4066Sahrens { \ 138*fa9e4066Sahrens (zb)->zb_level = level; \ 139*fa9e4066Sahrens (zb)->zb_blkid = blkid; \ 140*fa9e4066Sahrens } 141*fa9e4066Sahrens 142*fa9e4066Sahrens static int 143*fa9e4066Sahrens advance_objset(zseg_t *zseg, uint64_t objset, int advance) 144*fa9e4066Sahrens { 145*fa9e4066Sahrens zbookmark_t *zb = &zseg->seg_start; 146*fa9e4066Sahrens 147*fa9e4066Sahrens if (advance & ADVANCE_PRE) { 148*fa9e4066Sahrens if (objset >= ZB_MAXOBJSET) 149*fa9e4066Sahrens return (ERANGE); 150*fa9e4066Sahrens SET_BOOKMARK(zb, objset, 0, -1, 0); 151*fa9e4066Sahrens } else { 152*fa9e4066Sahrens if (objset >= ZB_MAXOBJSET) 153*fa9e4066Sahrens objset = 0; 154*fa9e4066Sahrens SET_BOOKMARK(zb, objset, 1, 0, 0); 155*fa9e4066Sahrens } 156*fa9e4066Sahrens 157*fa9e4066Sahrens if (compare_bookmark(zb, &zseg->seg_end, NULL, advance) > 0) 158*fa9e4066Sahrens return (ERANGE); 159*fa9e4066Sahrens 160*fa9e4066Sahrens return (EAGAIN); 161*fa9e4066Sahrens } 162*fa9e4066Sahrens 163*fa9e4066Sahrens static int 164*fa9e4066Sahrens advance_object(zseg_t *zseg, uint64_t object, int advance) 165*fa9e4066Sahrens { 166*fa9e4066Sahrens zbookmark_t *zb = &zseg->seg_start; 167*fa9e4066Sahrens 168*fa9e4066Sahrens if (advance & ADVANCE_PRE) { 169*fa9e4066Sahrens if (object >= ZB_MAXOBJECT) { 170*fa9e4066Sahrens SET_BOOKMARK(zb, zb->zb_objset + 1, 0, -1, 0); 171*fa9e4066Sahrens } else { 172*fa9e4066Sahrens SET_BOOKMARK(zb, zb->zb_objset, object, ZB_MAXLEVEL, 0); 173*fa9e4066Sahrens } 174*fa9e4066Sahrens } else { 175*fa9e4066Sahrens if (zb->zb_object == 0) { 176*fa9e4066Sahrens SET_BOOKMARK(zb, zb->zb_objset, 0, -1, 0); 177*fa9e4066Sahrens } else { 178*fa9e4066Sahrens if (object >= ZB_MAXOBJECT) 179*fa9e4066Sahrens object = 0; 180*fa9e4066Sahrens SET_BOOKMARK(zb, zb->zb_objset, object, 0, 0); 181*fa9e4066Sahrens } 182*fa9e4066Sahrens } 183*fa9e4066Sahrens 184*fa9e4066Sahrens if (compare_bookmark(zb, &zseg->seg_end, NULL, advance) > 0) 185*fa9e4066Sahrens return (ERANGE); 186*fa9e4066Sahrens 187*fa9e4066Sahrens return (EAGAIN); 188*fa9e4066Sahrens } 189*fa9e4066Sahrens 190*fa9e4066Sahrens static int 191*fa9e4066Sahrens advance_from_osphys(zseg_t *zseg, int advance) 192*fa9e4066Sahrens { 193*fa9e4066Sahrens zbookmark_t *zb = &zseg->seg_start; 194*fa9e4066Sahrens 195*fa9e4066Sahrens ASSERT(zb->zb_object == 0); 196*fa9e4066Sahrens ASSERT(zb->zb_level == -1); 197*fa9e4066Sahrens ASSERT(zb->zb_blkid == 0); 198*fa9e4066Sahrens 199*fa9e4066Sahrens if (advance & ADVANCE_PRE) { 200*fa9e4066Sahrens SET_BOOKMARK_LB(zb, ZB_MAXLEVEL, 0); 201*fa9e4066Sahrens } else { 202*fa9e4066Sahrens if (zb->zb_objset == 0) 203*fa9e4066Sahrens return (ERANGE); 204*fa9e4066Sahrens SET_BOOKMARK(zb, zb->zb_objset + 1, 1, 0, 0); 205*fa9e4066Sahrens } 206*fa9e4066Sahrens 207*fa9e4066Sahrens if (compare_bookmark(zb, &zseg->seg_end, NULL, advance) > 0) 208*fa9e4066Sahrens return (ERANGE); 209*fa9e4066Sahrens 210*fa9e4066Sahrens return (EAGAIN); 211*fa9e4066Sahrens } 212*fa9e4066Sahrens 213*fa9e4066Sahrens static int 214*fa9e4066Sahrens advance_block(zseg_t *zseg, dnode_phys_t *dnp, int rc, int advance) 215*fa9e4066Sahrens { 216*fa9e4066Sahrens zbookmark_t *zb = &zseg->seg_start; 217*fa9e4066Sahrens int wshift = dnp->dn_indblkshift - SPA_BLKPTRSHIFT; 218*fa9e4066Sahrens int maxlevel = dnp->dn_nlevels - 1; 219*fa9e4066Sahrens int level = zb->zb_level; 220*fa9e4066Sahrens uint64_t blkid = zb->zb_blkid; 221*fa9e4066Sahrens 222*fa9e4066Sahrens if (advance & ADVANCE_PRE) { 223*fa9e4066Sahrens if (level > 0 && rc == 0) { 224*fa9e4066Sahrens level--; 225*fa9e4066Sahrens blkid <<= wshift; 226*fa9e4066Sahrens } else { 227*fa9e4066Sahrens blkid++; 228*fa9e4066Sahrens 229*fa9e4066Sahrens if ((blkid << BP_SPAN_SHIFT(level, wshift)) > 230*fa9e4066Sahrens dnp->dn_maxblkid) 231*fa9e4066Sahrens return (ERANGE); 232*fa9e4066Sahrens 233*fa9e4066Sahrens while (level < maxlevel) { 234*fa9e4066Sahrens if (P2PHASE(blkid, 1ULL << wshift)) 235*fa9e4066Sahrens break; 236*fa9e4066Sahrens blkid >>= wshift; 237*fa9e4066Sahrens level++; 238*fa9e4066Sahrens } 239*fa9e4066Sahrens } 240*fa9e4066Sahrens } else { 241*fa9e4066Sahrens if (level >= maxlevel || P2PHASE(blkid + 1, 1ULL << wshift)) { 242*fa9e4066Sahrens blkid = (blkid + 1) << BP_SPAN_SHIFT(level, wshift); 243*fa9e4066Sahrens level = 0; 244*fa9e4066Sahrens } else { 245*fa9e4066Sahrens blkid >>= wshift; 246*fa9e4066Sahrens level++; 247*fa9e4066Sahrens } 248*fa9e4066Sahrens 249*fa9e4066Sahrens while ((blkid << BP_SPAN_SHIFT(level, wshift)) > 250*fa9e4066Sahrens dnp->dn_maxblkid) { 251*fa9e4066Sahrens if (level == maxlevel) 252*fa9e4066Sahrens return (ERANGE); 253*fa9e4066Sahrens blkid >>= wshift; 254*fa9e4066Sahrens level++; 255*fa9e4066Sahrens } 256*fa9e4066Sahrens } 257*fa9e4066Sahrens SET_BOOKMARK_LB(zb, level, blkid); 258*fa9e4066Sahrens 259*fa9e4066Sahrens if (compare_bookmark(zb, &zseg->seg_end, dnp, advance) > 0) 260*fa9e4066Sahrens return (ERANGE); 261*fa9e4066Sahrens 262*fa9e4066Sahrens return (EAGAIN); 263*fa9e4066Sahrens } 264*fa9e4066Sahrens 265*fa9e4066Sahrens static int 266*fa9e4066Sahrens traverse_callback(traverse_handle_t *th, zseg_t *zseg, traverse_blk_cache_t *bc) 267*fa9e4066Sahrens { 268*fa9e4066Sahrens /* 269*fa9e4066Sahrens * Before we issue the callback, prune against maxtxg. 270*fa9e4066Sahrens * 271*fa9e4066Sahrens * We prune against mintxg before we get here because it's a big win. 272*fa9e4066Sahrens * If a given block was born in txg 37, then we know that the entire 273*fa9e4066Sahrens * subtree below that block must have been born in txg 37 or earlier. 274*fa9e4066Sahrens * We can therefore lop off huge branches of the tree as we go. 275*fa9e4066Sahrens * 276*fa9e4066Sahrens * There's no corresponding optimization for maxtxg because knowing 277*fa9e4066Sahrens * that bp->blk_birth >= maxtxg doesn't imply anything about the bp's 278*fa9e4066Sahrens * children. In fact, the copy-on-write design of ZFS ensures that 279*fa9e4066Sahrens * top-level blocks will pretty much always be new. 280*fa9e4066Sahrens * 281*fa9e4066Sahrens * Therefore, in the name of simplicity we don't prune against 282*fa9e4066Sahrens * maxtxg until the last possible moment -- that being right now. 283*fa9e4066Sahrens */ 284*fa9e4066Sahrens if (bc->bc_errno == 0 && bc->bc_blkptr.blk_birth >= zseg->seg_maxtxg) 285*fa9e4066Sahrens return (0); 286*fa9e4066Sahrens 287*fa9e4066Sahrens if (bc->bc_errno == 0) { 288*fa9e4066Sahrens zbookmark_t *zb = &bc->bc_bookmark; 289*fa9e4066Sahrens zbookmark_t *szb = &zseg->seg_start; 290*fa9e4066Sahrens zbookmark_t *ezb = &zseg->seg_end; 291*fa9e4066Sahrens zbookmark_t *lzb = &th->th_lastcb; 292*fa9e4066Sahrens dnode_phys_t *dnp = bc->bc_dnode; 293*fa9e4066Sahrens 294*fa9e4066Sahrens /* 295*fa9e4066Sahrens * Debugging: verify that the order we visit things 296*fa9e4066Sahrens * agrees with the order defined by compare_bookmark(). 297*fa9e4066Sahrens */ 298*fa9e4066Sahrens ASSERT(compare_bookmark(zb, ezb, dnp, th->th_advance) <= 0); 299*fa9e4066Sahrens ASSERT(compare_bookmark(zb, szb, dnp, th->th_advance) == 0); 300*fa9e4066Sahrens ASSERT(compare_bookmark(lzb, zb, dnp, th->th_advance) < 0 || 301*fa9e4066Sahrens lzb->zb_level == ZB_NO_LEVEL); 302*fa9e4066Sahrens *lzb = *zb; 303*fa9e4066Sahrens } 304*fa9e4066Sahrens 305*fa9e4066Sahrens th->th_callbacks++; 306*fa9e4066Sahrens return (th->th_func(bc, th->th_spa, th->th_arg)); 307*fa9e4066Sahrens } 308*fa9e4066Sahrens 309*fa9e4066Sahrens static int 310*fa9e4066Sahrens traverse_read(traverse_handle_t *th, traverse_blk_cache_t *bc, blkptr_t *bp, 311*fa9e4066Sahrens dnode_phys_t *dnp) 312*fa9e4066Sahrens { 313*fa9e4066Sahrens zbookmark_t *zb = &bc->bc_bookmark; 314*fa9e4066Sahrens int error; 315*fa9e4066Sahrens 316*fa9e4066Sahrens th->th_hits++; 317*fa9e4066Sahrens 318*fa9e4066Sahrens bc->bc_dnode = dnp; 319*fa9e4066Sahrens bc->bc_errno = 0; 320*fa9e4066Sahrens 321*fa9e4066Sahrens if (BP_EQUAL(&bc->bc_blkptr, bp)) 322*fa9e4066Sahrens return (0); 323*fa9e4066Sahrens 324*fa9e4066Sahrens bc->bc_blkptr = *bp; 325*fa9e4066Sahrens 326*fa9e4066Sahrens if (bc->bc_data == NULL) 327*fa9e4066Sahrens return (0); 328*fa9e4066Sahrens 329*fa9e4066Sahrens if (BP_IS_HOLE(bp)) { 330*fa9e4066Sahrens ASSERT(th->th_advance & ADVANCE_HOLES); 331*fa9e4066Sahrens return (0); 332*fa9e4066Sahrens } 333*fa9e4066Sahrens 334*fa9e4066Sahrens if (compare_bookmark(zb, &th->th_noread, dnp, 0) == 0) { 335*fa9e4066Sahrens error = EIO; 336*fa9e4066Sahrens } else if (arc_tryread(th->th_spa, bp, bc->bc_data) == 0) { 337*fa9e4066Sahrens error = 0; 338*fa9e4066Sahrens th->th_arc_hits++; 339*fa9e4066Sahrens } else { 340*fa9e4066Sahrens error = zio_wait(zio_read(NULL, th->th_spa, bp, bc->bc_data, 341*fa9e4066Sahrens BP_GET_LSIZE(bp), NULL, NULL, ZIO_PRIORITY_SYNC_READ, 342*fa9e4066Sahrens th->th_zio_flags | ZIO_FLAG_DONT_CACHE)); 343*fa9e4066Sahrens 344*fa9e4066Sahrens if (BP_SHOULD_BYTESWAP(bp) && error == 0) 345*fa9e4066Sahrens (zb->zb_level > 0 ? byteswap_uint64_array : 346*fa9e4066Sahrens dmu_ot[BP_GET_TYPE(bp)].ot_byteswap)(bc->bc_data, 347*fa9e4066Sahrens BP_GET_LSIZE(bp)); 348*fa9e4066Sahrens th->th_reads++; 349*fa9e4066Sahrens } 350*fa9e4066Sahrens 351*fa9e4066Sahrens if (error) { 352*fa9e4066Sahrens bc->bc_errno = error; 353*fa9e4066Sahrens error = traverse_callback(th, NULL, bc); 354*fa9e4066Sahrens ASSERT(error == EAGAIN || error == EINTR || error == ERESTART); 355*fa9e4066Sahrens bc->bc_blkptr.blk_birth = -1ULL; 356*fa9e4066Sahrens } 357*fa9e4066Sahrens 358*fa9e4066Sahrens dprintf("cache %02x error %d <%llu, %llu, %d, %llx>\n", 359*fa9e4066Sahrens bc - &th->th_cache[0][0], error, 360*fa9e4066Sahrens zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid); 361*fa9e4066Sahrens 362*fa9e4066Sahrens return (error); 363*fa9e4066Sahrens } 364*fa9e4066Sahrens 365*fa9e4066Sahrens static int 366*fa9e4066Sahrens find_block(traverse_handle_t *th, zseg_t *zseg, dnode_phys_t *dnp, int depth) 367*fa9e4066Sahrens { 368*fa9e4066Sahrens zbookmark_t *zb = &zseg->seg_start; 369*fa9e4066Sahrens traverse_blk_cache_t *bc; 370*fa9e4066Sahrens blkptr_t *bp = dnp->dn_blkptr; 371*fa9e4066Sahrens int i, first, level; 372*fa9e4066Sahrens int nbp = dnp->dn_nblkptr; 373*fa9e4066Sahrens int minlevel = zb->zb_level; 374*fa9e4066Sahrens int maxlevel = dnp->dn_nlevels - 1; 375*fa9e4066Sahrens int wshift = dnp->dn_indblkshift - SPA_BLKPTRSHIFT; 376*fa9e4066Sahrens int bp_shift = BP_SPAN_SHIFT(maxlevel - minlevel, wshift); 377*fa9e4066Sahrens uint64_t blkid = zb->zb_blkid >> bp_shift; 378*fa9e4066Sahrens int do_holes = (th->th_advance & ADVANCE_HOLES) && depth == ZB_DN_CACHE; 379*fa9e4066Sahrens int rc; 380*fa9e4066Sahrens 381*fa9e4066Sahrens if (minlevel > maxlevel || blkid >= nbp) 382*fa9e4066Sahrens return (ERANGE); 383*fa9e4066Sahrens 384*fa9e4066Sahrens for (level = maxlevel; level >= minlevel; level--) { 385*fa9e4066Sahrens first = P2PHASE(blkid, 1ULL << wshift); 386*fa9e4066Sahrens 387*fa9e4066Sahrens for (i = first; i < nbp; i++) 388*fa9e4066Sahrens if (bp[i].blk_birth > zseg->seg_mintxg || 389*fa9e4066Sahrens BP_IS_HOLE(&bp[i]) && do_holes) 390*fa9e4066Sahrens break; 391*fa9e4066Sahrens 392*fa9e4066Sahrens if (i != first) { 393*fa9e4066Sahrens i--; 394*fa9e4066Sahrens SET_BOOKMARK_LB(zb, level, blkid + (i - first)); 395*fa9e4066Sahrens return (ENOTBLK); 396*fa9e4066Sahrens } 397*fa9e4066Sahrens 398*fa9e4066Sahrens bc = &th->th_cache[depth][level]; 399*fa9e4066Sahrens 400*fa9e4066Sahrens SET_BOOKMARK(&bc->bc_bookmark, zb->zb_objset, zb->zb_object, 401*fa9e4066Sahrens level, blkid); 402*fa9e4066Sahrens 403*fa9e4066Sahrens if (rc = traverse_read(th, bc, bp + i, dnp)) { 404*fa9e4066Sahrens if (rc != EAGAIN) { 405*fa9e4066Sahrens SET_BOOKMARK_LB(zb, level, blkid); 406*fa9e4066Sahrens } 407*fa9e4066Sahrens return (rc); 408*fa9e4066Sahrens } 409*fa9e4066Sahrens 410*fa9e4066Sahrens if (BP_IS_HOLE(&bp[i])) { 411*fa9e4066Sahrens SET_BOOKMARK_LB(zb, level, blkid); 412*fa9e4066Sahrens th->th_lastcb.zb_level = ZB_NO_LEVEL; 413*fa9e4066Sahrens return (0); 414*fa9e4066Sahrens } 415*fa9e4066Sahrens 416*fa9e4066Sahrens nbp = 1 << wshift; 417*fa9e4066Sahrens bp = bc->bc_data; 418*fa9e4066Sahrens bp_shift -= wshift; 419*fa9e4066Sahrens blkid = zb->zb_blkid >> bp_shift; 420*fa9e4066Sahrens } 421*fa9e4066Sahrens 422*fa9e4066Sahrens return (0); 423*fa9e4066Sahrens } 424*fa9e4066Sahrens 425*fa9e4066Sahrens static int 426*fa9e4066Sahrens get_dnode(traverse_handle_t *th, uint64_t objset, dnode_phys_t *mdn, 427*fa9e4066Sahrens uint64_t *objectp, dnode_phys_t **dnpp, uint64_t txg, int type, int depth) 428*fa9e4066Sahrens { 429*fa9e4066Sahrens zseg_t zseg; 430*fa9e4066Sahrens zbookmark_t *zb = &zseg.seg_start; 431*fa9e4066Sahrens uint64_t object = *objectp; 432*fa9e4066Sahrens int i, rc; 433*fa9e4066Sahrens 434*fa9e4066Sahrens SET_BOOKMARK(zb, objset, 0, 0, object / DNODES_PER_BLOCK); 435*fa9e4066Sahrens SET_BOOKMARK(&zseg.seg_end, objset, 0, 0, ZB_MAXBLKID); 436*fa9e4066Sahrens 437*fa9e4066Sahrens zseg.seg_mintxg = txg; 438*fa9e4066Sahrens zseg.seg_maxtxg = -1ULL; 439*fa9e4066Sahrens 440*fa9e4066Sahrens for (;;) { 441*fa9e4066Sahrens rc = find_block(th, &zseg, mdn, depth); 442*fa9e4066Sahrens 443*fa9e4066Sahrens if (rc == EAGAIN || rc == EINTR || rc == ERANGE) 444*fa9e4066Sahrens break; 445*fa9e4066Sahrens 446*fa9e4066Sahrens if (rc == 0 && zb->zb_level == 0) { 447*fa9e4066Sahrens dnode_phys_t *dnp = th->th_cache[depth][0].bc_data; 448*fa9e4066Sahrens for (i = 0; i < DNODES_PER_BLOCK; i++) { 449*fa9e4066Sahrens object = (zb->zb_blkid * DNODES_PER_BLOCK) + i; 450*fa9e4066Sahrens if (object >= *objectp && 451*fa9e4066Sahrens dnp[i].dn_type != DMU_OT_NONE && 452*fa9e4066Sahrens (type == -1 || dnp[i].dn_type == type)) { 453*fa9e4066Sahrens *objectp = object; 454*fa9e4066Sahrens *dnpp = &dnp[i]; 455*fa9e4066Sahrens return (0); 456*fa9e4066Sahrens } 457*fa9e4066Sahrens } 458*fa9e4066Sahrens } 459*fa9e4066Sahrens 460*fa9e4066Sahrens rc = advance_block(&zseg, mdn, rc, ADVANCE_PRE); 461*fa9e4066Sahrens 462*fa9e4066Sahrens if (rc == ERANGE) 463*fa9e4066Sahrens break; 464*fa9e4066Sahrens } 465*fa9e4066Sahrens 466*fa9e4066Sahrens if (rc == ERANGE) 467*fa9e4066Sahrens *objectp = ZB_MAXOBJECT; 468*fa9e4066Sahrens 469*fa9e4066Sahrens return (rc); 470*fa9e4066Sahrens } 471*fa9e4066Sahrens 472*fa9e4066Sahrens static int 473*fa9e4066Sahrens traverse_segment(traverse_handle_t *th, zseg_t *zseg, blkptr_t *mosbp) 474*fa9e4066Sahrens { 475*fa9e4066Sahrens zbookmark_t *zb = &zseg->seg_start; 476*fa9e4066Sahrens traverse_blk_cache_t *bc; 477*fa9e4066Sahrens dnode_phys_t *dn, *dn_tmp; 478*fa9e4066Sahrens int worklimit = 1000; 479*fa9e4066Sahrens int rc; 480*fa9e4066Sahrens 481*fa9e4066Sahrens dprintf("<%llu, %llu, %d, %llx>\n", 482*fa9e4066Sahrens zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid); 483*fa9e4066Sahrens 484*fa9e4066Sahrens bc = &th->th_cache[ZB_MOS_CACHE][ZB_MAXLEVEL - 1]; 485*fa9e4066Sahrens dn = &((objset_phys_t *)bc->bc_data)->os_meta_dnode; 486*fa9e4066Sahrens 487*fa9e4066Sahrens SET_BOOKMARK(&bc->bc_bookmark, 0, 0, -1, 0); 488*fa9e4066Sahrens 489*fa9e4066Sahrens rc = traverse_read(th, bc, mosbp, dn); 490*fa9e4066Sahrens 491*fa9e4066Sahrens if (rc) /* If we get ERESTART, we've got nowhere left to go */ 492*fa9e4066Sahrens return (rc == ERESTART ? EINTR : rc); 493*fa9e4066Sahrens 494*fa9e4066Sahrens ASSERT(dn->dn_nlevels < ZB_MAXLEVEL); 495*fa9e4066Sahrens 496*fa9e4066Sahrens if (zb->zb_objset != 0) { 497*fa9e4066Sahrens uint64_t objset = zb->zb_objset; 498*fa9e4066Sahrens dsl_dataset_phys_t *dsp; 499*fa9e4066Sahrens 500*fa9e4066Sahrens rc = get_dnode(th, 0, dn, &objset, &dn_tmp, 0, 501*fa9e4066Sahrens DMU_OT_DSL_OBJSET, ZB_MOS_CACHE); 502*fa9e4066Sahrens 503*fa9e4066Sahrens if (objset != zb->zb_objset) 504*fa9e4066Sahrens rc = advance_objset(zseg, objset, th->th_advance); 505*fa9e4066Sahrens 506*fa9e4066Sahrens if (rc != 0) 507*fa9e4066Sahrens return (rc); 508*fa9e4066Sahrens 509*fa9e4066Sahrens dsp = DN_BONUS(dn_tmp); 510*fa9e4066Sahrens 511*fa9e4066Sahrens bc = &th->th_cache[ZB_MDN_CACHE][ZB_MAXLEVEL - 1]; 512*fa9e4066Sahrens dn = &((objset_phys_t *)bc->bc_data)->os_meta_dnode; 513*fa9e4066Sahrens 514*fa9e4066Sahrens SET_BOOKMARK(&bc->bc_bookmark, objset, 0, -1, 0); 515*fa9e4066Sahrens 516*fa9e4066Sahrens rc = traverse_read(th, bc, &dsp->ds_bp, dn); 517*fa9e4066Sahrens 518*fa9e4066Sahrens if (rc != 0) { 519*fa9e4066Sahrens if (rc == ERESTART) 520*fa9e4066Sahrens rc = advance_objset(zseg, zb->zb_objset + 1, 521*fa9e4066Sahrens th->th_advance); 522*fa9e4066Sahrens return (rc); 523*fa9e4066Sahrens } 524*fa9e4066Sahrens 525*fa9e4066Sahrens if (th->th_advance & ADVANCE_PRUNE) 526*fa9e4066Sahrens zseg->seg_mintxg = 527*fa9e4066Sahrens MAX(zseg->seg_mintxg, dsp->ds_prev_snap_txg); 528*fa9e4066Sahrens } 529*fa9e4066Sahrens 530*fa9e4066Sahrens if (zb->zb_level == -1) { 531*fa9e4066Sahrens ASSERT(zb->zb_object == 0); 532*fa9e4066Sahrens 533*fa9e4066Sahrens if (bc->bc_blkptr.blk_birth > zseg->seg_mintxg) { 534*fa9e4066Sahrens rc = traverse_callback(th, zseg, bc); 535*fa9e4066Sahrens if (rc) { 536*fa9e4066Sahrens ASSERT(rc == EINTR); 537*fa9e4066Sahrens return (rc); 538*fa9e4066Sahrens } 539*fa9e4066Sahrens } 540*fa9e4066Sahrens 541*fa9e4066Sahrens return (advance_from_osphys(zseg, th->th_advance)); 542*fa9e4066Sahrens } 543*fa9e4066Sahrens 544*fa9e4066Sahrens if (zb->zb_object != 0) { 545*fa9e4066Sahrens uint64_t object = zb->zb_object; 546*fa9e4066Sahrens 547*fa9e4066Sahrens rc = get_dnode(th, zb->zb_objset, dn, &object, &dn_tmp, 548*fa9e4066Sahrens zseg->seg_mintxg, -1, ZB_MDN_CACHE); 549*fa9e4066Sahrens 550*fa9e4066Sahrens if (object != zb->zb_object) 551*fa9e4066Sahrens rc = advance_object(zseg, object, th->th_advance); 552*fa9e4066Sahrens 553*fa9e4066Sahrens if (rc != 0) 554*fa9e4066Sahrens return (rc); 555*fa9e4066Sahrens 556*fa9e4066Sahrens dn = dn_tmp; 557*fa9e4066Sahrens } 558*fa9e4066Sahrens 559*fa9e4066Sahrens if (zb->zb_level == ZB_MAXLEVEL) 560*fa9e4066Sahrens zb->zb_level = dn->dn_nlevels - 1; 561*fa9e4066Sahrens 562*fa9e4066Sahrens for (;;) { 563*fa9e4066Sahrens rc = find_block(th, zseg, dn, ZB_DN_CACHE); 564*fa9e4066Sahrens 565*fa9e4066Sahrens if (rc == EAGAIN || rc == EINTR || rc == ERANGE) 566*fa9e4066Sahrens break; 567*fa9e4066Sahrens 568*fa9e4066Sahrens if (rc == 0) { 569*fa9e4066Sahrens bc = &th->th_cache[ZB_DN_CACHE][zb->zb_level]; 570*fa9e4066Sahrens ASSERT(bc->bc_dnode == dn); 571*fa9e4066Sahrens ASSERT(bc->bc_blkptr.blk_birth <= mosbp->blk_birth); 572*fa9e4066Sahrens rc = traverse_callback(th, zseg, bc); 573*fa9e4066Sahrens if (rc) { 574*fa9e4066Sahrens ASSERT(rc == EINTR); 575*fa9e4066Sahrens return (rc); 576*fa9e4066Sahrens } 577*fa9e4066Sahrens if (BP_IS_HOLE(&bc->bc_blkptr)) { 578*fa9e4066Sahrens ASSERT(th->th_advance & ADVANCE_HOLES); 579*fa9e4066Sahrens rc = ENOTBLK; 580*fa9e4066Sahrens } 581*fa9e4066Sahrens } 582*fa9e4066Sahrens 583*fa9e4066Sahrens rc = advance_block(zseg, dn, rc, th->th_advance); 584*fa9e4066Sahrens 585*fa9e4066Sahrens if (rc == ERANGE) 586*fa9e4066Sahrens break; 587*fa9e4066Sahrens 588*fa9e4066Sahrens /* 589*fa9e4066Sahrens * Give spa_sync() a chance to run. 590*fa9e4066Sahrens */ 591*fa9e4066Sahrens if (spa_traverse_wanted(th->th_spa)) { 592*fa9e4066Sahrens th->th_syncs++; 593*fa9e4066Sahrens return (EAGAIN); 594*fa9e4066Sahrens } 595*fa9e4066Sahrens 596*fa9e4066Sahrens if (--worklimit == 0) 597*fa9e4066Sahrens return (EAGAIN); 598*fa9e4066Sahrens } 599*fa9e4066Sahrens 600*fa9e4066Sahrens if (rc == ERANGE) 601*fa9e4066Sahrens rc = advance_object(zseg, zb->zb_object + 1, th->th_advance); 602*fa9e4066Sahrens 603*fa9e4066Sahrens return (rc); 604*fa9e4066Sahrens } 605*fa9e4066Sahrens 606*fa9e4066Sahrens /* 607*fa9e4066Sahrens * It is the caller's responsibility to ensure that the dsl_dataset_t 608*fa9e4066Sahrens * doesn't go away during traversal. 609*fa9e4066Sahrens */ 610*fa9e4066Sahrens int 611*fa9e4066Sahrens traverse_dsl_dataset(dsl_dataset_t *ds, uint64_t txg_start, int advance, 612*fa9e4066Sahrens blkptr_cb_t func, void *arg) 613*fa9e4066Sahrens { 614*fa9e4066Sahrens spa_t *spa = ds->ds_dir->dd_pool->dp_spa; 615*fa9e4066Sahrens traverse_handle_t *th; 616*fa9e4066Sahrens int err; 617*fa9e4066Sahrens 618*fa9e4066Sahrens th = traverse_init(spa, func, arg, advance, ZIO_FLAG_MUSTSUCCEED); 619*fa9e4066Sahrens 620*fa9e4066Sahrens traverse_add_objset(th, txg_start, -1ULL, ds->ds_object); 621*fa9e4066Sahrens 622*fa9e4066Sahrens while ((err = traverse_more(th)) == EAGAIN) 623*fa9e4066Sahrens continue; 624*fa9e4066Sahrens 625*fa9e4066Sahrens traverse_fini(th); 626*fa9e4066Sahrens return (err); 627*fa9e4066Sahrens } 628*fa9e4066Sahrens 629*fa9e4066Sahrens int 630*fa9e4066Sahrens traverse_more(traverse_handle_t *th) 631*fa9e4066Sahrens { 632*fa9e4066Sahrens zseg_t *zseg = list_head(&th->th_seglist); 633*fa9e4066Sahrens uint64_t save_txg; /* XXX won't be necessary with real itinerary */ 634*fa9e4066Sahrens krwlock_t *rw = spa_traverse_rwlock(th->th_spa); 635*fa9e4066Sahrens blkptr_t *mosbp = spa_get_rootblkptr(th->th_spa); 636*fa9e4066Sahrens int rc; 637*fa9e4066Sahrens 638*fa9e4066Sahrens if (zseg == NULL) 639*fa9e4066Sahrens return (0); 640*fa9e4066Sahrens 641*fa9e4066Sahrens th->th_restarts++; 642*fa9e4066Sahrens 643*fa9e4066Sahrens save_txg = zseg->seg_mintxg; 644*fa9e4066Sahrens 645*fa9e4066Sahrens if (!(th->th_advance & ADVANCE_NOLOCK)) 646*fa9e4066Sahrens rw_enter(rw, RW_READER); 647*fa9e4066Sahrens 648*fa9e4066Sahrens rc = traverse_segment(th, zseg, mosbp); 649*fa9e4066Sahrens ASSERT(rc == ERANGE || rc == EAGAIN || rc == EINTR); 650*fa9e4066Sahrens 651*fa9e4066Sahrens if (!(th->th_advance & ADVANCE_NOLOCK)) 652*fa9e4066Sahrens rw_exit(rw); 653*fa9e4066Sahrens 654*fa9e4066Sahrens zseg->seg_mintxg = save_txg; 655*fa9e4066Sahrens 656*fa9e4066Sahrens if (rc == ERANGE) { 657*fa9e4066Sahrens list_remove(&th->th_seglist, zseg); 658*fa9e4066Sahrens kmem_free(zseg, sizeof (*zseg)); 659*fa9e4066Sahrens return (EAGAIN); 660*fa9e4066Sahrens } 661*fa9e4066Sahrens 662*fa9e4066Sahrens return (rc); 663*fa9e4066Sahrens } 664*fa9e4066Sahrens 665*fa9e4066Sahrens /* 666*fa9e4066Sahrens * Note: (mintxg, maxtxg) is an open interval; mintxg and maxtxg themselves 667*fa9e4066Sahrens * are not included. The blocks covered by this segment will all have 668*fa9e4066Sahrens * mintxg < birth < maxtxg. 669*fa9e4066Sahrens */ 670*fa9e4066Sahrens static void 671*fa9e4066Sahrens traverse_add_segment(traverse_handle_t *th, uint64_t mintxg, uint64_t maxtxg, 672*fa9e4066Sahrens uint64_t sobjset, uint64_t sobject, int slevel, uint64_t sblkid, 673*fa9e4066Sahrens uint64_t eobjset, uint64_t eobject, int elevel, uint64_t eblkid) 674*fa9e4066Sahrens { 675*fa9e4066Sahrens zseg_t *zseg; 676*fa9e4066Sahrens 677*fa9e4066Sahrens zseg = kmem_alloc(sizeof (zseg_t), KM_SLEEP); 678*fa9e4066Sahrens 679*fa9e4066Sahrens zseg->seg_mintxg = mintxg; 680*fa9e4066Sahrens zseg->seg_maxtxg = maxtxg; 681*fa9e4066Sahrens 682*fa9e4066Sahrens zseg->seg_start.zb_objset = sobjset; 683*fa9e4066Sahrens zseg->seg_start.zb_object = sobject; 684*fa9e4066Sahrens zseg->seg_start.zb_level = slevel; 685*fa9e4066Sahrens zseg->seg_start.zb_blkid = sblkid; 686*fa9e4066Sahrens 687*fa9e4066Sahrens zseg->seg_end.zb_objset = eobjset; 688*fa9e4066Sahrens zseg->seg_end.zb_object = eobject; 689*fa9e4066Sahrens zseg->seg_end.zb_level = elevel; 690*fa9e4066Sahrens zseg->seg_end.zb_blkid = eblkid; 691*fa9e4066Sahrens 692*fa9e4066Sahrens list_insert_tail(&th->th_seglist, zseg); 693*fa9e4066Sahrens } 694*fa9e4066Sahrens 695*fa9e4066Sahrens void 696*fa9e4066Sahrens traverse_add_dnode(traverse_handle_t *th, uint64_t mintxg, uint64_t maxtxg, 697*fa9e4066Sahrens uint64_t objset, uint64_t object) 698*fa9e4066Sahrens { 699*fa9e4066Sahrens if (th->th_advance & ADVANCE_PRE) 700*fa9e4066Sahrens traverse_add_segment(th, mintxg, maxtxg, 701*fa9e4066Sahrens objset, object, ZB_MAXLEVEL, 0, 702*fa9e4066Sahrens objset, object, 0, ZB_MAXBLKID); 703*fa9e4066Sahrens else 704*fa9e4066Sahrens traverse_add_segment(th, mintxg, maxtxg, 705*fa9e4066Sahrens objset, object, 0, 0, 706*fa9e4066Sahrens objset, object, 0, ZB_MAXBLKID); 707*fa9e4066Sahrens } 708*fa9e4066Sahrens 709*fa9e4066Sahrens void 710*fa9e4066Sahrens traverse_add_objset(traverse_handle_t *th, uint64_t mintxg, uint64_t maxtxg, 711*fa9e4066Sahrens uint64_t objset) 712*fa9e4066Sahrens { 713*fa9e4066Sahrens if (th->th_advance & ADVANCE_PRE) 714*fa9e4066Sahrens traverse_add_segment(th, mintxg, maxtxg, 715*fa9e4066Sahrens objset, 0, -1, 0, 716*fa9e4066Sahrens objset, ZB_MAXOBJECT, 0, ZB_MAXBLKID); 717*fa9e4066Sahrens else 718*fa9e4066Sahrens traverse_add_segment(th, mintxg, maxtxg, 719*fa9e4066Sahrens objset, 1, 0, 0, 720*fa9e4066Sahrens objset, 0, -1, 0); 721*fa9e4066Sahrens } 722*fa9e4066Sahrens 723*fa9e4066Sahrens void 724*fa9e4066Sahrens traverse_add_pool(traverse_handle_t *th, uint64_t mintxg, uint64_t maxtxg) 725*fa9e4066Sahrens { 726*fa9e4066Sahrens if (th->th_advance & ADVANCE_PRE) 727*fa9e4066Sahrens traverse_add_segment(th, mintxg, maxtxg, 728*fa9e4066Sahrens 0, 0, -1, 0, 729*fa9e4066Sahrens ZB_MAXOBJSET, ZB_MAXOBJECT, 0, ZB_MAXBLKID); 730*fa9e4066Sahrens else 731*fa9e4066Sahrens traverse_add_segment(th, mintxg, maxtxg, 732*fa9e4066Sahrens 1, 1, 0, 0, 733*fa9e4066Sahrens 0, 0, -1, 0); 734*fa9e4066Sahrens } 735*fa9e4066Sahrens 736*fa9e4066Sahrens traverse_handle_t * 737*fa9e4066Sahrens traverse_init(spa_t *spa, blkptr_cb_t func, void *arg, int advance, 738*fa9e4066Sahrens int zio_flags) 739*fa9e4066Sahrens { 740*fa9e4066Sahrens traverse_handle_t *th; 741*fa9e4066Sahrens int d, l; 742*fa9e4066Sahrens 743*fa9e4066Sahrens th = kmem_zalloc(sizeof (*th), KM_SLEEP); 744*fa9e4066Sahrens 745*fa9e4066Sahrens th->th_spa = spa; 746*fa9e4066Sahrens th->th_func = func; 747*fa9e4066Sahrens th->th_arg = arg; 748*fa9e4066Sahrens th->th_advance = advance; 749*fa9e4066Sahrens th->th_lastcb.zb_level = ZB_NO_LEVEL; 750*fa9e4066Sahrens th->th_noread.zb_level = ZB_NO_LEVEL; 751*fa9e4066Sahrens th->th_zio_flags = zio_flags; 752*fa9e4066Sahrens 753*fa9e4066Sahrens list_create(&th->th_seglist, sizeof (zseg_t), 754*fa9e4066Sahrens offsetof(zseg_t, seg_node)); 755*fa9e4066Sahrens 756*fa9e4066Sahrens for (d = 0; d < ZB_DEPTH; d++) { 757*fa9e4066Sahrens for (l = 0; l < ZB_MAXLEVEL; l++) { 758*fa9e4066Sahrens if ((advance & ADVANCE_DATA) || 759*fa9e4066Sahrens l != 0 || d != ZB_DN_CACHE) 760*fa9e4066Sahrens th->th_cache[d][l].bc_data = 761*fa9e4066Sahrens zio_buf_alloc(SPA_MAXBLOCKSIZE); 762*fa9e4066Sahrens } 763*fa9e4066Sahrens } 764*fa9e4066Sahrens 765*fa9e4066Sahrens return (th); 766*fa9e4066Sahrens } 767*fa9e4066Sahrens 768*fa9e4066Sahrens void 769*fa9e4066Sahrens traverse_fini(traverse_handle_t *th) 770*fa9e4066Sahrens { 771*fa9e4066Sahrens int d, l; 772*fa9e4066Sahrens zseg_t *zseg; 773*fa9e4066Sahrens 774*fa9e4066Sahrens for (d = 0; d < ZB_DEPTH; d++) 775*fa9e4066Sahrens for (l = 0; l < ZB_MAXLEVEL; l++) 776*fa9e4066Sahrens if (th->th_cache[d][l].bc_data != NULL) 777*fa9e4066Sahrens zio_buf_free(th->th_cache[d][l].bc_data, 778*fa9e4066Sahrens SPA_MAXBLOCKSIZE); 779*fa9e4066Sahrens 780*fa9e4066Sahrens while ((zseg = list_head(&th->th_seglist)) != NULL) { 781*fa9e4066Sahrens list_remove(&th->th_seglist, zseg); 782*fa9e4066Sahrens kmem_free(zseg, sizeof (*zseg)); 783*fa9e4066Sahrens } 784*fa9e4066Sahrens 785*fa9e4066Sahrens list_destroy(&th->th_seglist); 786*fa9e4066Sahrens 787*fa9e4066Sahrens dprintf("%llu hit, %llu ARC, %llu IO, %llu cb, %llu sync, %llu again\n", 788*fa9e4066Sahrens th->th_hits, th->th_arc_hits, th->th_reads, th->th_callbacks, 789*fa9e4066Sahrens th->th_syncs, th->th_restarts); 790*fa9e4066Sahrens 791*fa9e4066Sahrens kmem_free(th, sizeof (*th)); 792*fa9e4066Sahrens } 793