1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/zfs_context.h> 29 #include <sys/dmu_objset.h> 30 #include <sys/dmu_traverse.h> 31 #include <sys/dsl_dataset.h> 32 #include <sys/dsl_dir.h> 33 #include <sys/dsl_pool.h> 34 #include <sys/dnode.h> 35 #include <sys/spa.h> 36 #include <sys/zio.h> 37 #include <sys/dmu_impl.h> 38 39 #define BP_SPAN_SHIFT(level, width) ((level) * (width)) 40 41 #define BP_EQUAL(b1, b2) \ 42 (DVA_EQUAL(BP_IDENTITY(b1), BP_IDENTITY(b2)) && \ 43 (b1)->blk_birth == (b2)->blk_birth) 44 45 /* 46 * Compare two bookmarks. 47 * 48 * For ADVANCE_PRE, the visitation order is: 49 * 50 * objset 0, 1, 2, ..., ZB_MAXOBJSET. 51 * object 0, 1, 2, ..., ZB_MAXOBJECT. 52 * blkoff 0, 1, 2, ... 53 * level ZB_MAXLEVEL, ..., 2, 1, 0. 54 * 55 * where blkoff = blkid << BP_SPAN_SHIFT(level, width), and thus a valid 56 * ordering vector is: 57 * 58 * < objset, object, blkoff, -level > 59 * 60 * For ADVANCE_POST, the starting offsets aren't sequential but ending 61 * offsets [blkoff = (blkid + 1) << BP_SPAN_SHIFT(level, width)] are. 62 * The visitation order is: 63 * 64 * objset 1, 2, ..., ZB_MAXOBJSET, 0. 65 * object 1, 2, ..., ZB_MAXOBJECT, 0. 66 * blkoff 1, 2, ... 67 * level 0, 1, 2, ..., ZB_MAXLEVEL. 68 * 69 * and thus a valid ordering vector is: 70 * 71 * < objset - 1, object - 1, blkoff, level > 72 * 73 * Both orderings can be expressed as: 74 * 75 * < objset + bias, object + bias, blkoff, level ^ bias > 76 * 77 * where 'bias' is either 0 or -1 (for ADVANCE_PRE or ADVANCE_POST) 78 * and 'blkoff' is (blkid - bias) << BP_SPAN_SHIFT(level, wshift). 79 * 80 * Special case: an objset's osphys is represented as level -1 of object 0. 81 * It is always either the very first or very last block we visit in an objset. 82 * Therefore, if either bookmark's level is -1, level alone determines order. 83 */ 84 static int 85 compare_bookmark(zbookmark_t *szb, zbookmark_t *ezb, dnode_phys_t *dnp, 86 int advance) 87 { 88 int bias = (advance & ADVANCE_PRE) ? 0 : -1; 89 uint64_t sblkoff, eblkoff; 90 int slevel, elevel, wshift; 91 92 if (szb->zb_objset + bias < ezb->zb_objset + bias) 93 return (-1); 94 95 if (szb->zb_objset + bias > ezb->zb_objset + bias) 96 return (1); 97 98 slevel = szb->zb_level; 99 elevel = ezb->zb_level; 100 101 if ((slevel | elevel) < 0) 102 return ((slevel ^ bias) - (elevel ^ bias)); 103 104 if (szb->zb_object + bias < ezb->zb_object + bias) 105 return (-1); 106 107 if (szb->zb_object + bias > ezb->zb_object + bias) 108 return (1); 109 110 if (dnp == NULL) 111 return (0); 112 113 wshift = dnp->dn_indblkshift - SPA_BLKPTRSHIFT; 114 115 sblkoff = (szb->zb_blkid - bias) << BP_SPAN_SHIFT(slevel, wshift); 116 eblkoff = (ezb->zb_blkid - bias) << BP_SPAN_SHIFT(elevel, wshift); 117 118 if (sblkoff < eblkoff) 119 return (-1); 120 121 if (sblkoff > eblkoff) 122 return (1); 123 124 return ((elevel ^ bias) - (slevel ^ bias)); 125 } 126 127 #define SET_BOOKMARK(zb, objset, object, level, blkid) \ 128 { \ 129 (zb)->zb_objset = objset; \ 130 (zb)->zb_object = object; \ 131 (zb)->zb_level = level; \ 132 (zb)->zb_blkid = blkid; \ 133 } 134 135 #define SET_BOOKMARK_LB(zb, level, blkid) \ 136 { \ 137 (zb)->zb_level = level; \ 138 (zb)->zb_blkid = blkid; \ 139 } 140 141 static int 142 advance_objset(zseg_t *zseg, uint64_t objset, int advance) 143 { 144 zbookmark_t *zb = &zseg->seg_start; 145 146 if (advance & ADVANCE_PRE) { 147 if (objset >= ZB_MAXOBJSET) 148 return (ERANGE); 149 SET_BOOKMARK(zb, objset, 0, -1, 0); 150 } else { 151 if (objset >= ZB_MAXOBJSET) 152 objset = 0; 153 SET_BOOKMARK(zb, objset, 1, 0, 0); 154 } 155 156 if (compare_bookmark(zb, &zseg->seg_end, NULL, advance) > 0) 157 return (ERANGE); 158 159 return (EAGAIN); 160 } 161 162 static int 163 advance_object(zseg_t *zseg, uint64_t object, int advance) 164 { 165 zbookmark_t *zb = &zseg->seg_start; 166 167 if (advance & ADVANCE_PRE) { 168 if (object >= ZB_MAXOBJECT) { 169 SET_BOOKMARK(zb, zb->zb_objset + 1, 0, -1, 0); 170 } else { 171 SET_BOOKMARK(zb, zb->zb_objset, object, ZB_MAXLEVEL, 0); 172 } 173 } else { 174 if (zb->zb_object == 0) { 175 SET_BOOKMARK(zb, zb->zb_objset, 0, -1, 0); 176 } else { 177 if (object >= ZB_MAXOBJECT) 178 object = 0; 179 SET_BOOKMARK(zb, zb->zb_objset, object, 0, 0); 180 } 181 } 182 183 if (compare_bookmark(zb, &zseg->seg_end, NULL, advance) > 0) 184 return (ERANGE); 185 186 return (EAGAIN); 187 } 188 189 static int 190 advance_from_osphys(zseg_t *zseg, int advance) 191 { 192 zbookmark_t *zb = &zseg->seg_start; 193 194 ASSERT(zb->zb_object == 0); 195 ASSERT(zb->zb_level == -1); 196 ASSERT(zb->zb_blkid == 0); 197 198 if (advance & ADVANCE_PRE) { 199 SET_BOOKMARK_LB(zb, ZB_MAXLEVEL, 0); 200 } else { 201 if (zb->zb_objset == 0) 202 return (ERANGE); 203 SET_BOOKMARK(zb, zb->zb_objset + 1, 1, 0, 0); 204 } 205 206 if (compare_bookmark(zb, &zseg->seg_end, NULL, advance) > 0) 207 return (ERANGE); 208 209 return (EAGAIN); 210 } 211 212 static int 213 advance_block(zseg_t *zseg, dnode_phys_t *dnp, int rc, int advance) 214 { 215 zbookmark_t *zb = &zseg->seg_start; 216 int wshift = dnp->dn_indblkshift - SPA_BLKPTRSHIFT; 217 int maxlevel = dnp->dn_nlevels - 1; 218 int level = zb->zb_level; 219 uint64_t blkid = zb->zb_blkid; 220 221 if (advance & ADVANCE_PRE) { 222 if (level > 0 && rc == 0) { 223 level--; 224 blkid <<= wshift; 225 } else { 226 blkid++; 227 228 if ((blkid << BP_SPAN_SHIFT(level, wshift)) > 229 dnp->dn_maxblkid) 230 return (ERANGE); 231 232 while (level < maxlevel) { 233 if (P2PHASE(blkid, 1ULL << wshift)) 234 break; 235 blkid >>= wshift; 236 level++; 237 } 238 } 239 } else { 240 if (level >= maxlevel || P2PHASE(blkid + 1, 1ULL << wshift)) { 241 blkid = (blkid + 1) << BP_SPAN_SHIFT(level, wshift); 242 level = 0; 243 } else { 244 blkid >>= wshift; 245 level++; 246 } 247 248 while ((blkid << BP_SPAN_SHIFT(level, wshift)) > 249 dnp->dn_maxblkid) { 250 if (level == maxlevel) 251 return (ERANGE); 252 blkid >>= wshift; 253 level++; 254 } 255 } 256 SET_BOOKMARK_LB(zb, level, blkid); 257 258 if (compare_bookmark(zb, &zseg->seg_end, dnp, advance) > 0) 259 return (ERANGE); 260 261 return (EAGAIN); 262 } 263 264 static int 265 traverse_callback(traverse_handle_t *th, zseg_t *zseg, traverse_blk_cache_t *bc) 266 { 267 /* 268 * Before we issue the callback, prune against maxtxg. 269 * 270 * We prune against mintxg before we get here because it's a big win. 271 * If a given block was born in txg 37, then we know that the entire 272 * subtree below that block must have been born in txg 37 or earlier. 273 * We can therefore lop off huge branches of the tree as we go. 274 * 275 * There's no corresponding optimization for maxtxg because knowing 276 * that bp->blk_birth >= maxtxg doesn't imply anything about the bp's 277 * children. In fact, the copy-on-write design of ZFS ensures that 278 * top-level blocks will pretty much always be new. 279 * 280 * Therefore, in the name of simplicity we don't prune against 281 * maxtxg until the last possible moment -- that being right now. 282 */ 283 if (bc->bc_errno == 0 && bc->bc_blkptr.blk_birth >= zseg->seg_maxtxg) 284 return (0); 285 286 if (bc->bc_errno == 0) { 287 zbookmark_t *zb = &bc->bc_bookmark; 288 zbookmark_t *szb = &zseg->seg_start; 289 zbookmark_t *ezb = &zseg->seg_end; 290 zbookmark_t *lzb = &th->th_lastcb; 291 dnode_phys_t *dnp = bc->bc_dnode; 292 293 /* 294 * Debugging: verify that the order we visit things 295 * agrees with the order defined by compare_bookmark(). 296 */ 297 ASSERT(compare_bookmark(zb, ezb, dnp, th->th_advance) <= 0); 298 ASSERT(compare_bookmark(zb, szb, dnp, th->th_advance) == 0); 299 ASSERT(compare_bookmark(lzb, zb, dnp, th->th_advance) < 0 || 300 lzb->zb_level == ZB_NO_LEVEL); 301 *lzb = *zb; 302 } 303 304 th->th_callbacks++; 305 return (th->th_func(bc, th->th_spa, th->th_arg)); 306 } 307 308 static int 309 traverse_read(traverse_handle_t *th, traverse_blk_cache_t *bc, blkptr_t *bp, 310 dnode_phys_t *dnp) 311 { 312 zbookmark_t *zb = &bc->bc_bookmark; 313 int error; 314 315 th->th_hits++; 316 317 bc->bc_dnode = dnp; 318 bc->bc_errno = 0; 319 320 if (BP_EQUAL(&bc->bc_blkptr, bp)) 321 return (0); 322 323 bc->bc_blkptr = *bp; 324 325 if (bc->bc_data == NULL) 326 return (0); 327 328 if (BP_IS_HOLE(bp)) { 329 ASSERT(th->th_advance & ADVANCE_HOLES); 330 return (0); 331 } 332 333 if (compare_bookmark(zb, &th->th_noread, dnp, 0) == 0) { 334 error = EIO; 335 } else if (arc_tryread(th->th_spa, bp, bc->bc_data) == 0) { 336 error = 0; 337 th->th_arc_hits++; 338 } else { 339 error = zio_wait(zio_read(NULL, th->th_spa, bp, bc->bc_data, 340 BP_GET_LSIZE(bp), NULL, NULL, ZIO_PRIORITY_SYNC_READ, 341 th->th_zio_flags | ZIO_FLAG_DONT_CACHE, zb)); 342 343 if (BP_SHOULD_BYTESWAP(bp) && error == 0) 344 (zb->zb_level > 0 ? byteswap_uint64_array : 345 dmu_ot[BP_GET_TYPE(bp)].ot_byteswap)(bc->bc_data, 346 BP_GET_LSIZE(bp)); 347 th->th_reads++; 348 } 349 350 if (error) { 351 bc->bc_errno = error; 352 error = traverse_callback(th, NULL, bc); 353 ASSERT(error == EAGAIN || error == EINTR || error == ERESTART); 354 bc->bc_blkptr.blk_birth = -1ULL; 355 } 356 357 dprintf("cache %02x error %d <%llu, %llu, %d, %llx>\n", 358 bc - &th->th_cache[0][0], error, 359 zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid); 360 361 return (error); 362 } 363 364 static int 365 find_block(traverse_handle_t *th, zseg_t *zseg, dnode_phys_t *dnp, int depth) 366 { 367 zbookmark_t *zb = &zseg->seg_start; 368 traverse_blk_cache_t *bc; 369 blkptr_t *bp = dnp->dn_blkptr; 370 int i, first, level; 371 int nbp = dnp->dn_nblkptr; 372 int minlevel = zb->zb_level; 373 int maxlevel = dnp->dn_nlevels - 1; 374 int wshift = dnp->dn_indblkshift - SPA_BLKPTRSHIFT; 375 int bp_shift = BP_SPAN_SHIFT(maxlevel - minlevel, wshift); 376 uint64_t blkid = zb->zb_blkid >> bp_shift; 377 int do_holes = (th->th_advance & ADVANCE_HOLES) && depth == ZB_DN_CACHE; 378 int rc; 379 380 if (minlevel > maxlevel || blkid >= nbp) 381 return (ERANGE); 382 383 for (level = maxlevel; level >= minlevel; level--) { 384 first = P2PHASE(blkid, 1ULL << wshift); 385 386 for (i = first; i < nbp; i++) 387 if (bp[i].blk_birth > zseg->seg_mintxg || 388 BP_IS_HOLE(&bp[i]) && do_holes) 389 break; 390 391 if (i != first) { 392 i--; 393 SET_BOOKMARK_LB(zb, level, blkid + (i - first)); 394 return (ENOTBLK); 395 } 396 397 bc = &th->th_cache[depth][level]; 398 399 SET_BOOKMARK(&bc->bc_bookmark, zb->zb_objset, zb->zb_object, 400 level, blkid); 401 402 if (rc = traverse_read(th, bc, bp + i, dnp)) { 403 if (rc != EAGAIN) { 404 SET_BOOKMARK_LB(zb, level, blkid); 405 } 406 return (rc); 407 } 408 409 if (BP_IS_HOLE(&bp[i])) { 410 SET_BOOKMARK_LB(zb, level, blkid); 411 th->th_lastcb.zb_level = ZB_NO_LEVEL; 412 return (0); 413 } 414 415 nbp = 1 << wshift; 416 bp = bc->bc_data; 417 bp_shift -= wshift; 418 blkid = zb->zb_blkid >> bp_shift; 419 } 420 421 return (0); 422 } 423 424 static int 425 get_dnode(traverse_handle_t *th, uint64_t objset, dnode_phys_t *mdn, 426 uint64_t *objectp, dnode_phys_t **dnpp, uint64_t txg, int type, int depth) 427 { 428 zseg_t zseg; 429 zbookmark_t *zb = &zseg.seg_start; 430 uint64_t object = *objectp; 431 int i, rc; 432 433 SET_BOOKMARK(zb, objset, 0, 0, object / DNODES_PER_BLOCK); 434 SET_BOOKMARK(&zseg.seg_end, objset, 0, 0, ZB_MAXBLKID); 435 436 zseg.seg_mintxg = txg; 437 zseg.seg_maxtxg = -1ULL; 438 439 for (;;) { 440 rc = find_block(th, &zseg, mdn, depth); 441 442 if (rc == EAGAIN || rc == EINTR || rc == ERANGE) 443 break; 444 445 if (rc == 0 && zb->zb_level == 0) { 446 dnode_phys_t *dnp = th->th_cache[depth][0].bc_data; 447 for (i = 0; i < DNODES_PER_BLOCK; i++) { 448 object = (zb->zb_blkid * DNODES_PER_BLOCK) + i; 449 if (object >= *objectp && 450 dnp[i].dn_type != DMU_OT_NONE && 451 (type == -1 || dnp[i].dn_type == type)) { 452 *objectp = object; 453 *dnpp = &dnp[i]; 454 return (0); 455 } 456 } 457 } 458 459 rc = advance_block(&zseg, mdn, rc, ADVANCE_PRE); 460 461 if (rc == ERANGE) 462 break; 463 } 464 465 if (rc == ERANGE) 466 *objectp = ZB_MAXOBJECT; 467 468 return (rc); 469 } 470 471 /* ARGSUSED */ 472 static void 473 traverse_zil_block(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t maxtxg) 474 { 475 traverse_handle_t *th = arg; 476 traverse_blk_cache_t *bc = &th->th_zil_cache; 477 zbookmark_t *zb = &bc->bc_bookmark; 478 479 if (bp->blk_birth < maxtxg) { 480 zb->zb_object = 0; 481 zb->zb_blkid = bp->blk_cksum.zc_word[3]; 482 bc->bc_blkptr = *bp; 483 (void) th->th_func(bc, th->th_spa, th->th_arg); 484 } 485 } 486 487 /* ARGSUSED */ 488 static void 489 traverse_zil_record(zilog_t *zilog, lr_t *lrc, void *arg, uint64_t maxtxg) 490 { 491 traverse_handle_t *th = arg; 492 traverse_blk_cache_t *bc = &th->th_zil_cache; 493 zbookmark_t *zb = &bc->bc_bookmark; 494 495 if (lrc->lrc_txtype == TX_WRITE) { 496 lr_write_t *lr = (lr_write_t *)lrc; 497 blkptr_t *bp = &lr->lr_blkptr; 498 499 if (bp->blk_birth != 0 && bp->blk_birth < maxtxg) { 500 zb->zb_object = lr->lr_foid; 501 zb->zb_blkid = lr->lr_offset / BP_GET_LSIZE(bp); 502 bc->bc_blkptr = *bp; 503 (void) th->th_func(bc, th->th_spa, th->th_arg); 504 } 505 } 506 } 507 508 static void 509 traverse_zil(traverse_handle_t *th, traverse_blk_cache_t *bc, uint64_t maxtxg) 510 { 511 spa_t *spa = th->th_spa; 512 objset_phys_t *osphys = bc->bc_data; 513 dsl_pool_t *dp = spa_get_dsl(spa); 514 zilog_t *zilog; 515 516 ASSERT(bc == &th->th_cache[ZB_MDN_CACHE][ZB_MAXLEVEL - 1]); 517 ASSERT(bc->bc_bookmark.zb_level == -1); 518 519 th->th_zil_cache.bc_bookmark = bc->bc_bookmark; 520 521 zilog = zil_alloc(dp->dp_meta_objset, &osphys->os_zil_header); 522 523 zil_parse(zilog, traverse_zil_block, traverse_zil_record, th, maxtxg); 524 525 zil_free(zilog); 526 } 527 528 static int 529 traverse_segment(traverse_handle_t *th, zseg_t *zseg, blkptr_t *mosbp) 530 { 531 zbookmark_t *zb = &zseg->seg_start; 532 traverse_blk_cache_t *bc; 533 dnode_phys_t *dn, *dn_tmp; 534 int worklimit = 100; 535 int rc; 536 537 dprintf("<%llu, %llu, %d, %llx>\n", 538 zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid); 539 540 bc = &th->th_cache[ZB_MOS_CACHE][ZB_MAXLEVEL - 1]; 541 dn = &((objset_phys_t *)bc->bc_data)->os_meta_dnode; 542 543 SET_BOOKMARK(&bc->bc_bookmark, 0, 0, -1, 0); 544 545 rc = traverse_read(th, bc, mosbp, dn); 546 547 if (rc) /* If we get ERESTART, we've got nowhere left to go */ 548 return (rc == ERESTART ? EINTR : rc); 549 550 ASSERT(dn->dn_nlevels < ZB_MAXLEVEL); 551 552 if (zb->zb_objset != 0) { 553 uint64_t objset = zb->zb_objset; 554 dsl_dataset_phys_t *dsp; 555 556 rc = get_dnode(th, 0, dn, &objset, &dn_tmp, 0, 557 DMU_OT_DSL_DATASET, ZB_MOS_CACHE); 558 559 if (objset != zb->zb_objset) 560 rc = advance_objset(zseg, objset, th->th_advance); 561 562 if (rc != 0) 563 return (rc); 564 565 dsp = DN_BONUS(dn_tmp); 566 567 bc = &th->th_cache[ZB_MDN_CACHE][ZB_MAXLEVEL - 1]; 568 dn = &((objset_phys_t *)bc->bc_data)->os_meta_dnode; 569 570 SET_BOOKMARK(&bc->bc_bookmark, objset, 0, -1, 0); 571 572 rc = traverse_read(th, bc, &dsp->ds_bp, dn); 573 574 if (rc != 0) { 575 if (rc == ERESTART) 576 rc = advance_objset(zseg, zb->zb_objset + 1, 577 th->th_advance); 578 return (rc); 579 } 580 581 if (th->th_advance & ADVANCE_PRUNE) 582 zseg->seg_mintxg = 583 MAX(zseg->seg_mintxg, dsp->ds_prev_snap_txg); 584 } 585 586 if (zb->zb_level == -1) { 587 ASSERT(zb->zb_object == 0); 588 ASSERT(zb->zb_blkid == 0); 589 ASSERT(BP_GET_TYPE(&bc->bc_blkptr) == DMU_OT_OBJSET); 590 591 if (bc->bc_blkptr.blk_birth > zseg->seg_mintxg) { 592 rc = traverse_callback(th, zseg, bc); 593 if (rc) { 594 ASSERT(rc == EINTR); 595 return (rc); 596 } 597 if ((th->th_advance & ADVANCE_ZIL) && 598 zb->zb_objset != 0) 599 traverse_zil(th, bc, zseg->seg_maxtxg); 600 } 601 602 return (advance_from_osphys(zseg, th->th_advance)); 603 } 604 605 if (zb->zb_object != 0) { 606 uint64_t object = zb->zb_object; 607 608 rc = get_dnode(th, zb->zb_objset, dn, &object, &dn_tmp, 609 zseg->seg_mintxg, -1, ZB_MDN_CACHE); 610 611 if (object != zb->zb_object) 612 rc = advance_object(zseg, object, th->th_advance); 613 614 if (rc != 0) 615 return (rc); 616 617 dn = dn_tmp; 618 } 619 620 if (zb->zb_level == ZB_MAXLEVEL) 621 zb->zb_level = dn->dn_nlevels - 1; 622 623 for (;;) { 624 rc = find_block(th, zseg, dn, ZB_DN_CACHE); 625 626 if (rc == EAGAIN || rc == EINTR || rc == ERANGE) 627 break; 628 629 if (rc == 0) { 630 bc = &th->th_cache[ZB_DN_CACHE][zb->zb_level]; 631 ASSERT(bc->bc_dnode == dn); 632 ASSERT(bc->bc_blkptr.blk_birth <= mosbp->blk_birth); 633 rc = traverse_callback(th, zseg, bc); 634 if (rc) { 635 ASSERT(rc == EINTR); 636 return (rc); 637 } 638 if (BP_IS_HOLE(&bc->bc_blkptr)) { 639 ASSERT(th->th_advance & ADVANCE_HOLES); 640 rc = ENOTBLK; 641 } 642 } 643 644 rc = advance_block(zseg, dn, rc, th->th_advance); 645 646 if (rc == ERANGE) 647 break; 648 649 /* 650 * Give spa_sync() a chance to run. 651 */ 652 if (spa_traverse_wanted(th->th_spa)) { 653 th->th_syncs++; 654 return (EAGAIN); 655 } 656 657 if (--worklimit == 0) 658 return (EAGAIN); 659 } 660 661 if (rc == ERANGE) 662 rc = advance_object(zseg, zb->zb_object + 1, th->th_advance); 663 664 return (rc); 665 } 666 667 /* 668 * It is the caller's responsibility to ensure that the dsl_dataset_t 669 * doesn't go away during traversal. 670 */ 671 int 672 traverse_dsl_dataset(dsl_dataset_t *ds, uint64_t txg_start, int advance, 673 blkptr_cb_t func, void *arg) 674 { 675 spa_t *spa = ds->ds_dir->dd_pool->dp_spa; 676 traverse_handle_t *th; 677 int err; 678 679 th = traverse_init(spa, func, arg, advance, ZIO_FLAG_MUSTSUCCEED); 680 681 traverse_add_objset(th, txg_start, -1ULL, ds->ds_object); 682 683 while ((err = traverse_more(th)) == EAGAIN) 684 continue; 685 686 traverse_fini(th); 687 return (err); 688 } 689 690 int 691 traverse_more(traverse_handle_t *th) 692 { 693 zseg_t *zseg = list_head(&th->th_seglist); 694 uint64_t save_txg; /* XXX won't be necessary with real itinerary */ 695 krwlock_t *rw = spa_traverse_rwlock(th->th_spa); 696 blkptr_t *mosbp = spa_get_rootblkptr(th->th_spa); 697 int rc; 698 699 if (zseg == NULL) 700 return (0); 701 702 th->th_restarts++; 703 704 save_txg = zseg->seg_mintxg; 705 706 if (!(th->th_advance & ADVANCE_NOLOCK)) 707 rw_enter(rw, RW_READER); 708 709 rc = traverse_segment(th, zseg, mosbp); 710 ASSERT(rc == ERANGE || rc == EAGAIN || rc == EINTR); 711 712 if (!(th->th_advance & ADVANCE_NOLOCK)) 713 rw_exit(rw); 714 715 zseg->seg_mintxg = save_txg; 716 717 if (rc == ERANGE) { 718 list_remove(&th->th_seglist, zseg); 719 kmem_free(zseg, sizeof (*zseg)); 720 return (EAGAIN); 721 } 722 723 return (rc); 724 } 725 726 /* 727 * Note: (mintxg, maxtxg) is an open interval; mintxg and maxtxg themselves 728 * are not included. The blocks covered by this segment will all have 729 * mintxg < birth < maxtxg. 730 */ 731 static void 732 traverse_add_segment(traverse_handle_t *th, uint64_t mintxg, uint64_t maxtxg, 733 uint64_t sobjset, uint64_t sobject, int slevel, uint64_t sblkid, 734 uint64_t eobjset, uint64_t eobject, int elevel, uint64_t eblkid) 735 { 736 zseg_t *zseg; 737 738 zseg = kmem_alloc(sizeof (zseg_t), KM_SLEEP); 739 740 zseg->seg_mintxg = mintxg; 741 zseg->seg_maxtxg = maxtxg; 742 743 zseg->seg_start.zb_objset = sobjset; 744 zseg->seg_start.zb_object = sobject; 745 zseg->seg_start.zb_level = slevel; 746 zseg->seg_start.zb_blkid = sblkid; 747 748 zseg->seg_end.zb_objset = eobjset; 749 zseg->seg_end.zb_object = eobject; 750 zseg->seg_end.zb_level = elevel; 751 zseg->seg_end.zb_blkid = eblkid; 752 753 list_insert_tail(&th->th_seglist, zseg); 754 } 755 756 void 757 traverse_add_dnode(traverse_handle_t *th, uint64_t mintxg, uint64_t maxtxg, 758 uint64_t objset, uint64_t object) 759 { 760 if (th->th_advance & ADVANCE_PRE) 761 traverse_add_segment(th, mintxg, maxtxg, 762 objset, object, ZB_MAXLEVEL, 0, 763 objset, object, 0, ZB_MAXBLKID); 764 else 765 traverse_add_segment(th, mintxg, maxtxg, 766 objset, object, 0, 0, 767 objset, object, 0, ZB_MAXBLKID); 768 } 769 770 void 771 traverse_add_objset(traverse_handle_t *th, uint64_t mintxg, uint64_t maxtxg, 772 uint64_t objset) 773 { 774 if (th->th_advance & ADVANCE_PRE) 775 traverse_add_segment(th, mintxg, maxtxg, 776 objset, 0, -1, 0, 777 objset, ZB_MAXOBJECT, 0, ZB_MAXBLKID); 778 else 779 traverse_add_segment(th, mintxg, maxtxg, 780 objset, 1, 0, 0, 781 objset, 0, -1, 0); 782 } 783 784 void 785 traverse_add_pool(traverse_handle_t *th, uint64_t mintxg, uint64_t maxtxg) 786 { 787 if (th->th_advance & ADVANCE_PRE) 788 traverse_add_segment(th, mintxg, maxtxg, 789 0, 0, -1, 0, 790 ZB_MAXOBJSET, ZB_MAXOBJECT, 0, ZB_MAXBLKID); 791 else 792 traverse_add_segment(th, mintxg, maxtxg, 793 1, 1, 0, 0, 794 0, 0, -1, 0); 795 } 796 797 traverse_handle_t * 798 traverse_init(spa_t *spa, blkptr_cb_t func, void *arg, int advance, 799 int zio_flags) 800 { 801 traverse_handle_t *th; 802 int d, l; 803 804 th = kmem_zalloc(sizeof (*th), KM_SLEEP); 805 806 th->th_spa = spa; 807 th->th_func = func; 808 th->th_arg = arg; 809 th->th_advance = advance; 810 th->th_lastcb.zb_level = ZB_NO_LEVEL; 811 th->th_noread.zb_level = ZB_NO_LEVEL; 812 th->th_zio_flags = zio_flags; 813 814 list_create(&th->th_seglist, sizeof (zseg_t), 815 offsetof(zseg_t, seg_node)); 816 817 for (d = 0; d < ZB_DEPTH; d++) { 818 for (l = 0; l < ZB_MAXLEVEL; l++) { 819 if ((advance & ADVANCE_DATA) || 820 l != 0 || d != ZB_DN_CACHE) 821 th->th_cache[d][l].bc_data = 822 zio_buf_alloc(SPA_MAXBLOCKSIZE); 823 } 824 } 825 826 return (th); 827 } 828 829 void 830 traverse_fini(traverse_handle_t *th) 831 { 832 int d, l; 833 zseg_t *zseg; 834 835 for (d = 0; d < ZB_DEPTH; d++) 836 for (l = 0; l < ZB_MAXLEVEL; l++) 837 if (th->th_cache[d][l].bc_data != NULL) 838 zio_buf_free(th->th_cache[d][l].bc_data, 839 SPA_MAXBLOCKSIZE); 840 841 while ((zseg = list_head(&th->th_seglist)) != NULL) { 842 list_remove(&th->th_seglist, zseg); 843 kmem_free(zseg, sizeof (*zseg)); 844 } 845 846 list_destroy(&th->th_seglist); 847 848 dprintf("%llu hit, %llu ARC, %llu IO, %llu cb, %llu sync, %llu again\n", 849 th->th_hits, th->th_arc_hits, th->th_reads, th->th_callbacks, 850 th->th_syncs, th->th_restarts); 851 852 kmem_free(th, sizeof (*th)); 853 } 854