1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 29 /* 30 * This file contains the top half of the zfs directory structure 31 * implementation. The bottom half is in zap_leaf.c. 32 * 33 * The zdir is an extendable hash data structure. There is a table of 34 * pointers to buckets (zap_t->zd_data->zd_leafs). The buckets are 35 * each a constant size and hold a variable number of directory entries. 36 * The buckets (aka "leaf nodes") are implemented in zap_leaf.c. 37 * 38 * The pointer table holds a power of 2 number of pointers. 39 * (1<<zap_t->zd_data->zd_phys->zd_prefix_len). The bucket pointed to 40 * by the pointer at index i in the table holds entries whose hash value 41 * has a zd_prefix_len - bit prefix 42 */ 43 44 #include <sys/spa.h> 45 #include <sys/dmu.h> 46 #include <sys/zfs_context.h> 47 #include <sys/zap.h> 48 #include <sys/refcount.h> 49 #include <sys/zap_impl.h> 50 #include <sys/zap_leaf.h> 51 52 int fzap_default_block_shift = 14; /* 16k blocksize */ 53 54 static void zap_leaf_pageout(dmu_buf_t *db, void *vl); 55 static uint64_t zap_allocate_blocks(zap_t *zap, int nblocks); 56 57 58 void 59 fzap_byteswap(void *vbuf, size_t size) 60 { 61 uint64_t block_type; 62 63 block_type = *(uint64_t *)vbuf; 64 65 if (block_type == ZBT_LEAF || block_type == BSWAP_64(ZBT_LEAF)) 66 zap_leaf_byteswap(vbuf, size); 67 else { 68 /* it's a ptrtbl block */ 69 byteswap_uint64_array(vbuf, size); 70 } 71 } 72 73 void 74 fzap_upgrade(zap_t *zap, dmu_tx_t *tx) 75 { 76 dmu_buf_t *db; 77 zap_leaf_t *l; 78 int i; 79 zap_phys_t *zp; 80 81 ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); 82 zap->zap_ismicro = FALSE; 83 84 (void) dmu_buf_update_user(zap->zap_dbuf, zap, zap, 85 &zap->zap_f.zap_phys, zap_evict); 86 87 mutex_init(&zap->zap_f.zap_num_entries_mtx, 0, 0, 0); 88 zap->zap_f.zap_block_shift = highbit(zap->zap_dbuf->db_size) - 1; 89 90 zp = zap->zap_f.zap_phys; 91 /* 92 * explicitly zero it since it might be coming from an 93 * initialized microzap 94 */ 95 bzero(zap->zap_dbuf->db_data, zap->zap_dbuf->db_size); 96 zp->zap_block_type = ZBT_HEADER; 97 zp->zap_magic = ZAP_MAGIC; 98 99 zp->zap_ptrtbl.zt_shift = ZAP_EMBEDDED_PTRTBL_SHIFT(zap); 100 101 zp->zap_freeblk = 2; /* block 1 will be the first leaf */ 102 zp->zap_num_leafs = 1; 103 zp->zap_num_entries = 0; 104 zp->zap_salt = zap->zap_salt; 105 zp->zap_normflags = zap->zap_normflags; 106 107 /* block 1 will be the first leaf */ 108 for (i = 0; i < (1<<zp->zap_ptrtbl.zt_shift); i++) 109 ZAP_EMBEDDED_PTRTBL_ENT(zap, i) = 1; 110 111 /* 112 * set up block 1 - the first leaf 113 */ 114 VERIFY(0 == dmu_buf_hold(zap->zap_objset, zap->zap_object, 115 1<<FZAP_BLOCK_SHIFT(zap), FTAG, &db)); 116 dmu_buf_will_dirty(db, tx); 117 118 l = kmem_zalloc(sizeof (zap_leaf_t), KM_SLEEP); 119 l->l_dbuf = db; 120 l->l_phys = db->db_data; 121 122 zap_leaf_init(l, spa_version(dmu_objset_spa(zap->zap_objset))); 123 124 kmem_free(l, sizeof (zap_leaf_t)); 125 dmu_buf_rele(db, FTAG); 126 } 127 128 static int 129 zap_tryupgradedir(zap_t *zap, dmu_tx_t *tx) 130 { 131 if (RW_WRITE_HELD(&zap->zap_rwlock)) 132 return (1); 133 if (rw_tryupgrade(&zap->zap_rwlock)) { 134 dmu_buf_will_dirty(zap->zap_dbuf, tx); 135 return (1); 136 } 137 return (0); 138 } 139 140 /* 141 * Generic routines for dealing with the pointer & cookie tables. 142 */ 143 144 static int 145 zap_table_grow(zap_t *zap, zap_table_phys_t *tbl, 146 void (*transfer_func)(const uint64_t *src, uint64_t *dst, int n), 147 dmu_tx_t *tx) 148 { 149 uint64_t b, newblk; 150 dmu_buf_t *db_old, *db_new; 151 int err; 152 int bs = FZAP_BLOCK_SHIFT(zap); 153 int hepb = 1<<(bs-4); 154 /* hepb = half the number of entries in a block */ 155 156 ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); 157 ASSERT(tbl->zt_blk != 0); 158 ASSERT(tbl->zt_numblks > 0); 159 160 if (tbl->zt_nextblk != 0) { 161 newblk = tbl->zt_nextblk; 162 } else { 163 newblk = zap_allocate_blocks(zap, tbl->zt_numblks * 2); 164 tbl->zt_nextblk = newblk; 165 ASSERT3U(tbl->zt_blks_copied, ==, 0); 166 dmu_prefetch(zap->zap_objset, zap->zap_object, 167 tbl->zt_blk << bs, tbl->zt_numblks << bs); 168 } 169 170 /* 171 * Copy the ptrtbl from the old to new location. 172 */ 173 174 b = tbl->zt_blks_copied; 175 err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 176 (tbl->zt_blk + b) << bs, FTAG, &db_old); 177 if (err) 178 return (err); 179 180 /* first half of entries in old[b] go to new[2*b+0] */ 181 VERIFY(0 == dmu_buf_hold(zap->zap_objset, zap->zap_object, 182 (newblk + 2*b+0) << bs, FTAG, &db_new)); 183 dmu_buf_will_dirty(db_new, tx); 184 transfer_func(db_old->db_data, db_new->db_data, hepb); 185 dmu_buf_rele(db_new, FTAG); 186 187 /* second half of entries in old[b] go to new[2*b+1] */ 188 VERIFY(0 == dmu_buf_hold(zap->zap_objset, zap->zap_object, 189 (newblk + 2*b+1) << bs, FTAG, &db_new)); 190 dmu_buf_will_dirty(db_new, tx); 191 transfer_func((uint64_t *)db_old->db_data + hepb, 192 db_new->db_data, hepb); 193 dmu_buf_rele(db_new, FTAG); 194 195 dmu_buf_rele(db_old, FTAG); 196 197 tbl->zt_blks_copied++; 198 199 dprintf("copied block %llu of %llu\n", 200 tbl->zt_blks_copied, tbl->zt_numblks); 201 202 if (tbl->zt_blks_copied == tbl->zt_numblks) { 203 (void) dmu_free_range(zap->zap_objset, zap->zap_object, 204 tbl->zt_blk << bs, tbl->zt_numblks << bs, tx); 205 206 tbl->zt_blk = newblk; 207 tbl->zt_numblks *= 2; 208 tbl->zt_shift++; 209 tbl->zt_nextblk = 0; 210 tbl->zt_blks_copied = 0; 211 212 dprintf("finished; numblocks now %llu (%lluk entries)\n", 213 tbl->zt_numblks, 1<<(tbl->zt_shift-10)); 214 } 215 216 return (0); 217 } 218 219 static int 220 zap_table_store(zap_t *zap, zap_table_phys_t *tbl, uint64_t idx, uint64_t val, 221 dmu_tx_t *tx) 222 { 223 int err; 224 uint64_t blk, off; 225 int bs = FZAP_BLOCK_SHIFT(zap); 226 dmu_buf_t *db; 227 228 ASSERT(RW_LOCK_HELD(&zap->zap_rwlock)); 229 ASSERT(tbl->zt_blk != 0); 230 231 dprintf("storing %llx at index %llx\n", val, idx); 232 233 blk = idx >> (bs-3); 234 off = idx & ((1<<(bs-3))-1); 235 236 err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 237 (tbl->zt_blk + blk) << bs, FTAG, &db); 238 if (err) 239 return (err); 240 dmu_buf_will_dirty(db, tx); 241 242 if (tbl->zt_nextblk != 0) { 243 uint64_t idx2 = idx * 2; 244 uint64_t blk2 = idx2 >> (bs-3); 245 uint64_t off2 = idx2 & ((1<<(bs-3))-1); 246 dmu_buf_t *db2; 247 248 err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 249 (tbl->zt_nextblk + blk2) << bs, FTAG, &db2); 250 if (err) { 251 dmu_buf_rele(db, FTAG); 252 return (err); 253 } 254 dmu_buf_will_dirty(db2, tx); 255 ((uint64_t *)db2->db_data)[off2] = val; 256 ((uint64_t *)db2->db_data)[off2+1] = val; 257 dmu_buf_rele(db2, FTAG); 258 } 259 260 ((uint64_t *)db->db_data)[off] = val; 261 dmu_buf_rele(db, FTAG); 262 263 return (0); 264 } 265 266 static int 267 zap_table_load(zap_t *zap, zap_table_phys_t *tbl, uint64_t idx, uint64_t *valp) 268 { 269 uint64_t blk, off; 270 int err; 271 dmu_buf_t *db; 272 int bs = FZAP_BLOCK_SHIFT(zap); 273 274 ASSERT(RW_LOCK_HELD(&zap->zap_rwlock)); 275 276 blk = idx >> (bs-3); 277 off = idx & ((1<<(bs-3))-1); 278 279 err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 280 (tbl->zt_blk + blk) << bs, FTAG, &db); 281 if (err) 282 return (err); 283 *valp = ((uint64_t *)db->db_data)[off]; 284 dmu_buf_rele(db, FTAG); 285 286 if (tbl->zt_nextblk != 0) { 287 /* 288 * read the nextblk for the sake of i/o error checking, 289 * so that zap_table_load() will catch errors for 290 * zap_table_store. 291 */ 292 blk = (idx*2) >> (bs-3); 293 294 err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 295 (tbl->zt_nextblk + blk) << bs, FTAG, &db); 296 dmu_buf_rele(db, FTAG); 297 } 298 return (err); 299 } 300 301 /* 302 * Routines for growing the ptrtbl. 303 */ 304 305 static void 306 zap_ptrtbl_transfer(const uint64_t *src, uint64_t *dst, int n) 307 { 308 int i; 309 for (i = 0; i < n; i++) { 310 uint64_t lb = src[i]; 311 dst[2*i+0] = lb; 312 dst[2*i+1] = lb; 313 } 314 } 315 316 static int 317 zap_grow_ptrtbl(zap_t *zap, dmu_tx_t *tx) 318 { 319 /* In case things go horribly wrong. */ 320 if (zap->zap_f.zap_phys->zap_ptrtbl.zt_shift >= ZAP_HASHBITS-2) 321 return (ENOSPC); 322 323 if (zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks == 0) { 324 /* 325 * We are outgrowing the "embedded" ptrtbl (the one 326 * stored in the header block). Give it its own entire 327 * block, which will double the size of the ptrtbl. 328 */ 329 uint64_t newblk; 330 dmu_buf_t *db_new; 331 int err; 332 333 ASSERT3U(zap->zap_f.zap_phys->zap_ptrtbl.zt_shift, ==, 334 ZAP_EMBEDDED_PTRTBL_SHIFT(zap)); 335 ASSERT3U(zap->zap_f.zap_phys->zap_ptrtbl.zt_blk, ==, 0); 336 337 newblk = zap_allocate_blocks(zap, 1); 338 err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 339 newblk << FZAP_BLOCK_SHIFT(zap), FTAG, &db_new); 340 if (err) 341 return (err); 342 dmu_buf_will_dirty(db_new, tx); 343 zap_ptrtbl_transfer(&ZAP_EMBEDDED_PTRTBL_ENT(zap, 0), 344 db_new->db_data, 1 << ZAP_EMBEDDED_PTRTBL_SHIFT(zap)); 345 dmu_buf_rele(db_new, FTAG); 346 347 zap->zap_f.zap_phys->zap_ptrtbl.zt_blk = newblk; 348 zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks = 1; 349 zap->zap_f.zap_phys->zap_ptrtbl.zt_shift++; 350 351 ASSERT3U(1ULL << zap->zap_f.zap_phys->zap_ptrtbl.zt_shift, ==, 352 zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks << 353 (FZAP_BLOCK_SHIFT(zap)-3)); 354 355 return (0); 356 } else { 357 return (zap_table_grow(zap, &zap->zap_f.zap_phys->zap_ptrtbl, 358 zap_ptrtbl_transfer, tx)); 359 } 360 } 361 362 static void 363 zap_increment_num_entries(zap_t *zap, int delta, dmu_tx_t *tx) 364 { 365 dmu_buf_will_dirty(zap->zap_dbuf, tx); 366 mutex_enter(&zap->zap_f.zap_num_entries_mtx); 367 ASSERT(delta > 0 || zap->zap_f.zap_phys->zap_num_entries >= -delta); 368 zap->zap_f.zap_phys->zap_num_entries += delta; 369 mutex_exit(&zap->zap_f.zap_num_entries_mtx); 370 } 371 372 static uint64_t 373 zap_allocate_blocks(zap_t *zap, int nblocks) 374 { 375 uint64_t newblk; 376 ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); 377 newblk = zap->zap_f.zap_phys->zap_freeblk; 378 zap->zap_f.zap_phys->zap_freeblk += nblocks; 379 return (newblk); 380 } 381 382 static zap_leaf_t * 383 zap_create_leaf(zap_t *zap, dmu_tx_t *tx) 384 { 385 void *winner; 386 zap_leaf_t *l = kmem_alloc(sizeof (zap_leaf_t), KM_SLEEP); 387 388 ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); 389 390 rw_init(&l->l_rwlock, 0, 0, 0); 391 rw_enter(&l->l_rwlock, RW_WRITER); 392 l->l_blkid = zap_allocate_blocks(zap, 1); 393 l->l_dbuf = NULL; 394 l->l_phys = NULL; 395 396 VERIFY(0 == dmu_buf_hold(zap->zap_objset, zap->zap_object, 397 l->l_blkid << FZAP_BLOCK_SHIFT(zap), NULL, &l->l_dbuf)); 398 winner = dmu_buf_set_user(l->l_dbuf, l, &l->l_phys, zap_leaf_pageout); 399 ASSERT(winner == NULL); 400 dmu_buf_will_dirty(l->l_dbuf, tx); 401 402 zap_leaf_init(l, spa_version(dmu_objset_spa(zap->zap_objset))); 403 404 zap->zap_f.zap_phys->zap_num_leafs++; 405 406 return (l); 407 } 408 409 int 410 fzap_count(zap_t *zap, uint64_t *count) 411 { 412 ASSERT(!zap->zap_ismicro); 413 mutex_enter(&zap->zap_f.zap_num_entries_mtx); /* unnecessary */ 414 *count = zap->zap_f.zap_phys->zap_num_entries; 415 mutex_exit(&zap->zap_f.zap_num_entries_mtx); 416 return (0); 417 } 418 419 /* 420 * Routines for obtaining zap_leaf_t's 421 */ 422 423 void 424 zap_put_leaf(zap_leaf_t *l) 425 { 426 rw_exit(&l->l_rwlock); 427 dmu_buf_rele(l->l_dbuf, NULL); 428 } 429 430 _NOTE(ARGSUSED(0)) 431 static void 432 zap_leaf_pageout(dmu_buf_t *db, void *vl) 433 { 434 zap_leaf_t *l = vl; 435 436 rw_destroy(&l->l_rwlock); 437 kmem_free(l, sizeof (zap_leaf_t)); 438 } 439 440 static zap_leaf_t * 441 zap_open_leaf(uint64_t blkid, dmu_buf_t *db) 442 { 443 zap_leaf_t *l, *winner; 444 445 ASSERT(blkid != 0); 446 447 l = kmem_alloc(sizeof (zap_leaf_t), KM_SLEEP); 448 rw_init(&l->l_rwlock, 0, 0, 0); 449 rw_enter(&l->l_rwlock, RW_WRITER); 450 l->l_blkid = blkid; 451 l->l_bs = highbit(db->db_size)-1; 452 l->l_dbuf = db; 453 l->l_phys = NULL; 454 455 winner = dmu_buf_set_user(db, l, &l->l_phys, zap_leaf_pageout); 456 457 rw_exit(&l->l_rwlock); 458 if (winner != NULL) { 459 /* someone else set it first */ 460 zap_leaf_pageout(NULL, l); 461 l = winner; 462 } 463 464 /* 465 * lhr_pad was previously used for the next leaf in the leaf 466 * chain. There should be no chained leafs (as we have removed 467 * support for them). 468 */ 469 ASSERT3U(l->l_phys->l_hdr.lh_pad1, ==, 0); 470 471 /* 472 * There should be more hash entries than there can be 473 * chunks to put in the hash table 474 */ 475 ASSERT3U(ZAP_LEAF_HASH_NUMENTRIES(l), >, ZAP_LEAF_NUMCHUNKS(l) / 3); 476 477 /* The chunks should begin at the end of the hash table */ 478 ASSERT3P(&ZAP_LEAF_CHUNK(l, 0), ==, 479 &l->l_phys->l_hash[ZAP_LEAF_HASH_NUMENTRIES(l)]); 480 481 /* The chunks should end at the end of the block */ 482 ASSERT3U((uintptr_t)&ZAP_LEAF_CHUNK(l, ZAP_LEAF_NUMCHUNKS(l)) - 483 (uintptr_t)l->l_phys, ==, l->l_dbuf->db_size); 484 485 return (l); 486 } 487 488 static int 489 zap_get_leaf_byblk(zap_t *zap, uint64_t blkid, dmu_tx_t *tx, krw_t lt, 490 zap_leaf_t **lp) 491 { 492 dmu_buf_t *db; 493 zap_leaf_t *l; 494 int bs = FZAP_BLOCK_SHIFT(zap); 495 int err; 496 497 ASSERT(RW_LOCK_HELD(&zap->zap_rwlock)); 498 499 err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 500 blkid << bs, NULL, &db); 501 if (err) 502 return (err); 503 504 ASSERT3U(db->db_object, ==, zap->zap_object); 505 ASSERT3U(db->db_offset, ==, blkid << bs); 506 ASSERT3U(db->db_size, ==, 1 << bs); 507 ASSERT(blkid != 0); 508 509 l = dmu_buf_get_user(db); 510 511 if (l == NULL) 512 l = zap_open_leaf(blkid, db); 513 514 rw_enter(&l->l_rwlock, lt); 515 /* 516 * Must lock before dirtying, otherwise l->l_phys could change, 517 * causing ASSERT below to fail. 518 */ 519 if (lt == RW_WRITER) 520 dmu_buf_will_dirty(db, tx); 521 ASSERT3U(l->l_blkid, ==, blkid); 522 ASSERT3P(l->l_dbuf, ==, db); 523 ASSERT3P(l->l_phys, ==, l->l_dbuf->db_data); 524 ASSERT3U(l->l_phys->l_hdr.lh_block_type, ==, ZBT_LEAF); 525 ASSERT3U(l->l_phys->l_hdr.lh_magic, ==, ZAP_LEAF_MAGIC); 526 527 *lp = l; 528 return (0); 529 } 530 531 static int 532 zap_idx_to_blk(zap_t *zap, uint64_t idx, uint64_t *valp) 533 { 534 ASSERT(RW_LOCK_HELD(&zap->zap_rwlock)); 535 536 if (zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks == 0) { 537 ASSERT3U(idx, <, 538 (1ULL << zap->zap_f.zap_phys->zap_ptrtbl.zt_shift)); 539 *valp = ZAP_EMBEDDED_PTRTBL_ENT(zap, idx); 540 return (0); 541 } else { 542 return (zap_table_load(zap, &zap->zap_f.zap_phys->zap_ptrtbl, 543 idx, valp)); 544 } 545 } 546 547 static int 548 zap_set_idx_to_blk(zap_t *zap, uint64_t idx, uint64_t blk, dmu_tx_t *tx) 549 { 550 ASSERT(tx != NULL); 551 ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); 552 553 if (zap->zap_f.zap_phys->zap_ptrtbl.zt_blk == 0) { 554 ZAP_EMBEDDED_PTRTBL_ENT(zap, idx) = blk; 555 return (0); 556 } else { 557 return (zap_table_store(zap, &zap->zap_f.zap_phys->zap_ptrtbl, 558 idx, blk, tx)); 559 } 560 } 561 562 static int 563 zap_deref_leaf(zap_t *zap, uint64_t h, dmu_tx_t *tx, krw_t lt, zap_leaf_t **lp) 564 { 565 uint64_t idx, blk; 566 int err; 567 568 ASSERT(zap->zap_dbuf == NULL || 569 zap->zap_f.zap_phys == zap->zap_dbuf->db_data); 570 ASSERT3U(zap->zap_f.zap_phys->zap_magic, ==, ZAP_MAGIC); 571 idx = ZAP_HASH_IDX(h, zap->zap_f.zap_phys->zap_ptrtbl.zt_shift); 572 err = zap_idx_to_blk(zap, idx, &blk); 573 if (err != 0) 574 return (err); 575 err = zap_get_leaf_byblk(zap, blk, tx, lt, lp); 576 577 ASSERT(err || ZAP_HASH_IDX(h, (*lp)->l_phys->l_hdr.lh_prefix_len) == 578 (*lp)->l_phys->l_hdr.lh_prefix); 579 return (err); 580 } 581 582 static int 583 zap_expand_leaf(zap_name_t *zn, zap_leaf_t *l, dmu_tx_t *tx, zap_leaf_t **lp) 584 { 585 zap_t *zap = zn->zn_zap; 586 uint64_t hash = zn->zn_hash; 587 zap_leaf_t *nl; 588 int prefix_diff, i, err; 589 uint64_t sibling; 590 int old_prefix_len = l->l_phys->l_hdr.lh_prefix_len; 591 592 ASSERT3U(old_prefix_len, <=, zap->zap_f.zap_phys->zap_ptrtbl.zt_shift); 593 ASSERT(RW_LOCK_HELD(&zap->zap_rwlock)); 594 595 ASSERT3U(ZAP_HASH_IDX(hash, old_prefix_len), ==, 596 l->l_phys->l_hdr.lh_prefix); 597 598 if (zap_tryupgradedir(zap, tx) == 0 || 599 old_prefix_len == zap->zap_f.zap_phys->zap_ptrtbl.zt_shift) { 600 /* We failed to upgrade, or need to grow the pointer table */ 601 objset_t *os = zap->zap_objset; 602 uint64_t object = zap->zap_object; 603 604 zap_put_leaf(l); 605 zap_unlockdir(zap); 606 err = zap_lockdir(os, object, tx, RW_WRITER, 607 FALSE, FALSE, &zn->zn_zap); 608 zap = zn->zn_zap; 609 if (err) 610 return (err); 611 ASSERT(!zap->zap_ismicro); 612 613 while (old_prefix_len == 614 zap->zap_f.zap_phys->zap_ptrtbl.zt_shift) { 615 err = zap_grow_ptrtbl(zap, tx); 616 if (err) 617 return (err); 618 } 619 620 err = zap_deref_leaf(zap, hash, tx, RW_WRITER, &l); 621 if (err) 622 return (err); 623 624 if (l->l_phys->l_hdr.lh_prefix_len != old_prefix_len) { 625 /* it split while our locks were down */ 626 *lp = l; 627 return (0); 628 } 629 } 630 ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); 631 ASSERT3U(old_prefix_len, <, zap->zap_f.zap_phys->zap_ptrtbl.zt_shift); 632 ASSERT3U(ZAP_HASH_IDX(hash, old_prefix_len), ==, 633 l->l_phys->l_hdr.lh_prefix); 634 635 prefix_diff = zap->zap_f.zap_phys->zap_ptrtbl.zt_shift - 636 (old_prefix_len + 1); 637 sibling = (ZAP_HASH_IDX(hash, old_prefix_len + 1) | 1) << prefix_diff; 638 639 /* check for i/o errors before doing zap_leaf_split */ 640 for (i = 0; i < (1ULL<<prefix_diff); i++) { 641 uint64_t blk; 642 err = zap_idx_to_blk(zap, sibling+i, &blk); 643 if (err) 644 return (err); 645 ASSERT3U(blk, ==, l->l_blkid); 646 } 647 648 nl = zap_create_leaf(zap, tx); 649 zap_leaf_split(l, nl, spa_version(dmu_objset_spa(zap->zap_objset))); 650 651 /* set sibling pointers */ 652 for (i = 0; i < (1ULL<<prefix_diff); i++) { 653 err = zap_set_idx_to_blk(zap, sibling+i, nl->l_blkid, tx); 654 ASSERT3U(err, ==, 0); /* we checked for i/o errors above */ 655 } 656 657 if (hash & (1ULL << (64 - l->l_phys->l_hdr.lh_prefix_len))) { 658 /* we want the sibling */ 659 zap_put_leaf(l); 660 *lp = nl; 661 } else { 662 zap_put_leaf(nl); 663 *lp = l; 664 } 665 666 return (0); 667 } 668 669 static void 670 zap_put_leaf_maybe_grow_ptrtbl(zap_name_t *zn, zap_leaf_t *l, dmu_tx_t *tx) 671 { 672 zap_t *zap = zn->zn_zap; 673 int shift = zap->zap_f.zap_phys->zap_ptrtbl.zt_shift; 674 int leaffull = (l->l_phys->l_hdr.lh_prefix_len == shift && 675 l->l_phys->l_hdr.lh_nfree < ZAP_LEAF_LOW_WATER); 676 677 zap_put_leaf(l); 678 679 if (leaffull || zap->zap_f.zap_phys->zap_ptrtbl.zt_nextblk) { 680 int err; 681 682 /* 683 * We are in the middle of growing the pointer table, or 684 * this leaf will soon make us grow it. 685 */ 686 if (zap_tryupgradedir(zap, tx) == 0) { 687 objset_t *os = zap->zap_objset; 688 uint64_t zapobj = zap->zap_object; 689 690 zap_unlockdir(zap); 691 err = zap_lockdir(os, zapobj, tx, 692 RW_WRITER, FALSE, FALSE, &zn->zn_zap); 693 zap = zn->zn_zap; 694 if (err) 695 return; 696 } 697 698 /* could have finished growing while our locks were down */ 699 if (zap->zap_f.zap_phys->zap_ptrtbl.zt_shift == shift) 700 (void) zap_grow_ptrtbl(zap, tx); 701 } 702 } 703 704 705 static int 706 fzap_checksize(const char *name, uint64_t integer_size, uint64_t num_integers) 707 { 708 if (name && strlen(name) > ZAP_MAXNAMELEN) 709 return (E2BIG); 710 711 /* Only integer sizes supported by C */ 712 switch (integer_size) { 713 case 1: 714 case 2: 715 case 4: 716 case 8: 717 break; 718 default: 719 return (EINVAL); 720 } 721 722 if (integer_size * num_integers > ZAP_MAXVALUELEN) 723 return (E2BIG); 724 725 return (0); 726 } 727 728 /* 729 * Routines for manipulating attributes. 730 */ 731 int 732 fzap_lookup(zap_name_t *zn, 733 uint64_t integer_size, uint64_t num_integers, void *buf, 734 char *realname, int rn_len, boolean_t *ncp) 735 { 736 zap_leaf_t *l; 737 int err; 738 zap_entry_handle_t zeh; 739 740 err = fzap_checksize(zn->zn_name_orij, integer_size, num_integers); 741 if (err != 0) 742 return (err); 743 744 err = zap_deref_leaf(zn->zn_zap, zn->zn_hash, NULL, RW_READER, &l); 745 if (err != 0) 746 return (err); 747 err = zap_leaf_lookup(l, zn, &zeh); 748 if (err == 0) { 749 err = zap_entry_read(&zeh, integer_size, num_integers, buf); 750 (void) zap_entry_read_name(&zeh, rn_len, realname); 751 if (ncp) { 752 *ncp = zap_entry_normalization_conflict(&zeh, 753 zn, NULL, zn->zn_zap); 754 } 755 } 756 757 zap_put_leaf(l); 758 return (err); 759 } 760 761 int 762 fzap_add_cd(zap_name_t *zn, 763 uint64_t integer_size, uint64_t num_integers, 764 const void *val, uint32_t cd, dmu_tx_t *tx) 765 { 766 zap_leaf_t *l; 767 int err; 768 zap_entry_handle_t zeh; 769 zap_t *zap = zn->zn_zap; 770 771 ASSERT(RW_LOCK_HELD(&zap->zap_rwlock)); 772 ASSERT(!zap->zap_ismicro); 773 ASSERT(fzap_checksize(zn->zn_name_orij, 774 integer_size, num_integers) == 0); 775 776 err = zap_deref_leaf(zap, zn->zn_hash, tx, RW_WRITER, &l); 777 if (err != 0) 778 return (err); 779 retry: 780 err = zap_leaf_lookup(l, zn, &zeh); 781 if (err == 0) { 782 err = EEXIST; 783 goto out; 784 } 785 if (err != ENOENT) 786 goto out; 787 788 err = zap_entry_create(l, zn->zn_name_orij, zn->zn_hash, cd, 789 integer_size, num_integers, val, &zeh); 790 791 if (err == 0) { 792 zap_increment_num_entries(zap, 1, tx); 793 } else if (err == EAGAIN) { 794 err = zap_expand_leaf(zn, l, tx, &l); 795 zap = zn->zn_zap; /* zap_expand_leaf() may change zap */ 796 if (err == 0) 797 goto retry; 798 } 799 800 out: 801 if (zap != NULL) 802 zap_put_leaf_maybe_grow_ptrtbl(zn, l, tx); 803 return (err); 804 } 805 806 int 807 fzap_add(zap_name_t *zn, 808 uint64_t integer_size, uint64_t num_integers, 809 const void *val, dmu_tx_t *tx) 810 { 811 int err = fzap_checksize(zn->zn_name_orij, integer_size, num_integers); 812 if (err != 0) 813 return (err); 814 815 return (fzap_add_cd(zn, integer_size, num_integers, 816 val, ZAP_MAXCD, tx)); 817 } 818 819 int 820 fzap_update(zap_name_t *zn, 821 int integer_size, uint64_t num_integers, const void *val, dmu_tx_t *tx) 822 { 823 zap_leaf_t *l; 824 int err, create; 825 zap_entry_handle_t zeh; 826 zap_t *zap = zn->zn_zap; 827 828 ASSERT(RW_LOCK_HELD(&zap->zap_rwlock)); 829 err = fzap_checksize(zn->zn_name_orij, integer_size, num_integers); 830 if (err != 0) 831 return (err); 832 833 err = zap_deref_leaf(zap, zn->zn_hash, tx, RW_WRITER, &l); 834 if (err != 0) 835 return (err); 836 retry: 837 err = zap_leaf_lookup(l, zn, &zeh); 838 create = (err == ENOENT); 839 ASSERT(err == 0 || err == ENOENT); 840 841 if (create) { 842 err = zap_entry_create(l, zn->zn_name_orij, zn->zn_hash, 843 ZAP_MAXCD, integer_size, num_integers, val, &zeh); 844 if (err == 0) 845 zap_increment_num_entries(zap, 1, tx); 846 } else { 847 err = zap_entry_update(&zeh, integer_size, num_integers, val); 848 } 849 850 if (err == EAGAIN) { 851 err = zap_expand_leaf(zn, l, tx, &l); 852 zap = zn->zn_zap; /* zap_expand_leaf() may change zap */ 853 if (err == 0) 854 goto retry; 855 } 856 857 if (zap != NULL) 858 zap_put_leaf_maybe_grow_ptrtbl(zn, l, tx); 859 return (err); 860 } 861 862 int 863 fzap_length(zap_name_t *zn, 864 uint64_t *integer_size, uint64_t *num_integers) 865 { 866 zap_leaf_t *l; 867 int err; 868 zap_entry_handle_t zeh; 869 870 err = zap_deref_leaf(zn->zn_zap, zn->zn_hash, NULL, RW_READER, &l); 871 if (err != 0) 872 return (err); 873 err = zap_leaf_lookup(l, zn, &zeh); 874 if (err != 0) 875 goto out; 876 877 if (integer_size) 878 *integer_size = zeh.zeh_integer_size; 879 if (num_integers) 880 *num_integers = zeh.zeh_num_integers; 881 out: 882 zap_put_leaf(l); 883 return (err); 884 } 885 886 int 887 fzap_remove(zap_name_t *zn, dmu_tx_t *tx) 888 { 889 zap_leaf_t *l; 890 int err; 891 zap_entry_handle_t zeh; 892 893 err = zap_deref_leaf(zn->zn_zap, zn->zn_hash, tx, RW_WRITER, &l); 894 if (err != 0) 895 return (err); 896 err = zap_leaf_lookup(l, zn, &zeh); 897 if (err == 0) { 898 zap_entry_remove(&zeh); 899 zap_increment_num_entries(zn->zn_zap, -1, tx); 900 } 901 zap_put_leaf(l); 902 return (err); 903 } 904 905 int 906 zap_value_search(objset_t *os, uint64_t zapobj, uint64_t value, uint64_t mask, 907 char *name) 908 { 909 zap_cursor_t zc; 910 zap_attribute_t *za; 911 int err; 912 913 if (mask == 0) 914 mask = -1ULL; 915 916 za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP); 917 for (zap_cursor_init(&zc, os, zapobj); 918 (err = zap_cursor_retrieve(&zc, za)) == 0; 919 zap_cursor_advance(&zc)) { 920 if ((za->za_first_integer & mask) == (value & mask)) { 921 (void) strcpy(name, za->za_name); 922 break; 923 } 924 } 925 zap_cursor_fini(&zc); 926 kmem_free(za, sizeof (zap_attribute_t)); 927 return (err); 928 } 929 930 931 /* 932 * Routines for iterating over the attributes. 933 */ 934 935 int 936 fzap_cursor_retrieve(zap_t *zap, zap_cursor_t *zc, zap_attribute_t *za) 937 { 938 int err = ENOENT; 939 zap_entry_handle_t zeh; 940 zap_leaf_t *l; 941 942 /* retrieve the next entry at or after zc_hash/zc_cd */ 943 /* if no entry, return ENOENT */ 944 945 if (zc->zc_leaf && 946 (ZAP_HASH_IDX(zc->zc_hash, 947 zc->zc_leaf->l_phys->l_hdr.lh_prefix_len) != 948 zc->zc_leaf->l_phys->l_hdr.lh_prefix)) { 949 rw_enter(&zc->zc_leaf->l_rwlock, RW_READER); 950 zap_put_leaf(zc->zc_leaf); 951 zc->zc_leaf = NULL; 952 } 953 954 again: 955 if (zc->zc_leaf == NULL) { 956 err = zap_deref_leaf(zap, zc->zc_hash, NULL, RW_READER, 957 &zc->zc_leaf); 958 if (err != 0) 959 return (err); 960 } else { 961 rw_enter(&zc->zc_leaf->l_rwlock, RW_READER); 962 } 963 l = zc->zc_leaf; 964 965 err = zap_leaf_lookup_closest(l, zc->zc_hash, zc->zc_cd, &zeh); 966 967 if (err == ENOENT) { 968 uint64_t nocare = 969 (1ULL << (64 - l->l_phys->l_hdr.lh_prefix_len)) - 1; 970 zc->zc_hash = (zc->zc_hash & ~nocare) + nocare + 1; 971 zc->zc_cd = 0; 972 if (l->l_phys->l_hdr.lh_prefix_len == 0 || zc->zc_hash == 0) { 973 zc->zc_hash = -1ULL; 974 } else { 975 zap_put_leaf(zc->zc_leaf); 976 zc->zc_leaf = NULL; 977 goto again; 978 } 979 } 980 981 if (err == 0) { 982 zc->zc_hash = zeh.zeh_hash; 983 zc->zc_cd = zeh.zeh_cd; 984 za->za_integer_length = zeh.zeh_integer_size; 985 za->za_num_integers = zeh.zeh_num_integers; 986 if (zeh.zeh_num_integers == 0) { 987 za->za_first_integer = 0; 988 } else { 989 err = zap_entry_read(&zeh, 8, 1, &za->za_first_integer); 990 ASSERT(err == 0 || err == EOVERFLOW); 991 } 992 err = zap_entry_read_name(&zeh, 993 sizeof (za->za_name), za->za_name); 994 ASSERT(err == 0); 995 996 za->za_normalization_conflict = 997 zap_entry_normalization_conflict(&zeh, 998 NULL, za->za_name, zap); 999 } 1000 rw_exit(&zc->zc_leaf->l_rwlock); 1001 return (err); 1002 } 1003 1004 1005 static void 1006 zap_stats_ptrtbl(zap_t *zap, uint64_t *tbl, int len, zap_stats_t *zs) 1007 { 1008 int i, err; 1009 uint64_t lastblk = 0; 1010 1011 /* 1012 * NB: if a leaf has more pointers than an entire ptrtbl block 1013 * can hold, then it'll be accounted for more than once, since 1014 * we won't have lastblk. 1015 */ 1016 for (i = 0; i < len; i++) { 1017 zap_leaf_t *l; 1018 1019 if (tbl[i] == lastblk) 1020 continue; 1021 lastblk = tbl[i]; 1022 1023 err = zap_get_leaf_byblk(zap, tbl[i], NULL, RW_READER, &l); 1024 if (err == 0) { 1025 zap_leaf_stats(zap, l, zs); 1026 zap_put_leaf(l); 1027 } 1028 } 1029 } 1030 1031 void 1032 fzap_get_stats(zap_t *zap, zap_stats_t *zs) 1033 { 1034 int bs = FZAP_BLOCK_SHIFT(zap); 1035 zs->zs_blocksize = 1ULL << bs; 1036 1037 /* 1038 * Set zap_phys_t fields 1039 */ 1040 zs->zs_num_leafs = zap->zap_f.zap_phys->zap_num_leafs; 1041 zs->zs_num_entries = zap->zap_f.zap_phys->zap_num_entries; 1042 zs->zs_num_blocks = zap->zap_f.zap_phys->zap_freeblk; 1043 zs->zs_block_type = zap->zap_f.zap_phys->zap_block_type; 1044 zs->zs_magic = zap->zap_f.zap_phys->zap_magic; 1045 zs->zs_salt = zap->zap_f.zap_phys->zap_salt; 1046 1047 /* 1048 * Set zap_ptrtbl fields 1049 */ 1050 zs->zs_ptrtbl_len = 1ULL << zap->zap_f.zap_phys->zap_ptrtbl.zt_shift; 1051 zs->zs_ptrtbl_nextblk = zap->zap_f.zap_phys->zap_ptrtbl.zt_nextblk; 1052 zs->zs_ptrtbl_blks_copied = 1053 zap->zap_f.zap_phys->zap_ptrtbl.zt_blks_copied; 1054 zs->zs_ptrtbl_zt_blk = zap->zap_f.zap_phys->zap_ptrtbl.zt_blk; 1055 zs->zs_ptrtbl_zt_numblks = zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks; 1056 zs->zs_ptrtbl_zt_shift = zap->zap_f.zap_phys->zap_ptrtbl.zt_shift; 1057 1058 if (zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks == 0) { 1059 /* the ptrtbl is entirely in the header block. */ 1060 zap_stats_ptrtbl(zap, &ZAP_EMBEDDED_PTRTBL_ENT(zap, 0), 1061 1 << ZAP_EMBEDDED_PTRTBL_SHIFT(zap), zs); 1062 } else { 1063 int b; 1064 1065 dmu_prefetch(zap->zap_objset, zap->zap_object, 1066 zap->zap_f.zap_phys->zap_ptrtbl.zt_blk << bs, 1067 zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks << bs); 1068 1069 for (b = 0; b < zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks; 1070 b++) { 1071 dmu_buf_t *db; 1072 int err; 1073 1074 err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 1075 (zap->zap_f.zap_phys->zap_ptrtbl.zt_blk + b) << bs, 1076 FTAG, &db); 1077 if (err == 0) { 1078 zap_stats_ptrtbl(zap, db->db_data, 1079 1<<(bs-3), zs); 1080 dmu_buf_rele(db, FTAG); 1081 } 1082 } 1083 } 1084 } 1085