1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/zfs_context.h> 27 #include <sys/dmu.h> 28 #include <sys/dmu_impl.h> 29 #include <sys/dbuf.h> 30 #include <sys/dmu_objset.h> 31 #include <sys/dsl_dataset.h> 32 #include <sys/dsl_dir.h> 33 #include <sys/dmu_tx.h> 34 #include <sys/spa.h> 35 #include <sys/zio.h> 36 #include <sys/dmu_zfetch.h> 37 38 static void dbuf_destroy(dmu_buf_impl_t *db); 39 static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 40 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 41 static arc_done_func_t dbuf_write_ready; 42 static arc_done_func_t dbuf_write_done; 43 44 /* 45 * Global data structures and functions for the dbuf cache. 46 */ 47 static kmem_cache_t *dbuf_cache; 48 49 /* ARGSUSED */ 50 static int 51 dbuf_cons(void *vdb, void *unused, int kmflag) 52 { 53 dmu_buf_impl_t *db = vdb; 54 bzero(db, sizeof (dmu_buf_impl_t)); 55 56 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 57 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 58 refcount_create(&db->db_holds); 59 return (0); 60 } 61 62 /* ARGSUSED */ 63 static void 64 dbuf_dest(void *vdb, void *unused) 65 { 66 dmu_buf_impl_t *db = vdb; 67 mutex_destroy(&db->db_mtx); 68 cv_destroy(&db->db_changed); 69 refcount_destroy(&db->db_holds); 70 } 71 72 /* 73 * dbuf hash table routines 74 */ 75 static dbuf_hash_table_t dbuf_hash_table; 76 77 static uint64_t dbuf_hash_count; 78 79 static uint64_t 80 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 81 { 82 uintptr_t osv = (uintptr_t)os; 83 uint64_t crc = -1ULL; 84 85 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 86 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 87 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 88 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 89 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 90 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 91 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 92 93 crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 94 95 return (crc); 96 } 97 98 #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 99 100 #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 101 ((dbuf)->db.db_object == (obj) && \ 102 (dbuf)->db_objset == (os) && \ 103 (dbuf)->db_level == (level) && \ 104 (dbuf)->db_blkid == (blkid)) 105 106 dmu_buf_impl_t * 107 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 108 { 109 dbuf_hash_table_t *h = &dbuf_hash_table; 110 objset_impl_t *os = dn->dn_objset; 111 uint64_t obj = dn->dn_object; 112 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 113 uint64_t idx = hv & h->hash_table_mask; 114 dmu_buf_impl_t *db; 115 116 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 117 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 118 if (DBUF_EQUAL(db, os, obj, level, blkid)) { 119 mutex_enter(&db->db_mtx); 120 if (db->db_state != DB_EVICTING) { 121 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 122 return (db); 123 } 124 mutex_exit(&db->db_mtx); 125 } 126 } 127 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 128 return (NULL); 129 } 130 131 /* 132 * Insert an entry into the hash table. If there is already an element 133 * equal to elem in the hash table, then the already existing element 134 * will be returned and the new element will not be inserted. 135 * Otherwise returns NULL. 136 */ 137 static dmu_buf_impl_t * 138 dbuf_hash_insert(dmu_buf_impl_t *db) 139 { 140 dbuf_hash_table_t *h = &dbuf_hash_table; 141 objset_impl_t *os = db->db_objset; 142 uint64_t obj = db->db.db_object; 143 int level = db->db_level; 144 uint64_t blkid = db->db_blkid; 145 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 146 uint64_t idx = hv & h->hash_table_mask; 147 dmu_buf_impl_t *dbf; 148 149 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 150 for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 151 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 152 mutex_enter(&dbf->db_mtx); 153 if (dbf->db_state != DB_EVICTING) { 154 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 155 return (dbf); 156 } 157 mutex_exit(&dbf->db_mtx); 158 } 159 } 160 161 mutex_enter(&db->db_mtx); 162 db->db_hash_next = h->hash_table[idx]; 163 h->hash_table[idx] = db; 164 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 165 atomic_add_64(&dbuf_hash_count, 1); 166 167 return (NULL); 168 } 169 170 /* 171 * Remove an entry from the hash table. This operation will 172 * fail if there are any existing holds on the db. 173 */ 174 static void 175 dbuf_hash_remove(dmu_buf_impl_t *db) 176 { 177 dbuf_hash_table_t *h = &dbuf_hash_table; 178 uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 179 db->db_level, db->db_blkid); 180 uint64_t idx = hv & h->hash_table_mask; 181 dmu_buf_impl_t *dbf, **dbp; 182 183 /* 184 * We musn't hold db_mtx to maintin lock ordering: 185 * DBUF_HASH_MUTEX > db_mtx. 186 */ 187 ASSERT(refcount_is_zero(&db->db_holds)); 188 ASSERT(db->db_state == DB_EVICTING); 189 ASSERT(!MUTEX_HELD(&db->db_mtx)); 190 191 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 192 dbp = &h->hash_table[idx]; 193 while ((dbf = *dbp) != db) { 194 dbp = &dbf->db_hash_next; 195 ASSERT(dbf != NULL); 196 } 197 *dbp = db->db_hash_next; 198 db->db_hash_next = NULL; 199 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 200 atomic_add_64(&dbuf_hash_count, -1); 201 } 202 203 static arc_evict_func_t dbuf_do_evict; 204 205 static void 206 dbuf_evict_user(dmu_buf_impl_t *db) 207 { 208 ASSERT(MUTEX_HELD(&db->db_mtx)); 209 210 if (db->db_level != 0 || db->db_evict_func == NULL) 211 return; 212 213 if (db->db_user_data_ptr_ptr) 214 *db->db_user_data_ptr_ptr = db->db.db_data; 215 db->db_evict_func(&db->db, db->db_user_ptr); 216 db->db_user_ptr = NULL; 217 db->db_user_data_ptr_ptr = NULL; 218 db->db_evict_func = NULL; 219 } 220 221 void 222 dbuf_evict(dmu_buf_impl_t *db) 223 { 224 ASSERT(MUTEX_HELD(&db->db_mtx)); 225 ASSERT(db->db_buf == NULL); 226 ASSERT(db->db_data_pending == NULL); 227 228 dbuf_clear(db); 229 dbuf_destroy(db); 230 } 231 232 void 233 dbuf_init(void) 234 { 235 uint64_t hsize = 1ULL << 16; 236 dbuf_hash_table_t *h = &dbuf_hash_table; 237 int i; 238 239 /* 240 * The hash table is big enough to fill all of physical memory 241 * with an average 4K block size. The table will take up 242 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 243 */ 244 while (hsize * 4096 < physmem * PAGESIZE) 245 hsize <<= 1; 246 247 retry: 248 h->hash_table_mask = hsize - 1; 249 h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 250 if (h->hash_table == NULL) { 251 /* XXX - we should really return an error instead of assert */ 252 ASSERT(hsize > (1ULL << 10)); 253 hsize >>= 1; 254 goto retry; 255 } 256 257 dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 258 sizeof (dmu_buf_impl_t), 259 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 260 261 for (i = 0; i < DBUF_MUTEXES; i++) 262 mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 263 } 264 265 void 266 dbuf_fini(void) 267 { 268 dbuf_hash_table_t *h = &dbuf_hash_table; 269 int i; 270 271 for (i = 0; i < DBUF_MUTEXES; i++) 272 mutex_destroy(&h->hash_mutexes[i]); 273 kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 274 kmem_cache_destroy(dbuf_cache); 275 } 276 277 /* 278 * Other stuff. 279 */ 280 281 #ifdef ZFS_DEBUG 282 static void 283 dbuf_verify(dmu_buf_impl_t *db) 284 { 285 dnode_t *dn = db->db_dnode; 286 287 ASSERT(MUTEX_HELD(&db->db_mtx)); 288 289 if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 290 return; 291 292 ASSERT(db->db_objset != NULL); 293 if (dn == NULL) { 294 ASSERT(db->db_parent == NULL); 295 ASSERT(db->db_blkptr == NULL); 296 } else { 297 ASSERT3U(db->db.db_object, ==, dn->dn_object); 298 ASSERT3P(db->db_objset, ==, dn->dn_objset); 299 ASSERT3U(db->db_level, <, dn->dn_nlevels); 300 ASSERT(db->db_blkid == DB_BONUS_BLKID || 301 list_head(&dn->dn_dbufs)); 302 } 303 if (db->db_blkid == DB_BONUS_BLKID) { 304 ASSERT(dn != NULL); 305 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 306 ASSERT3U(db->db.db_offset, ==, DB_BONUS_BLKID); 307 } else { 308 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 309 } 310 311 if (db->db_level == 0) { 312 /* we can be momentarily larger in dnode_set_blksz() */ 313 if (db->db_blkid != DB_BONUS_BLKID && dn) { 314 ASSERT3U(db->db.db_size, >=, dn->dn_datablksz); 315 } 316 if (db->db.db_object == DMU_META_DNODE_OBJECT) { 317 dbuf_dirty_record_t *dr = db->db_data_pending; 318 /* 319 * it should only be modified in syncing 320 * context, so make sure we only have 321 * one copy of the data. 322 */ 323 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 324 } 325 } 326 327 /* verify db->db_blkptr */ 328 if (db->db_blkptr) { 329 if (db->db_parent == dn->dn_dbuf) { 330 /* db is pointed to by the dnode */ 331 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 332 if (db->db.db_object == DMU_META_DNODE_OBJECT) 333 ASSERT(db->db_parent == NULL); 334 else 335 ASSERT(db->db_parent != NULL); 336 ASSERT3P(db->db_blkptr, ==, 337 &dn->dn_phys->dn_blkptr[db->db_blkid]); 338 } else { 339 /* db is pointed to by an indirect block */ 340 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 341 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 342 ASSERT3U(db->db_parent->db.db_object, ==, 343 db->db.db_object); 344 /* 345 * dnode_grow_indblksz() can make this fail if we don't 346 * have the struct_rwlock. XXX indblksz no longer 347 * grows. safe to do this now? 348 */ 349 if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) { 350 ASSERT3P(db->db_blkptr, ==, 351 ((blkptr_t *)db->db_parent->db.db_data + 352 db->db_blkid % epb)); 353 } 354 } 355 } 356 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 357 db->db.db_data && db->db_blkid != DB_BONUS_BLKID && 358 db->db_state != DB_FILL && !dn->dn_free_txg) { 359 /* 360 * If the blkptr isn't set but they have nonzero data, 361 * it had better be dirty, otherwise we'll lose that 362 * data when we evict this buffer. 363 */ 364 if (db->db_dirtycnt == 0) { 365 uint64_t *buf = db->db.db_data; 366 int i; 367 368 for (i = 0; i < db->db.db_size >> 3; i++) { 369 ASSERT(buf[i] == 0); 370 } 371 } 372 } 373 } 374 #endif 375 376 static void 377 dbuf_update_data(dmu_buf_impl_t *db) 378 { 379 ASSERT(MUTEX_HELD(&db->db_mtx)); 380 if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 381 ASSERT(!refcount_is_zero(&db->db_holds)); 382 *db->db_user_data_ptr_ptr = db->db.db_data; 383 } 384 } 385 386 static void 387 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 388 { 389 ASSERT(MUTEX_HELD(&db->db_mtx)); 390 ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 391 db->db_buf = buf; 392 if (buf != NULL) { 393 ASSERT(buf->b_data != NULL); 394 db->db.db_data = buf->b_data; 395 if (!arc_released(buf)) 396 arc_set_callback(buf, dbuf_do_evict, db); 397 dbuf_update_data(db); 398 } else { 399 dbuf_evict_user(db); 400 db->db.db_data = NULL; 401 db->db_state = DB_UNCACHED; 402 } 403 } 404 405 uint64_t 406 dbuf_whichblock(dnode_t *dn, uint64_t offset) 407 { 408 if (dn->dn_datablkshift) { 409 return (offset >> dn->dn_datablkshift); 410 } else { 411 ASSERT3U(offset, <, dn->dn_datablksz); 412 return (0); 413 } 414 } 415 416 static void 417 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 418 { 419 dmu_buf_impl_t *db = vdb; 420 421 mutex_enter(&db->db_mtx); 422 ASSERT3U(db->db_state, ==, DB_READ); 423 /* 424 * All reads are synchronous, so we must have a hold on the dbuf 425 */ 426 ASSERT(refcount_count(&db->db_holds) > 0); 427 ASSERT(db->db_buf == NULL); 428 ASSERT(db->db.db_data == NULL); 429 if (db->db_level == 0 && db->db_freed_in_flight) { 430 /* we were freed in flight; disregard any error */ 431 arc_release(buf, db); 432 bzero(buf->b_data, db->db.db_size); 433 arc_buf_freeze(buf); 434 db->db_freed_in_flight = FALSE; 435 dbuf_set_data(db, buf); 436 db->db_state = DB_CACHED; 437 } else if (zio == NULL || zio->io_error == 0) { 438 dbuf_set_data(db, buf); 439 db->db_state = DB_CACHED; 440 } else { 441 ASSERT(db->db_blkid != DB_BONUS_BLKID); 442 ASSERT3P(db->db_buf, ==, NULL); 443 VERIFY(arc_buf_remove_ref(buf, db) == 1); 444 db->db_state = DB_UNCACHED; 445 } 446 cv_broadcast(&db->db_changed); 447 mutex_exit(&db->db_mtx); 448 dbuf_rele(db, NULL); 449 } 450 451 static void 452 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 453 { 454 dnode_t *dn = db->db_dnode; 455 zbookmark_t zb; 456 uint32_t aflags = ARC_NOWAIT; 457 arc_buf_t *pbuf; 458 459 ASSERT(!refcount_is_zero(&db->db_holds)); 460 /* We need the struct_rwlock to prevent db_blkptr from changing. */ 461 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 462 ASSERT(MUTEX_HELD(&db->db_mtx)); 463 ASSERT(db->db_state == DB_UNCACHED); 464 ASSERT(db->db_buf == NULL); 465 466 if (db->db_blkid == DB_BONUS_BLKID) { 467 int bonuslen = dn->dn_bonuslen; 468 469 ASSERT3U(bonuslen, <=, db->db.db_size); 470 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 471 arc_space_consume(DN_MAX_BONUSLEN); 472 if (bonuslen < DN_MAX_BONUSLEN) 473 bzero(db->db.db_data, DN_MAX_BONUSLEN); 474 bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, 475 bonuslen); 476 dbuf_update_data(db); 477 db->db_state = DB_CACHED; 478 mutex_exit(&db->db_mtx); 479 return; 480 } 481 482 /* 483 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 484 * processes the delete record and clears the bp while we are waiting 485 * for the dn_mtx (resulting in a "no" from block_freed). 486 */ 487 if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 488 (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 489 BP_IS_HOLE(db->db_blkptr)))) { 490 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 491 492 dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa, 493 db->db.db_size, db, type)); 494 bzero(db->db.db_data, db->db.db_size); 495 db->db_state = DB_CACHED; 496 *flags |= DB_RF_CACHED; 497 mutex_exit(&db->db_mtx); 498 return; 499 } 500 501 db->db_state = DB_READ; 502 mutex_exit(&db->db_mtx); 503 504 if (DBUF_IS_L2CACHEABLE(db)) 505 aflags |= ARC_L2CACHE; 506 507 zb.zb_objset = db->db_objset->os_dsl_dataset ? 508 db->db_objset->os_dsl_dataset->ds_object : 0; 509 zb.zb_object = db->db.db_object; 510 zb.zb_level = db->db_level; 511 zb.zb_blkid = db->db_blkid; 512 513 dbuf_add_ref(db, NULL); 514 /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */ 515 516 if (db->db_parent) 517 pbuf = db->db_parent->db_buf; 518 else 519 pbuf = db->db_objset->os_phys_buf; 520 521 (void) arc_read(zio, dn->dn_objset->os_spa, db->db_blkptr, pbuf, 522 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 523 (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 524 &aflags, &zb); 525 if (aflags & ARC_CACHED) 526 *flags |= DB_RF_CACHED; 527 } 528 529 int 530 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 531 { 532 int err = 0; 533 int havepzio = (zio != NULL); 534 int prefetch; 535 536 /* 537 * We don't have to hold the mutex to check db_state because it 538 * can't be freed while we have a hold on the buffer. 539 */ 540 ASSERT(!refcount_is_zero(&db->db_holds)); 541 542 if ((flags & DB_RF_HAVESTRUCT) == 0) 543 rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER); 544 545 prefetch = db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 546 (flags & DB_RF_NOPREFETCH) == 0 && db->db_dnode != NULL && 547 DBUF_IS_CACHEABLE(db); 548 549 mutex_enter(&db->db_mtx); 550 if (db->db_state == DB_CACHED) { 551 mutex_exit(&db->db_mtx); 552 if (prefetch) 553 dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 554 db->db.db_size, TRUE); 555 if ((flags & DB_RF_HAVESTRUCT) == 0) 556 rw_exit(&db->db_dnode->dn_struct_rwlock); 557 } else if (db->db_state == DB_UNCACHED) { 558 if (zio == NULL) { 559 zio = zio_root(db->db_dnode->dn_objset->os_spa, 560 NULL, NULL, ZIO_FLAG_CANFAIL); 561 } 562 dbuf_read_impl(db, zio, &flags); 563 564 /* dbuf_read_impl has dropped db_mtx for us */ 565 566 if (prefetch) 567 dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 568 db->db.db_size, flags & DB_RF_CACHED); 569 570 if ((flags & DB_RF_HAVESTRUCT) == 0) 571 rw_exit(&db->db_dnode->dn_struct_rwlock); 572 573 if (!havepzio) 574 err = zio_wait(zio); 575 } else { 576 mutex_exit(&db->db_mtx); 577 if (prefetch) 578 dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 579 db->db.db_size, TRUE); 580 if ((flags & DB_RF_HAVESTRUCT) == 0) 581 rw_exit(&db->db_dnode->dn_struct_rwlock); 582 583 mutex_enter(&db->db_mtx); 584 if ((flags & DB_RF_NEVERWAIT) == 0) { 585 while (db->db_state == DB_READ || 586 db->db_state == DB_FILL) { 587 ASSERT(db->db_state == DB_READ || 588 (flags & DB_RF_HAVESTRUCT) == 0); 589 cv_wait(&db->db_changed, &db->db_mtx); 590 } 591 if (db->db_state == DB_UNCACHED) 592 err = EIO; 593 } 594 mutex_exit(&db->db_mtx); 595 } 596 597 ASSERT(err || havepzio || db->db_state == DB_CACHED); 598 return (err); 599 } 600 601 static void 602 dbuf_noread(dmu_buf_impl_t *db) 603 { 604 ASSERT(!refcount_is_zero(&db->db_holds)); 605 ASSERT(db->db_blkid != DB_BONUS_BLKID); 606 mutex_enter(&db->db_mtx); 607 while (db->db_state == DB_READ || db->db_state == DB_FILL) 608 cv_wait(&db->db_changed, &db->db_mtx); 609 if (db->db_state == DB_UNCACHED) { 610 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 611 612 ASSERT(db->db_buf == NULL); 613 ASSERT(db->db.db_data == NULL); 614 dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 615 db->db.db_size, db, type)); 616 db->db_state = DB_FILL; 617 } else { 618 ASSERT3U(db->db_state, ==, DB_CACHED); 619 } 620 mutex_exit(&db->db_mtx); 621 } 622 623 /* 624 * This is our just-in-time copy function. It makes a copy of 625 * buffers, that have been modified in a previous transaction 626 * group, before we modify them in the current active group. 627 * 628 * This function is used in two places: when we are dirtying a 629 * buffer for the first time in a txg, and when we are freeing 630 * a range in a dnode that includes this buffer. 631 * 632 * Note that when we are called from dbuf_free_range() we do 633 * not put a hold on the buffer, we just traverse the active 634 * dbuf list for the dnode. 635 */ 636 static void 637 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 638 { 639 dbuf_dirty_record_t *dr = db->db_last_dirty; 640 641 ASSERT(MUTEX_HELD(&db->db_mtx)); 642 ASSERT(db->db.db_data != NULL); 643 ASSERT(db->db_level == 0); 644 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 645 646 if (dr == NULL || 647 (dr->dt.dl.dr_data != 648 ((db->db_blkid == DB_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 649 return; 650 651 /* 652 * If the last dirty record for this dbuf has not yet synced 653 * and its referencing the dbuf data, either: 654 * reset the reference to point to a new copy, 655 * or (if there a no active holders) 656 * just null out the current db_data pointer. 657 */ 658 ASSERT(dr->dr_txg >= txg - 2); 659 if (db->db_blkid == DB_BONUS_BLKID) { 660 /* Note that the data bufs here are zio_bufs */ 661 dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 662 arc_space_consume(DN_MAX_BONUSLEN); 663 bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 664 } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 665 int size = db->db.db_size; 666 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 667 dr->dt.dl.dr_data = arc_buf_alloc( 668 db->db_dnode->dn_objset->os_spa, size, db, type); 669 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 670 } else { 671 dbuf_set_data(db, NULL); 672 } 673 } 674 675 void 676 dbuf_unoverride(dbuf_dirty_record_t *dr) 677 { 678 dmu_buf_impl_t *db = dr->dr_dbuf; 679 uint64_t txg = dr->dr_txg; 680 681 ASSERT(MUTEX_HELD(&db->db_mtx)); 682 ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 683 ASSERT(db->db_level == 0); 684 685 if (db->db_blkid == DB_BONUS_BLKID || 686 dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 687 return; 688 689 /* free this block */ 690 if (!BP_IS_HOLE(&dr->dt.dl.dr_overridden_by)) { 691 /* XXX can get silent EIO here */ 692 (void) dsl_free(NULL, 693 spa_get_dsl(db->db_dnode->dn_objset->os_spa), 694 txg, &dr->dt.dl.dr_overridden_by, NULL, NULL, ARC_WAIT); 695 } 696 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 697 /* 698 * Release the already-written buffer, so we leave it in 699 * a consistent dirty state. Note that all callers are 700 * modifying the buffer, so they will immediately do 701 * another (redundant) arc_release(). Therefore, leave 702 * the buf thawed to save the effort of freezing & 703 * immediately re-thawing it. 704 */ 705 arc_release(dr->dt.dl.dr_data, db); 706 } 707 708 /* 709 * Evict (if its unreferenced) or clear (if its referenced) any level-0 710 * data blocks in the free range, so that any future readers will find 711 * empty blocks. Also, if we happen accross any level-1 dbufs in the 712 * range that have not already been marked dirty, mark them dirty so 713 * they stay in memory. 714 */ 715 void 716 dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx) 717 { 718 dmu_buf_impl_t *db, *db_next; 719 uint64_t txg = tx->tx_txg; 720 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 721 uint64_t first_l1 = start >> epbs; 722 uint64_t last_l1 = end >> epbs; 723 724 if (end > dn->dn_maxblkid) { 725 end = dn->dn_maxblkid; 726 last_l1 = end >> epbs; 727 } 728 dprintf_dnode(dn, "start=%llu end=%llu\n", start, end); 729 mutex_enter(&dn->dn_dbufs_mtx); 730 for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 731 db_next = list_next(&dn->dn_dbufs, db); 732 ASSERT(db->db_blkid != DB_BONUS_BLKID); 733 734 if (db->db_level == 1 && 735 db->db_blkid >= first_l1 && db->db_blkid <= last_l1) { 736 mutex_enter(&db->db_mtx); 737 if (db->db_last_dirty && 738 db->db_last_dirty->dr_txg < txg) { 739 dbuf_add_ref(db, FTAG); 740 mutex_exit(&db->db_mtx); 741 dbuf_will_dirty(db, tx); 742 dbuf_rele(db, FTAG); 743 } else { 744 mutex_exit(&db->db_mtx); 745 } 746 } 747 748 if (db->db_level != 0) 749 continue; 750 dprintf_dbuf(db, "found buf %s\n", ""); 751 if (db->db_blkid < start || db->db_blkid > end) 752 continue; 753 754 /* found a level 0 buffer in the range */ 755 if (dbuf_undirty(db, tx)) 756 continue; 757 758 mutex_enter(&db->db_mtx); 759 if (db->db_state == DB_UNCACHED || 760 db->db_state == DB_EVICTING) { 761 ASSERT(db->db.db_data == NULL); 762 mutex_exit(&db->db_mtx); 763 continue; 764 } 765 if (db->db_state == DB_READ || db->db_state == DB_FILL) { 766 /* will be handled in dbuf_read_done or dbuf_rele */ 767 db->db_freed_in_flight = TRUE; 768 mutex_exit(&db->db_mtx); 769 continue; 770 } 771 if (refcount_count(&db->db_holds) == 0) { 772 ASSERT(db->db_buf); 773 dbuf_clear(db); 774 continue; 775 } 776 /* The dbuf is referenced */ 777 778 if (db->db_last_dirty != NULL) { 779 dbuf_dirty_record_t *dr = db->db_last_dirty; 780 781 if (dr->dr_txg == txg) { 782 /* 783 * This buffer is "in-use", re-adjust the file 784 * size to reflect that this buffer may 785 * contain new data when we sync. 786 */ 787 if (db->db_blkid > dn->dn_maxblkid) 788 dn->dn_maxblkid = db->db_blkid; 789 dbuf_unoverride(dr); 790 } else { 791 /* 792 * This dbuf is not dirty in the open context. 793 * Either uncache it (if its not referenced in 794 * the open context) or reset its contents to 795 * empty. 796 */ 797 dbuf_fix_old_data(db, txg); 798 } 799 } 800 /* clear the contents if its cached */ 801 if (db->db_state == DB_CACHED) { 802 ASSERT(db->db.db_data != NULL); 803 arc_release(db->db_buf, db); 804 bzero(db->db.db_data, db->db.db_size); 805 arc_buf_freeze(db->db_buf); 806 } 807 808 mutex_exit(&db->db_mtx); 809 } 810 mutex_exit(&dn->dn_dbufs_mtx); 811 } 812 813 static int 814 dbuf_block_freeable(dmu_buf_impl_t *db) 815 { 816 dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 817 uint64_t birth_txg = 0; 818 819 /* 820 * We don't need any locking to protect db_blkptr: 821 * If it's syncing, then db_last_dirty will be set 822 * so we'll ignore db_blkptr. 823 */ 824 ASSERT(MUTEX_HELD(&db->db_mtx)); 825 if (db->db_last_dirty) 826 birth_txg = db->db_last_dirty->dr_txg; 827 else if (db->db_blkptr) 828 birth_txg = db->db_blkptr->blk_birth; 829 830 /* If we don't exist or are in a snapshot, we can't be freed */ 831 if (birth_txg) 832 return (ds == NULL || 833 dsl_dataset_block_freeable(ds, birth_txg)); 834 else 835 return (FALSE); 836 } 837 838 void 839 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 840 { 841 arc_buf_t *buf, *obuf; 842 int osize = db->db.db_size; 843 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 844 845 ASSERT(db->db_blkid != DB_BONUS_BLKID); 846 847 /* XXX does *this* func really need the lock? */ 848 ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)); 849 850 /* 851 * This call to dbuf_will_dirty() with the dn_struct_rwlock held 852 * is OK, because there can be no other references to the db 853 * when we are changing its size, so no concurrent DB_FILL can 854 * be happening. 855 */ 856 /* 857 * XXX we should be doing a dbuf_read, checking the return 858 * value and returning that up to our callers 859 */ 860 dbuf_will_dirty(db, tx); 861 862 /* create the data buffer for the new block */ 863 buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db, type); 864 865 /* copy old block data to the new block */ 866 obuf = db->db_buf; 867 bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 868 /* zero the remainder */ 869 if (size > osize) 870 bzero((uint8_t *)buf->b_data + osize, size - osize); 871 872 mutex_enter(&db->db_mtx); 873 dbuf_set_data(db, buf); 874 VERIFY(arc_buf_remove_ref(obuf, db) == 1); 875 db->db.db_size = size; 876 877 if (db->db_level == 0) { 878 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 879 db->db_last_dirty->dt.dl.dr_data = buf; 880 } 881 mutex_exit(&db->db_mtx); 882 883 dnode_willuse_space(db->db_dnode, size-osize, tx); 884 } 885 886 dbuf_dirty_record_t * 887 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 888 { 889 dnode_t *dn = db->db_dnode; 890 objset_impl_t *os = dn->dn_objset; 891 dbuf_dirty_record_t **drp, *dr; 892 int drop_struct_lock = FALSE; 893 boolean_t do_free_accounting = B_FALSE; 894 int txgoff = tx->tx_txg & TXG_MASK; 895 896 ASSERT(tx->tx_txg != 0); 897 ASSERT(!refcount_is_zero(&db->db_holds)); 898 DMU_TX_DIRTY_BUF(tx, db); 899 900 /* 901 * Shouldn't dirty a regular buffer in syncing context. Private 902 * objects may be dirtied in syncing context, but only if they 903 * were already pre-dirtied in open context. 904 * XXX We may want to prohibit dirtying in syncing context even 905 * if they did pre-dirty. 906 */ 907 ASSERT(!dmu_tx_is_syncing(tx) || 908 BP_IS_HOLE(dn->dn_objset->os_rootbp) || 909 dn->dn_object == DMU_META_DNODE_OBJECT || 910 dn->dn_objset->os_dsl_dataset == NULL || 911 dsl_dir_is_private(dn->dn_objset->os_dsl_dataset->ds_dir)); 912 913 /* 914 * We make this assert for private objects as well, but after we 915 * check if we're already dirty. They are allowed to re-dirty 916 * in syncing context. 917 */ 918 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 919 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 920 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 921 922 mutex_enter(&db->db_mtx); 923 /* 924 * XXX make this true for indirects too? The problem is that 925 * transactions created with dmu_tx_create_assigned() from 926 * syncing context don't bother holding ahead. 927 */ 928 ASSERT(db->db_level != 0 || 929 db->db_state == DB_CACHED || db->db_state == DB_FILL); 930 931 mutex_enter(&dn->dn_mtx); 932 /* 933 * Don't set dirtyctx to SYNC if we're just modifying this as we 934 * initialize the objset. 935 */ 936 if (dn->dn_dirtyctx == DN_UNDIRTIED && 937 !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 938 dn->dn_dirtyctx = 939 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 940 ASSERT(dn->dn_dirtyctx_firstset == NULL); 941 dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 942 } 943 mutex_exit(&dn->dn_mtx); 944 945 /* 946 * If this buffer is already dirty, we're done. 947 */ 948 drp = &db->db_last_dirty; 949 ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 950 db->db.db_object == DMU_META_DNODE_OBJECT); 951 while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 952 drp = &dr->dr_next; 953 if (dr && dr->dr_txg == tx->tx_txg) { 954 if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 955 /* 956 * If this buffer has already been written out, 957 * we now need to reset its state. 958 */ 959 dbuf_unoverride(dr); 960 if (db->db.db_object != DMU_META_DNODE_OBJECT) 961 arc_buf_thaw(db->db_buf); 962 } 963 mutex_exit(&db->db_mtx); 964 return (dr); 965 } 966 967 /* 968 * Only valid if not already dirty. 969 */ 970 ASSERT(dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 971 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 972 973 ASSERT3U(dn->dn_nlevels, >, db->db_level); 974 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 975 dn->dn_phys->dn_nlevels > db->db_level || 976 dn->dn_next_nlevels[txgoff] > db->db_level || 977 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 978 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 979 980 /* 981 * We should only be dirtying in syncing context if it's the 982 * mos, a spa os, or we're initializing the os. However, we are 983 * allowed to dirty in syncing context provided we already 984 * dirtied it in open context. Hence we must make this 985 * assertion only if we're not already dirty. 986 */ 987 ASSERT(!dmu_tx_is_syncing(tx) || 988 os->os_dsl_dataset == NULL || 989 !dsl_dir_is_private(os->os_dsl_dataset->ds_dir) || 990 !BP_IS_HOLE(os->os_rootbp)); 991 ASSERT(db->db.db_size != 0); 992 993 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 994 995 if (db->db_blkid != DB_BONUS_BLKID) { 996 /* 997 * Update the accounting. 998 * Note: we delay "free accounting" until after we drop 999 * the db_mtx. This keeps us from grabbing other locks 1000 * (and possibly deadlocking) in bp_get_dasize() while 1001 * also holding the db_mtx. 1002 */ 1003 dnode_willuse_space(dn, db->db.db_size, tx); 1004 do_free_accounting = dbuf_block_freeable(db); 1005 } 1006 1007 /* 1008 * If this buffer is dirty in an old transaction group we need 1009 * to make a copy of it so that the changes we make in this 1010 * transaction group won't leak out when we sync the older txg. 1011 */ 1012 dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1013 if (db->db_level == 0) { 1014 void *data_old = db->db_buf; 1015 1016 if (db->db_blkid == DB_BONUS_BLKID) { 1017 dbuf_fix_old_data(db, tx->tx_txg); 1018 data_old = db->db.db_data; 1019 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1020 /* 1021 * Release the data buffer from the cache so that we 1022 * can modify it without impacting possible other users 1023 * of this cached data block. Note that indirect 1024 * blocks and private objects are not released until the 1025 * syncing state (since they are only modified then). 1026 */ 1027 arc_release(db->db_buf, db); 1028 dbuf_fix_old_data(db, tx->tx_txg); 1029 data_old = db->db_buf; 1030 } 1031 ASSERT(data_old != NULL); 1032 dr->dt.dl.dr_data = data_old; 1033 } else { 1034 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1035 list_create(&dr->dt.di.dr_children, 1036 sizeof (dbuf_dirty_record_t), 1037 offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1038 } 1039 dr->dr_dbuf = db; 1040 dr->dr_txg = tx->tx_txg; 1041 dr->dr_next = *drp; 1042 *drp = dr; 1043 1044 /* 1045 * We could have been freed_in_flight between the dbuf_noread 1046 * and dbuf_dirty. We win, as though the dbuf_noread() had 1047 * happened after the free. 1048 */ 1049 if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 1050 mutex_enter(&dn->dn_mtx); 1051 dnode_clear_range(dn, db->db_blkid, 1, tx); 1052 mutex_exit(&dn->dn_mtx); 1053 db->db_freed_in_flight = FALSE; 1054 } 1055 1056 /* 1057 * This buffer is now part of this txg 1058 */ 1059 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1060 db->db_dirtycnt += 1; 1061 ASSERT3U(db->db_dirtycnt, <=, 3); 1062 1063 mutex_exit(&db->db_mtx); 1064 1065 if (db->db_blkid == DB_BONUS_BLKID) { 1066 mutex_enter(&dn->dn_mtx); 1067 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1068 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1069 mutex_exit(&dn->dn_mtx); 1070 dnode_setdirty(dn, tx); 1071 return (dr); 1072 } else if (do_free_accounting) { 1073 blkptr_t *bp = db->db_blkptr; 1074 int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1075 bp_get_dasize(os->os_spa, bp) : db->db.db_size; 1076 /* 1077 * This is only a guess -- if the dbuf is dirty 1078 * in a previous txg, we don't know how much 1079 * space it will use on disk yet. We should 1080 * really have the struct_rwlock to access 1081 * db_blkptr, but since this is just a guess, 1082 * it's OK if we get an odd answer. 1083 */ 1084 dnode_willuse_space(dn, -willfree, tx); 1085 } 1086 1087 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1088 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1089 drop_struct_lock = TRUE; 1090 } 1091 1092 if (db->db_level == 0) { 1093 dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 1094 ASSERT(dn->dn_maxblkid >= db->db_blkid); 1095 } 1096 1097 if (db->db_level+1 < dn->dn_nlevels) { 1098 dmu_buf_impl_t *parent = db->db_parent; 1099 dbuf_dirty_record_t *di; 1100 int parent_held = FALSE; 1101 1102 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1103 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1104 1105 parent = dbuf_hold_level(dn, db->db_level+1, 1106 db->db_blkid >> epbs, FTAG); 1107 parent_held = TRUE; 1108 } 1109 if (drop_struct_lock) 1110 rw_exit(&dn->dn_struct_rwlock); 1111 ASSERT3U(db->db_level+1, ==, parent->db_level); 1112 di = dbuf_dirty(parent, tx); 1113 if (parent_held) 1114 dbuf_rele(parent, FTAG); 1115 1116 mutex_enter(&db->db_mtx); 1117 /* possible race with dbuf_undirty() */ 1118 if (db->db_last_dirty == dr || 1119 dn->dn_object == DMU_META_DNODE_OBJECT) { 1120 mutex_enter(&di->dt.di.dr_mtx); 1121 ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1122 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1123 list_insert_tail(&di->dt.di.dr_children, dr); 1124 mutex_exit(&di->dt.di.dr_mtx); 1125 dr->dr_parent = di; 1126 } 1127 mutex_exit(&db->db_mtx); 1128 } else { 1129 ASSERT(db->db_level+1 == dn->dn_nlevels); 1130 ASSERT(db->db_blkid < dn->dn_nblkptr); 1131 ASSERT(db->db_parent == NULL || 1132 db->db_parent == db->db_dnode->dn_dbuf); 1133 mutex_enter(&dn->dn_mtx); 1134 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1135 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1136 mutex_exit(&dn->dn_mtx); 1137 if (drop_struct_lock) 1138 rw_exit(&dn->dn_struct_rwlock); 1139 } 1140 1141 dnode_setdirty(dn, tx); 1142 return (dr); 1143 } 1144 1145 static int 1146 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1147 { 1148 dnode_t *dn = db->db_dnode; 1149 uint64_t txg = tx->tx_txg; 1150 dbuf_dirty_record_t *dr, **drp; 1151 1152 ASSERT(txg != 0); 1153 ASSERT(db->db_blkid != DB_BONUS_BLKID); 1154 1155 mutex_enter(&db->db_mtx); 1156 1157 /* 1158 * If this buffer is not dirty, we're done. 1159 */ 1160 for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1161 if (dr->dr_txg <= txg) 1162 break; 1163 if (dr == NULL || dr->dr_txg < txg) { 1164 mutex_exit(&db->db_mtx); 1165 return (0); 1166 } 1167 ASSERT(dr->dr_txg == txg); 1168 1169 /* 1170 * If this buffer is currently held, we cannot undirty 1171 * it, since one of the current holders may be in the 1172 * middle of an update. Note that users of dbuf_undirty() 1173 * should not place a hold on the dbuf before the call. 1174 */ 1175 if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1176 mutex_exit(&db->db_mtx); 1177 /* Make sure we don't toss this buffer at sync phase */ 1178 mutex_enter(&dn->dn_mtx); 1179 dnode_clear_range(dn, db->db_blkid, 1, tx); 1180 mutex_exit(&dn->dn_mtx); 1181 return (0); 1182 } 1183 1184 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1185 1186 ASSERT(db->db.db_size != 0); 1187 1188 /* XXX would be nice to fix up dn_towrite_space[] */ 1189 1190 *drp = dr->dr_next; 1191 1192 if (dr->dr_parent) { 1193 mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1194 list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1195 mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1196 } else if (db->db_level+1 == dn->dn_nlevels) { 1197 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1198 mutex_enter(&dn->dn_mtx); 1199 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1200 mutex_exit(&dn->dn_mtx); 1201 } 1202 1203 if (db->db_level == 0) { 1204 dbuf_unoverride(dr); 1205 1206 ASSERT(db->db_buf != NULL); 1207 ASSERT(dr->dt.dl.dr_data != NULL); 1208 if (dr->dt.dl.dr_data != db->db_buf) 1209 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db) == 1); 1210 } else { 1211 ASSERT(db->db_buf != NULL); 1212 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1213 mutex_destroy(&dr->dt.di.dr_mtx); 1214 list_destroy(&dr->dt.di.dr_children); 1215 } 1216 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1217 1218 ASSERT(db->db_dirtycnt > 0); 1219 db->db_dirtycnt -= 1; 1220 1221 if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1222 arc_buf_t *buf = db->db_buf; 1223 1224 ASSERT(arc_released(buf)); 1225 dbuf_set_data(db, NULL); 1226 VERIFY(arc_buf_remove_ref(buf, db) == 1); 1227 dbuf_evict(db); 1228 return (1); 1229 } 1230 1231 mutex_exit(&db->db_mtx); 1232 return (0); 1233 } 1234 1235 #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1236 void 1237 dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1238 { 1239 int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1240 1241 ASSERT(tx->tx_txg != 0); 1242 ASSERT(!refcount_is_zero(&db->db_holds)); 1243 1244 if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) 1245 rf |= DB_RF_HAVESTRUCT; 1246 (void) dbuf_read(db, NULL, rf); 1247 (void) dbuf_dirty(db, tx); 1248 } 1249 1250 void 1251 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1252 { 1253 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1254 1255 ASSERT(db->db_blkid != DB_BONUS_BLKID); 1256 ASSERT(tx->tx_txg != 0); 1257 ASSERT(db->db_level == 0); 1258 ASSERT(!refcount_is_zero(&db->db_holds)); 1259 1260 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1261 dmu_tx_private_ok(tx)); 1262 1263 dbuf_noread(db); 1264 (void) dbuf_dirty(db, tx); 1265 } 1266 1267 #pragma weak dmu_buf_fill_done = dbuf_fill_done 1268 /* ARGSUSED */ 1269 void 1270 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1271 { 1272 mutex_enter(&db->db_mtx); 1273 DBUF_VERIFY(db); 1274 1275 if (db->db_state == DB_FILL) { 1276 if (db->db_level == 0 && db->db_freed_in_flight) { 1277 ASSERT(db->db_blkid != DB_BONUS_BLKID); 1278 /* we were freed while filling */ 1279 /* XXX dbuf_undirty? */ 1280 bzero(db->db.db_data, db->db.db_size); 1281 db->db_freed_in_flight = FALSE; 1282 } 1283 db->db_state = DB_CACHED; 1284 cv_broadcast(&db->db_changed); 1285 } 1286 mutex_exit(&db->db_mtx); 1287 } 1288 1289 /* 1290 * "Clear" the contents of this dbuf. This will mark the dbuf 1291 * EVICTING and clear *most* of its references. Unfortunetely, 1292 * when we are not holding the dn_dbufs_mtx, we can't clear the 1293 * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1294 * in this case. For callers from the DMU we will usually see: 1295 * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1296 * For the arc callback, we will usually see: 1297 * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1298 * Sometimes, though, we will get a mix of these two: 1299 * DMU: dbuf_clear()->arc_buf_evict() 1300 * ARC: dbuf_do_evict()->dbuf_destroy() 1301 */ 1302 void 1303 dbuf_clear(dmu_buf_impl_t *db) 1304 { 1305 dnode_t *dn = db->db_dnode; 1306 dmu_buf_impl_t *parent = db->db_parent; 1307 dmu_buf_impl_t *dndb = dn->dn_dbuf; 1308 int dbuf_gone = FALSE; 1309 1310 ASSERT(MUTEX_HELD(&db->db_mtx)); 1311 ASSERT(refcount_is_zero(&db->db_holds)); 1312 1313 dbuf_evict_user(db); 1314 1315 if (db->db_state == DB_CACHED) { 1316 ASSERT(db->db.db_data != NULL); 1317 if (db->db_blkid == DB_BONUS_BLKID) { 1318 zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 1319 arc_space_return(DN_MAX_BONUSLEN); 1320 } 1321 db->db.db_data = NULL; 1322 db->db_state = DB_UNCACHED; 1323 } 1324 1325 ASSERT3U(db->db_state, ==, DB_UNCACHED); 1326 ASSERT(db->db_data_pending == NULL); 1327 1328 db->db_state = DB_EVICTING; 1329 db->db_blkptr = NULL; 1330 1331 if (db->db_blkid != DB_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1332 list_remove(&dn->dn_dbufs, db); 1333 dnode_rele(dn, db); 1334 db->db_dnode = NULL; 1335 } 1336 1337 if (db->db_buf) 1338 dbuf_gone = arc_buf_evict(db->db_buf); 1339 1340 if (!dbuf_gone) 1341 mutex_exit(&db->db_mtx); 1342 1343 /* 1344 * If this dbuf is referened from an indirect dbuf, 1345 * decrement the ref count on the indirect dbuf. 1346 */ 1347 if (parent && parent != dndb) 1348 dbuf_rele(parent, db); 1349 } 1350 1351 static int 1352 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1353 dmu_buf_impl_t **parentp, blkptr_t **bpp) 1354 { 1355 int nlevels, epbs; 1356 1357 *parentp = NULL; 1358 *bpp = NULL; 1359 1360 ASSERT(blkid != DB_BONUS_BLKID); 1361 1362 if (dn->dn_phys->dn_nlevels == 0) 1363 nlevels = 1; 1364 else 1365 nlevels = dn->dn_phys->dn_nlevels; 1366 1367 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1368 1369 ASSERT3U(level * epbs, <, 64); 1370 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1371 if (level >= nlevels || 1372 (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1373 /* the buffer has no parent yet */ 1374 return (ENOENT); 1375 } else if (level < nlevels-1) { 1376 /* this block is referenced from an indirect block */ 1377 int err = dbuf_hold_impl(dn, level+1, 1378 blkid >> epbs, fail_sparse, NULL, parentp); 1379 if (err) 1380 return (err); 1381 err = dbuf_read(*parentp, NULL, 1382 (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1383 if (err) { 1384 dbuf_rele(*parentp, NULL); 1385 *parentp = NULL; 1386 return (err); 1387 } 1388 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1389 (blkid & ((1ULL << epbs) - 1)); 1390 return (0); 1391 } else { 1392 /* the block is referenced from the dnode */ 1393 ASSERT3U(level, ==, nlevels-1); 1394 ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1395 blkid < dn->dn_phys->dn_nblkptr); 1396 if (dn->dn_dbuf) { 1397 dbuf_add_ref(dn->dn_dbuf, NULL); 1398 *parentp = dn->dn_dbuf; 1399 } 1400 *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1401 return (0); 1402 } 1403 } 1404 1405 static dmu_buf_impl_t * 1406 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1407 dmu_buf_impl_t *parent, blkptr_t *blkptr) 1408 { 1409 objset_impl_t *os = dn->dn_objset; 1410 dmu_buf_impl_t *db, *odb; 1411 1412 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1413 ASSERT(dn->dn_type != DMU_OT_NONE); 1414 1415 db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1416 1417 db->db_objset = os; 1418 db->db.db_object = dn->dn_object; 1419 db->db_level = level; 1420 db->db_blkid = blkid; 1421 db->db_last_dirty = NULL; 1422 db->db_dirtycnt = 0; 1423 db->db_dnode = dn; 1424 db->db_parent = parent; 1425 db->db_blkptr = blkptr; 1426 1427 db->db_user_ptr = NULL; 1428 db->db_user_data_ptr_ptr = NULL; 1429 db->db_evict_func = NULL; 1430 db->db_immediate_evict = 0; 1431 db->db_freed_in_flight = 0; 1432 1433 if (blkid == DB_BONUS_BLKID) { 1434 ASSERT3P(parent, ==, dn->dn_dbuf); 1435 db->db.db_size = DN_MAX_BONUSLEN - 1436 (dn->dn_nblkptr-1) * sizeof (blkptr_t); 1437 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 1438 db->db.db_offset = DB_BONUS_BLKID; 1439 db->db_state = DB_UNCACHED; 1440 /* the bonus dbuf is not placed in the hash table */ 1441 arc_space_consume(sizeof (dmu_buf_impl_t)); 1442 return (db); 1443 } else { 1444 int blocksize = 1445 db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1446 db->db.db_size = blocksize; 1447 db->db.db_offset = db->db_blkid * blocksize; 1448 } 1449 1450 /* 1451 * Hold the dn_dbufs_mtx while we get the new dbuf 1452 * in the hash table *and* added to the dbufs list. 1453 * This prevents a possible deadlock with someone 1454 * trying to look up this dbuf before its added to the 1455 * dn_dbufs list. 1456 */ 1457 mutex_enter(&dn->dn_dbufs_mtx); 1458 db->db_state = DB_EVICTING; 1459 if ((odb = dbuf_hash_insert(db)) != NULL) { 1460 /* someone else inserted it first */ 1461 kmem_cache_free(dbuf_cache, db); 1462 mutex_exit(&dn->dn_dbufs_mtx); 1463 return (odb); 1464 } 1465 list_insert_head(&dn->dn_dbufs, db); 1466 db->db_state = DB_UNCACHED; 1467 mutex_exit(&dn->dn_dbufs_mtx); 1468 arc_space_consume(sizeof (dmu_buf_impl_t)); 1469 1470 if (parent && parent != dn->dn_dbuf) 1471 dbuf_add_ref(parent, db); 1472 1473 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1474 refcount_count(&dn->dn_holds) > 0); 1475 (void) refcount_add(&dn->dn_holds, db); 1476 1477 dprintf_dbuf(db, "db=%p\n", db); 1478 1479 return (db); 1480 } 1481 1482 static int 1483 dbuf_do_evict(void *private) 1484 { 1485 arc_buf_t *buf = private; 1486 dmu_buf_impl_t *db = buf->b_private; 1487 1488 if (!MUTEX_HELD(&db->db_mtx)) 1489 mutex_enter(&db->db_mtx); 1490 1491 ASSERT(refcount_is_zero(&db->db_holds)); 1492 1493 if (db->db_state != DB_EVICTING) { 1494 ASSERT(db->db_state == DB_CACHED); 1495 DBUF_VERIFY(db); 1496 db->db_buf = NULL; 1497 dbuf_evict(db); 1498 } else { 1499 mutex_exit(&db->db_mtx); 1500 dbuf_destroy(db); 1501 } 1502 return (0); 1503 } 1504 1505 static void 1506 dbuf_destroy(dmu_buf_impl_t *db) 1507 { 1508 ASSERT(refcount_is_zero(&db->db_holds)); 1509 1510 if (db->db_blkid != DB_BONUS_BLKID) { 1511 /* 1512 * If this dbuf is still on the dn_dbufs list, 1513 * remove it from that list. 1514 */ 1515 if (db->db_dnode) { 1516 dnode_t *dn = db->db_dnode; 1517 1518 mutex_enter(&dn->dn_dbufs_mtx); 1519 list_remove(&dn->dn_dbufs, db); 1520 mutex_exit(&dn->dn_dbufs_mtx); 1521 1522 dnode_rele(dn, db); 1523 db->db_dnode = NULL; 1524 } 1525 dbuf_hash_remove(db); 1526 } 1527 db->db_parent = NULL; 1528 db->db_buf = NULL; 1529 1530 ASSERT(!list_link_active(&db->db_link)); 1531 ASSERT(db->db.db_data == NULL); 1532 ASSERT(db->db_hash_next == NULL); 1533 ASSERT(db->db_blkptr == NULL); 1534 ASSERT(db->db_data_pending == NULL); 1535 1536 kmem_cache_free(dbuf_cache, db); 1537 arc_space_return(sizeof (dmu_buf_impl_t)); 1538 } 1539 1540 void 1541 dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1542 { 1543 dmu_buf_impl_t *db = NULL; 1544 blkptr_t *bp = NULL; 1545 1546 ASSERT(blkid != DB_BONUS_BLKID); 1547 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1548 1549 if (dnode_block_freed(dn, blkid)) 1550 return; 1551 1552 /* dbuf_find() returns with db_mtx held */ 1553 if (db = dbuf_find(dn, 0, blkid)) { 1554 if (refcount_count(&db->db_holds) > 0) { 1555 /* 1556 * This dbuf is active. We assume that it is 1557 * already CACHED, or else about to be either 1558 * read or filled. 1559 */ 1560 mutex_exit(&db->db_mtx); 1561 return; 1562 } 1563 mutex_exit(&db->db_mtx); 1564 db = NULL; 1565 } 1566 1567 if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1568 if (bp && !BP_IS_HOLE(bp)) { 1569 arc_buf_t *pbuf; 1570 uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1571 zbookmark_t zb; 1572 zb.zb_objset = dn->dn_objset->os_dsl_dataset ? 1573 dn->dn_objset->os_dsl_dataset->ds_object : 0; 1574 zb.zb_object = dn->dn_object; 1575 zb.zb_level = 0; 1576 zb.zb_blkid = blkid; 1577 1578 if (db) 1579 pbuf = db->db_buf; 1580 else 1581 pbuf = dn->dn_objset->os_phys_buf; 1582 1583 (void) arc_read(NULL, dn->dn_objset->os_spa, 1584 bp, pbuf, NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 1585 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 1586 &aflags, &zb); 1587 } 1588 if (db) 1589 dbuf_rele(db, NULL); 1590 } 1591 } 1592 1593 /* 1594 * Returns with db_holds incremented, and db_mtx not held. 1595 * Note: dn_struct_rwlock must be held. 1596 */ 1597 int 1598 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1599 void *tag, dmu_buf_impl_t **dbp) 1600 { 1601 dmu_buf_impl_t *db, *parent = NULL; 1602 1603 ASSERT(blkid != DB_BONUS_BLKID); 1604 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1605 ASSERT3U(dn->dn_nlevels, >, level); 1606 1607 *dbp = NULL; 1608 top: 1609 /* dbuf_find() returns with db_mtx held */ 1610 db = dbuf_find(dn, level, blkid); 1611 1612 if (db == NULL) { 1613 blkptr_t *bp = NULL; 1614 int err; 1615 1616 ASSERT3P(parent, ==, NULL); 1617 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1618 if (fail_sparse) { 1619 if (err == 0 && bp && BP_IS_HOLE(bp)) 1620 err = ENOENT; 1621 if (err) { 1622 if (parent) 1623 dbuf_rele(parent, NULL); 1624 return (err); 1625 } 1626 } 1627 if (err && err != ENOENT) 1628 return (err); 1629 db = dbuf_create(dn, level, blkid, parent, bp); 1630 } 1631 1632 if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1633 arc_buf_add_ref(db->db_buf, db); 1634 if (db->db_buf->b_data == NULL) { 1635 dbuf_clear(db); 1636 if (parent) { 1637 dbuf_rele(parent, NULL); 1638 parent = NULL; 1639 } 1640 goto top; 1641 } 1642 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1643 } 1644 1645 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1646 1647 /* 1648 * If this buffer is currently syncing out, and we are are 1649 * still referencing it from db_data, we need to make a copy 1650 * of it in case we decide we want to dirty it again in this txg. 1651 */ 1652 if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 1653 dn->dn_object != DMU_META_DNODE_OBJECT && 1654 db->db_state == DB_CACHED && db->db_data_pending) { 1655 dbuf_dirty_record_t *dr = db->db_data_pending; 1656 1657 if (dr->dt.dl.dr_data == db->db_buf) { 1658 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1659 1660 dbuf_set_data(db, 1661 arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 1662 db->db.db_size, db, type)); 1663 bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1664 db->db.db_size); 1665 } 1666 } 1667 1668 (void) refcount_add(&db->db_holds, tag); 1669 dbuf_update_data(db); 1670 DBUF_VERIFY(db); 1671 mutex_exit(&db->db_mtx); 1672 1673 /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1674 if (parent) 1675 dbuf_rele(parent, NULL); 1676 1677 ASSERT3P(db->db_dnode, ==, dn); 1678 ASSERT3U(db->db_blkid, ==, blkid); 1679 ASSERT3U(db->db_level, ==, level); 1680 *dbp = db; 1681 1682 return (0); 1683 } 1684 1685 dmu_buf_impl_t * 1686 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1687 { 1688 dmu_buf_impl_t *db; 1689 int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1690 return (err ? NULL : db); 1691 } 1692 1693 dmu_buf_impl_t * 1694 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1695 { 1696 dmu_buf_impl_t *db; 1697 int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1698 return (err ? NULL : db); 1699 } 1700 1701 void 1702 dbuf_create_bonus(dnode_t *dn) 1703 { 1704 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1705 1706 ASSERT(dn->dn_bonus == NULL); 1707 dn->dn_bonus = dbuf_create(dn, 0, DB_BONUS_BLKID, dn->dn_dbuf, NULL); 1708 } 1709 1710 #pragma weak dmu_buf_add_ref = dbuf_add_ref 1711 void 1712 dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 1713 { 1714 int64_t holds = refcount_add(&db->db_holds, tag); 1715 ASSERT(holds > 1); 1716 } 1717 1718 #pragma weak dmu_buf_rele = dbuf_rele 1719 void 1720 dbuf_rele(dmu_buf_impl_t *db, void *tag) 1721 { 1722 int64_t holds; 1723 1724 mutex_enter(&db->db_mtx); 1725 DBUF_VERIFY(db); 1726 1727 holds = refcount_remove(&db->db_holds, tag); 1728 ASSERT(holds >= 0); 1729 1730 /* 1731 * We can't freeze indirects if there is a possibility that they 1732 * may be modified in the current syncing context. 1733 */ 1734 if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 1735 arc_buf_freeze(db->db_buf); 1736 1737 if (holds == db->db_dirtycnt && 1738 db->db_level == 0 && db->db_immediate_evict) 1739 dbuf_evict_user(db); 1740 1741 if (holds == 0) { 1742 if (db->db_blkid == DB_BONUS_BLKID) { 1743 mutex_exit(&db->db_mtx); 1744 dnode_rele(db->db_dnode, db); 1745 } else if (db->db_buf == NULL) { 1746 /* 1747 * This is a special case: we never associated this 1748 * dbuf with any data allocated from the ARC. 1749 */ 1750 ASSERT3U(db->db_state, ==, DB_UNCACHED); 1751 dbuf_evict(db); 1752 } else if (arc_released(db->db_buf)) { 1753 arc_buf_t *buf = db->db_buf; 1754 /* 1755 * This dbuf has anonymous data associated with it. 1756 */ 1757 dbuf_set_data(db, NULL); 1758 VERIFY(arc_buf_remove_ref(buf, db) == 1); 1759 dbuf_evict(db); 1760 } else { 1761 VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0); 1762 if (!DBUF_IS_CACHEABLE(db)) 1763 dbuf_clear(db); 1764 else 1765 mutex_exit(&db->db_mtx); 1766 } 1767 } else { 1768 mutex_exit(&db->db_mtx); 1769 } 1770 } 1771 1772 #pragma weak dmu_buf_refcount = dbuf_refcount 1773 uint64_t 1774 dbuf_refcount(dmu_buf_impl_t *db) 1775 { 1776 return (refcount_count(&db->db_holds)); 1777 } 1778 1779 void * 1780 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1781 dmu_buf_evict_func_t *evict_func) 1782 { 1783 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1784 user_data_ptr_ptr, evict_func)); 1785 } 1786 1787 void * 1788 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1789 dmu_buf_evict_func_t *evict_func) 1790 { 1791 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1792 1793 db->db_immediate_evict = TRUE; 1794 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1795 user_data_ptr_ptr, evict_func)); 1796 } 1797 1798 void * 1799 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 1800 void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 1801 { 1802 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1803 ASSERT(db->db_level == 0); 1804 1805 ASSERT((user_ptr == NULL) == (evict_func == NULL)); 1806 1807 mutex_enter(&db->db_mtx); 1808 1809 if (db->db_user_ptr == old_user_ptr) { 1810 db->db_user_ptr = user_ptr; 1811 db->db_user_data_ptr_ptr = user_data_ptr_ptr; 1812 db->db_evict_func = evict_func; 1813 1814 dbuf_update_data(db); 1815 } else { 1816 old_user_ptr = db->db_user_ptr; 1817 } 1818 1819 mutex_exit(&db->db_mtx); 1820 return (old_user_ptr); 1821 } 1822 1823 void * 1824 dmu_buf_get_user(dmu_buf_t *db_fake) 1825 { 1826 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1827 ASSERT(!refcount_is_zero(&db->db_holds)); 1828 1829 return (db->db_user_ptr); 1830 } 1831 1832 static void 1833 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 1834 { 1835 /* ASSERT(dmu_tx_is_syncing(tx) */ 1836 ASSERT(MUTEX_HELD(&db->db_mtx)); 1837 1838 if (db->db_blkptr != NULL) 1839 return; 1840 1841 if (db->db_level == dn->dn_phys->dn_nlevels-1) { 1842 /* 1843 * This buffer was allocated at a time when there was 1844 * no available blkptrs from the dnode, or it was 1845 * inappropriate to hook it in (i.e., nlevels mis-match). 1846 */ 1847 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 1848 ASSERT(db->db_parent == NULL); 1849 db->db_parent = dn->dn_dbuf; 1850 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 1851 DBUF_VERIFY(db); 1852 } else { 1853 dmu_buf_impl_t *parent = db->db_parent; 1854 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 1855 1856 ASSERT(dn->dn_phys->dn_nlevels > 1); 1857 if (parent == NULL) { 1858 mutex_exit(&db->db_mtx); 1859 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1860 (void) dbuf_hold_impl(dn, db->db_level+1, 1861 db->db_blkid >> epbs, FALSE, db, &parent); 1862 rw_exit(&dn->dn_struct_rwlock); 1863 mutex_enter(&db->db_mtx); 1864 db->db_parent = parent; 1865 } 1866 db->db_blkptr = (blkptr_t *)parent->db.db_data + 1867 (db->db_blkid & ((1ULL << epbs) - 1)); 1868 DBUF_VERIFY(db); 1869 } 1870 } 1871 1872 static void 1873 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 1874 { 1875 dmu_buf_impl_t *db = dr->dr_dbuf; 1876 dnode_t *dn = db->db_dnode; 1877 zio_t *zio; 1878 1879 ASSERT(dmu_tx_is_syncing(tx)); 1880 1881 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 1882 1883 mutex_enter(&db->db_mtx); 1884 1885 ASSERT(db->db_level > 0); 1886 DBUF_VERIFY(db); 1887 1888 if (db->db_buf == NULL) { 1889 mutex_exit(&db->db_mtx); 1890 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 1891 mutex_enter(&db->db_mtx); 1892 } 1893 ASSERT3U(db->db_state, ==, DB_CACHED); 1894 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 1895 ASSERT(db->db_buf != NULL); 1896 1897 dbuf_check_blkptr(dn, db); 1898 1899 db->db_data_pending = dr; 1900 1901 mutex_exit(&db->db_mtx); 1902 dbuf_write(dr, db->db_buf, tx); 1903 1904 zio = dr->dr_zio; 1905 mutex_enter(&dr->dt.di.dr_mtx); 1906 dbuf_sync_list(&dr->dt.di.dr_children, tx); 1907 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1908 mutex_exit(&dr->dt.di.dr_mtx); 1909 zio_nowait(zio); 1910 } 1911 1912 static void 1913 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 1914 { 1915 arc_buf_t **datap = &dr->dt.dl.dr_data; 1916 dmu_buf_impl_t *db = dr->dr_dbuf; 1917 dnode_t *dn = db->db_dnode; 1918 objset_impl_t *os = dn->dn_objset; 1919 uint64_t txg = tx->tx_txg; 1920 int blksz; 1921 1922 ASSERT(dmu_tx_is_syncing(tx)); 1923 1924 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 1925 1926 mutex_enter(&db->db_mtx); 1927 /* 1928 * To be synced, we must be dirtied. But we 1929 * might have been freed after the dirty. 1930 */ 1931 if (db->db_state == DB_UNCACHED) { 1932 /* This buffer has been freed since it was dirtied */ 1933 ASSERT(db->db.db_data == NULL); 1934 } else if (db->db_state == DB_FILL) { 1935 /* This buffer was freed and is now being re-filled */ 1936 ASSERT(db->db.db_data != dr->dt.dl.dr_data); 1937 } else { 1938 ASSERT3U(db->db_state, ==, DB_CACHED); 1939 } 1940 DBUF_VERIFY(db); 1941 1942 /* 1943 * If this is a bonus buffer, simply copy the bonus data into the 1944 * dnode. It will be written out when the dnode is synced (and it 1945 * will be synced, since it must have been dirty for dbuf_sync to 1946 * be called). 1947 */ 1948 if (db->db_blkid == DB_BONUS_BLKID) { 1949 dbuf_dirty_record_t **drp; 1950 1951 ASSERT(*datap != NULL); 1952 ASSERT3U(db->db_level, ==, 0); 1953 ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 1954 bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 1955 if (*datap != db->db.db_data) { 1956 zio_buf_free(*datap, DN_MAX_BONUSLEN); 1957 arc_space_return(DN_MAX_BONUSLEN); 1958 } 1959 db->db_data_pending = NULL; 1960 drp = &db->db_last_dirty; 1961 while (*drp != dr) 1962 drp = &(*drp)->dr_next; 1963 ASSERT(dr->dr_next == NULL); 1964 *drp = dr->dr_next; 1965 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1966 ASSERT(db->db_dirtycnt > 0); 1967 db->db_dirtycnt -= 1; 1968 mutex_exit(&db->db_mtx); 1969 dbuf_rele(db, (void *)(uintptr_t)txg); 1970 return; 1971 } 1972 1973 /* 1974 * This function may have dropped the db_mtx lock allowing a dmu_sync 1975 * operation to sneak in. As a result, we need to ensure that we 1976 * don't check the dr_override_state until we have returned from 1977 * dbuf_check_blkptr. 1978 */ 1979 dbuf_check_blkptr(dn, db); 1980 1981 /* 1982 * If this buffer is in the middle of an immdiate write, 1983 * wait for the synchronous IO to complete. 1984 */ 1985 while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 1986 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 1987 cv_wait(&db->db_changed, &db->db_mtx); 1988 ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 1989 } 1990 1991 /* 1992 * If this dbuf has already been written out via an immediate write, 1993 * just complete the write by copying over the new block pointer and 1994 * updating the accounting via the write-completion functions. 1995 */ 1996 if (dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 1997 zio_t zio_fake; 1998 1999 zio_fake.io_private = &db; 2000 zio_fake.io_error = 0; 2001 zio_fake.io_bp = db->db_blkptr; 2002 zio_fake.io_bp_orig = *db->db_blkptr; 2003 zio_fake.io_txg = txg; 2004 zio_fake.io_flags = 0; 2005 2006 *db->db_blkptr = dr->dt.dl.dr_overridden_by; 2007 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 2008 db->db_data_pending = dr; 2009 dr->dr_zio = &zio_fake; 2010 mutex_exit(&db->db_mtx); 2011 2012 ASSERT(!DVA_EQUAL(BP_IDENTITY(zio_fake.io_bp), 2013 BP_IDENTITY(&zio_fake.io_bp_orig))); 2014 2015 if (BP_IS_OLDER(&zio_fake.io_bp_orig, txg)) 2016 (void) dsl_dataset_block_kill(os->os_dsl_dataset, 2017 &zio_fake.io_bp_orig, dn->dn_zio, tx); 2018 2019 dbuf_write_ready(&zio_fake, db->db_buf, db); 2020 dbuf_write_done(&zio_fake, db->db_buf, db); 2021 2022 return; 2023 } 2024 2025 blksz = arc_buf_size(*datap); 2026 2027 if (dn->dn_object != DMU_META_DNODE_OBJECT) { 2028 /* 2029 * If this buffer is currently "in use" (i.e., there are 2030 * active holds and db_data still references it), then make 2031 * a copy before we start the write so that any modifications 2032 * from the open txg will not leak into this write. 2033 * 2034 * NOTE: this copy does not need to be made for objects only 2035 * modified in the syncing context (e.g. DNONE_DNODE blocks). 2036 */ 2037 if (refcount_count(&db->db_holds) > 1 && *datap == db->db_buf) { 2038 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2039 *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2040 bcopy(db->db.db_data, (*datap)->b_data, blksz); 2041 } 2042 } 2043 2044 ASSERT(*datap != NULL); 2045 db->db_data_pending = dr; 2046 2047 mutex_exit(&db->db_mtx); 2048 2049 dbuf_write(dr, *datap, tx); 2050 2051 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2052 if (dn->dn_object == DMU_META_DNODE_OBJECT) 2053 list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2054 else 2055 zio_nowait(dr->dr_zio); 2056 } 2057 2058 void 2059 dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2060 { 2061 dbuf_dirty_record_t *dr; 2062 2063 while (dr = list_head(list)) { 2064 if (dr->dr_zio != NULL) { 2065 /* 2066 * If we find an already initialized zio then we 2067 * are processing the meta-dnode, and we have finished. 2068 * The dbufs for all dnodes are put back on the list 2069 * during processing, so that we can zio_wait() 2070 * these IOs after initiating all child IOs. 2071 */ 2072 ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2073 DMU_META_DNODE_OBJECT); 2074 break; 2075 } 2076 list_remove(list, dr); 2077 if (dr->dr_dbuf->db_level > 0) 2078 dbuf_sync_indirect(dr, tx); 2079 else 2080 dbuf_sync_leaf(dr, tx); 2081 } 2082 } 2083 2084 static void 2085 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2086 { 2087 dmu_buf_impl_t *db = dr->dr_dbuf; 2088 dnode_t *dn = db->db_dnode; 2089 objset_impl_t *os = dn->dn_objset; 2090 dmu_buf_impl_t *parent = db->db_parent; 2091 uint64_t txg = tx->tx_txg; 2092 zbookmark_t zb; 2093 writeprops_t wp = { 0 }; 2094 zio_t *zio; 2095 2096 if (!BP_IS_HOLE(db->db_blkptr) && 2097 (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE)) { 2098 /* 2099 * Private object buffers are released here rather 2100 * than in dbuf_dirty() since they are only modified 2101 * in the syncing context and we don't want the 2102 * overhead of making multiple copies of the data. 2103 */ 2104 arc_release(data, db); 2105 } else { 2106 ASSERT(arc_released(data)); 2107 /* XXX why do we need to thaw here? */ 2108 arc_buf_thaw(data); 2109 } 2110 2111 if (parent != dn->dn_dbuf) { 2112 ASSERT(parent && parent->db_data_pending); 2113 ASSERT(db->db_level == parent->db_level-1); 2114 ASSERT(arc_released(parent->db_buf)); 2115 zio = parent->db_data_pending->dr_zio; 2116 } else { 2117 ASSERT(db->db_level == dn->dn_phys->dn_nlevels-1); 2118 ASSERT3P(db->db_blkptr, ==, 2119 &dn->dn_phys->dn_blkptr[db->db_blkid]); 2120 zio = dn->dn_zio; 2121 } 2122 2123 ASSERT(db->db_level == 0 || data == db->db_buf); 2124 ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2125 ASSERT(zio); 2126 2127 zb.zb_objset = os->os_dsl_dataset ? os->os_dsl_dataset->ds_object : 0; 2128 zb.zb_object = db->db.db_object; 2129 zb.zb_level = db->db_level; 2130 zb.zb_blkid = db->db_blkid; 2131 2132 wp.wp_type = dn->dn_type; 2133 wp.wp_level = db->db_level; 2134 wp.wp_copies = os->os_copies; 2135 wp.wp_dncompress = dn->dn_compress; 2136 wp.wp_oscompress = os->os_compress; 2137 wp.wp_dnchecksum = dn->dn_checksum; 2138 wp.wp_oschecksum = os->os_checksum; 2139 2140 if (BP_IS_OLDER(db->db_blkptr, txg)) 2141 (void) dsl_dataset_block_kill( 2142 os->os_dsl_dataset, db->db_blkptr, zio, tx); 2143 2144 dr->dr_zio = arc_write(zio, os->os_spa, &wp, 2145 DBUF_IS_L2CACHEABLE(db), txg, db->db_blkptr, 2146 data, dbuf_write_ready, dbuf_write_done, db, 2147 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2148 } 2149 2150 /* ARGSUSED */ 2151 static void 2152 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2153 { 2154 dmu_buf_impl_t *db = vdb; 2155 dnode_t *dn = db->db_dnode; 2156 objset_impl_t *os = dn->dn_objset; 2157 blkptr_t *bp = zio->io_bp; 2158 blkptr_t *bp_orig = &zio->io_bp_orig; 2159 uint64_t fill = 0; 2160 int old_size, new_size, i; 2161 2162 ASSERT(db->db_blkptr == bp); 2163 2164 dprintf_dbuf_bp(db, bp_orig, "bp_orig: %s", ""); 2165 2166 old_size = bp_get_dasize(os->os_spa, bp_orig); 2167 new_size = bp_get_dasize(os->os_spa, bp); 2168 2169 dnode_diduse_space(dn, new_size - old_size); 2170 2171 if (BP_IS_HOLE(bp)) { 2172 dsl_dataset_t *ds = os->os_dsl_dataset; 2173 dmu_tx_t *tx = os->os_synctx; 2174 2175 if (bp_orig->blk_birth == tx->tx_txg) 2176 (void) dsl_dataset_block_kill(ds, bp_orig, zio, tx); 2177 ASSERT3U(bp->blk_fill, ==, 0); 2178 return; 2179 } 2180 2181 ASSERT(BP_GET_TYPE(bp) == dn->dn_type); 2182 ASSERT(BP_GET_LEVEL(bp) == db->db_level); 2183 2184 mutex_enter(&db->db_mtx); 2185 2186 if (db->db_level == 0) { 2187 mutex_enter(&dn->dn_mtx); 2188 if (db->db_blkid > dn->dn_phys->dn_maxblkid) 2189 dn->dn_phys->dn_maxblkid = db->db_blkid; 2190 mutex_exit(&dn->dn_mtx); 2191 2192 if (dn->dn_type == DMU_OT_DNODE) { 2193 dnode_phys_t *dnp = db->db.db_data; 2194 for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2195 i--, dnp++) { 2196 if (dnp->dn_type != DMU_OT_NONE) 2197 fill++; 2198 } 2199 } else { 2200 fill = 1; 2201 } 2202 } else { 2203 blkptr_t *ibp = db->db.db_data; 2204 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2205 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 2206 if (BP_IS_HOLE(ibp)) 2207 continue; 2208 ASSERT3U(BP_GET_LSIZE(ibp), ==, 2209 db->db_level == 1 ? dn->dn_datablksz : 2210 (1<<dn->dn_phys->dn_indblkshift)); 2211 fill += ibp->blk_fill; 2212 } 2213 } 2214 2215 bp->blk_fill = fill; 2216 2217 mutex_exit(&db->db_mtx); 2218 2219 if (zio->io_flags & ZIO_FLAG_IO_REWRITE) { 2220 ASSERT(DVA_EQUAL(BP_IDENTITY(bp), BP_IDENTITY(bp_orig))); 2221 } else { 2222 dsl_dataset_t *ds = os->os_dsl_dataset; 2223 dmu_tx_t *tx = os->os_synctx; 2224 2225 if (bp_orig->blk_birth == tx->tx_txg) 2226 (void) dsl_dataset_block_kill(ds, bp_orig, zio, tx); 2227 dsl_dataset_block_born(ds, bp, tx); 2228 } 2229 } 2230 2231 /* ARGSUSED */ 2232 static void 2233 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2234 { 2235 dmu_buf_impl_t *db = vdb; 2236 uint64_t txg = zio->io_txg; 2237 dbuf_dirty_record_t **drp, *dr; 2238 2239 ASSERT3U(zio->io_error, ==, 0); 2240 2241 mutex_enter(&db->db_mtx); 2242 2243 drp = &db->db_last_dirty; 2244 while ((dr = *drp) != db->db_data_pending) 2245 drp = &dr->dr_next; 2246 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2247 ASSERT(dr->dr_txg == txg); 2248 ASSERT(dr->dr_next == NULL); 2249 *drp = dr->dr_next; 2250 2251 if (db->db_level == 0) { 2252 ASSERT(db->db_blkid != DB_BONUS_BLKID); 2253 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 2254 2255 if (dr->dt.dl.dr_data != db->db_buf) 2256 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db) == 1); 2257 else if (!BP_IS_HOLE(db->db_blkptr)) 2258 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2259 else 2260 ASSERT(arc_released(db->db_buf)); 2261 } else { 2262 dnode_t *dn = db->db_dnode; 2263 2264 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2265 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2266 if (!BP_IS_HOLE(db->db_blkptr)) { 2267 int epbs = 2268 dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2269 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2270 db->db.db_size); 2271 ASSERT3U(dn->dn_phys->dn_maxblkid 2272 >> (db->db_level * epbs), >=, db->db_blkid); 2273 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2274 } 2275 mutex_destroy(&dr->dt.di.dr_mtx); 2276 list_destroy(&dr->dt.di.dr_children); 2277 } 2278 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2279 2280 cv_broadcast(&db->db_changed); 2281 ASSERT(db->db_dirtycnt > 0); 2282 db->db_dirtycnt -= 1; 2283 db->db_data_pending = NULL; 2284 mutex_exit(&db->db_mtx); 2285 2286 dprintf_dbuf_bp(db, zio->io_bp, "bp: %s", ""); 2287 2288 dbuf_rele(db, (void *)(uintptr_t)txg); 2289 } 2290