1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 24 * Copyright (c) 2012, 2017 by Delphix. All rights reserved. 25 * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 26 * Copyright (c) 2013, Joyent, Inc. All rights reserved. 27 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 28 * Copyright (c) 2014 Integros [integros.com] 29 */ 30 31 #include <sys/zfs_context.h> 32 #include <sys/dmu.h> 33 #include <sys/dmu_send.h> 34 #include <sys/dmu_impl.h> 35 #include <sys/dbuf.h> 36 #include <sys/dmu_objset.h> 37 #include <sys/dsl_dataset.h> 38 #include <sys/dsl_dir.h> 39 #include <sys/dmu_tx.h> 40 #include <sys/spa.h> 41 #include <sys/zio.h> 42 #include <sys/dmu_zfetch.h> 43 #include <sys/sa.h> 44 #include <sys/sa_impl.h> 45 #include <sys/zfeature.h> 46 #include <sys/blkptr.h> 47 #include <sys/range_tree.h> 48 #include <sys/callb.h> 49 #include <sys/abd.h> 50 #include <sys/vdev.h> 51 #include <sys/cityhash.h> 52 #include <sys/spa_impl.h> 53 54 uint_t zfs_dbuf_evict_key; 55 56 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 57 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 58 59 #ifndef __lint 60 extern inline void dmu_buf_init_user(dmu_buf_user_t *dbu, 61 dmu_buf_evict_func_t *evict_func_sync, 62 dmu_buf_evict_func_t *evict_func_async, 63 dmu_buf_t **clear_on_evict_dbufp); 64 #endif /* ! __lint */ 65 66 /* 67 * Global data structures and functions for the dbuf cache. 68 */ 69 static kmem_cache_t *dbuf_kmem_cache; 70 static taskq_t *dbu_evict_taskq; 71 72 static kthread_t *dbuf_cache_evict_thread; 73 static kmutex_t dbuf_evict_lock; 74 static kcondvar_t dbuf_evict_cv; 75 static boolean_t dbuf_evict_thread_exit; 76 77 /* 78 * There are two dbuf caches; each dbuf can only be in one of them at a time. 79 * 80 * 1. Cache of metadata dbufs, to help make read-heavy administrative commands 81 * from /sbin/zfs run faster. The "metadata cache" specifically stores dbufs 82 * that represent the metadata that describes filesystems/snapshots/ 83 * bookmarks/properties/etc. We only evict from this cache when we export a 84 * pool, to short-circuit as much I/O as possible for all administrative 85 * commands that need the metadata. There is no eviction policy for this 86 * cache, because we try to only include types in it which would occupy a 87 * very small amount of space per object but create a large impact on the 88 * performance of these commands. Instead, after it reaches a maximum size 89 * (which should only happen on very small memory systems with a very large 90 * number of filesystem objects), we stop taking new dbufs into the 91 * metadata cache, instead putting them in the normal dbuf cache. 92 * 93 * 2. LRU cache of dbufs. The "dbuf cache" maintains a list of dbufs that 94 * are not currently held but have been recently released. These dbufs 95 * are not eligible for arc eviction until they are aged out of the cache. 96 * Dbufs that are aged out of the cache will be immediately destroyed and 97 * become eligible for arc eviction. 98 * 99 * Dbufs are added to these caches once the last hold is released. If a dbuf is 100 * later accessed and still exists in the dbuf cache, then it will be removed 101 * from the cache and later re-added to the head of the cache. 102 * 103 * If a given dbuf meets the requirements for the metadata cache, it will go 104 * there, otherwise it will be considered for the generic LRU dbuf cache. The 105 * caches and the refcounts tracking their sizes are stored in an array indexed 106 * by those caches' matching enum values (from dbuf_cached_state_t). 107 */ 108 typedef struct dbuf_cache { 109 multilist_t *cache; 110 refcount_t size; 111 } dbuf_cache_t; 112 dbuf_cache_t dbuf_caches[DB_CACHE_MAX]; 113 114 /* Size limits for the caches */ 115 uint64_t dbuf_cache_max_bytes = 0; 116 uint64_t dbuf_metadata_cache_max_bytes = 0; 117 /* Set the default sizes of the caches to log2 fraction of arc size */ 118 int dbuf_cache_shift = 5; 119 int dbuf_metadata_cache_shift = 6; 120 121 /* 122 * For diagnostic purposes, this is incremented whenever we can't add 123 * something to the metadata cache because it's full, and instead put 124 * the data in the regular dbuf cache. 125 */ 126 uint64_t dbuf_metadata_cache_overflow; 127 128 /* 129 * The LRU dbuf cache uses a three-stage eviction policy: 130 * - A low water marker designates when the dbuf eviction thread 131 * should stop evicting from the dbuf cache. 132 * - When we reach the maximum size (aka mid water mark), we 133 * signal the eviction thread to run. 134 * - The high water mark indicates when the eviction thread 135 * is unable to keep up with the incoming load and eviction must 136 * happen in the context of the calling thread. 137 * 138 * The dbuf cache: 139 * (max size) 140 * low water mid water hi water 141 * +----------------------------------------+----------+----------+ 142 * | | | | 143 * | | | | 144 * | | | | 145 * | | | | 146 * +----------------------------------------+----------+----------+ 147 * stop signal evict 148 * evicting eviction directly 149 * thread 150 * 151 * The high and low water marks indicate the operating range for the eviction 152 * thread. The low water mark is, by default, 90% of the total size of the 153 * cache and the high water mark is at 110% (both of these percentages can be 154 * changed by setting dbuf_cache_lowater_pct and dbuf_cache_hiwater_pct, 155 * respectively). The eviction thread will try to ensure that the cache remains 156 * within this range by waking up every second and checking if the cache is 157 * above the low water mark. The thread can also be woken up by callers adding 158 * elements into the cache if the cache is larger than the mid water (i.e max 159 * cache size). Once the eviction thread is woken up and eviction is required, 160 * it will continue evicting buffers until it's able to reduce the cache size 161 * to the low water mark. If the cache size continues to grow and hits the high 162 * water mark, then callers adding elments to the cache will begin to evict 163 * directly from the cache until the cache is no longer above the high water 164 * mark. 165 */ 166 167 /* 168 * The percentage above and below the maximum cache size. 169 */ 170 uint_t dbuf_cache_hiwater_pct = 10; 171 uint_t dbuf_cache_lowater_pct = 10; 172 173 /* ARGSUSED */ 174 static int 175 dbuf_cons(void *vdb, void *unused, int kmflag) 176 { 177 dmu_buf_impl_t *db = vdb; 178 bzero(db, sizeof (dmu_buf_impl_t)); 179 180 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 181 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 182 multilist_link_init(&db->db_cache_link); 183 refcount_create(&db->db_holds); 184 185 return (0); 186 } 187 188 /* ARGSUSED */ 189 static void 190 dbuf_dest(void *vdb, void *unused) 191 { 192 dmu_buf_impl_t *db = vdb; 193 mutex_destroy(&db->db_mtx); 194 cv_destroy(&db->db_changed); 195 ASSERT(!multilist_link_active(&db->db_cache_link)); 196 refcount_destroy(&db->db_holds); 197 } 198 199 /* 200 * dbuf hash table routines 201 */ 202 static dbuf_hash_table_t dbuf_hash_table; 203 204 static uint64_t dbuf_hash_count; 205 206 /* 207 * We use Cityhash for this. It's fast, and has good hash properties without 208 * requiring any large static buffers. 209 */ 210 static uint64_t 211 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 212 { 213 return (cityhash4((uintptr_t)os, obj, (uint64_t)lvl, blkid)); 214 } 215 216 #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 217 ((dbuf)->db.db_object == (obj) && \ 218 (dbuf)->db_objset == (os) && \ 219 (dbuf)->db_level == (level) && \ 220 (dbuf)->db_blkid == (blkid)) 221 222 dmu_buf_impl_t * 223 dbuf_find(objset_t *os, uint64_t obj, uint8_t level, uint64_t blkid) 224 { 225 dbuf_hash_table_t *h = &dbuf_hash_table; 226 uint64_t hv = dbuf_hash(os, obj, level, blkid); 227 uint64_t idx = hv & h->hash_table_mask; 228 dmu_buf_impl_t *db; 229 230 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 231 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 232 if (DBUF_EQUAL(db, os, obj, level, blkid)) { 233 mutex_enter(&db->db_mtx); 234 if (db->db_state != DB_EVICTING) { 235 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 236 return (db); 237 } 238 mutex_exit(&db->db_mtx); 239 } 240 } 241 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 242 return (NULL); 243 } 244 245 static dmu_buf_impl_t * 246 dbuf_find_bonus(objset_t *os, uint64_t object) 247 { 248 dnode_t *dn; 249 dmu_buf_impl_t *db = NULL; 250 251 if (dnode_hold(os, object, FTAG, &dn) == 0) { 252 rw_enter(&dn->dn_struct_rwlock, RW_READER); 253 if (dn->dn_bonus != NULL) { 254 db = dn->dn_bonus; 255 mutex_enter(&db->db_mtx); 256 } 257 rw_exit(&dn->dn_struct_rwlock); 258 dnode_rele(dn, FTAG); 259 } 260 return (db); 261 } 262 263 /* 264 * Insert an entry into the hash table. If there is already an element 265 * equal to elem in the hash table, then the already existing element 266 * will be returned and the new element will not be inserted. 267 * Otherwise returns NULL. 268 */ 269 static dmu_buf_impl_t * 270 dbuf_hash_insert(dmu_buf_impl_t *db) 271 { 272 dbuf_hash_table_t *h = &dbuf_hash_table; 273 objset_t *os = db->db_objset; 274 uint64_t obj = db->db.db_object; 275 int level = db->db_level; 276 uint64_t blkid = db->db_blkid; 277 uint64_t hv = dbuf_hash(os, obj, level, blkid); 278 uint64_t idx = hv & h->hash_table_mask; 279 dmu_buf_impl_t *dbf; 280 281 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 282 for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 283 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 284 mutex_enter(&dbf->db_mtx); 285 if (dbf->db_state != DB_EVICTING) { 286 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 287 return (dbf); 288 } 289 mutex_exit(&dbf->db_mtx); 290 } 291 } 292 293 mutex_enter(&db->db_mtx); 294 db->db_hash_next = h->hash_table[idx]; 295 h->hash_table[idx] = db; 296 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 297 atomic_inc_64(&dbuf_hash_count); 298 299 return (NULL); 300 } 301 302 /* 303 * Remove an entry from the hash table. It must be in the EVICTING state. 304 */ 305 static void 306 dbuf_hash_remove(dmu_buf_impl_t *db) 307 { 308 dbuf_hash_table_t *h = &dbuf_hash_table; 309 uint64_t hv = dbuf_hash(db->db_objset, db->db.db_object, 310 db->db_level, db->db_blkid); 311 uint64_t idx = hv & h->hash_table_mask; 312 dmu_buf_impl_t *dbf, **dbp; 313 314 /* 315 * We musn't hold db_mtx to maintain lock ordering: 316 * DBUF_HASH_MUTEX > db_mtx. 317 */ 318 ASSERT(refcount_is_zero(&db->db_holds)); 319 ASSERT(db->db_state == DB_EVICTING); 320 ASSERT(!MUTEX_HELD(&db->db_mtx)); 321 322 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 323 dbp = &h->hash_table[idx]; 324 while ((dbf = *dbp) != db) { 325 dbp = &dbf->db_hash_next; 326 ASSERT(dbf != NULL); 327 } 328 *dbp = db->db_hash_next; 329 db->db_hash_next = NULL; 330 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 331 atomic_dec_64(&dbuf_hash_count); 332 } 333 334 typedef enum { 335 DBVU_EVICTING, 336 DBVU_NOT_EVICTING 337 } dbvu_verify_type_t; 338 339 static void 340 dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type) 341 { 342 #ifdef ZFS_DEBUG 343 int64_t holds; 344 345 if (db->db_user == NULL) 346 return; 347 348 /* Only data blocks support the attachment of user data. */ 349 ASSERT(db->db_level == 0); 350 351 /* Clients must resolve a dbuf before attaching user data. */ 352 ASSERT(db->db.db_data != NULL); 353 ASSERT3U(db->db_state, ==, DB_CACHED); 354 355 holds = refcount_count(&db->db_holds); 356 if (verify_type == DBVU_EVICTING) { 357 /* 358 * Immediate eviction occurs when holds == dirtycnt. 359 * For normal eviction buffers, holds is zero on 360 * eviction, except when dbuf_fix_old_data() calls 361 * dbuf_clear_data(). However, the hold count can grow 362 * during eviction even though db_mtx is held (see 363 * dmu_bonus_hold() for an example), so we can only 364 * test the generic invariant that holds >= dirtycnt. 365 */ 366 ASSERT3U(holds, >=, db->db_dirtycnt); 367 } else { 368 if (db->db_user_immediate_evict == TRUE) 369 ASSERT3U(holds, >=, db->db_dirtycnt); 370 else 371 ASSERT3U(holds, >, 0); 372 } 373 #endif 374 } 375 376 static void 377 dbuf_evict_user(dmu_buf_impl_t *db) 378 { 379 dmu_buf_user_t *dbu = db->db_user; 380 381 ASSERT(MUTEX_HELD(&db->db_mtx)); 382 383 if (dbu == NULL) 384 return; 385 386 dbuf_verify_user(db, DBVU_EVICTING); 387 db->db_user = NULL; 388 389 #ifdef ZFS_DEBUG 390 if (dbu->dbu_clear_on_evict_dbufp != NULL) 391 *dbu->dbu_clear_on_evict_dbufp = NULL; 392 #endif 393 394 /* 395 * There are two eviction callbacks - one that we call synchronously 396 * and one that we invoke via a taskq. The async one is useful for 397 * avoiding lock order reversals and limiting stack depth. 398 * 399 * Note that if we have a sync callback but no async callback, 400 * it's likely that the sync callback will free the structure 401 * containing the dbu. In that case we need to take care to not 402 * dereference dbu after calling the sync evict func. 403 */ 404 boolean_t has_async = (dbu->dbu_evict_func_async != NULL); 405 406 if (dbu->dbu_evict_func_sync != NULL) 407 dbu->dbu_evict_func_sync(dbu); 408 409 if (has_async) { 410 taskq_dispatch_ent(dbu_evict_taskq, dbu->dbu_evict_func_async, 411 dbu, 0, &dbu->dbu_tqent); 412 } 413 } 414 415 boolean_t 416 dbuf_is_metadata(dmu_buf_impl_t *db) 417 { 418 if (db->db_level > 0) { 419 return (B_TRUE); 420 } else { 421 boolean_t is_metadata; 422 423 DB_DNODE_ENTER(db); 424 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); 425 DB_DNODE_EXIT(db); 426 427 return (is_metadata); 428 } 429 } 430 431 /* 432 * This returns whether this dbuf should be stored in the metadata cache, which 433 * is based on whether it's from one of the dnode types that store data related 434 * to traversing dataset hierarchies. 435 */ 436 static boolean_t 437 dbuf_include_in_metadata_cache(dmu_buf_impl_t *db) 438 { 439 DB_DNODE_ENTER(db); 440 dmu_object_type_t type = DB_DNODE(db)->dn_type; 441 DB_DNODE_EXIT(db); 442 443 /* Check if this dbuf is one of the types we care about */ 444 if (DMU_OT_IS_METADATA_CACHED(type)) { 445 /* If we hit this, then we set something up wrong in dmu_ot */ 446 ASSERT(DMU_OT_IS_METADATA(type)); 447 448 /* 449 * Sanity check for small-memory systems: don't allocate too 450 * much memory for this purpose. 451 */ 452 if (refcount_count(&dbuf_caches[DB_DBUF_METADATA_CACHE].size) > 453 dbuf_metadata_cache_max_bytes) { 454 dbuf_metadata_cache_overflow++; 455 DTRACE_PROBE1(dbuf__metadata__cache__overflow, 456 dmu_buf_impl_t *, db); 457 return (B_FALSE); 458 } 459 460 return (B_TRUE); 461 } 462 463 return (B_FALSE); 464 } 465 466 /* 467 * This function *must* return indices evenly distributed between all 468 * sublists of the multilist. This is needed due to how the dbuf eviction 469 * code is laid out; dbuf_evict_thread() assumes dbufs are evenly 470 * distributed between all sublists and uses this assumption when 471 * deciding which sublist to evict from and how much to evict from it. 472 */ 473 unsigned int 474 dbuf_cache_multilist_index_func(multilist_t *ml, void *obj) 475 { 476 dmu_buf_impl_t *db = obj; 477 478 /* 479 * The assumption here, is the hash value for a given 480 * dmu_buf_impl_t will remain constant throughout it's lifetime 481 * (i.e. it's objset, object, level and blkid fields don't change). 482 * Thus, we don't need to store the dbuf's sublist index 483 * on insertion, as this index can be recalculated on removal. 484 * 485 * Also, the low order bits of the hash value are thought to be 486 * distributed evenly. Otherwise, in the case that the multilist 487 * has a power of two number of sublists, each sublists' usage 488 * would not be evenly distributed. 489 */ 490 return (dbuf_hash(db->db_objset, db->db.db_object, 491 db->db_level, db->db_blkid) % 492 multilist_get_num_sublists(ml)); 493 } 494 495 static inline boolean_t 496 dbuf_cache_above_hiwater(void) 497 { 498 uint64_t dbuf_cache_hiwater_bytes = 499 (dbuf_cache_max_bytes * dbuf_cache_hiwater_pct) / 100; 500 501 return (refcount_count(&dbuf_caches[DB_DBUF_CACHE].size) > 502 dbuf_cache_max_bytes + dbuf_cache_hiwater_bytes); 503 } 504 505 static inline boolean_t 506 dbuf_cache_above_lowater(void) 507 { 508 uint64_t dbuf_cache_lowater_bytes = 509 (dbuf_cache_max_bytes * dbuf_cache_lowater_pct) / 100; 510 511 return (refcount_count(&dbuf_caches[DB_DBUF_CACHE].size) > 512 dbuf_cache_max_bytes - dbuf_cache_lowater_bytes); 513 } 514 515 /* 516 * Evict the oldest eligible dbuf from the dbuf cache. 517 */ 518 static void 519 dbuf_evict_one(void) 520 { 521 int idx = multilist_get_random_index(dbuf_caches[DB_DBUF_CACHE].cache); 522 multilist_sublist_t *mls = multilist_sublist_lock( 523 dbuf_caches[DB_DBUF_CACHE].cache, idx); 524 525 ASSERT(!MUTEX_HELD(&dbuf_evict_lock)); 526 527 /* 528 * Set the thread's tsd to indicate that it's processing evictions. 529 * Once a thread stops evicting from the dbuf cache it will 530 * reset its tsd to NULL. 531 */ 532 ASSERT3P(tsd_get(zfs_dbuf_evict_key), ==, NULL); 533 (void) tsd_set(zfs_dbuf_evict_key, (void *)B_TRUE); 534 535 dmu_buf_impl_t *db = multilist_sublist_tail(mls); 536 while (db != NULL && mutex_tryenter(&db->db_mtx) == 0) { 537 db = multilist_sublist_prev(mls, db); 538 } 539 540 DTRACE_PROBE2(dbuf__evict__one, dmu_buf_impl_t *, db, 541 multilist_sublist_t *, mls); 542 543 if (db != NULL) { 544 multilist_sublist_remove(mls, db); 545 multilist_sublist_unlock(mls); 546 (void) refcount_remove_many(&dbuf_caches[DB_DBUF_CACHE].size, 547 db->db.db_size, db); 548 ASSERT3U(db->db_caching_status, ==, DB_DBUF_CACHE); 549 db->db_caching_status = DB_NO_CACHE; 550 dbuf_destroy(db); 551 } else { 552 multilist_sublist_unlock(mls); 553 } 554 (void) tsd_set(zfs_dbuf_evict_key, NULL); 555 } 556 557 /* 558 * The dbuf evict thread is responsible for aging out dbufs from the 559 * cache. Once the cache has reached it's maximum size, dbufs are removed 560 * and destroyed. The eviction thread will continue running until the size 561 * of the dbuf cache is at or below the maximum size. Once the dbuf is aged 562 * out of the cache it is destroyed and becomes eligible for arc eviction. 563 */ 564 /* ARGSUSED */ 565 static void 566 dbuf_evict_thread(void *unused) 567 { 568 callb_cpr_t cpr; 569 570 CALLB_CPR_INIT(&cpr, &dbuf_evict_lock, callb_generic_cpr, FTAG); 571 572 mutex_enter(&dbuf_evict_lock); 573 while (!dbuf_evict_thread_exit) { 574 while (!dbuf_cache_above_lowater() && !dbuf_evict_thread_exit) { 575 CALLB_CPR_SAFE_BEGIN(&cpr); 576 (void) cv_timedwait_hires(&dbuf_evict_cv, 577 &dbuf_evict_lock, SEC2NSEC(1), MSEC2NSEC(1), 0); 578 CALLB_CPR_SAFE_END(&cpr, &dbuf_evict_lock); 579 } 580 mutex_exit(&dbuf_evict_lock); 581 582 /* 583 * Keep evicting as long as we're above the low water mark 584 * for the cache. We do this without holding the locks to 585 * minimize lock contention. 586 */ 587 while (dbuf_cache_above_lowater() && !dbuf_evict_thread_exit) { 588 dbuf_evict_one(); 589 } 590 591 mutex_enter(&dbuf_evict_lock); 592 } 593 594 dbuf_evict_thread_exit = B_FALSE; 595 cv_broadcast(&dbuf_evict_cv); 596 CALLB_CPR_EXIT(&cpr); /* drops dbuf_evict_lock */ 597 thread_exit(); 598 } 599 600 /* 601 * Wake up the dbuf eviction thread if the dbuf cache is at its max size. 602 * If the dbuf cache is at its high water mark, then evict a dbuf from the 603 * dbuf cache using the callers context. 604 */ 605 static void 606 dbuf_evict_notify(void) 607 { 608 609 /* 610 * We use thread specific data to track when a thread has 611 * started processing evictions. This allows us to avoid deeply 612 * nested stacks that would have a call flow similar to this: 613 * 614 * dbuf_rele()-->dbuf_rele_and_unlock()-->dbuf_evict_notify() 615 * ^ | 616 * | | 617 * +-----dbuf_destroy()<--dbuf_evict_one()<--------+ 618 * 619 * The dbuf_eviction_thread will always have its tsd set until 620 * that thread exits. All other threads will only set their tsd 621 * if they are participating in the eviction process. This only 622 * happens if the eviction thread is unable to process evictions 623 * fast enough. To keep the dbuf cache size in check, other threads 624 * can evict from the dbuf cache directly. Those threads will set 625 * their tsd values so that we ensure that they only evict one dbuf 626 * from the dbuf cache. 627 */ 628 if (tsd_get(zfs_dbuf_evict_key) != NULL) 629 return; 630 631 /* 632 * We check if we should evict without holding the dbuf_evict_lock, 633 * because it's OK to occasionally make the wrong decision here, 634 * and grabbing the lock results in massive lock contention. 635 */ 636 if (refcount_count(&dbuf_caches[DB_DBUF_CACHE].size) > 637 dbuf_cache_max_bytes) { 638 if (dbuf_cache_above_hiwater()) 639 dbuf_evict_one(); 640 cv_signal(&dbuf_evict_cv); 641 } 642 } 643 644 void 645 dbuf_init(void) 646 { 647 uint64_t hsize = 1ULL << 16; 648 dbuf_hash_table_t *h = &dbuf_hash_table; 649 int i; 650 651 /* 652 * The hash table is big enough to fill all of physical memory 653 * with an average 4K block size. The table will take up 654 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 655 */ 656 while (hsize * 4096 < physmem * PAGESIZE) 657 hsize <<= 1; 658 659 retry: 660 h->hash_table_mask = hsize - 1; 661 h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 662 if (h->hash_table == NULL) { 663 /* XXX - we should really return an error instead of assert */ 664 ASSERT(hsize > (1ULL << 10)); 665 hsize >>= 1; 666 goto retry; 667 } 668 669 dbuf_kmem_cache = kmem_cache_create("dmu_buf_impl_t", 670 sizeof (dmu_buf_impl_t), 671 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 672 673 for (i = 0; i < DBUF_MUTEXES; i++) 674 mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 675 676 /* 677 * Setup the parameters for the dbuf caches. We set the sizes of the 678 * dbuf cache and the metadata cache to 1/32nd and 1/16th (default) 679 * of the size of the ARC, respectively. If the values are set in 680 * /etc/system and they're not greater than the size of the ARC, then 681 * we honor that value. 682 */ 683 if (dbuf_cache_max_bytes == 0 || 684 dbuf_cache_max_bytes >= arc_max_bytes()) { 685 dbuf_cache_max_bytes = arc_max_bytes() >> dbuf_cache_shift; 686 } 687 if (dbuf_metadata_cache_max_bytes == 0 || 688 dbuf_metadata_cache_max_bytes >= arc_max_bytes()) { 689 dbuf_metadata_cache_max_bytes = 690 arc_max_bytes() >> dbuf_metadata_cache_shift; 691 } 692 693 /* 694 * All entries are queued via taskq_dispatch_ent(), so min/maxalloc 695 * configuration is not required. 696 */ 697 dbu_evict_taskq = taskq_create("dbu_evict", 1, minclsyspri, 0, 0, 0); 698 699 for (dbuf_cached_state_t dcs = 0; dcs < DB_CACHE_MAX; dcs++) { 700 dbuf_caches[dcs].cache = 701 multilist_create(sizeof (dmu_buf_impl_t), 702 offsetof(dmu_buf_impl_t, db_cache_link), 703 dbuf_cache_multilist_index_func); 704 refcount_create(&dbuf_caches[dcs].size); 705 } 706 707 tsd_create(&zfs_dbuf_evict_key, NULL); 708 dbuf_evict_thread_exit = B_FALSE; 709 mutex_init(&dbuf_evict_lock, NULL, MUTEX_DEFAULT, NULL); 710 cv_init(&dbuf_evict_cv, NULL, CV_DEFAULT, NULL); 711 dbuf_cache_evict_thread = thread_create(NULL, 0, dbuf_evict_thread, 712 NULL, 0, &p0, TS_RUN, minclsyspri); 713 } 714 715 void 716 dbuf_fini(void) 717 { 718 dbuf_hash_table_t *h = &dbuf_hash_table; 719 int i; 720 721 for (i = 0; i < DBUF_MUTEXES; i++) 722 mutex_destroy(&h->hash_mutexes[i]); 723 kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 724 kmem_cache_destroy(dbuf_kmem_cache); 725 taskq_destroy(dbu_evict_taskq); 726 727 mutex_enter(&dbuf_evict_lock); 728 dbuf_evict_thread_exit = B_TRUE; 729 while (dbuf_evict_thread_exit) { 730 cv_signal(&dbuf_evict_cv); 731 cv_wait(&dbuf_evict_cv, &dbuf_evict_lock); 732 } 733 mutex_exit(&dbuf_evict_lock); 734 tsd_destroy(&zfs_dbuf_evict_key); 735 736 mutex_destroy(&dbuf_evict_lock); 737 cv_destroy(&dbuf_evict_cv); 738 739 for (dbuf_cached_state_t dcs = 0; dcs < DB_CACHE_MAX; dcs++) { 740 refcount_destroy(&dbuf_caches[dcs].size); 741 multilist_destroy(dbuf_caches[dcs].cache); 742 } 743 } 744 745 /* 746 * Other stuff. 747 */ 748 749 #ifdef ZFS_DEBUG 750 static void 751 dbuf_verify(dmu_buf_impl_t *db) 752 { 753 dnode_t *dn; 754 dbuf_dirty_record_t *dr; 755 756 ASSERT(MUTEX_HELD(&db->db_mtx)); 757 758 if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 759 return; 760 761 ASSERT(db->db_objset != NULL); 762 DB_DNODE_ENTER(db); 763 dn = DB_DNODE(db); 764 if (dn == NULL) { 765 ASSERT(db->db_parent == NULL); 766 ASSERT(db->db_blkptr == NULL); 767 } else { 768 ASSERT3U(db->db.db_object, ==, dn->dn_object); 769 ASSERT3P(db->db_objset, ==, dn->dn_objset); 770 ASSERT3U(db->db_level, <, dn->dn_nlevels); 771 ASSERT(db->db_blkid == DMU_BONUS_BLKID || 772 db->db_blkid == DMU_SPILL_BLKID || 773 !avl_is_empty(&dn->dn_dbufs)); 774 } 775 if (db->db_blkid == DMU_BONUS_BLKID) { 776 ASSERT(dn != NULL); 777 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 778 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 779 } else if (db->db_blkid == DMU_SPILL_BLKID) { 780 ASSERT(dn != NULL); 781 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 782 ASSERT0(db->db.db_offset); 783 } else { 784 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 785 } 786 787 for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 788 ASSERT(dr->dr_dbuf == db); 789 790 for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 791 ASSERT(dr->dr_dbuf == db); 792 793 /* 794 * We can't assert that db_size matches dn_datablksz because it 795 * can be momentarily different when another thread is doing 796 * dnode_set_blksz(). 797 */ 798 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 799 dr = db->db_data_pending; 800 /* 801 * It should only be modified in syncing context, so 802 * make sure we only have one copy of the data. 803 */ 804 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 805 } 806 807 /* verify db->db_blkptr */ 808 if (db->db_blkptr) { 809 if (db->db_parent == dn->dn_dbuf) { 810 /* db is pointed to by the dnode */ 811 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 812 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 813 ASSERT(db->db_parent == NULL); 814 else 815 ASSERT(db->db_parent != NULL); 816 if (db->db_blkid != DMU_SPILL_BLKID) 817 ASSERT3P(db->db_blkptr, ==, 818 &dn->dn_phys->dn_blkptr[db->db_blkid]); 819 } else { 820 /* db is pointed to by an indirect block */ 821 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 822 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 823 ASSERT3U(db->db_parent->db.db_object, ==, 824 db->db.db_object); 825 /* 826 * dnode_grow_indblksz() can make this fail if we don't 827 * have the struct_rwlock. XXX indblksz no longer 828 * grows. safe to do this now? 829 */ 830 if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 831 ASSERT3P(db->db_blkptr, ==, 832 ((blkptr_t *)db->db_parent->db.db_data + 833 db->db_blkid % epb)); 834 } 835 } 836 } 837 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 838 (db->db_buf == NULL || db->db_buf->b_data) && 839 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 840 db->db_state != DB_FILL && !dn->dn_free_txg) { 841 /* 842 * If the blkptr isn't set but they have nonzero data, 843 * it had better be dirty, otherwise we'll lose that 844 * data when we evict this buffer. 845 * 846 * There is an exception to this rule for indirect blocks; in 847 * this case, if the indirect block is a hole, we fill in a few 848 * fields on each of the child blocks (importantly, birth time) 849 * to prevent hole birth times from being lost when you 850 * partially fill in a hole. 851 */ 852 if (db->db_dirtycnt == 0) { 853 if (db->db_level == 0) { 854 uint64_t *buf = db->db.db_data; 855 int i; 856 857 for (i = 0; i < db->db.db_size >> 3; i++) { 858 ASSERT(buf[i] == 0); 859 } 860 } else { 861 blkptr_t *bps = db->db.db_data; 862 ASSERT3U(1 << DB_DNODE(db)->dn_indblkshift, ==, 863 db->db.db_size); 864 /* 865 * We want to verify that all the blkptrs in the 866 * indirect block are holes, but we may have 867 * automatically set up a few fields for them. 868 * We iterate through each blkptr and verify 869 * they only have those fields set. 870 */ 871 for (int i = 0; 872 i < db->db.db_size / sizeof (blkptr_t); 873 i++) { 874 blkptr_t *bp = &bps[i]; 875 ASSERT(ZIO_CHECKSUM_IS_ZERO( 876 &bp->blk_cksum)); 877 ASSERT( 878 DVA_IS_EMPTY(&bp->blk_dva[0]) && 879 DVA_IS_EMPTY(&bp->blk_dva[1]) && 880 DVA_IS_EMPTY(&bp->blk_dva[2])); 881 ASSERT0(bp->blk_fill); 882 ASSERT0(bp->blk_pad[0]); 883 ASSERT0(bp->blk_pad[1]); 884 ASSERT(!BP_IS_EMBEDDED(bp)); 885 ASSERT(BP_IS_HOLE(bp)); 886 ASSERT0(bp->blk_phys_birth); 887 } 888 } 889 } 890 } 891 DB_DNODE_EXIT(db); 892 } 893 #endif 894 895 static void 896 dbuf_clear_data(dmu_buf_impl_t *db) 897 { 898 ASSERT(MUTEX_HELD(&db->db_mtx)); 899 dbuf_evict_user(db); 900 ASSERT3P(db->db_buf, ==, NULL); 901 db->db.db_data = NULL; 902 if (db->db_state != DB_NOFILL) 903 db->db_state = DB_UNCACHED; 904 } 905 906 static void 907 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 908 { 909 ASSERT(MUTEX_HELD(&db->db_mtx)); 910 ASSERT(buf != NULL); 911 912 db->db_buf = buf; 913 ASSERT(buf->b_data != NULL); 914 db->db.db_data = buf->b_data; 915 } 916 917 /* 918 * Loan out an arc_buf for read. Return the loaned arc_buf. 919 */ 920 arc_buf_t * 921 dbuf_loan_arcbuf(dmu_buf_impl_t *db) 922 { 923 arc_buf_t *abuf; 924 925 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 926 mutex_enter(&db->db_mtx); 927 if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 928 int blksz = db->db.db_size; 929 spa_t *spa = db->db_objset->os_spa; 930 931 mutex_exit(&db->db_mtx); 932 abuf = arc_loan_buf(spa, B_FALSE, blksz); 933 bcopy(db->db.db_data, abuf->b_data, blksz); 934 } else { 935 abuf = db->db_buf; 936 arc_loan_inuse_buf(abuf, db); 937 db->db_buf = NULL; 938 dbuf_clear_data(db); 939 mutex_exit(&db->db_mtx); 940 } 941 return (abuf); 942 } 943 944 /* 945 * Calculate which level n block references the data at the level 0 offset 946 * provided. 947 */ 948 uint64_t 949 dbuf_whichblock(dnode_t *dn, int64_t level, uint64_t offset) 950 { 951 if (dn->dn_datablkshift != 0 && dn->dn_indblkshift != 0) { 952 /* 953 * The level n blkid is equal to the level 0 blkid divided by 954 * the number of level 0s in a level n block. 955 * 956 * The level 0 blkid is offset >> datablkshift = 957 * offset / 2^datablkshift. 958 * 959 * The number of level 0s in a level n is the number of block 960 * pointers in an indirect block, raised to the power of level. 961 * This is 2^(indblkshift - SPA_BLKPTRSHIFT)^level = 962 * 2^(level*(indblkshift - SPA_BLKPTRSHIFT)). 963 * 964 * Thus, the level n blkid is: offset / 965 * ((2^datablkshift)*(2^(level*(indblkshift - SPA_BLKPTRSHIFT))) 966 * = offset / 2^(datablkshift + level * 967 * (indblkshift - SPA_BLKPTRSHIFT)) 968 * = offset >> (datablkshift + level * 969 * (indblkshift - SPA_BLKPTRSHIFT)) 970 */ 971 return (offset >> (dn->dn_datablkshift + level * 972 (dn->dn_indblkshift - SPA_BLKPTRSHIFT))); 973 } else { 974 ASSERT3U(offset, <, dn->dn_datablksz); 975 return (0); 976 } 977 } 978 979 static void 980 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 981 { 982 dmu_buf_impl_t *db = vdb; 983 984 mutex_enter(&db->db_mtx); 985 ASSERT3U(db->db_state, ==, DB_READ); 986 /* 987 * All reads are synchronous, so we must have a hold on the dbuf 988 */ 989 ASSERT(refcount_count(&db->db_holds) > 0); 990 ASSERT(db->db_buf == NULL); 991 ASSERT(db->db.db_data == NULL); 992 if (db->db_level == 0 && db->db_freed_in_flight) { 993 /* we were freed in flight; disregard any error */ 994 arc_release(buf, db); 995 bzero(buf->b_data, db->db.db_size); 996 arc_buf_freeze(buf); 997 db->db_freed_in_flight = FALSE; 998 dbuf_set_data(db, buf); 999 db->db_state = DB_CACHED; 1000 } else if (zio == NULL || zio->io_error == 0) { 1001 dbuf_set_data(db, buf); 1002 db->db_state = DB_CACHED; 1003 } else { 1004 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1005 ASSERT3P(db->db_buf, ==, NULL); 1006 arc_buf_destroy(buf, db); 1007 db->db_state = DB_UNCACHED; 1008 } 1009 cv_broadcast(&db->db_changed); 1010 dbuf_rele_and_unlock(db, NULL); 1011 } 1012 1013 static void 1014 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 1015 { 1016 dnode_t *dn; 1017 zbookmark_phys_t zb; 1018 arc_flags_t aflags = ARC_FLAG_NOWAIT; 1019 1020 DB_DNODE_ENTER(db); 1021 dn = DB_DNODE(db); 1022 ASSERT(!refcount_is_zero(&db->db_holds)); 1023 /* We need the struct_rwlock to prevent db_blkptr from changing. */ 1024 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1025 ASSERT(MUTEX_HELD(&db->db_mtx)); 1026 ASSERT(db->db_state == DB_UNCACHED); 1027 ASSERT(db->db_buf == NULL); 1028 1029 if (db->db_blkid == DMU_BONUS_BLKID) { 1030 int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 1031 1032 ASSERT3U(bonuslen, <=, db->db.db_size); 1033 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 1034 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 1035 if (bonuslen < DN_MAX_BONUSLEN) 1036 bzero(db->db.db_data, DN_MAX_BONUSLEN); 1037 if (bonuslen) 1038 bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 1039 DB_DNODE_EXIT(db); 1040 db->db_state = DB_CACHED; 1041 mutex_exit(&db->db_mtx); 1042 return; 1043 } 1044 1045 /* 1046 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 1047 * processes the delete record and clears the bp while we are waiting 1048 * for the dn_mtx (resulting in a "no" from block_freed). 1049 */ 1050 if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 1051 (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 1052 BP_IS_HOLE(db->db_blkptr)))) { 1053 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1054 1055 dbuf_set_data(db, arc_alloc_buf(db->db_objset->os_spa, db, type, 1056 db->db.db_size)); 1057 bzero(db->db.db_data, db->db.db_size); 1058 1059 if (db->db_blkptr != NULL && db->db_level > 0 && 1060 BP_IS_HOLE(db->db_blkptr) && 1061 db->db_blkptr->blk_birth != 0) { 1062 blkptr_t *bps = db->db.db_data; 1063 for (int i = 0; i < ((1 << 1064 DB_DNODE(db)->dn_indblkshift) / sizeof (blkptr_t)); 1065 i++) { 1066 blkptr_t *bp = &bps[i]; 1067 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 1068 1 << dn->dn_indblkshift); 1069 BP_SET_LSIZE(bp, 1070 BP_GET_LEVEL(db->db_blkptr) == 1 ? 1071 dn->dn_datablksz : 1072 BP_GET_LSIZE(db->db_blkptr)); 1073 BP_SET_TYPE(bp, BP_GET_TYPE(db->db_blkptr)); 1074 BP_SET_LEVEL(bp, 1075 BP_GET_LEVEL(db->db_blkptr) - 1); 1076 BP_SET_BIRTH(bp, db->db_blkptr->blk_birth, 0); 1077 } 1078 } 1079 DB_DNODE_EXIT(db); 1080 db->db_state = DB_CACHED; 1081 mutex_exit(&db->db_mtx); 1082 return; 1083 } 1084 1085 DB_DNODE_EXIT(db); 1086 1087 db->db_state = DB_READ; 1088 mutex_exit(&db->db_mtx); 1089 1090 if (DBUF_IS_L2CACHEABLE(db)) 1091 aflags |= ARC_FLAG_L2CACHE; 1092 1093 SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 1094 db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 1095 db->db.db_object, db->db_level, db->db_blkid); 1096 1097 dbuf_add_ref(db, NULL); 1098 1099 (void) arc_read(zio, db->db_objset->os_spa, db->db_blkptr, 1100 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 1101 (flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 1102 &aflags, &zb); 1103 } 1104 1105 /* 1106 * This is our just-in-time copy function. It makes a copy of buffers that 1107 * have been modified in a previous transaction group before we access them in 1108 * the current active group. 1109 * 1110 * This function is used in three places: when we are dirtying a buffer for the 1111 * first time in a txg, when we are freeing a range in a dnode that includes 1112 * this buffer, and when we are accessing a buffer which was received compressed 1113 * and later referenced in a WRITE_BYREF record. 1114 * 1115 * Note that when we are called from dbuf_free_range() we do not put a hold on 1116 * the buffer, we just traverse the active dbuf list for the dnode. 1117 */ 1118 static void 1119 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 1120 { 1121 dbuf_dirty_record_t *dr = db->db_last_dirty; 1122 1123 ASSERT(MUTEX_HELD(&db->db_mtx)); 1124 ASSERT(db->db.db_data != NULL); 1125 ASSERT(db->db_level == 0); 1126 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 1127 1128 if (dr == NULL || 1129 (dr->dt.dl.dr_data != 1130 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 1131 return; 1132 1133 /* 1134 * If the last dirty record for this dbuf has not yet synced 1135 * and its referencing the dbuf data, either: 1136 * reset the reference to point to a new copy, 1137 * or (if there a no active holders) 1138 * just null out the current db_data pointer. 1139 */ 1140 ASSERT(dr->dr_txg >= txg - 2); 1141 if (db->db_blkid == DMU_BONUS_BLKID) { 1142 /* Note that the data bufs here are zio_bufs */ 1143 dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 1144 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 1145 bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 1146 } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1147 int size = arc_buf_size(db->db_buf); 1148 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1149 spa_t *spa = db->db_objset->os_spa; 1150 enum zio_compress compress_type = 1151 arc_get_compression(db->db_buf); 1152 1153 if (compress_type == ZIO_COMPRESS_OFF) { 1154 dr->dt.dl.dr_data = arc_alloc_buf(spa, db, type, size); 1155 } else { 1156 ASSERT3U(type, ==, ARC_BUFC_DATA); 1157 dr->dt.dl.dr_data = arc_alloc_compressed_buf(spa, db, 1158 size, arc_buf_lsize(db->db_buf), compress_type); 1159 } 1160 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 1161 } else { 1162 db->db_buf = NULL; 1163 dbuf_clear_data(db); 1164 } 1165 } 1166 1167 int 1168 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 1169 { 1170 int err = 0; 1171 boolean_t prefetch; 1172 dnode_t *dn; 1173 1174 /* 1175 * We don't have to hold the mutex to check db_state because it 1176 * can't be freed while we have a hold on the buffer. 1177 */ 1178 ASSERT(!refcount_is_zero(&db->db_holds)); 1179 1180 if (db->db_state == DB_NOFILL) 1181 return (SET_ERROR(EIO)); 1182 1183 DB_DNODE_ENTER(db); 1184 dn = DB_DNODE(db); 1185 if ((flags & DB_RF_HAVESTRUCT) == 0) 1186 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1187 1188 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1189 (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL && 1190 DBUF_IS_CACHEABLE(db); 1191 1192 mutex_enter(&db->db_mtx); 1193 if (db->db_state == DB_CACHED) { 1194 /* 1195 * If the arc buf is compressed, we need to decompress it to 1196 * read the data. This could happen during the "zfs receive" of 1197 * a stream which is compressed and deduplicated. 1198 */ 1199 if (db->db_buf != NULL && 1200 arc_get_compression(db->db_buf) != ZIO_COMPRESS_OFF) { 1201 dbuf_fix_old_data(db, 1202 spa_syncing_txg(dmu_objset_spa(db->db_objset))); 1203 err = arc_decompress(db->db_buf); 1204 dbuf_set_data(db, db->db_buf); 1205 } 1206 mutex_exit(&db->db_mtx); 1207 if (prefetch) 1208 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE); 1209 if ((flags & DB_RF_HAVESTRUCT) == 0) 1210 rw_exit(&dn->dn_struct_rwlock); 1211 DB_DNODE_EXIT(db); 1212 } else if (db->db_state == DB_UNCACHED) { 1213 spa_t *spa = dn->dn_objset->os_spa; 1214 boolean_t need_wait = B_FALSE; 1215 1216 if (zio == NULL && 1217 db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) { 1218 zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 1219 need_wait = B_TRUE; 1220 } 1221 dbuf_read_impl(db, zio, flags); 1222 1223 /* dbuf_read_impl has dropped db_mtx for us */ 1224 1225 if (prefetch) 1226 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE); 1227 1228 if ((flags & DB_RF_HAVESTRUCT) == 0) 1229 rw_exit(&dn->dn_struct_rwlock); 1230 DB_DNODE_EXIT(db); 1231 1232 if (need_wait) 1233 err = zio_wait(zio); 1234 } else { 1235 /* 1236 * Another reader came in while the dbuf was in flight 1237 * between UNCACHED and CACHED. Either a writer will finish 1238 * writing the buffer (sending the dbuf to CACHED) or the 1239 * first reader's request will reach the read_done callback 1240 * and send the dbuf to CACHED. Otherwise, a failure 1241 * occurred and the dbuf went to UNCACHED. 1242 */ 1243 mutex_exit(&db->db_mtx); 1244 if (prefetch) 1245 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE); 1246 if ((flags & DB_RF_HAVESTRUCT) == 0) 1247 rw_exit(&dn->dn_struct_rwlock); 1248 DB_DNODE_EXIT(db); 1249 1250 /* Skip the wait per the caller's request. */ 1251 mutex_enter(&db->db_mtx); 1252 if ((flags & DB_RF_NEVERWAIT) == 0) { 1253 while (db->db_state == DB_READ || 1254 db->db_state == DB_FILL) { 1255 ASSERT(db->db_state == DB_READ || 1256 (flags & DB_RF_HAVESTRUCT) == 0); 1257 DTRACE_PROBE2(blocked__read, dmu_buf_impl_t *, 1258 db, zio_t *, zio); 1259 cv_wait(&db->db_changed, &db->db_mtx); 1260 } 1261 if (db->db_state == DB_UNCACHED) 1262 err = SET_ERROR(EIO); 1263 } 1264 mutex_exit(&db->db_mtx); 1265 } 1266 1267 return (err); 1268 } 1269 1270 static void 1271 dbuf_noread(dmu_buf_impl_t *db) 1272 { 1273 ASSERT(!refcount_is_zero(&db->db_holds)); 1274 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1275 mutex_enter(&db->db_mtx); 1276 while (db->db_state == DB_READ || db->db_state == DB_FILL) 1277 cv_wait(&db->db_changed, &db->db_mtx); 1278 if (db->db_state == DB_UNCACHED) { 1279 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1280 spa_t *spa = db->db_objset->os_spa; 1281 1282 ASSERT(db->db_buf == NULL); 1283 ASSERT(db->db.db_data == NULL); 1284 dbuf_set_data(db, arc_alloc_buf(spa, db, type, db->db.db_size)); 1285 db->db_state = DB_FILL; 1286 } else if (db->db_state == DB_NOFILL) { 1287 dbuf_clear_data(db); 1288 } else { 1289 ASSERT3U(db->db_state, ==, DB_CACHED); 1290 } 1291 mutex_exit(&db->db_mtx); 1292 } 1293 1294 void 1295 dbuf_unoverride(dbuf_dirty_record_t *dr) 1296 { 1297 dmu_buf_impl_t *db = dr->dr_dbuf; 1298 blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 1299 uint64_t txg = dr->dr_txg; 1300 1301 ASSERT(MUTEX_HELD(&db->db_mtx)); 1302 /* 1303 * This assert is valid because dmu_sync() expects to be called by 1304 * a zilog's get_data while holding a range lock. This call only 1305 * comes from dbuf_dirty() callers who must also hold a range lock. 1306 */ 1307 ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 1308 ASSERT(db->db_level == 0); 1309 1310 if (db->db_blkid == DMU_BONUS_BLKID || 1311 dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 1312 return; 1313 1314 ASSERT(db->db_data_pending != dr); 1315 1316 /* free this block */ 1317 if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite) 1318 zio_free(db->db_objset->os_spa, txg, bp); 1319 1320 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 1321 dr->dt.dl.dr_nopwrite = B_FALSE; 1322 1323 /* 1324 * Release the already-written buffer, so we leave it in 1325 * a consistent dirty state. Note that all callers are 1326 * modifying the buffer, so they will immediately do 1327 * another (redundant) arc_release(). Therefore, leave 1328 * the buf thawed to save the effort of freezing & 1329 * immediately re-thawing it. 1330 */ 1331 arc_release(dr->dt.dl.dr_data, db); 1332 } 1333 1334 /* 1335 * Evict (if its unreferenced) or clear (if its referenced) any level-0 1336 * data blocks in the free range, so that any future readers will find 1337 * empty blocks. 1338 */ 1339 void 1340 dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid, 1341 dmu_tx_t *tx) 1342 { 1343 dmu_buf_impl_t db_search; 1344 dmu_buf_impl_t *db, *db_next; 1345 uint64_t txg = tx->tx_txg; 1346 avl_index_t where; 1347 1348 if (end_blkid > dn->dn_maxblkid && 1349 !(start_blkid == DMU_SPILL_BLKID || end_blkid == DMU_SPILL_BLKID)) 1350 end_blkid = dn->dn_maxblkid; 1351 dprintf_dnode(dn, "start=%llu end=%llu\n", start_blkid, end_blkid); 1352 1353 db_search.db_level = 0; 1354 db_search.db_blkid = start_blkid; 1355 db_search.db_state = DB_SEARCH; 1356 1357 mutex_enter(&dn->dn_dbufs_mtx); 1358 db = avl_find(&dn->dn_dbufs, &db_search, &where); 1359 ASSERT3P(db, ==, NULL); 1360 1361 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); 1362 1363 for (; db != NULL; db = db_next) { 1364 db_next = AVL_NEXT(&dn->dn_dbufs, db); 1365 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1366 1367 if (db->db_level != 0 || db->db_blkid > end_blkid) { 1368 break; 1369 } 1370 ASSERT3U(db->db_blkid, >=, start_blkid); 1371 1372 /* found a level 0 buffer in the range */ 1373 mutex_enter(&db->db_mtx); 1374 if (dbuf_undirty(db, tx)) { 1375 /* mutex has been dropped and dbuf destroyed */ 1376 continue; 1377 } 1378 1379 if (db->db_state == DB_UNCACHED || 1380 db->db_state == DB_NOFILL || 1381 db->db_state == DB_EVICTING) { 1382 ASSERT(db->db.db_data == NULL); 1383 mutex_exit(&db->db_mtx); 1384 continue; 1385 } 1386 if (db->db_state == DB_READ || db->db_state == DB_FILL) { 1387 /* will be handled in dbuf_read_done or dbuf_rele */ 1388 db->db_freed_in_flight = TRUE; 1389 mutex_exit(&db->db_mtx); 1390 continue; 1391 } 1392 if (refcount_count(&db->db_holds) == 0) { 1393 ASSERT(db->db_buf); 1394 dbuf_destroy(db); 1395 continue; 1396 } 1397 /* The dbuf is referenced */ 1398 1399 if (db->db_last_dirty != NULL) { 1400 dbuf_dirty_record_t *dr = db->db_last_dirty; 1401 1402 if (dr->dr_txg == txg) { 1403 /* 1404 * This buffer is "in-use", re-adjust the file 1405 * size to reflect that this buffer may 1406 * contain new data when we sync. 1407 */ 1408 if (db->db_blkid != DMU_SPILL_BLKID && 1409 db->db_blkid > dn->dn_maxblkid) 1410 dn->dn_maxblkid = db->db_blkid; 1411 dbuf_unoverride(dr); 1412 } else { 1413 /* 1414 * This dbuf is not dirty in the open context. 1415 * Either uncache it (if its not referenced in 1416 * the open context) or reset its contents to 1417 * empty. 1418 */ 1419 dbuf_fix_old_data(db, txg); 1420 } 1421 } 1422 /* clear the contents if its cached */ 1423 if (db->db_state == DB_CACHED) { 1424 ASSERT(db->db.db_data != NULL); 1425 arc_release(db->db_buf, db); 1426 bzero(db->db.db_data, db->db.db_size); 1427 arc_buf_freeze(db->db_buf); 1428 } 1429 1430 mutex_exit(&db->db_mtx); 1431 } 1432 mutex_exit(&dn->dn_dbufs_mtx); 1433 } 1434 1435 void 1436 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 1437 { 1438 arc_buf_t *buf, *obuf; 1439 int osize = db->db.db_size; 1440 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1441 dnode_t *dn; 1442 1443 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1444 1445 DB_DNODE_ENTER(db); 1446 dn = DB_DNODE(db); 1447 1448 /* XXX does *this* func really need the lock? */ 1449 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1450 1451 /* 1452 * This call to dmu_buf_will_dirty() with the dn_struct_rwlock held 1453 * is OK, because there can be no other references to the db 1454 * when we are changing its size, so no concurrent DB_FILL can 1455 * be happening. 1456 */ 1457 /* 1458 * XXX we should be doing a dbuf_read, checking the return 1459 * value and returning that up to our callers 1460 */ 1461 dmu_buf_will_dirty(&db->db, tx); 1462 1463 /* create the data buffer for the new block */ 1464 buf = arc_alloc_buf(dn->dn_objset->os_spa, db, type, size); 1465 1466 /* copy old block data to the new block */ 1467 obuf = db->db_buf; 1468 bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 1469 /* zero the remainder */ 1470 if (size > osize) 1471 bzero((uint8_t *)buf->b_data + osize, size - osize); 1472 1473 mutex_enter(&db->db_mtx); 1474 dbuf_set_data(db, buf); 1475 arc_buf_destroy(obuf, db); 1476 db->db.db_size = size; 1477 1478 if (db->db_level == 0) { 1479 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 1480 db->db_last_dirty->dt.dl.dr_data = buf; 1481 } 1482 mutex_exit(&db->db_mtx); 1483 1484 dmu_objset_willuse_space(dn->dn_objset, size - osize, tx); 1485 DB_DNODE_EXIT(db); 1486 } 1487 1488 void 1489 dbuf_release_bp(dmu_buf_impl_t *db) 1490 { 1491 objset_t *os = db->db_objset; 1492 1493 ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 1494 ASSERT(arc_released(os->os_phys_buf) || 1495 list_link_active(&os->os_dsl_dataset->ds_synced_link)); 1496 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 1497 1498 (void) arc_release(db->db_buf, db); 1499 } 1500 1501 /* 1502 * We already have a dirty record for this TXG, and we are being 1503 * dirtied again. 1504 */ 1505 static void 1506 dbuf_redirty(dbuf_dirty_record_t *dr) 1507 { 1508 dmu_buf_impl_t *db = dr->dr_dbuf; 1509 1510 ASSERT(MUTEX_HELD(&db->db_mtx)); 1511 1512 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 1513 /* 1514 * If this buffer has already been written out, 1515 * we now need to reset its state. 1516 */ 1517 dbuf_unoverride(dr); 1518 if (db->db.db_object != DMU_META_DNODE_OBJECT && 1519 db->db_state != DB_NOFILL) { 1520 /* Already released on initial dirty, so just thaw. */ 1521 ASSERT(arc_released(db->db_buf)); 1522 arc_buf_thaw(db->db_buf); 1523 } 1524 } 1525 } 1526 1527 dbuf_dirty_record_t * 1528 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1529 { 1530 dnode_t *dn; 1531 objset_t *os; 1532 dbuf_dirty_record_t **drp, *dr; 1533 int drop_struct_lock = FALSE; 1534 int txgoff = tx->tx_txg & TXG_MASK; 1535 1536 ASSERT(tx->tx_txg != 0); 1537 ASSERT(!refcount_is_zero(&db->db_holds)); 1538 DMU_TX_DIRTY_BUF(tx, db); 1539 1540 DB_DNODE_ENTER(db); 1541 dn = DB_DNODE(db); 1542 /* 1543 * Shouldn't dirty a regular buffer in syncing context. Private 1544 * objects may be dirtied in syncing context, but only if they 1545 * were already pre-dirtied in open context. 1546 */ 1547 #ifdef DEBUG 1548 if (dn->dn_objset->os_dsl_dataset != NULL) { 1549 rrw_enter(&dn->dn_objset->os_dsl_dataset->ds_bp_rwlock, 1550 RW_READER, FTAG); 1551 } 1552 ASSERT(!dmu_tx_is_syncing(tx) || 1553 BP_IS_HOLE(dn->dn_objset->os_rootbp) || 1554 DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1555 dn->dn_objset->os_dsl_dataset == NULL); 1556 if (dn->dn_objset->os_dsl_dataset != NULL) 1557 rrw_exit(&dn->dn_objset->os_dsl_dataset->ds_bp_rwlock, FTAG); 1558 #endif 1559 /* 1560 * We make this assert for private objects as well, but after we 1561 * check if we're already dirty. They are allowed to re-dirty 1562 * in syncing context. 1563 */ 1564 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1565 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1566 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1567 1568 mutex_enter(&db->db_mtx); 1569 /* 1570 * XXX make this true for indirects too? The problem is that 1571 * transactions created with dmu_tx_create_assigned() from 1572 * syncing context don't bother holding ahead. 1573 */ 1574 ASSERT(db->db_level != 0 || 1575 db->db_state == DB_CACHED || db->db_state == DB_FILL || 1576 db->db_state == DB_NOFILL); 1577 1578 mutex_enter(&dn->dn_mtx); 1579 /* 1580 * Don't set dirtyctx to SYNC if we're just modifying this as we 1581 * initialize the objset. 1582 */ 1583 if (dn->dn_dirtyctx == DN_UNDIRTIED) { 1584 if (dn->dn_objset->os_dsl_dataset != NULL) { 1585 rrw_enter(&dn->dn_objset->os_dsl_dataset->ds_bp_rwlock, 1586 RW_READER, FTAG); 1587 } 1588 if (!BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 1589 dn->dn_dirtyctx = (dmu_tx_is_syncing(tx) ? 1590 DN_DIRTY_SYNC : DN_DIRTY_OPEN); 1591 ASSERT(dn->dn_dirtyctx_firstset == NULL); 1592 dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 1593 } 1594 if (dn->dn_objset->os_dsl_dataset != NULL) { 1595 rrw_exit(&dn->dn_objset->os_dsl_dataset->ds_bp_rwlock, 1596 FTAG); 1597 } 1598 } 1599 mutex_exit(&dn->dn_mtx); 1600 1601 if (db->db_blkid == DMU_SPILL_BLKID) 1602 dn->dn_have_spill = B_TRUE; 1603 1604 /* 1605 * If this buffer is already dirty, we're done. 1606 */ 1607 drp = &db->db_last_dirty; 1608 ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 1609 db->db.db_object == DMU_META_DNODE_OBJECT); 1610 while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 1611 drp = &dr->dr_next; 1612 if (dr && dr->dr_txg == tx->tx_txg) { 1613 DB_DNODE_EXIT(db); 1614 1615 dbuf_redirty(dr); 1616 mutex_exit(&db->db_mtx); 1617 return (dr); 1618 } 1619 1620 /* 1621 * Only valid if not already dirty. 1622 */ 1623 ASSERT(dn->dn_object == 0 || 1624 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1625 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1626 1627 ASSERT3U(dn->dn_nlevels, >, db->db_level); 1628 1629 /* 1630 * We should only be dirtying in syncing context if it's the 1631 * mos or we're initializing the os or it's a special object. 1632 * However, we are allowed to dirty in syncing context provided 1633 * we already dirtied it in open context. Hence we must make 1634 * this assertion only if we're not already dirty. 1635 */ 1636 os = dn->dn_objset; 1637 VERIFY3U(tx->tx_txg, <=, spa_final_dirty_txg(os->os_spa)); 1638 #ifdef DEBUG 1639 if (dn->dn_objset->os_dsl_dataset != NULL) 1640 rrw_enter(&os->os_dsl_dataset->ds_bp_rwlock, RW_READER, FTAG); 1641 ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1642 os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1643 if (dn->dn_objset->os_dsl_dataset != NULL) 1644 rrw_exit(&os->os_dsl_dataset->ds_bp_rwlock, FTAG); 1645 #endif 1646 ASSERT(db->db.db_size != 0); 1647 1648 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1649 1650 if (db->db_blkid != DMU_BONUS_BLKID) { 1651 dmu_objset_willuse_space(os, db->db.db_size, tx); 1652 } 1653 1654 /* 1655 * If this buffer is dirty in an old transaction group we need 1656 * to make a copy of it so that the changes we make in this 1657 * transaction group won't leak out when we sync the older txg. 1658 */ 1659 dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1660 if (db->db_level == 0) { 1661 void *data_old = db->db_buf; 1662 1663 if (db->db_state != DB_NOFILL) { 1664 if (db->db_blkid == DMU_BONUS_BLKID) { 1665 dbuf_fix_old_data(db, tx->tx_txg); 1666 data_old = db->db.db_data; 1667 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1668 /* 1669 * Release the data buffer from the cache so 1670 * that we can modify it without impacting 1671 * possible other users of this cached data 1672 * block. Note that indirect blocks and 1673 * private objects are not released until the 1674 * syncing state (since they are only modified 1675 * then). 1676 */ 1677 arc_release(db->db_buf, db); 1678 dbuf_fix_old_data(db, tx->tx_txg); 1679 data_old = db->db_buf; 1680 } 1681 ASSERT(data_old != NULL); 1682 } 1683 dr->dt.dl.dr_data = data_old; 1684 } else { 1685 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1686 list_create(&dr->dt.di.dr_children, 1687 sizeof (dbuf_dirty_record_t), 1688 offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1689 } 1690 if (db->db_blkid != DMU_BONUS_BLKID && os->os_dsl_dataset != NULL) 1691 dr->dr_accounted = db->db.db_size; 1692 dr->dr_dbuf = db; 1693 dr->dr_txg = tx->tx_txg; 1694 dr->dr_next = *drp; 1695 *drp = dr; 1696 1697 /* 1698 * We could have been freed_in_flight between the dbuf_noread 1699 * and dbuf_dirty. We win, as though the dbuf_noread() had 1700 * happened after the free. 1701 */ 1702 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1703 db->db_blkid != DMU_SPILL_BLKID) { 1704 mutex_enter(&dn->dn_mtx); 1705 if (dn->dn_free_ranges[txgoff] != NULL) { 1706 range_tree_clear(dn->dn_free_ranges[txgoff], 1707 db->db_blkid, 1); 1708 } 1709 mutex_exit(&dn->dn_mtx); 1710 db->db_freed_in_flight = FALSE; 1711 } 1712 1713 /* 1714 * This buffer is now part of this txg 1715 */ 1716 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1717 db->db_dirtycnt += 1; 1718 ASSERT3U(db->db_dirtycnt, <=, 3); 1719 1720 mutex_exit(&db->db_mtx); 1721 1722 if (db->db_blkid == DMU_BONUS_BLKID || 1723 db->db_blkid == DMU_SPILL_BLKID) { 1724 mutex_enter(&dn->dn_mtx); 1725 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1726 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1727 mutex_exit(&dn->dn_mtx); 1728 dnode_setdirty(dn, tx); 1729 DB_DNODE_EXIT(db); 1730 return (dr); 1731 } 1732 1733 /* 1734 * The dn_struct_rwlock prevents db_blkptr from changing 1735 * due to a write from syncing context completing 1736 * while we are running, so we want to acquire it before 1737 * looking at db_blkptr. 1738 */ 1739 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1740 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1741 drop_struct_lock = TRUE; 1742 } 1743 1744 /* 1745 * We need to hold the dn_struct_rwlock to make this assertion, 1746 * because it protects dn_phys / dn_next_nlevels from changing. 1747 */ 1748 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1749 dn->dn_phys->dn_nlevels > db->db_level || 1750 dn->dn_next_nlevels[txgoff] > db->db_level || 1751 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1752 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1753 1754 /* 1755 * If we are overwriting a dedup BP, then unless it is snapshotted, 1756 * when we get to syncing context we will need to decrement its 1757 * refcount in the DDT. Prefetch the relevant DDT block so that 1758 * syncing context won't have to wait for the i/o. 1759 */ 1760 ddt_prefetch(os->os_spa, db->db_blkptr); 1761 1762 if (db->db_level == 0) { 1763 dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 1764 ASSERT(dn->dn_maxblkid >= db->db_blkid); 1765 } 1766 1767 if (db->db_level+1 < dn->dn_nlevels) { 1768 dmu_buf_impl_t *parent = db->db_parent; 1769 dbuf_dirty_record_t *di; 1770 int parent_held = FALSE; 1771 1772 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1773 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1774 1775 parent = dbuf_hold_level(dn, db->db_level+1, 1776 db->db_blkid >> epbs, FTAG); 1777 ASSERT(parent != NULL); 1778 parent_held = TRUE; 1779 } 1780 if (drop_struct_lock) 1781 rw_exit(&dn->dn_struct_rwlock); 1782 ASSERT3U(db->db_level+1, ==, parent->db_level); 1783 di = dbuf_dirty(parent, tx); 1784 if (parent_held) 1785 dbuf_rele(parent, FTAG); 1786 1787 mutex_enter(&db->db_mtx); 1788 /* 1789 * Since we've dropped the mutex, it's possible that 1790 * dbuf_undirty() might have changed this out from under us. 1791 */ 1792 if (db->db_last_dirty == dr || 1793 dn->dn_object == DMU_META_DNODE_OBJECT) { 1794 mutex_enter(&di->dt.di.dr_mtx); 1795 ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1796 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1797 list_insert_tail(&di->dt.di.dr_children, dr); 1798 mutex_exit(&di->dt.di.dr_mtx); 1799 dr->dr_parent = di; 1800 } 1801 mutex_exit(&db->db_mtx); 1802 } else { 1803 ASSERT(db->db_level+1 == dn->dn_nlevels); 1804 ASSERT(db->db_blkid < dn->dn_nblkptr); 1805 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); 1806 mutex_enter(&dn->dn_mtx); 1807 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1808 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1809 mutex_exit(&dn->dn_mtx); 1810 if (drop_struct_lock) 1811 rw_exit(&dn->dn_struct_rwlock); 1812 } 1813 1814 dnode_setdirty(dn, tx); 1815 DB_DNODE_EXIT(db); 1816 return (dr); 1817 } 1818 1819 /* 1820 * Undirty a buffer in the transaction group referenced by the given 1821 * transaction. Return whether this evicted the dbuf. 1822 */ 1823 static boolean_t 1824 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1825 { 1826 dnode_t *dn; 1827 uint64_t txg = tx->tx_txg; 1828 dbuf_dirty_record_t *dr, **drp; 1829 1830 ASSERT(txg != 0); 1831 1832 /* 1833 * Due to our use of dn_nlevels below, this can only be called 1834 * in open context, unless we are operating on the MOS. 1835 * From syncing context, dn_nlevels may be different from the 1836 * dn_nlevels used when dbuf was dirtied. 1837 */ 1838 ASSERT(db->db_objset == 1839 dmu_objset_pool(db->db_objset)->dp_meta_objset || 1840 txg != spa_syncing_txg(dmu_objset_spa(db->db_objset))); 1841 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1842 ASSERT0(db->db_level); 1843 ASSERT(MUTEX_HELD(&db->db_mtx)); 1844 1845 /* 1846 * If this buffer is not dirty, we're done. 1847 */ 1848 for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1849 if (dr->dr_txg <= txg) 1850 break; 1851 if (dr == NULL || dr->dr_txg < txg) 1852 return (B_FALSE); 1853 ASSERT(dr->dr_txg == txg); 1854 ASSERT(dr->dr_dbuf == db); 1855 1856 DB_DNODE_ENTER(db); 1857 dn = DB_DNODE(db); 1858 1859 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1860 1861 ASSERT(db->db.db_size != 0); 1862 1863 dsl_pool_undirty_space(dmu_objset_pool(dn->dn_objset), 1864 dr->dr_accounted, txg); 1865 1866 *drp = dr->dr_next; 1867 1868 /* 1869 * Note that there are three places in dbuf_dirty() 1870 * where this dirty record may be put on a list. 1871 * Make sure to do a list_remove corresponding to 1872 * every one of those list_insert calls. 1873 */ 1874 if (dr->dr_parent) { 1875 mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1876 list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1877 mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1878 } else if (db->db_blkid == DMU_SPILL_BLKID || 1879 db->db_level + 1 == dn->dn_nlevels) { 1880 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1881 mutex_enter(&dn->dn_mtx); 1882 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1883 mutex_exit(&dn->dn_mtx); 1884 } 1885 DB_DNODE_EXIT(db); 1886 1887 if (db->db_state != DB_NOFILL) { 1888 dbuf_unoverride(dr); 1889 1890 ASSERT(db->db_buf != NULL); 1891 ASSERT(dr->dt.dl.dr_data != NULL); 1892 if (dr->dt.dl.dr_data != db->db_buf) 1893 arc_buf_destroy(dr->dt.dl.dr_data, db); 1894 } 1895 1896 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1897 1898 ASSERT(db->db_dirtycnt > 0); 1899 db->db_dirtycnt -= 1; 1900 1901 if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1902 ASSERT(db->db_state == DB_NOFILL || arc_released(db->db_buf)); 1903 dbuf_destroy(db); 1904 return (B_TRUE); 1905 } 1906 1907 return (B_FALSE); 1908 } 1909 1910 void 1911 dmu_buf_will_dirty(dmu_buf_t *db_fake, dmu_tx_t *tx) 1912 { 1913 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1914 int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1915 1916 ASSERT(tx->tx_txg != 0); 1917 ASSERT(!refcount_is_zero(&db->db_holds)); 1918 1919 /* 1920 * Quick check for dirtyness. For already dirty blocks, this 1921 * reduces runtime of this function by >90%, and overall performance 1922 * by 50% for some workloads (e.g. file deletion with indirect blocks 1923 * cached). 1924 */ 1925 mutex_enter(&db->db_mtx); 1926 dbuf_dirty_record_t *dr; 1927 for (dr = db->db_last_dirty; 1928 dr != NULL && dr->dr_txg >= tx->tx_txg; dr = dr->dr_next) { 1929 /* 1930 * It's possible that it is already dirty but not cached, 1931 * because there are some calls to dbuf_dirty() that don't 1932 * go through dmu_buf_will_dirty(). 1933 */ 1934 if (dr->dr_txg == tx->tx_txg && db->db_state == DB_CACHED) { 1935 /* This dbuf is already dirty and cached. */ 1936 dbuf_redirty(dr); 1937 mutex_exit(&db->db_mtx); 1938 return; 1939 } 1940 } 1941 mutex_exit(&db->db_mtx); 1942 1943 DB_DNODE_ENTER(db); 1944 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) 1945 rf |= DB_RF_HAVESTRUCT; 1946 DB_DNODE_EXIT(db); 1947 (void) dbuf_read(db, NULL, rf); 1948 (void) dbuf_dirty(db, tx); 1949 } 1950 1951 void 1952 dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1953 { 1954 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1955 1956 db->db_state = DB_NOFILL; 1957 1958 dmu_buf_will_fill(db_fake, tx); 1959 } 1960 1961 void 1962 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1963 { 1964 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1965 1966 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1967 ASSERT(tx->tx_txg != 0); 1968 ASSERT(db->db_level == 0); 1969 ASSERT(!refcount_is_zero(&db->db_holds)); 1970 1971 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1972 dmu_tx_private_ok(tx)); 1973 1974 dbuf_noread(db); 1975 (void) dbuf_dirty(db, tx); 1976 } 1977 1978 #pragma weak dmu_buf_fill_done = dbuf_fill_done 1979 /* ARGSUSED */ 1980 void 1981 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1982 { 1983 mutex_enter(&db->db_mtx); 1984 DBUF_VERIFY(db); 1985 1986 if (db->db_state == DB_FILL) { 1987 if (db->db_level == 0 && db->db_freed_in_flight) { 1988 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1989 /* we were freed while filling */ 1990 /* XXX dbuf_undirty? */ 1991 bzero(db->db.db_data, db->db.db_size); 1992 db->db_freed_in_flight = FALSE; 1993 } 1994 db->db_state = DB_CACHED; 1995 cv_broadcast(&db->db_changed); 1996 } 1997 mutex_exit(&db->db_mtx); 1998 } 1999 2000 void 2001 dmu_buf_write_embedded(dmu_buf_t *dbuf, void *data, 2002 bp_embedded_type_t etype, enum zio_compress comp, 2003 int uncompressed_size, int compressed_size, int byteorder, 2004 dmu_tx_t *tx) 2005 { 2006 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 2007 struct dirty_leaf *dl; 2008 dmu_object_type_t type; 2009 2010 if (etype == BP_EMBEDDED_TYPE_DATA) { 2011 ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset), 2012 SPA_FEATURE_EMBEDDED_DATA)); 2013 } 2014 2015 DB_DNODE_ENTER(db); 2016 type = DB_DNODE(db)->dn_type; 2017 DB_DNODE_EXIT(db); 2018 2019 ASSERT0(db->db_level); 2020 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2021 2022 dmu_buf_will_not_fill(dbuf, tx); 2023 2024 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 2025 dl = &db->db_last_dirty->dt.dl; 2026 encode_embedded_bp_compressed(&dl->dr_overridden_by, 2027 data, comp, uncompressed_size, compressed_size); 2028 BPE_SET_ETYPE(&dl->dr_overridden_by, etype); 2029 BP_SET_TYPE(&dl->dr_overridden_by, type); 2030 BP_SET_LEVEL(&dl->dr_overridden_by, 0); 2031 BP_SET_BYTEORDER(&dl->dr_overridden_by, byteorder); 2032 2033 dl->dr_override_state = DR_OVERRIDDEN; 2034 dl->dr_overridden_by.blk_birth = db->db_last_dirty->dr_txg; 2035 } 2036 2037 /* 2038 * Directly assign a provided arc buf to a given dbuf if it's not referenced 2039 * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 2040 */ 2041 void 2042 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 2043 { 2044 ASSERT(!refcount_is_zero(&db->db_holds)); 2045 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2046 ASSERT(db->db_level == 0); 2047 ASSERT3U(dbuf_is_metadata(db), ==, arc_is_metadata(buf)); 2048 ASSERT(buf != NULL); 2049 ASSERT(arc_buf_lsize(buf) == db->db.db_size); 2050 ASSERT(tx->tx_txg != 0); 2051 2052 arc_return_buf(buf, db); 2053 ASSERT(arc_released(buf)); 2054 2055 mutex_enter(&db->db_mtx); 2056 2057 while (db->db_state == DB_READ || db->db_state == DB_FILL) 2058 cv_wait(&db->db_changed, &db->db_mtx); 2059 2060 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 2061 2062 if (db->db_state == DB_CACHED && 2063 refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 2064 mutex_exit(&db->db_mtx); 2065 (void) dbuf_dirty(db, tx); 2066 bcopy(buf->b_data, db->db.db_data, db->db.db_size); 2067 arc_buf_destroy(buf, db); 2068 xuio_stat_wbuf_copied(); 2069 return; 2070 } 2071 2072 xuio_stat_wbuf_nocopy(); 2073 if (db->db_state == DB_CACHED) { 2074 dbuf_dirty_record_t *dr = db->db_last_dirty; 2075 2076 ASSERT(db->db_buf != NULL); 2077 if (dr != NULL && dr->dr_txg == tx->tx_txg) { 2078 ASSERT(dr->dt.dl.dr_data == db->db_buf); 2079 if (!arc_released(db->db_buf)) { 2080 ASSERT(dr->dt.dl.dr_override_state == 2081 DR_OVERRIDDEN); 2082 arc_release(db->db_buf, db); 2083 } 2084 dr->dt.dl.dr_data = buf; 2085 arc_buf_destroy(db->db_buf, db); 2086 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 2087 arc_release(db->db_buf, db); 2088 arc_buf_destroy(db->db_buf, db); 2089 } 2090 db->db_buf = NULL; 2091 } 2092 ASSERT(db->db_buf == NULL); 2093 dbuf_set_data(db, buf); 2094 db->db_state = DB_FILL; 2095 mutex_exit(&db->db_mtx); 2096 (void) dbuf_dirty(db, tx); 2097 dmu_buf_fill_done(&db->db, tx); 2098 } 2099 2100 void 2101 dbuf_destroy(dmu_buf_impl_t *db) 2102 { 2103 dnode_t *dn; 2104 dmu_buf_impl_t *parent = db->db_parent; 2105 dmu_buf_impl_t *dndb; 2106 2107 ASSERT(MUTEX_HELD(&db->db_mtx)); 2108 ASSERT(refcount_is_zero(&db->db_holds)); 2109 2110 if (db->db_buf != NULL) { 2111 arc_buf_destroy(db->db_buf, db); 2112 db->db_buf = NULL; 2113 } 2114 2115 if (db->db_blkid == DMU_BONUS_BLKID) { 2116 ASSERT(db->db.db_data != NULL); 2117 zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 2118 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 2119 db->db_state = DB_UNCACHED; 2120 } 2121 2122 dbuf_clear_data(db); 2123 2124 if (multilist_link_active(&db->db_cache_link)) { 2125 ASSERT(db->db_caching_status == DB_DBUF_CACHE || 2126 db->db_caching_status == DB_DBUF_METADATA_CACHE); 2127 2128 multilist_remove(dbuf_caches[db->db_caching_status].cache, db); 2129 (void) refcount_remove_many( 2130 &dbuf_caches[db->db_caching_status].size, 2131 db->db.db_size, db); 2132 2133 db->db_caching_status = DB_NO_CACHE; 2134 } 2135 2136 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 2137 ASSERT(db->db_data_pending == NULL); 2138 2139 db->db_state = DB_EVICTING; 2140 db->db_blkptr = NULL; 2141 2142 /* 2143 * Now that db_state is DB_EVICTING, nobody else can find this via 2144 * the hash table. We can now drop db_mtx, which allows us to 2145 * acquire the dn_dbufs_mtx. 2146 */ 2147 mutex_exit(&db->db_mtx); 2148 2149 DB_DNODE_ENTER(db); 2150 dn = DB_DNODE(db); 2151 dndb = dn->dn_dbuf; 2152 if (db->db_blkid != DMU_BONUS_BLKID) { 2153 boolean_t needlock = !MUTEX_HELD(&dn->dn_dbufs_mtx); 2154 if (needlock) 2155 mutex_enter(&dn->dn_dbufs_mtx); 2156 avl_remove(&dn->dn_dbufs, db); 2157 atomic_dec_32(&dn->dn_dbufs_count); 2158 membar_producer(); 2159 DB_DNODE_EXIT(db); 2160 if (needlock) 2161 mutex_exit(&dn->dn_dbufs_mtx); 2162 /* 2163 * Decrementing the dbuf count means that the hold corresponding 2164 * to the removed dbuf is no longer discounted in dnode_move(), 2165 * so the dnode cannot be moved until after we release the hold. 2166 * The membar_producer() ensures visibility of the decremented 2167 * value in dnode_move(), since DB_DNODE_EXIT doesn't actually 2168 * release any lock. 2169 */ 2170 dnode_rele(dn, db); 2171 db->db_dnode_handle = NULL; 2172 2173 dbuf_hash_remove(db); 2174 } else { 2175 DB_DNODE_EXIT(db); 2176 } 2177 2178 ASSERT(refcount_is_zero(&db->db_holds)); 2179 2180 db->db_parent = NULL; 2181 2182 ASSERT(db->db_buf == NULL); 2183 ASSERT(db->db.db_data == NULL); 2184 ASSERT(db->db_hash_next == NULL); 2185 ASSERT(db->db_blkptr == NULL); 2186 ASSERT(db->db_data_pending == NULL); 2187 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); 2188 ASSERT(!multilist_link_active(&db->db_cache_link)); 2189 2190 kmem_cache_free(dbuf_kmem_cache, db); 2191 arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 2192 2193 /* 2194 * If this dbuf is referenced from an indirect dbuf, 2195 * decrement the ref count on the indirect dbuf. 2196 */ 2197 if (parent && parent != dndb) 2198 dbuf_rele(parent, db); 2199 } 2200 2201 /* 2202 * Note: While bpp will always be updated if the function returns success, 2203 * parentp will not be updated if the dnode does not have dn_dbuf filled in; 2204 * this happens when the dnode is the meta-dnode, or a userused or groupused 2205 * object. 2206 */ 2207 static int 2208 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 2209 dmu_buf_impl_t **parentp, blkptr_t **bpp) 2210 { 2211 *parentp = NULL; 2212 *bpp = NULL; 2213 2214 ASSERT(blkid != DMU_BONUS_BLKID); 2215 2216 if (blkid == DMU_SPILL_BLKID) { 2217 mutex_enter(&dn->dn_mtx); 2218 if (dn->dn_have_spill && 2219 (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 2220 *bpp = &dn->dn_phys->dn_spill; 2221 else 2222 *bpp = NULL; 2223 dbuf_add_ref(dn->dn_dbuf, NULL); 2224 *parentp = dn->dn_dbuf; 2225 mutex_exit(&dn->dn_mtx); 2226 return (0); 2227 } 2228 2229 int nlevels = 2230 (dn->dn_phys->dn_nlevels == 0) ? 1 : dn->dn_phys->dn_nlevels; 2231 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 2232 2233 ASSERT3U(level * epbs, <, 64); 2234 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2235 /* 2236 * This assertion shouldn't trip as long as the max indirect block size 2237 * is less than 1M. The reason for this is that up to that point, 2238 * the number of levels required to address an entire object with blocks 2239 * of size SPA_MINBLOCKSIZE satisfies nlevels * epbs + 1 <= 64. In 2240 * other words, if N * epbs + 1 > 64, then if (N-1) * epbs + 1 > 55 2241 * (i.e. we can address the entire object), objects will all use at most 2242 * N-1 levels and the assertion won't overflow. However, once epbs is 2243 * 13, 4 * 13 + 1 = 53, but 5 * 13 + 1 = 66. Then, 4 levels will not be 2244 * enough to address an entire object, so objects will have 5 levels, 2245 * but then this assertion will overflow. 2246 * 2247 * All this is to say that if we ever increase DN_MAX_INDBLKSHIFT, we 2248 * need to redo this logic to handle overflows. 2249 */ 2250 ASSERT(level >= nlevels || 2251 ((nlevels - level - 1) * epbs) + 2252 highbit64(dn->dn_phys->dn_nblkptr) <= 64); 2253 if (level >= nlevels || 2254 blkid >= ((uint64_t)dn->dn_phys->dn_nblkptr << 2255 ((nlevels - level - 1) * epbs)) || 2256 (fail_sparse && 2257 blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 2258 /* the buffer has no parent yet */ 2259 return (SET_ERROR(ENOENT)); 2260 } else if (level < nlevels-1) { 2261 /* this block is referenced from an indirect block */ 2262 int err = dbuf_hold_impl(dn, level+1, 2263 blkid >> epbs, fail_sparse, FALSE, NULL, parentp); 2264 if (err) 2265 return (err); 2266 err = dbuf_read(*parentp, NULL, 2267 (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 2268 if (err) { 2269 dbuf_rele(*parentp, NULL); 2270 *parentp = NULL; 2271 return (err); 2272 } 2273 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 2274 (blkid & ((1ULL << epbs) - 1)); 2275 if (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs))) 2276 ASSERT(BP_IS_HOLE(*bpp)); 2277 return (0); 2278 } else { 2279 /* the block is referenced from the dnode */ 2280 ASSERT3U(level, ==, nlevels-1); 2281 ASSERT(dn->dn_phys->dn_nblkptr == 0 || 2282 blkid < dn->dn_phys->dn_nblkptr); 2283 if (dn->dn_dbuf) { 2284 dbuf_add_ref(dn->dn_dbuf, NULL); 2285 *parentp = dn->dn_dbuf; 2286 } 2287 *bpp = &dn->dn_phys->dn_blkptr[blkid]; 2288 return (0); 2289 } 2290 } 2291 2292 static dmu_buf_impl_t * 2293 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 2294 dmu_buf_impl_t *parent, blkptr_t *blkptr) 2295 { 2296 objset_t *os = dn->dn_objset; 2297 dmu_buf_impl_t *db, *odb; 2298 2299 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2300 ASSERT(dn->dn_type != DMU_OT_NONE); 2301 2302 db = kmem_cache_alloc(dbuf_kmem_cache, KM_SLEEP); 2303 2304 db->db_objset = os; 2305 db->db.db_object = dn->dn_object; 2306 db->db_level = level; 2307 db->db_blkid = blkid; 2308 db->db_last_dirty = NULL; 2309 db->db_dirtycnt = 0; 2310 db->db_dnode_handle = dn->dn_handle; 2311 db->db_parent = parent; 2312 db->db_blkptr = blkptr; 2313 2314 db->db_user = NULL; 2315 db->db_user_immediate_evict = FALSE; 2316 db->db_freed_in_flight = FALSE; 2317 db->db_pending_evict = FALSE; 2318 2319 if (blkid == DMU_BONUS_BLKID) { 2320 ASSERT3P(parent, ==, dn->dn_dbuf); 2321 db->db.db_size = DN_MAX_BONUSLEN - 2322 (dn->dn_nblkptr-1) * sizeof (blkptr_t); 2323 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 2324 db->db.db_offset = DMU_BONUS_BLKID; 2325 db->db_state = DB_UNCACHED; 2326 db->db_caching_status = DB_NO_CACHE; 2327 /* the bonus dbuf is not placed in the hash table */ 2328 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 2329 return (db); 2330 } else if (blkid == DMU_SPILL_BLKID) { 2331 db->db.db_size = (blkptr != NULL) ? 2332 BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 2333 db->db.db_offset = 0; 2334 } else { 2335 int blocksize = 2336 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz; 2337 db->db.db_size = blocksize; 2338 db->db.db_offset = db->db_blkid * blocksize; 2339 } 2340 2341 /* 2342 * Hold the dn_dbufs_mtx while we get the new dbuf 2343 * in the hash table *and* added to the dbufs list. 2344 * This prevents a possible deadlock with someone 2345 * trying to look up this dbuf before its added to the 2346 * dn_dbufs list. 2347 */ 2348 mutex_enter(&dn->dn_dbufs_mtx); 2349 db->db_state = DB_EVICTING; 2350 if ((odb = dbuf_hash_insert(db)) != NULL) { 2351 /* someone else inserted it first */ 2352 kmem_cache_free(dbuf_kmem_cache, db); 2353 mutex_exit(&dn->dn_dbufs_mtx); 2354 return (odb); 2355 } 2356 avl_add(&dn->dn_dbufs, db); 2357 2358 db->db_state = DB_UNCACHED; 2359 db->db_caching_status = DB_NO_CACHE; 2360 mutex_exit(&dn->dn_dbufs_mtx); 2361 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 2362 2363 if (parent && parent != dn->dn_dbuf) 2364 dbuf_add_ref(parent, db); 2365 2366 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 2367 refcount_count(&dn->dn_holds) > 0); 2368 (void) refcount_add(&dn->dn_holds, db); 2369 atomic_inc_32(&dn->dn_dbufs_count); 2370 2371 dprintf_dbuf(db, "db=%p\n", db); 2372 2373 return (db); 2374 } 2375 2376 typedef struct dbuf_prefetch_arg { 2377 spa_t *dpa_spa; /* The spa to issue the prefetch in. */ 2378 zbookmark_phys_t dpa_zb; /* The target block to prefetch. */ 2379 int dpa_epbs; /* Entries (blkptr_t's) Per Block Shift. */ 2380 int dpa_curlevel; /* The current level that we're reading */ 2381 dnode_t *dpa_dnode; /* The dnode associated with the prefetch */ 2382 zio_priority_t dpa_prio; /* The priority I/Os should be issued at. */ 2383 zio_t *dpa_zio; /* The parent zio_t for all prefetches. */ 2384 arc_flags_t dpa_aflags; /* Flags to pass to the final prefetch. */ 2385 } dbuf_prefetch_arg_t; 2386 2387 /* 2388 * Actually issue the prefetch read for the block given. 2389 */ 2390 static void 2391 dbuf_issue_final_prefetch(dbuf_prefetch_arg_t *dpa, blkptr_t *bp) 2392 { 2393 if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) 2394 return; 2395 2396 arc_flags_t aflags = 2397 dpa->dpa_aflags | ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH; 2398 2399 ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp)); 2400 ASSERT3U(dpa->dpa_curlevel, ==, dpa->dpa_zb.zb_level); 2401 ASSERT(dpa->dpa_zio != NULL); 2402 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, bp, NULL, NULL, 2403 dpa->dpa_prio, ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2404 &aflags, &dpa->dpa_zb); 2405 } 2406 2407 /* 2408 * Called when an indirect block above our prefetch target is read in. This 2409 * will either read in the next indirect block down the tree or issue the actual 2410 * prefetch if the next block down is our target. 2411 */ 2412 static void 2413 dbuf_prefetch_indirect_done(zio_t *zio, arc_buf_t *abuf, void *private) 2414 { 2415 dbuf_prefetch_arg_t *dpa = private; 2416 2417 ASSERT3S(dpa->dpa_zb.zb_level, <, dpa->dpa_curlevel); 2418 ASSERT3S(dpa->dpa_curlevel, >, 0); 2419 2420 /* 2421 * The dpa_dnode is only valid if we are called with a NULL 2422 * zio. This indicates that the arc_read() returned without 2423 * first calling zio_read() to issue a physical read. Once 2424 * a physical read is made the dpa_dnode must be invalidated 2425 * as the locks guarding it may have been dropped. If the 2426 * dpa_dnode is still valid, then we want to add it to the dbuf 2427 * cache. To do so, we must hold the dbuf associated with the block 2428 * we just prefetched, read its contents so that we associate it 2429 * with an arc_buf_t, and then release it. 2430 */ 2431 if (zio != NULL) { 2432 ASSERT3S(BP_GET_LEVEL(zio->io_bp), ==, dpa->dpa_curlevel); 2433 if (zio->io_flags & ZIO_FLAG_RAW) { 2434 ASSERT3U(BP_GET_PSIZE(zio->io_bp), ==, zio->io_size); 2435 } else { 2436 ASSERT3U(BP_GET_LSIZE(zio->io_bp), ==, zio->io_size); 2437 } 2438 ASSERT3P(zio->io_spa, ==, dpa->dpa_spa); 2439 2440 dpa->dpa_dnode = NULL; 2441 } else if (dpa->dpa_dnode != NULL) { 2442 uint64_t curblkid = dpa->dpa_zb.zb_blkid >> 2443 (dpa->dpa_epbs * (dpa->dpa_curlevel - 2444 dpa->dpa_zb.zb_level)); 2445 dmu_buf_impl_t *db = dbuf_hold_level(dpa->dpa_dnode, 2446 dpa->dpa_curlevel, curblkid, FTAG); 2447 (void) dbuf_read(db, NULL, 2448 DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH | DB_RF_HAVESTRUCT); 2449 dbuf_rele(db, FTAG); 2450 } 2451 2452 dpa->dpa_curlevel--; 2453 2454 uint64_t nextblkid = dpa->dpa_zb.zb_blkid >> 2455 (dpa->dpa_epbs * (dpa->dpa_curlevel - dpa->dpa_zb.zb_level)); 2456 blkptr_t *bp = ((blkptr_t *)abuf->b_data) + 2457 P2PHASE(nextblkid, 1ULL << dpa->dpa_epbs); 2458 if (BP_IS_HOLE(bp) || (zio != NULL && zio->io_error != 0)) { 2459 kmem_free(dpa, sizeof (*dpa)); 2460 } else if (dpa->dpa_curlevel == dpa->dpa_zb.zb_level) { 2461 ASSERT3U(nextblkid, ==, dpa->dpa_zb.zb_blkid); 2462 dbuf_issue_final_prefetch(dpa, bp); 2463 kmem_free(dpa, sizeof (*dpa)); 2464 } else { 2465 arc_flags_t iter_aflags = ARC_FLAG_NOWAIT; 2466 zbookmark_phys_t zb; 2467 2468 /* flag if L2ARC eligible, l2arc_noprefetch then decides */ 2469 if (dpa->dpa_aflags & ARC_FLAG_L2CACHE) 2470 iter_aflags |= ARC_FLAG_L2CACHE; 2471 2472 ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp)); 2473 2474 SET_BOOKMARK(&zb, dpa->dpa_zb.zb_objset, 2475 dpa->dpa_zb.zb_object, dpa->dpa_curlevel, nextblkid); 2476 2477 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, 2478 bp, dbuf_prefetch_indirect_done, dpa, dpa->dpa_prio, 2479 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2480 &iter_aflags, &zb); 2481 } 2482 2483 arc_buf_destroy(abuf, private); 2484 } 2485 2486 /* 2487 * Issue prefetch reads for the given block on the given level. If the indirect 2488 * blocks above that block are not in memory, we will read them in 2489 * asynchronously. As a result, this call never blocks waiting for a read to 2490 * complete. 2491 */ 2492 void 2493 dbuf_prefetch(dnode_t *dn, int64_t level, uint64_t blkid, zio_priority_t prio, 2494 arc_flags_t aflags) 2495 { 2496 blkptr_t bp; 2497 int epbs, nlevels, curlevel; 2498 uint64_t curblkid; 2499 2500 ASSERT(blkid != DMU_BONUS_BLKID); 2501 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2502 2503 if (blkid > dn->dn_maxblkid) 2504 return; 2505 2506 if (dnode_block_freed(dn, blkid)) 2507 return; 2508 2509 /* 2510 * This dnode hasn't been written to disk yet, so there's nothing to 2511 * prefetch. 2512 */ 2513 nlevels = dn->dn_phys->dn_nlevels; 2514 if (level >= nlevels || dn->dn_phys->dn_nblkptr == 0) 2515 return; 2516 2517 epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2518 if (dn->dn_phys->dn_maxblkid < blkid << (epbs * level)) 2519 return; 2520 2521 dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object, 2522 level, blkid); 2523 if (db != NULL) { 2524 mutex_exit(&db->db_mtx); 2525 /* 2526 * This dbuf already exists. It is either CACHED, or 2527 * (we assume) about to be read or filled. 2528 */ 2529 return; 2530 } 2531 2532 /* 2533 * Find the closest ancestor (indirect block) of the target block 2534 * that is present in the cache. In this indirect block, we will 2535 * find the bp that is at curlevel, curblkid. 2536 */ 2537 curlevel = level; 2538 curblkid = blkid; 2539 while (curlevel < nlevels - 1) { 2540 int parent_level = curlevel + 1; 2541 uint64_t parent_blkid = curblkid >> epbs; 2542 dmu_buf_impl_t *db; 2543 2544 if (dbuf_hold_impl(dn, parent_level, parent_blkid, 2545 FALSE, TRUE, FTAG, &db) == 0) { 2546 blkptr_t *bpp = db->db_buf->b_data; 2547 bp = bpp[P2PHASE(curblkid, 1 << epbs)]; 2548 dbuf_rele(db, FTAG); 2549 break; 2550 } 2551 2552 curlevel = parent_level; 2553 curblkid = parent_blkid; 2554 } 2555 2556 if (curlevel == nlevels - 1) { 2557 /* No cached indirect blocks found. */ 2558 ASSERT3U(curblkid, <, dn->dn_phys->dn_nblkptr); 2559 bp = dn->dn_phys->dn_blkptr[curblkid]; 2560 } 2561 if (BP_IS_HOLE(&bp)) 2562 return; 2563 2564 ASSERT3U(curlevel, ==, BP_GET_LEVEL(&bp)); 2565 2566 zio_t *pio = zio_root(dmu_objset_spa(dn->dn_objset), NULL, NULL, 2567 ZIO_FLAG_CANFAIL); 2568 2569 dbuf_prefetch_arg_t *dpa = kmem_zalloc(sizeof (*dpa), KM_SLEEP); 2570 dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 2571 SET_BOOKMARK(&dpa->dpa_zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET, 2572 dn->dn_object, level, blkid); 2573 dpa->dpa_curlevel = curlevel; 2574 dpa->dpa_prio = prio; 2575 dpa->dpa_aflags = aflags; 2576 dpa->dpa_spa = dn->dn_objset->os_spa; 2577 dpa->dpa_dnode = dn; 2578 dpa->dpa_epbs = epbs; 2579 dpa->dpa_zio = pio; 2580 2581 /* flag if L2ARC eligible, l2arc_noprefetch then decides */ 2582 if (DNODE_LEVEL_IS_L2CACHEABLE(dn, level)) 2583 dpa->dpa_aflags |= ARC_FLAG_L2CACHE; 2584 2585 /* 2586 * If we have the indirect just above us, no need to do the asynchronous 2587 * prefetch chain; we'll just run the last step ourselves. If we're at 2588 * a higher level, though, we want to issue the prefetches for all the 2589 * indirect blocks asynchronously, so we can go on with whatever we were 2590 * doing. 2591 */ 2592 if (curlevel == level) { 2593 ASSERT3U(curblkid, ==, blkid); 2594 dbuf_issue_final_prefetch(dpa, &bp); 2595 kmem_free(dpa, sizeof (*dpa)); 2596 } else { 2597 arc_flags_t iter_aflags = ARC_FLAG_NOWAIT; 2598 zbookmark_phys_t zb; 2599 2600 /* flag if L2ARC eligible, l2arc_noprefetch then decides */ 2601 if (DNODE_LEVEL_IS_L2CACHEABLE(dn, level)) 2602 iter_aflags |= ARC_FLAG_L2CACHE; 2603 2604 SET_BOOKMARK(&zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET, 2605 dn->dn_object, curlevel, curblkid); 2606 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, 2607 &bp, dbuf_prefetch_indirect_done, dpa, prio, 2608 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2609 &iter_aflags, &zb); 2610 } 2611 /* 2612 * We use pio here instead of dpa_zio since it's possible that 2613 * dpa may have already been freed. 2614 */ 2615 zio_nowait(pio); 2616 } 2617 2618 /* 2619 * Returns with db_holds incremented, and db_mtx not held. 2620 * Note: dn_struct_rwlock must be held. 2621 */ 2622 int 2623 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, 2624 boolean_t fail_sparse, boolean_t fail_uncached, 2625 void *tag, dmu_buf_impl_t **dbp) 2626 { 2627 dmu_buf_impl_t *db, *parent = NULL; 2628 2629 ASSERT(blkid != DMU_BONUS_BLKID); 2630 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2631 ASSERT3U(dn->dn_nlevels, >, level); 2632 2633 *dbp = NULL; 2634 top: 2635 /* dbuf_find() returns with db_mtx held */ 2636 db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid); 2637 2638 if (db == NULL) { 2639 blkptr_t *bp = NULL; 2640 int err; 2641 2642 if (fail_uncached) 2643 return (SET_ERROR(ENOENT)); 2644 2645 ASSERT3P(parent, ==, NULL); 2646 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 2647 if (fail_sparse) { 2648 if (err == 0 && bp && BP_IS_HOLE(bp)) 2649 err = SET_ERROR(ENOENT); 2650 if (err) { 2651 if (parent) 2652 dbuf_rele(parent, NULL); 2653 return (err); 2654 } 2655 } 2656 if (err && err != ENOENT) 2657 return (err); 2658 db = dbuf_create(dn, level, blkid, parent, bp); 2659 } 2660 2661 if (fail_uncached && db->db_state != DB_CACHED) { 2662 mutex_exit(&db->db_mtx); 2663 return (SET_ERROR(ENOENT)); 2664 } 2665 2666 if (db->db_buf != NULL) 2667 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 2668 2669 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 2670 2671 /* 2672 * If this buffer is currently syncing out, and we are are 2673 * still referencing it from db_data, we need to make a copy 2674 * of it in case we decide we want to dirty it again in this txg. 2675 */ 2676 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 2677 dn->dn_object != DMU_META_DNODE_OBJECT && 2678 db->db_state == DB_CACHED && db->db_data_pending) { 2679 dbuf_dirty_record_t *dr = db->db_data_pending; 2680 2681 if (dr->dt.dl.dr_data == db->db_buf) { 2682 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2683 2684 dbuf_set_data(db, 2685 arc_alloc_buf(dn->dn_objset->os_spa, db, type, 2686 db->db.db_size)); 2687 bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 2688 db->db.db_size); 2689 } 2690 } 2691 2692 if (multilist_link_active(&db->db_cache_link)) { 2693 ASSERT(refcount_is_zero(&db->db_holds)); 2694 ASSERT(db->db_caching_status == DB_DBUF_CACHE || 2695 db->db_caching_status == DB_DBUF_METADATA_CACHE); 2696 2697 multilist_remove(dbuf_caches[db->db_caching_status].cache, db); 2698 (void) refcount_remove_many( 2699 &dbuf_caches[db->db_caching_status].size, 2700 db->db.db_size, db); 2701 2702 db->db_caching_status = DB_NO_CACHE; 2703 } 2704 (void) refcount_add(&db->db_holds, tag); 2705 DBUF_VERIFY(db); 2706 mutex_exit(&db->db_mtx); 2707 2708 /* NOTE: we can't rele the parent until after we drop the db_mtx */ 2709 if (parent) 2710 dbuf_rele(parent, NULL); 2711 2712 ASSERT3P(DB_DNODE(db), ==, dn); 2713 ASSERT3U(db->db_blkid, ==, blkid); 2714 ASSERT3U(db->db_level, ==, level); 2715 *dbp = db; 2716 2717 return (0); 2718 } 2719 2720 dmu_buf_impl_t * 2721 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 2722 { 2723 return (dbuf_hold_level(dn, 0, blkid, tag)); 2724 } 2725 2726 dmu_buf_impl_t * 2727 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 2728 { 2729 dmu_buf_impl_t *db; 2730 int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db); 2731 return (err ? NULL : db); 2732 } 2733 2734 void 2735 dbuf_create_bonus(dnode_t *dn) 2736 { 2737 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 2738 2739 ASSERT(dn->dn_bonus == NULL); 2740 dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 2741 } 2742 2743 int 2744 dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 2745 { 2746 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2747 dnode_t *dn; 2748 2749 if (db->db_blkid != DMU_SPILL_BLKID) 2750 return (SET_ERROR(ENOTSUP)); 2751 if (blksz == 0) 2752 blksz = SPA_MINBLOCKSIZE; 2753 ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset))); 2754 blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 2755 2756 DB_DNODE_ENTER(db); 2757 dn = DB_DNODE(db); 2758 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 2759 dbuf_new_size(db, blksz, tx); 2760 rw_exit(&dn->dn_struct_rwlock); 2761 DB_DNODE_EXIT(db); 2762 2763 return (0); 2764 } 2765 2766 void 2767 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 2768 { 2769 dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 2770 } 2771 2772 #pragma weak dmu_buf_add_ref = dbuf_add_ref 2773 void 2774 dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 2775 { 2776 int64_t holds = refcount_add(&db->db_holds, tag); 2777 ASSERT3S(holds, >, 1); 2778 } 2779 2780 #pragma weak dmu_buf_try_add_ref = dbuf_try_add_ref 2781 boolean_t 2782 dbuf_try_add_ref(dmu_buf_t *db_fake, objset_t *os, uint64_t obj, uint64_t blkid, 2783 void *tag) 2784 { 2785 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2786 dmu_buf_impl_t *found_db; 2787 boolean_t result = B_FALSE; 2788 2789 if (db->db_blkid == DMU_BONUS_BLKID) 2790 found_db = dbuf_find_bonus(os, obj); 2791 else 2792 found_db = dbuf_find(os, obj, 0, blkid); 2793 2794 if (found_db != NULL) { 2795 if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) { 2796 (void) refcount_add(&db->db_holds, tag); 2797 result = B_TRUE; 2798 } 2799 mutex_exit(&db->db_mtx); 2800 } 2801 return (result); 2802 } 2803 2804 /* 2805 * If you call dbuf_rele() you had better not be referencing the dnode handle 2806 * unless you have some other direct or indirect hold on the dnode. (An indirect 2807 * hold is a hold on one of the dnode's dbufs, including the bonus buffer.) 2808 * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the 2809 * dnode's parent dbuf evicting its dnode handles. 2810 */ 2811 void 2812 dbuf_rele(dmu_buf_impl_t *db, void *tag) 2813 { 2814 mutex_enter(&db->db_mtx); 2815 dbuf_rele_and_unlock(db, tag); 2816 } 2817 2818 void 2819 dmu_buf_rele(dmu_buf_t *db, void *tag) 2820 { 2821 dbuf_rele((dmu_buf_impl_t *)db, tag); 2822 } 2823 2824 /* 2825 * dbuf_rele() for an already-locked dbuf. This is necessary to allow 2826 * db_dirtycnt and db_holds to be updated atomically. 2827 */ 2828 void 2829 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 2830 { 2831 int64_t holds; 2832 2833 ASSERT(MUTEX_HELD(&db->db_mtx)); 2834 DBUF_VERIFY(db); 2835 2836 /* 2837 * Remove the reference to the dbuf before removing its hold on the 2838 * dnode so we can guarantee in dnode_move() that a referenced bonus 2839 * buffer has a corresponding dnode hold. 2840 */ 2841 holds = refcount_remove(&db->db_holds, tag); 2842 ASSERT(holds >= 0); 2843 2844 /* 2845 * We can't freeze indirects if there is a possibility that they 2846 * may be modified in the current syncing context. 2847 */ 2848 if (db->db_buf != NULL && 2849 holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) { 2850 arc_buf_freeze(db->db_buf); 2851 } 2852 2853 if (holds == db->db_dirtycnt && 2854 db->db_level == 0 && db->db_user_immediate_evict) 2855 dbuf_evict_user(db); 2856 2857 if (holds == 0) { 2858 if (db->db_blkid == DMU_BONUS_BLKID) { 2859 dnode_t *dn; 2860 boolean_t evict_dbuf = db->db_pending_evict; 2861 2862 /* 2863 * If the dnode moves here, we cannot cross this 2864 * barrier until the move completes. 2865 */ 2866 DB_DNODE_ENTER(db); 2867 2868 dn = DB_DNODE(db); 2869 atomic_dec_32(&dn->dn_dbufs_count); 2870 2871 /* 2872 * Decrementing the dbuf count means that the bonus 2873 * buffer's dnode hold is no longer discounted in 2874 * dnode_move(). The dnode cannot move until after 2875 * the dnode_rele() below. 2876 */ 2877 DB_DNODE_EXIT(db); 2878 2879 /* 2880 * Do not reference db after its lock is dropped. 2881 * Another thread may evict it. 2882 */ 2883 mutex_exit(&db->db_mtx); 2884 2885 if (evict_dbuf) 2886 dnode_evict_bonus(dn); 2887 2888 dnode_rele(dn, db); 2889 } else if (db->db_buf == NULL) { 2890 /* 2891 * This is a special case: we never associated this 2892 * dbuf with any data allocated from the ARC. 2893 */ 2894 ASSERT(db->db_state == DB_UNCACHED || 2895 db->db_state == DB_NOFILL); 2896 dbuf_destroy(db); 2897 } else if (arc_released(db->db_buf)) { 2898 /* 2899 * This dbuf has anonymous data associated with it. 2900 */ 2901 dbuf_destroy(db); 2902 } else { 2903 boolean_t do_arc_evict = B_FALSE; 2904 blkptr_t bp; 2905 spa_t *spa = dmu_objset_spa(db->db_objset); 2906 2907 if (!DBUF_IS_CACHEABLE(db) && 2908 db->db_blkptr != NULL && 2909 !BP_IS_HOLE(db->db_blkptr) && 2910 !BP_IS_EMBEDDED(db->db_blkptr)) { 2911 do_arc_evict = B_TRUE; 2912 bp = *db->db_blkptr; 2913 } 2914 2915 if (!DBUF_IS_CACHEABLE(db) || 2916 db->db_pending_evict) { 2917 dbuf_destroy(db); 2918 } else if (!multilist_link_active(&db->db_cache_link)) { 2919 ASSERT3U(db->db_caching_status, ==, 2920 DB_NO_CACHE); 2921 2922 dbuf_cached_state_t dcs = 2923 dbuf_include_in_metadata_cache(db) ? 2924 DB_DBUF_METADATA_CACHE : DB_DBUF_CACHE; 2925 db->db_caching_status = dcs; 2926 2927 multilist_insert(dbuf_caches[dcs].cache, db); 2928 (void) refcount_add_many(&dbuf_caches[dcs].size, 2929 db->db.db_size, db); 2930 mutex_exit(&db->db_mtx); 2931 2932 if (db->db_caching_status == DB_DBUF_CACHE) { 2933 dbuf_evict_notify(); 2934 } 2935 } 2936 2937 if (do_arc_evict) 2938 arc_freed(spa, &bp); 2939 } 2940 } else { 2941 mutex_exit(&db->db_mtx); 2942 } 2943 2944 } 2945 2946 #pragma weak dmu_buf_refcount = dbuf_refcount 2947 uint64_t 2948 dbuf_refcount(dmu_buf_impl_t *db) 2949 { 2950 return (refcount_count(&db->db_holds)); 2951 } 2952 2953 void * 2954 dmu_buf_replace_user(dmu_buf_t *db_fake, dmu_buf_user_t *old_user, 2955 dmu_buf_user_t *new_user) 2956 { 2957 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2958 2959 mutex_enter(&db->db_mtx); 2960 dbuf_verify_user(db, DBVU_NOT_EVICTING); 2961 if (db->db_user == old_user) 2962 db->db_user = new_user; 2963 else 2964 old_user = db->db_user; 2965 dbuf_verify_user(db, DBVU_NOT_EVICTING); 2966 mutex_exit(&db->db_mtx); 2967 2968 return (old_user); 2969 } 2970 2971 void * 2972 dmu_buf_set_user(dmu_buf_t *db_fake, dmu_buf_user_t *user) 2973 { 2974 return (dmu_buf_replace_user(db_fake, NULL, user)); 2975 } 2976 2977 void * 2978 dmu_buf_set_user_ie(dmu_buf_t *db_fake, dmu_buf_user_t *user) 2979 { 2980 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2981 2982 db->db_user_immediate_evict = TRUE; 2983 return (dmu_buf_set_user(db_fake, user)); 2984 } 2985 2986 void * 2987 dmu_buf_remove_user(dmu_buf_t *db_fake, dmu_buf_user_t *user) 2988 { 2989 return (dmu_buf_replace_user(db_fake, user, NULL)); 2990 } 2991 2992 void * 2993 dmu_buf_get_user(dmu_buf_t *db_fake) 2994 { 2995 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2996 2997 dbuf_verify_user(db, DBVU_NOT_EVICTING); 2998 return (db->db_user); 2999 } 3000 3001 void 3002 dmu_buf_user_evict_wait() 3003 { 3004 taskq_wait(dbu_evict_taskq); 3005 } 3006 3007 blkptr_t * 3008 dmu_buf_get_blkptr(dmu_buf_t *db) 3009 { 3010 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 3011 return (dbi->db_blkptr); 3012 } 3013 3014 objset_t * 3015 dmu_buf_get_objset(dmu_buf_t *db) 3016 { 3017 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 3018 return (dbi->db_objset); 3019 } 3020 3021 dnode_t * 3022 dmu_buf_dnode_enter(dmu_buf_t *db) 3023 { 3024 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 3025 DB_DNODE_ENTER(dbi); 3026 return (DB_DNODE(dbi)); 3027 } 3028 3029 void 3030 dmu_buf_dnode_exit(dmu_buf_t *db) 3031 { 3032 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 3033 DB_DNODE_EXIT(dbi); 3034 } 3035 3036 static void 3037 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 3038 { 3039 /* ASSERT(dmu_tx_is_syncing(tx) */ 3040 ASSERT(MUTEX_HELD(&db->db_mtx)); 3041 3042 if (db->db_blkptr != NULL) 3043 return; 3044 3045 if (db->db_blkid == DMU_SPILL_BLKID) { 3046 db->db_blkptr = &dn->dn_phys->dn_spill; 3047 BP_ZERO(db->db_blkptr); 3048 return; 3049 } 3050 if (db->db_level == dn->dn_phys->dn_nlevels-1) { 3051 /* 3052 * This buffer was allocated at a time when there was 3053 * no available blkptrs from the dnode, or it was 3054 * inappropriate to hook it in (i.e., nlevels mis-match). 3055 */ 3056 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 3057 ASSERT(db->db_parent == NULL); 3058 db->db_parent = dn->dn_dbuf; 3059 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 3060 DBUF_VERIFY(db); 3061 } else { 3062 dmu_buf_impl_t *parent = db->db_parent; 3063 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 3064 3065 ASSERT(dn->dn_phys->dn_nlevels > 1); 3066 if (parent == NULL) { 3067 mutex_exit(&db->db_mtx); 3068 rw_enter(&dn->dn_struct_rwlock, RW_READER); 3069 parent = dbuf_hold_level(dn, db->db_level + 1, 3070 db->db_blkid >> epbs, db); 3071 rw_exit(&dn->dn_struct_rwlock); 3072 mutex_enter(&db->db_mtx); 3073 db->db_parent = parent; 3074 } 3075 db->db_blkptr = (blkptr_t *)parent->db.db_data + 3076 (db->db_blkid & ((1ULL << epbs) - 1)); 3077 DBUF_VERIFY(db); 3078 } 3079 } 3080 3081 static void 3082 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 3083 { 3084 dmu_buf_impl_t *db = dr->dr_dbuf; 3085 dnode_t *dn; 3086 zio_t *zio; 3087 3088 ASSERT(dmu_tx_is_syncing(tx)); 3089 3090 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 3091 3092 mutex_enter(&db->db_mtx); 3093 3094 ASSERT(db->db_level > 0); 3095 DBUF_VERIFY(db); 3096 3097 /* Read the block if it hasn't been read yet. */ 3098 if (db->db_buf == NULL) { 3099 mutex_exit(&db->db_mtx); 3100 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 3101 mutex_enter(&db->db_mtx); 3102 } 3103 ASSERT3U(db->db_state, ==, DB_CACHED); 3104 ASSERT(db->db_buf != NULL); 3105 3106 DB_DNODE_ENTER(db); 3107 dn = DB_DNODE(db); 3108 /* Indirect block size must match what the dnode thinks it is. */ 3109 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 3110 dbuf_check_blkptr(dn, db); 3111 DB_DNODE_EXIT(db); 3112 3113 /* Provide the pending dirty record to child dbufs */ 3114 db->db_data_pending = dr; 3115 3116 mutex_exit(&db->db_mtx); 3117 3118 dbuf_write(dr, db->db_buf, tx); 3119 3120 zio = dr->dr_zio; 3121 mutex_enter(&dr->dt.di.dr_mtx); 3122 dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx); 3123 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 3124 mutex_exit(&dr->dt.di.dr_mtx); 3125 zio_nowait(zio); 3126 } 3127 3128 static void 3129 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 3130 { 3131 arc_buf_t **datap = &dr->dt.dl.dr_data; 3132 dmu_buf_impl_t *db = dr->dr_dbuf; 3133 dnode_t *dn; 3134 objset_t *os; 3135 uint64_t txg = tx->tx_txg; 3136 3137 ASSERT(dmu_tx_is_syncing(tx)); 3138 3139 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 3140 3141 mutex_enter(&db->db_mtx); 3142 /* 3143 * To be synced, we must be dirtied. But we 3144 * might have been freed after the dirty. 3145 */ 3146 if (db->db_state == DB_UNCACHED) { 3147 /* This buffer has been freed since it was dirtied */ 3148 ASSERT(db->db.db_data == NULL); 3149 } else if (db->db_state == DB_FILL) { 3150 /* This buffer was freed and is now being re-filled */ 3151 ASSERT(db->db.db_data != dr->dt.dl.dr_data); 3152 } else { 3153 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 3154 } 3155 DBUF_VERIFY(db); 3156 3157 DB_DNODE_ENTER(db); 3158 dn = DB_DNODE(db); 3159 3160 if (db->db_blkid == DMU_SPILL_BLKID) { 3161 mutex_enter(&dn->dn_mtx); 3162 dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 3163 mutex_exit(&dn->dn_mtx); 3164 } 3165 3166 /* 3167 * If this is a bonus buffer, simply copy the bonus data into the 3168 * dnode. It will be written out when the dnode is synced (and it 3169 * will be synced, since it must have been dirty for dbuf_sync to 3170 * be called). 3171 */ 3172 if (db->db_blkid == DMU_BONUS_BLKID) { 3173 dbuf_dirty_record_t **drp; 3174 3175 ASSERT(*datap != NULL); 3176 ASSERT0(db->db_level); 3177 ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 3178 bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 3179 DB_DNODE_EXIT(db); 3180 3181 if (*datap != db->db.db_data) { 3182 zio_buf_free(*datap, DN_MAX_BONUSLEN); 3183 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 3184 } 3185 db->db_data_pending = NULL; 3186 drp = &db->db_last_dirty; 3187 while (*drp != dr) 3188 drp = &(*drp)->dr_next; 3189 ASSERT(dr->dr_next == NULL); 3190 ASSERT(dr->dr_dbuf == db); 3191 *drp = dr->dr_next; 3192 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 3193 ASSERT(db->db_dirtycnt > 0); 3194 db->db_dirtycnt -= 1; 3195 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 3196 return; 3197 } 3198 3199 os = dn->dn_objset; 3200 3201 /* 3202 * This function may have dropped the db_mtx lock allowing a dmu_sync 3203 * operation to sneak in. As a result, we need to ensure that we 3204 * don't check the dr_override_state until we have returned from 3205 * dbuf_check_blkptr. 3206 */ 3207 dbuf_check_blkptr(dn, db); 3208 3209 /* 3210 * If this buffer is in the middle of an immediate write, 3211 * wait for the synchronous IO to complete. 3212 */ 3213 while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 3214 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 3215 cv_wait(&db->db_changed, &db->db_mtx); 3216 ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 3217 } 3218 3219 if (db->db_state != DB_NOFILL && 3220 dn->dn_object != DMU_META_DNODE_OBJECT && 3221 refcount_count(&db->db_holds) > 1 && 3222 dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 3223 *datap == db->db_buf) { 3224 /* 3225 * If this buffer is currently "in use" (i.e., there 3226 * are active holds and db_data still references it), 3227 * then make a copy before we start the write so that 3228 * any modifications from the open txg will not leak 3229 * into this write. 3230 * 3231 * NOTE: this copy does not need to be made for 3232 * objects only modified in the syncing context (e.g. 3233 * DNONE_DNODE blocks). 3234 */ 3235 int psize = arc_buf_size(*datap); 3236 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 3237 enum zio_compress compress_type = arc_get_compression(*datap); 3238 3239 if (compress_type == ZIO_COMPRESS_OFF) { 3240 *datap = arc_alloc_buf(os->os_spa, db, type, psize); 3241 } else { 3242 ASSERT3U(type, ==, ARC_BUFC_DATA); 3243 int lsize = arc_buf_lsize(*datap); 3244 *datap = arc_alloc_compressed_buf(os->os_spa, db, 3245 psize, lsize, compress_type); 3246 } 3247 bcopy(db->db.db_data, (*datap)->b_data, psize); 3248 } 3249 db->db_data_pending = dr; 3250 3251 mutex_exit(&db->db_mtx); 3252 3253 dbuf_write(dr, *datap, tx); 3254 3255 ASSERT(!list_link_active(&dr->dr_dirty_node)); 3256 if (dn->dn_object == DMU_META_DNODE_OBJECT) { 3257 list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 3258 DB_DNODE_EXIT(db); 3259 } else { 3260 /* 3261 * Although zio_nowait() does not "wait for an IO", it does 3262 * initiate the IO. If this is an empty write it seems plausible 3263 * that the IO could actually be completed before the nowait 3264 * returns. We need to DB_DNODE_EXIT() first in case 3265 * zio_nowait() invalidates the dbuf. 3266 */ 3267 DB_DNODE_EXIT(db); 3268 zio_nowait(dr->dr_zio); 3269 } 3270 } 3271 3272 void 3273 dbuf_sync_list(list_t *list, int level, dmu_tx_t *tx) 3274 { 3275 dbuf_dirty_record_t *dr; 3276 3277 while (dr = list_head(list)) { 3278 if (dr->dr_zio != NULL) { 3279 /* 3280 * If we find an already initialized zio then we 3281 * are processing the meta-dnode, and we have finished. 3282 * The dbufs for all dnodes are put back on the list 3283 * during processing, so that we can zio_wait() 3284 * these IOs after initiating all child IOs. 3285 */ 3286 ASSERT3U(dr->dr_dbuf->db.db_object, ==, 3287 DMU_META_DNODE_OBJECT); 3288 break; 3289 } 3290 if (dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID && 3291 dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID) { 3292 VERIFY3U(dr->dr_dbuf->db_level, ==, level); 3293 } 3294 list_remove(list, dr); 3295 if (dr->dr_dbuf->db_level > 0) 3296 dbuf_sync_indirect(dr, tx); 3297 else 3298 dbuf_sync_leaf(dr, tx); 3299 } 3300 } 3301 3302 /* ARGSUSED */ 3303 static void 3304 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 3305 { 3306 dmu_buf_impl_t *db = vdb; 3307 dnode_t *dn; 3308 blkptr_t *bp = zio->io_bp; 3309 blkptr_t *bp_orig = &zio->io_bp_orig; 3310 spa_t *spa = zio->io_spa; 3311 int64_t delta; 3312 uint64_t fill = 0; 3313 int i; 3314 3315 ASSERT3P(db->db_blkptr, !=, NULL); 3316 ASSERT3P(&db->db_data_pending->dr_bp_copy, ==, bp); 3317 3318 DB_DNODE_ENTER(db); 3319 dn = DB_DNODE(db); 3320 delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 3321 dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 3322 zio->io_prev_space_delta = delta; 3323 3324 if (bp->blk_birth != 0) { 3325 ASSERT((db->db_blkid != DMU_SPILL_BLKID && 3326 BP_GET_TYPE(bp) == dn->dn_type) || 3327 (db->db_blkid == DMU_SPILL_BLKID && 3328 BP_GET_TYPE(bp) == dn->dn_bonustype) || 3329 BP_IS_EMBEDDED(bp)); 3330 ASSERT(BP_GET_LEVEL(bp) == db->db_level); 3331 } 3332 3333 mutex_enter(&db->db_mtx); 3334 3335 #ifdef ZFS_DEBUG 3336 if (db->db_blkid == DMU_SPILL_BLKID) { 3337 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 3338 ASSERT(!(BP_IS_HOLE(bp)) && 3339 db->db_blkptr == &dn->dn_phys->dn_spill); 3340 } 3341 #endif 3342 3343 if (db->db_level == 0) { 3344 mutex_enter(&dn->dn_mtx); 3345 if (db->db_blkid > dn->dn_phys->dn_maxblkid && 3346 db->db_blkid != DMU_SPILL_BLKID) 3347 dn->dn_phys->dn_maxblkid = db->db_blkid; 3348 mutex_exit(&dn->dn_mtx); 3349 3350 if (dn->dn_type == DMU_OT_DNODE) { 3351 dnode_phys_t *dnp = db->db.db_data; 3352 for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 3353 i--, dnp++) { 3354 if (dnp->dn_type != DMU_OT_NONE) 3355 fill++; 3356 } 3357 } else { 3358 if (BP_IS_HOLE(bp)) { 3359 fill = 0; 3360 } else { 3361 fill = 1; 3362 } 3363 } 3364 } else { 3365 blkptr_t *ibp = db->db.db_data; 3366 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 3367 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 3368 if (BP_IS_HOLE(ibp)) 3369 continue; 3370 fill += BP_GET_FILL(ibp); 3371 } 3372 } 3373 DB_DNODE_EXIT(db); 3374 3375 if (!BP_IS_EMBEDDED(bp)) 3376 bp->blk_fill = fill; 3377 3378 mutex_exit(&db->db_mtx); 3379 3380 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 3381 *db->db_blkptr = *bp; 3382 rw_exit(&dn->dn_struct_rwlock); 3383 } 3384 3385 /* ARGSUSED */ 3386 /* 3387 * This function gets called just prior to running through the compression 3388 * stage of the zio pipeline. If we're an indirect block comprised of only 3389 * holes, then we want this indirect to be compressed away to a hole. In 3390 * order to do that we must zero out any information about the holes that 3391 * this indirect points to prior to before we try to compress it. 3392 */ 3393 static void 3394 dbuf_write_children_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 3395 { 3396 dmu_buf_impl_t *db = vdb; 3397 dnode_t *dn; 3398 blkptr_t *bp; 3399 unsigned int epbs, i; 3400 3401 ASSERT3U(db->db_level, >, 0); 3402 DB_DNODE_ENTER(db); 3403 dn = DB_DNODE(db); 3404 epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 3405 ASSERT3U(epbs, <, 31); 3406 3407 /* Determine if all our children are holes */ 3408 for (i = 0, bp = db->db.db_data; i < 1 << epbs; i++, bp++) { 3409 if (!BP_IS_HOLE(bp)) 3410 break; 3411 } 3412 3413 /* 3414 * If all the children are holes, then zero them all out so that 3415 * we may get compressed away. 3416 */ 3417 if (i == 1 << epbs) { 3418 /* 3419 * We only found holes. Grab the rwlock to prevent 3420 * anybody from reading the blocks we're about to 3421 * zero out. 3422 */ 3423 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 3424 bzero(db->db.db_data, db->db.db_size); 3425 rw_exit(&dn->dn_struct_rwlock); 3426 } 3427 DB_DNODE_EXIT(db); 3428 } 3429 3430 /* 3431 * The SPA will call this callback several times for each zio - once 3432 * for every physical child i/o (zio->io_phys_children times). This 3433 * allows the DMU to monitor the progress of each logical i/o. For example, 3434 * there may be 2 copies of an indirect block, or many fragments of a RAID-Z 3435 * block. There may be a long delay before all copies/fragments are completed, 3436 * so this callback allows us to retire dirty space gradually, as the physical 3437 * i/os complete. 3438 */ 3439 /* ARGSUSED */ 3440 static void 3441 dbuf_write_physdone(zio_t *zio, arc_buf_t *buf, void *arg) 3442 { 3443 dmu_buf_impl_t *db = arg; 3444 objset_t *os = db->db_objset; 3445 dsl_pool_t *dp = dmu_objset_pool(os); 3446 dbuf_dirty_record_t *dr; 3447 int delta = 0; 3448 3449 dr = db->db_data_pending; 3450 ASSERT3U(dr->dr_txg, ==, zio->io_txg); 3451 3452 /* 3453 * The callback will be called io_phys_children times. Retire one 3454 * portion of our dirty space each time we are called. Any rounding 3455 * error will be cleaned up by dsl_pool_sync()'s call to 3456 * dsl_pool_undirty_space(). 3457 */ 3458 delta = dr->dr_accounted / zio->io_phys_children; 3459 dsl_pool_undirty_space(dp, delta, zio->io_txg); 3460 } 3461 3462 /* ARGSUSED */ 3463 static void 3464 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 3465 { 3466 dmu_buf_impl_t *db = vdb; 3467 blkptr_t *bp_orig = &zio->io_bp_orig; 3468 blkptr_t *bp = db->db_blkptr; 3469 objset_t *os = db->db_objset; 3470 dmu_tx_t *tx = os->os_synctx; 3471 dbuf_dirty_record_t **drp, *dr; 3472 3473 ASSERT0(zio->io_error); 3474 ASSERT(db->db_blkptr == bp); 3475 3476 /* 3477 * For nopwrites and rewrites we ensure that the bp matches our 3478 * original and bypass all the accounting. 3479 */ 3480 if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) { 3481 ASSERT(BP_EQUAL(bp, bp_orig)); 3482 } else { 3483 dsl_dataset_t *ds = os->os_dsl_dataset; 3484 (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 3485 dsl_dataset_block_born(ds, bp, tx); 3486 } 3487 3488 mutex_enter(&db->db_mtx); 3489 3490 DBUF_VERIFY(db); 3491 3492 drp = &db->db_last_dirty; 3493 while ((dr = *drp) != db->db_data_pending) 3494 drp = &dr->dr_next; 3495 ASSERT(!list_link_active(&dr->dr_dirty_node)); 3496 ASSERT(dr->dr_dbuf == db); 3497 ASSERT(dr->dr_next == NULL); 3498 *drp = dr->dr_next; 3499 3500 #ifdef ZFS_DEBUG 3501 if (db->db_blkid == DMU_SPILL_BLKID) { 3502 dnode_t *dn; 3503 3504 DB_DNODE_ENTER(db); 3505 dn = DB_DNODE(db); 3506 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 3507 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 3508 db->db_blkptr == &dn->dn_phys->dn_spill); 3509 DB_DNODE_EXIT(db); 3510 } 3511 #endif 3512 3513 if (db->db_level == 0) { 3514 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 3515 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 3516 if (db->db_state != DB_NOFILL) { 3517 if (dr->dt.dl.dr_data != db->db_buf) 3518 arc_buf_destroy(dr->dt.dl.dr_data, db); 3519 } 3520 } else { 3521 dnode_t *dn; 3522 3523 DB_DNODE_ENTER(db); 3524 dn = DB_DNODE(db); 3525 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 3526 ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift); 3527 if (!BP_IS_HOLE(db->db_blkptr)) { 3528 int epbs = 3529 dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 3530 ASSERT3U(db->db_blkid, <=, 3531 dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)); 3532 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 3533 db->db.db_size); 3534 } 3535 DB_DNODE_EXIT(db); 3536 mutex_destroy(&dr->dt.di.dr_mtx); 3537 list_destroy(&dr->dt.di.dr_children); 3538 } 3539 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 3540 3541 cv_broadcast(&db->db_changed); 3542 ASSERT(db->db_dirtycnt > 0); 3543 db->db_dirtycnt -= 1; 3544 db->db_data_pending = NULL; 3545 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg); 3546 } 3547 3548 static void 3549 dbuf_write_nofill_ready(zio_t *zio) 3550 { 3551 dbuf_write_ready(zio, NULL, zio->io_private); 3552 } 3553 3554 static void 3555 dbuf_write_nofill_done(zio_t *zio) 3556 { 3557 dbuf_write_done(zio, NULL, zio->io_private); 3558 } 3559 3560 static void 3561 dbuf_write_override_ready(zio_t *zio) 3562 { 3563 dbuf_dirty_record_t *dr = zio->io_private; 3564 dmu_buf_impl_t *db = dr->dr_dbuf; 3565 3566 dbuf_write_ready(zio, NULL, db); 3567 } 3568 3569 static void 3570 dbuf_write_override_done(zio_t *zio) 3571 { 3572 dbuf_dirty_record_t *dr = zio->io_private; 3573 dmu_buf_impl_t *db = dr->dr_dbuf; 3574 blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 3575 3576 mutex_enter(&db->db_mtx); 3577 if (!BP_EQUAL(zio->io_bp, obp)) { 3578 if (!BP_IS_HOLE(obp)) 3579 dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 3580 arc_release(dr->dt.dl.dr_data, db); 3581 } 3582 mutex_exit(&db->db_mtx); 3583 dbuf_write_done(zio, NULL, db); 3584 3585 if (zio->io_abd != NULL) 3586 abd_put(zio->io_abd); 3587 } 3588 3589 typedef struct dbuf_remap_impl_callback_arg { 3590 objset_t *drica_os; 3591 uint64_t drica_blk_birth; 3592 dmu_tx_t *drica_tx; 3593 } dbuf_remap_impl_callback_arg_t; 3594 3595 static void 3596 dbuf_remap_impl_callback(uint64_t vdev, uint64_t offset, uint64_t size, 3597 void *arg) 3598 { 3599 dbuf_remap_impl_callback_arg_t *drica = arg; 3600 objset_t *os = drica->drica_os; 3601 spa_t *spa = dmu_objset_spa(os); 3602 dmu_tx_t *tx = drica->drica_tx; 3603 3604 ASSERT(dsl_pool_sync_context(spa_get_dsl(spa))); 3605 3606 if (os == spa_meta_objset(spa)) { 3607 spa_vdev_indirect_mark_obsolete(spa, vdev, offset, size, tx); 3608 } else { 3609 dsl_dataset_block_remapped(dmu_objset_ds(os), vdev, offset, 3610 size, drica->drica_blk_birth, tx); 3611 } 3612 } 3613 3614 static void 3615 dbuf_remap_impl(dnode_t *dn, blkptr_t *bp, dmu_tx_t *tx) 3616 { 3617 blkptr_t bp_copy = *bp; 3618 spa_t *spa = dmu_objset_spa(dn->dn_objset); 3619 dbuf_remap_impl_callback_arg_t drica; 3620 3621 ASSERT(dsl_pool_sync_context(spa_get_dsl(spa))); 3622 3623 drica.drica_os = dn->dn_objset; 3624 drica.drica_blk_birth = bp->blk_birth; 3625 drica.drica_tx = tx; 3626 if (spa_remap_blkptr(spa, &bp_copy, dbuf_remap_impl_callback, 3627 &drica)) { 3628 /* 3629 * The struct_rwlock prevents dbuf_read_impl() from 3630 * dereferencing the BP while we are changing it. To 3631 * avoid lock contention, only grab it when we are actually 3632 * changing the BP. 3633 */ 3634 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 3635 *bp = bp_copy; 3636 rw_exit(&dn->dn_struct_rwlock); 3637 } 3638 } 3639 3640 /* 3641 * Returns true if a dbuf_remap would modify the dbuf. We do this by attempting 3642 * to remap a copy of every bp in the dbuf. 3643 */ 3644 boolean_t 3645 dbuf_can_remap(const dmu_buf_impl_t *db) 3646 { 3647 spa_t *spa = dmu_objset_spa(db->db_objset); 3648 blkptr_t *bp = db->db.db_data; 3649 boolean_t ret = B_FALSE; 3650 3651 ASSERT3U(db->db_level, >, 0); 3652 ASSERT3S(db->db_state, ==, DB_CACHED); 3653 3654 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_DEVICE_REMOVAL)); 3655 3656 spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER); 3657 for (int i = 0; i < db->db.db_size >> SPA_BLKPTRSHIFT; i++) { 3658 blkptr_t bp_copy = bp[i]; 3659 if (spa_remap_blkptr(spa, &bp_copy, NULL, NULL)) { 3660 ret = B_TRUE; 3661 break; 3662 } 3663 } 3664 spa_config_exit(spa, SCL_VDEV, FTAG); 3665 3666 return (ret); 3667 } 3668 3669 boolean_t 3670 dnode_needs_remap(const dnode_t *dn) 3671 { 3672 spa_t *spa = dmu_objset_spa(dn->dn_objset); 3673 boolean_t ret = B_FALSE; 3674 3675 if (dn->dn_phys->dn_nlevels == 0) { 3676 return (B_FALSE); 3677 } 3678 3679 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_DEVICE_REMOVAL)); 3680 3681 spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER); 3682 for (int j = 0; j < dn->dn_phys->dn_nblkptr; j++) { 3683 blkptr_t bp_copy = dn->dn_phys->dn_blkptr[j]; 3684 if (spa_remap_blkptr(spa, &bp_copy, NULL, NULL)) { 3685 ret = B_TRUE; 3686 break; 3687 } 3688 } 3689 spa_config_exit(spa, SCL_VDEV, FTAG); 3690 3691 return (ret); 3692 } 3693 3694 /* 3695 * Remap any existing BP's to concrete vdevs, if possible. 3696 */ 3697 static void 3698 dbuf_remap(dnode_t *dn, dmu_buf_impl_t *db, dmu_tx_t *tx) 3699 { 3700 spa_t *spa = dmu_objset_spa(db->db_objset); 3701 ASSERT(dsl_pool_sync_context(spa_get_dsl(spa))); 3702 3703 if (!spa_feature_is_active(spa, SPA_FEATURE_DEVICE_REMOVAL)) 3704 return; 3705 3706 if (db->db_level > 0) { 3707 blkptr_t *bp = db->db.db_data; 3708 for (int i = 0; i < db->db.db_size >> SPA_BLKPTRSHIFT; i++) { 3709 dbuf_remap_impl(dn, &bp[i], tx); 3710 } 3711 } else if (db->db.db_object == DMU_META_DNODE_OBJECT) { 3712 dnode_phys_t *dnp = db->db.db_data; 3713 ASSERT3U(db->db_dnode_handle->dnh_dnode->dn_type, ==, 3714 DMU_OT_DNODE); 3715 for (int i = 0; i < db->db.db_size >> DNODE_SHIFT; i++) { 3716 for (int j = 0; j < dnp[i].dn_nblkptr; j++) { 3717 dbuf_remap_impl(dn, &dnp[i].dn_blkptr[j], tx); 3718 } 3719 } 3720 } 3721 } 3722 3723 3724 /* Issue I/O to commit a dirty buffer to disk. */ 3725 static void 3726 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 3727 { 3728 dmu_buf_impl_t *db = dr->dr_dbuf; 3729 dnode_t *dn; 3730 objset_t *os; 3731 dmu_buf_impl_t *parent = db->db_parent; 3732 uint64_t txg = tx->tx_txg; 3733 zbookmark_phys_t zb; 3734 zio_prop_t zp; 3735 zio_t *zio; 3736 int wp_flag = 0; 3737 3738 ASSERT(dmu_tx_is_syncing(tx)); 3739 3740 DB_DNODE_ENTER(db); 3741 dn = DB_DNODE(db); 3742 os = dn->dn_objset; 3743 3744 if (db->db_state != DB_NOFILL) { 3745 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 3746 /* 3747 * Private object buffers are released here rather 3748 * than in dbuf_dirty() since they are only modified 3749 * in the syncing context and we don't want the 3750 * overhead of making multiple copies of the data. 3751 */ 3752 if (BP_IS_HOLE(db->db_blkptr)) { 3753 arc_buf_thaw(data); 3754 } else { 3755 dbuf_release_bp(db); 3756 } 3757 dbuf_remap(dn, db, tx); 3758 } 3759 } 3760 3761 if (parent != dn->dn_dbuf) { 3762 /* Our parent is an indirect block. */ 3763 /* We have a dirty parent that has been scheduled for write. */ 3764 ASSERT(parent && parent->db_data_pending); 3765 /* Our parent's buffer is one level closer to the dnode. */ 3766 ASSERT(db->db_level == parent->db_level-1); 3767 /* 3768 * We're about to modify our parent's db_data by modifying 3769 * our block pointer, so the parent must be released. 3770 */ 3771 ASSERT(arc_released(parent->db_buf)); 3772 zio = parent->db_data_pending->dr_zio; 3773 } else { 3774 /* Our parent is the dnode itself. */ 3775 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 3776 db->db_blkid != DMU_SPILL_BLKID) || 3777 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 3778 if (db->db_blkid != DMU_SPILL_BLKID) 3779 ASSERT3P(db->db_blkptr, ==, 3780 &dn->dn_phys->dn_blkptr[db->db_blkid]); 3781 zio = dn->dn_zio; 3782 } 3783 3784 ASSERT(db->db_level == 0 || data == db->db_buf); 3785 ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 3786 ASSERT(zio); 3787 3788 SET_BOOKMARK(&zb, os->os_dsl_dataset ? 3789 os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 3790 db->db.db_object, db->db_level, db->db_blkid); 3791 3792 if (db->db_blkid == DMU_SPILL_BLKID) 3793 wp_flag = WP_SPILL; 3794 wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 3795 3796 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); 3797 DB_DNODE_EXIT(db); 3798 3799 /* 3800 * We copy the blkptr now (rather than when we instantiate the dirty 3801 * record), because its value can change between open context and 3802 * syncing context. We do not need to hold dn_struct_rwlock to read 3803 * db_blkptr because we are in syncing context. 3804 */ 3805 dr->dr_bp_copy = *db->db_blkptr; 3806 3807 if (db->db_level == 0 && 3808 dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 3809 /* 3810 * The BP for this block has been provided by open context 3811 * (by dmu_sync() or dmu_buf_write_embedded()). 3812 */ 3813 abd_t *contents = (data != NULL) ? 3814 abd_get_from_buf(data->b_data, arc_buf_size(data)) : NULL; 3815 3816 dr->dr_zio = zio_write(zio, os->os_spa, txg, &dr->dr_bp_copy, 3817 contents, db->db.db_size, db->db.db_size, &zp, 3818 dbuf_write_override_ready, NULL, NULL, 3819 dbuf_write_override_done, 3820 dr, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 3821 mutex_enter(&db->db_mtx); 3822 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 3823 zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 3824 dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite); 3825 mutex_exit(&db->db_mtx); 3826 } else if (db->db_state == DB_NOFILL) { 3827 ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF || 3828 zp.zp_checksum == ZIO_CHECKSUM_NOPARITY); 3829 dr->dr_zio = zio_write(zio, os->os_spa, txg, 3830 &dr->dr_bp_copy, NULL, db->db.db_size, db->db.db_size, &zp, 3831 dbuf_write_nofill_ready, NULL, NULL, 3832 dbuf_write_nofill_done, db, 3833 ZIO_PRIORITY_ASYNC_WRITE, 3834 ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 3835 } else { 3836 ASSERT(arc_released(data)); 3837 3838 /* 3839 * For indirect blocks, we want to setup the children 3840 * ready callback so that we can properly handle an indirect 3841 * block that only contains holes. 3842 */ 3843 arc_done_func_t *children_ready_cb = NULL; 3844 if (db->db_level != 0) 3845 children_ready_cb = dbuf_write_children_ready; 3846 3847 dr->dr_zio = arc_write(zio, os->os_spa, txg, 3848 &dr->dr_bp_copy, data, DBUF_IS_L2CACHEABLE(db), 3849 &zp, dbuf_write_ready, children_ready_cb, 3850 dbuf_write_physdone, dbuf_write_done, db, 3851 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 3852 } 3853 } 3854