1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 24 * Copyright (c) 2012, 2015 by Delphix. All rights reserved. 25 * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 26 * Copyright (c) 2013, Joyent, Inc. All rights reserved. 27 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 28 * Copyright (c) 2014 Integros [integros.com] 29 */ 30 31 #include <sys/zfs_context.h> 32 #include <sys/dmu.h> 33 #include <sys/dmu_send.h> 34 #include <sys/dmu_impl.h> 35 #include <sys/dbuf.h> 36 #include <sys/dmu_objset.h> 37 #include <sys/dsl_dataset.h> 38 #include <sys/dsl_dir.h> 39 #include <sys/dmu_tx.h> 40 #include <sys/spa.h> 41 #include <sys/zio.h> 42 #include <sys/dmu_zfetch.h> 43 #include <sys/sa.h> 44 #include <sys/sa_impl.h> 45 #include <sys/zfeature.h> 46 #include <sys/blkptr.h> 47 #include <sys/range_tree.h> 48 #include <sys/callb.h> 49 50 uint_t zfs_dbuf_evict_key; 51 52 /* 53 * Number of times that zfs_free_range() took the slow path while doing 54 * a zfs receive. A nonzero value indicates a potential performance problem. 55 */ 56 uint64_t zfs_free_range_recv_miss; 57 58 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 59 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 60 61 #ifndef __lint 62 extern inline void dmu_buf_init_user(dmu_buf_user_t *dbu, 63 dmu_buf_evict_func_t *evict_func, dmu_buf_t **clear_on_evict_dbufp); 64 #endif /* ! __lint */ 65 66 /* 67 * Global data structures and functions for the dbuf cache. 68 */ 69 static kmem_cache_t *dbuf_kmem_cache; 70 static taskq_t *dbu_evict_taskq; 71 72 static kthread_t *dbuf_cache_evict_thread; 73 static kmutex_t dbuf_evict_lock; 74 static kcondvar_t dbuf_evict_cv; 75 static boolean_t dbuf_evict_thread_exit; 76 77 /* 78 * LRU cache of dbufs. The dbuf cache maintains a list of dbufs that 79 * are not currently held but have been recently released. These dbufs 80 * are not eligible for arc eviction until they are aged out of the cache. 81 * Dbufs are added to the dbuf cache once the last hold is released. If a 82 * dbuf is later accessed and still exists in the dbuf cache, then it will 83 * be removed from the cache and later re-added to the head of the cache. 84 * Dbufs that are aged out of the cache will be immediately destroyed and 85 * become eligible for arc eviction. 86 */ 87 static multilist_t dbuf_cache; 88 static refcount_t dbuf_cache_size; 89 uint64_t dbuf_cache_max_bytes = 100 * 1024 * 1024; 90 91 /* Cap the size of the dbuf cache to log2 fraction of arc size. */ 92 int dbuf_cache_max_shift = 5; 93 94 /* 95 * The dbuf cache uses a three-stage eviction policy: 96 * - A low water marker designates when the dbuf eviction thread 97 * should stop evicting from the dbuf cache. 98 * - When we reach the maximum size (aka mid water mark), we 99 * signal the eviction thread to run. 100 * - The high water mark indicates when the eviction thread 101 * is unable to keep up with the incoming load and eviction must 102 * happen in the context of the calling thread. 103 * 104 * The dbuf cache: 105 * (max size) 106 * low water mid water hi water 107 * +----------------------------------------+----------+----------+ 108 * | | | | 109 * | | | | 110 * | | | | 111 * | | | | 112 * +----------------------------------------+----------+----------+ 113 * stop signal evict 114 * evicting eviction directly 115 * thread 116 * 117 * The high and low water marks indicate the operating range for the eviction 118 * thread. The low water mark is, by default, 90% of the total size of the 119 * cache and the high water mark is at 110% (both of these percentages can be 120 * changed by setting dbuf_cache_lowater_pct and dbuf_cache_hiwater_pct, 121 * respectively). The eviction thread will try to ensure that the cache remains 122 * within this range by waking up every second and checking if the cache is 123 * above the low water mark. The thread can also be woken up by callers adding 124 * elements into the cache if the cache is larger than the mid water (i.e max 125 * cache size). Once the eviction thread is woken up and eviction is required, 126 * it will continue evicting buffers until it's able to reduce the cache size 127 * to the low water mark. If the cache size continues to grow and hits the high 128 * water mark, then callers adding elments to the cache will begin to evict 129 * directly from the cache until the cache is no longer above the high water 130 * mark. 131 */ 132 133 /* 134 * The percentage above and below the maximum cache size. 135 */ 136 uint_t dbuf_cache_hiwater_pct = 10; 137 uint_t dbuf_cache_lowater_pct = 10; 138 139 /* ARGSUSED */ 140 static int 141 dbuf_cons(void *vdb, void *unused, int kmflag) 142 { 143 dmu_buf_impl_t *db = vdb; 144 bzero(db, sizeof (dmu_buf_impl_t)); 145 146 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 147 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 148 multilist_link_init(&db->db_cache_link); 149 refcount_create(&db->db_holds); 150 151 return (0); 152 } 153 154 /* ARGSUSED */ 155 static void 156 dbuf_dest(void *vdb, void *unused) 157 { 158 dmu_buf_impl_t *db = vdb; 159 mutex_destroy(&db->db_mtx); 160 cv_destroy(&db->db_changed); 161 ASSERT(!multilist_link_active(&db->db_cache_link)); 162 refcount_destroy(&db->db_holds); 163 } 164 165 /* 166 * dbuf hash table routines 167 */ 168 static dbuf_hash_table_t dbuf_hash_table; 169 170 static uint64_t dbuf_hash_count; 171 172 static uint64_t 173 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 174 { 175 uintptr_t osv = (uintptr_t)os; 176 uint64_t crc = -1ULL; 177 178 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 179 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 180 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 181 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 182 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 183 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 184 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 185 186 crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 187 188 return (crc); 189 } 190 191 #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 192 ((dbuf)->db.db_object == (obj) && \ 193 (dbuf)->db_objset == (os) && \ 194 (dbuf)->db_level == (level) && \ 195 (dbuf)->db_blkid == (blkid)) 196 197 dmu_buf_impl_t * 198 dbuf_find(objset_t *os, uint64_t obj, uint8_t level, uint64_t blkid) 199 { 200 dbuf_hash_table_t *h = &dbuf_hash_table; 201 uint64_t hv = dbuf_hash(os, obj, level, blkid); 202 uint64_t idx = hv & h->hash_table_mask; 203 dmu_buf_impl_t *db; 204 205 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 206 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 207 if (DBUF_EQUAL(db, os, obj, level, blkid)) { 208 mutex_enter(&db->db_mtx); 209 if (db->db_state != DB_EVICTING) { 210 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 211 return (db); 212 } 213 mutex_exit(&db->db_mtx); 214 } 215 } 216 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 217 return (NULL); 218 } 219 220 static dmu_buf_impl_t * 221 dbuf_find_bonus(objset_t *os, uint64_t object) 222 { 223 dnode_t *dn; 224 dmu_buf_impl_t *db = NULL; 225 226 if (dnode_hold(os, object, FTAG, &dn) == 0) { 227 rw_enter(&dn->dn_struct_rwlock, RW_READER); 228 if (dn->dn_bonus != NULL) { 229 db = dn->dn_bonus; 230 mutex_enter(&db->db_mtx); 231 } 232 rw_exit(&dn->dn_struct_rwlock); 233 dnode_rele(dn, FTAG); 234 } 235 return (db); 236 } 237 238 /* 239 * Insert an entry into the hash table. If there is already an element 240 * equal to elem in the hash table, then the already existing element 241 * will be returned and the new element will not be inserted. 242 * Otherwise returns NULL. 243 */ 244 static dmu_buf_impl_t * 245 dbuf_hash_insert(dmu_buf_impl_t *db) 246 { 247 dbuf_hash_table_t *h = &dbuf_hash_table; 248 objset_t *os = db->db_objset; 249 uint64_t obj = db->db.db_object; 250 int level = db->db_level; 251 uint64_t blkid = db->db_blkid; 252 uint64_t hv = dbuf_hash(os, obj, level, blkid); 253 uint64_t idx = hv & h->hash_table_mask; 254 dmu_buf_impl_t *dbf; 255 256 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 257 for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 258 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 259 mutex_enter(&dbf->db_mtx); 260 if (dbf->db_state != DB_EVICTING) { 261 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 262 return (dbf); 263 } 264 mutex_exit(&dbf->db_mtx); 265 } 266 } 267 268 mutex_enter(&db->db_mtx); 269 db->db_hash_next = h->hash_table[idx]; 270 h->hash_table[idx] = db; 271 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 272 atomic_inc_64(&dbuf_hash_count); 273 274 return (NULL); 275 } 276 277 /* 278 * Remove an entry from the hash table. It must be in the EVICTING state. 279 */ 280 static void 281 dbuf_hash_remove(dmu_buf_impl_t *db) 282 { 283 dbuf_hash_table_t *h = &dbuf_hash_table; 284 uint64_t hv = dbuf_hash(db->db_objset, db->db.db_object, 285 db->db_level, db->db_blkid); 286 uint64_t idx = hv & h->hash_table_mask; 287 dmu_buf_impl_t *dbf, **dbp; 288 289 /* 290 * We musn't hold db_mtx to maintain lock ordering: 291 * DBUF_HASH_MUTEX > db_mtx. 292 */ 293 ASSERT(refcount_is_zero(&db->db_holds)); 294 ASSERT(db->db_state == DB_EVICTING); 295 ASSERT(!MUTEX_HELD(&db->db_mtx)); 296 297 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 298 dbp = &h->hash_table[idx]; 299 while ((dbf = *dbp) != db) { 300 dbp = &dbf->db_hash_next; 301 ASSERT(dbf != NULL); 302 } 303 *dbp = db->db_hash_next; 304 db->db_hash_next = NULL; 305 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 306 atomic_dec_64(&dbuf_hash_count); 307 } 308 309 typedef enum { 310 DBVU_EVICTING, 311 DBVU_NOT_EVICTING 312 } dbvu_verify_type_t; 313 314 static void 315 dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type) 316 { 317 #ifdef ZFS_DEBUG 318 int64_t holds; 319 320 if (db->db_user == NULL) 321 return; 322 323 /* Only data blocks support the attachment of user data. */ 324 ASSERT(db->db_level == 0); 325 326 /* Clients must resolve a dbuf before attaching user data. */ 327 ASSERT(db->db.db_data != NULL); 328 ASSERT3U(db->db_state, ==, DB_CACHED); 329 330 holds = refcount_count(&db->db_holds); 331 if (verify_type == DBVU_EVICTING) { 332 /* 333 * Immediate eviction occurs when holds == dirtycnt. 334 * For normal eviction buffers, holds is zero on 335 * eviction, except when dbuf_fix_old_data() calls 336 * dbuf_clear_data(). However, the hold count can grow 337 * during eviction even though db_mtx is held (see 338 * dmu_bonus_hold() for an example), so we can only 339 * test the generic invariant that holds >= dirtycnt. 340 */ 341 ASSERT3U(holds, >=, db->db_dirtycnt); 342 } else { 343 if (db->db_user_immediate_evict == TRUE) 344 ASSERT3U(holds, >=, db->db_dirtycnt); 345 else 346 ASSERT3U(holds, >, 0); 347 } 348 #endif 349 } 350 351 static void 352 dbuf_evict_user(dmu_buf_impl_t *db) 353 { 354 dmu_buf_user_t *dbu = db->db_user; 355 356 ASSERT(MUTEX_HELD(&db->db_mtx)); 357 358 if (dbu == NULL) 359 return; 360 361 dbuf_verify_user(db, DBVU_EVICTING); 362 db->db_user = NULL; 363 364 #ifdef ZFS_DEBUG 365 if (dbu->dbu_clear_on_evict_dbufp != NULL) 366 *dbu->dbu_clear_on_evict_dbufp = NULL; 367 #endif 368 369 /* 370 * Invoke the callback from a taskq to avoid lock order reversals 371 * and limit stack depth. 372 */ 373 taskq_dispatch_ent(dbu_evict_taskq, dbu->dbu_evict_func, dbu, 0, 374 &dbu->dbu_tqent); 375 } 376 377 boolean_t 378 dbuf_is_metadata(dmu_buf_impl_t *db) 379 { 380 if (db->db_level > 0) { 381 return (B_TRUE); 382 } else { 383 boolean_t is_metadata; 384 385 DB_DNODE_ENTER(db); 386 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); 387 DB_DNODE_EXIT(db); 388 389 return (is_metadata); 390 } 391 } 392 393 /* 394 * This function *must* return indices evenly distributed between all 395 * sublists of the multilist. This is needed due to how the dbuf eviction 396 * code is laid out; dbuf_evict_thread() assumes dbufs are evenly 397 * distributed between all sublists and uses this assumption when 398 * deciding which sublist to evict from and how much to evict from it. 399 */ 400 unsigned int 401 dbuf_cache_multilist_index_func(multilist_t *ml, void *obj) 402 { 403 dmu_buf_impl_t *db = obj; 404 405 /* 406 * The assumption here, is the hash value for a given 407 * dmu_buf_impl_t will remain constant throughout it's lifetime 408 * (i.e. it's objset, object, level and blkid fields don't change). 409 * Thus, we don't need to store the dbuf's sublist index 410 * on insertion, as this index can be recalculated on removal. 411 * 412 * Also, the low order bits of the hash value are thought to be 413 * distributed evenly. Otherwise, in the case that the multilist 414 * has a power of two number of sublists, each sublists' usage 415 * would not be evenly distributed. 416 */ 417 return (dbuf_hash(db->db_objset, db->db.db_object, 418 db->db_level, db->db_blkid) % 419 multilist_get_num_sublists(ml)); 420 } 421 422 static inline boolean_t 423 dbuf_cache_above_hiwater(void) 424 { 425 uint64_t dbuf_cache_hiwater_bytes = 426 (dbuf_cache_max_bytes * dbuf_cache_hiwater_pct) / 100; 427 428 return (refcount_count(&dbuf_cache_size) > 429 dbuf_cache_max_bytes + dbuf_cache_hiwater_bytes); 430 } 431 432 static inline boolean_t 433 dbuf_cache_above_lowater(void) 434 { 435 uint64_t dbuf_cache_lowater_bytes = 436 (dbuf_cache_max_bytes * dbuf_cache_lowater_pct) / 100; 437 438 return (refcount_count(&dbuf_cache_size) > 439 dbuf_cache_max_bytes - dbuf_cache_lowater_bytes); 440 } 441 442 /* 443 * Evict the oldest eligible dbuf from the dbuf cache. 444 */ 445 static void 446 dbuf_evict_one(void) 447 { 448 int idx = multilist_get_random_index(&dbuf_cache); 449 multilist_sublist_t *mls = multilist_sublist_lock(&dbuf_cache, idx); 450 451 ASSERT(!MUTEX_HELD(&dbuf_evict_lock)); 452 453 /* 454 * Set the thread's tsd to indicate that it's processing evictions. 455 * Once a thread stops evicting from the dbuf cache it will 456 * reset its tsd to NULL. 457 */ 458 ASSERT3P(tsd_get(zfs_dbuf_evict_key), ==, NULL); 459 (void) tsd_set(zfs_dbuf_evict_key, (void *)B_TRUE); 460 461 dmu_buf_impl_t *db = multilist_sublist_tail(mls); 462 while (db != NULL && mutex_tryenter(&db->db_mtx) == 0) { 463 db = multilist_sublist_prev(mls, db); 464 } 465 466 DTRACE_PROBE2(dbuf__evict__one, dmu_buf_impl_t *, db, 467 multilist_sublist_t *, mls); 468 469 if (db != NULL) { 470 multilist_sublist_remove(mls, db); 471 multilist_sublist_unlock(mls); 472 (void) refcount_remove_many(&dbuf_cache_size, 473 db->db.db_size, db); 474 dbuf_destroy(db); 475 } else { 476 multilist_sublist_unlock(mls); 477 } 478 (void) tsd_set(zfs_dbuf_evict_key, NULL); 479 } 480 481 /* 482 * The dbuf evict thread is responsible for aging out dbufs from the 483 * cache. Once the cache has reached it's maximum size, dbufs are removed 484 * and destroyed. The eviction thread will continue running until the size 485 * of the dbuf cache is at or below the maximum size. Once the dbuf is aged 486 * out of the cache it is destroyed and becomes eligible for arc eviction. 487 */ 488 static void 489 dbuf_evict_thread(void) 490 { 491 callb_cpr_t cpr; 492 493 CALLB_CPR_INIT(&cpr, &dbuf_evict_lock, callb_generic_cpr, FTAG); 494 495 mutex_enter(&dbuf_evict_lock); 496 while (!dbuf_evict_thread_exit) { 497 while (!dbuf_cache_above_lowater() && !dbuf_evict_thread_exit) { 498 CALLB_CPR_SAFE_BEGIN(&cpr); 499 (void) cv_timedwait_hires(&dbuf_evict_cv, 500 &dbuf_evict_lock, SEC2NSEC(1), MSEC2NSEC(1), 0); 501 CALLB_CPR_SAFE_END(&cpr, &dbuf_evict_lock); 502 } 503 mutex_exit(&dbuf_evict_lock); 504 505 /* 506 * Keep evicting as long as we're above the low water mark 507 * for the cache. We do this without holding the locks to 508 * minimize lock contention. 509 */ 510 while (dbuf_cache_above_lowater() && !dbuf_evict_thread_exit) { 511 dbuf_evict_one(); 512 } 513 514 mutex_enter(&dbuf_evict_lock); 515 } 516 517 dbuf_evict_thread_exit = B_FALSE; 518 cv_broadcast(&dbuf_evict_cv); 519 CALLB_CPR_EXIT(&cpr); /* drops dbuf_evict_lock */ 520 thread_exit(); 521 } 522 523 /* 524 * Wake up the dbuf eviction thread if the dbuf cache is at its max size. 525 * If the dbuf cache is at its high water mark, then evict a dbuf from the 526 * dbuf cache using the callers context. 527 */ 528 static void 529 dbuf_evict_notify(void) 530 { 531 532 /* 533 * We use thread specific data to track when a thread has 534 * started processing evictions. This allows us to avoid deeply 535 * nested stacks that would have a call flow similar to this: 536 * 537 * dbuf_rele()-->dbuf_rele_and_unlock()-->dbuf_evict_notify() 538 * ^ | 539 * | | 540 * +-----dbuf_destroy()<--dbuf_evict_one()<--------+ 541 * 542 * The dbuf_eviction_thread will always have its tsd set until 543 * that thread exits. All other threads will only set their tsd 544 * if they are participating in the eviction process. This only 545 * happens if the eviction thread is unable to process evictions 546 * fast enough. To keep the dbuf cache size in check, other threads 547 * can evict from the dbuf cache directly. Those threads will set 548 * their tsd values so that we ensure that they only evict one dbuf 549 * from the dbuf cache. 550 */ 551 if (tsd_get(zfs_dbuf_evict_key) != NULL) 552 return; 553 554 if (refcount_count(&dbuf_cache_size) > dbuf_cache_max_bytes) { 555 boolean_t evict_now = B_FALSE; 556 557 mutex_enter(&dbuf_evict_lock); 558 if (refcount_count(&dbuf_cache_size) > dbuf_cache_max_bytes) { 559 evict_now = dbuf_cache_above_hiwater(); 560 cv_signal(&dbuf_evict_cv); 561 } 562 mutex_exit(&dbuf_evict_lock); 563 564 if (evict_now) { 565 dbuf_evict_one(); 566 } 567 } 568 } 569 570 void 571 dbuf_init(void) 572 { 573 uint64_t hsize = 1ULL << 16; 574 dbuf_hash_table_t *h = &dbuf_hash_table; 575 int i; 576 577 /* 578 * The hash table is big enough to fill all of physical memory 579 * with an average 4K block size. The table will take up 580 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 581 */ 582 while (hsize * 4096 < physmem * PAGESIZE) 583 hsize <<= 1; 584 585 retry: 586 h->hash_table_mask = hsize - 1; 587 h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 588 if (h->hash_table == NULL) { 589 /* XXX - we should really return an error instead of assert */ 590 ASSERT(hsize > (1ULL << 10)); 591 hsize >>= 1; 592 goto retry; 593 } 594 595 dbuf_kmem_cache = kmem_cache_create("dmu_buf_impl_t", 596 sizeof (dmu_buf_impl_t), 597 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 598 599 for (i = 0; i < DBUF_MUTEXES; i++) 600 mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 601 602 /* 603 * Setup the parameters for the dbuf cache. We cap the size of the 604 * dbuf cache to 1/32nd (default) of the size of the ARC. 605 */ 606 dbuf_cache_max_bytes = MIN(dbuf_cache_max_bytes, 607 arc_max_bytes() >> dbuf_cache_max_shift); 608 609 /* 610 * All entries are queued via taskq_dispatch_ent(), so min/maxalloc 611 * configuration is not required. 612 */ 613 dbu_evict_taskq = taskq_create("dbu_evict", 1, minclsyspri, 0, 0, 0); 614 615 multilist_create(&dbuf_cache, sizeof (dmu_buf_impl_t), 616 offsetof(dmu_buf_impl_t, db_cache_link), 617 zfs_arc_num_sublists_per_state, 618 dbuf_cache_multilist_index_func); 619 refcount_create(&dbuf_cache_size); 620 621 tsd_create(&zfs_dbuf_evict_key, NULL); 622 dbuf_evict_thread_exit = B_FALSE; 623 mutex_init(&dbuf_evict_lock, NULL, MUTEX_DEFAULT, NULL); 624 cv_init(&dbuf_evict_cv, NULL, CV_DEFAULT, NULL); 625 dbuf_cache_evict_thread = thread_create(NULL, 0, dbuf_evict_thread, 626 NULL, 0, &p0, TS_RUN, minclsyspri); 627 } 628 629 void 630 dbuf_fini(void) 631 { 632 dbuf_hash_table_t *h = &dbuf_hash_table; 633 int i; 634 635 for (i = 0; i < DBUF_MUTEXES; i++) 636 mutex_destroy(&h->hash_mutexes[i]); 637 kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 638 kmem_cache_destroy(dbuf_kmem_cache); 639 taskq_destroy(dbu_evict_taskq); 640 641 mutex_enter(&dbuf_evict_lock); 642 dbuf_evict_thread_exit = B_TRUE; 643 while (dbuf_evict_thread_exit) { 644 cv_signal(&dbuf_evict_cv); 645 cv_wait(&dbuf_evict_cv, &dbuf_evict_lock); 646 } 647 mutex_exit(&dbuf_evict_lock); 648 tsd_destroy(&zfs_dbuf_evict_key); 649 650 mutex_destroy(&dbuf_evict_lock); 651 cv_destroy(&dbuf_evict_cv); 652 653 refcount_destroy(&dbuf_cache_size); 654 multilist_destroy(&dbuf_cache); 655 } 656 657 /* 658 * Other stuff. 659 */ 660 661 #ifdef ZFS_DEBUG 662 static void 663 dbuf_verify(dmu_buf_impl_t *db) 664 { 665 dnode_t *dn; 666 dbuf_dirty_record_t *dr; 667 668 ASSERT(MUTEX_HELD(&db->db_mtx)); 669 670 if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 671 return; 672 673 ASSERT(db->db_objset != NULL); 674 DB_DNODE_ENTER(db); 675 dn = DB_DNODE(db); 676 if (dn == NULL) { 677 ASSERT(db->db_parent == NULL); 678 ASSERT(db->db_blkptr == NULL); 679 } else { 680 ASSERT3U(db->db.db_object, ==, dn->dn_object); 681 ASSERT3P(db->db_objset, ==, dn->dn_objset); 682 ASSERT3U(db->db_level, <, dn->dn_nlevels); 683 ASSERT(db->db_blkid == DMU_BONUS_BLKID || 684 db->db_blkid == DMU_SPILL_BLKID || 685 !avl_is_empty(&dn->dn_dbufs)); 686 } 687 if (db->db_blkid == DMU_BONUS_BLKID) { 688 ASSERT(dn != NULL); 689 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 690 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 691 } else if (db->db_blkid == DMU_SPILL_BLKID) { 692 ASSERT(dn != NULL); 693 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 694 ASSERT0(db->db.db_offset); 695 } else { 696 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 697 } 698 699 for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 700 ASSERT(dr->dr_dbuf == db); 701 702 for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 703 ASSERT(dr->dr_dbuf == db); 704 705 /* 706 * We can't assert that db_size matches dn_datablksz because it 707 * can be momentarily different when another thread is doing 708 * dnode_set_blksz(). 709 */ 710 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 711 dr = db->db_data_pending; 712 /* 713 * It should only be modified in syncing context, so 714 * make sure we only have one copy of the data. 715 */ 716 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 717 } 718 719 /* verify db->db_blkptr */ 720 if (db->db_blkptr) { 721 if (db->db_parent == dn->dn_dbuf) { 722 /* db is pointed to by the dnode */ 723 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 724 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 725 ASSERT(db->db_parent == NULL); 726 else 727 ASSERT(db->db_parent != NULL); 728 if (db->db_blkid != DMU_SPILL_BLKID) 729 ASSERT3P(db->db_blkptr, ==, 730 &dn->dn_phys->dn_blkptr[db->db_blkid]); 731 } else { 732 /* db is pointed to by an indirect block */ 733 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 734 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 735 ASSERT3U(db->db_parent->db.db_object, ==, 736 db->db.db_object); 737 /* 738 * dnode_grow_indblksz() can make this fail if we don't 739 * have the struct_rwlock. XXX indblksz no longer 740 * grows. safe to do this now? 741 */ 742 if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 743 ASSERT3P(db->db_blkptr, ==, 744 ((blkptr_t *)db->db_parent->db.db_data + 745 db->db_blkid % epb)); 746 } 747 } 748 } 749 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 750 (db->db_buf == NULL || db->db_buf->b_data) && 751 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 752 db->db_state != DB_FILL && !dn->dn_free_txg) { 753 /* 754 * If the blkptr isn't set but they have nonzero data, 755 * it had better be dirty, otherwise we'll lose that 756 * data when we evict this buffer. 757 * 758 * There is an exception to this rule for indirect blocks; in 759 * this case, if the indirect block is a hole, we fill in a few 760 * fields on each of the child blocks (importantly, birth time) 761 * to prevent hole birth times from being lost when you 762 * partially fill in a hole. 763 */ 764 if (db->db_dirtycnt == 0) { 765 if (db->db_level == 0) { 766 uint64_t *buf = db->db.db_data; 767 int i; 768 769 for (i = 0; i < db->db.db_size >> 3; i++) { 770 ASSERT(buf[i] == 0); 771 } 772 } else { 773 blkptr_t *bps = db->db.db_data; 774 ASSERT3U(1 << DB_DNODE(db)->dn_indblkshift, ==, 775 db->db.db_size); 776 /* 777 * We want to verify that all the blkptrs in the 778 * indirect block are holes, but we may have 779 * automatically set up a few fields for them. 780 * We iterate through each blkptr and verify 781 * they only have those fields set. 782 */ 783 for (int i = 0; 784 i < db->db.db_size / sizeof (blkptr_t); 785 i++) { 786 blkptr_t *bp = &bps[i]; 787 ASSERT(ZIO_CHECKSUM_IS_ZERO( 788 &bp->blk_cksum)); 789 ASSERT( 790 DVA_IS_EMPTY(&bp->blk_dva[0]) && 791 DVA_IS_EMPTY(&bp->blk_dva[1]) && 792 DVA_IS_EMPTY(&bp->blk_dva[2])); 793 ASSERT0(bp->blk_fill); 794 ASSERT0(bp->blk_pad[0]); 795 ASSERT0(bp->blk_pad[1]); 796 ASSERT(!BP_IS_EMBEDDED(bp)); 797 ASSERT(BP_IS_HOLE(bp)); 798 ASSERT0(bp->blk_phys_birth); 799 } 800 } 801 } 802 } 803 DB_DNODE_EXIT(db); 804 } 805 #endif 806 807 static void 808 dbuf_clear_data(dmu_buf_impl_t *db) 809 { 810 ASSERT(MUTEX_HELD(&db->db_mtx)); 811 dbuf_evict_user(db); 812 ASSERT3P(db->db_buf, ==, NULL); 813 db->db.db_data = NULL; 814 if (db->db_state != DB_NOFILL) 815 db->db_state = DB_UNCACHED; 816 } 817 818 static void 819 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 820 { 821 ASSERT(MUTEX_HELD(&db->db_mtx)); 822 ASSERT(buf != NULL); 823 824 db->db_buf = buf; 825 ASSERT(buf->b_data != NULL); 826 db->db.db_data = buf->b_data; 827 } 828 829 /* 830 * Loan out an arc_buf for read. Return the loaned arc_buf. 831 */ 832 arc_buf_t * 833 dbuf_loan_arcbuf(dmu_buf_impl_t *db) 834 { 835 arc_buf_t *abuf; 836 837 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 838 mutex_enter(&db->db_mtx); 839 if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 840 int blksz = db->db.db_size; 841 spa_t *spa = db->db_objset->os_spa; 842 843 mutex_exit(&db->db_mtx); 844 abuf = arc_loan_buf(spa, blksz); 845 bcopy(db->db.db_data, abuf->b_data, blksz); 846 } else { 847 abuf = db->db_buf; 848 arc_loan_inuse_buf(abuf, db); 849 db->db_buf = NULL; 850 dbuf_clear_data(db); 851 mutex_exit(&db->db_mtx); 852 } 853 return (abuf); 854 } 855 856 /* 857 * Calculate which level n block references the data at the level 0 offset 858 * provided. 859 */ 860 uint64_t 861 dbuf_whichblock(dnode_t *dn, int64_t level, uint64_t offset) 862 { 863 if (dn->dn_datablkshift != 0 && dn->dn_indblkshift != 0) { 864 /* 865 * The level n blkid is equal to the level 0 blkid divided by 866 * the number of level 0s in a level n block. 867 * 868 * The level 0 blkid is offset >> datablkshift = 869 * offset / 2^datablkshift. 870 * 871 * The number of level 0s in a level n is the number of block 872 * pointers in an indirect block, raised to the power of level. 873 * This is 2^(indblkshift - SPA_BLKPTRSHIFT)^level = 874 * 2^(level*(indblkshift - SPA_BLKPTRSHIFT)). 875 * 876 * Thus, the level n blkid is: offset / 877 * ((2^datablkshift)*(2^(level*(indblkshift - SPA_BLKPTRSHIFT))) 878 * = offset / 2^(datablkshift + level * 879 * (indblkshift - SPA_BLKPTRSHIFT)) 880 * = offset >> (datablkshift + level * 881 * (indblkshift - SPA_BLKPTRSHIFT)) 882 */ 883 return (offset >> (dn->dn_datablkshift + level * 884 (dn->dn_indblkshift - SPA_BLKPTRSHIFT))); 885 } else { 886 ASSERT3U(offset, <, dn->dn_datablksz); 887 return (0); 888 } 889 } 890 891 static void 892 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 893 { 894 dmu_buf_impl_t *db = vdb; 895 896 mutex_enter(&db->db_mtx); 897 ASSERT3U(db->db_state, ==, DB_READ); 898 /* 899 * All reads are synchronous, so we must have a hold on the dbuf 900 */ 901 ASSERT(refcount_count(&db->db_holds) > 0); 902 ASSERT(db->db_buf == NULL); 903 ASSERT(db->db.db_data == NULL); 904 if (db->db_level == 0 && db->db_freed_in_flight) { 905 /* we were freed in flight; disregard any error */ 906 arc_release(buf, db); 907 bzero(buf->b_data, db->db.db_size); 908 arc_buf_freeze(buf); 909 db->db_freed_in_flight = FALSE; 910 dbuf_set_data(db, buf); 911 db->db_state = DB_CACHED; 912 } else if (zio == NULL || zio->io_error == 0) { 913 dbuf_set_data(db, buf); 914 db->db_state = DB_CACHED; 915 } else { 916 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 917 ASSERT3P(db->db_buf, ==, NULL); 918 arc_buf_destroy(buf, db); 919 db->db_state = DB_UNCACHED; 920 } 921 cv_broadcast(&db->db_changed); 922 dbuf_rele_and_unlock(db, NULL); 923 } 924 925 static void 926 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 927 { 928 dnode_t *dn; 929 zbookmark_phys_t zb; 930 arc_flags_t aflags = ARC_FLAG_NOWAIT; 931 932 DB_DNODE_ENTER(db); 933 dn = DB_DNODE(db); 934 ASSERT(!refcount_is_zero(&db->db_holds)); 935 /* We need the struct_rwlock to prevent db_blkptr from changing. */ 936 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 937 ASSERT(MUTEX_HELD(&db->db_mtx)); 938 ASSERT(db->db_state == DB_UNCACHED); 939 ASSERT(db->db_buf == NULL); 940 941 if (db->db_blkid == DMU_BONUS_BLKID) { 942 int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 943 944 ASSERT3U(bonuslen, <=, db->db.db_size); 945 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 946 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 947 if (bonuslen < DN_MAX_BONUSLEN) 948 bzero(db->db.db_data, DN_MAX_BONUSLEN); 949 if (bonuslen) 950 bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 951 DB_DNODE_EXIT(db); 952 db->db_state = DB_CACHED; 953 mutex_exit(&db->db_mtx); 954 return; 955 } 956 957 /* 958 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 959 * processes the delete record and clears the bp while we are waiting 960 * for the dn_mtx (resulting in a "no" from block_freed). 961 */ 962 if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 963 (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 964 BP_IS_HOLE(db->db_blkptr)))) { 965 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 966 967 dbuf_set_data(db, arc_alloc_buf(db->db_objset->os_spa, 968 db->db.db_size, db, type)); 969 bzero(db->db.db_data, db->db.db_size); 970 971 if (db->db_blkptr != NULL && db->db_level > 0 && 972 BP_IS_HOLE(db->db_blkptr) && 973 db->db_blkptr->blk_birth != 0) { 974 blkptr_t *bps = db->db.db_data; 975 for (int i = 0; i < ((1 << 976 DB_DNODE(db)->dn_indblkshift) / sizeof (blkptr_t)); 977 i++) { 978 blkptr_t *bp = &bps[i]; 979 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 980 1 << dn->dn_indblkshift); 981 BP_SET_LSIZE(bp, 982 BP_GET_LEVEL(db->db_blkptr) == 1 ? 983 dn->dn_datablksz : 984 BP_GET_LSIZE(db->db_blkptr)); 985 BP_SET_TYPE(bp, BP_GET_TYPE(db->db_blkptr)); 986 BP_SET_LEVEL(bp, 987 BP_GET_LEVEL(db->db_blkptr) - 1); 988 BP_SET_BIRTH(bp, db->db_blkptr->blk_birth, 0); 989 } 990 } 991 DB_DNODE_EXIT(db); 992 db->db_state = DB_CACHED; 993 mutex_exit(&db->db_mtx); 994 return; 995 } 996 997 DB_DNODE_EXIT(db); 998 999 db->db_state = DB_READ; 1000 mutex_exit(&db->db_mtx); 1001 1002 if (DBUF_IS_L2CACHEABLE(db)) 1003 aflags |= ARC_FLAG_L2CACHE; 1004 1005 SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 1006 db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 1007 db->db.db_object, db->db_level, db->db_blkid); 1008 1009 dbuf_add_ref(db, NULL); 1010 1011 (void) arc_read(zio, db->db_objset->os_spa, db->db_blkptr, 1012 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 1013 (flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 1014 &aflags, &zb); 1015 } 1016 1017 int 1018 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 1019 { 1020 int err = 0; 1021 boolean_t havepzio = (zio != NULL); 1022 boolean_t prefetch; 1023 dnode_t *dn; 1024 1025 /* 1026 * We don't have to hold the mutex to check db_state because it 1027 * can't be freed while we have a hold on the buffer. 1028 */ 1029 ASSERT(!refcount_is_zero(&db->db_holds)); 1030 1031 if (db->db_state == DB_NOFILL) 1032 return (SET_ERROR(EIO)); 1033 1034 DB_DNODE_ENTER(db); 1035 dn = DB_DNODE(db); 1036 if ((flags & DB_RF_HAVESTRUCT) == 0) 1037 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1038 1039 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1040 (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL && 1041 DBUF_IS_CACHEABLE(db); 1042 1043 mutex_enter(&db->db_mtx); 1044 if (db->db_state == DB_CACHED) { 1045 mutex_exit(&db->db_mtx); 1046 if (prefetch) 1047 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE); 1048 if ((flags & DB_RF_HAVESTRUCT) == 0) 1049 rw_exit(&dn->dn_struct_rwlock); 1050 DB_DNODE_EXIT(db); 1051 } else if (db->db_state == DB_UNCACHED) { 1052 spa_t *spa = dn->dn_objset->os_spa; 1053 1054 if (zio == NULL) 1055 zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 1056 dbuf_read_impl(db, zio, flags); 1057 1058 /* dbuf_read_impl has dropped db_mtx for us */ 1059 1060 if (prefetch) 1061 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE); 1062 1063 if ((flags & DB_RF_HAVESTRUCT) == 0) 1064 rw_exit(&dn->dn_struct_rwlock); 1065 DB_DNODE_EXIT(db); 1066 1067 if (!havepzio) 1068 err = zio_wait(zio); 1069 } else { 1070 /* 1071 * Another reader came in while the dbuf was in flight 1072 * between UNCACHED and CACHED. Either a writer will finish 1073 * writing the buffer (sending the dbuf to CACHED) or the 1074 * first reader's request will reach the read_done callback 1075 * and send the dbuf to CACHED. Otherwise, a failure 1076 * occurred and the dbuf went to UNCACHED. 1077 */ 1078 mutex_exit(&db->db_mtx); 1079 if (prefetch) 1080 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE); 1081 if ((flags & DB_RF_HAVESTRUCT) == 0) 1082 rw_exit(&dn->dn_struct_rwlock); 1083 DB_DNODE_EXIT(db); 1084 1085 /* Skip the wait per the caller's request. */ 1086 mutex_enter(&db->db_mtx); 1087 if ((flags & DB_RF_NEVERWAIT) == 0) { 1088 while (db->db_state == DB_READ || 1089 db->db_state == DB_FILL) { 1090 ASSERT(db->db_state == DB_READ || 1091 (flags & DB_RF_HAVESTRUCT) == 0); 1092 DTRACE_PROBE2(blocked__read, dmu_buf_impl_t *, 1093 db, zio_t *, zio); 1094 cv_wait(&db->db_changed, &db->db_mtx); 1095 } 1096 if (db->db_state == DB_UNCACHED) 1097 err = SET_ERROR(EIO); 1098 } 1099 mutex_exit(&db->db_mtx); 1100 } 1101 1102 ASSERT(err || havepzio || db->db_state == DB_CACHED); 1103 return (err); 1104 } 1105 1106 static void 1107 dbuf_noread(dmu_buf_impl_t *db) 1108 { 1109 ASSERT(!refcount_is_zero(&db->db_holds)); 1110 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1111 mutex_enter(&db->db_mtx); 1112 while (db->db_state == DB_READ || db->db_state == DB_FILL) 1113 cv_wait(&db->db_changed, &db->db_mtx); 1114 if (db->db_state == DB_UNCACHED) { 1115 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1116 spa_t *spa = db->db_objset->os_spa; 1117 1118 ASSERT(db->db_buf == NULL); 1119 ASSERT(db->db.db_data == NULL); 1120 dbuf_set_data(db, arc_alloc_buf(spa, db->db.db_size, db, type)); 1121 db->db_state = DB_FILL; 1122 } else if (db->db_state == DB_NOFILL) { 1123 dbuf_clear_data(db); 1124 } else { 1125 ASSERT3U(db->db_state, ==, DB_CACHED); 1126 } 1127 mutex_exit(&db->db_mtx); 1128 } 1129 1130 /* 1131 * This is our just-in-time copy function. It makes a copy of 1132 * buffers, that have been modified in a previous transaction 1133 * group, before we modify them in the current active group. 1134 * 1135 * This function is used in two places: when we are dirtying a 1136 * buffer for the first time in a txg, and when we are freeing 1137 * a range in a dnode that includes this buffer. 1138 * 1139 * Note that when we are called from dbuf_free_range() we do 1140 * not put a hold on the buffer, we just traverse the active 1141 * dbuf list for the dnode. 1142 */ 1143 static void 1144 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 1145 { 1146 dbuf_dirty_record_t *dr = db->db_last_dirty; 1147 1148 ASSERT(MUTEX_HELD(&db->db_mtx)); 1149 ASSERT(db->db.db_data != NULL); 1150 ASSERT(db->db_level == 0); 1151 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 1152 1153 if (dr == NULL || 1154 (dr->dt.dl.dr_data != 1155 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 1156 return; 1157 1158 /* 1159 * If the last dirty record for this dbuf has not yet synced 1160 * and its referencing the dbuf data, either: 1161 * reset the reference to point to a new copy, 1162 * or (if there a no active holders) 1163 * just null out the current db_data pointer. 1164 */ 1165 ASSERT(dr->dr_txg >= txg - 2); 1166 if (db->db_blkid == DMU_BONUS_BLKID) { 1167 /* Note that the data bufs here are zio_bufs */ 1168 dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 1169 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 1170 bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 1171 } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1172 int size = db->db.db_size; 1173 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1174 spa_t *spa = db->db_objset->os_spa; 1175 1176 dr->dt.dl.dr_data = arc_alloc_buf(spa, size, db, type); 1177 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 1178 } else { 1179 db->db_buf = NULL; 1180 dbuf_clear_data(db); 1181 } 1182 } 1183 1184 void 1185 dbuf_unoverride(dbuf_dirty_record_t *dr) 1186 { 1187 dmu_buf_impl_t *db = dr->dr_dbuf; 1188 blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 1189 uint64_t txg = dr->dr_txg; 1190 1191 ASSERT(MUTEX_HELD(&db->db_mtx)); 1192 ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 1193 ASSERT(db->db_level == 0); 1194 1195 if (db->db_blkid == DMU_BONUS_BLKID || 1196 dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 1197 return; 1198 1199 ASSERT(db->db_data_pending != dr); 1200 1201 /* free this block */ 1202 if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite) 1203 zio_free(db->db_objset->os_spa, txg, bp); 1204 1205 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 1206 dr->dt.dl.dr_nopwrite = B_FALSE; 1207 1208 /* 1209 * Release the already-written buffer, so we leave it in 1210 * a consistent dirty state. Note that all callers are 1211 * modifying the buffer, so they will immediately do 1212 * another (redundant) arc_release(). Therefore, leave 1213 * the buf thawed to save the effort of freezing & 1214 * immediately re-thawing it. 1215 */ 1216 arc_release(dr->dt.dl.dr_data, db); 1217 } 1218 1219 /* 1220 * Evict (if its unreferenced) or clear (if its referenced) any level-0 1221 * data blocks in the free range, so that any future readers will find 1222 * empty blocks. 1223 * 1224 * This is a no-op if the dataset is in the middle of an incremental 1225 * receive; see comment below for details. 1226 */ 1227 void 1228 dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid, 1229 dmu_tx_t *tx) 1230 { 1231 dmu_buf_impl_t db_search; 1232 dmu_buf_impl_t *db, *db_next; 1233 uint64_t txg = tx->tx_txg; 1234 avl_index_t where; 1235 boolean_t freespill = 1236 (start_blkid == DMU_SPILL_BLKID || end_blkid == DMU_SPILL_BLKID); 1237 1238 if (end_blkid > dn->dn_maxblkid && !freespill) 1239 end_blkid = dn->dn_maxblkid; 1240 dprintf_dnode(dn, "start=%llu end=%llu\n", start_blkid, end_blkid); 1241 1242 db_search.db_level = 0; 1243 db_search.db_blkid = start_blkid; 1244 db_search.db_state = DB_SEARCH; 1245 1246 mutex_enter(&dn->dn_dbufs_mtx); 1247 if (start_blkid >= dn->dn_unlisted_l0_blkid && !freespill) { 1248 /* There can't be any dbufs in this range; no need to search. */ 1249 #ifdef DEBUG 1250 db = avl_find(&dn->dn_dbufs, &db_search, &where); 1251 ASSERT3P(db, ==, NULL); 1252 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); 1253 ASSERT(db == NULL || db->db_level > 0); 1254 #endif 1255 mutex_exit(&dn->dn_dbufs_mtx); 1256 return; 1257 } else if (dmu_objset_is_receiving(dn->dn_objset)) { 1258 /* 1259 * If we are receiving, we expect there to be no dbufs in 1260 * the range to be freed, because receive modifies each 1261 * block at most once, and in offset order. If this is 1262 * not the case, it can lead to performance problems, 1263 * so note that we unexpectedly took the slow path. 1264 */ 1265 atomic_inc_64(&zfs_free_range_recv_miss); 1266 } 1267 1268 db = avl_find(&dn->dn_dbufs, &db_search, &where); 1269 ASSERT3P(db, ==, NULL); 1270 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); 1271 1272 for (; db != NULL; db = db_next) { 1273 db_next = AVL_NEXT(&dn->dn_dbufs, db); 1274 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1275 1276 if (db->db_level != 0 || db->db_blkid > end_blkid) { 1277 break; 1278 } 1279 ASSERT3U(db->db_blkid, >=, start_blkid); 1280 1281 /* found a level 0 buffer in the range */ 1282 mutex_enter(&db->db_mtx); 1283 if (dbuf_undirty(db, tx)) { 1284 /* mutex has been dropped and dbuf destroyed */ 1285 continue; 1286 } 1287 1288 if (db->db_state == DB_UNCACHED || 1289 db->db_state == DB_NOFILL || 1290 db->db_state == DB_EVICTING) { 1291 ASSERT(db->db.db_data == NULL); 1292 mutex_exit(&db->db_mtx); 1293 continue; 1294 } 1295 if (db->db_state == DB_READ || db->db_state == DB_FILL) { 1296 /* will be handled in dbuf_read_done or dbuf_rele */ 1297 db->db_freed_in_flight = TRUE; 1298 mutex_exit(&db->db_mtx); 1299 continue; 1300 } 1301 if (refcount_count(&db->db_holds) == 0) { 1302 ASSERT(db->db_buf); 1303 dbuf_destroy(db); 1304 continue; 1305 } 1306 /* The dbuf is referenced */ 1307 1308 if (db->db_last_dirty != NULL) { 1309 dbuf_dirty_record_t *dr = db->db_last_dirty; 1310 1311 if (dr->dr_txg == txg) { 1312 /* 1313 * This buffer is "in-use", re-adjust the file 1314 * size to reflect that this buffer may 1315 * contain new data when we sync. 1316 */ 1317 if (db->db_blkid != DMU_SPILL_BLKID && 1318 db->db_blkid > dn->dn_maxblkid) 1319 dn->dn_maxblkid = db->db_blkid; 1320 dbuf_unoverride(dr); 1321 } else { 1322 /* 1323 * This dbuf is not dirty in the open context. 1324 * Either uncache it (if its not referenced in 1325 * the open context) or reset its contents to 1326 * empty. 1327 */ 1328 dbuf_fix_old_data(db, txg); 1329 } 1330 } 1331 /* clear the contents if its cached */ 1332 if (db->db_state == DB_CACHED) { 1333 ASSERT(db->db.db_data != NULL); 1334 arc_release(db->db_buf, db); 1335 bzero(db->db.db_data, db->db.db_size); 1336 arc_buf_freeze(db->db_buf); 1337 } 1338 1339 mutex_exit(&db->db_mtx); 1340 } 1341 mutex_exit(&dn->dn_dbufs_mtx); 1342 } 1343 1344 static int 1345 dbuf_block_freeable(dmu_buf_impl_t *db) 1346 { 1347 dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 1348 uint64_t birth_txg = 0; 1349 1350 /* 1351 * We don't need any locking to protect db_blkptr: 1352 * If it's syncing, then db_last_dirty will be set 1353 * so we'll ignore db_blkptr. 1354 * 1355 * This logic ensures that only block births for 1356 * filled blocks are considered. 1357 */ 1358 ASSERT(MUTEX_HELD(&db->db_mtx)); 1359 if (db->db_last_dirty && (db->db_blkptr == NULL || 1360 !BP_IS_HOLE(db->db_blkptr))) { 1361 birth_txg = db->db_last_dirty->dr_txg; 1362 } else if (db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) { 1363 birth_txg = db->db_blkptr->blk_birth; 1364 } 1365 1366 /* 1367 * If this block don't exist or is in a snapshot, it can't be freed. 1368 * Don't pass the bp to dsl_dataset_block_freeable() since we 1369 * are holding the db_mtx lock and might deadlock if we are 1370 * prefetching a dedup-ed block. 1371 */ 1372 if (birth_txg != 0) 1373 return (ds == NULL || 1374 dsl_dataset_block_freeable(ds, NULL, birth_txg)); 1375 else 1376 return (B_FALSE); 1377 } 1378 1379 void 1380 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 1381 { 1382 arc_buf_t *buf, *obuf; 1383 int osize = db->db.db_size; 1384 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1385 dnode_t *dn; 1386 1387 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1388 1389 DB_DNODE_ENTER(db); 1390 dn = DB_DNODE(db); 1391 1392 /* XXX does *this* func really need the lock? */ 1393 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1394 1395 /* 1396 * This call to dmu_buf_will_dirty() with the dn_struct_rwlock held 1397 * is OK, because there can be no other references to the db 1398 * when we are changing its size, so no concurrent DB_FILL can 1399 * be happening. 1400 */ 1401 /* 1402 * XXX we should be doing a dbuf_read, checking the return 1403 * value and returning that up to our callers 1404 */ 1405 dmu_buf_will_dirty(&db->db, tx); 1406 1407 /* create the data buffer for the new block */ 1408 buf = arc_alloc_buf(dn->dn_objset->os_spa, size, db, type); 1409 1410 /* copy old block data to the new block */ 1411 obuf = db->db_buf; 1412 bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 1413 /* zero the remainder */ 1414 if (size > osize) 1415 bzero((uint8_t *)buf->b_data + osize, size - osize); 1416 1417 mutex_enter(&db->db_mtx); 1418 dbuf_set_data(db, buf); 1419 arc_buf_destroy(obuf, db); 1420 db->db.db_size = size; 1421 1422 if (db->db_level == 0) { 1423 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 1424 db->db_last_dirty->dt.dl.dr_data = buf; 1425 } 1426 mutex_exit(&db->db_mtx); 1427 1428 dnode_willuse_space(dn, size-osize, tx); 1429 DB_DNODE_EXIT(db); 1430 } 1431 1432 void 1433 dbuf_release_bp(dmu_buf_impl_t *db) 1434 { 1435 objset_t *os = db->db_objset; 1436 1437 ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 1438 ASSERT(arc_released(os->os_phys_buf) || 1439 list_link_active(&os->os_dsl_dataset->ds_synced_link)); 1440 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 1441 1442 (void) arc_release(db->db_buf, db); 1443 } 1444 1445 /* 1446 * We already have a dirty record for this TXG, and we are being 1447 * dirtied again. 1448 */ 1449 static void 1450 dbuf_redirty(dbuf_dirty_record_t *dr) 1451 { 1452 dmu_buf_impl_t *db = dr->dr_dbuf; 1453 1454 ASSERT(MUTEX_HELD(&db->db_mtx)); 1455 1456 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 1457 /* 1458 * If this buffer has already been written out, 1459 * we now need to reset its state. 1460 */ 1461 dbuf_unoverride(dr); 1462 if (db->db.db_object != DMU_META_DNODE_OBJECT && 1463 db->db_state != DB_NOFILL) { 1464 /* Already released on initial dirty, so just thaw. */ 1465 ASSERT(arc_released(db->db_buf)); 1466 arc_buf_thaw(db->db_buf); 1467 } 1468 } 1469 } 1470 1471 dbuf_dirty_record_t * 1472 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1473 { 1474 dnode_t *dn; 1475 objset_t *os; 1476 dbuf_dirty_record_t **drp, *dr; 1477 int drop_struct_lock = FALSE; 1478 boolean_t do_free_accounting = B_FALSE; 1479 int txgoff = tx->tx_txg & TXG_MASK; 1480 1481 ASSERT(tx->tx_txg != 0); 1482 ASSERT(!refcount_is_zero(&db->db_holds)); 1483 DMU_TX_DIRTY_BUF(tx, db); 1484 1485 DB_DNODE_ENTER(db); 1486 dn = DB_DNODE(db); 1487 /* 1488 * Shouldn't dirty a regular buffer in syncing context. Private 1489 * objects may be dirtied in syncing context, but only if they 1490 * were already pre-dirtied in open context. 1491 */ 1492 ASSERT(!dmu_tx_is_syncing(tx) || 1493 BP_IS_HOLE(dn->dn_objset->os_rootbp) || 1494 DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1495 dn->dn_objset->os_dsl_dataset == NULL); 1496 /* 1497 * We make this assert for private objects as well, but after we 1498 * check if we're already dirty. They are allowed to re-dirty 1499 * in syncing context. 1500 */ 1501 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1502 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1503 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1504 1505 mutex_enter(&db->db_mtx); 1506 /* 1507 * XXX make this true for indirects too? The problem is that 1508 * transactions created with dmu_tx_create_assigned() from 1509 * syncing context don't bother holding ahead. 1510 */ 1511 ASSERT(db->db_level != 0 || 1512 db->db_state == DB_CACHED || db->db_state == DB_FILL || 1513 db->db_state == DB_NOFILL); 1514 1515 mutex_enter(&dn->dn_mtx); 1516 /* 1517 * Don't set dirtyctx to SYNC if we're just modifying this as we 1518 * initialize the objset. 1519 */ 1520 if (dn->dn_dirtyctx == DN_UNDIRTIED && 1521 !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 1522 dn->dn_dirtyctx = 1523 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 1524 ASSERT(dn->dn_dirtyctx_firstset == NULL); 1525 dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 1526 } 1527 mutex_exit(&dn->dn_mtx); 1528 1529 if (db->db_blkid == DMU_SPILL_BLKID) 1530 dn->dn_have_spill = B_TRUE; 1531 1532 /* 1533 * If this buffer is already dirty, we're done. 1534 */ 1535 drp = &db->db_last_dirty; 1536 ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 1537 db->db.db_object == DMU_META_DNODE_OBJECT); 1538 while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 1539 drp = &dr->dr_next; 1540 if (dr && dr->dr_txg == tx->tx_txg) { 1541 DB_DNODE_EXIT(db); 1542 1543 dbuf_redirty(dr); 1544 mutex_exit(&db->db_mtx); 1545 return (dr); 1546 } 1547 1548 /* 1549 * Only valid if not already dirty. 1550 */ 1551 ASSERT(dn->dn_object == 0 || 1552 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1553 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1554 1555 ASSERT3U(dn->dn_nlevels, >, db->db_level); 1556 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1557 dn->dn_phys->dn_nlevels > db->db_level || 1558 dn->dn_next_nlevels[txgoff] > db->db_level || 1559 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1560 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1561 1562 /* 1563 * We should only be dirtying in syncing context if it's the 1564 * mos or we're initializing the os or it's a special object. 1565 * However, we are allowed to dirty in syncing context provided 1566 * we already dirtied it in open context. Hence we must make 1567 * this assertion only if we're not already dirty. 1568 */ 1569 os = dn->dn_objset; 1570 ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1571 os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1572 ASSERT(db->db.db_size != 0); 1573 1574 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1575 1576 if (db->db_blkid != DMU_BONUS_BLKID) { 1577 /* 1578 * Update the accounting. 1579 * Note: we delay "free accounting" until after we drop 1580 * the db_mtx. This keeps us from grabbing other locks 1581 * (and possibly deadlocking) in bp_get_dsize() while 1582 * also holding the db_mtx. 1583 */ 1584 dnode_willuse_space(dn, db->db.db_size, tx); 1585 do_free_accounting = dbuf_block_freeable(db); 1586 } 1587 1588 /* 1589 * If this buffer is dirty in an old transaction group we need 1590 * to make a copy of it so that the changes we make in this 1591 * transaction group won't leak out when we sync the older txg. 1592 */ 1593 dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1594 if (db->db_level == 0) { 1595 void *data_old = db->db_buf; 1596 1597 if (db->db_state != DB_NOFILL) { 1598 if (db->db_blkid == DMU_BONUS_BLKID) { 1599 dbuf_fix_old_data(db, tx->tx_txg); 1600 data_old = db->db.db_data; 1601 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1602 /* 1603 * Release the data buffer from the cache so 1604 * that we can modify it without impacting 1605 * possible other users of this cached data 1606 * block. Note that indirect blocks and 1607 * private objects are not released until the 1608 * syncing state (since they are only modified 1609 * then). 1610 */ 1611 arc_release(db->db_buf, db); 1612 dbuf_fix_old_data(db, tx->tx_txg); 1613 data_old = db->db_buf; 1614 } 1615 ASSERT(data_old != NULL); 1616 } 1617 dr->dt.dl.dr_data = data_old; 1618 } else { 1619 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1620 list_create(&dr->dt.di.dr_children, 1621 sizeof (dbuf_dirty_record_t), 1622 offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1623 } 1624 if (db->db_blkid != DMU_BONUS_BLKID && os->os_dsl_dataset != NULL) 1625 dr->dr_accounted = db->db.db_size; 1626 dr->dr_dbuf = db; 1627 dr->dr_txg = tx->tx_txg; 1628 dr->dr_next = *drp; 1629 *drp = dr; 1630 1631 /* 1632 * We could have been freed_in_flight between the dbuf_noread 1633 * and dbuf_dirty. We win, as though the dbuf_noread() had 1634 * happened after the free. 1635 */ 1636 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1637 db->db_blkid != DMU_SPILL_BLKID) { 1638 mutex_enter(&dn->dn_mtx); 1639 if (dn->dn_free_ranges[txgoff] != NULL) { 1640 range_tree_clear(dn->dn_free_ranges[txgoff], 1641 db->db_blkid, 1); 1642 } 1643 mutex_exit(&dn->dn_mtx); 1644 db->db_freed_in_flight = FALSE; 1645 } 1646 1647 /* 1648 * This buffer is now part of this txg 1649 */ 1650 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1651 db->db_dirtycnt += 1; 1652 ASSERT3U(db->db_dirtycnt, <=, 3); 1653 1654 mutex_exit(&db->db_mtx); 1655 1656 if (db->db_blkid == DMU_BONUS_BLKID || 1657 db->db_blkid == DMU_SPILL_BLKID) { 1658 mutex_enter(&dn->dn_mtx); 1659 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1660 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1661 mutex_exit(&dn->dn_mtx); 1662 dnode_setdirty(dn, tx); 1663 DB_DNODE_EXIT(db); 1664 return (dr); 1665 } else if (do_free_accounting) { 1666 blkptr_t *bp = db->db_blkptr; 1667 int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1668 bp_get_dsize(os->os_spa, bp) : db->db.db_size; 1669 /* 1670 * This is only a guess -- if the dbuf is dirty 1671 * in a previous txg, we don't know how much 1672 * space it will use on disk yet. We should 1673 * really have the struct_rwlock to access 1674 * db_blkptr, but since this is just a guess, 1675 * it's OK if we get an odd answer. 1676 */ 1677 ddt_prefetch(os->os_spa, bp); 1678 dnode_willuse_space(dn, -willfree, tx); 1679 } 1680 1681 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1682 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1683 drop_struct_lock = TRUE; 1684 } 1685 1686 if (db->db_level == 0) { 1687 dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 1688 ASSERT(dn->dn_maxblkid >= db->db_blkid); 1689 } 1690 1691 if (db->db_level+1 < dn->dn_nlevels) { 1692 dmu_buf_impl_t *parent = db->db_parent; 1693 dbuf_dirty_record_t *di; 1694 int parent_held = FALSE; 1695 1696 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1697 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1698 1699 parent = dbuf_hold_level(dn, db->db_level+1, 1700 db->db_blkid >> epbs, FTAG); 1701 ASSERT(parent != NULL); 1702 parent_held = TRUE; 1703 } 1704 if (drop_struct_lock) 1705 rw_exit(&dn->dn_struct_rwlock); 1706 ASSERT3U(db->db_level+1, ==, parent->db_level); 1707 di = dbuf_dirty(parent, tx); 1708 if (parent_held) 1709 dbuf_rele(parent, FTAG); 1710 1711 mutex_enter(&db->db_mtx); 1712 /* 1713 * Since we've dropped the mutex, it's possible that 1714 * dbuf_undirty() might have changed this out from under us. 1715 */ 1716 if (db->db_last_dirty == dr || 1717 dn->dn_object == DMU_META_DNODE_OBJECT) { 1718 mutex_enter(&di->dt.di.dr_mtx); 1719 ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1720 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1721 list_insert_tail(&di->dt.di.dr_children, dr); 1722 mutex_exit(&di->dt.di.dr_mtx); 1723 dr->dr_parent = di; 1724 } 1725 mutex_exit(&db->db_mtx); 1726 } else { 1727 ASSERT(db->db_level+1 == dn->dn_nlevels); 1728 ASSERT(db->db_blkid < dn->dn_nblkptr); 1729 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); 1730 mutex_enter(&dn->dn_mtx); 1731 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1732 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1733 mutex_exit(&dn->dn_mtx); 1734 if (drop_struct_lock) 1735 rw_exit(&dn->dn_struct_rwlock); 1736 } 1737 1738 dnode_setdirty(dn, tx); 1739 DB_DNODE_EXIT(db); 1740 return (dr); 1741 } 1742 1743 /* 1744 * Undirty a buffer in the transaction group referenced by the given 1745 * transaction. Return whether this evicted the dbuf. 1746 */ 1747 static boolean_t 1748 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1749 { 1750 dnode_t *dn; 1751 uint64_t txg = tx->tx_txg; 1752 dbuf_dirty_record_t *dr, **drp; 1753 1754 ASSERT(txg != 0); 1755 1756 /* 1757 * Due to our use of dn_nlevels below, this can only be called 1758 * in open context, unless we are operating on the MOS. 1759 * From syncing context, dn_nlevels may be different from the 1760 * dn_nlevels used when dbuf was dirtied. 1761 */ 1762 ASSERT(db->db_objset == 1763 dmu_objset_pool(db->db_objset)->dp_meta_objset || 1764 txg != spa_syncing_txg(dmu_objset_spa(db->db_objset))); 1765 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1766 ASSERT0(db->db_level); 1767 ASSERT(MUTEX_HELD(&db->db_mtx)); 1768 1769 /* 1770 * If this buffer is not dirty, we're done. 1771 */ 1772 for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1773 if (dr->dr_txg <= txg) 1774 break; 1775 if (dr == NULL || dr->dr_txg < txg) 1776 return (B_FALSE); 1777 ASSERT(dr->dr_txg == txg); 1778 ASSERT(dr->dr_dbuf == db); 1779 1780 DB_DNODE_ENTER(db); 1781 dn = DB_DNODE(db); 1782 1783 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1784 1785 ASSERT(db->db.db_size != 0); 1786 1787 dsl_pool_undirty_space(dmu_objset_pool(dn->dn_objset), 1788 dr->dr_accounted, txg); 1789 1790 *drp = dr->dr_next; 1791 1792 /* 1793 * Note that there are three places in dbuf_dirty() 1794 * where this dirty record may be put on a list. 1795 * Make sure to do a list_remove corresponding to 1796 * every one of those list_insert calls. 1797 */ 1798 if (dr->dr_parent) { 1799 mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1800 list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1801 mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1802 } else if (db->db_blkid == DMU_SPILL_BLKID || 1803 db->db_level + 1 == dn->dn_nlevels) { 1804 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1805 mutex_enter(&dn->dn_mtx); 1806 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1807 mutex_exit(&dn->dn_mtx); 1808 } 1809 DB_DNODE_EXIT(db); 1810 1811 if (db->db_state != DB_NOFILL) { 1812 dbuf_unoverride(dr); 1813 1814 ASSERT(db->db_buf != NULL); 1815 ASSERT(dr->dt.dl.dr_data != NULL); 1816 if (dr->dt.dl.dr_data != db->db_buf) 1817 arc_buf_destroy(dr->dt.dl.dr_data, db); 1818 } 1819 1820 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1821 1822 ASSERT(db->db_dirtycnt > 0); 1823 db->db_dirtycnt -= 1; 1824 1825 if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1826 ASSERT(db->db_state == DB_NOFILL || arc_released(db->db_buf)); 1827 dbuf_destroy(db); 1828 return (B_TRUE); 1829 } 1830 1831 return (B_FALSE); 1832 } 1833 1834 void 1835 dmu_buf_will_dirty(dmu_buf_t *db_fake, dmu_tx_t *tx) 1836 { 1837 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1838 int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1839 1840 ASSERT(tx->tx_txg != 0); 1841 ASSERT(!refcount_is_zero(&db->db_holds)); 1842 1843 /* 1844 * Quick check for dirtyness. For already dirty blocks, this 1845 * reduces runtime of this function by >90%, and overall performance 1846 * by 50% for some workloads (e.g. file deletion with indirect blocks 1847 * cached). 1848 */ 1849 mutex_enter(&db->db_mtx); 1850 dbuf_dirty_record_t *dr; 1851 for (dr = db->db_last_dirty; 1852 dr != NULL && dr->dr_txg >= tx->tx_txg; dr = dr->dr_next) { 1853 /* 1854 * It's possible that it is already dirty but not cached, 1855 * because there are some calls to dbuf_dirty() that don't 1856 * go through dmu_buf_will_dirty(). 1857 */ 1858 if (dr->dr_txg == tx->tx_txg && db->db_state == DB_CACHED) { 1859 /* This dbuf is already dirty and cached. */ 1860 dbuf_redirty(dr); 1861 mutex_exit(&db->db_mtx); 1862 return; 1863 } 1864 } 1865 mutex_exit(&db->db_mtx); 1866 1867 DB_DNODE_ENTER(db); 1868 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) 1869 rf |= DB_RF_HAVESTRUCT; 1870 DB_DNODE_EXIT(db); 1871 (void) dbuf_read(db, NULL, rf); 1872 (void) dbuf_dirty(db, tx); 1873 } 1874 1875 void 1876 dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1877 { 1878 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1879 1880 db->db_state = DB_NOFILL; 1881 1882 dmu_buf_will_fill(db_fake, tx); 1883 } 1884 1885 void 1886 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1887 { 1888 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1889 1890 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1891 ASSERT(tx->tx_txg != 0); 1892 ASSERT(db->db_level == 0); 1893 ASSERT(!refcount_is_zero(&db->db_holds)); 1894 1895 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1896 dmu_tx_private_ok(tx)); 1897 1898 dbuf_noread(db); 1899 (void) dbuf_dirty(db, tx); 1900 } 1901 1902 #pragma weak dmu_buf_fill_done = dbuf_fill_done 1903 /* ARGSUSED */ 1904 void 1905 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1906 { 1907 mutex_enter(&db->db_mtx); 1908 DBUF_VERIFY(db); 1909 1910 if (db->db_state == DB_FILL) { 1911 if (db->db_level == 0 && db->db_freed_in_flight) { 1912 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1913 /* we were freed while filling */ 1914 /* XXX dbuf_undirty? */ 1915 bzero(db->db.db_data, db->db.db_size); 1916 db->db_freed_in_flight = FALSE; 1917 } 1918 db->db_state = DB_CACHED; 1919 cv_broadcast(&db->db_changed); 1920 } 1921 mutex_exit(&db->db_mtx); 1922 } 1923 1924 void 1925 dmu_buf_write_embedded(dmu_buf_t *dbuf, void *data, 1926 bp_embedded_type_t etype, enum zio_compress comp, 1927 int uncompressed_size, int compressed_size, int byteorder, 1928 dmu_tx_t *tx) 1929 { 1930 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 1931 struct dirty_leaf *dl; 1932 dmu_object_type_t type; 1933 1934 if (etype == BP_EMBEDDED_TYPE_DATA) { 1935 ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset), 1936 SPA_FEATURE_EMBEDDED_DATA)); 1937 } 1938 1939 DB_DNODE_ENTER(db); 1940 type = DB_DNODE(db)->dn_type; 1941 DB_DNODE_EXIT(db); 1942 1943 ASSERT0(db->db_level); 1944 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1945 1946 dmu_buf_will_not_fill(dbuf, tx); 1947 1948 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 1949 dl = &db->db_last_dirty->dt.dl; 1950 encode_embedded_bp_compressed(&dl->dr_overridden_by, 1951 data, comp, uncompressed_size, compressed_size); 1952 BPE_SET_ETYPE(&dl->dr_overridden_by, etype); 1953 BP_SET_TYPE(&dl->dr_overridden_by, type); 1954 BP_SET_LEVEL(&dl->dr_overridden_by, 0); 1955 BP_SET_BYTEORDER(&dl->dr_overridden_by, byteorder); 1956 1957 dl->dr_override_state = DR_OVERRIDDEN; 1958 dl->dr_overridden_by.blk_birth = db->db_last_dirty->dr_txg; 1959 } 1960 1961 /* 1962 * Directly assign a provided arc buf to a given dbuf if it's not referenced 1963 * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 1964 */ 1965 void 1966 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 1967 { 1968 ASSERT(!refcount_is_zero(&db->db_holds)); 1969 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1970 ASSERT(db->db_level == 0); 1971 ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 1972 ASSERT(buf != NULL); 1973 ASSERT(arc_buf_size(buf) == db->db.db_size); 1974 ASSERT(tx->tx_txg != 0); 1975 1976 arc_return_buf(buf, db); 1977 ASSERT(arc_released(buf)); 1978 1979 mutex_enter(&db->db_mtx); 1980 1981 while (db->db_state == DB_READ || db->db_state == DB_FILL) 1982 cv_wait(&db->db_changed, &db->db_mtx); 1983 1984 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 1985 1986 if (db->db_state == DB_CACHED && 1987 refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 1988 mutex_exit(&db->db_mtx); 1989 (void) dbuf_dirty(db, tx); 1990 bcopy(buf->b_data, db->db.db_data, db->db.db_size); 1991 arc_buf_destroy(buf, db); 1992 xuio_stat_wbuf_copied(); 1993 return; 1994 } 1995 1996 xuio_stat_wbuf_nocopy(); 1997 if (db->db_state == DB_CACHED) { 1998 dbuf_dirty_record_t *dr = db->db_last_dirty; 1999 2000 ASSERT(db->db_buf != NULL); 2001 if (dr != NULL && dr->dr_txg == tx->tx_txg) { 2002 ASSERT(dr->dt.dl.dr_data == db->db_buf); 2003 if (!arc_released(db->db_buf)) { 2004 ASSERT(dr->dt.dl.dr_override_state == 2005 DR_OVERRIDDEN); 2006 arc_release(db->db_buf, db); 2007 } 2008 dr->dt.dl.dr_data = buf; 2009 arc_buf_destroy(db->db_buf, db); 2010 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 2011 arc_release(db->db_buf, db); 2012 arc_buf_destroy(db->db_buf, db); 2013 } 2014 db->db_buf = NULL; 2015 } 2016 ASSERT(db->db_buf == NULL); 2017 dbuf_set_data(db, buf); 2018 db->db_state = DB_FILL; 2019 mutex_exit(&db->db_mtx); 2020 (void) dbuf_dirty(db, tx); 2021 dmu_buf_fill_done(&db->db, tx); 2022 } 2023 2024 void 2025 dbuf_destroy(dmu_buf_impl_t *db) 2026 { 2027 dnode_t *dn; 2028 dmu_buf_impl_t *parent = db->db_parent; 2029 dmu_buf_impl_t *dndb; 2030 2031 ASSERT(MUTEX_HELD(&db->db_mtx)); 2032 ASSERT(refcount_is_zero(&db->db_holds)); 2033 2034 if (db->db_buf != NULL) { 2035 arc_buf_destroy(db->db_buf, db); 2036 db->db_buf = NULL; 2037 } 2038 2039 if (db->db_blkid == DMU_BONUS_BLKID) { 2040 ASSERT(db->db.db_data != NULL); 2041 zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 2042 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 2043 db->db_state = DB_UNCACHED; 2044 } 2045 2046 dbuf_clear_data(db); 2047 2048 if (multilist_link_active(&db->db_cache_link)) { 2049 multilist_remove(&dbuf_cache, db); 2050 (void) refcount_remove_many(&dbuf_cache_size, 2051 db->db.db_size, db); 2052 } 2053 2054 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 2055 ASSERT(db->db_data_pending == NULL); 2056 2057 db->db_state = DB_EVICTING; 2058 db->db_blkptr = NULL; 2059 2060 /* 2061 * Now that db_state is DB_EVICTING, nobody else can find this via 2062 * the hash table. We can now drop db_mtx, which allows us to 2063 * acquire the dn_dbufs_mtx. 2064 */ 2065 mutex_exit(&db->db_mtx); 2066 2067 DB_DNODE_ENTER(db); 2068 dn = DB_DNODE(db); 2069 dndb = dn->dn_dbuf; 2070 if (db->db_blkid != DMU_BONUS_BLKID) { 2071 boolean_t needlock = !MUTEX_HELD(&dn->dn_dbufs_mtx); 2072 if (needlock) 2073 mutex_enter(&dn->dn_dbufs_mtx); 2074 avl_remove(&dn->dn_dbufs, db); 2075 atomic_dec_32(&dn->dn_dbufs_count); 2076 membar_producer(); 2077 DB_DNODE_EXIT(db); 2078 if (needlock) 2079 mutex_exit(&dn->dn_dbufs_mtx); 2080 /* 2081 * Decrementing the dbuf count means that the hold corresponding 2082 * to the removed dbuf is no longer discounted in dnode_move(), 2083 * so the dnode cannot be moved until after we release the hold. 2084 * The membar_producer() ensures visibility of the decremented 2085 * value in dnode_move(), since DB_DNODE_EXIT doesn't actually 2086 * release any lock. 2087 */ 2088 dnode_rele(dn, db); 2089 db->db_dnode_handle = NULL; 2090 2091 dbuf_hash_remove(db); 2092 } else { 2093 DB_DNODE_EXIT(db); 2094 } 2095 2096 ASSERT(refcount_is_zero(&db->db_holds)); 2097 2098 db->db_parent = NULL; 2099 2100 ASSERT(db->db_buf == NULL); 2101 ASSERT(db->db.db_data == NULL); 2102 ASSERT(db->db_hash_next == NULL); 2103 ASSERT(db->db_blkptr == NULL); 2104 ASSERT(db->db_data_pending == NULL); 2105 ASSERT(!multilist_link_active(&db->db_cache_link)); 2106 2107 kmem_cache_free(dbuf_kmem_cache, db); 2108 arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 2109 2110 /* 2111 * If this dbuf is referenced from an indirect dbuf, 2112 * decrement the ref count on the indirect dbuf. 2113 */ 2114 if (parent && parent != dndb) 2115 dbuf_rele(parent, db); 2116 } 2117 2118 /* 2119 * Note: While bpp will always be updated if the function returns success, 2120 * parentp will not be updated if the dnode does not have dn_dbuf filled in; 2121 * this happens when the dnode is the meta-dnode, or a userused or groupused 2122 * object. 2123 */ 2124 static int 2125 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 2126 dmu_buf_impl_t **parentp, blkptr_t **bpp) 2127 { 2128 int nlevels, epbs; 2129 2130 *parentp = NULL; 2131 *bpp = NULL; 2132 2133 ASSERT(blkid != DMU_BONUS_BLKID); 2134 2135 if (blkid == DMU_SPILL_BLKID) { 2136 mutex_enter(&dn->dn_mtx); 2137 if (dn->dn_have_spill && 2138 (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 2139 *bpp = &dn->dn_phys->dn_spill; 2140 else 2141 *bpp = NULL; 2142 dbuf_add_ref(dn->dn_dbuf, NULL); 2143 *parentp = dn->dn_dbuf; 2144 mutex_exit(&dn->dn_mtx); 2145 return (0); 2146 } 2147 2148 if (dn->dn_phys->dn_nlevels == 0) 2149 nlevels = 1; 2150 else 2151 nlevels = dn->dn_phys->dn_nlevels; 2152 2153 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 2154 2155 ASSERT3U(level * epbs, <, 64); 2156 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2157 if (level >= nlevels || 2158 (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 2159 /* the buffer has no parent yet */ 2160 return (SET_ERROR(ENOENT)); 2161 } else if (level < nlevels-1) { 2162 /* this block is referenced from an indirect block */ 2163 int err = dbuf_hold_impl(dn, level+1, 2164 blkid >> epbs, fail_sparse, FALSE, NULL, parentp); 2165 if (err) 2166 return (err); 2167 err = dbuf_read(*parentp, NULL, 2168 (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 2169 if (err) { 2170 dbuf_rele(*parentp, NULL); 2171 *parentp = NULL; 2172 return (err); 2173 } 2174 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 2175 (blkid & ((1ULL << epbs) - 1)); 2176 return (0); 2177 } else { 2178 /* the block is referenced from the dnode */ 2179 ASSERT3U(level, ==, nlevels-1); 2180 ASSERT(dn->dn_phys->dn_nblkptr == 0 || 2181 blkid < dn->dn_phys->dn_nblkptr); 2182 if (dn->dn_dbuf) { 2183 dbuf_add_ref(dn->dn_dbuf, NULL); 2184 *parentp = dn->dn_dbuf; 2185 } 2186 *bpp = &dn->dn_phys->dn_blkptr[blkid]; 2187 return (0); 2188 } 2189 } 2190 2191 static dmu_buf_impl_t * 2192 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 2193 dmu_buf_impl_t *parent, blkptr_t *blkptr) 2194 { 2195 objset_t *os = dn->dn_objset; 2196 dmu_buf_impl_t *db, *odb; 2197 2198 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2199 ASSERT(dn->dn_type != DMU_OT_NONE); 2200 2201 db = kmem_cache_alloc(dbuf_kmem_cache, KM_SLEEP); 2202 2203 db->db_objset = os; 2204 db->db.db_object = dn->dn_object; 2205 db->db_level = level; 2206 db->db_blkid = blkid; 2207 db->db_last_dirty = NULL; 2208 db->db_dirtycnt = 0; 2209 db->db_dnode_handle = dn->dn_handle; 2210 db->db_parent = parent; 2211 db->db_blkptr = blkptr; 2212 2213 db->db_user = NULL; 2214 db->db_user_immediate_evict = FALSE; 2215 db->db_freed_in_flight = FALSE; 2216 db->db_pending_evict = FALSE; 2217 2218 if (blkid == DMU_BONUS_BLKID) { 2219 ASSERT3P(parent, ==, dn->dn_dbuf); 2220 db->db.db_size = DN_MAX_BONUSLEN - 2221 (dn->dn_nblkptr-1) * sizeof (blkptr_t); 2222 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 2223 db->db.db_offset = DMU_BONUS_BLKID; 2224 db->db_state = DB_UNCACHED; 2225 /* the bonus dbuf is not placed in the hash table */ 2226 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 2227 return (db); 2228 } else if (blkid == DMU_SPILL_BLKID) { 2229 db->db.db_size = (blkptr != NULL) ? 2230 BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 2231 db->db.db_offset = 0; 2232 } else { 2233 int blocksize = 2234 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz; 2235 db->db.db_size = blocksize; 2236 db->db.db_offset = db->db_blkid * blocksize; 2237 } 2238 2239 /* 2240 * Hold the dn_dbufs_mtx while we get the new dbuf 2241 * in the hash table *and* added to the dbufs list. 2242 * This prevents a possible deadlock with someone 2243 * trying to look up this dbuf before its added to the 2244 * dn_dbufs list. 2245 */ 2246 mutex_enter(&dn->dn_dbufs_mtx); 2247 db->db_state = DB_EVICTING; 2248 if ((odb = dbuf_hash_insert(db)) != NULL) { 2249 /* someone else inserted it first */ 2250 kmem_cache_free(dbuf_kmem_cache, db); 2251 mutex_exit(&dn->dn_dbufs_mtx); 2252 return (odb); 2253 } 2254 avl_add(&dn->dn_dbufs, db); 2255 if (db->db_level == 0 && db->db_blkid >= 2256 dn->dn_unlisted_l0_blkid) 2257 dn->dn_unlisted_l0_blkid = db->db_blkid + 1; 2258 db->db_state = DB_UNCACHED; 2259 mutex_exit(&dn->dn_dbufs_mtx); 2260 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 2261 2262 if (parent && parent != dn->dn_dbuf) 2263 dbuf_add_ref(parent, db); 2264 2265 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 2266 refcount_count(&dn->dn_holds) > 0); 2267 (void) refcount_add(&dn->dn_holds, db); 2268 atomic_inc_32(&dn->dn_dbufs_count); 2269 2270 dprintf_dbuf(db, "db=%p\n", db); 2271 2272 return (db); 2273 } 2274 2275 typedef struct dbuf_prefetch_arg { 2276 spa_t *dpa_spa; /* The spa to issue the prefetch in. */ 2277 zbookmark_phys_t dpa_zb; /* The target block to prefetch. */ 2278 int dpa_epbs; /* Entries (blkptr_t's) Per Block Shift. */ 2279 int dpa_curlevel; /* The current level that we're reading */ 2280 dnode_t *dpa_dnode; /* The dnode associated with the prefetch */ 2281 zio_priority_t dpa_prio; /* The priority I/Os should be issued at. */ 2282 zio_t *dpa_zio; /* The parent zio_t for all prefetches. */ 2283 arc_flags_t dpa_aflags; /* Flags to pass to the final prefetch. */ 2284 } dbuf_prefetch_arg_t; 2285 2286 /* 2287 * Actually issue the prefetch read for the block given. 2288 */ 2289 static void 2290 dbuf_issue_final_prefetch(dbuf_prefetch_arg_t *dpa, blkptr_t *bp) 2291 { 2292 if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) 2293 return; 2294 2295 arc_flags_t aflags = 2296 dpa->dpa_aflags | ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH; 2297 2298 ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp)); 2299 ASSERT3U(dpa->dpa_curlevel, ==, dpa->dpa_zb.zb_level); 2300 ASSERT(dpa->dpa_zio != NULL); 2301 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, bp, NULL, NULL, 2302 dpa->dpa_prio, ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2303 &aflags, &dpa->dpa_zb); 2304 } 2305 2306 /* 2307 * Called when an indirect block above our prefetch target is read in. This 2308 * will either read in the next indirect block down the tree or issue the actual 2309 * prefetch if the next block down is our target. 2310 */ 2311 static void 2312 dbuf_prefetch_indirect_done(zio_t *zio, arc_buf_t *abuf, void *private) 2313 { 2314 dbuf_prefetch_arg_t *dpa = private; 2315 2316 ASSERT3S(dpa->dpa_zb.zb_level, <, dpa->dpa_curlevel); 2317 ASSERT3S(dpa->dpa_curlevel, >, 0); 2318 2319 /* 2320 * The dpa_dnode is only valid if we are called with a NULL 2321 * zio. This indicates that the arc_read() returned without 2322 * first calling zio_read() to issue a physical read. Once 2323 * a physical read is made the dpa_dnode must be invalidated 2324 * as the locks guarding it may have been dropped. If the 2325 * dpa_dnode is still valid, then we want to add it to the dbuf 2326 * cache. To do so, we must hold the dbuf associated with the block 2327 * we just prefetched, read its contents so that we associate it 2328 * with an arc_buf_t, and then release it. 2329 */ 2330 if (zio != NULL) { 2331 ASSERT3S(BP_GET_LEVEL(zio->io_bp), ==, dpa->dpa_curlevel); 2332 if (zio->io_flags & ZIO_FLAG_RAW) { 2333 ASSERT3U(BP_GET_PSIZE(zio->io_bp), ==, zio->io_size); 2334 } else { 2335 ASSERT3U(BP_GET_LSIZE(zio->io_bp), ==, zio->io_size); 2336 } 2337 ASSERT3P(zio->io_spa, ==, dpa->dpa_spa); 2338 2339 dpa->dpa_dnode = NULL; 2340 } else if (dpa->dpa_dnode != NULL) { 2341 uint64_t curblkid = dpa->dpa_zb.zb_blkid >> 2342 (dpa->dpa_epbs * (dpa->dpa_curlevel - 2343 dpa->dpa_zb.zb_level)); 2344 dmu_buf_impl_t *db = dbuf_hold_level(dpa->dpa_dnode, 2345 dpa->dpa_curlevel, curblkid, FTAG); 2346 (void) dbuf_read(db, NULL, 2347 DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH | DB_RF_HAVESTRUCT); 2348 dbuf_rele(db, FTAG); 2349 } 2350 2351 dpa->dpa_curlevel--; 2352 2353 uint64_t nextblkid = dpa->dpa_zb.zb_blkid >> 2354 (dpa->dpa_epbs * (dpa->dpa_curlevel - dpa->dpa_zb.zb_level)); 2355 blkptr_t *bp = ((blkptr_t *)abuf->b_data) + 2356 P2PHASE(nextblkid, 1ULL << dpa->dpa_epbs); 2357 if (BP_IS_HOLE(bp) || (zio != NULL && zio->io_error != 0)) { 2358 kmem_free(dpa, sizeof (*dpa)); 2359 } else if (dpa->dpa_curlevel == dpa->dpa_zb.zb_level) { 2360 ASSERT3U(nextblkid, ==, dpa->dpa_zb.zb_blkid); 2361 dbuf_issue_final_prefetch(dpa, bp); 2362 kmem_free(dpa, sizeof (*dpa)); 2363 } else { 2364 arc_flags_t iter_aflags = ARC_FLAG_NOWAIT; 2365 zbookmark_phys_t zb; 2366 2367 ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp)); 2368 2369 SET_BOOKMARK(&zb, dpa->dpa_zb.zb_objset, 2370 dpa->dpa_zb.zb_object, dpa->dpa_curlevel, nextblkid); 2371 2372 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, 2373 bp, dbuf_prefetch_indirect_done, dpa, dpa->dpa_prio, 2374 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2375 &iter_aflags, &zb); 2376 } 2377 2378 arc_buf_destroy(abuf, private); 2379 } 2380 2381 /* 2382 * Issue prefetch reads for the given block on the given level. If the indirect 2383 * blocks above that block are not in memory, we will read them in 2384 * asynchronously. As a result, this call never blocks waiting for a read to 2385 * complete. 2386 */ 2387 void 2388 dbuf_prefetch(dnode_t *dn, int64_t level, uint64_t blkid, zio_priority_t prio, 2389 arc_flags_t aflags) 2390 { 2391 blkptr_t bp; 2392 int epbs, nlevels, curlevel; 2393 uint64_t curblkid; 2394 2395 ASSERT(blkid != DMU_BONUS_BLKID); 2396 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2397 2398 if (blkid > dn->dn_maxblkid) 2399 return; 2400 2401 if (dnode_block_freed(dn, blkid)) 2402 return; 2403 2404 /* 2405 * This dnode hasn't been written to disk yet, so there's nothing to 2406 * prefetch. 2407 */ 2408 nlevels = dn->dn_phys->dn_nlevels; 2409 if (level >= nlevels || dn->dn_phys->dn_nblkptr == 0) 2410 return; 2411 2412 epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2413 if (dn->dn_phys->dn_maxblkid < blkid << (epbs * level)) 2414 return; 2415 2416 dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object, 2417 level, blkid); 2418 if (db != NULL) { 2419 mutex_exit(&db->db_mtx); 2420 /* 2421 * This dbuf already exists. It is either CACHED, or 2422 * (we assume) about to be read or filled. 2423 */ 2424 return; 2425 } 2426 2427 /* 2428 * Find the closest ancestor (indirect block) of the target block 2429 * that is present in the cache. In this indirect block, we will 2430 * find the bp that is at curlevel, curblkid. 2431 */ 2432 curlevel = level; 2433 curblkid = blkid; 2434 while (curlevel < nlevels - 1) { 2435 int parent_level = curlevel + 1; 2436 uint64_t parent_blkid = curblkid >> epbs; 2437 dmu_buf_impl_t *db; 2438 2439 if (dbuf_hold_impl(dn, parent_level, parent_blkid, 2440 FALSE, TRUE, FTAG, &db) == 0) { 2441 blkptr_t *bpp = db->db_buf->b_data; 2442 bp = bpp[P2PHASE(curblkid, 1 << epbs)]; 2443 dbuf_rele(db, FTAG); 2444 break; 2445 } 2446 2447 curlevel = parent_level; 2448 curblkid = parent_blkid; 2449 } 2450 2451 if (curlevel == nlevels - 1) { 2452 /* No cached indirect blocks found. */ 2453 ASSERT3U(curblkid, <, dn->dn_phys->dn_nblkptr); 2454 bp = dn->dn_phys->dn_blkptr[curblkid]; 2455 } 2456 if (BP_IS_HOLE(&bp)) 2457 return; 2458 2459 ASSERT3U(curlevel, ==, BP_GET_LEVEL(&bp)); 2460 2461 zio_t *pio = zio_root(dmu_objset_spa(dn->dn_objset), NULL, NULL, 2462 ZIO_FLAG_CANFAIL); 2463 2464 dbuf_prefetch_arg_t *dpa = kmem_zalloc(sizeof (*dpa), KM_SLEEP); 2465 dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 2466 SET_BOOKMARK(&dpa->dpa_zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET, 2467 dn->dn_object, level, blkid); 2468 dpa->dpa_curlevel = curlevel; 2469 dpa->dpa_prio = prio; 2470 dpa->dpa_aflags = aflags; 2471 dpa->dpa_spa = dn->dn_objset->os_spa; 2472 dpa->dpa_dnode = dn; 2473 dpa->dpa_epbs = epbs; 2474 dpa->dpa_zio = pio; 2475 2476 /* 2477 * If we have the indirect just above us, no need to do the asynchronous 2478 * prefetch chain; we'll just run the last step ourselves. If we're at 2479 * a higher level, though, we want to issue the prefetches for all the 2480 * indirect blocks asynchronously, so we can go on with whatever we were 2481 * doing. 2482 */ 2483 if (curlevel == level) { 2484 ASSERT3U(curblkid, ==, blkid); 2485 dbuf_issue_final_prefetch(dpa, &bp); 2486 kmem_free(dpa, sizeof (*dpa)); 2487 } else { 2488 arc_flags_t iter_aflags = ARC_FLAG_NOWAIT; 2489 zbookmark_phys_t zb; 2490 2491 SET_BOOKMARK(&zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET, 2492 dn->dn_object, curlevel, curblkid); 2493 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, 2494 &bp, dbuf_prefetch_indirect_done, dpa, prio, 2495 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2496 &iter_aflags, &zb); 2497 } 2498 /* 2499 * We use pio here instead of dpa_zio since it's possible that 2500 * dpa may have already been freed. 2501 */ 2502 zio_nowait(pio); 2503 } 2504 2505 /* 2506 * Returns with db_holds incremented, and db_mtx not held. 2507 * Note: dn_struct_rwlock must be held. 2508 */ 2509 int 2510 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, 2511 boolean_t fail_sparse, boolean_t fail_uncached, 2512 void *tag, dmu_buf_impl_t **dbp) 2513 { 2514 dmu_buf_impl_t *db, *parent = NULL; 2515 2516 ASSERT(blkid != DMU_BONUS_BLKID); 2517 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2518 ASSERT3U(dn->dn_nlevels, >, level); 2519 2520 *dbp = NULL; 2521 top: 2522 /* dbuf_find() returns with db_mtx held */ 2523 db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid); 2524 2525 if (db == NULL) { 2526 blkptr_t *bp = NULL; 2527 int err; 2528 2529 if (fail_uncached) 2530 return (SET_ERROR(ENOENT)); 2531 2532 ASSERT3P(parent, ==, NULL); 2533 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 2534 if (fail_sparse) { 2535 if (err == 0 && bp && BP_IS_HOLE(bp)) 2536 err = SET_ERROR(ENOENT); 2537 if (err) { 2538 if (parent) 2539 dbuf_rele(parent, NULL); 2540 return (err); 2541 } 2542 } 2543 if (err && err != ENOENT) 2544 return (err); 2545 db = dbuf_create(dn, level, blkid, parent, bp); 2546 } 2547 2548 if (fail_uncached && db->db_state != DB_CACHED) { 2549 mutex_exit(&db->db_mtx); 2550 return (SET_ERROR(ENOENT)); 2551 } 2552 2553 if (db->db_buf != NULL) 2554 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 2555 2556 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 2557 2558 /* 2559 * If this buffer is currently syncing out, and we are are 2560 * still referencing it from db_data, we need to make a copy 2561 * of it in case we decide we want to dirty it again in this txg. 2562 */ 2563 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 2564 dn->dn_object != DMU_META_DNODE_OBJECT && 2565 db->db_state == DB_CACHED && db->db_data_pending) { 2566 dbuf_dirty_record_t *dr = db->db_data_pending; 2567 2568 if (dr->dt.dl.dr_data == db->db_buf) { 2569 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2570 2571 dbuf_set_data(db, 2572 arc_alloc_buf(dn->dn_objset->os_spa, 2573 db->db.db_size, db, type)); 2574 bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 2575 db->db.db_size); 2576 } 2577 } 2578 2579 if (multilist_link_active(&db->db_cache_link)) { 2580 ASSERT(refcount_is_zero(&db->db_holds)); 2581 multilist_remove(&dbuf_cache, db); 2582 (void) refcount_remove_many(&dbuf_cache_size, 2583 db->db.db_size, db); 2584 } 2585 (void) refcount_add(&db->db_holds, tag); 2586 DBUF_VERIFY(db); 2587 mutex_exit(&db->db_mtx); 2588 2589 /* NOTE: we can't rele the parent until after we drop the db_mtx */ 2590 if (parent) 2591 dbuf_rele(parent, NULL); 2592 2593 ASSERT3P(DB_DNODE(db), ==, dn); 2594 ASSERT3U(db->db_blkid, ==, blkid); 2595 ASSERT3U(db->db_level, ==, level); 2596 *dbp = db; 2597 2598 return (0); 2599 } 2600 2601 dmu_buf_impl_t * 2602 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 2603 { 2604 return (dbuf_hold_level(dn, 0, blkid, tag)); 2605 } 2606 2607 dmu_buf_impl_t * 2608 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 2609 { 2610 dmu_buf_impl_t *db; 2611 int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db); 2612 return (err ? NULL : db); 2613 } 2614 2615 void 2616 dbuf_create_bonus(dnode_t *dn) 2617 { 2618 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 2619 2620 ASSERT(dn->dn_bonus == NULL); 2621 dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 2622 } 2623 2624 int 2625 dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 2626 { 2627 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2628 dnode_t *dn; 2629 2630 if (db->db_blkid != DMU_SPILL_BLKID) 2631 return (SET_ERROR(ENOTSUP)); 2632 if (blksz == 0) 2633 blksz = SPA_MINBLOCKSIZE; 2634 ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset))); 2635 blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 2636 2637 DB_DNODE_ENTER(db); 2638 dn = DB_DNODE(db); 2639 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 2640 dbuf_new_size(db, blksz, tx); 2641 rw_exit(&dn->dn_struct_rwlock); 2642 DB_DNODE_EXIT(db); 2643 2644 return (0); 2645 } 2646 2647 void 2648 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 2649 { 2650 dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 2651 } 2652 2653 #pragma weak dmu_buf_add_ref = dbuf_add_ref 2654 void 2655 dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 2656 { 2657 int64_t holds = refcount_add(&db->db_holds, tag); 2658 ASSERT3S(holds, >, 1); 2659 } 2660 2661 #pragma weak dmu_buf_try_add_ref = dbuf_try_add_ref 2662 boolean_t 2663 dbuf_try_add_ref(dmu_buf_t *db_fake, objset_t *os, uint64_t obj, uint64_t blkid, 2664 void *tag) 2665 { 2666 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2667 dmu_buf_impl_t *found_db; 2668 boolean_t result = B_FALSE; 2669 2670 if (db->db_blkid == DMU_BONUS_BLKID) 2671 found_db = dbuf_find_bonus(os, obj); 2672 else 2673 found_db = dbuf_find(os, obj, 0, blkid); 2674 2675 if (found_db != NULL) { 2676 if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) { 2677 (void) refcount_add(&db->db_holds, tag); 2678 result = B_TRUE; 2679 } 2680 mutex_exit(&db->db_mtx); 2681 } 2682 return (result); 2683 } 2684 2685 /* 2686 * If you call dbuf_rele() you had better not be referencing the dnode handle 2687 * unless you have some other direct or indirect hold on the dnode. (An indirect 2688 * hold is a hold on one of the dnode's dbufs, including the bonus buffer.) 2689 * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the 2690 * dnode's parent dbuf evicting its dnode handles. 2691 */ 2692 void 2693 dbuf_rele(dmu_buf_impl_t *db, void *tag) 2694 { 2695 mutex_enter(&db->db_mtx); 2696 dbuf_rele_and_unlock(db, tag); 2697 } 2698 2699 void 2700 dmu_buf_rele(dmu_buf_t *db, void *tag) 2701 { 2702 dbuf_rele((dmu_buf_impl_t *)db, tag); 2703 } 2704 2705 /* 2706 * dbuf_rele() for an already-locked dbuf. This is necessary to allow 2707 * db_dirtycnt and db_holds to be updated atomically. 2708 */ 2709 void 2710 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 2711 { 2712 int64_t holds; 2713 2714 ASSERT(MUTEX_HELD(&db->db_mtx)); 2715 DBUF_VERIFY(db); 2716 2717 /* 2718 * Remove the reference to the dbuf before removing its hold on the 2719 * dnode so we can guarantee in dnode_move() that a referenced bonus 2720 * buffer has a corresponding dnode hold. 2721 */ 2722 holds = refcount_remove(&db->db_holds, tag); 2723 ASSERT(holds >= 0); 2724 2725 /* 2726 * We can't freeze indirects if there is a possibility that they 2727 * may be modified in the current syncing context. 2728 */ 2729 if (db->db_buf != NULL && 2730 holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) { 2731 arc_buf_freeze(db->db_buf); 2732 } 2733 2734 if (holds == db->db_dirtycnt && 2735 db->db_level == 0 && db->db_user_immediate_evict) 2736 dbuf_evict_user(db); 2737 2738 if (holds == 0) { 2739 if (db->db_blkid == DMU_BONUS_BLKID) { 2740 dnode_t *dn; 2741 boolean_t evict_dbuf = db->db_pending_evict; 2742 2743 /* 2744 * If the dnode moves here, we cannot cross this 2745 * barrier until the move completes. 2746 */ 2747 DB_DNODE_ENTER(db); 2748 2749 dn = DB_DNODE(db); 2750 atomic_dec_32(&dn->dn_dbufs_count); 2751 2752 /* 2753 * Decrementing the dbuf count means that the bonus 2754 * buffer's dnode hold is no longer discounted in 2755 * dnode_move(). The dnode cannot move until after 2756 * the dnode_rele() below. 2757 */ 2758 DB_DNODE_EXIT(db); 2759 2760 /* 2761 * Do not reference db after its lock is dropped. 2762 * Another thread may evict it. 2763 */ 2764 mutex_exit(&db->db_mtx); 2765 2766 if (evict_dbuf) 2767 dnode_evict_bonus(dn); 2768 2769 dnode_rele(dn, db); 2770 } else if (db->db_buf == NULL) { 2771 /* 2772 * This is a special case: we never associated this 2773 * dbuf with any data allocated from the ARC. 2774 */ 2775 ASSERT(db->db_state == DB_UNCACHED || 2776 db->db_state == DB_NOFILL); 2777 dbuf_destroy(db); 2778 } else if (arc_released(db->db_buf)) { 2779 /* 2780 * This dbuf has anonymous data associated with it. 2781 */ 2782 dbuf_destroy(db); 2783 } else { 2784 boolean_t do_arc_evict = B_FALSE; 2785 blkptr_t bp; 2786 spa_t *spa = dmu_objset_spa(db->db_objset); 2787 2788 if (!DBUF_IS_CACHEABLE(db) && 2789 db->db_blkptr != NULL && 2790 !BP_IS_HOLE(db->db_blkptr) && 2791 !BP_IS_EMBEDDED(db->db_blkptr)) { 2792 do_arc_evict = B_TRUE; 2793 bp = *db->db_blkptr; 2794 } 2795 2796 if (!DBUF_IS_CACHEABLE(db) || 2797 db->db_pending_evict) { 2798 dbuf_destroy(db); 2799 } else if (!multilist_link_active(&db->db_cache_link)) { 2800 multilist_insert(&dbuf_cache, db); 2801 (void) refcount_add_many(&dbuf_cache_size, 2802 db->db.db_size, db); 2803 mutex_exit(&db->db_mtx); 2804 2805 dbuf_evict_notify(); 2806 } 2807 2808 if (do_arc_evict) 2809 arc_freed(spa, &bp); 2810 } 2811 } else { 2812 mutex_exit(&db->db_mtx); 2813 } 2814 2815 } 2816 2817 #pragma weak dmu_buf_refcount = dbuf_refcount 2818 uint64_t 2819 dbuf_refcount(dmu_buf_impl_t *db) 2820 { 2821 return (refcount_count(&db->db_holds)); 2822 } 2823 2824 void * 2825 dmu_buf_replace_user(dmu_buf_t *db_fake, dmu_buf_user_t *old_user, 2826 dmu_buf_user_t *new_user) 2827 { 2828 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2829 2830 mutex_enter(&db->db_mtx); 2831 dbuf_verify_user(db, DBVU_NOT_EVICTING); 2832 if (db->db_user == old_user) 2833 db->db_user = new_user; 2834 else 2835 old_user = db->db_user; 2836 dbuf_verify_user(db, DBVU_NOT_EVICTING); 2837 mutex_exit(&db->db_mtx); 2838 2839 return (old_user); 2840 } 2841 2842 void * 2843 dmu_buf_set_user(dmu_buf_t *db_fake, dmu_buf_user_t *user) 2844 { 2845 return (dmu_buf_replace_user(db_fake, NULL, user)); 2846 } 2847 2848 void * 2849 dmu_buf_set_user_ie(dmu_buf_t *db_fake, dmu_buf_user_t *user) 2850 { 2851 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2852 2853 db->db_user_immediate_evict = TRUE; 2854 return (dmu_buf_set_user(db_fake, user)); 2855 } 2856 2857 void * 2858 dmu_buf_remove_user(dmu_buf_t *db_fake, dmu_buf_user_t *user) 2859 { 2860 return (dmu_buf_replace_user(db_fake, user, NULL)); 2861 } 2862 2863 void * 2864 dmu_buf_get_user(dmu_buf_t *db_fake) 2865 { 2866 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2867 2868 dbuf_verify_user(db, DBVU_NOT_EVICTING); 2869 return (db->db_user); 2870 } 2871 2872 void 2873 dmu_buf_user_evict_wait() 2874 { 2875 taskq_wait(dbu_evict_taskq); 2876 } 2877 2878 boolean_t 2879 dmu_buf_freeable(dmu_buf_t *dbuf) 2880 { 2881 boolean_t res = B_FALSE; 2882 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 2883 2884 if (db->db_blkptr) 2885 res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, 2886 db->db_blkptr, db->db_blkptr->blk_birth); 2887 2888 return (res); 2889 } 2890 2891 blkptr_t * 2892 dmu_buf_get_blkptr(dmu_buf_t *db) 2893 { 2894 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 2895 return (dbi->db_blkptr); 2896 } 2897 2898 static void 2899 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 2900 { 2901 /* ASSERT(dmu_tx_is_syncing(tx) */ 2902 ASSERT(MUTEX_HELD(&db->db_mtx)); 2903 2904 if (db->db_blkptr != NULL) 2905 return; 2906 2907 if (db->db_blkid == DMU_SPILL_BLKID) { 2908 db->db_blkptr = &dn->dn_phys->dn_spill; 2909 BP_ZERO(db->db_blkptr); 2910 return; 2911 } 2912 if (db->db_level == dn->dn_phys->dn_nlevels-1) { 2913 /* 2914 * This buffer was allocated at a time when there was 2915 * no available blkptrs from the dnode, or it was 2916 * inappropriate to hook it in (i.e., nlevels mis-match). 2917 */ 2918 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 2919 ASSERT(db->db_parent == NULL); 2920 db->db_parent = dn->dn_dbuf; 2921 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 2922 DBUF_VERIFY(db); 2923 } else { 2924 dmu_buf_impl_t *parent = db->db_parent; 2925 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2926 2927 ASSERT(dn->dn_phys->dn_nlevels > 1); 2928 if (parent == NULL) { 2929 mutex_exit(&db->db_mtx); 2930 rw_enter(&dn->dn_struct_rwlock, RW_READER); 2931 parent = dbuf_hold_level(dn, db->db_level + 1, 2932 db->db_blkid >> epbs, db); 2933 rw_exit(&dn->dn_struct_rwlock); 2934 mutex_enter(&db->db_mtx); 2935 db->db_parent = parent; 2936 } 2937 db->db_blkptr = (blkptr_t *)parent->db.db_data + 2938 (db->db_blkid & ((1ULL << epbs) - 1)); 2939 DBUF_VERIFY(db); 2940 } 2941 } 2942 2943 static void 2944 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2945 { 2946 dmu_buf_impl_t *db = dr->dr_dbuf; 2947 dnode_t *dn; 2948 zio_t *zio; 2949 2950 ASSERT(dmu_tx_is_syncing(tx)); 2951 2952 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2953 2954 mutex_enter(&db->db_mtx); 2955 2956 ASSERT(db->db_level > 0); 2957 DBUF_VERIFY(db); 2958 2959 /* Read the block if it hasn't been read yet. */ 2960 if (db->db_buf == NULL) { 2961 mutex_exit(&db->db_mtx); 2962 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 2963 mutex_enter(&db->db_mtx); 2964 } 2965 ASSERT3U(db->db_state, ==, DB_CACHED); 2966 ASSERT(db->db_buf != NULL); 2967 2968 DB_DNODE_ENTER(db); 2969 dn = DB_DNODE(db); 2970 /* Indirect block size must match what the dnode thinks it is. */ 2971 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2972 dbuf_check_blkptr(dn, db); 2973 DB_DNODE_EXIT(db); 2974 2975 /* Provide the pending dirty record to child dbufs */ 2976 db->db_data_pending = dr; 2977 2978 mutex_exit(&db->db_mtx); 2979 dbuf_write(dr, db->db_buf, tx); 2980 2981 zio = dr->dr_zio; 2982 mutex_enter(&dr->dt.di.dr_mtx); 2983 dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx); 2984 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2985 mutex_exit(&dr->dt.di.dr_mtx); 2986 zio_nowait(zio); 2987 } 2988 2989 static void 2990 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2991 { 2992 arc_buf_t **datap = &dr->dt.dl.dr_data; 2993 dmu_buf_impl_t *db = dr->dr_dbuf; 2994 dnode_t *dn; 2995 objset_t *os; 2996 uint64_t txg = tx->tx_txg; 2997 2998 ASSERT(dmu_tx_is_syncing(tx)); 2999 3000 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 3001 3002 mutex_enter(&db->db_mtx); 3003 /* 3004 * To be synced, we must be dirtied. But we 3005 * might have been freed after the dirty. 3006 */ 3007 if (db->db_state == DB_UNCACHED) { 3008 /* This buffer has been freed since it was dirtied */ 3009 ASSERT(db->db.db_data == NULL); 3010 } else if (db->db_state == DB_FILL) { 3011 /* This buffer was freed and is now being re-filled */ 3012 ASSERT(db->db.db_data != dr->dt.dl.dr_data); 3013 } else { 3014 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 3015 } 3016 DBUF_VERIFY(db); 3017 3018 DB_DNODE_ENTER(db); 3019 dn = DB_DNODE(db); 3020 3021 if (db->db_blkid == DMU_SPILL_BLKID) { 3022 mutex_enter(&dn->dn_mtx); 3023 dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 3024 mutex_exit(&dn->dn_mtx); 3025 } 3026 3027 /* 3028 * If this is a bonus buffer, simply copy the bonus data into the 3029 * dnode. It will be written out when the dnode is synced (and it 3030 * will be synced, since it must have been dirty for dbuf_sync to 3031 * be called). 3032 */ 3033 if (db->db_blkid == DMU_BONUS_BLKID) { 3034 dbuf_dirty_record_t **drp; 3035 3036 ASSERT(*datap != NULL); 3037 ASSERT0(db->db_level); 3038 ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 3039 bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 3040 DB_DNODE_EXIT(db); 3041 3042 if (*datap != db->db.db_data) { 3043 zio_buf_free(*datap, DN_MAX_BONUSLEN); 3044 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 3045 } 3046 db->db_data_pending = NULL; 3047 drp = &db->db_last_dirty; 3048 while (*drp != dr) 3049 drp = &(*drp)->dr_next; 3050 ASSERT(dr->dr_next == NULL); 3051 ASSERT(dr->dr_dbuf == db); 3052 *drp = dr->dr_next; 3053 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 3054 ASSERT(db->db_dirtycnt > 0); 3055 db->db_dirtycnt -= 1; 3056 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 3057 return; 3058 } 3059 3060 os = dn->dn_objset; 3061 3062 /* 3063 * This function may have dropped the db_mtx lock allowing a dmu_sync 3064 * operation to sneak in. As a result, we need to ensure that we 3065 * don't check the dr_override_state until we have returned from 3066 * dbuf_check_blkptr. 3067 */ 3068 dbuf_check_blkptr(dn, db); 3069 3070 /* 3071 * If this buffer is in the middle of an immediate write, 3072 * wait for the synchronous IO to complete. 3073 */ 3074 while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 3075 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 3076 cv_wait(&db->db_changed, &db->db_mtx); 3077 ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 3078 } 3079 3080 if (db->db_state != DB_NOFILL && 3081 dn->dn_object != DMU_META_DNODE_OBJECT && 3082 refcount_count(&db->db_holds) > 1 && 3083 dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 3084 *datap == db->db_buf) { 3085 /* 3086 * If this buffer is currently "in use" (i.e., there 3087 * are active holds and db_data still references it), 3088 * then make a copy before we start the write so that 3089 * any modifications from the open txg will not leak 3090 * into this write. 3091 * 3092 * NOTE: this copy does not need to be made for 3093 * objects only modified in the syncing context (e.g. 3094 * DNONE_DNODE blocks). 3095 */ 3096 int blksz = arc_buf_size(*datap); 3097 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 3098 *datap = arc_alloc_buf(os->os_spa, blksz, db, type); 3099 bcopy(db->db.db_data, (*datap)->b_data, blksz); 3100 } 3101 db->db_data_pending = dr; 3102 3103 mutex_exit(&db->db_mtx); 3104 3105 dbuf_write(dr, *datap, tx); 3106 3107 ASSERT(!list_link_active(&dr->dr_dirty_node)); 3108 if (dn->dn_object == DMU_META_DNODE_OBJECT) { 3109 list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 3110 DB_DNODE_EXIT(db); 3111 } else { 3112 /* 3113 * Although zio_nowait() does not "wait for an IO", it does 3114 * initiate the IO. If this is an empty write it seems plausible 3115 * that the IO could actually be completed before the nowait 3116 * returns. We need to DB_DNODE_EXIT() first in case 3117 * zio_nowait() invalidates the dbuf. 3118 */ 3119 DB_DNODE_EXIT(db); 3120 zio_nowait(dr->dr_zio); 3121 } 3122 } 3123 3124 void 3125 dbuf_sync_list(list_t *list, int level, dmu_tx_t *tx) 3126 { 3127 dbuf_dirty_record_t *dr; 3128 3129 while (dr = list_head(list)) { 3130 if (dr->dr_zio != NULL) { 3131 /* 3132 * If we find an already initialized zio then we 3133 * are processing the meta-dnode, and we have finished. 3134 * The dbufs for all dnodes are put back on the list 3135 * during processing, so that we can zio_wait() 3136 * these IOs after initiating all child IOs. 3137 */ 3138 ASSERT3U(dr->dr_dbuf->db.db_object, ==, 3139 DMU_META_DNODE_OBJECT); 3140 break; 3141 } 3142 if (dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID && 3143 dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID) { 3144 VERIFY3U(dr->dr_dbuf->db_level, ==, level); 3145 } 3146 list_remove(list, dr); 3147 if (dr->dr_dbuf->db_level > 0) 3148 dbuf_sync_indirect(dr, tx); 3149 else 3150 dbuf_sync_leaf(dr, tx); 3151 } 3152 } 3153 3154 /* ARGSUSED */ 3155 static void 3156 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 3157 { 3158 dmu_buf_impl_t *db = vdb; 3159 dnode_t *dn; 3160 blkptr_t *bp = zio->io_bp; 3161 blkptr_t *bp_orig = &zio->io_bp_orig; 3162 spa_t *spa = zio->io_spa; 3163 int64_t delta; 3164 uint64_t fill = 0; 3165 int i; 3166 3167 ASSERT3P(db->db_blkptr, !=, NULL); 3168 ASSERT3P(&db->db_data_pending->dr_bp_copy, ==, bp); 3169 3170 DB_DNODE_ENTER(db); 3171 dn = DB_DNODE(db); 3172 delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 3173 dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 3174 zio->io_prev_space_delta = delta; 3175 3176 if (bp->blk_birth != 0) { 3177 ASSERT((db->db_blkid != DMU_SPILL_BLKID && 3178 BP_GET_TYPE(bp) == dn->dn_type) || 3179 (db->db_blkid == DMU_SPILL_BLKID && 3180 BP_GET_TYPE(bp) == dn->dn_bonustype) || 3181 BP_IS_EMBEDDED(bp)); 3182 ASSERT(BP_GET_LEVEL(bp) == db->db_level); 3183 } 3184 3185 mutex_enter(&db->db_mtx); 3186 3187 #ifdef ZFS_DEBUG 3188 if (db->db_blkid == DMU_SPILL_BLKID) { 3189 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 3190 ASSERT(!(BP_IS_HOLE(bp)) && 3191 db->db_blkptr == &dn->dn_phys->dn_spill); 3192 } 3193 #endif 3194 3195 if (db->db_level == 0) { 3196 mutex_enter(&dn->dn_mtx); 3197 if (db->db_blkid > dn->dn_phys->dn_maxblkid && 3198 db->db_blkid != DMU_SPILL_BLKID) 3199 dn->dn_phys->dn_maxblkid = db->db_blkid; 3200 mutex_exit(&dn->dn_mtx); 3201 3202 if (dn->dn_type == DMU_OT_DNODE) { 3203 dnode_phys_t *dnp = db->db.db_data; 3204 for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 3205 i--, dnp++) { 3206 if (dnp->dn_type != DMU_OT_NONE) 3207 fill++; 3208 } 3209 } else { 3210 if (BP_IS_HOLE(bp)) { 3211 fill = 0; 3212 } else { 3213 fill = 1; 3214 } 3215 } 3216 } else { 3217 blkptr_t *ibp = db->db.db_data; 3218 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 3219 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 3220 if (BP_IS_HOLE(ibp)) 3221 continue; 3222 fill += BP_GET_FILL(ibp); 3223 } 3224 } 3225 DB_DNODE_EXIT(db); 3226 3227 if (!BP_IS_EMBEDDED(bp)) 3228 bp->blk_fill = fill; 3229 3230 mutex_exit(&db->db_mtx); 3231 3232 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 3233 *db->db_blkptr = *bp; 3234 rw_exit(&dn->dn_struct_rwlock); 3235 } 3236 3237 /* ARGSUSED */ 3238 /* 3239 * This function gets called just prior to running through the compression 3240 * stage of the zio pipeline. If we're an indirect block comprised of only 3241 * holes, then we want this indirect to be compressed away to a hole. In 3242 * order to do that we must zero out any information about the holes that 3243 * this indirect points to prior to before we try to compress it. 3244 */ 3245 static void 3246 dbuf_write_children_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 3247 { 3248 dmu_buf_impl_t *db = vdb; 3249 dnode_t *dn; 3250 blkptr_t *bp; 3251 uint64_t i; 3252 int epbs; 3253 3254 ASSERT3U(db->db_level, >, 0); 3255 DB_DNODE_ENTER(db); 3256 dn = DB_DNODE(db); 3257 epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 3258 3259 /* Determine if all our children are holes */ 3260 for (i = 0, bp = db->db.db_data; i < 1 << epbs; i++, bp++) { 3261 if (!BP_IS_HOLE(bp)) 3262 break; 3263 } 3264 3265 /* 3266 * If all the children are holes, then zero them all out so that 3267 * we may get compressed away. 3268 */ 3269 if (i == 1 << epbs) { 3270 /* didn't find any non-holes */ 3271 bzero(db->db.db_data, db->db.db_size); 3272 } 3273 DB_DNODE_EXIT(db); 3274 } 3275 3276 /* 3277 * The SPA will call this callback several times for each zio - once 3278 * for every physical child i/o (zio->io_phys_children times). This 3279 * allows the DMU to monitor the progress of each logical i/o. For example, 3280 * there may be 2 copies of an indirect block, or many fragments of a RAID-Z 3281 * block. There may be a long delay before all copies/fragments are completed, 3282 * so this callback allows us to retire dirty space gradually, as the physical 3283 * i/os complete. 3284 */ 3285 /* ARGSUSED */ 3286 static void 3287 dbuf_write_physdone(zio_t *zio, arc_buf_t *buf, void *arg) 3288 { 3289 dmu_buf_impl_t *db = arg; 3290 objset_t *os = db->db_objset; 3291 dsl_pool_t *dp = dmu_objset_pool(os); 3292 dbuf_dirty_record_t *dr; 3293 int delta = 0; 3294 3295 dr = db->db_data_pending; 3296 ASSERT3U(dr->dr_txg, ==, zio->io_txg); 3297 3298 /* 3299 * The callback will be called io_phys_children times. Retire one 3300 * portion of our dirty space each time we are called. Any rounding 3301 * error will be cleaned up by dsl_pool_sync()'s call to 3302 * dsl_pool_undirty_space(). 3303 */ 3304 delta = dr->dr_accounted / zio->io_phys_children; 3305 dsl_pool_undirty_space(dp, delta, zio->io_txg); 3306 } 3307 3308 /* ARGSUSED */ 3309 static void 3310 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 3311 { 3312 dmu_buf_impl_t *db = vdb; 3313 blkptr_t *bp_orig = &zio->io_bp_orig; 3314 blkptr_t *bp = db->db_blkptr; 3315 objset_t *os = db->db_objset; 3316 dmu_tx_t *tx = os->os_synctx; 3317 dbuf_dirty_record_t **drp, *dr; 3318 3319 ASSERT0(zio->io_error); 3320 ASSERT(db->db_blkptr == bp); 3321 3322 /* 3323 * For nopwrites and rewrites we ensure that the bp matches our 3324 * original and bypass all the accounting. 3325 */ 3326 if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) { 3327 ASSERT(BP_EQUAL(bp, bp_orig)); 3328 } else { 3329 dsl_dataset_t *ds = os->os_dsl_dataset; 3330 (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 3331 dsl_dataset_block_born(ds, bp, tx); 3332 } 3333 3334 mutex_enter(&db->db_mtx); 3335 3336 DBUF_VERIFY(db); 3337 3338 drp = &db->db_last_dirty; 3339 while ((dr = *drp) != db->db_data_pending) 3340 drp = &dr->dr_next; 3341 ASSERT(!list_link_active(&dr->dr_dirty_node)); 3342 ASSERT(dr->dr_dbuf == db); 3343 ASSERT(dr->dr_next == NULL); 3344 *drp = dr->dr_next; 3345 3346 #ifdef ZFS_DEBUG 3347 if (db->db_blkid == DMU_SPILL_BLKID) { 3348 dnode_t *dn; 3349 3350 DB_DNODE_ENTER(db); 3351 dn = DB_DNODE(db); 3352 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 3353 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 3354 db->db_blkptr == &dn->dn_phys->dn_spill); 3355 DB_DNODE_EXIT(db); 3356 } 3357 #endif 3358 3359 if (db->db_level == 0) { 3360 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 3361 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 3362 if (db->db_state != DB_NOFILL) { 3363 if (dr->dt.dl.dr_data != db->db_buf) 3364 arc_buf_destroy(dr->dt.dl.dr_data, db); 3365 } 3366 } else { 3367 dnode_t *dn; 3368 3369 DB_DNODE_ENTER(db); 3370 dn = DB_DNODE(db); 3371 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 3372 ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift); 3373 if (!BP_IS_HOLE(db->db_blkptr)) { 3374 int epbs = 3375 dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 3376 ASSERT3U(db->db_blkid, <=, 3377 dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)); 3378 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 3379 db->db.db_size); 3380 } 3381 DB_DNODE_EXIT(db); 3382 mutex_destroy(&dr->dt.di.dr_mtx); 3383 list_destroy(&dr->dt.di.dr_children); 3384 } 3385 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 3386 3387 cv_broadcast(&db->db_changed); 3388 ASSERT(db->db_dirtycnt > 0); 3389 db->db_dirtycnt -= 1; 3390 db->db_data_pending = NULL; 3391 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg); 3392 } 3393 3394 static void 3395 dbuf_write_nofill_ready(zio_t *zio) 3396 { 3397 dbuf_write_ready(zio, NULL, zio->io_private); 3398 } 3399 3400 static void 3401 dbuf_write_nofill_done(zio_t *zio) 3402 { 3403 dbuf_write_done(zio, NULL, zio->io_private); 3404 } 3405 3406 static void 3407 dbuf_write_override_ready(zio_t *zio) 3408 { 3409 dbuf_dirty_record_t *dr = zio->io_private; 3410 dmu_buf_impl_t *db = dr->dr_dbuf; 3411 3412 dbuf_write_ready(zio, NULL, db); 3413 } 3414 3415 static void 3416 dbuf_write_override_done(zio_t *zio) 3417 { 3418 dbuf_dirty_record_t *dr = zio->io_private; 3419 dmu_buf_impl_t *db = dr->dr_dbuf; 3420 blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 3421 3422 mutex_enter(&db->db_mtx); 3423 if (!BP_EQUAL(zio->io_bp, obp)) { 3424 if (!BP_IS_HOLE(obp)) 3425 dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 3426 arc_release(dr->dt.dl.dr_data, db); 3427 } 3428 mutex_exit(&db->db_mtx); 3429 3430 dbuf_write_done(zio, NULL, db); 3431 } 3432 3433 /* Issue I/O to commit a dirty buffer to disk. */ 3434 static void 3435 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 3436 { 3437 dmu_buf_impl_t *db = dr->dr_dbuf; 3438 dnode_t *dn; 3439 objset_t *os; 3440 dmu_buf_impl_t *parent = db->db_parent; 3441 uint64_t txg = tx->tx_txg; 3442 zbookmark_phys_t zb; 3443 zio_prop_t zp; 3444 zio_t *zio; 3445 int wp_flag = 0; 3446 3447 ASSERT(dmu_tx_is_syncing(tx)); 3448 3449 DB_DNODE_ENTER(db); 3450 dn = DB_DNODE(db); 3451 os = dn->dn_objset; 3452 3453 if (db->db_state != DB_NOFILL) { 3454 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 3455 /* 3456 * Private object buffers are released here rather 3457 * than in dbuf_dirty() since they are only modified 3458 * in the syncing context and we don't want the 3459 * overhead of making multiple copies of the data. 3460 */ 3461 if (BP_IS_HOLE(db->db_blkptr)) { 3462 arc_buf_thaw(data); 3463 } else { 3464 dbuf_release_bp(db); 3465 } 3466 } 3467 } 3468 3469 if (parent != dn->dn_dbuf) { 3470 /* Our parent is an indirect block. */ 3471 /* We have a dirty parent that has been scheduled for write. */ 3472 ASSERT(parent && parent->db_data_pending); 3473 /* Our parent's buffer is one level closer to the dnode. */ 3474 ASSERT(db->db_level == parent->db_level-1); 3475 /* 3476 * We're about to modify our parent's db_data by modifying 3477 * our block pointer, so the parent must be released. 3478 */ 3479 ASSERT(arc_released(parent->db_buf)); 3480 zio = parent->db_data_pending->dr_zio; 3481 } else { 3482 /* Our parent is the dnode itself. */ 3483 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 3484 db->db_blkid != DMU_SPILL_BLKID) || 3485 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 3486 if (db->db_blkid != DMU_SPILL_BLKID) 3487 ASSERT3P(db->db_blkptr, ==, 3488 &dn->dn_phys->dn_blkptr[db->db_blkid]); 3489 zio = dn->dn_zio; 3490 } 3491 3492 ASSERT(db->db_level == 0 || data == db->db_buf); 3493 ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 3494 ASSERT(zio); 3495 3496 SET_BOOKMARK(&zb, os->os_dsl_dataset ? 3497 os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 3498 db->db.db_object, db->db_level, db->db_blkid); 3499 3500 if (db->db_blkid == DMU_SPILL_BLKID) 3501 wp_flag = WP_SPILL; 3502 wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 3503 3504 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); 3505 DB_DNODE_EXIT(db); 3506 3507 /* 3508 * We copy the blkptr now (rather than when we instantiate the dirty 3509 * record), because its value can change between open context and 3510 * syncing context. We do not need to hold dn_struct_rwlock to read 3511 * db_blkptr because we are in syncing context. 3512 */ 3513 dr->dr_bp_copy = *db->db_blkptr; 3514 3515 if (db->db_level == 0 && 3516 dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 3517 /* 3518 * The BP for this block has been provided by open context 3519 * (by dmu_sync() or dmu_buf_write_embedded()). 3520 */ 3521 void *contents = (data != NULL) ? data->b_data : NULL; 3522 3523 dr->dr_zio = zio_write(zio, os->os_spa, txg, 3524 &dr->dr_bp_copy, contents, db->db.db_size, &zp, 3525 dbuf_write_override_ready, NULL, NULL, 3526 dbuf_write_override_done, 3527 dr, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 3528 mutex_enter(&db->db_mtx); 3529 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 3530 zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 3531 dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite); 3532 mutex_exit(&db->db_mtx); 3533 } else if (db->db_state == DB_NOFILL) { 3534 ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF || 3535 zp.zp_checksum == ZIO_CHECKSUM_NOPARITY); 3536 dr->dr_zio = zio_write(zio, os->os_spa, txg, 3537 &dr->dr_bp_copy, NULL, db->db.db_size, &zp, 3538 dbuf_write_nofill_ready, NULL, NULL, 3539 dbuf_write_nofill_done, db, 3540 ZIO_PRIORITY_ASYNC_WRITE, 3541 ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 3542 } else { 3543 ASSERT(arc_released(data)); 3544 3545 /* 3546 * For indirect blocks, we want to setup the children 3547 * ready callback so that we can properly handle an indirect 3548 * block that only contains holes. 3549 */ 3550 arc_done_func_t *children_ready_cb = NULL; 3551 if (db->db_level != 0) 3552 children_ready_cb = dbuf_write_children_ready; 3553 3554 dr->dr_zio = arc_write(zio, os->os_spa, txg, 3555 &dr->dr_bp_copy, data, DBUF_IS_L2CACHEABLE(db), 3556 &zp, dbuf_write_ready, children_ready_cb, 3557 dbuf_write_physdone, dbuf_write_done, db, 3558 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 3559 } 3560 } 3561