1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5ea8dc4b6Seschrock * Common Development and Distribution License (the "License"). 6ea8dc4b6Seschrock * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 22a3f829aeSBill Moore * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23fa9e4066Sahrens * Use is subject to license terms. 24fa9e4066Sahrens */ 25fa9e4066Sahrens 26*283b8460SGeorge.Wilson /* 27*283b8460SGeorge.Wilson * Copyright (c) 2012 by Delphix. All rights reserved. 28*283b8460SGeorge.Wilson */ 29*283b8460SGeorge.Wilson 30fa9e4066Sahrens #include <sys/zfs_context.h> 31fa9e4066Sahrens #include <sys/vdev_impl.h> 32fa9e4066Sahrens #include <sys/zio.h> 33fa9e4066Sahrens #include <sys/avl.h> 34fa9e4066Sahrens 35614409b5Sahrens /* 36614409b5Sahrens * These tunables are for performance analysis. 37614409b5Sahrens */ 38614409b5Sahrens /* 39614409b5Sahrens * zfs_vdev_max_pending is the maximum number of i/os concurrently 40614409b5Sahrens * pending to each device. zfs_vdev_min_pending is the initial number 41614409b5Sahrens * of i/os pending to each device (before it starts ramping up to 42614409b5Sahrens * max_pending). 43614409b5Sahrens */ 44c33e334fSMatthew Ahrens int zfs_vdev_max_pending = 10; 45614409b5Sahrens int zfs_vdev_min_pending = 4; 46614409b5Sahrens 47d3d50737SRafael Vanoni /* deadline = pri + ddi_get_lbolt64() >> time_shift) */ 48614409b5Sahrens int zfs_vdev_time_shift = 6; 49614409b5Sahrens 50614409b5Sahrens /* exponential I/O issue ramp-up rate */ 51614409b5Sahrens int zfs_vdev_ramp_rate = 2; 52614409b5Sahrens 53614409b5Sahrens /* 54f94275ceSAdam Leventhal * To reduce IOPs, we aggregate small adjacent I/Os into one large I/O. 55f94275ceSAdam Leventhal * For read I/Os, we also aggregate across small adjacency gaps; for writes 56f94275ceSAdam Leventhal * we include spans of optional I/Os to aid aggregation at the disk even when 57f94275ceSAdam Leventhal * they aren't able to help us aggregate at this level. 58614409b5Sahrens */ 59614409b5Sahrens int zfs_vdev_aggregation_limit = SPA_MAXBLOCKSIZE; 606f708f7cSJeff Bonwick int zfs_vdev_read_gap_limit = 32 << 10; 61f94275ceSAdam Leventhal int zfs_vdev_write_gap_limit = 4 << 10; 62614409b5Sahrens 63fa9e4066Sahrens /* 64fa9e4066Sahrens * Virtual device vector for disk I/O scheduling. 65fa9e4066Sahrens */ 66fa9e4066Sahrens int 67fa9e4066Sahrens vdev_queue_deadline_compare(const void *x1, const void *x2) 68fa9e4066Sahrens { 69fa9e4066Sahrens const zio_t *z1 = x1; 70fa9e4066Sahrens const zio_t *z2 = x2; 71fa9e4066Sahrens 72fa9e4066Sahrens if (z1->io_deadline < z2->io_deadline) 73fa9e4066Sahrens return (-1); 74fa9e4066Sahrens if (z1->io_deadline > z2->io_deadline) 75fa9e4066Sahrens return (1); 76fa9e4066Sahrens 77fa9e4066Sahrens if (z1->io_offset < z2->io_offset) 78fa9e4066Sahrens return (-1); 79fa9e4066Sahrens if (z1->io_offset > z2->io_offset) 80fa9e4066Sahrens return (1); 81fa9e4066Sahrens 82fa9e4066Sahrens if (z1 < z2) 83fa9e4066Sahrens return (-1); 84fa9e4066Sahrens if (z1 > z2) 85fa9e4066Sahrens return (1); 86fa9e4066Sahrens 87fa9e4066Sahrens return (0); 88fa9e4066Sahrens } 89fa9e4066Sahrens 90fa9e4066Sahrens int 91fa9e4066Sahrens vdev_queue_offset_compare(const void *x1, const void *x2) 92fa9e4066Sahrens { 93fa9e4066Sahrens const zio_t *z1 = x1; 94fa9e4066Sahrens const zio_t *z2 = x2; 95fa9e4066Sahrens 96fa9e4066Sahrens if (z1->io_offset < z2->io_offset) 97fa9e4066Sahrens return (-1); 98fa9e4066Sahrens if (z1->io_offset > z2->io_offset) 99fa9e4066Sahrens return (1); 100fa9e4066Sahrens 101fa9e4066Sahrens if (z1 < z2) 102fa9e4066Sahrens return (-1); 103fa9e4066Sahrens if (z1 > z2) 104fa9e4066Sahrens return (1); 105fa9e4066Sahrens 106fa9e4066Sahrens return (0); 107fa9e4066Sahrens } 108fa9e4066Sahrens 109fa9e4066Sahrens void 110fa9e4066Sahrens vdev_queue_init(vdev_t *vd) 111fa9e4066Sahrens { 112fa9e4066Sahrens vdev_queue_t *vq = &vd->vdev_queue; 113fa9e4066Sahrens 114fa9e4066Sahrens mutex_init(&vq->vq_lock, NULL, MUTEX_DEFAULT, NULL); 115fa9e4066Sahrens 116fa9e4066Sahrens avl_create(&vq->vq_deadline_tree, vdev_queue_deadline_compare, 117fa9e4066Sahrens sizeof (zio_t), offsetof(struct zio, io_deadline_node)); 118fa9e4066Sahrens 119fa9e4066Sahrens avl_create(&vq->vq_read_tree, vdev_queue_offset_compare, 120fa9e4066Sahrens sizeof (zio_t), offsetof(struct zio, io_offset_node)); 121fa9e4066Sahrens 122fa9e4066Sahrens avl_create(&vq->vq_write_tree, vdev_queue_offset_compare, 123fa9e4066Sahrens sizeof (zio_t), offsetof(struct zio, io_offset_node)); 124fa9e4066Sahrens 125fa9e4066Sahrens avl_create(&vq->vq_pending_tree, vdev_queue_offset_compare, 126fa9e4066Sahrens sizeof (zio_t), offsetof(struct zio, io_offset_node)); 127fa9e4066Sahrens } 128fa9e4066Sahrens 129fa9e4066Sahrens void 130fa9e4066Sahrens vdev_queue_fini(vdev_t *vd) 131fa9e4066Sahrens { 132fa9e4066Sahrens vdev_queue_t *vq = &vd->vdev_queue; 133fa9e4066Sahrens 134fa9e4066Sahrens avl_destroy(&vq->vq_deadline_tree); 135fa9e4066Sahrens avl_destroy(&vq->vq_read_tree); 136fa9e4066Sahrens avl_destroy(&vq->vq_write_tree); 137fa9e4066Sahrens avl_destroy(&vq->vq_pending_tree); 138fa9e4066Sahrens 139fa9e4066Sahrens mutex_destroy(&vq->vq_lock); 140fa9e4066Sahrens } 141fa9e4066Sahrens 142ea8dc4b6Seschrock static void 143ea8dc4b6Seschrock vdev_queue_io_add(vdev_queue_t *vq, zio_t *zio) 144ea8dc4b6Seschrock { 145ea8dc4b6Seschrock avl_add(&vq->vq_deadline_tree, zio); 146ea8dc4b6Seschrock avl_add(zio->io_vdev_tree, zio); 147ea8dc4b6Seschrock } 148ea8dc4b6Seschrock 149ea8dc4b6Seschrock static void 150ea8dc4b6Seschrock vdev_queue_io_remove(vdev_queue_t *vq, zio_t *zio) 151ea8dc4b6Seschrock { 152ea8dc4b6Seschrock avl_remove(&vq->vq_deadline_tree, zio); 153ea8dc4b6Seschrock avl_remove(zio->io_vdev_tree, zio); 154ea8dc4b6Seschrock } 155ea8dc4b6Seschrock 156fa9e4066Sahrens static void 157fa9e4066Sahrens vdev_queue_agg_io_done(zio_t *aio) 158fa9e4066Sahrens { 159a3f829aeSBill Moore zio_t *pio; 160fa9e4066Sahrens 161a3f829aeSBill Moore while ((pio = zio_walk_parents(aio)) != NULL) 162fa9e4066Sahrens if (aio->io_type == ZIO_TYPE_READ) 163a3f829aeSBill Moore bcopy((char *)aio->io_data + (pio->io_offset - 164a3f829aeSBill Moore aio->io_offset), pio->io_data, pio->io_size); 165fa9e4066Sahrens 166fa9e4066Sahrens zio_buf_free(aio->io_data, aio->io_size); 167fa9e4066Sahrens } 168fa9e4066Sahrens 1696f708f7cSJeff Bonwick /* 1706f708f7cSJeff Bonwick * Compute the range spanned by two i/os, which is the endpoint of the last 1716f708f7cSJeff Bonwick * (lio->io_offset + lio->io_size) minus start of the first (fio->io_offset). 1726f708f7cSJeff Bonwick * Conveniently, the gap between fio and lio is given by -IO_SPAN(lio, fio); 1736f708f7cSJeff Bonwick * thus fio and lio are adjacent if and only if IO_SPAN(lio, fio) == 0. 1746f708f7cSJeff Bonwick */ 1756f708f7cSJeff Bonwick #define IO_SPAN(fio, lio) ((lio)->io_offset + (lio)->io_size - (fio)->io_offset) 1766f708f7cSJeff Bonwick #define IO_GAP(fio, lio) (-IO_SPAN(lio, fio)) 177fa9e4066Sahrens 178fa9e4066Sahrens static zio_t * 179e05725b1Sbonwick vdev_queue_io_to_issue(vdev_queue_t *vq, uint64_t pending_limit) 180fa9e4066Sahrens { 181f94275ceSAdam Leventhal zio_t *fio, *lio, *aio, *dio, *nio, *mio; 182a3f829aeSBill Moore avl_tree_t *t; 1838ad4d6ddSJeff Bonwick int flags; 1846f708f7cSJeff Bonwick uint64_t maxspan = zfs_vdev_aggregation_limit; 1856f708f7cSJeff Bonwick uint64_t maxgap; 186f94275ceSAdam Leventhal int stretch; 187fa9e4066Sahrens 188f94275ceSAdam Leventhal again: 189fa9e4066Sahrens ASSERT(MUTEX_HELD(&vq->vq_lock)); 190fa9e4066Sahrens 191fa9e4066Sahrens if (avl_numnodes(&vq->vq_pending_tree) >= pending_limit || 192fa9e4066Sahrens avl_numnodes(&vq->vq_deadline_tree) == 0) 193fa9e4066Sahrens return (NULL); 194fa9e4066Sahrens 195fa9e4066Sahrens fio = lio = avl_first(&vq->vq_deadline_tree); 196fa9e4066Sahrens 197a3f829aeSBill Moore t = fio->io_vdev_tree; 1988ad4d6ddSJeff Bonwick flags = fio->io_flags & ZIO_FLAG_AGG_INHERIT; 1996f708f7cSJeff Bonwick maxgap = (t == &vq->vq_read_tree) ? zfs_vdev_read_gap_limit : 0; 2008ad4d6ddSJeff Bonwick 2018ad4d6ddSJeff Bonwick if (!(flags & ZIO_FLAG_DONT_AGGREGATE)) { 2028ad4d6ddSJeff Bonwick /* 203f94275ceSAdam Leventhal * We can aggregate I/Os that are sufficiently adjacent and of 204f94275ceSAdam Leventhal * the same flavor, as expressed by the AGG_INHERIT flags. 205f94275ceSAdam Leventhal * The latter requirement is necessary so that certain 206f94275ceSAdam Leventhal * attributes of the I/O, such as whether it's a normal I/O 207f94275ceSAdam Leventhal * or a scrub/resilver, can be preserved in the aggregate. 208f94275ceSAdam Leventhal * We can include optional I/Os, but don't allow them 209f94275ceSAdam Leventhal * to begin a range as they add no benefit in that situation. 210f94275ceSAdam Leventhal */ 211f94275ceSAdam Leventhal 212f94275ceSAdam Leventhal /* 213f94275ceSAdam Leventhal * We keep track of the last non-optional I/O. 214f94275ceSAdam Leventhal */ 215f94275ceSAdam Leventhal mio = (fio->io_flags & ZIO_FLAG_OPTIONAL) ? NULL : fio; 216f94275ceSAdam Leventhal 217f94275ceSAdam Leventhal /* 218f94275ceSAdam Leventhal * Walk backwards through sufficiently contiguous I/Os 219f94275ceSAdam Leventhal * recording the last non-option I/O. 2208ad4d6ddSJeff Bonwick */ 221a3f829aeSBill Moore while ((dio = AVL_PREV(t, fio)) != NULL && 2228ad4d6ddSJeff Bonwick (dio->io_flags & ZIO_FLAG_AGG_INHERIT) == flags && 223f94275ceSAdam Leventhal IO_SPAN(dio, lio) <= maxspan && 224f94275ceSAdam Leventhal IO_GAP(dio, fio) <= maxgap) { 2258ad4d6ddSJeff Bonwick fio = dio; 226f94275ceSAdam Leventhal if (mio == NULL && !(fio->io_flags & ZIO_FLAG_OPTIONAL)) 227f94275ceSAdam Leventhal mio = fio; 228f94275ceSAdam Leventhal } 229f94275ceSAdam Leventhal 230f94275ceSAdam Leventhal /* 231f94275ceSAdam Leventhal * Skip any initial optional I/Os. 232f94275ceSAdam Leventhal */ 233f94275ceSAdam Leventhal while ((fio->io_flags & ZIO_FLAG_OPTIONAL) && fio != lio) { 234f94275ceSAdam Leventhal fio = AVL_NEXT(t, fio); 235f94275ceSAdam Leventhal ASSERT(fio != NULL); 236f94275ceSAdam Leventhal } 2376f708f7cSJeff Bonwick 238f94275ceSAdam Leventhal /* 239f94275ceSAdam Leventhal * Walk forward through sufficiently contiguous I/Os. 240f94275ceSAdam Leventhal */ 241a3f829aeSBill Moore while ((dio = AVL_NEXT(t, lio)) != NULL && 2428ad4d6ddSJeff Bonwick (dio->io_flags & ZIO_FLAG_AGG_INHERIT) == flags && 243f94275ceSAdam Leventhal IO_SPAN(fio, dio) <= maxspan && 244f94275ceSAdam Leventhal IO_GAP(lio, dio) <= maxgap) { 2458ad4d6ddSJeff Bonwick lio = dio; 246f94275ceSAdam Leventhal if (!(lio->io_flags & ZIO_FLAG_OPTIONAL)) 247f94275ceSAdam Leventhal mio = lio; 248f94275ceSAdam Leventhal } 249f94275ceSAdam Leventhal 250f94275ceSAdam Leventhal /* 251f94275ceSAdam Leventhal * Now that we've established the range of the I/O aggregation 252f94275ceSAdam Leventhal * we must decide what to do with trailing optional I/Os. 253f94275ceSAdam Leventhal * For reads, there's nothing to do. While we are unable to 254f94275ceSAdam Leventhal * aggregate further, it's possible that a trailing optional 255f94275ceSAdam Leventhal * I/O would allow the underlying device to aggregate with 256f94275ceSAdam Leventhal * subsequent I/Os. We must therefore determine if the next 257f94275ceSAdam Leventhal * non-optional I/O is close enough to make aggregation 258f94275ceSAdam Leventhal * worthwhile. 259f94275ceSAdam Leventhal */ 260f94275ceSAdam Leventhal stretch = B_FALSE; 261f94275ceSAdam Leventhal if (t != &vq->vq_read_tree && mio != NULL) { 262f94275ceSAdam Leventhal nio = lio; 263f94275ceSAdam Leventhal while ((dio = AVL_NEXT(t, nio)) != NULL && 264f94275ceSAdam Leventhal IO_GAP(nio, dio) == 0 && 265f94275ceSAdam Leventhal IO_GAP(mio, dio) <= zfs_vdev_write_gap_limit) { 266f94275ceSAdam Leventhal nio = dio; 267f94275ceSAdam Leventhal if (!(nio->io_flags & ZIO_FLAG_OPTIONAL)) { 268f94275ceSAdam Leventhal stretch = B_TRUE; 269f94275ceSAdam Leventhal break; 270f94275ceSAdam Leventhal } 271f94275ceSAdam Leventhal } 272f94275ceSAdam Leventhal } 273f94275ceSAdam Leventhal 274f94275ceSAdam Leventhal if (stretch) { 275f94275ceSAdam Leventhal /* This may be a no-op. */ 276f94275ceSAdam Leventhal VERIFY((dio = AVL_NEXT(t, lio)) != NULL); 277f94275ceSAdam Leventhal dio->io_flags &= ~ZIO_FLAG_OPTIONAL; 278f94275ceSAdam Leventhal } else { 279f94275ceSAdam Leventhal while (lio != mio && lio != fio) { 280f94275ceSAdam Leventhal ASSERT(lio->io_flags & ZIO_FLAG_OPTIONAL); 281f94275ceSAdam Leventhal lio = AVL_PREV(t, lio); 282f94275ceSAdam Leventhal ASSERT(lio != NULL); 283f94275ceSAdam Leventhal } 284f94275ceSAdam Leventhal } 285fa9e4066Sahrens } 286fa9e4066Sahrens 287fa9e4066Sahrens if (fio != lio) { 2886f708f7cSJeff Bonwick uint64_t size = IO_SPAN(fio, lio); 289614409b5Sahrens ASSERT(size <= zfs_vdev_aggregation_limit); 290fa9e4066Sahrens 291e14bb325SJeff Bonwick aio = zio_vdev_delegated_io(fio->io_vd, fio->io_offset, 29280eb36f2SGeorge Wilson zio_buf_alloc(size), size, fio->io_type, ZIO_PRIORITY_AGG, 2938ad4d6ddSJeff Bonwick flags | ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_QUEUE, 294fa9e4066Sahrens vdev_queue_agg_io_done, NULL); 295*283b8460SGeorge.Wilson aio->io_timestamp = fio->io_timestamp; 296fa9e4066Sahrens 2976f708f7cSJeff Bonwick nio = fio; 2986f708f7cSJeff Bonwick do { 2996f708f7cSJeff Bonwick dio = nio; 3006f708f7cSJeff Bonwick nio = AVL_NEXT(t, dio); 301fa9e4066Sahrens ASSERT(dio->io_type == aio->io_type); 302a3f829aeSBill Moore ASSERT(dio->io_vdev_tree == t); 303a3f829aeSBill Moore 304f94275ceSAdam Leventhal if (dio->io_flags & ZIO_FLAG_NODATA) { 305f94275ceSAdam Leventhal ASSERT(dio->io_type == ZIO_TYPE_WRITE); 306f94275ceSAdam Leventhal bzero((char *)aio->io_data + (dio->io_offset - 307f94275ceSAdam Leventhal aio->io_offset), dio->io_size); 308f94275ceSAdam Leventhal } else if (dio->io_type == ZIO_TYPE_WRITE) { 309a3f829aeSBill Moore bcopy(dio->io_data, (char *)aio->io_data + 310a3f829aeSBill Moore (dio->io_offset - aio->io_offset), 311a3f829aeSBill Moore dio->io_size); 312f94275ceSAdam Leventhal } 313a3f829aeSBill Moore 314a3f829aeSBill Moore zio_add_child(dio, aio); 315ea8dc4b6Seschrock vdev_queue_io_remove(vq, dio); 316fa9e4066Sahrens zio_vdev_io_bypass(dio); 317a3f829aeSBill Moore zio_execute(dio); 3186f708f7cSJeff Bonwick } while (dio != lio); 319fa9e4066Sahrens 320fa9e4066Sahrens avl_add(&vq->vq_pending_tree, aio); 321fa9e4066Sahrens 322fa9e4066Sahrens return (aio); 323fa9e4066Sahrens } 324fa9e4066Sahrens 325a3f829aeSBill Moore ASSERT(fio->io_vdev_tree == t); 326ea8dc4b6Seschrock vdev_queue_io_remove(vq, fio); 327fa9e4066Sahrens 328f94275ceSAdam Leventhal /* 329f94275ceSAdam Leventhal * If the I/O is or was optional and therefore has no data, we need to 330f94275ceSAdam Leventhal * simply discard it. We need to drop the vdev queue's lock to avoid a 331f94275ceSAdam Leventhal * deadlock that we could encounter since this I/O will complete 332f94275ceSAdam Leventhal * immediately. 333f94275ceSAdam Leventhal */ 334f94275ceSAdam Leventhal if (fio->io_flags & ZIO_FLAG_NODATA) { 335f94275ceSAdam Leventhal mutex_exit(&vq->vq_lock); 336f94275ceSAdam Leventhal zio_vdev_io_bypass(fio); 337f94275ceSAdam Leventhal zio_execute(fio); 338f94275ceSAdam Leventhal mutex_enter(&vq->vq_lock); 339f94275ceSAdam Leventhal goto again; 340f94275ceSAdam Leventhal } 341f94275ceSAdam Leventhal 342fa9e4066Sahrens avl_add(&vq->vq_pending_tree, fio); 343fa9e4066Sahrens 344fa9e4066Sahrens return (fio); 345fa9e4066Sahrens } 346fa9e4066Sahrens 347fa9e4066Sahrens zio_t * 348fa9e4066Sahrens vdev_queue_io(zio_t *zio) 349fa9e4066Sahrens { 350fa9e4066Sahrens vdev_queue_t *vq = &zio->io_vd->vdev_queue; 351fa9e4066Sahrens zio_t *nio; 352fa9e4066Sahrens 353fa9e4066Sahrens ASSERT(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE); 354fa9e4066Sahrens 355fa9e4066Sahrens if (zio->io_flags & ZIO_FLAG_DONT_QUEUE) 356fa9e4066Sahrens return (zio); 357fa9e4066Sahrens 358fa9e4066Sahrens zio->io_flags |= ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_QUEUE; 359fa9e4066Sahrens 360fa9e4066Sahrens if (zio->io_type == ZIO_TYPE_READ) 361fa9e4066Sahrens zio->io_vdev_tree = &vq->vq_read_tree; 362fa9e4066Sahrens else 363fa9e4066Sahrens zio->io_vdev_tree = &vq->vq_write_tree; 364fa9e4066Sahrens 365fa9e4066Sahrens mutex_enter(&vq->vq_lock); 366fa9e4066Sahrens 367*283b8460SGeorge.Wilson zio->io_timestamp = ddi_get_lbolt64(); 368*283b8460SGeorge.Wilson zio->io_deadline = (zio->io_timestamp >> zfs_vdev_time_shift) + 369d3d50737SRafael Vanoni zio->io_priority; 370fa9e4066Sahrens 371ea8dc4b6Seschrock vdev_queue_io_add(vq, zio); 372fa9e4066Sahrens 373e05725b1Sbonwick nio = vdev_queue_io_to_issue(vq, zfs_vdev_min_pending); 374fa9e4066Sahrens 375fa9e4066Sahrens mutex_exit(&vq->vq_lock); 376fa9e4066Sahrens 377e05725b1Sbonwick if (nio == NULL) 378e05725b1Sbonwick return (NULL); 379e05725b1Sbonwick 380e05725b1Sbonwick if (nio->io_done == vdev_queue_agg_io_done) { 381e05725b1Sbonwick zio_nowait(nio); 382e05725b1Sbonwick return (NULL); 383e05725b1Sbonwick } 384fa9e4066Sahrens 385e05725b1Sbonwick return (nio); 386fa9e4066Sahrens } 387fa9e4066Sahrens 388fa9e4066Sahrens void 389fa9e4066Sahrens vdev_queue_io_done(zio_t *zio) 390fa9e4066Sahrens { 391fa9e4066Sahrens vdev_queue_t *vq = &zio->io_vd->vdev_queue; 392fa9e4066Sahrens 393*283b8460SGeorge.Wilson if (zio_injection_enabled) 394*283b8460SGeorge.Wilson delay(SEC_TO_TICK(zio_handle_io_delay(zio))); 395*283b8460SGeorge.Wilson 396fa9e4066Sahrens mutex_enter(&vq->vq_lock); 397fa9e4066Sahrens 398fa9e4066Sahrens avl_remove(&vq->vq_pending_tree, zio); 399fa9e4066Sahrens 400*283b8460SGeorge.Wilson vq->vq_io_complete_ts = ddi_get_lbolt64(); 401*283b8460SGeorge.Wilson vq->vq_io_delta_ts = vq->vq_io_complete_ts - zio->io_timestamp; 402*283b8460SGeorge.Wilson 403e14bb325SJeff Bonwick for (int i = 0; i < zfs_vdev_ramp_rate; i++) { 404e14bb325SJeff Bonwick zio_t *nio = vdev_queue_io_to_issue(vq, zfs_vdev_max_pending); 405fa9e4066Sahrens if (nio == NULL) 406fa9e4066Sahrens break; 407fa9e4066Sahrens mutex_exit(&vq->vq_lock); 408e05725b1Sbonwick if (nio->io_done == vdev_queue_agg_io_done) { 409e05725b1Sbonwick zio_nowait(nio); 410e05725b1Sbonwick } else { 411fa9e4066Sahrens zio_vdev_io_reissue(nio); 412e05725b1Sbonwick zio_execute(nio); 413e05725b1Sbonwick } 414fa9e4066Sahrens mutex_enter(&vq->vq_lock); 415fa9e4066Sahrens } 416fa9e4066Sahrens 417fa9e4066Sahrens mutex_exit(&vq->vq_lock); 418fa9e4066Sahrens } 419