xref: /illumos-gate/usr/src/uts/common/fs/zfs/vdev_trim.c (revision af1d63ab)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2016 by Delphix. All rights reserved.
24  * Copyright (c) 2019 by Lawrence Livermore National Security, LLC.
25  * Copyright 2019 Joyent, Inc.
26  */
27 
28 #include <sys/spa.h>
29 #include <sys/spa_impl.h>
30 #include <sys/txg.h>
31 #include <sys/vdev_impl.h>
32 #include <sys/vdev_trim.h>
33 #include <sys/refcount.h>
34 #include <sys/metaslab_impl.h>
35 #include <sys/dsl_synctask.h>
36 #include <sys/zap.h>
37 #include <sys/dmu_tx.h>
38 
39 /*
40  * TRIM is a feature which is used to notify a SSD that some previously
41  * written space is no longer allocated by the pool.  This is useful because
42  * writes to a SSD must be performed to blocks which have first been erased.
43  * Ensuring the SSD always has a supply of erased blocks for new writes
44  * helps prevent the performance from deteriorating.
45  *
46  * There are two supported TRIM methods; manual and automatic.
47  *
48  * Manual TRIM:
49  *
50  * A manual TRIM is initiated by running the 'zpool trim' command.  A single
51  * 'vdev_trim' thread is created for each leaf vdev, and it is responsible for
52  * managing that vdev TRIM process.  This involves iterating over all the
53  * metaslabs, calculating the unallocated space ranges, and then issuing the
54  * required TRIM I/Os.
55  *
56  * While a metaslab is being actively trimmed it is not eligible to perform
57  * new allocations.  After traversing all of the metaslabs the thread is
58  * terminated.  Finally, both the requested options and current progress of
59  * the TRIM are regularly written to the pool.  This allows the TRIM to be
60  * suspended and resumed as needed.
61  *
62  * Automatic TRIM:
63  *
64  * An automatic TRIM is enabled by setting the 'autotrim' pool property
65  * to 'on'.  When enabled, a `vdev_autotrim' thread is created for each
66  * top-level (not leaf) vdev in the pool.  These threads perform the same
67  * core TRIM process as a manual TRIM, but with a few key differences.
68  *
69  * 1) Automatic TRIM happens continuously in the background and operates
70  *    solely on recently freed blocks (ms_trim not ms_allocatable).
71  *
72  * 2) Each thread is associated with a top-level (not leaf) vdev.  This has
73  *    the benefit of simplifying the threading model, it makes it easier
74  *    to coordinate administrative commands, and it ensures only a single
75  *    metaslab is disabled at a time.  Unlike manual TRIM, this means each
76  *    'vdev_autotrim' thread is responsible for issuing TRIM I/Os for its
77  *    children.
78  *
79  * 3) There is no automatic TRIM progress information stored on disk, nor
80  *    is it reported by 'zpool status'.
81  *
82  * While the automatic TRIM process is highly effective it is more likely
83  * than a manual TRIM to encounter tiny ranges.  Ranges less than or equal to
84  * 'zfs_trim_extent_bytes_min' (32k) are considered too small to efficiently
85  * TRIM and are skipped.  This means small amounts of freed space may not
86  * be automatically trimmed.
87  *
88  * Furthermore, devices with attached hot spares and devices being actively
89  * replaced are skipped.  This is done to avoid adding additional stress to
90  * a potentially unhealthy device and to minimize the required rebuild time.
91  *
92  * For this reason it may be beneficial to occasionally manually TRIM a pool
93  * even when automatic TRIM is enabled.
94  */
95 
96 /*
97  * Maximum size of TRIM I/O, ranges will be chunked in to 128MiB lengths.
98  */
99 unsigned int zfs_trim_extent_bytes_max = 128 * 1024 * 1024;
100 
101 /*
102  * Minimum size of TRIM I/O, extents smaller than 32Kib will be skipped.
103  */
104 unsigned int zfs_trim_extent_bytes_min = 32 * 1024;
105 
106 /*
107  * Skip uninitialized metaslabs during the TRIM process.  This option is
108  * useful for pools constructed from large thinly-provisioned devices where
109  * TRIM operations are slow.  As a pool ages an increasing fraction of
110  * the pools metaslabs will be initialized progressively degrading the
111  * usefulness of this option.  This setting is stored when starting a
112  * manual TRIM and will persist for the duration of the requested TRIM.
113  */
114 unsigned int zfs_trim_metaslab_skip = 0;
115 
116 /*
117  * Maximum number of queued TRIM I/Os per leaf vdev.  The number of
118  * concurrent TRIM I/Os issued to the device is controlled by the
119  * zfs_vdev_trim_min_active and zfs_vdev_trim_max_active module options.
120  */
121 unsigned int zfs_trim_queue_limit = 10;
122 
123 /*
124  * The minimum number of transaction groups between automatic trims of a
125  * metaslab.  This setting represents a trade-off between issuing more
126  * efficient TRIM operations, by allowing them to be aggregated longer,
127  * and issuing them promptly so the trimmed space is available.  Note
128  * that this value is a minimum; metaslabs can be trimmed less frequently
129  * when there are a large number of ranges which need to be trimmed.
130  *
131  * Increasing this value will allow frees to be aggregated for a longer
132  * time.  This can result is larger TRIM operations, and increased memory
133  * usage in order to track the ranges to be trimmed.  Decreasing this value
134  * has the opposite effect.  The default value of 32 was determined though
135  * testing to be a reasonable compromise.
136  */
137 unsigned int zfs_trim_txg_batch = 32;
138 
139 /*
140  * The trim_args are a control structure which describe how a leaf vdev
141  * should be trimmed.  The core elements are the vdev, the metaslab being
142  * trimmed and a range tree containing the extents to TRIM.  All provided
143  * ranges must be within the metaslab.
144  */
145 typedef struct trim_args {
146 	/*
147 	 * These fields are set by the caller of vdev_trim_ranges().
148 	 */
149 	vdev_t		*trim_vdev;		/* Leaf vdev to TRIM */
150 	metaslab_t	*trim_msp;		/* Disabled metaslab */
151 	range_tree_t	*trim_tree;		/* TRIM ranges (in metaslab) */
152 	trim_type_t	trim_type;		/* Manual or auto TRIM */
153 	uint64_t	trim_extent_bytes_max;	/* Maximum TRIM I/O size */
154 	uint64_t	trim_extent_bytes_min;	/* Minimum TRIM I/O size */
155 	enum trim_flag	trim_flags;		/* TRIM flags (secure) */
156 
157 	/*
158 	 * These fields are updated by vdev_trim_ranges().
159 	 */
160 	hrtime_t	trim_start_time;	/* Start time */
161 	uint64_t	trim_bytes_done;	/* Bytes trimmed */
162 } trim_args_t;
163 
164 /*
165  * Determines whether a vdev_trim_thread() should be stopped.
166  */
167 static boolean_t
168 vdev_trim_should_stop(vdev_t *vd)
169 {
170 	return (vd->vdev_trim_exit_wanted || !vdev_writeable(vd) ||
171 	    vd->vdev_detached || vd->vdev_top->vdev_removing);
172 }
173 
174 /*
175  * Determines whether a vdev_autotrim_thread() should be stopped.
176  */
177 static boolean_t
178 vdev_autotrim_should_stop(vdev_t *tvd)
179 {
180 	return (tvd->vdev_autotrim_exit_wanted ||
181 	    !vdev_writeable(tvd) || tvd->vdev_removing ||
182 	    spa_get_autotrim(tvd->vdev_spa) == SPA_AUTOTRIM_OFF);
183 }
184 
185 /*
186  * The sync task for updating the on-disk state of a manual TRIM.  This
187  * is scheduled by vdev_trim_change_state().
188  */
189 static void
190 vdev_trim_zap_update_sync(void *arg, dmu_tx_t *tx)
191 {
192 	/*
193 	 * We pass in the guid instead of the vdev_t since the vdev may
194 	 * have been freed prior to the sync task being processed.  This
195 	 * happens when a vdev is detached as we call spa_config_vdev_exit(),
196 	 * stop the trimming thread, schedule the sync task, and free
197 	 * the vdev. Later when the scheduled sync task is invoked, it would
198 	 * find that the vdev has been freed.
199 	 */
200 	uint64_t guid = *(uint64_t *)arg;
201 	uint64_t txg = dmu_tx_get_txg(tx);
202 	kmem_free(arg, sizeof (uint64_t));
203 
204 	vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);
205 	if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd))
206 		return;
207 
208 	uint64_t last_offset = vd->vdev_trim_offset[txg & TXG_MASK];
209 	vd->vdev_trim_offset[txg & TXG_MASK] = 0;
210 
211 	VERIFY3U(vd->vdev_leaf_zap, !=, 0);
212 
213 	objset_t *mos = vd->vdev_spa->spa_meta_objset;
214 
215 	if (last_offset > 0 || vd->vdev_trim_last_offset == UINT64_MAX) {
216 
217 		if (vd->vdev_trim_last_offset == UINT64_MAX)
218 			last_offset = 0;
219 
220 		vd->vdev_trim_last_offset = last_offset;
221 		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
222 		    VDEV_LEAF_ZAP_TRIM_LAST_OFFSET,
223 		    sizeof (last_offset), 1, &last_offset, tx));
224 	}
225 
226 	if (vd->vdev_trim_action_time > 0) {
227 		uint64_t val = (uint64_t)vd->vdev_trim_action_time;
228 		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
229 		    VDEV_LEAF_ZAP_TRIM_ACTION_TIME, sizeof (val),
230 		    1, &val, tx));
231 	}
232 
233 	if (vd->vdev_trim_rate > 0) {
234 		uint64_t rate = (uint64_t)vd->vdev_trim_rate;
235 
236 		if (rate == UINT64_MAX)
237 			rate = 0;
238 
239 		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
240 		    VDEV_LEAF_ZAP_TRIM_RATE, sizeof (rate), 1, &rate, tx));
241 	}
242 
243 	uint64_t partial = vd->vdev_trim_partial;
244 	if (partial == UINT64_MAX)
245 		partial = 0;
246 
247 	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,
248 	    sizeof (partial), 1, &partial, tx));
249 
250 	uint64_t secure = vd->vdev_trim_secure;
251 	if (secure == UINT64_MAX)
252 		secure = 0;
253 
254 	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,
255 	    sizeof (secure), 1, &secure, tx));
256 
257 
258 	uint64_t trim_state = vd->vdev_trim_state;
259 	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,
260 	    sizeof (trim_state), 1, &trim_state, tx));
261 }
262 
263 /*
264  * Update the on-disk state of a manual TRIM.  This is called to request
265  * that a TRIM be started/suspended/canceled, or to change one of the
266  * TRIM options (partial, secure, rate).
267  */
268 static void
269 vdev_trim_change_state(vdev_t *vd, vdev_trim_state_t new_state,
270     uint64_t rate, boolean_t partial, boolean_t secure)
271 {
272 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
273 	spa_t *spa = vd->vdev_spa;
274 
275 	if (new_state == vd->vdev_trim_state)
276 		return;
277 
278 	/*
279 	 * Copy the vd's guid, this will be freed by the sync task.
280 	 */
281 	uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
282 	*guid = vd->vdev_guid;
283 
284 	/*
285 	 * If we're suspending, then preserve the original start time.
286 	 */
287 	if (vd->vdev_trim_state != VDEV_TRIM_SUSPENDED) {
288 		vd->vdev_trim_action_time = gethrestime_sec();
289 	}
290 
291 	/*
292 	 * If we're activating, then preserve the requested rate and trim
293 	 * method.  Setting the last offset and rate to UINT64_MAX is used
294 	 * as a sentinel to indicate they should be reset to default values.
295 	 */
296 	if (new_state == VDEV_TRIM_ACTIVE) {
297 		if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE ||
298 		    vd->vdev_trim_state == VDEV_TRIM_CANCELED) {
299 			vd->vdev_trim_last_offset = UINT64_MAX;
300 			vd->vdev_trim_rate = UINT64_MAX;
301 			vd->vdev_trim_partial = UINT64_MAX;
302 			vd->vdev_trim_secure = UINT64_MAX;
303 		}
304 
305 		if (rate != 0)
306 			vd->vdev_trim_rate = rate;
307 
308 		if (partial != 0)
309 			vd->vdev_trim_partial = partial;
310 
311 		if (secure != 0)
312 			vd->vdev_trim_secure = secure;
313 	}
314 
315 	boolean_t resumed = !!(vd->vdev_trim_state == VDEV_TRIM_SUSPENDED);
316 	vd->vdev_trim_state = new_state;
317 
318 	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
319 	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
320 	dsl_sync_task_nowait(spa_get_dsl(spa), vdev_trim_zap_update_sync,
321 	    guid, 2, ZFS_SPACE_CHECK_NONE, tx);
322 
323 	switch (new_state) {
324 	case VDEV_TRIM_ACTIVE:
325 		spa_event_notify(spa, vd, NULL,
326 		    resumed ? ESC_ZFS_TRIM_RESUME : ESC_ZFS_TRIM_START);
327 		spa_history_log_internal(spa, "trim", tx,
328 		    "vdev=%s activated", vd->vdev_path);
329 		break;
330 	case VDEV_TRIM_SUSPENDED:
331 		spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_SUSPEND);
332 		spa_history_log_internal(spa, "trim", tx,
333 		    "vdev=%s suspended", vd->vdev_path);
334 		break;
335 	case VDEV_TRIM_CANCELED:
336 		spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_CANCEL);
337 		spa_history_log_internal(spa, "trim", tx,
338 		    "vdev=%s canceled", vd->vdev_path);
339 		break;
340 	case VDEV_TRIM_COMPLETE:
341 		spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_FINISH);
342 		spa_history_log_internal(spa, "trim", tx,
343 		    "vdev=%s complete", vd->vdev_path);
344 		break;
345 	default:
346 		panic("invalid state %llu", (unsigned long long)new_state);
347 	}
348 
349 	dmu_tx_commit(tx);
350 }
351 
352 /*
353  * The zio_done_func_t done callback for each manual TRIM issued.  It is
354  * responsible for updating the TRIM stats, reissuing failed TRIM I/Os,
355  * and limiting the number of in-flight TRIM I/Os.
356  */
357 static void
358 vdev_trim_cb(zio_t *zio)
359 {
360 	vdev_t *vd = zio->io_vd;
361 
362 	mutex_enter(&vd->vdev_trim_io_lock);
363 	if (zio->io_error == ENXIO && !vdev_writeable(vd)) {
364 		/*
365 		 * The I/O failed because the vdev was unavailable; roll the
366 		 * last offset back. (This works because spa_sync waits on
367 		 * spa_txg_zio before it runs sync tasks.)
368 		 */
369 		uint64_t *offset =
370 		    &vd->vdev_trim_offset[zio->io_txg & TXG_MASK];
371 		*offset = MIN(*offset, zio->io_offset);
372 	} else {
373 		if (zio->io_error != 0) {
374 			vd->vdev_stat.vs_trim_errors++;
375 			/*
376 			 * spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,
377 			 *  0, 0, 0, 0, 1, zio->io_orig_size);
378 			 */
379 		} else {
380 			/*
381 			 * spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,
382 			 *  1, zio->io_orig_size, 0, 0, 0, 0);
383 			 */
384 		}
385 
386 		vd->vdev_trim_bytes_done += zio->io_orig_size;
387 	}
388 
389 	ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_MANUAL], >, 0);
390 	vd->vdev_trim_inflight[TRIM_TYPE_MANUAL]--;
391 	cv_broadcast(&vd->vdev_trim_io_cv);
392 	mutex_exit(&vd->vdev_trim_io_lock);
393 
394 	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
395 }
396 
397 /*
398  * The zio_done_func_t done callback for each automatic TRIM issued.  It
399  * is responsible for updating the TRIM stats and limiting the number of
400  * in-flight TRIM I/Os.  Automatic TRIM I/Os are best effort and are
401  * never reissued on failure.
402  */
403 static void
404 vdev_autotrim_cb(zio_t *zio)
405 {
406 	vdev_t *vd = zio->io_vd;
407 
408 	mutex_enter(&vd->vdev_trim_io_lock);
409 
410 	if (zio->io_error != 0) {
411 		vd->vdev_stat.vs_trim_errors++;
412 		/*
413 		 * spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,
414 		 *  0, 0, 0, 0, 1, zio->io_orig_size);
415 		 */
416 	} else {
417 		/*
418 		 * spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,
419 		 *  1, zio->io_orig_size, 0, 0, 0, 0);
420 		 */
421 
422 		vd->vdev_autotrim_bytes_done += zio->io_orig_size;
423 	}
424 
425 	ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_AUTO], >, 0);
426 	vd->vdev_trim_inflight[TRIM_TYPE_AUTO]--;
427 	cv_broadcast(&vd->vdev_trim_io_cv);
428 	mutex_exit(&vd->vdev_trim_io_lock);
429 
430 	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
431 }
432 
433 /*
434  * Returns the average trim rate in bytes/sec for the ta->trim_vdev.
435  */
436 static uint64_t
437 vdev_trim_calculate_rate(trim_args_t *ta)
438 {
439 	return (ta->trim_bytes_done * 1000 /
440 	    (NSEC2MSEC(gethrtime() - ta->trim_start_time) + 1));
441 }
442 
443 /*
444  * Issues a physical TRIM and takes care of rate limiting (bytes/sec)
445  * and number of concurrent TRIM I/Os.
446  */
447 static int
448 vdev_trim_range(trim_args_t *ta, uint64_t start, uint64_t size)
449 {
450 	vdev_t *vd = ta->trim_vdev;
451 	spa_t *spa = vd->vdev_spa;
452 
453 	mutex_enter(&vd->vdev_trim_io_lock);
454 
455 	/*
456 	 * Limit manual TRIM I/Os to the requested rate.  This does not
457 	 * apply to automatic TRIM since no per vdev rate can be specified.
458 	 */
459 	if (ta->trim_type == TRIM_TYPE_MANUAL) {
460 		while (vd->vdev_trim_rate != 0 && !vdev_trim_should_stop(vd) &&
461 		    vdev_trim_calculate_rate(ta) > vd->vdev_trim_rate) {
462 			cv_timedwait_sig(&vd->vdev_trim_io_cv,
463 			    &vd->vdev_trim_io_lock, ddi_get_lbolt() +
464 			    MSEC_TO_TICK(10));
465 		}
466 	}
467 	ta->trim_bytes_done += size;
468 
469 	/* Limit in-flight trimming I/Os */
470 	while (vd->vdev_trim_inflight[0] + vd->vdev_trim_inflight[1] >=
471 	    zfs_trim_queue_limit) {
472 		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
473 	}
474 	vd->vdev_trim_inflight[ta->trim_type]++;
475 	mutex_exit(&vd->vdev_trim_io_lock);
476 
477 	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
478 	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
479 	uint64_t txg = dmu_tx_get_txg(tx);
480 
481 	spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER);
482 	mutex_enter(&vd->vdev_trim_lock);
483 
484 	if (ta->trim_type == TRIM_TYPE_MANUAL &&
485 	    vd->vdev_trim_offset[txg & TXG_MASK] == 0) {
486 		uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
487 		*guid = vd->vdev_guid;
488 
489 		/* This is the first write of this txg. */
490 		dsl_sync_task_nowait(spa_get_dsl(spa),
491 		    vdev_trim_zap_update_sync, guid, 2,
492 		    ZFS_SPACE_CHECK_RESERVED, tx);
493 	}
494 
495 	/*
496 	 * We know the vdev_t will still be around since all consumers of
497 	 * vdev_free must stop the trimming first.
498 	 */
499 	if ((ta->trim_type == TRIM_TYPE_MANUAL &&
500 	    vdev_trim_should_stop(vd)) ||
501 	    (ta->trim_type == TRIM_TYPE_AUTO &&
502 	    vdev_autotrim_should_stop(vd->vdev_top))) {
503 		mutex_enter(&vd->vdev_trim_io_lock);
504 		vd->vdev_trim_inflight[ta->trim_type]--;
505 		mutex_exit(&vd->vdev_trim_io_lock);
506 		spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
507 		mutex_exit(&vd->vdev_trim_lock);
508 		dmu_tx_commit(tx);
509 		return (SET_ERROR(EINTR));
510 	}
511 	mutex_exit(&vd->vdev_trim_lock);
512 
513 	if (ta->trim_type == TRIM_TYPE_MANUAL)
514 		vd->vdev_trim_offset[txg & TXG_MASK] = start + size;
515 
516 	zio_nowait(zio_trim(spa->spa_txg_zio[txg & TXG_MASK], vd,
517 	    start, size, ta->trim_type == TRIM_TYPE_MANUAL ?
518 	    vdev_trim_cb : vdev_autotrim_cb, NULL,
519 	    ZIO_PRIORITY_TRIM, ZIO_FLAG_CANFAIL, ta->trim_flags));
520 	/* vdev_trim_cb and vdev_autotrim_cb release SCL_STATE_ALL */
521 
522 	dmu_tx_commit(tx);
523 
524 	return (0);
525 }
526 
527 /*
528  * Issues TRIM I/Os for all ranges in the provided ta->trim_tree range tree.
529  * Additional parameters describing how the TRIM should be performed must
530  * be set in the trim_args structure.  See the trim_args definition for
531  * additional information.
532  */
533 static int
534 vdev_trim_ranges(trim_args_t *ta)
535 {
536 	vdev_t *vd = ta->trim_vdev;
537 	avl_tree_t *rt = &ta->trim_tree->rt_root;
538 	uint64_t extent_bytes_max = ta->trim_extent_bytes_max;
539 	uint64_t extent_bytes_min = ta->trim_extent_bytes_min;
540 	spa_t *spa = vd->vdev_spa;
541 
542 	ta->trim_start_time = gethrtime();
543 	ta->trim_bytes_done = 0;
544 
545 	for (range_seg_t *rs = avl_first(rt); rs != NULL;
546 	    rs = AVL_NEXT(rt, rs)) {
547 		uint64_t size = rs->rs_end - rs->rs_start;
548 
549 		if (extent_bytes_min && size < extent_bytes_min) {
550 			/*
551 			 * spa_iostats_trim_add(spa, ta->trim_type,
552 			 *  0, 0, 1, size, 0, 0);
553 			 */
554 			continue;
555 		}
556 
557 		/* Split range into legally-sized physical chunks */
558 		uint64_t writes_required = ((size - 1) / extent_bytes_max) + 1;
559 
560 		for (uint64_t w = 0; w < writes_required; w++) {
561 			int error;
562 
563 			error = vdev_trim_range(ta, VDEV_LABEL_START_SIZE +
564 			    rs->rs_start + (w * extent_bytes_max),
565 			    MIN(size - (w * extent_bytes_max),
566 			    extent_bytes_max));
567 			if (error != 0) {
568 				return (error);
569 			}
570 		}
571 	}
572 
573 	return (0);
574 }
575 
576 /*
577  * Calculates the completion percentage of a manual TRIM.
578  */
579 static void
580 vdev_trim_calculate_progress(vdev_t *vd)
581 {
582 	ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
583 	    spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
584 	ASSERT(vd->vdev_leaf_zap != 0);
585 
586 	vd->vdev_trim_bytes_est = 0;
587 	vd->vdev_trim_bytes_done = 0;
588 
589 	for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) {
590 		metaslab_t *msp = vd->vdev_top->vdev_ms[i];
591 		mutex_enter(&msp->ms_lock);
592 
593 		uint64_t ms_free = msp->ms_size -
594 		    metaslab_allocated_space(msp);
595 
596 		if (vd->vdev_top->vdev_ops == &vdev_raidz_ops)
597 			ms_free /= vd->vdev_top->vdev_children;
598 
599 		/*
600 		 * Convert the metaslab range to a physical range
601 		 * on our vdev. We use this to determine if we are
602 		 * in the middle of this metaslab range.
603 		 */
604 		range_seg_t logical_rs, physical_rs;
605 		logical_rs.rs_start = msp->ms_start;
606 		logical_rs.rs_end = msp->ms_start + msp->ms_size;
607 		vdev_xlate(vd, &logical_rs, &physical_rs);
608 
609 		if (vd->vdev_trim_last_offset <= physical_rs.rs_start) {
610 			vd->vdev_trim_bytes_est += ms_free;
611 			mutex_exit(&msp->ms_lock);
612 			continue;
613 		} else if (vd->vdev_trim_last_offset > physical_rs.rs_end) {
614 			vd->vdev_trim_bytes_done += ms_free;
615 			vd->vdev_trim_bytes_est += ms_free;
616 			mutex_exit(&msp->ms_lock);
617 			continue;
618 		}
619 
620 		/*
621 		 * If we get here, we're in the middle of trimming this
622 		 * metaslab.  Load it and walk the free tree for more
623 		 * accurate progress estimation.
624 		 */
625 		VERIFY0(metaslab_load(msp));
626 
627 		for (range_seg_t *rs = avl_first(&msp->ms_allocatable->rt_root);
628 		    rs; rs = AVL_NEXT(&msp->ms_allocatable->rt_root, rs)) {
629 			logical_rs.rs_start = rs->rs_start;
630 			logical_rs.rs_end = rs->rs_end;
631 			vdev_xlate(vd, &logical_rs, &physical_rs);
632 
633 			uint64_t size = physical_rs.rs_end -
634 			    physical_rs.rs_start;
635 			vd->vdev_trim_bytes_est += size;
636 			if (vd->vdev_trim_last_offset >= physical_rs.rs_end) {
637 				vd->vdev_trim_bytes_done += size;
638 			} else if (vd->vdev_trim_last_offset >
639 			    physical_rs.rs_start &&
640 			    vd->vdev_trim_last_offset <=
641 			    physical_rs.rs_end) {
642 				vd->vdev_trim_bytes_done +=
643 				    vd->vdev_trim_last_offset -
644 				    physical_rs.rs_start;
645 			}
646 		}
647 		mutex_exit(&msp->ms_lock);
648 	}
649 }
650 
651 /*
652  * Load from disk the vdev's manual TRIM information.  This includes the
653  * state, progress, and options provided when initiating the manual TRIM.
654  */
655 static int
656 vdev_trim_load(vdev_t *vd)
657 {
658 	int err = 0;
659 	ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
660 	    spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
661 	ASSERT(vd->vdev_leaf_zap != 0);
662 
663 	if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE ||
664 	    vd->vdev_trim_state == VDEV_TRIM_SUSPENDED) {
665 		err = zap_lookup(vd->vdev_spa->spa_meta_objset,
666 		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_LAST_OFFSET,
667 		    sizeof (vd->vdev_trim_last_offset), 1,
668 		    &vd->vdev_trim_last_offset);
669 		if (err == ENOENT) {
670 			vd->vdev_trim_last_offset = 0;
671 			err = 0;
672 		}
673 
674 		if (err == 0) {
675 			err = zap_lookup(vd->vdev_spa->spa_meta_objset,
676 			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_RATE,
677 			    sizeof (vd->vdev_trim_rate), 1,
678 			    &vd->vdev_trim_rate);
679 			if (err == ENOENT) {
680 				vd->vdev_trim_rate = 0;
681 				err = 0;
682 			}
683 		}
684 
685 		if (err == 0) {
686 			err = zap_lookup(vd->vdev_spa->spa_meta_objset,
687 			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,
688 			    sizeof (vd->vdev_trim_partial), 1,
689 			    &vd->vdev_trim_partial);
690 			if (err == ENOENT) {
691 				vd->vdev_trim_partial = 0;
692 				err = 0;
693 			}
694 		}
695 
696 		if (err == 0) {
697 			err = zap_lookup(vd->vdev_spa->spa_meta_objset,
698 			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,
699 			    sizeof (vd->vdev_trim_secure), 1,
700 			    &vd->vdev_trim_secure);
701 			if (err == ENOENT) {
702 				vd->vdev_trim_secure = 0;
703 				err = 0;
704 			}
705 		}
706 	}
707 
708 	vdev_trim_calculate_progress(vd);
709 
710 	return (err);
711 }
712 
713 /*
714  * Convert the logical range into a physical range and add it to the
715  * range tree passed in the trim_args_t.
716  */
717 static void
718 vdev_trim_range_add(void *arg, uint64_t start, uint64_t size)
719 {
720 	trim_args_t *ta = arg;
721 	vdev_t *vd = ta->trim_vdev;
722 	range_seg_t logical_rs, physical_rs;
723 	logical_rs.rs_start = start;
724 	logical_rs.rs_end = start + size;
725 
726 	/*
727 	 * Every range to be trimmed must be part of ms_allocatable.
728 	 * When ZFS_DEBUG_TRIM is set load the metaslab to verify this
729 	 * is always the case.
730 	 */
731 	if (zfs_flags & ZFS_DEBUG_TRIM) {
732 		metaslab_t *msp = ta->trim_msp;
733 		VERIFY0(metaslab_load(msp));
734 		VERIFY3B(msp->ms_loaded, ==, B_TRUE);
735 		VERIFY(range_tree_find(msp->ms_allocatable, start, size));
736 	}
737 
738 	ASSERT(vd->vdev_ops->vdev_op_leaf);
739 	vdev_xlate(vd, &logical_rs, &physical_rs);
740 
741 	IMPLY(vd->vdev_top == vd,
742 	    logical_rs.rs_start == physical_rs.rs_start);
743 	IMPLY(vd->vdev_top == vd,
744 	    logical_rs.rs_end == physical_rs.rs_end);
745 
746 	/*
747 	 * Only a manual trim will be traversing the vdev sequentially.
748 	 * For an auto trim all valid ranges should be added.
749 	 */
750 	if (ta->trim_type == TRIM_TYPE_MANUAL) {
751 
752 		/* Only add segments that we have not visited yet */
753 		if (physical_rs.rs_end <= vd->vdev_trim_last_offset)
754 			return;
755 
756 		/* Pick up where we left off mid-range. */
757 		if (vd->vdev_trim_last_offset > physical_rs.rs_start) {
758 			ASSERT3U(physical_rs.rs_end, >,
759 			    vd->vdev_trim_last_offset);
760 			physical_rs.rs_start = vd->vdev_trim_last_offset;
761 		}
762 	}
763 
764 	ASSERT3U(physical_rs.rs_end, >=, physical_rs.rs_start);
765 
766 	/*
767 	 * With raidz, it's possible that the logical range does not live on
768 	 * this leaf vdev. We only add the physical range to this vdev's if it
769 	 * has a length greater than 0.
770 	 */
771 	if (physical_rs.rs_end > physical_rs.rs_start) {
772 		range_tree_add(ta->trim_tree, physical_rs.rs_start,
773 		    physical_rs.rs_end - physical_rs.rs_start);
774 	} else {
775 		ASSERT3U(physical_rs.rs_end, ==, physical_rs.rs_start);
776 	}
777 }
778 
779 /*
780  * Each manual TRIM thread is responsible for trimming the unallocated
781  * space for each leaf vdev.  This is accomplished by sequentially iterating
782  * over its top-level metaslabs and issuing TRIM I/O for the space described
783  * by its ms_allocatable.  While a metaslab is undergoing trimming it is
784  * not eligible for new allocations.
785  */
786 static void
787 vdev_trim_thread(void *arg)
788 {
789 	vdev_t *vd = arg;
790 	spa_t *spa = vd->vdev_spa;
791 	trim_args_t ta;
792 	int error = 0;
793 
794 	/*
795 	 * The VDEV_LEAF_ZAP_TRIM_* entries may have been updated by
796 	 * vdev_trim().  Wait for the updated values to be reflected
797 	 * in the zap in order to start with the requested settings.
798 	 */
799 	txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
800 
801 	ASSERT(vdev_is_concrete(vd));
802 	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
803 
804 	vd->vdev_trim_last_offset = 0;
805 	vd->vdev_trim_rate = 0;
806 	vd->vdev_trim_partial = 0;
807 	vd->vdev_trim_secure = 0;
808 
809 	VERIFY0(vdev_trim_load(vd));
810 
811 	ta.trim_vdev = vd;
812 	ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max;
813 	ta.trim_extent_bytes_min = zfs_trim_extent_bytes_min;
814 	ta.trim_tree = range_tree_create(NULL, NULL);
815 	ta.trim_type = TRIM_TYPE_MANUAL;
816 	ta.trim_flags = 0;
817 
818 	/*
819 	 * When a secure TRIM has been requested infer that the intent
820 	 * is that everything must be trimmed.  Override the default
821 	 * minimum TRIM size to prevent ranges from being skipped.
822 	 */
823 	if (vd->vdev_trim_secure) {
824 		ta.trim_flags |= ZIO_TRIM_SECURE;
825 		ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE;
826 	}
827 
828 	uint64_t ms_count = 0;
829 	for (uint64_t i = 0; !vd->vdev_detached &&
830 	    i < vd->vdev_top->vdev_ms_count; i++) {
831 		metaslab_t *msp = vd->vdev_top->vdev_ms[i];
832 
833 		/*
834 		 * If we've expanded the top-level vdev or it's our
835 		 * first pass, calculate our progress.
836 		 */
837 		if (vd->vdev_top->vdev_ms_count != ms_count) {
838 			vdev_trim_calculate_progress(vd);
839 			ms_count = vd->vdev_top->vdev_ms_count;
840 		}
841 
842 		spa_config_exit(spa, SCL_CONFIG, FTAG);
843 		metaslab_disable(msp);
844 		mutex_enter(&msp->ms_lock);
845 		VERIFY0(metaslab_load(msp));
846 
847 		/*
848 		 * If a partial TRIM was requested skip metaslabs which have
849 		 * never been initialized and thus have never been written.
850 		 */
851 		if (msp->ms_sm == NULL && vd->vdev_trim_partial) {
852 			mutex_exit(&msp->ms_lock);
853 			metaslab_enable(msp, B_FALSE, B_FALSE);
854 			spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
855 			vdev_trim_calculate_progress(vd);
856 			continue;
857 		}
858 
859 		ta.trim_msp = msp;
860 		range_tree_walk(msp->ms_allocatable, vdev_trim_range_add, &ta);
861 		range_tree_vacate(msp->ms_trim, NULL, NULL);
862 		mutex_exit(&msp->ms_lock);
863 
864 		error = vdev_trim_ranges(&ta);
865 		metaslab_enable(msp, B_TRUE, B_FALSE);
866 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
867 
868 		range_tree_vacate(ta.trim_tree, NULL, NULL);
869 		if (error != 0)
870 			break;
871 	}
872 
873 	spa_config_exit(spa, SCL_CONFIG, FTAG);
874 	mutex_enter(&vd->vdev_trim_io_lock);
875 	while (vd->vdev_trim_inflight[0] > 0) {
876 		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
877 	}
878 	mutex_exit(&vd->vdev_trim_io_lock);
879 
880 	range_tree_destroy(ta.trim_tree);
881 
882 	mutex_enter(&vd->vdev_trim_lock);
883 	if (!vd->vdev_trim_exit_wanted && vdev_writeable(vd)) {
884 		vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE,
885 		    vd->vdev_trim_rate, vd->vdev_trim_partial,
886 		    vd->vdev_trim_secure);
887 	}
888 	ASSERT(vd->vdev_trim_thread != NULL || vd->vdev_trim_inflight[0] == 0);
889 
890 	/*
891 	 * Drop the vdev_trim_lock while we sync out the txg since it's
892 	 * possible that a device might be trying to come online and must
893 	 * check to see if it needs to restart a trim. That thread will be
894 	 * holding the spa_config_lock which would prevent the txg_wait_synced
895 	 * from completing.
896 	 */
897 	mutex_exit(&vd->vdev_trim_lock);
898 	txg_wait_synced(spa_get_dsl(spa), 0);
899 	mutex_enter(&vd->vdev_trim_lock);
900 
901 	vd->vdev_trim_thread = NULL;
902 	cv_broadcast(&vd->vdev_trim_cv);
903 	mutex_exit(&vd->vdev_trim_lock);
904 }
905 
906 /*
907  * Initiates a manual TRIM for the vdev_t.  Callers must hold vdev_trim_lock,
908  * the vdev_t must be a leaf and cannot already be manually trimming.
909  */
910 void
911 vdev_trim(vdev_t *vd, uint64_t rate, boolean_t partial, boolean_t secure)
912 {
913 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
914 	ASSERT(vd->vdev_ops->vdev_op_leaf);
915 	ASSERT(vdev_is_concrete(vd));
916 	ASSERT3P(vd->vdev_trim_thread, ==, NULL);
917 	ASSERT(!vd->vdev_detached);
918 	ASSERT(!vd->vdev_trim_exit_wanted);
919 	ASSERT(!vd->vdev_top->vdev_removing);
920 
921 	vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, rate, partial, secure);
922 	vd->vdev_trim_thread = thread_create(NULL, 0,
923 	    vdev_trim_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
924 }
925 
926 /*
927  * Wait for the trimming thread to be terminated (canceled or stopped).
928  */
929 static void
930 vdev_trim_stop_wait_impl(vdev_t *vd)
931 {
932 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
933 
934 	while (vd->vdev_trim_thread != NULL)
935 		cv_wait(&vd->vdev_trim_cv, &vd->vdev_trim_lock);
936 
937 	ASSERT3P(vd->vdev_trim_thread, ==, NULL);
938 	vd->vdev_trim_exit_wanted = B_FALSE;
939 }
940 
941 /*
942  * Wait for vdev trim threads which were listed to cleanly exit.
943  */
944 void
945 vdev_trim_stop_wait(spa_t *spa, list_t *vd_list)
946 {
947 	vdev_t *vd;
948 
949 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
950 
951 	while ((vd = list_remove_head(vd_list)) != NULL) {
952 		mutex_enter(&vd->vdev_trim_lock);
953 		vdev_trim_stop_wait_impl(vd);
954 		mutex_exit(&vd->vdev_trim_lock);
955 	}
956 }
957 
958 /*
959  * Stop trimming a device, with the resultant trimming state being tgt_state.
960  * For blocking behavior pass NULL for vd_list.  Otherwise, when a list_t is
961  * provided the stopping vdev is inserted in to the list.  Callers are then
962  * required to call vdev_trim_stop_wait() to block for all the trim threads
963  * to exit.  The caller must hold vdev_trim_lock and must not be writing to
964  * the spa config, as the trimming thread may try to enter the config as a
965  * reader before exiting.
966  */
967 void
968 vdev_trim_stop(vdev_t *vd, vdev_trim_state_t tgt_state, list_t *vd_list)
969 {
970 	ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER));
971 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
972 	ASSERT(vd->vdev_ops->vdev_op_leaf);
973 	ASSERT(vdev_is_concrete(vd));
974 
975 	/*
976 	 * Allow cancel requests to proceed even if the trim thread has
977 	 * stopped.
978 	 */
979 	if (vd->vdev_trim_thread == NULL && tgt_state != VDEV_TRIM_CANCELED)
980 		return;
981 
982 	vdev_trim_change_state(vd, tgt_state, 0, 0, 0);
983 	vd->vdev_trim_exit_wanted = B_TRUE;
984 
985 	if (vd_list == NULL) {
986 		vdev_trim_stop_wait_impl(vd);
987 	} else {
988 		ASSERT(MUTEX_HELD(&spa_namespace_lock));
989 		list_insert_tail(vd_list, vd);
990 	}
991 }
992 
993 /*
994  * Requests that all listed vdevs stop trimming.
995  */
996 static void
997 vdev_trim_stop_all_impl(vdev_t *vd, vdev_trim_state_t tgt_state,
998     list_t *vd_list)
999 {
1000 	if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) {
1001 		mutex_enter(&vd->vdev_trim_lock);
1002 		vdev_trim_stop(vd, tgt_state, vd_list);
1003 		mutex_exit(&vd->vdev_trim_lock);
1004 		return;
1005 	}
1006 
1007 	for (uint64_t i = 0; i < vd->vdev_children; i++) {
1008 		vdev_trim_stop_all_impl(vd->vdev_child[i], tgt_state,
1009 		    vd_list);
1010 	}
1011 }
1012 
1013 /*
1014  * Convenience function to stop trimming of a vdev tree and set all trim
1015  * thread pointers to NULL.
1016  */
1017 void
1018 vdev_trim_stop_all(vdev_t *vd, vdev_trim_state_t tgt_state)
1019 {
1020 	spa_t *spa = vd->vdev_spa;
1021 	list_t vd_list;
1022 
1023 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1024 
1025 	list_create(&vd_list, sizeof (vdev_t),
1026 	    offsetof(vdev_t, vdev_trim_node));
1027 
1028 	vdev_trim_stop_all_impl(vd, tgt_state, &vd_list);
1029 	vdev_trim_stop_wait(spa, &vd_list);
1030 
1031 	if (vd->vdev_spa->spa_sync_on) {
1032 		/* Make sure that our state has been synced to disk */
1033 		txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
1034 	}
1035 
1036 	list_destroy(&vd_list);
1037 }
1038 
1039 /*
1040  * Conditionally restarts a manual TRIM given its on-disk state.
1041  */
1042 void
1043 vdev_trim_restart(vdev_t *vd)
1044 {
1045 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1046 	ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
1047 
1048 	if (vd->vdev_leaf_zap != 0) {
1049 		mutex_enter(&vd->vdev_trim_lock);
1050 		uint64_t trim_state = VDEV_TRIM_NONE;
1051 		int err = zap_lookup(vd->vdev_spa->spa_meta_objset,
1052 		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,
1053 		    sizeof (trim_state), 1, &trim_state);
1054 		ASSERT(err == 0 || err == ENOENT);
1055 		vd->vdev_trim_state = trim_state;
1056 
1057 		uint64_t timestamp = 0;
1058 		err = zap_lookup(vd->vdev_spa->spa_meta_objset,
1059 		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_ACTION_TIME,
1060 		    sizeof (timestamp), 1, &timestamp);
1061 		ASSERT(err == 0 || err == ENOENT);
1062 		vd->vdev_trim_action_time = (time_t)timestamp;
1063 
1064 		if (vd->vdev_trim_state == VDEV_TRIM_SUSPENDED ||
1065 		    vd->vdev_offline) {
1066 			/* load progress for reporting, but don't resume */
1067 			VERIFY0(vdev_trim_load(vd));
1068 		} else if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE &&
1069 		    vdev_writeable(vd) && !vd->vdev_top->vdev_removing &&
1070 		    vd->vdev_trim_thread == NULL) {
1071 			VERIFY0(vdev_trim_load(vd));
1072 			vdev_trim(vd, vd->vdev_trim_rate,
1073 			    vd->vdev_trim_partial, vd->vdev_trim_secure);
1074 		}
1075 
1076 		mutex_exit(&vd->vdev_trim_lock);
1077 	}
1078 
1079 	for (uint64_t i = 0; i < vd->vdev_children; i++) {
1080 		vdev_trim_restart(vd->vdev_child[i]);
1081 	}
1082 }
1083 
1084 /*
1085  * Used by the automatic TRIM when ZFS_DEBUG_TRIM is set to verify that
1086  * every TRIM range is contained within ms_allocatable.
1087  */
1088 static void
1089 vdev_trim_range_verify(void *arg, uint64_t start, uint64_t size)
1090 {
1091 	trim_args_t *ta = arg;
1092 	metaslab_t *msp = ta->trim_msp;
1093 
1094 	VERIFY3B(msp->ms_loaded, ==, B_TRUE);
1095 	VERIFY3U(msp->ms_disabled, >, 0);
1096 	VERIFY(range_tree_find(msp->ms_allocatable, start, size) != NULL);
1097 }
1098 
1099 /*
1100  * Each automatic TRIM thread is responsible for managing the trimming of a
1101  * top-level vdev in the pool.  No automatic TRIM state is maintained on-disk.
1102  *
1103  * N.B. This behavior is different from a manual TRIM where a thread
1104  * is created for each leaf vdev, instead of each top-level vdev.
1105  */
1106 static void
1107 vdev_autotrim_thread(void *arg)
1108 {
1109 	vdev_t *vd = arg;
1110 	spa_t *spa = vd->vdev_spa;
1111 	int shift = 0;
1112 
1113 	mutex_enter(&vd->vdev_autotrim_lock);
1114 	ASSERT3P(vd->vdev_top, ==, vd);
1115 	ASSERT3P(vd->vdev_autotrim_thread, !=, NULL);
1116 	mutex_exit(&vd->vdev_autotrim_lock);
1117 	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1118 
1119 	uint64_t extent_bytes_max = zfs_trim_extent_bytes_max;
1120 	uint64_t extent_bytes_min = zfs_trim_extent_bytes_min;
1121 
1122 	while (!vdev_autotrim_should_stop(vd)) {
1123 		int txgs_per_trim = MAX(zfs_trim_txg_batch, 1);
1124 		boolean_t issued_trim = B_FALSE;
1125 
1126 		/*
1127 		 * All of the metaslabs are divided in to groups of size
1128 		 * num_metaslabs / zfs_trim_txg_batch.  Each of these groups
1129 		 * is composed of metaslabs which are spread evenly over the
1130 		 * device.
1131 		 *
1132 		 * For example, when zfs_trim_txg_batch = 32 (default) then
1133 		 * group 0 will contain metaslabs 0, 32, 64, ...;
1134 		 * group 1 will contain metaslabs 1, 33, 65, ...;
1135 		 * group 2 will contain metaslabs 2, 34, 66, ...; and so on.
1136 		 *
1137 		 * On each pass through the while() loop one of these groups
1138 		 * is selected.  This is accomplished by using a shift value
1139 		 * to select the starting metaslab, then striding over the
1140 		 * metaslabs using the zfs_trim_txg_batch size.  This is
1141 		 * done to accomplish two things.
1142 		 *
1143 		 * 1) By dividing the metaslabs into groups, and making sure
1144 		 *    that each group takes a minimum of one txg to process.
1145 		 *    Then zfs_trim_txg_batch controls the minimum number of
1146 		 *    txgs which must occur before a metaslab is revisited.
1147 		 *
1148 		 * 2) Selecting non-consecutive metaslabs distributes the
1149 		 *    TRIM commands for a group evenly over the entire device.
1150 		 *    This can be advantageous for certain types of devices.
1151 		 */
1152 		for (uint64_t i = shift % txgs_per_trim; i < vd->vdev_ms_count;
1153 		    i += txgs_per_trim) {
1154 			metaslab_t *msp = vd->vdev_ms[i];
1155 			range_tree_t *trim_tree;
1156 
1157 			spa_config_exit(spa, SCL_CONFIG, FTAG);
1158 			metaslab_disable(msp);
1159 			spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1160 
1161 			mutex_enter(&msp->ms_lock);
1162 
1163 			/*
1164 			 * Skip the metaslab when it has never been allocated
1165 			 * or when there are no recent frees to trim.
1166 			 */
1167 			if (msp->ms_sm == NULL ||
1168 			    range_tree_is_empty(msp->ms_trim)) {
1169 				mutex_exit(&msp->ms_lock);
1170 				metaslab_enable(msp, B_FALSE, B_FALSE);
1171 				continue;
1172 			}
1173 
1174 			/*
1175 			 * Skip the metaslab when it has already been disabled.
1176 			 * This may happen when a manual TRIM or initialize
1177 			 * operation is running concurrently.  In the case
1178 			 * of a manual TRIM, the ms_trim tree will have been
1179 			 * vacated.  Only ranges added after the manual TRIM
1180 			 * disabled the metaslab will be included in the tree.
1181 			 * These will be processed when the automatic TRIM
1182 			 * next revisits this metaslab.
1183 			 */
1184 			if (msp->ms_disabled > 1) {
1185 				mutex_exit(&msp->ms_lock);
1186 				metaslab_enable(msp, B_FALSE, B_FALSE);
1187 				continue;
1188 			}
1189 
1190 			/*
1191 			 * Allocate an empty range tree which is swapped in
1192 			 * for the existing ms_trim tree while it is processed.
1193 			 */
1194 			trim_tree = range_tree_create(NULL, NULL);
1195 			range_tree_swap(&msp->ms_trim, &trim_tree);
1196 			ASSERT(range_tree_is_empty(msp->ms_trim));
1197 
1198 			/*
1199 			 * There are two cases when constructing the per-vdev
1200 			 * trim trees for a metaslab.  If the top-level vdev
1201 			 * has no children then it is also a leaf and should
1202 			 * be trimmed.  Otherwise our children are the leaves
1203 			 * and a trim tree should be constructed for each.
1204 			 */
1205 			trim_args_t *tap;
1206 			uint64_t children = vd->vdev_children;
1207 			if (children == 0) {
1208 				children = 1;
1209 				tap = kmem_zalloc(sizeof (trim_args_t) *
1210 				    children, KM_SLEEP);
1211 				tap[0].trim_vdev = vd;
1212 			} else {
1213 				tap = kmem_zalloc(sizeof (trim_args_t) *
1214 				    children, KM_SLEEP);
1215 
1216 				for (uint64_t c = 0; c < children; c++) {
1217 					tap[c].trim_vdev = vd->vdev_child[c];
1218 				}
1219 			}
1220 
1221 			for (uint64_t c = 0; c < children; c++) {
1222 				trim_args_t *ta = &tap[c];
1223 				vdev_t *cvd = ta->trim_vdev;
1224 
1225 				ta->trim_msp = msp;
1226 				ta->trim_extent_bytes_max = extent_bytes_max;
1227 				ta->trim_extent_bytes_min = extent_bytes_min;
1228 				ta->trim_type = TRIM_TYPE_AUTO;
1229 				ta->trim_flags = 0;
1230 
1231 				if (cvd->vdev_detached ||
1232 				    !vdev_writeable(cvd) ||
1233 				    !cvd->vdev_has_trim ||
1234 				    cvd->vdev_trim_thread != NULL) {
1235 					continue;
1236 				}
1237 
1238 				/*
1239 				 * When a device has an attached hot spare, or
1240 				 * is being replaced it will not be trimmed.
1241 				 * This is done to avoid adding additional
1242 				 * stress to a potentially unhealthy device,
1243 				 * and to minimize the required rebuild time.
1244 				 */
1245 				if (!cvd->vdev_ops->vdev_op_leaf)
1246 					continue;
1247 
1248 				ta->trim_tree = range_tree_create(NULL, NULL);
1249 				range_tree_walk(trim_tree,
1250 				    vdev_trim_range_add, ta);
1251 			}
1252 
1253 			mutex_exit(&msp->ms_lock);
1254 			spa_config_exit(spa, SCL_CONFIG, FTAG);
1255 
1256 			/*
1257 			 * Issue the TRIM I/Os for all ranges covered by the
1258 			 * TRIM trees.  These ranges are safe to TRIM because
1259 			 * no new allocations will be performed until the call
1260 			 * to metaslab_enabled() below.
1261 			 */
1262 			for (uint64_t c = 0; c < children; c++) {
1263 				trim_args_t *ta = &tap[c];
1264 
1265 				/*
1266 				 * Always yield to a manual TRIM if one has
1267 				 * been started for the child vdev.
1268 				 */
1269 				if (ta->trim_tree == NULL ||
1270 				    ta->trim_vdev->vdev_trim_thread != NULL) {
1271 					continue;
1272 				}
1273 
1274 				/*
1275 				 * After this point metaslab_enable() must be
1276 				 * called with the sync flag set.  This is done
1277 				 * here because vdev_trim_ranges() is allowed
1278 				 * to be interrupted (EINTR) before issuing all
1279 				 * of the required TRIM I/Os.
1280 				 */
1281 				issued_trim = B_TRUE;
1282 
1283 				int error = vdev_trim_ranges(ta);
1284 				if (error)
1285 					break;
1286 			}
1287 
1288 			/*
1289 			 * Verify every range which was trimmed is still
1290 			 * contained within the ms_allocatable tree.
1291 			 */
1292 			if (zfs_flags & ZFS_DEBUG_TRIM) {
1293 				mutex_enter(&msp->ms_lock);
1294 				VERIFY0(metaslab_load(msp));
1295 				VERIFY3P(tap[0].trim_msp, ==, msp);
1296 				range_tree_walk(trim_tree,
1297 				    vdev_trim_range_verify, &tap[0]);
1298 				mutex_exit(&msp->ms_lock);
1299 			}
1300 
1301 			range_tree_vacate(trim_tree, NULL, NULL);
1302 			range_tree_destroy(trim_tree);
1303 
1304 			metaslab_enable(msp, issued_trim, B_FALSE);
1305 			spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1306 
1307 			for (uint64_t c = 0; c < children; c++) {
1308 				trim_args_t *ta = &tap[c];
1309 
1310 				if (ta->trim_tree == NULL)
1311 					continue;
1312 
1313 				range_tree_vacate(ta->trim_tree, NULL, NULL);
1314 				range_tree_destroy(ta->trim_tree);
1315 			}
1316 
1317 			kmem_free(tap, sizeof (trim_args_t) * children);
1318 		}
1319 
1320 		spa_config_exit(spa, SCL_CONFIG, FTAG);
1321 
1322 		/*
1323 		 * After completing the group of metaslabs wait for the next
1324 		 * open txg.  This is done to make sure that a minimum of
1325 		 * zfs_trim_txg_batch txgs will occur before these metaslabs
1326 		 * are trimmed again.
1327 		 */
1328 		txg_wait_open(spa_get_dsl(spa), 0, issued_trim);
1329 
1330 		shift++;
1331 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1332 	}
1333 
1334 	for (uint64_t c = 0; c < vd->vdev_children; c++) {
1335 		vdev_t *cvd = vd->vdev_child[c];
1336 		mutex_enter(&cvd->vdev_trim_io_lock);
1337 
1338 		while (cvd->vdev_trim_inflight[1] > 0) {
1339 			cv_wait(&cvd->vdev_trim_io_cv,
1340 			    &cvd->vdev_trim_io_lock);
1341 		}
1342 		mutex_exit(&cvd->vdev_trim_io_lock);
1343 	}
1344 
1345 	spa_config_exit(spa, SCL_CONFIG, FTAG);
1346 
1347 	/*
1348 	 * When exiting because the autotrim property was set to off, then
1349 	 * abandon any unprocessed ms_trim ranges to reclaim the memory.
1350 	 */
1351 	if (spa_get_autotrim(spa) == SPA_AUTOTRIM_OFF) {
1352 		for (uint64_t i = 0; i < vd->vdev_ms_count; i++) {
1353 			metaslab_t *msp = vd->vdev_ms[i];
1354 
1355 			mutex_enter(&msp->ms_lock);
1356 			range_tree_vacate(msp->ms_trim, NULL, NULL);
1357 			mutex_exit(&msp->ms_lock);
1358 		}
1359 	}
1360 
1361 	mutex_enter(&vd->vdev_autotrim_lock);
1362 	ASSERT(vd->vdev_autotrim_thread != NULL);
1363 	vd->vdev_autotrim_thread = NULL;
1364 	cv_broadcast(&vd->vdev_autotrim_cv);
1365 	mutex_exit(&vd->vdev_autotrim_lock);
1366 }
1367 
1368 /*
1369  * Starts an autotrim thread, if needed, for each top-level vdev which can be
1370  * trimmed.  A top-level vdev which has been evacuated will never be trimmed.
1371  */
1372 void
1373 vdev_autotrim(spa_t *spa)
1374 {
1375 	vdev_t *root_vd = spa->spa_root_vdev;
1376 
1377 	for (uint64_t i = 0; i < root_vd->vdev_children; i++) {
1378 		vdev_t *tvd = root_vd->vdev_child[i];
1379 
1380 		mutex_enter(&tvd->vdev_autotrim_lock);
1381 		if (vdev_writeable(tvd) && !tvd->vdev_removing &&
1382 		    tvd->vdev_autotrim_thread == NULL) {
1383 			ASSERT3P(tvd->vdev_top, ==, tvd);
1384 
1385 			tvd->vdev_autotrim_thread = thread_create(NULL, 0,
1386 			    vdev_autotrim_thread, tvd, 0, &p0, TS_RUN,
1387 			    maxclsyspri);
1388 			ASSERT(tvd->vdev_autotrim_thread != NULL);
1389 		}
1390 		mutex_exit(&tvd->vdev_autotrim_lock);
1391 	}
1392 }
1393 
1394 /*
1395  * Wait for the vdev_autotrim_thread associated with the passed top-level
1396  * vdev to be terminated (canceled or stopped).
1397  */
1398 void
1399 vdev_autotrim_stop_wait(vdev_t *tvd)
1400 {
1401 	mutex_enter(&tvd->vdev_autotrim_lock);
1402 	if (tvd->vdev_autotrim_thread != NULL) {
1403 		tvd->vdev_autotrim_exit_wanted = B_TRUE;
1404 
1405 		while (tvd->vdev_autotrim_thread != NULL) {
1406 			cv_wait(&tvd->vdev_autotrim_cv,
1407 			    &tvd->vdev_autotrim_lock);
1408 		}
1409 
1410 		ASSERT3P(tvd->vdev_autotrim_thread, ==, NULL);
1411 		tvd->vdev_autotrim_exit_wanted = B_FALSE;
1412 	}
1413 	mutex_exit(&tvd->vdev_autotrim_lock);
1414 }
1415 
1416 /*
1417  * Wait for all of the vdev_autotrim_thread associated with the pool to
1418  * be terminated (canceled or stopped).
1419  */
1420 void
1421 vdev_autotrim_stop_all(spa_t *spa)
1422 {
1423 	vdev_t *root_vd = spa->spa_root_vdev;
1424 
1425 	for (uint64_t i = 0; i < root_vd->vdev_children; i++)
1426 		vdev_autotrim_stop_wait(root_vd->vdev_child[i]);
1427 }
1428 
1429 /*
1430  * Conditionally restart all of the vdev_autotrim_thread's for the pool.
1431  */
1432 void
1433 vdev_autotrim_restart(spa_t *spa)
1434 {
1435 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1436 
1437 	if (spa->spa_autotrim)
1438 		vdev_autotrim(spa);
1439 }
1440