1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2016 by Delphix. All rights reserved.
24 * Copyright (c) 2019 by Lawrence Livermore National Security, LLC.
25 * Copyright 2019 Joyent, Inc.
26 */
27
28#include <sys/spa.h>
29#include <sys/spa_impl.h>
30#include <sys/txg.h>
31#include <sys/vdev_impl.h>
32#include <sys/vdev_trim.h>
33#include <sys/refcount.h>
34#include <sys/metaslab_impl.h>
35#include <sys/dsl_synctask.h>
36#include <sys/zap.h>
37#include <sys/dmu_tx.h>
38
39/*
40 * TRIM is a feature which is used to notify a SSD that some previously
41 * written space is no longer allocated by the pool.  This is useful because
42 * writes to a SSD must be performed to blocks which have first been erased.
43 * Ensuring the SSD always has a supply of erased blocks for new writes
44 * helps prevent the performance from deteriorating.
45 *
46 * There are two supported TRIM methods; manual and automatic.
47 *
48 * Manual TRIM:
49 *
50 * A manual TRIM is initiated by running the 'zpool trim' command.  A single
51 * 'vdev_trim' thread is created for each leaf vdev, and it is responsible for
52 * managing that vdev TRIM process.  This involves iterating over all the
53 * metaslabs, calculating the unallocated space ranges, and then issuing the
54 * required TRIM I/Os.
55 *
56 * While a metaslab is being actively trimmed it is not eligible to perform
57 * new allocations.  After traversing all of the metaslabs the thread is
58 * terminated.  Finally, both the requested options and current progress of
59 * the TRIM are regularly written to the pool.  This allows the TRIM to be
60 * suspended and resumed as needed.
61 *
62 * Automatic TRIM:
63 *
64 * An automatic TRIM is enabled by setting the 'autotrim' pool property
65 * to 'on'.  When enabled, a `vdev_autotrim' thread is created for each
66 * top-level (not leaf) vdev in the pool.  These threads perform the same
67 * core TRIM process as a manual TRIM, but with a few key differences.
68 *
69 * 1) Automatic TRIM happens continuously in the background and operates
70 *    solely on recently freed blocks (ms_trim not ms_allocatable).
71 *
72 * 2) Each thread is associated with a top-level (not leaf) vdev.  This has
73 *    the benefit of simplifying the threading model, it makes it easier
74 *    to coordinate administrative commands, and it ensures only a single
75 *    metaslab is disabled at a time.  Unlike manual TRIM, this means each
76 *    'vdev_autotrim' thread is responsible for issuing TRIM I/Os for its
77 *    children.
78 *
79 * 3) There is no automatic TRIM progress information stored on disk, nor
80 *    is it reported by 'zpool status'.
81 *
82 * While the automatic TRIM process is highly effective it is more likely
83 * than a manual TRIM to encounter tiny ranges.  Ranges less than or equal to
84 * 'zfs_trim_extent_bytes_min' (32k) are considered too small to efficiently
85 * TRIM and are skipped.  This means small amounts of freed space may not
86 * be automatically trimmed.
87 *
88 * Furthermore, devices with attached hot spares and devices being actively
89 * replaced are skipped.  This is done to avoid adding additional stress to
90 * a potentially unhealthy device and to minimize the required rebuild time.
91 *
92 * For this reason it may be beneficial to occasionally manually TRIM a pool
93 * even when automatic TRIM is enabled.
94 */
95
96/*
97 * Maximum size of TRIM I/O, ranges will be chunked in to 128MiB lengths.
98 */
99unsigned int zfs_trim_extent_bytes_max = 128 * 1024 * 1024;
100
101/*
102 * Minimum size of TRIM I/O, extents smaller than 32Kib will be skipped.
103 */
104unsigned int zfs_trim_extent_bytes_min = 32 * 1024;
105
106/*
107 * Skip uninitialized metaslabs during the TRIM process.  This option is
108 * useful for pools constructed from large thinly-provisioned devices where
109 * TRIM operations are slow.  As a pool ages an increasing fraction of
110 * the pools metaslabs will be initialized progressively degrading the
111 * usefulness of this option.  This setting is stored when starting a
112 * manual TRIM and will persist for the duration of the requested TRIM.
113 */
114unsigned int zfs_trim_metaslab_skip = 0;
115
116/*
117 * Maximum number of queued TRIM I/Os per leaf vdev.  The number of
118 * concurrent TRIM I/Os issued to the device is controlled by the
119 * zfs_vdev_trim_min_active and zfs_vdev_trim_max_active module options.
120 */
121unsigned int zfs_trim_queue_limit = 10;
122
123/*
124 * The minimum number of transaction groups between automatic trims of a
125 * metaslab.  This setting represents a trade-off between issuing more
126 * efficient TRIM operations, by allowing them to be aggregated longer,
127 * and issuing them promptly so the trimmed space is available.  Note
128 * that this value is a minimum; metaslabs can be trimmed less frequently
129 * when there are a large number of ranges which need to be trimmed.
130 *
131 * Increasing this value will allow frees to be aggregated for a longer
132 * time.  This can result is larger TRIM operations, and increased memory
133 * usage in order to track the ranges to be trimmed.  Decreasing this value
134 * has the opposite effect.  The default value of 32 was determined though
135 * testing to be a reasonable compromise.
136 */
137unsigned int zfs_trim_txg_batch = 32;
138
139/*
140 * The trim_args are a control structure which describe how a leaf vdev
141 * should be trimmed.  The core elements are the vdev, the metaslab being
142 * trimmed and a range tree containing the extents to TRIM.  All provided
143 * ranges must be within the metaslab.
144 */
145typedef struct trim_args {
146	/*
147	 * These fields are set by the caller of vdev_trim_ranges().
148	 */
149	vdev_t		*trim_vdev;		/* Leaf vdev to TRIM */
150	metaslab_t	*trim_msp;		/* Disabled metaslab */
151	range_tree_t	*trim_tree;		/* TRIM ranges (in metaslab) */
152	trim_type_t	trim_type;		/* Manual or auto TRIM */
153	uint64_t	trim_extent_bytes_max;	/* Maximum TRIM I/O size */
154	uint64_t	trim_extent_bytes_min;	/* Minimum TRIM I/O size */
155	enum trim_flag	trim_flags;		/* TRIM flags (secure) */
156
157	/*
158	 * These fields are updated by vdev_trim_ranges().
159	 */
160	hrtime_t	trim_start_time;	/* Start time */
161	uint64_t	trim_bytes_done;	/* Bytes trimmed */
162} trim_args_t;
163
164/*
165 * Determines whether a vdev_trim_thread() should be stopped.
166 */
167static boolean_t
168vdev_trim_should_stop(vdev_t *vd)
169{
170	return (vd->vdev_trim_exit_wanted || !vdev_writeable(vd) ||
171	    vd->vdev_detached || vd->vdev_top->vdev_removing);
172}
173
174/*
175 * Determines whether a vdev_autotrim_thread() should be stopped.
176 */
177static boolean_t
178vdev_autotrim_should_stop(vdev_t *tvd)
179{
180	return (tvd->vdev_autotrim_exit_wanted ||
181	    !vdev_writeable(tvd) || tvd->vdev_removing ||
182	    spa_get_autotrim(tvd->vdev_spa) == SPA_AUTOTRIM_OFF);
183}
184
185/*
186 * The sync task for updating the on-disk state of a manual TRIM.  This
187 * is scheduled by vdev_trim_change_state().
188 */
189static void
190vdev_trim_zap_update_sync(void *arg, dmu_tx_t *tx)
191{
192	/*
193	 * We pass in the guid instead of the vdev_t since the vdev may
194	 * have been freed prior to the sync task being processed.  This
195	 * happens when a vdev is detached as we call spa_config_vdev_exit(),
196	 * stop the trimming thread, schedule the sync task, and free
197	 * the vdev. Later when the scheduled sync task is invoked, it would
198	 * find that the vdev has been freed.
199	 */
200	uint64_t guid = *(uint64_t *)arg;
201	uint64_t txg = dmu_tx_get_txg(tx);
202	kmem_free(arg, sizeof (uint64_t));
203
204	vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);
205	if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd))
206		return;
207
208	uint64_t last_offset = vd->vdev_trim_offset[txg & TXG_MASK];
209	vd->vdev_trim_offset[txg & TXG_MASK] = 0;
210
211	VERIFY3U(vd->vdev_leaf_zap, !=, 0);
212
213	objset_t *mos = vd->vdev_spa->spa_meta_objset;
214
215	if (last_offset > 0 || vd->vdev_trim_last_offset == UINT64_MAX) {
216
217		if (vd->vdev_trim_last_offset == UINT64_MAX)
218			last_offset = 0;
219
220		vd->vdev_trim_last_offset = last_offset;
221		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
222		    VDEV_LEAF_ZAP_TRIM_LAST_OFFSET,
223		    sizeof (last_offset), 1, &last_offset, tx));
224	}
225
226	if (vd->vdev_trim_action_time > 0) {
227		uint64_t val = (uint64_t)vd->vdev_trim_action_time;
228		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
229		    VDEV_LEAF_ZAP_TRIM_ACTION_TIME, sizeof (val),
230		    1, &val, tx));
231	}
232
233	if (vd->vdev_trim_rate > 0) {
234		uint64_t rate = (uint64_t)vd->vdev_trim_rate;
235
236		if (rate == UINT64_MAX)
237			rate = 0;
238
239		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
240		    VDEV_LEAF_ZAP_TRIM_RATE, sizeof (rate), 1, &rate, tx));
241	}
242
243	uint64_t partial = vd->vdev_trim_partial;
244	if (partial == UINT64_MAX)
245		partial = 0;
246
247	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,
248	    sizeof (partial), 1, &partial, tx));
249
250	uint64_t secure = vd->vdev_trim_secure;
251	if (secure == UINT64_MAX)
252		secure = 0;
253
254	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,
255	    sizeof (secure), 1, &secure, tx));
256
257
258	uint64_t trim_state = vd->vdev_trim_state;
259	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,
260	    sizeof (trim_state), 1, &trim_state, tx));
261}
262
263/*
264 * Update the on-disk state of a manual TRIM.  This is called to request
265 * that a TRIM be started/suspended/canceled, or to change one of the
266 * TRIM options (partial, secure, rate).
267 */
268static void
269vdev_trim_change_state(vdev_t *vd, vdev_trim_state_t new_state,
270    uint64_t rate, boolean_t partial, boolean_t secure)
271{
272	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
273	spa_t *spa = vd->vdev_spa;
274
275	if (new_state == vd->vdev_trim_state)
276		return;
277
278	/*
279	 * Copy the vd's guid, this will be freed by the sync task.
280	 */
281	uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
282	*guid = vd->vdev_guid;
283
284	/*
285	 * If we're suspending, then preserve the original start time.
286	 */
287	if (vd->vdev_trim_state != VDEV_TRIM_SUSPENDED) {
288		vd->vdev_trim_action_time = gethrestime_sec();
289	}
290
291	/*
292	 * If we're activating, then preserve the requested rate and trim
293	 * method.  Setting the last offset and rate to UINT64_MAX is used
294	 * as a sentinel to indicate they should be reset to default values.
295	 */
296	if (new_state == VDEV_TRIM_ACTIVE) {
297		if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE ||
298		    vd->vdev_trim_state == VDEV_TRIM_CANCELED) {
299			vd->vdev_trim_last_offset = UINT64_MAX;
300			vd->vdev_trim_rate = UINT64_MAX;
301			vd->vdev_trim_partial = UINT64_MAX;
302			vd->vdev_trim_secure = UINT64_MAX;
303		}
304
305		if (rate != 0)
306			vd->vdev_trim_rate = rate;
307
308		if (partial != 0)
309			vd->vdev_trim_partial = partial;
310
311		if (secure != 0)
312			vd->vdev_trim_secure = secure;
313	}
314
315	boolean_t resumed = !!(vd->vdev_trim_state == VDEV_TRIM_SUSPENDED);
316	vd->vdev_trim_state = new_state;
317
318	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
319	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
320	dsl_sync_task_nowait(spa_get_dsl(spa), vdev_trim_zap_update_sync,
321	    guid, 2, ZFS_SPACE_CHECK_NONE, tx);
322
323	switch (new_state) {
324	case VDEV_TRIM_ACTIVE:
325		spa_event_notify(spa, vd, NULL,
326		    resumed ? ESC_ZFS_TRIM_RESUME : ESC_ZFS_TRIM_START);
327		spa_history_log_internal(spa, "trim", tx,
328		    "vdev=%s activated", vd->vdev_path);
329		break;
330	case VDEV_TRIM_SUSPENDED:
331		spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_SUSPEND);
332		spa_history_log_internal(spa, "trim", tx,
333		    "vdev=%s suspended", vd->vdev_path);
334		break;
335	case VDEV_TRIM_CANCELED:
336		spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_CANCEL);
337		spa_history_log_internal(spa, "trim", tx,
338		    "vdev=%s canceled", vd->vdev_path);
339		break;
340	case VDEV_TRIM_COMPLETE:
341		spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_FINISH);
342		spa_history_log_internal(spa, "trim", tx,
343		    "vdev=%s complete", vd->vdev_path);
344		break;
345	default:
346		panic("invalid state %llu", (unsigned long long)new_state);
347	}
348
349	dmu_tx_commit(tx);
350}
351
352/*
353 * The zio_done_func_t done callback for each manual TRIM issued.  It is
354 * responsible for updating the TRIM stats, reissuing failed TRIM I/Os,
355 * and limiting the number of in-flight TRIM I/Os.
356 */
357static void
358vdev_trim_cb(zio_t *zio)
359{
360	vdev_t *vd = zio->io_vd;
361
362	mutex_enter(&vd->vdev_trim_io_lock);
363	if (zio->io_error == ENXIO && !vdev_writeable(vd)) {
364		/*
365		 * The I/O failed because the vdev was unavailable; roll the
366		 * last offset back. (This works because spa_sync waits on
367		 * spa_txg_zio before it runs sync tasks.)
368		 */
369		uint64_t *offset =
370		    &vd->vdev_trim_offset[zio->io_txg & TXG_MASK];
371		*offset = MIN(*offset, zio->io_offset);
372	} else {
373		if (zio->io_error != 0) {
374			vd->vdev_stat.vs_trim_errors++;
375			/*
376			 * spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,
377			 *  0, 0, 0, 0, 1, zio->io_orig_size);
378			 */
379		} else {
380			/*
381			 * spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,
382			 *  1, zio->io_orig_size, 0, 0, 0, 0);
383			 */
384		}
385
386		vd->vdev_trim_bytes_done += zio->io_orig_size;
387	}
388
389	ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_MANUAL], >, 0);
390	vd->vdev_trim_inflight[TRIM_TYPE_MANUAL]--;
391	cv_broadcast(&vd->vdev_trim_io_cv);
392	mutex_exit(&vd->vdev_trim_io_lock);
393
394	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
395}
396
397/*
398 * The zio_done_func_t done callback for each automatic TRIM issued.  It
399 * is responsible for updating the TRIM stats and limiting the number of
400 * in-flight TRIM I/Os.  Automatic TRIM I/Os are best effort and are
401 * never reissued on failure.
402 */
403static void
404vdev_autotrim_cb(zio_t *zio)
405{
406	vdev_t *vd = zio->io_vd;
407
408	mutex_enter(&vd->vdev_trim_io_lock);
409
410	if (zio->io_error != 0) {
411		vd->vdev_stat.vs_trim_errors++;
412		/*
413		 * spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,
414		 *  0, 0, 0, 0, 1, zio->io_orig_size);
415		 */
416	} else {
417		/*
418		 * spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,
419		 *  1, zio->io_orig_size, 0, 0, 0, 0);
420		 */
421
422		vd->vdev_autotrim_bytes_done += zio->io_orig_size;
423	}
424
425	ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_AUTO], >, 0);
426	vd->vdev_trim_inflight[TRIM_TYPE_AUTO]--;
427	cv_broadcast(&vd->vdev_trim_io_cv);
428	mutex_exit(&vd->vdev_trim_io_lock);
429
430	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
431}
432
433/*
434 * Returns the average trim rate in bytes/sec for the ta->trim_vdev.
435 */
436static uint64_t
437vdev_trim_calculate_rate(trim_args_t *ta)
438{
439	return (ta->trim_bytes_done * 1000 /
440	    (NSEC2MSEC(gethrtime() - ta->trim_start_time) + 1));
441}
442
443/*
444 * Issues a physical TRIM and takes care of rate limiting (bytes/sec)
445 * and number of concurrent TRIM I/Os.
446 */
447static int
448vdev_trim_range(trim_args_t *ta, uint64_t start, uint64_t size)
449{
450	vdev_t *vd = ta->trim_vdev;
451	spa_t *spa = vd->vdev_spa;
452
453	mutex_enter(&vd->vdev_trim_io_lock);
454
455	/*
456	 * Limit manual TRIM I/Os to the requested rate.  This does not
457	 * apply to automatic TRIM since no per vdev rate can be specified.
458	 */
459	if (ta->trim_type == TRIM_TYPE_MANUAL) {
460		while (vd->vdev_trim_rate != 0 && !vdev_trim_should_stop(vd) &&
461		    vdev_trim_calculate_rate(ta) > vd->vdev_trim_rate) {
462			cv_timedwait_sig(&vd->vdev_trim_io_cv,
463			    &vd->vdev_trim_io_lock, ddi_get_lbolt() +
464			    MSEC_TO_TICK(10));
465		}
466	}
467	ta->trim_bytes_done += size;
468
469	/* Limit in-flight trimming I/Os */
470	while (vd->vdev_trim_inflight[0] + vd->vdev_trim_inflight[1] >=
471	    zfs_trim_queue_limit) {
472		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
473	}
474	vd->vdev_trim_inflight[ta->trim_type]++;
475	mutex_exit(&vd->vdev_trim_io_lock);
476
477	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
478	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
479	uint64_t txg = dmu_tx_get_txg(tx);
480
481	spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER);
482	mutex_enter(&vd->vdev_trim_lock);
483
484	if (ta->trim_type == TRIM_TYPE_MANUAL &&
485	    vd->vdev_trim_offset[txg & TXG_MASK] == 0) {
486		uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
487		*guid = vd->vdev_guid;
488
489		/* This is the first write of this txg. */
490		dsl_sync_task_nowait(spa_get_dsl(spa),
491		    vdev_trim_zap_update_sync, guid, 2,
492		    ZFS_SPACE_CHECK_RESERVED, tx);
493	}
494
495	/*
496	 * We know the vdev_t will still be around since all consumers of
497	 * vdev_free must stop the trimming first.
498	 */
499	if ((ta->trim_type == TRIM_TYPE_MANUAL &&
500	    vdev_trim_should_stop(vd)) ||
501	    (ta->trim_type == TRIM_TYPE_AUTO &&
502	    vdev_autotrim_should_stop(vd->vdev_top))) {
503		mutex_enter(&vd->vdev_trim_io_lock);
504		vd->vdev_trim_inflight[ta->trim_type]--;
505		mutex_exit(&vd->vdev_trim_io_lock);
506		spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
507		mutex_exit(&vd->vdev_trim_lock);
508		dmu_tx_commit(tx);
509		return (SET_ERROR(EINTR));
510	}
511	mutex_exit(&vd->vdev_trim_lock);
512
513	if (ta->trim_type == TRIM_TYPE_MANUAL)
514		vd->vdev_trim_offset[txg & TXG_MASK] = start + size;
515
516	zio_nowait(zio_trim(spa->spa_txg_zio[txg & TXG_MASK], vd,
517	    start, size, ta->trim_type == TRIM_TYPE_MANUAL ?
518	    vdev_trim_cb : vdev_autotrim_cb, NULL,
519	    ZIO_PRIORITY_TRIM, ZIO_FLAG_CANFAIL, ta->trim_flags));
520	/* vdev_trim_cb and vdev_autotrim_cb release SCL_STATE_ALL */
521
522	dmu_tx_commit(tx);
523
524	return (0);
525}
526
527/*
528 * Issues TRIM I/Os for all ranges in the provided ta->trim_tree range tree.
529 * Additional parameters describing how the TRIM should be performed must
530 * be set in the trim_args structure.  See the trim_args definition for
531 * additional information.
532 */
533static int
534vdev_trim_ranges(trim_args_t *ta)
535{
536	vdev_t *vd = ta->trim_vdev;
537	zfs_btree_t *t = &ta->trim_tree->rt_root;
538	zfs_btree_index_t idx;
539	uint64_t extent_bytes_max = ta->trim_extent_bytes_max;
540	uint64_t extent_bytes_min = ta->trim_extent_bytes_min;
541	spa_t *spa = vd->vdev_spa;
542
543	ta->trim_start_time = gethrtime();
544	ta->trim_bytes_done = 0;
545
546	for (range_seg_t *rs = zfs_btree_first(t, &idx); rs != NULL;
547	    rs = zfs_btree_next(t, &idx, &idx)) {
548		uint64_t size = rs_get_end(rs, ta->trim_tree) - rs_get_start(rs,
549		    ta->trim_tree);
550
551		if (extent_bytes_min && size < extent_bytes_min) {
552			/*
553			 * spa_iostats_trim_add(spa, ta->trim_type,
554			 *  0, 0, 1, size, 0, 0);
555			 */
556			continue;
557		}
558
559		/* Split range into legally-sized physical chunks */
560		uint64_t writes_required = ((size - 1) / extent_bytes_max) + 1;
561
562		for (uint64_t w = 0; w < writes_required; w++) {
563			int error;
564
565			error = vdev_trim_range(ta, VDEV_LABEL_START_SIZE +
566			    rs_get_start(rs, ta->trim_tree) +
567			    (w *extent_bytes_max), MIN(size -
568			    (w * extent_bytes_max), extent_bytes_max));
569			if (error != 0) {
570				return (error);
571			}
572		}
573	}
574
575	return (0);
576}
577
578/*
579 * Calculates the completion percentage of a manual TRIM.
580 */
581static void
582vdev_trim_calculate_progress(vdev_t *vd)
583{
584	ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
585	    spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
586	ASSERT(vd->vdev_leaf_zap != 0);
587
588	vd->vdev_trim_bytes_est = 0;
589	vd->vdev_trim_bytes_done = 0;
590
591	for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) {
592		metaslab_t *msp = vd->vdev_top->vdev_ms[i];
593		mutex_enter(&msp->ms_lock);
594
595		uint64_t ms_free = msp->ms_size -
596		    metaslab_allocated_space(msp);
597
598		if (vd->vdev_top->vdev_ops == &vdev_raidz_ops)
599			ms_free /= vd->vdev_top->vdev_children;
600
601		/*
602		 * Convert the metaslab range to a physical range
603		 * on our vdev. We use this to determine if we are
604		 * in the middle of this metaslab range.
605		 */
606		range_seg64_t logical_rs, physical_rs;
607		logical_rs.rs_start = msp->ms_start;
608		logical_rs.rs_end = msp->ms_start + msp->ms_size;
609		vdev_xlate(vd, &logical_rs, &physical_rs);
610
611		if (vd->vdev_trim_last_offset <= physical_rs.rs_start) {
612			vd->vdev_trim_bytes_est += ms_free;
613			mutex_exit(&msp->ms_lock);
614			continue;
615		} else if (vd->vdev_trim_last_offset > physical_rs.rs_end) {
616			vd->vdev_trim_bytes_done += ms_free;
617			vd->vdev_trim_bytes_est += ms_free;
618			mutex_exit(&msp->ms_lock);
619			continue;
620		}
621
622		/*
623		 * If we get here, we're in the middle of trimming this
624		 * metaslab.  Load it and walk the free tree for more
625		 * accurate progress estimation.
626		 */
627		VERIFY0(metaslab_load(msp));
628
629		range_tree_t *rt = msp->ms_allocatable;
630		zfs_btree_t *bt = &rt->rt_root;
631		zfs_btree_index_t idx;
632		for (range_seg_t *rs = zfs_btree_first(bt, &idx);
633		    rs != NULL; rs = zfs_btree_next(bt, &idx, &idx)) {
634			logical_rs.rs_start = rs_get_start(rs, rt);
635			logical_rs.rs_end = rs_get_end(rs, rt);
636			vdev_xlate(vd, &logical_rs, &physical_rs);
637
638			uint64_t size = physical_rs.rs_end -
639			    physical_rs.rs_start;
640			vd->vdev_trim_bytes_est += size;
641			if (vd->vdev_trim_last_offset >= physical_rs.rs_end) {
642				vd->vdev_trim_bytes_done += size;
643			} else if (vd->vdev_trim_last_offset >
644			    physical_rs.rs_start &&
645			    vd->vdev_trim_last_offset <=
646			    physical_rs.rs_end) {
647				vd->vdev_trim_bytes_done +=
648				    vd->vdev_trim_last_offset -
649				    physical_rs.rs_start;
650			}
651		}
652		mutex_exit(&msp->ms_lock);
653	}
654}
655
656/*
657 * Load from disk the vdev's manual TRIM information.  This includes the
658 * state, progress, and options provided when initiating the manual TRIM.
659 */
660static int
661vdev_trim_load(vdev_t *vd)
662{
663	int err = 0;
664	ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
665	    spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
666	ASSERT(vd->vdev_leaf_zap != 0);
667
668	if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE ||
669	    vd->vdev_trim_state == VDEV_TRIM_SUSPENDED) {
670		err = zap_lookup(vd->vdev_spa->spa_meta_objset,
671		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_LAST_OFFSET,
672		    sizeof (vd->vdev_trim_last_offset), 1,
673		    &vd->vdev_trim_last_offset);
674		if (err == ENOENT) {
675			vd->vdev_trim_last_offset = 0;
676			err = 0;
677		}
678
679		if (err == 0) {
680			err = zap_lookup(vd->vdev_spa->spa_meta_objset,
681			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_RATE,
682			    sizeof (vd->vdev_trim_rate), 1,
683			    &vd->vdev_trim_rate);
684			if (err == ENOENT) {
685				vd->vdev_trim_rate = 0;
686				err = 0;
687			}
688		}
689
690		if (err == 0) {
691			err = zap_lookup(vd->vdev_spa->spa_meta_objset,
692			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,
693			    sizeof (vd->vdev_trim_partial), 1,
694			    &vd->vdev_trim_partial);
695			if (err == ENOENT) {
696				vd->vdev_trim_partial = 0;
697				err = 0;
698			}
699		}
700
701		if (err == 0) {
702			err = zap_lookup(vd->vdev_spa->spa_meta_objset,
703			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,
704			    sizeof (vd->vdev_trim_secure), 1,
705			    &vd->vdev_trim_secure);
706			if (err == ENOENT) {
707				vd->vdev_trim_secure = 0;
708				err = 0;
709			}
710		}
711	}
712
713	vdev_trim_calculate_progress(vd);
714
715	return (err);
716}
717
718/*
719 * Convert the logical range into a physical range and add it to the
720 * range tree passed in the trim_args_t.
721 */
722static void
723vdev_trim_range_add(void *arg, uint64_t start, uint64_t size)
724{
725	trim_args_t *ta = arg;
726	vdev_t *vd = ta->trim_vdev;
727	range_seg64_t logical_rs, physical_rs;
728	logical_rs.rs_start = start;
729	logical_rs.rs_end = start + size;
730
731	/*
732	 * Every range to be trimmed must be part of ms_allocatable.
733	 * When ZFS_DEBUG_TRIM is set load the metaslab to verify this
734	 * is always the case.
735	 */
736	if (zfs_flags & ZFS_DEBUG_TRIM) {
737		metaslab_t *msp = ta->trim_msp;
738		VERIFY0(metaslab_load(msp));
739		VERIFY3B(msp->ms_loaded, ==, B_TRUE);
740		VERIFY(range_tree_contains(msp->ms_allocatable, start, size));
741	}
742
743	ASSERT(vd->vdev_ops->vdev_op_leaf);
744	vdev_xlate(vd, &logical_rs, &physical_rs);
745
746	IMPLY(vd->vdev_top == vd,
747	    logical_rs.rs_start == physical_rs.rs_start);
748	IMPLY(vd->vdev_top == vd,
749	    logical_rs.rs_end == physical_rs.rs_end);
750
751	/*
752	 * Only a manual trim will be traversing the vdev sequentially.
753	 * For an auto trim all valid ranges should be added.
754	 */
755	if (ta->trim_type == TRIM_TYPE_MANUAL) {
756
757		/* Only add segments that we have not visited yet */
758		if (physical_rs.rs_end <= vd->vdev_trim_last_offset)
759			return;
760
761		/* Pick up where we left off mid-range. */
762		if (vd->vdev_trim_last_offset > physical_rs.rs_start) {
763			ASSERT3U(physical_rs.rs_end, >,
764			    vd->vdev_trim_last_offset);
765			physical_rs.rs_start = vd->vdev_trim_last_offset;
766		}
767	}
768
769	ASSERT3U(physical_rs.rs_end, >=, physical_rs.rs_start);
770
771	/*
772	 * With raidz, it's possible that the logical range does not live on
773	 * this leaf vdev. We only add the physical range to this vdev's if it
774	 * has a length greater than 0.
775	 */
776	if (physical_rs.rs_end > physical_rs.rs_start) {
777		range_tree_add(ta->trim_tree, physical_rs.rs_start,
778		    physical_rs.rs_end - physical_rs.rs_start);
779	} else {
780		ASSERT3U(physical_rs.rs_end, ==, physical_rs.rs_start);
781	}
782}
783
784/*
785 * Each manual TRIM thread is responsible for trimming the unallocated
786 * space for each leaf vdev.  This is accomplished by sequentially iterating
787 * over its top-level metaslabs and issuing TRIM I/O for the space described
788 * by its ms_allocatable.  While a metaslab is undergoing trimming it is
789 * not eligible for new allocations.
790 */
791static void
792vdev_trim_thread(void *arg)
793{
794	vdev_t *vd = arg;
795	spa_t *spa = vd->vdev_spa;
796	trim_args_t ta;
797	int error = 0;
798
799	/*
800	 * The VDEV_LEAF_ZAP_TRIM_* entries may have been updated by
801	 * vdev_trim().  Wait for the updated values to be reflected
802	 * in the zap in order to start with the requested settings.
803	 */
804	txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
805
806	ASSERT(vdev_is_concrete(vd));
807	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
808
809	vd->vdev_trim_last_offset = 0;
810	vd->vdev_trim_rate = 0;
811	vd->vdev_trim_partial = 0;
812	vd->vdev_trim_secure = 0;
813
814	VERIFY0(vdev_trim_load(vd));
815
816	ta.trim_vdev = vd;
817	ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max;
818	ta.trim_extent_bytes_min = zfs_trim_extent_bytes_min;
819	ta.trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);
820	ta.trim_type = TRIM_TYPE_MANUAL;
821	ta.trim_flags = 0;
822
823	/*
824	 * When a secure TRIM has been requested infer that the intent
825	 * is that everything must be trimmed.  Override the default
826	 * minimum TRIM size to prevent ranges from being skipped.
827	 */
828	if (vd->vdev_trim_secure) {
829		ta.trim_flags |= ZIO_TRIM_SECURE;
830		ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE;
831	}
832
833	uint64_t ms_count = 0;
834	for (uint64_t i = 0; !vd->vdev_detached &&
835	    i < vd->vdev_top->vdev_ms_count; i++) {
836		metaslab_t *msp = vd->vdev_top->vdev_ms[i];
837
838		/*
839		 * If we've expanded the top-level vdev or it's our
840		 * first pass, calculate our progress.
841		 */
842		if (vd->vdev_top->vdev_ms_count != ms_count) {
843			vdev_trim_calculate_progress(vd);
844			ms_count = vd->vdev_top->vdev_ms_count;
845		}
846
847		spa_config_exit(spa, SCL_CONFIG, FTAG);
848		metaslab_disable(msp);
849		mutex_enter(&msp->ms_lock);
850		VERIFY0(metaslab_load(msp));
851
852		/*
853		 * If a partial TRIM was requested skip metaslabs which have
854		 * never been initialized and thus have never been written.
855		 */
856		if (msp->ms_sm == NULL && vd->vdev_trim_partial) {
857			mutex_exit(&msp->ms_lock);
858			metaslab_enable(msp, B_FALSE, B_FALSE);
859			spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
860			vdev_trim_calculate_progress(vd);
861			continue;
862		}
863
864		ta.trim_msp = msp;
865		range_tree_walk(msp->ms_allocatable, vdev_trim_range_add, &ta);
866		range_tree_vacate(msp->ms_trim, NULL, NULL);
867		mutex_exit(&msp->ms_lock);
868
869		error = vdev_trim_ranges(&ta);
870		metaslab_enable(msp, B_TRUE, B_FALSE);
871		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
872
873		range_tree_vacate(ta.trim_tree, NULL, NULL);
874		if (error != 0)
875			break;
876	}
877
878	spa_config_exit(spa, SCL_CONFIG, FTAG);
879	mutex_enter(&vd->vdev_trim_io_lock);
880	while (vd->vdev_trim_inflight[0] > 0) {
881		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
882	}
883	mutex_exit(&vd->vdev_trim_io_lock);
884
885	range_tree_destroy(ta.trim_tree);
886
887	mutex_enter(&vd->vdev_trim_lock);
888	if (!vd->vdev_trim_exit_wanted && vdev_writeable(vd)) {
889		vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE,
890		    vd->vdev_trim_rate, vd->vdev_trim_partial,
891		    vd->vdev_trim_secure);
892	}
893	ASSERT(vd->vdev_trim_thread != NULL || vd->vdev_trim_inflight[0] == 0);
894
895	/*
896	 * Drop the vdev_trim_lock while we sync out the txg since it's
897	 * possible that a device might be trying to come online and must
898	 * check to see if it needs to restart a trim. That thread will be
899	 * holding the spa_config_lock which would prevent the txg_wait_synced
900	 * from completing.
901	 */
902	mutex_exit(&vd->vdev_trim_lock);
903	txg_wait_synced(spa_get_dsl(spa), 0);
904	mutex_enter(&vd->vdev_trim_lock);
905
906	vd->vdev_trim_thread = NULL;
907	cv_broadcast(&vd->vdev_trim_cv);
908	mutex_exit(&vd->vdev_trim_lock);
909}
910
911/*
912 * Initiates a manual TRIM for the vdev_t.  Callers must hold vdev_trim_lock,
913 * the vdev_t must be a leaf and cannot already be manually trimming.
914 */
915void
916vdev_trim(vdev_t *vd, uint64_t rate, boolean_t partial, boolean_t secure)
917{
918	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
919	ASSERT(vd->vdev_ops->vdev_op_leaf);
920	ASSERT(vdev_is_concrete(vd));
921	ASSERT3P(vd->vdev_trim_thread, ==, NULL);
922	ASSERT(!vd->vdev_detached);
923	ASSERT(!vd->vdev_trim_exit_wanted);
924	ASSERT(!vd->vdev_top->vdev_removing);
925
926	vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, rate, partial, secure);
927	vd->vdev_trim_thread = thread_create(NULL, 0,
928	    vdev_trim_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
929}
930
931/*
932 * Wait for the trimming thread to be terminated (canceled or stopped).
933 */
934static void
935vdev_trim_stop_wait_impl(vdev_t *vd)
936{
937	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
938
939	while (vd->vdev_trim_thread != NULL)
940		cv_wait(&vd->vdev_trim_cv, &vd->vdev_trim_lock);
941
942	ASSERT3P(vd->vdev_trim_thread, ==, NULL);
943	vd->vdev_trim_exit_wanted = B_FALSE;
944}
945
946/*
947 * Wait for vdev trim threads which were listed to cleanly exit.
948 */
949void
950vdev_trim_stop_wait(spa_t *spa, list_t *vd_list)
951{
952	vdev_t *vd;
953
954	ASSERT(MUTEX_HELD(&spa_namespace_lock));
955
956	while ((vd = list_remove_head(vd_list)) != NULL) {
957		mutex_enter(&vd->vdev_trim_lock);
958		vdev_trim_stop_wait_impl(vd);
959		mutex_exit(&vd->vdev_trim_lock);
960	}
961}
962
963/*
964 * Stop trimming a device, with the resultant trimming state being tgt_state.
965 * For blocking behavior pass NULL for vd_list.  Otherwise, when a list_t is
966 * provided the stopping vdev is inserted in to the list.  Callers are then
967 * required to call vdev_trim_stop_wait() to block for all the trim threads
968 * to exit.  The caller must hold vdev_trim_lock and must not be writing to
969 * the spa config, as the trimming thread may try to enter the config as a
970 * reader before exiting.
971 */
972void
973vdev_trim_stop(vdev_t *vd, vdev_trim_state_t tgt_state, list_t *vd_list)
974{
975	ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER));
976	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
977	ASSERT(vd->vdev_ops->vdev_op_leaf);
978	ASSERT(vdev_is_concrete(vd));
979
980	/*
981	 * Allow cancel requests to proceed even if the trim thread has
982	 * stopped.
983	 */
984	if (vd->vdev_trim_thread == NULL && tgt_state != VDEV_TRIM_CANCELED)
985		return;
986
987	vdev_trim_change_state(vd, tgt_state, 0, 0, 0);
988	vd->vdev_trim_exit_wanted = B_TRUE;
989
990	if (vd_list == NULL) {
991		vdev_trim_stop_wait_impl(vd);
992	} else {
993		ASSERT(MUTEX_HELD(&spa_namespace_lock));
994		list_insert_tail(vd_list, vd);
995	}
996}
997
998/*
999 * Requests that all listed vdevs stop trimming.
1000 */
1001static void
1002vdev_trim_stop_all_impl(vdev_t *vd, vdev_trim_state_t tgt_state,
1003    list_t *vd_list)
1004{
1005	if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) {
1006		mutex_enter(&vd->vdev_trim_lock);
1007		vdev_trim_stop(vd, tgt_state, vd_list);
1008		mutex_exit(&vd->vdev_trim_lock);
1009		return;
1010	}
1011
1012	for (uint64_t i = 0; i < vd->vdev_children; i++) {
1013		vdev_trim_stop_all_impl(vd->vdev_child[i], tgt_state,
1014		    vd_list);
1015	}
1016}
1017
1018/*
1019 * Convenience function to stop trimming of a vdev tree and set all trim
1020 * thread pointers to NULL.
1021 */
1022void
1023vdev_trim_stop_all(vdev_t *vd, vdev_trim_state_t tgt_state)
1024{
1025	spa_t *spa = vd->vdev_spa;
1026	list_t vd_list;
1027
1028	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1029
1030	list_create(&vd_list, sizeof (vdev_t),
1031	    offsetof(vdev_t, vdev_trim_node));
1032
1033	vdev_trim_stop_all_impl(vd, tgt_state, &vd_list);
1034	vdev_trim_stop_wait(spa, &vd_list);
1035
1036	if (vd->vdev_spa->spa_sync_on) {
1037		/* Make sure that our state has been synced to disk */
1038		txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
1039	}
1040
1041	list_destroy(&vd_list);
1042}
1043
1044/*
1045 * Conditionally restarts a manual TRIM given its on-disk state.
1046 */
1047void
1048vdev_trim_restart(vdev_t *vd)
1049{
1050	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1051	ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
1052
1053	if (vd->vdev_leaf_zap != 0) {
1054		mutex_enter(&vd->vdev_trim_lock);
1055		uint64_t trim_state = VDEV_TRIM_NONE;
1056		int err = zap_lookup(vd->vdev_spa->spa_meta_objset,
1057		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,
1058		    sizeof (trim_state), 1, &trim_state);
1059		ASSERT(err == 0 || err == ENOENT);
1060		vd->vdev_trim_state = trim_state;
1061
1062		uint64_t timestamp = 0;
1063		err = zap_lookup(vd->vdev_spa->spa_meta_objset,
1064		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_ACTION_TIME,
1065		    sizeof (timestamp), 1, &timestamp);
1066		ASSERT(err == 0 || err == ENOENT);
1067		vd->vdev_trim_action_time = (time_t)timestamp;
1068
1069		if (vd->vdev_trim_state == VDEV_TRIM_SUSPENDED ||
1070		    vd->vdev_offline) {
1071			/* load progress for reporting, but don't resume */
1072			VERIFY0(vdev_trim_load(vd));
1073		} else if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE &&
1074		    vdev_writeable(vd) && !vd->vdev_top->vdev_removing &&
1075		    vd->vdev_trim_thread == NULL) {
1076			VERIFY0(vdev_trim_load(vd));
1077			vdev_trim(vd, vd->vdev_trim_rate,
1078			    vd->vdev_trim_partial, vd->vdev_trim_secure);
1079		}
1080
1081		mutex_exit(&vd->vdev_trim_lock);
1082	}
1083
1084	for (uint64_t i = 0; i < vd->vdev_children; i++) {
1085		vdev_trim_restart(vd->vdev_child[i]);
1086	}
1087}
1088
1089/*
1090 * Used by the automatic TRIM when ZFS_DEBUG_TRIM is set to verify that
1091 * every TRIM range is contained within ms_allocatable.
1092 */
1093static void
1094vdev_trim_range_verify(void *arg, uint64_t start, uint64_t size)
1095{
1096	trim_args_t *ta = arg;
1097	metaslab_t *msp = ta->trim_msp;
1098
1099	VERIFY3B(msp->ms_loaded, ==, B_TRUE);
1100	VERIFY3U(msp->ms_disabled, >, 0);
1101	VERIFY(range_tree_contains(msp->ms_allocatable, start, size));
1102}
1103
1104/*
1105 * Each automatic TRIM thread is responsible for managing the trimming of a
1106 * top-level vdev in the pool.  No automatic TRIM state is maintained on-disk.
1107 *
1108 * N.B. This behavior is different from a manual TRIM where a thread
1109 * is created for each leaf vdev, instead of each top-level vdev.
1110 */
1111static void
1112vdev_autotrim_thread(void *arg)
1113{
1114	vdev_t *vd = arg;
1115	spa_t *spa = vd->vdev_spa;
1116	int shift = 0;
1117
1118	mutex_enter(&vd->vdev_autotrim_lock);
1119	ASSERT3P(vd->vdev_top, ==, vd);
1120	ASSERT3P(vd->vdev_autotrim_thread, !=, NULL);
1121	mutex_exit(&vd->vdev_autotrim_lock);
1122	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1123
1124	uint64_t extent_bytes_max = zfs_trim_extent_bytes_max;
1125	uint64_t extent_bytes_min = zfs_trim_extent_bytes_min;
1126
1127	while (!vdev_autotrim_should_stop(vd)) {
1128		int txgs_per_trim = MAX(zfs_trim_txg_batch, 1);
1129		boolean_t issued_trim = B_FALSE;
1130
1131		/*
1132		 * All of the metaslabs are divided in to groups of size
1133		 * num_metaslabs / zfs_trim_txg_batch.  Each of these groups
1134		 * is composed of metaslabs which are spread evenly over the
1135		 * device.
1136		 *
1137		 * For example, when zfs_trim_txg_batch = 32 (default) then
1138		 * group 0 will contain metaslabs 0, 32, 64, ...;
1139		 * group 1 will contain metaslabs 1, 33, 65, ...;
1140		 * group 2 will contain metaslabs 2, 34, 66, ...; and so on.
1141		 *
1142		 * On each pass through the while() loop one of these groups
1143		 * is selected.  This is accomplished by using a shift value
1144		 * to select the starting metaslab, then striding over the
1145		 * metaslabs using the zfs_trim_txg_batch size.  This is
1146		 * done to accomplish two things.
1147		 *
1148		 * 1) By dividing the metaslabs into groups, and making sure
1149		 *    that each group takes a minimum of one txg to process.
1150		 *    Then zfs_trim_txg_batch controls the minimum number of
1151		 *    txgs which must occur before a metaslab is revisited.
1152		 *
1153		 * 2) Selecting non-consecutive metaslabs distributes the
1154		 *    TRIM commands for a group evenly over the entire device.
1155		 *    This can be advantageous for certain types of devices.
1156		 */
1157		for (uint64_t i = shift % txgs_per_trim; i < vd->vdev_ms_count;
1158		    i += txgs_per_trim) {
1159			metaslab_t *msp = vd->vdev_ms[i];
1160			range_tree_t *trim_tree;
1161
1162			spa_config_exit(spa, SCL_CONFIG, FTAG);
1163			metaslab_disable(msp);
1164			spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1165
1166			mutex_enter(&msp->ms_lock);
1167
1168			/*
1169			 * Skip the metaslab when it has never been allocated
1170			 * or when there are no recent frees to trim.
1171			 */
1172			if (msp->ms_sm == NULL ||
1173			    range_tree_is_empty(msp->ms_trim)) {
1174				mutex_exit(&msp->ms_lock);
1175				metaslab_enable(msp, B_FALSE, B_FALSE);
1176				continue;
1177			}
1178
1179			/*
1180			 * Skip the metaslab when it has already been disabled.
1181			 * This may happen when a manual TRIM or initialize
1182			 * operation is running concurrently.  In the case
1183			 * of a manual TRIM, the ms_trim tree will have been
1184			 * vacated.  Only ranges added after the manual TRIM
1185			 * disabled the metaslab will be included in the tree.
1186			 * These will be processed when the automatic TRIM
1187			 * next revisits this metaslab.
1188			 */
1189			if (msp->ms_disabled > 1) {
1190				mutex_exit(&msp->ms_lock);
1191				metaslab_enable(msp, B_FALSE, B_FALSE);
1192				continue;
1193			}
1194
1195			/*
1196			 * Allocate an empty range tree which is swapped in
1197			 * for the existing ms_trim tree while it is processed.
1198			 */
1199			trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL,
1200			    0, 0);
1201			range_tree_swap(&msp->ms_trim, &trim_tree);
1202			ASSERT(range_tree_is_empty(msp->ms_trim));
1203
1204			/*
1205			 * There are two cases when constructing the per-vdev
1206			 * trim trees for a metaslab.  If the top-level vdev
1207			 * has no children then it is also a leaf and should
1208			 * be trimmed.  Otherwise our children are the leaves
1209			 * and a trim tree should be constructed for each.
1210			 */
1211			trim_args_t *tap;
1212			uint64_t children = vd->vdev_children;
1213			if (children == 0) {
1214				children = 1;
1215				tap = kmem_zalloc(sizeof (trim_args_t) *
1216				    children, KM_SLEEP);
1217				tap[0].trim_vdev = vd;
1218			} else {
1219				tap = kmem_zalloc(sizeof (trim_args_t) *
1220				    children, KM_SLEEP);
1221
1222				for (uint64_t c = 0; c < children; c++) {
1223					tap[c].trim_vdev = vd->vdev_child[c];
1224				}
1225			}
1226
1227			for (uint64_t c = 0; c < children; c++) {
1228				trim_args_t *ta = &tap[c];
1229				vdev_t *cvd = ta->trim_vdev;
1230
1231				ta->trim_msp = msp;
1232				ta->trim_extent_bytes_max = extent_bytes_max;
1233				ta->trim_extent_bytes_min = extent_bytes_min;
1234				ta->trim_type = TRIM_TYPE_AUTO;
1235				ta->trim_flags = 0;
1236
1237				if (cvd->vdev_detached ||
1238				    !vdev_writeable(cvd) ||
1239				    !cvd->vdev_has_trim ||
1240				    cvd->vdev_trim_thread != NULL) {
1241					continue;
1242				}
1243
1244				/*
1245				 * When a device has an attached hot spare, or
1246				 * is being replaced it will not be trimmed.
1247				 * This is done to avoid adding additional
1248				 * stress to a potentially unhealthy device,
1249				 * and to minimize the required rebuild time.
1250				 */
1251				if (!cvd->vdev_ops->vdev_op_leaf)
1252					continue;
1253
1254				ta->trim_tree = range_tree_create(NULL,
1255				    RANGE_SEG64, NULL, 0, 0);
1256				range_tree_walk(trim_tree,
1257				    vdev_trim_range_add, ta);
1258			}
1259
1260			mutex_exit(&msp->ms_lock);
1261			spa_config_exit(spa, SCL_CONFIG, FTAG);
1262
1263			/*
1264			 * Issue the TRIM I/Os for all ranges covered by the
1265			 * TRIM trees.  These ranges are safe to TRIM because
1266			 * no new allocations will be performed until the call
1267			 * to metaslab_enabled() below.
1268			 */
1269			for (uint64_t c = 0; c < children; c++) {
1270				trim_args_t *ta = &tap[c];
1271
1272				/*
1273				 * Always yield to a manual TRIM if one has
1274				 * been started for the child vdev.
1275				 */
1276				if (ta->trim_tree == NULL ||
1277				    ta->trim_vdev->vdev_trim_thread != NULL) {
1278					continue;
1279				}
1280
1281				/*
1282				 * After this point metaslab_enable() must be
1283				 * called with the sync flag set.  This is done
1284				 * here because vdev_trim_ranges() is allowed
1285				 * to be interrupted (EINTR) before issuing all
1286				 * of the required TRIM I/Os.
1287				 */
1288				issued_trim = B_TRUE;
1289
1290				int error = vdev_trim_ranges(ta);
1291				if (error)
1292					break;
1293			}
1294
1295			/*
1296			 * Verify every range which was trimmed is still
1297			 * contained within the ms_allocatable tree.
1298			 */
1299			if (zfs_flags & ZFS_DEBUG_TRIM) {
1300				mutex_enter(&msp->ms_lock);
1301				VERIFY0(metaslab_load(msp));
1302				VERIFY3P(tap[0].trim_msp, ==, msp);
1303				range_tree_walk(trim_tree,
1304				    vdev_trim_range_verify, &tap[0]);
1305				mutex_exit(&msp->ms_lock);
1306			}
1307
1308			range_tree_vacate(trim_tree, NULL, NULL);
1309			range_tree_destroy(trim_tree);
1310
1311			metaslab_enable(msp, issued_trim, B_FALSE);
1312			spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1313
1314			for (uint64_t c = 0; c < children; c++) {
1315				trim_args_t *ta = &tap[c];
1316
1317				if (ta->trim_tree == NULL)
1318					continue;
1319
1320				range_tree_vacate(ta->trim_tree, NULL, NULL);
1321				range_tree_destroy(ta->trim_tree);
1322			}
1323
1324			kmem_free(tap, sizeof (trim_args_t) * children);
1325		}
1326
1327		spa_config_exit(spa, SCL_CONFIG, FTAG);
1328
1329		/*
1330		 * After completing the group of metaslabs wait for the next
1331		 * open txg.  This is done to make sure that a minimum of
1332		 * zfs_trim_txg_batch txgs will occur before these metaslabs
1333		 * are trimmed again.
1334		 */
1335		txg_wait_open(spa_get_dsl(spa), 0, issued_trim);
1336
1337		shift++;
1338		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1339	}
1340
1341	for (uint64_t c = 0; c < vd->vdev_children; c++) {
1342		vdev_t *cvd = vd->vdev_child[c];
1343		mutex_enter(&cvd->vdev_trim_io_lock);
1344
1345		while (cvd->vdev_trim_inflight[1] > 0) {
1346			cv_wait(&cvd->vdev_trim_io_cv,
1347			    &cvd->vdev_trim_io_lock);
1348		}
1349		mutex_exit(&cvd->vdev_trim_io_lock);
1350	}
1351
1352	spa_config_exit(spa, SCL_CONFIG, FTAG);
1353
1354	/*
1355	 * When exiting because the autotrim property was set to off, then
1356	 * abandon any unprocessed ms_trim ranges to reclaim the memory.
1357	 */
1358	if (spa_get_autotrim(spa) == SPA_AUTOTRIM_OFF) {
1359		for (uint64_t i = 0; i < vd->vdev_ms_count; i++) {
1360			metaslab_t *msp = vd->vdev_ms[i];
1361
1362			mutex_enter(&msp->ms_lock);
1363			range_tree_vacate(msp->ms_trim, NULL, NULL);
1364			mutex_exit(&msp->ms_lock);
1365		}
1366	}
1367
1368	mutex_enter(&vd->vdev_autotrim_lock);
1369	ASSERT(vd->vdev_autotrim_thread != NULL);
1370	vd->vdev_autotrim_thread = NULL;
1371	cv_broadcast(&vd->vdev_autotrim_cv);
1372	mutex_exit(&vd->vdev_autotrim_lock);
1373}
1374
1375/*
1376 * Starts an autotrim thread, if needed, for each top-level vdev which can be
1377 * trimmed.  A top-level vdev which has been evacuated will never be trimmed.
1378 */
1379void
1380vdev_autotrim(spa_t *spa)
1381{
1382	vdev_t *root_vd = spa->spa_root_vdev;
1383
1384	for (uint64_t i = 0; i < root_vd->vdev_children; i++) {
1385		vdev_t *tvd = root_vd->vdev_child[i];
1386
1387		mutex_enter(&tvd->vdev_autotrim_lock);
1388		if (vdev_writeable(tvd) && !tvd->vdev_removing &&
1389		    tvd->vdev_autotrim_thread == NULL) {
1390			ASSERT3P(tvd->vdev_top, ==, tvd);
1391
1392			tvd->vdev_autotrim_thread = thread_create(NULL, 0,
1393			    vdev_autotrim_thread, tvd, 0, &p0, TS_RUN,
1394			    maxclsyspri);
1395			ASSERT(tvd->vdev_autotrim_thread != NULL);
1396		}
1397		mutex_exit(&tvd->vdev_autotrim_lock);
1398	}
1399}
1400
1401/*
1402 * Wait for the vdev_autotrim_thread associated with the passed top-level
1403 * vdev to be terminated (canceled or stopped).
1404 */
1405void
1406vdev_autotrim_stop_wait(vdev_t *tvd)
1407{
1408	mutex_enter(&tvd->vdev_autotrim_lock);
1409	if (tvd->vdev_autotrim_thread != NULL) {
1410		tvd->vdev_autotrim_exit_wanted = B_TRUE;
1411
1412		while (tvd->vdev_autotrim_thread != NULL) {
1413			cv_wait(&tvd->vdev_autotrim_cv,
1414			    &tvd->vdev_autotrim_lock);
1415		}
1416
1417		ASSERT3P(tvd->vdev_autotrim_thread, ==, NULL);
1418		tvd->vdev_autotrim_exit_wanted = B_FALSE;
1419	}
1420	mutex_exit(&tvd->vdev_autotrim_lock);
1421}
1422
1423/*
1424 * Wait for all of the vdev_autotrim_thread associated with the pool to
1425 * be terminated (canceled or stopped).
1426 */
1427void
1428vdev_autotrim_stop_all(spa_t *spa)
1429{
1430	vdev_t *root_vd = spa->spa_root_vdev;
1431
1432	for (uint64_t i = 0; i < root_vd->vdev_children; i++)
1433		vdev_autotrim_stop_wait(root_vd->vdev_child[i]);
1434}
1435
1436/*
1437 * Conditionally restart all of the vdev_autotrim_thread's for the pool.
1438 */
1439void
1440vdev_autotrim_restart(spa_t *spa)
1441{
1442	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1443
1444	if (spa->spa_autotrim)
1445		vdev_autotrim(spa);
1446}
1447