1084fd14fSBrian Behlendorf /*
2084fd14fSBrian Behlendorf * CDDL HEADER START
3084fd14fSBrian Behlendorf *
4084fd14fSBrian Behlendorf * The contents of this file are subject to the terms of the
5084fd14fSBrian Behlendorf * Common Development and Distribution License (the "License").
6084fd14fSBrian Behlendorf * You may not use this file except in compliance with the License.
7084fd14fSBrian Behlendorf *
8084fd14fSBrian Behlendorf * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9084fd14fSBrian Behlendorf * or http://www.opensolaris.org/os/licensing.
10084fd14fSBrian Behlendorf * See the License for the specific language governing permissions
11084fd14fSBrian Behlendorf * and limitations under the License.
12084fd14fSBrian Behlendorf *
13084fd14fSBrian Behlendorf * When distributing Covered Code, include this CDDL HEADER in each
14084fd14fSBrian Behlendorf * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15084fd14fSBrian Behlendorf * If applicable, add the following below this CDDL HEADER, with the
16084fd14fSBrian Behlendorf * fields enclosed by brackets "[]" replaced with your own identifying
17084fd14fSBrian Behlendorf * information: Portions Copyright [yyyy] [name of copyright owner]
18084fd14fSBrian Behlendorf *
19084fd14fSBrian Behlendorf * CDDL HEADER END
20084fd14fSBrian Behlendorf */
21084fd14fSBrian Behlendorf
22084fd14fSBrian Behlendorf /*
23084fd14fSBrian Behlendorf * Copyright (c) 2016 by Delphix. All rights reserved.
24084fd14fSBrian Behlendorf * Copyright (c) 2019 by Lawrence Livermore National Security, LLC.
25084fd14fSBrian Behlendorf * Copyright 2019 Joyent, Inc.
26*a385ae06SJason King * Copyright 2023 RackTop Systems, Inc.
27084fd14fSBrian Behlendorf */
28084fd14fSBrian Behlendorf
29084fd14fSBrian Behlendorf #include <sys/spa.h>
30084fd14fSBrian Behlendorf #include <sys/spa_impl.h>
31084fd14fSBrian Behlendorf #include <sys/txg.h>
32084fd14fSBrian Behlendorf #include <sys/vdev_impl.h>
33084fd14fSBrian Behlendorf #include <sys/vdev_trim.h>
34084fd14fSBrian Behlendorf #include <sys/refcount.h>
35084fd14fSBrian Behlendorf #include <sys/metaslab_impl.h>
36084fd14fSBrian Behlendorf #include <sys/dsl_synctask.h>
37084fd14fSBrian Behlendorf #include <sys/zap.h>
38084fd14fSBrian Behlendorf #include <sys/dmu_tx.h>
39084fd14fSBrian Behlendorf
40084fd14fSBrian Behlendorf /*
41084fd14fSBrian Behlendorf * TRIM is a feature which is used to notify a SSD that some previously
42084fd14fSBrian Behlendorf * written space is no longer allocated by the pool. This is useful because
43084fd14fSBrian Behlendorf * writes to a SSD must be performed to blocks which have first been erased.
44084fd14fSBrian Behlendorf * Ensuring the SSD always has a supply of erased blocks for new writes
45084fd14fSBrian Behlendorf * helps prevent the performance from deteriorating.
46084fd14fSBrian Behlendorf *
47084fd14fSBrian Behlendorf * There are two supported TRIM methods; manual and automatic.
48084fd14fSBrian Behlendorf *
49084fd14fSBrian Behlendorf * Manual TRIM:
50084fd14fSBrian Behlendorf *
51084fd14fSBrian Behlendorf * A manual TRIM is initiated by running the 'zpool trim' command. A single
52084fd14fSBrian Behlendorf * 'vdev_trim' thread is created for each leaf vdev, and it is responsible for
53084fd14fSBrian Behlendorf * managing that vdev TRIM process. This involves iterating over all the
54084fd14fSBrian Behlendorf * metaslabs, calculating the unallocated space ranges, and then issuing the
55084fd14fSBrian Behlendorf * required TRIM I/Os.
56084fd14fSBrian Behlendorf *
57084fd14fSBrian Behlendorf * While a metaslab is being actively trimmed it is not eligible to perform
58084fd14fSBrian Behlendorf * new allocations. After traversing all of the metaslabs the thread is
59084fd14fSBrian Behlendorf * terminated. Finally, both the requested options and current progress of
60084fd14fSBrian Behlendorf * the TRIM are regularly written to the pool. This allows the TRIM to be
61084fd14fSBrian Behlendorf * suspended and resumed as needed.
62084fd14fSBrian Behlendorf *
63084fd14fSBrian Behlendorf * Automatic TRIM:
64084fd14fSBrian Behlendorf *
65084fd14fSBrian Behlendorf * An automatic TRIM is enabled by setting the 'autotrim' pool property
66084fd14fSBrian Behlendorf * to 'on'. When enabled, a `vdev_autotrim' thread is created for each
67084fd14fSBrian Behlendorf * top-level (not leaf) vdev in the pool. These threads perform the same
68084fd14fSBrian Behlendorf * core TRIM process as a manual TRIM, but with a few key differences.
69084fd14fSBrian Behlendorf *
70084fd14fSBrian Behlendorf * 1) Automatic TRIM happens continuously in the background and operates
71084fd14fSBrian Behlendorf * solely on recently freed blocks (ms_trim not ms_allocatable).
72084fd14fSBrian Behlendorf *
73084fd14fSBrian Behlendorf * 2) Each thread is associated with a top-level (not leaf) vdev. This has
74084fd14fSBrian Behlendorf * the benefit of simplifying the threading model, it makes it easier
75084fd14fSBrian Behlendorf * to coordinate administrative commands, and it ensures only a single
76084fd14fSBrian Behlendorf * metaslab is disabled at a time. Unlike manual TRIM, this means each
77084fd14fSBrian Behlendorf * 'vdev_autotrim' thread is responsible for issuing TRIM I/Os for its
78084fd14fSBrian Behlendorf * children.
79084fd14fSBrian Behlendorf *
80084fd14fSBrian Behlendorf * 3) There is no automatic TRIM progress information stored on disk, nor
81084fd14fSBrian Behlendorf * is it reported by 'zpool status'.
82084fd14fSBrian Behlendorf *
83084fd14fSBrian Behlendorf * While the automatic TRIM process is highly effective it is more likely
84084fd14fSBrian Behlendorf * than a manual TRIM to encounter tiny ranges. Ranges less than or equal to
85084fd14fSBrian Behlendorf * 'zfs_trim_extent_bytes_min' (32k) are considered too small to efficiently
86084fd14fSBrian Behlendorf * TRIM and are skipped. This means small amounts of freed space may not
87084fd14fSBrian Behlendorf * be automatically trimmed.
88084fd14fSBrian Behlendorf *
89084fd14fSBrian Behlendorf * Furthermore, devices with attached hot spares and devices being actively
90084fd14fSBrian Behlendorf * replaced are skipped. This is done to avoid adding additional stress to
91084fd14fSBrian Behlendorf * a potentially unhealthy device and to minimize the required rebuild time.
92084fd14fSBrian Behlendorf *
93084fd14fSBrian Behlendorf * For this reason it may be beneficial to occasionally manually TRIM a pool
94084fd14fSBrian Behlendorf * even when automatic TRIM is enabled.
95084fd14fSBrian Behlendorf */
96084fd14fSBrian Behlendorf
97084fd14fSBrian Behlendorf /*
98084fd14fSBrian Behlendorf * Maximum size of TRIM I/O, ranges will be chunked in to 128MiB lengths.
99084fd14fSBrian Behlendorf */
100084fd14fSBrian Behlendorf unsigned int zfs_trim_extent_bytes_max = 128 * 1024 * 1024;
101084fd14fSBrian Behlendorf
102084fd14fSBrian Behlendorf /*
103084fd14fSBrian Behlendorf * Minimum size of TRIM I/O, extents smaller than 32Kib will be skipped.
104084fd14fSBrian Behlendorf */
105084fd14fSBrian Behlendorf unsigned int zfs_trim_extent_bytes_min = 32 * 1024;
106084fd14fSBrian Behlendorf
107084fd14fSBrian Behlendorf /*
108084fd14fSBrian Behlendorf * Skip uninitialized metaslabs during the TRIM process. This option is
109084fd14fSBrian Behlendorf * useful for pools constructed from large thinly-provisioned devices where
110084fd14fSBrian Behlendorf * TRIM operations are slow. As a pool ages an increasing fraction of
111084fd14fSBrian Behlendorf * the pools metaslabs will be initialized progressively degrading the
112084fd14fSBrian Behlendorf * usefulness of this option. This setting is stored when starting a
113084fd14fSBrian Behlendorf * manual TRIM and will persist for the duration of the requested TRIM.
114084fd14fSBrian Behlendorf */
115084fd14fSBrian Behlendorf unsigned int zfs_trim_metaslab_skip = 0;
116084fd14fSBrian Behlendorf
117084fd14fSBrian Behlendorf /*
118084fd14fSBrian Behlendorf * Maximum number of queued TRIM I/Os per leaf vdev. The number of
119084fd14fSBrian Behlendorf * concurrent TRIM I/Os issued to the device is controlled by the
120084fd14fSBrian Behlendorf * zfs_vdev_trim_min_active and zfs_vdev_trim_max_active module options.
121084fd14fSBrian Behlendorf */
122084fd14fSBrian Behlendorf unsigned int zfs_trim_queue_limit = 10;
123084fd14fSBrian Behlendorf
124084fd14fSBrian Behlendorf /*
125084fd14fSBrian Behlendorf * The minimum number of transaction groups between automatic trims of a
126084fd14fSBrian Behlendorf * metaslab. This setting represents a trade-off between issuing more
127084fd14fSBrian Behlendorf * efficient TRIM operations, by allowing them to be aggregated longer,
128084fd14fSBrian Behlendorf * and issuing them promptly so the trimmed space is available. Note
129084fd14fSBrian Behlendorf * that this value is a minimum; metaslabs can be trimmed less frequently
130084fd14fSBrian Behlendorf * when there are a large number of ranges which need to be trimmed.
131084fd14fSBrian Behlendorf *
132084fd14fSBrian Behlendorf * Increasing this value will allow frees to be aggregated for a longer
133084fd14fSBrian Behlendorf * time. This can result is larger TRIM operations, and increased memory
134084fd14fSBrian Behlendorf * usage in order to track the ranges to be trimmed. Decreasing this value
135084fd14fSBrian Behlendorf * has the opposite effect. The default value of 32 was determined though
136084fd14fSBrian Behlendorf * testing to be a reasonable compromise.
137084fd14fSBrian Behlendorf */
138084fd14fSBrian Behlendorf unsigned int zfs_trim_txg_batch = 32;
139084fd14fSBrian Behlendorf
140084fd14fSBrian Behlendorf /*
141084fd14fSBrian Behlendorf * The trim_args are a control structure which describe how a leaf vdev
142084fd14fSBrian Behlendorf * should be trimmed. The core elements are the vdev, the metaslab being
143084fd14fSBrian Behlendorf * trimmed and a range tree containing the extents to TRIM. All provided
144084fd14fSBrian Behlendorf * ranges must be within the metaslab.
145084fd14fSBrian Behlendorf */
146084fd14fSBrian Behlendorf typedef struct trim_args {
147084fd14fSBrian Behlendorf /*
148084fd14fSBrian Behlendorf * These fields are set by the caller of vdev_trim_ranges().
149084fd14fSBrian Behlendorf */
150084fd14fSBrian Behlendorf vdev_t *trim_vdev; /* Leaf vdev to TRIM */
151084fd14fSBrian Behlendorf metaslab_t *trim_msp; /* Disabled metaslab */
152084fd14fSBrian Behlendorf range_tree_t *trim_tree; /* TRIM ranges (in metaslab) */
153084fd14fSBrian Behlendorf trim_type_t trim_type; /* Manual or auto TRIM */
154084fd14fSBrian Behlendorf uint64_t trim_extent_bytes_max; /* Maximum TRIM I/O size */
155084fd14fSBrian Behlendorf uint64_t trim_extent_bytes_min; /* Minimum TRIM I/O size */
156084fd14fSBrian Behlendorf enum trim_flag trim_flags; /* TRIM flags (secure) */
157084fd14fSBrian Behlendorf
158084fd14fSBrian Behlendorf /*
159084fd14fSBrian Behlendorf * These fields are updated by vdev_trim_ranges().
160084fd14fSBrian Behlendorf */
161084fd14fSBrian Behlendorf hrtime_t trim_start_time; /* Start time */
162084fd14fSBrian Behlendorf uint64_t trim_bytes_done; /* Bytes trimmed */
163084fd14fSBrian Behlendorf } trim_args_t;
164084fd14fSBrian Behlendorf
165084fd14fSBrian Behlendorf /*
166084fd14fSBrian Behlendorf * Determines whether a vdev_trim_thread() should be stopped.
167084fd14fSBrian Behlendorf */
168084fd14fSBrian Behlendorf static boolean_t
vdev_trim_should_stop(vdev_t * vd)169084fd14fSBrian Behlendorf vdev_trim_should_stop(vdev_t *vd)
170084fd14fSBrian Behlendorf {
171084fd14fSBrian Behlendorf return (vd->vdev_trim_exit_wanted || !vdev_writeable(vd) ||
172084fd14fSBrian Behlendorf vd->vdev_detached || vd->vdev_top->vdev_removing);
173084fd14fSBrian Behlendorf }
174084fd14fSBrian Behlendorf
175084fd14fSBrian Behlendorf /*
176084fd14fSBrian Behlendorf * Determines whether a vdev_autotrim_thread() should be stopped.
177084fd14fSBrian Behlendorf */
178084fd14fSBrian Behlendorf static boolean_t
vdev_autotrim_should_stop(vdev_t * tvd)179084fd14fSBrian Behlendorf vdev_autotrim_should_stop(vdev_t *tvd)
180084fd14fSBrian Behlendorf {
181084fd14fSBrian Behlendorf return (tvd->vdev_autotrim_exit_wanted ||
182084fd14fSBrian Behlendorf !vdev_writeable(tvd) || tvd->vdev_removing ||
183084fd14fSBrian Behlendorf spa_get_autotrim(tvd->vdev_spa) == SPA_AUTOTRIM_OFF);
184084fd14fSBrian Behlendorf }
185084fd14fSBrian Behlendorf
186084fd14fSBrian Behlendorf /*
187084fd14fSBrian Behlendorf * The sync task for updating the on-disk state of a manual TRIM. This
188084fd14fSBrian Behlendorf * is scheduled by vdev_trim_change_state().
189084fd14fSBrian Behlendorf */
190084fd14fSBrian Behlendorf static void
vdev_trim_zap_update_sync(void * arg,dmu_tx_t * tx)191084fd14fSBrian Behlendorf vdev_trim_zap_update_sync(void *arg, dmu_tx_t *tx)
192084fd14fSBrian Behlendorf {
193084fd14fSBrian Behlendorf /*
194084fd14fSBrian Behlendorf * We pass in the guid instead of the vdev_t since the vdev may
195084fd14fSBrian Behlendorf * have been freed prior to the sync task being processed. This
196084fd14fSBrian Behlendorf * happens when a vdev is detached as we call spa_config_vdev_exit(),
197084fd14fSBrian Behlendorf * stop the trimming thread, schedule the sync task, and free
198084fd14fSBrian Behlendorf * the vdev. Later when the scheduled sync task is invoked, it would
199084fd14fSBrian Behlendorf * find that the vdev has been freed.
200084fd14fSBrian Behlendorf */
201084fd14fSBrian Behlendorf uint64_t guid = *(uint64_t *)arg;
202084fd14fSBrian Behlendorf uint64_t txg = dmu_tx_get_txg(tx);
203084fd14fSBrian Behlendorf kmem_free(arg, sizeof (uint64_t));
204084fd14fSBrian Behlendorf
205084fd14fSBrian Behlendorf vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);
206084fd14fSBrian Behlendorf if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd))
207084fd14fSBrian Behlendorf return;
208084fd14fSBrian Behlendorf
209084fd14fSBrian Behlendorf uint64_t last_offset = vd->vdev_trim_offset[txg & TXG_MASK];
210084fd14fSBrian Behlendorf vd->vdev_trim_offset[txg & TXG_MASK] = 0;
211084fd14fSBrian Behlendorf
212084fd14fSBrian Behlendorf VERIFY3U(vd->vdev_leaf_zap, !=, 0);
213084fd14fSBrian Behlendorf
214084fd14fSBrian Behlendorf objset_t *mos = vd->vdev_spa->spa_meta_objset;
215084fd14fSBrian Behlendorf
216084fd14fSBrian Behlendorf if (last_offset > 0 || vd->vdev_trim_last_offset == UINT64_MAX) {
217084fd14fSBrian Behlendorf
218084fd14fSBrian Behlendorf if (vd->vdev_trim_last_offset == UINT64_MAX)
219084fd14fSBrian Behlendorf last_offset = 0;
220084fd14fSBrian Behlendorf
221084fd14fSBrian Behlendorf vd->vdev_trim_last_offset = last_offset;
222084fd14fSBrian Behlendorf VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
223084fd14fSBrian Behlendorf VDEV_LEAF_ZAP_TRIM_LAST_OFFSET,
224084fd14fSBrian Behlendorf sizeof (last_offset), 1, &last_offset, tx));
225084fd14fSBrian Behlendorf }
226084fd14fSBrian Behlendorf
227084fd14fSBrian Behlendorf if (vd->vdev_trim_action_time > 0) {
228084fd14fSBrian Behlendorf uint64_t val = (uint64_t)vd->vdev_trim_action_time;
229084fd14fSBrian Behlendorf VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
230084fd14fSBrian Behlendorf VDEV_LEAF_ZAP_TRIM_ACTION_TIME, sizeof (val),
231084fd14fSBrian Behlendorf 1, &val, tx));
232084fd14fSBrian Behlendorf }
233084fd14fSBrian Behlendorf
234084fd14fSBrian Behlendorf if (vd->vdev_trim_rate > 0) {
235084fd14fSBrian Behlendorf uint64_t rate = (uint64_t)vd->vdev_trim_rate;
236084fd14fSBrian Behlendorf
237084fd14fSBrian Behlendorf if (rate == UINT64_MAX)
238084fd14fSBrian Behlendorf rate = 0;
239084fd14fSBrian Behlendorf
240084fd14fSBrian Behlendorf VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
241084fd14fSBrian Behlendorf VDEV_LEAF_ZAP_TRIM_RATE, sizeof (rate), 1, &rate, tx));
242084fd14fSBrian Behlendorf }
243084fd14fSBrian Behlendorf
244084fd14fSBrian Behlendorf uint64_t partial = vd->vdev_trim_partial;
245084fd14fSBrian Behlendorf if (partial == UINT64_MAX)
246084fd14fSBrian Behlendorf partial = 0;
247084fd14fSBrian Behlendorf
248084fd14fSBrian Behlendorf VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,
249084fd14fSBrian Behlendorf sizeof (partial), 1, &partial, tx));
250084fd14fSBrian Behlendorf
251084fd14fSBrian Behlendorf uint64_t secure = vd->vdev_trim_secure;
252084fd14fSBrian Behlendorf if (secure == UINT64_MAX)
253084fd14fSBrian Behlendorf secure = 0;
254084fd14fSBrian Behlendorf
255084fd14fSBrian Behlendorf VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,
256084fd14fSBrian Behlendorf sizeof (secure), 1, &secure, tx));
257084fd14fSBrian Behlendorf
258084fd14fSBrian Behlendorf
259084fd14fSBrian Behlendorf uint64_t trim_state = vd->vdev_trim_state;
260084fd14fSBrian Behlendorf VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,
261084fd14fSBrian Behlendorf sizeof (trim_state), 1, &trim_state, tx));
262084fd14fSBrian Behlendorf }
263084fd14fSBrian Behlendorf
264084fd14fSBrian Behlendorf /*
265084fd14fSBrian Behlendorf * Update the on-disk state of a manual TRIM. This is called to request
266084fd14fSBrian Behlendorf * that a TRIM be started/suspended/canceled, or to change one of the
267084fd14fSBrian Behlendorf * TRIM options (partial, secure, rate).
268084fd14fSBrian Behlendorf */
269084fd14fSBrian Behlendorf static void
vdev_trim_change_state(vdev_t * vd,vdev_trim_state_t new_state,uint64_t rate,boolean_t partial,boolean_t secure)270084fd14fSBrian Behlendorf vdev_trim_change_state(vdev_t *vd, vdev_trim_state_t new_state,
271084fd14fSBrian Behlendorf uint64_t rate, boolean_t partial, boolean_t secure)
272084fd14fSBrian Behlendorf {
273084fd14fSBrian Behlendorf ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
274084fd14fSBrian Behlendorf spa_t *spa = vd->vdev_spa;
275084fd14fSBrian Behlendorf
276084fd14fSBrian Behlendorf if (new_state == vd->vdev_trim_state)
277084fd14fSBrian Behlendorf return;
278084fd14fSBrian Behlendorf
279084fd14fSBrian Behlendorf /*
280084fd14fSBrian Behlendorf * Copy the vd's guid, this will be freed by the sync task.
281084fd14fSBrian Behlendorf */
282084fd14fSBrian Behlendorf uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
283084fd14fSBrian Behlendorf *guid = vd->vdev_guid;
284084fd14fSBrian Behlendorf
285084fd14fSBrian Behlendorf /*
286084fd14fSBrian Behlendorf * If we're suspending, then preserve the original start time.
287084fd14fSBrian Behlendorf */
288084fd14fSBrian Behlendorf if (vd->vdev_trim_state != VDEV_TRIM_SUSPENDED) {
289084fd14fSBrian Behlendorf vd->vdev_trim_action_time = gethrestime_sec();
290084fd14fSBrian Behlendorf }
291084fd14fSBrian Behlendorf
292084fd14fSBrian Behlendorf /*
293084fd14fSBrian Behlendorf * If we're activating, then preserve the requested rate and trim
294084fd14fSBrian Behlendorf * method. Setting the last offset and rate to UINT64_MAX is used
295084fd14fSBrian Behlendorf * as a sentinel to indicate they should be reset to default values.
296084fd14fSBrian Behlendorf */
297084fd14fSBrian Behlendorf if (new_state == VDEV_TRIM_ACTIVE) {
298084fd14fSBrian Behlendorf if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE ||
299084fd14fSBrian Behlendorf vd->vdev_trim_state == VDEV_TRIM_CANCELED) {
300084fd14fSBrian Behlendorf vd->vdev_trim_last_offset = UINT64_MAX;
301084fd14fSBrian Behlendorf vd->vdev_trim_rate = UINT64_MAX;
302084fd14fSBrian Behlendorf vd->vdev_trim_partial = UINT64_MAX;
303084fd14fSBrian Behlendorf vd->vdev_trim_secure = UINT64_MAX;
304084fd14fSBrian Behlendorf }
305084fd14fSBrian Behlendorf
306084fd14fSBrian Behlendorf if (rate != 0)
307084fd14fSBrian Behlendorf vd->vdev_trim_rate = rate;
308084fd14fSBrian Behlendorf
309084fd14fSBrian Behlendorf if (partial != 0)
310084fd14fSBrian Behlendorf vd->vdev_trim_partial = partial;
311084fd14fSBrian Behlendorf
312084fd14fSBrian Behlendorf if (secure != 0)
313084fd14fSBrian Behlendorf vd->vdev_trim_secure = secure;
314084fd14fSBrian Behlendorf }
315084fd14fSBrian Behlendorf
316084fd14fSBrian Behlendorf boolean_t resumed = !!(vd->vdev_trim_state == VDEV_TRIM_SUSPENDED);
317084fd14fSBrian Behlendorf vd->vdev_trim_state = new_state;
318084fd14fSBrian Behlendorf
319084fd14fSBrian Behlendorf dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
320084fd14fSBrian Behlendorf VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
321084fd14fSBrian Behlendorf dsl_sync_task_nowait(spa_get_dsl(spa), vdev_trim_zap_update_sync,
322084fd14fSBrian Behlendorf guid, 2, ZFS_SPACE_CHECK_NONE, tx);
323084fd14fSBrian Behlendorf
324084fd14fSBrian Behlendorf switch (new_state) {
325084fd14fSBrian Behlendorf case VDEV_TRIM_ACTIVE:
326084fd14fSBrian Behlendorf spa_event_notify(spa, vd, NULL,
327084fd14fSBrian Behlendorf resumed ? ESC_ZFS_TRIM_RESUME : ESC_ZFS_TRIM_START);
328084fd14fSBrian Behlendorf spa_history_log_internal(spa, "trim", tx,
329084fd14fSBrian Behlendorf "vdev=%s activated", vd->vdev_path);
330084fd14fSBrian Behlendorf break;
331084fd14fSBrian Behlendorf case VDEV_TRIM_SUSPENDED:
332084fd14fSBrian Behlendorf spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_SUSPEND);
333084fd14fSBrian Behlendorf spa_history_log_internal(spa, "trim", tx,
334084fd14fSBrian Behlendorf "vdev=%s suspended", vd->vdev_path);
335084fd14fSBrian Behlendorf break;
336084fd14fSBrian Behlendorf case VDEV_TRIM_CANCELED:
337084fd14fSBrian Behlendorf spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_CANCEL);
338084fd14fSBrian Behlendorf spa_history_log_internal(spa, "trim", tx,
339084fd14fSBrian Behlendorf "vdev=%s canceled", vd->vdev_path);
340084fd14fSBrian Behlendorf break;
341084fd14fSBrian Behlendorf case VDEV_TRIM_COMPLETE:
342084fd14fSBrian Behlendorf spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_FINISH);
343084fd14fSBrian Behlendorf spa_history_log_internal(spa, "trim", tx,
344084fd14fSBrian Behlendorf "vdev=%s complete", vd->vdev_path);
345084fd14fSBrian Behlendorf break;
346084fd14fSBrian Behlendorf default:
347084fd14fSBrian Behlendorf panic("invalid state %llu", (unsigned long long)new_state);
348084fd14fSBrian Behlendorf }
349084fd14fSBrian Behlendorf
350084fd14fSBrian Behlendorf dmu_tx_commit(tx);
351084fd14fSBrian Behlendorf }
352084fd14fSBrian Behlendorf
353084fd14fSBrian Behlendorf /*
354084fd14fSBrian Behlendorf * The zio_done_func_t done callback for each manual TRIM issued. It is
355084fd14fSBrian Behlendorf * responsible for updating the TRIM stats, reissuing failed TRIM I/Os,
356084fd14fSBrian Behlendorf * and limiting the number of in-flight TRIM I/Os.
357084fd14fSBrian Behlendorf */
358084fd14fSBrian Behlendorf static void
vdev_trim_cb(zio_t * zio)359084fd14fSBrian Behlendorf vdev_trim_cb(zio_t *zio)
360084fd14fSBrian Behlendorf {
361084fd14fSBrian Behlendorf vdev_t *vd = zio->io_vd;
362084fd14fSBrian Behlendorf
363084fd14fSBrian Behlendorf mutex_enter(&vd->vdev_trim_io_lock);
364084fd14fSBrian Behlendorf if (zio->io_error == ENXIO && !vdev_writeable(vd)) {
365084fd14fSBrian Behlendorf /*
366084fd14fSBrian Behlendorf * The I/O failed because the vdev was unavailable; roll the
367084fd14fSBrian Behlendorf * last offset back. (This works because spa_sync waits on
368084fd14fSBrian Behlendorf * spa_txg_zio before it runs sync tasks.)
369084fd14fSBrian Behlendorf */
370084fd14fSBrian Behlendorf uint64_t *offset =
371084fd14fSBrian Behlendorf &vd->vdev_trim_offset[zio->io_txg & TXG_MASK];
372084fd14fSBrian Behlendorf *offset = MIN(*offset, zio->io_offset);
373084fd14fSBrian Behlendorf } else {
374084fd14fSBrian Behlendorf if (zio->io_error != 0) {
375084fd14fSBrian Behlendorf vd->vdev_stat.vs_trim_errors++;
376084fd14fSBrian Behlendorf /*
377084fd14fSBrian Behlendorf * spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,
378084fd14fSBrian Behlendorf * 0, 0, 0, 0, 1, zio->io_orig_size);
379084fd14fSBrian Behlendorf */
380084fd14fSBrian Behlendorf } else {
381084fd14fSBrian Behlendorf /*
382084fd14fSBrian Behlendorf * spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,
383084fd14fSBrian Behlendorf * 1, zio->io_orig_size, 0, 0, 0, 0);
384084fd14fSBrian Behlendorf */
385084fd14fSBrian Behlendorf }
386084fd14fSBrian Behlendorf
387084fd14fSBrian Behlendorf vd->vdev_trim_bytes_done += zio->io_orig_size;
388084fd14fSBrian Behlendorf }
389084fd14fSBrian Behlendorf
390084fd14fSBrian Behlendorf ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_MANUAL], >, 0);
391084fd14fSBrian Behlendorf vd->vdev_trim_inflight[TRIM_TYPE_MANUAL]--;
392084fd14fSBrian Behlendorf cv_broadcast(&vd->vdev_trim_io_cv);
393084fd14fSBrian Behlendorf mutex_exit(&vd->vdev_trim_io_lock);
394084fd14fSBrian Behlendorf
395084fd14fSBrian Behlendorf spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
396084fd14fSBrian Behlendorf }
397084fd14fSBrian Behlendorf
398084fd14fSBrian Behlendorf /*
399084fd14fSBrian Behlendorf * The zio_done_func_t done callback for each automatic TRIM issued. It
400084fd14fSBrian Behlendorf * is responsible for updating the TRIM stats and limiting the number of
401084fd14fSBrian Behlendorf * in-flight TRIM I/Os. Automatic TRIM I/Os are best effort and are
402084fd14fSBrian Behlendorf * never reissued on failure.
403084fd14fSBrian Behlendorf */
404084fd14fSBrian Behlendorf static void
vdev_autotrim_cb(zio_t * zio)405084fd14fSBrian Behlendorf vdev_autotrim_cb(zio_t *zio)
406084fd14fSBrian Behlendorf {
407084fd14fSBrian Behlendorf vdev_t *vd = zio->io_vd;
408084fd14fSBrian Behlendorf
409084fd14fSBrian Behlendorf mutex_enter(&vd->vdev_trim_io_lock);
410084fd14fSBrian Behlendorf
411084fd14fSBrian Behlendorf if (zio->io_error != 0) {
412084fd14fSBrian Behlendorf vd->vdev_stat.vs_trim_errors++;
413084fd14fSBrian Behlendorf /*
414084fd14fSBrian Behlendorf * spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,
415084fd14fSBrian Behlendorf * 0, 0, 0, 0, 1, zio->io_orig_size);
416084fd14fSBrian Behlendorf */
417084fd14fSBrian Behlendorf } else {
418084fd14fSBrian Behlendorf /*
419084fd14fSBrian Behlendorf * spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,
420084fd14fSBrian Behlendorf * 1, zio->io_orig_size, 0, 0, 0, 0);
421084fd14fSBrian Behlendorf */
422084fd14fSBrian Behlendorf
423084fd14fSBrian Behlendorf vd->vdev_autotrim_bytes_done += zio->io_orig_size;
424084fd14fSBrian Behlendorf }
425084fd14fSBrian Behlendorf
426084fd14fSBrian Behlendorf ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_AUTO], >, 0);
427084fd14fSBrian Behlendorf vd->vdev_trim_inflight[TRIM_TYPE_AUTO]--;
428084fd14fSBrian Behlendorf cv_broadcast(&vd->vdev_trim_io_cv);
429084fd14fSBrian Behlendorf mutex_exit(&vd->vdev_trim_io_lock);
430084fd14fSBrian Behlendorf
431084fd14fSBrian Behlendorf spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
432084fd14fSBrian Behlendorf }
433084fd14fSBrian Behlendorf
434084fd14fSBrian Behlendorf /*
435084fd14fSBrian Behlendorf * Returns the average trim rate in bytes/sec for the ta->trim_vdev.
436084fd14fSBrian Behlendorf */
437084fd14fSBrian Behlendorf static uint64_t
vdev_trim_calculate_rate(trim_args_t * ta)438084fd14fSBrian Behlendorf vdev_trim_calculate_rate(trim_args_t *ta)
439084fd14fSBrian Behlendorf {
440084fd14fSBrian Behlendorf return (ta->trim_bytes_done * 1000 /
441084fd14fSBrian Behlendorf (NSEC2MSEC(gethrtime() - ta->trim_start_time) + 1));
442084fd14fSBrian Behlendorf }
443084fd14fSBrian Behlendorf
444084fd14fSBrian Behlendorf /*
445084fd14fSBrian Behlendorf * Issues a physical TRIM and takes care of rate limiting (bytes/sec)
446084fd14fSBrian Behlendorf * and number of concurrent TRIM I/Os.
447084fd14fSBrian Behlendorf */
448084fd14fSBrian Behlendorf static int
vdev_trim_range(trim_args_t * ta,uint64_t start,uint64_t size)449084fd14fSBrian Behlendorf vdev_trim_range(trim_args_t *ta, uint64_t start, uint64_t size)
450084fd14fSBrian Behlendorf {
451084fd14fSBrian Behlendorf vdev_t *vd = ta->trim_vdev;
452084fd14fSBrian Behlendorf spa_t *spa = vd->vdev_spa;
453084fd14fSBrian Behlendorf
454084fd14fSBrian Behlendorf mutex_enter(&vd->vdev_trim_io_lock);
455084fd14fSBrian Behlendorf
456084fd14fSBrian Behlendorf /*
457084fd14fSBrian Behlendorf * Limit manual TRIM I/Os to the requested rate. This does not
458084fd14fSBrian Behlendorf * apply to automatic TRIM since no per vdev rate can be specified.
459084fd14fSBrian Behlendorf */
460084fd14fSBrian Behlendorf if (ta->trim_type == TRIM_TYPE_MANUAL) {
461084fd14fSBrian Behlendorf while (vd->vdev_trim_rate != 0 && !vdev_trim_should_stop(vd) &&
462084fd14fSBrian Behlendorf vdev_trim_calculate_rate(ta) > vd->vdev_trim_rate) {
463084fd14fSBrian Behlendorf cv_timedwait_sig(&vd->vdev_trim_io_cv,
464084fd14fSBrian Behlendorf &vd->vdev_trim_io_lock, ddi_get_lbolt() +
465084fd14fSBrian Behlendorf MSEC_TO_TICK(10));
466084fd14fSBrian Behlendorf }
467084fd14fSBrian Behlendorf }
468084fd14fSBrian Behlendorf ta->trim_bytes_done += size;
469084fd14fSBrian Behlendorf
470084fd14fSBrian Behlendorf /* Limit in-flight trimming I/Os */
471084fd14fSBrian Behlendorf while (vd->vdev_trim_inflight[0] + vd->vdev_trim_inflight[1] >=
472084fd14fSBrian Behlendorf zfs_trim_queue_limit) {
473084fd14fSBrian Behlendorf cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
474084fd14fSBrian Behlendorf }
475084fd14fSBrian Behlendorf vd->vdev_trim_inflight[ta->trim_type]++;
476084fd14fSBrian Behlendorf mutex_exit(&vd->vdev_trim_io_lock);
477084fd14fSBrian Behlendorf
478084fd14fSBrian Behlendorf dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
479084fd14fSBrian Behlendorf VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
480084fd14fSBrian Behlendorf uint64_t txg = dmu_tx_get_txg(tx);
481084fd14fSBrian Behlendorf
482084fd14fSBrian Behlendorf spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER);
483084fd14fSBrian Behlendorf mutex_enter(&vd->vdev_trim_lock);
484084fd14fSBrian Behlendorf
485084fd14fSBrian Behlendorf if (ta->trim_type == TRIM_TYPE_MANUAL &&
486084fd14fSBrian Behlendorf vd->vdev_trim_offset[txg & TXG_MASK] == 0) {
487084fd14fSBrian Behlendorf uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
488084fd14fSBrian Behlendorf *guid = vd->vdev_guid;
489084fd14fSBrian Behlendorf
490084fd14fSBrian Behlendorf /* This is the first write of this txg. */
491084fd14fSBrian Behlendorf dsl_sync_task_nowait(spa_get_dsl(spa),
492084fd14fSBrian Behlendorf vdev_trim_zap_update_sync, guid, 2,
493084fd14fSBrian Behlendorf ZFS_SPACE_CHECK_RESERVED, tx);
494084fd14fSBrian Behlendorf }
495084fd14fSBrian Behlendorf
496084fd14fSBrian Behlendorf /*
497084fd14fSBrian Behlendorf * We know the vdev_t will still be around since all consumers of
498084fd14fSBrian Behlendorf * vdev_free must stop the trimming first.
499084fd14fSBrian Behlendorf */
500084fd14fSBrian Behlendorf if ((ta->trim_type == TRIM_TYPE_MANUAL &&
501084fd14fSBrian Behlendorf vdev_trim_should_stop(vd)) ||
502084fd14fSBrian Behlendorf (ta->trim_type == TRIM_TYPE_AUTO &&
503084fd14fSBrian Behlendorf vdev_autotrim_should_stop(vd->vdev_top))) {
504084fd14fSBrian Behlendorf mutex_enter(&vd->vdev_trim_io_lock);
505084fd14fSBrian Behlendorf vd->vdev_trim_inflight[ta->trim_type]--;
506084fd14fSBrian Behlendorf mutex_exit(&vd->vdev_trim_io_lock);
507084fd14fSBrian Behlendorf spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
508084fd14fSBrian Behlendorf mutex_exit(&vd->vdev_trim_lock);
509084fd14fSBrian Behlendorf dmu_tx_commit(tx);
510084fd14fSBrian Behlendorf return (SET_ERROR(EINTR));
511084fd14fSBrian Behlendorf }
512084fd14fSBrian Behlendorf mutex_exit(&vd->vdev_trim_lock);
513084fd14fSBrian Behlendorf
514084fd14fSBrian Behlendorf if (ta->trim_type == TRIM_TYPE_MANUAL)
515084fd14fSBrian Behlendorf vd->vdev_trim_offset[txg & TXG_MASK] = start + size;
516084fd14fSBrian Behlendorf
517084fd14fSBrian Behlendorf zio_nowait(zio_trim(spa->spa_txg_zio[txg & TXG_MASK], vd,
518084fd14fSBrian Behlendorf start, size, ta->trim_type == TRIM_TYPE_MANUAL ?
519084fd14fSBrian Behlendorf vdev_trim_cb : vdev_autotrim_cb, NULL,
520084fd14fSBrian Behlendorf ZIO_PRIORITY_TRIM, ZIO_FLAG_CANFAIL, ta->trim_flags));
521084fd14fSBrian Behlendorf /* vdev_trim_cb and vdev_autotrim_cb release SCL_STATE_ALL */
522084fd14fSBrian Behlendorf
523084fd14fSBrian Behlendorf dmu_tx_commit(tx);
524084fd14fSBrian Behlendorf
525084fd14fSBrian Behlendorf return (0);
526084fd14fSBrian Behlendorf }
527084fd14fSBrian Behlendorf
528084fd14fSBrian Behlendorf /*
529084fd14fSBrian Behlendorf * Issues TRIM I/Os for all ranges in the provided ta->trim_tree range tree.
530084fd14fSBrian Behlendorf * Additional parameters describing how the TRIM should be performed must
531084fd14fSBrian Behlendorf * be set in the trim_args structure. See the trim_args definition for
532084fd14fSBrian Behlendorf * additional information.
533084fd14fSBrian Behlendorf */
534084fd14fSBrian Behlendorf static int
vdev_trim_ranges(trim_args_t * ta)535084fd14fSBrian Behlendorf vdev_trim_ranges(trim_args_t *ta)
536084fd14fSBrian Behlendorf {
537084fd14fSBrian Behlendorf vdev_t *vd = ta->trim_vdev;
5384d7988d6SPaul Dagnelie zfs_btree_t *t = &ta->trim_tree->rt_root;
5394d7988d6SPaul Dagnelie zfs_btree_index_t idx;
540084fd14fSBrian Behlendorf uint64_t extent_bytes_max = ta->trim_extent_bytes_max;
541084fd14fSBrian Behlendorf uint64_t extent_bytes_min = ta->trim_extent_bytes_min;
542084fd14fSBrian Behlendorf spa_t *spa = vd->vdev_spa;
543*a385ae06SJason King int error = 0;
544084fd14fSBrian Behlendorf
545084fd14fSBrian Behlendorf ta->trim_start_time = gethrtime();
546084fd14fSBrian Behlendorf ta->trim_bytes_done = 0;
547084fd14fSBrian Behlendorf
5484d7988d6SPaul Dagnelie for (range_seg_t *rs = zfs_btree_first(t, &idx); rs != NULL;
5494d7988d6SPaul Dagnelie rs = zfs_btree_next(t, &idx, &idx)) {
5504d7988d6SPaul Dagnelie uint64_t size = rs_get_end(rs, ta->trim_tree) - rs_get_start(rs,
5514d7988d6SPaul Dagnelie ta->trim_tree);
552084fd14fSBrian Behlendorf
553084fd14fSBrian Behlendorf if (extent_bytes_min && size < extent_bytes_min) {
554084fd14fSBrian Behlendorf /*
555084fd14fSBrian Behlendorf * spa_iostats_trim_add(spa, ta->trim_type,
556084fd14fSBrian Behlendorf * 0, 0, 1, size, 0, 0);
557084fd14fSBrian Behlendorf */
558084fd14fSBrian Behlendorf continue;
559084fd14fSBrian Behlendorf }
560084fd14fSBrian Behlendorf
561084fd14fSBrian Behlendorf /* Split range into legally-sized physical chunks */
562084fd14fSBrian Behlendorf uint64_t writes_required = ((size - 1) / extent_bytes_max) + 1;
563084fd14fSBrian Behlendorf
564084fd14fSBrian Behlendorf for (uint64_t w = 0; w < writes_required; w++) {
565084fd14fSBrian Behlendorf error = vdev_trim_range(ta, VDEV_LABEL_START_SIZE +
5664d7988d6SPaul Dagnelie rs_get_start(rs, ta->trim_tree) +
5674d7988d6SPaul Dagnelie (w *extent_bytes_max), MIN(size -
5684d7988d6SPaul Dagnelie (w * extent_bytes_max), extent_bytes_max));
569084fd14fSBrian Behlendorf if (error != 0) {
570*a385ae06SJason King goto done;
571084fd14fSBrian Behlendorf }
572084fd14fSBrian Behlendorf }
573084fd14fSBrian Behlendorf }
574084fd14fSBrian Behlendorf
575*a385ae06SJason King done:
576*a385ae06SJason King /*
577*a385ae06SJason King * Make sure all TRIMs for this metaslab have completed before
578*a385ae06SJason King * returning. TRIM zios have lower priority over regular or syncing
579*a385ae06SJason King * zios, so all TRIM zios for this metaslab must complete before the
580*a385ae06SJason King * metaslab is re-enabled. Otherwise it's possible write zios to
581*a385ae06SJason King * this metaslab could cut ahead of still queued TRIM zios for this
582*a385ae06SJason King * metaslab causing corruption if the ranges overlap.
583*a385ae06SJason King */
584*a385ae06SJason King mutex_enter(&vd->vdev_trim_io_lock);
585*a385ae06SJason King while (vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] > 0) {
586*a385ae06SJason King cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
587*a385ae06SJason King }
588*a385ae06SJason King mutex_exit(&vd->vdev_trim_io_lock);
589*a385ae06SJason King
590*a385ae06SJason King return (error);
591084fd14fSBrian Behlendorf }
592084fd14fSBrian Behlendorf
593084fd14fSBrian Behlendorf /*
594084fd14fSBrian Behlendorf * Calculates the completion percentage of a manual TRIM.
595084fd14fSBrian Behlendorf */
596084fd14fSBrian Behlendorf static void
vdev_trim_calculate_progress(vdev_t * vd)597084fd14fSBrian Behlendorf vdev_trim_calculate_progress(vdev_t *vd)
598084fd14fSBrian Behlendorf {
599084fd14fSBrian Behlendorf ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
600084fd14fSBrian Behlendorf spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
601084fd14fSBrian Behlendorf ASSERT(vd->vdev_leaf_zap != 0);
602084fd14fSBrian Behlendorf
603084fd14fSBrian Behlendorf vd->vdev_trim_bytes_est = 0;
604084fd14fSBrian Behlendorf vd->vdev_trim_bytes_done = 0;
605084fd14fSBrian Behlendorf
606084fd14fSBrian Behlendorf for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) {
607084fd14fSBrian Behlendorf metaslab_t *msp = vd->vdev_top->vdev_ms[i];
608084fd14fSBrian Behlendorf mutex_enter(&msp->ms_lock);
609084fd14fSBrian Behlendorf
610084fd14fSBrian Behlendorf uint64_t ms_free = msp->ms_size -
611084fd14fSBrian Behlendorf metaslab_allocated_space(msp);
612084fd14fSBrian Behlendorf
613084fd14fSBrian Behlendorf if (vd->vdev_top->vdev_ops == &vdev_raidz_ops)
614084fd14fSBrian Behlendorf ms_free /= vd->vdev_top->vdev_children;
615084fd14fSBrian Behlendorf
616084fd14fSBrian Behlendorf /*
617084fd14fSBrian Behlendorf * Convert the metaslab range to a physical range
618084fd14fSBrian Behlendorf * on our vdev. We use this to determine if we are
619084fd14fSBrian Behlendorf * in the middle of this metaslab range.
620084fd14fSBrian Behlendorf */
6214d7988d6SPaul Dagnelie range_seg64_t logical_rs, physical_rs;
622084fd14fSBrian Behlendorf logical_rs.rs_start = msp->ms_start;
623084fd14fSBrian Behlendorf logical_rs.rs_end = msp->ms_start + msp->ms_size;
624084fd14fSBrian Behlendorf vdev_xlate(vd, &logical_rs, &physical_rs);
625084fd14fSBrian Behlendorf
626084fd14fSBrian Behlendorf if (vd->vdev_trim_last_offset <= physical_rs.rs_start) {
627084fd14fSBrian Behlendorf vd->vdev_trim_bytes_est += ms_free;
628084fd14fSBrian Behlendorf mutex_exit(&msp->ms_lock);
629084fd14fSBrian Behlendorf continue;
630084fd14fSBrian Behlendorf } else if (vd->vdev_trim_last_offset > physical_rs.rs_end) {
631084fd14fSBrian Behlendorf vd->vdev_trim_bytes_done += ms_free;
632084fd14fSBrian Behlendorf vd->vdev_trim_bytes_est += ms_free;
633084fd14fSBrian Behlendorf mutex_exit(&msp->ms_lock);
634084fd14fSBrian Behlendorf continue;
635084fd14fSBrian Behlendorf }
636084fd14fSBrian Behlendorf
637084fd14fSBrian Behlendorf /*
638084fd14fSBrian Behlendorf * If we get here, we're in the middle of trimming this
639084fd14fSBrian Behlendorf * metaslab. Load it and walk the free tree for more
640084fd14fSBrian Behlendorf * accurate progress estimation.
641084fd14fSBrian Behlendorf */
642084fd14fSBrian Behlendorf VERIFY0(metaslab_load(msp));
643084fd14fSBrian Behlendorf
6444d7988d6SPaul Dagnelie range_tree_t *rt = msp->ms_allocatable;
6454d7988d6SPaul Dagnelie zfs_btree_t *bt = &rt->rt_root;
6464d7988d6SPaul Dagnelie zfs_btree_index_t idx;
6474d7988d6SPaul Dagnelie for (range_seg_t *rs = zfs_btree_first(bt, &idx);
6484d7988d6SPaul Dagnelie rs != NULL; rs = zfs_btree_next(bt, &idx, &idx)) {
6494d7988d6SPaul Dagnelie logical_rs.rs_start = rs_get_start(rs, rt);
6504d7988d6SPaul Dagnelie logical_rs.rs_end = rs_get_end(rs, rt);
651084fd14fSBrian Behlendorf vdev_xlate(vd, &logical_rs, &physical_rs);
652084fd14fSBrian Behlendorf
653084fd14fSBrian Behlendorf uint64_t size = physical_rs.rs_end -
654084fd14fSBrian Behlendorf physical_rs.rs_start;
655084fd14fSBrian Behlendorf vd->vdev_trim_bytes_est += size;
656084fd14fSBrian Behlendorf if (vd->vdev_trim_last_offset >= physical_rs.rs_end) {
657084fd14fSBrian Behlendorf vd->vdev_trim_bytes_done += size;
658084fd14fSBrian Behlendorf } else if (vd->vdev_trim_last_offset >
659084fd14fSBrian Behlendorf physical_rs.rs_start &&
660084fd14fSBrian Behlendorf vd->vdev_trim_last_offset <=
661084fd14fSBrian Behlendorf physical_rs.rs_end) {
662084fd14fSBrian Behlendorf vd->vdev_trim_bytes_done +=
663084fd14fSBrian Behlendorf vd->vdev_trim_last_offset -
664084fd14fSBrian Behlendorf physical_rs.rs_start;
665084fd14fSBrian Behlendorf }
666084fd14fSBrian Behlendorf }
667084fd14fSBrian Behlendorf mutex_exit(&msp->ms_lock);
668084fd14fSBrian Behlendorf }
669084fd14fSBrian Behlendorf }
670084fd14fSBrian Behlendorf
671084fd14fSBrian Behlendorf /*
672084fd14fSBrian Behlendorf * Load from disk the vdev's manual TRIM information. This includes the
673084fd14fSBrian Behlendorf * state, progress, and options provided when initiating the manual TRIM.
674084fd14fSBrian Behlendorf */
675084fd14fSBrian Behlendorf static int
vdev_trim_load(vdev_t * vd)676084fd14fSBrian Behlendorf vdev_trim_load(vdev_t *vd)
677084fd14fSBrian Behlendorf {
678084fd14fSBrian Behlendorf int err = 0;
679084fd14fSBrian Behlendorf ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
680084fd14fSBrian Behlendorf spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
681084fd14fSBrian Behlendorf ASSERT(vd->vdev_leaf_zap != 0);
682084fd14fSBrian Behlendorf
683084fd14fSBrian Behlendorf if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE ||
684084fd14fSBrian Behlendorf vd->vdev_trim_state == VDEV_TRIM_SUSPENDED) {
685084fd14fSBrian Behlendorf err = zap_lookup(vd->vdev_spa->spa_meta_objset,
686084fd14fSBrian Behlendorf vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_LAST_OFFSET,
687084fd14fSBrian Behlendorf sizeof (vd->vdev_trim_last_offset), 1,
688084fd14fSBrian Behlendorf &vd->vdev_trim_last_offset);
689084fd14fSBrian Behlendorf if (err == ENOENT) {
690084fd14fSBrian Behlendorf vd->vdev_trim_last_offset = 0;
691084fd14fSBrian Behlendorf err = 0;
692084fd14fSBrian Behlendorf }
693084fd14fSBrian Behlendorf
694084fd14fSBrian Behlendorf if (err == 0) {
695084fd14fSBrian Behlendorf err = zap_lookup(vd->vdev_spa->spa_meta_objset,
696084fd14fSBrian Behlendorf vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_RATE,
697084fd14fSBrian Behlendorf sizeof (vd->vdev_trim_rate), 1,
698084fd14fSBrian Behlendorf &vd->vdev_trim_rate);
699084fd14fSBrian Behlendorf if (err == ENOENT) {
700084fd14fSBrian Behlendorf vd->vdev_trim_rate = 0;
701084fd14fSBrian Behlendorf err = 0;
702084fd14fSBrian Behlendorf }
703084fd14fSBrian Behlendorf }
704084fd14fSBrian Behlendorf
705084fd14fSBrian Behlendorf if (err == 0) {
706084fd14fSBrian Behlendorf err = zap_lookup(vd->vdev_spa->spa_meta_objset,
707084fd14fSBrian Behlendorf vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,
708084fd14fSBrian Behlendorf sizeof (vd->vdev_trim_partial), 1,
709084fd14fSBrian Behlendorf &vd->vdev_trim_partial);
710084fd14fSBrian Behlendorf if (err == ENOENT) {
711084fd14fSBrian Behlendorf vd->vdev_trim_partial = 0;
712084fd14fSBrian Behlendorf err = 0;
713084fd14fSBrian Behlendorf }
714084fd14fSBrian Behlendorf }
715084fd14fSBrian Behlendorf
716084fd14fSBrian Behlendorf if (err == 0) {
717084fd14fSBrian Behlendorf err = zap_lookup(vd->vdev_spa->spa_meta_objset,
718084fd14fSBrian Behlendorf vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,
719084fd14fSBrian Behlendorf sizeof (vd->vdev_trim_secure), 1,
720084fd14fSBrian Behlendorf &vd->vdev_trim_secure);
721084fd14fSBrian Behlendorf if (err == ENOENT) {
722084fd14fSBrian Behlendorf vd->vdev_trim_secure = 0;
723084fd14fSBrian Behlendorf err = 0;
724084fd14fSBrian Behlendorf }
725084fd14fSBrian Behlendorf }
726084fd14fSBrian Behlendorf }
727084fd14fSBrian Behlendorf
728084fd14fSBrian Behlendorf vdev_trim_calculate_progress(vd);
729084fd14fSBrian Behlendorf
730084fd14fSBrian Behlendorf return (err);
731084fd14fSBrian Behlendorf }
732084fd14fSBrian Behlendorf
733084fd14fSBrian Behlendorf /*
734084fd14fSBrian Behlendorf * Convert the logical range into a physical range and add it to the
735084fd14fSBrian Behlendorf * range tree passed in the trim_args_t.
736084fd14fSBrian Behlendorf */
737084fd14fSBrian Behlendorf static void
vdev_trim_range_add(void * arg,uint64_t start,uint64_t size)738084fd14fSBrian Behlendorf vdev_trim_range_add(void *arg, uint64_t start, uint64_t size)
739084fd14fSBrian Behlendorf {
740084fd14fSBrian Behlendorf trim_args_t *ta = arg;
741084fd14fSBrian Behlendorf vdev_t *vd = ta->trim_vdev;
7424d7988d6SPaul Dagnelie range_seg64_t logical_rs, physical_rs;
743084fd14fSBrian Behlendorf logical_rs.rs_start = start;
744084fd14fSBrian Behlendorf logical_rs.rs_end = start + size;
745084fd14fSBrian Behlendorf
746084fd14fSBrian Behlendorf /*
747084fd14fSBrian Behlendorf * Every range to be trimmed must be part of ms_allocatable.
748084fd14fSBrian Behlendorf * When ZFS_DEBUG_TRIM is set load the metaslab to verify this
749084fd14fSBrian Behlendorf * is always the case.
750084fd14fSBrian Behlendorf */
751084fd14fSBrian Behlendorf if (zfs_flags & ZFS_DEBUG_TRIM) {
752084fd14fSBrian Behlendorf metaslab_t *msp = ta->trim_msp;
753084fd14fSBrian Behlendorf VERIFY0(metaslab_load(msp));
754084fd14fSBrian Behlendorf VERIFY3B(msp->ms_loaded, ==, B_TRUE);
7554d7988d6SPaul Dagnelie VERIFY(range_tree_contains(msp->ms_allocatable, start, size));
756084fd14fSBrian Behlendorf }
757084fd14fSBrian Behlendorf
758084fd14fSBrian Behlendorf ASSERT(vd->vdev_ops->vdev_op_leaf);
759084fd14fSBrian Behlendorf vdev_xlate(vd, &logical_rs, &physical_rs);
760084fd14fSBrian Behlendorf
761084fd14fSBrian Behlendorf IMPLY(vd->vdev_top == vd,
762084fd14fSBrian Behlendorf logical_rs.rs_start == physical_rs.rs_start);
763084fd14fSBrian Behlendorf IMPLY(vd->vdev_top == vd,
764084fd14fSBrian Behlendorf logical_rs.rs_end == physical_rs.rs_end);
765084fd14fSBrian Behlendorf
766084fd14fSBrian Behlendorf /*
767084fd14fSBrian Behlendorf * Only a manual trim will be traversing the vdev sequentially.
768084fd14fSBrian Behlendorf * For an auto trim all valid ranges should be added.
769084fd14fSBrian Behlendorf */
770084fd14fSBrian Behlendorf if (ta->trim_type == TRIM_TYPE_MANUAL) {
771084fd14fSBrian Behlendorf
772084fd14fSBrian Behlendorf /* Only add segments that we have not visited yet */
773084fd14fSBrian Behlendorf if (physical_rs.rs_end <= vd->vdev_trim_last_offset)
774084fd14fSBrian Behlendorf return;
775084fd14fSBrian Behlendorf
776084fd14fSBrian Behlendorf /* Pick up where we left off mid-range. */
777084fd14fSBrian Behlendorf if (vd->vdev_trim_last_offset > physical_rs.rs_start) {
778084fd14fSBrian Behlendorf ASSERT3U(physical_rs.rs_end, >,
779084fd14fSBrian Behlendorf vd->vdev_trim_last_offset);
780084fd14fSBrian Behlendorf physical_rs.rs_start = vd->vdev_trim_last_offset;
781084fd14fSBrian Behlendorf }
782084fd14fSBrian Behlendorf }
783084fd14fSBrian Behlendorf
784084fd14fSBrian Behlendorf ASSERT3U(physical_rs.rs_end, >=, physical_rs.rs_start);
785084fd14fSBrian Behlendorf
786084fd14fSBrian Behlendorf /*
787084fd14fSBrian Behlendorf * With raidz, it's possible that the logical range does not live on
788084fd14fSBrian Behlendorf * this leaf vdev. We only add the physical range to this vdev's if it
789084fd14fSBrian Behlendorf * has a length greater than 0.
790084fd14fSBrian Behlendorf */
791084fd14fSBrian Behlendorf if (physical_rs.rs_end > physical_rs.rs_start) {
792084fd14fSBrian Behlendorf range_tree_add(ta->trim_tree, physical_rs.rs_start,
793084fd14fSBrian Behlendorf physical_rs.rs_end - physical_rs.rs_start);
794084fd14fSBrian Behlendorf } else {
795084fd14fSBrian Behlendorf ASSERT3U(physical_rs.rs_end, ==, physical_rs.rs_start);
796084fd14fSBrian Behlendorf }
797084fd14fSBrian Behlendorf }
798084fd14fSBrian Behlendorf
799084fd14fSBrian Behlendorf /*
800084fd14fSBrian Behlendorf * Each manual TRIM thread is responsible for trimming the unallocated
801084fd14fSBrian Behlendorf * space for each leaf vdev. This is accomplished by sequentially iterating
802084fd14fSBrian Behlendorf * over its top-level metaslabs and issuing TRIM I/O for the space described
803084fd14fSBrian Behlendorf * by its ms_allocatable. While a metaslab is undergoing trimming it is
804084fd14fSBrian Behlendorf * not eligible for new allocations.
805084fd14fSBrian Behlendorf */
806084fd14fSBrian Behlendorf static void
vdev_trim_thread(void * arg)807084fd14fSBrian Behlendorf vdev_trim_thread(void *arg)
808084fd14fSBrian Behlendorf {
809084fd14fSBrian Behlendorf vdev_t *vd = arg;
810084fd14fSBrian Behlendorf spa_t *spa = vd->vdev_spa;
811084fd14fSBrian Behlendorf trim_args_t ta;
812084fd14fSBrian Behlendorf int error = 0;
813084fd14fSBrian Behlendorf
814084fd14fSBrian Behlendorf /*
815084fd14fSBrian Behlendorf * The VDEV_LEAF_ZAP_TRIM_* entries may have been updated by
816084fd14fSBrian Behlendorf * vdev_trim(). Wait for the updated values to be reflected
817084fd14fSBrian Behlendorf * in the zap in order to start with the requested settings.
818084fd14fSBrian Behlendorf */
819084fd14fSBrian Behlendorf txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
820084fd14fSBrian Behlendorf
821084fd14fSBrian Behlendorf ASSERT(vdev_is_concrete(vd));
822084fd14fSBrian Behlendorf spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
823084fd14fSBrian Behlendorf
824084fd14fSBrian Behlendorf vd->vdev_trim_last_offset = 0;
825084fd14fSBrian Behlendorf vd->vdev_trim_rate = 0;
826084fd14fSBrian Behlendorf vd->vdev_trim_partial = 0;
827084fd14fSBrian Behlendorf vd->vdev_trim_secure = 0;
828084fd14fSBrian Behlendorf
829084fd14fSBrian Behlendorf VERIFY0(vdev_trim_load(vd));
830084fd14fSBrian Behlendorf
831084fd14fSBrian Behlendorf ta.trim_vdev = vd;
832084fd14fSBrian Behlendorf ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max;
833084fd14fSBrian Behlendorf ta.trim_extent_bytes_min = zfs_trim_extent_bytes_min;
8344d7988d6SPaul Dagnelie ta.trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);
835084fd14fSBrian Behlendorf ta.trim_type = TRIM_TYPE_MANUAL;
836084fd14fSBrian Behlendorf ta.trim_flags = 0;
837084fd14fSBrian Behlendorf
838084fd14fSBrian Behlendorf /*
839084fd14fSBrian Behlendorf * When a secure TRIM has been requested infer that the intent
840084fd14fSBrian Behlendorf * is that everything must be trimmed. Override the default
841084fd14fSBrian Behlendorf * minimum TRIM size to prevent ranges from being skipped.
842084fd14fSBrian Behlendorf */
843084fd14fSBrian Behlendorf if (vd->vdev_trim_secure) {
844084fd14fSBrian Behlendorf ta.trim_flags |= ZIO_TRIM_SECURE;
845084fd14fSBrian Behlendorf ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE;
846084fd14fSBrian Behlendorf }
847084fd14fSBrian Behlendorf
848084fd14fSBrian Behlendorf uint64_t ms_count = 0;
849084fd14fSBrian Behlendorf for (uint64_t i = 0; !vd->vdev_detached &&
850084fd14fSBrian Behlendorf i < vd->vdev_top->vdev_ms_count; i++) {
851084fd14fSBrian Behlendorf metaslab_t *msp = vd->vdev_top->vdev_ms[i];
852084fd14fSBrian Behlendorf
853084fd14fSBrian Behlendorf /*
854084fd14fSBrian Behlendorf * If we've expanded the top-level vdev or it's our
855084fd14fSBrian Behlendorf * first pass, calculate our progress.
856084fd14fSBrian Behlendorf */
857084fd14fSBrian Behlendorf if (vd->vdev_top->vdev_ms_count != ms_count) {
858084fd14fSBrian Behlendorf vdev_trim_calculate_progress(vd);
859084fd14fSBrian Behlendorf ms_count = vd->vdev_top->vdev_ms_count;
860084fd14fSBrian Behlendorf }
861084fd14fSBrian Behlendorf
862084fd14fSBrian Behlendorf spa_config_exit(spa, SCL_CONFIG, FTAG);
863084fd14fSBrian Behlendorf metaslab_disable(msp);
864084fd14fSBrian Behlendorf mutex_enter(&msp->ms_lock);
865084fd14fSBrian Behlendorf VERIFY0(metaslab_load(msp));
866084fd14fSBrian Behlendorf
867084fd14fSBrian Behlendorf /*
868084fd14fSBrian Behlendorf * If a partial TRIM was requested skip metaslabs which have
869084fd14fSBrian Behlendorf * never been initialized and thus have never been written.
870084fd14fSBrian Behlendorf */
871084fd14fSBrian Behlendorf if (msp->ms_sm == NULL && vd->vdev_trim_partial) {
872084fd14fSBrian Behlendorf mutex_exit(&msp->ms_lock);
873af1d63abSPaul Dagnelie metaslab_enable(msp, B_FALSE, B_FALSE);
874084fd14fSBrian Behlendorf spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
875084fd14fSBrian Behlendorf vdev_trim_calculate_progress(vd);
876084fd14fSBrian Behlendorf continue;
877084fd14fSBrian Behlendorf }
878084fd14fSBrian Behlendorf
879084fd14fSBrian Behlendorf ta.trim_msp = msp;
880084fd14fSBrian Behlendorf range_tree_walk(msp->ms_allocatable, vdev_trim_range_add, &ta);
881084fd14fSBrian Behlendorf range_tree_vacate(msp->ms_trim, NULL, NULL);
882084fd14fSBrian Behlendorf mutex_exit(&msp->ms_lock);
883084fd14fSBrian Behlendorf
884084fd14fSBrian Behlendorf error = vdev_trim_ranges(&ta);
885af1d63abSPaul Dagnelie metaslab_enable(msp, B_TRUE, B_FALSE);
886084fd14fSBrian Behlendorf spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
887084fd14fSBrian Behlendorf
888084fd14fSBrian Behlendorf range_tree_vacate(ta.trim_tree, NULL, NULL);
889084fd14fSBrian Behlendorf if (error != 0)
890084fd14fSBrian Behlendorf break;
891084fd14fSBrian Behlendorf }
892084fd14fSBrian Behlendorf
893084fd14fSBrian Behlendorf spa_config_exit(spa, SCL_CONFIG, FTAG);
894084fd14fSBrian Behlendorf
895084fd14fSBrian Behlendorf range_tree_destroy(ta.trim_tree);
896084fd14fSBrian Behlendorf
897084fd14fSBrian Behlendorf mutex_enter(&vd->vdev_trim_lock);
898084fd14fSBrian Behlendorf if (!vd->vdev_trim_exit_wanted && vdev_writeable(vd)) {
899084fd14fSBrian Behlendorf vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE,
900084fd14fSBrian Behlendorf vd->vdev_trim_rate, vd->vdev_trim_partial,
901084fd14fSBrian Behlendorf vd->vdev_trim_secure);
902084fd14fSBrian Behlendorf }
903084fd14fSBrian Behlendorf ASSERT(vd->vdev_trim_thread != NULL || vd->vdev_trim_inflight[0] == 0);
904084fd14fSBrian Behlendorf
905084fd14fSBrian Behlendorf /*
906084fd14fSBrian Behlendorf * Drop the vdev_trim_lock while we sync out the txg since it's
907084fd14fSBrian Behlendorf * possible that a device might be trying to come online and must
908084fd14fSBrian Behlendorf * check to see if it needs to restart a trim. That thread will be
909084fd14fSBrian Behlendorf * holding the spa_config_lock which would prevent the txg_wait_synced
910084fd14fSBrian Behlendorf * from completing.
911084fd14fSBrian Behlendorf */
912084fd14fSBrian Behlendorf mutex_exit(&vd->vdev_trim_lock);
913084fd14fSBrian Behlendorf txg_wait_synced(spa_get_dsl(spa), 0);
914084fd14fSBrian Behlendorf mutex_enter(&vd->vdev_trim_lock);
915084fd14fSBrian Behlendorf
916084fd14fSBrian Behlendorf vd->vdev_trim_thread = NULL;
917084fd14fSBrian Behlendorf cv_broadcast(&vd->vdev_trim_cv);
918084fd14fSBrian Behlendorf mutex_exit(&vd->vdev_trim_lock);
919084fd14fSBrian Behlendorf }
920084fd14fSBrian Behlendorf
921084fd14fSBrian Behlendorf /*
922084fd14fSBrian Behlendorf * Initiates a manual TRIM for the vdev_t. Callers must hold vdev_trim_lock,
923084fd14fSBrian Behlendorf * the vdev_t must be a leaf and cannot already be manually trimming.
924084fd14fSBrian Behlendorf */
925084fd14fSBrian Behlendorf void
vdev_trim(vdev_t * vd,uint64_t rate,boolean_t partial,boolean_t secure)926084fd14fSBrian Behlendorf vdev_trim(vdev_t *vd, uint64_t rate, boolean_t partial, boolean_t secure)
927084fd14fSBrian Behlendorf {
928084fd14fSBrian Behlendorf ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
929084fd14fSBrian Behlendorf ASSERT(vd->vdev_ops->vdev_op_leaf);
930084fd14fSBrian Behlendorf ASSERT(vdev_is_concrete(vd));
931084fd14fSBrian Behlendorf ASSERT3P(vd->vdev_trim_thread, ==, NULL);
932084fd14fSBrian Behlendorf ASSERT(!vd->vdev_detached);
933084fd14fSBrian Behlendorf ASSERT(!vd->vdev_trim_exit_wanted);
934084fd14fSBrian Behlendorf ASSERT(!vd->vdev_top->vdev_removing);
935084fd14fSBrian Behlendorf
936084fd14fSBrian Behlendorf vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, rate, partial, secure);
937084fd14fSBrian Behlendorf vd->vdev_trim_thread = thread_create(NULL, 0,
938084fd14fSBrian Behlendorf vdev_trim_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
939084fd14fSBrian Behlendorf }
940084fd14fSBrian Behlendorf
941084fd14fSBrian Behlendorf /*
942084fd14fSBrian Behlendorf * Wait for the trimming thread to be terminated (canceled or stopped).
943084fd14fSBrian Behlendorf */
944084fd14fSBrian Behlendorf static void
vdev_trim_stop_wait_impl(vdev_t * vd)945084fd14fSBrian Behlendorf vdev_trim_stop_wait_impl(vdev_t *vd)
946084fd14fSBrian Behlendorf {
947084fd14fSBrian Behlendorf ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
948084fd14fSBrian Behlendorf
949084fd14fSBrian Behlendorf while (vd->vdev_trim_thread != NULL)
950084fd14fSBrian Behlendorf cv_wait(&vd->vdev_trim_cv, &vd->vdev_trim_lock);
951084fd14fSBrian Behlendorf
952084fd14fSBrian Behlendorf ASSERT3P(vd->vdev_trim_thread, ==, NULL);
953084fd14fSBrian Behlendorf vd->vdev_trim_exit_wanted = B_FALSE;
954084fd14fSBrian Behlendorf }
955084fd14fSBrian Behlendorf
956084fd14fSBrian Behlendorf /*
957084fd14fSBrian Behlendorf * Wait for vdev trim threads which were listed to cleanly exit.
958084fd14fSBrian Behlendorf */
959084fd14fSBrian Behlendorf void
vdev_trim_stop_wait(spa_t * spa,list_t * vd_list)960084fd14fSBrian Behlendorf vdev_trim_stop_wait(spa_t *spa, list_t *vd_list)
961084fd14fSBrian Behlendorf {
962084fd14fSBrian Behlendorf vdev_t *vd;
963084fd14fSBrian Behlendorf
964084fd14fSBrian Behlendorf ASSERT(MUTEX_HELD(&spa_namespace_lock));
965084fd14fSBrian Behlendorf
966084fd14fSBrian Behlendorf while ((vd = list_remove_head(vd_list)) != NULL) {
967084fd14fSBrian Behlendorf mutex_enter(&vd->vdev_trim_lock);
968084fd14fSBrian Behlendorf vdev_trim_stop_wait_impl(vd);
969084fd14fSBrian Behlendorf mutex_exit(&vd->vdev_trim_lock);
970084fd14fSBrian Behlendorf }
971084fd14fSBrian Behlendorf }
972084fd14fSBrian Behlendorf
973084fd14fSBrian Behlendorf /*
974084fd14fSBrian Behlendorf * Stop trimming a device, with the resultant trimming state being tgt_state.
975084fd14fSBrian Behlendorf * For blocking behavior pass NULL for vd_list. Otherwise, when a list_t is
976084fd14fSBrian Behlendorf * provided the stopping vdev is inserted in to the list. Callers are then
977084fd14fSBrian Behlendorf * required to call vdev_trim_stop_wait() to block for all the trim threads
978084fd14fSBrian Behlendorf * to exit. The caller must hold vdev_trim_lock and must not be writing to
979084fd14fSBrian Behlendorf * the spa config, as the trimming thread may try to enter the config as a
980084fd14fSBrian Behlendorf * reader before exiting.
981084fd14fSBrian Behlendorf */
982084fd14fSBrian Behlendorf void
vdev_trim_stop(vdev_t * vd,vdev_trim_state_t tgt_state,list_t * vd_list)983084fd14fSBrian Behlendorf vdev_trim_stop(vdev_t *vd, vdev_trim_state_t tgt_state, list_t *vd_list)
984084fd14fSBrian Behlendorf {
985084fd14fSBrian Behlendorf ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER));
986084fd14fSBrian Behlendorf ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
987084fd14fSBrian Behlendorf ASSERT(vd->vdev_ops->vdev_op_leaf);
988084fd14fSBrian Behlendorf ASSERT(vdev_is_concrete(vd));
989084fd14fSBrian Behlendorf
990084fd14fSBrian Behlendorf /*
991084fd14fSBrian Behlendorf * Allow cancel requests to proceed even if the trim thread has
992084fd14fSBrian Behlendorf * stopped.
993084fd14fSBrian Behlendorf */
994084fd14fSBrian Behlendorf if (vd->vdev_trim_thread == NULL && tgt_state != VDEV_TRIM_CANCELED)
995084fd14fSBrian Behlendorf return;
996084fd14fSBrian Behlendorf
997084fd14fSBrian Behlendorf vdev_trim_change_state(vd, tgt_state, 0, 0, 0);
998084fd14fSBrian Behlendorf vd->vdev_trim_exit_wanted = B_TRUE;
999084fd14fSBrian Behlendorf
1000084fd14fSBrian Behlendorf if (vd_list == NULL) {
1001084fd14fSBrian Behlendorf vdev_trim_stop_wait_impl(vd);
1002084fd14fSBrian Behlendorf } else {
1003084fd14fSBrian Behlendorf ASSERT(MUTEX_HELD(&spa_namespace_lock));
1004084fd14fSBrian Behlendorf list_insert_tail(vd_list, vd);
1005084fd14fSBrian Behlendorf }
1006084fd14fSBrian Behlendorf }
1007084fd14fSBrian Behlendorf
1008084fd14fSBrian Behlendorf /*
1009084fd14fSBrian Behlendorf * Requests that all listed vdevs stop trimming.
1010084fd14fSBrian Behlendorf */
1011084fd14fSBrian Behlendorf static void
vdev_trim_stop_all_impl(vdev_t * vd,vdev_trim_state_t tgt_state,list_t * vd_list)1012084fd14fSBrian Behlendorf vdev_trim_stop_all_impl(vdev_t *vd, vdev_trim_state_t tgt_state,
1013084fd14fSBrian Behlendorf list_t *vd_list)
1014084fd14fSBrian Behlendorf {
1015084fd14fSBrian Behlendorf if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) {
1016084fd14fSBrian Behlendorf mutex_enter(&vd->vdev_trim_lock);
1017084fd14fSBrian Behlendorf vdev_trim_stop(vd, tgt_state, vd_list);
1018084fd14fSBrian Behlendorf mutex_exit(&vd->vdev_trim_lock);
1019084fd14fSBrian Behlendorf return;
1020084fd14fSBrian Behlendorf }
1021084fd14fSBrian Behlendorf
1022084fd14fSBrian Behlendorf for (uint64_t i = 0; i < vd->vdev_children; i++) {
1023084fd14fSBrian Behlendorf vdev_trim_stop_all_impl(vd->vdev_child[i], tgt_state,
1024084fd14fSBrian Behlendorf vd_list);
1025084fd14fSBrian Behlendorf }
1026084fd14fSBrian Behlendorf }
1027084fd14fSBrian Behlendorf
1028084fd14fSBrian Behlendorf /*
1029084fd14fSBrian Behlendorf * Convenience function to stop trimming of a vdev tree and set all trim
1030084fd14fSBrian Behlendorf * thread pointers to NULL.
1031084fd14fSBrian Behlendorf */
1032084fd14fSBrian Behlendorf void
vdev_trim_stop_all(vdev_t * vd,vdev_trim_state_t tgt_state)1033084fd14fSBrian Behlendorf vdev_trim_stop_all(vdev_t *vd, vdev_trim_state_t tgt_state)
1034084fd14fSBrian Behlendorf {
1035084fd14fSBrian Behlendorf spa_t *spa = vd->vdev_spa;
1036084fd14fSBrian Behlendorf list_t vd_list;
1037084fd14fSBrian Behlendorf
1038084fd14fSBrian Behlendorf ASSERT(MUTEX_HELD(&spa_namespace_lock));
1039084fd14fSBrian Behlendorf
1040084fd14fSBrian Behlendorf list_create(&vd_list, sizeof (vdev_t),
1041084fd14fSBrian Behlendorf offsetof(vdev_t, vdev_trim_node));
1042084fd14fSBrian Behlendorf
1043084fd14fSBrian Behlendorf vdev_trim_stop_all_impl(vd, tgt_state, &vd_list);
1044084fd14fSBrian Behlendorf vdev_trim_stop_wait(spa, &vd_list);
1045084fd14fSBrian Behlendorf
1046084fd14fSBrian Behlendorf if (vd->vdev_spa->spa_sync_on) {
1047084fd14fSBrian Behlendorf /* Make sure that our state has been synced to disk */
1048084fd14fSBrian Behlendorf txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
1049084fd14fSBrian Behlendorf }
1050084fd14fSBrian Behlendorf
1051084fd14fSBrian Behlendorf list_destroy(&vd_list);
1052084fd14fSBrian Behlendorf }
1053084fd14fSBrian Behlendorf
1054084fd14fSBrian Behlendorf /*
1055084fd14fSBrian Behlendorf * Conditionally restarts a manual TRIM given its on-disk state.
1056084fd14fSBrian Behlendorf */
1057084fd14fSBrian Behlendorf void
vdev_trim_restart(vdev_t * vd)1058084fd14fSBrian Behlendorf vdev_trim_restart(vdev_t *vd)
1059084fd14fSBrian Behlendorf {
1060084fd14fSBrian Behlendorf ASSERT(MUTEX_HELD(&spa_namespace_lock));
1061084fd14fSBrian Behlendorf ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
1062084fd14fSBrian Behlendorf
1063084fd14fSBrian Behlendorf if (vd->vdev_leaf_zap != 0) {
1064084fd14fSBrian Behlendorf mutex_enter(&vd->vdev_trim_lock);
1065084fd14fSBrian Behlendorf uint64_t trim_state = VDEV_TRIM_NONE;
1066084fd14fSBrian Behlendorf int err = zap_lookup(vd->vdev_spa->spa_meta_objset,
1067084fd14fSBrian Behlendorf vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,
1068084fd14fSBrian Behlendorf sizeof (trim_state), 1, &trim_state);
1069084fd14fSBrian Behlendorf ASSERT(err == 0 || err == ENOENT);
1070084fd14fSBrian Behlendorf vd->vdev_trim_state = trim_state;
1071084fd14fSBrian Behlendorf
1072084fd14fSBrian Behlendorf uint64_t timestamp = 0;
1073084fd14fSBrian Behlendorf err = zap_lookup(vd->vdev_spa->spa_meta_objset,
1074084fd14fSBrian Behlendorf vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_ACTION_TIME,
1075084fd14fSBrian Behlendorf sizeof (timestamp), 1, ×tamp);
1076084fd14fSBrian Behlendorf ASSERT(err == 0 || err == ENOENT);
1077084fd14fSBrian Behlendorf vd->vdev_trim_action_time = (time_t)timestamp;
1078084fd14fSBrian Behlendorf
1079084fd14fSBrian Behlendorf if (vd->vdev_trim_state == VDEV_TRIM_SUSPENDED ||
1080084fd14fSBrian Behlendorf vd->vdev_offline) {
1081084fd14fSBrian Behlendorf /* load progress for reporting, but don't resume */
1082084fd14fSBrian Behlendorf VERIFY0(vdev_trim_load(vd));
1083084fd14fSBrian Behlendorf } else if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE &&
1084084fd14fSBrian Behlendorf vdev_writeable(vd) && !vd->vdev_top->vdev_removing &&
1085084fd14fSBrian Behlendorf vd->vdev_trim_thread == NULL) {
1086084fd14fSBrian Behlendorf VERIFY0(vdev_trim_load(vd));
1087084fd14fSBrian Behlendorf vdev_trim(vd, vd->vdev_trim_rate,
1088084fd14fSBrian Behlendorf vd->vdev_trim_partial, vd->vdev_trim_secure);
1089084fd14fSBrian Behlendorf }
1090084fd14fSBrian Behlendorf
1091084fd14fSBrian Behlendorf mutex_exit(&vd->vdev_trim_lock);
1092084fd14fSBrian Behlendorf }
1093084fd14fSBrian Behlendorf
1094084fd14fSBrian Behlendorf for (uint64_t i = 0; i < vd->vdev_children; i++) {
1095084fd14fSBrian Behlendorf vdev_trim_restart(vd->vdev_child[i]);
1096084fd14fSBrian Behlendorf }
1097084fd14fSBrian Behlendorf }
1098084fd14fSBrian Behlendorf
1099084fd14fSBrian Behlendorf /*
1100084fd14fSBrian Behlendorf * Used by the automatic TRIM when ZFS_DEBUG_TRIM is set to verify that
1101084fd14fSBrian Behlendorf * every TRIM range is contained within ms_allocatable.
1102084fd14fSBrian Behlendorf */
1103084fd14fSBrian Behlendorf static void
vdev_trim_range_verify(void * arg,uint64_t start,uint64_t size)1104084fd14fSBrian Behlendorf vdev_trim_range_verify(void *arg, uint64_t start, uint64_t size)
1105084fd14fSBrian Behlendorf {
1106084fd14fSBrian Behlendorf trim_args_t *ta = arg;
1107084fd14fSBrian Behlendorf metaslab_t *msp = ta->trim_msp;
1108084fd14fSBrian Behlendorf
1109084fd14fSBrian Behlendorf VERIFY3B(msp->ms_loaded, ==, B_TRUE);
1110084fd14fSBrian Behlendorf VERIFY3U(msp->ms_disabled, >, 0);
11114d7988d6SPaul Dagnelie VERIFY(range_tree_contains(msp->ms_allocatable, start, size));
1112084fd14fSBrian Behlendorf }
1113084fd14fSBrian Behlendorf
1114084fd14fSBrian Behlendorf /*
1115084fd14fSBrian Behlendorf * Each automatic TRIM thread is responsible for managing the trimming of a
1116084fd14fSBrian Behlendorf * top-level vdev in the pool. No automatic TRIM state is maintained on-disk.
1117084fd14fSBrian Behlendorf *
1118084fd14fSBrian Behlendorf * N.B. This behavior is different from a manual TRIM where a thread
1119084fd14fSBrian Behlendorf * is created for each leaf vdev, instead of each top-level vdev.
1120084fd14fSBrian Behlendorf */
1121084fd14fSBrian Behlendorf static void
vdev_autotrim_thread(void * arg)1122084fd14fSBrian Behlendorf vdev_autotrim_thread(void *arg)
1123084fd14fSBrian Behlendorf {
1124084fd14fSBrian Behlendorf vdev_t *vd = arg;
1125084fd14fSBrian Behlendorf spa_t *spa = vd->vdev_spa;
1126084fd14fSBrian Behlendorf int shift = 0;
1127084fd14fSBrian Behlendorf
1128084fd14fSBrian Behlendorf mutex_enter(&vd->vdev_autotrim_lock);
1129084fd14fSBrian Behlendorf ASSERT3P(vd->vdev_top, ==, vd);
1130084fd14fSBrian Behlendorf ASSERT3P(vd->vdev_autotrim_thread, !=, NULL);
1131084fd14fSBrian Behlendorf mutex_exit(&vd->vdev_autotrim_lock);
1132084fd14fSBrian Behlendorf spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1133084fd14fSBrian Behlendorf
1134084fd14fSBrian Behlendorf uint64_t extent_bytes_max = zfs_trim_extent_bytes_max;
1135084fd14fSBrian Behlendorf uint64_t extent_bytes_min = zfs_trim_extent_bytes_min;
1136084fd14fSBrian Behlendorf
1137084fd14fSBrian Behlendorf while (!vdev_autotrim_should_stop(vd)) {
1138084fd14fSBrian Behlendorf int txgs_per_trim = MAX(zfs_trim_txg_batch, 1);
1139084fd14fSBrian Behlendorf boolean_t issued_trim = B_FALSE;
1140084fd14fSBrian Behlendorf
1141084fd14fSBrian Behlendorf /*
1142084fd14fSBrian Behlendorf * All of the metaslabs are divided in to groups of size
1143084fd14fSBrian Behlendorf * num_metaslabs / zfs_trim_txg_batch. Each of these groups
1144084fd14fSBrian Behlendorf * is composed of metaslabs which are spread evenly over the
1145084fd14fSBrian Behlendorf * device.
1146084fd14fSBrian Behlendorf *
1147084fd14fSBrian Behlendorf * For example, when zfs_trim_txg_batch = 32 (default) then
1148084fd14fSBrian Behlendorf * group 0 will contain metaslabs 0, 32, 64, ...;
1149084fd14fSBrian Behlendorf * group 1 will contain metaslabs 1, 33, 65, ...;
1150084fd14fSBrian Behlendorf * group 2 will contain metaslabs 2, 34, 66, ...; and so on.
1151084fd14fSBrian Behlendorf *
1152084fd14fSBrian Behlendorf * On each pass through the while() loop one of these groups
1153084fd14fSBrian Behlendorf * is selected. This is accomplished by using a shift value
1154084fd14fSBrian Behlendorf * to select the starting metaslab, then striding over the
1155084fd14fSBrian Behlendorf * metaslabs using the zfs_trim_txg_batch size. This is
1156084fd14fSBrian Behlendorf * done to accomplish two things.
1157084fd14fSBrian Behlendorf *
1158084fd14fSBrian Behlendorf * 1) By dividing the metaslabs into groups, and making sure
1159084fd14fSBrian Behlendorf * that each group takes a minimum of one txg to process.
1160084fd14fSBrian Behlendorf * Then zfs_trim_txg_batch controls the minimum number of
1161084fd14fSBrian Behlendorf * txgs which must occur before a metaslab is revisited.
1162084fd14fSBrian Behlendorf *
1163084fd14fSBrian Behlendorf * 2) Selecting non-consecutive metaslabs distributes the
1164084fd14fSBrian Behlendorf * TRIM commands for a group evenly over the entire device.
1165084fd14fSBrian Behlendorf * This can be advantageous for certain types of devices.
1166084fd14fSBrian Behlendorf */
1167084fd14fSBrian Behlendorf for (uint64_t i = shift % txgs_per_trim; i < vd->vdev_ms_count;
1168084fd14fSBrian Behlendorf i += txgs_per_trim) {
1169084fd14fSBrian Behlendorf metaslab_t *msp = vd->vdev_ms[i];
1170084fd14fSBrian Behlendorf range_tree_t *trim_tree;
1171084fd14fSBrian Behlendorf
1172084fd14fSBrian Behlendorf spa_config_exit(spa, SCL_CONFIG, FTAG);
1173084fd14fSBrian Behlendorf metaslab_disable(msp);
1174084fd14fSBrian Behlendorf spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1175084fd14fSBrian Behlendorf
1176084fd14fSBrian Behlendorf mutex_enter(&msp->ms_lock);
1177084fd14fSBrian Behlendorf
1178084fd14fSBrian Behlendorf /*
1179084fd14fSBrian Behlendorf * Skip the metaslab when it has never been allocated
1180084fd14fSBrian Behlendorf * or when there are no recent frees to trim.
1181084fd14fSBrian Behlendorf */
1182084fd14fSBrian Behlendorf if (msp->ms_sm == NULL ||
1183084fd14fSBrian Behlendorf range_tree_is_empty(msp->ms_trim)) {
1184084fd14fSBrian Behlendorf mutex_exit(&msp->ms_lock);
1185af1d63abSPaul Dagnelie metaslab_enable(msp, B_FALSE, B_FALSE);
1186084fd14fSBrian Behlendorf continue;
1187084fd14fSBrian Behlendorf }
1188084fd14fSBrian Behlendorf
1189084fd14fSBrian Behlendorf /*
1190084fd14fSBrian Behlendorf * Skip the metaslab when it has already been disabled.
1191084fd14fSBrian Behlendorf * This may happen when a manual TRIM or initialize
1192084fd14fSBrian Behlendorf * operation is running concurrently. In the case
1193084fd14fSBrian Behlendorf * of a manual TRIM, the ms_trim tree will have been
1194084fd14fSBrian Behlendorf * vacated. Only ranges added after the manual TRIM
1195084fd14fSBrian Behlendorf * disabled the metaslab will be included in the tree.
1196084fd14fSBrian Behlendorf * These will be processed when the automatic TRIM
1197084fd14fSBrian Behlendorf * next revisits this metaslab.
1198084fd14fSBrian Behlendorf */
1199084fd14fSBrian Behlendorf if (msp->ms_disabled > 1) {
1200084fd14fSBrian Behlendorf mutex_exit(&msp->ms_lock);
1201af1d63abSPaul Dagnelie metaslab_enable(msp, B_FALSE, B_FALSE);
1202084fd14fSBrian Behlendorf continue;
1203084fd14fSBrian Behlendorf }
1204084fd14fSBrian Behlendorf
1205084fd14fSBrian Behlendorf /*
1206084fd14fSBrian Behlendorf * Allocate an empty range tree which is swapped in
1207084fd14fSBrian Behlendorf * for the existing ms_trim tree while it is processed.
1208084fd14fSBrian Behlendorf */
12094d7988d6SPaul Dagnelie trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL,
12104d7988d6SPaul Dagnelie 0, 0);
1211084fd14fSBrian Behlendorf range_tree_swap(&msp->ms_trim, &trim_tree);
1212084fd14fSBrian Behlendorf ASSERT(range_tree_is_empty(msp->ms_trim));
1213084fd14fSBrian Behlendorf
1214084fd14fSBrian Behlendorf /*
1215084fd14fSBrian Behlendorf * There are two cases when constructing the per-vdev
1216084fd14fSBrian Behlendorf * trim trees for a metaslab. If the top-level vdev
1217084fd14fSBrian Behlendorf * has no children then it is also a leaf and should
1218084fd14fSBrian Behlendorf * be trimmed. Otherwise our children are the leaves
1219084fd14fSBrian Behlendorf * and a trim tree should be constructed for each.
1220084fd14fSBrian Behlendorf */
1221084fd14fSBrian Behlendorf trim_args_t *tap;
1222084fd14fSBrian Behlendorf uint64_t children = vd->vdev_children;
1223084fd14fSBrian Behlendorf if (children == 0) {
1224084fd14fSBrian Behlendorf children = 1;
1225084fd14fSBrian Behlendorf tap = kmem_zalloc(sizeof (trim_args_t) *
1226084fd14fSBrian Behlendorf children, KM_SLEEP);
1227084fd14fSBrian Behlendorf tap[0].trim_vdev = vd;
1228084fd14fSBrian Behlendorf } else {
1229084fd14fSBrian Behlendorf tap = kmem_zalloc(sizeof (trim_args_t) *
1230084fd14fSBrian Behlendorf children, KM_SLEEP);
1231084fd14fSBrian Behlendorf
1232084fd14fSBrian Behlendorf for (uint64_t c = 0; c < children; c++) {
1233084fd14fSBrian Behlendorf tap[c].trim_vdev = vd->vdev_child[c];
1234084fd14fSBrian Behlendorf }
1235084fd14fSBrian Behlendorf }
1236084fd14fSBrian Behlendorf
1237084fd14fSBrian Behlendorf for (uint64_t c = 0; c < children; c++) {
1238084fd14fSBrian Behlendorf trim_args_t *ta = &tap[c];
1239084fd14fSBrian Behlendorf vdev_t *cvd = ta->trim_vdev;
1240084fd14fSBrian Behlendorf
1241084fd14fSBrian Behlendorf ta->trim_msp = msp;
1242084fd14fSBrian Behlendorf ta->trim_extent_bytes_max = extent_bytes_max;
1243084fd14fSBrian Behlendorf ta->trim_extent_bytes_min = extent_bytes_min;
1244084fd14fSBrian Behlendorf ta->trim_type = TRIM_TYPE_AUTO;
1245084fd14fSBrian Behlendorf ta->trim_flags = 0;
1246084fd14fSBrian Behlendorf
1247084fd14fSBrian Behlendorf if (cvd->vdev_detached ||
1248084fd14fSBrian Behlendorf !vdev_writeable(cvd) ||
1249084fd14fSBrian Behlendorf !cvd->vdev_has_trim ||
1250084fd14fSBrian Behlendorf cvd->vdev_trim_thread != NULL) {
1251084fd14fSBrian Behlendorf continue;
1252084fd14fSBrian Behlendorf }
1253084fd14fSBrian Behlendorf
1254084fd14fSBrian Behlendorf /*
1255084fd14fSBrian Behlendorf * When a device has an attached hot spare, or
1256084fd14fSBrian Behlendorf * is being replaced it will not be trimmed.
1257084fd14fSBrian Behlendorf * This is done to avoid adding additional
1258084fd14fSBrian Behlendorf * stress to a potentially unhealthy device,
1259084fd14fSBrian Behlendorf * and to minimize the required rebuild time.
1260084fd14fSBrian Behlendorf */
1261084fd14fSBrian Behlendorf if (!cvd->vdev_ops->vdev_op_leaf)
1262084fd14fSBrian Behlendorf continue;
1263084fd14fSBrian Behlendorf
12644d7988d6SPaul Dagnelie ta->trim_tree = range_tree_create(NULL,
12654d7988d6SPaul Dagnelie RANGE_SEG64, NULL, 0, 0);
1266084fd14fSBrian Behlendorf range_tree_walk(trim_tree,
1267084fd14fSBrian Behlendorf vdev_trim_range_add, ta);
1268084fd14fSBrian Behlendorf }
1269084fd14fSBrian Behlendorf
1270084fd14fSBrian Behlendorf mutex_exit(&msp->ms_lock);
1271084fd14fSBrian Behlendorf spa_config_exit(spa, SCL_CONFIG, FTAG);
1272084fd14fSBrian Behlendorf
1273084fd14fSBrian Behlendorf /*
1274084fd14fSBrian Behlendorf * Issue the TRIM I/Os for all ranges covered by the
1275084fd14fSBrian Behlendorf * TRIM trees. These ranges are safe to TRIM because
1276084fd14fSBrian Behlendorf * no new allocations will be performed until the call
1277084fd14fSBrian Behlendorf * to metaslab_enabled() below.
1278084fd14fSBrian Behlendorf */
1279084fd14fSBrian Behlendorf for (uint64_t c = 0; c < children; c++) {
1280084fd14fSBrian Behlendorf trim_args_t *ta = &tap[c];
1281084fd14fSBrian Behlendorf
1282084fd14fSBrian Behlendorf /*
1283084fd14fSBrian Behlendorf * Always yield to a manual TRIM if one has
1284084fd14fSBrian Behlendorf * been started for the child vdev.
1285084fd14fSBrian Behlendorf */
1286084fd14fSBrian Behlendorf if (ta->trim_tree == NULL ||
1287084fd14fSBrian Behlendorf ta->trim_vdev->vdev_trim_thread != NULL) {
1288084fd14fSBrian Behlendorf continue;
1289084fd14fSBrian Behlendorf }
1290084fd14fSBrian Behlendorf
1291084fd14fSBrian Behlendorf /*
1292084fd14fSBrian Behlendorf * After this point metaslab_enable() must be
1293084fd14fSBrian Behlendorf * called with the sync flag set. This is done
1294084fd14fSBrian Behlendorf * here because vdev_trim_ranges() is allowed
1295084fd14fSBrian Behlendorf * to be interrupted (EINTR) before issuing all
1296084fd14fSBrian Behlendorf * of the required TRIM I/Os.
1297084fd14fSBrian Behlendorf */
1298084fd14fSBrian Behlendorf issued_trim = B_TRUE;
1299084fd14fSBrian Behlendorf
1300084fd14fSBrian Behlendorf int error = vdev_trim_ranges(ta);
1301084fd14fSBrian Behlendorf if (error)
1302084fd14fSBrian Behlendorf break;
1303084fd14fSBrian Behlendorf }
1304084fd14fSBrian Behlendorf
1305084fd14fSBrian Behlendorf /*
1306084fd14fSBrian Behlendorf * Verify every range which was trimmed is still
1307084fd14fSBrian Behlendorf * contained within the ms_allocatable tree.
1308084fd14fSBrian Behlendorf */
1309084fd14fSBrian Behlendorf if (zfs_flags & ZFS_DEBUG_TRIM) {
1310084fd14fSBrian Behlendorf mutex_enter(&msp->ms_lock);
1311084fd14fSBrian Behlendorf VERIFY0(metaslab_load(msp));
1312084fd14fSBrian Behlendorf VERIFY3P(tap[0].trim_msp, ==, msp);
1313084fd14fSBrian Behlendorf range_tree_walk(trim_tree,
1314084fd14fSBrian Behlendorf vdev_trim_range_verify, &tap[0]);
1315084fd14fSBrian Behlendorf mutex_exit(&msp->ms_lock);
1316084fd14fSBrian Behlendorf }
1317084fd14fSBrian Behlendorf
1318084fd14fSBrian Behlendorf range_tree_vacate(trim_tree, NULL, NULL);
1319084fd14fSBrian Behlendorf range_tree_destroy(trim_tree);
1320084fd14fSBrian Behlendorf
1321af1d63abSPaul Dagnelie metaslab_enable(msp, issued_trim, B_FALSE);
1322084fd14fSBrian Behlendorf spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1323084fd14fSBrian Behlendorf
1324084fd14fSBrian Behlendorf for (uint64_t c = 0; c < children; c++) {
1325084fd14fSBrian Behlendorf trim_args_t *ta = &tap[c];
1326084fd14fSBrian Behlendorf
1327084fd14fSBrian Behlendorf if (ta->trim_tree == NULL)
1328084fd14fSBrian Behlendorf continue;
1329084fd14fSBrian Behlendorf
1330084fd14fSBrian Behlendorf range_tree_vacate(ta->trim_tree, NULL, NULL);
1331084fd14fSBrian Behlendorf range_tree_destroy(ta->trim_tree);
1332084fd14fSBrian Behlendorf }
1333084fd14fSBrian Behlendorf
1334084fd14fSBrian Behlendorf kmem_free(tap, sizeof (trim_args_t) * children);
1335084fd14fSBrian Behlendorf }
1336084fd14fSBrian Behlendorf
1337084fd14fSBrian Behlendorf spa_config_exit(spa, SCL_CONFIG, FTAG);
1338084fd14fSBrian Behlendorf
1339084fd14fSBrian Behlendorf /*
1340084fd14fSBrian Behlendorf * After completing the group of metaslabs wait for the next
1341084fd14fSBrian Behlendorf * open txg. This is done to make sure that a minimum of
1342084fd14fSBrian Behlendorf * zfs_trim_txg_batch txgs will occur before these metaslabs
1343084fd14fSBrian Behlendorf * are trimmed again.
1344084fd14fSBrian Behlendorf */
1345084fd14fSBrian Behlendorf txg_wait_open(spa_get_dsl(spa), 0, issued_trim);
1346084fd14fSBrian Behlendorf
1347084fd14fSBrian Behlendorf shift++;
1348084fd14fSBrian Behlendorf spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1349084fd14fSBrian Behlendorf }
1350084fd14fSBrian Behlendorf
1351084fd14fSBrian Behlendorf for (uint64_t c = 0; c < vd->vdev_children; c++) {
1352084fd14fSBrian Behlendorf vdev_t *cvd = vd->vdev_child[c];
1353084fd14fSBrian Behlendorf mutex_enter(&cvd->vdev_trim_io_lock);
1354084fd14fSBrian Behlendorf
1355084fd14fSBrian Behlendorf while (cvd->vdev_trim_inflight[1] > 0) {
1356084fd14fSBrian Behlendorf cv_wait(&cvd->vdev_trim_io_cv,
1357084fd14fSBrian Behlendorf &cvd->vdev_trim_io_lock);
1358084fd14fSBrian Behlendorf }
1359084fd14fSBrian Behlendorf mutex_exit(&cvd->vdev_trim_io_lock);
1360084fd14fSBrian Behlendorf }
1361084fd14fSBrian Behlendorf
1362084fd14fSBrian Behlendorf spa_config_exit(spa, SCL_CONFIG, FTAG);
1363084fd14fSBrian Behlendorf
1364084fd14fSBrian Behlendorf /*
1365084fd14fSBrian Behlendorf * When exiting because the autotrim property was set to off, then
1366084fd14fSBrian Behlendorf * abandon any unprocessed ms_trim ranges to reclaim the memory.
1367084fd14fSBrian Behlendorf */
1368084fd14fSBrian Behlendorf if (spa_get_autotrim(spa) == SPA_AUTOTRIM_OFF) {
1369084fd14fSBrian Behlendorf for (uint64_t i = 0; i < vd->vdev_ms_count; i++) {
1370084fd14fSBrian Behlendorf metaslab_t *msp = vd->vdev_ms[i];
1371084fd14fSBrian Behlendorf
1372084fd14fSBrian Behlendorf mutex_enter(&msp->ms_lock);
1373084fd14fSBrian Behlendorf range_tree_vacate(msp->ms_trim, NULL, NULL);
1374084fd14fSBrian Behlendorf mutex_exit(&msp->ms_lock);
1375084fd14fSBrian Behlendorf }
1376084fd14fSBrian Behlendorf }
1377084fd14fSBrian Behlendorf
1378084fd14fSBrian Behlendorf mutex_enter(&vd->vdev_autotrim_lock);
1379084fd14fSBrian Behlendorf ASSERT(vd->vdev_autotrim_thread != NULL);
1380084fd14fSBrian Behlendorf vd->vdev_autotrim_thread = NULL;
1381084fd14fSBrian Behlendorf cv_broadcast(&vd->vdev_autotrim_cv);
1382084fd14fSBrian Behlendorf mutex_exit(&vd->vdev_autotrim_lock);
1383084fd14fSBrian Behlendorf }
1384084fd14fSBrian Behlendorf
1385084fd14fSBrian Behlendorf /*
1386084fd14fSBrian Behlendorf * Starts an autotrim thread, if needed, for each top-level vdev which can be
1387084fd14fSBrian Behlendorf * trimmed. A top-level vdev which has been evacuated will never be trimmed.
1388084fd14fSBrian Behlendorf */
1389084fd14fSBrian Behlendorf void
vdev_autotrim(spa_t * spa)1390084fd14fSBrian Behlendorf vdev_autotrim(spa_t *spa)
1391084fd14fSBrian Behlendorf {
1392084fd14fSBrian Behlendorf vdev_t *root_vd = spa->spa_root_vdev;
1393084fd14fSBrian Behlendorf
1394084fd14fSBrian Behlendorf for (uint64_t i = 0; i < root_vd->vdev_children; i++) {
1395084fd14fSBrian Behlendorf vdev_t *tvd = root_vd->vdev_child[i];
1396084fd14fSBrian Behlendorf
1397084fd14fSBrian Behlendorf mutex_enter(&tvd->vdev_autotrim_lock);
1398084fd14fSBrian Behlendorf if (vdev_writeable(tvd) && !tvd->vdev_removing &&
1399084fd14fSBrian Behlendorf tvd->vdev_autotrim_thread == NULL) {
1400084fd14fSBrian Behlendorf ASSERT3P(tvd->vdev_top, ==, tvd);
1401084fd14fSBrian Behlendorf
1402084fd14fSBrian Behlendorf tvd->vdev_autotrim_thread = thread_create(NULL, 0,
1403084fd14fSBrian Behlendorf vdev_autotrim_thread, tvd, 0, &p0, TS_RUN,
1404084fd14fSBrian Behlendorf maxclsyspri);
1405084fd14fSBrian Behlendorf ASSERT(tvd->vdev_autotrim_thread != NULL);
1406084fd14fSBrian Behlendorf }
1407084fd14fSBrian Behlendorf mutex_exit(&tvd->vdev_autotrim_lock);
1408084fd14fSBrian Behlendorf }
1409084fd14fSBrian Behlendorf }
1410084fd14fSBrian Behlendorf
1411084fd14fSBrian Behlendorf /*
1412084fd14fSBrian Behlendorf * Wait for the vdev_autotrim_thread associated with the passed top-level
1413084fd14fSBrian Behlendorf * vdev to be terminated (canceled or stopped).
1414084fd14fSBrian Behlendorf */
1415084fd14fSBrian Behlendorf void
vdev_autotrim_stop_wait(vdev_t * tvd)1416084fd14fSBrian Behlendorf vdev_autotrim_stop_wait(vdev_t *tvd)
1417084fd14fSBrian Behlendorf {
1418084fd14fSBrian Behlendorf mutex_enter(&tvd->vdev_autotrim_lock);
1419084fd14fSBrian Behlendorf if (tvd->vdev_autotrim_thread != NULL) {
1420084fd14fSBrian Behlendorf tvd->vdev_autotrim_exit_wanted = B_TRUE;
1421084fd14fSBrian Behlendorf
1422084fd14fSBrian Behlendorf while (tvd->vdev_autotrim_thread != NULL) {
1423084fd14fSBrian Behlendorf cv_wait(&tvd->vdev_autotrim_cv,
1424084fd14fSBrian Behlendorf &tvd->vdev_autotrim_lock);
1425084fd14fSBrian Behlendorf }
1426084fd14fSBrian Behlendorf
1427084fd14fSBrian Behlendorf ASSERT3P(tvd->vdev_autotrim_thread, ==, NULL);
1428084fd14fSBrian Behlendorf tvd->vdev_autotrim_exit_wanted = B_FALSE;
1429084fd14fSBrian Behlendorf }
1430084fd14fSBrian Behlendorf mutex_exit(&tvd->vdev_autotrim_lock);
1431084fd14fSBrian Behlendorf }
1432084fd14fSBrian Behlendorf
1433084fd14fSBrian Behlendorf /*
1434084fd14fSBrian Behlendorf * Wait for all of the vdev_autotrim_thread associated with the pool to
1435084fd14fSBrian Behlendorf * be terminated (canceled or stopped).
1436084fd14fSBrian Behlendorf */
1437084fd14fSBrian Behlendorf void
vdev_autotrim_stop_all(spa_t * spa)1438084fd14fSBrian Behlendorf vdev_autotrim_stop_all(spa_t *spa)
1439084fd14fSBrian Behlendorf {
1440084fd14fSBrian Behlendorf vdev_t *root_vd = spa->spa_root_vdev;
1441084fd14fSBrian Behlendorf
1442084fd14fSBrian Behlendorf for (uint64_t i = 0; i < root_vd->vdev_children; i++)
1443084fd14fSBrian Behlendorf vdev_autotrim_stop_wait(root_vd->vdev_child[i]);
1444084fd14fSBrian Behlendorf }
1445084fd14fSBrian Behlendorf
1446084fd14fSBrian Behlendorf /*
1447084fd14fSBrian Behlendorf * Conditionally restart all of the vdev_autotrim_thread's for the pool.
1448084fd14fSBrian Behlendorf */
1449084fd14fSBrian Behlendorf void
vdev_autotrim_restart(spa_t * spa)1450084fd14fSBrian Behlendorf vdev_autotrim_restart(spa_t *spa)
1451084fd14fSBrian Behlendorf {
1452084fd14fSBrian Behlendorf ASSERT(MUTEX_HELD(&spa_namespace_lock));
1453084fd14fSBrian Behlendorf
1454084fd14fSBrian Behlendorf if (spa->spa_autotrim)
1455084fd14fSBrian Behlendorf vdev_autotrim(spa);
1456084fd14fSBrian Behlendorf }
1457