1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
25  */
26 
27 #include <sys/zfs_context.h>
28 #include <sys/spa_impl.h>
29 #include <sys/dmu.h>
30 #include <sys/dmu_tx.h>
31 #include <sys/zap.h>
32 #include <sys/vdev_impl.h>
33 #include <sys/metaslab.h>
34 #include <sys/metaslab_impl.h>
35 #include <sys/uberblock_impl.h>
36 #include <sys/txg.h>
37 #include <sys/avl.h>
38 #include <sys/bpobj.h>
39 #include <sys/dsl_pool.h>
40 #include <sys/dsl_synctask.h>
41 #include <sys/dsl_dir.h>
42 #include <sys/arc.h>
43 #include <sys/zfeature.h>
44 #include <sys/vdev_indirect_births.h>
45 #include <sys/vdev_indirect_mapping.h>
46 #include <sys/abd.h>
47 #include <sys/vdev_initialize.h>
48 #include <sys/vdev_trim.h>
49 
50 /*
51  * This file contains the necessary logic to remove vdevs from a
52  * storage pool.  Currently, the only devices that can be removed
53  * are log, cache, and spare devices; and top level vdevs from a pool
54  * w/o raidz.  (Note that members of a mirror can also be removed
55  * by the detach operation.)
56  *
57  * Log vdevs are removed by evacuating them and then turning the vdev
58  * into a hole vdev while holding spa config locks.
59  *
60  * Top level vdevs are removed and converted into an indirect vdev via
61  * a multi-step process:
62  *
63  *  - Disable allocations from this device (spa_vdev_remove_top).
64  *
65  *  - From a new thread (spa_vdev_remove_thread), copy data from
66  *    the removing vdev to a different vdev.  The copy happens in open
67  *    context (spa_vdev_copy_impl) and issues a sync task
68  *    (vdev_mapping_sync) so the sync thread can update the partial
69  *    indirect mappings in core and on disk.
70  *
71  *  - If a free happens during a removal, it is freed from the
72  *    removing vdev, and if it has already been copied, from the new
73  *    location as well (free_from_removing_vdev).
74  *
75  *  - After the removal is completed, the copy thread converts the vdev
76  *    into an indirect vdev (vdev_remove_complete) before instructing
77  *    the sync thread to destroy the space maps and finish the removal
78  *    (spa_finish_removal).
79  */
80 
81 typedef struct vdev_copy_arg {
82 	metaslab_t	*vca_msp;
83 	uint64_t	vca_outstanding_bytes;
84 	kcondvar_t	vca_cv;
85 	kmutex_t	vca_lock;
86 } vdev_copy_arg_t;
87 
88 /*
89  * The maximum amount of memory we can use for outstanding i/o while
90  * doing a device removal.  This determines how much i/o we can have
91  * in flight concurrently.
92  */
93 int zfs_remove_max_copy_bytes = 64 * 1024 * 1024;
94 
95 /*
96  * The largest contiguous segment that we will attempt to allocate when
97  * removing a device.  This can be no larger than SPA_MAXBLOCKSIZE.  If
98  * there is a performance problem with attempting to allocate large blocks,
99  * consider decreasing this.
100  *
101  * Note: we will issue I/Os of up to this size.  The mpt driver does not
102  * respond well to I/Os larger than 1MB, so we set this to 1MB.  (When
103  * mpt processes an I/O larger than 1MB, it needs to do an allocation of
104  * 2 physically contiguous pages; if this allocation fails, mpt will drop
105  * the I/O and hang the device.)
106  */
107 int zfs_remove_max_segment = 1024 * 1024;
108 
109 /*
110  * Allow a remap segment to span free chunks of at most this size. The main
111  * impact of a larger span is that we will read and write larger, more
112  * contiguous chunks, with more "unnecessary" data -- trading off bandwidth
113  * for iops.  The value here was chosen to align with
114  * zfs_vdev_read_gap_limit, which is a similar concept when doing regular
115  * reads (but there's no reason it has to be the same).
116  *
117  * Additionally, a higher span will have the following relatively minor
118  * effects:
119  *  - the mapping will be smaller, since one entry can cover more allocated
120  *    segments
121  *  - more of the fragmentation in the removing device will be preserved
122  *  - we'll do larger allocations, which may fail and fall back on smaller
123  *    allocations
124  */
125 int vdev_removal_max_span = 32 * 1024;
126 
127 /*
128  * This is used by the test suite so that it can ensure that certain
129  * actions happen while in the middle of a removal.
130  */
131 int zfs_removal_suspend_progress = 0;
132 
133 #define	VDEV_REMOVAL_ZAP_OBJS	"lzap"
134 
135 static void spa_vdev_remove_thread(void *arg);
136 
137 static void
spa_sync_removing_state(spa_t * spa,dmu_tx_t * tx)138 spa_sync_removing_state(spa_t *spa, dmu_tx_t *tx)
139 {
140 	VERIFY0(zap_update(spa->spa_dsl_pool->dp_meta_objset,
141 	    DMU_POOL_DIRECTORY_OBJECT,
142 	    DMU_POOL_REMOVING, sizeof (uint64_t),
143 	    sizeof (spa->spa_removing_phys) / sizeof (uint64_t),
144 	    &spa->spa_removing_phys, tx));
145 }
146 
147 static nvlist_t *
spa_nvlist_lookup_by_guid(nvlist_t ** nvpp,int count,uint64_t target_guid)148 spa_nvlist_lookup_by_guid(nvlist_t **nvpp, int count, uint64_t target_guid)
149 {
150 	for (int i = 0; i < count; i++) {
151 		uint64_t guid =
152 		    fnvlist_lookup_uint64(nvpp[i], ZPOOL_CONFIG_GUID);
153 
154 		if (guid == target_guid)
155 			return (nvpp[i]);
156 	}
157 
158 	return (NULL);
159 }
160 
161 static void
spa_vdev_remove_aux(nvlist_t * config,char * name,nvlist_t ** dev,int count,nvlist_t * dev_to_remove)162 spa_vdev_remove_aux(nvlist_t *config, char *name, nvlist_t **dev, int count,
163     nvlist_t *dev_to_remove)
164 {
165 	nvlist_t **newdev = NULL;
166 
167 	if (count > 1)
168 		newdev = kmem_alloc((count - 1) * sizeof (void *), KM_SLEEP);
169 
170 	for (int i = 0, j = 0; i < count; i++) {
171 		if (dev[i] == dev_to_remove)
172 			continue;
173 		VERIFY(nvlist_dup(dev[i], &newdev[j++], KM_SLEEP) == 0);
174 	}
175 
176 	VERIFY(nvlist_remove(config, name, DATA_TYPE_NVLIST_ARRAY) == 0);
177 	VERIFY(nvlist_add_nvlist_array(config, name, newdev, count - 1) == 0);
178 
179 	for (int i = 0; i < count - 1; i++)
180 		nvlist_free(newdev[i]);
181 
182 	if (count > 1)
183 		kmem_free(newdev, (count - 1) * sizeof (void *));
184 }
185 
186 static spa_vdev_removal_t *
spa_vdev_removal_create(vdev_t * vd)187 spa_vdev_removal_create(vdev_t *vd)
188 {
189 	spa_vdev_removal_t *svr = kmem_zalloc(sizeof (*svr), KM_SLEEP);
190 	mutex_init(&svr->svr_lock, NULL, MUTEX_DEFAULT, NULL);
191 	cv_init(&svr->svr_cv, NULL, CV_DEFAULT, NULL);
192 	svr->svr_allocd_segs = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);
193 	svr->svr_vdev_id = vd->vdev_id;
194 
195 	for (int i = 0; i < TXG_SIZE; i++) {
196 		svr->svr_frees[i] = range_tree_create(NULL, RANGE_SEG64, NULL,
197 		    0, 0);
198 		list_create(&svr->svr_new_segments[i],
199 		    sizeof (vdev_indirect_mapping_entry_t),
200 		    offsetof(vdev_indirect_mapping_entry_t, vime_node));
201 	}
202 
203 	return (svr);
204 }
205 
206 void
spa_vdev_removal_destroy(spa_vdev_removal_t * svr)207 spa_vdev_removal_destroy(spa_vdev_removal_t *svr)
208 {
209 	for (int i = 0; i < TXG_SIZE; i++) {
210 		ASSERT0(svr->svr_bytes_done[i]);
211 		ASSERT0(svr->svr_max_offset_to_sync[i]);
212 		range_tree_destroy(svr->svr_frees[i]);
213 		list_destroy(&svr->svr_new_segments[i]);
214 	}
215 
216 	range_tree_destroy(svr->svr_allocd_segs);
217 	mutex_destroy(&svr->svr_lock);
218 	cv_destroy(&svr->svr_cv);
219 	kmem_free(svr, sizeof (*svr));
220 }
221 
222 /*
223  * This is called as a synctask in the txg in which we will mark this vdev
224  * as removing (in the config stored in the MOS).
225  *
226  * It begins the evacuation of a toplevel vdev by:
227  * - initializing the spa_removing_phys which tracks this removal
228  * - computing the amount of space to remove for accounting purposes
229  * - dirtying all dbufs in the spa_config_object
230  * - creating the spa_vdev_removal
231  * - starting the spa_vdev_remove_thread
232  */
233 static void
vdev_remove_initiate_sync(void * arg,dmu_tx_t * tx)234 vdev_remove_initiate_sync(void *arg, dmu_tx_t *tx)
235 {
236 	int vdev_id = (uintptr_t)arg;
237 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
238 	vdev_t *vd = vdev_lookup_top(spa, vdev_id);
239 	vdev_indirect_config_t *vic = &vd->vdev_indirect_config;
240 	objset_t *mos = spa->spa_dsl_pool->dp_meta_objset;
241 	spa_vdev_removal_t *svr = NULL;
242 	uint64_t txg = dmu_tx_get_txg(tx);
243 
244 	ASSERT3P(vd->vdev_ops, !=, &vdev_raidz_ops);
245 	svr = spa_vdev_removal_create(vd);
246 
247 	ASSERT(vd->vdev_removing);
248 	ASSERT3P(vd->vdev_indirect_mapping, ==, NULL);
249 
250 	spa_feature_incr(spa, SPA_FEATURE_DEVICE_REMOVAL, tx);
251 	if (spa_feature_is_enabled(spa, SPA_FEATURE_OBSOLETE_COUNTS)) {
252 		/*
253 		 * By activating the OBSOLETE_COUNTS feature, we prevent
254 		 * the pool from being downgraded and ensure that the
255 		 * refcounts are precise.
256 		 */
257 		spa_feature_incr(spa, SPA_FEATURE_OBSOLETE_COUNTS, tx);
258 		uint64_t one = 1;
259 		VERIFY0(zap_add(spa->spa_meta_objset, vd->vdev_top_zap,
260 		    VDEV_TOP_ZAP_OBSOLETE_COUNTS_ARE_PRECISE, sizeof (one), 1,
261 		    &one, tx));
262 		ASSERT3U(vdev_obsolete_counts_are_precise(vd), !=, 0);
263 	}
264 
265 	vic->vic_mapping_object = vdev_indirect_mapping_alloc(mos, tx);
266 	vd->vdev_indirect_mapping =
267 	    vdev_indirect_mapping_open(mos, vic->vic_mapping_object);
268 	vic->vic_births_object = vdev_indirect_births_alloc(mos, tx);
269 	vd->vdev_indirect_births =
270 	    vdev_indirect_births_open(mos, vic->vic_births_object);
271 	spa->spa_removing_phys.sr_removing_vdev = vd->vdev_id;
272 	spa->spa_removing_phys.sr_start_time = gethrestime_sec();
273 	spa->spa_removing_phys.sr_end_time = 0;
274 	spa->spa_removing_phys.sr_state = DSS_SCANNING;
275 	spa->spa_removing_phys.sr_to_copy = 0;
276 	spa->spa_removing_phys.sr_copied = 0;
277 
278 	/*
279 	 * Note: We can't use vdev_stat's vs_alloc for sr_to_copy, because
280 	 * there may be space in the defer tree, which is free, but still
281 	 * counted in vs_alloc.
282 	 */
283 	for (uint64_t i = 0; i < vd->vdev_ms_count; i++) {
284 		metaslab_t *ms = vd->vdev_ms[i];
285 		if (ms->ms_sm == NULL)
286 			continue;
287 
288 		spa->spa_removing_phys.sr_to_copy +=
289 		    metaslab_allocated_space(ms);
290 
291 		/*
292 		 * Space which we are freeing this txg does not need to
293 		 * be copied.
294 		 */
295 		spa->spa_removing_phys.sr_to_copy -=
296 		    range_tree_space(ms->ms_freeing);
297 
298 		ASSERT0(range_tree_space(ms->ms_freed));
299 		for (int t = 0; t < TXG_SIZE; t++)
300 			ASSERT0(range_tree_space(ms->ms_allocating[t]));
301 	}
302 
303 	/*
304 	 * Sync tasks are called before metaslab_sync(), so there should
305 	 * be no already-synced metaslabs in the TXG_CLEAN list.
306 	 */
307 	ASSERT3P(txg_list_head(&vd->vdev_ms_list, TXG_CLEAN(txg)), ==, NULL);
308 
309 	spa_sync_removing_state(spa, tx);
310 
311 	/*
312 	 * All blocks that we need to read the most recent mapping must be
313 	 * stored on concrete vdevs.  Therefore, we must dirty anything that
314 	 * is read before spa_remove_init().  Specifically, the
315 	 * spa_config_object.  (Note that although we already modified the
316 	 * spa_config_object in spa_sync_removing_state, that may not have
317 	 * modified all blocks of the object.)
318 	 */
319 	dmu_object_info_t doi;
320 	VERIFY0(dmu_object_info(mos, DMU_POOL_DIRECTORY_OBJECT, &doi));
321 	for (uint64_t offset = 0; offset < doi.doi_max_offset; ) {
322 		dmu_buf_t *dbuf;
323 		VERIFY0(dmu_buf_hold(mos, DMU_POOL_DIRECTORY_OBJECT,
324 		    offset, FTAG, &dbuf, 0));
325 		dmu_buf_will_dirty(dbuf, tx);
326 		offset += dbuf->db_size;
327 		dmu_buf_rele(dbuf, FTAG);
328 	}
329 
330 	/*
331 	 * Now that we've allocated the im_object, dirty the vdev to ensure
332 	 * that the object gets written to the config on disk.
333 	 */
334 	vdev_config_dirty(vd);
335 
336 	zfs_dbgmsg("starting removal thread for vdev %llu (%p) in txg %llu "
337 	    "im_obj=%llu", vd->vdev_id, vd, dmu_tx_get_txg(tx),
338 	    vic->vic_mapping_object);
339 
340 	spa_history_log_internal(spa, "vdev remove started", tx,
341 	    "%s vdev %llu %s", spa_name(spa), vd->vdev_id,
342 	    (vd->vdev_path != NULL) ? vd->vdev_path : "-");
343 	/*
344 	 * Setting spa_vdev_removal causes subsequent frees to call
345 	 * free_from_removing_vdev().  Note that we don't need any locking
346 	 * because we are the sync thread, and metaslab_free_impl() is only
347 	 * called from syncing context (potentially from a zio taskq thread,
348 	 * but in any case only when there are outstanding free i/os, which
349 	 * there are not).
350 	 */
351 	ASSERT3P(spa->spa_vdev_removal, ==, NULL);
352 	spa->spa_vdev_removal = svr;
353 	svr->svr_thread = thread_create(NULL, 0,
354 	    spa_vdev_remove_thread, spa, 0, &p0, TS_RUN, minclsyspri);
355 }
356 
357 /*
358  * When we are opening a pool, we must read the mapping for each
359  * indirect vdev in order from most recently removed to least
360  * recently removed.  We do this because the blocks for the mapping
361  * of older indirect vdevs may be stored on more recently removed vdevs.
362  * In order to read each indirect mapping object, we must have
363  * initialized all more recently removed vdevs.
364  */
365 int
spa_remove_init(spa_t * spa)366 spa_remove_init(spa_t *spa)
367 {
368 	int error;
369 
370 	error = zap_lookup(spa->spa_dsl_pool->dp_meta_objset,
371 	    DMU_POOL_DIRECTORY_OBJECT,
372 	    DMU_POOL_REMOVING, sizeof (uint64_t),
373 	    sizeof (spa->spa_removing_phys) / sizeof (uint64_t),
374 	    &spa->spa_removing_phys);
375 
376 	if (error == ENOENT) {
377 		spa->spa_removing_phys.sr_state = DSS_NONE;
378 		spa->spa_removing_phys.sr_removing_vdev = -1;
379 		spa->spa_removing_phys.sr_prev_indirect_vdev = -1;
380 		spa->spa_indirect_vdevs_loaded = B_TRUE;
381 		return (0);
382 	} else if (error != 0) {
383 		return (error);
384 	}
385 
386 	if (spa->spa_removing_phys.sr_state == DSS_SCANNING) {
387 		/*
388 		 * We are currently removing a vdev.  Create and
389 		 * initialize a spa_vdev_removal_t from the bonus
390 		 * buffer of the removing vdevs vdev_im_object, and
391 		 * initialize its partial mapping.
392 		 */
393 		spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
394 		vdev_t *vd = vdev_lookup_top(spa,
395 		    spa->spa_removing_phys.sr_removing_vdev);
396 
397 		if (vd == NULL) {
398 			spa_config_exit(spa, SCL_STATE, FTAG);
399 			return (EINVAL);
400 		}
401 
402 		vdev_indirect_config_t *vic = &vd->vdev_indirect_config;
403 
404 		ASSERT(vdev_is_concrete(vd));
405 		spa_vdev_removal_t *svr = spa_vdev_removal_create(vd);
406 		ASSERT3U(svr->svr_vdev_id, ==, vd->vdev_id);
407 		ASSERT(vd->vdev_removing);
408 
409 		vd->vdev_indirect_mapping = vdev_indirect_mapping_open(
410 		    spa->spa_meta_objset, vic->vic_mapping_object);
411 		vd->vdev_indirect_births = vdev_indirect_births_open(
412 		    spa->spa_meta_objset, vic->vic_births_object);
413 		spa_config_exit(spa, SCL_STATE, FTAG);
414 
415 		spa->spa_vdev_removal = svr;
416 	}
417 
418 	spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
419 	uint64_t indirect_vdev_id =
420 	    spa->spa_removing_phys.sr_prev_indirect_vdev;
421 	while (indirect_vdev_id != UINT64_MAX) {
422 		vdev_t *vd = vdev_lookup_top(spa, indirect_vdev_id);
423 		vdev_indirect_config_t *vic = &vd->vdev_indirect_config;
424 
425 		ASSERT3P(vd->vdev_ops, ==, &vdev_indirect_ops);
426 		vd->vdev_indirect_mapping = vdev_indirect_mapping_open(
427 		    spa->spa_meta_objset, vic->vic_mapping_object);
428 		vd->vdev_indirect_births = vdev_indirect_births_open(
429 		    spa->spa_meta_objset, vic->vic_births_object);
430 
431 		indirect_vdev_id = vic->vic_prev_indirect_vdev;
432 	}
433 	spa_config_exit(spa, SCL_STATE, FTAG);
434 
435 	/*
436 	 * Now that we've loaded all the indirect mappings, we can allow
437 	 * reads from other blocks (e.g. via predictive prefetch).
438 	 */
439 	spa->spa_indirect_vdevs_loaded = B_TRUE;
440 	return (0);
441 }
442 
443 void
spa_restart_removal(spa_t * spa)444 spa_restart_removal(spa_t *spa)
445 {
446 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;
447 
448 	if (svr == NULL)
449 		return;
450 
451 	/*
452 	 * In general when this function is called there is no
453 	 * removal thread running. The only scenario where this
454 	 * is not true is during spa_import() where this function
455 	 * is called twice [once from spa_import_impl() and
456 	 * spa_async_resume()]. Thus, in the scenario where we
457 	 * import a pool that has an ongoing removal we don't
458 	 * want to spawn a second thread.
459 	 */
460 	if (svr->svr_thread != NULL)
461 		return;
462 
463 	if (!spa_writeable(spa))
464 		return;
465 
466 	zfs_dbgmsg("restarting removal of %llu", svr->svr_vdev_id);
467 	svr->svr_thread = thread_create(NULL, 0, spa_vdev_remove_thread, spa,
468 	    0, &p0, TS_RUN, minclsyspri);
469 }
470 
471 /*
472  * Process freeing from a device which is in the middle of being removed.
473  * We must handle this carefully so that we attempt to copy freed data,
474  * and we correctly free already-copied data.
475  */
476 void
free_from_removing_vdev(vdev_t * vd,uint64_t offset,uint64_t size)477 free_from_removing_vdev(vdev_t *vd, uint64_t offset, uint64_t size)
478 {
479 	spa_t *spa = vd->vdev_spa;
480 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;
481 	vdev_indirect_mapping_t *vim = vd->vdev_indirect_mapping;
482 	uint64_t txg = spa_syncing_txg(spa);
483 	uint64_t max_offset_yet = 0;
484 
485 	ASSERT(vd->vdev_indirect_config.vic_mapping_object != 0);
486 	ASSERT3U(vd->vdev_indirect_config.vic_mapping_object, ==,
487 	    vdev_indirect_mapping_object(vim));
488 	ASSERT3U(vd->vdev_id, ==, svr->svr_vdev_id);
489 
490 	mutex_enter(&svr->svr_lock);
491 
492 	/*
493 	 * Remove the segment from the removing vdev's spacemap.  This
494 	 * ensures that we will not attempt to copy this space (if the
495 	 * removal thread has not yet visited it), and also ensures
496 	 * that we know what is actually allocated on the new vdevs
497 	 * (needed if we cancel the removal).
498 	 *
499 	 * Note: we must do the metaslab_free_concrete() with the svr_lock
500 	 * held, so that the remove_thread can not load this metaslab and then
501 	 * visit this offset between the time that we metaslab_free_concrete()
502 	 * and when we check to see if it has been visited.
503 	 *
504 	 * Note: The checkpoint flag is set to false as having/taking
505 	 * a checkpoint and removing a device can't happen at the same
506 	 * time.
507 	 */
508 	ASSERT(!spa_has_checkpoint(spa));
509 	metaslab_free_concrete(vd, offset, size, B_FALSE);
510 
511 	uint64_t synced_size = 0;
512 	uint64_t synced_offset = 0;
513 	uint64_t max_offset_synced = vdev_indirect_mapping_max_offset(vim);
514 	if (offset < max_offset_synced) {
515 		/*
516 		 * The mapping for this offset is already on disk.
517 		 * Free from the new location.
518 		 *
519 		 * Note that we use svr_max_synced_offset because it is
520 		 * updated atomically with respect to the in-core mapping.
521 		 * By contrast, vim_max_offset is not.
522 		 *
523 		 * This block may be split between a synced entry and an
524 		 * in-flight or unvisited entry.  Only process the synced
525 		 * portion of it here.
526 		 */
527 		synced_size = MIN(size, max_offset_synced - offset);
528 		synced_offset = offset;
529 
530 		ASSERT3U(max_offset_yet, <=, max_offset_synced);
531 		max_offset_yet = max_offset_synced;
532 
533 		DTRACE_PROBE3(remove__free__synced,
534 		    spa_t *, spa,
535 		    uint64_t, offset,
536 		    uint64_t, synced_size);
537 
538 		size -= synced_size;
539 		offset += synced_size;
540 	}
541 
542 	/*
543 	 * Look at all in-flight txgs starting from the currently syncing one
544 	 * and see if a section of this free is being copied. By starting from
545 	 * this txg and iterating forward, we might find that this region
546 	 * was copied in two different txgs and handle it appropriately.
547 	 */
548 	for (int i = 0; i < TXG_CONCURRENT_STATES; i++) {
549 		int txgoff = (txg + i) & TXG_MASK;
550 		if (size > 0 && offset < svr->svr_max_offset_to_sync[txgoff]) {
551 			/*
552 			 * The mapping for this offset is in flight, and
553 			 * will be synced in txg+i.
554 			 */
555 			uint64_t inflight_size = MIN(size,
556 			    svr->svr_max_offset_to_sync[txgoff] - offset);
557 
558 			DTRACE_PROBE4(remove__free__inflight,
559 			    spa_t *, spa,
560 			    uint64_t, offset,
561 			    uint64_t, inflight_size,
562 			    uint64_t, txg + i);
563 
564 			/*
565 			 * We copy data in order of increasing offset.
566 			 * Therefore the max_offset_to_sync[] must increase
567 			 * (or be zero, indicating that nothing is being
568 			 * copied in that txg).
569 			 */
570 			if (svr->svr_max_offset_to_sync[txgoff] != 0) {
571 				ASSERT3U(svr->svr_max_offset_to_sync[txgoff],
572 				    >=, max_offset_yet);
573 				max_offset_yet =
574 				    svr->svr_max_offset_to_sync[txgoff];
575 			}
576 
577 			/*
578 			 * We've already committed to copying this segment:
579 			 * we have allocated space elsewhere in the pool for
580 			 * it and have an IO outstanding to copy the data. We
581 			 * cannot free the space before the copy has
582 			 * completed, or else the copy IO might overwrite any
583 			 * new data. To free that space, we record the
584 			 * segment in the appropriate svr_frees tree and free
585 			 * the mapped space later, in the txg where we have
586 			 * completed the copy and synced the mapping (see
587 			 * vdev_mapping_sync).
588 			 */
589 			range_tree_add(svr->svr_frees[txgoff],
590 			    offset, inflight_size);
591 			size -= inflight_size;
592 			offset += inflight_size;
593 
594 			/*
595 			 * This space is already accounted for as being
596 			 * done, because it is being copied in txg+i.
597 			 * However, if i!=0, then it is being copied in
598 			 * a future txg.  If we crash after this txg
599 			 * syncs but before txg+i syncs, then the space
600 			 * will be free.  Therefore we must account
601 			 * for the space being done in *this* txg
602 			 * (when it is freed) rather than the future txg
603 			 * (when it will be copied).
604 			 */
605 			ASSERT3U(svr->svr_bytes_done[txgoff], >=,
606 			    inflight_size);
607 			svr->svr_bytes_done[txgoff] -= inflight_size;
608 			svr->svr_bytes_done[txg & TXG_MASK] += inflight_size;
609 		}
610 	}
611 	ASSERT0(svr->svr_max_offset_to_sync[TXG_CLEAN(txg) & TXG_MASK]);
612 
613 	if (size > 0) {
614 		/*
615 		 * The copy thread has not yet visited this offset.  Ensure
616 		 * that it doesn't.
617 		 */
618 
619 		DTRACE_PROBE3(remove__free__unvisited,
620 		    spa_t *, spa,
621 		    uint64_t, offset,
622 		    uint64_t, size);
623 
624 		if (svr->svr_allocd_segs != NULL)
625 			range_tree_clear(svr->svr_allocd_segs, offset, size);
626 
627 		/*
628 		 * Since we now do not need to copy this data, for
629 		 * accounting purposes we have done our job and can count
630 		 * it as completed.
631 		 */
632 		svr->svr_bytes_done[txg & TXG_MASK] += size;
633 	}
634 	mutex_exit(&svr->svr_lock);
635 
636 	/*
637 	 * Now that we have dropped svr_lock, process the synced portion
638 	 * of this free.
639 	 */
640 	if (synced_size > 0) {
641 		vdev_indirect_mark_obsolete(vd, synced_offset, synced_size);
642 
643 		/*
644 		 * Note: this can only be called from syncing context,
645 		 * and the vdev_indirect_mapping is only changed from the
646 		 * sync thread, so we don't need svr_lock while doing
647 		 * metaslab_free_impl_cb.
648 		 */
649 		boolean_t checkpoint = B_FALSE;
650 		vdev_indirect_ops.vdev_op_remap(vd, synced_offset, synced_size,
651 		    metaslab_free_impl_cb, &checkpoint);
652 	}
653 }
654 
655 /*
656  * Stop an active removal and update the spa_removing phys.
657  */
658 static void
spa_finish_removal(spa_t * spa,dsl_scan_state_t state,dmu_tx_t * tx)659 spa_finish_removal(spa_t *spa, dsl_scan_state_t state, dmu_tx_t *tx)
660 {
661 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;
662 	ASSERT3U(dmu_tx_get_txg(tx), ==, spa_syncing_txg(spa));
663 
664 	/* Ensure the removal thread has completed before we free the svr. */
665 	spa_vdev_remove_suspend(spa);
666 
667 	ASSERT(state == DSS_FINISHED || state == DSS_CANCELED);
668 
669 	if (state == DSS_FINISHED) {
670 		spa_removing_phys_t *srp = &spa->spa_removing_phys;
671 		vdev_t *vd = vdev_lookup_top(spa, svr->svr_vdev_id);
672 		vdev_indirect_config_t *vic = &vd->vdev_indirect_config;
673 
674 		if (srp->sr_prev_indirect_vdev != UINT64_MAX) {
675 			vdev_t *pvd = vdev_lookup_top(spa,
676 			    srp->sr_prev_indirect_vdev);
677 			ASSERT3P(pvd->vdev_ops, ==, &vdev_indirect_ops);
678 		}
679 
680 		vic->vic_prev_indirect_vdev = srp->sr_prev_indirect_vdev;
681 		srp->sr_prev_indirect_vdev = vd->vdev_id;
682 	}
683 	spa->spa_removing_phys.sr_state = state;
684 	spa->spa_removing_phys.sr_end_time = gethrestime_sec();
685 
686 	spa->spa_vdev_removal = NULL;
687 	spa_vdev_removal_destroy(svr);
688 
689 	spa_sync_removing_state(spa, tx);
690 
691 	vdev_config_dirty(spa->spa_root_vdev);
692 }
693 
694 static void
free_mapped_segment_cb(void * arg,uint64_t offset,uint64_t size)695 free_mapped_segment_cb(void *arg, uint64_t offset, uint64_t size)
696 {
697 	vdev_t *vd = arg;
698 	vdev_indirect_mark_obsolete(vd, offset, size);
699 	boolean_t checkpoint = B_FALSE;
700 	vdev_indirect_ops.vdev_op_remap(vd, offset, size,
701 	    metaslab_free_impl_cb, &checkpoint);
702 }
703 
704 /*
705  * On behalf of the removal thread, syncs an incremental bit more of
706  * the indirect mapping to disk and updates the in-memory mapping.
707  * Called as a sync task in every txg that the removal thread makes progress.
708  */
709 static void
vdev_mapping_sync(void * arg,dmu_tx_t * tx)710 vdev_mapping_sync(void *arg, dmu_tx_t *tx)
711 {
712 	spa_vdev_removal_t *svr = arg;
713 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
714 	vdev_t *vd = vdev_lookup_top(spa, svr->svr_vdev_id);
715 	vdev_indirect_config_t *vic = &vd->vdev_indirect_config;
716 	uint64_t txg = dmu_tx_get_txg(tx);
717 	vdev_indirect_mapping_t *vim = vd->vdev_indirect_mapping;
718 
719 	ASSERT(vic->vic_mapping_object != 0);
720 	ASSERT3U(txg, ==, spa_syncing_txg(spa));
721 
722 	vdev_indirect_mapping_add_entries(vim,
723 	    &svr->svr_new_segments[txg & TXG_MASK], tx);
724 	vdev_indirect_births_add_entry(vd->vdev_indirect_births,
725 	    vdev_indirect_mapping_max_offset(vim), dmu_tx_get_txg(tx), tx);
726 
727 	/*
728 	 * Free the copied data for anything that was freed while the
729 	 * mapping entries were in flight.
730 	 */
731 	mutex_enter(&svr->svr_lock);
732 	range_tree_vacate(svr->svr_frees[txg & TXG_MASK],
733 	    free_mapped_segment_cb, vd);
734 	ASSERT3U(svr->svr_max_offset_to_sync[txg & TXG_MASK], >=,
735 	    vdev_indirect_mapping_max_offset(vim));
736 	svr->svr_max_offset_to_sync[txg & TXG_MASK] = 0;
737 	mutex_exit(&svr->svr_lock);
738 
739 	spa_sync_removing_state(spa, tx);
740 }
741 
742 typedef struct vdev_copy_segment_arg {
743 	spa_t *vcsa_spa;
744 	dva_t *vcsa_dest_dva;
745 	uint64_t vcsa_txg;
746 	range_tree_t *vcsa_obsolete_segs;
747 } vdev_copy_segment_arg_t;
748 
749 static void
unalloc_seg(void * arg,uint64_t start,uint64_t size)750 unalloc_seg(void *arg, uint64_t start, uint64_t size)
751 {
752 	vdev_copy_segment_arg_t *vcsa = arg;
753 	spa_t *spa = vcsa->vcsa_spa;
754 	blkptr_t bp = { 0 };
755 
756 	BP_SET_BIRTH(&bp, TXG_INITIAL, TXG_INITIAL);
757 	BP_SET_LSIZE(&bp, size);
758 	BP_SET_PSIZE(&bp, size);
759 	BP_SET_COMPRESS(&bp, ZIO_COMPRESS_OFF);
760 	BP_SET_CHECKSUM(&bp, ZIO_CHECKSUM_OFF);
761 	BP_SET_TYPE(&bp, DMU_OT_NONE);
762 	BP_SET_LEVEL(&bp, 0);
763 	BP_SET_DEDUP(&bp, 0);
764 	BP_SET_BYTEORDER(&bp, ZFS_HOST_BYTEORDER);
765 
766 	DVA_SET_VDEV(&bp.blk_dva[0], DVA_GET_VDEV(vcsa->vcsa_dest_dva));
767 	DVA_SET_OFFSET(&bp.blk_dva[0],
768 	    DVA_GET_OFFSET(vcsa->vcsa_dest_dva) + start);
769 	DVA_SET_ASIZE(&bp.blk_dva[0], size);
770 
771 	zio_free(spa, vcsa->vcsa_txg, &bp);
772 }
773 
774 /*
775  * All reads and writes associated with a call to spa_vdev_copy_segment()
776  * are done.
777  */
778 static void
spa_vdev_copy_segment_done(zio_t * zio)779 spa_vdev_copy_segment_done(zio_t *zio)
780 {
781 	vdev_copy_segment_arg_t *vcsa = zio->io_private;
782 
783 	range_tree_vacate(vcsa->vcsa_obsolete_segs,
784 	    unalloc_seg, vcsa);
785 	range_tree_destroy(vcsa->vcsa_obsolete_segs);
786 	kmem_free(vcsa, sizeof (*vcsa));
787 
788 	spa_config_exit(zio->io_spa, SCL_STATE, zio->io_spa);
789 }
790 
791 /*
792  * The write of the new location is done.
793  */
794 static void
spa_vdev_copy_segment_write_done(zio_t * zio)795 spa_vdev_copy_segment_write_done(zio_t *zio)
796 {
797 	vdev_copy_arg_t *vca = zio->io_private;
798 
799 	abd_free(zio->io_abd);
800 
801 	mutex_enter(&vca->vca_lock);
802 	vca->vca_outstanding_bytes -= zio->io_size;
803 	cv_signal(&vca->vca_cv);
804 	mutex_exit(&vca->vca_lock);
805 }
806 
807 /*
808  * The read of the old location is done.  The parent zio is the write to
809  * the new location.  Allow it to start.
810  */
811 static void
spa_vdev_copy_segment_read_done(zio_t * zio)812 spa_vdev_copy_segment_read_done(zio_t *zio)
813 {
814 	zio_nowait(zio_unique_parent(zio));
815 }
816 
817 /*
818  * If the old and new vdevs are mirrors, we will read both sides of the old
819  * mirror, and write each copy to the corresponding side of the new mirror.
820  * If the old and new vdevs have a different number of children, we will do
821  * this as best as possible.  Since we aren't verifying checksums, this
822  * ensures that as long as there's a good copy of the data, we'll have a
823  * good copy after the removal, even if there's silent damage to one side
824  * of the mirror. If we're removing a mirror that has some silent damage,
825  * we'll have exactly the same damage in the new location (assuming that
826  * the new location is also a mirror).
827  *
828  * We accomplish this by creating a tree of zio_t's, with as many writes as
829  * there are "children" of the new vdev (a non-redundant vdev counts as one
830  * child, a 2-way mirror has 2 children, etc). Each write has an associated
831  * read from a child of the old vdev. Typically there will be the same
832  * number of children of the old and new vdevs.  However, if there are more
833  * children of the new vdev, some child(ren) of the old vdev will be issued
834  * multiple reads.  If there are more children of the old vdev, some copies
835  * will be dropped.
836  *
837  * For example, the tree of zio_t's for a 2-way mirror is:
838  *
839  *                            null
840  *                           /    \
841  *    write(new vdev, child 0)      write(new vdev, child 1)
842  *      |                             |
843  *    read(old vdev, child 0)       read(old vdev, child 1)
844  *
845  * Child zio's complete before their parents complete.  However, zio's
846  * created with zio_vdev_child_io() may be issued before their children
847  * complete.  In this case we need to make sure that the children (reads)
848  * complete before the parents (writes) are *issued*.  We do this by not
849  * calling zio_nowait() on each write until its corresponding read has
850  * completed.
851  *
852  * The spa_config_lock must be held while zio's created by
853  * zio_vdev_child_io() are in progress, to ensure that the vdev tree does
854  * not change (e.g. due to a concurrent "zpool attach/detach"). The "null"
855  * zio is needed to release the spa_config_lock after all the reads and
856  * writes complete. (Note that we can't grab the config lock for each read,
857  * because it is not reentrant - we could deadlock with a thread waiting
858  * for a write lock.)
859  */
860 static void
spa_vdev_copy_one_child(vdev_copy_arg_t * vca,zio_t * nzio,vdev_t * source_vd,uint64_t source_offset,vdev_t * dest_child_vd,uint64_t dest_offset,int dest_id,uint64_t size)861 spa_vdev_copy_one_child(vdev_copy_arg_t *vca, zio_t *nzio,
862     vdev_t *source_vd, uint64_t source_offset,
863     vdev_t *dest_child_vd, uint64_t dest_offset, int dest_id, uint64_t size)
864 {
865 	ASSERT3U(spa_config_held(nzio->io_spa, SCL_ALL, RW_READER), !=, 0);
866 
867 	mutex_enter(&vca->vca_lock);
868 	vca->vca_outstanding_bytes += size;
869 	mutex_exit(&vca->vca_lock);
870 
871 	abd_t *abd = abd_alloc_for_io(size, B_FALSE);
872 
873 	vdev_t *source_child_vd;
874 	if (source_vd->vdev_ops == &vdev_mirror_ops && dest_id != -1) {
875 		/*
876 		 * Source and dest are both mirrors.  Copy from the same
877 		 * child id as we are copying to (wrapping around if there
878 		 * are more dest children than source children).
879 		 */
880 		source_child_vd =
881 		    source_vd->vdev_child[dest_id % source_vd->vdev_children];
882 	} else {
883 		source_child_vd = source_vd;
884 	}
885 
886 	zio_t *write_zio = zio_vdev_child_io(nzio, NULL,
887 	    dest_child_vd, dest_offset, abd, size,
888 	    ZIO_TYPE_WRITE, ZIO_PRIORITY_REMOVAL,
889 	    ZIO_FLAG_CANFAIL,
890 	    spa_vdev_copy_segment_write_done, vca);
891 
892 	zio_nowait(zio_vdev_child_io(write_zio, NULL,
893 	    source_child_vd, source_offset, abd, size,
894 	    ZIO_TYPE_READ, ZIO_PRIORITY_REMOVAL,
895 	    ZIO_FLAG_CANFAIL,
896 	    spa_vdev_copy_segment_read_done, vca));
897 }
898 
899 /*
900  * Allocate a new location for this segment, and create the zio_t's to
901  * read from the old location and write to the new location.
902  */
903 static int
spa_vdev_copy_segment(vdev_t * vd,range_tree_t * segs,uint64_t maxalloc,uint64_t txg,vdev_copy_arg_t * vca,zio_alloc_list_t * zal)904 spa_vdev_copy_segment(vdev_t *vd, range_tree_t *segs,
905     uint64_t maxalloc, uint64_t txg,
906     vdev_copy_arg_t *vca, zio_alloc_list_t *zal)
907 {
908 	metaslab_group_t *mg = vd->vdev_mg;
909 	spa_t *spa = vd->vdev_spa;
910 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;
911 	vdev_indirect_mapping_entry_t *entry;
912 	dva_t dst = { 0 };
913 	uint64_t start = range_tree_min(segs);
914 
915 	ASSERT3U(maxalloc, <=, SPA_MAXBLOCKSIZE);
916 
917 	uint64_t size = range_tree_span(segs);
918 	if (range_tree_span(segs) > maxalloc) {
919 		/*
920 		 * We can't allocate all the segments.  Prefer to end
921 		 * the allocation at the end of a segment, thus avoiding
922 		 * additional split blocks.
923 		 */
924 		range_seg_max_t search;
925 		zfs_btree_index_t where;
926 		rs_set_start(&search, segs, start + maxalloc);
927 		rs_set_end(&search, segs, start + maxalloc);
928 		(void) zfs_btree_find(&segs->rt_root, &search, &where);
929 		range_seg_t *rs = zfs_btree_prev(&segs->rt_root, &where,
930 		    &where);
931 		if (rs != NULL) {
932 			size = rs_get_end(rs, segs) - start;
933 		} else {
934 			/*
935 			 * There are no segments that end before maxalloc.
936 			 * I.e. the first segment is larger than maxalloc,
937 			 * so we must split it.
938 			 */
939 			size = maxalloc;
940 		}
941 	}
942 	ASSERT3U(size, <=, maxalloc);
943 
944 	/*
945 	 * An allocation class might not have any remaining vdevs or space
946 	 */
947 	metaslab_class_t *mc = mg->mg_class;
948 	if (mc != spa_normal_class(spa) && mc->mc_groups <= 1)
949 		mc = spa_normal_class(spa);
950 	int error = metaslab_alloc_dva(spa, mc, size, &dst, 0, NULL, txg, 0,
951 	    zal, 0);
952 	if (error == ENOSPC && mc != spa_normal_class(spa)) {
953 		error = metaslab_alloc_dva(spa, spa_normal_class(spa), size,
954 		    &dst, 0, NULL, txg, 0, zal, 0);
955 	}
956 	if (error != 0)
957 		return (error);
958 
959 	/*
960 	 * Determine the ranges that are not actually needed.  Offsets are
961 	 * relative to the start of the range to be copied (i.e. relative to the
962 	 * local variable "start").
963 	 */
964 	range_tree_t *obsolete_segs = range_tree_create(NULL, RANGE_SEG64, NULL,
965 	    0, 0);
966 
967 	zfs_btree_index_t where;
968 	range_seg_t *rs = zfs_btree_first(&segs->rt_root, &where);
969 	ASSERT3U(rs_get_start(rs, segs), ==, start);
970 	uint64_t prev_seg_end = rs_get_end(rs, segs);
971 	while ((rs = zfs_btree_next(&segs->rt_root, &where, &where)) != NULL) {
972 		if (rs_get_start(rs, segs) >= start + size) {
973 			break;
974 		} else {
975 			range_tree_add(obsolete_segs,
976 			    prev_seg_end - start,
977 			    rs_get_start(rs, segs) - prev_seg_end);
978 		}
979 		prev_seg_end = rs_get_end(rs, segs);
980 	}
981 	/* We don't end in the middle of an obsolete range */
982 	ASSERT3U(start + size, <=, prev_seg_end);
983 
984 	range_tree_clear(segs, start, size);
985 
986 	/*
987 	 * We can't have any padding of the allocated size, otherwise we will
988 	 * misunderstand what's allocated, and the size of the mapping.
989 	 * The caller ensures this will be true by passing in a size that is
990 	 * aligned to the worst (highest) ashift in the pool.
991 	 */
992 	ASSERT3U(DVA_GET_ASIZE(&dst), ==, size);
993 
994 	entry = kmem_zalloc(sizeof (vdev_indirect_mapping_entry_t), KM_SLEEP);
995 	DVA_MAPPING_SET_SRC_OFFSET(&entry->vime_mapping, start);
996 	entry->vime_mapping.vimep_dst = dst;
997 	if (spa_feature_is_enabled(spa, SPA_FEATURE_OBSOLETE_COUNTS)) {
998 		entry->vime_obsolete_count = range_tree_space(obsolete_segs);
999 	}
1000 
1001 	vdev_copy_segment_arg_t *vcsa = kmem_zalloc(sizeof (*vcsa), KM_SLEEP);
1002 	vcsa->vcsa_dest_dva = &entry->vime_mapping.vimep_dst;
1003 	vcsa->vcsa_obsolete_segs = obsolete_segs;
1004 	vcsa->vcsa_spa = spa;
1005 	vcsa->vcsa_txg = txg;
1006 
1007 	/*
1008 	 * See comment before spa_vdev_copy_one_child().
1009 	 */
1010 	spa_config_enter(spa, SCL_STATE, spa, RW_READER);
1011 	zio_t *nzio = zio_null(spa->spa_txg_zio[txg & TXG_MASK], spa, NULL,
1012 	    spa_vdev_copy_segment_done, vcsa, 0);
1013 	vdev_t *dest_vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dst));
1014 	if (dest_vd->vdev_ops == &vdev_mirror_ops) {
1015 		for (int i = 0; i < dest_vd->vdev_children; i++) {
1016 			vdev_t *child = dest_vd->vdev_child[i];
1017 			spa_vdev_copy_one_child(vca, nzio, vd, start,
1018 			    child, DVA_GET_OFFSET(&dst), i, size);
1019 		}
1020 	} else {
1021 		spa_vdev_copy_one_child(vca, nzio, vd, start,
1022 		    dest_vd, DVA_GET_OFFSET(&dst), -1, size);
1023 	}
1024 	zio_nowait(nzio);
1025 
1026 	list_insert_tail(&svr->svr_new_segments[txg & TXG_MASK], entry);
1027 	ASSERT3U(start + size, <=, vd->vdev_ms_count << vd->vdev_ms_shift);
1028 	vdev_dirty(vd, 0, NULL, txg);
1029 
1030 	return (0);
1031 }
1032 
1033 /*
1034  * Complete the removal of a toplevel vdev. This is called as a
1035  * synctask in the same txg that we will sync out the new config (to the
1036  * MOS object) which indicates that this vdev is indirect.
1037  */
1038 static void
vdev_remove_complete_sync(void * arg,dmu_tx_t * tx)1039 vdev_remove_complete_sync(void *arg, dmu_tx_t *tx)
1040 {
1041 	spa_vdev_removal_t *svr = arg;
1042 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
1043 	vdev_t *vd = vdev_lookup_top(spa, svr->svr_vdev_id);
1044 
1045 	ASSERT3P(vd->vdev_ops, ==, &vdev_indirect_ops);
1046 
1047 	for (int i = 0; i < TXG_SIZE; i++) {
1048 		ASSERT0(svr->svr_bytes_done[i]);
1049 	}
1050 
1051 	ASSERT3U(spa->spa_removing_phys.sr_copied, ==,
1052 	    spa->spa_removing_phys.sr_to_copy);
1053 
1054 	vdev_destroy_spacemaps(vd, tx);
1055 
1056 	/* destroy leaf zaps, if any */
1057 	ASSERT3P(svr->svr_zaplist, !=, NULL);
1058 	for (nvpair_t *pair = nvlist_next_nvpair(svr->svr_zaplist, NULL);
1059 	    pair != NULL;
1060 	    pair = nvlist_next_nvpair(svr->svr_zaplist, pair)) {
1061 		vdev_destroy_unlink_zap(vd, fnvpair_value_uint64(pair), tx);
1062 	}
1063 	fnvlist_free(svr->svr_zaplist);
1064 
1065 	spa_finish_removal(dmu_tx_pool(tx)->dp_spa, DSS_FINISHED, tx);
1066 	/* vd->vdev_path is not available here */
1067 	spa_history_log_internal(spa, "vdev remove completed",  tx,
1068 	    "%s vdev %llu", spa_name(spa), vd->vdev_id);
1069 }
1070 
1071 static void
vdev_remove_enlist_zaps(vdev_t * vd,nvlist_t * zlist)1072 vdev_remove_enlist_zaps(vdev_t *vd, nvlist_t *zlist)
1073 {
1074 	ASSERT3P(zlist, !=, NULL);
1075 	ASSERT3P(vd->vdev_ops, !=, &vdev_raidz_ops);
1076 
1077 	if (vd->vdev_leaf_zap != 0) {
1078 		char zkey[32];
1079 		(void) snprintf(zkey, sizeof (zkey), "%s-%"PRIu64,
1080 		    VDEV_REMOVAL_ZAP_OBJS, vd->vdev_leaf_zap);
1081 		fnvlist_add_uint64(zlist, zkey, vd->vdev_leaf_zap);
1082 	}
1083 
1084 	for (uint64_t id = 0; id < vd->vdev_children; id++) {
1085 		vdev_remove_enlist_zaps(vd->vdev_child[id], zlist);
1086 	}
1087 }
1088 
1089 static void
vdev_remove_replace_with_indirect(vdev_t * vd,uint64_t txg)1090 vdev_remove_replace_with_indirect(vdev_t *vd, uint64_t txg)
1091 {
1092 	vdev_t *ivd;
1093 	dmu_tx_t *tx;
1094 	spa_t *spa = vd->vdev_spa;
1095 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;
1096 
1097 	/*
1098 	 * First, build a list of leaf zaps to be destroyed.
1099 	 * This is passed to the sync context thread,
1100 	 * which does the actual unlinking.
1101 	 */
1102 	svr->svr_zaplist = fnvlist_alloc();
1103 	vdev_remove_enlist_zaps(vd, svr->svr_zaplist);
1104 
1105 	ivd = vdev_add_parent(vd, &vdev_indirect_ops);
1106 	ivd->vdev_removing = 0;
1107 
1108 	vd->vdev_leaf_zap = 0;
1109 
1110 	vdev_remove_child(ivd, vd);
1111 	vdev_compact_children(ivd);
1112 
1113 	ASSERT(!list_link_active(&vd->vdev_state_dirty_node));
1114 
1115 	tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
1116 	dsl_sync_task_nowait(spa->spa_dsl_pool, vdev_remove_complete_sync, svr,
1117 	    0, ZFS_SPACE_CHECK_NONE, tx);
1118 	dmu_tx_commit(tx);
1119 
1120 	/*
1121 	 * Indicate that this thread has exited.
1122 	 * After this, we can not use svr.
1123 	 */
1124 	mutex_enter(&svr->svr_lock);
1125 	svr->svr_thread = NULL;
1126 	cv_broadcast(&svr->svr_cv);
1127 	mutex_exit(&svr->svr_lock);
1128 }
1129 
1130 /*
1131  * Complete the removal of a toplevel vdev. This is called in open
1132  * context by the removal thread after we have copied all vdev's data.
1133  */
1134 static void
vdev_remove_complete(spa_t * spa)1135 vdev_remove_complete(spa_t *spa)
1136 {
1137 	uint64_t txg;
1138 
1139 	/*
1140 	 * Wait for any deferred frees to be synced before we call
1141 	 * vdev_metaslab_fini()
1142 	 */
1143 	txg_wait_synced(spa->spa_dsl_pool, 0);
1144 	txg = spa_vdev_enter(spa);
1145 	vdev_t *vd = vdev_lookup_top(spa, spa->spa_vdev_removal->svr_vdev_id);
1146 	ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
1147 	ASSERT3P(vd->vdev_trim_thread, ==, NULL);
1148 	ASSERT3P(vd->vdev_autotrim_thread, ==, NULL);
1149 
1150 	sysevent_t *ev = spa_event_create(spa, vd, NULL,
1151 	    ESC_ZFS_VDEV_REMOVE_DEV);
1152 
1153 	zfs_dbgmsg("finishing device removal for vdev %llu in txg %llu",
1154 	    vd->vdev_id, txg);
1155 
1156 	/*
1157 	 * Discard allocation state.
1158 	 */
1159 	if (vd->vdev_mg != NULL) {
1160 		vdev_metaslab_fini(vd);
1161 		metaslab_group_destroy(vd->vdev_mg);
1162 		vd->vdev_mg = NULL;
1163 		spa_log_sm_set_blocklimit(spa);
1164 	}
1165 	ASSERT0(vd->vdev_stat.vs_space);
1166 	ASSERT0(vd->vdev_stat.vs_dspace);
1167 
1168 	vdev_remove_replace_with_indirect(vd, txg);
1169 
1170 	/*
1171 	 * We now release the locks, allowing spa_sync to run and finish the
1172 	 * removal via vdev_remove_complete_sync in syncing context.
1173 	 *
1174 	 * Note that we hold on to the vdev_t that has been replaced.  Since
1175 	 * it isn't part of the vdev tree any longer, it can't be concurrently
1176 	 * manipulated, even while we don't have the config lock.
1177 	 */
1178 	(void) spa_vdev_exit(spa, NULL, txg, 0);
1179 
1180 	/*
1181 	 * Top ZAP should have been transferred to the indirect vdev in
1182 	 * vdev_remove_replace_with_indirect.
1183 	 */
1184 	ASSERT0(vd->vdev_top_zap);
1185 
1186 	/*
1187 	 * Leaf ZAP should have been moved in vdev_remove_replace_with_indirect.
1188 	 */
1189 	ASSERT0(vd->vdev_leaf_zap);
1190 
1191 	txg = spa_vdev_enter(spa);
1192 	(void) vdev_label_init(vd, 0, VDEV_LABEL_REMOVE);
1193 	/*
1194 	 * Request to update the config and the config cachefile.
1195 	 */
1196 	vdev_config_dirty(spa->spa_root_vdev);
1197 	(void) spa_vdev_exit(spa, vd, txg, 0);
1198 
1199 	spa_event_post(ev);
1200 }
1201 
1202 /*
1203  * Evacuates a segment of size at most max_alloc from the vdev
1204  * via repeated calls to spa_vdev_copy_segment. If an allocation
1205  * fails, the pool is probably too fragmented to handle such a
1206  * large size, so decrease max_alloc so that the caller will not try
1207  * this size again this txg.
1208  */
1209 static void
spa_vdev_copy_impl(vdev_t * vd,spa_vdev_removal_t * svr,vdev_copy_arg_t * vca,uint64_t * max_alloc,dmu_tx_t * tx)1210 spa_vdev_copy_impl(vdev_t *vd, spa_vdev_removal_t *svr, vdev_copy_arg_t *vca,
1211     uint64_t *max_alloc, dmu_tx_t *tx)
1212 {
1213 	uint64_t txg = dmu_tx_get_txg(tx);
1214 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
1215 
1216 	mutex_enter(&svr->svr_lock);
1217 
1218 	/*
1219 	 * Determine how big of a chunk to copy.  We can allocate up
1220 	 * to max_alloc bytes, and we can span up to vdev_removal_max_span
1221 	 * bytes of unallocated space at a time.  "segs" will track the
1222 	 * allocated segments that we are copying.  We may also be copying
1223 	 * free segments (of up to vdev_removal_max_span bytes).
1224 	 */
1225 	range_tree_t *segs = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);
1226 	for (;;) {
1227 		range_tree_t *rt = svr->svr_allocd_segs;
1228 		range_seg_t *rs = range_tree_first(rt);
1229 
1230 		if (rs == NULL)
1231 			break;
1232 
1233 		uint64_t seg_length;
1234 
1235 		if (range_tree_is_empty(segs)) {
1236 			/* need to truncate the first seg based on max_alloc */
1237 			seg_length = MIN(rs_get_end(rs, rt) - rs_get_start(rs,
1238 			    rt), *max_alloc);
1239 		} else {
1240 			if (rs_get_start(rs, rt) - range_tree_max(segs) >
1241 			    vdev_removal_max_span) {
1242 				/*
1243 				 * Including this segment would cause us to
1244 				 * copy a larger unneeded chunk than is allowed.
1245 				 */
1246 				break;
1247 			} else if (rs_get_end(rs, rt) - range_tree_min(segs) >
1248 			    *max_alloc) {
1249 				/*
1250 				 * This additional segment would extend past
1251 				 * max_alloc. Rather than splitting this
1252 				 * segment, leave it for the next mapping.
1253 				 */
1254 				break;
1255 			} else {
1256 				seg_length = rs_get_end(rs, rt) -
1257 				    rs_get_start(rs, rt);
1258 			}
1259 		}
1260 
1261 		range_tree_add(segs, rs_get_start(rs, rt), seg_length);
1262 		range_tree_remove(svr->svr_allocd_segs,
1263 		    rs_get_start(rs, rt), seg_length);
1264 	}
1265 
1266 	if (range_tree_is_empty(segs)) {
1267 		mutex_exit(&svr->svr_lock);
1268 		range_tree_destroy(segs);
1269 		return;
1270 	}
1271 
1272 	if (svr->svr_max_offset_to_sync[txg & TXG_MASK] == 0) {
1273 		dsl_sync_task_nowait(dmu_tx_pool(tx), vdev_mapping_sync,
1274 		    svr, 0, ZFS_SPACE_CHECK_NONE, tx);
1275 	}
1276 
1277 	svr->svr_max_offset_to_sync[txg & TXG_MASK] = range_tree_max(segs);
1278 
1279 	/*
1280 	 * Note: this is the amount of *allocated* space
1281 	 * that we are taking care of each txg.
1282 	 */
1283 	svr->svr_bytes_done[txg & TXG_MASK] += range_tree_space(segs);
1284 
1285 	mutex_exit(&svr->svr_lock);
1286 
1287 	zio_alloc_list_t zal;
1288 	metaslab_trace_init(&zal);
1289 	uint64_t thismax = SPA_MAXBLOCKSIZE;
1290 	while (!range_tree_is_empty(segs)) {
1291 		int error = spa_vdev_copy_segment(vd,
1292 		    segs, thismax, txg, vca, &zal);
1293 
1294 		if (error == ENOSPC) {
1295 			/*
1296 			 * Cut our segment in half, and don't try this
1297 			 * segment size again this txg.  Note that the
1298 			 * allocation size must be aligned to the highest
1299 			 * ashift in the pool, so that the allocation will
1300 			 * not be padded out to a multiple of the ashift,
1301 			 * which could cause us to think that this mapping
1302 			 * is larger than we intended.
1303 			 */
1304 			ASSERT3U(spa->spa_max_ashift, >=, SPA_MINBLOCKSHIFT);
1305 			ASSERT3U(spa->spa_max_ashift, ==, spa->spa_min_ashift);
1306 			uint64_t attempted =
1307 			    MIN(range_tree_span(segs), thismax);
1308 			thismax = P2ROUNDUP(attempted / 2,
1309 			    1 << spa->spa_max_ashift);
1310 			/*
1311 			 * The minimum-size allocation can not fail.
1312 			 */
1313 			ASSERT3U(attempted, >, 1 << spa->spa_max_ashift);
1314 			*max_alloc = attempted - (1 << spa->spa_max_ashift);
1315 		} else {
1316 			ASSERT0(error);
1317 
1318 			/*
1319 			 * We've performed an allocation, so reset the
1320 			 * alloc trace list.
1321 			 */
1322 			metaslab_trace_fini(&zal);
1323 			metaslab_trace_init(&zal);
1324 		}
1325 	}
1326 	metaslab_trace_fini(&zal);
1327 	range_tree_destroy(segs);
1328 }
1329 
1330 /*
1331  * The removal thread operates in open context.  It iterates over all
1332  * allocated space in the vdev, by loading each metaslab's spacemap.
1333  * For each contiguous segment of allocated space (capping the segment
1334  * size at SPA_MAXBLOCKSIZE), we:
1335  *    - Allocate space for it on another vdev.
1336  *    - Create a new mapping from the old location to the new location
1337  *      (as a record in svr_new_segments).
1338  *    - Initiate a logical read zio to get the data off the removing disk.
1339  *    - In the read zio's done callback, initiate a logical write zio to
1340  *      write it to the new vdev.
1341  * Note that all of this will take effect when a particular TXG syncs.
1342  * The sync thread ensures that all the phys reads and writes for the syncing
1343  * TXG have completed (see spa_txg_zio) and writes the new mappings to disk
1344  * (see vdev_mapping_sync()).
1345  */
1346 static void
spa_vdev_remove_thread(void * arg)1347 spa_vdev_remove_thread(void *arg)
1348 {
1349 	spa_t *spa = arg;
1350 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;
1351 	vdev_copy_arg_t vca;
1352 	uint64_t max_alloc = zfs_remove_max_segment;
1353 	uint64_t last_txg = 0;
1354 
1355 	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1356 	vdev_t *vd = vdev_lookup_top(spa, svr->svr_vdev_id);
1357 	vdev_indirect_mapping_t *vim = vd->vdev_indirect_mapping;
1358 	uint64_t start_offset = vdev_indirect_mapping_max_offset(vim);
1359 
1360 	ASSERT3P(vd->vdev_ops, !=, &vdev_indirect_ops);
1361 	ASSERT(vdev_is_concrete(vd));
1362 	ASSERT(vd->vdev_removing);
1363 	ASSERT(vd->vdev_indirect_config.vic_mapping_object != 0);
1364 	ASSERT(vim != NULL);
1365 
1366 	mutex_init(&vca.vca_lock, NULL, MUTEX_DEFAULT, NULL);
1367 	cv_init(&vca.vca_cv, NULL, CV_DEFAULT, NULL);
1368 	vca.vca_outstanding_bytes = 0;
1369 
1370 	mutex_enter(&svr->svr_lock);
1371 
1372 	/*
1373 	 * Start from vim_max_offset so we pick up where we left off
1374 	 * if we are restarting the removal after opening the pool.
1375 	 */
1376 	uint64_t msi;
1377 	for (msi = start_offset >> vd->vdev_ms_shift;
1378 	    msi < vd->vdev_ms_count && !svr->svr_thread_exit; msi++) {
1379 		metaslab_t *msp = vd->vdev_ms[msi];
1380 		ASSERT3U(msi, <=, vd->vdev_ms_count);
1381 
1382 		ASSERT0(range_tree_space(svr->svr_allocd_segs));
1383 
1384 		mutex_enter(&msp->ms_sync_lock);
1385 		mutex_enter(&msp->ms_lock);
1386 
1387 		/*
1388 		 * Assert nothing in flight -- ms_*tree is empty.
1389 		 */
1390 		for (int i = 0; i < TXG_SIZE; i++) {
1391 			ASSERT0(range_tree_space(msp->ms_allocating[i]));
1392 		}
1393 
1394 		/*
1395 		 * If the metaslab has ever been allocated from (ms_sm!=NULL),
1396 		 * read the allocated segments from the space map object
1397 		 * into svr_allocd_segs. Since we do this while holding
1398 		 * svr_lock and ms_sync_lock, concurrent frees (which
1399 		 * would have modified the space map) will wait for us
1400 		 * to finish loading the spacemap, and then take the
1401 		 * appropriate action (see free_from_removing_vdev()).
1402 		 */
1403 		if (msp->ms_sm != NULL) {
1404 			VERIFY0(space_map_load(msp->ms_sm,
1405 			    svr->svr_allocd_segs, SM_ALLOC));
1406 
1407 			range_tree_walk(msp->ms_unflushed_allocs,
1408 			    range_tree_add, svr->svr_allocd_segs);
1409 			range_tree_walk(msp->ms_unflushed_frees,
1410 			    range_tree_remove, svr->svr_allocd_segs);
1411 			range_tree_walk(msp->ms_freeing,
1412 			    range_tree_remove, svr->svr_allocd_segs);
1413 
1414 			/*
1415 			 * When we are resuming from a paused removal (i.e.
1416 			 * when importing a pool with a removal in progress),
1417 			 * discard any state that we have already processed.
1418 			 */
1419 			range_tree_clear(svr->svr_allocd_segs, 0, start_offset);
1420 		}
1421 		mutex_exit(&msp->ms_lock);
1422 		mutex_exit(&msp->ms_sync_lock);
1423 
1424 		vca.vca_msp = msp;
1425 		zfs_dbgmsg("copying %llu segments for metaslab %llu",
1426 		    zfs_btree_numnodes(&svr->svr_allocd_segs->rt_root),
1427 		    msp->ms_id);
1428 
1429 		while (!svr->svr_thread_exit &&
1430 		    !range_tree_is_empty(svr->svr_allocd_segs)) {
1431 
1432 			mutex_exit(&svr->svr_lock);
1433 
1434 			/*
1435 			 * We need to periodically drop the config lock so that
1436 			 * writers can get in.  Additionally, we can't wait
1437 			 * for a txg to sync while holding a config lock
1438 			 * (since a waiting writer could cause a 3-way deadlock
1439 			 * with the sync thread, which also gets a config
1440 			 * lock for reader).  So we can't hold the config lock
1441 			 * while calling dmu_tx_assign().
1442 			 */
1443 			spa_config_exit(spa, SCL_CONFIG, FTAG);
1444 
1445 			/*
1446 			 * This delay will pause the removal around the point
1447 			 * specified by zfs_removal_suspend_progress. We do this
1448 			 * solely from the test suite or during debugging.
1449 			 */
1450 			uint64_t bytes_copied =
1451 			    spa->spa_removing_phys.sr_copied;
1452 			for (int i = 0; i < TXG_SIZE; i++)
1453 				bytes_copied += svr->svr_bytes_done[i];
1454 			while (zfs_removal_suspend_progress &&
1455 			    !svr->svr_thread_exit)
1456 				delay(hz);
1457 
1458 			mutex_enter(&vca.vca_lock);
1459 			while (vca.vca_outstanding_bytes >
1460 			    zfs_remove_max_copy_bytes) {
1461 				cv_wait(&vca.vca_cv, &vca.vca_lock);
1462 			}
1463 			mutex_exit(&vca.vca_lock);
1464 
1465 			dmu_tx_t *tx =
1466 			    dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
1467 
1468 			VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
1469 			uint64_t txg = dmu_tx_get_txg(tx);
1470 
1471 			/*
1472 			 * Reacquire the vdev_config lock.  The vdev_t
1473 			 * that we're removing may have changed, e.g. due
1474 			 * to a vdev_attach or vdev_detach.
1475 			 */
1476 			spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1477 			vd = vdev_lookup_top(spa, svr->svr_vdev_id);
1478 
1479 			if (txg != last_txg)
1480 				max_alloc = zfs_remove_max_segment;
1481 			last_txg = txg;
1482 
1483 			spa_vdev_copy_impl(vd, svr, &vca, &max_alloc, tx);
1484 
1485 			dmu_tx_commit(tx);
1486 			mutex_enter(&svr->svr_lock);
1487 		}
1488 	}
1489 
1490 	mutex_exit(&svr->svr_lock);
1491 
1492 	spa_config_exit(spa, SCL_CONFIG, FTAG);
1493 
1494 	/*
1495 	 * Wait for all copies to finish before cleaning up the vca.
1496 	 */
1497 	txg_wait_synced(spa->spa_dsl_pool, 0);
1498 	ASSERT0(vca.vca_outstanding_bytes);
1499 
1500 	mutex_destroy(&vca.vca_lock);
1501 	cv_destroy(&vca.vca_cv);
1502 
1503 	if (svr->svr_thread_exit) {
1504 		mutex_enter(&svr->svr_lock);
1505 		range_tree_vacate(svr->svr_allocd_segs, NULL, NULL);
1506 		svr->svr_thread = NULL;
1507 		cv_broadcast(&svr->svr_cv);
1508 		mutex_exit(&svr->svr_lock);
1509 	} else {
1510 		ASSERT0(range_tree_space(svr->svr_allocd_segs));
1511 		vdev_remove_complete(spa);
1512 	}
1513 }
1514 
1515 void
spa_vdev_remove_suspend(spa_t * spa)1516 spa_vdev_remove_suspend(spa_t *spa)
1517 {
1518 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;
1519 
1520 	if (svr == NULL)
1521 		return;
1522 
1523 	mutex_enter(&svr->svr_lock);
1524 	svr->svr_thread_exit = B_TRUE;
1525 	while (svr->svr_thread != NULL)
1526 		cv_wait(&svr->svr_cv, &svr->svr_lock);
1527 	svr->svr_thread_exit = B_FALSE;
1528 	mutex_exit(&svr->svr_lock);
1529 }
1530 
1531 /* ARGSUSED */
1532 static int
spa_vdev_remove_cancel_check(void * arg,dmu_tx_t * tx)1533 spa_vdev_remove_cancel_check(void *arg, dmu_tx_t *tx)
1534 {
1535 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
1536 
1537 	if (spa->spa_vdev_removal == NULL)
1538 		return (ENOTACTIVE);
1539 	return (0);
1540 }
1541 
1542 /*
1543  * Cancel a removal by freeing all entries from the partial mapping
1544  * and marking the vdev as no longer being removing.
1545  */
1546 /* ARGSUSED */
1547 static void
spa_vdev_remove_cancel_sync(void * arg,dmu_tx_t * tx)1548 spa_vdev_remove_cancel_sync(void *arg, dmu_tx_t *tx)
1549 {
1550 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
1551 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;
1552 	vdev_t *vd = vdev_lookup_top(spa, svr->svr_vdev_id);
1553 	vdev_indirect_config_t *vic = &vd->vdev_indirect_config;
1554 	vdev_indirect_mapping_t *vim = vd->vdev_indirect_mapping;
1555 	objset_t *mos = spa->spa_meta_objset;
1556 
1557 	ASSERT3P(svr->svr_thread, ==, NULL);
1558 
1559 	spa_feature_decr(spa, SPA_FEATURE_DEVICE_REMOVAL, tx);
1560 	if (vdev_obsolete_counts_are_precise(vd)) {
1561 		spa_feature_decr(spa, SPA_FEATURE_OBSOLETE_COUNTS, tx);
1562 		VERIFY0(zap_remove(spa->spa_meta_objset, vd->vdev_top_zap,
1563 		    VDEV_TOP_ZAP_OBSOLETE_COUNTS_ARE_PRECISE, tx));
1564 	}
1565 
1566 	if (vdev_obsolete_sm_object(vd) != 0) {
1567 		ASSERT(vd->vdev_obsolete_sm != NULL);
1568 		ASSERT3U(vdev_obsolete_sm_object(vd), ==,
1569 		    space_map_object(vd->vdev_obsolete_sm));
1570 
1571 		space_map_free(vd->vdev_obsolete_sm, tx);
1572 		VERIFY0(zap_remove(spa->spa_meta_objset, vd->vdev_top_zap,
1573 		    VDEV_TOP_ZAP_INDIRECT_OBSOLETE_SM, tx));
1574 		space_map_close(vd->vdev_obsolete_sm);
1575 		vd->vdev_obsolete_sm = NULL;
1576 		spa_feature_decr(spa, SPA_FEATURE_OBSOLETE_COUNTS, tx);
1577 	}
1578 	for (int i = 0; i < TXG_SIZE; i++) {
1579 		ASSERT(list_is_empty(&svr->svr_new_segments[i]));
1580 		ASSERT3U(svr->svr_max_offset_to_sync[i], <=,
1581 		    vdev_indirect_mapping_max_offset(vim));
1582 	}
1583 
1584 	for (uint64_t msi = 0; msi < vd->vdev_ms_count; msi++) {
1585 		metaslab_t *msp = vd->vdev_ms[msi];
1586 
1587 		if (msp->ms_start >= vdev_indirect_mapping_max_offset(vim))
1588 			break;
1589 
1590 		ASSERT0(range_tree_space(svr->svr_allocd_segs));
1591 
1592 		mutex_enter(&msp->ms_lock);
1593 
1594 		/*
1595 		 * Assert nothing in flight -- ms_*tree is empty.
1596 		 */
1597 		for (int i = 0; i < TXG_SIZE; i++)
1598 			ASSERT0(range_tree_space(msp->ms_allocating[i]));
1599 		for (int i = 0; i < TXG_DEFER_SIZE; i++)
1600 			ASSERT0(range_tree_space(msp->ms_defer[i]));
1601 		ASSERT0(range_tree_space(msp->ms_freed));
1602 
1603 		if (msp->ms_sm != NULL) {
1604 			mutex_enter(&svr->svr_lock);
1605 			VERIFY0(space_map_load(msp->ms_sm,
1606 			    svr->svr_allocd_segs, SM_ALLOC));
1607 
1608 			range_tree_walk(msp->ms_unflushed_allocs,
1609 			    range_tree_add, svr->svr_allocd_segs);
1610 			range_tree_walk(msp->ms_unflushed_frees,
1611 			    range_tree_remove, svr->svr_allocd_segs);
1612 			range_tree_walk(msp->ms_freeing,
1613 			    range_tree_remove, svr->svr_allocd_segs);
1614 
1615 			/*
1616 			 * Clear everything past what has been synced,
1617 			 * because we have not allocated mappings for it yet.
1618 			 */
1619 			uint64_t syncd = vdev_indirect_mapping_max_offset(vim);
1620 			uint64_t sm_end = msp->ms_sm->sm_start +
1621 			    msp->ms_sm->sm_size;
1622 			if (sm_end > syncd)
1623 				range_tree_clear(svr->svr_allocd_segs,
1624 				    syncd, sm_end - syncd);
1625 
1626 			mutex_exit(&svr->svr_lock);
1627 		}
1628 		mutex_exit(&msp->ms_lock);
1629 
1630 		mutex_enter(&svr->svr_lock);
1631 		range_tree_vacate(svr->svr_allocd_segs,
1632 		    free_mapped_segment_cb, vd);
1633 		mutex_exit(&svr->svr_lock);
1634 	}
1635 
1636 	/*
1637 	 * Note: this must happen after we invoke free_mapped_segment_cb,
1638 	 * because it adds to the obsolete_segments.
1639 	 */
1640 	range_tree_vacate(vd->vdev_obsolete_segments, NULL, NULL);
1641 
1642 	ASSERT3U(vic->vic_mapping_object, ==,
1643 	    vdev_indirect_mapping_object(vd->vdev_indirect_mapping));
1644 	vdev_indirect_mapping_close(vd->vdev_indirect_mapping);
1645 	vd->vdev_indirect_mapping = NULL;
1646 	vdev_indirect_mapping_free(mos, vic->vic_mapping_object, tx);
1647 	vic->vic_mapping_object = 0;
1648 
1649 	ASSERT3U(vic->vic_births_object, ==,
1650 	    vdev_indirect_births_object(vd->vdev_indirect_births));
1651 	vdev_indirect_births_close(vd->vdev_indirect_births);
1652 	vd->vdev_indirect_births = NULL;
1653 	vdev_indirect_births_free(mos, vic->vic_births_object, tx);
1654 	vic->vic_births_object = 0;
1655 
1656 	/*
1657 	 * We may have processed some frees from the removing vdev in this
1658 	 * txg, thus increasing svr_bytes_done; discard that here to
1659 	 * satisfy the assertions in spa_vdev_removal_destroy().
1660 	 * Note that future txg's can not have any bytes_done, because
1661 	 * future TXG's are only modified from open context, and we have
1662 	 * already shut down the copying thread.
1663 	 */
1664 	svr->svr_bytes_done[dmu_tx_get_txg(tx) & TXG_MASK] = 0;
1665 	spa_finish_removal(spa, DSS_CANCELED, tx);
1666 
1667 	vd->vdev_removing = B_FALSE;
1668 	vdev_config_dirty(vd);
1669 
1670 	zfs_dbgmsg("canceled device removal for vdev %llu in %llu",
1671 	    vd->vdev_id, dmu_tx_get_txg(tx));
1672 	spa_history_log_internal(spa, "vdev remove canceled", tx,
1673 	    "%s vdev %llu %s", spa_name(spa),
1674 	    vd->vdev_id, (vd->vdev_path != NULL) ? vd->vdev_path : "-");
1675 }
1676 
1677 int
spa_vdev_remove_cancel(spa_t * spa)1678 spa_vdev_remove_cancel(spa_t *spa)
1679 {
1680 	spa_vdev_remove_suspend(spa);
1681 
1682 	if (spa->spa_vdev_removal == NULL)
1683 		return (ENOTACTIVE);
1684 
1685 	uint64_t vdid = spa->spa_vdev_removal->svr_vdev_id;
1686 
1687 	int error = dsl_sync_task(spa->spa_name, spa_vdev_remove_cancel_check,
1688 	    spa_vdev_remove_cancel_sync, NULL, 0,
1689 	    ZFS_SPACE_CHECK_EXTRA_RESERVED);
1690 
1691 	if (error == 0) {
1692 		spa_config_enter(spa, SCL_ALLOC | SCL_VDEV, FTAG, RW_WRITER);
1693 		vdev_t *vd = vdev_lookup_top(spa, vdid);
1694 		metaslab_group_activate(vd->vdev_mg);
1695 		spa_config_exit(spa, SCL_ALLOC | SCL_VDEV, FTAG);
1696 	}
1697 
1698 	return (error);
1699 }
1700 
1701 void
svr_sync(spa_t * spa,dmu_tx_t * tx)1702 svr_sync(spa_t *spa, dmu_tx_t *tx)
1703 {
1704 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;
1705 	int txgoff = dmu_tx_get_txg(tx) & TXG_MASK;
1706 
1707 	if (svr == NULL)
1708 		return;
1709 
1710 	/*
1711 	 * This check is necessary so that we do not dirty the
1712 	 * DIRECTORY_OBJECT via spa_sync_removing_state() when there
1713 	 * is nothing to do.  Dirtying it every time would prevent us
1714 	 * from syncing-to-convergence.
1715 	 */
1716 	if (svr->svr_bytes_done[txgoff] == 0)
1717 		return;
1718 
1719 	/*
1720 	 * Update progress accounting.
1721 	 */
1722 	spa->spa_removing_phys.sr_copied += svr->svr_bytes_done[txgoff];
1723 	svr->svr_bytes_done[txgoff] = 0;
1724 
1725 	spa_sync_removing_state(spa, tx);
1726 }
1727 
1728 static void
vdev_remove_make_hole_and_free(vdev_t * vd)1729 vdev_remove_make_hole_and_free(vdev_t *vd)
1730 {
1731 	uint64_t id = vd->vdev_id;
1732 	spa_t *spa = vd->vdev_spa;
1733 	vdev_t *rvd = spa->spa_root_vdev;
1734 
1735 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1736 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
1737 
1738 	vdev_free(vd);
1739 
1740 	vd = vdev_alloc_common(spa, id, 0, &vdev_hole_ops);
1741 	vdev_add_child(rvd, vd);
1742 	vdev_config_dirty(rvd);
1743 
1744 	/*
1745 	 * Reassess the health of our root vdev.
1746 	 */
1747 	vdev_reopen(rvd);
1748 }
1749 
1750 /*
1751  * Remove a log device.  The config lock is held for the specified TXG.
1752  */
1753 static int
spa_vdev_remove_log(vdev_t * vd,uint64_t * txg)1754 spa_vdev_remove_log(vdev_t *vd, uint64_t *txg)
1755 {
1756 	metaslab_group_t *mg = vd->vdev_mg;
1757 	spa_t *spa = vd->vdev_spa;
1758 	int error = 0;
1759 
1760 	ASSERT(vd->vdev_islog);
1761 	ASSERT(vd == vd->vdev_top);
1762 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1763 
1764 	/*
1765 	 * Stop allocating from this vdev.
1766 	 */
1767 	metaslab_group_passivate(mg);
1768 
1769 	/*
1770 	 * Wait for the youngest allocations and frees to sync,
1771 	 * and then wait for the deferral of those frees to finish.
1772 	 */
1773 	spa_vdev_config_exit(spa, NULL,
1774 	    *txg + TXG_CONCURRENT_STATES + TXG_DEFER_SIZE, 0, FTAG);
1775 
1776 	/*
1777 	 * Evacuate the device.  We don't hold the config lock as
1778 	 * writer since we need to do I/O but we do keep the
1779 	 * spa_namespace_lock held.  Once this completes the device
1780 	 * should no longer have any blocks allocated on it.
1781 	 */
1782 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1783 	if (vd->vdev_stat.vs_alloc != 0)
1784 		error = spa_reset_logs(spa);
1785 
1786 	*txg = spa_vdev_config_enter(spa);
1787 
1788 	if (error != 0) {
1789 		metaslab_group_activate(mg);
1790 		return (error);
1791 	}
1792 	ASSERT0(vd->vdev_stat.vs_alloc);
1793 
1794 	/*
1795 	 * The evacuation succeeded.  Remove any remaining MOS metadata
1796 	 * associated with this vdev, and wait for these changes to sync.
1797 	 */
1798 	vd->vdev_removing = B_TRUE;
1799 
1800 	vdev_dirty_leaves(vd, VDD_DTL, *txg);
1801 	vdev_config_dirty(vd);
1802 
1803 	/*
1804 	 * When the log space map feature is enabled we look at
1805 	 * the vdev's top_zap to find the on-disk flush data of
1806 	 * the metaslab we just flushed. Thus, while removing a
1807 	 * log vdev we make sure to call vdev_metaslab_fini()
1808 	 * first, which removes all metaslabs of this vdev from
1809 	 * spa_metaslabs_by_flushed before vdev_remove_empty()
1810 	 * destroys the top_zap of this log vdev.
1811 	 *
1812 	 * This avoids the scenario where we flush a metaslab
1813 	 * from the log vdev being removed that doesn't have a
1814 	 * top_zap and end up failing to lookup its on-disk flush
1815 	 * data.
1816 	 *
1817 	 * We don't call metaslab_group_destroy() right away
1818 	 * though (it will be called in vdev_free() later) as
1819 	 * during metaslab_sync() of metaslabs from other vdevs
1820 	 * we may touch the metaslab group of this vdev through
1821 	 * metaslab_class_histogram_verify()
1822 	 */
1823 	vdev_metaslab_fini(vd);
1824 	spa_log_sm_set_blocklimit(spa);
1825 
1826 	spa_history_log_internal(spa, "vdev remove", NULL,
1827 	    "%s vdev %llu (log) %s", spa_name(spa), vd->vdev_id,
1828 	    (vd->vdev_path != NULL) ? vd->vdev_path : "-");
1829 
1830 	/* Make sure these changes are sync'ed */
1831 	spa_vdev_config_exit(spa, NULL, *txg, 0, FTAG);
1832 
1833 	/* Stop initializing and TRIM */
1834 	vdev_initialize_stop_all(vd, VDEV_INITIALIZE_CANCELED);
1835 	vdev_trim_stop_all(vd, VDEV_TRIM_CANCELED);
1836 	vdev_autotrim_stop_wait(vd);
1837 
1838 	*txg = spa_vdev_config_enter(spa);
1839 
1840 	sysevent_t *ev = spa_event_create(spa, vd, NULL,
1841 	    ESC_ZFS_VDEV_REMOVE_DEV);
1842 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1843 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
1844 
1845 	/* The top ZAP should have been destroyed by vdev_remove_empty. */
1846 	ASSERT0(vd->vdev_top_zap);
1847 	/* The leaf ZAP should have been destroyed by vdev_dtl_sync. */
1848 	ASSERT0(vd->vdev_leaf_zap);
1849 
1850 	(void) vdev_label_init(vd, 0, VDEV_LABEL_REMOVE);
1851 
1852 	if (list_link_active(&vd->vdev_state_dirty_node))
1853 		vdev_state_clean(vd);
1854 	if (list_link_active(&vd->vdev_config_dirty_node))
1855 		vdev_config_clean(vd);
1856 
1857 	ASSERT0(vd->vdev_stat.vs_alloc);
1858 
1859 	/*
1860 	 * Clean up the vdev namespace.
1861 	 */
1862 	vdev_remove_make_hole_and_free(vd);
1863 
1864 	if (ev != NULL)
1865 		spa_event_post(ev);
1866 
1867 	return (0);
1868 }
1869 
1870 static int
spa_vdev_remove_top_check(vdev_t * vd)1871 spa_vdev_remove_top_check(vdev_t *vd)
1872 {
1873 	spa_t *spa = vd->vdev_spa;
1874 
1875 	if (vd != vd->vdev_top)
1876 		return (SET_ERROR(ENOTSUP));
1877 
1878 	if (!spa_feature_is_enabled(spa, SPA_FEATURE_DEVICE_REMOVAL))
1879 		return (SET_ERROR(ENOTSUP));
1880 
1881 	/* available space in the pool's normal class */
1882 	uint64_t available = dsl_dir_space_available(
1883 	    spa->spa_dsl_pool->dp_root_dir, NULL, 0, B_TRUE);
1884 
1885 	metaslab_class_t *mc = vd->vdev_mg->mg_class;
1886 
1887 	/*
1888 	 * When removing a vdev from an allocation class that has
1889 	 * remaining vdevs, include available space from the class.
1890 	 */
1891 	if (mc != spa_normal_class(spa) && mc->mc_groups > 1) {
1892 		uint64_t class_avail = metaslab_class_get_space(mc) -
1893 		    metaslab_class_get_alloc(mc);
1894 
1895 		/* add class space, adjusted for overhead */
1896 		available += (class_avail * 94) / 100;
1897 	}
1898 
1899 	/*
1900 	 * There has to be enough free space to remove the
1901 	 * device and leave double the "slop" space (i.e. we
1902 	 * must leave at least 3% of the pool free, in addition to
1903 	 * the normal slop space).
1904 	 */
1905 	if (available < vd->vdev_stat.vs_dspace + spa_get_slop_space(spa)) {
1906 		return (SET_ERROR(ENOSPC));
1907 	}
1908 
1909 	/*
1910 	 * There can not be a removal in progress.
1911 	 */
1912 	if (spa->spa_removing_phys.sr_state == DSS_SCANNING)
1913 		return (SET_ERROR(EBUSY));
1914 
1915 	/*
1916 	 * The device must have all its data.
1917 	 */
1918 	if (!vdev_dtl_empty(vd, DTL_MISSING) ||
1919 	    !vdev_dtl_empty(vd, DTL_OUTAGE))
1920 		return (SET_ERROR(EBUSY));
1921 
1922 	/*
1923 	 * The device must be healthy.
1924 	 */
1925 	if (!vdev_readable(vd))
1926 		return (SET_ERROR(EIO));
1927 
1928 	/*
1929 	 * All vdevs in normal class must have the same ashift.
1930 	 */
1931 	if (spa->spa_max_ashift != spa->spa_min_ashift) {
1932 		return (SET_ERROR(EINVAL));
1933 	}
1934 
1935 	/*
1936 	 * All vdevs in normal class must have the same ashift
1937 	 * and not be raidz.
1938 	 */
1939 	vdev_t *rvd = spa->spa_root_vdev;
1940 	int num_indirect = 0;
1941 	for (uint64_t id = 0; id < rvd->vdev_children; id++) {
1942 		vdev_t *cvd = rvd->vdev_child[id];
1943 		if (cvd->vdev_ashift != 0 && !cvd->vdev_islog)
1944 			ASSERT3U(cvd->vdev_ashift, ==, spa->spa_max_ashift);
1945 		if (cvd->vdev_ops == &vdev_indirect_ops)
1946 			num_indirect++;
1947 		if (!vdev_is_concrete(cvd))
1948 			continue;
1949 		if (cvd->vdev_ops == &vdev_raidz_ops)
1950 			return (SET_ERROR(EINVAL));
1951 		/*
1952 		 * Need the mirror to be mirror of leaf vdevs only
1953 		 */
1954 		if (cvd->vdev_ops == &vdev_mirror_ops) {
1955 			for (uint64_t cid = 0;
1956 			    cid < cvd->vdev_children; cid++) {
1957 				vdev_t *tmp = cvd->vdev_child[cid];
1958 				if (!tmp->vdev_ops->vdev_op_leaf)
1959 					return (SET_ERROR(EINVAL));
1960 			}
1961 		}
1962 	}
1963 
1964 	return (0);
1965 }
1966 
1967 /*
1968  * Initiate removal of a top-level vdev, reducing the total space in the pool.
1969  * The config lock is held for the specified TXG.  Once initiated,
1970  * evacuation of all allocated space (copying it to other vdevs) happens
1971  * in the background (see spa_vdev_remove_thread()), and can be canceled
1972  * (see spa_vdev_remove_cancel()).  If successful, the vdev will
1973  * be transformed to an indirect vdev (see spa_vdev_remove_complete()).
1974  */
1975 static int
spa_vdev_remove_top(vdev_t * vd,uint64_t * txg)1976 spa_vdev_remove_top(vdev_t *vd, uint64_t *txg)
1977 {
1978 	spa_t *spa = vd->vdev_spa;
1979 	int error;
1980 
1981 	/*
1982 	 * Check for errors up-front, so that we don't waste time
1983 	 * passivating the metaslab group and clearing the ZIL if there
1984 	 * are errors.
1985 	 */
1986 	error = spa_vdev_remove_top_check(vd);
1987 	if (error != 0)
1988 		return (error);
1989 
1990 	/*
1991 	 * Stop allocating from this vdev.  Note that we must check
1992 	 * that this is not the only device in the pool before
1993 	 * passivating, otherwise we will not be able to make
1994 	 * progress because we can't allocate from any vdevs.
1995 	 * The above check for sufficient free space serves this
1996 	 * purpose.
1997 	 */
1998 	metaslab_group_t *mg = vd->vdev_mg;
1999 	metaslab_group_passivate(mg);
2000 
2001 	/*
2002 	 * Wait for the youngest allocations and frees to sync,
2003 	 * and then wait for the deferral of those frees to finish.
2004 	 */
2005 	spa_vdev_config_exit(spa, NULL,
2006 	    *txg + TXG_CONCURRENT_STATES + TXG_DEFER_SIZE, 0, FTAG);
2007 
2008 	/*
2009 	 * We must ensure that no "stubby" log blocks are allocated
2010 	 * on the device to be removed.  These blocks could be
2011 	 * written at any time, including while we are in the middle
2012 	 * of copying them.
2013 	 */
2014 	error = spa_reset_logs(spa);
2015 
2016 	/*
2017 	 * We stop any initializing and TRIM that is currently in progress
2018 	 * but leave the state as "active". This will allow the process to
2019 	 * resume if the removal is canceled sometime later.
2020 	 */
2021 	vdev_initialize_stop_all(vd, VDEV_INITIALIZE_ACTIVE);
2022 	vdev_trim_stop_all(vd, VDEV_TRIM_ACTIVE);
2023 	vdev_autotrim_stop_wait(vd);
2024 
2025 	*txg = spa_vdev_config_enter(spa);
2026 
2027 	/*
2028 	 * Things might have changed while the config lock was dropped
2029 	 * (e.g. space usage).  Check for errors again.
2030 	 */
2031 	if (error == 0)
2032 		error = spa_vdev_remove_top_check(vd);
2033 
2034 	if (error != 0) {
2035 		metaslab_group_activate(mg);
2036 		spa_async_request(spa, SPA_ASYNC_INITIALIZE_RESTART);
2037 		spa_async_request(spa, SPA_ASYNC_TRIM_RESTART);
2038 		spa_async_request(spa, SPA_ASYNC_AUTOTRIM_RESTART);
2039 		return (error);
2040 	}
2041 
2042 	vd->vdev_removing = B_TRUE;
2043 
2044 	vdev_dirty_leaves(vd, VDD_DTL, *txg);
2045 	vdev_config_dirty(vd);
2046 	dmu_tx_t *tx = dmu_tx_create_assigned(spa->spa_dsl_pool, *txg);
2047 	dsl_sync_task_nowait(spa->spa_dsl_pool,
2048 	    vdev_remove_initiate_sync,
2049 	    (void *)(uintptr_t)vd->vdev_id, 0, ZFS_SPACE_CHECK_NONE, tx);
2050 	dmu_tx_commit(tx);
2051 
2052 	return (0);
2053 }
2054 
2055 /*
2056  * Remove a device from the pool.
2057  *
2058  * Removing a device from the vdev namespace requires several steps
2059  * and can take a significant amount of time.  As a result we use
2060  * the spa_vdev_config_[enter/exit] functions which allow us to
2061  * grab and release the spa_config_lock while still holding the namespace
2062  * lock.  During each step the configuration is synced out.
2063  */
2064 int
spa_vdev_remove(spa_t * spa,uint64_t guid,boolean_t unspare)2065 spa_vdev_remove(spa_t *spa, uint64_t guid, boolean_t unspare)
2066 {
2067 	vdev_t *vd;
2068 	nvlist_t **spares, **l2cache, *nv;
2069 	uint64_t txg = 0;
2070 	uint_t nspares, nl2cache;
2071 	int error = 0;
2072 	boolean_t locked = MUTEX_HELD(&spa_namespace_lock);
2073 	sysevent_t *ev = NULL;
2074 
2075 	ASSERT(spa_writeable(spa));
2076 
2077 	if (!locked)
2078 		txg = spa_vdev_enter(spa);
2079 
2080 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
2081 	if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) {
2082 		error = (spa_has_checkpoint(spa)) ?
2083 		    ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT;
2084 
2085 		if (!locked)
2086 			return (spa_vdev_exit(spa, NULL, txg, error));
2087 
2088 		return (error);
2089 	}
2090 
2091 	vd = spa_lookup_by_guid(spa, guid, B_FALSE);
2092 
2093 	if (spa->spa_spares.sav_vdevs != NULL &&
2094 	    nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
2095 	    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0 &&
2096 	    (nv = spa_nvlist_lookup_by_guid(spares, nspares, guid)) != NULL) {
2097 		/*
2098 		 * Only remove the hot spare if it's not currently in use
2099 		 * in this pool.
2100 		 */
2101 		if (vd == NULL || unspare) {
2102 			char *nvstr = fnvlist_lookup_string(nv,
2103 			    ZPOOL_CONFIG_PATH);
2104 			spa_history_log_internal(spa, "vdev remove", NULL,
2105 			    "%s vdev (%s) %s", spa_name(spa),
2106 			    VDEV_TYPE_SPARE, nvstr);
2107 			if (vd == NULL)
2108 				vd = spa_lookup_by_guid(spa, guid, B_TRUE);
2109 			ev = spa_event_create(spa, vd, NULL,
2110 			    ESC_ZFS_VDEV_REMOVE_AUX);
2111 			spa_vdev_remove_aux(spa->spa_spares.sav_config,
2112 			    ZPOOL_CONFIG_SPARES, spares, nspares, nv);
2113 			spa_load_spares(spa);
2114 			spa->spa_spares.sav_sync = B_TRUE;
2115 		} else {
2116 			error = SET_ERROR(EBUSY);
2117 		}
2118 	} else if (spa->spa_l2cache.sav_vdevs != NULL &&
2119 	    nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config,
2120 	    ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0 &&
2121 	    (nv = spa_nvlist_lookup_by_guid(l2cache, nl2cache, guid)) != NULL) {
2122 		char *nvstr = fnvlist_lookup_string(nv, ZPOOL_CONFIG_PATH);
2123 		spa_history_log_internal(spa, "vdev remove", NULL,
2124 		    "%s vdev (%s) %s", spa_name(spa), VDEV_TYPE_L2CACHE, nvstr);
2125 		/*
2126 		 * Cache devices can always be removed.
2127 		 */
2128 		vd = spa_lookup_by_guid(spa, guid, B_TRUE);
2129 		ev = spa_event_create(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE_AUX);
2130 		spa_vdev_remove_aux(spa->spa_l2cache.sav_config,
2131 		    ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache, nv);
2132 		spa_load_l2cache(spa);
2133 		spa->spa_l2cache.sav_sync = B_TRUE;
2134 	} else if (vd != NULL && vd->vdev_islog) {
2135 		ASSERT(!locked);
2136 		error = spa_vdev_remove_log(vd, &txg);
2137 	} else if (vd != NULL) {
2138 		ASSERT(!locked);
2139 		error = spa_vdev_remove_top(vd, &txg);
2140 	} else {
2141 		/*
2142 		 * There is no vdev of any kind with the specified guid.
2143 		 */
2144 		error = SET_ERROR(ENOENT);
2145 	}
2146 
2147 	if (!locked)
2148 		error = spa_vdev_exit(spa, NULL, txg, error);
2149 
2150 	if (ev != NULL) {
2151 		if (error != 0) {
2152 			spa_event_discard(ev);
2153 		} else {
2154 			spa_event_post(ev);
2155 		}
2156 	}
2157 
2158 	return (error);
2159 }
2160 
2161 int
spa_removal_get_stats(spa_t * spa,pool_removal_stat_t * prs)2162 spa_removal_get_stats(spa_t *spa, pool_removal_stat_t *prs)
2163 {
2164 	prs->prs_state = spa->spa_removing_phys.sr_state;
2165 
2166 	if (prs->prs_state == DSS_NONE)
2167 		return (SET_ERROR(ENOENT));
2168 
2169 	prs->prs_removing_vdev = spa->spa_removing_phys.sr_removing_vdev;
2170 	prs->prs_start_time = spa->spa_removing_phys.sr_start_time;
2171 	prs->prs_end_time = spa->spa_removing_phys.sr_end_time;
2172 	prs->prs_to_copy = spa->spa_removing_phys.sr_to_copy;
2173 	prs->prs_copied = spa->spa_removing_phys.sr_copied;
2174 
2175 	if (spa->spa_vdev_removal != NULL) {
2176 		for (int i = 0; i < TXG_SIZE; i++) {
2177 			prs->prs_copied +=
2178 			    spa->spa_vdev_removal->svr_bytes_done[i];
2179 		}
2180 	}
2181 
2182 	prs->prs_mapping_memory = 0;
2183 	uint64_t indirect_vdev_id =
2184 	    spa->spa_removing_phys.sr_prev_indirect_vdev;
2185 	while (indirect_vdev_id != -1) {
2186 		vdev_t *vd = spa->spa_root_vdev->vdev_child[indirect_vdev_id];
2187 		vdev_indirect_config_t *vic = &vd->vdev_indirect_config;
2188 		vdev_indirect_mapping_t *vim = vd->vdev_indirect_mapping;
2189 
2190 		ASSERT3P(vd->vdev_ops, ==, &vdev_indirect_ops);
2191 		prs->prs_mapping_memory += vdev_indirect_mapping_size(vim);
2192 		indirect_vdev_id = vic->vic_prev_indirect_vdev;
2193 	}
2194 
2195 	return (0);
2196 }
2197