1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2016 by Delphix. All rights reserved.
24 */
25
26#include <sys/spa.h>
27#include <sys/spa_impl.h>
28#include <sys/txg.h>
29#include <sys/vdev_impl.h>
30#include <sys/refcount.h>
31#include <sys/metaslab_impl.h>
32#include <sys/dsl_synctask.h>
33#include <sys/zap.h>
34#include <sys/dmu_tx.h>
35
36/*
37 * Value that is written to disk during initialization.
38 */
39uint64_t zfs_initialize_value = 0xdeadbeefdeadbeefULL;
40
41/* maximum number of I/Os outstanding per leaf vdev */
42int zfs_initialize_limit = 1;
43
44/* size of initializing writes; default 1MiB, see zfs_remove_max_segment */
45uint64_t zfs_initialize_chunk_size = 1024 * 1024;
46
47static boolean_t
48vdev_initialize_should_stop(vdev_t *vd)
49{
50	return (vd->vdev_initialize_exit_wanted || !vdev_writeable(vd) ||
51	    vd->vdev_detached || vd->vdev_top->vdev_removing);
52}
53
54static void
55vdev_initialize_zap_update_sync(void *arg, dmu_tx_t *tx)
56{
57	/*
58	 * We pass in the guid instead of the vdev_t since the vdev may
59	 * have been freed prior to the sync task being processed. This
60	 * happens when a vdev is detached as we call spa_config_vdev_exit(),
61	 * stop the initializing thread, schedule the sync task, and free
62	 * the vdev. Later when the scheduled sync task is invoked, it would
63	 * find that the vdev has been freed.
64	 */
65	uint64_t guid = *(uint64_t *)arg;
66	uint64_t txg = dmu_tx_get_txg(tx);
67	kmem_free(arg, sizeof (uint64_t));
68
69	vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);
70	if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd))
71		return;
72
73	uint64_t last_offset = vd->vdev_initialize_offset[txg & TXG_MASK];
74	vd->vdev_initialize_offset[txg & TXG_MASK] = 0;
75
76	VERIFY(vd->vdev_leaf_zap != 0);
77
78	objset_t *mos = vd->vdev_spa->spa_meta_objset;
79
80	if (last_offset > 0) {
81		vd->vdev_initialize_last_offset = last_offset;
82		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
83		    VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET,
84		    sizeof (last_offset), 1, &last_offset, tx));
85	}
86	if (vd->vdev_initialize_action_time > 0) {
87		uint64_t val = (uint64_t)vd->vdev_initialize_action_time;
88		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
89		    VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME, sizeof (val),
90		    1, &val, tx));
91	}
92
93	uint64_t initialize_state = vd->vdev_initialize_state;
94	VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
95	    VDEV_LEAF_ZAP_INITIALIZE_STATE, sizeof (initialize_state), 1,
96	    &initialize_state, tx));
97}
98
99static void
100vdev_initialize_change_state(vdev_t *vd, vdev_initializing_state_t new_state)
101{
102	ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
103	spa_t *spa = vd->vdev_spa;
104
105	if (new_state == vd->vdev_initialize_state)
106		return;
107
108	/*
109	 * Copy the vd's guid, this will be freed by the sync task.
110	 */
111	uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
112	*guid = vd->vdev_guid;
113
114	/*
115	 * If we're suspending, then preserving the original start time.
116	 */
117	if (vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED) {
118		vd->vdev_initialize_action_time = gethrestime_sec();
119	}
120	vd->vdev_initialize_state = new_state;
121
122	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
123	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
124	dsl_sync_task_nowait(spa_get_dsl(spa), vdev_initialize_zap_update_sync,
125	    guid, 2, ZFS_SPACE_CHECK_NONE, tx);
126
127	switch (new_state) {
128	case VDEV_INITIALIZE_ACTIVE:
129		spa_history_log_internal(spa, "initialize", tx,
130		    "vdev=%s activated", vd->vdev_path);
131		break;
132	case VDEV_INITIALIZE_SUSPENDED:
133		spa_history_log_internal(spa, "initialize", tx,
134		    "vdev=%s suspended", vd->vdev_path);
135		break;
136	case VDEV_INITIALIZE_CANCELED:
137		spa_history_log_internal(spa, "initialize", tx,
138		    "vdev=%s canceled", vd->vdev_path);
139		break;
140	case VDEV_INITIALIZE_COMPLETE:
141		spa_history_log_internal(spa, "initialize", tx,
142		    "vdev=%s complete", vd->vdev_path);
143		break;
144	default:
145		panic("invalid state %llu", (unsigned long long)new_state);
146	}
147
148	dmu_tx_commit(tx);
149}
150
151static void
152vdev_initialize_cb(zio_t *zio)
153{
154	vdev_t *vd = zio->io_vd;
155	mutex_enter(&vd->vdev_initialize_io_lock);
156	if (zio->io_error == ENXIO && !vdev_writeable(vd)) {
157		/*
158		 * The I/O failed because the vdev was unavailable; roll the
159		 * last offset back. (This works because spa_sync waits on
160		 * spa_txg_zio before it runs sync tasks.)
161		 */
162		uint64_t *off =
163		    &vd->vdev_initialize_offset[zio->io_txg & TXG_MASK];
164		*off = MIN(*off, zio->io_offset);
165	} else {
166		/*
167		 * Since initializing is best-effort, we ignore I/O errors and
168		 * rely on vdev_probe to determine if the errors are more
169		 * critical.
170		 */
171		if (zio->io_error != 0)
172			vd->vdev_stat.vs_initialize_errors++;
173
174		vd->vdev_initialize_bytes_done += zio->io_orig_size;
175	}
176	ASSERT3U(vd->vdev_initialize_inflight, >, 0);
177	vd->vdev_initialize_inflight--;
178	cv_broadcast(&vd->vdev_initialize_io_cv);
179	mutex_exit(&vd->vdev_initialize_io_lock);
180
181	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
182}
183
184/* Takes care of physical writing and limiting # of concurrent ZIOs. */
185static int
186vdev_initialize_write(vdev_t *vd, uint64_t start, uint64_t size, abd_t *data)
187{
188	spa_t *spa = vd->vdev_spa;
189
190	/* Limit inflight initializing I/Os */
191	mutex_enter(&vd->vdev_initialize_io_lock);
192	while (vd->vdev_initialize_inflight >= zfs_initialize_limit) {
193		cv_wait(&vd->vdev_initialize_io_cv,
194		    &vd->vdev_initialize_io_lock);
195	}
196	vd->vdev_initialize_inflight++;
197	mutex_exit(&vd->vdev_initialize_io_lock);
198
199	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
200	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
201	uint64_t txg = dmu_tx_get_txg(tx);
202
203	spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER);
204	mutex_enter(&vd->vdev_initialize_lock);
205
206	if (vd->vdev_initialize_offset[txg & TXG_MASK] == 0) {
207		uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
208		*guid = vd->vdev_guid;
209
210		/* This is the first write of this txg. */
211		dsl_sync_task_nowait(spa_get_dsl(spa),
212		    vdev_initialize_zap_update_sync, guid, 2,
213		    ZFS_SPACE_CHECK_RESERVED, tx);
214	}
215
216	/*
217	 * We know the vdev struct will still be around since all
218	 * consumers of vdev_free must stop the initialization first.
219	 */
220	if (vdev_initialize_should_stop(vd)) {
221		mutex_enter(&vd->vdev_initialize_io_lock);
222		ASSERT3U(vd->vdev_initialize_inflight, >, 0);
223		vd->vdev_initialize_inflight--;
224		mutex_exit(&vd->vdev_initialize_io_lock);
225		spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
226		mutex_exit(&vd->vdev_initialize_lock);
227		dmu_tx_commit(tx);
228		return (SET_ERROR(EINTR));
229	}
230	mutex_exit(&vd->vdev_initialize_lock);
231
232	vd->vdev_initialize_offset[txg & TXG_MASK] = start + size;
233	zio_nowait(zio_write_phys(spa->spa_txg_zio[txg & TXG_MASK], vd, start,
234	    size, data, ZIO_CHECKSUM_OFF, vdev_initialize_cb, NULL,
235	    ZIO_PRIORITY_INITIALIZING, ZIO_FLAG_CANFAIL, B_FALSE));
236	/* vdev_initialize_cb releases SCL_STATE_ALL */
237
238	dmu_tx_commit(tx);
239
240	return (0);
241}
242
243/*
244 * Callback to fill each ABD chunk with zfs_initialize_value. len must be
245 * divisible by sizeof (uint64_t), and buf must be 8-byte aligned. The ABD
246 * allocation will guarantee these for us.
247 */
248/* ARGSUSED */
249static int
250vdev_initialize_block_fill(void *buf, size_t len, void *unused)
251{
252	ASSERT0(len % sizeof (uint64_t));
253	for (uint64_t i = 0; i < len; i += sizeof (uint64_t)) {
254		*(uint64_t *)((char *)(buf) + i) = zfs_initialize_value;
255	}
256	return (0);
257}
258
259static abd_t *
260vdev_initialize_block_alloc()
261{
262	/* Allocate ABD for filler data */
263	abd_t *data = abd_alloc_for_io(zfs_initialize_chunk_size, B_FALSE);
264
265	ASSERT0(zfs_initialize_chunk_size % sizeof (uint64_t));
266	(void) abd_iterate_func(data, 0, zfs_initialize_chunk_size,
267	    vdev_initialize_block_fill, NULL);
268
269	return (data);
270}
271
272static void
273vdev_initialize_block_free(abd_t *data)
274{
275	abd_free(data);
276}
277
278static int
279vdev_initialize_ranges(vdev_t *vd, abd_t *data)
280{
281	avl_tree_t *rt = &vd->vdev_initialize_tree->rt_root;
282
283	for (range_seg_t *rs = avl_first(rt); rs != NULL;
284	    rs = AVL_NEXT(rt, rs)) {
285		uint64_t size = rs->rs_end - rs->rs_start;
286
287		/* Split range into legally-sized physical chunks */
288		uint64_t writes_required =
289		    ((size - 1) / zfs_initialize_chunk_size) + 1;
290
291		for (uint64_t w = 0; w < writes_required; w++) {
292			int error;
293
294			error = vdev_initialize_write(vd,
295			    VDEV_LABEL_START_SIZE + rs->rs_start +
296			    (w * zfs_initialize_chunk_size),
297			    MIN(size - (w * zfs_initialize_chunk_size),
298			    zfs_initialize_chunk_size), data);
299			if (error != 0)
300				return (error);
301		}
302	}
303	return (0);
304}
305
306static void
307vdev_initialize_calculate_progress(vdev_t *vd)
308{
309	ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
310	    spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
311	ASSERT(vd->vdev_leaf_zap != 0);
312
313	vd->vdev_initialize_bytes_est = 0;
314	vd->vdev_initialize_bytes_done = 0;
315
316	for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) {
317		metaslab_t *msp = vd->vdev_top->vdev_ms[i];
318		mutex_enter(&msp->ms_lock);
319
320		uint64_t ms_free = msp->ms_size -
321		    metaslab_allocated_space(msp);
322
323		if (vd->vdev_top->vdev_ops == &vdev_raidz_ops)
324			ms_free /= vd->vdev_top->vdev_children;
325
326		/*
327		 * Convert the metaslab range to a physical range
328		 * on our vdev. We use this to determine if we are
329		 * in the middle of this metaslab range.
330		 */
331		range_seg_t logical_rs, physical_rs;
332		logical_rs.rs_start = msp->ms_start;
333		logical_rs.rs_end = msp->ms_start + msp->ms_size;
334		vdev_xlate(vd, &logical_rs, &physical_rs);
335
336		if (vd->vdev_initialize_last_offset <= physical_rs.rs_start) {
337			vd->vdev_initialize_bytes_est += ms_free;
338			mutex_exit(&msp->ms_lock);
339			continue;
340		} else if (vd->vdev_initialize_last_offset >
341		    physical_rs.rs_end) {
342			vd->vdev_initialize_bytes_done += ms_free;
343			vd->vdev_initialize_bytes_est += ms_free;
344			mutex_exit(&msp->ms_lock);
345			continue;
346		}
347
348		/*
349		 * If we get here, we're in the middle of initializing this
350		 * metaslab. Load it and walk the free tree for more accurate
351		 * progress estimation.
352		 */
353		VERIFY0(metaslab_load(msp));
354
355		for (range_seg_t *rs = avl_first(&msp->ms_allocatable->rt_root);
356		    rs; rs = AVL_NEXT(&msp->ms_allocatable->rt_root, rs)) {
357			logical_rs.rs_start = rs->rs_start;
358			logical_rs.rs_end = rs->rs_end;
359			vdev_xlate(vd, &logical_rs, &physical_rs);
360
361			uint64_t size = physical_rs.rs_end -
362			    physical_rs.rs_start;
363			vd->vdev_initialize_bytes_est += size;
364			if (vd->vdev_initialize_last_offset >
365			    physical_rs.rs_end) {
366				vd->vdev_initialize_bytes_done += size;
367			} else if (vd->vdev_initialize_last_offset >
368			    physical_rs.rs_start &&
369			    vd->vdev_initialize_last_offset <
370			    physical_rs.rs_end) {
371				vd->vdev_initialize_bytes_done +=
372				    vd->vdev_initialize_last_offset -
373				    physical_rs.rs_start;
374			}
375		}
376		mutex_exit(&msp->ms_lock);
377	}
378}
379
380static int
381vdev_initialize_load(vdev_t *vd)
382{
383	int err = 0;
384	ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
385	    spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
386	ASSERT(vd->vdev_leaf_zap != 0);
387
388	if (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE ||
389	    vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED) {
390		err = zap_lookup(vd->vdev_spa->spa_meta_objset,
391		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET,
392		    sizeof (vd->vdev_initialize_last_offset), 1,
393		    &vd->vdev_initialize_last_offset);
394		if (err == ENOENT) {
395			vd->vdev_initialize_last_offset = 0;
396			err = 0;
397		}
398	}
399
400	vdev_initialize_calculate_progress(vd);
401	return (err);
402}
403
404
405/*
406 * Convert the logical range into a physical range and add it to our
407 * avl tree.
408 */
409void
410vdev_initialize_range_add(void *arg, uint64_t start, uint64_t size)
411{
412	vdev_t *vd = arg;
413	range_seg_t logical_rs, physical_rs;
414	logical_rs.rs_start = start;
415	logical_rs.rs_end = start + size;
416
417	ASSERT(vd->vdev_ops->vdev_op_leaf);
418	vdev_xlate(vd, &logical_rs, &physical_rs);
419
420	IMPLY(vd->vdev_top == vd,
421	    logical_rs.rs_start == physical_rs.rs_start);
422	IMPLY(vd->vdev_top == vd,
423	    logical_rs.rs_end == physical_rs.rs_end);
424
425	/* Only add segments that we have not visited yet */
426	if (physical_rs.rs_end <= vd->vdev_initialize_last_offset)
427		return;
428
429	/* Pick up where we left off mid-range. */
430	if (vd->vdev_initialize_last_offset > physical_rs.rs_start) {
431		zfs_dbgmsg("range write: vd %s changed (%llu, %llu) to "
432		    "(%llu, %llu)", vd->vdev_path,
433		    (u_longlong_t)physical_rs.rs_start,
434		    (u_longlong_t)physical_rs.rs_end,
435		    (u_longlong_t)vd->vdev_initialize_last_offset,
436		    (u_longlong_t)physical_rs.rs_end);
437		ASSERT3U(physical_rs.rs_end, >,
438		    vd->vdev_initialize_last_offset);
439		physical_rs.rs_start = vd->vdev_initialize_last_offset;
440	}
441	ASSERT3U(physical_rs.rs_end, >=, physical_rs.rs_start);
442
443	/*
444	 * With raidz, it's possible that the logical range does not live on
445	 * this leaf vdev. We only add the physical range to this vdev's if it
446	 * has a length greater than 0.
447	 */
448	if (physical_rs.rs_end > physical_rs.rs_start) {
449		range_tree_add(vd->vdev_initialize_tree, physical_rs.rs_start,
450		    physical_rs.rs_end - physical_rs.rs_start);
451	} else {
452		ASSERT3U(physical_rs.rs_end, ==, physical_rs.rs_start);
453	}
454}
455
456static void
457vdev_initialize_thread(void *arg)
458{
459	vdev_t *vd = arg;
460	spa_t *spa = vd->vdev_spa;
461	int error = 0;
462	uint64_t ms_count = 0;
463
464	ASSERT(vdev_is_concrete(vd));
465	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
466
467	vd->vdev_initialize_last_offset = 0;
468	VERIFY0(vdev_initialize_load(vd));
469
470	abd_t *deadbeef = vdev_initialize_block_alloc();
471
472	vd->vdev_initialize_tree = range_tree_create(NULL, NULL);
473
474	for (uint64_t i = 0; !vd->vdev_detached &&
475	    i < vd->vdev_top->vdev_ms_count; i++) {
476		metaslab_t *msp = vd->vdev_top->vdev_ms[i];
477
478		/*
479		 * If we've expanded the top-level vdev or it's our
480		 * first pass, calculate our progress.
481		 */
482		if (vd->vdev_top->vdev_ms_count != ms_count) {
483			vdev_initialize_calculate_progress(vd);
484			ms_count = vd->vdev_top->vdev_ms_count;
485		}
486
487		spa_config_exit(spa, SCL_CONFIG, FTAG);
488		metaslab_disable(msp);
489		mutex_enter(&msp->ms_lock);
490		VERIFY0(metaslab_load(msp));
491
492		range_tree_walk(msp->ms_allocatable, vdev_initialize_range_add,
493		    vd);
494		mutex_exit(&msp->ms_lock);
495
496		error = vdev_initialize_ranges(vd, deadbeef);
497		metaslab_enable(msp, B_TRUE);
498		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
499
500		range_tree_vacate(vd->vdev_initialize_tree, NULL, NULL);
501		if (error != 0)
502			break;
503	}
504
505	spa_config_exit(spa, SCL_CONFIG, FTAG);
506	mutex_enter(&vd->vdev_initialize_io_lock);
507	while (vd->vdev_initialize_inflight > 0) {
508		cv_wait(&vd->vdev_initialize_io_cv,
509		    &vd->vdev_initialize_io_lock);
510	}
511	mutex_exit(&vd->vdev_initialize_io_lock);
512
513	range_tree_destroy(vd->vdev_initialize_tree);
514	vdev_initialize_block_free(deadbeef);
515	vd->vdev_initialize_tree = NULL;
516
517	mutex_enter(&vd->vdev_initialize_lock);
518	if (!vd->vdev_initialize_exit_wanted && vdev_writeable(vd)) {
519		vdev_initialize_change_state(vd, VDEV_INITIALIZE_COMPLETE);
520	}
521	ASSERT(vd->vdev_initialize_thread != NULL ||
522	    vd->vdev_initialize_inflight == 0);
523
524	/*
525	 * Drop the vdev_initialize_lock while we sync out the
526	 * txg since it's possible that a device might be trying to
527	 * come online and must check to see if it needs to restart an
528	 * initialization. That thread will be holding the spa_config_lock
529	 * which would prevent the txg_wait_synced from completing.
530	 */
531	mutex_exit(&vd->vdev_initialize_lock);
532	txg_wait_synced(spa_get_dsl(spa), 0);
533	mutex_enter(&vd->vdev_initialize_lock);
534
535	vd->vdev_initialize_thread = NULL;
536	cv_broadcast(&vd->vdev_initialize_cv);
537	mutex_exit(&vd->vdev_initialize_lock);
538}
539
540/*
541 * Initiates a device. Caller must hold vdev_initialize_lock.
542 * Device must be a leaf and not already be initializing.
543 */
544void
545vdev_initialize(vdev_t *vd)
546{
547	ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
548	ASSERT(vd->vdev_ops->vdev_op_leaf);
549	ASSERT(vdev_is_concrete(vd));
550	ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
551	ASSERT(!vd->vdev_detached);
552	ASSERT(!vd->vdev_initialize_exit_wanted);
553	ASSERT(!vd->vdev_top->vdev_removing);
554
555	vdev_initialize_change_state(vd, VDEV_INITIALIZE_ACTIVE);
556	vd->vdev_initialize_thread = thread_create(NULL, 0,
557	    vdev_initialize_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
558}
559
560/*
561 * Wait for the initialize thread to be terminated (cancelled or stopped).
562 */
563static void
564vdev_initialize_stop_wait_impl(vdev_t *vd)
565{
566	ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
567
568	while (vd->vdev_initialize_thread != NULL)
569		cv_wait(&vd->vdev_initialize_cv, &vd->vdev_initialize_lock);
570
571	ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
572	vd->vdev_initialize_exit_wanted = B_FALSE;
573}
574
575/*
576 * Wait for vdev initialize threads which were either to cleanly exit.
577 */
578void
579vdev_initialize_stop_wait(spa_t *spa, list_t *vd_list)
580{
581	vdev_t *vd;
582
583	ASSERT(MUTEX_HELD(&spa_namespace_lock));
584
585	while ((vd = list_remove_head(vd_list)) != NULL) {
586		mutex_enter(&vd->vdev_initialize_lock);
587		vdev_initialize_stop_wait_impl(vd);
588		mutex_exit(&vd->vdev_initialize_lock);
589	}
590}
591
592/*
593 * Stop initializing a device, with the resultant initializing state being
594 * tgt_state.  For blocking behavior pass NULL for vd_list.  Otherwise, when
595 * a list_t is provided the stopping vdev is inserted in to the list.  Callers
596 * are then required to call vdev_initialize_stop_wait() to block for all the
597 * initialization threads to exit.  The caller must hold vdev_initialize_lock
598 * and must not be writing to the spa config, as the initializing thread may
599 * try to enter the config as a reader before exiting.
600 */
601void
602vdev_initialize_stop(vdev_t *vd, vdev_initializing_state_t tgt_state,
603    list_t *vd_list)
604{
605	ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER));
606	ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
607	ASSERT(vd->vdev_ops->vdev_op_leaf);
608	ASSERT(vdev_is_concrete(vd));
609
610	/*
611	 * Allow cancel requests to proceed even if the initialize thread
612	 * has stopped.
613	 */
614	if (vd->vdev_initialize_thread == NULL &&
615	    tgt_state != VDEV_INITIALIZE_CANCELED) {
616		return;
617	}
618
619	vdev_initialize_change_state(vd, tgt_state);
620	vd->vdev_initialize_exit_wanted = B_TRUE;
621
622	if (vd_list == NULL) {
623		vdev_initialize_stop_wait_impl(vd);
624	} else {
625		ASSERT(MUTEX_HELD(&spa_namespace_lock));
626		list_insert_tail(vd_list, vd);
627	}
628}
629
630static void
631vdev_initialize_stop_all_impl(vdev_t *vd, vdev_initializing_state_t tgt_state,
632    list_t *vd_list)
633{
634	if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) {
635		mutex_enter(&vd->vdev_initialize_lock);
636		vdev_initialize_stop(vd, tgt_state, vd_list);
637		mutex_exit(&vd->vdev_initialize_lock);
638		return;
639	}
640
641	for (uint64_t i = 0; i < vd->vdev_children; i++) {
642		vdev_initialize_stop_all_impl(vd->vdev_child[i], tgt_state,
643		    vd_list);
644	}
645}
646
647/*
648 * Convenience function to stop initializing of a vdev tree and set all
649 * initialize thread pointers to NULL.
650 */
651void
652vdev_initialize_stop_all(vdev_t *vd, vdev_initializing_state_t tgt_state)
653{
654	spa_t *spa = vd->vdev_spa;
655	list_t vd_list;
656
657	ASSERT(MUTEX_HELD(&spa_namespace_lock));
658
659	list_create(&vd_list, sizeof (vdev_t),
660	    offsetof(vdev_t, vdev_initialize_node));
661
662	vdev_initialize_stop_all_impl(vd, tgt_state, &vd_list);
663	vdev_initialize_stop_wait(spa, &vd_list);
664
665	if (vd->vdev_spa->spa_sync_on) {
666		/* Make sure that our state has been synced to disk */
667		txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
668	}
669
670	list_destroy(&vd_list);
671}
672
673void
674vdev_initialize_restart(vdev_t *vd)
675{
676	ASSERT(MUTEX_HELD(&spa_namespace_lock));
677	ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
678
679	if (vd->vdev_leaf_zap != 0) {
680		mutex_enter(&vd->vdev_initialize_lock);
681		uint64_t initialize_state = VDEV_INITIALIZE_NONE;
682		int err = zap_lookup(vd->vdev_spa->spa_meta_objset,
683		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_STATE,
684		    sizeof (initialize_state), 1, &initialize_state);
685		ASSERT(err == 0 || err == ENOENT);
686		vd->vdev_initialize_state = initialize_state;
687
688		uint64_t timestamp = 0;
689		err = zap_lookup(vd->vdev_spa->spa_meta_objset,
690		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME,
691		    sizeof (timestamp), 1, &timestamp);
692		ASSERT(err == 0 || err == ENOENT);
693		vd->vdev_initialize_action_time = (time_t)timestamp;
694
695		if (vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED ||
696		    vd->vdev_offline) {
697			/* load progress for reporting, but don't resume */
698			VERIFY0(vdev_initialize_load(vd));
699		} else if (vd->vdev_initialize_state ==
700		    VDEV_INITIALIZE_ACTIVE && vdev_writeable(vd) &&
701		    !vd->vdev_top->vdev_removing &&
702		    vd->vdev_initialize_thread == NULL) {
703			vdev_initialize(vd);
704		}
705
706		mutex_exit(&vd->vdev_initialize_lock);
707	}
708
709	for (uint64_t i = 0; i < vd->vdev_children; i++) {
710		vdev_initialize_restart(vd->vdev_child[i]);
711	}
712}
713