1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
25 * Copyright (c) 2012 by Frederik Wessels. All rights reserved.
26 * Copyright (c) 2013 by Prasad Joshi (sTec). All rights reserved.
27 * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
28 * Copyright 2016 Nexenta Systems, Inc.
29 * Copyright (c) 2017 Datto Inc.
30 * Copyright (c) 2017, Intel Corporation.
31 * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>
32 * Copyright 2019 Joyent, Inc.
33 * Copyright (c) 2012 by Cyril Plisko. All rights reserved.
34 * Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
35 */
36
37#include <assert.h>
38#include <ctype.h>
39#include <dirent.h>
40#include <errno.h>
41#include <fcntl.h>
42#include <getopt.h>
43#include <libgen.h>
44#include <libintl.h>
45#include <libuutil.h>
46#include <locale.h>
47#include <stdio.h>
48#include <stdlib.h>
49#include <string.h>
50#include <strings.h>
51#include <unistd.h>
52#include <priv.h>
53#include <pwd.h>
54#include <zone.h>
55#include <zfs_prop.h>
56#include <sys/fs/zfs.h>
57#include <sys/stat.h>
58#include <sys/debug.h>
59
60#include <libzfs.h>
61
62#include "zpool_util.h"
63#include "zfs_comutil.h"
64#include "zfeature_common.h"
65
66#include "statcommon.h"
67
68static int zpool_do_create(int, char **);
69static int zpool_do_destroy(int, char **);
70
71static int zpool_do_add(int, char **);
72static int zpool_do_remove(int, char **);
73static int zpool_do_labelclear(int, char **);
74
75static int zpool_do_checkpoint(int, char **);
76
77static int zpool_do_list(int, char **);
78static int zpool_do_iostat(int, char **);
79static int zpool_do_status(int, char **);
80
81static int zpool_do_online(int, char **);
82static int zpool_do_offline(int, char **);
83static int zpool_do_clear(int, char **);
84static int zpool_do_reopen(int, char **);
85
86static int zpool_do_reguid(int, char **);
87
88static int zpool_do_attach(int, char **);
89static int zpool_do_detach(int, char **);
90static int zpool_do_replace(int, char **);
91static int zpool_do_split(int, char **);
92
93static int zpool_do_initialize(int, char **);
94static int zpool_do_scrub(int, char **);
95static int zpool_do_resilver(int, char **);
96static int zpool_do_trim(int, char **);
97
98static int zpool_do_import(int, char **);
99static int zpool_do_export(int, char **);
100
101static int zpool_do_upgrade(int, char **);
102
103static int zpool_do_history(int, char **);
104
105static int zpool_do_get(int, char **);
106static int zpool_do_set(int, char **);
107
108static int zpool_do_sync(int, char **);
109
110/*
111 * These libumem hooks provide a reasonable set of defaults for the allocator's
112 * debugging facilities.
113 */
114
115#ifdef DEBUG
116const char *
117_umem_debug_init(void)
118{
119	return ("default,verbose"); /* $UMEM_DEBUG setting */
120}
121
122const char *
123_umem_logging_init(void)
124{
125	return ("fail,contents"); /* $UMEM_LOGGING setting */
126}
127#endif
128
129typedef enum {
130	HELP_ADD,
131	HELP_ATTACH,
132	HELP_CLEAR,
133	HELP_CREATE,
134	HELP_CHECKPOINT,
135	HELP_DESTROY,
136	HELP_DETACH,
137	HELP_EXPORT,
138	HELP_HISTORY,
139	HELP_IMPORT,
140	HELP_IOSTAT,
141	HELP_LABELCLEAR,
142	HELP_LIST,
143	HELP_OFFLINE,
144	HELP_ONLINE,
145	HELP_REPLACE,
146	HELP_REMOVE,
147	HELP_INITIALIZE,
148	HELP_SCRUB,
149	HELP_RESILVER,
150	HELP_TRIM,
151	HELP_STATUS,
152	HELP_UPGRADE,
153	HELP_GET,
154	HELP_SET,
155	HELP_SPLIT,
156	HELP_SYNC,
157	HELP_REGUID,
158	HELP_REOPEN
159} zpool_help_t;
160
161
162typedef struct zpool_command {
163	const char	*name;
164	int		(*func)(int, char **);
165	zpool_help_t	usage;
166} zpool_command_t;
167
168/*
169 * Master command table.  Each ZFS command has a name, associated function, and
170 * usage message.  The usage messages need to be internationalized, so we have
171 * to have a function to return the usage message based on a command index.
172 *
173 * These commands are organized according to how they are displayed in the usage
174 * message.  An empty command (one with a NULL name) indicates an empty line in
175 * the generic usage message.
176 */
177static zpool_command_t command_table[] = {
178	{ "create",	zpool_do_create,	HELP_CREATE		},
179	{ "destroy",	zpool_do_destroy,	HELP_DESTROY		},
180	{ NULL },
181	{ "add",	zpool_do_add,		HELP_ADD		},
182	{ "remove",	zpool_do_remove,	HELP_REMOVE		},
183	{ NULL },
184	{ "labelclear",	zpool_do_labelclear,	HELP_LABELCLEAR		},
185	{ NULL },
186	{ "checkpoint",	zpool_do_checkpoint,	HELP_CHECKPOINT		},
187	{ NULL },
188	{ "list",	zpool_do_list,		HELP_LIST		},
189	{ "iostat",	zpool_do_iostat,	HELP_IOSTAT		},
190	{ "status",	zpool_do_status,	HELP_STATUS		},
191	{ NULL },
192	{ "online",	zpool_do_online,	HELP_ONLINE		},
193	{ "offline",	zpool_do_offline,	HELP_OFFLINE		},
194	{ "clear",	zpool_do_clear,		HELP_CLEAR		},
195	{ "reopen",	zpool_do_reopen,	HELP_REOPEN		},
196	{ NULL },
197	{ "attach",	zpool_do_attach,	HELP_ATTACH		},
198	{ "detach",	zpool_do_detach,	HELP_DETACH		},
199	{ "replace",	zpool_do_replace,	HELP_REPLACE		},
200	{ "split",	zpool_do_split,		HELP_SPLIT		},
201	{ NULL },
202	{ "initialize",	zpool_do_initialize,	HELP_INITIALIZE		},
203	{ "resilver",	zpool_do_resilver,	HELP_RESILVER		},
204	{ "scrub",	zpool_do_scrub,		HELP_SCRUB		},
205	{ "trim",	zpool_do_trim,		HELP_TRIM		},
206	{ NULL },
207	{ "import",	zpool_do_import,	HELP_IMPORT		},
208	{ "export",	zpool_do_export,	HELP_EXPORT		},
209	{ "upgrade",	zpool_do_upgrade,	HELP_UPGRADE		},
210	{ "reguid",	zpool_do_reguid,	HELP_REGUID		},
211	{ NULL },
212	{ "history",	zpool_do_history,	HELP_HISTORY		},
213	{ "get",	zpool_do_get,		HELP_GET		},
214	{ "set",	zpool_do_set,		HELP_SET		},
215	{ "sync",	zpool_do_sync,		HELP_SYNC		},
216};
217
218#define	NCOMMAND	(sizeof (command_table) / sizeof (command_table[0]))
219
220#define	VDEV_ALLOC_CLASS_LOGS	"logs"
221
222static zpool_command_t *current_command;
223static char history_str[HIS_MAX_RECORD_LEN];
224static boolean_t log_history = B_TRUE;
225static uint_t timestamp_fmt = NODATE;
226
227static const char *
228get_usage(zpool_help_t idx)
229{
230	switch (idx) {
231	case HELP_ADD:
232		return (gettext("\tadd [-fgLnP] [-o property=value] "
233		    "<pool> <vdev> ...\n"));
234	case HELP_ATTACH:
235		return (gettext("\tattach [-f] [-o property=value] "
236		    "<pool> <device> <new-device>\n"));
237	case HELP_CLEAR:
238		return (gettext("\tclear [-nF] <pool> [device]\n"));
239	case HELP_CREATE:
240		return (gettext("\tcreate [-fnd] [-B] "
241		    "[-o property=value] ... \n"
242		    "\t    [-O file-system-property=value] ...\n"
243		    "\t    [-m mountpoint] [-R root] [-t tempname] "
244		    "<pool> <vdev> ...\n"));
245	case HELP_CHECKPOINT:
246		return (gettext("\tcheckpoint [--discard] <pool> ...\n"));
247	case HELP_DESTROY:
248		return (gettext("\tdestroy [-f] <pool>\n"));
249	case HELP_DETACH:
250		return (gettext("\tdetach <pool> <device>\n"));
251	case HELP_EXPORT:
252		return (gettext("\texport [-f] <pool> ...\n"));
253	case HELP_HISTORY:
254		return (gettext("\thistory [-il] [<pool>] ...\n"));
255	case HELP_IMPORT:
256		return (gettext("\timport [-d dir] [-D]\n"
257		    "\timport [-d dir | -c cachefile] [-F [-n]] [-l] "
258		    "<pool | id>\n"
259		    "\timport [-o mntopts] [-o property=value] ... \n"
260		    "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
261		    "[-R root] [-F [-n]] -a\n"
262		    "\timport [-o mntopts] [-o property=value] ... \n"
263		    "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
264		    "[-R root] [-F [-n]] [-t]\n"
265		    "\t    [--rewind-to-checkpoint] <pool | id> [newpool]\n"));
266	case HELP_IOSTAT:
267		return (gettext("\tiostat [-gLPv] [-T d|u] [pool] ... "
268		    "[interval [count]]\n"));
269	case HELP_LABELCLEAR:
270		return (gettext("\tlabelclear [-f] <vdev>\n"));
271	case HELP_LIST:
272		return (gettext("\tlist [-gHLpPv] [-o property[,...]] "
273		    "[-T d|u] [pool] ... [interval [count]]\n"));
274	case HELP_OFFLINE:
275		return (gettext("\toffline [-t] <pool> <device> ...\n"));
276	case HELP_ONLINE:
277		return (gettext("\tonline <pool> <device> ...\n"));
278	case HELP_REPLACE:
279		return (gettext("\treplace [-f] <pool> <device> "
280		    "[new-device]\n"));
281	case HELP_REMOVE:
282		return (gettext("\tremove [-nps] <pool> <device> ...\n"));
283	case HELP_REOPEN:
284		return (gettext("\treopen <pool>\n"));
285	case HELP_INITIALIZE:
286		return (gettext("\tinitialize [-c | -s] <pool> "
287		    "[<device> ...]\n"));
288	case HELP_SCRUB:
289		return (gettext("\tscrub [-s | -p] <pool> ...\n"));
290	case HELP_RESILVER:
291		return (gettext("\tresilver <pool> ...\n"));
292	case HELP_TRIM:
293		return (gettext("\ttrim [-d] [-r <rate>] [-c | -s] <pool> "
294		    "[<device> ...]\n"));
295	case HELP_STATUS:
296		return (gettext("\tstatus [-igLPvxD] [-T d|u] [pool] ... "
297		    "[interval [count]]\n"));
298	case HELP_UPGRADE:
299		return (gettext("\tupgrade\n"
300		    "\tupgrade -v\n"
301		    "\tupgrade [-V version] <-a | pool ...>\n"));
302	case HELP_GET:
303		return (gettext("\tget [-Hp] [-o \"all\" | field[,...]] "
304		    "<\"all\" | property[,...]> <pool> ...\n"));
305	case HELP_SET:
306		return (gettext("\tset <property=value> <pool> \n"));
307	case HELP_SPLIT:
308		return (gettext("\tsplit [-gLlnP] [-R altroot] [-o mntopts]\n"
309		    "\t    [-o property=value] <pool> <newpool> "
310		    "[<device> ...]\n"));
311	case HELP_REGUID:
312		return (gettext("\treguid <pool>\n"));
313	case HELP_SYNC:
314		return (gettext("\tsync [pool] ...\n"));
315	}
316
317	abort();
318	/* NOTREACHED */
319}
320
321static void
322zpool_collect_leaves(zpool_handle_t *zhp, nvlist_t *nvroot, nvlist_t *res)
323{
324	uint_t children = 0;
325	nvlist_t **child;
326	uint_t i;
327
328	(void) nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
329	    &child, &children);
330
331	if (children == 0) {
332		char *path = zpool_vdev_name(g_zfs, zhp, nvroot, 0);
333
334		if (strcmp(path, VDEV_TYPE_INDIRECT) != 0)
335			fnvlist_add_boolean(res, path);
336
337		free(path);
338		return;
339	}
340
341	for (i = 0; i < children; i++) {
342		zpool_collect_leaves(zhp, child[i], res);
343	}
344}
345
346/*
347 * Callback routine that will print out a pool property value.
348 */
349static int
350print_prop_cb(int prop, void *cb)
351{
352	FILE *fp = cb;
353
354	(void) fprintf(fp, "\t%-19s  ", zpool_prop_to_name(prop));
355
356	if (zpool_prop_readonly(prop))
357		(void) fprintf(fp, "  NO   ");
358	else
359		(void) fprintf(fp, " YES   ");
360
361	if (zpool_prop_values(prop) == NULL)
362		(void) fprintf(fp, "-\n");
363	else
364		(void) fprintf(fp, "%s\n", zpool_prop_values(prop));
365
366	return (ZPROP_CONT);
367}
368
369/*
370 * Display usage message.  If we're inside a command, display only the usage for
371 * that command.  Otherwise, iterate over the entire command table and display
372 * a complete usage message.
373 */
374void
375usage(boolean_t requested)
376{
377	FILE *fp = requested ? stdout : stderr;
378
379	if (current_command == NULL) {
380		int i;
381
382		(void) fprintf(fp, gettext("usage: zpool command args ...\n"));
383		(void) fprintf(fp,
384		    gettext("where 'command' is one of the following:\n\n"));
385
386		for (i = 0; i < NCOMMAND; i++) {
387			if (command_table[i].name == NULL)
388				(void) fprintf(fp, "\n");
389			else
390				(void) fprintf(fp, "%s",
391				    get_usage(command_table[i].usage));
392		}
393	} else {
394		(void) fprintf(fp, gettext("usage:\n"));
395		(void) fprintf(fp, "%s", get_usage(current_command->usage));
396	}
397
398	if (current_command != NULL &&
399	    ((strcmp(current_command->name, "set") == 0) ||
400	    (strcmp(current_command->name, "get") == 0) ||
401	    (strcmp(current_command->name, "list") == 0))) {
402
403		(void) fprintf(fp,
404		    gettext("\nthe following properties are supported:\n"));
405
406		(void) fprintf(fp, "\n\t%-19s  %s   %s\n\n",
407		    "PROPERTY", "EDIT", "VALUES");
408
409		/* Iterate over all properties */
410		(void) zprop_iter(print_prop_cb, fp, B_FALSE, B_TRUE,
411		    ZFS_TYPE_POOL);
412
413		(void) fprintf(fp, "\t%-19s   ", "feature@...");
414		(void) fprintf(fp, "YES   disabled | enabled | active\n");
415
416		(void) fprintf(fp, gettext("\nThe feature@ properties must be "
417		    "appended with a feature name.\nSee zpool-features(5).\n"));
418	}
419
420	/*
421	 * See comments at end of main().
422	 */
423	if (getenv("ZFS_ABORT") != NULL) {
424		(void) printf("dumping core by request\n");
425		abort();
426	}
427
428	exit(requested ? 0 : 2);
429}
430
431/*
432 * print a pool vdev config for dry runs
433 */
434static void
435print_vdev_tree(zpool_handle_t *zhp, const char *name, nvlist_t *nv, int indent,
436    const char *match, int name_flags)
437{
438	nvlist_t **child;
439	uint_t c, children;
440	char *vname;
441	boolean_t printed = B_FALSE;
442
443	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
444	    &child, &children) != 0) {
445		if (name != NULL)
446			(void) printf("\t%*s%s\n", indent, "", name);
447		return;
448	}
449
450	for (c = 0; c < children; c++) {
451		uint64_t is_log = B_FALSE;
452		char *class = "";
453
454		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
455		    &is_log);
456		if (is_log)
457			class = VDEV_ALLOC_BIAS_LOG;
458		(void) nvlist_lookup_string(child[c],
459		    ZPOOL_CONFIG_ALLOCATION_BIAS, &class);
460		if (strcmp(match, class) != 0)
461			continue;
462
463		if (!printed && name != NULL) {
464			(void) printf("\t%*s%s\n", indent, "", name);
465			printed = B_TRUE;
466		}
467		vname = zpool_vdev_name(g_zfs, zhp, child[c], name_flags);
468		print_vdev_tree(zhp, vname, child[c], indent + 2, "",
469		    name_flags);
470		free(vname);
471	}
472}
473
474static boolean_t
475prop_list_contains_feature(nvlist_t *proplist)
476{
477	nvpair_t *nvp;
478	for (nvp = nvlist_next_nvpair(proplist, NULL); NULL != nvp;
479	    nvp = nvlist_next_nvpair(proplist, nvp)) {
480		if (zpool_prop_feature(nvpair_name(nvp)))
481			return (B_TRUE);
482	}
483	return (B_FALSE);
484}
485
486/*
487 * Add a property pair (name, string-value) into a property nvlist.
488 */
489static int
490add_prop_list(const char *propname, char *propval, nvlist_t **props,
491    boolean_t poolprop)
492{
493	zpool_prop_t prop = ZPOOL_PROP_INVAL;
494	zfs_prop_t fprop;
495	nvlist_t *proplist;
496	const char *normnm;
497	char *strval;
498
499	if (*props == NULL &&
500	    nvlist_alloc(props, NV_UNIQUE_NAME, 0) != 0) {
501		(void) fprintf(stderr,
502		    gettext("internal error: out of memory\n"));
503		return (1);
504	}
505
506	proplist = *props;
507
508	if (poolprop) {
509		const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
510
511		if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
512		    !zpool_prop_feature(propname)) {
513			(void) fprintf(stderr, gettext("property '%s' is "
514			    "not a valid pool property\n"), propname);
515			return (2);
516		}
517
518		/*
519		 * feature@ properties and version should not be specified
520		 * at the same time.
521		 */
522		if ((prop == ZPOOL_PROP_INVAL && zpool_prop_feature(propname) &&
523		    nvlist_exists(proplist, vname)) ||
524		    (prop == ZPOOL_PROP_VERSION &&
525		    prop_list_contains_feature(proplist))) {
526			(void) fprintf(stderr, gettext("'feature@' and "
527			    "'version' properties cannot be specified "
528			    "together\n"));
529			return (2);
530		}
531
532
533		if (zpool_prop_feature(propname))
534			normnm = propname;
535		else
536			normnm = zpool_prop_to_name(prop);
537	} else {
538		if ((fprop = zfs_name_to_prop(propname)) != ZPROP_INVAL) {
539			normnm = zfs_prop_to_name(fprop);
540		} else {
541			normnm = propname;
542		}
543	}
544
545	if (nvlist_lookup_string(proplist, normnm, &strval) == 0 &&
546	    prop != ZPOOL_PROP_CACHEFILE) {
547		(void) fprintf(stderr, gettext("property '%s' "
548		    "specified multiple times\n"), propname);
549		return (2);
550	}
551
552	if (nvlist_add_string(proplist, normnm, propval) != 0) {
553		(void) fprintf(stderr, gettext("internal "
554		    "error: out of memory\n"));
555		return (1);
556	}
557
558	return (0);
559}
560
561/*
562 * Set a default property pair (name, string-value) in a property nvlist
563 */
564static int
565add_prop_list_default(const char *propname, char *propval, nvlist_t **props,
566    boolean_t poolprop)
567{
568	char *pval;
569
570	if (nvlist_lookup_string(*props, propname, &pval) == 0)
571		return (0);
572
573	return (add_prop_list(propname, propval, props, poolprop));
574}
575
576/*
577 * zpool add [-fgLnP] [-o property=value] <pool> <vdev> ...
578 *
579 *	-f	Force addition of devices, even if they appear in use
580 *	-g	Display guid for individual vdev name.
581 *	-L	Follow links when resolving vdev path name.
582 *	-n	Do not add the devices, but display the resulting layout if
583 *		they were to be added.
584 *	-P	Display full path for vdev name.
585 *	-o	Set property=value.
586 *
587 * Adds the given vdevs to 'pool'.  As with create, the bulk of this work is
588 * handled by get_vdev_spec(), which constructs the nvlist needed to pass to
589 * libzfs.
590 */
591int
592zpool_do_add(int argc, char **argv)
593{
594	boolean_t force = B_FALSE;
595	boolean_t dryrun = B_FALSE;
596	int name_flags = 0;
597	int c;
598	nvlist_t *nvroot;
599	char *poolname;
600	zpool_boot_label_t boot_type;
601	uint64_t boot_size;
602	int ret;
603	zpool_handle_t *zhp;
604	nvlist_t *config;
605	nvlist_t *props = NULL;
606	char *propval;
607
608	/* check options */
609	while ((c = getopt(argc, argv, "fgLnPo:")) != -1) {
610		switch (c) {
611		case 'f':
612			force = B_TRUE;
613			break;
614		case 'g':
615			name_flags |= VDEV_NAME_GUID;
616			break;
617		case 'L':
618			name_flags |= VDEV_NAME_FOLLOW_LINKS;
619			break;
620		case 'n':
621			dryrun = B_TRUE;
622			break;
623		case 'P':
624			name_flags |= VDEV_NAME_PATH;
625			break;
626		case 'o':
627			if ((propval = strchr(optarg, '=')) == NULL) {
628				(void) fprintf(stderr, gettext("missing "
629				    "'=' for -o option\n"));
630				usage(B_FALSE);
631			}
632			*propval = '\0';
633			propval++;
634
635			if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
636			    (add_prop_list(optarg, propval, &props, B_TRUE)))
637				usage(B_FALSE);
638			break;
639		case '?':
640			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
641			    optopt);
642			usage(B_FALSE);
643		}
644	}
645
646	argc -= optind;
647	argv += optind;
648
649	/* get pool name and check number of arguments */
650	if (argc < 1) {
651		(void) fprintf(stderr, gettext("missing pool name argument\n"));
652		usage(B_FALSE);
653	}
654	if (argc < 2) {
655		(void) fprintf(stderr, gettext("missing vdev specification\n"));
656		usage(B_FALSE);
657	}
658
659	poolname = argv[0];
660
661	argc--;
662	argv++;
663
664	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
665		return (1);
666
667	if ((config = zpool_get_config(zhp, NULL)) == NULL) {
668		(void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
669		    poolname);
670		zpool_close(zhp);
671		return (1);
672	}
673
674	if (zpool_is_bootable(zhp))
675		boot_type = ZPOOL_COPY_BOOT_LABEL;
676	else
677		boot_type = ZPOOL_NO_BOOT_LABEL;
678
679	/* unless manually specified use "ashift" pool property (if set) */
680	if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
681		int intval;
682		zprop_source_t src;
683		char strval[ZPOOL_MAXPROPLEN];
684
685		intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
686		if (src != ZPROP_SRC_DEFAULT) {
687			(void) sprintf(strval, "%" PRId32, intval);
688			verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
689			    &props, B_TRUE) == 0);
690		}
691	}
692
693	/* pass off to get_vdev_spec for processing */
694	boot_size = zpool_get_prop_int(zhp, ZPOOL_PROP_BOOTSIZE, NULL);
695	nvroot = make_root_vdev(zhp, props, force, !force, B_FALSE, dryrun,
696	    boot_type, boot_size, argc, argv);
697	if (nvroot == NULL) {
698		zpool_close(zhp);
699		return (1);
700	}
701
702	if (dryrun) {
703		nvlist_t *poolnvroot;
704
705		verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
706		    &poolnvroot) == 0);
707
708		(void) printf(gettext("would update '%s' to the following "
709		    "configuration:\n"), zpool_get_name(zhp));
710
711		/* print original main pool and new tree */
712		print_vdev_tree(zhp, poolname, poolnvroot, 0, "",
713		    name_flags | VDEV_NAME_TYPE_ID);
714		print_vdev_tree(zhp, NULL, nvroot, 0, "", name_flags);
715
716		/* print other classes: 'dedup', 'special', and 'log' */
717		print_vdev_tree(zhp, "dedup", poolnvroot, 0,
718		    VDEV_ALLOC_BIAS_DEDUP, name_flags);
719		print_vdev_tree(zhp, NULL, nvroot, 0, VDEV_ALLOC_BIAS_DEDUP,
720		    name_flags);
721
722		print_vdev_tree(zhp, "special", poolnvroot, 0,
723		    VDEV_ALLOC_BIAS_SPECIAL, name_flags);
724		print_vdev_tree(zhp, NULL, nvroot, 0, VDEV_ALLOC_BIAS_SPECIAL,
725		    name_flags);
726
727		print_vdev_tree(zhp, "logs", poolnvroot, 0, VDEV_ALLOC_BIAS_LOG,
728		    name_flags);
729		print_vdev_tree(zhp, NULL, nvroot, 0, VDEV_ALLOC_BIAS_LOG,
730		    name_flags);
731
732		ret = 0;
733	} else {
734		ret = (zpool_add(zhp, nvroot) != 0);
735	}
736
737	nvlist_free(props);
738	nvlist_free(nvroot);
739	zpool_close(zhp);
740
741	return (ret);
742}
743
744/*
745 * zpool remove  <pool> <vdev> ...
746 *
747 * Removes the given vdev from the pool.
748 */
749int
750zpool_do_remove(int argc, char **argv)
751{
752	char *poolname;
753	int i, ret = 0;
754	zpool_handle_t *zhp;
755	boolean_t stop = B_FALSE;
756	boolean_t noop = B_FALSE;
757	boolean_t parsable = B_FALSE;
758	char c;
759
760	/* check options */
761	while ((c = getopt(argc, argv, "nps")) != -1) {
762		switch (c) {
763		case 'n':
764			noop = B_TRUE;
765			break;
766		case 'p':
767			parsable = B_TRUE;
768			break;
769		case 's':
770			stop = B_TRUE;
771			break;
772		case '?':
773			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
774			    optopt);
775			usage(B_FALSE);
776		}
777	}
778
779	argc -= optind;
780	argv += optind;
781
782	/* get pool name and check number of arguments */
783	if (argc < 1) {
784		(void) fprintf(stderr, gettext("missing pool name argument\n"));
785		usage(B_FALSE);
786	}
787
788	poolname = argv[0];
789
790	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
791		return (1);
792
793	if (stop && noop) {
794		(void) fprintf(stderr, gettext("stop request ignored\n"));
795		return (0);
796	}
797
798	if (stop) {
799		if (argc > 1) {
800			(void) fprintf(stderr, gettext("too many arguments\n"));
801			usage(B_FALSE);
802		}
803		if (zpool_vdev_remove_cancel(zhp) != 0)
804			ret = 1;
805	} else {
806		if (argc < 2) {
807			(void) fprintf(stderr, gettext("missing device\n"));
808			usage(B_FALSE);
809		}
810
811		for (i = 1; i < argc; i++) {
812			if (noop) {
813				uint64_t size;
814
815				if (zpool_vdev_indirect_size(zhp, argv[i],
816				    &size) != 0) {
817					ret = 1;
818					break;
819				}
820				if (parsable) {
821					(void) printf("%s %llu\n",
822					    argv[i], size);
823				} else {
824					char valstr[32];
825					zfs_nicenum(size, valstr,
826					    sizeof (valstr));
827					(void) printf("Memory that will be "
828					    "used after removing %s: %s\n",
829					    argv[i], valstr);
830				}
831			} else {
832				if (zpool_vdev_remove(zhp, argv[i]) != 0)
833					ret = 1;
834			}
835		}
836	}
837
838	return (ret);
839}
840
841/*
842 * zpool labelclear [-f] <vdev>
843 *
844 *	-f	Force clearing the label for the vdevs which are members of
845 *		the exported or foreign pools.
846 *
847 * Verifies that the vdev is not active and zeros out the label information
848 * on the device.
849 */
850int
851zpool_do_labelclear(int argc, char **argv)
852{
853	char vdev[MAXPATHLEN];
854	char *name = NULL;
855	struct stat st;
856	int c, fd, ret = 0;
857	nvlist_t *config;
858	pool_state_t state;
859	boolean_t inuse = B_FALSE;
860	boolean_t force = B_FALSE;
861
862	/* check options */
863	while ((c = getopt(argc, argv, "f")) != -1) {
864		switch (c) {
865		case 'f':
866			force = B_TRUE;
867			break;
868		default:
869			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
870			    optopt);
871			usage(B_FALSE);
872		}
873	}
874
875	argc -= optind;
876	argv += optind;
877
878	/* get vdev name */
879	if (argc < 1) {
880		(void) fprintf(stderr, gettext("missing vdev name\n"));
881		usage(B_FALSE);
882	}
883	if (argc > 1) {
884		(void) fprintf(stderr, gettext("too many arguments\n"));
885		usage(B_FALSE);
886	}
887
888	/*
889	 * Check if we were given absolute path and use it as is.
890	 * Otherwise if the provided vdev name doesn't point to a file,
891	 * try prepending dsk path and appending s0.
892	 */
893	(void) strlcpy(vdev, argv[0], sizeof (vdev));
894	if (vdev[0] != '/' && stat(vdev, &st) != 0) {
895		char *s;
896
897		(void) snprintf(vdev, sizeof (vdev), "%s/%s",
898		    ZFS_DISK_ROOT, argv[0]);
899		if ((s = strrchr(argv[0], 's')) == NULL ||
900		    !isdigit(*(s + 1)))
901			(void) strlcat(vdev, "s0", sizeof (vdev));
902		if (stat(vdev, &st) != 0) {
903			(void) fprintf(stderr, gettext(
904			    "failed to find device %s, try specifying absolute "
905			    "path instead\n"), argv[0]);
906			return (1);
907		}
908	}
909
910	if ((fd = open(vdev, O_RDWR)) < 0) {
911		(void) fprintf(stderr, gettext("failed to open %s: %s\n"),
912		    vdev, strerror(errno));
913		return (1);
914	}
915
916	if (zpool_read_label(fd, &config) != 0) {
917		(void) fprintf(stderr,
918		    gettext("failed to read label from %s\n"), vdev);
919		return (1);
920	}
921	nvlist_free(config);
922
923	ret = zpool_in_use(g_zfs, fd, &state, &name, &inuse);
924	if (ret != 0) {
925		(void) fprintf(stderr,
926		    gettext("failed to check state for %s\n"), vdev);
927		return (1);
928	}
929
930	if (!inuse)
931		goto wipe_label;
932
933	switch (state) {
934	default:
935	case POOL_STATE_ACTIVE:
936	case POOL_STATE_SPARE:
937	case POOL_STATE_L2CACHE:
938		(void) fprintf(stderr, gettext(
939		    "%s is a member (%s) of pool \"%s\"\n"),
940		    vdev, zpool_pool_state_to_name(state), name);
941		ret = 1;
942		goto errout;
943
944	case POOL_STATE_EXPORTED:
945		if (force)
946			break;
947		(void) fprintf(stderr, gettext(
948		    "use '-f' to override the following error:\n"
949		    "%s is a member of exported pool \"%s\"\n"),
950		    vdev, name);
951		ret = 1;
952		goto errout;
953
954	case POOL_STATE_POTENTIALLY_ACTIVE:
955		if (force)
956			break;
957		(void) fprintf(stderr, gettext(
958		    "use '-f' to override the following error:\n"
959		    "%s is a member of potentially active pool \"%s\"\n"),
960		    vdev, name);
961		ret = 1;
962		goto errout;
963
964	case POOL_STATE_DESTROYED:
965		/* inuse should never be set for a destroyed pool */
966		assert(0);
967		break;
968	}
969
970wipe_label:
971	ret = zpool_clear_label(fd);
972	if (ret != 0) {
973		(void) fprintf(stderr,
974		    gettext("failed to clear label for %s\n"), vdev);
975	}
976
977errout:
978	free(name);
979	(void) close(fd);
980
981	return (ret);
982}
983
984/*
985 * zpool create [-fnd] [-B] [-o property=value] ...
986 *		[-O file-system-property=value] ...
987 *		[-R root] [-m mountpoint] [-t tempname] <pool> <dev> ...
988 *
989 *	-B	Create boot partition.
990 *	-f	Force creation, even if devices appear in use
991 *	-n	Do not create the pool, but display the resulting layout if it
992 *		were to be created.
993 *	-R	Create a pool under an alternate root
994 *	-m	Set default mountpoint for the root dataset.  By default it's
995 *		'/<pool>'
996 *	-t	Use the temporary name until the pool is exported.
997 *	-o	Set property=value.
998 *	-d	Don't automatically enable all supported pool features
999 *		(individual features can be enabled with -o).
1000 *	-O	Set fsproperty=value in the pool's root file system
1001 *
1002 * Creates the named pool according to the given vdev specification.  The
1003 * bulk of the vdev processing is done in get_vdev_spec() in zpool_vdev.c.  Once
1004 * we get the nvlist back from get_vdev_spec(), we either print out the contents
1005 * (if '-n' was specified), or pass it to libzfs to do the creation.
1006 */
1007
1008#define	SYSTEM256	(256 * 1024 * 1024)
1009int
1010zpool_do_create(int argc, char **argv)
1011{
1012	boolean_t force = B_FALSE;
1013	boolean_t dryrun = B_FALSE;
1014	boolean_t enable_all_pool_feat = B_TRUE;
1015	zpool_boot_label_t boot_type = ZPOOL_NO_BOOT_LABEL;
1016	uint64_t boot_size = 0;
1017	int c;
1018	nvlist_t *nvroot = NULL;
1019	char *poolname;
1020	char *tname = NULL;
1021	int ret = 1;
1022	char *altroot = NULL;
1023	char *mountpoint = NULL;
1024	nvlist_t *fsprops = NULL;
1025	nvlist_t *props = NULL;
1026	char *propval;
1027
1028	/* check options */
1029	while ((c = getopt(argc, argv, ":fndBR:m:o:O:t:")) != -1) {
1030		switch (c) {
1031		case 'f':
1032			force = B_TRUE;
1033			break;
1034		case 'n':
1035			dryrun = B_TRUE;
1036			break;
1037		case 'd':
1038			enable_all_pool_feat = B_FALSE;
1039			break;
1040		case 'B':
1041			/*
1042			 * We should create the system partition.
1043			 * Also make sure the size is set.
1044			 */
1045			boot_type = ZPOOL_CREATE_BOOT_LABEL;
1046			if (boot_size == 0)
1047				boot_size = SYSTEM256;
1048			break;
1049		case 'R':
1050			altroot = optarg;
1051			if (add_prop_list(zpool_prop_to_name(
1052			    ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
1053				goto errout;
1054			if (add_prop_list_default(zpool_prop_to_name(
1055			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
1056				goto errout;
1057			break;
1058		case 'm':
1059			/* Equivalent to -O mountpoint=optarg */
1060			mountpoint = optarg;
1061			break;
1062		case 'o':
1063			if ((propval = strchr(optarg, '=')) == NULL) {
1064				(void) fprintf(stderr, gettext("missing "
1065				    "'=' for -o option\n"));
1066				goto errout;
1067			}
1068			*propval = '\0';
1069			propval++;
1070
1071			if (add_prop_list(optarg, propval, &props, B_TRUE))
1072				goto errout;
1073
1074			/*
1075			 * Get bootsize value for make_root_vdev().
1076			 */
1077			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_BOOTSIZE) {
1078				if (zfs_nicestrtonum(g_zfs, propval,
1079				    &boot_size) < 0 || boot_size == 0) {
1080					(void) fprintf(stderr,
1081					    gettext("bad boot partition size "
1082					    "'%s': %s\n"),  propval,
1083					    libzfs_error_description(g_zfs));
1084					goto errout;
1085				}
1086			}
1087
1088			/*
1089			 * If the user is creating a pool that doesn't support
1090			 * feature flags, don't enable any features.
1091			 */
1092			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_VERSION) {
1093				char *end;
1094				u_longlong_t ver;
1095
1096				ver = strtoull(propval, &end, 10);
1097				if (*end == '\0' &&
1098				    ver < SPA_VERSION_FEATURES) {
1099					enable_all_pool_feat = B_FALSE;
1100				}
1101			}
1102			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_ALTROOT)
1103				altroot = propval;
1104			break;
1105		case 'O':
1106			if ((propval = strchr(optarg, '=')) == NULL) {
1107				(void) fprintf(stderr, gettext("missing "
1108				    "'=' for -O option\n"));
1109				goto errout;
1110			}
1111			*propval = '\0';
1112			propval++;
1113
1114			/*
1115			 * Mountpoints are checked and then added later.
1116			 * Uniquely among properties, they can be specified
1117			 * more than once, to avoid conflict with -m.
1118			 */
1119			if (0 == strcmp(optarg,
1120			    zfs_prop_to_name(ZFS_PROP_MOUNTPOINT))) {
1121				mountpoint = propval;
1122			} else if (add_prop_list(optarg, propval, &fsprops,
1123			    B_FALSE)) {
1124				goto errout;
1125			}
1126			break;
1127		case 't':
1128			/*
1129			 * Sanity check temporary pool name.
1130			 */
1131			if (strchr(optarg, '/') != NULL) {
1132				(void) fprintf(stderr, gettext("cannot create "
1133				    "'%s': invalid character '/' in temporary "
1134				    "name\n"), optarg);
1135				(void) fprintf(stderr, gettext("use 'zfs "
1136				    "create' to create a dataset\n"));
1137				goto errout;
1138			}
1139
1140			if (add_prop_list(zpool_prop_to_name(
1141			    ZPOOL_PROP_TNAME), optarg, &props, B_TRUE))
1142				goto errout;
1143			if (add_prop_list_default(zpool_prop_to_name(
1144			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
1145				goto errout;
1146			tname = optarg;
1147			break;
1148		case ':':
1149			(void) fprintf(stderr, gettext("missing argument for "
1150			    "'%c' option\n"), optopt);
1151			goto badusage;
1152		case '?':
1153			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
1154			    optopt);
1155			goto badusage;
1156		}
1157	}
1158
1159	argc -= optind;
1160	argv += optind;
1161
1162	/* get pool name and check number of arguments */
1163	if (argc < 1) {
1164		(void) fprintf(stderr, gettext("missing pool name argument\n"));
1165		goto badusage;
1166	}
1167	if (argc < 2) {
1168		(void) fprintf(stderr, gettext("missing vdev specification\n"));
1169		goto badusage;
1170	}
1171
1172	poolname = argv[0];
1173
1174	/*
1175	 * As a special case, check for use of '/' in the name, and direct the
1176	 * user to use 'zfs create' instead.
1177	 */
1178	if (strchr(poolname, '/') != NULL) {
1179		(void) fprintf(stderr, gettext("cannot create '%s': invalid "
1180		    "character '/' in pool name\n"), poolname);
1181		(void) fprintf(stderr, gettext("use 'zfs create' to "
1182		    "create a dataset\n"));
1183		goto errout;
1184	}
1185
1186	/*
1187	 * Make sure the bootsize is set when ZPOOL_CREATE_BOOT_LABEL is used,
1188	 * and not set otherwise.
1189	 */
1190	if (boot_type == ZPOOL_CREATE_BOOT_LABEL) {
1191		const char *propname;
1192		char *strptr, *buf = NULL;
1193		int rv;
1194
1195		propname = zpool_prop_to_name(ZPOOL_PROP_BOOTSIZE);
1196		if (nvlist_lookup_string(props, propname, &strptr) != 0) {
1197			(void) asprintf(&buf, "%" PRIu64, boot_size);
1198			if (buf == NULL) {
1199				(void) fprintf(stderr,
1200				    gettext("internal error: out of memory\n"));
1201				goto errout;
1202			}
1203			rv = add_prop_list(propname, buf, &props, B_TRUE);
1204			free(buf);
1205			if (rv != 0)
1206				goto errout;
1207		}
1208	} else {
1209		const char *propname;
1210		char *strptr;
1211
1212		propname = zpool_prop_to_name(ZPOOL_PROP_BOOTSIZE);
1213		if (nvlist_lookup_string(props, propname, &strptr) == 0) {
1214			(void) fprintf(stderr, gettext("error: setting boot "
1215			    "partition size requires option '-B'\n"));
1216			goto errout;
1217		}
1218	}
1219
1220	/* pass off to get_vdev_spec for bulk processing */
1221	nvroot = make_root_vdev(NULL, props, force, !force, B_FALSE, dryrun,
1222	    boot_type, boot_size, argc - 1, argv + 1);
1223	if (nvroot == NULL)
1224		goto errout;
1225
1226	/* make_root_vdev() allows 0 toplevel children if there are spares */
1227	if (!zfs_allocatable_devs(nvroot)) {
1228		(void) fprintf(stderr, gettext("invalid vdev "
1229		    "specification: at least one toplevel vdev must be "
1230		    "specified\n"));
1231		goto errout;
1232	}
1233
1234	if (altroot != NULL && altroot[0] != '/') {
1235		(void) fprintf(stderr, gettext("invalid alternate root '%s': "
1236		    "must be an absolute path\n"), altroot);
1237		goto errout;
1238	}
1239
1240	/*
1241	 * Check the validity of the mountpoint and direct the user to use the
1242	 * '-m' mountpoint option if it looks like its in use.
1243	 */
1244	if (mountpoint == NULL ||
1245	    (strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0 &&
1246	    strcmp(mountpoint, ZFS_MOUNTPOINT_NONE) != 0)) {
1247		char buf[MAXPATHLEN];
1248		DIR *dirp;
1249
1250		if (mountpoint && mountpoint[0] != '/') {
1251			(void) fprintf(stderr, gettext("invalid mountpoint "
1252			    "'%s': must be an absolute path, 'legacy', or "
1253			    "'none'\n"), mountpoint);
1254			goto errout;
1255		}
1256
1257		if (mountpoint == NULL) {
1258			if (altroot != NULL)
1259				(void) snprintf(buf, sizeof (buf), "%s/%s",
1260				    altroot, poolname);
1261			else
1262				(void) snprintf(buf, sizeof (buf), "/%s",
1263				    poolname);
1264		} else {
1265			if (altroot != NULL)
1266				(void) snprintf(buf, sizeof (buf), "%s%s",
1267				    altroot, mountpoint);
1268			else
1269				(void) snprintf(buf, sizeof (buf), "%s",
1270				    mountpoint);
1271		}
1272
1273		if ((dirp = opendir(buf)) == NULL && errno != ENOENT) {
1274			(void) fprintf(stderr, gettext("mountpoint '%s' : "
1275			    "%s\n"), buf, strerror(errno));
1276			(void) fprintf(stderr, gettext("use '-m' "
1277			    "option to provide a different default\n"));
1278			goto errout;
1279		} else if (dirp) {
1280			int count = 0;
1281
1282			while (count < 3 && readdir(dirp) != NULL)
1283				count++;
1284			(void) closedir(dirp);
1285
1286			if (count > 2) {
1287				(void) fprintf(stderr, gettext("mountpoint "
1288				    "'%s' exists and is not empty\n"), buf);
1289				(void) fprintf(stderr, gettext("use '-m' "
1290				    "option to provide a "
1291				    "different default\n"));
1292				goto errout;
1293			}
1294		}
1295	}
1296
1297	/*
1298	 * Now that the mountpoint's validity has been checked, ensure that
1299	 * the property is set appropriately prior to creating the pool.
1300	 */
1301	if (mountpoint != NULL) {
1302		ret = add_prop_list(zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1303		    mountpoint, &fsprops, B_FALSE);
1304		if (ret != 0)
1305			goto errout;
1306	}
1307
1308	ret = 1;
1309	if (dryrun) {
1310		/*
1311		 * For a dry run invocation, print out a basic message and run
1312		 * through all the vdevs in the list and print out in an
1313		 * appropriate hierarchy.
1314		 */
1315		(void) printf(gettext("would create '%s' with the "
1316		    "following layout:\n\n"), poolname);
1317
1318		print_vdev_tree(NULL, poolname, nvroot, 0, "", 0);
1319		print_vdev_tree(NULL, "dedup", nvroot, 0,
1320		    VDEV_ALLOC_BIAS_DEDUP, 0);
1321		print_vdev_tree(NULL, "special", nvroot, 0,
1322		    VDEV_ALLOC_BIAS_SPECIAL, 0);
1323		print_vdev_tree(NULL, "logs", nvroot, 0,
1324		    VDEV_ALLOC_BIAS_LOG, 0);
1325
1326		ret = 0;
1327	} else {
1328		/*
1329		 * Hand off to libzfs.
1330		 */
1331		if (enable_all_pool_feat) {
1332			spa_feature_t i;
1333			for (i = 0; i < SPA_FEATURES; i++) {
1334				char propname[MAXPATHLEN];
1335				zfeature_info_t *feat = &spa_feature_table[i];
1336
1337				(void) snprintf(propname, sizeof (propname),
1338				    "feature@%s", feat->fi_uname);
1339
1340				/*
1341				 * Skip feature if user specified it manually
1342				 * on the command line.
1343				 */
1344				if (nvlist_exists(props, propname))
1345					continue;
1346
1347				ret = add_prop_list(propname,
1348				    ZFS_FEATURE_ENABLED, &props, B_TRUE);
1349				if (ret != 0)
1350					goto errout;
1351			}
1352		}
1353
1354		ret = 1;
1355		if (zpool_create(g_zfs, poolname,
1356		    nvroot, props, fsprops) == 0) {
1357			zfs_handle_t *pool = zfs_open(g_zfs,
1358			    tname ? tname : poolname, ZFS_TYPE_FILESYSTEM);
1359			if (pool != NULL) {
1360				if (zfs_mount(pool, NULL, 0) == 0)
1361					ret = zfs_shareall(pool);
1362				zfs_close(pool);
1363			}
1364		} else if (libzfs_errno(g_zfs) == EZFS_INVALIDNAME) {
1365			(void) fprintf(stderr, gettext("pool name may have "
1366			    "been omitted\n"));
1367		}
1368	}
1369
1370errout:
1371	nvlist_free(nvroot);
1372	nvlist_free(fsprops);
1373	nvlist_free(props);
1374	return (ret);
1375badusage:
1376	nvlist_free(fsprops);
1377	nvlist_free(props);
1378	usage(B_FALSE);
1379	return (2);
1380}
1381
1382/*
1383 * zpool destroy <pool>
1384 *
1385 *	-f	Forcefully unmount any datasets
1386 *
1387 * Destroy the given pool.  Automatically unmounts any datasets in the pool.
1388 */
1389int
1390zpool_do_destroy(int argc, char **argv)
1391{
1392	boolean_t force = B_FALSE;
1393	int c;
1394	char *pool;
1395	zpool_handle_t *zhp;
1396	int ret;
1397
1398	/* check options */
1399	while ((c = getopt(argc, argv, "f")) != -1) {
1400		switch (c) {
1401		case 'f':
1402			force = B_TRUE;
1403			break;
1404		case '?':
1405			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
1406			    optopt);
1407			usage(B_FALSE);
1408		}
1409	}
1410
1411	argc -= optind;
1412	argv += optind;
1413
1414	/* check arguments */
1415	if (argc < 1) {
1416		(void) fprintf(stderr, gettext("missing pool argument\n"));
1417		usage(B_FALSE);
1418	}
1419	if (argc > 1) {
1420		(void) fprintf(stderr, gettext("too many arguments\n"));
1421		usage(B_FALSE);
1422	}
1423
1424	pool = argv[0];
1425
1426	if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
1427		/*
1428		 * As a special case, check for use of '/' in the name, and
1429		 * direct the user to use 'zfs destroy' instead.
1430		 */
1431		if (strchr(pool, '/') != NULL)
1432			(void) fprintf(stderr, gettext("use 'zfs destroy' to "
1433			    "destroy a dataset\n"));
1434		return (1);
1435	}
1436
1437	if (zpool_disable_datasets(zhp, force) != 0) {
1438		(void) fprintf(stderr, gettext("could not destroy '%s': "
1439		    "could not unmount datasets\n"), zpool_get_name(zhp));
1440		return (1);
1441	}
1442
1443	/* The history must be logged as part of the export */
1444	log_history = B_FALSE;
1445
1446	ret = (zpool_destroy(zhp, history_str) != 0);
1447
1448	zpool_close(zhp);
1449
1450	return (ret);
1451}
1452
1453/*
1454 * zpool export [-f] <pool> ...
1455 *
1456 *	-f	Forcefully unmount datasets
1457 *
1458 * Export the given pools.  By default, the command will attempt to cleanly
1459 * unmount any active datasets within the pool.  If the '-f' flag is specified,
1460 * then the datasets will be forcefully unmounted.
1461 */
1462int
1463zpool_do_export(int argc, char **argv)
1464{
1465	boolean_t force = B_FALSE;
1466	boolean_t hardforce = B_FALSE;
1467	int c;
1468	zpool_handle_t *zhp;
1469	int ret;
1470	int i;
1471
1472	/* check options */
1473	while ((c = getopt(argc, argv, "fF")) != -1) {
1474		switch (c) {
1475		case 'f':
1476			force = B_TRUE;
1477			break;
1478		case 'F':
1479			hardforce = B_TRUE;
1480			break;
1481		case '?':
1482			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
1483			    optopt);
1484			usage(B_FALSE);
1485		}
1486	}
1487
1488	argc -= optind;
1489	argv += optind;
1490
1491	/* check arguments */
1492	if (argc < 1) {
1493		(void) fprintf(stderr, gettext("missing pool argument\n"));
1494		usage(B_FALSE);
1495	}
1496
1497	ret = 0;
1498	for (i = 0; i < argc; i++) {
1499		if ((zhp = zpool_open_canfail(g_zfs, argv[i])) == NULL) {
1500			ret = 1;
1501			continue;
1502		}
1503
1504		if (zpool_disable_datasets(zhp, force) != 0) {
1505			ret = 1;
1506			zpool_close(zhp);
1507			continue;
1508		}
1509
1510		/* The history must be logged as part of the export */
1511		log_history = B_FALSE;
1512
1513		if (hardforce) {
1514			if (zpool_export_force(zhp, history_str) != 0)
1515				ret = 1;
1516		} else if (zpool_export(zhp, force, history_str) != 0) {
1517			ret = 1;
1518		}
1519
1520		zpool_close(zhp);
1521	}
1522
1523	return (ret);
1524}
1525
1526/*
1527 * Given a vdev configuration, determine the maximum width needed for the device
1528 * name column.
1529 */
1530static int
1531max_width(zpool_handle_t *zhp, nvlist_t *nv, int depth, int max,
1532    int name_flags)
1533{
1534	char *name;
1535	nvlist_t **child;
1536	uint_t c, children;
1537	int ret;
1538
1539	name = zpool_vdev_name(g_zfs, zhp, nv, name_flags | VDEV_NAME_TYPE_ID);
1540	if (strlen(name) + depth > max)
1541		max = strlen(name) + depth;
1542
1543	free(name);
1544
1545	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
1546	    &child, &children) == 0) {
1547		for (c = 0; c < children; c++)
1548			if ((ret = max_width(zhp, child[c], depth + 2,
1549			    max, name_flags)) > max)
1550				max = ret;
1551	}
1552
1553	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
1554	    &child, &children) == 0) {
1555		for (c = 0; c < children; c++)
1556			if ((ret = max_width(zhp, child[c], depth + 2,
1557			    max, name_flags)) > max)
1558				max = ret;
1559	}
1560
1561	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1562	    &child, &children) == 0) {
1563		for (c = 0; c < children; c++)
1564			if ((ret = max_width(zhp, child[c], depth + 2,
1565			    max, name_flags)) > max)
1566				max = ret;
1567	}
1568
1569	return (max);
1570}
1571
1572typedef struct spare_cbdata {
1573	uint64_t	cb_guid;
1574	zpool_handle_t	*cb_zhp;
1575} spare_cbdata_t;
1576
1577static boolean_t
1578find_vdev(nvlist_t *nv, uint64_t search)
1579{
1580	uint64_t guid;
1581	nvlist_t **child;
1582	uint_t c, children;
1583
1584	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
1585	    search == guid)
1586		return (B_TRUE);
1587
1588	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1589	    &child, &children) == 0) {
1590		for (c = 0; c < children; c++)
1591			if (find_vdev(child[c], search))
1592				return (B_TRUE);
1593	}
1594
1595	return (B_FALSE);
1596}
1597
1598static int
1599find_spare(zpool_handle_t *zhp, void *data)
1600{
1601	spare_cbdata_t *cbp = data;
1602	nvlist_t *config, *nvroot;
1603
1604	config = zpool_get_config(zhp, NULL);
1605	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
1606	    &nvroot) == 0);
1607
1608	if (find_vdev(nvroot, cbp->cb_guid)) {
1609		cbp->cb_zhp = zhp;
1610		return (1);
1611	}
1612
1613	zpool_close(zhp);
1614	return (0);
1615}
1616
1617typedef struct status_cbdata {
1618	int		cb_count;
1619	int		cb_name_flags;
1620	int		cb_namewidth;
1621	boolean_t	cb_allpools;
1622	boolean_t	cb_verbose;
1623	boolean_t	cb_explain;
1624	boolean_t	cb_first;
1625	boolean_t	cb_dedup_stats;
1626	boolean_t	cb_print_status;
1627	boolean_t	cb_print_vdev_init;
1628	boolean_t	cb_print_vdev_trim;
1629} status_cbdata_t;
1630
1631/*
1632 * Print vdev initialization status for leaves
1633 */
1634static void
1635print_status_initialize(vdev_stat_t *vs, boolean_t verbose)
1636{
1637	if (verbose) {
1638		if ((vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE ||
1639		    vs->vs_initialize_state == VDEV_INITIALIZE_SUSPENDED ||
1640		    vs->vs_initialize_state == VDEV_INITIALIZE_COMPLETE) &&
1641		    !vs->vs_scan_removing) {
1642			char zbuf[1024];
1643			char tbuf[256];
1644			struct tm zaction_ts;
1645
1646			time_t t = vs->vs_initialize_action_time;
1647			int initialize_pct = 100;
1648			if (vs->vs_initialize_state !=
1649			    VDEV_INITIALIZE_COMPLETE) {
1650				initialize_pct = (vs->vs_initialize_bytes_done *
1651				    100 / (vs->vs_initialize_bytes_est + 1));
1652			}
1653
1654			(void) localtime_r(&t, &zaction_ts);
1655			(void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
1656
1657			switch (vs->vs_initialize_state) {
1658			case VDEV_INITIALIZE_SUSPENDED:
1659				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1660				    gettext("suspended, started at"), tbuf);
1661				break;
1662			case VDEV_INITIALIZE_ACTIVE:
1663				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1664				    gettext("started at"), tbuf);
1665				break;
1666			case VDEV_INITIALIZE_COMPLETE:
1667				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1668				    gettext("completed at"), tbuf);
1669				break;
1670			}
1671
1672			(void) printf(gettext("  (%d%% initialized%s)"),
1673			    initialize_pct, zbuf);
1674		} else {
1675			(void) printf(gettext("  (uninitialized)"));
1676		}
1677	} else if (vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE) {
1678		(void) printf(gettext("  (initializing)"));
1679	}
1680}
1681
1682/*
1683 * Print vdev TRIM status for leaves
1684 */
1685static void
1686print_status_trim(vdev_stat_t *vs, boolean_t verbose)
1687{
1688	if (verbose) {
1689		if ((vs->vs_trim_state == VDEV_TRIM_ACTIVE ||
1690		    vs->vs_trim_state == VDEV_TRIM_SUSPENDED ||
1691		    vs->vs_trim_state == VDEV_TRIM_COMPLETE) &&
1692		    !vs->vs_scan_removing) {
1693			char zbuf[1024];
1694			char tbuf[256];
1695			struct tm zaction_ts;
1696
1697			time_t t = vs->vs_trim_action_time;
1698			int trim_pct = 100;
1699			if (vs->vs_trim_state != VDEV_TRIM_COMPLETE) {
1700				trim_pct = (vs->vs_trim_bytes_done *
1701				    100 / (vs->vs_trim_bytes_est + 1));
1702			}
1703
1704			(void) localtime_r(&t, &zaction_ts);
1705			(void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
1706
1707			switch (vs->vs_trim_state) {
1708			case VDEV_TRIM_SUSPENDED:
1709				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1710				    gettext("suspended, started at"), tbuf);
1711				break;
1712			case VDEV_TRIM_ACTIVE:
1713				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1714				    gettext("started at"), tbuf);
1715				break;
1716			case VDEV_TRIM_COMPLETE:
1717				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1718				    gettext("completed at"), tbuf);
1719				break;
1720			}
1721
1722			(void) printf(gettext("  (%d%% trimmed%s)"),
1723			    trim_pct, zbuf);
1724		} else if (vs->vs_trim_notsup) {
1725			(void) printf(gettext("  (trim unsupported)"));
1726		} else {
1727			(void) printf(gettext("  (untrimmed)"));
1728		}
1729	} else if (vs->vs_trim_state == VDEV_TRIM_ACTIVE) {
1730		(void) printf(gettext("  (trimming)"));
1731	}
1732}
1733
1734/*
1735 * Print out configuration state as requested by status_callback.
1736 */
1737static void
1738print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
1739    nvlist_t *nv, int depth, boolean_t isspare)
1740{
1741	nvlist_t **child, *root;
1742	uint_t c, children;
1743	pool_scan_stat_t *ps = NULL;
1744	vdev_stat_t *vs;
1745	char rbuf[6], wbuf[6], cbuf[6];
1746	char *vname;
1747	uint64_t notpresent;
1748	spare_cbdata_t spare_cb;
1749	const char *state;
1750	char *type;
1751
1752	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1753	    &child, &children) != 0)
1754		children = 0;
1755
1756	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
1757	    (uint64_t **)&vs, &c) == 0);
1758
1759	verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
1760
1761	if (strcmp(type, VDEV_TYPE_INDIRECT) == 0)
1762		return;
1763
1764	state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
1765	if (isspare) {
1766		/*
1767		 * For hot spares, we use the terms 'INUSE' and 'AVAILABLE' for
1768		 * online drives.
1769		 */
1770		if (vs->vs_aux == VDEV_AUX_SPARED)
1771			state = "INUSE";
1772		else if (vs->vs_state == VDEV_STATE_HEALTHY)
1773			state = "AVAIL";
1774	}
1775
1776	(void) printf("\t%*s%-*s  %-8s", depth, "", cb->cb_namewidth - depth,
1777	    name, state);
1778
1779	if (!isspare) {
1780		zfs_nicenum(vs->vs_read_errors, rbuf, sizeof (rbuf));
1781		zfs_nicenum(vs->vs_write_errors, wbuf, sizeof (wbuf));
1782		zfs_nicenum(vs->vs_checksum_errors, cbuf, sizeof (cbuf));
1783		(void) printf(" %5s %5s %5s", rbuf, wbuf, cbuf);
1784	}
1785
1786	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
1787	    &notpresent) == 0) {
1788		char *path;
1789		verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0);
1790		(void) printf("  was %s", path);
1791	} else if (vs->vs_aux != 0) {
1792		(void) printf("  ");
1793
1794		switch (vs->vs_aux) {
1795		case VDEV_AUX_OPEN_FAILED:
1796			(void) printf(gettext("cannot open"));
1797			break;
1798
1799		case VDEV_AUX_BAD_GUID_SUM:
1800			(void) printf(gettext("missing device"));
1801			break;
1802
1803		case VDEV_AUX_NO_REPLICAS:
1804			(void) printf(gettext("insufficient replicas"));
1805			break;
1806
1807		case VDEV_AUX_VERSION_NEWER:
1808			(void) printf(gettext("newer version"));
1809			break;
1810
1811		case VDEV_AUX_UNSUP_FEAT:
1812			(void) printf(gettext("unsupported feature(s)"));
1813			break;
1814
1815		case VDEV_AUX_SPARED:
1816			verify(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID,
1817			    &spare_cb.cb_guid) == 0);
1818			if (zpool_iter(g_zfs, find_spare, &spare_cb) == 1) {
1819				if (strcmp(zpool_get_name(spare_cb.cb_zhp),
1820				    zpool_get_name(zhp)) == 0)
1821					(void) printf(gettext("currently in "
1822					    "use"));
1823				else
1824					(void) printf(gettext("in use by "
1825					    "pool '%s'"),
1826					    zpool_get_name(spare_cb.cb_zhp));
1827				zpool_close(spare_cb.cb_zhp);
1828			} else {
1829				(void) printf(gettext("currently in use"));
1830			}
1831			break;
1832
1833		case VDEV_AUX_ERR_EXCEEDED:
1834			(void) printf(gettext("too many errors"));
1835			break;
1836
1837		case VDEV_AUX_IO_FAILURE:
1838			(void) printf(gettext("experienced I/O failures"));
1839			break;
1840
1841		case VDEV_AUX_BAD_LOG:
1842			(void) printf(gettext("bad intent log"));
1843			break;
1844
1845		case VDEV_AUX_EXTERNAL:
1846			(void) printf(gettext("external device fault"));
1847			break;
1848
1849		case VDEV_AUX_SPLIT_POOL:
1850			(void) printf(gettext("split into new pool"));
1851			break;
1852
1853		case VDEV_AUX_ACTIVE:
1854			(void) printf(gettext("currently in use"));
1855			break;
1856
1857		case VDEV_AUX_CHILDREN_OFFLINE:
1858			(void) printf(gettext("all children offline"));
1859			break;
1860
1861		default:
1862			(void) printf(gettext("corrupted data"));
1863			break;
1864		}
1865	}
1866
1867	/* The root vdev has the scrub/resilver stats */
1868	root = fnvlist_lookup_nvlist(zpool_get_config(zhp, NULL),
1869	    ZPOOL_CONFIG_VDEV_TREE);
1870	(void) nvlist_lookup_uint64_array(root, ZPOOL_CONFIG_SCAN_STATS,
1871	    (uint64_t **)&ps, &c);
1872
1873	if (ps != NULL && ps->pss_state == DSS_SCANNING && children == 0) {
1874		if (vs->vs_scan_processed != 0) {
1875			(void) printf(gettext("  (%s)"),
1876			    (ps->pss_func == POOL_SCAN_RESILVER) ?
1877			    "resilvering" : "repairing");
1878		} else if (vs->vs_resilver_deferred) {
1879			(void) printf(gettext("  (awaiting resilver)"));
1880		}
1881	}
1882
1883	/* Display vdev initialization and trim status for leaves */
1884	if (children == 0) {
1885		print_status_initialize(vs, cb->cb_print_vdev_init);
1886		print_status_trim(vs, cb->cb_print_vdev_trim);
1887	}
1888
1889	(void) printf("\n");
1890
1891	for (c = 0; c < children; c++) {
1892		uint64_t islog = B_FALSE, ishole = B_FALSE;
1893
1894		/* Don't print logs or holes here */
1895		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
1896		    &islog);
1897		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
1898		    &ishole);
1899		if (islog || ishole)
1900			continue;
1901		/* Only print normal classes here */
1902		if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
1903			continue;
1904
1905		vname = zpool_vdev_name(g_zfs, zhp, child[c],
1906		    cb->cb_name_flags | VDEV_NAME_TYPE_ID);
1907
1908		print_status_config(zhp, cb, vname, child[c], depth + 2,
1909		    isspare);
1910		free(vname);
1911	}
1912}
1913
1914/*
1915 * Print the configuration of an exported pool.  Iterate over all vdevs in the
1916 * pool, printing out the name and status for each one.
1917 */
1918static void
1919print_import_config(status_cbdata_t *cb, const char *name, nvlist_t *nv,
1920    int depth)
1921{
1922	nvlist_t **child;
1923	uint_t c, children;
1924	vdev_stat_t *vs;
1925	char *type, *vname;
1926
1927	verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
1928	if (strcmp(type, VDEV_TYPE_MISSING) == 0 ||
1929	    strcmp(type, VDEV_TYPE_HOLE) == 0)
1930		return;
1931
1932	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
1933	    (uint64_t **)&vs, &c) == 0);
1934
1935	(void) printf("\t%*s%-*s", depth, "", cb->cb_namewidth - depth, name);
1936	(void) printf("  %s", zpool_state_to_name(vs->vs_state, vs->vs_aux));
1937
1938	if (vs->vs_aux != 0) {
1939		(void) printf("  ");
1940
1941		switch (vs->vs_aux) {
1942		case VDEV_AUX_OPEN_FAILED:
1943			(void) printf(gettext("cannot open"));
1944			break;
1945
1946		case VDEV_AUX_BAD_GUID_SUM:
1947			(void) printf(gettext("missing device"));
1948			break;
1949
1950		case VDEV_AUX_NO_REPLICAS:
1951			(void) printf(gettext("insufficient replicas"));
1952			break;
1953
1954		case VDEV_AUX_VERSION_NEWER:
1955			(void) printf(gettext("newer version"));
1956			break;
1957
1958		case VDEV_AUX_UNSUP_FEAT:
1959			(void) printf(gettext("unsupported feature(s)"));
1960			break;
1961
1962		case VDEV_AUX_ERR_EXCEEDED:
1963			(void) printf(gettext("too many errors"));
1964			break;
1965
1966		case VDEV_AUX_ACTIVE:
1967			(void) printf(gettext("currently in use"));
1968			break;
1969
1970		case VDEV_AUX_CHILDREN_OFFLINE:
1971			(void) printf(gettext("all children offline"));
1972			break;
1973
1974		default:
1975			(void) printf(gettext("corrupted data"));
1976			break;
1977		}
1978	}
1979	(void) printf("\n");
1980
1981	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1982	    &child, &children) != 0)
1983		return;
1984
1985	for (c = 0; c < children; c++) {
1986		uint64_t is_log = B_FALSE;
1987
1988		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
1989		    &is_log);
1990		if (is_log)
1991			continue;
1992		if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
1993			continue;
1994
1995		vname = zpool_vdev_name(g_zfs, NULL, child[c],
1996		    cb->cb_name_flags | VDEV_NAME_TYPE_ID);
1997		print_import_config(cb, vname, child[c], depth + 2);
1998		free(vname);
1999	}
2000
2001	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
2002	    &child, &children) == 0) {
2003		(void) printf(gettext("\tcache\n"));
2004		for (c = 0; c < children; c++) {
2005			vname = zpool_vdev_name(g_zfs, NULL, child[c],
2006			    cb->cb_name_flags);
2007			(void) printf("\t  %s\n", vname);
2008			free(vname);
2009		}
2010	}
2011
2012	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
2013	    &child, &children) == 0) {
2014		(void) printf(gettext("\tspares\n"));
2015		for (c = 0; c < children; c++) {
2016			vname = zpool_vdev_name(g_zfs, NULL, child[c],
2017			    cb->cb_name_flags);
2018			(void) printf("\t  %s\n", vname);
2019			free(vname);
2020		}
2021	}
2022}
2023
2024/*
2025 * Print specialized class vdevs.
2026 *
2027 * These are recorded as top level vdevs in the main pool child array
2028 * but with "is_log" set to 1 or an "alloc_bias" string. We use either
2029 * print_status_config() or print_import_config() to print the top level
2030 * class vdevs then any of their children (eg mirrored slogs) are printed
2031 * recursively - which works because only the top level vdev is marked.
2032 */
2033static void
2034print_class_vdevs(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t *nv,
2035    const char *class)
2036{
2037	uint_t c, children;
2038	nvlist_t **child;
2039	boolean_t printed = B_FALSE;
2040
2041	assert(zhp != NULL || !cb->cb_verbose);
2042
2043	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, &child,
2044	    &children) != 0)
2045		return;
2046
2047	for (c = 0; c < children; c++) {
2048		uint64_t is_log = B_FALSE;
2049		char *bias = NULL;
2050		char *type = NULL;
2051
2052		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2053		    &is_log);
2054
2055		if (is_log) {
2056			bias = VDEV_ALLOC_CLASS_LOGS;
2057		} else {
2058			(void) nvlist_lookup_string(child[c],
2059			    ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
2060			(void) nvlist_lookup_string(child[c],
2061			    ZPOOL_CONFIG_TYPE, &type);
2062		}
2063
2064		if (bias == NULL || strcmp(bias, class) != 0)
2065			continue;
2066		if (!is_log && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
2067			continue;
2068
2069		if (!printed) {
2070			(void) printf("\t%s\t\n", gettext(class));
2071			printed = B_TRUE;
2072		}
2073
2074		char *name = zpool_vdev_name(g_zfs, zhp, child[c],
2075		    cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2076		if (cb->cb_print_status)
2077			print_status_config(zhp, cb, name, child[c], 2,
2078			    B_FALSE);
2079		else
2080			print_import_config(cb, name, child[c], 2);
2081		free(name);
2082	}
2083}
2084
2085/*
2086 * Display the status for the given pool.
2087 */
2088static void
2089show_import(nvlist_t *config)
2090{
2091	uint64_t pool_state;
2092	vdev_stat_t *vs;
2093	char *name;
2094	uint64_t guid;
2095	uint64_t hostid = 0;
2096	char *msgid;
2097	char *hostname = "unknown";
2098	nvlist_t *nvroot, *nvinfo;
2099	int reason;
2100	zpool_errata_t errata;
2101	const char *health;
2102	uint_t vsc;
2103	char *comment;
2104	status_cbdata_t cb = { 0 };
2105
2106	verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
2107	    &name) == 0);
2108	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
2109	    &guid) == 0);
2110	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
2111	    &pool_state) == 0);
2112	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
2113	    &nvroot) == 0);
2114
2115	verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
2116	    (uint64_t **)&vs, &vsc) == 0);
2117	health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
2118
2119	reason = zpool_import_status(config, &msgid, &errata);
2120
2121	(void) printf(gettext("   pool: %s\n"), name);
2122	(void) printf(gettext("     id: %llu\n"), (u_longlong_t)guid);
2123	(void) printf(gettext("  state: %s"), health);
2124	if (pool_state == POOL_STATE_DESTROYED)
2125		(void) printf(gettext(" (DESTROYED)"));
2126	(void) printf("\n");
2127
2128	switch (reason) {
2129	case ZPOOL_STATUS_MISSING_DEV_R:
2130	case ZPOOL_STATUS_MISSING_DEV_NR:
2131	case ZPOOL_STATUS_BAD_GUID_SUM:
2132		(void) printf(gettext(" status: One or more devices are "
2133		    "missing from the system.\n"));
2134		break;
2135
2136	case ZPOOL_STATUS_CORRUPT_LABEL_R:
2137	case ZPOOL_STATUS_CORRUPT_LABEL_NR:
2138		(void) printf(gettext(" status: One or more devices contains "
2139		    "corrupted data.\n"));
2140		break;
2141
2142	case ZPOOL_STATUS_CORRUPT_DATA:
2143		(void) printf(
2144		    gettext(" status: The pool data is corrupted.\n"));
2145		break;
2146
2147	case ZPOOL_STATUS_OFFLINE_DEV:
2148		(void) printf(gettext(" status: One or more devices "
2149		    "are offlined.\n"));
2150		break;
2151
2152	case ZPOOL_STATUS_CORRUPT_POOL:
2153		(void) printf(gettext(" status: The pool metadata is "
2154		    "corrupted.\n"));
2155		break;
2156
2157	case ZPOOL_STATUS_VERSION_OLDER:
2158		(void) printf(gettext(" status: The pool is formatted using a "
2159		    "legacy on-disk version.\n"));
2160		break;
2161
2162	case ZPOOL_STATUS_VERSION_NEWER:
2163		(void) printf(gettext(" status: The pool is formatted using an "
2164		    "incompatible version.\n"));
2165		break;
2166
2167	case ZPOOL_STATUS_FEAT_DISABLED:
2168		(void) printf(gettext(" status: Some supported features are "
2169		    "not enabled on the pool.\n"));
2170		break;
2171
2172	case ZPOOL_STATUS_UNSUP_FEAT_READ:
2173		(void) printf(gettext("status: The pool uses the following "
2174		    "feature(s) not supported on this system:\n"));
2175		zpool_print_unsup_feat(config);
2176		break;
2177
2178	case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
2179		(void) printf(gettext("status: The pool can only be accessed "
2180		    "in read-only mode on this system. It\n\tcannot be "
2181		    "accessed in read-write mode because it uses the "
2182		    "following\n\tfeature(s) not supported on this system:\n"));
2183		zpool_print_unsup_feat(config);
2184		break;
2185
2186	case ZPOOL_STATUS_HOSTID_ACTIVE:
2187		(void) printf(gettext(" status: The pool is currently "
2188		    "imported by another system.\n"));
2189		break;
2190
2191	case ZPOOL_STATUS_HOSTID_REQUIRED:
2192		(void) printf(gettext(" status: The pool has the "
2193		    "multihost property on.  It cannot\n\tbe safely imported "
2194		    "when the system hostid is not set.\n"));
2195		break;
2196
2197	case ZPOOL_STATUS_HOSTID_MISMATCH:
2198		(void) printf(gettext(" status: The pool was last accessed by "
2199		    "another system.\n"));
2200		break;
2201
2202	case ZPOOL_STATUS_FAULTED_DEV_R:
2203	case ZPOOL_STATUS_FAULTED_DEV_NR:
2204		(void) printf(gettext(" status: One or more devices are "
2205		    "faulted.\n"));
2206		break;
2207
2208	case ZPOOL_STATUS_BAD_LOG:
2209		(void) printf(gettext(" status: An intent log record cannot be "
2210		    "read.\n"));
2211		break;
2212
2213	case ZPOOL_STATUS_RESILVERING:
2214		(void) printf(gettext(" status: One or more devices were being "
2215		    "resilvered.\n"));
2216		break;
2217
2218	case ZPOOL_STATUS_ERRATA:
2219		(void) printf(gettext(" status: Errata #%d detected.\n"),
2220		    errata);
2221		break;
2222
2223	default:
2224		/*
2225		 * No other status can be seen when importing pools.
2226		 */
2227		assert(reason == ZPOOL_STATUS_OK);
2228	}
2229
2230	/*
2231	 * Print out an action according to the overall state of the pool.
2232	 */
2233	if (vs->vs_state == VDEV_STATE_HEALTHY) {
2234		if (reason == ZPOOL_STATUS_VERSION_OLDER ||
2235		    reason == ZPOOL_STATUS_FEAT_DISABLED) {
2236			(void) printf(gettext(" action: The pool can be "
2237			    "imported using its name or numeric identifier, "
2238			    "though\n\tsome features will not be available "
2239			    "without an explicit 'zpool upgrade'.\n"));
2240		} else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
2241			(void) printf(gettext(" action: The pool can be "
2242			    "imported using its name or numeric "
2243			    "identifier and\n\tthe '-f' flag.\n"));
2244		} else if (reason == ZPOOL_STATUS_ERRATA) {
2245			switch (errata) {
2246			case ZPOOL_ERRATA_NONE:
2247				break;
2248
2249			case ZPOOL_ERRATA_ZOL_2094_SCRUB:
2250				(void) printf(gettext(" action: The pool can "
2251				    "be imported using its name or numeric "
2252				    "identifier,\n\thowever there is a compat"
2253				    "ibility issue which should be corrected"
2254				    "\n\tby running 'zpool scrub'\n"));
2255				break;
2256
2257			case ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY:
2258				(void) printf(gettext(" action: The pool can"
2259				    "not be imported with this version of ZFS "
2260				    "due to\n\tan active asynchronous destroy. "
2261				    "Revert to an earlier version\n\tand "
2262				    "allow the destroy to complete before "
2263				    "updating.\n"));
2264				break;
2265
2266			case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
2267				(void) printf(gettext(" action: Existing "
2268				    "encrypted datasets contain an on-disk "
2269				    "incompatibility, which\n\tneeds to be "
2270				    "corrected. Backup these datasets to new "
2271				    "encrypted datasets\n\tand destroy the "
2272				    "old ones.\n"));
2273				break;
2274			case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
2275				(void) printf(gettext(" action: Any existing "
2276				    "encrypted datasets contain an on-disk "
2277				    "incompatibility which\n\tmay cause "
2278				    "on-disk corruption with 'zfs recv' and "
2279				    "which needs to be\n\tcorrected. Enable "
2280				    "the bookmark_v2 feature and backup "
2281				    "these datasets to new encrypted "
2282				    "datasets and\n\tdestroy the old ones. "
2283				    "If this pool does not contain any "
2284				    "encrypted datasets, simply enable\n\t"
2285				    "the bookmark_v2 feature.\n"));
2286				break;
2287			default:
2288				/*
2289				 * All errata must contain an action message.
2290				 */
2291				assert(0);
2292			}
2293		} else {
2294			(void) printf(gettext(" action: The pool can be "
2295			    "imported using its name or numeric "
2296			    "identifier.\n"));
2297		}
2298	} else if (vs->vs_state == VDEV_STATE_DEGRADED) {
2299		(void) printf(gettext(" action: The pool can be imported "
2300		    "despite missing or damaged devices.  The\n\tfault "
2301		    "tolerance of the pool may be compromised if imported.\n"));
2302	} else {
2303		switch (reason) {
2304		case ZPOOL_STATUS_VERSION_NEWER:
2305			(void) printf(gettext(" action: The pool cannot be "
2306			    "imported.  Access the pool on a system running "
2307			    "newer\n\tsoftware, or recreate the pool from "
2308			    "backup.\n"));
2309			break;
2310		case ZPOOL_STATUS_UNSUP_FEAT_READ:
2311			(void) printf(gettext("action: The pool cannot be "
2312			    "imported. Access the pool on a system that "
2313			    "supports\n\tthe required feature(s), or recreate "
2314			    "the pool from backup.\n"));
2315			break;
2316		case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
2317			(void) printf(gettext("action: The pool cannot be "
2318			    "imported in read-write mode. Import the pool "
2319			    "with\n"
2320			    "\t\"-o readonly=on\", access the pool on a system "
2321			    "that supports the\n\trequired feature(s), or "
2322			    "recreate the pool from backup.\n"));
2323			break;
2324		case ZPOOL_STATUS_MISSING_DEV_R:
2325		case ZPOOL_STATUS_MISSING_DEV_NR:
2326		case ZPOOL_STATUS_BAD_GUID_SUM:
2327			(void) printf(gettext(" action: The pool cannot be "
2328			    "imported. Attach the missing\n\tdevices and try "
2329			    "again.\n"));
2330			break;
2331		case ZPOOL_STATUS_HOSTID_ACTIVE:
2332			VERIFY0(nvlist_lookup_nvlist(config,
2333			    ZPOOL_CONFIG_LOAD_INFO, &nvinfo));
2334
2335			if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
2336				hostname = fnvlist_lookup_string(nvinfo,
2337				    ZPOOL_CONFIG_MMP_HOSTNAME);
2338
2339			if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
2340				hostid = fnvlist_lookup_uint64(nvinfo,
2341				    ZPOOL_CONFIG_MMP_HOSTID);
2342
2343			(void) printf(gettext(" action: The pool must be "
2344			    "exported from %s (hostid=%lx)\n\tbefore it "
2345			    "can be safely imported.\n"), hostname,
2346			    (unsigned long) hostid);
2347			break;
2348		case ZPOOL_STATUS_HOSTID_REQUIRED:
2349			(void) printf(gettext(" action: Check the SMF "
2350			    "svc:/system/hostid service.\n"));
2351			break;
2352		default:
2353			(void) printf(gettext(" action: The pool cannot be "
2354			    "imported due to damaged devices or data.\n"));
2355		}
2356	}
2357
2358	/* Print the comment attached to the pool. */
2359	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
2360		(void) printf(gettext("comment: %s\n"), comment);
2361
2362	/*
2363	 * If the state is "closed" or "can't open", and the aux state
2364	 * is "corrupt data":
2365	 */
2366	if (((vs->vs_state == VDEV_STATE_CLOSED) ||
2367	    (vs->vs_state == VDEV_STATE_CANT_OPEN)) &&
2368	    (vs->vs_aux == VDEV_AUX_CORRUPT_DATA)) {
2369		if (pool_state == POOL_STATE_DESTROYED)
2370			(void) printf(gettext("\tThe pool was destroyed, "
2371			    "but can be imported using the '-Df' flags.\n"));
2372		else if (pool_state != POOL_STATE_EXPORTED)
2373			(void) printf(gettext("\tThe pool may be active on "
2374			    "another system, but can be imported using\n\t"
2375			    "the '-f' flag.\n"));
2376	}
2377
2378	if (msgid != NULL)
2379		(void) printf(gettext("   see: http://illumos.org/msg/%s\n"),
2380		    msgid);
2381
2382	(void) printf(gettext(" config:\n\n"));
2383
2384	cb.cb_namewidth = max_width(NULL, nvroot, 0, 0, 0);
2385	if (cb.cb_namewidth < 10)
2386		cb.cb_namewidth = 10;
2387
2388	print_import_config(&cb, name, nvroot, 0);
2389
2390	print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_DEDUP);
2391	print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
2392	print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_CLASS_LOGS);
2393
2394	if (reason == ZPOOL_STATUS_BAD_GUID_SUM) {
2395		(void) printf(gettext("\n\tAdditional devices are known to "
2396		    "be part of this pool, though their\n\texact "
2397		    "configuration cannot be determined.\n"));
2398	}
2399}
2400
2401static boolean_t
2402zfs_force_import_required(nvlist_t *config)
2403{
2404	uint64_t state;
2405	uint64_t hostid = 0;
2406	nvlist_t *nvinfo;
2407
2408	state = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE);
2409	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid);
2410
2411	if (state != POOL_STATE_EXPORTED && hostid != get_system_hostid())
2412		return (B_TRUE);
2413
2414	nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
2415	if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE)) {
2416		mmp_state_t mmp_state = fnvlist_lookup_uint64(nvinfo,
2417		    ZPOOL_CONFIG_MMP_STATE);
2418
2419		if (mmp_state != MMP_STATE_INACTIVE)
2420			return (B_TRUE);
2421	}
2422
2423	return (B_FALSE);
2424}
2425
2426/*
2427 * Perform the import for the given configuration.  This passes the heavy
2428 * lifting off to zpool_import_props(), and then mounts the datasets contained
2429 * within the pool.
2430 */
2431static int
2432do_import(nvlist_t *config, const char *newname, const char *mntopts,
2433    nvlist_t *props, int flags)
2434{
2435	int ret = 0;
2436	zpool_handle_t *zhp;
2437	char *name;
2438	uint64_t version;
2439
2440	name = fnvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME);
2441	version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION);
2442
2443	if (!SPA_VERSION_IS_SUPPORTED(version)) {
2444		(void) fprintf(stderr, gettext("cannot import '%s': pool "
2445		    "is formatted using an unsupported ZFS version\n"), name);
2446		return (1);
2447	} else if (zfs_force_import_required(config) &&
2448	    !(flags & ZFS_IMPORT_ANY_HOST)) {
2449		mmp_state_t mmp_state = MMP_STATE_INACTIVE;
2450		nvlist_t *nvinfo;
2451
2452		nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
2453		if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE))
2454			mmp_state = fnvlist_lookup_uint64(nvinfo,
2455			    ZPOOL_CONFIG_MMP_STATE);
2456
2457		if (mmp_state == MMP_STATE_ACTIVE) {
2458			char *hostname = "<unknown>";
2459			uint64_t hostid = 0;
2460
2461			if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
2462				hostname = fnvlist_lookup_string(nvinfo,
2463				    ZPOOL_CONFIG_MMP_HOSTNAME);
2464
2465			if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
2466				hostid = fnvlist_lookup_uint64(nvinfo,
2467				    ZPOOL_CONFIG_MMP_HOSTID);
2468
2469			(void) fprintf(stderr, gettext("cannot import '%s': "
2470			    "pool is imported on %s (hostid: "
2471			    "0x%lx)\nExport the pool on the other system, "
2472			    "then run 'zpool import'.\n"),
2473			    name, hostname, (unsigned long) hostid);
2474		} else if (mmp_state == MMP_STATE_NO_HOSTID) {
2475			(void) fprintf(stderr, gettext("Cannot import '%s': "
2476			    "pool has the multihost property on and the\n"
2477			    "system's hostid is not set.\n"), name);
2478		} else {
2479			char *hostname = "<unknown>";
2480			uint64_t timestamp = 0;
2481			uint64_t hostid = 0;
2482
2483			if (nvlist_exists(config, ZPOOL_CONFIG_HOSTNAME))
2484				hostname = fnvlist_lookup_string(config,
2485				    ZPOOL_CONFIG_HOSTNAME);
2486
2487			if (nvlist_exists(config, ZPOOL_CONFIG_TIMESTAMP))
2488				timestamp = fnvlist_lookup_uint64(config,
2489				    ZPOOL_CONFIG_TIMESTAMP);
2490
2491			if (nvlist_exists(config, ZPOOL_CONFIG_HOSTID))
2492				hostid = fnvlist_lookup_uint64(config,
2493				    ZPOOL_CONFIG_HOSTID);
2494
2495			(void) fprintf(stderr, gettext("cannot import '%s': "
2496			    "pool was previously in use from another system.\n"
2497			    "Last accessed by %s (hostid=%lx) at %s"
2498			    "The pool can be imported, use 'zpool import -f' "
2499			    "to import the pool.\n"), name, hostname,
2500			    (unsigned long)hostid, ctime((time_t *)&timestamp));
2501
2502		}
2503
2504		return (1);
2505	}
2506
2507	if (zpool_import_props(g_zfs, config, newname, props, flags) != 0)
2508		return (1);
2509
2510	if (newname != NULL)
2511		name = (char *)newname;
2512
2513	if ((zhp = zpool_open_canfail(g_zfs, name)) == NULL)
2514		return (1);
2515
2516	/*
2517	 * Loading keys is best effort. We don't want to return immediately
2518	 * if it fails but we do want to give the error to the caller.
2519	 */
2520	if (flags & ZFS_IMPORT_LOAD_KEYS) {
2521		ret = zfs_crypto_attempt_load_keys(g_zfs, name);
2522		if (ret != 0)
2523			ret = 1;
2524	}
2525
2526	if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
2527	    !(flags & ZFS_IMPORT_ONLY) &&
2528	    zpool_enable_datasets(zhp, mntopts, 0) != 0) {
2529		zpool_close(zhp);
2530		return (1);
2531	}
2532
2533	zpool_close(zhp);
2534	return (ret);
2535}
2536
2537/*
2538 * zpool checkpoint <pool>
2539 *       checkpoint --discard <pool>
2540 *
2541 *	-d	Discard the checkpoint from a checkpointed
2542 *	--discard  pool.
2543 *
2544 * Checkpoints the specified pool, by taking a "snapshot" of its
2545 * current state. A pool can only have one checkpoint at a time.
2546 */
2547int
2548zpool_do_checkpoint(int argc, char **argv)
2549{
2550	boolean_t discard;
2551	char *pool;
2552	zpool_handle_t *zhp;
2553	int c, err;
2554
2555	struct option long_options[] = {
2556		{"discard", no_argument, NULL, 'd'},
2557		{0, 0, 0, 0}
2558	};
2559
2560	discard = B_FALSE;
2561	while ((c = getopt_long(argc, argv, ":d", long_options, NULL)) != -1) {
2562		switch (c) {
2563		case 'd':
2564			discard = B_TRUE;
2565			break;
2566		case '?':
2567			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
2568			    optopt);
2569			usage(B_FALSE);
2570		}
2571	}
2572
2573	argc -= optind;
2574	argv += optind;
2575
2576	if (argc < 1) {
2577		(void) fprintf(stderr, gettext("missing pool argument\n"));
2578		usage(B_FALSE);
2579	}
2580
2581	if (argc > 1) {
2582		(void) fprintf(stderr, gettext("too many arguments\n"));
2583		usage(B_FALSE);
2584	}
2585
2586	pool = argv[0];
2587
2588	if ((zhp = zpool_open(g_zfs, pool)) == NULL) {
2589		/* As a special case, check for use of '/' in the name */
2590		if (strchr(pool, '/') != NULL)
2591			(void) fprintf(stderr, gettext("'zpool checkpoint' "
2592			    "doesn't work on datasets. To save the state "
2593			    "of a dataset from a specific point in time "
2594			    "please use 'zfs snapshot'\n"));
2595		return (1);
2596	}
2597
2598	if (discard)
2599		err = (zpool_discard_checkpoint(zhp) != 0);
2600	else
2601		err = (zpool_checkpoint(zhp) != 0);
2602
2603	zpool_close(zhp);
2604
2605	return (err);
2606}
2607
2608#define	CHECKPOINT_OPT	1024
2609
2610/*
2611 * zpool import [-d dir] [-D]
2612 *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
2613 *              [-d dir | -c cachefile] [-f] -a
2614 *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
2615 *              [-d dir | -c cachefile] [-f] [-n] [-F] [-t]
2616 *              <pool | id> [newpool]
2617 *
2618 *	-c	Read pool information from a cachefile instead of searching
2619 *		devices.
2620 *
2621 *	-d	Scan in a specific directory, other than /dev/dsk.  More than
2622 *		one directory can be specified using multiple '-d' options.
2623 *
2624 *	-D	Scan for previously destroyed pools or import all or only
2625 *		specified destroyed pools.
2626 *
2627 *	-R	Temporarily import the pool, with all mountpoints relative to
2628 *		the given root.  The pool will remain exported when the machine
2629 *		is rebooted.
2630 *
2631 *	-V	Import even in the presence of faulted vdevs.  This is an
2632 *		intentionally undocumented option for testing purposes, and
2633 *		treats the pool configuration as complete, leaving any bad
2634 *		vdevs in the FAULTED state. In other words, it does verbatim
2635 *		import.
2636 *
2637 *	-f	Force import, even if it appears that the pool is active.
2638 *
2639 *	-F	Attempt rewind if necessary.
2640 *
2641 *	-n	See if rewind would work, but don't actually rewind.
2642 *
2643 *	-N	Import the pool but don't mount datasets.
2644 *
2645 *	-t	Use newpool as a temporary pool name instead of renaming
2646 *		the pool.
2647 *
2648 *	-T	Specify a starting txg to use for import. This option is
2649 *		intentionally undocumented option for testing purposes.
2650 *
2651 *	-a	Import all pools found.
2652 *
2653 *	-o	Set property=value and/or temporary mount options (without '=').
2654 *  -l	Load encryption keys while importing.
2655 *
2656 *	--rewind-to-checkpoint
2657 *		Import the pool and revert back to the checkpoint.
2658 *
2659 * The import command scans for pools to import, and import pools based on pool
2660 * name and GUID.  The pool can also be renamed as part of the import process.
2661 */
2662int
2663zpool_do_import(int argc, char **argv)
2664{
2665	char **searchdirs = NULL;
2666	int nsearch = 0;
2667	int c;
2668	int err = 0;
2669	nvlist_t *pools = NULL;
2670	boolean_t do_all = B_FALSE;
2671	boolean_t do_destroyed = B_FALSE;
2672	char *mntopts = NULL;
2673	nvpair_t *elem;
2674	nvlist_t *config;
2675	uint64_t searchguid = 0;
2676	char *searchname = NULL;
2677	char *propval;
2678	nvlist_t *found_config;
2679	nvlist_t *policy = NULL;
2680	nvlist_t *props = NULL;
2681	boolean_t first;
2682	int flags = ZFS_IMPORT_NORMAL;
2683	uint32_t rewind_policy = ZPOOL_NO_REWIND;
2684	boolean_t dryrun = B_FALSE;
2685	boolean_t do_rewind = B_FALSE;
2686	boolean_t xtreme_rewind = B_FALSE;
2687	uint64_t pool_state, txg = -1ULL;
2688	char *cachefile = NULL;
2689	importargs_t idata = { 0 };
2690	char *endptr;
2691
2692
2693	struct option long_options[] = {
2694		{"rewind-to-checkpoint", no_argument, NULL, CHECKPOINT_OPT},
2695		{0, 0, 0, 0}
2696	};
2697
2698	/* check options */
2699	while ((c = getopt_long(argc, argv, ":aCc:d:DEfFlmnNo:rR:tT:VX",
2700	    long_options, NULL)) != -1) {
2701		switch (c) {
2702		case 'a':
2703			do_all = B_TRUE;
2704			break;
2705		case 'c':
2706			cachefile = optarg;
2707			break;
2708		case 'd':
2709			if (searchdirs == NULL) {
2710				searchdirs = safe_malloc(sizeof (char *));
2711			} else {
2712				char **tmp = safe_malloc((nsearch + 1) *
2713				    sizeof (char *));
2714				bcopy(searchdirs, tmp, nsearch *
2715				    sizeof (char *));
2716				free(searchdirs);
2717				searchdirs = tmp;
2718			}
2719			searchdirs[nsearch++] = optarg;
2720			break;
2721		case 'D':
2722			do_destroyed = B_TRUE;
2723			break;
2724		case 'f':
2725			flags |= ZFS_IMPORT_ANY_HOST;
2726			break;
2727		case 'F':
2728			do_rewind = B_TRUE;
2729			break;
2730		case 'l':
2731			flags |= ZFS_IMPORT_LOAD_KEYS;
2732			break;
2733		case 'm':
2734			flags |= ZFS_IMPORT_MISSING_LOG;
2735			break;
2736		case 'n':
2737			dryrun = B_TRUE;
2738			break;
2739		case 'N':
2740			flags |= ZFS_IMPORT_ONLY;
2741			break;
2742		case 'o':
2743			if ((propval = strchr(optarg, '=')) != NULL) {
2744				*propval = '\0';
2745				propval++;
2746				if (add_prop_list(optarg, propval,
2747				    &props, B_TRUE))
2748					goto error;
2749			} else {
2750				mntopts = optarg;
2751			}
2752			break;
2753		case 'R':
2754			if (add_prop_list(zpool_prop_to_name(
2755			    ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
2756				goto error;
2757			if (add_prop_list_default(zpool_prop_to_name(
2758			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
2759				goto error;
2760			break;
2761		case 't':
2762			flags |= ZFS_IMPORT_TEMP_NAME;
2763			if (add_prop_list_default(zpool_prop_to_name(
2764			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
2765				goto error;
2766			break;
2767		case 'T':
2768			errno = 0;
2769			txg = strtoull(optarg, &endptr, 0);
2770			if (errno != 0 || *endptr != '\0') {
2771				(void) fprintf(stderr,
2772				    gettext("invalid txg value\n"));
2773				usage(B_FALSE);
2774			}
2775			rewind_policy = ZPOOL_DO_REWIND | ZPOOL_EXTREME_REWIND;
2776			break;
2777		case 'V':
2778			flags |= ZFS_IMPORT_VERBATIM;
2779			break;
2780		case 'X':
2781			xtreme_rewind = B_TRUE;
2782			break;
2783		case CHECKPOINT_OPT:
2784			flags |= ZFS_IMPORT_CHECKPOINT;
2785			break;
2786		case ':':
2787			(void) fprintf(stderr, gettext("missing argument for "
2788			    "'%c' option\n"), optopt);
2789			usage(B_FALSE);
2790			break;
2791		case '?':
2792			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
2793			    optopt);
2794			usage(B_FALSE);
2795		}
2796	}
2797
2798	argc -= optind;
2799	argv += optind;
2800
2801	if (cachefile && nsearch != 0) {
2802		(void) fprintf(stderr, gettext("-c is incompatible with -d\n"));
2803		usage(B_FALSE);
2804	}
2805
2806	if ((flags & ZFS_IMPORT_LOAD_KEYS) && (flags & ZFS_IMPORT_ONLY)) {
2807		(void) fprintf(stderr, gettext("-l is incompatible with -N\n"));
2808		usage(B_FALSE);
2809	}
2810
2811	if ((flags & ZFS_IMPORT_LOAD_KEYS) && !do_all && argc == 0) {
2812		(void) fprintf(stderr, gettext("-l is only meaningful during "
2813		    "an import\n"));
2814		usage(B_FALSE);
2815	}
2816
2817	if ((dryrun || xtreme_rewind) && !do_rewind) {
2818		(void) fprintf(stderr,
2819		    gettext("-n or -X only meaningful with -F\n"));
2820		usage(B_FALSE);
2821	}
2822	if (dryrun)
2823		rewind_policy = ZPOOL_TRY_REWIND;
2824	else if (do_rewind)
2825		rewind_policy = ZPOOL_DO_REWIND;
2826	if (xtreme_rewind)
2827		rewind_policy |= ZPOOL_EXTREME_REWIND;
2828
2829	/* In the future, we can capture further policy and include it here */
2830	if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
2831	    nvlist_add_uint64(policy, ZPOOL_LOAD_REQUEST_TXG, txg) != 0 ||
2832	    nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
2833	    rewind_policy) != 0)
2834		goto error;
2835
2836	if (searchdirs == NULL) {
2837		searchdirs = safe_malloc(sizeof (char *));
2838		searchdirs[0] = ZFS_DISK_ROOT;
2839		nsearch = 1;
2840	}
2841
2842	/* check argument count */
2843	if (do_all) {
2844		if (argc != 0) {
2845			(void) fprintf(stderr, gettext("too many arguments\n"));
2846			usage(B_FALSE);
2847		}
2848	} else {
2849		if (argc > 2) {
2850			(void) fprintf(stderr, gettext("too many arguments\n"));
2851			usage(B_FALSE);
2852		}
2853
2854		/*
2855		 * Check for the SYS_CONFIG privilege.  We do this explicitly
2856		 * here because otherwise any attempt to discover pools will
2857		 * silently fail.
2858		 */
2859		if (argc == 0 && !priv_ineffect(PRIV_SYS_CONFIG)) {
2860			(void) fprintf(stderr, gettext("cannot "
2861			    "discover pools: permission denied\n"));
2862			free(searchdirs);
2863			nvlist_free(policy);
2864			return (1);
2865		}
2866	}
2867
2868	/*
2869	 * Depending on the arguments given, we do one of the following:
2870	 *
2871	 *	<none>	Iterate through all pools and display information about
2872	 *		each one.
2873	 *
2874	 *	-a	Iterate through all pools and try to import each one.
2875	 *
2876	 *	<id>	Find the pool that corresponds to the given GUID/pool
2877	 *		name and import that one.
2878	 *
2879	 *	-D	Above options applies only to destroyed pools.
2880	 */
2881	if (argc != 0) {
2882		char *endptr;
2883
2884		errno = 0;
2885		searchguid = strtoull(argv[0], &endptr, 10);
2886		if (errno != 0 || *endptr != '\0') {
2887			searchname = argv[0];
2888			searchguid = 0;
2889		}
2890		found_config = NULL;
2891
2892		/*
2893		 * User specified a name or guid.  Ensure it's unique.
2894		 */
2895		idata.unique = B_TRUE;
2896	}
2897
2898
2899	idata.path = searchdirs;
2900	idata.paths = nsearch;
2901	idata.poolname = searchname;
2902	idata.guid = searchguid;
2903	idata.cachefile = cachefile;
2904	idata.policy = policy;
2905
2906	pools = zpool_search_import(g_zfs, &idata);
2907
2908	if (pools != NULL && idata.exists &&
2909	    (argc == 1 || strcmp(argv[0], argv[1]) == 0)) {
2910		(void) fprintf(stderr, gettext("cannot import '%s': "
2911		    "a pool with that name already exists\n"),
2912		    argv[0]);
2913		(void) fprintf(stderr, gettext("use the form 'zpool import "
2914		    "[-t] <pool | id> <newpool>' to give it a new temporary "
2915		    "or permanent name\n"));
2916		err = 1;
2917	} else if (pools == NULL && idata.exists) {
2918		(void) fprintf(stderr, gettext("cannot import '%s': "
2919		    "a pool with that name is already created/imported,\n"),
2920		    argv[0]);
2921		(void) fprintf(stderr, gettext("and no additional pools "
2922		    "with that name were found\n"));
2923		err = 1;
2924	} else if (pools == NULL) {
2925		if (argc != 0) {
2926			(void) fprintf(stderr, gettext("cannot import '%s': "
2927			    "no such pool available\n"), argv[0]);
2928		}
2929		err = 1;
2930	}
2931
2932	if (err == 1) {
2933		free(searchdirs);
2934		nvlist_free(policy);
2935		return (1);
2936	}
2937
2938	/*
2939	 * At this point we have a list of import candidate configs. Even if
2940	 * we were searching by pool name or guid, we still need to
2941	 * post-process the list to deal with pool state and possible
2942	 * duplicate names.
2943	 */
2944	err = 0;
2945	elem = NULL;
2946	first = B_TRUE;
2947	while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) {
2948
2949		verify(nvpair_value_nvlist(elem, &config) == 0);
2950
2951		verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
2952		    &pool_state) == 0);
2953		if (!do_destroyed && pool_state == POOL_STATE_DESTROYED)
2954			continue;
2955		if (do_destroyed && pool_state != POOL_STATE_DESTROYED)
2956			continue;
2957
2958		verify(nvlist_add_nvlist(config, ZPOOL_LOAD_POLICY,
2959		    policy) == 0);
2960
2961		if (argc == 0) {
2962			if (first)
2963				first = B_FALSE;
2964			else if (!do_all)
2965				(void) printf("\n");
2966
2967			if (do_all) {
2968				err |= do_import(config, NULL, mntopts,
2969				    props, flags);
2970			} else {
2971				show_import(config);
2972			}
2973		} else if (searchname != NULL) {
2974			char *name;
2975
2976			/*
2977			 * We are searching for a pool based on name.
2978			 */
2979			verify(nvlist_lookup_string(config,
2980			    ZPOOL_CONFIG_POOL_NAME, &name) == 0);
2981
2982			if (strcmp(name, searchname) == 0) {
2983				if (found_config != NULL) {
2984					(void) fprintf(stderr, gettext(
2985					    "cannot import '%s': more than "
2986					    "one matching pool\n"), searchname);
2987					(void) fprintf(stderr, gettext(
2988					    "import by numeric ID instead\n"));
2989					err = B_TRUE;
2990				}
2991				found_config = config;
2992			}
2993		} else {
2994			uint64_t guid;
2995
2996			/*
2997			 * Search for a pool by guid.
2998			 */
2999			verify(nvlist_lookup_uint64(config,
3000			    ZPOOL_CONFIG_POOL_GUID, &guid) == 0);
3001
3002			if (guid == searchguid)
3003				found_config = config;
3004		}
3005	}
3006
3007	/*
3008	 * If we were searching for a specific pool, verify that we found a
3009	 * pool, and then do the import.
3010	 */
3011	if (argc != 0 && err == 0) {
3012		if (found_config == NULL) {
3013			(void) fprintf(stderr, gettext("cannot import '%s': "
3014			    "no such pool available\n"), argv[0]);
3015			err = B_TRUE;
3016		} else {
3017			err |= do_import(found_config, argc == 1 ? NULL :
3018			    argv[1], mntopts, props, flags);
3019		}
3020	}
3021
3022	/*
3023	 * If we were just looking for pools, report an error if none were
3024	 * found.
3025	 */
3026	if (argc == 0 && first)
3027		(void) fprintf(stderr,
3028		    gettext("no pools available to import\n"));
3029
3030error:
3031	nvlist_free(props);
3032	nvlist_free(pools);
3033	nvlist_free(policy);
3034	free(searchdirs);
3035
3036	return (err ? 1 : 0);
3037}
3038
3039/*
3040 * zpool sync [-f] [pool] ...
3041 *
3042 * -f (undocumented) force uberblock (and config including zpool cache file)
3043 *    update.
3044 *
3045 * Sync the specified pool(s).
3046 * Without arguments "zpool sync" will sync all pools.
3047 * This command initiates TXG sync(s) and will return after the TXG(s) commit.
3048 *
3049 */
3050static int
3051zpool_do_sync(int argc, char **argv)
3052{
3053	int ret;
3054	boolean_t force = B_FALSE;
3055
3056	/* check options */
3057	while ((ret  = getopt(argc, argv, "f")) != -1) {
3058		switch (ret) {
3059		case 'f':
3060			force = B_TRUE;
3061			break;
3062		case '?':
3063			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3064			    optopt);
3065			usage(B_FALSE);
3066		}
3067	}
3068
3069	argc -= optind;
3070	argv += optind;
3071
3072	/* if argc == 0 we will execute zpool_sync_one on all pools */
3073	ret = for_each_pool(argc, argv, B_FALSE, NULL, zpool_sync_one, &force);
3074
3075	return (ret);
3076}
3077
3078typedef struct iostat_cbdata {
3079	boolean_t cb_verbose;
3080	int cb_name_flags;
3081	int cb_namewidth;
3082	int cb_iteration;
3083	boolean_t cb_scripted;
3084	zpool_list_t *cb_list;
3085} iostat_cbdata_t;
3086
3087static void
3088print_iostat_separator(iostat_cbdata_t *cb)
3089{
3090	int i = 0;
3091
3092	for (i = 0; i < cb->cb_namewidth; i++)
3093		(void) printf("-");
3094	(void) printf("  -----  -----  -----  -----  -----  -----\n");
3095}
3096
3097static void
3098print_iostat_header(iostat_cbdata_t *cb)
3099{
3100	(void) printf("%*s     capacity     operations    bandwidth\n",
3101	    cb->cb_namewidth, "");
3102	(void) printf("%-*s  alloc   free   read  write   read  write\n",
3103	    cb->cb_namewidth, "pool");
3104	print_iostat_separator(cb);
3105}
3106
3107/*
3108 * Display a single statistic.
3109 */
3110static void
3111print_one_stat(uint64_t value)
3112{
3113	char buf[64];
3114
3115	zfs_nicenum(value, buf, sizeof (buf));
3116	(void) printf("  %5s", buf);
3117}
3118
3119static const char *class_name[] = {
3120	VDEV_ALLOC_BIAS_DEDUP,
3121	VDEV_ALLOC_BIAS_SPECIAL,
3122	VDEV_ALLOC_CLASS_LOGS
3123};
3124
3125/*
3126 * Print out all the statistics for the given vdev.  This can either be the
3127 * toplevel configuration, or called recursively.  If 'name' is NULL, then this
3128 * is a verbose output, and we don't want to display the toplevel pool stats.
3129 */
3130static void
3131print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
3132    nvlist_t *newnv, iostat_cbdata_t *cb, int depth)
3133{
3134	nvlist_t **oldchild, **newchild;
3135	uint_t c, children;
3136	vdev_stat_t *oldvs, *newvs;
3137	vdev_stat_t zerovs = { 0 };
3138	char *vname;
3139	uint64_t tdelta;
3140	double scale;
3141
3142	if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
3143		return;
3144
3145	if (oldnv != NULL) {
3146		verify(nvlist_lookup_uint64_array(oldnv,
3147		    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&oldvs, &c) == 0);
3148	} else {
3149		oldvs = &zerovs;
3150	}
3151
3152	verify(nvlist_lookup_uint64_array(newnv, ZPOOL_CONFIG_VDEV_STATS,
3153	    (uint64_t **)&newvs, &c) == 0);
3154
3155	if (strlen(name) + depth > cb->cb_namewidth)
3156		(void) printf("%*s%s", depth, "", name);
3157	else
3158		(void) printf("%*s%s%*s", depth, "", name,
3159		    (int)(cb->cb_namewidth - strlen(name) - depth), "");
3160
3161	tdelta = newvs->vs_timestamp - oldvs->vs_timestamp;
3162
3163	if (tdelta == 0)
3164		scale = 1.0;
3165	else
3166		scale = (double)NANOSEC / tdelta;
3167
3168	/* only toplevel vdevs have capacity stats */
3169	if (newvs->vs_space == 0) {
3170		(void) printf("      -      -");
3171	} else {
3172		print_one_stat(newvs->vs_alloc);
3173		print_one_stat(newvs->vs_space - newvs->vs_alloc);
3174	}
3175
3176	print_one_stat((uint64_t)(scale * (newvs->vs_ops[ZIO_TYPE_READ] -
3177	    oldvs->vs_ops[ZIO_TYPE_READ])));
3178
3179	print_one_stat((uint64_t)(scale * (newvs->vs_ops[ZIO_TYPE_WRITE] -
3180	    oldvs->vs_ops[ZIO_TYPE_WRITE])));
3181
3182	print_one_stat((uint64_t)(scale * (newvs->vs_bytes[ZIO_TYPE_READ] -
3183	    oldvs->vs_bytes[ZIO_TYPE_READ])));
3184
3185	print_one_stat((uint64_t)(scale * (newvs->vs_bytes[ZIO_TYPE_WRITE] -
3186	    oldvs->vs_bytes[ZIO_TYPE_WRITE])));
3187
3188	(void) printf("\n");
3189
3190	if (!cb->cb_verbose)
3191		return;
3192
3193	if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_CHILDREN,
3194	    &newchild, &children) != 0)
3195		return;
3196
3197	if (oldnv && nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_CHILDREN,
3198	    &oldchild, &c) != 0)
3199		return;
3200
3201	/*
3202	 * print normal top-level devices
3203	 */
3204	for (c = 0; c < children; c++) {
3205		uint64_t ishole = B_FALSE, islog = B_FALSE;
3206
3207		(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_HOLE,
3208		    &ishole);
3209
3210		(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_LOG,
3211		    &islog);
3212
3213		if (ishole || islog)
3214			continue;
3215
3216		if (nvlist_exists(newchild[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
3217			continue;
3218
3219		vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
3220		    cb->cb_name_flags);
3221		print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL,
3222		    newchild[c], cb, depth + 2);
3223		free(vname);
3224	}
3225
3226	/*
3227	 * print all other top-level devices
3228	 */
3229	for (uint_t n = 0; n < 3; n++) {
3230		boolean_t printed = B_FALSE;
3231		for (c = 0; c < children; c++) {
3232			uint64_t islog = B_FALSE;
3233			char *bias = NULL;
3234			char *type = NULL;
3235
3236			(void) nvlist_lookup_uint64(newchild[c],
3237			    ZPOOL_CONFIG_IS_LOG, &islog);
3238			if (islog) {
3239				bias = VDEV_ALLOC_CLASS_LOGS;
3240			} else {
3241				(void) nvlist_lookup_string(newchild[c],
3242				    ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
3243				(void) nvlist_lookup_string(newchild[c],
3244				    ZPOOL_CONFIG_TYPE, &type);
3245			}
3246			if (bias == NULL || strcmp(bias, class_name[n]) != 0)
3247				continue;
3248			if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
3249				continue;
3250
3251			if (!printed) {
3252				if (!cb->cb_scripted) {
3253					(void) printf(
3254					    "%-*s      -      -      -      -"
3255					    "      -      -",
3256					    cb->cb_namewidth, class_name[n]);
3257				}
3258				printf("\n");
3259				printed = B_TRUE;
3260			}
3261
3262			vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
3263			    cb->cb_name_flags);
3264			print_vdev_stats(zhp, vname, oldnv ?
3265			    oldchild[c] : NULL, newchild[c], cb, depth + 2);
3266			free(vname);
3267		}
3268
3269	}
3270
3271	/*
3272	 * Include level 2 ARC devices in iostat output
3273	 */
3274	if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_L2CACHE,
3275	    &newchild, &children) != 0)
3276		return;
3277
3278	if (oldnv && nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_L2CACHE,
3279	    &oldchild, &c) != 0)
3280		return;
3281
3282	if (children > 0) {
3283		(void) printf("%-*s      -      -      -      -      -      "
3284		    "-\n", cb->cb_namewidth, "cache");
3285		for (c = 0; c < children; c++) {
3286			vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
3287			    cb->cb_name_flags);
3288			print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL,
3289			    newchild[c], cb, depth + 2);
3290			free(vname);
3291		}
3292	}
3293}
3294
3295static int
3296refresh_iostat(zpool_handle_t *zhp, void *data)
3297{
3298	iostat_cbdata_t *cb = data;
3299	boolean_t missing;
3300
3301	/*
3302	 * If the pool has disappeared, remove it from the list and continue.
3303	 */
3304	if (zpool_refresh_stats(zhp, &missing) != 0)
3305		return (-1);
3306
3307	if (missing)
3308		pool_list_remove(cb->cb_list, zhp);
3309
3310	return (0);
3311}
3312
3313/*
3314 * Callback to print out the iostats for the given pool.
3315 */
3316int
3317print_iostat(zpool_handle_t *zhp, void *data)
3318{
3319	iostat_cbdata_t *cb = data;
3320	nvlist_t *oldconfig, *newconfig;
3321	nvlist_t *oldnvroot, *newnvroot;
3322
3323	newconfig = zpool_get_config(zhp, &oldconfig);
3324
3325	if (cb->cb_iteration == 1)
3326		oldconfig = NULL;
3327
3328	verify(nvlist_lookup_nvlist(newconfig, ZPOOL_CONFIG_VDEV_TREE,
3329	    &newnvroot) == 0);
3330
3331	if (oldconfig == NULL)
3332		oldnvroot = NULL;
3333	else
3334		verify(nvlist_lookup_nvlist(oldconfig, ZPOOL_CONFIG_VDEV_TREE,
3335		    &oldnvroot) == 0);
3336
3337	/*
3338	 * Print out the statistics for the pool.
3339	 */
3340	print_vdev_stats(zhp, zpool_get_name(zhp), oldnvroot, newnvroot, cb, 0);
3341
3342	if (cb->cb_verbose)
3343		print_iostat_separator(cb);
3344
3345	return (0);
3346}
3347
3348int
3349get_namewidth(zpool_handle_t *zhp, void *data)
3350{
3351	iostat_cbdata_t *cb = data;
3352	nvlist_t *config, *nvroot;
3353
3354	if ((config = zpool_get_config(zhp, NULL)) != NULL) {
3355		verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
3356		    &nvroot) == 0);
3357		if (!cb->cb_verbose)
3358			cb->cb_namewidth = strlen(zpool_get_name(zhp));
3359		else
3360			cb->cb_namewidth = max_width(zhp, nvroot, 0,
3361			    cb->cb_namewidth, cb->cb_name_flags);
3362	}
3363
3364	/*
3365	 * The width must fall into the range [10,38].  The upper limit is the
3366	 * maximum we can have and still fit in 80 columns.
3367	 */
3368	if (cb->cb_namewidth < 10)
3369		cb->cb_namewidth = 10;
3370	if (cb->cb_namewidth > 38)
3371		cb->cb_namewidth = 38;
3372
3373	return (0);
3374}
3375
3376/*
3377 * Parse the input string, get the 'interval' and 'count' value if there is one.
3378 */
3379static void
3380get_interval_count(int *argcp, char **argv, unsigned long *iv,
3381    unsigned long *cnt)
3382{
3383	unsigned long interval = 0, count = 0;
3384	int argc = *argcp, errno;
3385
3386	/*
3387	 * Determine if the last argument is an integer or a pool name
3388	 */
3389	if (argc > 0 && isdigit(argv[argc - 1][0])) {
3390		char *end;
3391
3392		errno = 0;
3393		interval = strtoul(argv[argc - 1], &end, 10);
3394
3395		if (*end == '\0' && errno == 0) {
3396			if (interval == 0) {
3397				(void) fprintf(stderr, gettext("interval "
3398				    "cannot be zero\n"));
3399				usage(B_FALSE);
3400			}
3401			/*
3402			 * Ignore the last parameter
3403			 */
3404			argc--;
3405		} else {
3406			/*
3407			 * If this is not a valid number, just plow on.  The
3408			 * user will get a more informative error message later
3409			 * on.
3410			 */
3411			interval = 0;
3412		}
3413	}
3414
3415	/*
3416	 * If the last argument is also an integer, then we have both a count
3417	 * and an interval.
3418	 */
3419	if (argc > 0 && isdigit(argv[argc - 1][0])) {
3420		char *end;
3421
3422		errno = 0;
3423		count = interval;
3424		interval = strtoul(argv[argc - 1], &end, 10);
3425
3426		if (*end == '\0' && errno == 0) {
3427			if (interval == 0) {
3428				(void) fprintf(stderr, gettext("interval "
3429				    "cannot be zero\n"));
3430				usage(B_FALSE);
3431			}
3432
3433			/*
3434			 * Ignore the last parameter
3435			 */
3436			argc--;
3437		} else {
3438			interval = 0;
3439		}
3440	}
3441
3442	*iv = interval;
3443	*cnt = count;
3444	*argcp = argc;
3445}
3446
3447static void
3448get_timestamp_arg(char c)
3449{
3450	if (c == 'u')
3451		timestamp_fmt = UDATE;
3452	else if (c == 'd')
3453		timestamp_fmt = DDATE;
3454	else
3455		usage(B_FALSE);
3456}
3457
3458/*
3459 * zpool iostat [-gLPv] [-T d|u] [pool] ... [interval [count]]
3460 *
3461 *	-g	Display guid for individual vdev name.
3462 *	-L	Follow links when resolving vdev path name.
3463 *	-P	Display full path for vdev name.
3464 *	-v	Display statistics for individual vdevs
3465 *	-T	Display a timestamp in date(1) or Unix format
3466 *
3467 * This command can be tricky because we want to be able to deal with pool
3468 * creation/destruction as well as vdev configuration changes.  The bulk of this
3469 * processing is handled by the pool_list_* routines in zpool_iter.c.  We rely
3470 * on pool_list_update() to detect the addition of new pools.  Configuration
3471 * changes are all handled within libzfs.
3472 */
3473int
3474zpool_do_iostat(int argc, char **argv)
3475{
3476	int c;
3477	int ret;
3478	int npools;
3479	unsigned long interval = 0, count = 0;
3480	zpool_list_t *list;
3481	boolean_t verbose = B_FALSE;
3482	boolean_t guid = B_FALSE;
3483	boolean_t follow_links = B_FALSE;
3484	boolean_t full_name = B_FALSE;
3485	iostat_cbdata_t cb = { 0 };
3486
3487	/* check options */
3488	while ((c = getopt(argc, argv, "gLPT:v")) != -1) {
3489		switch (c) {
3490		case 'g':
3491			guid = B_TRUE;
3492			break;
3493		case 'L':
3494			follow_links = B_TRUE;
3495			break;
3496		case 'P':
3497			full_name = B_TRUE;
3498			break;
3499		case 'T':
3500			get_timestamp_arg(*optarg);
3501			break;
3502		case 'v':
3503			verbose = B_TRUE;
3504			break;
3505		case '?':
3506			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3507			    optopt);
3508			usage(B_FALSE);
3509		}
3510	}
3511
3512	argc -= optind;
3513	argv += optind;
3514
3515	get_interval_count(&argc, argv, &interval, &count);
3516
3517	/*
3518	 * Construct the list of all interesting pools.
3519	 */
3520	ret = 0;
3521	if ((list = pool_list_get(argc, argv, NULL, &ret)) == NULL)
3522		return (1);
3523
3524	if (pool_list_count(list) == 0 && argc != 0) {
3525		pool_list_free(list);
3526		return (1);
3527	}
3528
3529	if (pool_list_count(list) == 0 && interval == 0) {
3530		pool_list_free(list);
3531		(void) fprintf(stderr, gettext("no pools available\n"));
3532		return (1);
3533	}
3534
3535	/*
3536	 * Enter the main iostat loop.
3537	 */
3538	cb.cb_list = list;
3539	cb.cb_verbose = verbose;
3540	if (guid)
3541		cb.cb_name_flags |= VDEV_NAME_GUID;
3542	if (follow_links)
3543		cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
3544	if (full_name)
3545		cb.cb_name_flags |= VDEV_NAME_PATH;
3546	cb.cb_iteration = 0;
3547	cb.cb_namewidth = 0;
3548
3549	for (;;) {
3550		pool_list_update(list);
3551
3552		if ((npools = pool_list_count(list)) == 0)
3553			break;
3554
3555		/*
3556		 * Refresh all statistics.  This is done as an explicit step
3557		 * before calculating the maximum name width, so that any
3558		 * configuration changes are properly accounted for.
3559		 */
3560		(void) pool_list_iter(list, B_FALSE, refresh_iostat, &cb);
3561
3562		/*
3563		 * Iterate over all pools to determine the maximum width
3564		 * for the pool / device name column across all pools.
3565		 */
3566		cb.cb_namewidth = 0;
3567		(void) pool_list_iter(list, B_FALSE, get_namewidth, &cb);
3568
3569		if (timestamp_fmt != NODATE)
3570			print_timestamp(timestamp_fmt);
3571
3572		/*
3573		 * If it's the first time, or verbose mode, print the header.
3574		 */
3575		if (++cb.cb_iteration == 1 || verbose)
3576			print_iostat_header(&cb);
3577
3578		(void) pool_list_iter(list, B_FALSE, print_iostat, &cb);
3579
3580		/*
3581		 * If there's more than one pool, and we're not in verbose mode
3582		 * (which prints a separator for us), then print a separator.
3583		 */
3584		if (npools > 1 && !verbose)
3585			print_iostat_separator(&cb);
3586
3587		if (verbose)
3588			(void) printf("\n");
3589
3590		/*
3591		 * Flush the output so that redirection to a file isn't buffered
3592		 * indefinitely.
3593		 */
3594		(void) fflush(stdout);
3595
3596		if (interval == 0)
3597			break;
3598
3599		if (count != 0 && --count == 0)
3600			break;
3601
3602		(void) sleep(interval);
3603	}
3604
3605	pool_list_free(list);
3606
3607	return (ret);
3608}
3609
3610typedef struct list_cbdata {
3611	boolean_t	cb_verbose;
3612	int		cb_name_flags;
3613	int		cb_namewidth;
3614	boolean_t	cb_scripted;
3615	zprop_list_t	*cb_proplist;
3616	boolean_t	cb_literal;
3617} list_cbdata_t;
3618
3619
3620/*
3621 * Given a list of columns to display, output appropriate headers for each one.
3622 */
3623static void
3624print_header(list_cbdata_t *cb)
3625{
3626	zprop_list_t *pl = cb->cb_proplist;
3627	char headerbuf[ZPOOL_MAXPROPLEN];
3628	const char *header;
3629	boolean_t first = B_TRUE;
3630	boolean_t right_justify;
3631	size_t width = 0;
3632
3633	for (; pl != NULL; pl = pl->pl_next) {
3634		width = pl->pl_width;
3635		if (first && cb->cb_verbose) {
3636			/*
3637			 * Reset the width to accommodate the verbose listing
3638			 * of devices.
3639			 */
3640			width = cb->cb_namewidth;
3641		}
3642
3643		if (!first)
3644			(void) printf("  ");
3645		else
3646			first = B_FALSE;
3647
3648		right_justify = B_FALSE;
3649		if (pl->pl_prop != ZPROP_INVAL) {
3650			header = zpool_prop_column_name(pl->pl_prop);
3651			right_justify = zpool_prop_align_right(pl->pl_prop);
3652		} else {
3653			int i;
3654
3655			for (i = 0; pl->pl_user_prop[i] != '\0'; i++)
3656				headerbuf[i] = toupper(pl->pl_user_prop[i]);
3657			headerbuf[i] = '\0';
3658			header = headerbuf;
3659		}
3660
3661		if (pl->pl_next == NULL && !right_justify)
3662			(void) printf("%s", header);
3663		else if (right_justify)
3664			(void) printf("%*s", width, header);
3665		else
3666			(void) printf("%-*s", width, header);
3667
3668	}
3669
3670	(void) printf("\n");
3671}
3672
3673/*
3674 * Given a pool and a list of properties, print out all the properties according
3675 * to the described layout. Used by zpool_do_list().
3676 */
3677static void
3678print_pool(zpool_handle_t *zhp, list_cbdata_t *cb)
3679{
3680	zprop_list_t *pl = cb->cb_proplist;
3681	boolean_t first = B_TRUE;
3682	char property[ZPOOL_MAXPROPLEN];
3683	char *propstr;
3684	boolean_t right_justify;
3685	size_t width;
3686
3687	for (; pl != NULL; pl = pl->pl_next) {
3688
3689		width = pl->pl_width;
3690		if (first && cb->cb_verbose) {
3691			/*
3692			 * Reset the width to accommodate the verbose listing
3693			 * of devices.
3694			 */
3695			width = cb->cb_namewidth;
3696		}
3697
3698		if (!first) {
3699			if (cb->cb_scripted)
3700				(void) printf("\t");
3701			else
3702				(void) printf("  ");
3703		} else {
3704			first = B_FALSE;
3705		}
3706
3707		right_justify = B_FALSE;
3708		if (pl->pl_prop != ZPROP_INVAL) {
3709			if (zpool_get_prop(zhp, pl->pl_prop, property,
3710			    sizeof (property), NULL, cb->cb_literal) != 0)
3711				propstr = "-";
3712			else
3713				propstr = property;
3714
3715			right_justify = zpool_prop_align_right(pl->pl_prop);
3716		} else if ((zpool_prop_feature(pl->pl_user_prop) ||
3717		    zpool_prop_unsupported(pl->pl_user_prop)) &&
3718		    zpool_prop_get_feature(zhp, pl->pl_user_prop, property,
3719		    sizeof (property)) == 0) {
3720			propstr = property;
3721		} else {
3722			propstr = "-";
3723		}
3724
3725
3726		/*
3727		 * If this is being called in scripted mode, or if this is the
3728		 * last column and it is left-justified, don't include a width
3729		 * format specifier.
3730		 */
3731		if (cb->cb_scripted || (pl->pl_next == NULL && !right_justify))
3732			(void) printf("%s", propstr);
3733		else if (right_justify)
3734			(void) printf("%*s", width, propstr);
3735		else
3736			(void) printf("%-*s", width, propstr);
3737	}
3738
3739	(void) printf("\n");
3740}
3741
3742static void
3743print_one_column(zpool_prop_t prop, uint64_t value, boolean_t scripted,
3744    boolean_t valid)
3745{
3746	char propval[64];
3747	boolean_t fixed;
3748	size_t width = zprop_width(prop, &fixed, ZFS_TYPE_POOL);
3749
3750	switch (prop) {
3751	case ZPOOL_PROP_EXPANDSZ:
3752	case ZPOOL_PROP_CHECKPOINT:
3753		if (value == 0)
3754			(void) strlcpy(propval, "-", sizeof (propval));
3755		else
3756			zfs_nicenum(value, propval, sizeof (propval));
3757		break;
3758	case ZPOOL_PROP_FRAGMENTATION:
3759		if (value == ZFS_FRAG_INVALID) {
3760			(void) strlcpy(propval, "-", sizeof (propval));
3761		} else {
3762			(void) snprintf(propval, sizeof (propval), "%llu%%",
3763			    value);
3764		}
3765		break;
3766	case ZPOOL_PROP_CAPACITY:
3767		(void) snprintf(propval, sizeof (propval),
3768		    value < 1000 ? "%1.2f%%" : value < 10000 ?
3769		    "%2.1f%%" : "%3.0f%%", value / 100.0);
3770		break;
3771	default:
3772		zfs_nicenum(value, propval, sizeof (propval));
3773	}
3774
3775	if (!valid)
3776		(void) strlcpy(propval, "-", sizeof (propval));
3777
3778	if (scripted)
3779		(void) printf("\t%s", propval);
3780	else
3781		(void) printf("  %*s", width, propval);
3782}
3783
3784/*
3785 * print static default line per vdev
3786 */
3787void
3788print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
3789    list_cbdata_t *cb, int depth)
3790{
3791	nvlist_t **child;
3792	vdev_stat_t *vs;
3793	uint_t c, children;
3794	char *vname;
3795	boolean_t scripted = cb->cb_scripted;
3796	uint64_t islog = B_FALSE;
3797	char *dashes = "%-*s      -      -      -         -      -      -\n";
3798
3799	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
3800	    (uint64_t **)&vs, &c) == 0);
3801
3802	if (name != NULL) {
3803		boolean_t toplevel = (vs->vs_space != 0);
3804		uint64_t cap;
3805
3806		if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
3807			return;
3808
3809		if (scripted)
3810			(void) printf("\t%s", name);
3811		else if (strlen(name) + depth > cb->cb_namewidth)
3812			(void) printf("%*s%s", depth, "", name);
3813		else
3814			(void) printf("%*s%s%*s", depth, "", name,
3815			    (int)(cb->cb_namewidth - strlen(name) - depth), "");
3816
3817		/*
3818		 * Print the properties for the individual vdevs. Some
3819		 * properties are only applicable to toplevel vdevs. The
3820		 * 'toplevel' boolean value is passed to the print_one_column()
3821		 * to indicate that the value is valid.
3822		 */
3823		print_one_column(ZPOOL_PROP_SIZE, vs->vs_space, scripted,
3824		    toplevel);
3825		print_one_column(ZPOOL_PROP_ALLOCATED, vs->vs_alloc, scripted,
3826		    toplevel);
3827		print_one_column(ZPOOL_PROP_FREE, vs->vs_space - vs->vs_alloc,
3828		    scripted, toplevel);
3829		print_one_column(ZPOOL_PROP_CHECKPOINT,
3830		    vs->vs_checkpoint_space, scripted, toplevel);
3831		print_one_column(ZPOOL_PROP_EXPANDSZ, vs->vs_esize, scripted,
3832		    B_TRUE);
3833		print_one_column(ZPOOL_PROP_FRAGMENTATION,
3834		    vs->vs_fragmentation, scripted,
3835		    (vs->vs_fragmentation != ZFS_FRAG_INVALID && toplevel));
3836		cap = (vs->vs_space == 0) ? 0 :
3837		    (vs->vs_alloc * 10000 / vs->vs_space);
3838		print_one_column(ZPOOL_PROP_CAPACITY, cap, scripted, toplevel);
3839		(void) printf("\n");
3840	}
3841
3842	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
3843	    &child, &children) != 0)
3844		return;
3845
3846	/* list the normal vdevs first */
3847	for (c = 0; c < children; c++) {
3848		uint64_t ishole = B_FALSE;
3849
3850		if (nvlist_lookup_uint64(child[c],
3851		    ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
3852			continue;
3853
3854		if (nvlist_lookup_uint64(child[c],
3855		    ZPOOL_CONFIG_IS_LOG, &islog) == 0 && islog)
3856			continue;
3857
3858		if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
3859			continue;
3860
3861		vname = zpool_vdev_name(g_zfs, zhp, child[c],
3862		    cb->cb_name_flags);
3863		print_list_stats(zhp, vname, child[c], cb, depth + 2);
3864		free(vname);
3865	}
3866
3867	/* list the classes: 'logs', 'dedup', and 'special' */
3868	for (uint_t n = 0; n < 3; n++) {
3869		boolean_t printed = B_FALSE;
3870
3871		for (c = 0; c < children; c++) {
3872			char *bias = NULL;
3873			char *type = NULL;
3874
3875			if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
3876			    &islog) == 0 && islog) {
3877				bias = VDEV_ALLOC_CLASS_LOGS;
3878			} else {
3879				(void) nvlist_lookup_string(child[c],
3880				    ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
3881				(void) nvlist_lookup_string(child[c],
3882				    ZPOOL_CONFIG_TYPE, &type);
3883			}
3884			if (bias == NULL || strcmp(bias, class_name[n]) != 0)
3885				continue;
3886			if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
3887				continue;
3888
3889			if (!printed) {
3890				/* LINTED E_SEC_PRINTF_VAR_FMT */
3891				(void) printf(dashes, cb->cb_namewidth,
3892				    class_name[n]);
3893				printed = B_TRUE;
3894			}
3895			vname = zpool_vdev_name(g_zfs, zhp, child[c],
3896			    cb->cb_name_flags);
3897			print_list_stats(zhp, vname, child[c], cb, depth + 2);
3898			free(vname);
3899		}
3900	}
3901
3902	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
3903	    &child, &children) == 0 && children > 0) {
3904		/* LINTED E_SEC_PRINTF_VAR_FMT */
3905		(void) printf(dashes, cb->cb_namewidth, "cache");
3906		for (c = 0; c < children; c++) {
3907			vname = zpool_vdev_name(g_zfs, zhp, child[c],
3908			    cb->cb_name_flags);
3909			print_list_stats(zhp, vname, child[c], cb, depth + 2);
3910			free(vname);
3911		}
3912	}
3913
3914	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES, &child,
3915	    &children) == 0 && children > 0) {
3916		/* LINTED E_SEC_PRINTF_VAR_FMT */
3917		(void) printf(dashes, cb->cb_namewidth, "spare");
3918		for (c = 0; c < children; c++) {
3919			vname = zpool_vdev_name(g_zfs, zhp, child[c],
3920			    cb->cb_name_flags);
3921			print_list_stats(zhp, vname, child[c], cb, depth + 2);
3922			free(vname);
3923		}
3924	}
3925}
3926
3927/*
3928 * Generic callback function to list a pool.
3929 */
3930int
3931list_callback(zpool_handle_t *zhp, void *data)
3932{
3933	list_cbdata_t *cbp = data;
3934	nvlist_t *config;
3935	nvlist_t *nvroot;
3936
3937	config = zpool_get_config(zhp, NULL);
3938
3939	if (cbp->cb_verbose) {
3940		config = zpool_get_config(zhp, NULL);
3941
3942		verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
3943		    &nvroot) == 0);
3944	}
3945
3946	if (cbp->cb_verbose)
3947		cbp->cb_namewidth = max_width(zhp, nvroot, 0, 0,
3948		    cbp->cb_name_flags);
3949
3950	print_pool(zhp, cbp);
3951
3952	if (cbp->cb_verbose)
3953		print_list_stats(zhp, NULL, nvroot, cbp, 0);
3954
3955	return (0);
3956}
3957
3958/*
3959 * zpool list [-gHLP] [-o prop[,prop]*] [-T d|u] [pool] ... [interval [count]]
3960 *
3961 *	-g	Display guid for individual vdev name.
3962 *	-H	Scripted mode.  Don't display headers, and separate properties
3963 *		by a single tab.
3964 *	-L	Follow links when resolving vdev path name.
3965 *	-o	List of properties to display.  Defaults to
3966 *		"name,size,allocated,free,expandsize,fragmentation,capacity,"
3967 *		"dedupratio,health,altroot"
3968 *	-p	Diplay values in parsable (exact) format.
3969 *	-P	Display full path for vdev name.
3970 *	-T	Display a timestamp in date(1) or Unix format
3971 *
3972 * List all pools in the system, whether or not they're healthy.  Output space
3973 * statistics for each one, as well as health status summary.
3974 */
3975int
3976zpool_do_list(int argc, char **argv)
3977{
3978	int c;
3979	int ret;
3980	list_cbdata_t cb = { 0 };
3981	static char default_props[] =
3982	    "name,size,allocated,free,checkpoint,expandsize,fragmentation,"
3983	    "capacity,dedupratio,health,altroot";
3984	char *props = default_props;
3985	unsigned long interval = 0, count = 0;
3986	zpool_list_t *list;
3987	boolean_t first = B_TRUE;
3988
3989	/* check options */
3990	while ((c = getopt(argc, argv, ":gHLo:pPT:v")) != -1) {
3991		switch (c) {
3992		case 'g':
3993			cb.cb_name_flags |= VDEV_NAME_GUID;
3994			break;
3995		case 'H':
3996			cb.cb_scripted = B_TRUE;
3997			break;
3998		case 'L':
3999			cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
4000			break;
4001		case 'o':
4002			props = optarg;
4003			break;
4004		case 'P':
4005			cb.cb_name_flags |= VDEV_NAME_PATH;
4006			break;
4007		case 'p':
4008			cb.cb_literal = B_TRUE;
4009			break;
4010		case 'T':
4011			get_timestamp_arg(*optarg);
4012			break;
4013		case 'v':
4014			cb.cb_verbose = B_TRUE;
4015			cb.cb_namewidth = 8;	/* 8 until precalc is avail */
4016			break;
4017		case ':':
4018			(void) fprintf(stderr, gettext("missing argument for "
4019			    "'%c' option\n"), optopt);
4020			usage(B_FALSE);
4021			break;
4022		case '?':
4023			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
4024			    optopt);
4025			usage(B_FALSE);
4026		}
4027	}
4028
4029	argc -= optind;
4030	argv += optind;
4031
4032	get_interval_count(&argc, argv, &interval, &count);
4033
4034	if (zprop_get_list(g_zfs, props, &cb.cb_proplist, ZFS_TYPE_POOL) != 0)
4035		usage(B_FALSE);
4036
4037	for (;;) {
4038		if ((list = pool_list_get(argc, argv, &cb.cb_proplist,
4039		    &ret)) == NULL)
4040			return (1);
4041
4042		if (pool_list_count(list) == 0)
4043			break;
4044
4045		cb.cb_namewidth = 0;
4046		(void) pool_list_iter(list, B_FALSE, get_namewidth, &cb);
4047
4048		if (timestamp_fmt != NODATE)
4049			print_timestamp(timestamp_fmt);
4050
4051		if (!cb.cb_scripted && (first || cb.cb_verbose)) {
4052			print_header(&cb);
4053			first = B_FALSE;
4054		}
4055		ret = pool_list_iter(list, B_TRUE, list_callback, &cb);
4056
4057		if (interval == 0)
4058			break;
4059
4060		if (count != 0 && --count == 0)
4061			break;
4062
4063		pool_list_free(list);
4064		(void) sleep(interval);
4065	}
4066
4067	if (argc == 0 && !cb.cb_scripted && pool_list_count(list) == 0) {
4068		(void) printf(gettext("no pools available\n"));
4069		ret = 0;
4070	}
4071
4072	pool_list_free(list);
4073	zprop_free_list(cb.cb_proplist);
4074	return (ret);
4075}
4076
4077static int
4078zpool_do_attach_or_replace(int argc, char **argv, int replacing)
4079{
4080	boolean_t force = B_FALSE;
4081	int c;
4082	nvlist_t *nvroot;
4083	char *poolname, *old_disk, *new_disk;
4084	zpool_handle_t *zhp;
4085	zpool_boot_label_t boot_type;
4086	uint64_t boot_size;
4087	nvlist_t *props = NULL;
4088	char *propval;
4089	int ret;
4090
4091	/* check options */
4092	while ((c = getopt(argc, argv, "fo:")) != -1) {
4093		switch (c) {
4094		case 'f':
4095			force = B_TRUE;
4096			break;
4097		case 'o':
4098			if ((propval = strchr(optarg, '=')) == NULL) {
4099				(void) fprintf(stderr, gettext("missing "
4100				    "'=' for -o option\n"));
4101				usage(B_FALSE);
4102			}
4103			*propval = '\0';
4104			propval++;
4105
4106			if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
4107			    (add_prop_list(optarg, propval, &props, B_TRUE)))
4108				usage(B_FALSE);
4109			break;
4110		case '?':
4111			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
4112			    optopt);
4113			usage(B_FALSE);
4114		}
4115	}
4116
4117	argc -= optind;
4118	argv += optind;
4119
4120	/* get pool name and check number of arguments */
4121	if (argc < 1) {
4122		(void) fprintf(stderr, gettext("missing pool name argument\n"));
4123		usage(B_FALSE);
4124	}
4125
4126	poolname = argv[0];
4127
4128	if (argc < 2) {
4129		(void) fprintf(stderr,
4130		    gettext("missing <device> specification\n"));
4131		usage(B_FALSE);
4132	}
4133
4134	old_disk = argv[1];
4135
4136	if (argc < 3) {
4137		if (!replacing) {
4138			(void) fprintf(stderr,
4139			    gettext("missing <new_device> specification\n"));
4140			usage(B_FALSE);
4141		}
4142		new_disk = old_disk;
4143		argc -= 1;
4144		argv += 1;
4145	} else {
4146		new_disk = argv[2];
4147		argc -= 2;
4148		argv += 2;
4149	}
4150
4151	if (argc > 1) {
4152		(void) fprintf(stderr, gettext("too many arguments\n"));
4153		usage(B_FALSE);
4154	}
4155
4156	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
4157		return (1);
4158
4159	if (zpool_get_config(zhp, NULL) == NULL) {
4160		(void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
4161		    poolname);
4162		zpool_close(zhp);
4163		return (1);
4164	}
4165
4166	if (zpool_is_bootable(zhp))
4167		boot_type = ZPOOL_COPY_BOOT_LABEL;
4168	else
4169		boot_type = ZPOOL_NO_BOOT_LABEL;
4170
4171	boot_size = zpool_get_prop_int(zhp, ZPOOL_PROP_BOOTSIZE, NULL);
4172
4173	/* unless manually specified use "ashift" pool property (if set) */
4174	if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
4175		int intval;
4176		zprop_source_t src;
4177		char strval[ZPOOL_MAXPROPLEN];
4178
4179		intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
4180		if (src != ZPROP_SRC_DEFAULT) {
4181			(void) sprintf(strval, "%" PRId32, intval);
4182			verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
4183			    &props, B_TRUE) == 0);
4184		}
4185	}
4186
4187	nvroot = make_root_vdev(zhp, props, force, B_FALSE, replacing, B_FALSE,
4188	    boot_type, boot_size, argc, argv);
4189	if (nvroot == NULL) {
4190		zpool_close(zhp);
4191		return (1);
4192	}
4193
4194	ret = zpool_vdev_attach(zhp, old_disk, new_disk, nvroot, replacing);
4195
4196	nvlist_free(nvroot);
4197	zpool_close(zhp);
4198
4199	return (ret);
4200}
4201
4202/*
4203 * zpool replace [-f] <pool> <device> <new_device>
4204 *
4205 *	-f	Force attach, even if <new_device> appears to be in use.
4206 *
4207 * Replace <device> with <new_device>.
4208 */
4209/* ARGSUSED */
4210int
4211zpool_do_replace(int argc, char **argv)
4212{
4213	return (zpool_do_attach_or_replace(argc, argv, B_TRUE));
4214}
4215
4216/*
4217 * zpool attach [-f] [-o property=value] <pool> <device> <new_device>
4218 *
4219 *	-f	Force attach, even if <new_device> appears to be in use.
4220 *	-o	Set property=value.
4221 *
4222 * Attach <new_device> to the mirror containing <device>.  If <device> is not
4223 * part of a mirror, then <device> will be transformed into a mirror of
4224 * <device> and <new_device>.  In either case, <new_device> will begin life
4225 * with a DTL of [0, now], and will immediately begin to resilver itself.
4226 */
4227int
4228zpool_do_attach(int argc, char **argv)
4229{
4230	return (zpool_do_attach_or_replace(argc, argv, B_FALSE));
4231}
4232
4233/*
4234 * zpool detach [-f] <pool> <device>
4235 *
4236 *	-f	Force detach of <device>, even if DTLs argue against it
4237 *		(not supported yet)
4238 *
4239 * Detach a device from a mirror.  The operation will be refused if <device>
4240 * is the last device in the mirror, or if the DTLs indicate that this device
4241 * has the only valid copy of some data.
4242 */
4243/* ARGSUSED */
4244int
4245zpool_do_detach(int argc, char **argv)
4246{
4247	int c;
4248	char *poolname, *path;
4249	zpool_handle_t *zhp;
4250	int ret;
4251
4252	/* check options */
4253	while ((c = getopt(argc, argv, "f")) != -1) {
4254		switch (c) {
4255		case 'f':
4256		case '?':
4257			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
4258			    optopt);
4259			usage(B_FALSE);
4260		}
4261	}
4262
4263	argc -= optind;
4264	argv += optind;
4265
4266	/* get pool name and check number of arguments */
4267	if (argc < 1) {
4268		(void) fprintf(stderr, gettext("missing pool name argument\n"));
4269		usage(B_FALSE);
4270	}
4271
4272	if (argc < 2) {
4273		(void) fprintf(stderr,
4274		    gettext("missing <device> specification\n"));
4275		usage(B_FALSE);
4276	}
4277
4278	poolname = argv[0];
4279	path = argv[1];
4280
4281	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
4282		return (1);
4283
4284	ret = zpool_vdev_detach(zhp, path);
4285
4286	zpool_close(zhp);
4287
4288	return (ret);
4289}
4290
4291/*
4292 * zpool split [-gLnP] [-o prop=val] ...
4293 *		[-o mntopt] ...
4294 *		[-R altroot] <pool> <newpool> [<device> ...]
4295 *
4296 *	-g      Display guid for individual vdev name.
4297 *	-L	Follow links when resolving vdev path name.
4298 *	-n	Do not split the pool, but display the resulting layout if
4299 *		it were to be split.
4300 *	-o	Set property=value, or set mount options.
4301 *	-P	Display full path for vdev name.
4302 *	-R	Mount the split-off pool under an alternate root.
4303 *	-l	Load encryption keys while importing.
4304 *
4305 * Splits the named pool and gives it the new pool name.  Devices to be split
4306 * off may be listed, provided that no more than one device is specified
4307 * per top-level vdev mirror.  The newly split pool is left in an exported
4308 * state unless -R is specified.
4309 *
4310 * Restrictions: the top-level of the pool pool must only be made up of
4311 * mirrors; all devices in the pool must be healthy; no device may be
4312 * undergoing a resilvering operation.
4313 */
4314int
4315zpool_do_split(int argc, char **argv)
4316{
4317	char *srcpool, *newpool, *propval;
4318	char *mntopts = NULL;
4319	splitflags_t flags;
4320	int c, ret = 0;
4321	boolean_t loadkeys = B_FALSE;
4322	zpool_handle_t *zhp;
4323	nvlist_t *config, *props = NULL;
4324
4325	flags.dryrun = B_FALSE;
4326	flags.import = B_FALSE;
4327	flags.name_flags = 0;
4328
4329	/* check options */
4330	while ((c = getopt(argc, argv, ":gLR:lno:P")) != -1) {
4331		switch (c) {
4332		case 'g':
4333			flags.name_flags |= VDEV_NAME_GUID;
4334			break;
4335		case 'L':
4336			flags.name_flags |= VDEV_NAME_FOLLOW_LINKS;
4337			break;
4338		case 'R':
4339			flags.import = B_TRUE;
4340			if (add_prop_list(
4341			    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), optarg,
4342			    &props, B_TRUE) != 0) {
4343				nvlist_free(props);
4344				usage(B_FALSE);
4345			}
4346			break;
4347		case 'l':
4348			loadkeys = B_TRUE;
4349			break;
4350		case 'n':
4351			flags.dryrun = B_TRUE;
4352			break;
4353		case 'o':
4354			if ((propval = strchr(optarg, '=')) != NULL) {
4355				*propval = '\0';
4356				propval++;
4357				if (add_prop_list(optarg, propval,
4358				    &props, B_TRUE) != 0) {
4359					nvlist_free(props);
4360					usage(B_FALSE);
4361				}
4362			} else {
4363				mntopts = optarg;
4364			}
4365			break;
4366		case 'P':
4367			flags.name_flags |= VDEV_NAME_PATH;
4368			break;
4369		case ':':
4370			(void) fprintf(stderr, gettext("missing argument for "
4371			    "'%c' option\n"), optopt);
4372			usage(B_FALSE);
4373			break;
4374		case '?':
4375			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
4376			    optopt);
4377			usage(B_FALSE);
4378			break;
4379		}
4380	}
4381
4382	if (!flags.import && mntopts != NULL) {
4383		(void) fprintf(stderr, gettext("setting mntopts is only "
4384		    "valid when importing the pool\n"));
4385		usage(B_FALSE);
4386	}
4387
4388	if (!flags.import && loadkeys) {
4389		(void) fprintf(stderr, gettext("loading keys is only "
4390		    "valid when importing the pool\n"));
4391		usage(B_FALSE);
4392	}
4393
4394	argc -= optind;
4395	argv += optind;
4396
4397	if (argc < 1) {
4398		(void) fprintf(stderr, gettext("Missing pool name\n"));
4399		usage(B_FALSE);
4400	}
4401	if (argc < 2) {
4402		(void) fprintf(stderr, gettext("Missing new pool name\n"));
4403		usage(B_FALSE);
4404	}
4405
4406	srcpool = argv[0];
4407	newpool = argv[1];
4408
4409	argc -= 2;
4410	argv += 2;
4411
4412	if ((zhp = zpool_open(g_zfs, srcpool)) == NULL)
4413		return (1);
4414
4415	config = split_mirror_vdev(zhp, newpool, props, flags, argc, argv);
4416	if (config == NULL) {
4417		ret = 1;
4418	} else {
4419		if (flags.dryrun) {
4420			(void) printf(gettext("would create '%s' with the "
4421			    "following layout:\n\n"), newpool);
4422			print_vdev_tree(NULL, newpool, config, 0, "",
4423			    flags.name_flags);
4424		}
4425		nvlist_free(config);
4426	}
4427
4428	zpool_close(zhp);
4429
4430	if (ret != 0 || flags.dryrun || !flags.import)
4431		return (ret);
4432
4433	/*
4434	 * The split was successful. Now we need to open the new
4435	 * pool and import it.
4436	 */
4437	if ((zhp = zpool_open_canfail(g_zfs, newpool)) == NULL)
4438		return (1);
4439
4440	if (loadkeys) {
4441		ret = zfs_crypto_attempt_load_keys(g_zfs, newpool);
4442		if (ret != 0)
4443			ret = 1;
4444	}
4445
4446	if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
4447	    zpool_enable_datasets(zhp, mntopts, 0) != 0) {
4448		ret = 1;
4449		(void) fprintf(stderr, gettext("Split was successful, but "
4450		    "the datasets could not all be mounted\n"));
4451		(void) fprintf(stderr, gettext("Try doing '%s' with a "
4452		    "different altroot\n"), "zpool import");
4453	}
4454	zpool_close(zhp);
4455
4456	return (ret);
4457}
4458
4459
4460
4461/*
4462 * zpool online <pool> <device> ...
4463 */
4464int
4465zpool_do_online(int argc, char **argv)
4466{
4467	int c, i;
4468	char *poolname;
4469	zpool_handle_t *zhp;
4470	int ret = 0;
4471	vdev_state_t newstate;
4472	int flags = 0;
4473
4474	/* check options */
4475	while ((c = getopt(argc, argv, "et")) != -1) {
4476		switch (c) {
4477		case 'e':
4478			flags |= ZFS_ONLINE_EXPAND;
4479			break;
4480		case 't':
4481		case '?':
4482			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
4483			    optopt);
4484			usage(B_FALSE);
4485		}
4486	}
4487
4488	argc -= optind;
4489	argv += optind;
4490
4491	/* get pool name and check number of arguments */
4492	if (argc < 1) {
4493		(void) fprintf(stderr, gettext("missing pool name\n"));
4494		usage(B_FALSE);
4495	}
4496	if (argc < 2) {
4497		(void) fprintf(stderr, gettext("missing device name\n"));
4498		usage(B_FALSE);
4499	}
4500
4501	poolname = argv[0];
4502
4503	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
4504		return (1);
4505
4506	for (i = 1; i < argc; i++) {
4507		if (zpool_vdev_online(zhp, argv[i], flags, &newstate) == 0) {
4508			if (newstate != VDEV_STATE_HEALTHY) {
4509				(void) printf(gettext("warning: device '%s' "
4510				    "onlined, but remains in faulted state\n"),
4511				    argv[i]);
4512				if (newstate == VDEV_STATE_FAULTED)
4513					(void) printf(gettext("use 'zpool "
4514					    "clear' to restore a faulted "
4515					    "device\n"));
4516				else
4517					(void) printf(gettext("use 'zpool "
4518					    "replace' to replace devices "
4519					    "that are no longer present\n"));
4520			}
4521		} else {
4522			ret = 1;
4523		}
4524	}
4525
4526	zpool_close(zhp);
4527
4528	return (ret);
4529}
4530
4531/*
4532 * zpool offline [-ft] <pool> <device> ...
4533 *
4534 *	-f	Force the device into the offline state, even if doing
4535 *		so would appear to compromise pool availability.
4536 *		(not supported yet)
4537 *
4538 *	-t	Only take the device off-line temporarily.  The offline
4539 *		state will not be persistent across reboots.
4540 */
4541/* ARGSUSED */
4542int
4543zpool_do_offline(int argc, char **argv)
4544{
4545	int c, i;
4546	char *poolname;
4547	zpool_handle_t *zhp;
4548	int ret = 0;
4549	boolean_t istmp = B_FALSE;
4550
4551	/* check options */
4552	while ((c = getopt(argc, argv, "ft")) != -1) {
4553		switch (c) {
4554		case 't':
4555			istmp = B_TRUE;
4556			break;
4557		case 'f':
4558		case '?':
4559			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
4560			    optopt);
4561			usage(B_FALSE);
4562		}
4563	}
4564
4565	argc -= optind;
4566	argv += optind;
4567
4568	/* get pool name and check number of arguments */
4569	if (argc < 1) {
4570		(void) fprintf(stderr, gettext("missing pool name\n"));
4571		usage(B_FALSE);
4572	}
4573	if (argc < 2) {
4574		(void) fprintf(stderr, gettext("missing device name\n"));
4575		usage(B_FALSE);
4576	}
4577
4578	poolname = argv[0];
4579
4580	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
4581		return (1);
4582
4583	for (i = 1; i < argc; i++) {
4584		if (zpool_vdev_offline(zhp, argv[i], istmp) != 0)
4585			ret = 1;
4586	}
4587
4588	zpool_close(zhp);
4589
4590	return (ret);
4591}
4592
4593/*
4594 * zpool clear <pool> [device]
4595 *
4596 * Clear all errors associated with a pool or a particular device.
4597 */
4598int
4599zpool_do_clear(int argc, char **argv)
4600{
4601	int c;
4602	int ret = 0;
4603	boolean_t dryrun = B_FALSE;
4604	boolean_t do_rewind = B_FALSE;
4605	boolean_t xtreme_rewind = B_FALSE;
4606	uint32_t rewind_policy = ZPOOL_NO_REWIND;
4607	nvlist_t *policy = NULL;
4608	zpool_handle_t *zhp;
4609	char *pool, *device;
4610
4611	/* check options */
4612	while ((c = getopt(argc, argv, "FnX")) != -1) {
4613		switch (c) {
4614		case 'F':
4615			do_rewind = B_TRUE;
4616			break;
4617		case 'n':
4618			dryrun = B_TRUE;
4619			break;
4620		case 'X':
4621			xtreme_rewind = B_TRUE;
4622			break;
4623		case '?':
4624			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
4625			    optopt);
4626			usage(B_FALSE);
4627		}
4628	}
4629
4630	argc -= optind;
4631	argv += optind;
4632
4633	if (argc < 1) {
4634		(void) fprintf(stderr, gettext("missing pool name\n"));
4635		usage(B_FALSE);
4636	}
4637
4638	if (argc > 2) {
4639		(void) fprintf(stderr, gettext("too many arguments\n"));
4640		usage(B_FALSE);
4641	}
4642
4643	if ((dryrun || xtreme_rewind) && !do_rewind) {
4644		(void) fprintf(stderr,
4645		    gettext("-n or -X only meaningful with -F\n"));
4646		usage(B_FALSE);
4647	}
4648	if (dryrun)
4649		rewind_policy = ZPOOL_TRY_REWIND;
4650	else if (do_rewind)
4651		rewind_policy = ZPOOL_DO_REWIND;
4652	if (xtreme_rewind)
4653		rewind_policy |= ZPOOL_EXTREME_REWIND;
4654
4655	/* In future, further rewind policy choices can be passed along here */
4656	if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
4657	    nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
4658	    rewind_policy) != 0) {
4659		return (1);
4660	}
4661
4662	pool = argv[0];
4663	device = argc == 2 ? argv[1] : NULL;
4664
4665	if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
4666		nvlist_free(policy);
4667		return (1);
4668	}
4669
4670	if (zpool_clear(zhp, device, policy) != 0)
4671		ret = 1;
4672
4673	zpool_close(zhp);
4674
4675	nvlist_free(policy);
4676
4677	return (ret);
4678}
4679
4680/*
4681 * zpool reguid <pool>
4682 */
4683int
4684zpool_do_reguid(int argc, char **argv)
4685{
4686	int c;
4687	char *poolname;
4688	zpool_handle_t *zhp;
4689	int ret = 0;
4690
4691	/* check options */
4692	while ((c = getopt(argc, argv, "")) != -1) {
4693		switch (c) {
4694		case '?':
4695			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
4696			    optopt);
4697			usage(B_FALSE);
4698		}
4699	}
4700
4701	argc -= optind;
4702	argv += optind;
4703
4704	/* get pool name and check number of arguments */
4705	if (argc < 1) {
4706		(void) fprintf(stderr, gettext("missing pool name\n"));
4707		usage(B_FALSE);
4708	}
4709
4710	if (argc > 1) {
4711		(void) fprintf(stderr, gettext("too many arguments\n"));
4712		usage(B_FALSE);
4713	}
4714
4715	poolname = argv[0];
4716	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
4717		return (1);
4718
4719	ret = zpool_reguid(zhp);
4720
4721	zpool_close(zhp);
4722	return (ret);
4723}
4724
4725
4726/*
4727 * zpool reopen <pool>
4728 *
4729 * Reopen the pool so that the kernel can update the sizes of all vdevs.
4730 */
4731int
4732zpool_do_reopen(int argc, char **argv)
4733{
4734	int c;
4735	int ret = 0;
4736	zpool_handle_t *zhp;
4737	char *pool;
4738
4739	/* check options */
4740	while ((c = getopt(argc, argv, "")) != -1) {
4741		switch (c) {
4742		case '?':
4743			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
4744			    optopt);
4745			usage(B_FALSE);
4746		}
4747	}
4748
4749	argc--;
4750	argv++;
4751
4752	if (argc < 1) {
4753		(void) fprintf(stderr, gettext("missing pool name\n"));
4754		usage(B_FALSE);
4755	}
4756
4757	if (argc > 1) {
4758		(void) fprintf(stderr, gettext("too many arguments\n"));
4759		usage(B_FALSE);
4760	}
4761
4762	pool = argv[0];
4763	if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL)
4764		return (1);
4765
4766	ret = zpool_reopen(zhp);
4767	zpool_close(zhp);
4768	return (ret);
4769}
4770
4771typedef struct scrub_cbdata {
4772	int	cb_type;
4773	int	cb_argc;
4774	char	**cb_argv;
4775	pool_scrub_cmd_t cb_scrub_cmd;
4776} scrub_cbdata_t;
4777
4778static boolean_t
4779zpool_has_checkpoint(zpool_handle_t *zhp)
4780{
4781	nvlist_t *config, *nvroot;
4782
4783	config = zpool_get_config(zhp, NULL);
4784
4785	if (config != NULL) {
4786		pool_checkpoint_stat_t *pcs = NULL;
4787		uint_t c;
4788
4789		nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
4790		(void) nvlist_lookup_uint64_array(nvroot,
4791		    ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
4792
4793		if (pcs == NULL || pcs->pcs_state == CS_NONE)
4794			return (B_FALSE);
4795
4796		assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS ||
4797		    pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
4798		return (B_TRUE);
4799	}
4800
4801	return (B_FALSE);
4802}
4803
4804int
4805scrub_callback(zpool_handle_t *zhp, void *data)
4806{
4807	scrub_cbdata_t *cb = data;
4808	int err;
4809
4810	/*
4811	 * Ignore faulted pools.
4812	 */
4813	if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
4814		(void) fprintf(stderr, gettext("cannot scan '%s': pool is "
4815		    "currently unavailable\n"), zpool_get_name(zhp));
4816		return (1);
4817	}
4818
4819	err = zpool_scan(zhp, cb->cb_type, cb->cb_scrub_cmd);
4820
4821	if (err == 0 && zpool_has_checkpoint(zhp) &&
4822	    cb->cb_type == POOL_SCAN_SCRUB) {
4823		(void) printf(gettext("warning: will not scrub state that "
4824		    "belongs to the checkpoint of pool '%s'\n"),
4825		    zpool_get_name(zhp));
4826	}
4827
4828	return (err != 0);
4829}
4830
4831/*
4832 * zpool scrub [-s | -p] <pool> ...
4833 *
4834 *	-s	Stop.  Stops any in-progress scrub.
4835 *	-p	Pause. Pause in-progress scrub.
4836 */
4837int
4838zpool_do_scrub(int argc, char **argv)
4839{
4840	int c;
4841	scrub_cbdata_t cb;
4842
4843	cb.cb_type = POOL_SCAN_SCRUB;
4844	cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
4845
4846	/* check options */
4847	while ((c = getopt(argc, argv, "sp")) != -1) {
4848		switch (c) {
4849		case 's':
4850			cb.cb_type = POOL_SCAN_NONE;
4851			break;
4852		case 'p':
4853			cb.cb_scrub_cmd = POOL_SCRUB_PAUSE;
4854			break;
4855		case '?':
4856			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
4857			    optopt);
4858			usage(B_FALSE);
4859		}
4860	}
4861
4862	if (cb.cb_type == POOL_SCAN_NONE &&
4863	    cb.cb_scrub_cmd == POOL_SCRUB_PAUSE) {
4864		(void) fprintf(stderr, gettext("invalid option combination: "
4865		    "-s and -p are mutually exclusive\n"));
4866		usage(B_FALSE);
4867	}
4868
4869	cb.cb_argc = argc;
4870	cb.cb_argv = argv;
4871	argc -= optind;
4872	argv += optind;
4873
4874	if (argc < 1) {
4875		(void) fprintf(stderr, gettext("missing pool name argument\n"));
4876		usage(B_FALSE);
4877	}
4878
4879	return (for_each_pool(argc, argv, B_TRUE, NULL, scrub_callback, &cb));
4880}
4881
4882/*
4883 * zpool resilver <pool> ...
4884 *
4885 *	Restarts any in-progress resilver
4886 */
4887int
4888zpool_do_resilver(int argc, char **argv)
4889{
4890	int c;
4891	scrub_cbdata_t cb;
4892
4893	cb.cb_type = POOL_SCAN_RESILVER;
4894	cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
4895	cb.cb_argc = argc;
4896	cb.cb_argv = argv;
4897
4898	/* check options */
4899	while ((c = getopt(argc, argv, "")) != -1) {
4900		switch (c) {
4901		case '?':
4902			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
4903			    optopt);
4904			usage(B_FALSE);
4905		}
4906	}
4907
4908	argc -= optind;
4909	argv += optind;
4910
4911	if (argc < 1) {
4912		(void) fprintf(stderr, gettext("missing pool name argument\n"));
4913		usage(B_FALSE);
4914	}
4915
4916	return (for_each_pool(argc, argv, B_TRUE, NULL, scrub_callback, &cb));
4917}
4918
4919/*
4920 * zpool trim [-d] [-r <rate>] [-c | -s] <pool> [<device> ...]
4921 *
4922 *	-c		Cancel. Ends any in-progress trim.
4923 *	-d		Secure trim.  Requires kernel and device support.
4924 *	-r <rate>	Sets the TRIM rate in bytes (per second). Supports
4925 *			adding a multiplier suffix such as 'k' or 'm'.
4926 *	-s		Suspend. TRIM can then be restarted with no flags.
4927 */
4928int
4929zpool_do_trim(int argc, char **argv)
4930{
4931	struct option long_options[] = {
4932		{"cancel",	no_argument,		NULL,	'c'},
4933		{"secure",	no_argument,		NULL,	'd'},
4934		{"rate",	required_argument,	NULL,	'r'},
4935		{"suspend",	no_argument,		NULL,	's'},
4936		{0, 0, 0, 0}
4937	};
4938
4939	pool_trim_func_t cmd_type = POOL_TRIM_START;
4940	uint64_t rate = 0;
4941	boolean_t secure = B_FALSE;
4942
4943	int c;
4944	while ((c = getopt_long(argc, argv, "cdr:s", long_options, NULL))
4945	    != -1) {
4946		switch (c) {
4947		case 'c':
4948			if (cmd_type != POOL_TRIM_START &&
4949			    cmd_type != POOL_TRIM_CANCEL) {
4950				(void) fprintf(stderr, gettext("-c cannot be "
4951				    "combined with other options\n"));
4952				usage(B_FALSE);
4953			}
4954			cmd_type = POOL_TRIM_CANCEL;
4955			break;
4956		case 'd':
4957			if (cmd_type != POOL_TRIM_START) {
4958				(void) fprintf(stderr, gettext("-d cannot be "
4959				    "combined with the -c or -s options\n"));
4960				usage(B_FALSE);
4961			}
4962			secure = B_TRUE;
4963			break;
4964		case 'r':
4965			if (cmd_type != POOL_TRIM_START) {
4966				(void) fprintf(stderr, gettext("-r cannot be "
4967				    "combined with the -c or -s options\n"));
4968				usage(B_FALSE);
4969			}
4970			if (zfs_nicestrtonum(NULL, optarg, &rate) == -1) {
4971				(void) fprintf(stderr,
4972				    gettext("invalid value for rate\n"));
4973				usage(B_FALSE);
4974			}
4975			break;
4976		case 's':
4977			if (cmd_type != POOL_TRIM_START &&
4978			    cmd_type != POOL_TRIM_SUSPEND) {
4979				(void) fprintf(stderr, gettext("-s cannot be "
4980				    "combined with other options\n"));
4981				usage(B_FALSE);
4982			}
4983			cmd_type = POOL_TRIM_SUSPEND;
4984			break;
4985		case '?':
4986			if (optopt != 0) {
4987				(void) fprintf(stderr,
4988				    gettext("invalid option '%c'\n"), optopt);
4989			} else {
4990				(void) fprintf(stderr,
4991				    gettext("invalid option '%s'\n"),
4992				    argv[optind - 1]);
4993			}
4994			usage(B_FALSE);
4995		}
4996	}
4997
4998	argc -= optind;
4999	argv += optind;
5000
5001	if (argc < 1) {
5002		(void) fprintf(stderr, gettext("missing pool name argument\n"));
5003		usage(B_FALSE);
5004		return (-1);
5005	}
5006
5007	char *poolname = argv[0];
5008	zpool_handle_t *zhp = zpool_open(g_zfs, poolname);
5009	if (zhp == NULL)
5010		return (-1);
5011
5012	trimflags_t trim_flags = {
5013		.secure = secure,
5014		.rate = rate,
5015	};
5016
5017	nvlist_t *vdevs = fnvlist_alloc();
5018	if (argc == 1) {
5019		/* no individual leaf vdevs specified, so add them all */
5020		nvlist_t *config = zpool_get_config(zhp, NULL);
5021		nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
5022		    ZPOOL_CONFIG_VDEV_TREE);
5023		zpool_collect_leaves(zhp, nvroot, vdevs);
5024		trim_flags.fullpool = B_TRUE;
5025	} else {
5026		trim_flags.fullpool = B_FALSE;
5027		for (int i = 1; i < argc; i++) {
5028			fnvlist_add_boolean(vdevs, argv[i]);
5029		}
5030	}
5031
5032	int error = zpool_trim(zhp, cmd_type, vdevs, &trim_flags);
5033
5034	fnvlist_free(vdevs);
5035	zpool_close(zhp);
5036
5037	return (error);
5038}
5039
5040/*
5041 * zpool initialize [-c | -s] <pool> [<vdev> ...]
5042 * Initialize all unused blocks in the specified vdevs, or all vdevs in the pool
5043 * if none specified.
5044 *
5045 *	-c	Cancel. Ends active initializing.
5046 *	-s	Suspend. Initializing can then be restarted with no flags.
5047 */
5048int
5049zpool_do_initialize(int argc, char **argv)
5050{
5051	int c;
5052	char *poolname;
5053	zpool_handle_t *zhp;
5054	nvlist_t *vdevs;
5055	int err = 0;
5056
5057	struct option long_options[] = {
5058		{"cancel",	no_argument,		NULL, 'c'},
5059		{"suspend",	no_argument,		NULL, 's'},
5060		{0, 0, 0, 0}
5061	};
5062
5063	pool_initialize_func_t cmd_type = POOL_INITIALIZE_START;
5064	while ((c = getopt_long(argc, argv, "cs", long_options, NULL)) != -1) {
5065		switch (c) {
5066		case 'c':
5067			if (cmd_type != POOL_INITIALIZE_START &&
5068			    cmd_type != POOL_INITIALIZE_CANCEL) {
5069				(void) fprintf(stderr, gettext("-c cannot be "
5070				    "combined with other options\n"));
5071				usage(B_FALSE);
5072			}
5073			cmd_type = POOL_INITIALIZE_CANCEL;
5074			break;
5075		case 's':
5076			if (cmd_type != POOL_INITIALIZE_START &&
5077			    cmd_type != POOL_INITIALIZE_SUSPEND) {
5078				(void) fprintf(stderr, gettext("-s cannot be "
5079				    "combined with other options\n"));
5080				usage(B_FALSE);
5081			}
5082			cmd_type = POOL_INITIALIZE_SUSPEND;
5083			break;
5084		case '?':
5085			if (optopt != 0) {
5086				(void) fprintf(stderr,
5087				    gettext("invalid option '%c'\n"), optopt);
5088			} else {
5089				(void) fprintf(stderr,
5090				    gettext("invalid option '%s'\n"),
5091				    argv[optind - 1]);
5092			}
5093			usage(B_FALSE);
5094		}
5095	}
5096
5097	argc -= optind;
5098	argv += optind;
5099
5100	if (argc < 1) {
5101		(void) fprintf(stderr, gettext("missing pool name argument\n"));
5102		usage(B_FALSE);
5103		return (-1);
5104	}
5105
5106	poolname = argv[0];
5107	zhp = zpool_open(g_zfs, poolname);
5108	if (zhp == NULL)
5109		return (-1);
5110
5111	vdevs = fnvlist_alloc();
5112	if (argc == 1) {
5113		/* no individual leaf vdevs specified, so add them all */
5114		nvlist_t *config = zpool_get_config(zhp, NULL);
5115		nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
5116		    ZPOOL_CONFIG_VDEV_TREE);
5117		zpool_collect_leaves(zhp, nvroot, vdevs);
5118	} else {
5119		for (int i = 1; i < argc; i++) {
5120			fnvlist_add_boolean(vdevs, argv[i]);
5121		}
5122	}
5123
5124	err = zpool_initialize(zhp, cmd_type, vdevs);
5125
5126	fnvlist_free(vdevs);
5127	zpool_close(zhp);
5128
5129	return (err);
5130}
5131
5132/*
5133 * Print out detailed scrub status.
5134 */
5135static void
5136print_scan_status(pool_scan_stat_t *ps)
5137{
5138	time_t start, end, pause;
5139	uint64_t total_secs_left;
5140	uint64_t elapsed, secs_left, mins_left, hours_left, days_left;
5141	uint64_t pass_scanned, scanned, pass_issued, issued, total;
5142	uint_t scan_rate, issue_rate;
5143	double fraction_done;
5144	char processed_buf[7], scanned_buf[7], issued_buf[7], total_buf[7];
5145	char srate_buf[7], irate_buf[7];
5146
5147	(void) printf(gettext("  scan: "));
5148
5149	/* If there's never been a scan, there's not much to say. */
5150	if (ps == NULL || ps->pss_func == POOL_SCAN_NONE ||
5151	    ps->pss_func >= POOL_SCAN_FUNCS) {
5152		(void) printf(gettext("none requested\n"));
5153		return;
5154	}
5155
5156	start = ps->pss_start_time;
5157	end = ps->pss_end_time;
5158	pause = ps->pss_pass_scrub_pause;
5159
5160	zfs_nicenum(ps->pss_processed, processed_buf, sizeof (processed_buf));
5161
5162	assert(ps->pss_func == POOL_SCAN_SCRUB ||
5163	    ps->pss_func == POOL_SCAN_RESILVER);
5164
5165	/*
5166	 * Scan is finished or canceled.
5167	 */
5168	if (ps->pss_state == DSS_FINISHED) {
5169		total_secs_left = end - start;
5170		days_left = total_secs_left / 60 / 60 / 24;
5171		hours_left = (total_secs_left / 60 / 60) % 24;
5172		mins_left = (total_secs_left / 60) % 60;
5173		secs_left = (total_secs_left % 60);
5174
5175		if (ps->pss_func == POOL_SCAN_SCRUB) {
5176			(void) printf(gettext("scrub repaired %s "
5177			    "in %llu days %02llu:%02llu:%02llu "
5178			    "with %llu errors on %s"), processed_buf,
5179			    (u_longlong_t)days_left, (u_longlong_t)hours_left,
5180			    (u_longlong_t)mins_left, (u_longlong_t)secs_left,
5181			    (u_longlong_t)ps->pss_errors, ctime(&end));
5182		} else if (ps->pss_func == POOL_SCAN_RESILVER) {
5183			(void) printf(gettext("resilvered %s "
5184			    "in %llu days %02llu:%02llu:%02llu "
5185			    "with %llu errors on %s"), processed_buf,
5186			    (u_longlong_t)days_left, (u_longlong_t)hours_left,
5187			    (u_longlong_t)mins_left, (u_longlong_t)secs_left,
5188			    (u_longlong_t)ps->pss_errors, ctime(&end));
5189		}
5190		return;
5191	} else if (ps->pss_state == DSS_CANCELED) {
5192		if (ps->pss_func == POOL_SCAN_SCRUB) {
5193			(void) printf(gettext("scrub canceled on %s"),
5194			    ctime(&end));
5195		} else if (ps->pss_func == POOL_SCAN_RESILVER) {
5196			(void) printf(gettext("resilver canceled on %s"),
5197			    ctime(&end));
5198		}
5199		return;
5200	}
5201
5202	assert(ps->pss_state == DSS_SCANNING);
5203
5204	/* Scan is in progress. Resilvers can't be paused. */
5205	if (ps->pss_func == POOL_SCAN_SCRUB) {
5206		if (pause == 0) {
5207			(void) printf(gettext("scrub in progress since %s"),
5208			    ctime(&start));
5209		} else {
5210			(void) printf(gettext("scrub paused since %s"),
5211			    ctime(&pause));
5212			(void) printf(gettext("\tscrub started on %s"),
5213			    ctime(&start));
5214		}
5215	} else if (ps->pss_func == POOL_SCAN_RESILVER) {
5216		(void) printf(gettext("resilver in progress since %s"),
5217		    ctime(&start));
5218	}
5219
5220	scanned = ps->pss_examined;
5221	pass_scanned = ps->pss_pass_exam;
5222	issued = ps->pss_issued;
5223	pass_issued = ps->pss_pass_issued;
5224	total = ps->pss_to_examine;
5225
5226	/* we are only done with a block once we have issued the IO for it  */
5227	fraction_done = (double)issued / total;
5228
5229	/* elapsed time for this pass, rounding up to 1 if it's 0 */
5230	elapsed = time(NULL) - ps->pss_pass_start;
5231	elapsed -= ps->pss_pass_scrub_spent_paused;
5232	elapsed = (elapsed != 0) ? elapsed : 1;
5233
5234	scan_rate = pass_scanned / elapsed;
5235	issue_rate = pass_issued / elapsed;
5236	total_secs_left = (issue_rate != 0 && total >= issued) ?
5237	    ((total - issued) / issue_rate) : UINT64_MAX;
5238
5239	days_left = total_secs_left / 60 / 60 / 24;
5240	hours_left = (total_secs_left / 60 / 60) % 24;
5241	mins_left = (total_secs_left / 60) % 60;
5242	secs_left = (total_secs_left % 60);
5243
5244	/* format all of the numbers we will be reporting */
5245	zfs_nicenum(scanned, scanned_buf, sizeof (scanned_buf));
5246	zfs_nicenum(issued, issued_buf, sizeof (issued_buf));
5247	zfs_nicenum(total, total_buf, sizeof (total_buf));
5248	zfs_nicenum(scan_rate, srate_buf, sizeof (srate_buf));
5249	zfs_nicenum(issue_rate, irate_buf, sizeof (irate_buf));
5250
5251	/* do not print estimated time if we have a paused scrub */
5252	if (pause == 0) {
5253		(void) printf(gettext("\t%s scanned at %s/s, "
5254		    "%s issued at %s/s, %s total\n"),
5255		    scanned_buf, srate_buf, issued_buf, irate_buf, total_buf);
5256	} else {
5257		(void) printf(gettext("\t%s scanned, %s issued, %s total\n"),
5258		    scanned_buf, issued_buf, total_buf);
5259	}
5260
5261	if (ps->pss_func == POOL_SCAN_RESILVER) {
5262		(void) printf(gettext("\t%s resilvered, %.2f%% done"),
5263		    processed_buf, 100 * fraction_done);
5264	} else if (ps->pss_func == POOL_SCAN_SCRUB) {
5265		(void) printf(gettext("\t%s repaired, %.2f%% done"),
5266		    processed_buf, 100 * fraction_done);
5267	}
5268
5269	if (pause == 0) {
5270		if (total_secs_left != UINT64_MAX &&
5271		    issue_rate >= 10 * 1024 * 1024) {
5272			(void) printf(gettext(", %llu days "
5273			    "%02llu:%02llu:%02llu to go\n"),
5274			    (u_longlong_t)days_left, (u_longlong_t)hours_left,
5275			    (u_longlong_t)mins_left, (u_longlong_t)secs_left);
5276		} else {
5277			(void) printf(gettext(", no estimated "
5278			    "completion time\n"));
5279		}
5280	} else {
5281		(void) printf(gettext("\n"));
5282	}
5283}
5284
5285/*
5286 * As we don't scrub checkpointed blocks, we want to warn the
5287 * user that we skipped scanning some blocks if a checkpoint exists
5288 * or existed at any time during the scan.
5289 */
5290static void
5291print_checkpoint_scan_warning(pool_scan_stat_t *ps, pool_checkpoint_stat_t *pcs)
5292{
5293	if (ps == NULL || pcs == NULL)
5294		return;
5295
5296	if (pcs->pcs_state == CS_NONE ||
5297	    pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
5298		return;
5299
5300	assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS);
5301
5302	if (ps->pss_state == DSS_NONE)
5303		return;
5304
5305	if ((ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) &&
5306	    ps->pss_end_time < pcs->pcs_start_time)
5307		return;
5308
5309	if (ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) {
5310		(void) printf(gettext("    scan warning: skipped blocks "
5311		    "that are only referenced by the checkpoint.\n"));
5312	} else {
5313		assert(ps->pss_state == DSS_SCANNING);
5314		(void) printf(gettext("    scan warning: skipping blocks "
5315		    "that are only referenced by the checkpoint.\n"));
5316	}
5317}
5318
5319/*
5320 * Print out detailed removal status.
5321 */
5322static void
5323print_removal_status(zpool_handle_t *zhp, pool_removal_stat_t *prs)
5324{
5325	char copied_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
5326	time_t start, end;
5327	nvlist_t *config, *nvroot;
5328	nvlist_t **child;
5329	uint_t children;
5330	char *vdev_name;
5331
5332	if (prs == NULL || prs->prs_state == DSS_NONE)
5333		return;
5334
5335	/*
5336	 * Determine name of vdev.
5337	 */
5338	config = zpool_get_config(zhp, NULL);
5339	nvroot = fnvlist_lookup_nvlist(config,
5340	    ZPOOL_CONFIG_VDEV_TREE);
5341	verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
5342	    &child, &children) == 0);
5343	assert(prs->prs_removing_vdev < children);
5344	vdev_name = zpool_vdev_name(g_zfs, zhp,
5345	    child[prs->prs_removing_vdev], B_TRUE);
5346
5347	(void) printf(gettext("remove: "));
5348
5349	start = prs->prs_start_time;
5350	end = prs->prs_end_time;
5351	zfs_nicenum(prs->prs_copied, copied_buf, sizeof (copied_buf));
5352
5353	/*
5354	 * Removal is finished or canceled.
5355	 */
5356	if (prs->prs_state == DSS_FINISHED) {
5357		uint64_t minutes_taken = (end - start) / 60;
5358
5359		(void) printf(gettext("Removal of vdev %llu copied %s "
5360		    "in %lluh%um, completed on %s"),
5361		    (longlong_t)prs->prs_removing_vdev,
5362		    copied_buf,
5363		    (u_longlong_t)(minutes_taken / 60),
5364		    (uint_t)(minutes_taken % 60),
5365		    ctime((time_t *)&end));
5366	} else if (prs->prs_state == DSS_CANCELED) {
5367		(void) printf(gettext("Removal of %s canceled on %s"),
5368		    vdev_name, ctime(&end));
5369	} else {
5370		uint64_t copied, total, elapsed, mins_left, hours_left;
5371		double fraction_done;
5372		uint_t rate;
5373
5374		assert(prs->prs_state == DSS_SCANNING);
5375
5376		/*
5377		 * Removal is in progress.
5378		 */
5379		(void) printf(gettext(
5380		    "Evacuation of %s in progress since %s"),
5381		    vdev_name, ctime(&start));
5382
5383		copied = prs->prs_copied > 0 ? prs->prs_copied : 1;
5384		total = prs->prs_to_copy;
5385		fraction_done = (double)copied / total;
5386
5387		/* elapsed time for this pass */
5388		elapsed = time(NULL) - prs->prs_start_time;
5389		elapsed = elapsed > 0 ? elapsed : 1;
5390		rate = copied / elapsed;
5391		rate = rate > 0 ? rate : 1;
5392		mins_left = ((total - copied) / rate) / 60;
5393		hours_left = mins_left / 60;
5394
5395		zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
5396		zfs_nicenum(total, total_buf, sizeof (total_buf));
5397		zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
5398
5399		/*
5400		 * do not print estimated time if hours_left is more than
5401		 * 30 days
5402		 */
5403		(void) printf(gettext("    %s copied out of %s at %s/s, "
5404		    "%.2f%% done"),
5405		    examined_buf, total_buf, rate_buf, 100 * fraction_done);
5406		if (hours_left < (30 * 24)) {
5407			(void) printf(gettext(", %lluh%um to go\n"),
5408			    (u_longlong_t)hours_left, (uint_t)(mins_left % 60));
5409		} else {
5410			(void) printf(gettext(
5411			    ", (copy is slow, no estimated time)\n"));
5412		}
5413	}
5414
5415	if (prs->prs_mapping_memory > 0) {
5416		char mem_buf[7];
5417		zfs_nicenum(prs->prs_mapping_memory, mem_buf, sizeof (mem_buf));
5418		(void) printf(gettext("    %s memory used for "
5419		    "removed device mappings\n"),
5420		    mem_buf);
5421	}
5422}
5423
5424static void
5425print_checkpoint_status(pool_checkpoint_stat_t *pcs)
5426{
5427	time_t start;
5428	char space_buf[7];
5429
5430	if (pcs == NULL || pcs->pcs_state == CS_NONE)
5431		return;
5432
5433	(void) printf(gettext("checkpoint: "));
5434
5435	start = pcs->pcs_start_time;
5436	zfs_nicenum(pcs->pcs_space, space_buf, sizeof (space_buf));
5437
5438	if (pcs->pcs_state == CS_CHECKPOINT_EXISTS) {
5439		char *date = ctime(&start);
5440
5441		/*
5442		 * ctime() adds a newline at the end of the generated
5443		 * string, thus the weird format specifier and the
5444		 * strlen() call used to chop it off from the output.
5445		 */
5446		(void) printf(gettext("created %.*s, consumes %s\n"),
5447		    strlen(date) - 1, date, space_buf);
5448		return;
5449	}
5450
5451	assert(pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
5452
5453	(void) printf(gettext("discarding, %s remaining.\n"),
5454	    space_buf);
5455}
5456
5457static void
5458print_error_log(zpool_handle_t *zhp)
5459{
5460	nvlist_t *nverrlist = NULL;
5461	nvpair_t *elem;
5462	char *pathname;
5463	size_t len = MAXPATHLEN * 2;
5464
5465	if (zpool_get_errlog(zhp, &nverrlist) != 0) {
5466		(void) printf("errors: List of errors unavailable "
5467		    "(insufficient privileges)\n");
5468		return;
5469	}
5470
5471	(void) printf("errors: Permanent errors have been "
5472	    "detected in the following files:\n\n");
5473
5474	pathname = safe_malloc(len);
5475	elem = NULL;
5476	while ((elem = nvlist_next_nvpair(nverrlist, elem)) != NULL) {
5477		nvlist_t *nv;
5478		uint64_t dsobj, obj;
5479
5480		verify(nvpair_value_nvlist(elem, &nv) == 0);
5481		verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_DATASET,
5482		    &dsobj) == 0);
5483		verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_OBJECT,
5484		    &obj) == 0);
5485		zpool_obj_to_path(zhp, dsobj, obj, pathname, len);
5486		(void) printf("%7s %s\n", "", pathname);
5487	}
5488	free(pathname);
5489	nvlist_free(nverrlist);
5490}
5491
5492static void
5493print_spares(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **spares,
5494    uint_t nspares)
5495{
5496	uint_t i;
5497	char *name;
5498
5499	if (nspares == 0)
5500		return;
5501
5502	(void) printf(gettext("\tspares\n"));
5503
5504	for (i = 0; i < nspares; i++) {
5505		name = zpool_vdev_name(g_zfs, zhp, spares[i],
5506		    cb->cb_name_flags);
5507		print_status_config(zhp, cb, name, spares[i], 2, B_TRUE);
5508		free(name);
5509	}
5510}
5511
5512static void
5513print_l2cache(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **l2cache,
5514    uint_t nl2cache)
5515{
5516	uint_t i;
5517	char *name;
5518
5519	if (nl2cache == 0)
5520		return;
5521
5522	(void) printf(gettext("\tcache\n"));
5523
5524	for (i = 0; i < nl2cache; i++) {
5525		name = zpool_vdev_name(g_zfs, zhp, l2cache[i],
5526		    cb->cb_name_flags);
5527		print_status_config(zhp, cb, name, l2cache[i], 2, B_FALSE);
5528		free(name);
5529	}
5530}
5531
5532static void
5533print_dedup_stats(nvlist_t *config)
5534{
5535	ddt_histogram_t *ddh;
5536	ddt_stat_t *dds;
5537	ddt_object_t *ddo;
5538	uint_t c;
5539
5540	/*
5541	 * If the pool was faulted then we may not have been able to
5542	 * obtain the config. Otherwise, if we have anything in the dedup
5543	 * table continue processing the stats.
5544	 */
5545	if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS,
5546	    (uint64_t **)&ddo, &c) != 0)
5547		return;
5548
5549	(void) printf("\n");
5550	(void) printf(gettext(" dedup: "));
5551	if (ddo->ddo_count == 0) {
5552		(void) printf(gettext("no DDT entries\n"));
5553		return;
5554	}
5555
5556	(void) printf("DDT entries %llu, size %llu on disk, %llu in core\n",
5557	    (u_longlong_t)ddo->ddo_count,
5558	    (u_longlong_t)ddo->ddo_dspace,
5559	    (u_longlong_t)ddo->ddo_mspace);
5560
5561	verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_STATS,
5562	    (uint64_t **)&dds, &c) == 0);
5563	verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_HISTOGRAM,
5564	    (uint64_t **)&ddh, &c) == 0);
5565	zpool_dump_ddt(dds, ddh);
5566}
5567
5568/*
5569 * Display a summary of pool status.  Displays a summary such as:
5570 *
5571 *        pool: tank
5572 *	status: DEGRADED
5573 *	reason: One or more devices ...
5574 *         see: http://illumos.org/msg/ZFS-xxxx-01
5575 *	config:
5576 *		mirror		DEGRADED
5577 *                c1t0d0	OK
5578 *                c2t0d0	UNAVAIL
5579 *
5580 * When given the '-v' option, we print out the complete config.  If the '-e'
5581 * option is specified, then we print out error rate information as well.
5582 */
5583int
5584status_callback(zpool_handle_t *zhp, void *data)
5585{
5586	status_cbdata_t *cbp = data;
5587	nvlist_t *config, *nvroot;
5588	char *msgid;
5589	int reason;
5590	zpool_errata_t errata;
5591	const char *health;
5592	uint_t c;
5593	vdev_stat_t *vs;
5594
5595	config = zpool_get_config(zhp, NULL);
5596	reason = zpool_get_status(zhp, &msgid, &errata);
5597
5598	cbp->cb_count++;
5599
5600	/*
5601	 * If we were given 'zpool status -x', only report those pools with
5602	 * problems.
5603	 */
5604	if (cbp->cb_explain &&
5605	    (reason == ZPOOL_STATUS_OK ||
5606	    reason == ZPOOL_STATUS_VERSION_OLDER ||
5607	    reason == ZPOOL_STATUS_FEAT_DISABLED)) {
5608		if (!cbp->cb_allpools) {
5609			(void) printf(gettext("pool '%s' is healthy\n"),
5610			    zpool_get_name(zhp));
5611			if (cbp->cb_first)
5612				cbp->cb_first = B_FALSE;
5613		}
5614		return (0);
5615	}
5616
5617	if (cbp->cb_first)
5618		cbp->cb_first = B_FALSE;
5619	else
5620		(void) printf("\n");
5621
5622	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
5623	verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
5624	    (uint64_t **)&vs, &c) == 0);
5625	health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
5626
5627	(void) printf(gettext("  pool: %s\n"), zpool_get_name(zhp));
5628	(void) printf(gettext(" state: %s\n"), health);
5629
5630	switch (reason) {
5631	case ZPOOL_STATUS_MISSING_DEV_R:
5632		(void) printf(gettext("status: One or more devices could not "
5633		    "be opened.  Sufficient replicas exist for\n\tthe pool to "
5634		    "continue functioning in a degraded state.\n"));
5635		(void) printf(gettext("action: Attach the missing device and "
5636		    "online it using 'zpool online'.\n"));
5637		break;
5638
5639	case ZPOOL_STATUS_MISSING_DEV_NR:
5640		(void) printf(gettext("status: One or more devices could not "
5641		    "be opened.  There are insufficient\n\treplicas for the "
5642		    "pool to continue functioning.\n"));
5643		(void) printf(gettext("action: Attach the missing device and "
5644		    "online it using 'zpool online'.\n"));
5645		break;
5646
5647	case ZPOOL_STATUS_CORRUPT_LABEL_R:
5648		(void) printf(gettext("status: One or more devices could not "
5649		    "be used because the label is missing or\n\tinvalid.  "
5650		    "Sufficient replicas exist for the pool to continue\n\t"
5651		    "functioning in a degraded state.\n"));
5652		(void) printf(gettext("action: Replace the device using "
5653		    "'zpool replace'.\n"));
5654		break;
5655
5656	case ZPOOL_STATUS_CORRUPT_LABEL_NR:
5657		(void) printf(gettext("status: One or more devices could not "
5658		    "be used because the label is missing \n\tor invalid.  "
5659		    "There are insufficient replicas for the pool to "
5660		    "continue\n\tfunctioning.\n"));
5661		zpool_explain_recover(zpool_get_handle(zhp),
5662		    zpool_get_name(zhp), reason, config);
5663		break;
5664
5665	case ZPOOL_STATUS_FAILING_DEV:
5666		(void) printf(gettext("status: One or more devices has "
5667		    "experienced an unrecoverable error.  An\n\tattempt was "
5668		    "made to correct the error.  Applications are "
5669		    "unaffected.\n"));
5670		(void) printf(gettext("action: Determine if the device needs "
5671		    "to be replaced, and clear the errors\n\tusing "
5672		    "'zpool clear' or replace the device with 'zpool "
5673		    "replace'.\n"));
5674		break;
5675
5676	case ZPOOL_STATUS_OFFLINE_DEV:
5677		(void) printf(gettext("status: One or more devices has "
5678		    "been taken offline by the administrator.\n\tSufficient "
5679		    "replicas exist for the pool to continue functioning in "
5680		    "a\n\tdegraded state.\n"));
5681		(void) printf(gettext("action: Online the device using "
5682		    "'zpool online' or replace the device with\n\t'zpool "
5683		    "replace'.\n"));
5684		break;
5685
5686	case ZPOOL_STATUS_REMOVED_DEV:
5687		(void) printf(gettext("status: One or more devices has "
5688		    "been removed by the administrator.\n\tSufficient "
5689		    "replicas exist for the pool to continue functioning in "
5690		    "a\n\tdegraded state.\n"));
5691		(void) printf(gettext("action: Online the device using "
5692		    "'zpool online' or replace the device with\n\t'zpool "
5693		    "replace'.\n"));
5694		break;
5695
5696	case ZPOOL_STATUS_RESILVERING:
5697		(void) printf(gettext("status: One or more devices is "
5698		    "currently being resilvered.  The pool will\n\tcontinue "
5699		    "to function, possibly in a degraded state.\n"));
5700		(void) printf(gettext("action: Wait for the resilver to "
5701		    "complete.\n"));
5702		break;
5703
5704	case ZPOOL_STATUS_CORRUPT_DATA:
5705		(void) printf(gettext("status: One or more devices has "
5706		    "experienced an error resulting in data\n\tcorruption.  "
5707		    "Applications may be affected.\n"));
5708		(void) printf(gettext("action: Restore the file in question "
5709		    "if possible.  Otherwise restore the\n\tentire pool from "
5710		    "backup.\n"));
5711		break;
5712
5713	case ZPOOL_STATUS_CORRUPT_POOL:
5714		(void) printf(gettext("status: The pool metadata is corrupted "
5715		    "and the pool cannot be opened.\n"));
5716		zpool_explain_recover(zpool_get_handle(zhp),
5717		    zpool_get_name(zhp), reason, config);
5718		break;
5719
5720	case ZPOOL_STATUS_VERSION_OLDER:
5721		(void) printf(gettext("status: The pool is formatted using a "
5722		    "legacy on-disk format.  The pool can\n\tstill be used, "
5723		    "but some features are unavailable.\n"));
5724		(void) printf(gettext("action: Upgrade the pool using 'zpool "
5725		    "upgrade'.  Once this is done, the\n\tpool will no longer "
5726		    "be accessible on software that does not support feature\n"
5727		    "\tflags.\n"));
5728		break;
5729
5730	case ZPOOL_STATUS_VERSION_NEWER:
5731		(void) printf(gettext("status: The pool has been upgraded to a "
5732		    "newer, incompatible on-disk version.\n\tThe pool cannot "
5733		    "be accessed on this system.\n"));
5734		(void) printf(gettext("action: Access the pool from a system "
5735		    "running more recent software, or\n\trestore the pool from "
5736		    "backup.\n"));
5737		break;
5738
5739	case ZPOOL_STATUS_FEAT_DISABLED:
5740		(void) printf(gettext("status: Some supported features are not "
5741		    "enabled on the pool. The pool can\n\tstill be used, but "
5742		    "some features are unavailable.\n"));
5743		(void) printf(gettext("action: Enable all features using "
5744		    "'zpool upgrade'. Once this is done,\n\tthe pool may no "
5745		    "longer be accessible by software that does not support\n\t"
5746		    "the features. See zpool-features(5) for details.\n"));
5747		break;
5748
5749	case ZPOOL_STATUS_UNSUP_FEAT_READ:
5750		(void) printf(gettext("status: The pool cannot be accessed on "
5751		    "this system because it uses the\n\tfollowing feature(s) "
5752		    "not supported on this system:\n"));
5753		zpool_print_unsup_feat(config);
5754		(void) printf("\n");
5755		(void) printf(gettext("action: Access the pool from a system "
5756		    "that supports the required feature(s),\n\tor restore the "
5757		    "pool from backup.\n"));
5758		break;
5759
5760	case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
5761		(void) printf(gettext("status: The pool can only be accessed "
5762		    "in read-only mode on this system. It\n\tcannot be "
5763		    "accessed in read-write mode because it uses the "
5764		    "following\n\tfeature(s) not supported on this system:\n"));
5765		zpool_print_unsup_feat(config);
5766		(void) printf("\n");
5767		(void) printf(gettext("action: The pool cannot be accessed in "
5768		    "read-write mode. Import the pool with\n"
5769		    "\t\"-o readonly=on\", access the pool from a system that "
5770		    "supports the\n\trequired feature(s), or restore the "
5771		    "pool from backup.\n"));
5772		break;
5773
5774	case ZPOOL_STATUS_FAULTED_DEV_R:
5775		(void) printf(gettext("status: One or more devices are "
5776		    "faulted in response to persistent errors.\n\tSufficient "
5777		    "replicas exist for the pool to continue functioning "
5778		    "in a\n\tdegraded state.\n"));
5779		(void) printf(gettext("action: Replace the faulted device, "
5780		    "or use 'zpool clear' to mark the device\n\trepaired.\n"));
5781		break;
5782
5783	case ZPOOL_STATUS_FAULTED_DEV_NR:
5784		(void) printf(gettext("status: One or more devices are "
5785		    "faulted in response to persistent errors.  There are "
5786		    "insufficient replicas for the pool to\n\tcontinue "
5787		    "functioning.\n"));
5788		(void) printf(gettext("action: Destroy and re-create the pool "
5789		    "from a backup source.  Manually marking the device\n"
5790		    "\trepaired using 'zpool clear' may allow some data "
5791		    "to be recovered.\n"));
5792		break;
5793
5794	case ZPOOL_STATUS_IO_FAILURE_MMP:
5795		(void) printf(gettext("status: The pool is suspended because "
5796		    "multihost writes failed or were delayed;\n\tanother "
5797		    "system could import the pool undetected.\n"));
5798		(void) printf(gettext("action: Make sure the pool's devices "
5799		    "are connected, then reboot your system and\n\timport the "
5800		    "pool.\n"));
5801		break;
5802
5803	case ZPOOL_STATUS_IO_FAILURE_WAIT:
5804	case ZPOOL_STATUS_IO_FAILURE_CONTINUE:
5805		(void) printf(gettext("status: One or more devices are "
5806		    "faulted in response to IO failures.\n"));
5807		(void) printf(gettext("action: Make sure the affected devices "
5808		    "are connected, then run 'zpool clear'.\n"));
5809		break;
5810
5811	case ZPOOL_STATUS_BAD_LOG:
5812		(void) printf(gettext("status: An intent log record "
5813		    "could not be read.\n"
5814		    "\tWaiting for adminstrator intervention to fix the "
5815		    "faulted pool.\n"));
5816		(void) printf(gettext("action: Either restore the affected "
5817		    "device(s) and run 'zpool online',\n"
5818		    "\tor ignore the intent log records by running "
5819		    "'zpool clear'.\n"));
5820		break;
5821
5822	case ZPOOL_STATUS_ERRATA:
5823		(void) printf(gettext("status: Errata #%d detected.\n"),
5824		    errata);
5825
5826		switch (errata) {
5827		case ZPOOL_ERRATA_NONE:
5828			break;
5829
5830		case ZPOOL_ERRATA_ZOL_2094_SCRUB:
5831			(void) printf(gettext("action: To correct the issue "
5832			    "run 'zpool scrub'.\n"));
5833			break;
5834
5835		case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
5836			(void) printf(gettext("\tExisting encrypted datasets "
5837			    "contain an on-disk incompatibility\n\twhich "
5838			    "needs to be corrected.\n"));
5839			(void) printf(gettext("action: To correct the issue "
5840			    "backup existing encrypted datasets to new\n\t"
5841			    "encrypted datasets and destroy the old ones. "
5842			    "'zfs mount -o ro' can\n\tbe used to temporarily "
5843			    "mount existing encrypted datasets readonly.\n"));
5844			break;
5845
5846		case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
5847			(void) printf(gettext("\tExisting encrypted datasets "
5848			    "contain an on-disk incompatibility\n\twhich "
5849			    "needs to be corrected.\n"));
5850			(void) printf(gettext("action: To correct the issue "
5851			    "enable the bookmark_v2 feature and "
5852			    "backup\n\tany existing encrypted datasets to "
5853			    "new encrypted datasets and\n\tdestroy the old "
5854			    "ones. If this pool does not contain any\n\t"
5855			    "encrypted datasets, simply enable the "
5856			    "bookmark_v2 feature\n"));
5857			break;
5858
5859		default:
5860			/*
5861			 * All errata which allow the pool to be imported
5862			 * must contain an action message.
5863			 */
5864			assert(0);
5865		}
5866		break;
5867
5868	default:
5869		/*
5870		 * The remaining errors can't actually be generated, yet.
5871		 */
5872		assert(reason == ZPOOL_STATUS_OK);
5873	}
5874
5875	if (msgid != NULL)
5876		(void) printf(gettext("   see: http://illumos.org/msg/%s\n"),
5877		    msgid);
5878
5879	if (config != NULL) {
5880		uint64_t nerr;
5881		nvlist_t **spares, **l2cache;
5882		uint_t nspares, nl2cache;
5883		pool_checkpoint_stat_t *pcs = NULL;
5884		pool_scan_stat_t *ps = NULL;
5885		pool_removal_stat_t *prs = NULL;
5886
5887		(void) nvlist_lookup_uint64_array(nvroot,
5888		    ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
5889		(void) nvlist_lookup_uint64_array(nvroot,
5890		    ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&ps, &c);
5891		(void) nvlist_lookup_uint64_array(nvroot,
5892		    ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
5893
5894		print_scan_status(ps);
5895		print_checkpoint_scan_warning(ps, pcs);
5896		print_removal_status(zhp, prs);
5897		print_checkpoint_status(pcs);
5898
5899		cbp->cb_namewidth = max_width(zhp, nvroot, 0, 0,
5900		    cbp->cb_name_flags);
5901		if (cbp->cb_namewidth < 10)
5902			cbp->cb_namewidth = 10;
5903
5904		(void) printf(gettext("config:\n\n"));
5905		(void) printf(gettext("\t%-*s  %-8s %5s %5s %5s\n"),
5906		    cbp->cb_namewidth, "NAME", "STATE", "READ", "WRITE",
5907		    "CKSUM");
5908
5909		print_status_config(zhp, cbp, zpool_get_name(zhp), nvroot, 0,
5910		    B_FALSE);
5911
5912		print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_DEDUP);
5913		print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
5914		print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_CLASS_LOGS);
5915
5916		if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
5917		    &l2cache, &nl2cache) == 0)
5918			print_l2cache(zhp, cbp, l2cache, nl2cache);
5919
5920		if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
5921		    &spares, &nspares) == 0)
5922			print_spares(zhp, cbp, spares, nspares);
5923
5924		if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
5925		    &nerr) == 0) {
5926			nvlist_t *nverrlist = NULL;
5927
5928			/*
5929			 * If the approximate error count is small, get a
5930			 * precise count by fetching the entire log and
5931			 * uniquifying the results.
5932			 */
5933			if (nerr > 0 && nerr < 100 && !cbp->cb_verbose &&
5934			    zpool_get_errlog(zhp, &nverrlist) == 0) {
5935				nvpair_t *elem;
5936
5937				elem = NULL;
5938				nerr = 0;
5939				while ((elem = nvlist_next_nvpair(nverrlist,
5940				    elem)) != NULL) {
5941					nerr++;
5942				}
5943			}
5944			nvlist_free(nverrlist);
5945
5946			(void) printf("\n");
5947
5948			if (nerr == 0)
5949				(void) printf(gettext("errors: No known data "
5950				    "errors\n"));
5951			else if (!cbp->cb_verbose)
5952				(void) printf(gettext("errors: %llu data "
5953				    "errors, use '-v' for a list\n"),
5954				    (u_longlong_t)nerr);
5955			else
5956				print_error_log(zhp);
5957		}
5958
5959		if (cbp->cb_dedup_stats)
5960			print_dedup_stats(config);
5961	} else {
5962		(void) printf(gettext("config: The configuration cannot be "
5963		    "determined.\n"));
5964	}
5965
5966	return (0);
5967}
5968
5969/*
5970 * zpool status [-igLPtvx] [-T d|u] [pool] ... [interval [count]]
5971 *
5972 *	-i	Display vdev initialization status.
5973 *	-g	Display guid for individual vdev name.
5974 *	-L	Follow links when resolving vdev path name.
5975 *	-P	Display full path for vdev name.
5976 *	-v	Display complete error logs
5977 *	-x	Display only pools with potential problems
5978 *	-D	Display dedup status (undocumented)
5979 *	-t	Display vdev TRIM status.
5980 *	-T	Display a timestamp in date(1) or Unix format
5981 *
5982 * Describes the health status of all pools or some subset.
5983 */
5984int
5985zpool_do_status(int argc, char **argv)
5986{
5987	int c;
5988	int ret;
5989	unsigned long interval = 0, count = 0;
5990	status_cbdata_t cb = { 0 };
5991
5992	/* check options */
5993	while ((c = getopt(argc, argv, "igLPvxDtT:")) != -1) {
5994		switch (c) {
5995		case 'i':
5996			cb.cb_print_vdev_init = B_TRUE;
5997			break;
5998		case 'g':
5999			cb.cb_name_flags |= VDEV_NAME_GUID;
6000			break;
6001		case 'L':
6002			cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
6003			break;
6004		case 'P':
6005			cb.cb_name_flags |= VDEV_NAME_PATH;
6006			break;
6007		case 'v':
6008			cb.cb_verbose = B_TRUE;
6009			break;
6010		case 'x':
6011			cb.cb_explain = B_TRUE;
6012			break;
6013		case 'D':
6014			cb.cb_dedup_stats = B_TRUE;
6015			break;
6016		case 't':
6017			cb.cb_print_vdev_trim = B_TRUE;
6018			break;
6019		case 'T':
6020			get_timestamp_arg(*optarg);
6021			break;
6022		case '?':
6023			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
6024			    optopt);
6025			usage(B_FALSE);
6026		}
6027	}
6028
6029	argc -= optind;
6030	argv += optind;
6031
6032	get_interval_count(&argc, argv, &interval, &count);
6033
6034	if (argc == 0)
6035		cb.cb_allpools = B_TRUE;
6036
6037	cb.cb_first = B_TRUE;
6038	cb.cb_print_status = B_TRUE;
6039
6040	for (;;) {
6041		if (timestamp_fmt != NODATE)
6042			print_timestamp(timestamp_fmt);
6043
6044		ret = for_each_pool(argc, argv, B_TRUE, NULL,
6045		    status_callback, &cb);
6046
6047		if (argc == 0 && cb.cb_count == 0)
6048			(void) printf(gettext("no pools available\n"));
6049		else if (cb.cb_explain && cb.cb_first && cb.cb_allpools)
6050			(void) printf(gettext("all pools are healthy\n"));
6051
6052		if (ret != 0)
6053			return (ret);
6054
6055		if (interval == 0)
6056			break;
6057
6058		if (count != 0 && --count == 0)
6059			break;
6060
6061		(void) sleep(interval);
6062	}
6063
6064	return (0);
6065}
6066
6067typedef struct upgrade_cbdata {
6068	int	cb_first;
6069	int	cb_argc;
6070	uint64_t cb_version;
6071	char	**cb_argv;
6072} upgrade_cbdata_t;
6073
6074static int
6075upgrade_version(zpool_handle_t *zhp, uint64_t version)
6076{
6077	int ret;
6078	nvlist_t *config;
6079	uint64_t oldversion;
6080
6081	config = zpool_get_config(zhp, NULL);
6082	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
6083	    &oldversion) == 0);
6084
6085	assert(SPA_VERSION_IS_SUPPORTED(oldversion));
6086	assert(oldversion < version);
6087
6088	ret = zpool_upgrade(zhp, version);
6089	if (ret != 0)
6090		return (ret);
6091
6092	if (version >= SPA_VERSION_FEATURES) {
6093		(void) printf(gettext("Successfully upgraded "
6094		    "'%s' from version %llu to feature flags.\n"),
6095		    zpool_get_name(zhp), oldversion);
6096	} else {
6097		(void) printf(gettext("Successfully upgraded "
6098		    "'%s' from version %llu to version %llu.\n"),
6099		    zpool_get_name(zhp), oldversion, version);
6100	}
6101
6102	return (0);
6103}
6104
6105static int
6106upgrade_enable_all(zpool_handle_t *zhp, int *countp)
6107{
6108	int i, ret, count;
6109	boolean_t firstff = B_TRUE;
6110	nvlist_t *enabled = zpool_get_features(zhp);
6111
6112	count = 0;
6113	for (i = 0; i < SPA_FEATURES; i++) {
6114		const char *fname = spa_feature_table[i].fi_uname;
6115		const char *fguid = spa_feature_table[i].fi_guid;
6116		if (!nvlist_exists(enabled, fguid)) {
6117			char *propname;
6118			verify(-1 != asprintf(&propname, "feature@%s", fname));
6119			ret = zpool_set_prop(zhp, propname,
6120			    ZFS_FEATURE_ENABLED);
6121			if (ret != 0) {
6122				free(propname);
6123				return (ret);
6124			}
6125			count++;
6126
6127			if (firstff) {
6128				(void) printf(gettext("Enabled the "
6129				    "following features on '%s':\n"),
6130				    zpool_get_name(zhp));
6131				firstff = B_FALSE;
6132			}
6133			(void) printf(gettext("  %s\n"), fname);
6134			free(propname);
6135		}
6136	}
6137
6138	if (countp != NULL)
6139		*countp = count;
6140	return (0);
6141}
6142
6143static int
6144upgrade_cb(zpool_handle_t *zhp, void *arg)
6145{
6146	upgrade_cbdata_t *cbp = arg;
6147	nvlist_t *config;
6148	uint64_t version;
6149	boolean_t printnl = B_FALSE;
6150	int ret;
6151
6152	config = zpool_get_config(zhp, NULL);
6153	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
6154	    &version) == 0);
6155
6156	assert(SPA_VERSION_IS_SUPPORTED(version));
6157
6158	if (version < cbp->cb_version) {
6159		cbp->cb_first = B_FALSE;
6160		ret = upgrade_version(zhp, cbp->cb_version);
6161		if (ret != 0)
6162			return (ret);
6163		printnl = B_TRUE;
6164
6165		/*
6166		 * If they did "zpool upgrade -a", then we could
6167		 * be doing ioctls to different pools.  We need
6168		 * to log this history once to each pool, and bypass
6169		 * the normal history logging that happens in main().
6170		 */
6171		(void) zpool_log_history(g_zfs, history_str);
6172		log_history = B_FALSE;
6173	}
6174
6175	if (cbp->cb_version >= SPA_VERSION_FEATURES) {
6176		int count;
6177		ret = upgrade_enable_all(zhp, &count);
6178		if (ret != 0)
6179			return (ret);
6180
6181		if (count > 0) {
6182			cbp->cb_first = B_FALSE;
6183			printnl = B_TRUE;
6184		}
6185	}
6186
6187	if (printnl) {
6188		(void) printf(gettext("\n"));
6189	}
6190
6191	return (0);
6192}
6193
6194static int
6195upgrade_list_older_cb(zpool_handle_t *zhp, void *arg)
6196{
6197	upgrade_cbdata_t *cbp = arg;
6198	nvlist_t *config;
6199	uint64_t version;
6200
6201	config = zpool_get_config(zhp, NULL);
6202	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
6203	    &version) == 0);
6204
6205	assert(SPA_VERSION_IS_SUPPORTED(version));
6206
6207	if (version < SPA_VERSION_FEATURES) {
6208		if (cbp->cb_first) {
6209			(void) printf(gettext("The following pools are "
6210			    "formatted with legacy version numbers and can\n"
6211			    "be upgraded to use feature flags.  After "
6212			    "being upgraded, these pools\nwill no "
6213			    "longer be accessible by software that does not "
6214			    "support feature\nflags.\n\n"));
6215			(void) printf(gettext("VER  POOL\n"));
6216			(void) printf(gettext("---  ------------\n"));
6217			cbp->cb_first = B_FALSE;
6218		}
6219
6220		(void) printf("%2llu   %s\n", (u_longlong_t)version,
6221		    zpool_get_name(zhp));
6222	}
6223
6224	return (0);
6225}
6226
6227static int
6228upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
6229{
6230	upgrade_cbdata_t *cbp = arg;
6231	nvlist_t *config;
6232	uint64_t version;
6233
6234	config = zpool_get_config(zhp, NULL);
6235	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
6236	    &version) == 0);
6237
6238	if (version >= SPA_VERSION_FEATURES) {
6239		int i;
6240		boolean_t poolfirst = B_TRUE;
6241		nvlist_t *enabled = zpool_get_features(zhp);
6242
6243		for (i = 0; i < SPA_FEATURES; i++) {
6244			const char *fguid = spa_feature_table[i].fi_guid;
6245			const char *fname = spa_feature_table[i].fi_uname;
6246			if (!nvlist_exists(enabled, fguid)) {
6247				if (cbp->cb_first) {
6248					(void) printf(gettext("\nSome "
6249					    "supported features are not "
6250					    "enabled on the following pools. "
6251					    "Once a\nfeature is enabled the "
6252					    "pool may become incompatible with "
6253					    "software\nthat does not support "
6254					    "the feature. See "
6255					    "zpool-features(5) for "
6256					    "details.\n\n"));
6257					(void) printf(gettext("POOL  "
6258					    "FEATURE\n"));
6259					(void) printf(gettext("------"
6260					    "---------\n"));
6261					cbp->cb_first = B_FALSE;
6262				}
6263
6264				if (poolfirst) {
6265					(void) printf(gettext("%s\n"),
6266					    zpool_get_name(zhp));
6267					poolfirst = B_FALSE;
6268				}
6269
6270				(void) printf(gettext("      %s\n"), fname);
6271			}
6272		}
6273	}
6274
6275	return (0);
6276}
6277
6278/* ARGSUSED */
6279static int
6280upgrade_one(zpool_handle_t *zhp, void *data)
6281{
6282	boolean_t printnl = B_FALSE;
6283	upgrade_cbdata_t *cbp = data;
6284	uint64_t cur_version;
6285	int ret;
6286
6287	if (strcmp("log", zpool_get_name(zhp)) == 0) {
6288		(void) printf(gettext("'log' is now a reserved word\n"
6289		    "Pool 'log' must be renamed using export and import"
6290		    " to upgrade.\n"));
6291		return (1);
6292	}
6293
6294	cur_version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
6295	if (cur_version > cbp->cb_version) {
6296		(void) printf(gettext("Pool '%s' is already formatted "
6297		    "using more current version '%llu'.\n\n"),
6298		    zpool_get_name(zhp), cur_version);
6299		return (0);
6300	}
6301
6302	if (cbp->cb_version != SPA_VERSION && cur_version == cbp->cb_version) {
6303		(void) printf(gettext("Pool '%s' is already formatted "
6304		    "using version %llu.\n\n"), zpool_get_name(zhp),
6305		    cbp->cb_version);
6306		return (0);
6307	}
6308
6309	if (cur_version != cbp->cb_version) {
6310		printnl = B_TRUE;
6311		ret = upgrade_version(zhp, cbp->cb_version);
6312		if (ret != 0)
6313			return (ret);
6314	}
6315
6316	if (cbp->cb_version >= SPA_VERSION_FEATURES) {
6317		int count = 0;
6318		ret = upgrade_enable_all(zhp, &count);
6319		if (ret != 0)
6320			return (ret);
6321
6322		if (count != 0) {
6323			printnl = B_TRUE;
6324		} else if (cur_version == SPA_VERSION) {
6325			(void) printf(gettext("Pool '%s' already has all "
6326			    "supported features enabled.\n"),
6327			    zpool_get_name(zhp));
6328		}
6329	}
6330
6331	if (printnl) {
6332		(void) printf(gettext("\n"));
6333	}
6334
6335	return (0);
6336}
6337
6338/*
6339 * zpool upgrade
6340 * zpool upgrade -v
6341 * zpool upgrade [-V version] <-a | pool ...>
6342 *
6343 * With no arguments, display downrev'd ZFS pool available for upgrade.
6344 * Individual pools can be upgraded by specifying the pool, and '-a' will
6345 * upgrade all pools.
6346 */
6347int
6348zpool_do_upgrade(int argc, char **argv)
6349{
6350	int c;
6351	upgrade_cbdata_t cb = { 0 };
6352	int ret = 0;
6353	boolean_t showversions = B_FALSE;
6354	boolean_t upgradeall = B_FALSE;
6355	char *end;
6356
6357
6358	/* check options */
6359	while ((c = getopt(argc, argv, ":avV:")) != -1) {
6360		switch (c) {
6361		case 'a':
6362			upgradeall = B_TRUE;
6363			break;
6364		case 'v':
6365			showversions = B_TRUE;
6366			break;
6367		case 'V':
6368			cb.cb_version = strtoll(optarg, &end, 10);
6369			if (*end != '\0' ||
6370			    !SPA_VERSION_IS_SUPPORTED(cb.cb_version)) {
6371				(void) fprintf(stderr,
6372				    gettext("invalid version '%s'\n"), optarg);
6373				usage(B_FALSE);
6374			}
6375			break;
6376		case ':':
6377			(void) fprintf(stderr, gettext("missing argument for "
6378			    "'%c' option\n"), optopt);
6379			usage(B_FALSE);
6380			break;
6381		case '?':
6382			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
6383			    optopt);
6384			usage(B_FALSE);
6385		}
6386	}
6387
6388	cb.cb_argc = argc;
6389	cb.cb_argv = argv;
6390	argc -= optind;
6391	argv += optind;
6392
6393	if (cb.cb_version == 0) {
6394		cb.cb_version = SPA_VERSION;
6395	} else if (!upgradeall && argc == 0) {
6396		(void) fprintf(stderr, gettext("-V option is "
6397		    "incompatible with other arguments\n"));
6398		usage(B_FALSE);
6399	}
6400
6401	if (showversions) {
6402		if (upgradeall || argc != 0) {
6403			(void) fprintf(stderr, gettext("-v option is "
6404			    "incompatible with other arguments\n"));
6405			usage(B_FALSE);
6406		}
6407	} else if (upgradeall) {
6408		if (argc != 0) {
6409			(void) fprintf(stderr, gettext("-a option should not "
6410			    "be used along with a pool name\n"));
6411			usage(B_FALSE);
6412		}
6413	}
6414
6415	(void) printf(gettext("This system supports ZFS pool feature "
6416	    "flags.\n\n"));
6417	if (showversions) {
6418		int i;
6419
6420		(void) printf(gettext("The following features are "
6421		    "supported:\n\n"));
6422		(void) printf(gettext("FEAT DESCRIPTION\n"));
6423		(void) printf("----------------------------------------------"
6424		    "---------------\n");
6425		for (i = 0; i < SPA_FEATURES; i++) {
6426			zfeature_info_t *fi = &spa_feature_table[i];
6427			const char *ro =
6428			    (fi->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
6429			    " (read-only compatible)" : "";
6430
6431			(void) printf("%-37s%s\n", fi->fi_uname, ro);
6432			(void) printf("     %s\n", fi->fi_desc);
6433		}
6434		(void) printf("\n");
6435
6436		(void) printf(gettext("The following legacy versions are also "
6437		    "supported:\n\n"));
6438		(void) printf(gettext("VER  DESCRIPTION\n"));
6439		(void) printf("---  -----------------------------------------"
6440		    "---------------\n");
6441		(void) printf(gettext(" 1   Initial ZFS version\n"));
6442		(void) printf(gettext(" 2   Ditto blocks "
6443		    "(replicated metadata)\n"));
6444		(void) printf(gettext(" 3   Hot spares and double parity "
6445		    "RAID-Z\n"));
6446		(void) printf(gettext(" 4   zpool history\n"));
6447		(void) printf(gettext(" 5   Compression using the gzip "
6448		    "algorithm\n"));
6449		(void) printf(gettext(" 6   bootfs pool property\n"));
6450		(void) printf(gettext(" 7   Separate intent log devices\n"));
6451		(void) printf(gettext(" 8   Delegated administration\n"));
6452		(void) printf(gettext(" 9   refquota and refreservation "
6453		    "properties\n"));
6454		(void) printf(gettext(" 10  Cache devices\n"));
6455		(void) printf(gettext(" 11  Improved scrub performance\n"));
6456		(void) printf(gettext(" 12  Snapshot properties\n"));
6457		(void) printf(gettext(" 13  snapused property\n"));
6458		(void) printf(gettext(" 14  passthrough-x aclinherit\n"));
6459		(void) printf(gettext(" 15  user/group space accounting\n"));
6460		(void) printf(gettext(" 16  stmf property support\n"));
6461		(void) printf(gettext(" 17  Triple-parity RAID-Z\n"));
6462		(void) printf(gettext(" 18  Snapshot user holds\n"));
6463		(void) printf(gettext(" 19  Log device removal\n"));
6464		(void) printf(gettext(" 20  Compression using zle "
6465		    "(zero-length encoding)\n"));
6466		(void) printf(gettext(" 21  Deduplication\n"));
6467		(void) printf(gettext(" 22  Received properties\n"));
6468		(void) printf(gettext(" 23  Slim ZIL\n"));
6469		(void) printf(gettext(" 24  System attributes\n"));
6470		(void) printf(gettext(" 25  Improved scrub stats\n"));
6471		(void) printf(gettext(" 26  Improved snapshot deletion "
6472		    "performance\n"));
6473		(void) printf(gettext(" 27  Improved snapshot creation "
6474		    "performance\n"));
6475		(void) printf(gettext(" 28  Multiple vdev replacements\n"));
6476		(void) printf(gettext("\nFor more information on a particular "
6477		    "version, including supported releases,\n"));
6478		(void) printf(gettext("see the ZFS Administration Guide.\n\n"));
6479	} else if (argc == 0 && upgradeall) {
6480		cb.cb_first = B_TRUE;
6481		ret = zpool_iter(g_zfs, upgrade_cb, &cb);
6482		if (ret == 0 && cb.cb_first) {
6483			if (cb.cb_version == SPA_VERSION) {
6484				(void) printf(gettext("All pools are already "
6485				    "formatted using feature flags.\n\n"));
6486				(void) printf(gettext("Every feature flags "
6487				    "pool already has all supported features "
6488				    "enabled.\n"));
6489			} else {
6490				(void) printf(gettext("All pools are already "
6491				    "formatted with version %llu or higher.\n"),
6492				    cb.cb_version);
6493			}
6494		}
6495	} else if (argc == 0) {
6496		cb.cb_first = B_TRUE;
6497		ret = zpool_iter(g_zfs, upgrade_list_older_cb, &cb);
6498		assert(ret == 0);
6499
6500		if (cb.cb_first) {
6501			(void) printf(gettext("All pools are formatted "
6502			    "using feature flags.\n\n"));
6503		} else {
6504			(void) printf(gettext("\nUse 'zpool upgrade -v' "
6505			    "for a list of available legacy versions.\n"));
6506		}
6507
6508		cb.cb_first = B_TRUE;
6509		ret = zpool_iter(g_zfs, upgrade_list_disabled_cb, &cb);
6510		assert(ret == 0);
6511
6512		if (cb.cb_first) {
6513			(void) printf(gettext("Every feature flags pool has "
6514			    "all supported features enabled.\n"));
6515		} else {
6516			(void) printf(gettext("\n"));
6517		}
6518	} else {
6519		ret = for_each_pool(argc, argv, B_FALSE, NULL,
6520		    upgrade_one, &cb);
6521	}
6522
6523	return (ret);
6524}
6525
6526typedef struct hist_cbdata {
6527	boolean_t first;
6528	boolean_t longfmt;
6529	boolean_t internal;
6530} hist_cbdata_t;
6531
6532/*
6533 * Print out the command history for a specific pool.
6534 */
6535static int
6536get_history_one(zpool_handle_t *zhp, void *data)
6537{
6538	nvlist_t *nvhis;
6539	nvlist_t **records;
6540	uint_t numrecords;
6541	int ret, i;
6542	hist_cbdata_t *cb = (hist_cbdata_t *)data;
6543
6544	cb->first = B_FALSE;
6545
6546	(void) printf(gettext("History for '%s':\n"), zpool_get_name(zhp));
6547
6548	if ((ret = zpool_get_history(zhp, &nvhis)) != 0)
6549		return (ret);
6550
6551	verify(nvlist_lookup_nvlist_array(nvhis, ZPOOL_HIST_RECORD,
6552	    &records, &numrecords) == 0);
6553	for (i = 0; i < numrecords; i++) {
6554		nvlist_t *rec = records[i];
6555		char tbuf[30] = "";
6556
6557		if (nvlist_exists(rec, ZPOOL_HIST_TIME)) {
6558			time_t tsec;
6559			struct tm t;
6560
6561			tsec = fnvlist_lookup_uint64(records[i],
6562			    ZPOOL_HIST_TIME);
6563			(void) localtime_r(&tsec, &t);
6564			(void) strftime(tbuf, sizeof (tbuf), "%F.%T", &t);
6565		}
6566
6567		if (nvlist_exists(rec, ZPOOL_HIST_CMD)) {
6568			(void) printf("%s %s", tbuf,
6569			    fnvlist_lookup_string(rec, ZPOOL_HIST_CMD));
6570		} else if (nvlist_exists(rec, ZPOOL_HIST_INT_EVENT)) {
6571			int ievent =
6572			    fnvlist_lookup_uint64(rec, ZPOOL_HIST_INT_EVENT);
6573			if (!cb->internal)
6574				continue;
6575			if (ievent >= ZFS_NUM_LEGACY_HISTORY_EVENTS) {
6576				(void) printf("%s unrecognized record:\n",
6577				    tbuf);
6578				dump_nvlist(rec, 4);
6579				continue;
6580			}
6581			(void) printf("%s [internal %s txg:%lld] %s", tbuf,
6582			    zfs_history_event_names[ievent],
6583			    fnvlist_lookup_uint64(rec, ZPOOL_HIST_TXG),
6584			    fnvlist_lookup_string(rec, ZPOOL_HIST_INT_STR));
6585		} else if (nvlist_exists(rec, ZPOOL_HIST_INT_NAME)) {
6586			if (!cb->internal)
6587				continue;
6588			(void) printf("%s [txg:%lld] %s", tbuf,
6589			    fnvlist_lookup_uint64(rec, ZPOOL_HIST_TXG),
6590			    fnvlist_lookup_string(rec, ZPOOL_HIST_INT_NAME));
6591			if (nvlist_exists(rec, ZPOOL_HIST_DSNAME)) {
6592				(void) printf(" %s (%llu)",
6593				    fnvlist_lookup_string(rec,
6594				    ZPOOL_HIST_DSNAME),
6595				    fnvlist_lookup_uint64(rec,
6596				    ZPOOL_HIST_DSID));
6597			}
6598			(void) printf(" %s", fnvlist_lookup_string(rec,
6599			    ZPOOL_HIST_INT_STR));
6600		} else if (nvlist_exists(rec, ZPOOL_HIST_IOCTL)) {
6601			if (!cb->internal)
6602				continue;
6603			(void) printf("%s ioctl %s\n", tbuf,
6604			    fnvlist_lookup_string(rec, ZPOOL_HIST_IOCTL));
6605			if (nvlist_exists(rec, ZPOOL_HIST_INPUT_NVL)) {
6606				(void) printf("    input:\n");
6607				dump_nvlist(fnvlist_lookup_nvlist(rec,
6608				    ZPOOL_HIST_INPUT_NVL), 8);
6609			}
6610			if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_NVL)) {
6611				(void) printf("    output:\n");
6612				dump_nvlist(fnvlist_lookup_nvlist(rec,
6613				    ZPOOL_HIST_OUTPUT_NVL), 8);
6614			}
6615			if (nvlist_exists(rec, ZPOOL_HIST_ERRNO)) {
6616				(void) printf("    errno: %lld\n",
6617				    fnvlist_lookup_int64(rec,
6618				    ZPOOL_HIST_ERRNO));
6619			}
6620		} else {
6621			if (!cb->internal)
6622				continue;
6623			(void) printf("%s unrecognized record:\n", tbuf);
6624			dump_nvlist(rec, 4);
6625		}
6626
6627		if (!cb->longfmt) {
6628			(void) printf("\n");
6629			continue;
6630		}
6631		(void) printf(" [");
6632		if (nvlist_exists(rec, ZPOOL_HIST_WHO)) {
6633			uid_t who = fnvlist_lookup_uint64(rec, ZPOOL_HIST_WHO);
6634			struct passwd *pwd = getpwuid(who);
6635			(void) printf("user %d ", (int)who);
6636			if (pwd != NULL)
6637				(void) printf("(%s) ", pwd->pw_name);
6638		}
6639		if (nvlist_exists(rec, ZPOOL_HIST_HOST)) {
6640			(void) printf("on %s",
6641			    fnvlist_lookup_string(rec, ZPOOL_HIST_HOST));
6642		}
6643		if (nvlist_exists(rec, ZPOOL_HIST_ZONE)) {
6644			(void) printf(":%s",
6645			    fnvlist_lookup_string(rec, ZPOOL_HIST_ZONE));
6646		}
6647		(void) printf("]");
6648		(void) printf("\n");
6649	}
6650	(void) printf("\n");
6651	nvlist_free(nvhis);
6652
6653	return (ret);
6654}
6655
6656/*
6657 * zpool history <pool>
6658 *
6659 * Displays the history of commands that modified pools.
6660 */
6661int
6662zpool_do_history(int argc, char **argv)
6663{
6664	hist_cbdata_t cbdata = { 0 };
6665	int ret;
6666	int c;
6667
6668	cbdata.first = B_TRUE;
6669	/* check options */
6670	while ((c = getopt(argc, argv, "li")) != -1) {
6671		switch (c) {
6672		case 'l':
6673			cbdata.longfmt = B_TRUE;
6674			break;
6675		case 'i':
6676			cbdata.internal = B_TRUE;
6677			break;
6678		case '?':
6679			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
6680			    optopt);
6681			usage(B_FALSE);
6682		}
6683	}
6684	argc -= optind;
6685	argv += optind;
6686
6687	ret = for_each_pool(argc, argv, B_FALSE,  NULL, get_history_one,
6688	    &cbdata);
6689
6690	if (argc == 0 && cbdata.first == B_TRUE) {
6691		(void) printf(gettext("no pools available\n"));
6692		return (0);
6693	}
6694
6695	return (ret);
6696}
6697
6698static int
6699get_callback(zpool_handle_t *zhp, void *data)
6700{
6701