xref: /illumos-gate/usr/src/cmd/zpool/zpool_main.c (revision d2aa06e8e95fc3abd82008cdd0959cbbdb2d50fb)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
25  * Copyright (c) 2012 by Frederik Wessels. All rights reserved.
26  * Copyright (c) 2013 by Prasad Joshi (sTec). All rights reserved.
27  * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
28  * Copyright 2016 Nexenta Systems, Inc.
29  * Copyright (c) 2017 Datto Inc.
30  */
31 
32 #include <assert.h>
33 #include <ctype.h>
34 #include <dirent.h>
35 #include <errno.h>
36 #include <fcntl.h>
37 #include <libgen.h>
38 #include <libintl.h>
39 #include <libuutil.h>
40 #include <locale.h>
41 #include <stdio.h>
42 #include <stdlib.h>
43 #include <string.h>
44 #include <strings.h>
45 #include <unistd.h>
46 #include <priv.h>
47 #include <pwd.h>
48 #include <zone.h>
49 #include <zfs_prop.h>
50 #include <sys/fs/zfs.h>
51 #include <sys/stat.h>
52 
53 #include <libzfs.h>
54 
55 #include "zpool_util.h"
56 #include "zfs_comutil.h"
57 #include "zfeature_common.h"
58 
59 #include "statcommon.h"
60 
61 static int zpool_do_create(int, char **);
62 static int zpool_do_destroy(int, char **);
63 
64 static int zpool_do_add(int, char **);
65 static int zpool_do_remove(int, char **);
66 static int zpool_do_labelclear(int, char **);
67 
68 static int zpool_do_list(int, char **);
69 static int zpool_do_iostat(int, char **);
70 static int zpool_do_status(int, char **);
71 
72 static int zpool_do_online(int, char **);
73 static int zpool_do_offline(int, char **);
74 static int zpool_do_clear(int, char **);
75 static int zpool_do_reopen(int, char **);
76 
77 static int zpool_do_reguid(int, char **);
78 
79 static int zpool_do_attach(int, char **);
80 static int zpool_do_detach(int, char **);
81 static int zpool_do_replace(int, char **);
82 static int zpool_do_split(int, char **);
83 
84 static int zpool_do_scrub(int, char **);
85 
86 static int zpool_do_import(int, char **);
87 static int zpool_do_export(int, char **);
88 
89 static int zpool_do_upgrade(int, char **);
90 
91 static int zpool_do_history(int, char **);
92 
93 static int zpool_do_get(int, char **);
94 static int zpool_do_set(int, char **);
95 
96 /*
97  * These libumem hooks provide a reasonable set of defaults for the allocator's
98  * debugging facilities.
99  */
100 
101 #ifdef DEBUG
102 const char *
103 _umem_debug_init(void)
104 {
105 	return ("default,verbose"); /* $UMEM_DEBUG setting */
106 }
107 
108 const char *
109 _umem_logging_init(void)
110 {
111 	return ("fail,contents"); /* $UMEM_LOGGING setting */
112 }
113 #endif
114 
115 typedef enum {
116 	HELP_ADD,
117 	HELP_ATTACH,
118 	HELP_CLEAR,
119 	HELP_CREATE,
120 	HELP_DESTROY,
121 	HELP_DETACH,
122 	HELP_EXPORT,
123 	HELP_HISTORY,
124 	HELP_IMPORT,
125 	HELP_IOSTAT,
126 	HELP_LABELCLEAR,
127 	HELP_LIST,
128 	HELP_OFFLINE,
129 	HELP_ONLINE,
130 	HELP_REPLACE,
131 	HELP_REMOVE,
132 	HELP_SCRUB,
133 	HELP_STATUS,
134 	HELP_UPGRADE,
135 	HELP_GET,
136 	HELP_SET,
137 	HELP_SPLIT,
138 	HELP_REGUID,
139 	HELP_REOPEN
140 } zpool_help_t;
141 
142 
143 typedef struct zpool_command {
144 	const char	*name;
145 	int		(*func)(int, char **);
146 	zpool_help_t	usage;
147 } zpool_command_t;
148 
149 /*
150  * Master command table.  Each ZFS command has a name, associated function, and
151  * usage message.  The usage messages need to be internationalized, so we have
152  * to have a function to return the usage message based on a command index.
153  *
154  * These commands are organized according to how they are displayed in the usage
155  * message.  An empty command (one with a NULL name) indicates an empty line in
156  * the generic usage message.
157  */
158 static zpool_command_t command_table[] = {
159 	{ "create",	zpool_do_create,	HELP_CREATE		},
160 	{ "destroy",	zpool_do_destroy,	HELP_DESTROY		},
161 	{ NULL },
162 	{ "add",	zpool_do_add,		HELP_ADD		},
163 	{ "remove",	zpool_do_remove,	HELP_REMOVE		},
164 	{ NULL },
165 	{ "labelclear",	zpool_do_labelclear,	HELP_LABELCLEAR		},
166 	{ NULL },
167 	{ "list",	zpool_do_list,		HELP_LIST		},
168 	{ "iostat",	zpool_do_iostat,	HELP_IOSTAT		},
169 	{ "status",	zpool_do_status,	HELP_STATUS		},
170 	{ NULL },
171 	{ "online",	zpool_do_online,	HELP_ONLINE		},
172 	{ "offline",	zpool_do_offline,	HELP_OFFLINE		},
173 	{ "clear",	zpool_do_clear,		HELP_CLEAR		},
174 	{ "reopen",	zpool_do_reopen,	HELP_REOPEN		},
175 	{ NULL },
176 	{ "attach",	zpool_do_attach,	HELP_ATTACH		},
177 	{ "detach",	zpool_do_detach,	HELP_DETACH		},
178 	{ "replace",	zpool_do_replace,	HELP_REPLACE		},
179 	{ "split",	zpool_do_split,		HELP_SPLIT		},
180 	{ NULL },
181 	{ "scrub",	zpool_do_scrub,		HELP_SCRUB		},
182 	{ NULL },
183 	{ "import",	zpool_do_import,	HELP_IMPORT		},
184 	{ "export",	zpool_do_export,	HELP_EXPORT		},
185 	{ "upgrade",	zpool_do_upgrade,	HELP_UPGRADE		},
186 	{ "reguid",	zpool_do_reguid,	HELP_REGUID		},
187 	{ NULL },
188 	{ "history",	zpool_do_history,	HELP_HISTORY		},
189 	{ "get",	zpool_do_get,		HELP_GET		},
190 	{ "set",	zpool_do_set,		HELP_SET		},
191 };
192 
193 #define	NCOMMAND	(sizeof (command_table) / sizeof (command_table[0]))
194 
195 static zpool_command_t *current_command;
196 static char history_str[HIS_MAX_RECORD_LEN];
197 static boolean_t log_history = B_TRUE;
198 static uint_t timestamp_fmt = NODATE;
199 
200 static const char *
201 get_usage(zpool_help_t idx)
202 {
203 	switch (idx) {
204 	case HELP_ADD:
205 		return (gettext("\tadd [-fn] <pool> <vdev> ...\n"));
206 	case HELP_ATTACH:
207 		return (gettext("\tattach [-f] <pool> <device> "
208 		    "<new-device>\n"));
209 	case HELP_CLEAR:
210 		return (gettext("\tclear [-nF] <pool> [device]\n"));
211 	case HELP_CREATE:
212 		return (gettext("\tcreate [-fnd] [-B] "
213 		    "[-o property=value] ... \n"
214 		    "\t    [-O file-system-property=value] ... \n"
215 		    "\t    [-m mountpoint] [-R root] <pool> <vdev> ...\n"));
216 	case HELP_DESTROY:
217 		return (gettext("\tdestroy [-f] <pool>\n"));
218 	case HELP_DETACH:
219 		return (gettext("\tdetach <pool> <device>\n"));
220 	case HELP_EXPORT:
221 		return (gettext("\texport [-f] <pool> ...\n"));
222 	case HELP_HISTORY:
223 		return (gettext("\thistory [-il] [<pool>] ...\n"));
224 	case HELP_IMPORT:
225 		return (gettext("\timport [-d dir] [-D]\n"
226 		    "\timport [-d dir | -c cachefile] [-F [-n]] <pool | id>\n"
227 		    "\timport [-o mntopts] [-o property=value] ... \n"
228 		    "\t    [-d dir | -c cachefile] [-D] [-f] [-m] [-N] "
229 		    "[-R root] [-F [-n]] -a\n"
230 		    "\timport [-o mntopts] [-o property=value] ... \n"
231 		    "\t    [-d dir | -c cachefile] [-D] [-f] [-m] [-N] "
232 		    "[-R root] [-F [-n]]\n"
233 		    "\t    <pool | id> [newpool]\n"));
234 	case HELP_IOSTAT:
235 		return (gettext("\tiostat [-v] [-T d|u] [pool] ... [interval "
236 		    "[count]]\n"));
237 	case HELP_LABELCLEAR:
238 		return (gettext("\tlabelclear [-f] <vdev>\n"));
239 	case HELP_LIST:
240 		return (gettext("\tlist [-Hp] [-o property[,...]] "
241 		    "[-T d|u] [pool] ... [interval [count]]\n"));
242 	case HELP_OFFLINE:
243 		return (gettext("\toffline [-t] <pool> <device> ...\n"));
244 	case HELP_ONLINE:
245 		return (gettext("\tonline <pool> <device> ...\n"));
246 	case HELP_REPLACE:
247 		return (gettext("\treplace [-f] <pool> <device> "
248 		    "[new-device]\n"));
249 	case HELP_REMOVE:
250 		return (gettext("\tremove [-nps] <pool> <device> ...\n"));
251 	case HELP_REOPEN:
252 		return (gettext("\treopen <pool>\n"));
253 	case HELP_SCRUB:
254 		return (gettext("\tscrub [-s | -p] <pool> ...\n"));
255 	case HELP_STATUS:
256 		return (gettext("\tstatus [-vx] [-T d|u] [pool] ... [interval "
257 		    "[count]]\n"));
258 	case HELP_UPGRADE:
259 		return (gettext("\tupgrade\n"
260 		    "\tupgrade -v\n"
261 		    "\tupgrade [-V version] <-a | pool ...>\n"));
262 	case HELP_GET:
263 		return (gettext("\tget [-Hp] [-o \"all\" | field[,...]] "
264 		    "<\"all\" | property[,...]> <pool> ...\n"));
265 	case HELP_SET:
266 		return (gettext("\tset <property=value> <pool> \n"));
267 	case HELP_SPLIT:
268 		return (gettext("\tsplit [-n] [-R altroot] [-o mntopts]\n"
269 		    "\t    [-o property=value] <pool> <newpool> "
270 		    "[<device> ...]\n"));
271 	case HELP_REGUID:
272 		return (gettext("\treguid <pool>\n"));
273 	}
274 
275 	abort();
276 	/* NOTREACHED */
277 }
278 
279 
280 /*
281  * Callback routine that will print out a pool property value.
282  */
283 static int
284 print_prop_cb(int prop, void *cb)
285 {
286 	FILE *fp = cb;
287 
288 	(void) fprintf(fp, "\t%-15s  ", zpool_prop_to_name(prop));
289 
290 	if (zpool_prop_readonly(prop))
291 		(void) fprintf(fp, "  NO   ");
292 	else
293 		(void) fprintf(fp, " YES   ");
294 
295 	if (zpool_prop_values(prop) == NULL)
296 		(void) fprintf(fp, "-\n");
297 	else
298 		(void) fprintf(fp, "%s\n", zpool_prop_values(prop));
299 
300 	return (ZPROP_CONT);
301 }
302 
303 /*
304  * Display usage message.  If we're inside a command, display only the usage for
305  * that command.  Otherwise, iterate over the entire command table and display
306  * a complete usage message.
307  */
308 void
309 usage(boolean_t requested)
310 {
311 	FILE *fp = requested ? stdout : stderr;
312 
313 	if (current_command == NULL) {
314 		int i;
315 
316 		(void) fprintf(fp, gettext("usage: zpool command args ...\n"));
317 		(void) fprintf(fp,
318 		    gettext("where 'command' is one of the following:\n\n"));
319 
320 		for (i = 0; i < NCOMMAND; i++) {
321 			if (command_table[i].name == NULL)
322 				(void) fprintf(fp, "\n");
323 			else
324 				(void) fprintf(fp, "%s",
325 				    get_usage(command_table[i].usage));
326 		}
327 	} else {
328 		(void) fprintf(fp, gettext("usage:\n"));
329 		(void) fprintf(fp, "%s", get_usage(current_command->usage));
330 	}
331 
332 	if (current_command != NULL &&
333 	    ((strcmp(current_command->name, "set") == 0) ||
334 	    (strcmp(current_command->name, "get") == 0) ||
335 	    (strcmp(current_command->name, "list") == 0))) {
336 
337 		(void) fprintf(fp,
338 		    gettext("\nthe following properties are supported:\n"));
339 
340 		(void) fprintf(fp, "\n\t%-15s  %s   %s\n\n",
341 		    "PROPERTY", "EDIT", "VALUES");
342 
343 		/* Iterate over all properties */
344 		(void) zprop_iter(print_prop_cb, fp, B_FALSE, B_TRUE,
345 		    ZFS_TYPE_POOL);
346 
347 		(void) fprintf(fp, "\t%-15s   ", "feature@...");
348 		(void) fprintf(fp, "YES   disabled | enabled | active\n");
349 
350 		(void) fprintf(fp, gettext("\nThe feature@ properties must be "
351 		    "appended with a feature name.\nSee zpool-features(5).\n"));
352 	}
353 
354 	/*
355 	 * See comments at end of main().
356 	 */
357 	if (getenv("ZFS_ABORT") != NULL) {
358 		(void) printf("dumping core by request\n");
359 		abort();
360 	}
361 
362 	exit(requested ? 0 : 2);
363 }
364 
365 void
366 print_vdev_tree(zpool_handle_t *zhp, const char *name, nvlist_t *nv, int indent,
367     boolean_t print_logs)
368 {
369 	nvlist_t **child;
370 	uint_t c, children;
371 	char *vname;
372 
373 	if (name != NULL)
374 		(void) printf("\t%*s%s\n", indent, "", name);
375 
376 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
377 	    &child, &children) != 0)
378 		return;
379 
380 	for (c = 0; c < children; c++) {
381 		uint64_t is_log = B_FALSE;
382 
383 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
384 		    &is_log);
385 		if ((is_log && !print_logs) || (!is_log && print_logs))
386 			continue;
387 
388 		vname = zpool_vdev_name(g_zfs, zhp, child[c], B_FALSE);
389 		print_vdev_tree(zhp, vname, child[c], indent + 2,
390 		    B_FALSE);
391 		free(vname);
392 	}
393 }
394 
395 static boolean_t
396 prop_list_contains_feature(nvlist_t *proplist)
397 {
398 	nvpair_t *nvp;
399 	for (nvp = nvlist_next_nvpair(proplist, NULL); NULL != nvp;
400 	    nvp = nvlist_next_nvpair(proplist, nvp)) {
401 		if (zpool_prop_feature(nvpair_name(nvp)))
402 			return (B_TRUE);
403 	}
404 	return (B_FALSE);
405 }
406 
407 /*
408  * Add a property pair (name, string-value) into a property nvlist.
409  */
410 static int
411 add_prop_list(const char *propname, char *propval, nvlist_t **props,
412     boolean_t poolprop)
413 {
414 	zpool_prop_t prop = ZPOOL_PROP_INVAL;
415 	zfs_prop_t fprop;
416 	nvlist_t *proplist;
417 	const char *normnm;
418 	char *strval;
419 
420 	if (*props == NULL &&
421 	    nvlist_alloc(props, NV_UNIQUE_NAME, 0) != 0) {
422 		(void) fprintf(stderr,
423 		    gettext("internal error: out of memory\n"));
424 		return (1);
425 	}
426 
427 	proplist = *props;
428 
429 	if (poolprop) {
430 		const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
431 
432 		if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
433 		    !zpool_prop_feature(propname)) {
434 			(void) fprintf(stderr, gettext("property '%s' is "
435 			    "not a valid pool property\n"), propname);
436 			return (2);
437 		}
438 
439 		/*
440 		 * feature@ properties and version should not be specified
441 		 * at the same time.
442 		 */
443 		if ((prop == ZPOOL_PROP_INVAL && zpool_prop_feature(propname) &&
444 		    nvlist_exists(proplist, vname)) ||
445 		    (prop == ZPOOL_PROP_VERSION &&
446 		    prop_list_contains_feature(proplist))) {
447 			(void) fprintf(stderr, gettext("'feature@' and "
448 			    "'version' properties cannot be specified "
449 			    "together\n"));
450 			return (2);
451 		}
452 
453 
454 		if (zpool_prop_feature(propname))
455 			normnm = propname;
456 		else
457 			normnm = zpool_prop_to_name(prop);
458 	} else {
459 		if ((fprop = zfs_name_to_prop(propname)) != ZPROP_INVAL) {
460 			normnm = zfs_prop_to_name(fprop);
461 		} else {
462 			normnm = propname;
463 		}
464 	}
465 
466 	if (nvlist_lookup_string(proplist, normnm, &strval) == 0 &&
467 	    prop != ZPOOL_PROP_CACHEFILE) {
468 		(void) fprintf(stderr, gettext("property '%s' "
469 		    "specified multiple times\n"), propname);
470 		return (2);
471 	}
472 
473 	if (nvlist_add_string(proplist, normnm, propval) != 0) {
474 		(void) fprintf(stderr, gettext("internal "
475 		    "error: out of memory\n"));
476 		return (1);
477 	}
478 
479 	return (0);
480 }
481 
482 /*
483  * zpool add [-fn] <pool> <vdev> ...
484  *
485  *	-f	Force addition of devices, even if they appear in use
486  *	-n	Do not add the devices, but display the resulting layout if
487  *		they were to be added.
488  *
489  * Adds the given vdevs to 'pool'.  As with create, the bulk of this work is
490  * handled by get_vdev_spec(), which constructs the nvlist needed to pass to
491  * libzfs.
492  */
493 int
494 zpool_do_add(int argc, char **argv)
495 {
496 	boolean_t force = B_FALSE;
497 	boolean_t dryrun = B_FALSE;
498 	int c;
499 	nvlist_t *nvroot;
500 	char *poolname;
501 	zpool_boot_label_t boot_type;
502 	uint64_t boot_size;
503 	int ret;
504 	zpool_handle_t *zhp;
505 	nvlist_t *config;
506 
507 	/* check options */
508 	while ((c = getopt(argc, argv, "fn")) != -1) {
509 		switch (c) {
510 		case 'f':
511 			force = B_TRUE;
512 			break;
513 		case 'n':
514 			dryrun = B_TRUE;
515 			break;
516 		case '?':
517 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
518 			    optopt);
519 			usage(B_FALSE);
520 		}
521 	}
522 
523 	argc -= optind;
524 	argv += optind;
525 
526 	/* get pool name and check number of arguments */
527 	if (argc < 1) {
528 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
529 		usage(B_FALSE);
530 	}
531 	if (argc < 2) {
532 		(void) fprintf(stderr, gettext("missing vdev specification\n"));
533 		usage(B_FALSE);
534 	}
535 
536 	poolname = argv[0];
537 
538 	argc--;
539 	argv++;
540 
541 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
542 		return (1);
543 
544 	if ((config = zpool_get_config(zhp, NULL)) == NULL) {
545 		(void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
546 		    poolname);
547 		zpool_close(zhp);
548 		return (1);
549 	}
550 
551 	if (zpool_is_bootable(zhp))
552 		boot_type = ZPOOL_COPY_BOOT_LABEL;
553 	else
554 		boot_type = ZPOOL_NO_BOOT_LABEL;
555 
556 	/* pass off to get_vdev_spec for processing */
557 	boot_size = zpool_get_prop_int(zhp, ZPOOL_PROP_BOOTSIZE, NULL);
558 	nvroot = make_root_vdev(zhp, force, !force, B_FALSE, dryrun,
559 	    boot_type, boot_size, argc, argv);
560 	if (nvroot == NULL) {
561 		zpool_close(zhp);
562 		return (1);
563 	}
564 
565 	if (dryrun) {
566 		nvlist_t *poolnvroot;
567 
568 		verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
569 		    &poolnvroot) == 0);
570 
571 		(void) printf(gettext("would update '%s' to the following "
572 		    "configuration:\n"), zpool_get_name(zhp));
573 
574 		/* print original main pool and new tree */
575 		print_vdev_tree(zhp, poolname, poolnvroot, 0, B_FALSE);
576 		print_vdev_tree(zhp, NULL, nvroot, 0, B_FALSE);
577 
578 		/* Do the same for the logs */
579 		if (num_logs(poolnvroot) > 0) {
580 			print_vdev_tree(zhp, "logs", poolnvroot, 0, B_TRUE);
581 			print_vdev_tree(zhp, NULL, nvroot, 0, B_TRUE);
582 		} else if (num_logs(nvroot) > 0) {
583 			print_vdev_tree(zhp, "logs", nvroot, 0, B_TRUE);
584 		}
585 
586 		ret = 0;
587 	} else {
588 		ret = (zpool_add(zhp, nvroot) != 0);
589 	}
590 
591 	nvlist_free(nvroot);
592 	zpool_close(zhp);
593 
594 	return (ret);
595 }
596 
597 /*
598  * zpool remove  <pool> <vdev> ...
599  *
600  * Removes the given vdev from the pool.
601  */
602 int
603 zpool_do_remove(int argc, char **argv)
604 {
605 	char *poolname;
606 	int i, ret = 0;
607 	zpool_handle_t *zhp;
608 	boolean_t stop = B_FALSE;
609 	boolean_t noop = B_FALSE;
610 	boolean_t parsable = B_FALSE;
611 	char c;
612 
613 	/* check options */
614 	while ((c = getopt(argc, argv, "nps")) != -1) {
615 		switch (c) {
616 		case 'n':
617 			noop = B_TRUE;
618 			break;
619 		case 'p':
620 			parsable = B_TRUE;
621 			break;
622 		case 's':
623 			stop = B_TRUE;
624 			break;
625 		case '?':
626 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
627 			    optopt);
628 			usage(B_FALSE);
629 		}
630 	}
631 
632 	argc -= optind;
633 	argv += optind;
634 
635 	/* get pool name and check number of arguments */
636 	if (argc < 1) {
637 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
638 		usage(B_FALSE);
639 	}
640 
641 	poolname = argv[0];
642 
643 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
644 		return (1);
645 
646 	if (stop && noop) {
647 		(void) fprintf(stderr, gettext("stop request ignored\n"));
648 		return (0);
649 	}
650 
651 	if (stop) {
652 		if (argc > 1) {
653 			(void) fprintf(stderr, gettext("too many arguments\n"));
654 			usage(B_FALSE);
655 		}
656 		if (zpool_vdev_remove_cancel(zhp) != 0)
657 			ret = 1;
658 	} else {
659 		if (argc < 2) {
660 			(void) fprintf(stderr, gettext("missing device\n"));
661 			usage(B_FALSE);
662 		}
663 
664 		for (i = 1; i < argc; i++) {
665 			if (noop) {
666 				uint64_t size;
667 
668 				if (zpool_vdev_indirect_size(zhp, argv[i],
669 				    &size) != 0) {
670 					ret = 1;
671 					break;
672 				}
673 				if (parsable) {
674 					(void) printf("%s %llu\n",
675 					    argv[i], size);
676 				} else {
677 					char valstr[32];
678 					zfs_nicenum(size, valstr,
679 					    sizeof (valstr));
680 					(void) printf("Memory that will be "
681 					    "used after removing %s: %s\n",
682 					    argv[i], valstr);
683 				}
684 			} else {
685 				if (zpool_vdev_remove(zhp, argv[i]) != 0)
686 					ret = 1;
687 			}
688 		}
689 	}
690 
691 	return (ret);
692 }
693 
694 /*
695  * zpool labelclear [-f] <vdev>
696  *
697  *	-f	Force clearing the label for the vdevs which are members of
698  *		the exported or foreign pools.
699  *
700  * Verifies that the vdev is not active and zeros out the label information
701  * on the device.
702  */
703 int
704 zpool_do_labelclear(int argc, char **argv)
705 {
706 	char vdev[MAXPATHLEN];
707 	char *name = NULL;
708 	struct stat st;
709 	int c, fd, ret = 0;
710 	nvlist_t *config;
711 	pool_state_t state;
712 	boolean_t inuse = B_FALSE;
713 	boolean_t force = B_FALSE;
714 
715 	/* check options */
716 	while ((c = getopt(argc, argv, "f")) != -1) {
717 		switch (c) {
718 		case 'f':
719 			force = B_TRUE;
720 			break;
721 		default:
722 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
723 			    optopt);
724 			usage(B_FALSE);
725 		}
726 	}
727 
728 	argc -= optind;
729 	argv += optind;
730 
731 	/* get vdev name */
732 	if (argc < 1) {
733 		(void) fprintf(stderr, gettext("missing vdev name\n"));
734 		usage(B_FALSE);
735 	}
736 	if (argc > 1) {
737 		(void) fprintf(stderr, gettext("too many arguments\n"));
738 		usage(B_FALSE);
739 	}
740 
741 	/*
742 	 * Check if we were given absolute path and use it as is.
743 	 * Otherwise if the provided vdev name doesn't point to a file,
744 	 * try prepending dsk path and appending s0.
745 	 */
746 	(void) strlcpy(vdev, argv[0], sizeof (vdev));
747 	if (vdev[0] != '/' && stat(vdev, &st) != 0) {
748 		char *s;
749 
750 		(void) snprintf(vdev, sizeof (vdev), "%s/%s",
751 		    ZFS_DISK_ROOT, argv[0]);
752 		if ((s = strrchr(argv[0], 's')) == NULL ||
753 		    !isdigit(*(s + 1)))
754 			(void) strlcat(vdev, "s0", sizeof (vdev));
755 		if (stat(vdev, &st) != 0) {
756 			(void) fprintf(stderr, gettext(
757 			    "failed to find device %s, try specifying absolute "
758 			    "path instead\n"), argv[0]);
759 			return (1);
760 		}
761 	}
762 
763 	if ((fd = open(vdev, O_RDWR)) < 0) {
764 		(void) fprintf(stderr, gettext("failed to open %s: %s\n"),
765 		    vdev, strerror(errno));
766 		return (1);
767 	}
768 
769 	if (zpool_read_label(fd, &config) != 0) {
770 		(void) fprintf(stderr,
771 		    gettext("failed to read label from %s\n"), vdev);
772 		return (1);
773 	}
774 	nvlist_free(config);
775 
776 	ret = zpool_in_use(g_zfs, fd, &state, &name, &inuse);
777 	if (ret != 0) {
778 		(void) fprintf(stderr,
779 		    gettext("failed to check state for %s\n"), vdev);
780 		return (1);
781 	}
782 
783 	if (!inuse)
784 		goto wipe_label;
785 
786 	switch (state) {
787 	default:
788 	case POOL_STATE_ACTIVE:
789 	case POOL_STATE_SPARE:
790 	case POOL_STATE_L2CACHE:
791 		(void) fprintf(stderr, gettext(
792 		    "%s is a member (%s) of pool \"%s\"\n"),
793 		    vdev, zpool_pool_state_to_name(state), name);
794 		ret = 1;
795 		goto errout;
796 
797 	case POOL_STATE_EXPORTED:
798 		if (force)
799 			break;
800 		(void) fprintf(stderr, gettext(
801 		    "use '-f' to override the following error:\n"
802 		    "%s is a member of exported pool \"%s\"\n"),
803 		    vdev, name);
804 		ret = 1;
805 		goto errout;
806 
807 	case POOL_STATE_POTENTIALLY_ACTIVE:
808 		if (force)
809 			break;
810 		(void) fprintf(stderr, gettext(
811 		    "use '-f' to override the following error:\n"
812 		    "%s is a member of potentially active pool \"%s\"\n"),
813 		    vdev, name);
814 		ret = 1;
815 		goto errout;
816 
817 	case POOL_STATE_DESTROYED:
818 		/* inuse should never be set for a destroyed pool */
819 		assert(0);
820 		break;
821 	}
822 
823 wipe_label:
824 	ret = zpool_clear_label(fd);
825 	if (ret != 0) {
826 		(void) fprintf(stderr,
827 		    gettext("failed to clear label for %s\n"), vdev);
828 	}
829 
830 errout:
831 	free(name);
832 	(void) close(fd);
833 
834 	return (ret);
835 }
836 
837 /*
838  * zpool create [-fnd] [-B] [-o property=value] ...
839  *		[-O file-system-property=value] ...
840  *		[-R root] [-m mountpoint] <pool> <dev> ...
841  *
842  *	-B	Create boot partition.
843  *	-f	Force creation, even if devices appear in use
844  *	-n	Do not create the pool, but display the resulting layout if it
845  *		were to be created.
846  *      -R	Create a pool under an alternate root
847  *      -m	Set default mountpoint for the root dataset.  By default it's
848  *		'/<pool>'
849  *	-o	Set property=value.
850  *	-d	Don't automatically enable all supported pool features
851  *		(individual features can be enabled with -o).
852  *	-O	Set fsproperty=value in the pool's root file system
853  *
854  * Creates the named pool according to the given vdev specification.  The
855  * bulk of the vdev processing is done in get_vdev_spec() in zpool_vdev.c.  Once
856  * we get the nvlist back from get_vdev_spec(), we either print out the contents
857  * (if '-n' was specified), or pass it to libzfs to do the creation.
858  */
859 
860 #define	SYSTEM256	(256 * 1024 * 1024)
861 int
862 zpool_do_create(int argc, char **argv)
863 {
864 	boolean_t force = B_FALSE;
865 	boolean_t dryrun = B_FALSE;
866 	boolean_t enable_all_pool_feat = B_TRUE;
867 	zpool_boot_label_t boot_type = ZPOOL_NO_BOOT_LABEL;
868 	uint64_t boot_size = 0;
869 	int c;
870 	nvlist_t *nvroot = NULL;
871 	char *poolname;
872 	int ret = 1;
873 	char *altroot = NULL;
874 	char *mountpoint = NULL;
875 	nvlist_t *fsprops = NULL;
876 	nvlist_t *props = NULL;
877 	char *propval;
878 
879 	/* check options */
880 	while ((c = getopt(argc, argv, ":fndBR:m:o:O:")) != -1) {
881 		switch (c) {
882 		case 'f':
883 			force = B_TRUE;
884 			break;
885 		case 'n':
886 			dryrun = B_TRUE;
887 			break;
888 		case 'd':
889 			enable_all_pool_feat = B_FALSE;
890 			break;
891 		case 'B':
892 			/*
893 			 * We should create the system partition.
894 			 * Also make sure the size is set.
895 			 */
896 			boot_type = ZPOOL_CREATE_BOOT_LABEL;
897 			if (boot_size == 0)
898 				boot_size = SYSTEM256;
899 			break;
900 		case 'R':
901 			altroot = optarg;
902 			if (add_prop_list(zpool_prop_to_name(
903 			    ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
904 				goto errout;
905 			if (nvlist_lookup_string(props,
906 			    zpool_prop_to_name(ZPOOL_PROP_CACHEFILE),
907 			    &propval) == 0)
908 				break;
909 			if (add_prop_list(zpool_prop_to_name(
910 			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
911 				goto errout;
912 			break;
913 		case 'm':
914 			/* Equivalent to -O mountpoint=optarg */
915 			mountpoint = optarg;
916 			break;
917 		case 'o':
918 			if ((propval = strchr(optarg, '=')) == NULL) {
919 				(void) fprintf(stderr, gettext("missing "
920 				    "'=' for -o option\n"));
921 				goto errout;
922 			}
923 			*propval = '\0';
924 			propval++;
925 
926 			if (add_prop_list(optarg, propval, &props, B_TRUE))
927 				goto errout;
928 
929 			/*
930 			 * Get bootsize value for make_root_vdev().
931 			 */
932 			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_BOOTSIZE) {
933 				if (zfs_nicestrtonum(g_zfs, propval,
934 				    &boot_size) < 0 || boot_size == 0) {
935 					(void) fprintf(stderr,
936 					    gettext("bad boot partition size "
937 					    "'%s': %s\n"),  propval,
938 					    libzfs_error_description(g_zfs));
939 					goto errout;
940 				}
941 			}
942 
943 			/*
944 			 * If the user is creating a pool that doesn't support
945 			 * feature flags, don't enable any features.
946 			 */
947 			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_VERSION) {
948 				char *end;
949 				u_longlong_t ver;
950 
951 				ver = strtoull(propval, &end, 10);
952 				if (*end == '\0' &&
953 				    ver < SPA_VERSION_FEATURES) {
954 					enable_all_pool_feat = B_FALSE;
955 				}
956 			}
957 			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_ALTROOT)
958 				altroot = propval;
959 			break;
960 		case 'O':
961 			if ((propval = strchr(optarg, '=')) == NULL) {
962 				(void) fprintf(stderr, gettext("missing "
963 				    "'=' for -O option\n"));
964 				goto errout;
965 			}
966 			*propval = '\0';
967 			propval++;
968 
969 			/*
970 			 * Mountpoints are checked and then added later.
971 			 * Uniquely among properties, they can be specified
972 			 * more than once, to avoid conflict with -m.
973 			 */
974 			if (0 == strcmp(optarg,
975 			    zfs_prop_to_name(ZFS_PROP_MOUNTPOINT))) {
976 				mountpoint = propval;
977 			} else if (add_prop_list(optarg, propval, &fsprops,
978 			    B_FALSE)) {
979 				goto errout;
980 			}
981 			break;
982 		case ':':
983 			(void) fprintf(stderr, gettext("missing argument for "
984 			    "'%c' option\n"), optopt);
985 			goto badusage;
986 		case '?':
987 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
988 			    optopt);
989 			goto badusage;
990 		}
991 	}
992 
993 	argc -= optind;
994 	argv += optind;
995 
996 	/* get pool name and check number of arguments */
997 	if (argc < 1) {
998 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
999 		goto badusage;
1000 	}
1001 	if (argc < 2) {
1002 		(void) fprintf(stderr, gettext("missing vdev specification\n"));
1003 		goto badusage;
1004 	}
1005 
1006 	poolname = argv[0];
1007 
1008 	/*
1009 	 * As a special case, check for use of '/' in the name, and direct the
1010 	 * user to use 'zfs create' instead.
1011 	 */
1012 	if (strchr(poolname, '/') != NULL) {
1013 		(void) fprintf(stderr, gettext("cannot create '%s': invalid "
1014 		    "character '/' in pool name\n"), poolname);
1015 		(void) fprintf(stderr, gettext("use 'zfs create' to "
1016 		    "create a dataset\n"));
1017 		goto errout;
1018 	}
1019 
1020 	/*
1021 	 * Make sure the bootsize is set when ZPOOL_CREATE_BOOT_LABEL is used,
1022 	 * and not set otherwise.
1023 	 */
1024 	if (boot_type == ZPOOL_CREATE_BOOT_LABEL) {
1025 		const char *propname;
1026 		char *strptr, *buf = NULL;
1027 		int rv;
1028 
1029 		propname = zpool_prop_to_name(ZPOOL_PROP_BOOTSIZE);
1030 		if (nvlist_lookup_string(props, propname, &strptr) != 0) {
1031 			(void) asprintf(&buf, "%" PRIu64, boot_size);
1032 			if (buf == NULL) {
1033 				(void) fprintf(stderr,
1034 				    gettext("internal error: out of memory\n"));
1035 				goto errout;
1036 			}
1037 			rv = add_prop_list(propname, buf, &props, B_TRUE);
1038 			free(buf);
1039 			if (rv != 0)
1040 				goto errout;
1041 		}
1042 	} else {
1043 		const char *propname;
1044 		char *strptr;
1045 
1046 		propname = zpool_prop_to_name(ZPOOL_PROP_BOOTSIZE);
1047 		if (nvlist_lookup_string(props, propname, &strptr) == 0) {
1048 			(void) fprintf(stderr, gettext("error: setting boot "
1049 			    "partition size requires option '-B'\n"));
1050 			goto errout;
1051 		}
1052 	}
1053 
1054 	/* pass off to get_vdev_spec for bulk processing */
1055 	nvroot = make_root_vdev(NULL, force, !force, B_FALSE, dryrun,
1056 	    boot_type, boot_size, argc - 1, argv + 1);
1057 	if (nvroot == NULL)
1058 		goto errout;
1059 
1060 	/* make_root_vdev() allows 0 toplevel children if there are spares */
1061 	if (!zfs_allocatable_devs(nvroot)) {
1062 		(void) fprintf(stderr, gettext("invalid vdev "
1063 		    "specification: at least one toplevel vdev must be "
1064 		    "specified\n"));
1065 		goto errout;
1066 	}
1067 
1068 	if (altroot != NULL && altroot[0] != '/') {
1069 		(void) fprintf(stderr, gettext("invalid alternate root '%s': "
1070 		    "must be an absolute path\n"), altroot);
1071 		goto errout;
1072 	}
1073 
1074 	/*
1075 	 * Check the validity of the mountpoint and direct the user to use the
1076 	 * '-m' mountpoint option if it looks like its in use.
1077 	 */
1078 	if (mountpoint == NULL ||
1079 	    (strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0 &&
1080 	    strcmp(mountpoint, ZFS_MOUNTPOINT_NONE) != 0)) {
1081 		char buf[MAXPATHLEN];
1082 		DIR *dirp;
1083 
1084 		if (mountpoint && mountpoint[0] != '/') {
1085 			(void) fprintf(stderr, gettext("invalid mountpoint "
1086 			    "'%s': must be an absolute path, 'legacy', or "
1087 			    "'none'\n"), mountpoint);
1088 			goto errout;
1089 		}
1090 
1091 		if (mountpoint == NULL) {
1092 			if (altroot != NULL)
1093 				(void) snprintf(buf, sizeof (buf), "%s/%s",
1094 				    altroot, poolname);
1095 			else
1096 				(void) snprintf(buf, sizeof (buf), "/%s",
1097 				    poolname);
1098 		} else {
1099 			if (altroot != NULL)
1100 				(void) snprintf(buf, sizeof (buf), "%s%s",
1101 				    altroot, mountpoint);
1102 			else
1103 				(void) snprintf(buf, sizeof (buf), "%s",
1104 				    mountpoint);
1105 		}
1106 
1107 		if ((dirp = opendir(buf)) == NULL && errno != ENOENT) {
1108 			(void) fprintf(stderr, gettext("mountpoint '%s' : "
1109 			    "%s\n"), buf, strerror(errno));
1110 			(void) fprintf(stderr, gettext("use '-m' "
1111 			    "option to provide a different default\n"));
1112 			goto errout;
1113 		} else if (dirp) {
1114 			int count = 0;
1115 
1116 			while (count < 3 && readdir(dirp) != NULL)
1117 				count++;
1118 			(void) closedir(dirp);
1119 
1120 			if (count > 2) {
1121 				(void) fprintf(stderr, gettext("mountpoint "
1122 				    "'%s' exists and is not empty\n"), buf);
1123 				(void) fprintf(stderr, gettext("use '-m' "
1124 				    "option to provide a "
1125 				    "different default\n"));
1126 				goto errout;
1127 			}
1128 		}
1129 	}
1130 
1131 	/*
1132 	 * Now that the mountpoint's validity has been checked, ensure that
1133 	 * the property is set appropriately prior to creating the pool.
1134 	 */
1135 	if (mountpoint != NULL) {
1136 		ret = add_prop_list(zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1137 		    mountpoint, &fsprops, B_FALSE);
1138 		if (ret != 0)
1139 			goto errout;
1140 	}
1141 
1142 	ret = 1;
1143 	if (dryrun) {
1144 		/*
1145 		 * For a dry run invocation, print out a basic message and run
1146 		 * through all the vdevs in the list and print out in an
1147 		 * appropriate hierarchy.
1148 		 */
1149 		(void) printf(gettext("would create '%s' with the "
1150 		    "following layout:\n\n"), poolname);
1151 
1152 		print_vdev_tree(NULL, poolname, nvroot, 0, B_FALSE);
1153 		if (num_logs(nvroot) > 0)
1154 			print_vdev_tree(NULL, "logs", nvroot, 0, B_TRUE);
1155 
1156 		ret = 0;
1157 	} else {
1158 		/*
1159 		 * Hand off to libzfs.
1160 		 */
1161 		if (enable_all_pool_feat) {
1162 			spa_feature_t i;
1163 			for (i = 0; i < SPA_FEATURES; i++) {
1164 				char propname[MAXPATHLEN];
1165 				zfeature_info_t *feat = &spa_feature_table[i];
1166 
1167 				(void) snprintf(propname, sizeof (propname),
1168 				    "feature@%s", feat->fi_uname);
1169 
1170 				/*
1171 				 * Skip feature if user specified it manually
1172 				 * on the command line.
1173 				 */
1174 				if (nvlist_exists(props, propname))
1175 					continue;
1176 
1177 				ret = add_prop_list(propname,
1178 				    ZFS_FEATURE_ENABLED, &props, B_TRUE);
1179 				if (ret != 0)
1180 					goto errout;
1181 			}
1182 		}
1183 
1184 		ret = 1;
1185 		if (zpool_create(g_zfs, poolname,
1186 		    nvroot, props, fsprops) == 0) {
1187 			zfs_handle_t *pool = zfs_open(g_zfs, poolname,
1188 			    ZFS_TYPE_FILESYSTEM);
1189 			if (pool != NULL) {
1190 				if (zfs_mount(pool, NULL, 0) == 0)
1191 					ret = zfs_shareall(pool);
1192 				zfs_close(pool);
1193 			}
1194 		} else if (libzfs_errno(g_zfs) == EZFS_INVALIDNAME) {
1195 			(void) fprintf(stderr, gettext("pool name may have "
1196 			    "been omitted\n"));
1197 		}
1198 	}
1199 
1200 errout:
1201 	nvlist_free(nvroot);
1202 	nvlist_free(fsprops);
1203 	nvlist_free(props);
1204 	return (ret);
1205 badusage:
1206 	nvlist_free(fsprops);
1207 	nvlist_free(props);
1208 	usage(B_FALSE);
1209 	return (2);
1210 }
1211 
1212 /*
1213  * zpool destroy <pool>
1214  *
1215  * 	-f	Forcefully unmount any datasets
1216  *
1217  * Destroy the given pool.  Automatically unmounts any datasets in the pool.
1218  */
1219 int
1220 zpool_do_destroy(int argc, char **argv)
1221 {
1222 	boolean_t force = B_FALSE;
1223 	int c;
1224 	char *pool;
1225 	zpool_handle_t *zhp;
1226 	int ret;
1227 
1228 	/* check options */
1229 	while ((c = getopt(argc, argv, "f")) != -1) {
1230 		switch (c) {
1231 		case 'f':
1232 			force = B_TRUE;
1233 			break;
1234 		case '?':
1235 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
1236 			    optopt);
1237 			usage(B_FALSE);
1238 		}
1239 	}
1240 
1241 	argc -= optind;
1242 	argv += optind;
1243 
1244 	/* check arguments */
1245 	if (argc < 1) {
1246 		(void) fprintf(stderr, gettext("missing pool argument\n"));
1247 		usage(B_FALSE);
1248 	}
1249 	if (argc > 1) {
1250 		(void) fprintf(stderr, gettext("too many arguments\n"));
1251 		usage(B_FALSE);
1252 	}
1253 
1254 	pool = argv[0];
1255 
1256 	if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
1257 		/*
1258 		 * As a special case, check for use of '/' in the name, and
1259 		 * direct the user to use 'zfs destroy' instead.
1260 		 */
1261 		if (strchr(pool, '/') != NULL)
1262 			(void) fprintf(stderr, gettext("use 'zfs destroy' to "
1263 			    "destroy a dataset\n"));
1264 		return (1);
1265 	}
1266 
1267 	if (zpool_disable_datasets(zhp, force) != 0) {
1268 		(void) fprintf(stderr, gettext("could not destroy '%s': "
1269 		    "could not unmount datasets\n"), zpool_get_name(zhp));
1270 		return (1);
1271 	}
1272 
1273 	/* The history must be logged as part of the export */
1274 	log_history = B_FALSE;
1275 
1276 	ret = (zpool_destroy(zhp, history_str) != 0);
1277 
1278 	zpool_close(zhp);
1279 
1280 	return (ret);
1281 }
1282 
1283 /*
1284  * zpool export [-f] <pool> ...
1285  *
1286  *	-f	Forcefully unmount datasets
1287  *
1288  * Export the given pools.  By default, the command will attempt to cleanly
1289  * unmount any active datasets within the pool.  If the '-f' flag is specified,
1290  * then the datasets will be forcefully unmounted.
1291  */
1292 int
1293 zpool_do_export(int argc, char **argv)
1294 {
1295 	boolean_t force = B_FALSE;
1296 	boolean_t hardforce = B_FALSE;
1297 	int c;
1298 	zpool_handle_t *zhp;
1299 	int ret;
1300 	int i;
1301 
1302 	/* check options */
1303 	while ((c = getopt(argc, argv, "fF")) != -1) {
1304 		switch (c) {
1305 		case 'f':
1306 			force = B_TRUE;
1307 			break;
1308 		case 'F':
1309 			hardforce = B_TRUE;
1310 			break;
1311 		case '?':
1312 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
1313 			    optopt);
1314 			usage(B_FALSE);
1315 		}
1316 	}
1317 
1318 	argc -= optind;
1319 	argv += optind;
1320 
1321 	/* check arguments */
1322 	if (argc < 1) {
1323 		(void) fprintf(stderr, gettext("missing pool argument\n"));
1324 		usage(B_FALSE);
1325 	}
1326 
1327 	ret = 0;
1328 	for (i = 0; i < argc; i++) {
1329 		if ((zhp = zpool_open_canfail(g_zfs, argv[i])) == NULL) {
1330 			ret = 1;
1331 			continue;
1332 		}
1333 
1334 		if (zpool_disable_datasets(zhp, force) != 0) {
1335 			ret = 1;
1336 			zpool_close(zhp);
1337 			continue;
1338 		}
1339 
1340 		/* The history must be logged as part of the export */
1341 		log_history = B_FALSE;
1342 
1343 		if (hardforce) {
1344 			if (zpool_export_force(zhp, history_str) != 0)
1345 				ret = 1;
1346 		} else if (zpool_export(zhp, force, history_str) != 0) {
1347 			ret = 1;
1348 		}
1349 
1350 		zpool_close(zhp);
1351 	}
1352 
1353 	return (ret);
1354 }
1355 
1356 /*
1357  * Given a vdev configuration, determine the maximum width needed for the device
1358  * name column.
1359  */
1360 static int
1361 max_width(zpool_handle_t *zhp, nvlist_t *nv, int depth, int max)
1362 {
1363 	char *name = zpool_vdev_name(g_zfs, zhp, nv, B_TRUE);
1364 	nvlist_t **child;
1365 	uint_t c, children;
1366 	int ret;
1367 
1368 	if (strlen(name) + depth > max)
1369 		max = strlen(name) + depth;
1370 
1371 	free(name);
1372 
1373 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
1374 	    &child, &children) == 0) {
1375 		for (c = 0; c < children; c++)
1376 			if ((ret = max_width(zhp, child[c], depth + 2,
1377 			    max)) > max)
1378 				max = ret;
1379 	}
1380 
1381 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
1382 	    &child, &children) == 0) {
1383 		for (c = 0; c < children; c++)
1384 			if ((ret = max_width(zhp, child[c], depth + 2,
1385 			    max)) > max)
1386 				max = ret;
1387 	}
1388 
1389 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1390 	    &child, &children) == 0) {
1391 		for (c = 0; c < children; c++)
1392 			if ((ret = max_width(zhp, child[c], depth + 2,
1393 			    max)) > max)
1394 				max = ret;
1395 	}
1396 
1397 
1398 	return (max);
1399 }
1400 
1401 typedef struct spare_cbdata {
1402 	uint64_t	cb_guid;
1403 	zpool_handle_t	*cb_zhp;
1404 } spare_cbdata_t;
1405 
1406 static boolean_t
1407 find_vdev(nvlist_t *nv, uint64_t search)
1408 {
1409 	uint64_t guid;
1410 	nvlist_t **child;
1411 	uint_t c, children;
1412 
1413 	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
1414 	    search == guid)
1415 		return (B_TRUE);
1416 
1417 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1418 	    &child, &children) == 0) {
1419 		for (c = 0; c < children; c++)
1420 			if (find_vdev(child[c], search))
1421 				return (B_TRUE);
1422 	}
1423 
1424 	return (B_FALSE);
1425 }
1426 
1427 static int
1428 find_spare(zpool_handle_t *zhp, void *data)
1429 {
1430 	spare_cbdata_t *cbp = data;
1431 	nvlist_t *config, *nvroot;
1432 
1433 	config = zpool_get_config(zhp, NULL);
1434 	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
1435 	    &nvroot) == 0);
1436 
1437 	if (find_vdev(nvroot, cbp->cb_guid)) {
1438 		cbp->cb_zhp = zhp;
1439 		return (1);
1440 	}
1441 
1442 	zpool_close(zhp);
1443 	return (0);
1444 }
1445 
1446 /*
1447  * Print out configuration state as requested by status_callback.
1448  */
1449 void
1450 print_status_config(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
1451     int namewidth, int depth, boolean_t isspare)
1452 {
1453 	nvlist_t **child;
1454 	uint_t c, children;
1455 	pool_scan_stat_t *ps = NULL;
1456 	vdev_stat_t *vs;
1457 	char rbuf[6], wbuf[6], cbuf[6];
1458 	char *vname;
1459 	uint64_t notpresent;
1460 	spare_cbdata_t cb;
1461 	const char *state;
1462 	char *type;
1463 
1464 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1465 	    &child, &children) != 0)
1466 		children = 0;
1467 
1468 	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
1469 	    (uint64_t **)&vs, &c) == 0);
1470 
1471 	verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
1472 
1473 	if (strcmp(type, VDEV_TYPE_INDIRECT) == 0)
1474 		return;
1475 
1476 	state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
1477 	if (isspare) {
1478 		/*
1479 		 * For hot spares, we use the terms 'INUSE' and 'AVAILABLE' for
1480 		 * online drives.
1481 		 */
1482 		if (vs->vs_aux == VDEV_AUX_SPARED)
1483 			state = "INUSE";
1484 		else if (vs->vs_state == VDEV_STATE_HEALTHY)
1485 			state = "AVAIL";
1486 	}
1487 
1488 	(void) printf("\t%*s%-*s  %-8s", depth, "", namewidth - depth,
1489 	    name, state);
1490 
1491 	if (!isspare) {
1492 		zfs_nicenum(vs->vs_read_errors, rbuf, sizeof (rbuf));
1493 		zfs_nicenum(vs->vs_write_errors, wbuf, sizeof (wbuf));
1494 		zfs_nicenum(vs->vs_checksum_errors, cbuf, sizeof (cbuf));
1495 		(void) printf(" %5s %5s %5s", rbuf, wbuf, cbuf);
1496 	}
1497 
1498 	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
1499 	    &notpresent) == 0) {
1500 		char *path;
1501 		verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0);
1502 		(void) printf("  was %s", path);
1503 	} else if (vs->vs_aux != 0) {
1504 		(void) printf("  ");
1505 
1506 		switch (vs->vs_aux) {
1507 		case VDEV_AUX_OPEN_FAILED:
1508 			(void) printf(gettext("cannot open"));
1509 			break;
1510 
1511 		case VDEV_AUX_BAD_GUID_SUM:
1512 			(void) printf(gettext("missing device"));
1513 			break;
1514 
1515 		case VDEV_AUX_NO_REPLICAS:
1516 			(void) printf(gettext("insufficient replicas"));
1517 			break;
1518 
1519 		case VDEV_AUX_VERSION_NEWER:
1520 			(void) printf(gettext("newer version"));
1521 			break;
1522 
1523 		case VDEV_AUX_UNSUP_FEAT:
1524 			(void) printf(gettext("unsupported feature(s)"));
1525 			break;
1526 
1527 		case VDEV_AUX_SPARED:
1528 			verify(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID,
1529 			    &cb.cb_guid) == 0);
1530 			if (zpool_iter(g_zfs, find_spare, &cb) == 1) {
1531 				if (strcmp(zpool_get_name(cb.cb_zhp),
1532 				    zpool_get_name(zhp)) == 0)
1533 					(void) printf(gettext("currently in "
1534 					    "use"));
1535 				else
1536 					(void) printf(gettext("in use by "
1537 					    "pool '%s'"),
1538 					    zpool_get_name(cb.cb_zhp));
1539 				zpool_close(cb.cb_zhp);
1540 			} else {
1541 				(void) printf(gettext("currently in use"));
1542 			}
1543 			break;
1544 
1545 		case VDEV_AUX_ERR_EXCEEDED:
1546 			(void) printf(gettext("too many errors"));
1547 			break;
1548 
1549 		case VDEV_AUX_IO_FAILURE:
1550 			(void) printf(gettext("experienced I/O failures"));
1551 			break;
1552 
1553 		case VDEV_AUX_BAD_LOG:
1554 			(void) printf(gettext("bad intent log"));
1555 			break;
1556 
1557 		case VDEV_AUX_EXTERNAL:
1558 			(void) printf(gettext("external device fault"));
1559 			break;
1560 
1561 		case VDEV_AUX_SPLIT_POOL:
1562 			(void) printf(gettext("split into new pool"));
1563 			break;
1564 
1565 		default:
1566 			(void) printf(gettext("corrupted data"));
1567 			break;
1568 		}
1569 	}
1570 
1571 	(void) nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_SCAN_STATS,
1572 	    (uint64_t **)&ps, &c);
1573 
1574 	if (ps && ps->pss_state == DSS_SCANNING &&
1575 	    vs->vs_scan_processed != 0 && children == 0) {
1576 		(void) printf(gettext("  (%s)"),
1577 		    (ps->pss_func == POOL_SCAN_RESILVER) ?
1578 		    "resilvering" : "repairing");
1579 	}
1580 
1581 	(void) printf("\n");
1582 
1583 	for (c = 0; c < children; c++) {
1584 		uint64_t islog = B_FALSE, ishole = B_FALSE;
1585 
1586 		/* Don't print logs or holes here */
1587 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
1588 		    &islog);
1589 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
1590 		    &ishole);
1591 		if (islog || ishole)
1592 			continue;
1593 		vname = zpool_vdev_name(g_zfs, zhp, child[c], B_TRUE);
1594 		print_status_config(zhp, vname, child[c],
1595 		    namewidth, depth + 2, isspare);
1596 		free(vname);
1597 	}
1598 }
1599 
1600 
1601 /*
1602  * Print the configuration of an exported pool.  Iterate over all vdevs in the
1603  * pool, printing out the name and status for each one.
1604  */
1605 void
1606 print_import_config(const char *name, nvlist_t *nv, int namewidth, int depth)
1607 {
1608 	nvlist_t **child;
1609 	uint_t c, children;
1610 	vdev_stat_t *vs;
1611 	char *type, *vname;
1612 
1613 	verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
1614 	if (strcmp(type, VDEV_TYPE_MISSING) == 0 ||
1615 	    strcmp(type, VDEV_TYPE_HOLE) == 0)
1616 		return;
1617 
1618 	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
1619 	    (uint64_t **)&vs, &c) == 0);
1620 
1621 	(void) printf("\t%*s%-*s", depth, "", namewidth - depth, name);
1622 	(void) printf("  %s", zpool_state_to_name(vs->vs_state, vs->vs_aux));
1623 
1624 	if (vs->vs_aux != 0) {
1625 		(void) printf("  ");
1626 
1627 		switch (vs->vs_aux) {
1628 		case VDEV_AUX_OPEN_FAILED:
1629 			(void) printf(gettext("cannot open"));
1630 			break;
1631 
1632 		case VDEV_AUX_BAD_GUID_SUM:
1633 			(void) printf(gettext("missing device"));
1634 			break;
1635 
1636 		case VDEV_AUX_NO_REPLICAS:
1637 			(void) printf(gettext("insufficient replicas"));
1638 			break;
1639 
1640 		case VDEV_AUX_VERSION_NEWER:
1641 			(void) printf(gettext("newer version"));
1642 			break;
1643 
1644 		case VDEV_AUX_UNSUP_FEAT:
1645 			(void) printf(gettext("unsupported feature(s)"));
1646 			break;
1647 
1648 		case VDEV_AUX_ERR_EXCEEDED:
1649 			(void) printf(gettext("too many errors"));
1650 			break;
1651 
1652 		default:
1653 			(void) printf(gettext("corrupted data"));
1654 			break;
1655 		}
1656 	}
1657 	(void) printf("\n");
1658 
1659 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1660 	    &child, &children) != 0)
1661 		return;
1662 
1663 	for (c = 0; c < children; c++) {
1664 		uint64_t is_log = B_FALSE;
1665 
1666 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
1667 		    &is_log);
1668 		if (is_log)
1669 			continue;
1670 
1671 		vname = zpool_vdev_name(g_zfs, NULL, child[c], B_TRUE);
1672 		print_import_config(vname, child[c], namewidth, depth + 2);
1673 		free(vname);
1674 	}
1675 
1676 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
1677 	    &child, &children) == 0) {
1678 		(void) printf(gettext("\tcache\n"));
1679 		for (c = 0; c < children; c++) {
1680 			vname = zpool_vdev_name(g_zfs, NULL, child[c], B_FALSE);
1681 			(void) printf("\t  %s\n", vname);
1682 			free(vname);
1683 		}
1684 	}
1685 
1686 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
1687 	    &child, &children) == 0) {
1688 		(void) printf(gettext("\tspares\n"));
1689 		for (c = 0; c < children; c++) {
1690 			vname = zpool_vdev_name(g_zfs, NULL, child[c], B_FALSE);
1691 			(void) printf("\t  %s\n", vname);
1692 			free(vname);
1693 		}
1694 	}
1695 }
1696 
1697 /*
1698  * Print log vdevs.
1699  * Logs are recorded as top level vdevs in the main pool child array
1700  * but with "is_log" set to 1. We use either print_status_config() or
1701  * print_import_config() to print the top level logs then any log
1702  * children (eg mirrored slogs) are printed recursively - which
1703  * works because only the top level vdev is marked "is_log"
1704  */
1705 static void
1706 print_logs(zpool_handle_t *zhp, nvlist_t *nv, int namewidth, boolean_t verbose)
1707 {
1708 	uint_t c, children;
1709 	nvlist_t **child;
1710 
1711 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, &child,
1712 	    &children) != 0)
1713 		return;
1714 
1715 	(void) printf(gettext("\tlogs\n"));
1716 
1717 	for (c = 0; c < children; c++) {
1718 		uint64_t is_log = B_FALSE;
1719 		char *name;
1720 
1721 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
1722 		    &is_log);
1723 		if (!is_log)
1724 			continue;
1725 		name = zpool_vdev_name(g_zfs, zhp, child[c], B_TRUE);
1726 		if (verbose)
1727 			print_status_config(zhp, name, child[c], namewidth,
1728 			    2, B_FALSE);
1729 		else
1730 			print_import_config(name, child[c], namewidth, 2);
1731 		free(name);
1732 	}
1733 }
1734 
1735 /*
1736  * Display the status for the given pool.
1737  */
1738 static void
1739 show_import(nvlist_t *config)
1740 {
1741 	uint64_t pool_state;
1742 	vdev_stat_t *vs;
1743 	char *name;
1744 	uint64_t guid;
1745 	char *msgid;
1746 	nvlist_t *nvroot;
1747 	int reason;
1748 	const char *health;
1749 	uint_t vsc;
1750 	int namewidth;
1751 	char *comment;
1752 
1753 	verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
1754 	    &name) == 0);
1755 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
1756 	    &guid) == 0);
1757 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
1758 	    &pool_state) == 0);
1759 	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
1760 	    &nvroot) == 0);
1761 
1762 	verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
1763 	    (uint64_t **)&vs, &vsc) == 0);
1764 	health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
1765 
1766 	reason = zpool_import_status(config, &msgid);
1767 
1768 	(void) printf(gettext("   pool: %s\n"), name);
1769 	(void) printf(gettext("     id: %llu\n"), (u_longlong_t)guid);
1770 	(void) printf(gettext("  state: %s"), health);
1771 	if (pool_state == POOL_STATE_DESTROYED)
1772 		(void) printf(gettext(" (DESTROYED)"));
1773 	(void) printf("\n");
1774 
1775 	switch (reason) {
1776 	case ZPOOL_STATUS_MISSING_DEV_R:
1777 	case ZPOOL_STATUS_MISSING_DEV_NR:
1778 	case ZPOOL_STATUS_BAD_GUID_SUM:
1779 		(void) printf(gettext(" status: One or more devices are "
1780 		    "missing from the system.\n"));
1781 		break;
1782 
1783 	case ZPOOL_STATUS_CORRUPT_LABEL_R:
1784 	case ZPOOL_STATUS_CORRUPT_LABEL_NR:
1785 		(void) printf(gettext(" status: One or more devices contains "
1786 		    "corrupted data.\n"));
1787 		break;
1788 
1789 	case ZPOOL_STATUS_CORRUPT_DATA:
1790 		(void) printf(
1791 		    gettext(" status: The pool data is corrupted.\n"));
1792 		break;
1793 
1794 	case ZPOOL_STATUS_OFFLINE_DEV:
1795 		(void) printf(gettext(" status: One or more devices "
1796 		    "are offlined.\n"));
1797 		break;
1798 
1799 	case ZPOOL_STATUS_CORRUPT_POOL:
1800 		(void) printf(gettext(" status: The pool metadata is "
1801 		    "corrupted.\n"));
1802 		break;
1803 
1804 	case ZPOOL_STATUS_VERSION_OLDER:
1805 		(void) printf(gettext(" status: The pool is formatted using a "
1806 		    "legacy on-disk version.\n"));
1807 		break;
1808 
1809 	case ZPOOL_STATUS_VERSION_NEWER:
1810 		(void) printf(gettext(" status: The pool is formatted using an "
1811 		    "incompatible version.\n"));
1812 		break;
1813 
1814 	case ZPOOL_STATUS_FEAT_DISABLED:
1815 		(void) printf(gettext(" status: Some supported features are "
1816 		    "not enabled on the pool.\n"));
1817 		break;
1818 
1819 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
1820 		(void) printf(gettext("status: The pool uses the following "
1821 		    "feature(s) not supported on this sytem:\n"));
1822 		zpool_print_unsup_feat(config);
1823 		break;
1824 
1825 	case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
1826 		(void) printf(gettext("status: The pool can only be accessed "
1827 		    "in read-only mode on this system. It\n\tcannot be "
1828 		    "accessed in read-write mode because it uses the "
1829 		    "following\n\tfeature(s) not supported on this system:\n"));
1830 		zpool_print_unsup_feat(config);
1831 		break;
1832 
1833 	case ZPOOL_STATUS_HOSTID_MISMATCH:
1834 		(void) printf(gettext(" status: The pool was last accessed by "
1835 		    "another system.\n"));
1836 		break;
1837 
1838 	case ZPOOL_STATUS_FAULTED_DEV_R:
1839 	case ZPOOL_STATUS_FAULTED_DEV_NR:
1840 		(void) printf(gettext(" status: One or more devices are "
1841 		    "faulted.\n"));
1842 		break;
1843 
1844 	case ZPOOL_STATUS_BAD_LOG:
1845 		(void) printf(gettext(" status: An intent log record cannot be "
1846 		    "read.\n"));
1847 		break;
1848 
1849 	case ZPOOL_STATUS_RESILVERING:
1850 		(void) printf(gettext(" status: One or more devices were being "
1851 		    "resilvered.\n"));
1852 		break;
1853 
1854 	default:
1855 		/*
1856 		 * No other status can be seen when importing pools.
1857 		 */
1858 		assert(reason == ZPOOL_STATUS_OK);
1859 	}
1860 
1861 	/*
1862 	 * Print out an action according to the overall state of the pool.
1863 	 */
1864 	if (vs->vs_state == VDEV_STATE_HEALTHY) {
1865 		if (reason == ZPOOL_STATUS_VERSION_OLDER ||
1866 		    reason == ZPOOL_STATUS_FEAT_DISABLED) {
1867 			(void) printf(gettext(" action: The pool can be "
1868 			    "imported using its name or numeric identifier, "
1869 			    "though\n\tsome features will not be available "
1870 			    "without an explicit 'zpool upgrade'.\n"));
1871 		} else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
1872 			(void) printf(gettext(" action: The pool can be "
1873 			    "imported using its name or numeric "
1874 			    "identifier and\n\tthe '-f' flag.\n"));
1875 		} else {
1876 			(void) printf(gettext(" action: The pool can be "
1877 			    "imported using its name or numeric "
1878 			    "identifier.\n"));
1879 		}
1880 	} else if (vs->vs_state == VDEV_STATE_DEGRADED) {
1881 		(void) printf(gettext(" action: The pool can be imported "
1882 		    "despite missing or damaged devices.  The\n\tfault "
1883 		    "tolerance of the pool may be compromised if imported.\n"));
1884 	} else {
1885 		switch (reason) {
1886 		case ZPOOL_STATUS_VERSION_NEWER:
1887 			(void) printf(gettext(" action: The pool cannot be "
1888 			    "imported.  Access the pool on a system running "
1889 			    "newer\n\tsoftware, or recreate the pool from "
1890 			    "backup.\n"));
1891 			break;
1892 		case ZPOOL_STATUS_UNSUP_FEAT_READ:
1893 			(void) printf(gettext("action: The pool cannot be "
1894 			    "imported. Access the pool on a system that "
1895 			    "supports\n\tthe required feature(s), or recreate "
1896 			    "the pool from backup.\n"));
1897 			break;
1898 		case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
1899 			(void) printf(gettext("action: The pool cannot be "
1900 			    "imported in read-write mode. Import the pool "
1901 			    "with\n"
1902 			    "\t\"-o readonly=on\", access the pool on a system "
1903 			    "that supports the\n\trequired feature(s), or "
1904 			    "recreate the pool from backup.\n"));
1905 			break;
1906 		case ZPOOL_STATUS_MISSING_DEV_R:
1907 		case ZPOOL_STATUS_MISSING_DEV_NR:
1908 		case ZPOOL_STATUS_BAD_GUID_SUM:
1909 			(void) printf(gettext(" action: The pool cannot be "
1910 			    "imported. Attach the missing\n\tdevices and try "
1911 			    "again.\n"));
1912 			break;
1913 		default:
1914 			(void) printf(gettext(" action: The pool cannot be "
1915 			    "imported due to damaged devices or data.\n"));
1916 		}
1917 	}
1918 
1919 	/* Print the comment attached to the pool. */
1920 	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
1921 		(void) printf(gettext("comment: %s\n"), comment);
1922 
1923 	/*
1924 	 * If the state is "closed" or "can't open", and the aux state
1925 	 * is "corrupt data":
1926 	 */
1927 	if (((vs->vs_state == VDEV_STATE_CLOSED) ||
1928 	    (vs->vs_state == VDEV_STATE_CANT_OPEN)) &&
1929 	    (vs->vs_aux == VDEV_AUX_CORRUPT_DATA)) {
1930 		if (pool_state == POOL_STATE_DESTROYED)
1931 			(void) printf(gettext("\tThe pool was destroyed, "
1932 			    "but can be imported using the '-Df' flags.\n"));
1933 		else if (pool_state != POOL_STATE_EXPORTED)
1934 			(void) printf(gettext("\tThe pool may be active on "
1935 			    "another system, but can be imported using\n\t"
1936 			    "the '-f' flag.\n"));
1937 	}
1938 
1939 	if (msgid != NULL)
1940 		(void) printf(gettext("   see: http://illumos.org/msg/%s\n"),
1941 		    msgid);
1942 
1943 	(void) printf(gettext(" config:\n\n"));
1944 
1945 	namewidth = max_width(NULL, nvroot, 0, 0);
1946 	if (namewidth < 10)
1947 		namewidth = 10;
1948 
1949 	print_import_config(name, nvroot, namewidth, 0);
1950 	if (num_logs(nvroot) > 0)
1951 		print_logs(NULL, nvroot, namewidth, B_FALSE);
1952 
1953 	if (reason == ZPOOL_STATUS_BAD_GUID_SUM) {
1954 		(void) printf(gettext("\n\tAdditional devices are known to "
1955 		    "be part of this pool, though their\n\texact "
1956 		    "configuration cannot be determined.\n"));
1957 	}
1958 }
1959 
1960 /*
1961  * Perform the import for the given configuration.  This passes the heavy
1962  * lifting off to zpool_import_props(), and then mounts the datasets contained
1963  * within the pool.
1964  */
1965 static int
1966 do_import(nvlist_t *config, const char *newname, const char *mntopts,
1967     nvlist_t *props, int flags)
1968 {
1969 	zpool_handle_t *zhp;
1970 	char *name;
1971 	uint64_t state;
1972 	uint64_t version;
1973 
1974 	verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
1975 	    &name) == 0);
1976 
1977 	verify(nvlist_lookup_uint64(config,
1978 	    ZPOOL_CONFIG_POOL_STATE, &state) == 0);
1979 	verify(nvlist_lookup_uint64(config,
1980 	    ZPOOL_CONFIG_VERSION, &version) == 0);
1981 	if (!SPA_VERSION_IS_SUPPORTED(version)) {
1982 		(void) fprintf(stderr, gettext("cannot import '%s': pool "
1983 		    "is formatted using an unsupported ZFS version\n"), name);
1984 		return (1);
1985 	} else if (state != POOL_STATE_EXPORTED &&
1986 	    !(flags & ZFS_IMPORT_ANY_HOST)) {
1987 		uint64_t hostid;
1988 
1989 		if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID,
1990 		    &hostid) == 0) {
1991 			if ((unsigned long)hostid != gethostid()) {
1992 				char *hostname;
1993 				uint64_t timestamp;
1994 				time_t t;
1995 
1996 				verify(nvlist_lookup_string(config,
1997 				    ZPOOL_CONFIG_HOSTNAME, &hostname) == 0);
1998 				verify(nvlist_lookup_uint64(config,
1999 				    ZPOOL_CONFIG_TIMESTAMP, &timestamp) == 0);
2000 				t = timestamp;
2001 				(void) fprintf(stderr, gettext("cannot import "
2002 				    "'%s': pool may be in use from other "
2003 				    "system, it was last accessed by %s "
2004 				    "(hostid: 0x%lx) on %s"), name, hostname,
2005 				    (unsigned long)hostid,
2006 				    asctime(localtime(&t)));
2007 				(void) fprintf(stderr, gettext("use '-f' to "
2008 				    "import anyway\n"));
2009 				return (1);
2010 			}
2011 		} else {
2012 			(void) fprintf(stderr, gettext("cannot import '%s': "
2013 			    "pool may be in use from other system\n"), name);
2014 			(void) fprintf(stderr, gettext("use '-f' to import "
2015 			    "anyway\n"));
2016 			return (1);
2017 		}
2018 	}
2019 
2020 	if (zpool_import_props(g_zfs, config, newname, props, flags) != 0)
2021 		return (1);
2022 
2023 	if (newname != NULL)
2024 		name = (char *)newname;
2025 
2026 	if ((zhp = zpool_open_canfail(g_zfs, name)) == NULL)
2027 		return (1);
2028 
2029 	if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
2030 	    !(flags & ZFS_IMPORT_ONLY) &&
2031 	    zpool_enable_datasets(zhp, mntopts, 0) != 0) {
2032 		zpool_close(zhp);
2033 		return (1);
2034 	}
2035 
2036 	zpool_close(zhp);
2037 	return (0);
2038 }
2039 
2040 /*
2041  * zpool import [-d dir] [-D]
2042  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D]
2043  *              [-d dir | -c cachefile] [-f] -a
2044  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D]
2045  *              [-d dir | -c cachefile] [-f] [-n] [-F] <pool | id> [newpool]
2046  *
2047  *	 -c	Read pool information from a cachefile instead of searching
2048  *		devices.
2049  *
2050  *       -d	Scan in a specific directory, other than /dev/dsk.  More than
2051  *		one directory can be specified using multiple '-d' options.
2052  *
2053  *       -D     Scan for previously destroyed pools or import all or only
2054  *              specified destroyed pools.
2055  *
2056  *       -R	Temporarily import the pool, with all mountpoints relative to
2057  *		the given root.  The pool will remain exported when the machine
2058  *		is rebooted.
2059  *
2060  *       -V	Import even in the presence of faulted vdevs.  This is an
2061  *       	intentionally undocumented option for testing purposes, and
2062  *       	treats the pool configuration as complete, leaving any bad
2063  *		vdevs in the FAULTED state. In other words, it does verbatim
2064  *		import.
2065  *
2066  *       -f	Force import, even if it appears that the pool is active.
2067  *
2068  *       -F     Attempt rewind if necessary.
2069  *
2070  *       -n     See if rewind would work, but don't actually rewind.
2071  *
2072  *       -N     Import the pool but don't mount datasets.
2073  *
2074  *       -T     Specify a starting txg to use for import. This option is
2075  *       	intentionally undocumented option for testing purposes.
2076  *
2077  *       -a	Import all pools found.
2078  *
2079  *       -o	Set property=value and/or temporary mount options (without '=').
2080  *
2081  * The import command scans for pools to import, and import pools based on pool
2082  * name and GUID.  The pool can also be renamed as part of the import process.
2083  */
2084 int
2085 zpool_do_import(int argc, char **argv)
2086 {
2087 	char **searchdirs = NULL;
2088 	int nsearch = 0;
2089 	int c;
2090 	int err = 0;
2091 	nvlist_t *pools = NULL;
2092 	boolean_t do_all = B_FALSE;
2093 	boolean_t do_destroyed = B_FALSE;
2094 	char *mntopts = NULL;
2095 	nvpair_t *elem;
2096 	nvlist_t *config;
2097 	uint64_t searchguid = 0;
2098 	char *searchname = NULL;
2099 	char *propval;
2100 	nvlist_t *found_config;
2101 	nvlist_t *policy = NULL;
2102 	nvlist_t *props = NULL;
2103 	boolean_t first;
2104 	int flags = ZFS_IMPORT_NORMAL;
2105 	uint32_t rewind_policy = ZPOOL_NO_REWIND;
2106 	boolean_t dryrun = B_FALSE;
2107 	boolean_t do_rewind = B_FALSE;
2108 	boolean_t xtreme_rewind = B_FALSE;
2109 	uint64_t pool_state, txg = -1ULL;
2110 	char *cachefile = NULL;
2111 	importargs_t idata = { 0 };
2112 	char *endptr;
2113 
2114 	/* check options */
2115 	while ((c = getopt(argc, argv, ":aCc:d:DEfFmnNo:rR:T:VX")) != -1) {
2116 		switch (c) {
2117 		case 'a':
2118 			do_all = B_TRUE;
2119 			break;
2120 		case 'c':
2121 			cachefile = optarg;
2122 			break;
2123 		case 'd':
2124 			if (searchdirs == NULL) {
2125 				searchdirs = safe_malloc(sizeof (char *));
2126 			} else {
2127 				char **tmp = safe_malloc((nsearch + 1) *
2128 				    sizeof (char *));
2129 				bcopy(searchdirs, tmp, nsearch *
2130 				    sizeof (char *));
2131 				free(searchdirs);
2132 				searchdirs = tmp;
2133 			}
2134 			searchdirs[nsearch++] = optarg;
2135 			break;
2136 		case 'D':
2137 			do_destroyed = B_TRUE;
2138 			break;
2139 		case 'f':
2140 			flags |= ZFS_IMPORT_ANY_HOST;
2141 			break;
2142 		case 'F':
2143 			do_rewind = B_TRUE;
2144 			break;
2145 		case 'm':
2146 			flags |= ZFS_IMPORT_MISSING_LOG;
2147 			break;
2148 		case 'n':
2149 			dryrun = B_TRUE;
2150 			break;
2151 		case 'N':
2152 			flags |= ZFS_IMPORT_ONLY;
2153 			break;
2154 		case 'o':
2155 			if ((propval = strchr(optarg, '=')) != NULL) {
2156 				*propval = '\0';
2157 				propval++;
2158 				if (add_prop_list(optarg, propval,
2159 				    &props, B_TRUE))
2160 					goto error;
2161 			} else {
2162 				mntopts = optarg;
2163 			}
2164 			break;
2165 		case 'R':
2166 			if (add_prop_list(zpool_prop_to_name(
2167 			    ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
2168 				goto error;
2169 			if (nvlist_lookup_string(props,
2170 			    zpool_prop_to_name(ZPOOL_PROP_CACHEFILE),
2171 			    &propval) == 0)
2172 				break;
2173 			if (add_prop_list(zpool_prop_to_name(
2174 			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
2175 				goto error;
2176 			break;
2177 		case 'T':
2178 			errno = 0;
2179 			txg = strtoull(optarg, &endptr, 0);
2180 			if (errno != 0 || *endptr != '\0') {
2181 				(void) fprintf(stderr,
2182 				    gettext("invalid txg value\n"));
2183 				usage(B_FALSE);
2184 			}
2185 			rewind_policy = ZPOOL_DO_REWIND | ZPOOL_EXTREME_REWIND;
2186 			break;
2187 		case 'V':
2188 			flags |= ZFS_IMPORT_VERBATIM;
2189 			break;
2190 		case 'X':
2191 			xtreme_rewind = B_TRUE;
2192 			break;
2193 		case ':':
2194 			(void) fprintf(stderr, gettext("missing argument for "
2195 			    "'%c' option\n"), optopt);
2196 			usage(B_FALSE);
2197 			break;
2198 		case '?':
2199 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
2200 			    optopt);
2201 			usage(B_FALSE);
2202 		}
2203 	}
2204 
2205 	argc -= optind;
2206 	argv += optind;
2207 
2208 	if (cachefile && nsearch != 0) {
2209 		(void) fprintf(stderr, gettext("-c is incompatible with -d\n"));
2210 		usage(B_FALSE);
2211 	}
2212 
2213 	if ((dryrun || xtreme_rewind) && !do_rewind) {
2214 		(void) fprintf(stderr,
2215 		    gettext("-n or -X only meaningful with -F\n"));
2216 		usage(B_FALSE);
2217 	}
2218 	if (dryrun)
2219 		rewind_policy = ZPOOL_TRY_REWIND;
2220 	else if (do_rewind)
2221 		rewind_policy = ZPOOL_DO_REWIND;
2222 	if (xtreme_rewind)
2223 		rewind_policy |= ZPOOL_EXTREME_REWIND;
2224 
2225 	/* In the future, we can capture further policy and include it here */
2226 	if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
2227 	    nvlist_add_uint64(policy, ZPOOL_REWIND_REQUEST_TXG, txg) != 0 ||
2228 	    nvlist_add_uint32(policy, ZPOOL_REWIND_REQUEST, rewind_policy) != 0)
2229 		goto error;
2230 
2231 	if (searchdirs == NULL) {
2232 		searchdirs = safe_malloc(sizeof (char *));
2233 		searchdirs[0] = ZFS_DISK_ROOT;
2234 		nsearch = 1;
2235 	}
2236 
2237 	/* check argument count */
2238 	if (do_all) {
2239 		if (argc != 0) {
2240 			(void) fprintf(stderr, gettext("too many arguments\n"));
2241 			usage(B_FALSE);
2242 		}
2243 	} else {
2244 		if (argc > 2) {
2245 			(void) fprintf(stderr, gettext("too many arguments\n"));
2246 			usage(B_FALSE);
2247 		}
2248 
2249 		/*
2250 		 * Check for the SYS_CONFIG privilege.  We do this explicitly
2251 		 * here because otherwise any attempt to discover pools will
2252 		 * silently fail.
2253 		 */
2254 		if (argc == 0 && !priv_ineffect(PRIV_SYS_CONFIG)) {
2255 			(void) fprintf(stderr, gettext("cannot "
2256 			    "discover pools: permission denied\n"));
2257 			free(searchdirs);
2258 			nvlist_free(policy);
2259 			return (1);
2260 		}
2261 	}
2262 
2263 	/*
2264 	 * Depending on the arguments given, we do one of the following:
2265 	 *
2266 	 *	<none>	Iterate through all pools and display information about
2267 	 *		each one.
2268 	 *
2269 	 *	-a	Iterate through all pools and try to import each one.
2270 	 *
2271 	 *	<id>	Find the pool that corresponds to the given GUID/pool
2272 	 *		name and import that one.
2273 	 *
2274 	 *	-D	Above options applies only to destroyed pools.
2275 	 */
2276 	if (argc != 0) {
2277 		char *endptr;
2278 
2279 		errno = 0;
2280 		searchguid = strtoull(argv[0], &endptr, 10);
2281 		if (errno != 0 || *endptr != '\0') {
2282 			searchname = argv[0];
2283 			searchguid = 0;
2284 		}
2285 		found_config = NULL;
2286 
2287 		/*
2288 		 * User specified a name or guid.  Ensure it's unique.
2289 		 */
2290 		idata.unique = B_TRUE;
2291 	}
2292 
2293 
2294 	idata.path = searchdirs;
2295 	idata.paths = nsearch;
2296 	idata.poolname = searchname;
2297 	idata.guid = searchguid;
2298 	idata.cachefile = cachefile;
2299 
2300 	pools = zpool_search_import(g_zfs, &idata);
2301 
2302 	if (pools != NULL && idata.exists &&
2303 	    (argc == 1 || strcmp(argv[0], argv[1]) == 0)) {
2304 		(void) fprintf(stderr, gettext("cannot import '%s': "
2305 		    "a pool with that name already exists\n"),
2306 		    argv[0]);
2307 		(void) fprintf(stderr, gettext("use the form '%s "
2308 		    "<pool | id> <newpool>' to give it a new name\n"),
2309 		    "zpool import");
2310 		err = 1;
2311 	} else if (pools == NULL && idata.exists) {
2312 		(void) fprintf(stderr, gettext("cannot import '%s': "
2313 		    "a pool with that name is already created/imported,\n"),
2314 		    argv[0]);
2315 		(void) fprintf(stderr, gettext("and no additional pools "
2316 		    "with that name were found\n"));
2317 		err = 1;
2318 	} else if (pools == NULL) {
2319 		if (argc != 0) {
2320 			(void) fprintf(stderr, gettext("cannot import '%s': "
2321 			    "no such pool available\n"), argv[0]);
2322 		}
2323 		err = 1;
2324 	}
2325 
2326 	if (err == 1) {
2327 		free(searchdirs);
2328 		nvlist_free(policy);
2329 		return (1);
2330 	}
2331 
2332 	/*
2333 	 * At this point we have a list of import candidate configs. Even if
2334 	 * we were searching by pool name or guid, we still need to
2335 	 * post-process the list to deal with pool state and possible
2336 	 * duplicate names.
2337 	 */
2338 	err = 0;
2339 	elem = NULL;
2340 	first = B_TRUE;
2341 	while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) {
2342 
2343 		verify(nvpair_value_nvlist(elem, &config) == 0);
2344 
2345 		verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
2346 		    &pool_state) == 0);
2347 		if (!do_destroyed && pool_state == POOL_STATE_DESTROYED)
2348 			continue;
2349 		if (do_destroyed && pool_state != POOL_STATE_DESTROYED)
2350 			continue;
2351 
2352 		verify(nvlist_add_nvlist(config, ZPOOL_REWIND_POLICY,
2353 		    policy) == 0);
2354 
2355 		if (argc == 0) {
2356 			if (first)
2357 				first = B_FALSE;
2358 			else if (!do_all)
2359 				(void) printf("\n");
2360 
2361 			if (do_all) {
2362 				err |= do_import(config, NULL, mntopts,
2363 				    props, flags);
2364 			} else {
2365 				show_import(config);
2366 			}
2367 		} else if (searchname != NULL) {
2368 			char *name;
2369 
2370 			/*
2371 			 * We are searching for a pool based on name.
2372 			 */
2373 			verify(nvlist_lookup_string(config,
2374 			    ZPOOL_CONFIG_POOL_NAME, &name) == 0);
2375 
2376 			if (strcmp(name, searchname) == 0) {
2377 				if (found_config != NULL) {
2378 					(void) fprintf(stderr, gettext(
2379 					    "cannot import '%s': more than "
2380 					    "one matching pool\n"), searchname);
2381 					(void) fprintf(stderr, gettext(
2382 					    "import by numeric ID instead\n"));
2383 					err = B_TRUE;
2384 				}
2385 				found_config = config;
2386 			}
2387 		} else {
2388 			uint64_t guid;
2389 
2390 			/*
2391 			 * Search for a pool by guid.
2392 			 */
2393 			verify(nvlist_lookup_uint64(config,
2394 			    ZPOOL_CONFIG_POOL_GUID, &guid) == 0);
2395 
2396 			if (guid == searchguid)
2397 				found_config = config;
2398 		}
2399 	}
2400 
2401 	/*
2402 	 * If we were searching for a specific pool, verify that we found a
2403 	 * pool, and then do the import.
2404 	 */
2405 	if (argc != 0 && err == 0) {
2406 		if (found_config == NULL) {
2407 			(void) fprintf(stderr, gettext("cannot import '%s': "
2408 			    "no such pool available\n"), argv[0]);
2409 			err = B_TRUE;
2410 		} else {
2411 			err |= do_import(found_config, argc == 1 ? NULL :
2412 			    argv[1], mntopts, props, flags);
2413 		}
2414 	}
2415 
2416 	/*
2417 	 * If we were just looking for pools, report an error if none were
2418 	 * found.
2419 	 */
2420 	if (argc == 0 && first)
2421 		(void) fprintf(stderr,
2422 		    gettext("no pools available to import\n"));
2423 
2424 error:
2425 	nvlist_free(props);
2426 	nvlist_free(pools);
2427 	nvlist_free(policy);
2428 	free(searchdirs);
2429 
2430 	return (err ? 1 : 0);
2431 }
2432 
2433 typedef struct iostat_cbdata {
2434 	boolean_t cb_verbose;
2435 	int cb_namewidth;
2436 	int cb_iteration;
2437 	zpool_list_t *cb_list;
2438 } iostat_cbdata_t;
2439 
2440 static void
2441 print_iostat_separator(iostat_cbdata_t *cb)
2442 {
2443 	int i = 0;
2444 
2445 	for (i = 0; i < cb->cb_namewidth; i++)
2446 		(void) printf("-");
2447 	(void) printf("  -----  -----  -----  -----  -----  -----\n");
2448 }
2449 
2450 static void
2451 print_iostat_header(iostat_cbdata_t *cb)
2452 {
2453 	(void) printf("%*s     capacity     operations    bandwidth\n",
2454 	    cb->cb_namewidth, "");
2455 	(void) printf("%-*s  alloc   free   read  write   read  write\n",
2456 	    cb->cb_namewidth, "pool");
2457 	print_iostat_separator(cb);
2458 }
2459 
2460 /*
2461  * Display a single statistic.
2462  */
2463 static void
2464 print_one_stat(uint64_t value)
2465 {
2466 	char buf[64];
2467 
2468 	zfs_nicenum(value, buf, sizeof (buf));
2469 	(void) printf("  %5s", buf);
2470 }
2471 
2472 /*
2473  * Print out all the statistics for the given vdev.  This can either be the
2474  * toplevel configuration, or called recursively.  If 'name' is NULL, then this
2475  * is a verbose output, and we don't want to display the toplevel pool stats.
2476  */
2477 void
2478 print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
2479     nvlist_t *newnv, iostat_cbdata_t *cb, int depth)
2480 {
2481 	nvlist_t **oldchild, **newchild;
2482 	uint_t c, children;
2483 	vdev_stat_t *oldvs, *newvs;
2484 	vdev_stat_t zerovs = { 0 };
2485 	uint64_t tdelta;
2486 	double scale;
2487 	char *vname;
2488 
2489 	if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
2490 		return;
2491 
2492 	if (oldnv != NULL) {
2493 		verify(nvlist_lookup_uint64_array(oldnv,
2494 		    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&oldvs, &c) == 0);
2495 	} else {
2496 		oldvs = &zerovs;
2497 	}
2498 
2499 	verify(nvlist_lookup_uint64_array(newnv, ZPOOL_CONFIG_VDEV_STATS,
2500 	    (uint64_t **)&newvs, &c) == 0);
2501 
2502 	if (strlen(name) + depth > cb->cb_namewidth)
2503 		(void) printf("%*s%s", depth, "", name);
2504 	else
2505 		(void) printf("%*s%s%*s", depth, "", name,
2506 		    (int)(cb->cb_namewidth - strlen(name) - depth), "");
2507 
2508 	tdelta = newvs->vs_timestamp - oldvs->vs_timestamp;
2509 
2510 	if (tdelta == 0)
2511 		scale = 1.0;
2512 	else
2513 		scale = (double)NANOSEC / tdelta;
2514 
2515 	/* only toplevel vdevs have capacity stats */
2516 	if (newvs->vs_space == 0) {
2517 		(void) printf("      -      -");
2518 	} else {
2519 		print_one_stat(newvs->vs_alloc);
2520 		print_one_stat(newvs->vs_space - newvs->vs_alloc);
2521 	}
2522 
2523 	print_one_stat((uint64_t)(scale * (newvs->vs_ops[ZIO_TYPE_READ] -
2524 	    oldvs->vs_ops[ZIO_TYPE_READ])));
2525 
2526 	print_one_stat((uint64_t)(scale * (newvs->vs_ops[ZIO_TYPE_WRITE] -
2527 	    oldvs->vs_ops[ZIO_TYPE_WRITE])));
2528 
2529 	print_one_stat((uint64_t)(scale * (newvs->vs_bytes[ZIO_TYPE_READ] -
2530 	    oldvs->vs_bytes[ZIO_TYPE_READ])));
2531 
2532 	print_one_stat((uint64_t)(scale * (newvs->vs_bytes[ZIO_TYPE_WRITE] -
2533 	    oldvs->vs_bytes[ZIO_TYPE_WRITE])));
2534 
2535 	(void) printf("\n");
2536 
2537 	if (!cb->cb_verbose)
2538 		return;
2539 
2540 	if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_CHILDREN,
2541 	    &newchild, &children) != 0)
2542 		return;
2543 
2544 	if (oldnv && nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_CHILDREN,
2545 	    &oldchild, &c) != 0)
2546 		return;
2547 
2548 	for (c = 0; c < children; c++) {
2549 		uint64_t ishole = B_FALSE, islog = B_FALSE;
2550 
2551 		(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_HOLE,
2552 		    &ishole);
2553 
2554 		(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_LOG,
2555 		    &islog);
2556 
2557 		if (ishole || islog)
2558 			continue;
2559 
2560 		vname = zpool_vdev_name(g_zfs, zhp, newchild[c], B_FALSE);
2561 		print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL,
2562 		    newchild[c], cb, depth + 2);
2563 		free(vname);
2564 	}
2565 
2566 	/*
2567 	 * Log device section
2568 	 */
2569 
2570 	if (num_logs(newnv) > 0) {
2571 		(void) printf("%-*s      -      -      -      -      -      "
2572 		    "-\n", cb->cb_namewidth, "logs");
2573 
2574 		for (c = 0; c < children; c++) {
2575 			uint64_t islog = B_FALSE;
2576 			(void) nvlist_lookup_uint64(newchild[c],
2577 			    ZPOOL_CONFIG_IS_LOG, &islog);
2578 
2579 			if (islog) {
2580 				vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
2581 				    B_FALSE);
2582 				print_vdev_stats(zhp, vname, oldnv ?
2583 				    oldchild[c] : NULL, newchild[c],
2584 				    cb, depth + 2);
2585 				free(vname);
2586 			}
2587 		}
2588 
2589 	}
2590 
2591 	/*
2592 	 * Include level 2 ARC devices in iostat output
2593 	 */
2594 	if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_L2CACHE,
2595 	    &newchild, &children) != 0)
2596 		return;
2597 
2598 	if (oldnv && nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_L2CACHE,
2599 	    &oldchild, &c) != 0)
2600 		return;
2601 
2602 	if (children > 0) {
2603 		(void) printf("%-*s      -      -      -      -      -      "
2604 		    "-\n", cb->cb_namewidth, "cache");
2605 		for (c = 0; c < children; c++) {
2606 			vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
2607 			    B_FALSE);
2608 			print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL,
2609 			    newchild[c], cb, depth + 2);
2610 			free(vname);
2611 		}
2612 	}
2613 }
2614 
2615 static int
2616 refresh_iostat(zpool_handle_t *zhp, void *data)
2617 {
2618 	iostat_cbdata_t *cb = data;
2619 	boolean_t missing;
2620 
2621 	/*
2622 	 * If the pool has disappeared, remove it from the list and continue.
2623 	 */
2624 	if (zpool_refresh_stats(zhp, &missing) != 0)
2625 		return (-1);
2626 
2627 	if (missing)
2628 		pool_list_remove(cb->cb_list, zhp);
2629 
2630 	return (0);
2631 }
2632 
2633 /*
2634  * Callback to print out the iostats for the given pool.
2635  */
2636 int
2637 print_iostat(zpool_handle_t *zhp, void *data)
2638 {
2639 	iostat_cbdata_t *cb = data;
2640 	nvlist_t *oldconfig, *newconfig;
2641 	nvlist_t *oldnvroot, *newnvroot;
2642 
2643 	newconfig = zpool_get_config(zhp, &oldconfig);
2644 
2645 	if (cb->cb_iteration == 1)
2646 		oldconfig = NULL;
2647 
2648 	verify(nvlist_lookup_nvlist(newconfig, ZPOOL_CONFIG_VDEV_TREE,
2649 	    &newnvroot) == 0);
2650 
2651 	if (oldconfig == NULL)
2652 		oldnvroot = NULL;
2653 	else
2654 		verify(nvlist_lookup_nvlist(oldconfig, ZPOOL_CONFIG_VDEV_TREE,
2655 		    &oldnvroot) == 0);
2656 
2657 	/*
2658 	 * Print out the statistics for the pool.
2659 	 */
2660 	print_vdev_stats(zhp, zpool_get_name(zhp), oldnvroot, newnvroot, cb, 0);
2661 
2662 	if (cb->cb_verbose)
2663 		print_iostat_separator(cb);
2664 
2665 	return (0);
2666 }
2667 
2668 int
2669 get_namewidth(zpool_handle_t *zhp, void *data)
2670 {
2671 	iostat_cbdata_t *cb = data;
2672 	nvlist_t *config, *nvroot;
2673 
2674 	if ((config = zpool_get_config(zhp, NULL)) != NULL) {
2675 		verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
2676 		    &nvroot) == 0);
2677 		if (!cb->cb_verbose)
2678 			cb->cb_namewidth = strlen(zpool_get_name(zhp));
2679 		else
2680 			cb->cb_namewidth = max_width(zhp, nvroot, 0,
2681 			    cb->cb_namewidth);
2682 	}
2683 
2684 	/*
2685 	 * The width must fall into the range [10,38].  The upper limit is the
2686 	 * maximum we can have and still fit in 80 columns.
2687 	 */
2688 	if (cb->cb_namewidth < 10)
2689 		cb->cb_namewidth = 10;
2690 	if (cb->cb_namewidth > 38)
2691 		cb->cb_namewidth = 38;
2692 
2693 	return (0);
2694 }
2695 
2696 /*
2697  * Parse the input string, get the 'interval' and 'count' value if there is one.
2698  */
2699 static void
2700 get_interval_count(int *argcp, char **argv, unsigned long *iv,
2701     unsigned long *cnt)
2702 {
2703 	unsigned long interval = 0, count = 0;
2704 	int argc = *argcp, errno;
2705 
2706 	/*
2707 	 * Determine if the last argument is an integer or a pool name
2708 	 */
2709 	if (argc > 0 && isdigit(argv[argc - 1][0])) {
2710 		char *end;
2711 
2712 		errno = 0;
2713 		interval = strtoul(argv[argc - 1], &end, 10);
2714 
2715 		if (*end == '\0' && errno == 0) {
2716 			if (interval == 0) {
2717 				(void) fprintf(stderr, gettext("interval "
2718 				    "cannot be zero\n"));
2719 				usage(B_FALSE);
2720 			}
2721 			/*
2722 			 * Ignore the last parameter
2723 			 */
2724 			argc--;
2725 		} else {
2726 			/*
2727 			 * If this is not a valid number, just plow on.  The
2728 			 * user will get a more informative error message later
2729 			 * on.
2730 			 */
2731 			interval = 0;
2732 		}
2733 	}
2734 
2735 	/*
2736 	 * If the last argument is also an integer, then we have both a count
2737 	 * and an interval.
2738 	 */
2739 	if (argc > 0 && isdigit(argv[argc - 1][0])) {
2740 		char *end;
2741 
2742 		errno = 0;
2743 		count = interval;
2744 		interval = strtoul(argv[argc - 1], &end, 10);
2745 
2746 		if (*end == '\0' && errno == 0) {
2747 			if (interval == 0) {
2748 				(void) fprintf(stderr, gettext("interval "
2749 				    "cannot be zero\n"));
2750 				usage(B_FALSE);
2751 			}
2752 
2753 			/*
2754 			 * Ignore the last parameter
2755 			 */
2756 			argc--;
2757 		} else {
2758 			interval = 0;
2759 		}
2760 	}
2761 
2762 	*iv = interval;
2763 	*cnt = count;
2764 	*argcp = argc;
2765 }
2766 
2767 static void
2768 get_timestamp_arg(char c)
2769 {
2770 	if (c == 'u')
2771 		timestamp_fmt = UDATE;
2772 	else if (c == 'd')
2773 		timestamp_fmt = DDATE;
2774 	else
2775 		usage(B_FALSE);
2776 }
2777 
2778 /*
2779  * zpool iostat [-v] [-T d|u] [pool] ... [interval [count]]
2780  *
2781  *	-v	Display statistics for individual vdevs
2782  *	-T	Display a timestamp in date(1) or Unix format
2783  *
2784  * This command can be tricky because we want to be able to deal with pool
2785  * creation/destruction as well as vdev configuration changes.  The bulk of this
2786  * processing is handled by the pool_list_* routines in zpool_iter.c.  We rely
2787  * on pool_list_update() to detect the addition of new pools.  Configuration
2788  * changes are all handled within libzfs.
2789  */
2790 int
2791 zpool_do_iostat(int argc, char **argv)
2792 {
2793 	int c;
2794 	int ret;
2795 	int npools;
2796 	unsigned long interval = 0, count = 0;
2797 	zpool_list_t *list;
2798 	boolean_t verbose = B_FALSE;
2799 	iostat_cbdata_t cb;
2800 
2801 	/* check options */
2802 	while ((c = getopt(argc, argv, "T:v")) != -1) {
2803 		switch (c) {
2804 		case 'T':
2805 			get_timestamp_arg(*optarg);
2806 			break;
2807 		case 'v':
2808 			verbose = B_TRUE;
2809 			break;
2810 		case '?':
2811 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
2812 			    optopt);
2813 			usage(B_FALSE);
2814 		}
2815 	}
2816 
2817 	argc -= optind;
2818 	argv += optind;
2819 
2820 	get_interval_count(&argc, argv, &interval, &count);
2821 
2822 	/*
2823 	 * Construct the list of all interesting pools.
2824 	 */
2825 	ret = 0;
2826 	if ((list = pool_list_get(argc, argv, NULL, &ret)) == NULL)
2827 		return (1);
2828 
2829 	if (pool_list_count(list) == 0 && argc != 0) {
2830 		pool_list_free(list);
2831 		return (1);
2832 	}
2833 
2834 	if (pool_list_count(list) == 0 && interval == 0) {
2835 		pool_list_free(list);
2836 		(void) fprintf(stderr, gettext("no pools available\n"));
2837 		return (1);
2838 	}
2839 
2840 	/*
2841 	 * Enter the main iostat loop.
2842 	 */
2843 	cb.cb_list = list;
2844 	cb.cb_verbose = verbose;
2845 	cb.cb_iteration = 0;
2846 	cb.cb_namewidth = 0;
2847 
2848 	for (;;) {
2849 		pool_list_update(list);
2850 
2851 		if ((npools = pool_list_count(list)) == 0)
2852 			break;
2853 
2854 		/*
2855 		 * Refresh all statistics.  This is done as an explicit step
2856 		 * before calculating the maximum name width, so that any
2857 		 * configuration changes are properly accounted for.
2858 		 */
2859 		(void) pool_list_iter(list, B_FALSE, refresh_iostat, &cb);
2860 
2861 		/*
2862 		 * Iterate over all pools to determine the maximum width
2863 		 * for the pool / device name column across all pools.
2864 		 */
2865 		cb.cb_namewidth = 0;
2866 		(void) pool_list_iter(list, B_FALSE, get_namewidth, &cb);
2867 
2868 		if (timestamp_fmt != NODATE)
2869 			print_timestamp(timestamp_fmt);
2870 
2871 		/*
2872 		 * If it's the first time, or verbose mode, print the header.
2873 		 */
2874 		if (++cb.cb_iteration == 1 || verbose)
2875 			print_iostat_header(&cb);
2876 
2877 		(void) pool_list_iter(list, B_FALSE, print_iostat, &cb);
2878 
2879 		/*
2880 		 * If there's more than one pool, and we're not in verbose mode
2881 		 * (which prints a separator for us), then print a separator.
2882 		 */
2883 		if (npools > 1 && !verbose)
2884 			print_iostat_separator(&cb);
2885 
2886 		if (verbose)
2887 			(void) printf("\n");
2888 
2889 		/*
2890 		 * Flush the output so that redirection to a file isn't buffered
2891 		 * indefinitely.
2892 		 */
2893 		(void) fflush(stdout);
2894 
2895 		if (interval == 0)
2896 			break;
2897 
2898 		if (count != 0 && --count == 0)
2899 			break;
2900 
2901 		(void) sleep(interval);
2902 	}
2903 
2904 	pool_list_free(list);
2905 
2906 	return (ret);
2907 }
2908 
2909 typedef struct list_cbdata {
2910 	boolean_t	cb_verbose;
2911 	int		cb_namewidth;
2912 	boolean_t	cb_scripted;
2913 	zprop_list_t	*cb_proplist;
2914 	boolean_t	cb_literal;
2915 } list_cbdata_t;
2916 
2917 /*
2918  * Given a list of columns to display, output appropriate headers for each one.
2919  */
2920 static void
2921 print_header(list_cbdata_t *cb)
2922 {
2923 	zprop_list_t *pl = cb->cb_proplist;
2924 	char headerbuf[ZPOOL_MAXPROPLEN];
2925 	const char *header;
2926 	boolean_t first = B_TRUE;
2927 	boolean_t right_justify;
2928 	size_t width = 0;
2929 
2930 	for (; pl != NULL; pl = pl->pl_next) {
2931 		width = pl->pl_width;
2932 		if (first && cb->cb_verbose) {
2933 			/*
2934 			 * Reset the width to accommodate the verbose listing
2935 			 * of devices.
2936 			 */
2937 			width = cb->cb_namewidth;
2938 		}
2939 
2940 		if (!first)
2941 			(void) printf("  ");
2942 		else
2943 			first = B_FALSE;
2944 
2945 		right_justify = B_FALSE;
2946 		if (pl->pl_prop != ZPROP_INVAL) {
2947 			header = zpool_prop_column_name(pl->pl_prop);
2948 			right_justify = zpool_prop_align_right(pl->pl_prop);
2949 		} else {
2950 			int i;
2951 
2952 			for (i = 0; pl->pl_user_prop[i] != '\0'; i++)
2953 				headerbuf[i] = toupper(pl->pl_user_prop[i]);
2954 			headerbuf[i] = '\0';
2955 			header = headerbuf;
2956 		}
2957 
2958 		if (pl->pl_next == NULL && !right_justify)
2959 			(void) printf("%s", header);
2960 		else if (right_justify)
2961 			(void) printf("%*s", width, header);
2962 		else
2963 			(void) printf("%-*s", width, header);
2964 
2965 	}
2966 
2967 	(void) printf("\n");
2968 }
2969 
2970 /*
2971  * Given a pool and a list of properties, print out all the properties according
2972  * to the described layout.
2973  */
2974 static void
2975 print_pool(zpool_handle_t *zhp, list_cbdata_t *cb)
2976 {
2977 	zprop_list_t *pl = cb->cb_proplist;
2978 	boolean_t first = B_TRUE;
2979 	char property[ZPOOL_MAXPROPLEN];
2980 	char *propstr;
2981 	boolean_t right_justify;
2982 	size_t width;
2983 
2984 	for (; pl != NULL; pl = pl->pl_next) {
2985 
2986 		width = pl->pl_width;
2987 		if (first && cb->cb_verbose) {
2988 			/*
2989 			 * Reset the width to accommodate the verbose listing
2990 			 * of devices.
2991 			 */
2992 			width = cb->cb_namewidth;
2993 		}
2994 
2995 		if (!first) {
2996 			if (cb->cb_scripted)
2997 				(void) printf("\t");
2998 			else
2999 				(void) printf("  ");
3000 		} else {
3001 			first = B_FALSE;
3002 		}
3003 
3004 		right_justify = B_FALSE;
3005 		if (pl->pl_prop != ZPROP_INVAL) {
3006 			if (zpool_get_prop(zhp, pl->pl_prop, property,
3007 			    sizeof (property), NULL, cb->cb_literal) != 0)
3008 				propstr = "-";
3009 			else
3010 				propstr = property;
3011 
3012 			right_justify = zpool_prop_align_right(pl->pl_prop);
3013 		} else if ((zpool_prop_feature(pl->pl_user_prop) ||
3014 		    zpool_prop_unsupported(pl->pl_user_prop)) &&
3015 		    zpool_prop_get_feature(zhp, pl->pl_user_prop, property,
3016 		    sizeof (property)) == 0) {
3017 			propstr = property;
3018 		} else {
3019 			propstr = "-";
3020 		}
3021 
3022 
3023 		/*
3024 		 * If this is being called in scripted mode, or if this is the
3025 		 * last column and it is left-justified, don't include a width
3026 		 * format specifier.
3027 		 */
3028 		if (cb->cb_scripted || (pl->pl_next == NULL && !right_justify))
3029 			(void) printf("%s", propstr);
3030 		else if (right_justify)
3031 			(void) printf("%*s", width, propstr);
3032 		else
3033 			(void) printf("%-*s", width, propstr);
3034 	}
3035 
3036 	(void) printf("\n");
3037 }
3038 
3039 static void
3040 print_one_column(zpool_prop_t prop, uint64_t value, boolean_t scripted,
3041     boolean_t valid)
3042 {
3043 	char propval[64];
3044 	boolean_t fixed;
3045 	size_t width = zprop_width(prop, &fixed, ZFS_TYPE_POOL);
3046 
3047 	switch (prop) {
3048 	case ZPOOL_PROP_EXPANDSZ:
3049 		if (value == 0)
3050 			(void) strlcpy(propval, "-", sizeof (propval));
3051 		else
3052 			zfs_nicenum(value, propval, sizeof (propval));
3053 		break;
3054 	case ZPOOL_PROP_FRAGMENTATION:
3055 		if (value == ZFS_FRAG_INVALID) {
3056 			(void) strlcpy(propval, "-", sizeof (propval));
3057 		} else {
3058 			(void) snprintf(propval, sizeof (propval), "%llu%%",
3059 			    value);
3060 		}
3061 		break;
3062 	case ZPOOL_PROP_CAPACITY:
3063 		(void) snprintf(propval, sizeof (propval), "%llu%%", value);
3064 		break;
3065 	default:
3066 		zfs_nicenum(value, propval, sizeof (propval));
3067 	}
3068 
3069 	if (!valid)
3070 		(void) strlcpy(propval, "-", sizeof (propval));
3071 
3072 	if (scripted)
3073 		(void) printf("\t%s", propval);
3074 	else
3075 		(void) printf("  %*s", width, propval);
3076 }
3077 
3078 void
3079 print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
3080     list_cbdata_t *cb, int depth)
3081 {
3082 	nvlist_t **child;
3083 	vdev_stat_t *vs;
3084 	uint_t c, children;
3085 	char *vname;
3086 	boolean_t scripted = cb->cb_scripted;
3087 	uint64_t islog = B_FALSE;
3088 	boolean_t haslog = B_FALSE;
3089 	char *dashes = "%-*s      -      -      -         -      -      -\n";
3090 
3091 	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
3092 	    (uint64_t **)&vs, &c) == 0);
3093 
3094 	if (name != NULL) {
3095 		boolean_t toplevel = (vs->vs_space != 0);
3096 		uint64_t cap;
3097 
3098 		if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
3099 			return;
3100 
3101 		if (scripted)
3102 			(void) printf("\t%s", name);
3103 		else if (strlen(name) + depth > cb->cb_namewidth)
3104 			(void) printf("%*s%s", depth, "", name);
3105 		else
3106 			(void) printf("%*s%s%*s", depth, "", name,
3107 			    (int)(cb->cb_namewidth - strlen(name) - depth), "");
3108 
3109 		/*
3110 		 * Print the properties for the individual vdevs. Some
3111 		 * properties are only applicable to toplevel vdevs. The
3112 		 * 'toplevel' boolean value is passed to the print_one_column()
3113 		 * to indicate that the value is valid.
3114 		 */
3115 		print_one_column(ZPOOL_PROP_SIZE, vs->vs_space, scripted,
3116 		    toplevel);
3117 		print_one_column(ZPOOL_PROP_ALLOCATED, vs->vs_alloc, scripted,
3118 		    toplevel);
3119 		print_one_column(ZPOOL_PROP_FREE, vs->vs_space - vs->vs_alloc,
3120 		    scripted, toplevel);
3121 		print_one_column(ZPOOL_PROP_EXPANDSZ, vs->vs_esize, scripted,
3122 		    B_TRUE);
3123 		print_one_column(ZPOOL_PROP_FRAGMENTATION,
3124 		    vs->vs_fragmentation, scripted,
3125 		    (vs->vs_fragmentation != ZFS_FRAG_INVALID && toplevel));
3126 		cap = (vs->vs_space == 0) ? 0 :
3127 		    (vs->vs_alloc * 100 / vs->vs_space);
3128 		print_one_column(ZPOOL_PROP_CAPACITY, cap, scripted, toplevel);
3129 		(void) printf("\n");
3130 	}
3131 
3132 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
3133 	    &child, &children) != 0)
3134 		return;
3135 
3136 	for (c = 0; c < children; c++) {
3137 		uint64_t ishole = B_FALSE;
3138 
3139 		if (nvlist_lookup_uint64(child[c],
3140 		    ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
3141 			continue;
3142 
3143 		if (nvlist_lookup_uint64(child[c],
3144 		    ZPOOL_CONFIG_IS_LOG, &islog) == 0 && islog) {
3145 			haslog = B_TRUE;
3146 			continue;
3147 		}
3148 
3149 		vname = zpool_vdev_name(g_zfs, zhp, child[c], B_FALSE);
3150 		print_list_stats(zhp, vname, child[c], cb, depth + 2);
3151 		free(vname);
3152 	}
3153 
3154 	if (haslog == B_TRUE) {
3155 		/* LINTED E_SEC_PRINTF_VAR_FMT */
3156 		(void) printf(dashes, cb->cb_namewidth, "log");
3157 		for (c = 0; c < children; c++) {
3158 			if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
3159 			    &islog) != 0 || !islog)
3160 				continue;
3161 			vname = zpool_vdev_name(g_zfs, zhp, child[c], B_FALSE);
3162 			print_list_stats(zhp, vname, child[c], cb, depth + 2);
3163 			free(vname);
3164 		}
3165 	}
3166 
3167 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
3168 	    &child, &children) == 0 && children > 0) {
3169 		/* LINTED E_SEC_PRINTF_VAR_FMT */
3170 		(void) printf(dashes, cb->cb_namewidth, "cache");
3171 		for (c = 0; c < children; c++) {
3172 			vname = zpool_vdev_name(g_zfs, zhp, child[c], B_FALSE);
3173 			print_list_stats(zhp, vname, child[c], cb, depth + 2);
3174 			free(vname);
3175 		}
3176 	}
3177 
3178 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES, &child,
3179 	    &children) == 0 && children > 0) {
3180 		/* LINTED E_SEC_PRINTF_VAR_FMT */
3181 		(void) printf(dashes, cb->cb_namewidth, "spare");
3182 		for (c = 0; c < children; c++) {
3183 			vname = zpool_vdev_name(g_zfs, zhp, child[c], B_FALSE);
3184 			print_list_stats(zhp, vname, child[c], cb, depth + 2);
3185 			free(vname);
3186 		}
3187 	}
3188 }
3189 
3190 
3191 /*
3192  * Generic callback function to list a pool.
3193  */
3194 int
3195 list_callback(zpool_handle_t *zhp, void *data)
3196 {
3197 	list_cbdata_t *cbp = data;
3198 	nvlist_t *config;
3199 	nvlist_t *nvroot;
3200 
3201 	config = zpool_get_config(zhp, NULL);
3202 
3203 	print_pool(zhp, cbp);
3204 	if (!cbp->cb_verbose)
3205 		return (0);
3206 
3207 	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
3208 	    &nvroot) == 0);
3209 	print_list_stats(zhp, NULL, nvroot, cbp, 0);
3210 
3211 	return (0);
3212 }
3213 
3214 /*
3215  * zpool list [-Hp] [-o prop[,prop]*] [-T d|u] [pool] ... [interval [count]]
3216  *
3217  *	-H	Scripted mode.  Don't display headers, and separate properties
3218  *		by a single tab.
3219  *	-o	List of properties to display.  Defaults to
3220  *		"name,size,allocated,free,expandsize,fragmentation,capacity,"
3221  *		"dedupratio,health,altroot"
3222  * 	-p	Diplay values in parsable (exact) format.
3223  *	-T	Display a timestamp in date(1) or Unix format
3224  *
3225  * List all pools in the system, whether or not they're healthy.  Output space
3226  * statistics for each one, as well as health status summary.
3227  */
3228 int
3229 zpool_do_list(int argc, char **argv)
3230 {
3231 	int c;
3232 	int ret;
3233 	list_cbdata_t cb = { 0 };
3234 	static char default_props[] =
3235 	    "name,size,allocated,free,expandsize,fragmentation,capacity,"
3236 	    "dedupratio,health,altroot";
3237 	char *props = default_props;
3238 	unsigned long interval = 0, count = 0;
3239 	zpool_list_t *list;
3240 	boolean_t first = B_TRUE;
3241 
3242 	/* check options */
3243 	while ((c = getopt(argc, argv, ":Ho:pT:v")) != -1) {
3244 		switch (c) {
3245 		case 'H':
3246 			cb.cb_scripted = B_TRUE;
3247 			break;
3248 		case 'o':
3249 			props = optarg;
3250 			break;
3251 		case 'p':
3252 			cb.cb_literal = B_TRUE;
3253 			break;
3254 		case 'T':
3255 			get_timestamp_arg(*optarg);
3256 			break;
3257 		case 'v':
3258 			cb.cb_verbose = B_TRUE;
3259 			break;
3260 		case ':':
3261 			(void) fprintf(stderr, gettext("missing argument for "
3262 			    "'%c' option\n"), optopt);
3263 			usage(B_FALSE);
3264 			break;
3265 		case '?':
3266 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3267 			    optopt);
3268 			usage(B_FALSE);
3269 		}
3270 	}
3271 
3272 	argc -= optind;
3273 	argv += optind;
3274 
3275 	get_interval_count(&argc, argv, &interval, &count);
3276 
3277 	if (zprop_get_list(g_zfs, props, &cb.cb_proplist, ZFS_TYPE_POOL) != 0)
3278 		usage(B_FALSE);
3279 
3280 	for (;;) {
3281 		if ((list = pool_list_get(argc, argv, &cb.cb_proplist,
3282 		    &ret)) == NULL)
3283 			return (1);
3284 
3285 		if (pool_list_count(list) == 0)
3286 			break;
3287 
3288 		cb.cb_namewidth = 0;
3289 		(void) pool_list_iter(list, B_FALSE, get_namewidth, &cb);
3290 
3291 		if (timestamp_fmt != NODATE)
3292 			print_timestamp(timestamp_fmt);
3293 
3294 		if (!cb.cb_scripted && (first || cb.cb_verbose)) {
3295 			print_header(&cb);
3296 			first = B_FALSE;
3297 		}
3298 		ret = pool_list_iter(list, B_TRUE, list_callback, &cb);
3299 
3300 		if (interval == 0)
3301 			break;
3302 
3303 		if (count != 0 && --count == 0)
3304 			break;
3305 
3306 		pool_list_free(list);
3307 		(void) sleep(interval);
3308 	}
3309 
3310 	if (argc == 0 && !cb.cb_scripted && pool_list_count(list) == 0) {
3311 		(void) printf(gettext("no pools available\n"));
3312 		ret = 0;
3313 	}
3314 
3315 	pool_list_free(list);
3316 	zprop_free_list(cb.cb_proplist);
3317 	return (ret);
3318 }
3319 
3320 static int
3321 zpool_do_attach_or_replace(int argc, char **argv, int replacing)
3322 {
3323 	boolean_t force = B_FALSE;
3324 	int c;
3325 	nvlist_t *nvroot;
3326 	char *poolname, *old_disk, *new_disk;
3327 	zpool_handle_t *zhp;
3328 	zpool_boot_label_t boot_type;
3329 	uint64_t boot_size;
3330 	int ret;
3331 
3332 	/* check options */
3333 	while ((c = getopt(argc, argv, "f")) != -1) {
3334 		switch (c) {
3335 		case 'f':
3336 			force = B_TRUE;
3337 			break;
3338 		case '?':
3339 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3340 			    optopt);
3341 			usage(B_FALSE);
3342 		}
3343 	}
3344 
3345 	argc -= optind;
3346 	argv += optind;
3347 
3348 	/* get pool name and check number of arguments */
3349 	if (argc < 1) {
3350 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
3351 		usage(B_FALSE);
3352 	}
3353 
3354 	poolname = argv[0];
3355 
3356 	if (argc < 2) {
3357 		(void) fprintf(stderr,
3358 		    gettext("missing <device> specification\n"));
3359 		usage(B_FALSE);
3360 	}
3361 
3362 	old_disk = argv[1];
3363 
3364 	if (argc < 3) {
3365 		if (!replacing) {
3366 			(void) fprintf(stderr,
3367 			    gettext("missing <new_device> specification\n"));
3368 			usage(B_FALSE);
3369 		}
3370 		new_disk = old_disk;
3371 		argc -= 1;
3372 		argv += 1;
3373 	} else {
3374 		new_disk = argv[2];
3375 		argc -= 2;
3376 		argv += 2;
3377 	}
3378 
3379 	if (argc > 1) {
3380 		(void) fprintf(stderr, gettext("too many arguments\n"));
3381 		usage(B_FALSE);
3382 	}
3383 
3384 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
3385 		return (1);
3386 
3387 	if (zpool_get_config(zhp, NULL) == NULL) {
3388 		(void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
3389 		    poolname);
3390 		zpool_close(zhp);
3391 		return (1);
3392 	}
3393 
3394 	if (zpool_is_bootable(zhp))
3395 		boot_type = ZPOOL_COPY_BOOT_LABEL;
3396 	else
3397 		boot_type = ZPOOL_NO_BOOT_LABEL;
3398 
3399 	boot_size = zpool_get_prop_int(zhp, ZPOOL_PROP_BOOTSIZE, NULL);
3400 	nvroot = make_root_vdev(zhp, force, B_FALSE, replacing, B_FALSE,
3401 	    boot_type, boot_size, argc, argv);
3402 	if (nvroot == NULL) {
3403 		zpool_close(zhp);
3404 		return (1);
3405 	}
3406 
3407 	ret = zpool_vdev_attach(zhp, old_disk, new_disk, nvroot, replacing);
3408 
3409 	nvlist_free(nvroot);
3410 	zpool_close(zhp);
3411 
3412 	return (ret);
3413 }
3414 
3415 /*
3416  * zpool replace [-f] <pool> <device> <new_device>
3417  *
3418  *	-f	Force attach, even if <new_device> appears to be in use.
3419  *
3420  * Replace <device> with <new_device>.
3421  */
3422 /* ARGSUSED */
3423 int
3424 zpool_do_replace(int argc, char **argv)
3425 {
3426 	return (zpool_do_attach_or_replace(argc, argv, B_TRUE));
3427 }
3428 
3429 /*
3430  * zpool attach [-f] <pool> <device> <new_device>
3431  *
3432  *	-f	Force attach, even if <new_device> appears to be in use.
3433  *
3434  * Attach <new_device> to the mirror containing <device>.  If <device> is not
3435  * part of a mirror, then <device> will be transformed into a mirror of
3436  * <device> and <new_device>.  In either case, <new_device> will begin life
3437  * with a DTL of [0, now], and will immediately begin to resilver itself.
3438  */
3439 int
3440 zpool_do_attach(int argc, char **argv)
3441 {
3442 	return (zpool_do_attach_or_replace(argc, argv, B_FALSE));
3443 }
3444 
3445 /*
3446  * zpool detach [-f] <pool> <device>
3447  *
3448  *	-f	Force detach of <device>, even if DTLs argue against it
3449  *		(not supported yet)
3450  *
3451  * Detach a device from a mirror.  The operation will be refused if <device>
3452  * is the last device in the mirror, or if the DTLs indicate that this device
3453  * has the only valid copy of some data.
3454  */
3455 /* ARGSUSED */
3456 int
3457 zpool_do_detach(int argc, char **argv)
3458 {
3459 	int c;
3460 	char *poolname, *path;
3461 	zpool_handle_t *zhp;
3462 	int ret;
3463 
3464 	/* check options */
3465 	while ((c = getopt(argc, argv, "f")) != -1) {
3466 		switch (c) {
3467 		case 'f':
3468 		case '?':
3469 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3470 			    optopt);
3471 			usage(B_FALSE);
3472 		}
3473 	}
3474 
3475 	argc -= optind;
3476 	argv += optind;
3477 
3478 	/* get pool name and check number of arguments */
3479 	if (argc < 1) {
3480 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
3481 		usage(B_FALSE);
3482 	}
3483 
3484 	if (argc < 2) {
3485 		(void) fprintf(stderr,
3486 		    gettext("missing <device> specification\n"));
3487 		usage(B_FALSE);
3488 	}
3489 
3490 	poolname = argv[0];
3491 	path = argv[1];
3492 
3493 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
3494 		return (1);
3495 
3496 	ret = zpool_vdev_detach(zhp, path);
3497 
3498 	zpool_close(zhp);
3499 
3500 	return (ret);
3501 }
3502 
3503 /*
3504  * zpool split [-n] [-o prop=val] ...
3505  *		[-o mntopt] ...
3506  *		[-R altroot] <pool> <newpool> [<device> ...]
3507  *
3508  *	-n	Do not split the pool, but display the resulting layout if
3509  *		it were to be split.
3510  *	-o	Set property=value, or set mount options.
3511  *	-R	Mount the split-off pool under an alternate root.
3512  *
3513  * Splits the named pool and gives it the new pool name.  Devices to be split
3514  * off may be listed, provided that no more than one device is specified
3515  * per top-level vdev mirror.  The newly split pool is left in an exported
3516  * state unless -R is specified.
3517  *
3518  * Restrictions: the top-level of the pool pool must only be made up of
3519  * mirrors; all devices in the pool must be healthy; no device may be
3520  * undergoing a resilvering operation.
3521  */
3522 int
3523 zpool_do_split(int argc, char **argv)
3524 {
3525 	char *srcpool, *newpool, *propval;
3526 	char *mntopts = NULL;
3527 	splitflags_t flags;
3528 	int c, ret = 0;
3529 	zpool_handle_t *zhp;
3530 	nvlist_t *config, *props = NULL;
3531 
3532 	flags.dryrun = B_FALSE;
3533 	flags.import = B_FALSE;
3534 
3535 	/* check options */
3536 	while ((c = getopt(argc, argv, ":R:no:")) != -1) {
3537 		switch (c) {
3538 		case 'R':
3539 			flags.import = B_TRUE;
3540 			if (add_prop_list(
3541 			    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), optarg,
3542 			    &props, B_TRUE) != 0) {
3543 				nvlist_free(props);
3544 				usage(B_FALSE);
3545 			}
3546 			break;
3547 		case 'n':
3548 			flags.dryrun = B_TRUE;
3549 			break;
3550 		case 'o':
3551 			if ((propval = strchr(optarg, '=')) != NULL) {
3552 				*propval = '\0';
3553 				propval++;
3554 				if (add_prop_list(optarg, propval,
3555 				    &props, B_TRUE) != 0) {
3556 					nvlist_free(props);
3557 					usage(B_FALSE);
3558 				}
3559 			} else {
3560 				mntopts = optarg;
3561 			}
3562 			break;
3563 		case ':':
3564 			(void) fprintf(stderr, gettext("missing argument for "
3565 			    "'%c' option\n"), optopt);
3566 			usage(B_FALSE);
3567 			break;
3568 		case '?':
3569 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3570 			    optopt);
3571 			usage(B_FALSE);
3572 			break;
3573 		}
3574 	}
3575 
3576 	if (!flags.import && mntopts != NULL) {
3577 		(void) fprintf(stderr, gettext("setting mntopts is only "
3578 		    "valid when importing the pool\n"));
3579 		usage(B_FALSE);
3580 	}
3581 
3582 	argc -= optind;
3583 	argv += optind;
3584 
3585 	if (argc < 1) {
3586 		(void) fprintf(stderr, gettext("Missing pool name\n"));
3587 		usage(B_FALSE);
3588 	}
3589 	if (argc < 2) {
3590 		(void) fprintf(stderr, gettext("Missing new pool name\n"));
3591 		usage(B_FALSE);
3592 	}
3593 
3594 	srcpool = argv[0];
3595 	newpool = argv[1];
3596 
3597 	argc -= 2;
3598 	argv += 2;
3599 
3600 	if ((zhp = zpool_open(g_zfs, srcpool)) == NULL)
3601 		return (1);
3602 
3603 	config = split_mirror_vdev(zhp, newpool, props, flags, argc, argv);
3604 	if (config == NULL) {
3605 		ret = 1;
3606 	} else {
3607 		if (flags.dryrun) {
3608 			(void) printf(gettext("would create '%s' with the "
3609 			    "following layout:\n\n"), newpool);
3610 			print_vdev_tree(NULL, newpool, config, 0, B_FALSE);
3611 		}
3612 		nvlist_free(config);
3613 	}
3614 
3615 	zpool_close(zhp);
3616 
3617 	if (ret != 0 || flags.dryrun || !flags.import)
3618 		return (ret);
3619 
3620 	/*
3621 	 * The split was successful. Now we need to open the new
3622 	 * pool and import it.
3623 	 */
3624 	if ((zhp = zpool_open_canfail(g_zfs, newpool)) == NULL)
3625 		return (1);
3626 	if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
3627 	    zpool_enable_datasets(zhp, mntopts, 0) != 0) {
3628 		ret = 1;
3629 		(void) fprintf(stderr, gettext("Split was successful, but "
3630 		    "the datasets could not all be mounted\n"));
3631 		(void) fprintf(stderr, gettext("Try doing '%s' with a "
3632 		    "different altroot\n"), "zpool import");
3633 	}
3634 	zpool_close(zhp);
3635 
3636 	return (ret);
3637 }
3638 
3639 
3640 
3641 /*
3642  * zpool online <pool> <device> ...
3643  */
3644 int
3645 zpool_do_online(int argc, char **argv)
3646 {
3647 	int c, i;
3648 	char *poolname;
3649 	zpool_handle_t *zhp;
3650 	int ret = 0;
3651 	vdev_state_t newstate;
3652 	int flags = 0;
3653 
3654 	/* check options */
3655 	while ((c = getopt(argc, argv, "et")) != -1) {
3656 		switch (c) {
3657 		case 'e':
3658 			flags |= ZFS_ONLINE_EXPAND;
3659 			break;
3660 		case 't':
3661 		case '?':
3662 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3663 			    optopt);
3664 			usage(B_FALSE);
3665 		}
3666 	}
3667 
3668 	argc -= optind;
3669 	argv += optind;
3670 
3671 	/* get pool name and check number of arguments */
3672 	if (argc < 1) {
3673 		(void) fprintf(stderr, gettext("missing pool name\n"));
3674 		usage(B_FALSE);
3675 	}
3676 	if (argc < 2) {
3677 		(void) fprintf(stderr, gettext("missing device name\n"));
3678 		usage(B_FALSE);
3679 	}
3680 
3681 	poolname = argv[0];
3682 
3683 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
3684 		return (1);
3685 
3686 	for (i = 1; i < argc; i++) {
3687 		if (zpool_vdev_online(zhp, argv[i], flags, &newstate) == 0) {
3688 			if (newstate != VDEV_STATE_HEALTHY) {
3689 				(void) printf(gettext("warning: device '%s' "
3690 				    "onlined, but remains in faulted state\n"),
3691 				    argv[i]);
3692 				if (newstate == VDEV_STATE_FAULTED)
3693 					(void) printf(gettext("use 'zpool "
3694 					    "clear' to restore a faulted "
3695 					    "device\n"));
3696 				else
3697 					(void) printf(gettext("use 'zpool "
3698 					    "replace' to replace devices "
3699 					    "that are no longer present\n"));
3700 			}
3701 		} else {
3702 			ret = 1;
3703 		}
3704 	}
3705 
3706 	zpool_close(zhp);
3707 
3708 	return (ret);
3709 }
3710 
3711 /*
3712  * zpool offline [-ft] <pool> <device> ...
3713  *
3714  *	-f	Force the device into the offline state, even if doing
3715  *		so would appear to compromise pool availability.
3716  *		(not supported yet)
3717  *
3718  *	-t	Only take the device off-line temporarily.  The offline
3719  *		state will not be persistent across reboots.
3720  */
3721 /* ARGSUSED */
3722 int
3723 zpool_do_offline(int argc, char **argv)
3724 {
3725 	int c, i;
3726 	char *poolname;
3727 	zpool_handle_t *zhp;
3728 	int ret = 0;
3729 	boolean_t istmp = B_FALSE;
3730 
3731 	/* check options */
3732 	while ((c = getopt(argc, argv, "ft")) != -1) {
3733 		switch (c) {
3734 		case 't':
3735 			istmp = B_TRUE;
3736 			break;
3737 		case 'f':
3738 		case '?':
3739 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3740 			    optopt);
3741 			usage(B_FALSE);
3742 		}
3743 	}
3744 
3745 	argc -= optind;
3746 	argv += optind;
3747 
3748 	/* get pool name and check number of arguments */
3749 	if (argc < 1) {
3750 		(void) fprintf(stderr, gettext("missing pool name\n"));
3751 		usage(B_FALSE);
3752 	}
3753 	if (argc < 2) {
3754 		(void) fprintf(stderr, gettext("missing device name\n"));
3755 		usage(B_FALSE);
3756 	}
3757 
3758 	poolname = argv[0];
3759 
3760 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
3761 		return (1);
3762 
3763 	for (i = 1; i < argc; i++) {
3764 		if (zpool_vdev_offline(zhp, argv[i], istmp) != 0)
3765 			ret = 1;
3766 	}
3767 
3768 	zpool_close(zhp);
3769 
3770 	return (ret);
3771 }
3772 
3773 /*
3774  * zpool clear <pool> [device]
3775  *
3776  * Clear all errors associated with a pool or a particular device.
3777  */
3778 int
3779 zpool_do_clear(int argc, char **argv)
3780 {
3781 	int c;
3782 	int ret = 0;
3783 	boolean_t dryrun = B_FALSE;
3784 	boolean_t do_rewind = B_FALSE;
3785 	boolean_t xtreme_rewind = B_FALSE;
3786 	uint32_t rewind_policy = ZPOOL_NO_REWIND;
3787 	nvlist_t *policy = NULL;
3788 	zpool_handle_t *zhp;
3789 	char *pool, *device;
3790 
3791 	/* check options */
3792 	while ((c = getopt(argc, argv, "FnX")) != -1) {
3793 		switch (c) {
3794 		case 'F':
3795 			do_rewind = B_TRUE;
3796 			break;
3797 		case 'n':
3798 			dryrun = B_TRUE;
3799 			break;
3800 		case 'X':
3801 			xtreme_rewind = B_TRUE;
3802 			break;
3803 		case '?':
3804 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3805 			    optopt);
3806 			usage(B_FALSE);
3807 		}
3808 	}
3809 
3810 	argc -= optind;
3811 	argv += optind;
3812 
3813 	if (argc < 1) {
3814 		(void) fprintf(stderr, gettext("missing pool name\n"));
3815 		usage(B_FALSE);
3816 	}
3817 
3818 	if (argc > 2) {
3819 		(void) fprintf(stderr, gettext("too many arguments\n"));
3820 		usage(B_FALSE);
3821 	}
3822 
3823 	if ((dryrun || xtreme_rewind) && !do_rewind) {
3824 		(void) fprintf(stderr,
3825 		    gettext("-n or -X only meaningful with -F\n"));
3826 		usage(B_FALSE);
3827 	}
3828 	if (dryrun)
3829 		rewind_policy = ZPOOL_TRY_REWIND;
3830 	else if (do_rewind)
3831 		rewind_policy = ZPOOL_DO_REWIND;
3832 	if (xtreme_rewind)
3833 		rewind_policy |= ZPOOL_EXTREME_REWIND;
3834 
3835 	/* In future, further rewind policy choices can be passed along here */
3836 	if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
3837 	    nvlist_add_uint32(policy, ZPOOL_REWIND_REQUEST, rewind_policy) != 0)
3838 		return (1);
3839 
3840 	pool = argv[0];
3841 	device = argc == 2 ? argv[1] : NULL;
3842 
3843 	if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
3844 		nvlist_free(policy);
3845 		return (1);
3846 	}
3847 
3848 	if (zpool_clear(zhp, device, policy) != 0)
3849 		ret = 1;
3850 
3851 	zpool_close(zhp);
3852 
3853 	nvlist_free(policy);
3854 
3855 	return (ret);
3856 }
3857 
3858 /*
3859  * zpool reguid <pool>
3860  */
3861 int
3862 zpool_do_reguid(int argc, char **argv)
3863 {
3864 	int c;
3865 	char *poolname;
3866 	zpool_handle_t *zhp;
3867 	int ret = 0;
3868 
3869 	/* check options */
3870 	while ((c = getopt(argc, argv, "")) != -1) {
3871 		switch (c) {
3872 		case '?':
3873 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3874 			    optopt);
3875 			usage(B_FALSE);
3876 		}
3877 	}
3878 
3879 	argc -= optind;
3880 	argv += optind;
3881 
3882 	/* get pool name and check number of arguments */
3883 	if (argc < 1) {
3884 		(void) fprintf(stderr, gettext("missing pool name\n"));
3885 		usage(B_FALSE);
3886 	}
3887 
3888 	if (argc > 1) {
3889 		(void) fprintf(stderr, gettext("too many arguments\n"));
3890 		usage(B_FALSE);
3891 	}
3892 
3893 	poolname = argv[0];
3894 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
3895 		return (1);
3896 
3897 	ret = zpool_reguid(zhp);
3898 
3899 	zpool_close(zhp);
3900 	return (ret);
3901 }
3902 
3903 
3904 /*
3905  * zpool reopen <pool>
3906  *
3907  * Reopen the pool so that the kernel can update the sizes of all vdevs.
3908  */
3909 int
3910 zpool_do_reopen(int argc, char **argv)
3911 {
3912 	int c;
3913 	int ret = 0;
3914 	zpool_handle_t *zhp;
3915 	char *pool;
3916 
3917 	/* check options */
3918 	while ((c = getopt(argc, argv, "")) != -1) {
3919 		switch (c) {
3920 		case '?':
3921 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3922 			    optopt);
3923 			usage(B_FALSE);
3924 		}
3925 	}
3926 
3927 	argc--;
3928 	argv++;
3929 
3930 	if (argc < 1) {
3931 		(void) fprintf(stderr, gettext("missing pool name\n"));
3932 		usage(B_FALSE);
3933 	}
3934 
3935 	if (argc > 1) {
3936 		(void) fprintf(stderr, gettext("too many arguments\n"));
3937 		usage(B_FALSE);
3938 	}
3939 
3940 	pool = argv[0];
3941 	if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL)
3942 		return (1);
3943 
3944 	ret = zpool_reopen(zhp);
3945 	zpool_close(zhp);
3946 	return (ret);
3947 }
3948 
3949 typedef struct scrub_cbdata {
3950 	int	cb_type;
3951 	int	cb_argc;
3952 	char	**cb_argv;
3953 	pool_scrub_cmd_t cb_scrub_cmd;
3954 } scrub_cbdata_t;
3955 
3956 int
3957 scrub_callback(zpool_handle_t *zhp, void *data)
3958 {
3959 	scrub_cbdata_t *cb = data;
3960 	int err;
3961 
3962 	/*
3963 	 * Ignore faulted pools.
3964 	 */
3965 	if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
3966 		(void) fprintf(stderr, gettext("cannot scrub '%s': pool is "
3967 		    "currently unavailable\n"), zpool_get_name(zhp));
3968 		return (1);
3969 	}
3970 
3971 	err = zpool_scan(zhp, cb->cb_type, cb->cb_scrub_cmd);
3972 
3973 	return (err != 0);
3974 }
3975 
3976 /*
3977  * zpool scrub [-s | -p] <pool> ...
3978  *
3979  *	-s	Stop.  Stops any in-progress scrub.
3980  *	-p	Pause. Pause in-progress scrub.
3981  */
3982 int
3983 zpool_do_scrub(int argc, char **argv)
3984 {
3985 	int c;
3986 	scrub_cbdata_t cb;
3987 
3988 	cb.cb_type = POOL_SCAN_SCRUB;
3989 	cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
3990 
3991 	/* check options */
3992 	while ((c = getopt(argc, argv, "sp")) != -1) {
3993 		switch (c) {
3994 		case 's':
3995 			cb.cb_type = POOL_SCAN_NONE;
3996 			break;
3997 		case 'p':
3998 			cb.cb_scrub_cmd = POOL_SCRUB_PAUSE;
3999 			break;
4000 		case '?':
4001 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
4002 			    optopt);
4003 			usage(B_FALSE);
4004 		}
4005 	}
4006 
4007 	if (cb.cb_type == POOL_SCAN_NONE &&
4008 	    cb.cb_scrub_cmd == POOL_SCRUB_PAUSE) {
4009 		(void) fprintf(stderr, gettext("invalid option combination: "
4010 		    "-s and -p are mutually exclusive\n"));
4011 		usage(B_FALSE);
4012 	}
4013 
4014 	cb.cb_argc = argc;
4015 	cb.cb_argv = argv;
4016 	argc -= optind;
4017 	argv += optind;
4018 
4019 	if (argc < 1) {
4020 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
4021 		usage(B_FALSE);
4022 	}
4023 
4024 	return (for_each_pool(argc, argv, B_TRUE, NULL, scrub_callback, &cb));
4025 }
4026 
4027 typedef struct status_cbdata {
4028 	int		cb_count;
4029 	boolean_t	cb_allpools;
4030 	boolean_t	cb_verbose;
4031 	boolean_t	cb_explain;
4032 	boolean_t	cb_first;
4033 	boolean_t	cb_dedup_stats;
4034 } status_cbdata_t;
4035 
4036 /*
4037  * Print out detailed scrub status.
4038  */
4039 static void
4040 print_scan_status(pool_scan_stat_t *ps)
4041 {
4042 	time_t start, end, pause;
4043 	uint64_t elapsed, mins_left, hours_left;
4044 	uint64_t pass_exam, examined, total;
4045 	uint_t rate;
4046 	double fraction_done;
4047 	char processed_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
4048 
4049 	(void) printf(gettext("  scan: "));
4050 
4051 	/* If there's never been a scan, there's not much to say. */
4052 	if (ps == NULL || ps->pss_func == POOL_SCAN_NONE ||
4053 	    ps->pss_func >= POOL_SCAN_FUNCS) {
4054 		(void) printf(gettext("none requested\n"));
4055 		return;
4056 	}
4057 
4058 	start = ps->pss_start_time;
4059 	end = ps->pss_end_time;
4060 	pause = ps->pss_pass_scrub_pause;
4061 	zfs_nicenum(ps->pss_processed, processed_buf, sizeof (processed_buf));
4062 
4063 	assert(ps->pss_func == POOL_SCAN_SCRUB ||
4064 	    ps->pss_func == POOL_SCAN_RESILVER);
4065 	/*
4066 	 * Scan is finished or canceled.
4067 	 */
4068 	if (ps->pss_state == DSS_FINISHED) {
4069 		uint64_t minutes_taken = (end - start) / 60;
4070 		char *fmt = NULL;
4071 
4072 		if (ps->pss_func == POOL_SCAN_SCRUB) {
4073 			fmt = gettext("scrub repaired %s in %lluh%um with "
4074 			    "%llu errors on %s");
4075 		} else if (ps->pss_func == POOL_SCAN_RESILVER) {
4076 			fmt = gettext("resilvered %s in %lluh%um with "
4077 			    "%llu errors on %s");
4078 		}
4079 		/* LINTED */
4080 		(void) printf(fmt, processed_buf,
4081 		    (u_longlong_t)(minutes_taken / 60),
4082 		    (uint_t)(minutes_taken % 60),
4083 		    (u_longlong_t)ps->pss_errors,
4084 		    ctime((time_t *)&end));
4085 		return;
4086 	} else if (ps->pss_state == DSS_CANCELED) {
4087 		if (ps->pss_func == POOL_SCAN_SCRUB) {
4088 			(void) printf(gettext("scrub canceled on %s"),
4089 			    ctime(&end));
4090 		} else if (ps->pss_func == POOL_SCAN_RESILVER) {
4091 			(void) printf(gettext("resilver canceled on %s"),
4092 			    ctime(&end));
4093 		}
4094 		return;
4095 	}
4096 
4097 	assert(ps->pss_state == DSS_SCANNING);
4098 
4099 	/*
4100 	 * Scan is in progress.
4101 	 */
4102 	if (ps->pss_func == POOL_SCAN_SCRUB) {
4103 		if (pause == 0) {
4104 			(void) printf(gettext("scrub in progress since %s"),
4105 			    ctime(&start));
4106 		} else {
4107 			char buf[32];
4108 			struct tm *p = localtime(&pause);
4109 			(void) strftime(buf, sizeof (buf), "%a %b %e %T %Y", p);
4110 			(void) printf(gettext("scrub paused since %s\n"), buf);
4111 			(void) printf(gettext("\tscrub started on   %s"),
4112 			    ctime(&start));
4113 		}
4114 	} else if (ps->pss_func == POOL_SCAN_RESILVER) {
4115 		(void) printf(gettext("resilver in progress since %s"),
4116 		    ctime(&start));
4117 	}
4118 
4119 	examined = ps->pss_examined ? ps->pss_examined : 1;
4120 	total = ps->pss_to_examine;
4121 	fraction_done = (double)examined / total;
4122 
4123 	/* elapsed time for this pass */
4124 	elapsed = time(NULL) - ps->pss_pass_start;
4125 	elapsed -= ps->pss_pass_scrub_spent_paused;
4126 	elapsed = elapsed ? elapsed : 1;
4127 	pass_exam = ps->pss_pass_exam ? ps->pss_pass_exam : 1;
4128 	rate = pass_exam / elapsed;
4129 	rate = rate ? rate : 1;
4130 	mins_left = ((total - examined) / rate) / 60;
4131 	hours_left = mins_left / 60;
4132 
4133 	zfs_nicenum(examined, examined_buf, sizeof (examined_buf));
4134 	zfs_nicenum(total, total_buf, sizeof (total_buf));
4135 
4136 	/*
4137 	 * do not print estimated time if hours_left is more than 30 days
4138 	 * or we have a paused scrub
4139 	 */
4140 	if (pause == 0) {
4141 		zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
4142 		(void) printf(gettext("\t%s scanned out of %s at %s/s"),
4143 		    examined_buf, total_buf, rate_buf);
4144 		if (hours_left < (30 * 24)) {
4145 			(void) printf(gettext(", %lluh%um to go\n"),
4146 			    (u_longlong_t)hours_left, (uint_t)(mins_left % 60));
4147 		} else {
4148 			(void) printf(gettext(
4149 			    ", (scan is slow, no estimated time)\n"));
4150 		}
4151 	} else {
4152 		(void) printf(gettext("\t%s scanned out of %s\n"),
4153 		    examined_buf, total_buf);
4154 	}
4155 
4156 	if (ps->pss_func == POOL_SCAN_RESILVER) {
4157 		(void) printf(gettext("    %s resilvered, %.2f%% done\n"),
4158 		    processed_buf, 100 * fraction_done);
4159 	} else if (ps->pss_func == POOL_SCAN_SCRUB) {
4160 		(void) printf(gettext("    %s repaired, %.2f%% done\n"),
4161 		    processed_buf, 100 * fraction_done);
4162 	}
4163 }
4164 
4165 /*
4166  * Print out detailed removal status.
4167  */
4168 static void
4169 print_removal_status(zpool_handle_t *zhp, pool_removal_stat_t *prs)
4170 {
4171 	char copied_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
4172 	time_t start, end;
4173 	nvlist_t *config, *nvroot;
4174 	nvlist_t **child;
4175 	uint_t children;
4176 	char *vdev_name;
4177 
4178 	if (prs == NULL || prs->prs_state == DSS_NONE)
4179 		return;
4180 
4181 	/*
4182 	 * Determine name of vdev.
4183 	 */
4184 	config = zpool_get_config(zhp, NULL);
4185 	nvroot = fnvlist_lookup_nvlist(config,
4186 	    ZPOOL_CONFIG_VDEV_TREE);
4187 	verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
4188 	    &child, &children) == 0);
4189 	assert(prs->prs_removing_vdev < children);
4190 	vdev_name = zpool_vdev_name(g_zfs, zhp,
4191 	    child[prs->prs_removing_vdev], B_TRUE);
4192 
4193 	(void) printf(gettext("remove: "));
4194 
4195 	start = prs->prs_start_time;
4196 	end = prs->prs_end_time;
4197 	zfs_nicenum(prs->prs_copied, copied_buf, sizeof (copied_buf));
4198 
4199 	/*
4200 	 * Removal is finished or canceled.
4201 	 */
4202 	if (prs->prs_state == DSS_FINISHED) {
4203 		uint64_t minutes_taken = (end - start) / 60;
4204 
4205 		(void) printf(gettext("Removal of vdev %llu copied %s "
4206 		    "in %lluh%um, completed on %s"),
4207 		    (longlong_t)prs->prs_removing_vdev,
4208 		    copied_buf,
4209 		    (u_longlong_t)(minutes_taken / 60),
4210 		    (uint_t)(minutes_taken % 60),
4211 		    ctime((time_t *)&end));
4212 	} else if (prs->prs_state == DSS_CANCELED) {
4213 		(void) printf(gettext("Removal of %s canceled on %s"),
4214 		    vdev_name, ctime(&end));
4215 	} else {
4216 		uint64_t copied, total, elapsed, mins_left, hours_left;
4217 		double fraction_done;
4218 		uint_t rate;
4219 
4220 		assert(prs->prs_state == DSS_SCANNING);
4221 
4222 		/*
4223 		 * Removal is in progress.
4224 		 */
4225 		(void) printf(gettext(
4226 		    "Evacuation of %s in progress since %s"),
4227 		    vdev_name, ctime(&start));
4228 
4229 		copied = prs->prs_copied > 0 ? prs->prs_copied : 1;
4230 		total = prs->prs_to_copy;
4231 		fraction_done = (double)copied / total;
4232 
4233 		/* elapsed time for this pass */
4234 		elapsed = time(NULL) - prs->prs_start_time;
4235 		elapsed = elapsed > 0 ? elapsed : 1;
4236 		rate = copied / elapsed;
4237 		rate = rate > 0 ? rate : 1;
4238 		mins_left = ((total - copied) / rate) / 60;
4239 		hours_left = mins_left / 60;
4240 
4241 		zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
4242 		zfs_nicenum(total, total_buf, sizeof (total_buf));
4243 		zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
4244 
4245 		/*
4246 		 * do not print estimated time if hours_left is more than
4247 		 * 30 days
4248 		 */
4249 		(void) printf(gettext("    %s copied out of %s at %s/s, "
4250 		    "%.2f%% done"),
4251 		    examined_buf, total_buf, rate_buf, 100 * fraction_done);
4252 		if (hours_left < (30 * 24)) {
4253 			(void) printf(gettext(", %lluh%um to go\n"),
4254 			    (u_longlong_t)hours_left, (uint_t)(mins_left % 60));
4255 		} else {
4256 			(void) printf(gettext(
4257 			    ", (copy is slow, no estimated time)\n"));
4258 		}
4259 	}
4260 
4261 	if (prs->prs_mapping_memory > 0) {
4262 		char mem_buf[7];
4263 		zfs_nicenum(prs->prs_mapping_memory, mem_buf, sizeof (mem_buf));
4264 		(void) printf(gettext("    %s memory used for "
4265 		    "removed device mappings\n"),
4266 		    mem_buf);
4267 	}
4268 }
4269 
4270 static void
4271 print_error_log(zpool_handle_t *zhp)
4272 {
4273 	nvlist_t *nverrlist = NULL;
4274 	nvpair_t *elem;
4275 	char *pathname;
4276 	size_t len = MAXPATHLEN * 2;
4277 
4278 	if (zpool_get_errlog(zhp, &nverrlist) != 0) {
4279 		(void) printf("errors: List of errors unavailable "
4280 		    "(insufficient privileges)\n");
4281 		return;
4282 	}
4283 
4284 	(void) printf("errors: Permanent errors have been "
4285 	    "detected in the following files:\n\n");
4286 
4287 	pathname = safe_malloc(len);
4288 	elem = NULL;
4289 	while ((elem = nvlist_next_nvpair(nverrlist, elem)) != NULL) {
4290 		nvlist_t *nv;
4291 		uint64_t dsobj, obj;
4292 
4293 		verify(nvpair_value_nvlist(elem, &nv) == 0);
4294 		verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_DATASET,
4295 		    &dsobj) == 0);
4296 		verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_OBJECT,
4297 		    &obj) == 0);
4298 		zpool_obj_to_path(zhp, dsobj, obj, pathname, len);
4299 		(void) printf("%7s %s\n", "", pathname);
4300 	}
4301 	free(pathname);
4302 	nvlist_free(nverrlist);
4303 }
4304 
4305 static void
4306 print_spares(zpool_handle_t *zhp, nvlist_t **spares, uint_t nspares,
4307     int namewidth)
4308 {
4309 	uint_t i;
4310 	char *name;
4311 
4312 	if (nspares == 0)
4313 		return;
4314 
4315 	(void) printf(gettext("\tspares\n"));
4316 
4317 	for (i = 0; i < nspares; i++) {
4318 		name = zpool_vdev_name(g_zfs, zhp, spares[i], B_FALSE);
4319 		print_status_config(zhp, name, spares[i],
4320 		    namewidth, 2, B_TRUE);
4321 		free(name);
4322 	}
4323 }
4324 
4325 static void
4326 print_l2cache(zpool_handle_t *zhp, nvlist_t **l2cache, uint_t nl2cache,
4327     int namewidth)
4328 {
4329 	uint_t i;
4330 	char *name;
4331 
4332 	if (nl2cache == 0)
4333 		return;
4334 
4335 	(void) printf(gettext("\tcache\n"));
4336 
4337 	for (i = 0; i < nl2cache; i++) {
4338 		name = zpool_vdev_name(g_zfs, zhp, l2cache[i], B_FALSE);
4339 		print_status_config(zhp, name, l2cache[i],
4340 		    namewidth, 2, B_FALSE);
4341 		free(name);
4342 	}
4343 }
4344 
4345 static void
4346 print_dedup_stats(nvlist_t *config)
4347 {
4348 	ddt_histogram_t *ddh;
4349 	ddt_stat_t *dds;
4350 	ddt_object_t *ddo;
4351 	uint_t c;
4352 
4353 	/*
4354 	 * If the pool was faulted then we may not have been able to
4355 	 * obtain the config. Otherwise, if we have anything in the dedup
4356 	 * table continue processing the stats.
4357 	 */
4358 	if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS,
4359 	    (uint64_t **)&ddo, &c) != 0)
4360 		return;
4361 
4362 	(void) printf("\n");
4363 	(void) printf(gettext(" dedup: "));
4364 	if (ddo->ddo_count == 0) {
4365 		(void) printf(gettext("no DDT entries\n"));
4366 		return;
4367 	}
4368 
4369 	(void) printf("DDT entries %llu, size %llu on disk, %llu in core\n",
4370 	    (u_longlong_t)ddo->ddo_count,
4371 	    (u_longlong_t)ddo->ddo_dspace,
4372 	    (u_longlong_t)ddo->ddo_mspace);
4373 
4374 	verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_STATS,
4375 	    (uint64_t **)&dds, &c) == 0);
4376 	verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_HISTOGRAM,
4377 	    (uint64_t **)&ddh, &c) == 0);
4378 	zpool_dump_ddt(dds, ddh);
4379 }
4380 
4381 /*
4382  * Display a summary of pool status.  Displays a summary such as:
4383  *
4384  *        pool: tank
4385  *	status: DEGRADED
4386  *	reason: One or more devices ...
4387  *         see: http://illumos.org/msg/ZFS-xxxx-01
4388  *	config:
4389  *		mirror		DEGRADED
4390  *                c1t0d0	OK
4391  *                c2t0d0	UNAVAIL
4392  *
4393  * When given the '-v' option, we print out the complete config.  If the '-e'
4394  * option is specified, then we print out error rate information as well.
4395  */
4396 int
4397 status_callback(zpool_handle_t *zhp, void *data)
4398 {
4399 	status_cbdata_t *cbp = data;
4400 	nvlist_t *config, *nvroot;
4401 	char *msgid;
4402 	int reason;
4403 	const char *health;
4404 	uint_t c;
4405 	vdev_stat_t *vs;
4406 
4407 	config = zpool_get_config(zhp, NULL);
4408 	reason = zpool_get_status(zhp, &msgid);
4409 
4410 	cbp->cb_count++;
4411 
4412 	/*
4413 	 * If we were given 'zpool status -x', only report those pools with
4414 	 * problems.
4415 	 */
4416 	if (cbp->cb_explain &&
4417 	    (reason == ZPOOL_STATUS_OK ||
4418 	    reason == ZPOOL_STATUS_VERSION_OLDER ||
4419 	    reason == ZPOOL_STATUS_FEAT_DISABLED)) {
4420 		if (!cbp->cb_allpools) {
4421 			(void) printf(gettext("pool '%s' is healthy\n"),
4422 			    zpool_get_name(zhp));
4423 			if (cbp->cb_first)
4424 				cbp->cb_first = B_FALSE;
4425 		}
4426 		return (0);
4427 	}
4428 
4429 	if (cbp->cb_first)
4430 		cbp->cb_first = B_FALSE;
4431 	else
4432 		(void) printf("\n");
4433 
4434 	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
4435 	verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
4436 	    (uint64_t **)&vs, &c) == 0);
4437 	health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
4438 
4439 	(void) printf(gettext("  pool: %s\n"), zpool_get_name(zhp));
4440 	(void) printf(gettext(" state: %s\n"), health);
4441 
4442 	switch (reason) {
4443 	case ZPOOL_STATUS_MISSING_DEV_R:
4444 		(void) printf(gettext("status: One or more devices could not "
4445 		    "be opened.  Sufficient replicas exist for\n\tthe pool to "
4446 		    "continue functioning in a degraded state.\n"));
4447 		(void) printf(gettext("action: Attach the missing device and "
4448 		    "online it using 'zpool online'.\n"));
4449 		break;
4450 
4451 	case ZPOOL_STATUS_MISSING_DEV_NR:
4452 		(void) printf(gettext("status: One or more devices could not "
4453 		    "be opened.  There are insufficient\n\treplicas for the "
4454 		    "pool to continue functioning.\n"));
4455 		(void) printf(gettext("action: Attach the missing device and "
4456 		    "online it using 'zpool online'.\n"));
4457 		break;
4458 
4459 	case ZPOOL_STATUS_CORRUPT_LABEL_R:
4460 		(void) printf(gettext("status: One or more devices could not "
4461 		    "be used because the label is missing or\n\tinvalid.  "
4462 		    "Sufficient replicas exist for the pool to continue\n\t"
4463 		    "functioning in a degraded state.\n"));
4464 		(void) printf(gettext("action: Replace the device using "
4465 		    "'zpool replace'.\n"));
4466 		break;
4467 
4468 	case ZPOOL_STATUS_CORRUPT_LABEL_NR:
4469 		(void) printf(gettext("status: One or more devices could not "
4470 		    "be used because the label is missing \n\tor invalid.  "
4471 		    "There are insufficient replicas for the pool to "
4472 		    "continue\n\tfunctioning.\n"));
4473 		zpool_explain_recover(zpool_get_handle(zhp),
4474 		    zpool_get_name(zhp), reason, config);
4475 		break;
4476 
4477 	case ZPOOL_STATUS_FAILING_DEV:
4478 		(void) printf(gettext("status: One or more devices has "
4479 		    "experienced an unrecoverable error.  An\n\tattempt was "
4480 		    "made to correct the error.  Applications are "
4481 		    "unaffected.\n"));
4482 		(void) printf(gettext("action: Determine if the device needs "
4483 		    "to be replaced, and clear the errors\n\tusing "
4484 		    "'zpool clear' or replace the device with 'zpool "
4485 		    "replace'.\n"));
4486 		break;
4487 
4488 	case ZPOOL_STATUS_OFFLINE_DEV:
4489 		(void) printf(gettext("status: One or more devices has "
4490 		    "been taken offline by the administrator.\n\tSufficient "
4491 		    "replicas exist for the pool to continue functioning in "
4492 		    "a\n\tdegraded state.\n"));
4493 		(void) printf(gettext("action: Online the device using "
4494 		    "'zpool online' or replace the device with\n\t'zpool "
4495 		    "replace'.\n"));
4496 		break;
4497 
4498 	case ZPOOL_STATUS_REMOVED_DEV:
4499 		(void) printf(gettext("status: One or more devices has "
4500 		    "been removed by the administrator.\n\tSufficient "
4501 		    "replicas exist for the pool to continue functioning in "
4502 		    "a\n\tdegraded state.\n"));
4503 		(void) printf(gettext("action: Online the device using "
4504 		    "'zpool online' or replace the device with\n\t'zpool "
4505 		    "replace'.\n"));
4506 		break;
4507 
4508 	case ZPOOL_STATUS_RESILVERING:
4509 		(void) printf(gettext("status: One or more devices is "
4510 		    "currently being resilvered.  The pool will\n\tcontinue "
4511 		    "to function, possibly in a degraded state.\n"));
4512 		(void) printf(gettext("action: Wait for the resilver to "
4513 		    "complete.\n"));
4514 		break;
4515 
4516 	case ZPOOL_STATUS_CORRUPT_DATA:
4517 		(void) printf(gettext("status: One or more devices has "
4518 		    "experienced an error resulting in data\n\tcorruption.  "
4519 		    "Applications may be affected.\n"));
4520 		(void) printf(gettext("action: Restore the file in question "
4521 		    "if possible.  Otherwise restore the\n\tentire pool from "
4522 		    "backup.\n"));
4523 		break;
4524 
4525 	case ZPOOL_STATUS_CORRUPT_POOL:
4526 		(void) printf(gettext("status: The pool metadata is corrupted "
4527 		    "and the pool cannot be opened.\n"));
4528 		zpool_explain_recover(zpool_get_handle(zhp),
4529 		    zpool_get_name(zhp), reason, config);
4530 		break;
4531 
4532 	case ZPOOL_STATUS_VERSION_OLDER:
4533 		(void) printf(gettext("status: The pool is formatted using a "
4534 		    "legacy on-disk format.  The pool can\n\tstill be used, "
4535 		    "but some features are unavailable.\n"));
4536 		(void) printf(gettext("action: Upgrade the pool using 'zpool "
4537 		    "upgrade'.  Once this is done, the\n\tpool will no longer "
4538 		    "be accessible on software that does not support feature\n"
4539 		    "\tflags.\n"));
4540 		break;
4541 
4542 	case ZPOOL_STATUS_VERSION_NEWER:
4543 		(void) printf(gettext("status: The pool has been upgraded to a "
4544 		    "newer, incompatible on-disk version.\n\tThe pool cannot "
4545 		    "be accessed on this system.\n"));
4546 		(void) printf(gettext("action: Access the pool from a system "
4547 		    "running more recent software, or\n\trestore the pool from "
4548 		    "backup.\n"));
4549 		break;
4550 
4551 	case ZPOOL_STATUS_FEAT_DISABLED:
4552 		(void) printf(gettext("status: Some supported features are not "
4553 		    "enabled on the pool. The pool can\n\tstill be used, but "
4554 		    "some features are unavailable.\n"));
4555 		(void) printf(gettext("action: Enable all features using "
4556 		    "'zpool upgrade'. Once this is done,\n\tthe pool may no "
4557 		    "longer be accessible by software that does not support\n\t"
4558 		    "the features. See zpool-features(5) for details.\n"));
4559 		break;
4560 
4561 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
4562 		(void) printf(gettext("status: The pool cannot be accessed on "
4563 		    "this system because it uses the\n\tfollowing feature(s) "
4564 		    "not supported on this system:\n"));
4565 		zpool_print_unsup_feat(config);
4566 		(void) printf("\n");
4567 		(void) printf(gettext("action: Access the pool from a system "
4568 		    "that supports the required feature(s),\n\tor restore the "
4569 		    "pool from backup.\n"));
4570 		break;
4571 
4572 	case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
4573 		(void) printf(gettext("status: The pool can only be accessed "
4574 		    "in read-only mode on this system. It\n\tcannot be "
4575 		    "accessed in read-write mode because it uses the "
4576 		    "following\n\tfeature(s) not supported on this system:\n"));
4577 		zpool_print_unsup_feat(config);
4578 		(void) printf("\n");
4579 		(void) printf(gettext("action: The pool cannot be accessed in "
4580 		    "read-write mode. Import the pool with\n"
4581 		    "\t\"-o readonly=on\", access the pool from a system that "
4582 		    "supports the\n\trequired feature(s), or restore the "
4583 		    "pool from backup.\n"));
4584 		break;
4585 
4586 	case ZPOOL_STATUS_FAULTED_DEV_R:
4587 		(void) printf(gettext("status: One or more devices are "
4588 		    "faulted in response to persistent errors.\n\tSufficient "
4589 		    "replicas exist for the pool to continue functioning "
4590 		    "in a\n\tdegraded state.\n"));
4591 		(void) printf(gettext("action: Replace the faulted device, "
4592 		    "or use 'zpool clear' to mark the device\n\trepaired.\n"));
4593 		break;
4594 
4595 	case ZPOOL_STATUS_FAULTED_DEV_NR:
4596 		(void) printf(gettext("status: One or more devices are "
4597 		    "faulted in response to persistent errors.  There are "
4598 		    "insufficient replicas for the pool to\n\tcontinue "
4599 		    "functioning.\n"));
4600 		(void) printf(gettext("action: Destroy and re-create the pool "
4601 		    "from a backup source.  Manually marking the device\n"
4602 		    "\trepaired using 'zpool clear' may allow some data "
4603 		    "to be recovered.\n"));
4604 		break;
4605 
4606 	case ZPOOL_STATUS_IO_FAILURE_WAIT:
4607 	case ZPOOL_STATUS_IO_FAILURE_CONTINUE:
4608 		(void) printf(gettext("status: One or more devices are "
4609 		    "faulted in response to IO failures.\n"));
4610 		(void) printf(gettext("action: Make sure the affected devices "
4611 		    "are connected, then run 'zpool clear'.\n"));
4612 		break;
4613 
4614 	case ZPOOL_STATUS_BAD_LOG:
4615 		(void) printf(gettext("status: An intent log record "
4616 		    "could not be read.\n"
4617 		    "\tWaiting for adminstrator intervention to fix the "
4618 		    "faulted pool.\n"));
4619 		(void) printf(gettext("action: Either restore the affected "
4620 		    "device(s) and run 'zpool online',\n"
4621 		    "\tor ignore the intent log records by running "
4622 		    "'zpool clear'.\n"));
4623 		break;
4624 
4625 	default:
4626 		/*
4627 		 * The remaining errors can't actually be generated, yet.
4628 		 */
4629 		assert(reason == ZPOOL_STATUS_OK);
4630 	}
4631 
4632 	if (msgid != NULL)
4633 		(void) printf(gettext("   see: http://illumos.org/msg/%s\n"),
4634 		    msgid);
4635 
4636 	if (config != NULL) {
4637 		int namewidth;
4638 		uint64_t nerr;
4639 		nvlist_t **spares, **l2cache;
4640 		uint_t nspares, nl2cache;
4641 		pool_scan_stat_t *ps = NULL;
4642 		pool_removal_stat_t *prs = NULL;
4643 
4644 		(void) nvlist_lookup_uint64_array(nvroot,
4645 		    ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&ps, &c);
4646 		print_scan_status(ps);
4647 
4648 		(void) nvlist_lookup_uint64_array(nvroot,
4649 		    ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
4650 		print_removal_status(zhp, prs);
4651 
4652 		namewidth = max_width(zhp, nvroot, 0, 0);
4653 		if (namewidth < 10)
4654 			namewidth = 10;
4655 
4656 		(void) printf(gettext("config:\n\n"));
4657 		(void) printf(gettext("\t%-*s  %-8s %5s %5s %5s\n"), namewidth,
4658 		    "NAME", "STATE", "READ", "WRITE", "CKSUM");
4659 		print_status_config(zhp, zpool_get_name(zhp), nvroot,
4660 		    namewidth, 0, B_FALSE);
4661 
4662 		if (num_logs(nvroot) > 0)
4663 			print_logs(zhp, nvroot, namewidth, B_TRUE);
4664 		if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
4665 		    &l2cache, &nl2cache) == 0)
4666 			print_l2cache(zhp, l2cache, nl2cache, namewidth);
4667 
4668 		if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
4669 		    &spares, &nspares) == 0)
4670 			print_spares(zhp, spares, nspares, namewidth);
4671 
4672 		if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
4673 		    &nerr) == 0) {
4674 			nvlist_t *nverrlist = NULL;
4675 
4676 			/*
4677 			 * If the approximate error count is small, get a
4678 			 * precise count by fetching the entire log and
4679 			 * uniquifying the results.
4680 			 */
4681 			if (nerr > 0 && nerr < 100 && !cbp->cb_verbose &&
4682 			    zpool_get_errlog(zhp, &nverrlist) == 0) {
4683 				nvpair_t *elem;
4684 
4685 				elem = NULL;
4686 				nerr = 0;
4687 				while ((elem = nvlist_next_nvpair(nverrlist,
4688 				    elem)) != NULL) {
4689 					nerr++;
4690 				}
4691 			}
4692 			nvlist_free(nverrlist);
4693 
4694 			(void) printf("\n");
4695 
4696 			if (nerr == 0)
4697 				(void) printf(gettext("errors: No known data "
4698 				    "errors\n"));
4699 			else if (!cbp->cb_verbose)
4700 				(void) printf(gettext("errors: %llu data "
4701 				    "errors, use '-v' for a list\n"),
4702 				    (u_longlong_t)nerr);
4703 			else
4704 				print_error_log(zhp);
4705 		}
4706 
4707 		if (cbp->cb_dedup_stats)
4708 			print_dedup_stats(config);
4709 	} else {
4710 		(void) printf(gettext("config: The configuration cannot be "
4711 		    "determined.\n"));
4712 	}
4713 
4714 	return (0);
4715 }
4716 
4717 /*
4718  * zpool status [-vx] [-T d|u] [pool] ... [interval [count]]
4719  *
4720  *	-v	Display complete error logs
4721  *	-x	Display only pools with potential problems
4722  *	-D	Display dedup status (undocumented)
4723  *	-T	Display a timestamp in date(1) or Unix format
4724  *
4725  * Describes the health status of all pools or some subset.
4726  */
4727 int
4728 zpool_do_status(int argc, char **argv)
4729 {
4730 	int c;
4731 	int ret;
4732 	unsigned long interval = 0, count = 0;
4733 	status_cbdata_t cb = { 0 };
4734 
4735 	/* check options */
4736 	while ((c = getopt(argc, argv, "vxDT:")) != -1) {
4737 		switch (c) {
4738 		case 'v':
4739 			cb.cb_verbose = B_TRUE;
4740 			break;
4741 		case 'x':
4742 			cb.cb_explain = B_TRUE;
4743 			break;
4744 		case 'D':
4745 			cb.cb_dedup_stats = B_TRUE;
4746 			break;
4747 		case 'T':
4748 			get_timestamp_arg(*optarg);
4749 			break;
4750 		case '?':
4751 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
4752 			    optopt);
4753 			usage(B_FALSE);
4754 		}
4755 	}
4756 
4757 	argc -= optind;
4758 	argv += optind;
4759 
4760 	get_interval_count(&argc, argv, &interval, &count);
4761 
4762 	if (argc == 0)
4763 		cb.cb_allpools = B_TRUE;
4764 
4765 	cb.cb_first = B_TRUE;
4766 
4767 	for (;;) {
4768 		if (timestamp_fmt != NODATE)
4769 			print_timestamp(timestamp_fmt);
4770 
4771 		ret = for_each_pool(argc, argv, B_TRUE, NULL,
4772 		    status_callback, &cb);
4773 
4774 		if (argc == 0 && cb.cb_count == 0)
4775 			(void) printf(gettext("no pools available\n"));
4776 		else if (cb.cb_explain && cb.cb_first && cb.cb_allpools)
4777 			(void) printf(gettext("all pools are healthy\n"));
4778 
4779 		if (ret != 0)
4780 			return (ret);
4781 
4782 		if (interval == 0)
4783 			break;
4784 
4785 		if (count != 0 && --count == 0)
4786 			break;
4787 
4788 		(void) sleep(interval);
4789 	}
4790 
4791 	return (0);
4792 }
4793 
4794 typedef struct upgrade_cbdata {
4795 	int	cb_first;
4796 	int	cb_argc;
4797 	uint64_t cb_version;
4798 	char	**cb_argv;
4799 } upgrade_cbdata_t;
4800 
4801 static int
4802 upgrade_version(zpool_handle_t *zhp, uint64_t version)
4803 {
4804 	int ret;
4805 	nvlist_t *config;
4806 	uint64_t oldversion;
4807 
4808 	config = zpool_get_config(zhp, NULL);
4809 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
4810 	    &oldversion) == 0);
4811 
4812 	assert(SPA_VERSION_IS_SUPPORTED(oldversion));
4813 	assert(oldversion < version);
4814 
4815 	ret = zpool_upgrade(zhp, version);
4816 	if (ret != 0)
4817 		return (ret);
4818 
4819 	if (version >= SPA_VERSION_FEATURES) {
4820 		(void) printf(gettext("Successfully upgraded "
4821 		    "'%s' from version %llu to feature flags.\n"),
4822 		    zpool_get_name(zhp), oldversion);
4823 	} else {
4824 		(void) printf(gettext("Successfully upgraded "
4825 		    "'%s' from version %llu to version %llu.\n"),
4826 		    zpool_get_name(zhp), oldversion, version);
4827 	}
4828 
4829 	return (0);
4830 }
4831 
4832 static int
4833 upgrade_enable_all(zpool_handle_t *zhp, int *countp)
4834 {
4835 	int i, ret, count;
4836 	boolean_t firstff = B_TRUE;
4837 	nvlist_t *enabled = zpool_get_features(zhp);
4838 
4839 	count = 0;
4840 	for (i = 0; i < SPA_FEATURES; i++) {
4841 		const char *fname = spa_feature_table[i].fi_uname;
4842 		const char *fguid = spa_feature_table[i].fi_guid;
4843 		if (!nvlist_exists(enabled, fguid)) {
4844 			char *propname;
4845 			verify(-1 != asprintf(&propname, "feature@%s", fname));
4846 			ret = zpool_set_prop(zhp, propname,
4847 			    ZFS_FEATURE_ENABLED);
4848 			if (ret != 0) {
4849 				free(propname);
4850 				return (ret);
4851 			}
4852 			count++;
4853 
4854 			if (firstff) {
4855 				(void) printf(gettext("Enabled the "
4856 				    "following features on '%s':\n"),
4857 				    zpool_get_name(zhp));
4858 				firstff = B_FALSE;
4859 			}
4860 			(void) printf(gettext("  %s\n"), fname);
4861 			free(propname);
4862 		}
4863 	}
4864 
4865 	if (countp != NULL)
4866 		*countp = count;
4867 	return (0);
4868 }
4869 
4870 static int
4871 upgrade_cb(zpool_handle_t *zhp, void *arg)
4872 {
4873 	upgrade_cbdata_t *cbp = arg;
4874 	nvlist_t *config;
4875 	uint64_t version;
4876 	boolean_t printnl = B_FALSE;
4877 	int ret;
4878 
4879 	config = zpool_get_config(zhp, NULL);
4880 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
4881 	    &version) == 0);
4882 
4883 	assert(SPA_VERSION_IS_SUPPORTED(version));
4884 
4885 	if (version < cbp->cb_version) {
4886 		cbp->cb_first = B_FALSE;
4887 		ret = upgrade_version(zhp, cbp->cb_version);
4888 		if (ret != 0)
4889 			return (ret);
4890 		printnl = B_TRUE;
4891 
4892 		/*
4893 		 * If they did "zpool upgrade -a", then we could
4894 		 * be doing ioctls to different pools.  We need
4895 		 * to log this history once to each pool, and bypass
4896 		 * the normal history logging that happens in main().
4897 		 */
4898 		(void) zpool_log_history(g_zfs, history_str);
4899 		log_history = B_FALSE;
4900 	}
4901 
4902 	if (cbp->cb_version >= SPA_VERSION_FEATURES) {
4903 		int count;
4904 		ret = upgrade_enable_all(zhp, &count);
4905 		if (ret != 0)
4906 			return (ret);
4907 
4908 		if (count > 0) {
4909 			cbp->cb_first = B_FALSE;
4910 			printnl = B_TRUE;
4911 		}
4912 	}
4913 
4914 	if (printnl) {
4915 		(void) printf(gettext("\n"));
4916 	}
4917 
4918 	return (0);
4919 }
4920 
4921 static int
4922 upgrade_list_older_cb(zpool_handle_t *zhp, void *arg)
4923 {
4924 	upgrade_cbdata_t *cbp = arg;
4925 	nvlist_t *config;
4926 	uint64_t version;
4927 
4928 	config = zpool_get_config(zhp, NULL);
4929 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
4930 	    &version) == 0);
4931 
4932 	assert(SPA_VERSION_IS_SUPPORTED(version));
4933 
4934 	if (version < SPA_VERSION_FEATURES) {
4935 		if (cbp->cb_first) {
4936 			(void) printf(gettext("The following pools are "
4937 			    "formatted with legacy version numbers and can\n"
4938 			    "be upgraded to use feature flags.  After "
4939 			    "being upgraded, these pools\nwill no "
4940 			    "longer be accessible by software that does not "
4941 			    "support feature\nflags.\n\n"));
4942 			(void) printf(gettext("VER  POOL\n"));
4943 			(void) printf(gettext("---  ------------\n"));
4944 			cbp->cb_first = B_FALSE;
4945 		}
4946 
4947 		(void) printf("%2llu   %s\n", (u_longlong_t)version,
4948 		    zpool_get_name(zhp));
4949 	}
4950 
4951 	return (0);
4952 }
4953 
4954 static int
4955 upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
4956 {
4957 	upgrade_cbdata_t *cbp = arg;
4958 	nvlist_t *config;
4959 	uint64_t version;
4960 
4961 	config = zpool_get_config(zhp, NULL);
4962 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
4963 	    &version) == 0);
4964 
4965 	if (version >= SPA_VERSION_FEATURES) {
4966 		int i;
4967 		boolean_t poolfirst = B_TRUE;
4968 		nvlist_t *enabled = zpool_get_features(zhp);
4969 
4970 		for (i = 0; i < SPA_FEATURES; i++) {
4971 			const char *fguid = spa_feature_table[i].fi_guid;
4972 			const char *fname = spa_feature_table[i].fi_uname;
4973 			if (!nvlist_exists(enabled, fguid)) {
4974 				if (cbp->cb_first) {
4975 					(void) printf(gettext("\nSome "
4976 					    "supported features are not "
4977 					    "enabled on the following pools. "
4978 					    "Once a\nfeature is enabled the "
4979 					    "pool may become incompatible with "
4980 					    "software\nthat does not support "
4981 					    "the feature. See "
4982 					    "zpool-features(5) for "
4983 					    "details.\n\n"));
4984 					(void) printf(gettext("POOL  "
4985 					    "FEATURE\n"));
4986 					(void) printf(gettext("------"
4987 					    "---------\n"));
4988 					cbp->cb_first = B_FALSE;
4989 				}
4990 
4991 				if (poolfirst) {
4992 					(void) printf(gettext("%s\n"),
4993 					    zpool_get_name(zhp));
4994 					poolfirst = B_FALSE;
4995 				}
4996 
4997 				(void) printf(gettext("      %s\n"), fname);
4998 			}
4999 		}
5000 	}
5001 
5002 	return (0);
5003 }
5004 
5005 /* ARGSUSED */
5006 static int
5007 upgrade_one(zpool_handle_t *zhp, void *data)
5008 {
5009 	boolean_t printnl = B_FALSE;
5010 	upgrade_cbdata_t *cbp = data;
5011 	uint64_t cur_version;
5012 	int ret;
5013 
5014 	if (strcmp("log", zpool_get_name(zhp)) == 0) {
5015 		(void) printf(gettext("'log' is now a reserved word\n"
5016 		    "Pool 'log' must be renamed using export and import"
5017 		    " to upgrade.\n"));
5018 		return (1);
5019 	}
5020 
5021 	cur_version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
5022 	if (cur_version > cbp->cb_version) {
5023 		(void) printf(gettext("Pool '%s' is already formatted "
5024 		    "using more current version '%llu'.\n\n"),
5025 		    zpool_get_name(zhp), cur_version);
5026 		return (0);
5027 	}
5028 
5029 	if (cbp->cb_version != SPA_VERSION && cur_version == cbp->cb_version) {
5030 		(void) printf(gettext("Pool '%s' is already formatted "
5031 		    "using version %llu.\n\n"), zpool_get_name(zhp),
5032 		    cbp->cb_version);
5033 		return (0);
5034 	}
5035 
5036 	if (cur_version != cbp->cb_version) {
5037 		printnl = B_TRUE;
5038 		ret = upgrade_version(zhp, cbp->cb_version);
5039 		if (ret != 0)
5040 			return (ret);
5041 	}
5042 
5043 	if (cbp->cb_version >= SPA_VERSION_FEATURES) {
5044 		int count = 0;
5045 		ret = upgrade_enable_all(zhp, &count);
5046 		if (ret != 0)
5047 			return (ret);
5048 
5049 		if (count != 0) {
5050 			printnl = B_TRUE;
5051 		} else if (cur_version == SPA_VERSION) {
5052 			(void) printf(gettext("Pool '%s' already has all "
5053 			    "supported features enabled.\n"),
5054 			    zpool_get_name(zhp));
5055 		}
5056 	}
5057 
5058 	if (printnl) {
5059 		(void) printf(gettext("\n"));
5060 	}
5061 
5062 	return (0);
5063 }
5064 
5065 /*
5066  * zpool upgrade
5067  * zpool upgrade -v
5068  * zpool upgrade [-V version] <-a | pool ...>
5069  *
5070  * With no arguments, display downrev'd ZFS pool available for upgrade.
5071  * Individual pools can be upgraded by specifying the pool, and '-a' will
5072  * upgrade all pools.
5073  */
5074 int
5075 zpool_do_upgrade(int argc, char **argv)
5076 {
5077 	int c;
5078 	upgrade_cbdata_t cb = { 0 };
5079 	int ret = 0;
5080 	boolean_t showversions = B_FALSE;
5081 	boolean_t upgradeall = B_FALSE;
5082 	char *end;
5083 
5084 
5085 	/* check options */
5086 	while ((c = getopt(argc, argv, ":avV:")) != -1) {
5087 		switch (c) {
5088 		case 'a':
5089 			upgradeall = B_TRUE;
5090 			break;
5091 		case 'v':
5092 			showversions = B_TRUE;
5093 			break;
5094 		case 'V':
5095 			cb.cb_version = strtoll(optarg, &end, 10);
5096 			if (*end != '\0' ||
5097 			    !SPA_VERSION_IS_SUPPORTED(cb.cb_version)) {
5098 				(void) fprintf(stderr,
5099 				    gettext("invalid version '%s'\n"), optarg);
5100 				usage(B_FALSE);
5101 			}
5102 			break;
5103 		case ':':
5104 			(void) fprintf(stderr, gettext("missing argument for "
5105 			    "'%c' option\n"), optopt);
5106 			usage(B_FALSE);
5107 			break;
5108 		case '?':
5109 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
5110 			    optopt);
5111 			usage(B_FALSE);
5112 		}
5113 	}
5114 
5115 	cb.cb_argc = argc;
5116 	cb.cb_argv = argv;
5117 	argc -= optind;
5118 	argv += optind;
5119 
5120 	if (cb.cb_version == 0) {
5121 		cb.cb_version = SPA_VERSION;
5122 	} else if (!upgradeall && argc == 0) {
5123 		(void) fprintf(stderr, gettext("-V option is "
5124 		    "incompatible with other arguments\n"));
5125 		usage(B_FALSE);
5126 	}
5127 
5128 	if (showversions) {
5129 		if (upgradeall || argc != 0) {
5130 			(void) fprintf(stderr, gettext("-v option is "
5131 			    "incompatible with other arguments\n"));
5132 			usage(B_FALSE);
5133 		}
5134 	} else if (upgradeall) {
5135 		if (argc != 0) {
5136 			(void) fprintf(stderr, gettext("-a option should not "
5137 			    "be used along with a pool name\n"));
5138 			usage(B_FALSE);
5139 		}
5140 	}
5141 
5142 	(void) printf(gettext("This system supports ZFS pool feature "
5143 	    "flags.\n\n"));
5144 	if (showversions) {
5145 		int i;
5146 
5147 		(void) printf(gettext("The following features are "
5148 		    "supported:\n\n"));
5149 		(void) printf(gettext("FEAT DESCRIPTION\n"));
5150 		(void) printf("----------------------------------------------"
5151 		    "---------------\n");
5152 		for (i = 0; i < SPA_FEATURES; i++) {
5153 			zfeature_info_t *fi = &spa_feature_table[i];
5154 			const char *ro =
5155 			    (fi->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
5156 			    " (read-only compatible)" : "";
5157 
5158 			(void) printf("%-37s%s\n", fi->fi_uname, ro);
5159 			(void) printf("     %s\n", fi->fi_desc);
5160 		}
5161 		(void) printf("\n");
5162 
5163 		(void) printf(gettext("The following legacy versions are also "
5164 		    "supported:\n\n"));
5165 		(void) printf(gettext("VER  DESCRIPTION\n"));
5166 		(void) printf("---  -----------------------------------------"
5167 		    "---------------\n");
5168 		(void) printf(gettext(" 1   Initial ZFS version\n"));
5169 		(void) printf(gettext(" 2   Ditto blocks "
5170 		    "(replicated metadata)\n"));
5171 		(void) printf(gettext(" 3   Hot spares and double parity "
5172 		    "RAID-Z\n"));
5173 		(void) printf(gettext(" 4   zpool history\n"));
5174 		(void) printf(gettext(" 5   Compression using the gzip "
5175 		    "algorithm\n"));
5176 		(void) printf(gettext(" 6   bootfs pool property\n"));
5177 		(void) printf(gettext(" 7   Separate intent log devices\n"));
5178 		(void) printf(gettext(" 8   Delegated administration\n"));
5179 		(void) printf(gettext(" 9   refquota and refreservation "
5180 		    "properties\n"));
5181 		(void) printf(gettext(" 10  Cache devices\n"));
5182 		(void) printf(gettext(" 11  Improved scrub performance\n"));
5183 		(void) printf(gettext(" 12  Snapshot properties\n"));
5184 		(void) printf(gettext(" 13  snapused property\n"));
5185 		(void) printf(gettext(" 14  passthrough-x aclinherit\n"));
5186 		(void) printf(gettext(" 15  user/group space accounting\n"));
5187 		(void) printf(gettext(" 16  stmf property support\n"));
5188 		(void) printf(gettext(" 17  Triple-parity RAID-Z\n"));
5189 		(void) printf(gettext(" 18  Snapshot user holds\n"));
5190 		(void) printf(gettext(" 19  Log device removal\n"));
5191 		(void) printf(gettext(" 20  Compression using zle "
5192 		    "(zero-length encoding)\n"));
5193 		(void) printf(gettext(" 21  Deduplication\n"));
5194 		(void) printf(gettext(" 22  Received properties\n"));
5195 		(void) printf(gettext(" 23  Slim ZIL\n"));
5196 		(void) printf(gettext(" 24  System attributes\n"));
5197 		(void) printf(gettext(" 25  Improved scrub stats\n"));
5198 		(void) printf(gettext(" 26  Improved snapshot deletion "
5199 		    "performance\n"));
5200 		(void) printf(gettext(" 27  Improved snapshot creation "
5201 		    "performance\n"));
5202 		(void) printf(gettext(" 28  Multiple vdev replacements\n"));
5203 		(void) printf(gettext("\nFor more information on a particular "
5204 		    "version, including supported releases,\n"));
5205 		(void) printf(gettext("see the ZFS Administration Guide.\n\n"));
5206 	} else if (argc == 0 && upgradeall) {
5207 		cb.cb_first = B_TRUE;
5208 		ret = zpool_iter(g_zfs, upgrade_cb, &cb);
5209 		if (ret == 0 && cb.cb_first) {
5210 			if (cb.cb_version == SPA_VERSION) {
5211 				(void) printf(gettext("All pools are already "
5212 				    "formatted using feature flags.\n\n"));
5213 				(void) printf(gettext("Every feature flags "
5214 				    "pool already has all supported features "
5215 				    "enabled.\n"));
5216 			} else {
5217 				(void) printf(gettext("All pools are already "
5218 				    "formatted with version %llu or higher.\n"),
5219 				    cb.cb_version);
5220 			}
5221 		}
5222 	} else if (argc == 0) {
5223 		cb.cb_first = B_TRUE;
5224 		ret = zpool_iter(g_zfs, upgrade_list_older_cb, &cb);
5225 		assert(ret == 0);
5226 
5227 		if (cb.cb_first) {
5228 			(void) printf(gettext("All pools are formatted "
5229 			    "using feature flags.\n\n"));
5230 		} else {
5231 			(void) printf(gettext("\nUse 'zpool upgrade -v' "
5232 			    "for a list of available legacy versions.\n"));
5233 		}
5234 
5235 		cb.cb_first = B_TRUE;
5236 		ret = zpool_iter(g_zfs, upgrade_list_disabled_cb, &cb);
5237 		assert(ret == 0);
5238 
5239 		if (cb.cb_first) {
5240 			(void) printf(gettext("Every feature flags pool has "
5241 			    "all supported features enabled.\n"));
5242 		} else {
5243 			(void) printf(gettext("\n"));
5244 		}
5245 	} else {
5246 		ret = for_each_pool(argc, argv, B_FALSE, NULL,
5247 		    upgrade_one, &cb);
5248 	}
5249 
5250 	return (ret);
5251 }
5252 
5253 typedef struct hist_cbdata {
5254 	boolean_t first;
5255 	boolean_t longfmt;
5256 	boolean_t internal;
5257 } hist_cbdata_t;
5258 
5259 /*
5260  * Print out the command history for a specific pool.
5261  */
5262 static int
5263 get_history_one(zpool_handle_t *zhp, void *data)
5264 {
5265 	nvlist_t *nvhis;
5266 	nvlist_t **records;
5267 	uint_t numrecords;
5268 	int ret, i;
5269 	hist_cbdata_t *cb = (hist_cbdata_t *)data;
5270 
5271 	cb->first = B_FALSE;
5272 
5273 	(void) printf(gettext("History for '%s':\n"), zpool_get_name(zhp));
5274 
5275 	if ((ret = zpool_get_history(zhp, &nvhis)) != 0)
5276 		return (ret);
5277 
5278 	verify(nvlist_lookup_nvlist_array(nvhis, ZPOOL_HIST_RECORD,
5279 	    &records, &numrecords) == 0);
5280 	for (i = 0; i < numrecords; i++) {
5281 		nvlist_t *rec = records[i];
5282 		char tbuf[30] = "";
5283 
5284 		if (nvlist_exists(rec, ZPOOL_HIST_TIME)) {
5285 			time_t tsec;
5286 			struct tm t;
5287 
5288 			tsec = fnvlist_lookup_uint64(records[i],
5289 			    ZPOOL_HIST_TIME);
5290 			(void) localtime_r(&tsec, &t);
5291 			(void) strftime(tbuf, sizeof (tbuf), "%F.%T", &t);
5292 		}
5293 
5294 		if (nvlist_exists(rec, ZPOOL_HIST_CMD)) {
5295 			(void) printf("%s %s", tbuf,
5296 			    fnvlist_lookup_string(rec, ZPOOL_HIST_CMD));
5297 		} else if (nvlist_exists(rec, ZPOOL_HIST_INT_EVENT)) {
5298 			int ievent =
5299 			    fnvlist_lookup_uint64(rec, ZPOOL_HIST_INT_EVENT);
5300 			if (!cb->internal)
5301 				continue;
5302 			if (ievent >= ZFS_NUM_LEGACY_HISTORY_EVENTS) {
5303 				(void) printf("%s unrecognized record:\n",
5304 				    tbuf);
5305 				dump_nvlist(rec, 4);
5306 				continue;
5307 			}
5308 			(void) printf("%s [internal %s txg:%lld] %s", tbuf,
5309 			    zfs_history_event_names[ievent],
5310 			    fnvlist_lookup_uint64(rec, ZPOOL_HIST_TXG),
5311 			    fnvlist_lookup_string(rec, ZPOOL_HIST_INT_STR));
5312 		} else if (nvlist_exists(rec, ZPOOL_HIST_INT_NAME)) {
5313 			if (!cb->internal)
5314 				continue;
5315 			(void) printf("%s [txg:%lld] %s", tbuf,
5316 			    fnvlist_lookup_uint64(rec, ZPOOL_HIST_TXG),
5317 			    fnvlist_lookup_string(rec, ZPOOL_HIST_INT_NAME));
5318 			if (nvlist_exists(rec, ZPOOL_HIST_DSNAME)) {
5319 				(void) printf(" %s (%llu)",
5320 				    fnvlist_lookup_string(rec,
5321 				    ZPOOL_HIST_DSNAME),
5322 				    fnvlist_lookup_uint64(rec,
5323 				    ZPOOL_HIST_DSID));
5324 			}
5325 			(void) printf(" %s", fnvlist_lookup_string(rec,
5326 			    ZPOOL_HIST_INT_STR));
5327 		} else if (nvlist_exists(rec, ZPOOL_HIST_IOCTL)) {
5328 			if (!cb->internal)
5329 				continue;
5330 			(void) printf("%s ioctl %s\n", tbuf,
5331 			    fnvlist_lookup_string(rec, ZPOOL_HIST_IOCTL));
5332 			if (nvlist_exists(rec, ZPOOL_HIST_INPUT_NVL)) {
5333 				(void) printf("    input:\n");
5334 				dump_nvlist(fnvlist_lookup_nvlist(rec,
5335 				    ZPOOL_HIST_INPUT_NVL), 8);
5336 			}
5337 			if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_NVL)) {
5338 				(void) printf("    output:\n");
5339 				dump_nvlist(fnvlist_lookup_nvlist(rec,
5340 				    ZPOOL_HIST_OUTPUT_NVL), 8);
5341 			}
5342 			if (nvlist_exists(rec, ZPOOL_HIST_ERRNO)) {
5343 				(void) printf("    errno: %lld\n",
5344 				    fnvlist_lookup_int64(rec,
5345 				    ZPOOL_HIST_ERRNO));
5346 			}
5347 		} else {
5348 			if (!cb->internal)
5349 				continue;
5350 			(void) printf("%s unrecognized record:\n", tbuf);
5351 			dump_nvlist(rec, 4);
5352 		}
5353 
5354 		if (!cb->longfmt) {
5355 			(void) printf("\n");
5356 			continue;
5357 		}
5358 		(void) printf(" [");
5359 		if (nvlist_exists(rec, ZPOOL_HIST_WHO)) {
5360 			uid_t who = fnvlist_lookup_uint64(rec, ZPOOL_HIST_WHO);
5361 			struct passwd *pwd = getpwuid(who);
5362 			(void) printf("user %d ", (int)who);
5363 			if (pwd != NULL)
5364 				(void) printf("(%s) ", pwd->pw_name);
5365 		}
5366 		if (nvlist_exists(rec, ZPOOL_HIST_HOST)) {
5367 			(void) printf("on %s",
5368 			    fnvlist_lookup_string(rec, ZPOOL_HIST_HOST));
5369 		}
5370 		if (nvlist_exists(rec, ZPOOL_HIST_ZONE)) {
5371 			(void) printf(":%s",
5372 			    fnvlist_lookup_string(rec, ZPOOL_HIST_ZONE));
5373 		}
5374 		(void) printf("]");
5375 		(void) printf("\n");
5376 	}
5377 	(void) printf("\n");
5378 	nvlist_free(nvhis);
5379 
5380 	return (ret);
5381 }
5382 
5383 /*
5384  * zpool history <pool>
5385  *
5386  * Displays the history of commands that modified pools.
5387  */
5388 int
5389 zpool_do_history(int argc, char **argv)
5390 {
5391 	hist_cbdata_t cbdata = { 0 };
5392 	int ret;
5393 	int c;
5394 
5395 	cbdata.first = B_TRUE;
5396 	/* check options */
5397 	while ((c = getopt(argc, argv, "li")) != -1) {
5398 		switch (c) {
5399 		case 'l':
5400 			cbdata.longfmt = B_TRUE;
5401 			break;
5402 		case 'i':
5403 			cbdata.internal = B_TRUE;
5404 			break;
5405 		case '?':
5406 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
5407 			    optopt);
5408 			usage(B_FALSE);
5409 		}
5410 	}
5411 	argc -= optind;
5412 	argv += optind;
5413 
5414 	ret = for_each_pool(argc, argv, B_FALSE,  NULL, get_history_one,
5415 	    &cbdata);
5416 
5417 	if (argc == 0 && cbdata.first == B_TRUE) {
5418 		(void) printf(gettext("no pools available\n"));
5419 		return (0);
5420 	}
5421 
5422 	return (ret);
5423 }
5424 
5425 static int
5426 get_callback(zpool_handle_t *zhp, void *data)
5427 {
5428 	zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
5429 	char value[MAXNAMELEN];
5430 	zprop_source_t srctype;
5431 	zprop_list_t *pl;
5432 
5433 	for (pl = cbp->cb_proplist; pl != NULL; pl = pl->pl_next) {
5434 
5435 		/*
5436 		 * Skip the special fake placeholder. This will also skip
5437 		 * over the name property when 'all' is specified.
5438 		 */
5439 		if (pl->pl_prop == ZPOOL_PROP_NAME &&
5440 		    pl == cbp->cb_proplist)
5441 			continue;
5442 
5443 		if (pl->pl_prop == ZPROP_INVAL &&
5444 		    (zpool_prop_feature(pl->pl_user_prop) ||
5445 		    zpool_prop_unsupported(pl->pl_user_prop))) {
5446 			srctype = ZPROP_SRC_LOCAL;
5447 
5448 			if (zpool_prop_get_feature(zhp, pl->pl_user_prop,
5449 			    value, sizeof (value)) == 0) {
5450 				zprop_print_one_property(zpool_get_name(zhp),
5451 				    cbp, pl->pl_user_prop, value, srctype,
5452 				    NULL, NULL);
5453 			}
5454 		} else {
5455 			if (zpool_get_prop(zhp, pl->pl_prop, value,
5456 			    sizeof (value), &srctype, cbp->cb_literal) != 0)
5457 				continue;
5458 
5459 			zprop_print_one_property(zpool_get_name(zhp), cbp,
5460 			    zpool_prop_to_name(pl->pl_prop), value, srctype,
5461 			    NULL, NULL);
5462 		}
5463 	}
5464 	return (0);
5465 }
5466 
5467 /*
5468  * zpool get [-Hp] [-o "all" | field[,...]] <"all" | property[,...]> <pool> ...
5469  *
5470  *	-H	Scripted mode.  Don't display headers, and separate properties
5471  *		by a single tab.
5472  *	-o	List of columns to display.  Defaults to
5473  *		"name,property,value,source".
5474  * 	-p	Diplay values in parsable (exact) format.
5475  *
5476  * Get properties of pools in the system. Output space statistics
5477  * for each one as well as other attributes.
5478  */
5479 int
5480 zpool_do_get(int argc, char **argv)
5481 {
5482 	zprop_get_cbdata_t cb = { 0 };
5483 	zprop_list_t fake_name = { 0 };
5484 	int ret;
5485 	int c, i;
5486 	char *value;
5487 
5488 	cb.cb_first = B_TRUE;
5489 
5490 	/*
5491 	 * Set up default columns and sources.
5492 	 */
5493 	cb.cb_sources = ZPROP_SRC_ALL;
5494 	cb.cb_columns[0] = GET_COL_NAME;
5495 	cb.cb_columns[1] = GET_COL_PROPERTY;
5496 	cb.cb_columns[2] = GET_COL_VALUE;
5497 	cb.cb_columns[3] = GET_COL_SOURCE;
5498 	cb.cb_type = ZFS_TYPE_POOL;
5499 
5500 	/* check options */
5501 	while ((c = getopt(argc, argv, ":Hpo:")) != -1) {
5502 		switch (c) {
5503 		case 'p':
5504 			cb.cb_literal = B_TRUE;
5505 			break;
5506 		case 'H':
5507 			cb.cb_scripted = B_TRUE;
5508 			break;
5509 		case 'o':
5510 			bzero(&cb.cb_columns, sizeof (cb.cb_columns));
5511 			i = 0;
5512 			while (*optarg != '\0') {
5513 				static char *col_subopts[] =
5514 				{ "name", "property", "value", "source",
5515 				"all", NULL };
5516 
5517 				if (i == ZFS_GET_NCOLS) {
5518 					(void) fprintf(stderr, gettext("too "
5519 					"many fields given to -o "
5520 					"option\n"));
5521 					usage(B_FALSE);
5522 				}
5523 
5524 				switch (getsubopt(&optarg, col_subopts,
5525 				    &value)) {
5526 				case 0:
5527 					cb.cb_columns[i++] = GET_COL_NAME;
5528 					break;
5529 				case 1:
5530 					cb.cb_columns[i++] = GET_COL_PROPERTY;
5531 					break;
5532 				case 2:
5533 					cb.cb_columns[i++] = GET_COL_VALUE;
5534 					break;
5535 				case 3:
5536 					cb.cb_columns[i++] = GET_COL_SOURCE;
5537 					break;
5538 				case 4:
5539 					if (i > 0) {
5540 						(void) fprintf(stderr,
5541 						    gettext("\"all\" conflicts "
5542 						    "with specific fields "
5543 						    "given to -o option\n"));
5544 						usage(B_FALSE);
5545 					}
5546 					cb.cb_columns[0] = GET_COL_NAME;
5547 					cb.cb_columns[1] = GET_COL_PROPERTY;
5548 					cb.cb_columns[2] = GET_COL_VALUE;
5549 					cb.cb_columns[3] = GET_COL_SOURCE;
5550 					i = ZFS_GET_NCOLS;
5551 					break;
5552 				default:
5553 					(void) fprintf(stderr,
5554 					    gettext("invalid column name "
5555 					    "'%s'\n"), value);
5556 					usage(B_FALSE);
5557 				}
5558 			}
5559 			break;
5560 		case '?':
5561 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
5562 			    optopt);
5563 			usage(B_FALSE);
5564 		}
5565 	}
5566 
5567 	argc -= optind;
5568 	argv += optind;
5569 
5570 	if (argc < 1) {
5571 		(void) fprintf(stderr, gettext("missing property "
5572 		    "argument\n"));
5573 		usage(B_FALSE);
5574 	}
5575 
5576 	if (zprop_get_list(g_zfs, argv[0], &cb.cb_proplist,
5577 	    ZFS_TYPE_POOL) != 0)
5578 		usage(B_FALSE);
5579 
5580 	argc--;
5581 	argv++;
5582 
5583 	if (cb.cb_proplist != NULL) {
5584 		fake_name.pl_prop = ZPOOL_PROP_NAME;
5585 		fake_name.pl_width = strlen(gettext("NAME"));
5586 		fake_name.pl_next = cb.cb_proplist;
5587 		cb.cb_proplist = &fake_name;
5588 	}
5589 
5590 	ret = for_each_pool(argc, argv, B_TRUE, &cb.cb_proplist,
5591 	    get_callback, &cb);
5592 
5593 	if (cb.cb_proplist == &fake_name)
5594 		zprop_free_list(fake_name.pl_next);
5595 	else
5596 		zprop_free_list(cb.cb_proplist);
5597 
5598 	return (ret);
5599 }
5600 
5601 typedef struct set_cbdata {
5602 	char *cb_propname;
5603 	char *cb_value;
5604 	boolean_t cb_any_successful;
5605 } set_cbdata_t;
5606 
5607 int
5608 set_callback(zpool_handle_t *zhp, void *data)
5609 {
5610 	int error;
5611 	set_cbdata_t *cb = (set_cbdata_t *)data;
5612 
5613 	error = zpool_set_prop(zhp, cb->cb_propname, cb->cb_value);
5614 
5615 	if (!error)
5616 		cb->cb_any_successful = B_TRUE;
5617 
5618 	return (error);
5619 }
5620 
5621 int
5622 zpool_do_set(int argc, char **argv)
5623 {
5624 	set_cbdata_t cb = { 0 };
5625 	int error;
5626 
5627 	if (argc > 1 && argv[1][0] == '-') {
5628 		(void) fprintf(stderr, gettext("invalid option '%c'\n"),
5629 		    argv[1][1]);
5630 		usage(B_FALSE);
5631 	}
5632 
5633 	if (argc < 2) {
5634 		(void) fprintf(stderr, gettext("missing property=value "
5635 		    "argument\n"));
5636 		usage(B_FALSE);
5637 	}
5638 
5639 	if (argc < 3) {
5640 		(void) fprintf(stderr, gettext("missing pool name\n"));
5641 		usage(B_FALSE);
5642 	}
5643 
5644 	if (argc > 3) {
5645 		(void) fprintf(stderr, gettext("too many pool names\n"));
5646 		usage(B_FALSE);
5647 	}
5648 
5649 	cb.cb_propname = argv[1];
5650 	cb.cb_value = strchr(cb.cb_propname, '=');
5651 	if (cb.cb_value == NULL) {
5652 		(void) fprintf(stderr, gettext("missing value in "
5653 		    "property=value argument\n"));
5654 		usage(B_FALSE);
5655 	}
5656 
5657 	*(cb.cb_value) = '\0';
5658 	cb.cb_value++;
5659 
5660 	error = for_each_pool(argc - 2, argv + 2, B_TRUE, NULL,
5661 	    set_callback, &cb);
5662 
5663 	return (error);
5664 }
5665 
5666 static int
5667 find_command_idx(char *command, int *idx)
5668 {
5669 	int i;
5670 
5671 	for (i = 0; i < NCOMMAND; i++) {
5672 		if (command_table[i].name == NULL)
5673 			continue;
5674 
5675 		if (strcmp(command, command_table[i].name) == 0) {
5676 			*idx = i;
5677 			return (0);
5678 		}
5679 	}
5680 	return (1);
5681 }
5682 
5683 int
5684 main(int argc, char **argv)
5685 {
5686 	int ret = 0;
5687 	int i;
5688 	char *cmdname;
5689 
5690 	(void) setlocale(LC_ALL, "");
5691 	(void) textdomain(TEXT_DOMAIN);
5692 
5693 	if ((g_zfs = libzfs_init()) == NULL) {
5694 		(void) fprintf(stderr, gettext("internal error: failed to "
5695 		    "initialize ZFS library\n"));
5696 		return (1);
5697 	}
5698 
5699 	libzfs_print_on_error(g_zfs, B_TRUE);
5700 
5701 	opterr = 0;
5702 
5703 	/*
5704 	 * Make sure the user has specified some command.
5705 	 */
5706 	if (argc < 2) {
5707 		(void) fprintf(stderr, gettext("missing command\n"));
5708 		usage(B_FALSE);
5709 	}
5710 
5711 	cmdname = argv[1];
5712 
5713 	/*
5714 	 * Special case '-?'
5715 	 */
5716 	if (strcmp(cmdname, "-?") == 0)
5717 		usage(B_TRUE);
5718 
5719 	zfs_save_arguments(argc, argv, history_str, sizeof (history_str));
5720 
5721 	/*
5722 	 * Run the appropriate command.
5723 	 */
5724 	if (find_command_idx(cmdname, &i) == 0) {
5725 		current_command = &command_table[i];
5726 		ret = command_table[i].func(argc - 1, argv + 1);
5727 	} else if (strchr(cmdname, '=')) {
5728 		verify(find_command_idx("set", &i) == 0);
5729 		current_command = &command_table[i];
5730 		ret = command_table[i].func(argc, argv);
5731 	} else if (strcmp(cmdname, "freeze") == 0 && argc == 3) {
5732 		/*
5733 		 * 'freeze' is a vile debugging abomination, so we treat
5734 		 * it as such.
5735 		 */
5736 		char buf[16384];
5737 		int fd = open(ZFS_DEV, O_RDWR);
5738 		(void) strcpy((void *)buf, argv[2]);
5739 		return (!!ioctl(fd, ZFS_IOC_POOL_FREEZE, buf));
5740 	} else {
5741 		(void) fprintf(stderr, gettext("unrecognized "
5742 		    "command '%s'\n"), cmdname);
5743 		usage(B_FALSE);
5744 	}
5745 
5746 	if (ret == 0 && log_history)
5747 		(void) zpool_log_history(g_zfs, history_str);
5748 
5749 	libzfs_fini(g_zfs);
5750 
5751 	/*
5752 	 * The 'ZFS_ABORT' environment variable causes us to dump core on exit
5753 	 * for the purposes of running ::findleaks.
5754 	 */
5755 	if (getenv("ZFS_ABORT") != NULL) {
5756 		(void) printf("dumping core by request\n");
5757 		abort();
5758 	}
5759 
5760 	return (ret);
5761 }
5762