xref: /illumos-gate/usr/src/cmd/ztest/zloop.bash (revision 1672d299)
12c601221SChris Williamson#!/bin/bash
22c601221SChris Williamson
32c601221SChris Williamson#
42c601221SChris Williamson# CDDL HEADER START
52c601221SChris Williamson#
62c601221SChris Williamson# This file and its contents are supplied under the terms of the
72c601221SChris Williamson# Common Development and Distribution License ("CDDL"), version 1.0.
82c601221SChris Williamson# You may only use this file in accordance with the terms of version
92c601221SChris Williamson# 1.0 of the CDDL.
102c601221SChris Williamson#
112c601221SChris Williamson# A full copy of the text of the CDDL should have accompanied this
122c601221SChris Williamson# source.  A copy of the CDDL is also available via the Internet at
132c601221SChris Williamson# http://www.illumos.org/license/CDDL.
142c601221SChris Williamson#
152c601221SChris Williamson# CDDL HEADER END
162c601221SChris Williamson#
172c601221SChris Williamson
182c601221SChris Williamson#
19213fcdcbSJohn Wren Kennedy# Copyright (c) 2015, 2017 by Delphix. All rights reserved.
202c601221SChris Williamson#
212c601221SChris Williamson
222c601221SChris Williamsonset -x
232c601221SChris Williamsonexport UMEM_DEBUG=default,verbose
242c601221SChris Williamsonexport UMEM_LOGGING=transaction,contents
252c601221SChris Williamsonset +x
262c601221SChris Williamson
272c601221SChris WilliamsonDEFAULTWORKDIR=/var/tmp
282c601221SChris WilliamsonDEFAULTCOREDIR=/var/tmp/zloop
292c601221SChris Williamson
302c601221SChris Williamsonfunction usage
312c601221SChris Williamson{
322c601221SChris Williamson	echo -e "\n$0 [-t <timeout>] [-c <dump directory>]" \
332c601221SChris Williamson	    "[ -- [extra ztest parameters]]\n" \
342c601221SChris Williamson	    "\n" \
352c601221SChris Williamson	    "  This script runs ztest repeatedly with randomized arguments.\n" \
362c601221SChris Williamson	    "  If a crash is encountered, the ztest logs, any associated\n" \
372c601221SChris Williamson	    "  vdev files, and core file (if one exists) are moved to the\n" \
382c601221SChris Williamson	    "  output directory ($DEFAULTCOREDIR by default). Any options\n" \
392c601221SChris Williamson	    "  after the -- end-of-options marker will be passed to ztest.\n" \
402c601221SChris Williamson	    "\n" \
412c601221SChris Williamson	    "  Options:\n" \
422c601221SChris Williamson	    "    -t  Total time to loop for, in seconds. If not provided,\n" \
432c601221SChris Williamson	    "        zloop runs forever.\n" \
442c601221SChris Williamson	    "    -f  Specify working directory for ztest vdev files.\n" \
452c601221SChris Williamson	    "    -c  Specify a core dump directory to use.\n" \
462c601221SChris Williamson	    "    -h  Print this help message.\n" \
472c601221SChris Williamson	    "" >&2
482c601221SChris Williamson}
492c601221SChris Williamson
502c601221SChris Williamsonfunction or_die
512c601221SChris Williamson{
522c601221SChris Williamson	$@
532c601221SChris Williamson	if [[ $? -ne 0 ]]; then
542c601221SChris Williamson		echo "Command failed: $@"
552c601221SChris Williamson		exit 1
562c601221SChris Williamson	fi
572c601221SChris Williamson}
582c601221SChris Williamson
592c601221SChris Williamsonfunction store_core
602c601221SChris Williamson{
612c601221SChris Williamson	if [[ $ztrc -ne 0 ]] || [[ -f core ]]; then
622c601221SChris Williamson		coreid=$(/bin/date "+zloop-%y%m%d-%H%M%S")
632c601221SChris Williamson		foundcrashes=$(($foundcrashes + 1))
642c601221SChris Williamson
652c601221SChris Williamson		dest=$coredir/$coreid
662c601221SChris Williamson		or_die /bin/mkdir $dest
672c601221SChris Williamson		or_die /bin/mkdir $dest/vdev
682c601221SChris Williamson
692c601221SChris Williamson		echo "*** ztest crash found - moving logs to $coredir/$coreid"
702c601221SChris Williamson
712c601221SChris Williamson		or_die /bin/mv ztest.history $dest/
722c601221SChris Williamson		or_die /bin/mv ztest.out $dest/
732c601221SChris Williamson		or_die /bin/mv $workdir/ztest* $dest/vdev/
742c601221SChris Williamson		or_die /bin/mv $workdir/zpool.cache $dest/vdev/
752c601221SChris Williamson
762c601221SChris Williamson		# check for core
772c601221SChris Williamson		if [[ -f core ]]; then
782c601221SChris Williamson			corestatus=$(mdb -e "::status" core)
792c601221SChris Williamson			corestack=$(mdb -e "::stack" core)
802c601221SChris Williamson
812c601221SChris Williamson			# Dump core + logs to stored directory
822c601221SChris Williamson			echo "$corestatus" >>$dest/status
832c601221SChris Williamson			echo "$corestack" >>$dest/status
842c601221SChris Williamson			or_die /bin/mv core $dest/
852c601221SChris Williamson
862c601221SChris Williamson			# Record info in cores logfile
872c601221SChris Williamson			echo "*** core @ $coredir/$coreid/core:" | /bin/tee -a ztest.cores
882c601221SChris Williamson			echo "$corestatus" | /bin/tee -a ztest.cores
892c601221SChris Williamson			echo "$corestack" | /bin/tee -a ztest.cores
902c601221SChris Williamson			echo "" | /bin/tee -a ztest.cores
912c601221SChris Williamson		fi
922c601221SChris Williamson		echo "continuing..."
932c601221SChris Williamson	fi
942c601221SChris Williamson}
952c601221SChris Williamson
962c601221SChris Williamson# parse arguments
972c601221SChris Williamson# expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
982c601221SChris Williamsoncoredir=$DEFAULTCOREDIR
992c601221SChris Williamsonworkdir=$DEFAULTWORKDIR
1002c601221SChris Williamsontimeout=0
1012c601221SChris Williamsonwhile getopts ":ht:c:f:" opt; do
1022c601221SChris Williamson	case $opt in
1032c601221SChris Williamson		t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
1042c601221SChris Williamson		c ) [[ $OPTARG ]] && coredir=$OPTARG ;;
1052c601221SChris Williamson		f ) [[ $OPTARG ]] && workdir=$(/usr/bin/readlink -f $OPTARG) ;;
1062c601221SChris Williamson		h ) usage
1072c601221SChris Williamson		    exit 2
1082c601221SChris Williamson		    ;;
1092c601221SChris Williamson		* ) echo "Invalid argument: -$OPTARG";
1102c601221SChris Williamson		    usage
1112c601221SChris Williamson		    exit 1
1122c601221SChris Williamson	esac
1132c601221SChris Williamsondone
1142c601221SChris Williamson# pass remaining arguments on to ztest
1152c601221SChris Williamsonshift $((OPTIND - 1))
1162c601221SChris Williamson
1172c601221SChris Williamsonif [[ -f core ]]; then
1182c601221SChris Williamson	echo "There's a core dump here you might want to look at first."
1192c601221SChris Williamson	exit 1
1202c601221SChris Williamsonfi
1212c601221SChris Williamson
1222c601221SChris Williamsonif [[ ! -d $coredir ]]; then
1232c601221SChris Williamson	echo "core dump directory ($coredir) does not exist, creating it."
1242c601221SChris Williamson	or_die /bin/mkdir -p $coredir
1252c601221SChris Williamsonfi
1262c601221SChris Williamson
1272c601221SChris Williamsonif [[ ! -w $coredir ]]; then
1282c601221SChris Williamson	echo "core dump directory ($coredir) is not writable."
1292c601221SChris Williamson	exit 1
1302c601221SChris Williamsonfi
1312c601221SChris Williamson
1322c601221SChris Williamsonor_die /bin/rm -f ztest.history
1332c601221SChris Williamsonor_die /bin/rm -f ztest.cores
1342c601221SChris Williamson
135213fcdcbSJohn Wren Kennedy# Allow core files to be written to cwd if that's currently disabled.
136213fcdcbSJohn Wren Kennedysudo coreadm -e process
137213fcdcbSJohn Wren Kennedy
1382c601221SChris Williamsonztrc=0		# ztest return value
1392c601221SChris Williamsonfoundcrashes=0	# number of crashes found so far
1402c601221SChris Williamsonstarttime=$(/bin/date +%s)
1412c601221SChris Williamsoncurtime=$starttime
1422c601221SChris Williamson
1432c601221SChris Williamson# if no timeout was specified, loop forever.
1442c601221SChris Williamsonwhile [[ $timeout -eq 0 ]] || [[ $curtime -le $(($starttime + $timeout)) ]]; do
1452c601221SChris Williamson	zopt="-VVVVV"
1462c601221SChris Williamson
1472c601221SChris Williamson	# switch between common arrangements & fully randomized
1482c601221SChris Williamson	if [[ $((RANDOM % 2)) -eq 0 ]]; then
1492c601221SChris Williamson		mirrors=2
1502c601221SChris Williamson		raidz=0
1512c601221SChris Williamson		parity=1
1522c601221SChris Williamson		vdevs=2
1532c601221SChris Williamson	else
1542c601221SChris Williamson		mirrors=$(((RANDOM % 3) * 1))
1552c601221SChris Williamson		parity=$(((RANDOM % 3) + 1))
1562c601221SChris Williamson		raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
1572c601221SChris Williamson		vdevs=$(((RANDOM % 3) + 3))
1582c601221SChris Williamson	fi
1592c601221SChris Williamson	align=$(((RANDOM % 2) * 3 + 9))
1602c601221SChris Williamson	runtime=$((RANDOM % 100))
1612c601221SChris Williamson	passtime=$((RANDOM % (runtime / 3 + 1) + 10))
1622c601221SChris Williamson	size=128m
1632c601221SChris Williamson
1642c601221SChris Williamson	zopt="$zopt -m $mirrors"
1652c601221SChris Williamson	zopt="$zopt -r $raidz"
1662c601221SChris Williamson	zopt="$zopt -R $parity"
1672c601221SChris Williamson	zopt="$zopt -v $vdevs"
1682c601221SChris Williamson	zopt="$zopt -a $align"
1692c601221SChris Williamson	zopt="$zopt -T $runtime"
1702c601221SChris Williamson	zopt="$zopt -P $passtime"
1712c601221SChris Williamson	zopt="$zopt -s $size"
1722c601221SChris Williamson	zopt="$zopt -f $workdir"
1732c601221SChris Williamson
1742c601221SChris Williamson	cmd="ztest $zopt $@"
1752c601221SChris Williamson	desc="$(/bin/date '+%m/%d %T') $cmd"
1762c601221SChris Williamson	echo "$desc" | /bin/tee -a ztest.history
1772c601221SChris Williamson	echo "$desc" >>ztest.out
178*1672d299SYuri Pankov	/usr/bin/$cmd >>ztest.out 2>&1
1792c601221SChris Williamson	ztrc=$?
1802c601221SChris Williamson	/bin/egrep '===|WARNING' ztest.out >>ztest.history
181*1672d299SYuri Pankov	/usr/sbin/zdb -U $workdir/zpool.cache -DD ztest >>ztest.ddt 2>&1
1822c601221SChris Williamson
1832c601221SChris Williamson	store_core
1842c601221SChris Williamson
1852c601221SChris Williamson	curtime=$(/bin/date +%s)
1862c601221SChris Williamsondone
1872c601221SChris Williamson
1882c601221SChris Williamsonecho "zloop finished, $foundcrashes crashes found"
1892c601221SChris Williamson
1902c601221SChris Williamson/bin/uptime >>ztest.out
1912c601221SChris Williamson
1922c601221SChris Williamsonif [[ $foundcrashes -gt 0 ]]; then
1932c601221SChris Williamson	exit 1
1942c601221SChris Williamsonfi
195