xref: /illumos-gate/usr/src/cmd/ztest/zloop.bash (revision 213fcdcb) 
1#!/bin/bash
2
3#
4# CDDL HEADER START
5#
6# This file and its contents are supplied under the terms of the
7# Common Development and Distribution License ("CDDL"), version 1.0.
8# You may only use this file in accordance with the terms of version
9# 1.0 of the CDDL.
10#
11# A full copy of the text of the CDDL should have accompanied this
12# source.  A copy of the CDDL is also available via the Internet at
13# http://www.illumos.org/license/CDDL.
14#
15# CDDL HEADER END
16#
17
18#
19# Copyright (c) 2015, 2017 by Delphix. All rights reserved.
20#
21
22set -x
23export BITS=64
24export UMEM_DEBUG=default,verbose
25export UMEM_LOGGING=transaction,contents
26set +x
27
28sparc_32=sparc
29sparc_64=sparcv9
30i386_32=i86
31i386_64=amd64
32ARCH=`uname -p`
33eval 'ARCHBITS=${'"${ARCH}_${BITS}"'}'
34BIN=$ROOT/usr/bin/${ARCHBITS}
35SBIN=$ROOT/usr/sbin/${ARCHBITS}
36DEFAULTWORKDIR=/var/tmp
37DEFAULTCOREDIR=/var/tmp/zloop
38
39function usage
40{
41	echo -e "\n$0 [-t <timeout>] [-c <dump directory>]" \
42	    "[ -- [extra ztest parameters]]\n" \
43	    "\n" \
44	    "  This script runs ztest repeatedly with randomized arguments.\n" \
45	    "  If a crash is encountered, the ztest logs, any associated\n" \
46	    "  vdev files, and core file (if one exists) are moved to the\n" \
47	    "  output directory ($DEFAULTCOREDIR by default). Any options\n" \
48	    "  after the -- end-of-options marker will be passed to ztest.\n" \
49	    "\n" \
50	    "  Options:\n" \
51	    "    -t  Total time to loop for, in seconds. If not provided,\n" \
52	    "        zloop runs forever.\n" \
53	    "    -f  Specify working directory for ztest vdev files.\n" \
54	    "    -c  Specify a core dump directory to use.\n" \
55	    "    -h  Print this help message.\n" \
56	    "" >&2
57}
58
59function or_die
60{
61	$@
62	if [[ $? -ne 0 ]]; then
63		echo "Command failed: $@"
64		exit 1
65	fi
66}
67
68function store_core
69{
70	if [[ $ztrc -ne 0 ]] || [[ -f core ]]; then
71		coreid=$(/bin/date "+zloop-%y%m%d-%H%M%S")
72		foundcrashes=$(($foundcrashes + 1))
73
74		dest=$coredir/$coreid
75		or_die /bin/mkdir $dest
76		or_die /bin/mkdir $dest/vdev
77
78		echo "*** ztest crash found - moving logs to $coredir/$coreid"
79
80		or_die /bin/mv ztest.history $dest/
81		or_die /bin/mv ztest.out $dest/
82		or_die /bin/mv $workdir/ztest* $dest/vdev/
83		or_die /bin/mv $workdir/zpool.cache $dest/vdev/
84
85		# check for core
86		if [[ -f core ]]; then
87			corestatus=$(mdb -e "::status" core)
88			corestack=$(mdb -e "::stack" core)
89
90			# Dump core + logs to stored directory
91			echo "$corestatus" >>$dest/status
92			echo "$corestack" >>$dest/status
93			or_die /bin/mv core $dest/
94
95			# Record info in cores logfile
96			echo "*** core @ $coredir/$coreid/core:" | /bin/tee -a ztest.cores
97			echo "$corestatus" | /bin/tee -a ztest.cores
98			echo "$corestack" | /bin/tee -a ztest.cores
99			echo "" | /bin/tee -a ztest.cores
100		fi
101		echo "continuing..."
102	fi
103}
104
105set -x
106export PATH=${BIN}:${SBIN}
107export LD_LIBRARY_PATH=$ROOT/lib/$BITS:$ROOT/usr/lib/$BITS
108set +x
109
110# parse arguments
111# expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
112coredir=$DEFAULTCOREDIR
113workdir=$DEFAULTWORKDIR
114timeout=0
115while getopts ":ht:c:f:" opt; do
116	case $opt in
117		t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
118		c ) [[ $OPTARG ]] && coredir=$OPTARG ;;
119		f ) [[ $OPTARG ]] && workdir=$(/usr/bin/readlink -f $OPTARG) ;;
120		h ) usage
121		    exit 2
122		    ;;
123		* ) echo "Invalid argument: -$OPTARG";
124		    usage
125		    exit 1
126	esac
127done
128# pass remaining arguments on to ztest
129shift $((OPTIND - 1))
130
131if [[ -f core ]]; then
132	echo "There's a core dump here you might want to look at first."
133	exit 1
134fi
135
136if [[ ! -d $coredir ]]; then
137	echo "core dump directory ($coredir) does not exist, creating it."
138	or_die /bin/mkdir -p $coredir
139fi
140
141if [[ ! -w $coredir ]]; then
142	echo "core dump directory ($coredir) is not writable."
143	exit 1
144fi
145
146or_die /bin/rm -f ztest.history
147or_die /bin/rm -f ztest.cores
148
149# Allow core files to be written to cwd if that's currently disabled.
150sudo coreadm -e process
151
152ztrc=0		# ztest return value
153foundcrashes=0	# number of crashes found so far
154starttime=$(/bin/date +%s)
155curtime=$starttime
156
157# if no timeout was specified, loop forever.
158while [[ $timeout -eq 0 ]] || [[ $curtime -le $(($starttime + $timeout)) ]]; do
159	zopt="-VVVVV"
160
161	# switch between common arrangements & fully randomized
162	if [[ $((RANDOM % 2)) -eq 0 ]]; then
163		mirrors=2
164		raidz=0
165		parity=1
166		vdevs=2
167	else
168		mirrors=$(((RANDOM % 3) * 1))
169		parity=$(((RANDOM % 3) + 1))
170		raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
171		vdevs=$(((RANDOM % 3) + 3))
172	fi
173	align=$(((RANDOM % 2) * 3 + 9))
174	runtime=$((RANDOM % 100))
175	passtime=$((RANDOM % (runtime / 3 + 1) + 10))
176	size=128m
177
178	zopt="$zopt -m $mirrors"
179	zopt="$zopt -r $raidz"
180	zopt="$zopt -R $parity"
181	zopt="$zopt -v $vdevs"
182	zopt="$zopt -a $align"
183	zopt="$zopt -T $runtime"
184	zopt="$zopt -P $passtime"
185	zopt="$zopt -s $size"
186	zopt="$zopt -f $workdir"
187
188	cmd="ztest $zopt $@"
189	desc="$(/bin/date '+%m/%d %T') $cmd"
190	echo "$desc" | /bin/tee -a ztest.history
191	echo "$desc" >>ztest.out
192	$BIN/$cmd >>ztest.out 2>&1
193	ztrc=$?
194	/bin/egrep '===|WARNING' ztest.out >>ztest.history
195	$SBIN/zdb -U $workdir/zpool.cache -DD ztest >>ztest.ddt 2>&1
196
197	store_core
198
199	curtime=$(/bin/date +%s)
200done
201
202echo "zloop finished, $foundcrashes crashes found"
203
204/bin/uptime >>ztest.out
205
206if [[ $foundcrashes -gt 0 ]]; then
207	exit 1
208fi
209