12c601221SChris Williamson#!/bin/bash 22c601221SChris Williamson 32c601221SChris Williamson# 42c601221SChris Williamson# CDDL HEADER START 52c601221SChris Williamson# 62c601221SChris Williamson# This file and its contents are supplied under the terms of the 72c601221SChris Williamson# Common Development and Distribution License ("CDDL"), version 1.0. 82c601221SChris Williamson# You may only use this file in accordance with the terms of version 92c601221SChris Williamson# 1.0 of the CDDL. 102c601221SChris Williamson# 112c601221SChris Williamson# A full copy of the text of the CDDL should have accompanied this 122c601221SChris Williamson# source. A copy of the CDDL is also available via the Internet at 132c601221SChris Williamson# http://www.illumos.org/license/CDDL. 142c601221SChris Williamson# 152c601221SChris Williamson# CDDL HEADER END 162c601221SChris Williamson# 172c601221SChris Williamson 182c601221SChris Williamson# 19*213fcdcbSJohn Wren Kennedy# Copyright (c) 2015, 2017 by Delphix. All rights reserved. 202c601221SChris Williamson# 212c601221SChris Williamson 222c601221SChris Williamsonset -x 232c601221SChris Williamsonexport BITS=64 242c601221SChris Williamsonexport UMEM_DEBUG=default,verbose 252c601221SChris Williamsonexport UMEM_LOGGING=transaction,contents 262c601221SChris Williamsonset +x 272c601221SChris Williamson 282c601221SChris Williamsonsparc_32=sparc 292c601221SChris Williamsonsparc_64=sparcv9 302c601221SChris Williamsoni386_32=i86 312c601221SChris Williamsoni386_64=amd64 322c601221SChris WilliamsonARCH=`uname -p` 332c601221SChris Williamsoneval 'ARCHBITS=${'"${ARCH}_${BITS}"'}' 342c601221SChris WilliamsonBIN=$ROOT/usr/bin/${ARCHBITS} 352c601221SChris WilliamsonSBIN=$ROOT/usr/sbin/${ARCHBITS} 362c601221SChris WilliamsonDEFAULTWORKDIR=/var/tmp 372c601221SChris WilliamsonDEFAULTCOREDIR=/var/tmp/zloop 382c601221SChris Williamson 392c601221SChris Williamsonfunction usage 402c601221SChris Williamson{ 412c601221SChris Williamson echo -e "\n$0 [-t <timeout>] [-c <dump directory>]" \ 422c601221SChris Williamson "[ -- [extra ztest parameters]]\n" \ 432c601221SChris Williamson "\n" \ 442c601221SChris Williamson " This script runs ztest repeatedly with randomized arguments.\n" \ 452c601221SChris Williamson " If a crash is encountered, the ztest logs, any associated\n" \ 462c601221SChris Williamson " vdev files, and core file (if one exists) are moved to the\n" \ 472c601221SChris Williamson " output directory ($DEFAULTCOREDIR by default). Any options\n" \ 482c601221SChris Williamson " after the -- end-of-options marker will be passed to ztest.\n" \ 492c601221SChris Williamson "\n" \ 502c601221SChris Williamson " Options:\n" \ 512c601221SChris Williamson " -t Total time to loop for, in seconds. If not provided,\n" \ 522c601221SChris Williamson " zloop runs forever.\n" \ 532c601221SChris Williamson " -f Specify working directory for ztest vdev files.\n" \ 542c601221SChris Williamson " -c Specify a core dump directory to use.\n" \ 552c601221SChris Williamson " -h Print this help message.\n" \ 562c601221SChris Williamson "" >&2 572c601221SChris Williamson} 582c601221SChris Williamson 592c601221SChris Williamsonfunction or_die 602c601221SChris Williamson{ 612c601221SChris Williamson $@ 622c601221SChris Williamson if [[ $? -ne 0 ]]; then 632c601221SChris Williamson echo "Command failed: $@" 642c601221SChris Williamson exit 1 652c601221SChris Williamson fi 662c601221SChris Williamson} 672c601221SChris Williamson 682c601221SChris Williamsonfunction store_core 692c601221SChris Williamson{ 702c601221SChris Williamson if [[ $ztrc -ne 0 ]] || [[ -f core ]]; then 712c601221SChris Williamson coreid=$(/bin/date "+zloop-%y%m%d-%H%M%S") 722c601221SChris Williamson foundcrashes=$(($foundcrashes + 1)) 732c601221SChris Williamson 742c601221SChris Williamson dest=$coredir/$coreid 752c601221SChris Williamson or_die /bin/mkdir $dest 762c601221SChris Williamson or_die /bin/mkdir $dest/vdev 772c601221SChris Williamson 782c601221SChris Williamson echo "*** ztest crash found - moving logs to $coredir/$coreid" 792c601221SChris Williamson 802c601221SChris Williamson or_die /bin/mv ztest.history $dest/ 812c601221SChris Williamson or_die /bin/mv ztest.out $dest/ 822c601221SChris Williamson or_die /bin/mv $workdir/ztest* $dest/vdev/ 832c601221SChris Williamson or_die /bin/mv $workdir/zpool.cache $dest/vdev/ 842c601221SChris Williamson 852c601221SChris Williamson # check for core 862c601221SChris Williamson if [[ -f core ]]; then 872c601221SChris Williamson corestatus=$(mdb -e "::status" core) 882c601221SChris Williamson corestack=$(mdb -e "::stack" core) 892c601221SChris Williamson 902c601221SChris Williamson # Dump core + logs to stored directory 912c601221SChris Williamson echo "$corestatus" >>$dest/status 922c601221SChris Williamson echo "$corestack" >>$dest/status 932c601221SChris Williamson or_die /bin/mv core $dest/ 942c601221SChris Williamson 952c601221SChris Williamson # Record info in cores logfile 962c601221SChris Williamson echo "*** core @ $coredir/$coreid/core:" | /bin/tee -a ztest.cores 972c601221SChris Williamson echo "$corestatus" | /bin/tee -a ztest.cores 982c601221SChris Williamson echo "$corestack" | /bin/tee -a ztest.cores 992c601221SChris Williamson echo "" | /bin/tee -a ztest.cores 1002c601221SChris Williamson fi 1012c601221SChris Williamson echo "continuing..." 1022c601221SChris Williamson fi 1032c601221SChris Williamson} 1042c601221SChris Williamson 1052c601221SChris Williamsonset -x 1062c601221SChris Williamsonexport PATH=${BIN}:${SBIN} 1072c601221SChris Williamsonexport LD_LIBRARY_PATH=$ROOT/lib/$BITS:$ROOT/usr/lib/$BITS 1082c601221SChris Williamsonset +x 1092c601221SChris Williamson 1102c601221SChris Williamson# parse arguments 1112c601221SChris Williamson# expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args] 1122c601221SChris Williamsoncoredir=$DEFAULTCOREDIR 1132c601221SChris Williamsonworkdir=$DEFAULTWORKDIR 1142c601221SChris Williamsontimeout=0 1152c601221SChris Williamsonwhile getopts ":ht:c:f:" opt; do 1162c601221SChris Williamson case $opt in 1172c601221SChris Williamson t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;; 1182c601221SChris Williamson c ) [[ $OPTARG ]] && coredir=$OPTARG ;; 1192c601221SChris Williamson f ) [[ $OPTARG ]] && workdir=$(/usr/bin/readlink -f $OPTARG) ;; 1202c601221SChris Williamson h ) usage 1212c601221SChris Williamson exit 2 1222c601221SChris Williamson ;; 1232c601221SChris Williamson * ) echo "Invalid argument: -$OPTARG"; 1242c601221SChris Williamson usage 1252c601221SChris Williamson exit 1 1262c601221SChris Williamson esac 1272c601221SChris Williamsondone 1282c601221SChris Williamson# pass remaining arguments on to ztest 1292c601221SChris Williamsonshift $((OPTIND - 1)) 1302c601221SChris Williamson 1312c601221SChris Williamsonif [[ -f core ]]; then 1322c601221SChris Williamson echo "There's a core dump here you might want to look at first." 1332c601221SChris Williamson exit 1 1342c601221SChris Williamsonfi 1352c601221SChris Williamson 1362c601221SChris Williamsonif [[ ! -d $coredir ]]; then 1372c601221SChris Williamson echo "core dump directory ($coredir) does not exist, creating it." 1382c601221SChris Williamson or_die /bin/mkdir -p $coredir 1392c601221SChris Williamsonfi 1402c601221SChris Williamson 1412c601221SChris Williamsonif [[ ! -w $coredir ]]; then 1422c601221SChris Williamson echo "core dump directory ($coredir) is not writable." 1432c601221SChris Williamson exit 1 1442c601221SChris Williamsonfi 1452c601221SChris Williamson 1462c601221SChris Williamsonor_die /bin/rm -f ztest.history 1472c601221SChris Williamsonor_die /bin/rm -f ztest.cores 1482c601221SChris Williamson 149*213fcdcbSJohn Wren Kennedy# Allow core files to be written to cwd if that's currently disabled. 150*213fcdcbSJohn Wren Kennedysudo coreadm -e process 151*213fcdcbSJohn Wren Kennedy 1522c601221SChris Williamsonztrc=0 # ztest return value 1532c601221SChris Williamsonfoundcrashes=0 # number of crashes found so far 1542c601221SChris Williamsonstarttime=$(/bin/date +%s) 1552c601221SChris Williamsoncurtime=$starttime 1562c601221SChris Williamson 1572c601221SChris Williamson# if no timeout was specified, loop forever. 1582c601221SChris Williamsonwhile [[ $timeout -eq 0 ]] || [[ $curtime -le $(($starttime + $timeout)) ]]; do 1592c601221SChris Williamson zopt="-VVVVV" 1602c601221SChris Williamson 1612c601221SChris Williamson # switch between common arrangements & fully randomized 1622c601221SChris Williamson if [[ $((RANDOM % 2)) -eq 0 ]]; then 1632c601221SChris Williamson mirrors=2 1642c601221SChris Williamson raidz=0 1652c601221SChris Williamson parity=1 1662c601221SChris Williamson vdevs=2 1672c601221SChris Williamson else 1682c601221SChris Williamson mirrors=$(((RANDOM % 3) * 1)) 1692c601221SChris Williamson parity=$(((RANDOM % 3) + 1)) 1702c601221SChris Williamson raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2))) 1712c601221SChris Williamson vdevs=$(((RANDOM % 3) + 3)) 1722c601221SChris Williamson fi 1732c601221SChris Williamson align=$(((RANDOM % 2) * 3 + 9)) 1742c601221SChris Williamson runtime=$((RANDOM % 100)) 1752c601221SChris Williamson passtime=$((RANDOM % (runtime / 3 + 1) + 10)) 1762c601221SChris Williamson size=128m 1772c601221SChris Williamson 1782c601221SChris Williamson zopt="$zopt -m $mirrors" 1792c601221SChris Williamson zopt="$zopt -r $raidz" 1802c601221SChris Williamson zopt="$zopt -R $parity" 1812c601221SChris Williamson zopt="$zopt -v $vdevs" 1822c601221SChris Williamson zopt="$zopt -a $align" 1832c601221SChris Williamson zopt="$zopt -T $runtime" 1842c601221SChris Williamson zopt="$zopt -P $passtime" 1852c601221SChris Williamson zopt="$zopt -s $size" 1862c601221SChris Williamson zopt="$zopt -f $workdir" 1872c601221SChris Williamson 1882c601221SChris Williamson cmd="ztest $zopt $@" 1892c601221SChris Williamson desc="$(/bin/date '+%m/%d %T') $cmd" 1902c601221SChris Williamson echo "$desc" | /bin/tee -a ztest.history 1912c601221SChris Williamson echo "$desc" >>ztest.out 1922c601221SChris Williamson $BIN/$cmd >>ztest.out 2>&1 1932c601221SChris Williamson ztrc=$? 1942c601221SChris Williamson /bin/egrep '===|WARNING' ztest.out >>ztest.history 1951271e4b1SPrakash Surya $SBIN/zdb -U $workdir/zpool.cache -DD ztest >>ztest.ddt 2>&1 1962c601221SChris Williamson 1972c601221SChris Williamson store_core 1982c601221SChris Williamson 1992c601221SChris Williamson curtime=$(/bin/date +%s) 2002c601221SChris Williamsondone 2012c601221SChris Williamson 2022c601221SChris Williamsonecho "zloop finished, $foundcrashes crashes found" 2032c601221SChris Williamson 2042c601221SChris Williamson/bin/uptime >>ztest.out 2052c601221SChris Williamson 2062c601221SChris Williamsonif [[ $foundcrashes -gt 0 ]]; then 2072c601221SChris Williamson exit 1 2082c601221SChris Williamsonfi 209