12c601221SChris Williamson#!/bin/bash 22c601221SChris Williamson 32c601221SChris Williamson# 42c601221SChris Williamson# CDDL HEADER START 52c601221SChris Williamson# 62c601221SChris Williamson# This file and its contents are supplied under the terms of the 72c601221SChris Williamson# Common Development and Distribution License ("CDDL"), version 1.0. 82c601221SChris Williamson# You may only use this file in accordance with the terms of version 92c601221SChris Williamson# 1.0 of the CDDL. 102c601221SChris Williamson# 112c601221SChris Williamson# A full copy of the text of the CDDL should have accompanied this 122c601221SChris Williamson# source. A copy of the CDDL is also available via the Internet at 132c601221SChris Williamson# http://www.illumos.org/license/CDDL. 142c601221SChris Williamson# 152c601221SChris Williamson# CDDL HEADER END 162c601221SChris Williamson# 172c601221SChris Williamson 182c601221SChris Williamson# 19213fcdcbSJohn Wren Kennedy# Copyright (c) 2015, 2017 by Delphix. All rights reserved. 202c601221SChris Williamson# 212c601221SChris Williamson 222c601221SChris Williamsonset -x 232c601221SChris Williamsonexport UMEM_DEBUG=default,verbose 242c601221SChris Williamsonexport UMEM_LOGGING=transaction,contents 252c601221SChris Williamsonset +x 262c601221SChris Williamson 272c601221SChris WilliamsonDEFAULTWORKDIR=/var/tmp 282c601221SChris WilliamsonDEFAULTCOREDIR=/var/tmp/zloop 292c601221SChris Williamson 302c601221SChris Williamsonfunction usage 312c601221SChris Williamson{ 322c601221SChris Williamson echo -e "\n$0 [-t <timeout>] [-c <dump directory>]" \ 332c601221SChris Williamson "[ -- [extra ztest parameters]]\n" \ 342c601221SChris Williamson "\n" \ 352c601221SChris Williamson " This script runs ztest repeatedly with randomized arguments.\n" \ 362c601221SChris Williamson " If a crash is encountered, the ztest logs, any associated\n" \ 372c601221SChris Williamson " vdev files, and core file (if one exists) are moved to the\n" \ 382c601221SChris Williamson " output directory ($DEFAULTCOREDIR by default). Any options\n" \ 392c601221SChris Williamson " after the -- end-of-options marker will be passed to ztest.\n" \ 402c601221SChris Williamson "\n" \ 412c601221SChris Williamson " Options:\n" \ 422c601221SChris Williamson " -t Total time to loop for, in seconds. If not provided,\n" \ 432c601221SChris Williamson " zloop runs forever.\n" \ 442c601221SChris Williamson " -f Specify working directory for ztest vdev files.\n" \ 452c601221SChris Williamson " -c Specify a core dump directory to use.\n" \ 462c601221SChris Williamson " -h Print this help message.\n" \ 472c601221SChris Williamson "" >&2 482c601221SChris Williamson} 492c601221SChris Williamson 502c601221SChris Williamsonfunction or_die 512c601221SChris Williamson{ 522c601221SChris Williamson $@ 532c601221SChris Williamson if [[ $? -ne 0 ]]; then 542c601221SChris Williamson echo "Command failed: $@" 552c601221SChris Williamson exit 1 562c601221SChris Williamson fi 572c601221SChris Williamson} 582c601221SChris Williamson 592c601221SChris Williamsonfunction store_core 602c601221SChris Williamson{ 612c601221SChris Williamson if [[ $ztrc -ne 0 ]] || [[ -f core ]]; then 622c601221SChris Williamson coreid=$(/bin/date "+zloop-%y%m%d-%H%M%S") 632c601221SChris Williamson foundcrashes=$(($foundcrashes + 1)) 642c601221SChris Williamson 652c601221SChris Williamson dest=$coredir/$coreid 662c601221SChris Williamson or_die /bin/mkdir $dest 672c601221SChris Williamson or_die /bin/mkdir $dest/vdev 682c601221SChris Williamson 692c601221SChris Williamson echo "*** ztest crash found - moving logs to $coredir/$coreid" 702c601221SChris Williamson 712c601221SChris Williamson or_die /bin/mv ztest.history $dest/ 722c601221SChris Williamson or_die /bin/mv ztest.out $dest/ 732c601221SChris Williamson or_die /bin/mv $workdir/ztest* $dest/vdev/ 742c601221SChris Williamson or_die /bin/mv $workdir/zpool.cache $dest/vdev/ 752c601221SChris Williamson 762c601221SChris Williamson # check for core 772c601221SChris Williamson if [[ -f core ]]; then 782c601221SChris Williamson corestatus=$(mdb -e "::status" core) 792c601221SChris Williamson corestack=$(mdb -e "::stack" core) 802c601221SChris Williamson 812c601221SChris Williamson # Dump core + logs to stored directory 822c601221SChris Williamson echo "$corestatus" >>$dest/status 832c601221SChris Williamson echo "$corestack" >>$dest/status 842c601221SChris Williamson or_die /bin/mv core $dest/ 852c601221SChris Williamson 862c601221SChris Williamson # Record info in cores logfile 872c601221SChris Williamson echo "*** core @ $coredir/$coreid/core:" | /bin/tee -a ztest.cores 882c601221SChris Williamson echo "$corestatus" | /bin/tee -a ztest.cores 892c601221SChris Williamson echo "$corestack" | /bin/tee -a ztest.cores 902c601221SChris Williamson echo "" | /bin/tee -a ztest.cores 912c601221SChris Williamson fi 922c601221SChris Williamson echo "continuing..." 932c601221SChris Williamson fi 942c601221SChris Williamson} 952c601221SChris Williamson 962c601221SChris Williamson# parse arguments 972c601221SChris Williamson# expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args] 982c601221SChris Williamsoncoredir=$DEFAULTCOREDIR 992c601221SChris Williamsonworkdir=$DEFAULTWORKDIR 1002c601221SChris Williamsontimeout=0 1012c601221SChris Williamsonwhile getopts ":ht:c:f:" opt; do 1022c601221SChris Williamson case $opt in 1032c601221SChris Williamson t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;; 1042c601221SChris Williamson c ) [[ $OPTARG ]] && coredir=$OPTARG ;; 1052c601221SChris Williamson f ) [[ $OPTARG ]] && workdir=$(/usr/bin/readlink -f $OPTARG) ;; 1062c601221SChris Williamson h ) usage 1072c601221SChris Williamson exit 2 1082c601221SChris Williamson ;; 1092c601221SChris Williamson * ) echo "Invalid argument: -$OPTARG"; 1102c601221SChris Williamson usage 1112c601221SChris Williamson exit 1 1122c601221SChris Williamson esac 1132c601221SChris Williamsondone 1142c601221SChris Williamson# pass remaining arguments on to ztest 1152c601221SChris Williamsonshift $((OPTIND - 1)) 1162c601221SChris Williamson 1172c601221SChris Williamsonif [[ -f core ]]; then 1182c601221SChris Williamson echo "There's a core dump here you might want to look at first." 1192c601221SChris Williamson exit 1 1202c601221SChris Williamsonfi 1212c601221SChris Williamson 1222c601221SChris Williamsonif [[ ! -d $coredir ]]; then 1232c601221SChris Williamson echo "core dump directory ($coredir) does not exist, creating it." 1242c601221SChris Williamson or_die /bin/mkdir -p $coredir 1252c601221SChris Williamsonfi 1262c601221SChris Williamson 1272c601221SChris Williamsonif [[ ! -w $coredir ]]; then 1282c601221SChris Williamson echo "core dump directory ($coredir) is not writable." 1292c601221SChris Williamson exit 1 1302c601221SChris Williamsonfi 1312c601221SChris Williamson 1322c601221SChris Williamsonor_die /bin/rm -f ztest.history 1332c601221SChris Williamsonor_die /bin/rm -f ztest.cores 1342c601221SChris Williamson 135213fcdcbSJohn Wren Kennedy# Allow core files to be written to cwd if that's currently disabled. 136213fcdcbSJohn Wren Kennedysudo coreadm -e process 137213fcdcbSJohn Wren Kennedy 1382c601221SChris Williamsonztrc=0 # ztest return value 1392c601221SChris Williamsonfoundcrashes=0 # number of crashes found so far 1402c601221SChris Williamsonstarttime=$(/bin/date +%s) 1412c601221SChris Williamsoncurtime=$starttime 1422c601221SChris Williamson 1432c601221SChris Williamson# if no timeout was specified, loop forever. 1442c601221SChris Williamsonwhile [[ $timeout -eq 0 ]] || [[ $curtime -le $(($starttime + $timeout)) ]]; do 1452c601221SChris Williamson zopt="-VVVVV" 1462c601221SChris Williamson 1472c601221SChris Williamson # switch between common arrangements & fully randomized 1482c601221SChris Williamson if [[ $((RANDOM % 2)) -eq 0 ]]; then 1492c601221SChris Williamson mirrors=2 1502c601221SChris Williamson raidz=0 1512c601221SChris Williamson parity=1 1522c601221SChris Williamson vdevs=2 1532c601221SChris Williamson else 1542c601221SChris Williamson mirrors=$(((RANDOM % 3) * 1)) 1552c601221SChris Williamson parity=$(((RANDOM % 3) + 1)) 1562c601221SChris Williamson raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2))) 1572c601221SChris Williamson vdevs=$(((RANDOM % 3) + 3)) 1582c601221SChris Williamson fi 1592c601221SChris Williamson align=$(((RANDOM % 2) * 3 + 9)) 1602c601221SChris Williamson runtime=$((RANDOM % 100)) 1612c601221SChris Williamson passtime=$((RANDOM % (runtime / 3 + 1) + 10)) 1622c601221SChris Williamson size=128m 1632c601221SChris Williamson 1642c601221SChris Williamson zopt="$zopt -m $mirrors" 1652c601221SChris Williamson zopt="$zopt -r $raidz" 1662c601221SChris Williamson zopt="$zopt -R $parity" 1672c601221SChris Williamson zopt="$zopt -v $vdevs" 1682c601221SChris Williamson zopt="$zopt -a $align" 1692c601221SChris Williamson zopt="$zopt -T $runtime" 1702c601221SChris Williamson zopt="$zopt -P $passtime" 1712c601221SChris Williamson zopt="$zopt -s $size" 1722c601221SChris Williamson zopt="$zopt -f $workdir" 1732c601221SChris Williamson 1742c601221SChris Williamson cmd="ztest $zopt $@" 1752c601221SChris Williamson desc="$(/bin/date '+%m/%d %T') $cmd" 1762c601221SChris Williamson echo "$desc" | /bin/tee -a ztest.history 1772c601221SChris Williamson echo "$desc" >>ztest.out 178*1672d299SYuri Pankov /usr/bin/$cmd >>ztest.out 2>&1 1792c601221SChris Williamson ztrc=$? 1802c601221SChris Williamson /bin/egrep '===|WARNING' ztest.out >>ztest.history 181*1672d299SYuri Pankov /usr/sbin/zdb -U $workdir/zpool.cache -DD ztest >>ztest.ddt 2>&1 1822c601221SChris Williamson 1832c601221SChris Williamson store_core 1842c601221SChris Williamson 1852c601221SChris Williamson curtime=$(/bin/date +%s) 1862c601221SChris Williamsondone 1872c601221SChris Williamson 1882c601221SChris Williamsonecho "zloop finished, $foundcrashes crashes found" 1892c601221SChris Williamson 1902c601221SChris Williamson/bin/uptime >>ztest.out 1912c601221SChris Williamson 1922c601221SChris Williamsonif [[ $foundcrashes -gt 0 ]]; then 1932c601221SChris Williamson exit 1 1942c601221SChris Williamsonfi 195