11f5207b7SJohn Levon#!/bin/sh 21f5207b7SJohn Levon 31f5207b7SJohn Levon#set -x 41f5207b7SJohn Levon 51f5207b7SJohn Levoncd $(dirname "$0") 61f5207b7SJohn Levon 71f5207b7SJohn Levondefault_path=".." 81f5207b7SJohn Levondefault_cmd="sparse \$file" 9*c85f09ccSJohn Levondefault_args="$SPARSE_TEST_ARGS" 10*c85f09ccSJohn Levontests_list="" 111f5207b7SJohn Levonprog_name=`basename $0` 121f5207b7SJohn Levon 131f5207b7SJohn Levonif [ ! -x "$default_path/sparse-llvm" ]; then 14*c85f09ccSJohn Levon disabled_cmds="sparsec sparsei sparse-llvm sparse-llvm-dis" 151f5207b7SJohn Levonfi 161f5207b7SJohn Levon 171f5207b7SJohn Levon# flags: 181f5207b7SJohn Levon# - some tests gave an unexpected result 191f5207b7SJohn Levonfailed=0 201f5207b7SJohn Levon 211f5207b7SJohn Levon# counts: 221f5207b7SJohn Levon# - tests that have not been converted to test-suite format 231f5207b7SJohn Levon# - tests that are disabled 241f5207b7SJohn Levon# - tests that passed 251f5207b7SJohn Levon# - tests that failed 261f5207b7SJohn Levon# - tests that failed but are known to fail 271f5207b7SJohn Levonunhandled_tests=0 281f5207b7SJohn Levondisabled_tests=0 291f5207b7SJohn Levonok_tests=0 301f5207b7SJohn Levonko_tests=0 311f5207b7SJohn Levonknown_ko_tests=0 321f5207b7SJohn Levon 331f5207b7SJohn Levon# defaults to not verbose 341f5207b7SJohn Levon[ -z "$V" ] && V=0 35*c85f09ccSJohn Levonvquiet="" 36*c85f09ccSJohn Levonquiet=0 37*c85f09ccSJohn Levonabort=0 38*c85f09ccSJohn Levon 39*c85f09ccSJohn Levon 40*c85f09ccSJohn Levon## 41*c85f09ccSJohn Levon# verbose(string) - prints string if we are in verbose mode 42*c85f09ccSJohn Levonverbose() 43*c85f09ccSJohn Levon{ 44*c85f09ccSJohn Levon [ "$V" -eq "1" ] && echo " $1" 45*c85f09ccSJohn Levon return 0 46*c85f09ccSJohn Levon} 47*c85f09ccSJohn Levon 48*c85f09ccSJohn Levon## 49*c85f09ccSJohn Levon# warning(string) - prints a warning 50*c85f09ccSJohn Levonwarning() 51*c85f09ccSJohn Levon{ 52*c85f09ccSJohn Levon [ "$quiet" -ne 1 ] && echo "warning: $1" 53*c85f09ccSJohn Levon return 0 54*c85f09ccSJohn Levon} 55*c85f09ccSJohn Levon 56*c85f09ccSJohn Levon## 57*c85f09ccSJohn Levon# error(string[, die]) - prints an error and exits with value die if given 58*c85f09ccSJohn Levonerror() 59*c85f09ccSJohn Levon{ 60*c85f09ccSJohn Levon [ "$quiet" -ne 1 ] && echo "error: $1" 61*c85f09ccSJohn Levon [ -n "$2" ] && exit $2 62*c85f09ccSJohn Levon return 0 63*c85f09ccSJohn Levon} 64*c85f09ccSJohn Levon 651f5207b7SJohn Levon 661f5207b7SJohn Levon## 671f5207b7SJohn Levon# get_tag_value(file) - get the 'check-<...>' tags & values 681f5207b7SJohn Levonget_tag_value() 691f5207b7SJohn Levon{ 701f5207b7SJohn Levon check_name="" 711f5207b7SJohn Levon check_command="$default_cmd" 721f5207b7SJohn Levon check_exit_value=0 731f5207b7SJohn Levon check_timeout=0 741f5207b7SJohn Levon check_known_to_fail=0 751f5207b7SJohn Levon check_error_ignore=0 761f5207b7SJohn Levon check_output_ignore=0 771f5207b7SJohn Levon check_output_contains=0 781f5207b7SJohn Levon check_output_excludes=0 791f5207b7SJohn Levon check_output_pattern=0 80*c85f09ccSJohn Levon check_arch_ignore="" 81*c85f09ccSJohn Levon check_arch_only="" 82*c85f09ccSJohn Levon check_assert="" 83*c85f09ccSJohn Levon check_cpp_if="" 841f5207b7SJohn Levon 851f5207b7SJohn Levon lines=$(grep 'check-[a-z-]*' $1 | \ 861f5207b7SJohn Levon sed -e 's/^.*\(check-[a-z-]*:*\) *\(.*\)$/\1 \2/') 871f5207b7SJohn Levon 881f5207b7SJohn Levon while read tag val; do 891f5207b7SJohn Levon #echo "-> tag: '$tag'" 901f5207b7SJohn Levon #echo "-> val: '$val'" 911f5207b7SJohn Levon case $tag in 921f5207b7SJohn Levon check-name:) check_name="$val" ;; 931f5207b7SJohn Levon check-command:) check_command="$val" ;; 941f5207b7SJohn Levon check-exit-value:) check_exit_value="$val" ;; 951f5207b7SJohn Levon check-timeout:) [ -z "$val" ] && val=1 961f5207b7SJohn Levon check_timeout="$val" ;; 971f5207b7SJohn Levon check-known-to-fail) check_known_to_fail=1 ;; 981f5207b7SJohn Levon check-error-ignore) check_error_ignore=1 ;; 991f5207b7SJohn Levon check-output-ignore) check_output_ignore=1 ;; 1001f5207b7SJohn Levon check-output-contains:) check_output_contains=1 ;; 1011f5207b7SJohn Levon check-output-excludes:) check_output_excludes=1 ;; 102*c85f09ccSJohn Levon check-output-pattern) check_output_pattern=1 ;; 103*c85f09ccSJohn Levon check-arch-ignore:) arch=$(uname -m) 104*c85f09ccSJohn Levon check_arch_ignore="$val" ;; 105*c85f09ccSJohn Levon check-arch-only:) arch=$(uname -m) 106*c85f09ccSJohn Levon check_arch_only="$val" ;; 107*c85f09ccSJohn Levon check-assert:) check_assert="$val" ;; 108*c85f09ccSJohn Levon check-cpp-if:) check_cpp_if="$val" ;; 109*c85f09ccSJohn Levon 110*c85f09ccSJohn Levon check-description:) ;; # ignore 111*c85f09ccSJohn Levon check-note:) ;; # ignore 112*c85f09ccSJohn Levon check-warning:) ;; # ignore 113*c85f09ccSJohn Levon check-error-start) ;; # ignore 114*c85f09ccSJohn Levon check-error-end) ;; # ignore 115*c85f09ccSJohn Levon check-output-start) ;; # ignore 116*c85f09ccSJohn Levon check-output-end) ;; # ignore 117*c85f09ccSJohn Levon check-should-pass) ;; # ignore, unused annotation 118*c85f09ccSJohn Levon check-should-fail) ;; # ignore, unused annotation 119*c85f09ccSJohn Levon check-should-warn) ;; # ignore, unused annotation 120*c85f09ccSJohn Levon check-*) error "$1: unknown tag '$tag'" 1 ;; 1211f5207b7SJohn Levon esac 1221f5207b7SJohn Levon done << EOT 1231f5207b7SJohn Levon $lines 1241f5207b7SJohn LevonEOT 1251f5207b7SJohn Levon} 1261f5207b7SJohn Levon 1271f5207b7SJohn Levon## 1281f5207b7SJohn Levon# helper for has_(each|none)_patterns() 1291f5207b7SJohn Levonhas_patterns() 1301f5207b7SJohn Levon{ 1311f5207b7SJohn Levon ifile="$1" 1321f5207b7SJohn Levon patt="$2" 1331f5207b7SJohn Levon ofile="$3" 1341f5207b7SJohn Levon cmp="$4" 135*c85f09ccSJohn Levon msg="$5" 1361f5207b7SJohn Levon grep "$patt:" "$ifile" | \ 1371f5207b7SJohn Levon sed -e "s/^.*$patt: *\(.*\)$/\1/" | \ 1381f5207b7SJohn Levon while read val; do 1391f5207b7SJohn Levon grep -s -q "$val" "$ofile" 1401f5207b7SJohn Levon if [ "$?" $cmp 0 ]; then 141*c85f09ccSJohn Levon error " Pattern '$val' unexpectedly $msg" 1421f5207b7SJohn Levon return 1 1431f5207b7SJohn Levon fi 1441f5207b7SJohn Levon done 1451f5207b7SJohn Levon 1461f5207b7SJohn Levon return $? 1471f5207b7SJohn Levon} 1481f5207b7SJohn Levon 1491f5207b7SJohn Levon## 1501f5207b7SJohn Levon# has_each_patterns(ifile tag ofile) - does ofile contains some 1511f5207b7SJohn Levon# of the patterns given by ifile's tags? 1521f5207b7SJohn Levon# 1531f5207b7SJohn Levon# returns 0 if all present, 1 otherwise 1541f5207b7SJohn Levonhas_each_patterns() 1551f5207b7SJohn Levon{ 156*c85f09ccSJohn Levon has_patterns "$1" "$2" "$4" -ne "$3" 1571f5207b7SJohn Levon} 1581f5207b7SJohn Levon 1591f5207b7SJohn Levon## 1601f5207b7SJohn Levon# has_none_patterns(ifile tag ofile) - does ofile contains some 1611f5207b7SJohn Levon# of the patterns given by ifile's tags? 1621f5207b7SJohn Levon# 1631f5207b7SJohn Levon# returns 1 if any present, 0 otherwise 1641f5207b7SJohn Levonhas_none_patterns() 1651f5207b7SJohn Levon{ 166*c85f09ccSJohn Levon has_patterns "$1" "$2" "$4" -eq "$3" 1671f5207b7SJohn Levon} 1681f5207b7SJohn Levon 1691f5207b7SJohn Levon## 170*c85f09ccSJohn Levon# minmax_patterns(ifile tag ofile) - does ofile contains the 1711f5207b7SJohn Levon# the patterns given by ifile's tags 1721f5207b7SJohn Levon# the right number of time? 173*c85f09ccSJohn Levonminmax_patterns() 1741f5207b7SJohn Levon{ 1751f5207b7SJohn Levon ifile="$1" 1761f5207b7SJohn Levon patt="$2" 1771f5207b7SJohn Levon ofile="$3" 178*c85f09ccSJohn Levon grep "$patt([0-9-]*\(, *\)*[0-9-]*):" "$ifile" | \ 179*c85f09ccSJohn Levon sed -e "s/^.*$patt(\([0-9]*\)): *\(.*\)/\1 eq \2/" \ 180*c85f09ccSJohn Levon -e "s/^.*$patt(\([0-9-]*\), *\([0-9-]*\)): *\(.*\)/\1 \2 \3/" | \ 181*c85f09ccSJohn Levon while read min max pat; do 1821f5207b7SJohn Levon n=$(grep -s "$pat" "$ofile" | wc -l) 183*c85f09ccSJohn Levon if [ "$max" = "eq" ]; then 184*c85f09ccSJohn Levon if [ "$n" -ne "$min" ]; then 185*c85f09ccSJohn Levon error " Pattern '$pat' expected $min times but got $n times" 1861f5207b7SJohn Levon return 1 187*c85f09ccSJohn Levon fi 188*c85f09ccSJohn Levon continue 189*c85f09ccSJohn Levon fi 190*c85f09ccSJohn Levon if [ "$min" != '-' ]; then 191*c85f09ccSJohn Levon if [ "$n" -lt "$min" ]; then 192*c85f09ccSJohn Levon error " Pattern '$pat' expected min $min times but got $n times" 193*c85f09ccSJohn Levon return 1 194*c85f09ccSJohn Levon fi 195*c85f09ccSJohn Levon fi 196*c85f09ccSJohn Levon if [ "$max" != '-' ]; then 197*c85f09ccSJohn Levon if [ "$n" -gt "$max" ]; then 198*c85f09ccSJohn Levon error " Pattern '$pat' expected max $max times but got $n times" 199*c85f09ccSJohn Levon return 1 200*c85f09ccSJohn Levon fi 2011f5207b7SJohn Levon fi 2021f5207b7SJohn Levon done 2031f5207b7SJohn Levon 2041f5207b7SJohn Levon return $? 2051f5207b7SJohn Levon} 2061f5207b7SJohn Levon 2071f5207b7SJohn Levon## 208*c85f09ccSJohn Levon# arg_file(filename) - checks if filename exists 209*c85f09ccSJohn Levonarg_file() 2101f5207b7SJohn Levon{ 211*c85f09ccSJohn Levon [ -z "$1" ] && { 212*c85f09ccSJohn Levon do_usage 213*c85f09ccSJohn Levon exit 1 214*c85f09ccSJohn Levon } 215*c85f09ccSJohn Levon [ -e "$1" ] || { 216*c85f09ccSJohn Levon error "Can't open file $1" 217*c85f09ccSJohn Levon exit 1 218*c85f09ccSJohn Levon } 2191f5207b7SJohn Levon return 0 2201f5207b7SJohn Levon} 2211f5207b7SJohn Levon 2221f5207b7SJohn Levon 223*c85f09ccSJohn Levon## 2241f5207b7SJohn Levondo_usage() 2251f5207b7SJohn Levon{ 2261f5207b7SJohn Levonecho "$prog_name - a tiny automatic testing script" 227*c85f09ccSJohn Levonecho "Usage: $prog_name [option(s)] [command] [arguments]" 228*c85f09ccSJohn Levonecho 229*c85f09ccSJohn Levonecho "options:" 230*c85f09ccSJohn Levonecho " -a|--abort Abort the tests as soon as one fails." 231*c85f09ccSJohn Levonecho " -q|--quiet Be extra quiet while running the tests." 232*c85f09ccSJohn Levonecho " --args='...' Add these options to the test command." 2331f5207b7SJohn Levonecho 2341f5207b7SJohn Levonecho "commands:" 235*c85f09ccSJohn Levonecho " [file ...] Runs the test suite on the given file(s)." 236*c85f09ccSJohn Levonecho " If a directory is given, run only those files." 237*c85f09ccSJohn Levonecho " If no file is given, run the whole testsuite." 238*c85f09ccSJohn Levonecho " single file Run the test in 'file'." 239*c85f09ccSJohn Levonecho " format file [name [cmd]] Help writing a new test case using cmd." 2401f5207b7SJohn Levonecho 241*c85f09ccSJohn Levonecho " [command] help Print usage." 242*c85f09ccSJohn Levon} 243*c85f09ccSJohn Levon 244*c85f09ccSJohn Levondisable() 245*c85f09ccSJohn Levon{ 246*c85f09ccSJohn Levon disabled_tests=$(($disabled_tests + 1)) 247*c85f09ccSJohn Levon if [ -z "$vquiet" ]; then 248*c85f09ccSJohn Levon echo " SKIP $1 ($2)" 249*c85f09ccSJohn Levon fi 2501f5207b7SJohn Levon} 2511f5207b7SJohn Levon 2521f5207b7SJohn Levon## 2531f5207b7SJohn Levon# do_test(file) - tries to validate a test case 2541f5207b7SJohn Levon# 2551f5207b7SJohn Levon# it "parses" file, looking for check-* tags and tries to validate 2561f5207b7SJohn Levon# the test against an expected result 2571f5207b7SJohn Levon# returns: 2581f5207b7SJohn Levon# - 0 if the test passed, 2591f5207b7SJohn Levon# - 1 if it failed, 2601f5207b7SJohn Levon# - 2 if it is not a "test-suite" test. 2611f5207b7SJohn Levon# - 3 if the test is disabled. 2621f5207b7SJohn Levondo_test() 2631f5207b7SJohn Levon{ 2641f5207b7SJohn Levon test_failed=0 2651f5207b7SJohn Levon file="$1" 266*c85f09ccSJohn Levon quiet=0 2671f5207b7SJohn Levon 2681f5207b7SJohn Levon get_tag_value $file 2691f5207b7SJohn Levon 2701f5207b7SJohn Levon # can this test be handled by test-suite ? 2711f5207b7SJohn Levon # (it has to have a check-name key in it) 2721f5207b7SJohn Levon if [ "$check_name" = "" ]; then 273*c85f09ccSJohn Levon warning "$file: test unhandled" 2741f5207b7SJohn Levon unhandled_tests=$(($unhandled_tests + 1)) 2751f5207b7SJohn Levon return 2 2761f5207b7SJohn Levon fi 2771f5207b7SJohn Levon test_name="$check_name" 2781f5207b7SJohn Levon 2791f5207b7SJohn Levon # does the test provide a specific command ? 2801f5207b7SJohn Levon if [ "$check_command" = "" ]; then 2811f5207b7SJohn Levon check_command="$defaut_command" 2821f5207b7SJohn Levon fi 2831f5207b7SJohn Levon 2841f5207b7SJohn Levon # check for disabled commands 2851f5207b7SJohn Levon set -- $check_command 2861f5207b7SJohn Levon base_cmd=$1 2871f5207b7SJohn Levon for i in $disabled_cmds; do 2881f5207b7SJohn Levon if [ "$i" = "$base_cmd" ] ; then 289*c85f09ccSJohn Levon disable "$test_name" "$file" 2901f5207b7SJohn Levon return 3 2911f5207b7SJohn Levon fi 2921f5207b7SJohn Levon done 293*c85f09ccSJohn Levon if [ "$check_arch_ignore" != "" ]; then 294*c85f09ccSJohn Levon if echo $arch | egrep -q -w "$check_arch_ignore"; then 295*c85f09ccSJohn Levon disable "$test_name" "$file" 296*c85f09ccSJohn Levon return 3 297*c85f09ccSJohn Levon fi 298*c85f09ccSJohn Levon fi 299*c85f09ccSJohn Levon if [ "$check_arch_only" != "" ]; then 300*c85f09ccSJohn Levon if ! (echo $arch | egrep -q -w "$check_arch_only"); then 301*c85f09ccSJohn Levon disable "$test_name" "$file" 302*c85f09ccSJohn Levon return 3 303*c85f09ccSJohn Levon fi 304*c85f09ccSJohn Levon fi 305*c85f09ccSJohn Levon if [ "$check_assert" != "" ]; then 306*c85f09ccSJohn Levon res=$(../sparse - 2>&1 >/dev/null <<- EOF 307*c85f09ccSJohn Levon _Static_assert($check_assert, "$check_assert"); 308*c85f09ccSJohn Levon EOF 309*c85f09ccSJohn Levon ) 310*c85f09ccSJohn Levon if [ "$res" != "" ]; then 311*c85f09ccSJohn Levon disable "$test_name" "$file" 312*c85f09ccSJohn Levon return 3 313*c85f09ccSJohn Levon fi 314*c85f09ccSJohn Levon fi 315*c85f09ccSJohn Levon if [ "$check_cpp_if" != "" ]; then 316*c85f09ccSJohn Levon res=$(../sparse -E - 2>/dev/null <<- EOF 317*c85f09ccSJohn Levon #if !($check_cpp_if) 318*c85f09ccSJohn Levon fail 319*c85f09ccSJohn Levon #endif 320*c85f09ccSJohn Levon EOF 321*c85f09ccSJohn Levon ) 322*c85f09ccSJohn Levon if [ "$res" != "" ]; then 323*c85f09ccSJohn Levon disable "$test_name" "$file" 324*c85f09ccSJohn Levon return 3 325*c85f09ccSJohn Levon fi 326*c85f09ccSJohn Levon fi 3271f5207b7SJohn Levon 328*c85f09ccSJohn Levon if [ -z "$vquiet" ]; then 329*c85f09ccSJohn Levon echo " TEST $test_name ($file)" 330*c85f09ccSJohn Levon fi 3311f5207b7SJohn Levon 332*c85f09ccSJohn Levon verbose "Using command : $(echo "$@")" 3331f5207b7SJohn Levon 3341f5207b7SJohn Levon # grab the expected exit value 3351f5207b7SJohn Levon expected_exit_value=$check_exit_value 3361f5207b7SJohn Levon verbose "Expecting exit value: $expected_exit_value" 3371f5207b7SJohn Levon 3381f5207b7SJohn Levon # do we want a timeout? 339*c85f09ccSJohn Levon pre_cmd="" 3401f5207b7SJohn Levon if [ $check_timeout -ne 0 ]; then 341*c85f09ccSJohn Levon pre_cmd="timeout -k 1s $check_timeout" 3421f5207b7SJohn Levon fi 3431f5207b7SJohn Levon 344*c85f09ccSJohn Levon shift 345*c85f09ccSJohn Levon # launch the test command and 3461f5207b7SJohn Levon # grab the actual output & exit value 347*c85f09ccSJohn Levon eval $pre_cmd $default_path/$base_cmd $default_args "$@" \ 348*c85f09ccSJohn Levon 1> $file.output.got 2> $file.error.got 3491f5207b7SJohn Levon actual_exit_value=$? 3501f5207b7SJohn Levon 3511f5207b7SJohn Levon must_fail=$check_known_to_fail 3521f5207b7SJohn Levon [ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1 3531f5207b7SJohn Levon known_ko_tests=$(($known_ko_tests + $must_fail)) 3541f5207b7SJohn Levon 355*c85f09ccSJohn Levon for stream in error output; do 3561f5207b7SJohn Levon eval ignore=\$check_${stream}_ignore 3571f5207b7SJohn Levon [ $ignore -eq 1 ] && continue 3581f5207b7SJohn Levon 3591f5207b7SJohn Levon # grab the expected output 3601f5207b7SJohn Levon sed -n "/check-$stream-start/,/check-$stream-end/p" $file \ 3611f5207b7SJohn Levon | grep -v check-$stream > "$file".$stream.expected 3621f5207b7SJohn Levon 3631f5207b7SJohn Levon diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff 3641f5207b7SJohn Levon if [ "$?" -ne "0" ]; then 3651f5207b7SJohn Levon error "actual $stream text does not match expected $stream text." 3661f5207b7SJohn Levon error "see $file.$stream.* for further investigation." 3671f5207b7SJohn Levon [ $quiet -ne 1 ] && cat "$file".$stream.diff 3681f5207b7SJohn Levon test_failed=1 3691f5207b7SJohn Levon fi 3701f5207b7SJohn Levon done 3711f5207b7SJohn Levon 3721f5207b7SJohn Levon if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then 3731f5207b7SJohn Levon error "Actual exit value does not match the expected one." 3741f5207b7SJohn Levon error "expected $expected_exit_value, got $actual_exit_value." 3751f5207b7SJohn Levon test_failed=1 3761f5207b7SJohn Levon fi 3771f5207b7SJohn Levon 3781f5207b7SJohn Levon # verify the 'check-output-contains/excludes' tags 3791f5207b7SJohn Levon if [ $check_output_contains -eq 1 ]; then 380*c85f09ccSJohn Levon has_each_patterns "$file" 'check-output-contains' absent $file.output.got 3811f5207b7SJohn Levon if [ "$?" -ne "0" ]; then 3821f5207b7SJohn Levon test_failed=1 3831f5207b7SJohn Levon fi 3841f5207b7SJohn Levon fi 3851f5207b7SJohn Levon if [ $check_output_excludes -eq 1 ]; then 386*c85f09ccSJohn Levon has_none_patterns "$file" 'check-output-excludes' present $file.output.got 3871f5207b7SJohn Levon if [ "$?" -ne "0" ]; then 3881f5207b7SJohn Levon test_failed=1 3891f5207b7SJohn Levon fi 3901f5207b7SJohn Levon fi 3911f5207b7SJohn Levon if [ $check_output_pattern -eq 1 ]; then 392*c85f09ccSJohn Levon # verify the 'check-output-pattern(...)' tags 393*c85f09ccSJohn Levon minmax_patterns "$file" 'check-output-pattern' $file.output.got 3941f5207b7SJohn Levon if [ "$?" -ne "0" ]; then 3951f5207b7SJohn Levon test_failed=1 3961f5207b7SJohn Levon fi 3971f5207b7SJohn Levon fi 3981f5207b7SJohn Levon 3991f5207b7SJohn Levon if [ "$must_fail" -eq "1" ]; then 4001f5207b7SJohn Levon if [ "$test_failed" -eq "1" ]; then 401*c85f09ccSJohn Levon [ -z "$vquiet" ] && \ 402*c85f09ccSJohn Levon echo "info: XFAIL: test '$file' is known to fail" 4031f5207b7SJohn Levon else 404*c85f09ccSJohn Levon echo "error: XPASS: test '$file' is known to fail but succeed!" 4051f5207b7SJohn Levon fi 406*c85f09ccSJohn Levon else 407*c85f09ccSJohn Levon if [ "$test_failed" -eq "1" ]; then 408*c85f09ccSJohn Levon echo "error: FAIL: test '$file' failed" 409*c85f09ccSJohn Levon else 410*c85f09ccSJohn Levon [ "$V" -ne "0" ] && \ 411*c85f09ccSJohn Levon echo "info: PASS: test '$file' passed" 412*c85f09ccSJohn Levon fi 413*c85f09ccSJohn Levon fi 414*c85f09ccSJohn Levon 415*c85f09ccSJohn Levon if [ "$test_failed" -ne "$must_fail" ]; then 416*c85f09ccSJohn Levon [ $abort -eq 1 ] && exit 1 417*c85f09ccSJohn Levon test_failed=1 418*c85f09ccSJohn Levon failed=1 4191f5207b7SJohn Levon fi 4201f5207b7SJohn Levon 4211f5207b7SJohn Levon if [ "$test_failed" -eq "1" ]; then 4221f5207b7SJohn Levon ko_tests=$(($ko_tests + 1)) 4231f5207b7SJohn Levon else 4241f5207b7SJohn Levon ok_tests=$(($ok_tests + 1)) 4251f5207b7SJohn Levon rm -f $file.{error,output}.{expected,got,diff} 4261f5207b7SJohn Levon fi 4271f5207b7SJohn Levon return $test_failed 4281f5207b7SJohn Levon} 4291f5207b7SJohn Levon 4301f5207b7SJohn Levondo_test_suite() 4311f5207b7SJohn Levon{ 4321f5207b7SJohn Levon for i in $tests_list; do 4331f5207b7SJohn Levon do_test "$i" 4341f5207b7SJohn Levon done 4351f5207b7SJohn Levon 436*c85f09ccSJohn Levon OK=OK 437*c85f09ccSJohn Levon [ $failed -eq 0 ] || OK=KO 438*c85f09ccSJohn Levon 4391f5207b7SJohn Levon # prints some numbers 4401f5207b7SJohn Levon tests_nr=$(($ok_tests + $ko_tests)) 441*c85f09ccSJohn Levon echo "$OK: out of $tests_nr tests, $ok_tests passed, $ko_tests failed" 442*c85f09ccSJohn Levon if [ "$known_ko_tests" -ne 0 ]; then 443*c85f09ccSJohn Levon echo " $known_ko_tests of them are known to fail" 444*c85f09ccSJohn Levon fi 4451f5207b7SJohn Levon if [ "$unhandled_tests" -ne "0" ]; then 446*c85f09ccSJohn Levon echo " $unhandled_tests tests could not be handled by $prog_name" 4471f5207b7SJohn Levon fi 4481f5207b7SJohn Levon if [ "$disabled_tests" -ne "0" ]; then 449*c85f09ccSJohn Levon echo " $disabled_tests tests were disabled" 4501f5207b7SJohn Levon fi 4511f5207b7SJohn Levon} 4521f5207b7SJohn Levon 4531f5207b7SJohn Levon## 454*c85f09ccSJohn Levondo_format_help() { 455*c85f09ccSJohn Levonecho "Usage: $prog_name [option(s)] [--]format file [name [cmd]]" 456*c85f09ccSJohn Levonecho 457*c85f09ccSJohn Levonecho "options:" 458*c85f09ccSJohn Levonecho " -a append the created test to the input file" 459*c85f09ccSJohn Levonecho " -f write a test known to fail" 460*c85f09ccSJohn Levonecho " -l write a test for linearized code" 461*c85f09ccSJohn Levonecho 462*c85f09ccSJohn Levonecho "argument(s):" 463*c85f09ccSJohn Levonecho " file file containing the test case(s)" 464*c85f09ccSJohn Levonecho " name name for the test case (defaults to file)" 465*c85f09ccSJohn Levonecho " cmd command to be used (defaults to 'sparse \$file')" 466*c85f09ccSJohn Levon} 467*c85f09ccSJohn Levon 468*c85f09ccSJohn Levon## 469*c85f09ccSJohn Levon# do_format([options,] file[, name[, cmd]]) - helps a test writer to format test-suite tags 4701f5207b7SJohn Levondo_format() 4711f5207b7SJohn Levon{ 472*c85f09ccSJohn Levon def_cmd="$default_cmd" 473*c85f09ccSJohn Levon append=0 474*c85f09ccSJohn Levon linear=0 475*c85f09ccSJohn Levon fail=0 476*c85f09ccSJohn Levon 477*c85f09ccSJohn Levon while [ $# -gt 1 ] ; do 478*c85f09ccSJohn Levon case "$1" in 479*c85f09ccSJohn Levon -a) 480*c85f09ccSJohn Levon append=1 ;; 481*c85f09ccSJohn Levon -f) 482*c85f09ccSJohn Levon fail=1 ;; 483*c85f09ccSJohn Levon -l) 484*c85f09ccSJohn Levon def_cmd='test-linearize -Wno-decl $file' 485*c85f09ccSJohn Levon linear=1 ;; 486*c85f09ccSJohn Levon help|-*) 487*c85f09ccSJohn Levon do_format_help 488*c85f09ccSJohn Levon return 0 489*c85f09ccSJohn Levon ;; 490*c85f09ccSJohn Levon *) break ;; 491*c85f09ccSJohn Levon esac 492*c85f09ccSJohn Levon shift 493*c85f09ccSJohn Levon continue 494*c85f09ccSJohn Levon done 495*c85f09ccSJohn Levon 496*c85f09ccSJohn Levon arg_file "$1" || return 1 497*c85f09ccSJohn Levon 4981f5207b7SJohn Levon file="$1" 499*c85f09ccSJohn Levon fname="$2" 500*c85f09ccSJohn Levon [ -z "$fname" ] && fname="$(basename "$1" .c)" 501*c85f09ccSJohn Levon fcmd="$3" 502*c85f09ccSJohn Levon [ -z "$fcmd" ] && fcmd="$def_cmd" 503*c85f09ccSJohn Levon 5041f5207b7SJohn Levon cmd=`eval echo $default_path/$fcmd` 5051f5207b7SJohn Levon $cmd 1> $file.output.got 2> $file.error.got 5061f5207b7SJohn Levon fexit_value=$? 507*c85f09ccSJohn Levon [ $append != 0 ] && exec >> $file 5081f5207b7SJohn Levon cat <<_EOF 509*c85f09ccSJohn Levon 5101f5207b7SJohn Levon/* 5111f5207b7SJohn Levon * check-name: $fname 5121f5207b7SJohn Levon_EOF 5131f5207b7SJohn Levon if [ "$fcmd" != "$default_cmd" ]; then 5141f5207b7SJohn Levon echo " * check-command: $fcmd" 5151f5207b7SJohn Levon fi 5161f5207b7SJohn Levon if [ "$fexit_value" -ne "0" ]; then 5171f5207b7SJohn Levon echo " * check-exit-value: $fexit_value" 5181f5207b7SJohn Levon fi 519*c85f09ccSJohn Levon if [ $fail != 0 ]; then 520*c85f09ccSJohn Levon echo " * check-known-to-fail" 521*c85f09ccSJohn Levon fi 522*c85f09ccSJohn Levon if [ $linear != 0 ]; then 523*c85f09ccSJohn Levon echo ' *' 524*c85f09ccSJohn Levon echo ' * check-output-ignore' 525*c85f09ccSJohn Levon echo ' * check-output-contains: xyz\\\\.' 526*c85f09ccSJohn Levon echo ' * check-output-excludes: \\\\.' 527*c85f09ccSJohn Levon fi 5281f5207b7SJohn Levon for stream in output error; do 5291f5207b7SJohn Levon if [ -s "$file.$stream.got" ]; then 5301f5207b7SJohn Levon echo " *" 5311f5207b7SJohn Levon echo " * check-$stream-start" 5321f5207b7SJohn Levon cat "$file.$stream.got" 5331f5207b7SJohn Levon echo " * check-$stream-end" 5341f5207b7SJohn Levon fi 5351f5207b7SJohn Levon done 5361f5207b7SJohn Levon echo " */" 5371f5207b7SJohn Levon return 0 5381f5207b7SJohn Levon} 5391f5207b7SJohn Levon 540*c85f09ccSJohn Levon## allow flags from environment 541*c85f09ccSJohn Levonset -- $SPARSE_TEST_FLAGS "$@" 5421f5207b7SJohn Levon 543*c85f09ccSJohn Levon## process the flags 544*c85f09ccSJohn Levonwhile [ "$#" -gt "0" ]; do 545*c85f09ccSJohn Levon case "$1" in 546*c85f09ccSJohn Levon -a|--abort) 547*c85f09ccSJohn Levon abort=1 548*c85f09ccSJohn Levon ;; 549*c85f09ccSJohn Levon -q|--quiet) 550*c85f09ccSJohn Levon vquiet=1 5511f5207b7SJohn Levon ;; 552*c85f09ccSJohn Levon --args=*) 553*c85f09ccSJohn Levon default_args="${1#--args=}"; 554*c85f09ccSJohn Levon ;; 555*c85f09ccSJohn Levon 556*c85f09ccSJohn Levon single|--single) 5571f5207b7SJohn Levon arg_file "$2" 5581f5207b7SJohn Levon do_test "$2" 5591f5207b7SJohn Levon case "$?" in 5601f5207b7SJohn Levon 0) echo "$2 passed !";; 5611f5207b7SJohn Levon 1) echo "$2 failed !";; 5621f5207b7SJohn Levon 2) echo "$2 can't be handled by $prog_name";; 5631f5207b7SJohn Levon esac 564*c85f09ccSJohn Levon exit $failed 5651f5207b7SJohn Levon ;; 566*c85f09ccSJohn Levon format|--format) 567*c85f09ccSJohn Levon shift 568*c85f09ccSJohn Levon do_format "$@" 569*c85f09ccSJohn Levon exit 0 5701f5207b7SJohn Levon ;; 571*c85f09ccSJohn Levon help) 5721f5207b7SJohn Levon do_usage 5731f5207b7SJohn Levon exit 1 5741f5207b7SJohn Levon ;; 5751f5207b7SJohn Levon 576*c85f09ccSJohn Levon *.c|*.cdoc) 577*c85f09ccSJohn Levon tests_list="$tests_list $1" 578*c85f09ccSJohn Levon ;; 579*c85f09ccSJohn Levon *) 580*c85f09ccSJohn Levon if [ ! -d "$1" ]; then 581*c85f09ccSJohn Levon do_usage 582*c85f09ccSJohn Levon exit 1 583*c85f09ccSJohn Levon fi 584*c85f09ccSJohn Levon tests_list="$tests_list $(find "$1" -name '*.c' | sort)" 585*c85f09ccSJohn Levon ;; 586*c85f09ccSJohn Levon esac 587*c85f09ccSJohn Levon shift 588*c85f09ccSJohn Levondone 589*c85f09ccSJohn Levon 590*c85f09ccSJohn Levonif [ -z "$tests_list" ]; then 591*c85f09ccSJohn Levon tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort` 592*c85f09ccSJohn Levonfi 593*c85f09ccSJohn Levon 594*c85f09ccSJohn Levondo_test_suite 5951f5207b7SJohn Levonexit $failed 5961f5207b7SJohn Levon 597