11f5207b7SJohn Levon#!/bin/sh
21f5207b7SJohn Levon
31f5207b7SJohn Levon#set -x
41f5207b7SJohn Levon
51f5207b7SJohn Levoncd $(dirname "$0")
61f5207b7SJohn Levon
71f5207b7SJohn Levondefault_path=".."
81f5207b7SJohn Levondefault_cmd="sparse \$file"
9*c85f09ccSJohn Levondefault_args="$SPARSE_TEST_ARGS"
10*c85f09ccSJohn Levontests_list=""
111f5207b7SJohn Levonprog_name=`basename $0`
121f5207b7SJohn Levon
131f5207b7SJohn Levonif [ ! -x "$default_path/sparse-llvm" ]; then
14*c85f09ccSJohn Levon	disabled_cmds="sparsec sparsei sparse-llvm sparse-llvm-dis"
151f5207b7SJohn Levonfi
161f5207b7SJohn Levon
171f5207b7SJohn Levon# flags:
181f5207b7SJohn Levon#	- some tests gave an unexpected result
191f5207b7SJohn Levonfailed=0
201f5207b7SJohn Levon
211f5207b7SJohn Levon# counts:
221f5207b7SJohn Levon#	- tests that have not been converted to test-suite format
231f5207b7SJohn Levon#	- tests that are disabled
241f5207b7SJohn Levon#	- tests that passed
251f5207b7SJohn Levon#	- tests that failed
261f5207b7SJohn Levon#	- tests that failed but are known to fail
271f5207b7SJohn Levonunhandled_tests=0
281f5207b7SJohn Levondisabled_tests=0
291f5207b7SJohn Levonok_tests=0
301f5207b7SJohn Levonko_tests=0
311f5207b7SJohn Levonknown_ko_tests=0
321f5207b7SJohn Levon
331f5207b7SJohn Levon# defaults to not verbose
341f5207b7SJohn Levon[ -z "$V" ] && V=0
35*c85f09ccSJohn Levonvquiet=""
36*c85f09ccSJohn Levonquiet=0
37*c85f09ccSJohn Levonabort=0
38*c85f09ccSJohn Levon
39*c85f09ccSJohn Levon
40*c85f09ccSJohn Levon##
41*c85f09ccSJohn Levon# verbose(string) - prints string if we are in verbose mode
42*c85f09ccSJohn Levonverbose()
43*c85f09ccSJohn Levon{
44*c85f09ccSJohn Levon	[ "$V" -eq "1" ] && echo "        $1"
45*c85f09ccSJohn Levon	return 0
46*c85f09ccSJohn Levon}
47*c85f09ccSJohn Levon
48*c85f09ccSJohn Levon##
49*c85f09ccSJohn Levon# warning(string) - prints a warning
50*c85f09ccSJohn Levonwarning()
51*c85f09ccSJohn Levon{
52*c85f09ccSJohn Levon	[ "$quiet" -ne 1 ] && echo "warning: $1"
53*c85f09ccSJohn Levon	return 0
54*c85f09ccSJohn Levon}
55*c85f09ccSJohn Levon
56*c85f09ccSJohn Levon##
57*c85f09ccSJohn Levon# error(string[, die]) - prints an error and exits with value die if given
58*c85f09ccSJohn Levonerror()
59*c85f09ccSJohn Levon{
60*c85f09ccSJohn Levon	[ "$quiet" -ne 1 ] && echo "error: $1"
61*c85f09ccSJohn Levon	[ -n "$2" ] && exit $2
62*c85f09ccSJohn Levon	return 0
63*c85f09ccSJohn Levon}
64*c85f09ccSJohn Levon
651f5207b7SJohn Levon
661f5207b7SJohn Levon##
671f5207b7SJohn Levon# get_tag_value(file) - get the 'check-<...>' tags & values
681f5207b7SJohn Levonget_tag_value()
691f5207b7SJohn Levon{
701f5207b7SJohn Levon	check_name=""
711f5207b7SJohn Levon	check_command="$default_cmd"
721f5207b7SJohn Levon	check_exit_value=0
731f5207b7SJohn Levon	check_timeout=0
741f5207b7SJohn Levon	check_known_to_fail=0
751f5207b7SJohn Levon	check_error_ignore=0
761f5207b7SJohn Levon	check_output_ignore=0
771f5207b7SJohn Levon	check_output_contains=0
781f5207b7SJohn Levon	check_output_excludes=0
791f5207b7SJohn Levon	check_output_pattern=0
80*c85f09ccSJohn Levon	check_arch_ignore=""
81*c85f09ccSJohn Levon	check_arch_only=""
82*c85f09ccSJohn Levon	check_assert=""
83*c85f09ccSJohn Levon	check_cpp_if=""
841f5207b7SJohn Levon
851f5207b7SJohn Levon	lines=$(grep 'check-[a-z-]*' $1 | \
861f5207b7SJohn Levon		sed -e 's/^.*\(check-[a-z-]*:*\) *\(.*\)$/\1 \2/')
871f5207b7SJohn Levon
881f5207b7SJohn Levon	while read tag val; do
891f5207b7SJohn Levon		#echo "-> tag: '$tag'"
901f5207b7SJohn Levon		#echo "-> val: '$val'"
911f5207b7SJohn Levon		case $tag in
921f5207b7SJohn Levon		check-name:)		check_name="$val" ;;
931f5207b7SJohn Levon		check-command:)		check_command="$val" ;;
941f5207b7SJohn Levon		check-exit-value:)	check_exit_value="$val" ;;
951f5207b7SJohn Levon		check-timeout:)		[ -z "$val" ] && val=1
961f5207b7SJohn Levon					check_timeout="$val" ;;
971f5207b7SJohn Levon		check-known-to-fail)	check_known_to_fail=1 ;;
981f5207b7SJohn Levon		check-error-ignore)	check_error_ignore=1 ;;
991f5207b7SJohn Levon		check-output-ignore)	check_output_ignore=1 ;;
1001f5207b7SJohn Levon		check-output-contains:)	check_output_contains=1 ;;
1011f5207b7SJohn Levon		check-output-excludes:)	check_output_excludes=1 ;;
102*c85f09ccSJohn Levon		check-output-pattern)	check_output_pattern=1 ;;
103*c85f09ccSJohn Levon		check-arch-ignore:)	arch=$(uname -m)
104*c85f09ccSJohn Levon					check_arch_ignore="$val" ;;
105*c85f09ccSJohn Levon		check-arch-only:)	arch=$(uname -m)
106*c85f09ccSJohn Levon					check_arch_only="$val" ;;
107*c85f09ccSJohn Levon		check-assert:)		check_assert="$val" ;;
108*c85f09ccSJohn Levon		check-cpp-if:)		check_cpp_if="$val" ;;
109*c85f09ccSJohn Levon
110*c85f09ccSJohn Levon		check-description:)	;;	# ignore
111*c85f09ccSJohn Levon		check-note:)		;;	# ignore
112*c85f09ccSJohn Levon		check-warning:)		;;	# ignore
113*c85f09ccSJohn Levon		check-error-start)	;;	# ignore
114*c85f09ccSJohn Levon		check-error-end)	;;	# ignore
115*c85f09ccSJohn Levon		check-output-start)	;;	# ignore
116*c85f09ccSJohn Levon		check-output-end)	;;	# ignore
117*c85f09ccSJohn Levon		check-should-pass)	;;	# ignore, unused annotation
118*c85f09ccSJohn Levon		check-should-fail)	;;	# ignore, unused annotation
119*c85f09ccSJohn Levon		check-should-warn)	;;	# ignore, unused annotation
120*c85f09ccSJohn Levon		check-*)		error "$1: unknown tag '$tag'" 1 ;;
1211f5207b7SJohn Levon		esac
1221f5207b7SJohn Levon	done << EOT
1231f5207b7SJohn Levon	$lines
1241f5207b7SJohn LevonEOT
1251f5207b7SJohn Levon}
1261f5207b7SJohn Levon
1271f5207b7SJohn Levon##
1281f5207b7SJohn Levon# helper for has_(each|none)_patterns()
1291f5207b7SJohn Levonhas_patterns()
1301f5207b7SJohn Levon{
1311f5207b7SJohn Levon	ifile="$1"
1321f5207b7SJohn Levon	patt="$2"
1331f5207b7SJohn Levon	ofile="$3"
1341f5207b7SJohn Levon	cmp="$4"
135*c85f09ccSJohn Levon	msg="$5"
1361f5207b7SJohn Levon	grep "$patt:" "$ifile" | \
1371f5207b7SJohn Levon	sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
1381f5207b7SJohn Levon	while read val; do
1391f5207b7SJohn Levon		grep -s -q "$val" "$ofile"
1401f5207b7SJohn Levon		if [ "$?" $cmp 0 ]; then
141*c85f09ccSJohn Levon			error "	Pattern '$val' unexpectedly $msg"
1421f5207b7SJohn Levon			return 1
1431f5207b7SJohn Levon		fi
1441f5207b7SJohn Levon	done
1451f5207b7SJohn Levon
1461f5207b7SJohn Levon	return $?
1471f5207b7SJohn Levon}
1481f5207b7SJohn Levon
1491f5207b7SJohn Levon##
1501f5207b7SJohn Levon# has_each_patterns(ifile tag ofile) - does ofile contains some
1511f5207b7SJohn Levon#                        of the patterns given by ifile's tags?
1521f5207b7SJohn Levon#
1531f5207b7SJohn Levon# returns 0 if all present, 1 otherwise
1541f5207b7SJohn Levonhas_each_patterns()
1551f5207b7SJohn Levon{
156*c85f09ccSJohn Levon	has_patterns "$1" "$2" "$4" -ne "$3"
1571f5207b7SJohn Levon}
1581f5207b7SJohn Levon
1591f5207b7SJohn Levon##
1601f5207b7SJohn Levon# has_none_patterns(ifile tag ofile) - does ofile contains some
1611f5207b7SJohn Levon#                        of the patterns given by ifile's tags?
1621f5207b7SJohn Levon#
1631f5207b7SJohn Levon# returns 1 if any present, 0 otherwise
1641f5207b7SJohn Levonhas_none_patterns()
1651f5207b7SJohn Levon{
166*c85f09ccSJohn Levon	has_patterns "$1" "$2" "$4" -eq "$3"
1671f5207b7SJohn Levon}
1681f5207b7SJohn Levon
1691f5207b7SJohn Levon##
170*c85f09ccSJohn Levon# minmax_patterns(ifile tag ofile) - does ofile contains the
1711f5207b7SJohn Levon#                        the patterns given by ifile's tags
1721f5207b7SJohn Levon#                        the right number of time?
173*c85f09ccSJohn Levonminmax_patterns()
1741f5207b7SJohn Levon{
1751f5207b7SJohn Levon	ifile="$1"
1761f5207b7SJohn Levon	patt="$2"
1771f5207b7SJohn Levon	ofile="$3"
178*c85f09ccSJohn Levon	grep "$patt([0-9-]*\(, *\)*[0-9-]*):" "$ifile" | \
179*c85f09ccSJohn Levon	sed -e "s/^.*$patt(\([0-9]*\)): *\(.*\)/\1 eq \2/" \
180*c85f09ccSJohn Levon	    -e "s/^.*$patt(\([0-9-]*\), *\([0-9-]*\)): *\(.*\)/\1 \2 \3/" | \
181*c85f09ccSJohn Levon	while read min max pat; do
1821f5207b7SJohn Levon		n=$(grep -s "$pat" "$ofile" | wc -l)
183*c85f09ccSJohn Levon		if [ "$max" = "eq" ]; then
184*c85f09ccSJohn Levon		    if [ "$n" -ne "$min" ]; then
185*c85f09ccSJohn Levon			error "	Pattern '$pat' expected $min times but got $n times"
1861f5207b7SJohn Levon			return 1
187*c85f09ccSJohn Levon		    fi
188*c85f09ccSJohn Levon		    continue
189*c85f09ccSJohn Levon		fi
190*c85f09ccSJohn Levon		if [ "$min" != '-' ]; then
191*c85f09ccSJohn Levon		    if [ "$n" -lt "$min" ]; then
192*c85f09ccSJohn Levon			error "	Pattern '$pat' expected min $min times but got $n times"
193*c85f09ccSJohn Levon			return 1
194*c85f09ccSJohn Levon		    fi
195*c85f09ccSJohn Levon		fi
196*c85f09ccSJohn Levon		if [ "$max" != '-' ]; then
197*c85f09ccSJohn Levon		    if [ "$n" -gt "$max" ]; then
198*c85f09ccSJohn Levon			error "	Pattern '$pat' expected max $max times but got $n times"
199*c85f09ccSJohn Levon			return 1
200*c85f09ccSJohn Levon		    fi
2011f5207b7SJohn Levon		fi
2021f5207b7SJohn Levon	done
2031f5207b7SJohn Levon
2041f5207b7SJohn Levon	return $?
2051f5207b7SJohn Levon}
2061f5207b7SJohn Levon
2071f5207b7SJohn Levon##
208*c85f09ccSJohn Levon# arg_file(filename) - checks if filename exists
209*c85f09ccSJohn Levonarg_file()
2101f5207b7SJohn Levon{
211*c85f09ccSJohn Levon	[ -z "$1" ] && {
212*c85f09ccSJohn Levon		do_usage
213*c85f09ccSJohn Levon		exit 1
214*c85f09ccSJohn Levon	}
215*c85f09ccSJohn Levon	[ -e "$1" ] || {
216*c85f09ccSJohn Levon		error "Can't open file $1"
217*c85f09ccSJohn Levon		exit 1
218*c85f09ccSJohn Levon	}
2191f5207b7SJohn Levon	return 0
2201f5207b7SJohn Levon}
2211f5207b7SJohn Levon
2221f5207b7SJohn Levon
223*c85f09ccSJohn Levon##
2241f5207b7SJohn Levondo_usage()
2251f5207b7SJohn Levon{
2261f5207b7SJohn Levonecho "$prog_name - a tiny automatic testing script"
227*c85f09ccSJohn Levonecho "Usage: $prog_name [option(s)] [command] [arguments]"
228*c85f09ccSJohn Levonecho
229*c85f09ccSJohn Levonecho "options:"
230*c85f09ccSJohn Levonecho "    -a|--abort                 Abort the tests as soon as one fails."
231*c85f09ccSJohn Levonecho "    -q|--quiet                 Be extra quiet while running the tests."
232*c85f09ccSJohn Levonecho "    --args='...'               Add these options to the test command."
2331f5207b7SJohn Levonecho
2341f5207b7SJohn Levonecho "commands:"
235*c85f09ccSJohn Levonecho "    [file ...]                 Runs the test suite on the given file(s)."
236*c85f09ccSJohn Levonecho "                               If a directory is given, run only those files."
237*c85f09ccSJohn Levonecho "                               If no file is given, run the whole testsuite."
238*c85f09ccSJohn Levonecho "    single file                Run the test in 'file'."
239*c85f09ccSJohn Levonecho "    format file [name [cmd]]   Help writing a new test case using cmd."
2401f5207b7SJohn Levonecho
241*c85f09ccSJohn Levonecho "    [command] help             Print usage."
242*c85f09ccSJohn Levon}
243*c85f09ccSJohn Levon
244*c85f09ccSJohn Levondisable()
245*c85f09ccSJohn Levon{
246*c85f09ccSJohn Levon	disabled_tests=$(($disabled_tests + 1))
247*c85f09ccSJohn Levon	if [ -z "$vquiet" ]; then
248*c85f09ccSJohn Levon		echo "  SKIP    $1 ($2)"
249*c85f09ccSJohn Levon	fi
2501f5207b7SJohn Levon}
2511f5207b7SJohn Levon
2521f5207b7SJohn Levon##
2531f5207b7SJohn Levon# do_test(file) - tries to validate a test case
2541f5207b7SJohn Levon#
2551f5207b7SJohn Levon# it "parses" file, looking for check-* tags and tries to validate
2561f5207b7SJohn Levon# the test against an expected result
2571f5207b7SJohn Levon# returns:
2581f5207b7SJohn Levon#	- 0 if the test passed,
2591f5207b7SJohn Levon#	- 1 if it failed,
2601f5207b7SJohn Levon#	- 2 if it is not a "test-suite" test.
2611f5207b7SJohn Levon#	- 3 if the test is disabled.
2621f5207b7SJohn Levondo_test()
2631f5207b7SJohn Levon{
2641f5207b7SJohn Levon	test_failed=0
2651f5207b7SJohn Levon	file="$1"
266*c85f09ccSJohn Levon	quiet=0
2671f5207b7SJohn Levon
2681f5207b7SJohn Levon	get_tag_value $file
2691f5207b7SJohn Levon
2701f5207b7SJohn Levon	# can this test be handled by test-suite ?
2711f5207b7SJohn Levon	# (it has to have a check-name key in it)
2721f5207b7SJohn Levon	if [ "$check_name" = "" ]; then
273*c85f09ccSJohn Levon		warning "$file: test unhandled"
2741f5207b7SJohn Levon		unhandled_tests=$(($unhandled_tests + 1))
2751f5207b7SJohn Levon		return 2
2761f5207b7SJohn Levon	fi
2771f5207b7SJohn Levon	test_name="$check_name"
2781f5207b7SJohn Levon
2791f5207b7SJohn Levon	# does the test provide a specific command ?
2801f5207b7SJohn Levon	if [ "$check_command" = "" ]; then
2811f5207b7SJohn Levon		check_command="$defaut_command"
2821f5207b7SJohn Levon	fi
2831f5207b7SJohn Levon
2841f5207b7SJohn Levon	# check for disabled commands
2851f5207b7SJohn Levon	set -- $check_command
2861f5207b7SJohn Levon	base_cmd=$1
2871f5207b7SJohn Levon	for i in $disabled_cmds; do
2881f5207b7SJohn Levon		if [ "$i" = "$base_cmd" ] ; then
289*c85f09ccSJohn Levon			disable "$test_name" "$file"
2901f5207b7SJohn Levon			return 3
2911f5207b7SJohn Levon		fi
2921f5207b7SJohn Levon	done
293*c85f09ccSJohn Levon	if [ "$check_arch_ignore" != "" ]; then
294*c85f09ccSJohn Levon		if echo $arch | egrep -q -w "$check_arch_ignore"; then
295*c85f09ccSJohn Levon			disable "$test_name" "$file"
296*c85f09ccSJohn Levon			return 3
297*c85f09ccSJohn Levon		fi
298*c85f09ccSJohn Levon	fi
299*c85f09ccSJohn Levon	if [ "$check_arch_only" != "" ]; then
300*c85f09ccSJohn Levon		if ! (echo $arch | egrep -q -w "$check_arch_only"); then
301*c85f09ccSJohn Levon			disable "$test_name" "$file"
302*c85f09ccSJohn Levon			return 3
303*c85f09ccSJohn Levon		fi
304*c85f09ccSJohn Levon	fi
305*c85f09ccSJohn Levon	if [ "$check_assert" != "" ]; then
306*c85f09ccSJohn Levon		res=$(../sparse - 2>&1 >/dev/null <<- EOF
307*c85f09ccSJohn Levon			_Static_assert($check_assert, "$check_assert");
308*c85f09ccSJohn Levon			EOF
309*c85f09ccSJohn Levon		)
310*c85f09ccSJohn Levon		if [ "$res" != "" ]; then
311*c85f09ccSJohn Levon			disable "$test_name" "$file"
312*c85f09ccSJohn Levon			return 3
313*c85f09ccSJohn Levon		fi
314*c85f09ccSJohn Levon	fi
315*c85f09ccSJohn Levon	if [ "$check_cpp_if" != "" ]; then
316*c85f09ccSJohn Levon		res=$(../sparse -E - 2>/dev/null <<- EOF
317*c85f09ccSJohn Levon			#if !($check_cpp_if)
318*c85f09ccSJohn Levon			fail
319*c85f09ccSJohn Levon			#endif
320*c85f09ccSJohn Levon			EOF
321*c85f09ccSJohn Levon		)
322*c85f09ccSJohn Levon		if [ "$res" != "" ]; then
323*c85f09ccSJohn Levon			disable "$test_name" "$file"
324*c85f09ccSJohn Levon			return 3
325*c85f09ccSJohn Levon		fi
326*c85f09ccSJohn Levon	fi
3271f5207b7SJohn Levon
328*c85f09ccSJohn Levon	if [ -z "$vquiet" ]; then
329*c85f09ccSJohn Levon		echo "  TEST    $test_name ($file)"
330*c85f09ccSJohn Levon	fi
3311f5207b7SJohn Levon
332*c85f09ccSJohn Levon	verbose "Using command       : $(echo "$@")"
3331f5207b7SJohn Levon
3341f5207b7SJohn Levon	# grab the expected exit value
3351f5207b7SJohn Levon	expected_exit_value=$check_exit_value
3361f5207b7SJohn Levon	verbose "Expecting exit value: $expected_exit_value"
3371f5207b7SJohn Levon
3381f5207b7SJohn Levon	# do we want a timeout?
339*c85f09ccSJohn Levon	pre_cmd=""
3401f5207b7SJohn Levon	if [ $check_timeout -ne 0 ]; then
341*c85f09ccSJohn Levon		pre_cmd="timeout -k 1s $check_timeout"
3421f5207b7SJohn Levon	fi
3431f5207b7SJohn Levon
344*c85f09ccSJohn Levon	shift
345*c85f09ccSJohn Levon	# launch the test command and
3461f5207b7SJohn Levon	# grab the actual output & exit value
347*c85f09ccSJohn Levon	eval $pre_cmd $default_path/$base_cmd $default_args "$@" \
348*c85f09ccSJohn Levon		1> $file.output.got 2> $file.error.got
3491f5207b7SJohn Levon	actual_exit_value=$?
3501f5207b7SJohn Levon
3511f5207b7SJohn Levon	must_fail=$check_known_to_fail
3521f5207b7SJohn Levon	[ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1
3531f5207b7SJohn Levon	known_ko_tests=$(($known_ko_tests + $must_fail))
3541f5207b7SJohn Levon
355*c85f09ccSJohn Levon	for stream in error output; do
3561f5207b7SJohn Levon		eval ignore=\$check_${stream}_ignore
3571f5207b7SJohn Levon		[ $ignore -eq 1 ] && continue
3581f5207b7SJohn Levon
3591f5207b7SJohn Levon		# grab the expected output
3601f5207b7SJohn Levon		sed -n "/check-$stream-start/,/check-$stream-end/p" $file \
3611f5207b7SJohn Levon		| grep -v check-$stream > "$file".$stream.expected
3621f5207b7SJohn Levon
3631f5207b7SJohn Levon		diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
3641f5207b7SJohn Levon		if [ "$?" -ne "0" ]; then
3651f5207b7SJohn Levon			error "actual $stream text does not match expected $stream text."
3661f5207b7SJohn Levon			error  "see $file.$stream.* for further investigation."
3671f5207b7SJohn Levon			[ $quiet -ne 1 ] && cat "$file".$stream.diff
3681f5207b7SJohn Levon			test_failed=1
3691f5207b7SJohn Levon		fi
3701f5207b7SJohn Levon	done
3711f5207b7SJohn Levon
3721f5207b7SJohn Levon	if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
3731f5207b7SJohn Levon		error "Actual exit value does not match the expected one."
3741f5207b7SJohn Levon		error "expected $expected_exit_value, got $actual_exit_value."
3751f5207b7SJohn Levon		test_failed=1
3761f5207b7SJohn Levon	fi
3771f5207b7SJohn Levon
3781f5207b7SJohn Levon	# verify the 'check-output-contains/excludes' tags
3791f5207b7SJohn Levon	if [ $check_output_contains -eq 1 ]; then
380*c85f09ccSJohn Levon		has_each_patterns "$file" 'check-output-contains' absent $file.output.got
3811f5207b7SJohn Levon		if [ "$?" -ne "0" ]; then
3821f5207b7SJohn Levon			test_failed=1
3831f5207b7SJohn Levon		fi
3841f5207b7SJohn Levon	fi
3851f5207b7SJohn Levon	if [ $check_output_excludes -eq 1 ]; then
386*c85f09ccSJohn Levon		has_none_patterns "$file" 'check-output-excludes' present $file.output.got
3871f5207b7SJohn Levon		if [ "$?" -ne "0" ]; then
3881f5207b7SJohn Levon			test_failed=1
3891f5207b7SJohn Levon		fi
3901f5207b7SJohn Levon	fi
3911f5207b7SJohn Levon	if [ $check_output_pattern -eq 1 ]; then
392*c85f09ccSJohn Levon		# verify the 'check-output-pattern(...)' tags
393*c85f09ccSJohn Levon		minmax_patterns "$file" 'check-output-pattern' $file.output.got
3941f5207b7SJohn Levon		if [ "$?" -ne "0" ]; then
3951f5207b7SJohn Levon			test_failed=1
3961f5207b7SJohn Levon		fi
3971f5207b7SJohn Levon	fi
3981f5207b7SJohn Levon
3991f5207b7SJohn Levon	if [ "$must_fail" -eq "1" ]; then
4001f5207b7SJohn Levon		if [ "$test_failed" -eq "1" ]; then
401*c85f09ccSJohn Levon			[ -z "$vquiet" ] && \
402*c85f09ccSJohn Levon			echo "info: XFAIL: test '$file' is known to fail"
4031f5207b7SJohn Levon		else
404*c85f09ccSJohn Levon			echo "error: XPASS: test '$file' is known to fail but succeed!"
4051f5207b7SJohn Levon		fi
406*c85f09ccSJohn Levon	else
407*c85f09ccSJohn Levon		if [ "$test_failed" -eq "1" ]; then
408*c85f09ccSJohn Levon			echo "error: FAIL: test '$file' failed"
409*c85f09ccSJohn Levon		else
410*c85f09ccSJohn Levon			[ "$V" -ne "0" ] && \
411*c85f09ccSJohn Levon			echo "info: PASS: test '$file' passed"
412*c85f09ccSJohn Levon		fi
413*c85f09ccSJohn Levon	fi
414*c85f09ccSJohn Levon
415*c85f09ccSJohn Levon	if [ "$test_failed" -ne "$must_fail" ]; then
416*c85f09ccSJohn Levon		[ $abort -eq 1 ] && exit 1
417*c85f09ccSJohn Levon		test_failed=1
418*c85f09ccSJohn Levon		failed=1
4191f5207b7SJohn Levon	fi
4201f5207b7SJohn Levon
4211f5207b7SJohn Levon	if [ "$test_failed" -eq "1" ]; then
4221f5207b7SJohn Levon		ko_tests=$(($ko_tests + 1))
4231f5207b7SJohn Levon	else
4241f5207b7SJohn Levon		ok_tests=$(($ok_tests + 1))
4251f5207b7SJohn Levon		rm -f $file.{error,output}.{expected,got,diff}
4261f5207b7SJohn Levon	fi
4271f5207b7SJohn Levon	return $test_failed
4281f5207b7SJohn Levon}
4291f5207b7SJohn Levon
4301f5207b7SJohn Levondo_test_suite()
4311f5207b7SJohn Levon{
4321f5207b7SJohn Levon	for i in $tests_list; do
4331f5207b7SJohn Levon		do_test "$i"
4341f5207b7SJohn Levon	done
4351f5207b7SJohn Levon
436*c85f09ccSJohn Levon	OK=OK
437*c85f09ccSJohn Levon	[ $failed -eq 0 ] || OK=KO
438*c85f09ccSJohn Levon
4391f5207b7SJohn Levon	# prints some numbers
4401f5207b7SJohn Levon	tests_nr=$(($ok_tests + $ko_tests))
441*c85f09ccSJohn Levon	echo "$OK: out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
442*c85f09ccSJohn Levon	if [ "$known_ko_tests" -ne 0 ]; then
443*c85f09ccSJohn Levon		echo "	$known_ko_tests of them are known to fail"
444*c85f09ccSJohn Levon	fi
4451f5207b7SJohn Levon	if [ "$unhandled_tests" -ne "0" ]; then
446*c85f09ccSJohn Levon		echo "	$unhandled_tests tests could not be handled by $prog_name"
4471f5207b7SJohn Levon	fi
4481f5207b7SJohn Levon	if [ "$disabled_tests" -ne "0" ]; then
449*c85f09ccSJohn Levon		echo "	$disabled_tests tests were disabled"
4501f5207b7SJohn Levon	fi
4511f5207b7SJohn Levon}
4521f5207b7SJohn Levon
4531f5207b7SJohn Levon##
454*c85f09ccSJohn Levondo_format_help() {
455*c85f09ccSJohn Levonecho "Usage: $prog_name [option(s)] [--]format file [name [cmd]]"
456*c85f09ccSJohn Levonecho
457*c85f09ccSJohn Levonecho "options:"
458*c85f09ccSJohn Levonecho "    -a                         append the created test to the input file"
459*c85f09ccSJohn Levonecho "    -f                         write a test known to fail"
460*c85f09ccSJohn Levonecho "    -l                         write a test for linearized code"
461*c85f09ccSJohn Levonecho
462*c85f09ccSJohn Levonecho "argument(s):"
463*c85f09ccSJohn Levonecho "    file                       file containing the test case(s)"
464*c85f09ccSJohn Levonecho "    name                       name for the test case (defaults to file)"
465*c85f09ccSJohn Levonecho "    cmd                        command to be used (defaults to 'sparse \$file')"
466*c85f09ccSJohn Levon}
467*c85f09ccSJohn Levon
468*c85f09ccSJohn Levon##
469*c85f09ccSJohn Levon# do_format([options,] file[, name[, cmd]]) - helps a test writer to format test-suite tags
4701f5207b7SJohn Levondo_format()
4711f5207b7SJohn Levon{
472*c85f09ccSJohn Levon	def_cmd="$default_cmd"
473*c85f09ccSJohn Levon	append=0
474*c85f09ccSJohn Levon	linear=0
475*c85f09ccSJohn Levon	fail=0
476*c85f09ccSJohn Levon
477*c85f09ccSJohn Levon	while [ $# -gt 1 ] ; do
478*c85f09ccSJohn Levon		case "$1" in
479*c85f09ccSJohn Levon		-a)
480*c85f09ccSJohn Levon			append=1 ;;
481*c85f09ccSJohn Levon		-f)
482*c85f09ccSJohn Levon			fail=1 ;;
483*c85f09ccSJohn Levon		-l)
484*c85f09ccSJohn Levon			def_cmd='test-linearize -Wno-decl $file'
485*c85f09ccSJohn Levon			linear=1 ;;
486*c85f09ccSJohn Levon		help|-*)
487*c85f09ccSJohn Levon			do_format_help
488*c85f09ccSJohn Levon			return 0
489*c85f09ccSJohn Levon			;;
490*c85f09ccSJohn Levon		*)	break ;;
491*c85f09ccSJohn Levon		esac
492*c85f09ccSJohn Levon		shift
493*c85f09ccSJohn Levon		continue
494*c85f09ccSJohn Levon	done
495*c85f09ccSJohn Levon
496*c85f09ccSJohn Levon	arg_file "$1" || return 1
497*c85f09ccSJohn Levon
4981f5207b7SJohn Levon	file="$1"
499*c85f09ccSJohn Levon	fname="$2"
500*c85f09ccSJohn Levon	[ -z "$fname" ] && fname="$(basename "$1" .c)"
501*c85f09ccSJohn Levon	fcmd="$3"
502*c85f09ccSJohn Levon	[ -z "$fcmd" ] && fcmd="$def_cmd"
503*c85f09ccSJohn Levon
5041f5207b7SJohn Levon	cmd=`eval echo $default_path/$fcmd`
5051f5207b7SJohn Levon	$cmd 1> $file.output.got 2> $file.error.got
5061f5207b7SJohn Levon	fexit_value=$?
507*c85f09ccSJohn Levon	[ $append != 0 ] && exec >> $file
5081f5207b7SJohn Levon	cat <<_EOF
509*c85f09ccSJohn Levon
5101f5207b7SJohn Levon/*
5111f5207b7SJohn Levon * check-name: $fname
5121f5207b7SJohn Levon_EOF
5131f5207b7SJohn Levon	if [ "$fcmd" != "$default_cmd" ]; then
5141f5207b7SJohn Levon		echo " * check-command: $fcmd"
5151f5207b7SJohn Levon	fi
5161f5207b7SJohn Levon	if [ "$fexit_value" -ne "0" ]; then
5171f5207b7SJohn Levon		echo " * check-exit-value: $fexit_value"
5181f5207b7SJohn Levon	fi
519*c85f09ccSJohn Levon	if [ $fail != 0 ]; then
520*c85f09ccSJohn Levon		echo " * check-known-to-fail"
521*c85f09ccSJohn Levon	fi
522*c85f09ccSJohn Levon	if [ $linear != 0 ]; then
523*c85f09ccSJohn Levon		echo ' *'
524*c85f09ccSJohn Levon		echo ' * check-output-ignore'
525*c85f09ccSJohn Levon		echo ' * check-output-contains: xyz\\\\.'
526*c85f09ccSJohn Levon		echo ' * check-output-excludes: \\\\.'
527*c85f09ccSJohn Levon	fi
5281f5207b7SJohn Levon	for stream in output error; do
5291f5207b7SJohn Levon		if [ -s "$file.$stream.got" ]; then
5301f5207b7SJohn Levon			echo " *"
5311f5207b7SJohn Levon			echo " * check-$stream-start"
5321f5207b7SJohn Levon			cat "$file.$stream.got"
5331f5207b7SJohn Levon			echo " * check-$stream-end"
5341f5207b7SJohn Levon		fi
5351f5207b7SJohn Levon	done
5361f5207b7SJohn Levon	echo " */"
5371f5207b7SJohn Levon	return 0
5381f5207b7SJohn Levon}
5391f5207b7SJohn Levon
540*c85f09ccSJohn Levon## allow flags from environment
541*c85f09ccSJohn Levonset -- $SPARSE_TEST_FLAGS "$@"
5421f5207b7SJohn Levon
543*c85f09ccSJohn Levon## process the flags
544*c85f09ccSJohn Levonwhile [ "$#" -gt "0" ]; do
545*c85f09ccSJohn Levon	case "$1" in
546*c85f09ccSJohn Levon	-a|--abort)
547*c85f09ccSJohn Levon		abort=1
548*c85f09ccSJohn Levon		;;
549*c85f09ccSJohn Levon	-q|--quiet)
550*c85f09ccSJohn Levon		vquiet=1
5511f5207b7SJohn Levon		;;
552*c85f09ccSJohn Levon	--args=*)
553*c85f09ccSJohn Levon		default_args="${1#--args=}";
554*c85f09ccSJohn Levon		;;
555*c85f09ccSJohn Levon
556*c85f09ccSJohn Levon	single|--single)
5571f5207b7SJohn Levon		arg_file "$2"
5581f5207b7SJohn Levon		do_test "$2"
5591f5207b7SJohn Levon		case "$?" in
5601f5207b7SJohn Levon			0) echo "$2 passed !";;
5611f5207b7SJohn Levon			1) echo "$2 failed !";;
5621f5207b7SJohn Levon			2) echo "$2 can't be handled by $prog_name";;
5631f5207b7SJohn Levon		esac
564*c85f09ccSJohn Levon		exit $failed
5651f5207b7SJohn Levon		;;
566*c85f09ccSJohn Levon	format|--format)
567*c85f09ccSJohn Levon		shift
568*c85f09ccSJohn Levon		do_format "$@"
569*c85f09ccSJohn Levon		exit 0
5701f5207b7SJohn Levon		;;
571*c85f09ccSJohn Levon	help)
5721f5207b7SJohn Levon		do_usage
5731f5207b7SJohn Levon		exit 1
5741f5207b7SJohn Levon		;;
5751f5207b7SJohn Levon
576*c85f09ccSJohn Levon	*.c|*.cdoc)
577*c85f09ccSJohn Levon		tests_list="$tests_list $1"
578*c85f09ccSJohn Levon		;;
579*c85f09ccSJohn Levon	*)
580*c85f09ccSJohn Levon		if [ ! -d "$1" ]; then
581*c85f09ccSJohn Levon			do_usage
582*c85f09ccSJohn Levon			exit 1
583*c85f09ccSJohn Levon		fi
584*c85f09ccSJohn Levon		tests_list="$tests_list $(find "$1" -name '*.c' | sort)"
585*c85f09ccSJohn Levon		;;
586*c85f09ccSJohn Levon	esac
587*c85f09ccSJohn Levon	shift
588*c85f09ccSJohn Levondone
589*c85f09ccSJohn Levon
590*c85f09ccSJohn Levonif [ -z "$tests_list" ]; then
591*c85f09ccSJohn Levon	tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
592*c85f09ccSJohn Levonfi
593*c85f09ccSJohn Levon
594*c85f09ccSJohn Levondo_test_suite
5951f5207b7SJohn Levonexit $failed
5961f5207b7SJohn Levon