1#!/bin/sh
2
3#set -x
4
5cd $(dirname "$0")
6
7default_path=".."
8default_cmd="sparse \$file"
9default_args="$SPARSE_TEST_ARGS"
10tests_list=""
11prog_name=`basename $0`
12
13if [ ! -x "$default_path/sparse-llvm" ]; then
14	disabled_cmds="sparsec sparsei sparse-llvm sparse-llvm-dis"
15fi
16
17# flags:
18#	- some tests gave an unexpected result
19failed=0
20
21# counts:
22#	- tests that have not been converted to test-suite format
23#	- tests that are disabled
24#	- tests that passed
25#	- tests that failed
26#	- tests that failed but are known to fail
27unhandled_tests=0
28disabled_tests=0
29ok_tests=0
30ko_tests=0
31known_ko_tests=0
32
33# defaults to not verbose
34[ -z "$V" ] && V=0
35vquiet=""
36quiet=0
37abort=0
38
39
40##
41# verbose(string) - prints string if we are in verbose mode
42verbose()
43{
44	[ "$V" -eq "1" ] && echo "        $1"
45	return 0
46}
47
48##
49# warning(string) - prints a warning
50warning()
51{
52	[ "$quiet" -ne 1 ] && echo "warning: $1"
53	return 0
54}
55
56##
57# error(string[, die]) - prints an error and exits with value die if given
58error()
59{
60	[ "$quiet" -ne 1 ] && echo "error: $1"
61	[ -n "$2" ] && exit $2
62	return 0
63}
64
65
66##
67# get_tag_value(file) - get the 'check-<...>' tags & values
68get_tag_value()
69{
70	check_name=""
71	check_command="$default_cmd"
72	check_exit_value=0
73	check_timeout=0
74	check_known_to_fail=0
75	check_error_ignore=0
76	check_output_ignore=0
77	check_output_contains=0
78	check_output_excludes=0
79	check_output_pattern=0
80	check_arch_ignore=""
81	check_arch_only=""
82	check_assert=""
83	check_cpp_if=""
84
85	lines=$(grep 'check-[a-z-]*' $1 | \
86		sed -e 's/^.*\(check-[a-z-]*:*\) *\(.*\)$/\1 \2/')
87
88	while read tag val; do
89		#echo "-> tag: '$tag'"
90		#echo "-> val: '$val'"
91		case $tag in
92		check-name:)		check_name="$val" ;;
93		check-command:)		check_command="$val" ;;
94		check-exit-value:)	check_exit_value="$val" ;;
95		check-timeout:)		[ -z "$val" ] && val=1
96					check_timeout="$val" ;;
97		check-known-to-fail)	check_known_to_fail=1 ;;
98		check-error-ignore)	check_error_ignore=1 ;;
99		check-output-ignore)	check_output_ignore=1 ;;
100		check-output-contains:)	check_output_contains=1 ;;
101		check-output-excludes:)	check_output_excludes=1 ;;
102		check-output-pattern)	check_output_pattern=1 ;;
103		check-arch-ignore:)	arch=$(uname -m)
104					check_arch_ignore="$val" ;;
105		check-arch-only:)	arch=$(uname -m)
106					check_arch_only="$val" ;;
107		check-assert:)		check_assert="$val" ;;
108		check-cpp-if:)		check_cpp_if="$val" ;;
109
110		check-description:)	;;	# ignore
111		check-note:)		;;	# ignore
112		check-warning:)		;;	# ignore
113		check-error-start)	;;	# ignore
114		check-error-end)	;;	# ignore
115		check-output-start)	;;	# ignore
116		check-output-end)	;;	# ignore
117		check-should-pass)	;;	# ignore, unused annotation
118		check-should-fail)	;;	# ignore, unused annotation
119		check-should-warn)	;;	# ignore, unused annotation
120		check-*)		error "$1: unknown tag '$tag'" 1 ;;
121		esac
122	done << EOT
123	$lines
124EOT
125}
126
127##
128# helper for has_(each|none)_patterns()
129has_patterns()
130{
131	ifile="$1"
132	patt="$2"
133	ofile="$3"
134	cmp="$4"
135	msg="$5"
136	grep "$patt:" "$ifile" | \
137	sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
138	while read val; do
139		grep -s -q "$val" "$ofile"
140		if [ "$?" $cmp 0 ]; then
141			error "	Pattern '$val' unexpectedly $msg"
142			return 1
143		fi
144	done
145
146	return $?
147}
148
149##
150# has_each_patterns(ifile tag ofile) - does ofile contains some
151#                        of the patterns given by ifile's tags?
152#
153# returns 0 if all present, 1 otherwise
154has_each_patterns()
155{
156	has_patterns "$1" "$2" "$4" -ne "$3"
157}
158
159##
160# has_none_patterns(ifile tag ofile) - does ofile contains some
161#                        of the patterns given by ifile's tags?
162#
163# returns 1 if any present, 0 otherwise
164has_none_patterns()
165{
166	has_patterns "$1" "$2" "$4" -eq "$3"
167}
168
169##
170# minmax_patterns(ifile tag ofile) - does ofile contains the
171#                        the patterns given by ifile's tags
172#                        the right number of time?
173minmax_patterns()
174{
175	ifile="$1"
176	patt="$2"
177	ofile="$3"
178	grep "$patt([0-9-]*\(, *\)*[0-9-]*):" "$ifile" | \
179	sed -e "s/^.*$patt(\([0-9]*\)): *\(.*\)/\1 eq \2/" \
180	    -e "s/^.*$patt(\([0-9-]*\), *\([0-9-]*\)): *\(.*\)/\1 \2 \3/" | \
181	while read min max pat; do
182		n=$(grep -s "$pat" "$ofile" | wc -l)
183		if [ "$max" = "eq" ]; then
184		    if [ "$n" -ne "$min" ]; then
185			error "	Pattern '$pat' expected $min times but got $n times"
186			return 1
187		    fi
188		    continue
189		fi
190		if [ "$min" != '-' ]; then
191		    if [ "$n" -lt "$min" ]; then
192			error "	Pattern '$pat' expected min $min times but got $n times"
193			return 1
194		    fi
195		fi
196		if [ "$max" != '-' ]; then
197		    if [ "$n" -gt "$max" ]; then
198			error "	Pattern '$pat' expected max $max times but got $n times"
199			return 1
200		    fi
201		fi
202	done
203
204	return $?
205}
206
207##
208# arg_file(filename) - checks if filename exists
209arg_file()
210{
211	[ -z "$1" ] && {
212		do_usage
213		exit 1
214	}
215	[ -e "$1" ] || {
216		error "Can't open file $1"
217		exit 1
218	}
219	return 0
220}
221
222
223##
224do_usage()
225{
226echo "$prog_name - a tiny automatic testing script"
227echo "Usage: $prog_name [option(s)] [command] [arguments]"
228echo
229echo "options:"
230echo "    -a|--abort                 Abort the tests as soon as one fails."
231echo "    -q|--quiet                 Be extra quiet while running the tests."
232echo "    --args='...'               Add these options to the test command."
233echo
234echo "commands:"
235echo "    [file ...]                 Runs the test suite on the given file(s)."
236echo "                               If a directory is given, run only those files."
237echo "                               If no file is given, run the whole testsuite."
238echo "    single file                Run the test in 'file'."
239echo "    format file [name [cmd]]   Help writing a new test case using cmd."
240echo
241echo "    [command] help             Print usage."
242}
243
244disable()
245{
246	disabled_tests=$(($disabled_tests + 1))
247	if [ -z "$vquiet" ]; then
248		echo "  SKIP    $1 ($2)"
249	fi
250}
251
252##
253# do_test(file) - tries to validate a test case
254#
255# it "parses" file, looking for check-* tags and tries to validate
256# the test against an expected result
257# returns:
258#	- 0 if the test passed,
259#	- 1 if it failed,
260#	- 2 if it is not a "test-suite" test.
261#	- 3 if the test is disabled.
262do_test()
263{
264	test_failed=0
265	file="$1"
266	quiet=0
267
268	get_tag_value $file
269
270	# can this test be handled by test-suite ?
271	# (it has to have a check-name key in it)
272	if [ "$check_name" = "" ]; then
273		warning "$file: test unhandled"
274		unhandled_tests=$(($unhandled_tests + 1))
275		return 2
276	fi
277	test_name="$check_name"
278
279	# does the test provide a specific command ?
280	if [ "$check_command" = "" ]; then
281		check_command="$defaut_command"
282	fi
283
284	# check for disabled commands
285	set -- $check_command
286	base_cmd=$1
287	for i in $disabled_cmds; do
288		if [ "$i" = "$base_cmd" ] ; then
289			disable "$test_name" "$file"
290			return 3
291		fi
292	done
293	if [ "$check_arch_ignore" != "" ]; then
294		if echo $arch | egrep -q -w "$check_arch_ignore"; then
295			disable "$test_name" "$file"
296			return 3
297		fi
298	fi
299	if [ "$check_arch_only" != "" ]; then
300		if ! (echo $arch | egrep -q -w "$check_arch_only"); then
301			disable "$test_name" "$file"
302			return 3
303		fi
304	fi
305	if [ "$check_assert" != "" ]; then
306		res=$(../sparse - 2>&1 >/dev/null <<- EOF
307			_Static_assert($check_assert, "$check_assert");
308			EOF
309		)
310		if [ "$res" != "" ]; then
311			disable "$test_name" "$file"
312			return 3
313		fi
314	fi
315	if [ "$check_cpp_if" != "" ]; then
316		res=$(../sparse -E - 2>/dev/null <<- EOF
317			#if !($check_cpp_if)
318			fail
319			#endif
320			EOF
321		)
322		if [ "$res" != "" ]; then
323			disable "$test_name" "$file"
324			return 3
325		fi
326	fi
327
328	if [ -z "$vquiet" ]; then
329		echo "  TEST    $test_name ($file)"
330	fi
331
332	verbose "Using command       : $(echo "$@")"
333
334	# grab the expected exit value
335	expected_exit_value=$check_exit_value
336	verbose "Expecting exit value: $expected_exit_value"
337
338	# do we want a timeout?
339	pre_cmd=""
340	if [ $check_timeout -ne 0 ]; then
341		pre_cmd="timeout -k 1s $check_timeout"
342	fi
343
344	shift
345	# launch the test command and
346	# grab the actual output & exit value
347	eval $pre_cmd $default_path/$base_cmd $default_args "$@" \
348		1> $file.output.got 2> $file.error.got
349	actual_exit_value=$?
350
351	must_fail=$check_known_to_fail
352	[ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1
353	known_ko_tests=$(($known_ko_tests + $must_fail))
354
355	for stream in error output; do
356		eval ignore=\$check_${stream}_ignore
357		[ $ignore -eq 1 ] && continue
358
359		# grab the expected output
360		sed -n "/check-$stream-start/,/check-$stream-end/p" $file \
361		| grep -v check-$stream > "$file".$stream.expected
362
363		diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
364		if [ "$?" -ne "0" ]; then
365			error "actual $stream text does not match expected $stream text."
366			error  "see $file.$stream.* for further investigation."
367			[ $quiet -ne 1 ] && cat "$file".$stream.diff
368			test_failed=1
369		fi
370	done
371
372	if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
373		error "Actual exit value does not match the expected one."
374		error "expected $expected_exit_value, got $actual_exit_value."
375		test_failed=1
376	fi
377
378	# verify the 'check-output-contains/excludes' tags
379	if [ $check_output_contains -eq 1 ]; then
380		has_each_patterns "$file" 'check-output-contains' absent $file.output.got
381		if [ "$?" -ne "0" ]; then
382			test_failed=1
383		fi
384	fi
385	if [ $check_output_excludes -eq 1 ]; then
386		has_none_patterns "$file" 'check-output-excludes' present $file.output.got
387		if [ "$?" -ne "0" ]; then
388			test_failed=1
389		fi
390	fi
391	if [ $check_output_pattern -eq 1 ]; then
392		# verify the 'check-output-pattern(...)' tags
393		minmax_patterns "$file" 'check-output-pattern' $file.output.got
394		if [ "$?" -ne "0" ]; then
395			test_failed=1
396		fi
397	fi
398
399	if [ "$must_fail" -eq "1" ]; then
400		if [ "$test_failed" -eq "1" ]; then
401			[ -z "$vquiet" ] && \
402			echo "info: XFAIL: test '$file' is known to fail"
403		else
404			echo "error: XPASS: test '$file' is known to fail but succeed!"
405		fi
406	else
407		if [ "$test_failed" -eq "1" ]; then
408			echo "error: FAIL: test '$file' failed"
409		else
410			[ "$V" -ne "0" ] && \
411			echo "info: PASS: test '$file' passed"
412		fi
413	fi
414
415	if [ "$test_failed" -ne "$must_fail" ]; then
416		[ $abort -eq 1 ] && exit 1
417		test_failed=1
418		failed=1
419	fi
420
421	if [ "$test_failed" -eq "1" ]; then
422		ko_tests=$(($ko_tests + 1))
423	else
424		ok_tests=$(($ok_tests + 1))
425		rm -f $file.{error,output}.{expected,got,diff}
426	fi
427	return $test_failed
428}
429
430do_test_suite()
431{
432	for i in $tests_list; do
433		do_test "$i"
434	done
435
436	OK=OK
437	[ $failed -eq 0 ] || OK=KO
438
439	# prints some numbers
440	tests_nr=$(($ok_tests + $ko_tests))
441	echo "$OK: out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
442	if [ "$known_ko_tests" -ne 0 ]; then
443		echo "	$known_ko_tests of them are known to fail"
444	fi
445	if [ "$unhandled_tests" -ne "0" ]; then
446		echo "	$unhandled_tests tests could not be handled by $prog_name"
447	fi
448	if [ "$disabled_tests" -ne "0" ]; then
449		echo "	$disabled_tests tests were disabled"
450	fi
451}
452
453##
454do_format_help() {
455echo "Usage: $prog_name [option(s)] [--]format file [name [cmd]]"
456echo
457echo "options:"
458echo "    -a                         append the created test to the input file"
459echo "    -f                         write a test known to fail"
460echo "    -l                         write a test for linearized code"
461echo
462echo "argument(s):"
463echo "    file                       file containing the test case(s)"
464echo "    name                       name for the test case (defaults to file)"
465echo "    cmd                        command to be used (defaults to 'sparse \$file')"
466}
467
468##
469# do_format([options,] file[, name[, cmd]]) - helps a test writer to format test-suite tags
470do_format()
471{
472	def_cmd="$default_cmd"
473	append=0
474	linear=0
475	fail=0
476
477	while [ $# -gt 1 ] ; do
478		case "$1" in
479		-a)
480			append=1 ;;
481		-f)
482			fail=1 ;;
483		-l)
484			def_cmd='test-linearize -Wno-decl $file'
485			linear=1 ;;
486		help|-*)
487			do_format_help
488			return 0
489			;;
490		*)	break ;;
491		esac
492		shift
493		continue
494	done
495
496	arg_file "$1" || return 1
497
498	file="$1"
499	fname="$2"
500	[ -z "$fname" ] && fname="$(basename "$1" .c)"
501	fcmd="$3"
502	[ -z "$fcmd" ] && fcmd="$def_cmd"
503
504	cmd=`eval echo $default_path/$fcmd`
505	$cmd 1> $file.output.got 2> $file.error.got
506	fexit_value=$?
507	[ $append != 0 ] && exec >> $file
508	cat <<_EOF
509
510/*
511 * check-name: $fname
512_EOF
513	if [ "$fcmd" != "$default_cmd" ]; then
514		echo " * check-command: $fcmd"
515	fi
516	if [ "$fexit_value" -ne "0" ]; then
517		echo " * check-exit-value: $fexit_value"
518	fi
519	if [ $fail != 0 ]; then
520		echo " * check-known-to-fail"
521	fi
522	if [ $linear != 0 ]; then
523		echo ' *'
524		echo ' * check-output-ignore'
525		echo ' * check-output-contains: xyz\\\\.'
526		echo ' * check-output-excludes: \\\\.'
527	fi
528	for stream in output error; do
529		if [ -s "$file.$stream.got" ]; then
530			echo " *"
531			echo " * check-$stream-start"
532			cat "$file.$stream.got"
533			echo " * check-$stream-end"
534		fi
535	done
536	echo " */"
537	return 0
538}
539
540## allow flags from environment
541set -- $SPARSE_TEST_FLAGS "$@"
542
543## process the flags
544while [ "$#" -gt "0" ]; do
545	case "$1" in
546	-a|--abort)
547		abort=1
548		;;
549	-q|--quiet)
550		vquiet=1
551		;;
552	--args=*)
553		default_args="${1#--args=}";
554		;;
555
556	single|--single)
557		arg_file "$2"
558		do_test "$2"
559		case "$?" in
560			0) echo "$2 passed !";;
561			1) echo "$2 failed !";;
562			2) echo "$2 can't be handled by $prog_name";;
563		esac
564		exit $failed
565		;;
566	format|--format)
567		shift
568		do_format "$@"
569		exit 0
570		;;
571	help)
572		do_usage
573		exit 1
574		;;
575
576	*.c|*.cdoc)
577		tests_list="$tests_list $1"
578		;;
579	*)
580		if [ ! -d "$1" ]; then
581			do_usage
582			exit 1
583		fi
584		tests_list="$tests_list $(find "$1" -name '*.c' | sort)"
585		;;
586	esac
587	shift
588done
589
590if [ -z "$tests_list" ]; then
591	tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
592fi
593
594do_test_suite
595exit $failed
596
597