1#!/bin/sh
2
3#set -x
4
5cd $(dirname "$0")
6
7default_path=".."
8default_cmd="sparse \$file"
9tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
10prog_name=`basename $0`
11
12if [ ! -x "$default_path/sparse-llvm" ]; then
13	disabled_cmds="sparsec sparsei sparse-llvm"
14fi
15
16# flags:
17#	- some tests gave an unexpected result
18failed=0
19
20# counts:
21#	- tests that have not been converted to test-suite format
22#	- tests that are disabled
23#	- tests that passed
24#	- tests that failed
25#	- tests that failed but are known to fail
26unhandled_tests=0
27disabled_tests=0
28ok_tests=0
29ko_tests=0
30known_ko_tests=0
31
32# defaults to not verbose
33[ -z "$V" ] && V=0
34[ $V -eq 0 ] && quiet=1 || quiet=0
35
36##
37# get_tag_value(file) - get the 'check-<...>' tags & values
38get_tag_value()
39{
40	check_name=""
41	check_command="$default_cmd"
42	check_exit_value=0
43	check_timeout=0
44	check_known_to_fail=0
45	check_error_ignore=0
46	check_output_ignore=0
47	check_output_contains=0
48	check_output_excludes=0
49	check_output_pattern=0
50
51	lines=$(grep 'check-[a-z-]*' $1 | \
52		sed -e 's/^.*\(check-[a-z-]*:*\) *\(.*\)$/\1 \2/')
53
54	while read tag val; do
55		#echo "-> tag: '$tag'"
56		#echo "-> val: '$val'"
57		case $tag in
58		check-name:)		check_name="$val" ;;
59		check-command:)		check_command="$val" ;;
60		check-exit-value:)	check_exit_value="$val" ;;
61		check-timeout:)		[ -z "$val" ] && val=1
62					check_timeout="$val" ;;
63		check-known-to-fail)	check_known_to_fail=1 ;;
64		check-error-ignore)	check_error_ignore=1 ;;
65		check-output-ignore)	check_output_ignore=1 ;;
66		check-output-contains:)	check_output_contains=1 ;;
67		check-output-excludes:)	check_output_excludes=1 ;;
68		check-output-pattern-)	check_output_pattern=1 ;;
69		esac
70	done << EOT
71	$lines
72EOT
73}
74
75##
76# helper for has_(each|none)_patterns()
77has_patterns()
78{
79	ifile="$1"
80	patt="$2"
81	ofile="$3"
82	cmp="$4"
83	grep "$patt:" "$ifile" | \
84	sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
85	while read val; do
86		grep -s -q "$val" "$ofile"
87		if [ "$?" $cmp 0 ]; then
88			return 1
89		fi
90	done
91
92	return $?
93}
94
95##
96# has_each_patterns(ifile tag ofile) - does ofile contains some
97#                        of the patterns given by ifile's tags?
98#
99# returns 0 if all present, 1 otherwise
100has_each_patterns()
101{
102	has_patterns "$1" "$2" "$3" -ne
103}
104
105##
106# has_none_patterns(ifile tag ofile) - does ofile contains some
107#                        of the patterns given by ifile's tags?
108#
109# returns 1 if any present, 0 otherwise
110has_none_patterns()
111{
112	has_patterns "$1" "$2" "$3" -eq
113}
114
115##
116# nbr_patterns(ifile tag ofile) - does ofile contains the
117#                        the patterns given by ifile's tags
118#                        the right number of time?
119nbr_patterns()
120{
121	ifile="$1"
122	patt="$2"
123	ofile="$3"
124	grep "$patt-[0-9][0-9]*-times:" "$ifile" | \
125	sed -e "s/^.*$patt-\([0-9][0-9]*\)-times: *\(.*\)/\1 \2/" | \
126	while read nbr pat; do
127		n=$(grep -s "$pat" "$ofile" | wc -l)
128		if [ "$n" -ne "$nbr" ]; then
129			return 1
130		fi
131	done
132
133	return $?
134}
135
136##
137# verbose(string) - prints string if we are in verbose mode
138verbose()
139{
140	[ "$V" -eq "1" ] && echo "        $1"
141	return 0
142}
143
144##
145# error(string[, die]) - prints an error and exits with value die if given
146error()
147{
148	[ "$quiet" -ne 1 ] && echo "error: $1"
149	[ -n "$2" ] && exit $2
150	return 0
151}
152
153do_usage()
154{
155echo "$prog_name - a tiny automatic testing script"
156echo "Usage: $prog_name [command] [command arguments]"
157echo
158echo "commands:"
159echo "    none                       runs the whole test suite"
160echo "    single file                runs the test in 'file'"
161echo "    format file [name [cmd]]   helps writing a new test case using cmd"
162echo
163echo "    help                       prints usage"
164}
165
166##
167# do_test(file) - tries to validate a test case
168#
169# it "parses" file, looking for check-* tags and tries to validate
170# the test against an expected result
171# returns:
172#	- 0 if the test passed,
173#	- 1 if it failed,
174#	- 2 if it is not a "test-suite" test.
175#	- 3 if the test is disabled.
176do_test()
177{
178	test_failed=0
179	file="$1"
180
181	get_tag_value $file
182
183	# can this test be handled by test-suite ?
184	# (it has to have a check-name key in it)
185	if [ "$check_name" = "" ]; then
186		echo "warning: test '$file' unhandled"
187		unhandled_tests=$(($unhandled_tests + 1))
188		return 2
189	fi
190	test_name="$check_name"
191
192	# does the test provide a specific command ?
193	if [ "$check_command" = "" ]; then
194		check_command="$defaut_command"
195	fi
196
197	# check for disabled commands
198	set -- $check_command
199	base_cmd=$1
200	for i in $disabled_cmds; do
201		if [ "$i" = "$base_cmd" ] ; then
202			disabled_tests=$(($disabled_tests + 1))
203			echo "     DISABLE $test_name ($file)"
204			return 3
205		fi
206	done
207
208	cmd=`eval echo $default_path/$check_command`
209
210	echo "     TEST    $test_name ($file)"
211
212	verbose "Using command       : $cmd"
213
214	# grab the expected exit value
215	expected_exit_value=$check_exit_value
216	verbose "Expecting exit value: $expected_exit_value"
217
218	# do we want a timeout?
219	if [ $check_timeout -ne 0 ]; then
220		cmd="timeout -k 1s $check_timeout $cmd"
221	fi
222
223	# grab the actual output & exit value
224	$cmd 1> $file.output.got 2> $file.error.got
225	actual_exit_value=$?
226
227	must_fail=$check_known_to_fail
228	quiet=0
229	[ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1
230	known_ko_tests=$(($known_ko_tests + $must_fail))
231
232	for stream in output error; do
233		eval ignore=\$check_${stream}_ignore
234		[ $ignore -eq 1 ] && continue
235
236		# grab the expected output
237		sed -n "/check-$stream-start/,/check-$stream-end/p" $file \
238		| grep -v check-$stream > "$file".$stream.expected
239
240		diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
241		if [ "$?" -ne "0" ]; then
242			error "actual $stream text does not match expected $stream text."
243			error  "see $file.$stream.* for further investigation."
244			[ $quiet -ne 1 ] && cat "$file".$stream.diff
245			test_failed=1
246		fi
247	done
248
249	if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
250		error "Actual exit value does not match the expected one."
251		error "expected $expected_exit_value, got $actual_exit_value."
252		test_failed=1
253	fi
254
255	# verify the 'check-output-contains/excludes' tags
256	if [ $check_output_contains -eq 1 ]; then
257		has_each_patterns "$file" 'check-output-contains' $file.output.got
258		if [ "$?" -ne "0" ]; then
259			error "Actual output doesn't contain some of the expected patterns."
260			test_failed=1
261		fi
262	fi
263	if [ $check_output_excludes -eq 1 ]; then
264		has_none_patterns "$file" 'check-output-excludes' $file.output.got
265		if [ "$?" -ne "0" ]; then
266			error "Actual output contains some patterns which are not expected."
267			test_failed=1
268		fi
269	fi
270	if [ $check_output_pattern -eq 1 ]; then
271		# verify the 'check-output-pattern-X-times' tags
272		nbr_patterns "$file" 'check-output-pattern' $file.output.got
273		if [ "$?" -ne "0" ]; then
274			error "Actual output doesn't contain the pattern the expected number."
275			test_failed=1
276		fi
277	fi
278
279	[ "$test_failed" -eq "$must_fail" ] || failed=1
280
281	if [ "$must_fail" -eq "1" ]; then
282		if [ "$test_failed" -eq "1" ]; then
283			echo "info: test '$file' is known to fail"
284		else
285			echo "error: test '$file' is known to fail but succeed!"
286			test_failed=1
287		fi
288	fi
289
290	if [ "$test_failed" -eq "1" ]; then
291		ko_tests=$(($ko_tests + 1))
292	else
293		ok_tests=$(($ok_tests + 1))
294		rm -f $file.{error,output}.{expected,got,diff}
295	fi
296	return $test_failed
297}
298
299do_test_suite()
300{
301	for i in $tests_list; do
302		do_test "$i"
303	done
304
305	# prints some numbers
306	tests_nr=$(($ok_tests + $ko_tests))
307	echo -n "Out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
308	echo " ($known_ko_tests of them are known to fail)"
309	if [ "$unhandled_tests" -ne "0" ]; then
310		echo "$unhandled_tests tests could not be handled by $prog_name"
311	fi
312	if [ "$disabled_tests" -ne "0" ]; then
313		echo "$disabled_tests tests were disabled"
314	fi
315}
316
317##
318# do_format(file[, name[, cmd]]) - helps a test writer to format test-suite tags
319do_format()
320{
321	if [ -z "$2" ]; then
322		fname="$1"
323		fcmd=$default_cmd
324	elif [ -z "$3" ]; then
325		fname="$2"
326		fcmd=$default_cmd
327	else
328		fname="$2"
329		fcmd="$3"
330	fi
331	file="$1"
332	cmd=`eval echo $default_path/$fcmd`
333	$cmd 1> $file.output.got 2> $file.error.got
334	fexit_value=$?
335	cat <<_EOF
336/*
337 * check-name: $fname
338_EOF
339	if [ "$fcmd" != "$default_cmd" ]; then
340		echo " * check-command: $fcmd"
341	fi
342	if [ "$fexit_value" -ne "0" ]; then
343		echo " * check-exit-value: $fexit_value"
344	fi
345	for stream in output error; do
346		if [ -s "$file.$stream.got" ]; then
347			echo " *"
348			echo " * check-$stream-start"
349			cat "$file.$stream.got"
350			echo " * check-$stream-end"
351		fi
352	done
353	echo " */"
354	return 0
355}
356
357##
358# arg_file(filename) - checks if filename exists
359arg_file()
360{
361	[ -z "$1" ] && {
362		do_usage
363		exit 1
364	}
365	[ -e "$1" ] || {
366		error "Can't open file $1"
367		exit 1
368	}
369	return 0
370}
371
372case "$1" in
373	'')
374		do_test_suite
375		;;
376	single)
377		arg_file "$2"
378		do_test "$2"
379		case "$?" in
380			0) echo "$2 passed !";;
381			1) echo "$2 failed !";;
382			2) echo "$2 can't be handled by $prog_name";;
383		esac
384		;;
385	format)
386		arg_file "$2"
387		do_format "$2" "$3" "$4"
388		;;
389	help | *)
390		do_usage
391		exit 1
392		;;
393esac
394
395exit $failed
396
397