mirror of
https://gcc.gnu.org/git/gcc.git
synced 2024-11-23 02:44:18 +08:00
586e32583a
When compare_tests compares both C and C++ tests in c-c++-common, they get the same identifier, so expected differences in results across languages become undesirably noisy. This patch adds tool identifiers to tests, so that runs by different tools are not confused by the compare logic. It also fixes a bug in reporting differences, that would attempt to print an undefined fname (the definitions are in subshell loops), and adjusts the target insertion to match tabs in addition to blanks after colons. for contrib/ChangeLog * compare_tests: Add tool to test lines. Match tabs besides blanks to insert tool and target. Don't print undefined fname.
206 lines
5.9 KiB
Bash
Executable File
206 lines
5.9 KiB
Bash
Executable File
#!/bin/sh
|
|
# This script automatically test the given tool with the tool's test cases,
|
|
# reporting anything of interest.
|
|
|
|
# Written by Mike Stump <mrs@cygnus.com>
|
|
# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
|
|
|
|
usage()
|
|
{
|
|
if [ -n "$1" ] ; then
|
|
echo "$0: Error: $1" >&2
|
|
echo >&2
|
|
fi
|
|
cat >&2 <<EOUSAGE
|
|
Usage: $0 [-strict] PREVIOUS CURRENT
|
|
|
|
Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
|
|
|
|
If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
|
|
|
|
Unless -strict is given, these discrepancies are not counted as errors:
|
|
missing/extra .sum files when comparing directories
|
|
tests that failed in PREVIOUS but pass in CURRENT
|
|
tests that were not in PREVIOUS but appear in CURRENT
|
|
tests in PREVIOUS that are missing in CURRENT
|
|
|
|
Exit with the following values:
|
|
0 if there is nothing of interest
|
|
1 if there are errors when comparing single test case files
|
|
N for the number of errors found when comparing directories
|
|
EOUSAGE
|
|
exit 2
|
|
}
|
|
|
|
export LC_ALL=C
|
|
|
|
tool=gxx
|
|
|
|
TMPDIR=${TMPDIR:-/tmp}
|
|
tmp1=$TMPDIR/$tool-testing.$$a
|
|
tmp2=$TMPDIR/$tool-testing.$$b
|
|
now_s=$TMPDIR/$tool-testing.$$d
|
|
before_s=$TMPDIR/$tool-testing.$$e
|
|
lst1=$TMPDIR/$tool-lst1.$$
|
|
lst2=$TMPDIR/$tool-lst2.$$
|
|
lst3=$TMPDIR/$tool-lst3.$$
|
|
lst4=$TMPDIR/$tool-lst4.$$
|
|
lst5=$TMPDIR/$tool-lst5.$$
|
|
sum1=$TMPDIR/$tool-sum1.$$
|
|
sum2=$TMPDIR/$tool-sum2.$$
|
|
tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
|
|
|
|
[ "$1" = "-strict" ] && strict=$1 && shift
|
|
[ "$1" = "-?" ] && usage
|
|
[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
|
|
|
|
trap "rm -f $tmps" 0 1 2 3 5 9 13 15
|
|
exit_status=0
|
|
|
|
if [ -d "$1" -a -d "$2" ] ; then
|
|
find "$1/" -name '*.sum' >$lst1
|
|
find "$2/" -name '*.sum' >$lst2
|
|
echo "# Comparing directories"
|
|
echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
|
|
echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
|
|
echo
|
|
# remove leading directory components to compare
|
|
sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
|
|
sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
|
|
comm -23 $lst3 $lst4 >$lst5
|
|
if [ -s $lst5 ] ; then
|
|
echo "# Extra sum files in Dir1=$1"
|
|
sed -e "s|^|< $1/|" $lst5
|
|
echo
|
|
[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
|
|
fi
|
|
comm -13 $lst3 $lst4 >$lst5
|
|
if [ -s $lst5 ] ; then
|
|
echo "# Extra sum files in Dir2=$2"
|
|
sed -e "s|^|> $2/|" $lst5
|
|
echo
|
|
[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
|
|
fi
|
|
comm -12 $lst3 $lst4 | sort -u >$lst5
|
|
if [ ! -s $lst5 ] ; then
|
|
echo "# No common sum files"
|
|
exit_status=`expr $exit_status + 1`
|
|
exit $exit_status
|
|
fi
|
|
cmnsums=`cat $lst5 | wc -l`
|
|
echo "# Comparing $cmnsums common sum files"
|
|
( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
|
|
( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
|
|
echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
|
|
${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
|
|
ret=$?
|
|
if [ $ret -ne 0 ]; then
|
|
exit_status=`expr $exit_status + 1`
|
|
echo "## Differences found"
|
|
fi
|
|
if [ $exit_status -ne 0 ]; then
|
|
echo "# $exit_status differences in $cmnsums common sum files found"
|
|
else
|
|
echo "# No differences found in $cmnsums common sum files"
|
|
fi
|
|
exit $exit_status
|
|
elif [ -d "$1" -o -d "$2" ] ; then
|
|
usage "Must specify either two directories or two files"
|
|
fi
|
|
|
|
sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^[ ]*=== [^ ]* tests ===$/ {tool = $2} /^Running target / {target = $3} { if (tool != "") { sub(/:[ ]/, "&"tool": " ); }; if (target != "unix") { sub(/:[ ]/, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1
|
|
sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^[ ]*=== [^ ]* tests ===$/ {tool = $2} /^Running target / {target = $3} { if (tool != "") { sub(/:[ ]/, "&"tool": " ); }; if (target != "unix") { sub(/:[ ]/, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2
|
|
|
|
before=$tmp1
|
|
now=$tmp2
|
|
|
|
|
|
if sort -k 2 </dev/null >/dev/null 2>&1; then
|
|
skip1='-k 2'
|
|
else
|
|
skip1='+1'
|
|
fi
|
|
|
|
sort -t ':' $skip1 "$now" > "$now_s"
|
|
sort -t ':' $skip1 "$before" > "$before_s"
|
|
|
|
grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
|
|
grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2
|
|
|
|
grep -s . $tmp2 >/dev/null
|
|
if [ $? = 0 ]; then
|
|
num=`cat $tmp2 | wc -l`
|
|
echo "Tests that now fail, but worked before ($num tests):"
|
|
echo
|
|
cat $tmp2
|
|
echo
|
|
exit_status=1
|
|
fi
|
|
|
|
grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
|
|
grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2
|
|
|
|
grep -s . $tmp2 >/dev/null
|
|
if [ $? = 0 ]; then
|
|
num=`cat $tmp2 | wc -l`
|
|
echo "Tests that now work, but didn't before ($num tests):"
|
|
echo
|
|
cat $tmp2
|
|
[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
|
|
echo
|
|
fi
|
|
|
|
grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
|
|
grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2
|
|
|
|
grep -s . $tmp2 >/dev/null
|
|
if [ $? = 0 ]; then
|
|
num=`cat $tmp2 | wc -l`
|
|
echo "New tests that FAIL ($num tests):"
|
|
echo
|
|
cat $tmp2
|
|
echo
|
|
exit_status=1
|
|
fi
|
|
|
|
grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
|
|
grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2
|
|
|
|
grep -s . $tmp2 >/dev/null
|
|
if [ $? = 0 ]; then
|
|
num=`cat $tmp2 | wc -l`
|
|
echo "New tests that PASS ($num tests):"
|
|
echo
|
|
cat $tmp2
|
|
[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
|
|
echo
|
|
fi
|
|
|
|
grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
|
|
grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2
|
|
|
|
grep -s . $tmp2 >/dev/null
|
|
if [ $? = 0 ]; then
|
|
num=`cat $tmp2 | wc -l`
|
|
echo "Old tests that passed, that have disappeared ($num tests): (Eeek!)"
|
|
echo
|
|
cat $tmp2
|
|
[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
|
|
echo
|
|
fi
|
|
|
|
grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
|
|
grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2
|
|
|
|
grep -s . $tmp2 >/dev/null
|
|
if [ $? = 0 ]; then
|
|
num=`cat $tmp2 | wc -l`
|
|
echo "Old tests that failed, that have disappeared ($num tests): (Eeek!)"
|
|
echo
|
|
cat $tmp2
|
|
[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
|
|
echo
|
|
fi
|
|
|
|
exit $exit_status
|