diff options
Diffstat (limited to 'contrib/test-tool')
-rwxr-xr-x | contrib/test-tool | 378 |
1 files changed, 378 insertions, 0 deletions
diff --git a/contrib/test-tool b/contrib/test-tool new file mode 100755 index 0000000..32ab887 --- /dev/null +++ b/contrib/test-tool @@ -0,0 +1,378 @@ +#!/bin/sh +# This script automatically test the given tool with the tool's test cases, +# reporting anything of interest. + +# exits with 1 if there is nothing of interest +# exits with 0 if there is something interesting +# exits with 2 if an error occurred + +# Syntax: test-tool [-expectedpass] [-keepoutput] [-noupdate] g++|gcc|gdb|... +# +# -expectedpass: Turn XFAIL into "pass", XPASS into "fail". +# The default is XFAIL->fail, XPASS->pass. +# -keepoutput: Save "make check" output in test-$tool.log. +# -noupdate: Don't update log files. + +# Limitations, don't run this multiple times in one day, unless the -noupdate +# flag is given. + +# Written by Mike Stump <mrs@cygnus.com> + +expectedpass=no +keepoutput=no +update=yes +tool="" + +# See if cp -p works. +pwd=`pwd` +cd /tmp +rm -f test-tool-$$-1 test-tool-$$-2 +touch test-tool-$$-1 +cp -p test-tool-$$-1 test-tool-$$-2 2>/dev/null +if [ $? = 0 -a -f test-tool-$$-2 ] ; then + CP="cp -p" +else + CP=cp +fi +rm -f test-tool-$$-1 test-tool-$$-2 +cd $pwd + +for arg in $* +do + case $arg in + -expectedpass) expectedpass=yes ;; + -keepoutput) keepoutput=yes ;; + -noupdate) update=no ;; + -*) + echo "Usage: [-expectedpass] [-keepoutput] [-noupdate] tool_name" >&2 + exit 2 + ;; + *) + if [ "$tool" != "" ]; then + echo "Usage: [-expectedpass] [-keepoutput] [-noupdate] tool_name" >&2 + exit 2 + fi + tool=$arg + ;; + esac +done + +# FIXME: It sure would be nice if `testdir' wasn't necessary. :-( + +case $tool in + g++) + devoname=gcc + checktarget=check-g++ + testdir=testsuite + ;; + gcc) + devoname=gcc + checktarget=check-gcc + testdir=testsuite + ;; + ld|gld) + devoname=ld + checktarget=check + testdir=. + ;; + binutils) + devoname=binutils + checktarget=check + testdir=. + ;; + *) + devoname=$tool + checktarget=check + testdir=testsuite + ;; +esac + +# Default DEVOSRCDIR +if [ "$DEVOSRCDIR" = "" ]; then + DEVOSRCDIR=$HOME/devo ; export DEVOSRCDIR +fi + +# Check DEVOSRCDIR +if [ ! -d "$DEVOSRCDIR" ]; then + echo "$0: no directory $DEVOSRCDIR" >&2 + exit 2 +fi + +# Default DEVOBINDIR +if [ "$DEVOBINDIR" = "" ]; then + CPU=`$DEVOSRCDIR/config.guess` + if [ $? != 0 ]; then + echo "$0: cannot run config.guess" >&2 + exit 2 + fi + DEVOBINDIR=$HOME/$CPU ; export DEVOBINDIR +fi + +# Check DEVOBINDIR +if [ ! -d "$DEVOBINDIR" ]; then + echo "$0: no directory $DEVOBINDIR" >&2 + exit 2 +fi + +# Specialize DEVOSRCDIR +if [ -d "$DEVOSRCDIR/$devoname" ]; then + DEVOSRCDIR=$DEVOSRCDIR/$devoname +else + echo "$0: Cannot find source directory." >&2 + exit 2 +fi + +# Default LOGDIR +if [ "$LOGDIR" = "" ]; then + LOGDIR=$HOME/logs ; export LOGDIR +fi + +# Check LOGDIR +if [ ! -d "$LOGDIR" ]; then + echo "$0: no directory $LOGDIR" >&2 + exit 2 +fi + +# Specialize DEVOBINDIR +if [ -d "$DEVOBINDIR/$devoname" ]; then + DEVOBINDIR=$DEVOBINDIR/$devoname +else + echo "$0: Cannot find binary directory." >&2 + exit 2 +fi + +# Binary directory +cd $DEVOBINDIR || exit 2 + +TMPDIR=${TMPDIR-/tmp} + +tmp=$TMPDIR/$tool-testing.$$a +tmp1=$TMPDIR/$tool-testing.$$b +tmp2=$TMPDIR/$tool-testing.$$c +now_s=$TMPDIR/$tool-testing.$$d +before_s=$TMPDIR/$tool-testing.$$e + +if [ "$keepoutput" = yes ]; then + rm -f test-$tool.log + make RUNTESTFLAGS="-v -v" $checktarget >test-$tool.log 2>&1 +else + make RUNTESTFLAGS="-v -v" $checktarget >/dev/null 2>&1 +fi + +# Check for DEJAGNU errors that prevented any output at all. +if [ ! -f $testdir/$tool.sum ]; then + echo "Tests didn't run, probably because of a framework error." + if [ "$keepoutput" = yes ]; then + echo + tail -20 test-$tool.log + else + echo "Unable to determine why. Rerun with -keepoutput." + fi + exit 2 +fi + +# Canonicalize XFAIL and XPASS so the rest of the script can ignore them. +if [ "$expectedpass" = yes ]; then + sed 's/^XFAIL/PASS(XFAIL)/; s/^XPASS/FAIL(XPASS)/' <$testdir/$tool.sum >$testdir/$tool.1.sum || exit 2 +else + sed 's/^XFAIL/FAIL(XFAIL)/; s/^XPASS/PASS(XPASS)/' <$testdir/$tool.sum >$testdir/$tool.1.sum || exit 2 +fi +mv $testdir/$tool.1.sum $testdir/$tool.sum + +patterns="$LOGDIR/$tool-??????.sum $LOGDIR/$tool-??????.sum.gz $LOGDIR/$tool-??????????.sum $LOGDIR/$tool-??????????.sum.gz" +before=`ls -1t $patterns 2>/dev/null | sed 1q` + +todayname=`date '+%y%m%d'` +if [ "$update" = no ]; then + now=$testdir/$tool.sum +else + mv -f $testdir/$tool.sum $LOGDIR/$tool-$todayname.sum || exit 2 + mv -f $testdir/$tool.log $LOGDIR/$tool-$todayname.log || exit 2 + + # Say where the logs are stored so they appear in email messages. + echo + echo "Log files: $LOGDIR/$tool-$todayname.*" + echo + + now="$LOGDIR/$tool-$todayname.sum"; +fi + +trap "rm -f $tmp $tmp1 $tmp2 $now_s $before_s" 0 1 2 3 5 9 13 15 + +case $before in + "") before="cat /dev/null" ;; + *.gz) before="gunzip -c $before" ;; + *) before="cat $before" ;; +esac + +# First, the test summary. +egrep '^# of |===.*Summary.*===' "$now" || echo "(No test summary?)" +echo + +# Pick out the PASS/FAIL/Ufoo messages. +# We grep for them again later but that's for robustness' sake. +grep '^[PFU][A-Z()]*:' "$now" | sort -t ':' +1 > "$now_s" +$before | grep '^[PFU][A-Z()]*:' | sort -t ':' +1 > "$before_s" + +grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]*//' >$tmp1 +grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]*//' | comm -12 $tmp1 - >$tmp2 + +grep -s . $tmp2 >/dev/null +if [ $? = 0 ]; then + echo "Tests that now unexpectedly fail, but worked before:" + echo + cat $tmp2 + showchangelog=1 + echo +fi + +grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]*//' >$tmp1 +grep '^[PFU][A-Z()]*:' "$before_s" | sed 's/^[^:]*:[ ]*//' | comm -23 $tmp1 - >$tmp2 + +grep -s . $tmp2 >/dev/null +if [ $? = 0 ]; then + echo "New tests that unexpectedly FAIL:" + echo + cat $tmp2 + echo +fi + +grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]*//' >$tmp1 +grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]*//' | comm -12 $tmp1 - >$tmp2 + +grep -s . $tmp2 >/dev/null +if [ $? = 0 ]; then + echo "Tests that still don't work:" + echo + cat $tmp2 + echo +fi + +grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]*//' >$tmp1 +grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]*//' | comm -12 $tmp1 - >$tmp2 + +grep -s . $tmp2 >/dev/null +if [ $? = 0 ]; then + echo "Tests that now work, but didn't before:" + echo + cat $tmp2 + echo +fi + +grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]*//' >$tmp1 +grep '^[PFU][A-Z()]*:' "$before_s" | sed 's/^[^:]*:[ ]*//' | comm -23 $tmp1 - >$tmp2 + +grep -s . $tmp2 >/dev/null +if [ $? = 0 ]; then + echo "New tests that PASS:" + echo + cat $tmp2 + echo +fi + +grep '^[PFU][A-Z()]*:' "$now_s" | sed 's/^[^:]*:[ ]*//' >$tmp1 +grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]*//' | comm -13 $tmp1 - >$tmp2 + +grep -s . $tmp2 >/dev/null +if [ $? = 0 ]; then + echo "Old tests that passed, that have disappeared: (Eeek!)" + echo + cat $tmp2 + echo +fi + +grep '^[PFU][A-Z()]*:' "$now_s" | sed 's/^[^:]*:[ ]*//' >$tmp1 +grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]*//' | comm -13 $tmp1 - >$tmp2 + +grep -s . $tmp2 >/dev/null +if [ $? = 0 ]; then + echo "Old tests that failed, that have disappeared: (Eeek!)" + echo + cat $tmp2 + echo +fi + +egrep '^(ERROR|WARNING):' "$now" >$tmp1 + +if grep -s . $tmp1 > /dev/null; then + echo "Errors and warnings:" + echo + cat $tmp1 + echo +fi + +if [ "$tool" = g++ ]; then + if [ -f $DEVOBINDIR/libio/run-make-check ]; then + cd $DEVOBINDIR/libio + make check >$TMPDIR/clgpp$$ 2>&1 + if [ $? != 0 ]; then + echo + echo "libio fails to make check:" + tail -20 $TMPDIR/clgpp$$ + fi + fi + if [ -f $DEVOBINDIR/libstdc++/run-make-check ]; then + cd $DEVOBINDIR/libstdc++ + make check >$TMPDIR/clgpp$$ 2>&1 + if [ $? != 0 ]; then + echo + echo "libstdc++ fails to make check:" + tail -20 $TMPDIR/clgpp$$ + fi + fi + if [ -f $DEVOBINDIR/libg++/run-make-check ]; then + cd $DEVOBINDIR/libg++ + make check >$TMPDIR/clgpp$$ 2>&1 + if [ $? != 0 ]; then + echo + echo "libg++ fails to make check:" + tail -20 $TMPDIR/clgpp$$ + fi + fi + rm -f $TMPDIR/clgpp$$ + cd $DEVOBINDIR +fi + +if [ "$devoname" != "" ]; then + if [ "$showchangelog" = 1 ]; then + echo "Here is what's new in the ChangeLog:" + echo + diff -c $LOGDIR/$devoname.ChangeLog $DEVOSRCDIR/ChangeLog + echo + if [ "$tool" = g++ ]; then + echo + echo "Here is what's new in the ChangeLog.egcs:" + echo + diff -c $LOGDIR/gcc.ChangeLog.egcs $DEVOSRCDIR/ChangeLog.egcs + + echo + echo "Here is what's new in the cp/ChangeLog:" + echo + diff -c $LOGDIR/g++.ChangeLog $DEVOSRCDIR/cp/ChangeLog + fi + echo + fi + if [ "$update" != no ]; then + # save the old ChangeLog as a reference for next time + rm -f $LOGDIR/$devoname.ChangeLog.BAK + mv $LOGDIR/$devoname.ChangeLog $LOGDIR/$devoname.ChangeLog.BAK 2>/dev/null + $CP $DEVOSRCDIR/ChangeLog $LOGDIR/$devoname.ChangeLog + if [ "$tool" = g++ ]; then + rm -f $LOGDIR/gcc.ChangeLog.egcs.BAK + mv $LOGDIR/gcc.ChangeLog.egcs $LOGDIR/gcc.ChangeLog.egcs.BAK 2>/dev/null + $CP $DEVOSRCDIR/ChangeLog.egcs $LOGDIR/gcc.ChangeLog.egcs + + rm -f $LOGDIR/g++.ChangeLog.BAK + mv $LOGDIR/g++.ChangeLog $LOGDIR/g++.ChangeLog.BAK 2>/dev/null + $CP $DEVOSRCDIR/cp/ChangeLog $LOGDIR/g++.ChangeLog + fi + fi +fi + +$before | diff - $now | grep -s . >/dev/null +if [ $? = 0 ]; then + echo "Details:" + echo + $before | diff - $now + echo +fi |