aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--ChangeLog14
-rw-r--r--INSTALL6
-rw-r--r--Makeconfig7
-rw-r--r--Makefile12
-rw-r--r--NEWS7
-rw-r--r--manual/install.texi6
-rwxr-xr-xscripts/evaluate-test.sh9
7 files changed, 59 insertions, 2 deletions
diff --git a/ChangeLog b/ChangeLog
index 99d2cae..fb61ad2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+2014-03-14 Joseph Myers <joseph@codesourcery.com>
+
+ * scripts/evaluate-test.sh: Handle fourth argument to determine
+ whether test run should stop on failure.
+ * Makeconfig (stop-on-test-failure): New variable.
+ (evaluate-test): Pass fourth argument to evaluate-test.sh based on
+ $(stop-on-test-failure).
+ * Makefile (tests): Give a summary of results from testing and
+ exit with failure status if they include an ERROR or FAIL.
+ (xtests): Likewise.
+ * manual/install.texi (Configuring and compiling): Mention
+ stop-on-test-failure=y.
+ * INSTALL: Regenerated.
+
2014-03-14 Roland McGrath <roland@hack.frob.com>
* scripts/versionlist.awk: New file.
diff --git a/INSTALL b/INSTALL
index f845940..13fb678 100644
--- a/INSTALL
+++ b/INSTALL
@@ -203,6 +203,12 @@ The tests (and later installation) use some pre-existing files of the
system such as `/etc/passwd', `/etc/nsswitch.conf' and others. These
files must all contain correct and sensible content.
+ Normally, `make check' will run all the tests before reporting all
+problems found and exiting with error status if any problems occurred.
+You can specify `stop-on-test-failure=y' when running `make check' to
+make the test run stop and exit with an error status immediately when a
+failure occurs.
+
To format the `GNU C Library Reference Manual' for printing, type
`make dvi'. You need a working TeX installation to do this. The
distribution builds the on-line formatted version of the manual, as
diff --git a/Makeconfig b/Makeconfig
index fa9fb63..3338ab6 100644
--- a/Makeconfig
+++ b/Makeconfig
@@ -601,6 +601,12 @@ run-built-tests = yes
endif
endif
+# Whether to stop immediately when a test fails. Nonempty means to
+# stop, empty means not to stop.
+ifndef stop-on-test-failure
+stop-on-test-failure =
+endif
+
# How to run a program we just linked with our library.
# The program binary is assumed to be $(word 2,$^).
built-program-file = $(dir $(word 2,$^))$(notdir $(word 2,$^))
@@ -1092,6 +1098,7 @@ test-xfail-name = $(strip $(patsubst %.out, %, $(patsubst $(objpfx)%, %, $@)))
# XPASS or XFAIL rather than PASS or FAIL.
evaluate-test = $(..)scripts/evaluate-test.sh $(test-name) $$? \
$(if $(test-xfail-$(test-xfail-name)),true,false) \
+ $(if $(stop-on-test-failure),true,false) \
> $(common-objpfx)$(test-name).test-result
endif # Makeconfig not yet included
diff --git a/Makefile b/Makefile
index 8214dda..e026fb1 100644
--- a/Makefile
+++ b/Makefile
@@ -324,10 +324,22 @@ tests: $(tests-special)
$(..)scripts/merge-test-results.sh -t $(objpfx) subdir-tests.sum \
$(sort $(subdirs) .) \
> $(objpfx)tests.sum
+ @grep '^ERROR:' $(objpfx)tests.sum || true
+ @grep '^FAIL:' $(objpfx)tests.sum || true
+ @echo "Summary of test results:"
+ @sed 's/:.*//' < $(objpfx)tests.sum | sort | uniq -c
+ @if grep -q '^ERROR:' $(objpfx)tests.sum; then exit 1; fi
+ @if grep -q '^FAIL:' $(objpfx)tests.sum; then exit 1; fi
xtests:
$(..)scripts/merge-test-results.sh -t $(objpfx) subdir-xtests.sum \
$(sort $(subdirs)) \
> $(objpfx)xtests.sum
+ @grep '^ERROR:' $(objpfx)xtests.sum || true
+ @grep '^FAIL:' $(objpfx)xtests.sum || true
+ @echo "Summary of test results for extra tests:"
+ @sed 's/:.*//' < $(objpfx)xtests.sum | sort | uniq -c
+ @if grep -q '^ERROR:' $(objpfx)xtests.sum; then exit 1; fi
+ @if grep -q '^FAIL:' $(objpfx)xtests.sum; then exit 1; fi
# The realclean target is just like distclean for the parent, but we want
# the subdirs to know the difference in case they care.
diff --git a/NEWS b/NEWS
index 443b146..c69d40a 100644
--- a/NEWS
+++ b/NEWS
@@ -13,6 +13,13 @@ Version 2.20
16611, 16613, 16623, 16632, 16639, 16642, 16670, 16674, 16677, 16683,
16689, 16695, 16701, 16706, 16707.
+* Running the testsuite no longer terminates as soon as a test fails.
+ Instead, a file tests.sum (xtests.sum from "make xcheck") is generated,
+ with PASS or FAIL lines for individual tests. A summary of the results is
+ printed, including a list of failing lists, and "make check" exits with
+ error status if there were any unexpected failures. "make check
+ stop-on-test-failure=y" may be used to keep the old behavior.
+
* The am33 port, which had not worked for several years, has been removed
from ports.
diff --git a/manual/install.texi b/manual/install.texi
index 8562bdc..d2b8c1e 100644
--- a/manual/install.texi
+++ b/manual/install.texi
@@ -235,6 +235,12 @@ The tests (and later installation) use some pre-existing files of the
system such as @file{/etc/passwd}, @file{/etc/nsswitch.conf} and others.
These files must all contain correct and sensible content.
+Normally, @code{make check} will run all the tests before reporting
+all problems found and exiting with error status if any problems
+occurred. You can specify @samp{stop-on-test-failure=y} when running
+@code{make check} to make the test run stop and exit with an error
+status immediately when a failure occurs.
+
To format the @cite{GNU C Library Reference Manual} for printing, type
@w{@code{make dvi}}. You need a working @TeX{} installation to do
this. The distribution builds the on-line formatted version of the
diff --git a/scripts/evaluate-test.sh b/scripts/evaluate-test.sh
index c8f5012..2a5c156 100755
--- a/scripts/evaluate-test.sh
+++ b/scripts/evaluate-test.sh
@@ -17,12 +17,13 @@
# License along with the GNU C Library; if not, see
# <http://www.gnu.org/licenses/>.
-# usage: evaluate-test.sh test_name rc xfail
+# usage: evaluate-test.sh test_name rc xfail stop_on_failure
test_name=$1
rc=$2
orig_rc=$rc
xfail=$3
+stop_on_failure=$4
if [ $rc -eq 0 ]; then
result="PASS"
@@ -37,4 +38,8 @@ fi
echo "$result: $test_name"
echo "original exit status $orig_rc"
-exit $rc
+if $stop_on_failure; then
+ exit $rc
+else
+ exit 0
+fi