diff options
-rw-r--r-- | ChangeLog | 21 | ||||
-rw-r--r-- | Makefile.am | 24 | ||||
-rw-r--r-- | Makefile.in | 65 | ||||
-rw-r--r-- | NEWS | 3 | ||||
-rw-r--r-- | commands/report-card.awk | 238 | ||||
-rw-r--r-- | doc/dejagnu-report-card.1 | 146 | ||||
-rw-r--r-- | doc/dejagnu.texi | 45 | ||||
-rw-r--r-- | doc/version.texi | 4 | ||||
-rw-r--r-- | testsuite/lib/bohman_ssd.exp | 225 | ||||
-rw-r--r-- | testsuite/lib/report-card.exp | 39 | ||||
-rw-r--r-- | testsuite/report-card.all/onetest.exp | 209 | ||||
-rw-r--r-- | testsuite/report-card.all/passes.exp | 276 |
12 files changed, 1262 insertions, 33 deletions
@@ -1,3 +1,24 @@ +2019-01-02 Jacob Bachmeyer <jcb62281@gmail.com> + + * NEWS: Document report card. + + * Makefile.am (clean-local): Add target. + (clean-local-check): Add target; mark as PHONY. + (commands_DATA): Add "report-card" scripts. + (dist_man_MANS): Add dejagnu-report-card.1 and split. + (DEJATOOL): Add "report-card" tool. + (TESTSUITE_FILES): Add testsuite for "report-card" tool. + + * commands/report-card.awk: New command script. + + * doc/dejagnu.texi (Invoking dejagnu report card): New node. + * doc/dejagnu-report-card.1: New man page. + + * testsuite/lib/bohman_ssd.exp: New file. + * testsuite/lib/report-card.exp: New file. + * testsuite/report-card.all/onetest.exp: New file. + * testsuite/report-card.all/passes.exp: New file. + 2019-01-02 Ben Elliston <bje@gnu.org> * Makefile.am (DISTCLEANFILES): Don't use this. diff --git a/Makefile.am b/Makefile.am index 99a7eec..d78993c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,4 @@ -# Copyright (C) 1992-2016, 2018 Free Software Foundation, Inc. +# Copyright (C) 1992-2016, 2018, 2019 Free Software Foundation, Inc. # # This file is part of DejaGnu. # @@ -28,6 +28,14 @@ EXTRA_DIST = ChangeLog-1992 MAINTAINERS dejagnu runtest \ CLEANFILES = options-init.exp stats-init.exp +clean-local: clean-local-check +.PHONY: clean-local-check +clean-local-check: + -rm -rf testsuite/launcher.all/command/bin + -rm -rf testsuite/launcher.all/command/share + -rm -rf testsuite/report-card.all/onetest + -rm -rf testsuite/report-card.all/passes + # Give a reassuring message so that users know the "build" worked. all-local: @echo "Done. Now run 'make install'." @@ -60,7 +68,8 @@ pkgdata_DATA = \ commandsdir = $(pkgdatadir)/commands commands_DATA = \ - commands/help.sh + commands/help.sh \ + commands/report-card.awk configdir = $(pkgdatadir)/config config_DATA = \ @@ -157,6 +166,8 @@ TESTSUITE_FILES = \ testsuite/launcher.all/help.exp \ testsuite/launcher.all/interp.exp \ testsuite/launcher.all/verbose.exp \ + testsuite/report-card.all/onetest.exp \ + testsuite/report-card.all/passes.exp \ testsuite/runtest.libs/topdir/subdir1/subsubdir1/subsubfile1 \ testsuite/runtest.libs/topdir/subdir1/subfile1 \ testsuite/runtest.libs/topdir/subdir1/subfile2 \ @@ -173,14 +184,16 @@ TESTSUITE_FILES = \ testsuite/runtest.main/options/testsuite/null.test/null.exp \ testsuite/runtest.main/stats.exp \ testsuite/runtest.main/stats/testsuite/stat.test/stats-sub.exp \ + testsuite/lib/bohman_ssd.exp \ testsuite/lib/launcher.exp \ testsuite/lib/libdejagnu.exp \ testsuite/lib/libsup.exp \ + testsuite/lib/report-card.exp \ testsuite/lib/runtest.exp \ testsuite/lib/util-defs.exp \ testsuite/libdejagnu/tunit.exp -DEJATOOL = launcher libdejagnu runtest +DEJATOOL = launcher libdejagnu report-card runtest RUNTEST = ${top_srcdir}/runtest @@ -191,5 +204,8 @@ unit_SOURCES = testsuite/libdejagnu/unit.cc # Documentation. TEXINFO_TEX = doc/texinfo.tex -dist_man_MANS = doc/dejagnu.1 doc/dejagnu-help.1 doc/runtest.1 +dist_man_MANS = doc/dejagnu.1 \ + doc/dejagnu-help.1 \ + doc/dejagnu-report-card.1 \ + doc/runtest.1 info_TEXINFOS = doc/dejagnu.texi diff --git a/Makefile.in b/Makefile.in index 1a6d1ef..1cd211c 100644 --- a/Makefile.in +++ b/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# Copyright (C) 1992-2016, 2018 Free Software Foundation, Inc. +# Copyright (C) 1992-2016, 2018, 2019 Free Software Foundation, Inc. # # This file is part of DejaGnu. # @@ -409,7 +409,8 @@ pkgdata_DATA = \ commandsdir = $(pkgdatadir)/commands commands_DATA = \ - commands/help.sh + commands/help.sh \ + commands/report-card.awk configdir = $(pkgdatadir)/config config_DATA = \ @@ -505,6 +506,8 @@ TESTSUITE_FILES = \ testsuite/launcher.all/help.exp \ testsuite/launcher.all/interp.exp \ testsuite/launcher.all/verbose.exp \ + testsuite/report-card.all/onetest.exp \ + testsuite/report-card.all/passes.exp \ testsuite/runtest.libs/topdir/subdir1/subsubdir1/subsubfile1 \ testsuite/runtest.libs/topdir/subdir1/subfile1 \ testsuite/runtest.libs/topdir/subdir1/subfile2 \ @@ -521,21 +524,27 @@ TESTSUITE_FILES = \ testsuite/runtest.main/options/testsuite/null.test/null.exp \ testsuite/runtest.main/stats.exp \ testsuite/runtest.main/stats/testsuite/stat.test/stats-sub.exp \ + testsuite/lib/bohman_ssd.exp \ testsuite/lib/launcher.exp \ testsuite/lib/libdejagnu.exp \ testsuite/lib/libsup.exp \ + testsuite/lib/report-card.exp \ testsuite/lib/runtest.exp \ testsuite/lib/util-defs.exp \ testsuite/libdejagnu/tunit.exp -DEJATOOL = launcher libdejagnu runtest +DEJATOOL = launcher libdejagnu report-card runtest RUNTEST = ${top_srcdir}/runtest AM_CXXFLAGS = -I$(top_srcdir) -g unit_SOURCES = testsuite/libdejagnu/unit.cc # Documentation. TEXINFO_TEX = doc/texinfo.tex -dist_man_MANS = doc/dejagnu.1 doc/dejagnu-help.1 doc/runtest.1 +dist_man_MANS = doc/dejagnu.1 \ + doc/dejagnu-help.1 \ + doc/dejagnu-report-card.1 \ + doc/runtest.1 + info_TEXINFOS = doc/dejagnu.texi all: all-am @@ -1304,7 +1313,7 @@ maintainer-clean-generic: @echo "it deletes files that may require special tools to rebuild." clean: clean-am -clean-am: clean-aminfo clean-checkPROGRAMS clean-generic \ +clean-am: clean-aminfo clean-checkPROGRAMS clean-generic clean-local \ mostlyclean-am distclean: distclean-am @@ -1480,25 +1489,25 @@ uninstall-man: uninstall-man1 .PHONY: CTAGS GTAGS TAGS all all-am all-local am--refresh check \ check-DEJAGNU check-am clean clean-aminfo clean-checkPROGRAMS \ - clean-cscope clean-generic cscope cscopelist-am ctags ctags-am \ - dist dist-all dist-bzip2 dist-gzip dist-info dist-lzip \ - dist-shar dist-tarZ dist-xz dist-zip distcheck distclean \ - distclean-DEJAGNU distclean-compile distclean-generic \ - distclean-tags distcleancheck distdir distuninstallcheck dvi \ - dvi-am html html-am info info-am install install-am \ - install-baseboardDATA install-binSCRIPTS install-commandsDATA \ - install-configDATA install-data install-data-am \ - install-djlibexecSCRIPTS install-dvi install-dvi-am \ - install-exec install-exec-am install-html install-html-am \ - install-includeHEADERS install-info install-info-am \ - install-man install-man1 install-pdf install-pdf-am \ - install-pkgdataDATA install-ps install-ps-am install-strip \ - installcheck installcheck-am installdirs maintainer-clean \ - maintainer-clean-aminfo maintainer-clean-generic \ - maintainer-clean-vti mostlyclean mostlyclean-aminfo \ - mostlyclean-compile mostlyclean-generic mostlyclean-vti pdf \ - pdf-am ps ps-am tags tags-am uninstall uninstall-am \ - uninstall-baseboardDATA uninstall-binSCRIPTS \ + clean-cscope clean-generic clean-local cscope cscopelist-am \ + ctags ctags-am dist dist-all dist-bzip2 dist-gzip dist-info \ + dist-lzip dist-shar dist-tarZ dist-xz dist-zip distcheck \ + distclean distclean-DEJAGNU distclean-compile \ + distclean-generic distclean-tags distcleancheck distdir \ + distuninstallcheck dvi dvi-am html html-am info info-am \ + install install-am install-baseboardDATA install-binSCRIPTS \ + install-commandsDATA install-configDATA install-data \ + install-data-am install-djlibexecSCRIPTS install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-includeHEADERS install-info \ + install-info-am install-man install-man1 install-pdf \ + install-pdf-am install-pkgdataDATA install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-aminfo \ + maintainer-clean-generic maintainer-clean-vti mostlyclean \ + mostlyclean-aminfo mostlyclean-compile mostlyclean-generic \ + mostlyclean-vti pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-baseboardDATA uninstall-binSCRIPTS \ uninstall-commandsDATA uninstall-configDATA \ uninstall-djlibexecSCRIPTS uninstall-dvi-am uninstall-html-am \ uninstall-includeHEADERS uninstall-info-am uninstall-man \ @@ -1509,6 +1518,14 @@ uninstall-man: uninstall-man1 export DEJAGNU +clean-local: clean-local-check +.PHONY: clean-local-check +clean-local-check: + -rm -rf testsuite/launcher.all/command/bin + -rm -rf testsuite/launcher.all/command/share + -rm -rf testsuite/report-card.all/onetest + -rm -rf testsuite/report-card.all/passes + # Give a reassuring message so that users know the "build" worked. all-local: @echo "Done. Now run 'make install'." @@ -19,6 +19,9 @@ Changes since 1.6.2: auxiliary commands not directly involved with running tests. The "runtest" command will remain for that purpose for the forseeable future. +8. The first auxiliary command is added: "report card". The "dejagnu + report card" command reads DejaGnu summary files and produces a + compact tabular summary across multiple tools. Changes since 1.6.1: diff --git a/commands/report-card.awk b/commands/report-card.awk new file mode 100644 index 0000000..b04c0e9 --- /dev/null +++ b/commands/report-card.awk @@ -0,0 +1,238 @@ +# report-card.awk -- Test summary tool +# Copyright (C) 2018 Free Software Foundation, Inc. +# +# This file is part of DejaGnu. +# +# DejaGnu is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# DejaGnu is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with DejaGnu; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA. + +# This file was written by Jacob Bachmeyer. + +# ##help +# #Usage: dejagnu report card [<option>|<tool>|<file>]... +# #Usage: dejagnu report-card [<option>|<tool>|<file>]... +# # --verbose, -v Emit additional messages +# ##end + +# Arrays storing lists in this program store items in numbered keys, with a +# count in the "C" key, similar to Awk's ARGV/ARGC. + +# The Tools array stores a list of tools in 1..N. + +# The Passes array stores a global list of passes seen, a per-tool list of +# passes seen, and a global index of passes seen if DejaGnu's multipass +# support is used. +# Key prefixes: +# "" -- global list: 1..N; "C" +# "t", <tool> -- per-tool list: 1..N; "C" +# Key patterns: +# "p", <pass> -- count of tools using <pass> + +# The Totals array stores counts of test results, indexed by tool and pass. +# A summarization step adds per-tool, per-pass, and grand totals. +# Key patterns: +# "tp", <Tool>, <Pass>, <result> +# "t", <Tool>, <result> +# "p", <Pass>, <result> +# <result> + +## +## Get list of files to scan + +BEGIN { + Tools["C"] = 1 + Passes["", "C"] = 1 + ToolWidth = 0 + PassWidth = 0 + Verbose = 0 + # remove arguments from ARGV + for (i = 1; i < ARGC; i++) { + if (ARGV[i] ~ /^-/) { + if (ARGV[i] ~ /^--?v(erb.*)?$/) + Verbose++ + else if (ARGV[i] == "--") + break + delete ARGV[i] + } + } + if (ARGV[i] == "--") + delete ARGV[i] + if (Verbose) print "Verbose level is "Verbose + # adjust filenames in ARGV + FileCount = 0 + for (i = 1; i < ARGC; i++) { + if (i in ARGV) FileCount++ + else continue + if (ARGV[i] ~ /\.sum$/) continue + else if (ARGV[i] ~ /\.log$/) sub(/\.log$/, ".sum", ARGV[i]) + else if (ARGV[i] ~/\.$/) sub(/\.$/, ".sum", ARGV[i]) + else ARGV[i] = (ARGV[i]".sum") + } + if (FileCount == 0) { + cmd_ls_files = "ls -1 *.sum" + while (cmd_ls_files | getline File) { + FileCount++ + ARGV[ARGC++] = File + } + close(cmd_ls_files) + } + if (Verbose > 2) { + print "Reading "FileCount" file(s)" + for (i = 1; i < ARGC; i++) + if (i in ARGV) + print " "ARGV[i] + } +} + +## +## Read files and collect data + +FNR == 1 { + if (Verbose) + print "Reading `"FILENAME"' ..." + Pass = "" + Tool = File = FILENAME + sub(/\.sum$/, "", Tool) + if (length(Tool) > ToolWidth) + ToolWidth = length(Tool) + Tools[Tools["C"]++] = Tool + Passes["t", Tool, "C"] = 1 + Passes["t", Tool, 1] = "" # will be overwritten if multipass is used +} + +/^Running pass `[^']*' .../ { + Pass = $3 + sub(/^`/, "", Pass) + sub(/'$/, "", Pass) + if (("p", Pass) in Passes) + Passes["p", Pass]++ + else { + if (length(Pass) > PassWidth) + PassWidth = length(Pass) + Passes["", Passes["", "C"]++] = Pass + Passes["p", Pass] = 1 + } + Passes["t", Tool, Passes["t", Tool, "C"]++] = Pass +} + +$1 ~ /:$/ { sub(/:$/, "", $1); Totals["tp", Tool, Pass, $1]++ } + +## +## Compute totals + +END { + $0 = ("PASS FAIL KPASS KFAIL XPASS XFAIL UNSUPPORTED UNRESOLVED UNTESTED") + for (i = 1; i in Tools; i++) + for (j = 1; ("t", Tools[i], j) in Passes; j++) + for (k = 1; k <= NF; k++) { + Totals[$k] \ + += Totals["tp", Tools[i], Passes["t", Tools[i], j], $k] + Totals["t", Tools[i], $k] \ + += Totals["tp", Tools[i], Passes["t", Tools[i], j], $k] + Totals["p", Passes["t", Tools[i], j], $k] \ + += Totals["tp", Tools[i], Passes["t", Tools[i], j], $k] + } +} + +## +## Compute total name column width + +END { + if (Passes["", "C"] > 1) + NameWidth = ToolWidth + 3 + PassWidth + else + NameWidth = ToolWidth +} + +## +## Emit header + +END { + printf "%*s __________________________________________________\n", \ + NameWidth, "" + printf "%*s / %6s %6s %6s %6s %6s %6s %6s\n", NameWidth, "", \ + "PASS", "FAIL", "?PASS", "?FAIL", "UNSUP", "UNRES", "UNTEST" + printf "%*s |--------------------------------------------------\n", \ + NameWidth, "" +} + +## +## Emit counts + +END { + for (i = 1; i in Tools; i++) { + Tool = Tools[i] + for (j = 1; ("t", Tool, j) in Passes; j++) { + Pass = Passes["t", Tool, j] + if (Passes["t", Tool, "C"] > 1) + printf "%*s / %-*s | ", ToolWidth, Tool, PassWidth, Pass + else if (Passes["", "C"] > 1) + printf "%*s %*s | ", ToolWidth, Tool, PassWidth, "" + else + printf "%*s | ", NameWidth, Tool + # Passes["t", <tool>, 1] is a pass name or a null string if + # <tool> did not use multipass. + printf " %6d %6d %6d %6d %6d %6d %6d%s%s\n", \ + Totals["tp", Tool, Pass, "PASS"], \ + Totals["tp", Tool, Pass, "FAIL"], \ + Totals["tp", Tool, Pass, "KPASS"] \ + + Totals["tp", Tool, Pass, "XPASS"], \ + Totals["tp", Tool, Pass, "KFAIL"] \ + + Totals["tp", Tool, Pass, "XFAIL"], \ + Totals["tp", Tool, Pass, "UNSUPPORTED"], \ + Totals["tp", Tool, Pass, "UNRESOLVED"], \ + Totals["tp", Tool, Pass, "UNTESTED"], \ + (Totals["tp", Tool, Pass, "ERROR" ] > 0 ? " !E!" : ""), \ + (Totals["tp", Tool, Pass, "WARNING"] > 0 ? " !W!" : "") + } + } +} + +## +## Emit pass totals + +END { + if (Passes["", "C"] > 1) { + printf "%*s |--------------------------------------------------\n", \ + NameWidth, "" + for (i = 1; ("", i) in Passes; i++) + printf "%*s %-*s | %6d %6d %6d %6d %6d %6d %6d\n", \ + ToolWidth, "", PassWidth, Passes["", i], \ + Totals["p", Passes["", i], "PASS"], \ + Totals["p", Passes["", i], "FAIL"], \ + Totals["p", Passes["", i], "KPASS"] \ + + Totals["p", Passes["", i], "XPASS"], \ + Totals["p", Passes["", i], "KFAIL"] \ + + Totals["p", Passes["", i], "XFAIL"], \ + Totals["p", Passes["", i], "UNSUPPORTED"], \ + Totals["p", Passes["", i], "UNRESOLVED"], \ + Totals["p", Passes["", i], "UNTESTED"] + } +} + +## +## Emit grand totals + +END { + printf "%*s |--------------------------------------------------\n", \ + NameWidth, "" + printf "%*s | %6d %6d %6d %6d %6d %6d %6d\n", NameWidth, "", \ + Totals["PASS"], Totals["FAIL"], \ + Totals["KPASS"] + Totals["XPASS"], Totals["KFAIL"] + Totals["XFAIL"], \ + Totals["UNSUPPORTED"], Totals["UNRESOLVED"], Totals["UNTESTED"] + printf "%*s \\__________________________________________________\n", \ + NameWidth, "" +} + +#EOF diff --git a/doc/dejagnu-report-card.1 b/doc/dejagnu-report-card.1 new file mode 100644 index 0000000..2e69fd0 --- /dev/null +++ b/doc/dejagnu-report-card.1 @@ -0,0 +1,146 @@ +.\" Copyright (C) 2018 Free Software Foundation, Inc. +.\" You may distribute this file under the terms of the GNU Free +.\" Documentation License. +.Dd December 31, 2018 +.Os GNU +.Dt DEJAGNU-REPORT-CARD 1 URM +.Sh NAME +.Nm dejagnu\ report\ card +.Nd summarize results from testing multiple tools +.Sh SYNOPSIS +.Nm dejagnu\ report\ card +.Oo Ao Ar option Ac \*(Ba Ao Ar tool Ac \*(Ba Ao Ar file Ac Oc ... +.Sh DESCRIPTION +The +.Nm +command displays results from testing multiple tools in a tabular format. +The produced table lists, for each tool (and if multiple passes were run, +each pass) the number of tests passed, failed, unsupported, unresolved, and +untested. Tests that are expected to fail are counted in separate columns +from tests expected to pass, but "known" failures and "expected" failures +are summarized together. If a test generated warnings or errors, a tag +.Ql !W! +or +.Ql !E! +is appended at the end of the relevant line. +.Pp +Aside from options, the argument list may include tool or file names. The +.Nm +command prefers to read DejaGnu summary files and will translate names accordingly: +.Bl -tag -width ".Pa *.sum" +.It Pa *.sum +Used as-is. +.It Pa *.log +Rewritten to +.Pa *.sum +with the same stem. +.It Pa *. +The string +.Pa sum +is appended to select a summary file. This processing is done for +convenience when using Readline file name completion in a shell, which will +complete to the dot. +.It Pa * +Taken as a tool name; +.Pa .sum +is appended. +.El +.Sh OPTIONS +.Bl -tag -width ".Fl v , -verbose" +.It Fl v , -verbose +Emit additional output describing the operation of +.Nm +itself. +.El +.Sh FILES +The +.Nm +command produces its output by reading the summary files produced by +DejaGnu and counting "PASS", "FAIL", etc. +.Pp +If no names are given as arguments, all files matching +.Pa *.sum +in the current directory are read. +.Sh EXAMPLES +.Ss A simple example from DejaGnu's own testsuite +.Bd -literal +$ dejagnu report card +\ __________________________________________________ +\ / PASS FAIL ?PASS ?FAIL UNSUP UNRES UNTEST +\ |-------------------------------------------------- +\ launcher | 52 0 0 0 0 0 0 +libdejagnu | 5 0 0 0 0 0 0 +\ runtest | 135 0 0 0 0 0 0 +\ |-------------------------------------------------- +\ | 192 0 0 0 0 0 0 +\ \\__________________________________________________ +.Ed +.Pp +Three tools were tested, with a total of 192 tests, all expected to pass. +In this example, all tests did pass, so all other columns are zero. The +.Ql ?PASS +and +.Ql ?FAIL +columns count tests known or expected to fail that either unexpectedly +passed or failed as expected. The remaining three columns count the +exceptional results for unsupported tests, unresolved tests and stub tests +that simply declare themselves untested. +.Pp +.ne 16v +.Ss The same example after tests were added for dejagnu-report-card +.Bd -literal +$ dejagnu report-card +\ __________________________________________________ +\ / PASS FAIL ?PASS ?FAIL UNSUP UNRES UNTEST +\ |-------------------------------------------------- +\ launcher | 52 0 0 0 0 0 0 +\ libdejagnu | 5 0 0 0 0 0 0 +report-card / awk | 36 0 0 0 0 0 0 +report-card / sh | 36 0 0 0 0 0 0 +report-card / tcl | 36 0 0 0 0 0 0 +\ runtest | 135 0 0 0 0 0 0 +\ |-------------------------------------------------- +\ awk | 36 0 0 0 0 0 0 +\ sh | 36 0 0 0 0 0 0 +\ tcl | 36 0 0 0 0 0 0 +\ |-------------------------------------------------- +\ | 300 0 0 0 0 0 0 +\ \\__________________________________________________ +.Ed +.Pp +The +.Ql report-card +tool has been added, with three passes, one for each implementation. (The +shell and Tcl implementations were later dropped to reduce future +maintenance burden.) As before, all tests passed as expected. The +interesting difference from the previous example is the use of DejaGnu's +multipass testing feature and the additional per-pass summary lines added. +For this example, only the +.Ql report-card +tool uses multipass testing, so each pass total is simply the count of +tests for +.Ql report-card +instead of a distinct total. +.Pp +Also note that the command used to invoke +.Nm +is slightly different here. The +.Xr dejagnu 1 +launcher will also accept multiple words joined with dashes into a single +argument. This allows individual words in a command name to be separated +with either dashes or spaces on the command line interchangeably. +.Sh SEE ALSO +.Xr dejagnu 1 +.Xr runtest 1 +.Pp +The full documentation for DejaGnu is maintained as a Texinfo manual. If the +.Nm info +program is properly installed at your site, the command +.Li info dejagnu +should give you access to the complete manual. +.Sh AUTHORS +.An Jacob Bachmeyer +.\".Sh BUGS +.\" LocalWords: Dt dejagnu URM Nm Ao Oo Oc DejaGnu Xr runtest DejaGnu's Bd Ql +.\" LocalWords: testsuite UNSUP UNRES UNTEST libdejagnu Readline Ss tcl awk +.\" LocalWords: ne multipass diff --git a/doc/dejagnu.texi b/doc/dejagnu.texi index e07a40a..11d433e 100644 --- a/doc/dejagnu.texi +++ b/doc/dejagnu.texi @@ -80,6 +80,7 @@ Running other DejaGnu commands * Invoking dejagnu:: Command line options for the launcher itself. * Invoking dejagnu help:: Reading man pages for dejagnu subcommands. +* Invoking dejagnu report card:: Summarizing test results from many tools. Customizing DejaGnu @@ -1026,7 +1027,8 @@ then runs the requested command. @menu * Invoking dejagnu:: Command line options for the launcher itself. -* Invoking dejagnu help:: Reading man pages for dejagnu subcommands. +* Invoking dejagnu help:: Reading man pages for dejagnu subcommands. +* Invoking dejagnu report card:: Summarizing test results from many tools. @end menu @node Invoking dejagnu, Invoking dejagnu help, Running other DejaGnu commands, Running other DejaGnu commands @@ -1089,7 +1091,7 @@ invoked command. All arguments after the command name are passed to the invoked command. -@node Invoking dejagnu help, , Invoking dejagnu, Running other DejaGnu commands +@node Invoking dejagnu help, Invoking dejagnu report card, Invoking dejagnu, Running other DejaGnu commands @section Invoking @command{dejagnu help} @cindex dejagnu help, invoking @@ -1116,6 +1118,42 @@ inner workings of the @command{dejagnu help} command to be produced. The @option{--path}, @option{-w}, and @option{-W} options are passed to @command{man}. +@node Invoking dejagnu report card, , Invoking dejagnu help, Running other DejaGnu commands +@section Invoking @command{dejagnu report card} +@cindex dejagnu report card, invoking +@cindex dejagnu report-card, invoking + +The @command{dejagnu report card} tool produces a tabular summary of +the results from test runs by reading the summary files that DejaGnu +produces. + +@example +@command{dejagnu report card} [<option>|<tool>|<file>]... +@end example + +The @option{--verbose} option causes additional output describing the +inner workings of the @command{dejagnu report card} command to be produced. + +Aside from options, the command may include a list of tools or files. +Names ending in @samp{.sum} are used as-is. Names ending in +@samp{.log} are changed to instead refer to the summary file. Names +ending with a simple dot (@samp{.}) have @samp{sum} appended, for +convenience when using Readline filename completion in a shell, which +will complete to the dot, since there are both @samp{.sum} and +@samp{.log} files produced for each tool tested. Lastly, all other +names are taken as tool names and @samp{.sum} is appended to refer to +the summary file produced by DejaGnu. + +The relevant summary files are read and an ASCII-art table is +produced. The table has columns for counts of tests passed, failed, +unsupported, unresolved, and untested. Tests that are expected to +pass and tests that are expected to fail are counted in separate +columns, but known failures (@samp{KFAIL} and @samp{KPASS}) are +summarized together with expected failures (@samp{XFAIL} and +@samp{XPASS}) in two additional columns: @samp{?PASS} and +@samp{?FAIL}. Additionally, if a test produced any warnings or +errors, tags @samp{!W!} or @samp{!E!} are added at the end of the row. + @node Customizing DejaGnu, Extending DejaGnu, Running other DejaGnu commands, Top @chapter Customizing DejaGnu @cindex customization @@ -5610,4 +5648,5 @@ This makes @code{runtest} exit. Abbreviation: @kbd{q}. @bye @c LocalWords: subdirectory prepend prepended testsuite filename Expect's svn -@c LocalWords: DejaGnu CVS RCS SCCS prepending subcommands +@c LocalWords: DejaGnu CVS RCS SCCS prepending subcommands Tcl Awk Readline +@c LocalWords: POSIX KFAIL KPASS XFAIL XPASS diff --git a/doc/version.texi b/doc/version.texi index e3c9a08..f2d1f35 100644 --- a/doc/version.texi +++ b/doc/version.texi @@ -1,4 +1,4 @@ -@set UPDATED 20 December 2018 -@set UPDATED-MONTH December 2018 +@set UPDATED 2 January 2019 +@set UPDATED-MONTH January 2019 @set EDITION 1.6.3-git @set VERSION 1.6.3-git diff --git a/testsuite/lib/bohman_ssd.exp b/testsuite/lib/bohman_ssd.exp new file mode 100644 index 0000000..25b1072 --- /dev/null +++ b/testsuite/lib/bohman_ssd.exp @@ -0,0 +1,225 @@ +# Copyright (C) 2018 Free Software Foundation, Inc. +# +# This file is part of DejaGnu. +# +# DejaGnu is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# DejaGnu is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with DejaGnu; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA. + +# This file was written by Jacob Bachmeyer. + +# This library provides functions for generating subset-sum-distinct sets +# using a construction published by Tom Bohman in: +# T. Bohman, A construction for sets of integers with distinct subset sums, +# The Electronic. Journal of Combinatorics 5 (1998) /#R3 +# <URL:http://www.combinatorics.org/Volume_5/PDF/v5i1r3.pdf>, +# retrieved 2018-12-28 SHA-1 1c35035427b3406a44f7290f13ec8fbc3d105041 +namespace eval ::math_utils::Bohman_SSD { + + # b_n(i) + proc b { n i } { + if { $n <= 1 } { error "invalid parameter n: $n" } + if { $i <= 2*$n } { error "invalid parameter i: $i" } + + if { $i >= 2*$n + 4 } { + return [expr { round(sqrt(2*($i + 2 - 2*$n))) }] + } elseif { $i == 2*$n + 3 } { + return [expr { $n + 2 }] + } else { # $i == 2*$n + 1 || $i == 2*$n + 2 + return [expr { $n + 1 }] + } + } + + variable d_memo + array unset d_memo + array set d_memo {} + + # d_n(i) + proc d { n i } { + variable d_memo + if { [info exists d_memo($n,$i)] } { return $d_memo($n,$i) } + + if { $n <= 1 } { error "invalid parameter n: $n" } + if { $i < 1 } { error "invalid parameter i: $i" } + + if { $i == $n } { + return 1 + } elseif { $i < $n } { + set j [expr { $n - $i }] + return [expr { 2 * round(pow(4,($j - 1))) }] + } elseif { $i <= 2*$n } { + set j [expr { $i - $n }] + return [expr { round(pow(4,($j - 1))) }] + } else { # $i > 2*$n + set sum 0 + for { set j [expr { $i - [b $n $i] }] } { $j < $i } { incr j } { + incr sum [d $n $j] + } + set d_memo($n,$i) $sum + return $sum + } + } + + # S_{n,m} returns list + proc S { n m } { + if { $n <= 1 } { error "invalid parameter n: $n" } + if { $m < 2*$n } { error "invalid parameter m: $m" } + + set dv [list] + for { set i 1 } { $i <= $m } { incr i } { lappend dv [d $n $i] } + set sum 0 + foreach d $dv { incr sum $d } + set result [list] + foreach d $dv { + lappend result $sum + incr sum -$d + } + return $result + } + + # b'_n(i) + proc bp { n i } { + if { $n < 1 } { error "invalid parameter n: $n" } + if { $i <= 2*$n + 1 } { error "invalid parameter i: $i" } + + if { $i >= 2*$n + 5 } { + return [expr { round(sqrt(2*($i + 1 - 2*$n))) }] + } elseif { $i == 2*$n + 2 } { + return [expr { $n + 1 }] + } else { # $i == 2*$n + 3 || $i == 2*$n + 4 + return [expr { $n + 2 }] + } + } + + variable dp_memo + array unset dp_memo + array set dp_memo {} + + # d'_n(i) + proc dp { n i } { + variable dp_memo + if { [info exists dp_memo($n,$i)] } { return $dp_memo($n,$i) } + + if { $n < 1 } { error "invalid parameter n: $n" } + if { $i < 1 } { error "invalid parameter i: $i" } + + if { $i == $n + 1 } { + return 1 + } elseif { $i < $n + 1 } { + set j [expr { $n + 1 - $i }] + return [expr { round(pow(4,($j - 1))) }] + } elseif { $i <= 2*$n + 1 } { + set j [expr { $i - $n - 1 }] + return [expr { 2 * round(pow(4,($j - 1))) }] + } else { # $i > 2*$n + 1 + set sum 0 + for { set j [expr { $i - [bp $n $i] }] } { $j < $i } { incr j } { + incr sum [dp $n $j] + } + set dp_memo($n,$i) $sum + return $sum + } + } + # The example for d'_3 in the paper is wrong starting at i=11. The + # paper says that it is 200, but it is actually 300. + + # S'_{n,m} returns list + proc Sp { n m } { + if { $n < 1 } { error "invalid parameter n: $n" } + if { $m < 2*$n + 1 } { error "invalid parameter m: $m" } + + set dv [list] + for { set i 1 } { $i <= $m } { incr i } { lappend dv [dp $n $i] } + set sum 0 + foreach d $dv { incr sum $d } + set result [list] + foreach d $dv { + lappend result $sum + incr sum -$d + } + return $result + } + + # Given a list of numbers, verify that all sums of all subsets are in + # fact unique. + # + # This is a brute force search and not based on Bohman's paper. This + # quickly becomes impractical for large lists, requiring inordinate + # amounts of both time and space. + proc check { base } { + set bound [expr { int(pow(2,[llength $base])) }] + for { set i 0 } { $i < $bound } { incr i } { + set R $i + set sum 0 + foreach v $base { + if { $R & 1 } { incr sum $v } + set R [expr { $R >> 1 }] + } + if { [info exists output($sum)] } { + # emit counterexample + set cexl [list] + set R $i + foreach v $base { + if { $R & 1 } { lappend cexl $v } + set R [expr { $R >> 1 }] + } + set cex [join $cexl "+"] + append cex "=" $sum "=" + set cexl [list] + set R $output($sum) + foreach v $base { + if { $R & 1 } { lappend cexl $v } + set R [expr { $R >> 1 }] + } + append cex [join $cexl "+"] + error "list is not subset-sum-distinct: $cex" + } + set output($sum) $i + } + return 1 + } + + # Given a list of numbers and a sum of a subset of that list, find a + # subset that produces the given sum. If the list of numbers is + # subset-sum-distinct, this will return the unique solution. + # Otherwise, an unspecified solution is returned. If the sum is not + # actually a sum of a subset of the list, an empty list is returned. + # + # This is a brute force search and not based on Bohman's paper. This + # requires constant space, but quickly becomes impractical for large + # lists, requiring inordinate time to complete. + proc summands { base goal } { + set bound [expr { int(pow(2,[llength $base])) }] + for { set i 0 } { $i < $bound } { incr i } { + set R $i + set sum 0 + foreach v $base { + if { $R & 1 } { incr sum $v } + set R [expr { $R >> 1 }] + } + if { $sum == $goal } { + set resl [list] + set R $i + foreach v $base { + if { $R & 1 } { lappend resl $v } + set R [expr { $R >> 1 }] + } + return $resl + } + } + return [list] + } + +} + +#EOF diff --git a/testsuite/lib/report-card.exp b/testsuite/lib/report-card.exp new file mode 100644 index 0000000..7fa8838 --- /dev/null +++ b/testsuite/lib/report-card.exp @@ -0,0 +1,39 @@ +# Copyright (C) 2018 Free Software Foundation, Inc. +# +# This file is part of DejaGnu. +# +# DejaGnu is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# DejaGnu is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with DejaGnu; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA. + +# This file was written by Jacob Bachmeyer. + +# Ensure that the dejagnu(1) launcher is available for testing. +if { ![info exists LAUNCHER] } { + set LAUNCHER \ + [file join [file dirname [testsuite file -source -top]] dejagnu] +} +verbose "Using LAUNCHER $LAUNCHER" 2 + +if { [which $LAUNCHER] == 0 } { + perror "Can't find LAUNCHER = $LAUNCHER" + exit 2 +} + +# stub: dejagnu-report-card is non-interactive +proc report-card_exit {} {} + +# stub: dejagnu-report-card does not have a separate version number +proc report-card_version {} {} + +#EOF diff --git a/testsuite/report-card.all/onetest.exp b/testsuite/report-card.all/onetest.exp new file mode 100644 index 0000000..b2ae814 --- /dev/null +++ b/testsuite/report-card.all/onetest.exp @@ -0,0 +1,209 @@ +# Copyright (C) 2018 Free Software Foundation, Inc. +# +# This file is part of DejaGnu. +# +# DejaGnu is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# DejaGnu is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with DejaGnu; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA. + +# This file was written by Jacob Bachmeyer. + +set header_column_names { PASS FAIL ?PASS ?FAIL UNSUP UNRES UNTEST } +set separator_count 0 +set re_digit_columns {} +for { set i 0 } { $i < 7 } { incr i } { + append re_digit_columns {[[:space:]]+([[:digit:]]+)} +} + +set test_names { pass fail kpass kfail xpass xfail + unsupported unresolved untested + note warning error } +set test_results { PASS FAIL KPASS KFAIL XPASS XFAIL + UNSUPPORTED UNRESOLVED UNTESTED + NOTE WARNING ERROR } + +foreach name $test_names result $test_results { + set fd [open [testsuite file -object -test onetest one-${name}.sum] w] + puts $fd "${result}: one test" + close $fd +} + +set stty_init { -onlcr -onlret } + +spawn /bin/sh -c \ + "cd [testsuite file -object -test onetest]\ + && exec $LAUNCHER report-card" + +# check header +expect { + -re {^[[:space:]]+_+[\r\n]+} { + # discard initial header line + exp_continue + } + -re {^[[:space:]]+/([^\r\n]*)[\r\n]+} { + # check column labels + foreach want $header_column_names have $expect_out(1,string) { + if { $have eq $want } { + pass "header item $want" + } else { + fail "header item $want" + } + } + exp_continue + } + -re {^[[:space:]]+\|-+[\r\n]+} { + incr separator_count + } +} + +# check results +array unset scoreboard +array set scoreboard { + pass 0 fail 0 kpass 0 kfail 0 xpass 0 xfail 0 + unsupported 0 unresolved 0 untested 0 + note 0 warning 0 error 0 +} +array unset column_subexp_map +array set column_subexp_map { + pass 2 fail 3 kpass 4 kfail 5 xpass 4 xfail 5 + unsupported 6 unresolved 7 untested 8 + note 0 warning 9 error 9 +} +set re_table_row {^[[:space:]]*one-([[:alpha:]]+)[[:space:]]+\|} +append re_table_row $re_digit_columns +append re_table_row {((?:[[:space:]]+![EW]!)*)[\r\n]+} +expect { + -re $re_table_row { + for { set i 2 } { $i < 9 } { incr i } { + if { $expect_out($i,string)\ + == ( $i == $column_subexp_map($expect_out(1,string))\ + ? 1 : 0 ) } { + incr scoreboard($expect_out(1,string)) + } else { + incr scoreboard($expect_out(1,string)) -1 + } + } + set have_warning_tag [string match "*!W!*" $expect_out(9,string)] + set have_error_tag [string match "*!E!*" $expect_out(9,string)] + if { $column_subexp_map($expect_out(1,string)) == 9 } { + # testing an after-row tag + switch -- $expect_out(1,string) { + warning { + incr scoreboard(warning) \ + [expr { $have_warning_tag ? 1 : -1 }] + incr scoreboard(error) \ + [expr { $have_error_tag ? -1 : 1 }] + } + error { + incr scoreboard(warning) \ + [expr { $have_warning_tag ? -1 : 1 }] + incr scoreboard(error) \ + [expr { $have_error_tag ? 1 : -1 }] + } + default { error "unknown tag $expect_out(1,string)" } + } + } else { + incr scoreboard(warning) [expr { $have_warning_tag ? -1 : 1 }] + incr scoreboard(error) [expr { $have_error_tag ? -1 : 1 }] + } + exp_continue + } + -re {^[[:space:]]+\|-+[\r\n]+} { + incr separator_count + } +} +foreach result [lsort [array names scoreboard]] { + verbose -log "scoreboard($result) = $scoreboard($result)" +} +foreach result [array names scoreboard] { + if { $scoreboard($result) == ( 7 + ( $column_subexp_map($result) == 9\ + ? [llength $test_names] : 0 ) ) } { + pass "count result $result" + } else { + fail "count result $result" + } +} + +# check totals +set column_totals { pad 1 1 2 2 1 1 1 } +set re_totals_row {^[[:space:]]+\|} +append re_totals_row $re_digit_columns +append re_totals_row {[\r\n]+} +set totals_matched 0 +expect { + -re $re_totals_row { + for { set i 1 } { $i < 8 } { incr i } { + if { [lindex $column_totals $i] == $expect_out($i,string) } { + incr totals_matched + } + } + exp_continue + } + -re {^[[:space:]]+\|-+[\r\n]+} { + incr separator_count + } + -re {^[[:space:]]+\\_+[\r\n]+} { + # all done + } +} + +if { $totals_matched == 7 } { + pass "expected total count" +} else { + fail "expected total count" +} + +if { $separator_count == 2 } { + pass "expected separator lines" +} else { + fail "expected separator lines" +} + +# Ensure that totals map correctly by reading each file one at a time +foreach name $test_names { + set separator_count 0 + spawn /bin/sh -c \ + "cd [testsuite file -object -test onetest]\ + && exec $LAUNCHER report-card one-${name}.sum" + # skip header + expect { + -re {^[[:space:]]+_+[\r\n]+} { exp_continue } + -re {^[[:space:]]+/([^\r\n]*)[\r\n]+} { exp_continue } + -re {^[[:space:]]+\|-+[\r\n]+} { incr separator_count } + } + # capture the item line + expect -re {^one-[^|]+(\|[[:space:][:digit:]]*)[[:space:]!EW]*[\r\n]+} { + regsub {[[:space:]]*$} $expect_out(1,string) "" item_line + } + # skip the separator + expect -re {^[[:space:]]+\|-+[\r\n]+} { incr separator_count } + # capture the totals line + expect -re {^[[:space:]]+(\|[[:space:][:digit:]]*)[\r\n]+} { + regsub {[[:space:]]*$} $expect_out(1,string) "" totals_line + } + # skip the footer + expect -re {.+} { exp_continue } + # do the item and totals lines match? + if { $item_line eq $totals_line } { + pass "verify total for $name" + } else { + fail "verify total for $name" + } + if { $separator_count == 2 } { + pass "expected separator lines for $name" + } else { + fail "expected separator lines for $name" + } +} + +#EOF diff --git a/testsuite/report-card.all/passes.exp b/testsuite/report-card.all/passes.exp new file mode 100644 index 0000000..012e9ac --- /dev/null +++ b/testsuite/report-card.all/passes.exp @@ -0,0 +1,276 @@ +# Copyright (C) 2018 Free Software Foundation, Inc. +# +# This file is part of DejaGnu. +# +# DejaGnu is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# DejaGnu is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with DejaGnu; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA. + +# This file was written by Jacob Bachmeyer. + +load_lib bohman_ssd.exp + +set header_column_names { PASS FAIL ?PASS ?FAIL UNSUP UNRES UNTEST } +set result_column_map { + PASS FAIL { KPASS XPASS } { KFAIL XFAIL } + UNSUPPORTED UNRESOLVED UNTESTED +} + +set test_results { PASS FAIL KPASS KFAIL XPASS XFAIL + UNSUPPORTED UNRESOLVED UNTESTED } + +# each entry: { {mode n} { suffix_tag... } { pass... } { { result name }... } } +array unset tuplemap +array set tuplemap { + basic { {S 3} { a b } { foo bar } + { { PASS pass } { FAIL fail } } } + kxpass { {S 2} { a b } { foo bar } + { { KPASS kpass } { XPASS xpass } } } + kxfail { {Sp 2} { a b } { foo bar } + { { KFAIL kfail } { XFAIL xfail } } } + unresult { {S 2} { a b } { foo bar } + { { UNSUPPORTED unsupported } + { UNRESOLVED unresolved } { UNTESTED untested } } } +} + +# Given: TUPLES: { { result ... }... }, PASSES: { pass... } +# Return: Cartesian product TUPLES x PASSES: { { result pass ... }... } +proc build_tuple_list { tuples passes } { + set result [list] + foreach cell $tuples { + foreach pass $passes { + lappend result [linsert $cell 1 $pass] + } + } + return $result +} + +# Given: TUPLES: { { result pass name }... }, MODE: S | Sp, N +# Return: { { result pass name count }... } where COUNT is from an SSD-set +proc annotate_tuple_list { tuples mode n } { + set m [llength $tuples] + set ssd [switch -- $mode { + S { ::math_utils::Bohman_SSD::S $n $m } + Sp { ::math_utils::Bohman_SSD::Sp $n $m } + }] + set result [list] + foreach cell $tuples ssdterm $ssd { + lappend result [linsert $cell end $ssdterm] + } + return $result +} + +# Given: TUPLES: { { result pass name count }... }; (RESULT,PASS) not unique +# Return: { { result pass expected_total }... } where (RESULT,PASS) is unique +proc compute_expected_pass_totals { tuples } { + foreach cell $tuples { set count([lrange $cell 0 1]) 0 } + foreach cell $tuples { incr count([lrange $cell 0 1]) [lindex $cell 3] } + set result [list] + foreach name [lsort [array names count]] { + lappend result [concat $name $count($name)] + } + return $result +} + +# Given: TUPLES: { { result pass name count }... }; (RESULT,PASS) not unique +# Return: { { result expected_grand_total }... } +proc compute_expected_grand_totals { tuples } { + foreach cell $tuples { set count([lindex $cell 0]) 0 } + foreach cell $tuples { incr count([lindex $cell 0]) [lindex $cell 3] } + set result [list] + foreach name [lsort [array names count]] { + lappend result [list $name $count($name)] + } + return $result +} + +# Given: TUPLES: { { result pass ... }... } where (RESULT,PASS) repeats later +# Return: { { { result pass ... }... }... }; (RESULT,PASS) unique per sublist +proc split_tuple_list { tuples } { + set result [list] + set sublist [list] + foreach cell $tuples { + if { [info exists seen([lrange $cell 0 1])] } { + # split here + lappend result $sublist + set sublist [list] + array unset seen + } + lappend sublist $cell + set seen([lrange $cell 0 1]) 1 + } + lappend result $sublist + return $result +} + +# TUPLES is: { { result pass name count }... } +proc write_file { basename tuples } { + set fd [open [testsuite file -object -test passes ${basename}.sum] w] + set pass {} + foreach cell [lsort -index 1 $tuples] { + if { $pass ne [lindex $cell 1] } { + puts $fd "Running pass `[lindex $cell 1]' ..." + set pass [lindex $cell 1] + } + for { set i 1 } { $i <= [lindex $cell 3] } { incr i } { + puts $fd "[lindex $cell 0]: [lindex $cell 1]:\ + [lindex $cell 2] test ${i}/[lindex $cell 3]" + } + } + close $fd +} + +proc run_multipass_output_test { filetag } { + global LAUNCHER + global header_column_names + global result_column_map + global test_results + global tuplemap + + set ssdpar [lindex $tuplemap($filetag) 0] + set tags [lindex $tuplemap($filetag) 1] + set passes [lindex $tuplemap($filetag) 2] + set results {} + foreach dummy $tags { lappend results [lindex $tuplemap($filetag) 3] } + set results [join $results] + + # initialize totals arrays to zero + foreach result $test_results { set have_grand_totals($result) 0 } + array set want_grand_totals [array get have_grand_totals] + foreach cell [build_tuple_list $test_results $passes] { + set have_pass_totals([join [lrange $cell 0 1] ","]) 0 + } + array set want_pass_totals [array get have_pass_totals] + + # get the test list + set list [build_tuple_list $results $passes] + set list [annotate_tuple_list $list [lindex $ssdpar 0] [lindex $ssdpar 1]] + + # compute expected totals + # note that this only fills non-zero array positions + foreach cell [compute_expected_pass_totals $list] { + set want_pass_totals([join [lrange $cell 0 1] ","]) [lindex $cell 2] + } + array set want_grand_totals [join [compute_expected_grand_totals $list]] + + # write the test data files and store expected per-file counts + foreach tag $tags fileset [split_tuple_list $list] { + # write test file + write_file "${filetag}-${tag}" $fileset + # initialize test results for this file + foreach result $test_results { + foreach pass $passes { + set want_file_counts(${filetag}-${tag},$result,$pass) 0 + set have_file_counts(${filetag}-${tag},$result,$pass) 0 + } + } + # store expected results for this file + foreach cell $fileset { + set want_file_counts(${filetag}-${tag},[join [lrange $cell 0 1] \ + ","]) [lindex $cell 3] + } + } + + # run the dejagnu-report-card tool + set separator_count 0 + spawn /bin/sh -c \ + "cd [testsuite file -object -test passes]\ + && exec $LAUNCHER report-card ${filetag}-*.sum" + + # skip header + expect { + -re {^[[:space:]]+_+[\r\n]+} { exp_continue } + -re {^[[:space:]]+/([^\r\n]*)[\r\n]+} { exp_continue } + -re {^[[:space:]]+\|-+[\r\n]+} { incr separator_count } + } + + # read individual file lines + set re_file_row {^[[:space:]]*} + append re_file_row {(} $filetag {-[[:alpha:]]+)[[:space:]]+} + append re_file_row {/[[:space:]]+([[:alpha:]]+)[[:space:]]+\|} + append re_file_row {[[:space:]]*([[:digit:][:space:]]+)[\r\n]+} + expect { + -re $re_file_row { + foreach column $result_column_map colname $header_column_names \ + have $expect_out(3,string) { + set want 0 + foreach rs $column { + set tmp $expect_out(1,string),$rs,$expect_out(2,string) + incr want $want_file_counts($tmp) + } + if { $have == $want } { + pass "count $colname\ + for pass $expect_out(2,string)\ + in file $expect_out(1,string)" + } else { + fail "count $colname\ + for pass $expect_out(2,string)\ + in file $expect_out(1,string)" + } + } + exp_continue + } + -re {^[[:space:]]+\|-+[\r\n]+} { incr separator_count } + } + + # read pass totals lines + set re_pass_row {^[[:space:]]+([[:alpha:]]+)[[:space:]]+\|} + append re_pass_row {[[:space:]]*([[:digit:][:space:]]+)[\r\n]+} + expect { + -re $re_pass_row { + foreach column $result_column_map colname $header_column_names \ + have $expect_out(2,string) { + set want 0 + foreach rs $column { + incr want $want_pass_totals($rs,$expect_out(1,string)) + } + if { $have == $want } { + pass "total $colname for pass $expect_out(1,string)" + } else { + fail "total $colname for pass $expect_out(1,string)" + } + } + exp_continue + } + -re {^[[:space:]]+\|-+[\r\n]+} { incr separator_count } + } + + # read grand totals line + expect -re {^[[:space:]]+\|[[:space:]]*([[:digit:][:space:]]+)[\r\n]+} { + foreach column $result_column_map colname $header_column_names \ + have $expect_out(1,string) { + set want 0 + foreach rs $column { incr want $want_grand_totals($rs) } + if { $have == $want } { + pass "grand total $colname" + } else { + fail "grand total $colname" + } + } + } + + # skip the footer + expect -re {.+} { exp_continue } + + if { $separator_count == 3 } { + pass "expected separator lines" + } else { + fail "expected separator lines" + } +} + +foreach filetag [lsort [array names tuplemap]] { + run_multipass_output_test $filetag +} + +#EOF |