diff options
Diffstat (limited to 'contrib')
| -rw-r--r-- | contrib/ChangeLog | 5 | ||||
| -rw-r--r-- | contrib/dg-extract-results.py | 592 | ||||
| -rwxr-xr-x | contrib/dg-extract-results.sh | 459 |
3 files changed, 1056 insertions, 0 deletions
diff --git a/contrib/ChangeLog b/contrib/ChangeLog new file mode 100644 index 0000000..8ce6765 --- /dev/null +++ b/contrib/ChangeLog @@ -0,0 +1,5 @@ +2018-08-06 Rainer Orth <ro@CeBiTec.Uni-Bielefeld.DE> + + * dg-extract-results.sh: Move from gdb/testsuite. + Update from gcc repo. + * dg-extract-results.py: New from gcc repo. diff --git a/contrib/dg-extract-results.py b/contrib/dg-extract-results.py new file mode 100644 index 0000000..4b02a5b --- /dev/null +++ b/contrib/dg-extract-results.py @@ -0,0 +1,592 @@ +#!/usr/bin/python +# +# Copyright (C) 2014 Free Software Foundation, Inc. +# +# This script is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. + +import sys +import getopt +import re +import io +from datetime import datetime +from operator import attrgetter + +# True if unrecognised lines should cause a fatal error. Might want to turn +# this on by default later. +strict = False + +# True if the order of .log segments should match the .sum file, false if +# they should keep the original order. +sort_logs = True + +# A version of open() that is safe against whatever binary output +# might be added to the log. +def safe_open (filename): + if sys.version_info >= (3, 0): + return open (filename, 'r', errors = 'surrogateescape') + return open (filename, 'r') + +# Force stdout to handle escape sequences from a safe_open file. +if sys.version_info >= (3, 0): + sys.stdout = io.TextIOWrapper (sys.stdout.buffer, + errors = 'surrogateescape') + +class Named: + def __init__ (self, name): + self.name = name + +class ToolRun (Named): + def __init__ (self, name): + Named.__init__ (self, name) + # The variations run for this tool, mapped by --target_board name. + self.variations = dict() + + # Return the VariationRun for variation NAME. + def get_variation (self, name): + if name not in self.variations: + self.variations[name] = VariationRun (name) + return self.variations[name] + +class VariationRun (Named): + def __init__ (self, name): + Named.__init__ (self, name) + # A segment of text before the harness runs start, describing which + # baseboard files were loaded for the target. + self.header = None + # The harnesses run for this variation, mapped by filename. + self.harnesses = dict() + # A list giving the number of times each type of result has + # been seen. + self.counts = [] + + # Return the HarnessRun for harness NAME. + def get_harness (self, name): + if name not in self.harnesses: + self.harnesses[name] = HarnessRun (name) + return self.harnesses[name] + +class HarnessRun (Named): + def __init__ (self, name): + Named.__init__ (self, name) + # Segments of text that make up the harness run, mapped by a test-based + # key that can be used to order them. + self.segments = dict() + # Segments of text that make up the harness run but which have + # no recognized test results. These are typically harnesses that + # are completely skipped for the target. + self.empty = [] + # A list of results. Each entry is a pair in which the first element + # is a unique sorting key and in which the second is the full + # PASS/FAIL line. + self.results = [] + + # Add a segment of text to the harness run. If the segment includes + # test results, KEY is an example of one of them, and can be used to + # combine the individual segments in order. If the segment has no + # test results (e.g. because the harness doesn't do anything for the + # current configuration) then KEY is None instead. In that case + # just collect the segments in the order that we see them. + def add_segment (self, key, segment): + if key: + assert key not in self.segments + self.segments[key] = segment + else: + self.empty.append (segment) + +class Segment: + def __init__ (self, filename, start): + self.filename = filename + self.start = start + self.lines = 0 + +class Prog: + def __init__ (self): + # The variations specified on the command line. + self.variations = [] + # The variations seen in the input files. + self.known_variations = set() + # The tools specified on the command line. + self.tools = [] + # Whether to create .sum rather than .log output. + self.do_sum = True + # Regexps used while parsing. + self.test_run_re = re.compile (r'^Test Run By (\S+) on (.*)$') + self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$') + self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED' + r'|WARNING|ERROR|UNSUPPORTED|UNTESTED' + r'|KFAIL):\s*(.+)') + self.completed_re = re.compile (r'.* completed at (.*)') + # Pieces of text to write at the head of the output. + # start_line is a pair in which the first element is a datetime + # and in which the second is the associated 'Test Run By' line. + self.start_line = None + self.native_line = '' + self.target_line = '' + self.host_line = '' + self.acats_premable = '' + # Pieces of text to write at the end of the output. + # end_line is like start_line but for the 'runtest completed' line. + self.acats_failures = [] + self.version_output = '' + self.end_line = None + # Known summary types. + self.count_names = [ + '# of DejaGnu errors\t\t', + '# of expected passes\t\t', + '# of unexpected failures\t', + '# of unexpected successes\t', + '# of expected failures\t\t', + '# of unknown successes\t\t', + '# of known failures\t\t', + '# of untested testcases\t\t', + '# of unresolved testcases\t', + '# of unsupported tests\t\t' + ] + self.runs = dict() + + def usage (self): + name = sys.argv[0] + sys.stderr.write ('Usage: ' + name + + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ... + + tool The tool (e.g. g++, libffi) for which to create a + new test summary file. If not specified then output + is created for all tools. + variant-list One or more test variant names. If the list is + not specified then one is constructed from all + variants in the files for <tool>. + sum-file A test summary file with the format of those + created by runtest from DejaGnu. + If -L is used, merge *.log files instead of *.sum. In this + mode the exact order of lines may not be preserved, just different + Running *.exp chunks should be in correct order. +''') + sys.exit (1) + + def fatal (self, what, string): + if not what: + what = sys.argv[0] + sys.stderr.write (what + ': ' + string + '\n') + sys.exit (1) + + # Parse the command-line arguments. + def parse_cmdline (self): + try: + (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L') + if len (self.files) == 0: + self.usage() + for (option, value) in options: + if option == '-l': + self.variations.append (value) + elif option == '-t': + self.tools.append (value) + else: + self.do_sum = False + except getopt.GetoptError as e: + self.fatal (None, e.msg) + + # Try to parse time string TIME, returning an arbitrary time on failure. + # Getting this right is just a nice-to-have so failures should be silent. + def parse_time (self, time): + try: + return datetime.strptime (time, '%c') + except ValueError: + return datetime.now() + + # Parse an integer and abort on failure. + def parse_int (self, filename, value): + try: + return int (value) + except ValueError: + self.fatal (filename, 'expected an integer, got: ' + value) + + # Return a list that represents no test results. + def zero_counts (self): + return [0 for x in self.count_names] + + # Return the ToolRun for tool NAME. + def get_tool (self, name): + if name not in self.runs: + self.runs[name] = ToolRun (name) + return self.runs[name] + + # Add the result counts in list FROMC to TOC. + def accumulate_counts (self, toc, fromc): + for i in range (len (self.count_names)): + toc[i] += fromc[i] + + # Parse the list of variations after 'Schedule of variations:'. + # Return the number seen. + def parse_variations (self, filename, file): + num_variations = 0 + while True: + line = file.readline() + if line == '': + self.fatal (filename, 'could not parse variation list') + if line == '\n': + break + self.known_variations.add (line.strip()) + num_variations += 1 + return num_variations + + # Parse from the first line after 'Running target ...' to the end + # of the run's summary. + def parse_run (self, filename, file, tool, variation, num_variations): + header = None + harness = None + segment = None + final_using = 0 + + # If this is the first run for this variation, add any text before + # the first harness to the header. + if not variation.header: + segment = Segment (filename, file.tell()) + variation.header = segment + + # Parse the rest of the summary (the '# of ' lines). + if len (variation.counts) == 0: + variation.counts = self.zero_counts() + + # Parse up until the first line of the summary. + if num_variations == 1: + end = '\t\t=== ' + tool.name + ' Summary ===\n' + else: + end = ('\t\t=== ' + tool.name + ' Summary for ' + + variation.name + ' ===\n') + while True: + line = file.readline() + if line == '': + self.fatal (filename, 'no recognised summary line') + if line == end: + break + + # Look for the start of a new harness. + if line.startswith ('Running ') and line.endswith (' ...\n'): + # Close off the current harness segment, if any. + if harness: + segment.lines -= final_using + harness.add_segment (first_key, segment) + name = line[len ('Running '):-len(' ...\n')] + harness = variation.get_harness (name) + segment = Segment (filename, file.tell()) + first_key = None + final_using = 0 + continue + + # Record test results. Associate the first test result with + # the harness segment, so that if a run for a particular harness + # has been split up, we can reassemble the individual segments + # in a sensible order. + # + # dejagnu sometimes issues warnings about the testing environment + # before running any tests. Treat them as part of the header + # rather than as a test result. + match = self.result_re.match (line) + if match and (harness or not line.startswith ('WARNING:')): + if not harness: + self.fatal (filename, 'saw test result before harness name') + name = match.group (2) + # Ugly hack to get the right order for gfortran. + if name.startswith ('gfortran.dg/g77/'): + name = 'h' + name + key = (name, len (harness.results)) + harness.results.append ((key, line)) + if not first_key and sort_logs: + first_key = key + if line.startswith ('ERROR: (DejaGnu)'): + for i in range (len (self.count_names)): + if 'DejaGnu errors' in self.count_names[i]: + variation.counts[i] += 1 + break + + # 'Using ...' lines are only interesting in a header. Splitting + # the test up into parallel runs leads to more 'Using ...' lines + # than there would be in a single log. + if line.startswith ('Using '): + final_using += 1 + else: + final_using = 0 + + # Add other text to the current segment, if any. + if segment: + segment.lines += 1 + + # Close off the final harness segment, if any. + if harness: + segment.lines -= final_using + harness.add_segment (first_key, segment) + + while True: + before = file.tell() + line = file.readline() + if line == '': + break + if line == '\n': + continue + if not line.startswith ('# '): + file.seek (before) + break + found = False + for i in range (len (self.count_names)): + if line.startswith (self.count_names[i]): + count = line[len (self.count_names[i]):-1].strip() + variation.counts[i] += self.parse_int (filename, count) + found = True + break + if not found: + self.fatal (filename, 'unknown test result: ' + line[:-1]) + + # Parse an acats run, which uses a different format from dejagnu. + # We have just skipped over '=== acats configuration ==='. + def parse_acats_run (self, filename, file): + # Parse the preamble, which describes the configuration and logs + # the creation of support files. + record = (self.acats_premable == '') + if record: + self.acats_premable = '\t\t=== acats configuration ===\n' + while True: + line = file.readline() + if line == '': + self.fatal (filename, 'could not parse acats preamble') + if line == '\t\t=== acats tests ===\n': + break + if record: + self.acats_premable += line + + # Parse the test results themselves, using a dummy variation name. + tool = self.get_tool ('acats') + variation = tool.get_variation ('none') + self.parse_run (filename, file, tool, variation, 1) + + # Parse the failure list. + while True: + before = file.tell() + line = file.readline() + if line.startswith ('*** FAILURES: '): + self.acats_failures.append (line[len ('*** FAILURES: '):-1]) + continue + file.seek (before) + break + + # Parse the final summary at the end of a log in order to capture + # the version output that follows it. + def parse_final_summary (self, filename, file): + record = (self.version_output == '') + while True: + line = file.readline() + if line == '': + break + if line.startswith ('# of '): + continue + if record: + self.version_output += line + if line == '\n': + break + + # Parse a .log or .sum file. + def parse_file (self, filename, file): + tool = None + target = None + num_variations = 1 + while True: + line = file.readline() + if line == '': + return + + # Parse the list of variations, which comes before the test + # runs themselves. + if line.startswith ('Schedule of variations:'): + num_variations = self.parse_variations (filename, file) + continue + + # Parse a testsuite run for one tool/variation combination. + if line.startswith ('Running target '): + name = line[len ('Running target '):-1] + if not tool: + self.fatal (filename, 'could not parse tool name') + if name not in self.known_variations: + self.fatal (filename, 'unknown target: ' + name) + self.parse_run (filename, file, tool, + tool.get_variation (name), + num_variations) + # If there is only one variation then there is no separate + # summary for it. Record any following version output. + if num_variations == 1: + self.parse_final_summary (filename, file) + continue + + # Parse the start line. In the case where several files are being + # parsed, pick the one with the earliest time. + match = self.test_run_re.match (line) + if match: + time = self.parse_time (match.group (2)) + if not self.start_line or self.start_line[0] > time: + self.start_line = (time, line) + continue + + # Parse the form used for native testing. + if line.startswith ('Native configuration is '): + self.native_line = line + continue + + # Parse the target triplet. + if line.startswith ('Target is '): + self.target_line = line + continue + + # Parse the host triplet. + if line.startswith ('Host is '): + self.host_line = line + continue + + # Parse the acats premable. + if line == '\t\t=== acats configuration ===\n': + self.parse_acats_run (filename, file) + continue + + # Parse the tool name. + match = self.tool_re.match (line) + if match: + tool = self.get_tool (match.group (1)) + continue + + # Skip over the final summary (which we instead create from + # individual runs) and parse the version output. + if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n': + if file.readline() != '\n': + self.fatal (filename, 'expected blank line after summary') + self.parse_final_summary (filename, file) + continue + + # Parse the completion line. In the case where several files + # are being parsed, pick the one with the latest time. + match = self.completed_re.match (line) + if match: + time = self.parse_time (match.group (1)) + if not self.end_line or self.end_line[0] < time: + self.end_line = (time, line) + continue + + # Sanity check to make sure that important text doesn't get + # dropped accidentally. + if strict and line.strip() != '': + self.fatal (filename, 'unrecognised line: ' + line[:-1]) + + # Output a segment of text. + def output_segment (self, segment): + with safe_open (segment.filename) as file: + file.seek (segment.start) + for i in range (segment.lines): + sys.stdout.write (file.readline()) + + # Output a summary giving the number of times each type of result has + # been seen. + def output_summary (self, tool, counts): + for i in range (len (self.count_names)): + name = self.count_names[i] + # dejagnu only prints result types that were seen at least once, + # but acats always prints a number of unexpected failures. + if (counts[i] > 0 + or (tool.name == 'acats' + and name.startswith ('# of unexpected failures'))): + sys.stdout.write ('%s%d\n' % (name, counts[i])) + + # Output unified .log or .sum information for a particular variation, + # with a summary at the end. + def output_variation (self, tool, variation): + self.output_segment (variation.header) + for harness in sorted (variation.harnesses.values(), + key = attrgetter ('name')): + sys.stdout.write ('Running ' + harness.name + ' ...\n') + if self.do_sum: + harness.results.sort() + for (key, line) in harness.results: + sys.stdout.write (line) + else: + # Rearrange the log segments into test order (but without + # rearranging text within those segments). + for key in sorted (harness.segments.keys()): + self.output_segment (harness.segments[key]) + for segment in harness.empty: + self.output_segment (segment) + if len (self.variations) > 1: + sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for ' + + variation.name + ' ===\n\n') + self.output_summary (tool, variation.counts) + + # Output unified .log or .sum information for a particular tool, + # with a summary at the end. + def output_tool (self, tool): + counts = self.zero_counts() + if tool.name == 'acats': + # acats doesn't use variations, so just output everything. + # It also has a different approach to whitespace. + sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n') + for variation in tool.variations.values(): + self.output_variation (tool, variation) + self.accumulate_counts (counts, variation.counts) + sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n') + else: + # Output the results in the usual dejagnu runtest format. + sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n' + 'Schedule of variations:\n') + for name in self.variations: + if name in tool.variations: + sys.stdout.write (' ' + name + '\n') + sys.stdout.write ('\n') + for name in self.variations: + if name in tool.variations: + variation = tool.variations[name] + sys.stdout.write ('Running target ' + + variation.name + '\n') + self.output_variation (tool, variation) + self.accumulate_counts (counts, variation.counts) + sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n') + self.output_summary (tool, counts) + + def main (self): + self.parse_cmdline() + try: + # Parse the input files. + for filename in self.files: + with safe_open (filename) as file: + self.parse_file (filename, file) + + # Decide what to output. + if len (self.variations) == 0: + self.variations = sorted (self.known_variations) + else: + for name in self.variations: + if name not in self.known_variations: + self.fatal (None, 'no results for ' + name) + if len (self.tools) == 0: + self.tools = sorted (self.runs.keys()) + + # Output the header. + if self.start_line: + sys.stdout.write (self.start_line[1]) + sys.stdout.write (self.native_line) + sys.stdout.write (self.target_line) + sys.stdout.write (self.host_line) + sys.stdout.write (self.acats_premable) + + # Output the main body. + for name in self.tools: + if name not in self.runs: + self.fatal (None, 'no results for ' + name) + self.output_tool (self.runs[name]) + + # Output the footer. + if len (self.acats_failures) > 0: + sys.stdout.write ('*** FAILURES: ' + + ' '.join (self.acats_failures) + '\n') + sys.stdout.write (self.version_output) + if self.end_line: + sys.stdout.write (self.end_line[1]) + except IOError as e: + self.fatal (e.filename, e.strerror) + +Prog().main() diff --git a/contrib/dg-extract-results.sh b/contrib/dg-extract-results.sh new file mode 100755 index 0000000..6ee3d26 --- /dev/null +++ b/contrib/dg-extract-results.sh @@ -0,0 +1,459 @@ +#! /bin/sh + +# For a specified tool and optional list of test variants, extract +# test results from one or more test summary (.sum) files and combine +# the results into a new test summary file, sent to the standard output. +# The resulting file can be used with test result comparison scripts for +# results from tests that were run in parallel. See usage() below. + +# Copyright (C) 2008, 2009, 2010, 2012 Free Software Foundation +# Contributed by Janis Johnson <janis187@us.ibm.com> +# +# This file is part of GCC. +# +# GCC is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# GCC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GCC; see the file COPYING. If not, write to +# the Free Software Foundation, 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. + +PROGNAME=dg-extract-results.sh + +# Try to use the python version if possible, since it tends to be faster. +PYTHON_VER=`echo "$0" | sed 's/sh$/py/'` +if test "$PYTHON_VER" != "$0" && + test -f "$PYTHON_VER" && + python -c 'import sys, getopt, re, io, datetime, operator; sys.exit (0 if sys.version_info >= (2, 6) else 1)' \ + > /dev/null 2> /dev/null; then + exec python $PYTHON_VER "$@" +fi + +usage() { + cat <<EOF >&2 +Usage: $PROGNAME [-t tool] [-l variant-list] [-L] sum-file ... + + tool The tool (e.g. g++, libffi) for which to create a + new test summary file. If not specified then all + specified sum files must be for the same tool. + variant-list One or more test variant names. If the list is + not specified then one is constructed from all + variants in the files for <tool>. + sum-file A test summary file with the format of those + created by runtest from DejaGnu. + If -L is used, merge *.log files instead of *.sum. In this + mode the exact order of lines may not be preserved, just different + Running *.exp chunks should be in correct order. +EOF +} + +# Write a message to the standard error. + +msg() { + echo "$@" >&2 +} + +# Parse the command-line options. + +VARIANTS="" +TOOL="" +MODE="sum" + +while getopts "l:t:L" ARG; do + case $ARG in + l) VARIANTS="${VARIANTS} ${OPTARG}";; + t) test -z "$TOOL" || (msg "${PROGNAME}: only one tool can be specified"; exit 1); + TOOL="${OPTARG}";; + L) MODE="log";; + \?) usage; exit 0;; + esac +done +shift `expr ${OPTIND} - 1` + +if test $# -lt 1 ; then + usage + exit 1 +fi + +TMPDIR=${TMPDIR-/tmp} +SUM_FILES="$@" +FIRST_SUM=$1 +TMP= +trap 'EXIT_STATUS=$?; rm -rf $TMP && exit $EXIT_STATUS' 0 +# Create a (secure) tmp directory for tmp files. +{ + TMP=`(umask 077 && mktemp -d -q "${TMPDIR}/dg-combine-results-$$-XXXXXX") 2>/dev/null` && + test -n "$TMP" && test -d "$TMP" +} || +{ + TMP=${TMPDIR}/dg-combine-results-$$-$RANDOM + (umask 077 && mkdir $TMP) +} || +{ + msg "${PROGNAME}: cannot create a temporary directory" + { (exit 1); exit 1; } +} + +# Find a good awk. + +if test -z "$AWK" ; then + for AWK in gawk nawk awk + do + if type $AWK 2>&1 | grep 'not found' > /dev/null 2>&1 ; then + : + else + break + fi + done +fi + +# Verify that the specified summary files exist. + +ERROR=0 +for FILE in $SUM_FILES +do + if ! test -f $FILE ; then + msg "${PROGNAME}: file $FILE does not exist." + ERROR=1 + fi +done +test $ERROR -eq 0 || exit 1 + +# Test if grep supports the '--text' option + +GREP=grep + +if echo -e '\x00foo\x00' | $GREP --text foo > /dev/null 2>&1 ; then + GREP="grep --text" +else + # Our grep does not recognize the '--text' option. We have to + # treat our files in order to remove any non-printable character. + for file in $SUM_FILES ; do + mv $file ${file}.orig + cat -v ${file}.orig > $file + done +fi + +if [ -z "$TOOL" ]; then + # If no tool was specified, all specified summary files must be for + # the same tool. + + CNT=`$GREP '=== .* tests ===' $SUM_FILES | $AWK '{ print $3 }' | sort -u | wc -l` + if [ $CNT -eq 1 ]; then + TOOL=`$GREP '=== .* tests ===' $FIRST_SUM | $AWK '{ print $2 }'` + else + msg "${PROGNAME}: sum files are for multiple tools, specify a tool" + msg "" + usage + exit 1 + fi +else + # Ignore the specified summary files that are not for this tool. This + # should keep the relevant files in the same order. + + SUM_FILES=`$GREP -l "=== $TOOL" $SUM_FILES` + if test -z "$SUM_FILES" ; then + msg "${PROGNAME}: none of the specified files are results for $TOOL" + exit 1 + fi +fi + +if [ "$TOOL" = acats ]; then + # Acats *.sum or *.log files aren't dejagnu generated, and they have + # somewhat different format. + ACATS_AWK=${TMP}/acats.awk + cat <<EOF > $ACATS_AWK +BEGIN { + print_prologue=1; curfile=""; insummary=0 + passcnt=0; failcnt=0; unsupcnt=0; failures="" +} +/^[ \t]*=== acats configuration ===/ { + insummary=0 + if (print_prologue) print + next +} +/^[ \t]*=== acats tests ===/ { + if (print_prologue) print + print_prologue=0 + next +} +/^Running chapter / { + if (curfile) close (curfile) + curfile="${TMP}/chapter-"\$3 + print >> curfile + next +} +/^[ \t]*=== acats Summary ===/ { + if (curfile) close (curfile) + curfile="" + insummary=1 + next +} +/^# of expected passes/ { if (insummary == 1) passcnt += \$5; next; } +/^# of unexpected failures/ { if (insummary == 1) failcnt += \$5; next; } +/^# of unsupported tests/ { if (insummary == 1) unsupcnt += \$5; next; } +/^\*\*\* FAILURES: / { + if (insummary == 1) { + if (failures) sub(/^\*\*\* FAILURES:/,"") + failures=failures""\$0 + } +} +{ + if (print_prologue) { print; next } + if (curfile) print >> curfile +} +END { + system ("cat ${TMP}/chapter-*") + print " === acats Summary ===" + print "# of expected passes " passcnt + print "# of unexpected failures " failcnt + if (unsupcnt) print "# of unsupported tests " unsupcnt + if (failures) print failures +} +EOF + + rm -f ${TMP}/chapter-* + $AWK -f $ACATS_AWK $SUM_FILES + exit 0 +fi + +# If no variants were specified, find all variants in the remaining +# summary files. Otherwise, ignore specified variants that aren't in +# any of those summary files. + +if test -z "$VARIANTS" ; then + VAR_AWK=${TMP}/variants.awk + cat <<EOF > $VAR_AWK +/^Schedule of variations:/ { in_vars=1; next } +/^$/ { in_vars=0 } +/^Running target/ { exit } +{ if (in_vars==1) print \$1; else next } +EOF + + touch ${TMP}/varlist + for FILE in $SUM_FILES; do + $AWK -f $VAR_AWK $FILE >> ${TMP}/varlist + done + VARIANTS="`sort -u ${TMP}/varlist`" +else + VARS="$VARIANTS" + VARIANTS="" + for VAR in $VARS + do + $GREP "Running target $VAR" $SUM_FILES > /dev/null && VARIANTS="$VARIANTS $VAR" + done +fi + +# Find out if we have more than one variant, or any at all. + +VARIANT_COUNT=0 +for VAR in $VARIANTS +do + VARIANT_COUNT=`expr $VARIANT_COUNT + 1` +done + +if test $VARIANT_COUNT -eq 0 ; then + msg "${PROGNAME}: no file for $TOOL has results for the specified variants" + exit 1 +fi + +cat $SUM_FILES \ + | $AWK '/^Running/ { if ($2 != "target" && $3 == "...") print "EXPFILE: "$2 } ' \ + | sort -u > ${TMP}/expfiles + +# Write the begining of the combined summary file. + +head -n 2 $FIRST_SUM +echo +echo " === $TOOL tests ===" +echo +echo "Schedule of variations:" +for VAR in $VARIANTS +do + echo " $VAR" +done +echo + +# For each test variant for the tool, copy test reports from each of the +# summary files. Set up two awk scripts from within the loop to +# initialize VAR and TOOL with the script, rather than assuming that the +# available version of awk can pass variables from the command line. + +for VAR in $VARIANTS +do + GUTS_AWK=${TMP}/guts.awk + cat << EOF > $GUTS_AWK +BEGIN { + variant="$VAR" + firstvar=1 + expfileno=1 + cnt=0 + print_using=0 + need_close=0 +} +/^EXPFILE: / { + expfiles[expfileno] = \$2 + expfilesr[\$2] = expfileno + expfileno = expfileno + 1 +} +/^Running target / { + curvar = \$3 + if (variant == curvar && firstvar == 1) { print; print_using=1; firstvar = 0 } + next +} +/^Using / { + if (variant == curvar && print_using) { print; next } +} +/^Running .*\\.exp \\.\\.\\./ { + print_using=0 + if (variant == curvar) { + if (need_close) close(curfile) + curfile="${TMP}/list"expfilesr[\$2] + expfileseen[\$2]=expfileseen[\$2] + 1 + need_close=0 + testname="00" + next + } +} +/^\t\t=== .* ===$/ { curvar = ""; next } +/^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED|WARNING|ERROR|UNSUPPORTED|UNTESTED|KFAIL):/ { + testname=\$2 + # Ugly hack for gfortran.dg/dg.exp + if ("$TOOL" == "gfortran" && testname ~ /^gfortran.dg\/g77\//) + testname="h"testname +} +/^$/ { if ("$MODE" == "sum") next } +{ if (variant == curvar && curfile) { + if ("$MODE" == "sum") { + printf "%s %08d|", testname, cnt >> curfile + cnt = cnt + 1 + } + filewritten[curfile]=1 + need_close=1 + print >> curfile + } else + next +} +END { + n=1 + while (n < expfileno) { + if (expfileseen[expfiles[n]]) { + print "Running "expfiles[n]" ..." + if (filewritten["${TMP}/list"n]) { + if (expfileseen[expfiles[n]] == 1) + cmd="cat" + else + cmd="LC_ALL=C sort" + if ("$MODE" == "sum") + system (cmd" ${TMP}/list"n" | sed -n 's/^[^ ]* [^ |]*|//p'") + else + system ("cat ${TMP}/list"n) + } + } + n = n + 1 + } +} +EOF + + SUMS_AWK=${TMP}/sums.awk + rm -f $SUMS_AWK + cat << EOF > $SUMS_AWK +BEGIN { + variant="$VAR" + tool="$TOOL" + passcnt=0; failcnt=0; untstcnt=0; xpasscnt=0; xfailcnt=0; kpasscnt=0; kfailcnt=0; unsupcnt=0; unrescnt=0; dgerrorcnt=0; + curvar=""; insummary=0 +} +/^Running target / { curvar = \$3; next } +/^ERROR: \(DejaGnu\)/ { if (variant == curvar) dgerrorcnt += 1 } +/^# of / { if (variant == curvar) insummary = 1 } +/^# of expected passes/ { if (insummary == 1) passcnt += \$5; next; } +/^# of unexpected successes/ { if (insummary == 1) xpasscnt += \$5; next; } +/^# of unexpected failures/ { if (insummary == 1) failcnt += \$5; next; } +/^# of expected failures/ { if (insummary == 1) xfailcnt += \$5; next; } +/^# of unknown successes/ { if (insummary == 1) kpasscnt += \$5; next; } +/^# of known failures/ { if (insummary == 1) kfailcnt += \$5; next; } +/^# of untested testcases/ { if (insummary == 1) untstcnt += \$5; next; } +/^# of unresolved testcases/ { if (insummary == 1) unrescnt += \$5; next; } +/^# of unsupported tests/ { if (insummary == 1) unsupcnt += \$5; next; } +/^$/ { if (insummary == 1) + { insummary = 0; curvar = "" } + next + } +{ next } +END { + printf ("\t\t=== %s Summary for %s ===\n\n", tool, variant) + if (dgerrorcnt != 0) printf ("# of DejaGnu errors\t\t%d\n", dgerrorcnt) + if (passcnt != 0) printf ("# of expected passes\t\t%d\n", passcnt) + if (failcnt != 0) printf ("# of unexpected failures\t%d\n", failcnt) + if (xpasscnt != 0) printf ("# of unexpected successes\t%d\n", xpasscnt) + if (xfailcnt != 0) printf ("# of expected failures\t\t%d\n", xfailcnt) + if (kpasscnt != 0) printf ("# of unknown successes\t\t%d\n", kpasscnt) + if (kfailcnt != 0) printf ("# of known failures\t\t%d\n", kfailcnt) + if (untstcnt != 0) printf ("# of untested testcases\t\t%d\n", untstcnt) + if (unrescnt != 0) printf ("# of unresolved testcases\t%d\n", unrescnt) + if (unsupcnt != 0) printf ("# of unsupported tests\t\t%d\n", unsupcnt) +} +EOF + + PVAR=`echo $VAR | sed 's,/,.,g'` + TMPFILE=${TMP}/var-$PVAR + rm -f $TMPFILE + rm -f ${TMP}/list* + cat ${TMP}/expfiles $SUM_FILES | $AWK -f $GUTS_AWK + cat $SUM_FILES | $AWK -f $SUMS_AWK > $TMPFILE + # If there are multiple variants, output the counts for this one; + # otherwise there will just be the final counts at the end. + test $VARIANT_COUNT -eq 1 || cat $TMPFILE +done + +# Set up an awk script to get the combined summary counts for the tool. + +TOTAL_AWK=${TMP}/total.awk +cat << EOF > $TOTAL_AWK +BEGIN { + tool="$TOOL" + passcnt=0; failcnt=0; untstcnt=0; xpasscnt=0; xfailcnt=0; kfailcnt=0; unsupcnt=0; unrescnt=0; dgerrorcnt=0 +} +/^# of DejaGnu errors/ { dgerrorcnt += \$5 } +/^# of expected passes/ { passcnt += \$5 } +/^# of unexpected failures/ { failcnt += \$5 } +/^# of unexpected successes/ { xpasscnt += \$5 } +/^# of expected failures/ { xfailcnt += \$5 } +/^# of unknown successes/ { kpasscnt += \$5 } +/^# of known failures/ { kfailcnt += \$5 } +/^# of untested testcases/ { untstcnt += \$5 } +/^# of unresolved testcases/ { unrescnt += \$5 } +/^# of unsupported tests/ { unsupcnt += \$5 } +END { + printf ("\n\t\t=== %s Summary ===\n\n", tool) + if (dgerrorcnt != 0) printf ("# of DejaGnu errors\t\t%d\n", dgerrorcnt) + if (passcnt != 0) printf ("# of expected passes\t\t%d\n", passcnt) + if (failcnt != 0) printf ("# of unexpected failures\t%d\n", failcnt) + if (xpasscnt != 0) printf ("# of unexpected successes\t%d\n", xpasscnt) + if (xfailcnt != 0) printf ("# of expected failures\t\t%d\n", xfailcnt) + if (kpasscnt != 0) printf ("# of unknown successes\t\t%d\n", kpasscnt) + if (kfailcnt != 0) printf ("# of known failures\t\t%d\n", kfailcnt) + if (untstcnt != 0) printf ("# of untested testcases\t\t%d\n", untstcnt) + if (unrescnt != 0) printf ("# of unresolved testcases\t%d\n", unrescnt) + if (unsupcnt != 0) printf ("# of unsupported tests\t\t%d\n", unsupcnt) +} +EOF + +# Find the total summaries for the tool and add to the end of the output. +cat ${TMP}/var-* | $AWK -f $TOTAL_AWK + +# This is ugly, but if there's version output from the compiler under test +# at the end of the file, we want it. The other thing that might be there +# is the final summary counts. +tail -2 $FIRST_SUM | $GREP '^#' > /dev/null || tail -2 $FIRST_SUM + +exit 0 |