diff options
Diffstat (limited to 'scripts/testsuite-filter')
| -rwxr-xr-x | scripts/testsuite-filter | 263 |
1 files changed, 263 insertions, 0 deletions
diff --git a/scripts/testsuite-filter b/scripts/testsuite-filter new file mode 100755 index 0000000..51eabb2 --- /dev/null +++ b/scripts/testsuite-filter @@ -0,0 +1,263 @@ +#!/usr/bin/env python + + +from __future__ import print_function +import sys +import os +import re +import collections + +debug = False + + +class Arch: + def __init__(self, arch): + arch = re.sub("^rv(32|64|128)g", 'rv\\1imafd', arch) + m = re.match('rv(32|64|128)', arch) + self.base_arch = m.group(0) + self.ext = [] + i = len(self.base_arch) + + while i < len(arch): + ext = arch[i] + if ext in ['x', 's', 'z']: + extlen = 1 + while (i+extlen) < len(arch) and arch[i+extlen] != '_': + extlen += 1 + self.ext.append(arch[i:i + extlen]) + i += extlen + elif ext == '_': + i += 1 + else: + self.ext.append(ext) + i += 1 + + +def usage(): + print("%s <toolname> <libc>" \ + " <white-list-base-dir> <testsuite.sum>" % sys.argv[0]) + + +def get_white_list_files(raw_arch, abi, libc, white_list_base_dir): + """ Return white file list according the arch, abi, libc name and component. + """ + white_list_files = [] + arch = Arch(raw_arch) + + def append_if_exist(filename): + if debug: + print ("Try append: %s" % filename) + filepath = os.path.join(white_list_base_dir, filename) + if os.path.exists(filepath): + white_list_files.append(filepath) + + libc_filename = "common.log" + append_if_exist(libc_filename) + + libc_filename = "%s.log" % (libc) + append_if_exist(libc_filename) + + filename = "%s.log" % (arch.base_arch) + append_if_exist(filename) + + filename = "%s.log" % (abi) + append_if_exist(filename) + + filename = "%s.%s.log" % (arch.base_arch, abi) + append_if_exist(filename) + + filename = "%s.%s.log" % (libc, arch.base_arch) + append_if_exist(filename) + + filename = "%s.%s.log" % (libc, abi) + append_if_exist(filename) + + filename = "%s.%s.%s.log" % (libc, arch.base_arch, abi) + append_if_exist(filename) + + for ext in arch.ext: + filename = "%s.log" % (ext) + append_if_exist(filename) + + filename = "%s.%s.log" % (arch.base_arch, ext) + append_if_exist(filename) + + filename = "%s.%s.log" % (ext, abi) + append_if_exist(filename) + + filename = "%s.%s.%s.log" % (arch.base_arch, ext, abi) + append_if_exist(filename) + + filename = "%s.%s.log" % (libc, ext) + append_if_exist(filename) + + filename = "%s.%s.%s.log" % (libc, arch.base_arch, ext) + append_if_exist(filename) + + filename = "%s.%s.%s.log" % (libc, ext, abi) + append_if_exist(filename) + + filename = "%s.%s.%s.%s.log" % (libc, arch.base_arch, ext, abi) + append_if_exist(filename) + + return white_list_files + + +def read_white_lists(white_list_files): + white_lists = dict() + for fname in white_list_files: + with open(fname) as f: + content = f.readlines() + for l in content: + l = l.strip() + if len(l) == 0: + continue + if l[0] == '#': + continue + try: + key = l.split(' ')[1] + except: + print ("Corrupt whitelist file?") + print ("Each line must contail <STATUS>: .*") + print ("e.g. FAIL: g++.dg/pr83239.C") + print ("Or starts with # for comment") + white_lists[key] = l + return white_lists + + +def read_gcc_sum(sum_files): + unexpected_results = dict() + for sum_file in sum_files: + with open(sum_file) as f: + content = f.readlines() + current_target = None + variations = [] + scan_variations = False + unexpected_result = dict() + tool = os.path.basename(sum_file).split(".")[0] + for l in content: + if l.startswith("Schedule of variations"): + scan_variations = True + continue + if scan_variations and l.startswith(" "): + variations.append(l.strip()) + continue + scan_variations = False + + if l.startswith("Running target"): + # Parsing current running target. + current_target = l.split(" ")[-1].strip() + unexpected_result[current_target] = list() + elif l.startswith("FAIL") or l.startswith("XPASS"): + unexpected_result[current_target].append(l.strip()) + unexpected_results[tool] = unexpected_result + # tool -> variation(target) -> list of unexpected result + return unexpected_results + + +def get_white_list(arch, abi, libc, white_list_base_dir): + white_list_files = \ + get_white_list_files(arch, abi, libc, white_list_base_dir) + white_list = read_white_lists(white_list_files) + return white_list + + + +def filter_result(tool, libc, white_list_base_dir, unexpected_results): + summary = dict() + any_fail = False + # Filter with white list. + for testtool, variation_unexpected_result in unexpected_results.iteritems(): + for variation, unexpected_result in variation_unexpected_result.iteritems(): + # Extract variation to arch/abi + arch = "" + abi = "" + cmodel = "" + for info in variation.split('/'): + if info.startswith('-march'): + arch = info[7:] + elif info.startswith('-mabi'): + abi = info[6:] + elif info.startswith('-mcmodel'): + cmodel = info[9:] + + white_list = \ + get_white_list(arch, abi, libc, + os.path.join(white_list_base_dir, tool)) + # filter! + config = (arch, abi, cmodel) + fail_count = 0 + case_count = set() + unexpected_result_list = [] + for ur in unexpected_result: + key = ur.split(' ')[1] + if key in white_list and \ + ur.startswith(white_list[key]): + # This item can be ignored + continue + else: + unexpected_result_list.append(ur) + fail_count += 1 + case_count.add(key) + any_fail = True + if config not in summary: + summary[config] = dict() + summary[config][testtool] = (fail_count, len(case_count)) + + if len(unexpected_result_list) != 0: + print ("\t\t=== %s: Unexpected fails for %s %s %s ===" \ + % (testtool, arch, abi, cmodel)) + for ur in unexpected_result_list: + print (ur) + + + # Generate summary report. + if tool == 'gcc': + toollist = ['gcc', 'g++', 'gfortran'] + else: + raise Exception("Unsupported tool `%s`" % tool) + + bar_item = map(lambda x: "%13s" % x, toollist) + bar = " |".join(bar_item) + print ("\n ========= Summary of gcc testsuite =========") + print (" | # of unexpected case / # of unique unexpected case") + print (" |%s |" % bar) + for config, result in summary.iteritems(): + arch, abi, cmodel = config + print (" %10s/ %6s/ %6s |" % (arch, abi, cmodel), end='') + for tool in toollist: + if tool not in summary[config]: + print ("%13s |" % '-', end='') + continue + + fail_count, case_count = summary[config][tool] + print ("%5d / %5d |" % (fail_count, case_count), end='') + print ("") + if any_fail: + return 1 + else: + return 0 + + +def main(): + if len(sys.argv) < 5: + usage() + sys.exit() + tool, libc, white_list_base_dir, sum_files = sys.argv[1:5] + + rv = 0 + + sum_files = sum_files.split(',') + unexpected_results = read_gcc_sum(sum_files) + if tool in ['gcc']: + rv = filter_result(tool, libc, white_list_base_dir, + unexpected_results) + else: + print ("Unsupported tool: `%s`" % tool) + rv = 1 + + sys.exit(rv) + + +if __name__ == '__main__': + main() |