diff options
Diffstat (limited to 'bin/run_tests.py')
| -rwxr-xr-x | bin/run_tests.py | 727 |
1 files changed, 727 insertions, 0 deletions
diff --git a/bin/run_tests.py b/bin/run_tests.py new file mode 100755 index 0000000..ebde47b --- /dev/null +++ b/bin/run_tests.py @@ -0,0 +1,727 @@ +#! /usr/bin/env python3 +# vim: set tabstop=4 shiftwidth=4 expandtab set textwidth=79 +# ===========================================================================79 +# Filename: run_tests.py +# +# Description: Converting run_tests.sh to python +# +# Based on run_test.sh +# +# Same as run_test.sh: +# 1. output into .xml file is the same +# 2. colorization of output +# +# Differences with run_test.sh +# 1. Does not depend upon .elf file extension for filetype. +# Uses the unix 'file' command to get filetype +# 2. Added usage function +# 3. Added run switches (which didn't exist in run_test.sh) +# See the print_usage() function for a descriptionm of +# the supported switches. +# 4. Default search directories, rather than just one. +# 5. Removed use of $RISCV env var. +# 6. requires python3 +# 7. Added support to ignore tests via the --test_ignore_pyfile +# switch +# 8. Added support to supply switches to the riscv_sim command +# via the --test_switch_pyfile. +# +# Author(s): Bill McSpadden (bill@riscv.org) +# +# History: See revision control log +# ===========================================================================79 + +# ===========================================================================79 +# Necessary imports for this script: +import os # needed for os command interactions +import glob # needed for file list gathering useing wildcards +import re # regular expression +import sys # needed for command line arguments +import getopt # needed for command line arguments +import collections # needed for dequeues +import subprocess # needed for subprocesses where stdout is needed +from pathlib import Path +from inspect import currentframe, getframeinfo +from abc import ABC, abstractmethod +from copy import deepcopy +# Necessary imports for this script +# ===========================================================================79 + + +# ===========================================================================79 +# General data for use in this script. +# ===========================================================================79 +# Data structure for sim command line arguments +# TODO: make the key a regex +sim_test_command_line_switch_dict = { } +my_ignore_test_tuple = [] + +# Allowed command line options +opts, args = getopt.getopt ( + sys.argv[1:], + "dhuo:", + [ + "help", + "usage", + "outfile=", + "test_dir=", + "32bit=", + "64bit=", + "c_sim=", + "ocaml_sim=", + "sailcov=", + "clean_build=", + "test_dir=", + "test_switch_pyfile=", + "test_ignore_pyfile=", + "debug" + ] + ) + +# Variables to be overridden with command line switches +xml_outfile = "./tests.xml" +run_32bit_tests = True +run_64bit_tests = True +run_csim = True +run_ocamlsim = False +sailcov = False +clean_build = True +test_dir_list = [ "isa", "riscv-tests" ] +sail_riscv_rootdir = 'SAIL_RISCV_ROOTDIR' +test_switch_pyfile = '' +test_ignore_pyfile = '' +debug = False + +# Variables for tracking test status and test output +RED = '\033[0;91m' +GREEN = '\033[0;92m' +YELLOW = '\033[0;93m' +NC = '\033[0m' +test_pass = 0 +test_fail = 0 +all_pass = 0 +all_fail = 0 +SUITE_XML = "" +SUITES_XML = "" +# ===========================================================================79 + +# ===========================================================================79 +# Function prototypes: +# ==================================== +# Print Levels: +# print() Normal python print function. Goes to stdout +# debug_print() Print debug information. Goes to stdout. +# error_print() Print error message. Goes to stdout. Generally most errors should be fatal errors +# fatal_print() Print fatal error message. Exit with status 1. Generally most errors should be fatal errors. Goes to stdout. +# TRACE() For bringup debug only. TRACE() instances should be removed. + +def debug_print (text = "") : + if debug : + cf = currentframe() + of = cf.f_back + fi = getframeinfo(of) + filename = os.path.basename(fi.filename) + print("debug: file: " + filename + " line: " + str(of.f_lineno) + " : " + text) + return + +def error_print (text = "") : + cf = currentframe() + of = cf.f_back + fi = getframeinfo(of) + filename = os.path.basename(fi.filename) + print("error: file: " + filename + " line: " + str(of.f_lineno) + " : " + text) + return + +def fatal_print (text = "") : + cf = currentframe() + of = cf.f_back + fi = getframeinfo(of) + filename = os.path.basename(fi.filename) + print("fatal error: file: " + filename + " line: " + str(of.f_lineno) + " : " + text) + sys.exit(1) + return # never taken + +def TRACE(text = "") : + cf = currentframe() + of = cf.f_back + fi = getframeinfo(of) + filename = os.path.basename(fi.filename) + print("TRACE: file: " + filename + " line: " + str(of.f_lineno) + " : " + text) + return +# Print Levels: +# ==================================== + +# ==================================== +# Support for command line options +def print_usage(invocation) : + print(invocation + " usage: " + invocation + " [<options>]") + print(" Typically, invoke this script in the directory above where the elf-file tests live.") + print(" The script looks into test_dir, finds all of the elf files and then runs the simulator") + print(" with each elf file.") + print("") + print(" Output logs are put into [dir]/<testname>.cout (for C sim) or [dir/]<testname>.out (for ocaml sim).") + print("") + print(" Some tests require specific command line switches to properly run. To add these") + print(" command line switches, you must use the '--test_switch_pyfile=<file>' switch.") + print("") + print(" options:") + print(" -h --help -u -usage print out help/usage message") + print(" -o/--outfile=<file> name of xml tests results file to be generated. default: ./tests.xml ") + print(" --32bit=[yes|y|no|n] run 32-bit tests. default: yes") + print(" --64bit=[yes|y|no|n] run 64-bit tests. default: yes") + print(" --c_sim=[yes|y|no|n] run the C simulator. default: yes") + print(" --ocaml_sim=[yes|y|no|n] run the Ocaml simulator. default: no") + print(" --sailcov=[yes|y|no|n] compile and run to get Sail model coverage. default: no. ") + print(" NOTE: sets 'clean_build' to yes. Coverage is gathered seperately for ") + print(" 32 and 64 bit models") + print(" --clean_build=[yes|y|no|n] do a 'make clean' before running 32/64/c_sim/ocaml_sim set of tests. default: yes") + print(" --test_dir=<dir> directory where test elf files live. default: ./isa ./riscv-tests") + print(" --test_switch_pyfile=<file.py> a python fragment file that allows the user to pass in command line switches to the") + print(" riscv_sim command on a per-test basis. The format of the file should be:") + print(" sim_test_command_line_switch_dict = {") + print(" \"<testname_A>\" : \"<switch> [<switch> ...] \",") + print(" \"<testname_B>\" : \"<switch> [<switch> ...] \",") + print(" }") + print(" --test_ignore_pyfile=<file.py> <file> contains a tuple (immutable list) of tests to be ignored. The format of the file should be:") + print(" ignore_test_tuple = [") + print(" \"<testname_A>\",") + print(" \"<testname_B>\",") + print(" ]") + print(" -d,--debug turn on debug output") + +def process_command_line_args(opts) : + global xml_outfile + global run_32bit_tests + global run_32bit_tests + global run_64bit_tests + global run_csim + global run_ocamlsim + global sailcov + global clean_build + global test_dir_list + global test_switch_pyfile + global test_ignore_pyfile + global debug + + for opt, arg in opts : + if opt in ('-h', '--help', '-u', '--usage') : + print_usage(sys.argv[0]) + sys.exit(0) + elif opt in ('-o', "--outfile") : + xml_outfile = arg + elif opt in ('--32bit') : + if arg in ('yes', 'y') : + run_32bit_tests = True + elif arg in ('no', 'n') : + run_32bit_tests = False + else : + fatal_print("invalid argument to '--32bit' switch: " + arg) + elif opt in ('--64bit') : + if arg in ('yes', 'y') : + run_64bit_tests = True + elif arg in ('no', 'n') : + run_64bit_tests = False + else : + fatal_print("invalid argument to '--64bit' switch: " + arg) + elif opt in ('--c_sim') : + if arg in ('yes', 'y') : + run_csim = True + elif arg in ('no', 'n') : + run_csim = False + else : + fatal_print("invalid argument to '--run_csim' switch: " + arg) + elif opt in ('--ocaml_sim') : + if arg in ('yes', 'y') : + run_ocamlsim = True + elif arg in ('no', 'n') : + run_ocamlsim = False + else : + fatal_print("invalid argument to '--run_ocamlsim' switch: " + arg) + elif opt in ('--sailcov') : + if arg in ('yes', 'y') : + sailcov = True + elif arg in ('no', 'n') : + sailcov = False + else : + fatal_print("invalid argument to '--sailcov' switch: " + arg) + sys.exit(1) + elif opt in ('--clean_build') : + if arg in ('yes', 'y') : + clean_build = True + elif arg in ('no', 'n') : + clean_build = False + else : + fatal_print("invalid argument to '--run_ocamlsim' switch: " + arg) + sys.exit(1) + elif opt in ('--test_dir') : + if not os.path.exists(arg) : + fatal_print("test_dir path, '" + arg + "', does not exist") + test_dir_list = [] + test_dir_list.append(arg) + elif opt in ('--test_switch_pyfile') : + if not os.path.isfile(arg) : + fatal_print("--test_switch_pyfile argument error. file, '" + arg + "', does not exist") + test_switch_pyfile = arg + elif opt in ('--test_ignore_pyfile') : + if not os.path.isfile(arg) : + fatal_print("--test_ignore_pyfile argument error. file, '" + arg + "', does not exist") + sys.exit(1) + test_ignore_pyfile = arg + elif opt in ('-d', '--debug') : + debug = True + else : + fatal_print("unexpected command line option: " + opt) + +# print_optional_settings AFTER the inmplicit overrides have happened +def print_optional_settings() : + global xml_outfile + global run_32bit_tests + global run_32bit_tests + global run_64bit_tests + global run_csim + global run_ocamlsim + global sailcov + global clean_build + global test_dir_list + global test_switch_pyfile + global test_ignore_pyfile + global debug + + print('================================================================') + print('Run time variable settings: ') + print(' {:32}'.format('debug: ') + str(debug)) + print(' {:32}'.format('outfile: ') + xml_outfile) + print(' {:32}'.format('run_32bit_tests: ') + str(run_32bit_tests)) + print(' {:32}'.format('run_64bit_tests: ') + str(run_64bit_tests)) + print(' {:32}'.format('run_csim: ') + str(run_csim)) + print(' {:32}'.format('run_ocamlsim: ') + str(run_ocamlsim)) + print(' {:32}'.format('sailcov: ') + str(sailcov)) + print(' {:32}'.format('clean_build: ') + str(clean_build)) + print(' {:32}'.format('test_dir_list: ') + str(test_dir_list)) + print(' {:32}'.format('test_ignore_pyfile: ') + test_ignore_pyfile) + print(' {:32}'.format('ignore_test: ') + str(my_ignore_test_tuple)) + print(' {:32}'.format('test_switch_pyfile: ') + test_switch_pyfile) + print(' {:32}'.format('sim_test_comand_line_switch: ') + str(sim_test_command_line_switch)) + print('================================================================') + +# Support for command line options +# ==================================== + +# ==================================== +# Functions from run_tests.sh +def green(test_str, ok_fail_str) : + global test_pass + global SUITE_XML + global GREEN + global NC + test_pass += 1 + print(test_str + ':' + GREEN + ok_fail_str + NC) + SUITE_XML += ' <testcase name="' + test_str + '"/>\n' + +def yellow(test_str, ok_fail_str) : + global test_fail + global SUITE_XML + global YELLOW + global NC + test_fail += 1 + print(test_str + ':' + YELLOW + ok_fail_str + NC) + SUITE_XML += ' <testcase name="' + test_str + '">\n <failure message="' + ok_fail_str + '">' + ok_fail_str + '</failure>\n </testcase>\n' + +def red(test_str, ok_fail_str) : + global test_fail + global SUITE_XML + global RED + global NC + test_fail += 1 + print(test_str + ':' + RED + ok_fail_str + NC) + SUITE_XML += ' <testcase name="' + test_str + '">\n <failure message="' + ok_fail_str + '">' + ok_fail_str + '</failure>\n </testcase>\n' + +def finish_suite(suite_name) : + global test_pass + global test_fail + global all_pass + global all_fail + global SUITE_XML + global SUITES_XML + + print(suite_name + ': Passed ' + str(test_pass) + ' out of ' + str(test_pass + test_fail) + '\n\n') + date_tmp = subprocess.check_output("date", shell=True, text=True) + date = date_tmp.rstrip() + SUITES_XML += ' <testsuite name="' + suite_name + '" tests=' + str(test_pass + test_fail ) + '" failures="' + str(test_fail) + '" timestamp="' + date + '">\n' + SUITE_XML + ' </testsuite>\n' + SUITE_XML="" + all_pass += test_pass + all_fail += test_fail + test_pass = 0 + test_fail = 0 +# Functions from run_tests.sh +# ==================================== + + +# ==================================== +# Functions for determining file types +# +# TODO: there MUST be an equivalent to the 'file' command in python. +# Replace the 'file' command with a python equivalent. + +def is_elf(filename) : + cmd = "file -b " + filename + " | awk 'BEGIN { FS = \",\" } ; { print $1 } ' | grep -q \"ELF\" " + ret = os.system(cmd) + if ret == 0 : + return 1 + else : + return 0 + +def is_32bit(filename) : + cmd = "file -b " + filename + " | awk 'BEGIN { FS = \",\" } ; { print $1 } ' | grep -q \"32-bit\" " + ret = os.system(cmd) + if ret == 0 : + return 1 + else : + return 0 + +def is_64bit(filename) : + cmd = "file -b " + filename + " | awk 'BEGIN { FS = \",\" } ; { print $1 } ' | grep -q \"64-bit\" " + ret = os.system(cmd) + if ret == 0 : + return 1 + else : + return 0 + +def is_riscv(filename) : + cmd = "file -b " + filename + " | awk 'BEGIN { FS = \",\" } ; { print $2 } ' | grep -q \"RISC-V\" " + ret = os.system(cmd) + if ret == 0 : + return 1 + else : + return 0 + +def is_riscv_elf(filename) : + return is_elf(filename) and is_riscv(filename) + +def is_riscv_elf_32(filename) : + return is_riscv_elf(filename) and is_32bit(filename) + +def is_riscv_elf_64(filename) : + return is_riscv_elf(filename) and is_64bit(filename) + +def ignore_test(testname) : + for t in my_ignore_test_tuple : + debug_print("ignore testname: " + os.path.basename(testname) + " t: " + t) + if t == os.path.basename(testname) : + return True + else : + continue + return False +# Functions for determining file types +# ==================================== + +# Function prototypes +# ===========================================================================79 + +# ===========================================================================79 +# Start of execution.... + +debug_print("starting...") +debug_print("abspath to this script: " + os.path.abspath(sys.argv[0])) +debug_print("opts: " + str(opts)) + +process_command_line_args(opts) + +# ==================================== +# Implicit overrides of program varaibles +if sailcov : + clean_build = True + +if test_switch_pyfile : + exec(open(test_switch_pyfile).read()) + # check to see if variable set + if 'sim_test_command_line_switch_dict' in locals() : + sim_test_command_line_switch = sim_test_command_line_switch_dict; + else : + fatal_print("the python variable, sim_test_command_line_switch_dict, is not properly set in " + test_switch_pyfile) + sys.exit(1) + +if test_ignore_pyfile : + exec(open(test_ignore_pyfile).read()) + if 'ignore_test_tuple' in locals() : + my_ignore_test_tuple = ignore_test_tuple + else : + fatal_print("the python variable, ignore_test_tuple, is not properly set in " + test_ignore_pyfile) + +# Debug print out of important program variables +if debug : + print_optional_settings() + +# TODO: check that only 1 dir in test_dir_list exists +for d in test_dir_list : + if os.path.exists(d) : + TESTDIR = d + else : + pass + +debug_print('TESTDIR : ' + TESTDIR) + +# DIR points to the invocation directory. +DIR = os.getcwd() +SEARCH_DIR = DIR +while SEARCH_DIR != '/' : + if os.path.isfile(SEARCH_DIR + '/' + sail_riscv_rootdir) : + RISCVDIR = SEARCH_DIR + break + if SEARCH_DIR == '/' : + fatal_print("can't find root directory of repository") + SEARCH_DIR = os.path.dirname(SEARCH_DIR) +debug_print("RISCVDIR: " + RISCVDIR) + +if sailcov : + MAKE_SAILCOV = "SAILCOV=true" +else : + MAKE_SAILCOV = "" + +if os.path.isfile(DIR + xml_outfile) != False : + os.remove(DIR + xml_outfile) + +# TODO: Do you really want to run the tests from the RISCVDIR? +# TODO: check for success/failure of chdir +os.chdir(RISCVDIR) + +debug_print("DIR + '/' + TESTDIR + '/' + * :" + DIR + '/' + TESTDIR + '/' + "*") + +# Do 'make clean' to avoid cross-arch pollution. + +if clean_build : + cmd = "make ARCH=RV32 clean" + ret_val = os.system(cmd) + if ret_val != 0 : + fatal_print("non-zero exit value from command: '" + cmd + "'") + else : + pass +else : + pass + +if run_ocamlsim : + if run_32bit_tests : + print("Building 32-bit RISCV specification...") + cmd = "make ARCH=RV32 ocaml_emulator/riscv_ocaml_sim_RV32" + ret_val = os.system(cmd) + if ret_val == 0 : + green("Building 32-bit RISCV OCaml emulator", "ok") + else : + debug_print("non-zero exit value from command: '" + cmd + "'") + red("Building 32-bit RISCV OCaml emulator","fail") + +if run_32bit_tests and run_ocamlsim : + for test in glob.glob(DIR + '/' + TESTDIR + '/' + "*") : + debug_print("test: " + test) + if not is_riscv_elf_32(test) : + continue + if ignore_test(test) : + debug_print("ignoring test: " + test) + continue + # skip F/D tests on OCaml for now + pat = re.compile('(rv32ud)') + mo = pat.search(test) + if mo != None : + continue + pat = re.compile('(rv32uf)') + mo = pat.search(test) + if mo != None : + continue + outfile = test + ".out" + sim_switch = "" + for key in sim_test_command_line_switch : + pat = re.compile(key) + mo = pat.search(test) + if mo != None: + sim_switch = sim_test_command_line_switch[key] + break + cmd = "timeout 5 " + RISCVDIR + "/ocaml_emulator/riscv_ocaml_sim_RV32" + " " + sim_switch + " " + test + " > " + outfile + " 2>&1 && grep -q SUCCESS " + outfile + ret_val = os.system(cmd) + if ret_val == 0 : + green("OCaml-32 " + os.path.basename(test), "ok") + else : + red("OCaml-32 " + os.path.basename(test), "fail") +else : + pass + +finish_suite("32-bit RISCV OCaml-simulator tests") + +if clean_build : + cmd = "make ARCH=RV32 clean" + ret_val = os.system(cmd) + if ret_val != 0 : + fatal_print("non-zero exit value from command: '" + cmd + "'") + sys.exit(1) + else : + pass +else : + pass + + +print("Building 32-bit RISCV specification...") +if run_csim : + if run_32bit_tests : + cmd = "make ARCH=RV32 " + MAKE_SAILCOV + " c_emulator/riscv_sim_RV32" + ret_val = os.system(cmd) + if ret_val == 0 : + green("Building 32-bit RISCV C emulator", "ok") + else : + red("Building 32-bit RISCV C emulator","fail") + error_print("non-zero exit value from command: '" + cmd + "'") + +if run_32bit_tests and run_csim : + for test in glob.glob(DIR + '/' + TESTDIR + '/' + "*") : + if not is_riscv_elf_32(test) : + continue + if ignore_test(test) : + debug_print("ignoring test: " + test) + continue + outfile = test + ".cout" + sim_switch = "" + for key in sim_test_command_line_switch : + pat = re.compile(key) + mo = pat.search(test) + if mo != None: + sim_switch = sim_test_command_line_switch[key] + break + + if sailcov : + run_sailcov = " --sailcov-file sailcov_RV32" + else : + run_sailcov = "" + + cmd = "timeout 5 " + RISCVDIR + "/c_emulator/riscv_sim_RV32" + run_sailcov + " " + sim_switch + " " + test + " > " + outfile + " 2>&1 && grep -q SUCCESS " + outfile + debug_print("cmd: '" + cmd + "'") + ret_val = os.system(cmd) + if ret_val == 0 : + green("C-32 " + os.path.basename(test), "ok") + else : + red("C-32 " + os.path.basename(test), "fail") +else : + pass + +finish_suite("32-bit RISCV C-simulator tests") + +if clean_build : + cmd = "make ARCH=RV64 clean" + ret_val = os.system(cmd) + if ret_val != 0 : + fatal_print("non-zero exit value from command: '" + cmd + "'") + else : + pass +else : + pass + +print("Building 64-bit RISCV specification...") +if run_ocamlsim : + if run_64-bit_tests : + cmd = "make ARCH=RV64 ocaml_emulator/riscv_ocaml_sim_RV64" + ret_val = os.system(cmd) + if ret_val == 0 : + green("Building 64-bit RISCV OCaml emulator", "ok") + else : + error_print("non-zero exit value from command: '" + cmd + "'") + red("Building 64-bit RISCV OCaml emulator","fail") + +if run_64bit_tests and run_ocamlsim : + for test in glob.glob(DIR + '/' + TESTDIR + '/' + "*") : + debug_print("test: " + test) + if not is_riscv_elf_64(test) : + continue + if ignore_test(test) : + debug_print("ignoring test: " + test) + continue + # skip F/D tests on OCaml for now + pat = re.compile('(rv64ud)') + mo = pat.search(test) + if mo != None : + continue + pat = re.compile('(rv64uf)') + mo = pat.search(test) + if mo != None : + continue + outfile = test + ".out" + sim_switch = "" + for key in sim_test_command_line_switch : + pat = re.compile(key) + mo = pat.search(test) + if mo != None: + sim_switch = sim_test_command_line_switch[key] + break + cmd = "timeout 5 " + RISCVDIR + "/ocaml_emulator/riscv_ocaml_sim_RV64" + " " + sim_switch + " " + test + " > " + outfile + " 2>&1 && grep -q SUCCESS " + outfile + ret_val = os.system(cmd) + if ret_val == 0 : + green("OCaml-64 " + os.path.basename(test), "ok") + else : + red("OCaml-64 " + os.path.basename(test), "fail") +else : + pass + +finish_suite("64-bit RISCV OCaml-simulator tests") + + +if clean_build : + cmd = "make ARCH=RV64 clean" + ret_val = os.system(cmd) + if ret_val != 0 : + fatal_print("non-zero exit value from command: '" + cmd + "'") + else : + pass +else : + pass + +print("Building 64-bit RISCV specification...") +if run_csim : + if run_64bit_tests : + cmd = "make ARCH=RV64 " + MAKE_SAILCOV + " c_emulator/riscv_sim_RV64" + ret_val = os.system(cmd) + if ret_val == 0 : + green("Building 64-bit RISCV C emulator", "ok") + else : + red("Building 64-bit RISCV C emulator","fail") + error_print("non-zero exit value from command: '" + cmd + "'") + +if run_64bit_tests and run_csim : + for test in glob.glob(DIR + '/' + TESTDIR + '/' + "*") : + debug_print("test: " + test) + if not is_riscv_elf_64(test) : + continue + if ignore_test(test) : + debug_print("ignoring test: " + test) + continue + outfile = test + ".cout" + sim_switch = "" + for key in sim_test_command_line_switch : + pat = re.compile(key) + mo = pat.search(test) + if mo != None: + sim_switch = sim_test_command_line_switch[key] + break + + if sailcov : + run_sailcov = " --sailcov-file sailcov_RV64" + else : + run_sailcov = "" + + cmd = "timeout 5 " + RISCVDIR + "/c_emulator/riscv_sim_RV64" + run_sailcov + " " + sim_switch + " " + test + " > " + outfile + " 2>&1 && grep -q SUCCESS " + outfile + ret_val = os.system(cmd) + if ret_val == 0 : + green("C-64 " + os.path.basename(test), "ok") + else : + red("C-64 " + os.path.basename(test), "fail") +else : + pass + +finish_suite("64-bit RISCV C-simulator tests") + +print('Passed ' + str(all_pass) + ' out of ' + str(all_pass + all_fail) + '\n\n') +XML = '<testsuites tests="' + str(all_pass + all_fail) + '" failures="' + str(all_fail) + '">\n' + SUITES_XML + '</testsuites>\n' + +xml_outfile_fh = open(DIR + '/' + xml_outfile, 'w') +print(XML, file = xml_outfile_fh) +xml_outfile_fh.close() + +if all_fail > 0 : + sys.exit(1) +else : + sys.exit(0) |