Run ISA tests in CI (#134)

* test: Ignore generated XML output

* run_tests: Build RVFI emulators too

Can't run tests with them though as they're built for direct instruction
injection (RVFI-DII) via an instruction stream over a network socket,
not fetching instructions from memory, so this remains just a build
test.

* run_tests/run_fp_tests: Print summary and give meaningful exit code

* run_tests/run_fp_tests: Include tests and failures in top-level XML entity

* run_tests/run_fp_tests: Use failure not error for XML output

The former is the standard tag for normal test failures, the latter is
for catastrophic things like test harness errors.

* Run ISA tests in CI

3 files changed, 61 insertions, 20 deletions

diff --git a/test/.gitignore b/test/.gitignore
new file mode 100644
index 0000000..733e73b
--- /dev/null
+++ b/test/.gitignore
@@ -0,0 +1 @@
+/tests.xml
diff --git a/test/run_fp_tests.sh b/test/run_fp_tests.sh
index 91e7c80..bbc7adb 100755
--- a/test/run_fp_tests.sh
+++ b/test/run_fp_tests.sh
@@ -14,39 +14,41 @@ rm -f $DIR/tests.xml
 
 pass=0
 fail=0
-XML=""
+all_pass=0
+all_fail=0
+SUITE_XML=""
+SUITES_XML=""
 
 function green {
     (( pass += 1 ))
     printf "$1: ${GREEN}$2${NC}\n"
-    XML+="    <testcase name=\"$1\"/>\n"
+    SUITE_XML+="    <testcase name=\"$1\"/>\n"
 }
 
 function yellow {
     (( fail += 1 ))
     printf "$1: ${YELLOW}$2${NC}\n"
-    XML+="    <testcase name=\"$1\">\n      <error message=\"$2\">$2</error>\n    </testcase>\n"
+    SUITE_XML+="    <testcase name=\"$1\">\n      <failure message=\"$2\">$2</failure>\n    </testcase>\n"
 }
 
 function red {
     (( fail += 1 ))
     printf "$1: ${RED}$2${NC}\n"
-    XML+="    <testcase name=\"$1\">\n      <error message=\"$2\">$2</error>\n    </testcase>\n"
+    SUITE_XML+="    <testcase name=\"$1\">\n      <failure message=\"$2\">$2</failure>\n    </testcase>\n"
 }
 
 function finish_suite {
     printf "$1: Passed ${pass} out of $(( pass + fail ))\n\n"
-    XML="  <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$XML  </testsuite>\n"
-    printf "$XML" >> $DIR/tests.xml
-    XML=""
+    SUITES_XML+="  <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$SUITE_XML  </testsuite>\n"
+    SUITE_XML=""
+    (( all_pass += pass )) || :
+    (( all_fail += fail )) || :
     pass=0
     fail=0
 }
 
 SAILLIBDIR="$DIR/../../lib/"
 
-printf "<testsuites>\n" >> $DIR/tests.xml
-
 cd $RISCVDIR
 
 # Do 'make clean' to avoid cross-arch pollution.
@@ -85,4 +87,11 @@ for test in $DIR/riscv-tests/rv32u{f,d}*.elf $DIR/riscv-tests/rv32mi-p-csr.elf;
 done
 finish_suite "32-bit RISCV C tests"
 
-printf "</testsuites>\n" >> $DIR/tests.xml
+printf "Passed ${all_pass} out of $(( all_pass + all_fail ))\n\n"
+XML="<testsuites tests=\"$(( all_pass + all_fail ))\" failures=\"${all_fail}\">\n$SUITES_XML</testsuites>\n"
+printf "$XML" > $DIR/tests.xml
+
+if [ $all_fail -gt 0 ]
+then
+    exit 1
+fi
diff --git a/test/run_tests.sh b/test/run_tests.sh
index e2a7043..8c0f03f 100755
--- a/test/run_tests.sh
+++ b/test/run_tests.sh
@@ -14,39 +14,41 @@ rm -f $DIR/tests.xml
 
 pass=0
 fail=0
-XML=""
+all_pass=0
+all_fail=0
+SUITE_XML=""
+SUITES_XML=""
 
 function green {
     (( pass += 1 ))
     printf "$1: ${GREEN}$2${NC}\n"
-    XML+="    <testcase name=\"$1\"/>\n"
+    SUITE_XML+="    <testcase name=\"$1\"/>\n"
 }
 
 function yellow {
     (( fail += 1 ))
     printf "$1: ${YELLOW}$2${NC}\n"
-    XML+="    <testcase name=\"$1\">\n      <error message=\"$2\">$2</error>\n    </testcase>\n"
+    SUITE_XML+="    <testcase name=\"$1\">\n      <failure message=\"$2\">$2</failure>\n    </testcase>\n"
 }
 
 function red {
     (( fail += 1 ))
     printf "$1: ${RED}$2${NC}\n"
-    XML+="    <testcase name=\"$1\">\n      <error message=\"$2\">$2</error>\n    </testcase>\n"
+    SUITE_XML+="    <testcase name=\"$1\">\n      <failure message=\"$2\">$2</failure>\n    </testcase>\n"
 }
 
 function finish_suite {
     printf "$1: Passed ${pass} out of $(( pass + fail ))\n\n"
-    XML="  <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$XML  </testsuite>\n"
-    printf "$XML" >> $DIR/tests.xml
-    XML=""
+    SUITES_XML+="  <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$SUITE_XML  </testsuite>\n"
+    SUITE_XML=""
+    (( all_pass += pass )) || :
+    (( all_fail += fail )) || :
     pass=0
     fail=0
 }
 
 SAILLIBDIR="$DIR/../../lib/"
 
-printf "<testsuites>\n" >> $DIR/tests.xml
-
 cd $RISCVDIR
 
 # Do 'make clean' to avoid cross-arch pollution.
@@ -141,4 +143,33 @@ for test in $DIR/riscv-tests/rv64*.elf; do
 done
 finish_suite "64-bit RISCV C tests"
 
-printf "</testsuites>\n" >> $DIR/tests.xml
+# Do 'make clean' to avoid cross-arch pollution.
+make clean
+
+if ARCH=RV32 make c_emulator/riscv_rvfi_RV32;
+then
+    green "Building 32-bit RISCV RVFI C emulator" "ok"
+else
+    red "Building 32-bit RISCV RVFI C emulator" "fail"
+fi
+finish_suite "32-bit RISCV RVFI C tests"
+
+# Do 'make clean' to avoid cross-arch pollution.
+make clean
+
+if ARCH=RV64 make c_emulator/riscv_rvfi_RV64;
+then
+    green "Building 64-bit RISCV RVFI C emulator" "ok"
+else
+    red "Building 64-bit RISCV RVFI C emulator" "fail"
+fi
+finish_suite "64-bit RISCV RVFI C tests"
+
+printf "Passed ${all_pass} out of $(( all_pass + all_fail ))\n\n"
+XML="<testsuites tests=\"$(( all_pass + all_fail ))\" failures=\"${all_fail}\">\n$SUITES_XML</testsuites>\n"
+printf "$XML" > $DIR/tests.xml
+
+if [ $all_fail -gt 0 ]
+then
+    exit 1
+fi