diff options
author | Jessica Clarke <jrtc27@jrtc27.com> | 2022-02-16 09:40:56 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-02-16 09:40:56 +0000 |
commit | 50f4bbaa5bd397ff5a71b7b39fa543d656934b80 (patch) | |
tree | 6a9daf8f4a60cc67d55ecbb890e68895fcfe7c77 | |
parent | 1e3906f0eee16666fec67e7f221e0e2e82747052 (diff) | |
download | sail-riscv-50f4bbaa5bd397ff5a71b7b39fa543d656934b80.zip sail-riscv-50f4bbaa5bd397ff5a71b7b39fa543d656934b80.tar.gz sail-riscv-50f4bbaa5bd397ff5a71b7b39fa543d656934b80.tar.bz2 |
Run ISA tests in CI (#134)
* test: Ignore generated XML output
* run_tests: Build RVFI emulators too
Can't run tests with them though as they're built for direct instruction
injection (RVFI-DII) via an instruction stream over a network socket,
not fetching instructions from memory, so this remains just a build
test.
* run_tests/run_fp_tests: Print summary and give meaningful exit code
* run_tests/run_fp_tests: Include tests and failures in top-level XML entity
* run_tests/run_fp_tests: Use failure not error for XML output
The former is the standard tag for normal test failures, the latter is
for catastrophic things like test harness errors.
* Run ISA tests in CI
-rw-r--r-- | .github/workflows/compile.yml | 26 | ||||
-rw-r--r-- | .github/workflows/test-results.yml | 51 | ||||
-rw-r--r-- | test/.gitignore | 1 | ||||
-rwxr-xr-x | test/run_fp_tests.sh | 29 | ||||
-rwxr-xr-x | test/run_tests.sh | 51 |
5 files changed, 130 insertions, 28 deletions
diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 48bd305..8ef4ca8 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -6,10 +6,10 @@ jobs: build: runs-on: [ubuntu-18.04] steps: - - name: Install opam2 - run: | - sudo add-apt-repository -y ppa:avsm/ppa - sudo apt install -y opam zlib1g-dev pkg-config libgmp-dev z3 + - name: Add opam2 PPA + run: sudo add-apt-repository -y ppa:avsm/ppa + - name: Install packages + run: sudo apt install -y opam zlib1g-dev pkg-config libgmp-dev z3 device-tree-compiler - name: Init opam run: opam init --disable-sandboxing -y - name: Install sail @@ -18,7 +18,17 @@ jobs: uses: actions/checkout@HEAD with: submodules: true - - name: Build RV32 simulators - run: eval $(opam env) && make ARCH=RV32 -j2 csim rvfi osim - - name: Build RV64 simulators - run: eval $(opam env) && make ARCH=RV64 -j2 csim rvfi osim + - name: Build and test simulators + run: eval $(opam env) && test/run_tests.sh + - name: Upload test results + if: always() + uses: actions/upload-artifact@v2 + with: + name: tests.xml + path: test/tests.xml + - name: Upload event payload + if: always() + uses: actions/upload-artifact@v2 + with: + name: event.json + path: ${{ github.event_path }} diff --git a/.github/workflows/test-results.yml b/.github/workflows/test-results.yml new file mode 100644 index 0000000..65071cd --- /dev/null +++ b/.github/workflows/test-results.yml @@ -0,0 +1,51 @@ +name: Publish test results + +on: + workflow_run: + workflows: ["CI"] + types: + - completed + +jobs: + publish-test-results: + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion != 'skipped' + steps: + - name: Download artifacts + uses: actions/github-script@v3.1.0 + with: + script: | + var fs = require('fs'); + var artifacts = await github.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{github.event.workflow_run.id }}, + }); + var matchArtifacts = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == 'tests.xml' || artifact.name == 'event.json' + }); + var count = matchArtifacts.length; + for (var i = 0; i < count; i++) { + var matchArtifact = matchArtifacts[i]; + var download = await github.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + var name = matchArtifact.name; + var dest = name + '.zip' + fs.writeFileSync('${{github.workspace}}/' + dest, Buffer.from(download.data)); + console.log("Downloaded", name, "as", dest); + } + - name: Extract test results + run: unzip tests.xml.zip + - name: Extract event payload + run: unzip event.json.zip + - name: Publish test results + uses: EnricoMi/publish-unit-test-result-action@v1 + with: + commit: ${{ github.event.workflow_run.head_sha }} + event_file: event.json + event_name: ${{ github.event.workflow_run.event }} + files: tests.xml diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 0000000..733e73b --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +/tests.xml diff --git a/test/run_fp_tests.sh b/test/run_fp_tests.sh index 91e7c80..bbc7adb 100755 --- a/test/run_fp_tests.sh +++ b/test/run_fp_tests.sh @@ -14,39 +14,41 @@ rm -f $DIR/tests.xml pass=0 fail=0 -XML="" +all_pass=0 +all_fail=0 +SUITE_XML="" +SUITES_XML="" function green { (( pass += 1 )) printf "$1: ${GREEN}$2${NC}\n" - XML+=" <testcase name=\"$1\"/>\n" + SUITE_XML+=" <testcase name=\"$1\"/>\n" } function yellow { (( fail += 1 )) printf "$1: ${YELLOW}$2${NC}\n" - XML+=" <testcase name=\"$1\">\n <error message=\"$2\">$2</error>\n </testcase>\n" + SUITE_XML+=" <testcase name=\"$1\">\n <failure message=\"$2\">$2</failure>\n </testcase>\n" } function red { (( fail += 1 )) printf "$1: ${RED}$2${NC}\n" - XML+=" <testcase name=\"$1\">\n <error message=\"$2\">$2</error>\n </testcase>\n" + SUITE_XML+=" <testcase name=\"$1\">\n <failure message=\"$2\">$2</failure>\n </testcase>\n" } function finish_suite { printf "$1: Passed ${pass} out of $(( pass + fail ))\n\n" - XML=" <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$XML </testsuite>\n" - printf "$XML" >> $DIR/tests.xml - XML="" + SUITES_XML+=" <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$SUITE_XML </testsuite>\n" + SUITE_XML="" + (( all_pass += pass )) || : + (( all_fail += fail )) || : pass=0 fail=0 } SAILLIBDIR="$DIR/../../lib/" -printf "<testsuites>\n" >> $DIR/tests.xml - cd $RISCVDIR # Do 'make clean' to avoid cross-arch pollution. @@ -85,4 +87,11 @@ for test in $DIR/riscv-tests/rv32u{f,d}*.elf $DIR/riscv-tests/rv32mi-p-csr.elf; done finish_suite "32-bit RISCV C tests" -printf "</testsuites>\n" >> $DIR/tests.xml +printf "Passed ${all_pass} out of $(( all_pass + all_fail ))\n\n" +XML="<testsuites tests=\"$(( all_pass + all_fail ))\" failures=\"${all_fail}\">\n$SUITES_XML</testsuites>\n" +printf "$XML" > $DIR/tests.xml + +if [ $all_fail -gt 0 ] +then + exit 1 +fi diff --git a/test/run_tests.sh b/test/run_tests.sh index e2a7043..8c0f03f 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -14,39 +14,41 @@ rm -f $DIR/tests.xml pass=0 fail=0 -XML="" +all_pass=0 +all_fail=0 +SUITE_XML="" +SUITES_XML="" function green { (( pass += 1 )) printf "$1: ${GREEN}$2${NC}\n" - XML+=" <testcase name=\"$1\"/>\n" + SUITE_XML+=" <testcase name=\"$1\"/>\n" } function yellow { (( fail += 1 )) printf "$1: ${YELLOW}$2${NC}\n" - XML+=" <testcase name=\"$1\">\n <error message=\"$2\">$2</error>\n </testcase>\n" + SUITE_XML+=" <testcase name=\"$1\">\n <failure message=\"$2\">$2</failure>\n </testcase>\n" } function red { (( fail += 1 )) printf "$1: ${RED}$2${NC}\n" - XML+=" <testcase name=\"$1\">\n <error message=\"$2\">$2</error>\n </testcase>\n" + SUITE_XML+=" <testcase name=\"$1\">\n <failure message=\"$2\">$2</failure>\n </testcase>\n" } function finish_suite { printf "$1: Passed ${pass} out of $(( pass + fail ))\n\n" - XML=" <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$XML </testsuite>\n" - printf "$XML" >> $DIR/tests.xml - XML="" + SUITES_XML+=" <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$SUITE_XML </testsuite>\n" + SUITE_XML="" + (( all_pass += pass )) || : + (( all_fail += fail )) || : pass=0 fail=0 } SAILLIBDIR="$DIR/../../lib/" -printf "<testsuites>\n" >> $DIR/tests.xml - cd $RISCVDIR # Do 'make clean' to avoid cross-arch pollution. @@ -141,4 +143,33 @@ for test in $DIR/riscv-tests/rv64*.elf; do done finish_suite "64-bit RISCV C tests" -printf "</testsuites>\n" >> $DIR/tests.xml +# Do 'make clean' to avoid cross-arch pollution. +make clean + +if ARCH=RV32 make c_emulator/riscv_rvfi_RV32; +then + green "Building 32-bit RISCV RVFI C emulator" "ok" +else + red "Building 32-bit RISCV RVFI C emulator" "fail" +fi +finish_suite "32-bit RISCV RVFI C tests" + +# Do 'make clean' to avoid cross-arch pollution. +make clean + +if ARCH=RV64 make c_emulator/riscv_rvfi_RV64; +then + green "Building 64-bit RISCV RVFI C emulator" "ok" +else + red "Building 64-bit RISCV RVFI C emulator" "fail" +fi +finish_suite "64-bit RISCV RVFI C tests" + +printf "Passed ${all_pass} out of $(( all_pass + all_fail ))\n\n" +XML="<testsuites tests=\"$(( all_pass + all_fail ))\" failures=\"${all_fail}\">\n$SUITES_XML</testsuites>\n" +printf "$XML" > $DIR/tests.xml + +if [ $all_fail -gt 0 ] +then + exit 1 +fi |