From 2a64ed855e6db81c3df4659e1066f777b9bacc8c Mon Sep 17 00:00:00 2001 From: Jon Turney Date: Fri, 27 Oct 2017 14:39:10 +0100 Subject: Verify that failing tests are failing with an error, not a python exception PR #2527 suggests "making failing tests more strict about failing gracefully". To achive this, make meson exit with distinct exit statuses for meson errors and python exceptions, and check the exit status is as expected for failing tests. I can't see how to write a test for this, within the current framework. You can test this change by reverting the fix (but not the test) from commit 1a159db8 and verifying that 'test cases/failing/66 string as link target' fails. --- run_project_tests.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'run_project_tests.py') diff --git a/run_project_tests.py b/run_project_tests.py index 1d17000..9bbbbd0 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -321,9 +321,12 @@ def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler, backen mesonlog = no_meson_log_msg gen_time = time.time() - gen_start if should_fail == 'meson': - if returncode != 0: + if returncode == 1: return TestResult('', BuildStep.configure, stdo, stde, mesonlog, gen_time) - return TestResult('Test that should have failed succeeded', BuildStep.configure, stdo, stde, mesonlog, gen_time) + elif returncode != 0: + return TestResult('Test exited with unexpected status {}'.format(returncode), BuildStep.configure, stdo, stde, mesonlog, gen_time) + else: + return TestResult('Test that should have failed succeeded', BuildStep.configure, stdo, stde, mesonlog, gen_time) if returncode != 0: return TestResult('Generating the build system failed.', BuildStep.configure, stdo, stde, mesonlog, gen_time) # Touch the meson.build file to force a regenerate so we can test that -- cgit v1.1