Verify that failing tests are failing with an error, not a python exception

PR #2527 suggests "making failing tests more strict about failing
gracefully".

To achive this, make meson exit with distinct exit statuses for meson errors
and python exceptions, and check the exit status is as expected for failing
tests.

I can't see how to write a test for this, within the current framework.

You can test this change by reverting the fix (but not the test) from commit
1a159db8 and verifying that 'test cases/failing/66 string as link target'
fails.

4 files changed, 26 insertions, 3 deletions

diff --git a/docs/markdown/Running-Meson.md b/docs/markdown/Running-Meson.md
index 23d5e97..14b2d19 100644
--- a/docs/markdown/Running-Meson.md
+++ b/docs/markdown/Running-Meson.md
@@ -146,3 +146,9 @@ Meson has a standard command line help feature. It can be accessed
 with the following command.
 
     meson --help
+
+Exit status
+==
+
+Meson exits with status 0 if successful, 1 for problems with the command line or
+meson.build file, and 2 for internal errors.
diff --git a/man/meson.1 b/man/meson.1
index 929bc6e..4429fa2 100644
--- a/man/meson.1
+++ b/man/meson.1
@@ -202,6 +202,19 @@ show available versions of the specified project
 \fBstatus\fR
 show installed and available versions of currently used subprojects
 
+.SH EXIT STATUS
+
+.TP
+.B 0
+Successful.
+.TP
+.B 1
+Usage error, or an error parsing or executing meson.build.
+.TP
+.B 2
+Internal error.
+.TP
+
 .SH SEE ALSO
 
 http://mesonbuild.com/
diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py
index 073e505..7966d70 100644
--- a/mesonbuild/mesonmain.py
+++ b/mesonbuild/mesonmain.py
@@ -376,11 +376,12 @@ def run(original_args, mainfile=None):
             mlog.log("\nA full log can be found at", mlog.bold(logfile))
             if os.environ.get('MESON_FORCE_BACKTRACE'):
                 raise
+            return 1
         else:
             if os.environ.get('MESON_FORCE_BACKTRACE'):
                 raise
             traceback.print_exc()
-        return 1
+            return 2
     finally:
         mlog.shutdown()
 
diff --git a/run_project_tests.py b/run_project_tests.py
index 1d17000..9bbbbd0 100755
--- a/run_project_tests.py
+++ b/run_project_tests.py
@@ -321,9 +321,12 @@ def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler, backen
         mesonlog = no_meson_log_msg
     gen_time = time.time() - gen_start
     if should_fail == 'meson':
-        if returncode != 0:
+        if returncode == 1:
             return TestResult('', BuildStep.configure, stdo, stde, mesonlog, gen_time)
-        return TestResult('Test that should have failed succeeded', BuildStep.configure, stdo, stde, mesonlog, gen_time)
+        elif returncode != 0:
+            return TestResult('Test exited with unexpected status {}'.format(returncode), BuildStep.configure, stdo, stde, mesonlog, gen_time)
+        else:
+            return TestResult('Test that should have failed succeeded', BuildStep.configure, stdo, stde, mesonlog, gen_time)
     if returncode != 0:
         return TestResult('Generating the build system failed.', BuildStep.configure, stdo, stde, mesonlog, gen_time)
     # Touch the meson.build file to force a regenerate so we can test that