Merge pull request #2863 from jon-turney/exit-status-on-exception

Verify that failing tests are failing with an error, not a python exception

10 files changed, 75 insertions, 9 deletions

diff --git a/docs/markdown/Running-Meson.md b/docs/markdown/Running-Meson.md
index 23d5e97..14b2d19 100644
--- a/docs/markdown/Running-Meson.md
+++ b/docs/markdown/Running-Meson.md
@@ -146,3 +146,9 @@ Meson has a standard command line help feature. It can be accessed
 with the following command.
 
     meson --help
+
+Exit status
+==
+
+Meson exits with status 0 if successful, 1 for problems with the command line or
+meson.build file, and 2 for internal errors.
diff --git a/man/meson.1 b/man/meson.1
index 929bc6e..4429fa2 100644
--- a/man/meson.1
+++ b/man/meson.1
@@ -202,6 +202,19 @@ show available versions of the specified project
 \fBstatus\fR
 show installed and available versions of currently used subprojects
 
+.SH EXIT STATUS
+
+.TP
+.B 0
+Successful.
+.TP
+.B 1
+Usage error, or an error parsing or executing meson.build.
+.TP
+.B 2
+Internal error.
+.TP
+
 .SH SEE ALSO
 
 http://mesonbuild.com/
diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py
index 52c670a..e553423 100644
--- a/mesonbuild/environment.py
+++ b/mesonbuild/environment.py
@@ -695,10 +695,11 @@ class Environment:
         for compiler in compilers:
             if isinstance(compiler, str):
                 compiler = [compiler]
+            arg = ['--version']
             try:
-                p, out = Popen_safe(compiler + ['--version'])[0:2]
+                p, out = Popen_safe(compiler + arg)[0:2]
             except OSError as e:
-                popen_exceptions[compiler] = e
+                popen_exceptions[' '.join(compiler + arg)] = e
                 continue
 
             version = search_version(out)
diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py
index 4bd7d7f..8041526 100644
--- a/mesonbuild/interpreter.py
+++ b/mesonbuild/interpreter.py
@@ -1504,6 +1504,8 @@ class Interpreter(InterpreterBase):
                            'test': self.func_test,
                            'vcs_tag': self.func_vcs_tag,
                            })
+        if 'MESON_UNIT_TEST' in os.environ:
+            self.funcs.update({'exception': self.func_exception})
 
     def holderify(self, item):
         if isinstance(item, list):
@@ -1984,6 +1986,11 @@ to directly access options of other subprojects.''')
         self.validate_arguments(args, 1, [str])
         raise InterpreterException('Problem encountered: ' + args[0])
 
+    @noKwargs
+    def func_exception(self, node, args, kwargs):
+        self.validate_arguments(args, 0, [])
+        raise Exception()
+
     def detect_compilers(self, lang, need_cross_compiler):
         cross_comp = None
         if lang == 'c':
diff --git a/mesonbuild/interpreterbase.py b/mesonbuild/interpreterbase.py
index 6618dc8..0539b14 100644
--- a/mesonbuild/interpreterbase.py
+++ b/mesonbuild/interpreterbase.py
@@ -267,9 +267,8 @@ class InterpreterBase:
 
     def validate_comparison_types(self, val1, val2):
         if type(val1) != type(val2):
-            mlog.warning('''Trying to compare values of different types ({}, {}).
-The result of this is undefined and will become a hard error
-in a future Meson release.'''.format(type(val1).__name__, type(val2).__name__))
+            return False
+        return True
 
     def evaluate_comparison(self, node):
         val1 = self.evaluate_statement(node.left)
@@ -278,11 +277,23 @@ in a future Meson release.'''.format(type(val1).__name__, type(val2).__name__))
         val2 = self.evaluate_statement(node.right)
         if is_disabler(val2):
             return val2
-        self.validate_comparison_types(val1, val2)
+        valid = self.validate_comparison_types(val1, val2)
+        # Ordering comparisons of different types isn't allowed since PR #1810
+        # (0.41.0).  Since PR #2884 we also warn about equality comparisons of
+        # different types, which will one day become an error.
+        if not valid and (node.ctype == '==' or node.ctype == '!='):
+            mlog.warning('''Trying to compare values of different types ({}, {}) using {}.
+The result of this is undefined and will become a hard error in a future Meson release.'''
+                         .format(type(val1).__name__, type(val2).__name__, node.ctype), location=node)
         if node.ctype == '==':
             return val1 == val2
         elif node.ctype == '!=':
             return val1 != val2
+        elif not valid:
+            raise InterpreterException(
+                'Values of different types ({}, {}) cannot be compared using {}.'.format(type(val1).__name__,
+                                                                                         type(val2).__name__,
+                                                                                         node.ctype))
         elif not self.is_elementary_type(val1):
             raise InterpreterException('{} can only be compared for equality.'.format(node.left.value))
         elif not self.is_elementary_type(val2):
diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py
index 073e505..7966d70 100644
--- a/mesonbuild/mesonmain.py
+++ b/mesonbuild/mesonmain.py
@@ -376,11 +376,12 @@ def run(original_args, mainfile=None):
             mlog.log("\nA full log can be found at", mlog.bold(logfile))
             if os.environ.get('MESON_FORCE_BACKTRACE'):
                 raise
+            return 1
         else:
             if os.environ.get('MESON_FORCE_BACKTRACE'):
                 raise
             traceback.print_exc()
-        return 1
+            return 2
     finally:
         mlog.shutdown()
 
diff --git a/run_project_tests.py b/run_project_tests.py
index 1d17000..9bbbbd0 100755
--- a/run_project_tests.py
+++ b/run_project_tests.py
@@ -321,9 +321,12 @@ def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler, backen
         mesonlog = no_meson_log_msg
     gen_time = time.time() - gen_start
     if should_fail == 'meson':
-        if returncode != 0:
+        if returncode == 1:
             return TestResult('', BuildStep.configure, stdo, stde, mesonlog, gen_time)
-        return TestResult('Test that should have failed succeeded', BuildStep.configure, stdo, stde, mesonlog, gen_time)
+        elif returncode != 0:
+            return TestResult('Test exited with unexpected status {}'.format(returncode), BuildStep.configure, stdo, stde, mesonlog, gen_time)
+        else:
+            return TestResult('Test that should have failed succeeded', BuildStep.configure, stdo, stde, mesonlog, gen_time)
     if returncode != 0:
         return TestResult('Generating the build system failed.', BuildStep.configure, stdo, stde, mesonlog, gen_time)
     # Touch the meson.build file to force a regenerate so we can test that
diff --git a/run_unittests.py b/run_unittests.py
index 103847a..69f467b 100755
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -1933,6 +1933,17 @@ class FailureTests(BasePlatformTests):
         self.assertRegex(out, r'Also couldn\'t find a fallback subproject in '
                          '.*subprojects.*failingsubproj.*for the dependency.*somedep')
 
+    def test_exception_exit_status(self):
+        '''
+        Test exit status on python exception
+        '''
+        tdir = os.path.join(self.unit_test_dir, '21 exit status')
+        os.environ['MESON_UNIT_TEST'] = '1'
+        with self.assertRaises(subprocess.CalledProcessError) as cm:
+            self.init(tdir, inprocess=False)
+        self.assertEqual(cm.exception.returncode, 2)
+        self.wipe()
+
 
 class WindowsTests(BasePlatformTests):
     '''
diff --git a/test cases/common/19 comparison/meson.build b/test cases/common/19 comparison/meson.build
index c4cff9f..fb641ed 100644
--- a/test cases/common/19 comparison/meson.build
+++ b/test cases/common/19 comparison/meson.build
@@ -126,3 +126,14 @@ test('equalfalse', exe13)
 test('equaltrue', exe14)
 test('nequaltrue', exe15)
 test('nequalfalse', exe16)
+
+# Equality comparisons of different elementary types
+# (these all cause warnings currently, will become an error in future)
+
+assert([] != 'st', 'not equal')
+assert([] != 1, 'not equal')
+assert(2 != 'st', 'not equal')
+
+assert(not ([] == 'st'), 'not equal')
+assert(not ([] == 1), 'not equal')
+assert(not (2 == 'st'), 'not equal')
diff --git a/test cases/unit/21 exit status/meson.build b/test cases/unit/21 exit status/meson.build
new file mode 100644
index 0000000..4f5485b
--- /dev/null
+++ b/test cases/unit/21 exit status/meson.build
@@ -0,0 +1,2 @@
+project('exit status')
+exception()