Merge pull request #3424 from NickeZ/nickez/fail-fast

Add option to fail fast in tests

5 files changed, 127 insertions, 83 deletions

diff --git a/run_cross_test.py b/run_cross_test.py
index 7191402..6e04fa2 100755
--- a/run_cross_test.py
+++ b/run_cross_test.py
@@ -25,14 +25,16 @@ Eventually migrate to something fancier.'''
 import sys
 import os
 from pathlib import Path
+import argparse
 
 from run_project_tests import gather_tests, run_tests, StopException, setup_commands
 from run_project_tests import failing_logs
 
-def runtests(cross_file):
+def runtests(cross_file, failfast):
     commontests = [('common', gather_tests(Path('test cases', 'common')), False)]
     try:
-        (passing_tests, failing_tests, skipped_tests) = run_tests(commontests, 'meson-cross-test-run', ['--cross', cross_file])
+        (passing_tests, failing_tests, skipped_tests) = \
+            run_tests(commontests, 'meson-cross-test-run', failfast, ['--cross', cross_file])
     except StopException:
         pass
     print('\nTotal passed cross tests:', passing_tests)
@@ -40,11 +42,17 @@ def runtests(cross_file):
     print('Total skipped cross tests:', skipped_tests)
     if failing_tests > 0 and ('TRAVIS' in os.environ or 'APPVEYOR' in os.environ):
         print('\nMesonlogs of failing tests\n')
-        for l in failing_logs:
-            print(l, '\n')
-    sys.exit(failing_tests)
+        for log in failing_logs:
+            print(log, '\n')
+    return failing_tests
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--failfast', action='store_true')
+    parser.add_argument('cross_file')
+    options = parser.parse_args()
+    setup_commands('ninja')
+    return runtests(options.cross_file, options.failfast)
 
 if __name__ == '__main__':
-    setup_commands('ninja')
-    cross_file = sys.argv[1]
-    runtests(cross_file)
+    sys.exit(main())
diff --git a/run_meson_command_tests.py b/run_meson_command_tests.py
index fd33856..390868a 100755
--- a/run_meson_command_tests.py
+++ b/run_meson_command_tests.py
@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
 import os
 import tempfile
 import unittest
@@ -23,11 +24,6 @@ from pathlib import Path
 
 from mesonbuild.mesonlib import windows_proof_rmtree, python_command, is_windows
 
-# Find the meson.py adjacent to us
-meson_py = Path(__file__).resolve().parent / 'meson.py'
-if not meson_py.is_file():
-    raise RuntimeError("meson.py not found: test must only run from git")
-
 def get_pypath():
     import sysconfig
     pypath = sysconfig.get_path('purelib', vars={'base': ''})
@@ -176,8 +172,7 @@ class CommandTests(unittest.TestCase):
         builddir = str(self.tmpdir / 'build4')
         (bindir / 'meson').rename(bindir / 'meson.real')
         wrapper = (bindir / 'meson')
-        with open(str(wrapper), 'w') as f:
-            f.write('#!/bin/sh\n\nmeson.real "$@"')
+        wrapper.open('w').write('#!/bin/sh\n\nmeson.real "$@"')
         wrapper.chmod(0o755)
         meson_setup = [str(wrapper), 'setup']
         meson_command = meson_setup + self.meson_args
@@ -195,5 +190,6 @@ class CommandTests(unittest.TestCase):
         zipapp.create_archive(source=source, target=target, interpreter=python_command[0], main=None)
         self._run([target.as_posix(), '--help'])
 
+
 if __name__ == '__main__':
-    unittest.main(buffer=True)
+    sys.exit(unittest.main(buffer=True))
diff --git a/run_project_tests.py b/run_project_tests.py
index 876d135..6474185 100755
--- a/run_project_tests.py
+++ b/run_project_tests.py
@@ -36,11 +36,12 @@ import argparse
 import xml.etree.ElementTree as ET
 import time
 import multiprocessing
-from concurrent.futures import ProcessPoolExecutor
+from concurrent.futures import ProcessPoolExecutor, CancelledError
 import re
 from run_tests import get_fake_options, run_configure, get_meson_script
 from run_tests import get_backend_commands, get_backend_args_for_dir, Backend
 from run_tests import ensure_backend_detects_changes
+from run_tests import guess_backend
 
 
 class BuildStep(Enum):
@@ -101,26 +102,7 @@ signal.signal(signal.SIGTERM, stop_handler)
 def setup_commands(optbackend):
     global do_debug, backend, backend_flags
     global compile_commands, clean_commands, test_commands, install_commands, uninstall_commands
-    backend = optbackend
-    msbuild_exe = shutil.which('msbuild')
-    # Auto-detect backend if unspecified
-    if backend is None:
-        if msbuild_exe is not None:
-            backend = 'vs' # Meson will auto-detect VS version to use
-        else:
-            backend = 'ninja'
-    # Set backend arguments for Meson
-    if backend.startswith('vs'):
-        backend_flags = ['--backend=' + backend]
-        backend = Backend.vs
-    elif backend == 'xcode':
-        backend_flags = ['--backend=xcode']
-        backend = Backend.xcode
-    elif backend == 'ninja':
-        backend_flags = ['--backend=ninja']
-        backend = Backend.ninja
-    else:
-        raise RuntimeError('Unknown backend: {!r}'.format(backend))
+    backend, backend_flags = guess_backend(optbackend, shutil.which('msbuild'))
     compile_commands, clean_commands, test_commands, install_commands, \
         uninstall_commands = get_backend_commands(backend, do_debug)
 
@@ -523,14 +505,14 @@ def detect_tests_to_run():
     gathered_tests = [(name, gather_tests(Path('test cases', subdir)), skip) for name, subdir, skip in all_tests]
     return gathered_tests
 
-def run_tests(all_tests, log_name_base, extra_args):
+def run_tests(all_tests, log_name_base, failfast, extra_args):
     global logfile
     txtname = log_name_base + '.txt'
     with open(txtname, 'w', encoding='utf-8', errors='ignore') as lf:
         logfile = lf
-        return _run_tests(all_tests, log_name_base, extra_args)
+        return _run_tests(all_tests, log_name_base, failfast, extra_args)
 
-def _run_tests(all_tests, log_name_base, extra_args):
+def _run_tests(all_tests, log_name_base, failfast, extra_args):
     global stop, executor, futures, system_compiler
     xmlname = log_name_base + '.xml'
     junit_root = ET.Element('testsuites')
@@ -578,7 +560,10 @@ def _run_tests(all_tests, log_name_base, extra_args):
             futures.append((testname, t, result))
         for (testname, t, result) in futures:
             sys.stdout.flush()
-            result = result.result()
+            try:
+                result = result.result()
+            except CancelledError:
+                continue
             if (result is None) or (('MESON_SKIP_TEST' in result.stdo) and (skippable(name, t.as_posix()))):
                 print(yellow('Skipping:'), t.as_posix())
                 current_test = ET.SubElement(current_suite, 'testcase', {'name': testname,
@@ -599,6 +584,10 @@ def _run_tests(all_tests, log_name_base, extra_args):
                     else:
                         failing_logs.append(result.stdo)
                     failing_logs.append(result.stde)
+                    if failfast:
+                        print("Cancelling the rest of the tests")
+                        for (_, _, res) in futures:
+                            res.cancel()
                 else:
                     print('Succeeded test%s: %s' % (without_install, t.as_posix()))
                     passing_tests += 1
@@ -616,6 +605,10 @@ def _run_tests(all_tests, log_name_base, extra_args):
                 stdoel.text = result.stdo
                 stdeel = ET.SubElement(current_test, 'system-err')
                 stdeel.text = result.stde
+
+        if failfast and failing_tests > 0:
+            break
+
     print("\nTotal configuration time: %.2fs" % conf_time)
     print("Total build time: %.2fs" % build_time)
     print("Total test time: %.2fs" % test_time)
@@ -709,6 +702,8 @@ if __name__ == '__main__':
                         help='arguments that are passed directly to Meson (remember to have -- before these).')
     parser.add_argument('--backend', default=None, dest='backend',
                         choices=backendlist)
+    parser.add_argument('--failfast', action='store_true',
+                        help='Stop running if test case fails')
     options = parser.parse_args()
     setup_commands(options.backend)
 
@@ -720,7 +715,7 @@ if __name__ == '__main__':
     check_meson_commands_work()
     try:
         all_tests = detect_tests_to_run()
-        (passing_tests, failing_tests, skipped_tests) = run_tests(all_tests, 'meson-test-run', options.extra_args)
+        (passing_tests, failing_tests, skipped_tests) = run_tests(all_tests, 'meson-test-run', options.failfast, options.extra_args)
     except StopException:
         pass
     print('\nTotal passed tests:', green(str(passing_tests)))
diff --git a/run_tests.py b/run_tests.py
index af926ea..1e03fa7 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -21,17 +21,40 @@ import shutil
 import subprocess
 import tempfile
 import platform
+import argparse
 from io import StringIO
 from enum import Enum
 from glob import glob
 from pathlib import Path
-
 import mesonbuild
 from mesonbuild import mesonlib
 from mesonbuild import mesonmain
 from mesonbuild import mtest
 from mesonbuild import mlog
 from mesonbuild.environment import Environment, detect_ninja
+from mesonbuild.coredata import backendlist
+
+def guess_backend(backend, msbuild_exe):
+    # Auto-detect backend if unspecified
+    backend_flags = []
+    if backend is None:
+        if msbuild_exe is not None:
+            backend = 'vs' # Meson will auto-detect VS version to use
+        else:
+            backend = 'ninja'
+    # Set backend arguments for Meson
+    if backend.startswith('vs'):
+        backend_flags = ['--backend=' + backend]
+        backend = Backend.vs
+    elif backend == 'xcode':
+        backend_flags = ['--backend=xcode']
+        backend = Backend.xcode
+    elif backend == 'ninja':
+        backend_flags = ['--backend=ninja']
+        backend = Backend.ninja
+    else:
+        raise RuntimeError('Unknown backend: {!r}'.format(backend))
+    return (backend, backend_flags)
 
 
 # Fake classes and objects for mocking
@@ -106,9 +129,9 @@ def find_vcxproj_with_target(builddir, target):
     import re, fnmatch
     t, ext = os.path.splitext(target)
     if ext:
-        p = '<TargetName>{}</TargetName>\s*<TargetExt>\{}</TargetExt>'.format(t, ext)
+        p = r'<TargetName>{}</TargetName>\s*<TargetExt>\{}</TargetExt>'.format(t, ext)
     else:
-        p = '<TargetName>{}</TargetName>'.format(t)
+        p = r'<TargetName>{}</TargetName>'.format(t)
     for root, dirs, files in os.walk(builddir):
         for f in fnmatch.filter(files, '*.vcxproj'):
             f = os.path.join(builddir, f)
@@ -218,32 +241,25 @@ def print_system_info():
     print('System:', platform.system())
     print('')
 
-if __name__ == '__main__':
+def main():
     print_system_info()
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cov', action='store_true')
+    parser.add_argument('--backend', default=None, dest='backend',
+                        choices=backendlist)
+    parser.add_argument('--cross', default=False, dest='cross', action='store_true')
+    parser.add_argument('--failfast', action='store_true')
+    (options, _) = parser.parse_known_args()
     # Enable coverage early...
-    enable_coverage = '--cov' in sys.argv
+    enable_coverage = options.cov
     if enable_coverage:
         os.makedirs('.coverage', exist_ok=True)
         sys.argv.remove('--cov')
         import coverage
         coverage.process_startup()
     returncode = 0
-    # Iterate over list in reverse order to find the last --backend arg
-    backend = Backend.ninja
-    cross = False
-    # FIXME: PLEASE convert to argparse
-    for arg in reversed(sys.argv[1:]):
-        if arg.startswith('--backend'):
-            if arg.startswith('--backend=vs'):
-                backend = Backend.vs
-            elif arg == '--backend=xcode':
-                backend = Backend.xcode
-        if arg.startswith('--cross'):
-            cross = True
-            if arg == '--cross=mingw':
-                cross = 'mingw'
-            elif arg == '--cross=arm':
-                cross = 'arm'
+    cross = options.cross
+    backend, _ = guess_backend(options.backend, shutil.which('msbuild'))
     # Running on a developer machine? Be nice!
     if not mesonlib.is_windows() and not mesonlib.is_haiku() and 'TRAVIS' not in os.environ:
         os.nice(20)
@@ -267,26 +283,50 @@ if __name__ == '__main__':
     # Can't pass arguments to unit tests, so set the backend to use in the environment
     env = os.environ.copy()
     env['MESON_UNIT_TEST_BACKEND'] = backend.name
-    with tempfile.TemporaryDirectory() as td:
+    with tempfile.TemporaryDirectory() as temp_dir:
         # Enable coverage on all subsequent processes.
         if enable_coverage:
-            with open(os.path.join(td, 'usercustomize.py'), 'w') as f:
-                f.write('import coverage\n'
-                        'coverage.process_startup()\n')
+            Path(temp_dir, 'usercustomize.py').open('w').write(
+                'import coverage\n'
+                'coverage.process_startup()\n')
             env['COVERAGE_PROCESS_START'] = '.coveragerc'
-            env['PYTHONPATH'] = os.pathsep.join([td] + env.get('PYTHONPATH', []))
+            if 'PYTHONPATH' in env:
+                env['PYTHONPATH'] = os.pathsep.join([temp_dir, env.get('PYTHONPATH')])
+            else:
+                env['PYTHONPATH'] = temp_dir
         if not cross:
-            returncode += subprocess.call(mesonlib.python_command + ['run_meson_command_tests.py', '-v'], env=env)
-            returncode += subprocess.call(mesonlib.python_command + ['run_unittests.py', '-v'], env=env)
-            returncode += subprocess.call(mesonlib.python_command + ['run_project_tests.py'] + sys.argv[1:], env=env)
+            cmd = mesonlib.python_command + ['run_meson_command_tests.py', '-v']
+            if options.failfast:
+                cmd += ['--failfast']
+            returncode += subprocess.call(cmd, env=env)
+            if options.failfast and returncode != 0:
+                return returncode
+            cmd = mesonlib.python_command + ['run_unittests.py', '-v']
+            if options.failfast:
+                cmd += ['--failfast']
+            returncode += subprocess.call(cmd, env=env)
+            if options.failfast and returncode != 0:
+                return returncode
+            cmd = mesonlib.python_command + ['run_project_tests.py'] + sys.argv[1:]
+            returncode += subprocess.call(cmd, env=env)
         else:
             cross_test_args = mesonlib.python_command + ['run_cross_test.py']
-            if cross is True or cross == 'arm':
-                print(mlog.bold('Running armhf cross tests.').get_text(mlog.colorize_console))
-                print()
-                returncode += subprocess.call(cross_test_args + ['cross/ubuntu-armhf.txt'], env=env)
-            if cross is True or cross == 'mingw':
-                print(mlog.bold('Running mingw-w64 64-bit cross tests.').get_text(mlog.colorize_console))
-                print()
-                returncode += subprocess.call(cross_test_args + ['cross/linux-mingw-w64-64bit.txt'], env=env)
-    sys.exit(returncode)
+            print(mlog.bold('Running armhf cross tests.').get_text(mlog.colorize_console))
+            print()
+            cmd = cross_test_args + ['cross/ubuntu-armhf.txt']
+            if options.failfast:
+                cmd += ['--failfast']
+            returncode += subprocess.call(cmd, env=env)
+            if options.failfast and returncode != 0:
+                return returncode
+            print(mlog.bold('Running mingw-w64 64-bit cross tests.')
+                  .get_text(mlog.colorize_console))
+            print()
+            cmd = cross_test_args + ['cross/linux-mingw-w64-64bit.txt']
+            if options.failfast:
+                cmd += ['--failfast']
+            returncode += subprocess.call(cmd, env=env)
+    return returncode
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/run_unittests.py b/run_unittests.py
index 9ffce52..9b7e45a 100755
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -16,11 +16,13 @@
 import stat
 import shlex
 import subprocess
-import re, json
+import re
+import json
 import tempfile
 import textwrap
 import os
 import shutil
+import sys
 import unittest
 import platform
 from itertools import chain
@@ -4329,7 +4331,7 @@ def should_run_cross_arm_tests():
 def should_run_cross_mingw_tests():
     return shutil.which('x86_64-w64-mingw32-gcc') and not (is_windows() or is_cygwin())
 
-if __name__ == '__main__':
+def main():
     unset_envs()
     cases = ['InternalTests', 'DataTests', 'AllPlatformTests', 'FailureTests', 'PythonTests']
     if not is_windows():
@@ -4343,4 +4345,7 @@ if __name__ == '__main__':
     if is_osx():
         cases += ['DarwinTests']
 
-    unittest.main(defaultTest=cases, buffer=True)
+    return unittest.main(defaultTest=cases, buffer=True)
+
+if __name__ == '__main__':
+    sys.exit(main())