Merge pull request #8389 from dcbaker/submit/single-test-runner

Add a script to run a single meson functional test case (with test.json support)

4 files changed, 244 insertions, 129 deletions

diff --git a/run_mypy.py b/run_mypy.py
index e6900c7..1c886f8 100755
--- a/run_mypy.py
+++ b/run_mypy.py
@@ -39,6 +39,7 @@ modules = [
     'mesonbuild/optinterpreter.py',
 
     'run_mypy.py',
+    'run_single_test.py',
     'tools'
 ]
 
diff --git a/run_project_tests.py b/run_project_tests.py
index 14f135a..b6879ee 100755
--- a/run_project_tests.py
+++ b/run_project_tests.py
@@ -461,6 +461,8 @@ def create_deterministic_builddir(test: TestDef, use_tmpdir: bool) -> str:
         src_dir += test.name
     rel_dirname = 'b ' + hashlib.sha256(src_dir.encode(errors='ignore')).hexdigest()[0:10]
     abs_pathname = os.path.join(tempfile.gettempdir() if use_tmpdir else os.getcwd(), rel_dirname)
+    if os.path.exists(abs_pathname):
+        mesonlib.windows_proof_rmtree(abs_pathname)
     os.mkdir(abs_pathname)
     return abs_pathname
 
@@ -474,7 +476,7 @@ def format_parameter_file(file_basename: str, test: TestDef, test_build_dir: str
 
     return destination
 
-def detect_parameter_files(test: TestDef, test_build_dir: str) -> (Path, Path):
+def detect_parameter_files(test: TestDef, test_build_dir: str) -> T.Tuple[Path, Path]:
     nativefile = test.path / 'nativefile.ini'
     crossfile = test.path / 'crossfile.ini'
 
@@ -486,7 +488,9 @@ def detect_parameter_files(test: TestDef, test_build_dir: str) -> (Path, Path):
 
     return nativefile, crossfile
 
-def run_test(test: TestDef, extra_args, compiler, backend, flags, commands, should_fail, use_tmp: bool):
+def run_test(test: TestDef, extra_args: T.List[str], compiler: str, backend: Backend,
+            flags: T.List[str], commands: T.Tuple[T.List[str], T.List[str], T.List[str], T.List[str]],
+            should_fail: bool, use_tmp: bool) -> T.Optional[TestResult]:
     if test.skip:
         return None
     build_dir = create_deterministic_builddir(test, use_tmp)
@@ -501,7 +505,10 @@ def run_test(test: TestDef, extra_args, compiler, backend, flags, commands, shou
     finally:
         mesonlib.windows_proof_rmtree(build_dir)
 
-def _run_test(test: TestDef, test_build_dir: str, install_dir: str, extra_args, compiler, backend, flags, commands, should_fail):
+def _run_test(test: TestDef, test_build_dir: str, install_dir: str,
+              extra_args: T.List[str], compiler: str, backend: Backend,
+              flags: T.List[str], commands: T.Tuple[T.List[str], T.List[str], T.List[str], T.List[str]],
+              should_fail: bool) -> TestResult:
     compile_commands, clean_commands, install_commands, uninstall_commands = commands
     gen_start = time.time()
     # Configure in-process
@@ -620,130 +627,140 @@ def _run_test(test: TestDef, test_build_dir: str, install_dir: str, extra_args,
 
     return testresult
 
-def gather_tests(testdir: Path, stdout_mandatory: bool) -> T.List[TestDef]:
-    tests = [t.name for t in testdir.iterdir() if t.is_dir()]
-    tests = [t for t in tests if not t.startswith('.')]  # Filter non-tests files (dot files, etc)
-    test_defs = [TestDef(testdir / t, None, []) for t in tests]
-    all_tests = []  # type: T.List[TestDef]
-    for t in test_defs:
-        test_def = {}
-        test_def_file = t.path / 'test.json'
-        if test_def_file.is_file():
-            test_def = json.loads(test_def_file.read_text())
-
-        # Handle additional environment variables
-        env = {}  # type: T.Dict[str, str]
-        if 'env' in test_def:
-            assert isinstance(test_def['env'], dict)
-            env = test_def['env']
-            for key, val in env.items():
-                val = val.replace('@ROOT@', t.path.resolve().as_posix())
-                val = val.replace('@PATH@', t.env.get('PATH', ''))
-                env[key] = val
-
-        # Handle installed files
-        installed = []  # type: T.List[InstalledFile]
-        if 'installed' in test_def:
-            installed = [InstalledFile(x) for x in test_def['installed']]
-
-        # Handle expected output
-        stdout = test_def.get('stdout', [])
-        if stdout_mandatory and not stdout:
-            raise RuntimeError("{} must contain a non-empty stdout key".format(test_def_file))
-
-        # Handle the do_not_set_opts list
-        do_not_set_opts = test_def.get('do_not_set_opts', [])  # type: T.List[str]
-
-        # Skip tests if the tool requirements are not met
-        if 'tools' in test_def:
-            assert isinstance(test_def['tools'], dict)
-            for tool, vers_req in test_def['tools'].items():
-                if tool not in tool_vers_map:
-                    t.skip = True
-                elif not mesonlib.version_compare(tool_vers_map[tool], vers_req):
-                    t.skip = True
-
-        # Skip the matrix code and just update the existing test
-        if 'matrix' not in test_def:
-            t.env.update(env)
-            t.installed_files = installed
-            t.do_not_set_opts = do_not_set_opts
-            t.stdout = stdout
-            all_tests += [t]
-            continue
 
-        # 'matrix; entry is present, so build multiple tests from matrix definition
-        opt_list = []  # type: T.List[T.List[T.Tuple[str, bool]]]
-        matrix = test_def['matrix']
-        assert "options" in matrix
-        for key, val in matrix["options"].items():
-            assert isinstance(val, list)
-            tmp_opts = []  # type: T.List[T.Tuple[str, bool]]
-            for i in val:
-                assert isinstance(i, dict)
-                assert "val" in i
-                skip = False
-
-                # Skip the matrix entry if environment variable is present
-                if 'skip_on_env' in i:
-                    for skip_env_var in i['skip_on_env']:
-                        if skip_env_var in os.environ:
-                            skip = True
-
-                # Only run the test if all compiler ID's match
-                if 'compilers' in i:
-                    for lang, id_list in i['compilers'].items():
-                        if lang not in compiler_id_map or compiler_id_map[lang] not in id_list:
-                            skip = True
-                            break
-
-                # Add an empty matrix entry
-                if i['val'] is None:
-                    tmp_opts += [(None, skip)]
-                    continue
+def load_test_json(t: TestDef, stdout_mandatory: bool) -> T.List[TestDef]:
+    all_tests: T.List[TestDef] = []
+    test_def = {}
+    test_def_file = t.path / 'test.json'
+    if test_def_file.is_file():
+        test_def = json.loads(test_def_file.read_text())
+
+    # Handle additional environment variables
+    env = {}  # type: T.Dict[str, str]
+    if 'env' in test_def:
+        assert isinstance(test_def['env'], dict)
+        env = test_def['env']
+        for key, val in env.items():
+            val = val.replace('@ROOT@', t.path.resolve().as_posix())
+            val = val.replace('@PATH@', t.env.get('PATH', ''))
+            env[key] = val
+
+    # Handle installed files
+    installed = []  # type: T.List[InstalledFile]
+    if 'installed' in test_def:
+        installed = [InstalledFile(x) for x in test_def['installed']]
+
+    # Handle expected output
+    stdout = test_def.get('stdout', [])
+    if stdout_mandatory and not stdout:
+        raise RuntimeError("{} must contain a non-empty stdout key".format(test_def_file))
+
+    # Handle the do_not_set_opts list
+    do_not_set_opts = test_def.get('do_not_set_opts', [])  # type: T.List[str]
+
+    # Skip tests if the tool requirements are not met
+    if 'tools' in test_def:
+        assert isinstance(test_def['tools'], dict)
+        for tool, vers_req in test_def['tools'].items():
+            if tool not in tool_vers_map:
+                t.skip = True
+            elif not mesonlib.version_compare(tool_vers_map[tool], vers_req):
+                t.skip = True
+
+    # Skip the matrix code and just update the existing test
+    if 'matrix' not in test_def:
+        t.env.update(env)
+        t.installed_files = installed
+        t.do_not_set_opts = do_not_set_opts
+        t.stdout = stdout
+        return [t]
+
+    new_opt_list: T.List[T.List[T.Tuple[str, bool]]]
+
+    # 'matrix; entry is present, so build multiple tests from matrix definition
+    opt_list = []  # type: T.List[T.List[T.Tuple[str, bool]]]
+    matrix = test_def['matrix']
+    assert "options" in matrix
+    for key, val in matrix["options"].items():
+        assert isinstance(val, list)
+        tmp_opts = []  # type: T.List[T.Tuple[str, bool]]
+        for i in val:
+            assert isinstance(i, dict)
+            assert "val" in i
+            skip = False
+
+            # Skip the matrix entry if environment variable is present
+            if 'skip_on_env' in i:
+                for skip_env_var in i['skip_on_env']:
+                    if skip_env_var in os.environ:
+                        skip = True
+
+            # Only run the test if all compiler ID's match
+            if 'compilers' in i:
+                for lang, id_list in i['compilers'].items():
+                    if lang not in compiler_id_map or compiler_id_map[lang] not in id_list:
+                        skip = True
+                        break
 
-                tmp_opts += [('{}={}'.format(key, i['val']), skip)]
+            # Add an empty matrix entry
+            if i['val'] is None:
+                tmp_opts += [(None, skip)]
+                continue
 
-            if opt_list:
-                new_opt_list = []  # type: T.List[T.List[T.Tuple[str, bool]]]
-                for i in opt_list:
-                    for j in tmp_opts:
-                        new_opt_list += [[*i, j]]
-                opt_list = new_opt_list
-            else:
-                opt_list = [[x] for x in tmp_opts]
+            tmp_opts += [('{}={}'.format(key, i['val']), skip)]
 
-        # Exclude specific configurations
-        if 'exclude' in matrix:
-            assert isinstance(matrix['exclude'], list)
-            new_opt_list = []  # type: T.List[T.List[T.Tuple[str, bool]]]
+        if opt_list:
+            new_opt_list = []
             for i in opt_list:
-                exclude = False
-                opt_names = [x[0] for x in i]
-                for j in matrix['exclude']:
-                    ex_list = ['{}={}'.format(k, v) for k, v in j.items()]
-                    if all([x in opt_names for x in ex_list]):
-                        exclude = True
-                        break
-
-                if not exclude:
-                    new_opt_list += [i]
-
+                for j in tmp_opts:
+                    new_opt_list += [[*i, j]]
             opt_list = new_opt_list
+        else:
+            opt_list = [[x] for x in tmp_opts]
 
+    # Exclude specific configurations
+    if 'exclude' in matrix:
+        assert isinstance(matrix['exclude'], list)
+        new_opt_list = []
         for i in opt_list:
-            name = ' '.join([x[0] for x in i if x[0] is not None])
-            opts = ['-D' + x[0] for x in i if x[0] is not None]
-            skip = any([x[1] for x in i])
-            test = TestDef(t.path, name, opts, skip or t.skip)
-            test.env.update(env)
-            test.installed_files = installed
-            test.do_not_set_opts = do_not_set_opts
-            test.stdout = stdout
-            all_tests += [test]
+            exclude = False
+            opt_names = [x[0] for x in i]
+            for j in matrix['exclude']:
+                ex_list = ['{}={}'.format(k, v) for k, v in j.items()]
+                if all([x in opt_names for x in ex_list]):
+                    exclude = True
+                    break
+
+            if not exclude:
+                new_opt_list += [i]
+
+        opt_list = new_opt_list
+
+    for i in opt_list:
+        name = ' '.join([x[0] for x in i if x[0] is not None])
+        opts = ['-D' + x[0] for x in i if x[0] is not None]
+        skip = any([x[1] for x in i])
+        test = TestDef(t.path, name, opts, skip or t.skip)
+        test.env.update(env)
+        test.installed_files = installed
+        test.do_not_set_opts = do_not_set_opts
+        test.stdout = stdout
+        all_tests.append(test)
+
+    return all_tests
+
+
+def gather_tests(testdir: Path, stdout_mandatory: bool) -> T.List[TestDef]:
+    tests = [t.name for t in testdir.iterdir() if t.is_dir()]
+    tests = [t for t in tests if not t.startswith('.')]  # Filter non-tests files (dot files, etc)
+    test_defs = [TestDef(testdir / t, None, []) for t in tests]
+    all_tests: T.List[TestDef] = []
+    for t in test_defs:
+        all_tests.extend(load_test_json(t, stdout_mandatory))
 
     return sorted(all_tests)
 
+
 def have_d_compiler():
     if shutil.which("ldc2"):
         return True
diff --git a/run_single_test.py b/run_single_test.py
new file mode 100755
index 0000000..9b3ed18
--- /dev/null
+++ b/run_single_test.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+# SPDX-license-identifier: Apache-2.0
+# Copyright © 2021 Intel Corporation
+
+"""Script for running a single project test.
+
+This script is meant for Meson developers who want to run a single project
+test, with all of the rules from the test.json file loaded.
+"""
+
+import argparse
+import pathlib
+import shutil
+import typing as T
+
+from mesonbuild import environment
+from mesonbuild import mlog
+from mesonbuild import mesonlib
+from run_project_tests import TestDef, load_test_json, run_test, BuildStep
+from run_tests import get_backend_commands, guess_backend, get_fake_options
+
+if T.TYPE_CHECKING:
+    try:
+        from typing import Protocol
+    except ImportError:
+        # Mypy gets grump about this even though it's fine
+        from typing_extensions import Protocol  # type: ignore
+
+    class ArgumentType(Protocol):
+
+        """Typing information for command line arguments."""
+
+        case: pathlib.Path
+        subtests: T.List[int]
+        backend: str
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('case', type=pathlib.Path, help='The test case to run')
+    parser.add_argument('--subtest', type=int, action='append', dest='subtests', help='which subtests to run')
+    parser.add_argument('--backend', action='store', help="Which backend to use")
+    args = T.cast('ArgumentType', parser.parse_args())
+
+    test = TestDef(args.case, args.case.stem, [])
+    tests = load_test_json(test, False)
+    if args.subtests:
+        tests = [t for i, t in enumerate(tests) if i in args.subtests]
+
+    with mesonlib.TemporaryDirectoryWinProof() as build_dir:
+        fake_opts = get_fake_options('/')
+        env = environment.Environment(None, build_dir, fake_opts)
+        try:
+            comp = env.compiler_from_language('c', mesonlib.MachineChoice.HOST).get_id()
+        except mesonlib.MesonException:
+            raise RuntimeError('Could not detect C compiler')
+
+    backend, backend_args = guess_backend(args.backend, shutil.which('msbuild'))
+    _cmds = get_backend_commands(backend, False)
+    commands = (_cmds[0], _cmds[1], _cmds[3], _cmds[4])
+
+    results = [run_test(t, t.args, comp, backend, backend_args, commands, False, True) for t in tests]
+    failed = False
+    for test, result in zip(tests, results):
+        if result is None:
+            msg = mlog.yellow('SKIP:')
+        elif result.msg:
+            msg = mlog.red('FAIL:')
+            failed = True
+        else:
+            msg = mlog.green('PASS:')
+        mlog.log(msg, test.display_name())
+        if result.msg:
+            mlog.log('reason:', result.msg)
+            if result.step is BuildStep.configure:
+                # For configure failures, instead of printing stdout,
+                # print the meson log if available since it's a superset
+                # of stdout and often has very useful information.
+                mlog.log(result.mlog)
+            else:
+                mlog.log(result.stdo)
+            for cmd_res in result.cicmds:
+                mlog.log(cmd_res)
+            mlog.log(result.stde)
+
+    exit(1 if failed else 0)
+
+if __name__ == "__main__":
+    main()
diff --git a/run_tests.py b/run_tests.py
index 24d6a51..1e01aa9 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -27,6 +27,8 @@ from enum import Enum
 from glob import glob
 from pathlib import Path
 from unittest import mock
+import typing as T
+
 from mesonbuild import compilers
 from mesonbuild import dependencies
 from mesonbuild import mesonlib
@@ -57,26 +59,27 @@ else:
 if NINJA_CMD is None:
     raise RuntimeError('Could not find Ninja v1.7 or newer')
 
-def guess_backend(backend, msbuild_exe: str):
+def guess_backend(backend_str: str, msbuild_exe: str) -> T.Tuple['Backend', T.List[str]]:
     # Auto-detect backend if unspecified
     backend_flags = []
-    if backend is None:
+    if backend_str is None:
         if msbuild_exe is not None and (mesonlib.is_windows() and not _using_intelcl()):
-            backend = 'vs' # Meson will auto-detect VS version to use
+            backend_str = 'vs' # Meson will auto-detect VS version to use
         else:
-            backend = 'ninja'
+            backend_str = 'ninja'
+
     # Set backend arguments for Meson
-    if backend.startswith('vs'):
-        backend_flags = ['--backend=' + backend]
+    if backend_str.startswith('vs'):
+        backend_flags = ['--backend=' + backend_str]
         backend = Backend.vs
-    elif backend == 'xcode':
+    elif backend_str == 'xcode':
         backend_flags = ['--backend=xcode']
         backend = Backend.xcode
-    elif backend == 'ninja':
+    elif backend_str == 'ninja':
         backend_flags = ['--backend=ninja']
         backend = Backend.ninja
     else:
-        raise RuntimeError('Unknown backend: {!r}'.format(backend))
+        raise RuntimeError('Unknown backend: {!r}'.format(backend_str))
     return (backend, backend_flags)
 
 
@@ -115,7 +118,8 @@ class FakeCompilerOptions:
     def __init__(self):
         self.value = []
 
-def get_fake_options(prefix=''):
+# TODO: use a typing.Protocol here
+def get_fake_options(prefix: str = '') -> argparse.Namespace:
     opts = argparse.Namespace()
     opts.native_file = []
     opts.cross_file = None
@@ -208,9 +212,13 @@ def get_builddir_target_args(backend, builddir, target):
         raise AssertionError('Unknown backend: {!r}'.format(backend))
     return target_args + dir_args
 
-def get_backend_commands(backend, debug=False):
-    install_cmd = []
-    uninstall_cmd = []
+def get_backend_commands(backend: Backend, debug: bool = False) -> \
+        T.Tuple[T.List[str], T.List[str], T.List[str], T.List[str], T.List[str]]:
+    install_cmd: T.List[str] = []
+    uninstall_cmd: T.List[str] = []
+    clean_cmd: T.List[str]
+    cmd: T.List[str]
+    test_cmd: T.List[str]
     if backend is Backend.vs:
         cmd = ['msbuild']
         clean_cmd = cmd + ['/target:Clean']