diff options
| -rwxr-xr-x | run_mypy.py | 1 | ||||
| -rwxr-xr-x | run_project_tests.py | 249 | ||||
| -rwxr-xr-x | run_single_test.py | 89 | ||||
| -rwxr-xr-x | run_tests.py | 34 |
4 files changed, 244 insertions, 129 deletions
diff --git a/run_mypy.py b/run_mypy.py index e6900c7..1c886f8 100755 --- a/run_mypy.py +++ b/run_mypy.py @@ -39,6 +39,7 @@ modules = [ 'mesonbuild/optinterpreter.py', 'run_mypy.py', + 'run_single_test.py', 'tools' ] diff --git a/run_project_tests.py b/run_project_tests.py index 14f135a..b6879ee 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -461,6 +461,8 @@ def create_deterministic_builddir(test: TestDef, use_tmpdir: bool) -> str: src_dir += test.name rel_dirname = 'b ' + hashlib.sha256(src_dir.encode(errors='ignore')).hexdigest()[0:10] abs_pathname = os.path.join(tempfile.gettempdir() if use_tmpdir else os.getcwd(), rel_dirname) + if os.path.exists(abs_pathname): + mesonlib.windows_proof_rmtree(abs_pathname) os.mkdir(abs_pathname) return abs_pathname @@ -474,7 +476,7 @@ def format_parameter_file(file_basename: str, test: TestDef, test_build_dir: str return destination -def detect_parameter_files(test: TestDef, test_build_dir: str) -> (Path, Path): +def detect_parameter_files(test: TestDef, test_build_dir: str) -> T.Tuple[Path, Path]: nativefile = test.path / 'nativefile.ini' crossfile = test.path / 'crossfile.ini' @@ -486,7 +488,9 @@ def detect_parameter_files(test: TestDef, test_build_dir: str) -> (Path, Path): return nativefile, crossfile -def run_test(test: TestDef, extra_args, compiler, backend, flags, commands, should_fail, use_tmp: bool): +def run_test(test: TestDef, extra_args: T.List[str], compiler: str, backend: Backend, + flags: T.List[str], commands: T.Tuple[T.List[str], T.List[str], T.List[str], T.List[str]], + should_fail: bool, use_tmp: bool) -> T.Optional[TestResult]: if test.skip: return None build_dir = create_deterministic_builddir(test, use_tmp) @@ -501,7 +505,10 @@ def run_test(test: TestDef, extra_args, compiler, backend, flags, commands, shou finally: mesonlib.windows_proof_rmtree(build_dir) -def _run_test(test: TestDef, test_build_dir: str, install_dir: str, extra_args, compiler, backend, flags, commands, should_fail): +def _run_test(test: TestDef, test_build_dir: str, install_dir: str, + extra_args: T.List[str], compiler: str, backend: Backend, + flags: T.List[str], commands: T.Tuple[T.List[str], T.List[str], T.List[str], T.List[str]], + should_fail: bool) -> TestResult: compile_commands, clean_commands, install_commands, uninstall_commands = commands gen_start = time.time() # Configure in-process @@ -620,130 +627,140 @@ def _run_test(test: TestDef, test_build_dir: str, install_dir: str, extra_args, return testresult -def gather_tests(testdir: Path, stdout_mandatory: bool) -> T.List[TestDef]: - tests = [t.name for t in testdir.iterdir() if t.is_dir()] - tests = [t for t in tests if not t.startswith('.')] # Filter non-tests files (dot files, etc) - test_defs = [TestDef(testdir / t, None, []) for t in tests] - all_tests = [] # type: T.List[TestDef] - for t in test_defs: - test_def = {} - test_def_file = t.path / 'test.json' - if test_def_file.is_file(): - test_def = json.loads(test_def_file.read_text()) - - # Handle additional environment variables - env = {} # type: T.Dict[str, str] - if 'env' in test_def: - assert isinstance(test_def['env'], dict) - env = test_def['env'] - for key, val in env.items(): - val = val.replace('@ROOT@', t.path.resolve().as_posix()) - val = val.replace('@PATH@', t.env.get('PATH', '')) - env[key] = val - - # Handle installed files - installed = [] # type: T.List[InstalledFile] - if 'installed' in test_def: - installed = [InstalledFile(x) for x in test_def['installed']] - - # Handle expected output - stdout = test_def.get('stdout', []) - if stdout_mandatory and not stdout: - raise RuntimeError("{} must contain a non-empty stdout key".format(test_def_file)) - - # Handle the do_not_set_opts list - do_not_set_opts = test_def.get('do_not_set_opts', []) # type: T.List[str] - - # Skip tests if the tool requirements are not met - if 'tools' in test_def: - assert isinstance(test_def['tools'], dict) - for tool, vers_req in test_def['tools'].items(): - if tool not in tool_vers_map: - t.skip = True - elif not mesonlib.version_compare(tool_vers_map[tool], vers_req): - t.skip = True - - # Skip the matrix code and just update the existing test - if 'matrix' not in test_def: - t.env.update(env) - t.installed_files = installed - t.do_not_set_opts = do_not_set_opts - t.stdout = stdout - all_tests += [t] - continue - # 'matrix; entry is present, so build multiple tests from matrix definition - opt_list = [] # type: T.List[T.List[T.Tuple[str, bool]]] - matrix = test_def['matrix'] - assert "options" in matrix - for key, val in matrix["options"].items(): - assert isinstance(val, list) - tmp_opts = [] # type: T.List[T.Tuple[str, bool]] - for i in val: - assert isinstance(i, dict) - assert "val" in i - skip = False - - # Skip the matrix entry if environment variable is present - if 'skip_on_env' in i: - for skip_env_var in i['skip_on_env']: - if skip_env_var in os.environ: - skip = True - - # Only run the test if all compiler ID's match - if 'compilers' in i: - for lang, id_list in i['compilers'].items(): - if lang not in compiler_id_map or compiler_id_map[lang] not in id_list: - skip = True - break - - # Add an empty matrix entry - if i['val'] is None: - tmp_opts += [(None, skip)] - continue +def load_test_json(t: TestDef, stdout_mandatory: bool) -> T.List[TestDef]: + all_tests: T.List[TestDef] = [] + test_def = {} + test_def_file = t.path / 'test.json' + if test_def_file.is_file(): + test_def = json.loads(test_def_file.read_text()) + + # Handle additional environment variables + env = {} # type: T.Dict[str, str] + if 'env' in test_def: + assert isinstance(test_def['env'], dict) + env = test_def['env'] + for key, val in env.items(): + val = val.replace('@ROOT@', t.path.resolve().as_posix()) + val = val.replace('@PATH@', t.env.get('PATH', '')) + env[key] = val + + # Handle installed files + installed = [] # type: T.List[InstalledFile] + if 'installed' in test_def: + installed = [InstalledFile(x) for x in test_def['installed']] + + # Handle expected output + stdout = test_def.get('stdout', []) + if stdout_mandatory and not stdout: + raise RuntimeError("{} must contain a non-empty stdout key".format(test_def_file)) + + # Handle the do_not_set_opts list + do_not_set_opts = test_def.get('do_not_set_opts', []) # type: T.List[str] + + # Skip tests if the tool requirements are not met + if 'tools' in test_def: + assert isinstance(test_def['tools'], dict) + for tool, vers_req in test_def['tools'].items(): + if tool not in tool_vers_map: + t.skip = True + elif not mesonlib.version_compare(tool_vers_map[tool], vers_req): + t.skip = True + + # Skip the matrix code and just update the existing test + if 'matrix' not in test_def: + t.env.update(env) + t.installed_files = installed + t.do_not_set_opts = do_not_set_opts + t.stdout = stdout + return [t] + + new_opt_list: T.List[T.List[T.Tuple[str, bool]]] + + # 'matrix; entry is present, so build multiple tests from matrix definition + opt_list = [] # type: T.List[T.List[T.Tuple[str, bool]]] + matrix = test_def['matrix'] + assert "options" in matrix + for key, val in matrix["options"].items(): + assert isinstance(val, list) + tmp_opts = [] # type: T.List[T.Tuple[str, bool]] + for i in val: + assert isinstance(i, dict) + assert "val" in i + skip = False + + # Skip the matrix entry if environment variable is present + if 'skip_on_env' in i: + for skip_env_var in i['skip_on_env']: + if skip_env_var in os.environ: + skip = True + + # Only run the test if all compiler ID's match + if 'compilers' in i: + for lang, id_list in i['compilers'].items(): + if lang not in compiler_id_map or compiler_id_map[lang] not in id_list: + skip = True + break - tmp_opts += [('{}={}'.format(key, i['val']), skip)] + # Add an empty matrix entry + if i['val'] is None: + tmp_opts += [(None, skip)] + continue - if opt_list: - new_opt_list = [] # type: T.List[T.List[T.Tuple[str, bool]]] - for i in opt_list: - for j in tmp_opts: - new_opt_list += [[*i, j]] - opt_list = new_opt_list - else: - opt_list = [[x] for x in tmp_opts] + tmp_opts += [('{}={}'.format(key, i['val']), skip)] - # Exclude specific configurations - if 'exclude' in matrix: - assert isinstance(matrix['exclude'], list) - new_opt_list = [] # type: T.List[T.List[T.Tuple[str, bool]]] + if opt_list: + new_opt_list = [] for i in opt_list: - exclude = False - opt_names = [x[0] for x in i] - for j in matrix['exclude']: - ex_list = ['{}={}'.format(k, v) for k, v in j.items()] - if all([x in opt_names for x in ex_list]): - exclude = True - break - - if not exclude: - new_opt_list += [i] - + for j in tmp_opts: + new_opt_list += [[*i, j]] opt_list = new_opt_list + else: + opt_list = [[x] for x in tmp_opts] + # Exclude specific configurations + if 'exclude' in matrix: + assert isinstance(matrix['exclude'], list) + new_opt_list = [] for i in opt_list: - name = ' '.join([x[0] for x in i if x[0] is not None]) - opts = ['-D' + x[0] for x in i if x[0] is not None] - skip = any([x[1] for x in i]) - test = TestDef(t.path, name, opts, skip or t.skip) - test.env.update(env) - test.installed_files = installed - test.do_not_set_opts = do_not_set_opts - test.stdout = stdout - all_tests += [test] + exclude = False + opt_names = [x[0] for x in i] + for j in matrix['exclude']: + ex_list = ['{}={}'.format(k, v) for k, v in j.items()] + if all([x in opt_names for x in ex_list]): + exclude = True + break + + if not exclude: + new_opt_list += [i] + + opt_list = new_opt_list + + for i in opt_list: + name = ' '.join([x[0] for x in i if x[0] is not None]) + opts = ['-D' + x[0] for x in i if x[0] is not None] + skip = any([x[1] for x in i]) + test = TestDef(t.path, name, opts, skip or t.skip) + test.env.update(env) + test.installed_files = installed + test.do_not_set_opts = do_not_set_opts + test.stdout = stdout + all_tests.append(test) + + return all_tests + + +def gather_tests(testdir: Path, stdout_mandatory: bool) -> T.List[TestDef]: + tests = [t.name for t in testdir.iterdir() if t.is_dir()] + tests = [t for t in tests if not t.startswith('.')] # Filter non-tests files (dot files, etc) + test_defs = [TestDef(testdir / t, None, []) for t in tests] + all_tests: T.List[TestDef] = [] + for t in test_defs: + all_tests.extend(load_test_json(t, stdout_mandatory)) return sorted(all_tests) + def have_d_compiler(): if shutil.which("ldc2"): return True diff --git a/run_single_test.py b/run_single_test.py new file mode 100755 index 0000000..9b3ed18 --- /dev/null +++ b/run_single_test.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# SPDX-license-identifier: Apache-2.0 +# Copyright © 2021 Intel Corporation + +"""Script for running a single project test. + +This script is meant for Meson developers who want to run a single project +test, with all of the rules from the test.json file loaded. +""" + +import argparse +import pathlib +import shutil +import typing as T + +from mesonbuild import environment +from mesonbuild import mlog +from mesonbuild import mesonlib +from run_project_tests import TestDef, load_test_json, run_test, BuildStep +from run_tests import get_backend_commands, guess_backend, get_fake_options + +if T.TYPE_CHECKING: + try: + from typing import Protocol + except ImportError: + # Mypy gets grump about this even though it's fine + from typing_extensions import Protocol # type: ignore + + class ArgumentType(Protocol): + + """Typing information for command line arguments.""" + + case: pathlib.Path + subtests: T.List[int] + backend: str + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('case', type=pathlib.Path, help='The test case to run') + parser.add_argument('--subtest', type=int, action='append', dest='subtests', help='which subtests to run') + parser.add_argument('--backend', action='store', help="Which backend to use") + args = T.cast('ArgumentType', parser.parse_args()) + + test = TestDef(args.case, args.case.stem, []) + tests = load_test_json(test, False) + if args.subtests: + tests = [t for i, t in enumerate(tests) if i in args.subtests] + + with mesonlib.TemporaryDirectoryWinProof() as build_dir: + fake_opts = get_fake_options('/') + env = environment.Environment(None, build_dir, fake_opts) + try: + comp = env.compiler_from_language('c', mesonlib.MachineChoice.HOST).get_id() + except mesonlib.MesonException: + raise RuntimeError('Could not detect C compiler') + + backend, backend_args = guess_backend(args.backend, shutil.which('msbuild')) + _cmds = get_backend_commands(backend, False) + commands = (_cmds[0], _cmds[1], _cmds[3], _cmds[4]) + + results = [run_test(t, t.args, comp, backend, backend_args, commands, False, True) for t in tests] + failed = False + for test, result in zip(tests, results): + if result is None: + msg = mlog.yellow('SKIP:') + elif result.msg: + msg = mlog.red('FAIL:') + failed = True + else: + msg = mlog.green('PASS:') + mlog.log(msg, test.display_name()) + if result.msg: + mlog.log('reason:', result.msg) + if result.step is BuildStep.configure: + # For configure failures, instead of printing stdout, + # print the meson log if available since it's a superset + # of stdout and often has very useful information. + mlog.log(result.mlog) + else: + mlog.log(result.stdo) + for cmd_res in result.cicmds: + mlog.log(cmd_res) + mlog.log(result.stde) + + exit(1 if failed else 0) + +if __name__ == "__main__": + main() diff --git a/run_tests.py b/run_tests.py index 24d6a51..1e01aa9 100755 --- a/run_tests.py +++ b/run_tests.py @@ -27,6 +27,8 @@ from enum import Enum from glob import glob from pathlib import Path from unittest import mock +import typing as T + from mesonbuild import compilers from mesonbuild import dependencies from mesonbuild import mesonlib @@ -57,26 +59,27 @@ else: if NINJA_CMD is None: raise RuntimeError('Could not find Ninja v1.7 or newer') -def guess_backend(backend, msbuild_exe: str): +def guess_backend(backend_str: str, msbuild_exe: str) -> T.Tuple['Backend', T.List[str]]: # Auto-detect backend if unspecified backend_flags = [] - if backend is None: + if backend_str is None: if msbuild_exe is not None and (mesonlib.is_windows() and not _using_intelcl()): - backend = 'vs' # Meson will auto-detect VS version to use + backend_str = 'vs' # Meson will auto-detect VS version to use else: - backend = 'ninja' + backend_str = 'ninja' + # Set backend arguments for Meson - if backend.startswith('vs'): - backend_flags = ['--backend=' + backend] + if backend_str.startswith('vs'): + backend_flags = ['--backend=' + backend_str] backend = Backend.vs - elif backend == 'xcode': + elif backend_str == 'xcode': backend_flags = ['--backend=xcode'] backend = Backend.xcode - elif backend == 'ninja': + elif backend_str == 'ninja': backend_flags = ['--backend=ninja'] backend = Backend.ninja else: - raise RuntimeError('Unknown backend: {!r}'.format(backend)) + raise RuntimeError('Unknown backend: {!r}'.format(backend_str)) return (backend, backend_flags) @@ -115,7 +118,8 @@ class FakeCompilerOptions: def __init__(self): self.value = [] -def get_fake_options(prefix=''): +# TODO: use a typing.Protocol here +def get_fake_options(prefix: str = '') -> argparse.Namespace: opts = argparse.Namespace() opts.native_file = [] opts.cross_file = None @@ -208,9 +212,13 @@ def get_builddir_target_args(backend, builddir, target): raise AssertionError('Unknown backend: {!r}'.format(backend)) return target_args + dir_args -def get_backend_commands(backend, debug=False): - install_cmd = [] - uninstall_cmd = [] +def get_backend_commands(backend: Backend, debug: bool = False) -> \ + T.Tuple[T.List[str], T.List[str], T.List[str], T.List[str], T.List[str]]: + install_cmd: T.List[str] = [] + uninstall_cmd: T.List[str] = [] + clean_cmd: T.List[str] + cmd: T.List[str] + test_cmd: T.List[str] if backend is Backend.vs: cmd = ['msbuild'] clean_cmd = cmd + ['/target:Clean'] |