aboutsummaryrefslogtreecommitdiff
path: root/run_project_tests.py
diff options
context:
space:
mode:
authorJussi Pakkanen <jpakkane@gmail.com>2020-05-26 02:52:12 +0300
committerGitHub <noreply@github.com>2020-05-26 02:52:12 +0300
commit1e7f1ad4c96945b5a849896bd368ceb40d0bfd1a (patch)
tree88338057d1154c532aa5993335579641b0b17ca1 /run_project_tests.py
parent6f199db95b45dca4db7a7f2d478760f7789e21a3 (diff)
parent6b1e1ffa773a7d781429b4a00218f5d49b78b872 (diff)
downloadmeson-1e7f1ad4c96945b5a849896bd368ceb40d0bfd1a.zip
meson-1e7f1ad4c96945b5a849896bd368ceb40d0bfd1a.tar.gz
meson-1e7f1ad4c96945b5a849896bd368ceb40d0bfd1a.tar.bz2
Merge pull request #7197 from jon-turney/test-output-check-mandatory
Make the expected output check mandatory for failing-meson and warning-meson tests
Diffstat (limited to 'run_project_tests.py')
-rwxr-xr-xrun_project_tests.py85
1 files changed, 45 insertions, 40 deletions
diff --git a/run_project_tests.py b/run_project_tests.py
index 927d0fe..5fc8aa2 100755
--- a/run_project_tests.py
+++ b/run_project_tests.py
@@ -603,18 +603,16 @@ def _run_test(test: TestDef, test_build_dir: str, install_dir: str, extra_args,
return testresult
-def gather_tests(testdir: Path) -> T.List[TestDef]:
+def gather_tests(testdir: Path, stdout_mandatory: bool) -> T.List[TestDef]:
tests = [t.name for t in testdir.iterdir() if t.is_dir()]
tests = [t for t in tests if not t.startswith('.')] # Filter non-tests files (dot files, etc)
test_defs = [TestDef(testdir / t, None, []) for t in tests]
all_tests = [] # type: T.List[TestDef]
for t in test_defs:
+ test_def = {}
test_def_file = t.path / 'test.json'
- if not test_def_file.is_file():
- all_tests += [t]
- continue
-
- test_def = json.loads(test_def_file.read_text())
+ if test_def_file.is_file():
+ test_def = json.loads(test_def_file.read_text())
# Handle additional environment variables
env = {} # type: T.Dict[str, str]
@@ -632,6 +630,8 @@ def gather_tests(testdir: Path) -> T.List[TestDef]:
# Handle expected output
stdout = test_def.get('stdout', [])
+ if stdout_mandatory and not stdout:
+ raise RuntimeError("{} must contain a non-empty stdout key".format(test_def_file))
# Handle the do_not_set_opts list
do_not_set_opts = test_def.get('do_not_set_opts', []) # type: T.List[str]
@@ -906,45 +906,50 @@ def detect_tests_to_run(only: T.List[str], use_tmp: bool) -> T.List[T.Tuple[str,
shutil.which('pgfortran') or
shutil.which('ifort'))
- # Name, subdirectory, skip condition.
+ class TestCategory:
+ def __init__(self, category: str, subdir: str, skip: bool = False, stdout_mandatory: bool = False):
+ self.category = category # category name
+ self.subdir = subdir # subdirectory
+ self.skip = skip # skip condition
+ self.stdout_mandatory = stdout_mandatory # expected stdout is mandatory for tests in this categroy
+
all_tests = [
- ('cmake', 'cmake', not shutil.which('cmake') or (os.environ.get('compiler') == 'msvc2015' and under_ci)),
- ('common', 'common', False),
- ('warning-meson', 'warning', False),
- ('failing-meson', 'failing', False),
- ('failing-build', 'failing build', False),
- ('failing-test', 'failing test', False),
- ('keyval', 'keyval', False),
-
- ('platform-osx', 'osx', not mesonlib.is_osx()),
- ('platform-windows', 'windows', not mesonlib.is_windows() and not mesonlib.is_cygwin()),
- ('platform-linux', 'linuxlike', mesonlib.is_osx() or mesonlib.is_windows()),
-
- ('java', 'java', backend is not Backend.ninja or mesonlib.is_osx() or not have_java()),
- ('C#', 'csharp', skip_csharp(backend)),
- ('vala', 'vala', backend is not Backend.ninja or not shutil.which(os.environ.get('VALAC', 'valac'))),
- ('rust', 'rust', should_skip_rust(backend)),
- ('d', 'd', backend is not Backend.ninja or not have_d_compiler()),
- ('objective c', 'objc', backend not in (Backend.ninja, Backend.xcode) or not have_objc_compiler(options.use_tmpdir)),
- ('objective c++', 'objcpp', backend not in (Backend.ninja, Backend.xcode) or not have_objcpp_compiler(options.use_tmpdir)),
- ('fortran', 'fortran', skip_fortran or backend != Backend.ninja),
- ('swift', 'swift', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('swiftc')),
+ TestCategory('cmake', 'cmake', not shutil.which('cmake') or (os.environ.get('compiler') == 'msvc2015' and under_ci)),
+ TestCategory('common', 'common'),
+ TestCategory('warning-meson', 'warning', stdout_mandatory=True),
+ TestCategory('failing-meson', 'failing', stdout_mandatory=True),
+ TestCategory('failing-build', 'failing build'),
+ TestCategory('failing-test', 'failing test'),
+ TestCategory('keyval', 'keyval'),
+ TestCategory('platform-osx', 'osx', not mesonlib.is_osx()),
+ TestCategory('platform-windows', 'windows', not mesonlib.is_windows() and not mesonlib.is_cygwin()),
+ TestCategory('platform-linux', 'linuxlike', mesonlib.is_osx() or mesonlib.is_windows()),
+ TestCategory('java', 'java', backend is not Backend.ninja or mesonlib.is_osx() or not have_java()),
+ TestCategory('C#', 'csharp', skip_csharp(backend)),
+ TestCategory('vala', 'vala', backend is not Backend.ninja or not shutil.which(os.environ.get('VALAC', 'valac'))),
+ TestCategory('rust', 'rust', should_skip_rust(backend)),
+ TestCategory('d', 'd', backend is not Backend.ninja or not have_d_compiler()),
+ TestCategory('objective c', 'objc', backend not in (Backend.ninja, Backend.xcode) or not have_objc_compiler(options.use_tmpdir)),
+ TestCategory('objective c++', 'objcpp', backend not in (Backend.ninja, Backend.xcode) or not have_objcpp_compiler(options.use_tmpdir)),
+ TestCategory('fortran', 'fortran', skip_fortran or backend != Backend.ninja),
+ TestCategory('swift', 'swift', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('swiftc')),
# CUDA tests on Windows: use Ninja backend: python run_project_tests.py --only cuda --backend ninja
- ('cuda', 'cuda', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('nvcc')),
- ('python3', 'python3', backend is not Backend.ninja),
- ('python', 'python', backend is not Backend.ninja),
- ('fpga', 'fpga', shutil.which('yosys') is None),
- ('frameworks', 'frameworks', False),
- ('nasm', 'nasm', False),
- ('wasm', 'wasm', shutil.which('emcc') is None or backend is not Backend.ninja),
+ TestCategory('cuda', 'cuda', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('nvcc')),
+ TestCategory('python3', 'python3', backend is not Backend.ninja),
+ TestCategory('python', 'python', backend is not Backend.ninja),
+ TestCategory('fpga', 'fpga', shutil.which('yosys') is None),
+ TestCategory('frameworks', 'frameworks'),
+ TestCategory('nasm', 'nasm'),
+ TestCategory('wasm', 'wasm', shutil.which('emcc') is None or backend is not Backend.ninja),
]
- names = [t[0] for t in all_tests]
- assert names == ALL_TESTS, 'argparse("--only", choices=ALL_TESTS) need to be updated to match all_tests names'
+ categories = [t.category for t in all_tests]
+ assert categories == ALL_TESTS, 'argparse("--only", choices=ALL_TESTS) need to be updated to match all_tests categories'
+
if only:
- ind = [names.index(o) for o in only]
- all_tests = [all_tests[i] for i in ind]
- gathered_tests = [(name, gather_tests(Path('test cases', subdir)), skip) for name, subdir, skip in all_tests]
+ all_tests = [t for t in all_tests if t.category in only]
+
+ gathered_tests = [(t.category, gather_tests(Path('test cases', t.subdir), t.stdout_mandatory), t.skip) for t in all_tests]
return gathered_tests
def run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]],