Revert "Merge pull request #7172 from jon-turney/test-output-check-mandatory"

This reverts commit 0871b1032c53287a1ed3ce5108799fb0daccaec5, reversing
changes made to 9dc3ca2c1c9fbb47e731551c6432df144f725261.

11 files changed, 40 insertions, 115 deletions

diff --git a/run_project_tests.py b/run_project_tests.py
index 22c0205..bcfe05c 100755
--- a/run_project_tests.py
+++ b/run_project_tests.py
@@ -594,16 +594,18 @@ def _run_test(test: TestDef, test_build_dir: str, install_dir: str, extra_args,
 
     return testresult
 
-def gather_tests(testdir: Path, stdout_mandatory: bool) -> T.List[TestDef]:
+def gather_tests(testdir: Path) -> T.List[TestDef]:
     tests = [t.name for t in testdir.iterdir() if t.is_dir()]
     tests = [t for t in tests if not t.startswith('.')]  # Filter non-tests files (dot files, etc)
     test_defs = [TestDef(testdir / t, None, []) for t in tests]
     all_tests = []  # type: T.List[TestDef]
     for t in test_defs:
-        test_def = {}
         test_def_file = t.path / 'test.json'
-        if test_def_file.is_file():
-            test_def = json.loads(test_def_file.read_text())
+        if not test_def_file.is_file():
+            all_tests += [t]
+            continue
+
+        test_def = json.loads(test_def_file.read_text())
 
         # Handle additional environment variables
         env = {}  # type: T.Dict[str, str]
@@ -621,8 +623,6 @@ def gather_tests(testdir: Path, stdout_mandatory: bool) -> T.List[TestDef]:
 
         # Handle expected output
         stdout = test_def.get('stdout', [])
-        if stdout_mandatory and not stdout:
-            raise RuntimeError("{} must contain a non-empty stdout key".format(test_def_file))
 
         # Handle the do_not_set_opts list
         do_not_set_opts = test_def.get('do_not_set_opts', [])  # type: T.List[str]
@@ -897,50 +897,45 @@ def detect_tests_to_run(only: T.List[str], use_tmp: bool) -> T.List[T.Tuple[str,
                        shutil.which('pgfortran') or
                        shutil.which('ifort'))
 
-    class TestCategory:
-        def __init__(self, category: str, subdir: str, skip: bool = False, stdout_mandatory: bool = False):
-            self.category = category                  # category name
-            self.subdir = subdir                      # subdirectory
-            self.skip = skip                          # skip condition
-            self.stdout_mandatory = stdout_mandatory  # expected stdout is mandatory for tests in this categroy
-
+    # Name, subdirectory, skip condition.
     all_tests = [
-        TestCategory('cmake', 'cmake', not shutil.which('cmake') or (os.environ.get('compiler') == 'msvc2015' and under_ci)),
-        TestCategory('common', 'common'),
-        TestCategory('warning-meson', 'warning', stdout_mandatory=True),
-        TestCategory('failing-meson', 'failing', stdout_mandatory=True),
-        TestCategory('failing-build', 'failing build'),
-        TestCategory('failing-test',  'failing test'),
-        TestCategory('keyval', 'keyval'),
-        TestCategory('platform-osx', 'osx', not mesonlib.is_osx()),
-        TestCategory('platform-windows', 'windows', not mesonlib.is_windows() and not mesonlib.is_cygwin()),
-        TestCategory('platform-linux', 'linuxlike', mesonlib.is_osx() or mesonlib.is_windows()),
-        TestCategory('java', 'java', backend is not Backend.ninja or mesonlib.is_osx() or not have_java()),
-        TestCategory('C#', 'csharp', skip_csharp(backend)),
-        TestCategory('vala', 'vala', backend is not Backend.ninja or not shutil.which(os.environ.get('VALAC', 'valac'))),
-        TestCategory('rust', 'rust', should_skip_rust(backend)),
-        TestCategory('d', 'd', backend is not Backend.ninja or not have_d_compiler()),
-        TestCategory('objective c', 'objc', backend not in (Backend.ninja, Backend.xcode) or not have_objc_compiler(options.use_tmpdir)),
-        TestCategory('objective c++', 'objcpp', backend not in (Backend.ninja, Backend.xcode) or not have_objcpp_compiler(options.use_tmpdir)),
-        TestCategory('fortran', 'fortran', skip_fortran or backend != Backend.ninja),
-        TestCategory('swift', 'swift', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('swiftc')),
+        ('cmake', 'cmake', not shutil.which('cmake') or (os.environ.get('compiler') == 'msvc2015' and under_ci)),
+        ('common', 'common', False),
+        ('warning-meson', 'warning', False),
+        ('failing-meson', 'failing', False),
+        ('failing-build', 'failing build', False),
+        ('failing-test',  'failing test', False),
+        ('keyval', 'keyval', False),
+
+        ('platform-osx', 'osx', not mesonlib.is_osx()),
+        ('platform-windows', 'windows', not mesonlib.is_windows() and not mesonlib.is_cygwin()),
+        ('platform-linux', 'linuxlike', mesonlib.is_osx() or mesonlib.is_windows()),
+
+        ('java', 'java', backend is not Backend.ninja or mesonlib.is_osx() or not have_java()),
+        ('C#', 'csharp', skip_csharp(backend)),
+        ('vala', 'vala', backend is not Backend.ninja or not shutil.which(os.environ.get('VALAC', 'valac'))),
+        ('rust', 'rust', should_skip_rust(backend)),
+        ('d', 'd', backend is not Backend.ninja or not have_d_compiler()),
+        ('objective c', 'objc', backend not in (Backend.ninja, Backend.xcode) or not have_objc_compiler(options.use_tmpdir)),
+        ('objective c++', 'objcpp', backend not in (Backend.ninja, Backend.xcode) or not have_objcpp_compiler(options.use_tmpdir)),
+        ('fortran', 'fortran', skip_fortran or backend != Backend.ninja),
+        ('swift', 'swift', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('swiftc')),
         # CUDA tests on Windows: use Ninja backend:  python run_project_tests.py --only cuda --backend ninja
-        TestCategory('cuda', 'cuda', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('nvcc')),
-        TestCategory('python3', 'python3', backend is not Backend.ninja),
-        TestCategory('python', 'python', backend is not Backend.ninja),
-        TestCategory('fpga', 'fpga', shutil.which('yosys') is None),
-        TestCategory('frameworks', 'frameworks'),
-        TestCategory('nasm', 'nasm'),
-        TestCategory('wasm', 'wasm', shutil.which('emcc') is None or backend is not Backend.ninja),
+        ('cuda', 'cuda', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('nvcc')),
+        ('python3', 'python3', backend is not Backend.ninja),
+        ('python', 'python', backend is not Backend.ninja),
+        ('fpga', 'fpga', shutil.which('yosys') is None),
+        ('frameworks', 'frameworks', False),
+        ('nasm', 'nasm', False),
+        ('wasm', 'wasm', shutil.which('emcc') is None or backend is not Backend.ninja),
     ]
 
-    categories = [t.category for t in all_tests]
-    assert categories == ALL_TESTS, 'argparse("--only", choices=ALL_TESTS) need to be updated to match all_tests categories'
-
+    names = [t[0] for t in all_tests]
+    assert names == ALL_TESTS, 'argparse("--only", choices=ALL_TESTS) need to be updated to match all_tests names'
     if only:
-        all_tests = [t for t in all_tests if t.category in only]
-
-    gathered_tests = [(t.category, gather_tests(Path('test cases', t.subdir), t.stdout_mandatory), t.skip) for t in all_tests]
+        ind = [names.index(o) for o in only]
+        all_tests = [all_tests[i] for i in ind]
+    gathered_tests = [(name, gather_tests(Path('test cases', subdir)), skip) for name, subdir, skip in all_tests]
     return gathered_tests
 
 def run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]],
diff --git a/test cases/failing/100 fallback consistency/test.json b/test cases/failing/100 fallback consistency/test.json
deleted file mode 100644
index a783d8c..0000000
--- a/test cases/failing/100 fallback consistency/test.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-  "stdout": [
-    {
-      "line": "test cases/failing/100 fallback consistency/meson.build:7:0: ERROR: Inconsistency: Subproject has overridden the dependency with another variable than 'dep2'"
-    }
-  ]
-}
diff --git a/test cases/failing/101 no native compiler/test.json b/test cases/failing/101 no native compiler/test.json
deleted file mode 100644
index c7b5d1c..0000000
--- a/test cases/failing/101 no native compiler/test.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-  "stdout": [
-    {
-      "line": "test cases/failing/101 no native compiler/meson.build:12:0: ERROR: No host machine compiler for \"main.c\""
-    }
-  ]
-}
diff --git a/test cases/failing/102 subdir parse error/test.json b/test cases/failing/102 subdir parse error/test.json
deleted file mode 100644
index 06fd4d3..0000000
--- a/test cases/failing/102 subdir parse error/test.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-  "stdout": [
-    {
-      "line": "test cases/failing/102 subdir parse error/subdir/meson.build:1:0: ERROR: Plusassignment target must be an id."
-    }
-  ]
-}
diff --git a/test cases/failing/103 invalid option file/test.json b/test cases/failing/103 invalid option file/test.json
deleted file mode 100644
index 20dbec3..0000000
--- a/test cases/failing/103 invalid option file/test.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-  "stdout": [
-    {
-      "line": "test cases/failing/103 invalid option file/meson_options.txt:1:0: ERROR: lexer"
-    }
-  ]
-}
diff --git a/test cases/failing/104 no lang/test.json b/test cases/failing/104 no lang/test.json
deleted file mode 100644
index 62999be..0000000
--- a/test cases/failing/104 no lang/test.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-  "stdout": [
-    {
-      "line": "test cases/failing/104 no lang/meson.build:2:0: ERROR: No host machine compiler for \"main.c\""
-    }
-  ]
-}
diff --git a/test cases/failing/105 no glib-compile-resources/test.json b/test cases/failing/105 no glib-compile-resources/test.json
deleted file mode 100644
index 67dc7e4..0000000
--- a/test cases/failing/105 no glib-compile-resources/test.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-  "stdout": [
-    {
-      "line": "test cases/failing/105 no glib-compile-resources/meson.build:8:0: ERROR: Could not execute glib-compile-resources."
-    }
-  ]
-}
diff --git a/test cases/failing/36 pkgconfig dependency impossible conditions/test.json b/test cases/failing/36 pkgconfig dependency impossible conditions/test.json
deleted file mode 100644
index 2ce62ac..0000000
--- a/test cases/failing/36 pkgconfig dependency impossible conditions/test.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-  "stdout": [
-    {
-      "line": "test cases/failing/36 pkgconfig dependency impossible conditions/meson.build:7:0: ERROR: Dependency 'zlib' was already checked and was not found"
-    }
-  ]
-}
diff --git a/test cases/failing/67 subproj different versions/test.json b/test cases/failing/67 subproj different versions/test.json
deleted file mode 100644
index d16daf9..0000000
--- a/test cases/failing/67 subproj different versions/test.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-  "stdout": [
-    {
-      "line": "test cases/failing/67 subproj different versions/subprojects/b/meson.build:3:0: ERROR: Dependency 'c' was already checked and was not found"
-    }
-  ]
-}
diff --git a/test cases/failing/84 gtest dependency with version/test.json b/test cases/failing/84 gtest dependency with version/test.json
deleted file mode 100644
index e1bbcac..0000000
--- a/test cases/failing/84 gtest dependency with version/test.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-  "stdout": [
-    {
-      "line": "test cases/failing/84 gtest dependency with version/meson.build:8:0: ERROR: Dependency 'gtest' was already checked and was not found"
-    }
-  ]
-}
diff --git a/test cases/failing/98 fallback consistency/test.json b/test cases/failing/98 fallback consistency/test.json
deleted file mode 100644
index fd77bad..0000000
--- a/test cases/failing/98 fallback consistency/test.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-  "stdout": [
-    {
-      "line": "test cases/failing/98 fallback consistency/meson.build:4:0: ERROR: Inconsistency: Subproject has overridden the dependency with another variable than 'dep2'"
-    }
-  ]
-}