Merge pull request #7064 from dcbaker/gtest-protocol

Add support for Gtest as a test protocol

5 files changed, 108 insertions, 34 deletions

diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py
index 0aaf66a..2727abe 100644
--- a/mesonbuild/backend/backends.py
+++ b/mesonbuild/backend/backends.py
@@ -12,23 +12,54 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os, pickle, re
+from collections import OrderedDict
+from functools import lru_cache
+import enum
+import json
+import os
+import pickle
+import re
+import shlex
+import subprocess
 import textwrap
+import typing as T
+
 from .. import build
 from .. import dependencies
 from .. import mesonlib
 from .. import mlog
-import json
-import subprocess
-from ..mesonlib import MachineChoice, MesonException, OrderedSet, OptionOverrideProxy
-from ..mesonlib import classify_unity_sources, unholder
-from ..mesonlib import File
 from ..compilers import CompilerArgs, VisualStudioLikeCompiler
-from ..interpreter import Interpreter
-from collections import OrderedDict
-import shlex
-from functools import lru_cache
-import typing as T
+from ..mesonlib import (
+    File, MachineChoice, MesonException, OrderedSet, OptionOverrideProxy,
+    classify_unity_sources, unholder
+)
+
+if T.TYPE_CHECKING:
+    from ..interpreter import Interpreter
+
+
+class TestProtocol(enum.Enum):
+
+    EXITCODE = 0
+    TAP = 1
+    GTEST = 2
+
+    @classmethod
+    def from_str(cls, string: str) -> 'TestProtocol':
+        if string == 'exitcode':
+            return cls.EXITCODE
+        elif string == 'tap':
+            return cls.TAP
+        elif string == 'gtest':
+            return cls.GTEST
+        raise MesonException('unknown test format {}'.format(string))
+
+    def __str__(self) -> str:
+        if self is self.EXITCODE:
+            return 'exitcode'
+        elif self is self.GTEST:
+            return 'gtest'
+        return 'tap'
 
 
 class CleanTrees:
@@ -83,11 +114,11 @@ class ExecutableSerialisation:
 
 class TestSerialisation:
     def __init__(self, name: str, project: str, suite: str, fname: T.List[str],
-                 is_cross_built: bool, exe_wrapper: T.Optional[build.Executable],
+                 is_cross_built: bool, exe_wrapper: T.Optional[dependencies.ExternalProgram],
                  needs_exe_wrapper: bool, is_parallel: bool, cmd_args: T.List[str],
                  env: build.EnvironmentVariables, should_fail: bool,
                  timeout: T.Optional[int], workdir: T.Optional[str],
-                 extra_paths: T.List[str], protocol: str, priority: int):
+                 extra_paths: T.List[str], protocol: TestProtocol, priority: int):
         self.name = name
         self.project_name = project
         self.suite = suite
@@ -107,7 +138,7 @@ class TestSerialisation:
         self.priority = priority
         self.needs_exe_wrapper = needs_exe_wrapper
 
-def get_backend_from_name(backend: str, build: T.Optional[build.Build] = None, interpreter: T.Optional[Interpreter] = None) -> T.Optional['Backend']:
+def get_backend_from_name(backend: str, build: T.Optional[build.Build] = None, interpreter: T.Optional['Interpreter'] = None) -> T.Optional['Backend']:
     if backend == 'ninja':
         from . import ninjabackend
         return ninjabackend.NinjaBackend(build, interpreter)
@@ -134,7 +165,7 @@ def get_backend_from_name(backend: str, build: T.Optional[build.Build] = None, i
 # This class contains the basic functionality that is needed by all backends.
 # Feel free to move stuff in and out of it as you see fit.
 class Backend:
-    def __init__(self, build: T.Optional[build.Build], interpreter: T.Optional[Interpreter]):
+    def __init__(self, build: T.Optional[build.Build], interpreter: T.Optional['Interpreter']):
         # Make it possible to construct a dummy backend
         # This is used for introspection without a build directory
         if build is None:
diff --git a/mesonbuild/build.py b/mesonbuild/build.py
index aff1d5f..fbf2b17 100644
--- a/mesonbuild/build.py
+++ b/mesonbuild/build.py
@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import copy, os, re
 from collections import OrderedDict, defaultdict
-import itertools, pathlib
+from functools import lru_cache
+import copy
 import hashlib
+import itertools, pathlib
+import os
 import pickle
-from functools import lru_cache
+import re
 import typing as T
 
 from . import environment
diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py
index 6ec4c0e..a3e9dee 100644
--- a/mesonbuild/interpreter.py
+++ b/mesonbuild/interpreter.py
@@ -33,6 +33,7 @@ from .interpreterbase import FeatureNew, FeatureDeprecated, FeatureNewKwargs
 from .interpreterbase import ObjectHolder
 from .modules import ModuleReturnValue
 from .cmake import CMakeInterpreter
+from .backend.backends import TestProtocol
 
 from pathlib import Path, PurePath
 import os
@@ -979,7 +980,7 @@ class Test(InterpreterObject):
         self.should_fail = should_fail
         self.timeout = timeout
         self.workdir = workdir
-        self.protocol = protocol
+        self.protocol = TestProtocol.from_str(protocol)
         self.priority = priority
 
     def get_exe(self):
@@ -3821,6 +3822,8 @@ This will become a hard error in the future.''' % kwargs['input'], location=self
     @FeatureNewKwargs('test', '0.52.0', ['priority'])
     @permittedKwargs(permitted_kwargs['test'])
     def func_test(self, node, args, kwargs):
+        if kwargs.get('protocol') == 'gtest':
+            FeatureNew('"gtest" protocol for tests', '0.55.0').use(self.subproject)
         self.add_test(node, args, kwargs, True)
 
     def unpack_env_kwarg(self, kwargs) -> build.EnvironmentVariables:
@@ -3872,8 +3875,8 @@ This will become a hard error in the future.''' % kwargs['input'], location=self
         if not isinstance(timeout, int):
             raise InterpreterException('Timeout must be an integer.')
         protocol = kwargs.get('protocol', 'exitcode')
-        if protocol not in ('exitcode', 'tap'):
-            raise InterpreterException('Protocol must be "exitcode" or "tap".')
+        if protocol not in {'exitcode', 'tap', 'gtest'}:
+            raise InterpreterException('Protocol must be "exitcode", "tap", or "gtest".')
         suite = []
         prj = self.subproject if self.is_subproject() else self.build.project_name
         for s in mesonlib.stringlistify(kwargs.get('suite', '')):
diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py
index d5516d4..54e302b 100644
--- a/mesonbuild/mintro.py
+++ b/mesonbuild/mintro.py
@@ -328,7 +328,7 @@ def get_test_list(testdata) -> T.List[T.Dict[str, T.Union[str, int, T.List[str],
         to['suite'] = t.suite
         to['is_parallel'] = t.is_parallel
         to['priority'] = t.priority
-        to['protocol'] = t.protocol
+        to['protocol'] = str(t.protocol)
         result.append(to)
     return result
 
diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 3239736..4592c90 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -43,6 +43,7 @@ from . import environment
 from . import mlog
 from .dependencies import ExternalProgram
 from .mesonlib import MesonException, get_wine_shortpath, split_args
+from .backend.backends import TestProtocol
 
 if T.TYPE_CHECKING:
     from .backend.backends import TestSerialisation
@@ -94,6 +95,9 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
     parser.add_argument('--wrapper', default=None, dest='wrapper', type=split_args,
                         help='wrapper to run tests with (e.g. Valgrind)')
     parser.add_argument('-C', default='.', dest='wd',
+                        # https://github.com/python/typeshed/issues/3107
+                        # https://github.com/python/mypy/issues/7177
+                        type=os.path.abspath,  # type: ignore
                         help='directory to cd into before running')
     parser.add_argument('--suite', default=[], dest='include_suites', action='append', metavar='SUITE',
                         help='Only run tests belonging to the given suite.')
@@ -348,6 +352,19 @@ class JunitBuilder:
 
     def log(self, name: str, test: 'TestRun') -> None:
         """Log a single test case."""
+        if test.junit is not None:
+            for suite in test.junit.findall('.//testsuite'):
+                # Assume that we don't need to merge anything here...
+                suite.attrib['name'] = '{}.{}.{}'.format(test.project, name, suite.attrib['name'])
+
+                # GTest can inject invalid attributes
+                for case in suite.findall('.//testcase[@result]'):
+                    del case.attrib['result']
+                for case in suite.findall('.//testcase[@timestamp]'):
+                    del case.attrib['timestamp']
+                self.root.append(suite)
+            return
+
         # In this case we have a test binary with multiple results.
         # We want to record this so that each result is recorded
         # separately
@@ -429,10 +446,24 @@ class JunitBuilder:
 class TestRun:
 
     @classmethod
+    def make_gtest(cls, test: 'TestSerialisation', test_env: T.Dict[str, str],
+                   returncode: int, starttime: float, duration: float,
+                   stdo: T.Optional[str], stde: T.Optional[str],
+                   cmd: T.Optional[T.List[str]]) -> 'TestRun':
+        filename = '{}.xml'.format(test.name)
+        if test.workdir:
+            filename = os.path.join(test.workdir, filename)
+        tree = et.parse(filename)
+
+        return cls.make_exitcode(
+            test, test_env, returncode, starttime, duration, stdo, stde, cmd,
+            junit=tree)
+
+    @classmethod
     def make_exitcode(cls, test: 'TestSerialisation', test_env: T.Dict[str, str],
                       returncode: int, starttime: float, duration: float,
                       stdo: T.Optional[str], stde: T.Optional[str],
-                      cmd: T.Optional[T.List[str]]) -> 'TestRun':
+                      cmd: T.Optional[T.List[str]], **kwargs) -> 'TestRun':
         if returncode == GNU_SKIP_RETURNCODE:
             res = TestResult.SKIP
         elif returncode == GNU_ERROR_RETURNCODE:
@@ -441,15 +472,15 @@ class TestRun:
             res = TestResult.EXPECTEDFAIL if bool(returncode) else TestResult.UNEXPECTEDPASS
         else:
             res = TestResult.FAIL if bool(returncode) else TestResult.OK
-        return cls(test, test_env, res, [], returncode, starttime, duration, stdo, stde, cmd)
+        return cls(test, test_env, res, [], returncode, starttime, duration, stdo, stde, cmd, **kwargs)
 
     @classmethod
     def make_tap(cls, test: 'TestSerialisation', test_env: T.Dict[str, str],
                  returncode: int, starttime: float, duration: float,
                  stdo: str, stde: str,
                  cmd: T.Optional[T.List[str]]) -> 'TestRun':
-        res = None    # T.Optional[TestResult]
-        results = []  # T.List[TestResult]
+        res = None    # type: T.Optional[TestResult]
+        results = []  # type: T.List[TestResult]
         failed = False
 
         for i in TAPParser(io.StringIO(stdo)).parse():
@@ -485,7 +516,7 @@ class TestRun:
                  res: TestResult, results: T.List[TestResult], returncode:
                  int, starttime: float, duration: float,
                  stdo: T.Optional[str], stde: T.Optional[str],
-                 cmd: T.Optional[T.List[str]]):
+                 cmd: T.Optional[T.List[str]], *, junit: T.Optional[et.ElementTree] = None):
         assert isinstance(res, TestResult)
         self.res = res
         self.results = results  # May be an empty list
@@ -498,6 +529,7 @@ class TestRun:
         self.env = test_env
         self.should_fail = test.should_fail
         self.project = test.project_name
+        self.junit = junit
 
     def get_log(self) -> str:
         res = '--- command ---\n'
@@ -544,9 +576,7 @@ def write_json_log(jsonlogfile: T.TextIO, test_name: str, result: TestRun) -> No
     jsonlogfile.write(json.dumps(jresult) + '\n')
 
 def run_with_mono(fname: str) -> bool:
-    if fname.endswith('.exe') and not (is_windows() or is_cygwin()):
-        return True
-    return False
+    return fname.endswith('.exe') and not (is_windows() or is_cygwin())
 
 def load_benchmarks(build_dir: str) -> T.List['TestSerialisation']:
     datafile = Path(build_dir) / 'meson-private' / 'meson_benchmark_setup.dat'
@@ -633,7 +663,7 @@ class SingleTestRunner:
         if not self.options.verbose:
             stdout = tempfile.TemporaryFile("wb+")
             stderr = tempfile.TemporaryFile("wb+") if self.options.split else stdout
-        if self.test.protocol == 'tap' and stderr is stdout:
+        if self.test.protocol is TestProtocol.TAP and stderr is stdout:
             stdout = tempfile.TemporaryFile("wb+")
 
         # Let gdb handle ^C instead of us
@@ -653,7 +683,14 @@ class SingleTestRunner:
                 # errors avoid not being able to use the terminal.
                 os.setsid()  # type: ignore
 
-        p = subprocess.Popen(cmd,
+        extra_cmd = []  # type: T.List[str]
+        if self.test.protocol is TestProtocol.GTEST:
+            gtestname = '{}.xml'.format(self.test.name)
+            if self.test.workdir:
+                gtestname = '{}:{}'.format(self.test.workdir, self.test.name)
+            extra_cmd.append('--gtest_output=xml:{}'.format(gtestname))
+
+        p = subprocess.Popen(cmd + extra_cmd,
                              stdout=stdout,
                              stderr=stderr,
                              env=self.env,
@@ -743,8 +780,10 @@ class SingleTestRunner:
         if timed_out:
             return TestRun(self.test, self.test_env, TestResult.TIMEOUT, [], p.returncode, starttime, duration, stdo, stde, cmd)
         else:
-            if self.test.protocol == 'exitcode':
+            if self.test.protocol is TestProtocol.EXITCODE:
                 return TestRun.make_exitcode(self.test, self.test_env, p.returncode, starttime, duration, stdo, stde, cmd)
+            elif self.test.protocol is TestProtocol.GTEST:
+                return TestRun.make_gtest(self.test, self.test_env, p.returncode, starttime, duration, stdo, stde, cmd)
             else:
                 if self.options.verbose:
                     print(stdo, end='')
@@ -1162,7 +1201,6 @@ def run(options: argparse.Namespace) -> int:
         if not exe.found():
             print('Could not find requested program: {!r}'.format(check_bin))
             return 1
-    options.wd = os.path.abspath(options.wd)
 
     if not options.list and not options.no_rebuild:
         if not rebuild_all(options.wd):