diff options
author | Dylan Baker <dylan@pnwbakers.com> | 2020-04-30 15:36:17 -0700 |
---|---|---|
committer | Dylan Baker <dylan@pnwbakers.com> | 2020-05-04 11:33:19 -0700 |
commit | 083c5f635741a29f93f95c817601dbc66207699d (patch) | |
tree | b0bcbb26bc160bb0fd6dcc496e733c5317a555da | |
parent | 0c51762463abd72526ac84f3cfeaa286186ae1d7 (diff) | |
download | meson-083c5f635741a29f93f95c817601dbc66207699d.zip meson-083c5f635741a29f93f95c817601dbc66207699d.tar.gz meson-083c5f635741a29f93f95c817601dbc66207699d.tar.bz2 |
Add native support for gtest tests
Gtest can output junit results with a command line switch. We can parse
this to get more detailed results than the returncode, and put those in
our own Junit output. We basically just throw away the top level
'testsuites' object, then fixup the names of the tests, and shove that
into our junit.
-rw-r--r-- | docs/markdown/Reference-manual.md | 13 | ||||
-rw-r--r-- | docs/markdown/snippets/gtest_protocol.md | 6 | ||||
-rw-r--r-- | mesonbuild/backend/backends.py | 5 | ||||
-rw-r--r-- | mesonbuild/interpreter.py | 6 | ||||
-rw-r--r-- | mesonbuild/mtest.py | 54 | ||||
-rwxr-xr-x | run_unittests.py | 10 | ||||
-rw-r--r-- | test cases/frameworks/2 gtest/meson.build | 4 |
7 files changed, 78 insertions, 20 deletions
diff --git a/docs/markdown/Reference-manual.md b/docs/markdown/Reference-manual.md index 963af9d..15a438b 100644 --- a/docs/markdown/Reference-manual.md +++ b/docs/markdown/Reference-manual.md @@ -1662,11 +1662,14 @@ test(..., env: nomalloc, ...) before test is executed even if they have `build_by_default : false`. Since 0.46.0 -- `protocol` specifies how the test results are parsed and can be one - of `exitcode` (the executable's exit code is used by the test harness - to record the outcome of the test) or `tap` ([Test Anything - Protocol](https://www.testanything.org/)). For more on the Meson test - harness protocol read [Unit Tests](Unit-tests.md). Since 0.50.0 +- `protocol` *(Since 0.50.0)* specifies how the test results are parsed and can + be one of `exitcode`, `tap`, or `gtest`. For more information about test + harness protocol read [Unit Tests](Unit-tests.md). The following values are + accepted: + - `exitcode`: the executable's exit code is used by the test harness + to record the outcome of the test) + - `tap` ([Test Anything Protocol](https://www.testanything.org/)) + - `gtest`. *(Since 0.55.0)* for Google Tests. - `priority` specifies the priority of a test. Tests with a higher priority are *started* before tests with a lower priority. diff --git a/docs/markdown/snippets/gtest_protocol.md b/docs/markdown/snippets/gtest_protocol.md new file mode 100644 index 0000000..14f3af9 --- /dev/null +++ b/docs/markdown/snippets/gtest_protocol.md @@ -0,0 +1,6 @@ +## Test protocol for gtest + +Due to the popularity of Gtest (google test) among C and C++ developers meson +now supports a special protocol for gtest. With this protocol meson injects +arguments to gtests to output JUnit, reads that JUnit, and adds the output to +the JUnit it generates. diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index ad01011..d41cef1 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -42,6 +42,7 @@ class TestProtocol(enum.Enum): EXITCODE = 0 TAP = 1 + GTEST = 2 @classmethod def from_str(cls, string: str) -> 'TestProtocol': @@ -49,11 +50,15 @@ class TestProtocol(enum.Enum): return cls.EXITCODE elif string == 'tap': return cls.TAP + elif string == 'gtest': + return cls.GTEST raise MesonException('unknown test format {}'.format(string)) def __str__(self) -> str: if self is self.EXITCODE: return 'exitcode' + elif self is self.GTEST: + return 'gtest' return 'tap' diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index 7b8ca63..c0be92a 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -3772,6 +3772,8 @@ This will become a hard error in the future.''' % kwargs['input'], location=self @FeatureNewKwargs('test', '0.52.0', ['priority']) @permittedKwargs(permitted_kwargs['test']) def func_test(self, node, args, kwargs): + if kwargs.get('protocol') == 'gtest': + FeatureNew('"gtest" protocol for tests', '0.55.0').use(self.subproject) self.add_test(node, args, kwargs, True) def unpack_env_kwarg(self, kwargs) -> build.EnvironmentVariables: @@ -3823,8 +3825,8 @@ This will become a hard error in the future.''' % kwargs['input'], location=self if not isinstance(timeout, int): raise InterpreterException('Timeout must be an integer.') protocol = kwargs.get('protocol', 'exitcode') - if protocol not in ('exitcode', 'tap'): - raise InterpreterException('Protocol must be "exitcode" or "tap".') + if protocol not in {'exitcode', 'tap', 'gtest'}: + raise InterpreterException('Protocol must be "exitcode", "tap", or "gtest".') suite = [] prj = self.subproject if self.is_subproject() else self.build.project_name for s in mesonlib.stringlistify(kwargs.get('suite', '')): diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index 69da400..4592c90 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -94,7 +94,10 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: help='List available tests.') parser.add_argument('--wrapper', default=None, dest='wrapper', type=split_args, help='wrapper to run tests with (e.g. Valgrind)') - parser.add_argument('-C', default='.', dest='wd', type=os.path.abspath, + parser.add_argument('-C', default='.', dest='wd', + # https://github.com/python/typeshed/issues/3107 + # https://github.com/python/mypy/issues/7177 + type=os.path.abspath, # type: ignore help='directory to cd into before running') parser.add_argument('--suite', default=[], dest='include_suites', action='append', metavar='SUITE', help='Only run tests belonging to the given suite.') @@ -349,6 +352,19 @@ class JunitBuilder: def log(self, name: str, test: 'TestRun') -> None: """Log a single test case.""" + if test.junit is not None: + for suite in test.junit.findall('.//testsuite'): + # Assume that we don't need to merge anything here... + suite.attrib['name'] = '{}.{}.{}'.format(test.project, name, suite.attrib['name']) + + # GTest can inject invalid attributes + for case in suite.findall('.//testcase[@result]'): + del case.attrib['result'] + for case in suite.findall('.//testcase[@timestamp]'): + del case.attrib['timestamp'] + self.root.append(suite) + return + # In this case we have a test binary with multiple results. # We want to record this so that each result is recorded # separately @@ -430,10 +446,24 @@ class JunitBuilder: class TestRun: @classmethod + def make_gtest(cls, test: 'TestSerialisation', test_env: T.Dict[str, str], + returncode: int, starttime: float, duration: float, + stdo: T.Optional[str], stde: T.Optional[str], + cmd: T.Optional[T.List[str]]) -> 'TestRun': + filename = '{}.xml'.format(test.name) + if test.workdir: + filename = os.path.join(test.workdir, filename) + tree = et.parse(filename) + + return cls.make_exitcode( + test, test_env, returncode, starttime, duration, stdo, stde, cmd, + junit=tree) + + @classmethod def make_exitcode(cls, test: 'TestSerialisation', test_env: T.Dict[str, str], returncode: int, starttime: float, duration: float, stdo: T.Optional[str], stde: T.Optional[str], - cmd: T.Optional[T.List[str]]) -> 'TestRun': + cmd: T.Optional[T.List[str]], **kwargs) -> 'TestRun': if returncode == GNU_SKIP_RETURNCODE: res = TestResult.SKIP elif returncode == GNU_ERROR_RETURNCODE: @@ -442,15 +472,15 @@ class TestRun: res = TestResult.EXPECTEDFAIL if bool(returncode) else TestResult.UNEXPECTEDPASS else: res = TestResult.FAIL if bool(returncode) else TestResult.OK - return cls(test, test_env, res, [], returncode, starttime, duration, stdo, stde, cmd) + return cls(test, test_env, res, [], returncode, starttime, duration, stdo, stde, cmd, **kwargs) @classmethod def make_tap(cls, test: 'TestSerialisation', test_env: T.Dict[str, str], returncode: int, starttime: float, duration: float, stdo: str, stde: str, cmd: T.Optional[T.List[str]]) -> 'TestRun': - res = None # T.Optional[TestResult] - results = [] # T.List[TestResult] + res = None # type: T.Optional[TestResult] + results = [] # type: T.List[TestResult] failed = False for i in TAPParser(io.StringIO(stdo)).parse(): @@ -486,7 +516,7 @@ class TestRun: res: TestResult, results: T.List[TestResult], returncode: int, starttime: float, duration: float, stdo: T.Optional[str], stde: T.Optional[str], - cmd: T.Optional[T.List[str]]): + cmd: T.Optional[T.List[str]], *, junit: T.Optional[et.ElementTree] = None): assert isinstance(res, TestResult) self.res = res self.results = results # May be an empty list @@ -499,6 +529,7 @@ class TestRun: self.env = test_env self.should_fail = test.should_fail self.project = test.project_name + self.junit = junit def get_log(self) -> str: res = '--- command ---\n' @@ -652,7 +683,14 @@ class SingleTestRunner: # errors avoid not being able to use the terminal. os.setsid() # type: ignore - p = subprocess.Popen(cmd, + extra_cmd = [] # type: T.List[str] + if self.test.protocol is TestProtocol.GTEST: + gtestname = '{}.xml'.format(self.test.name) + if self.test.workdir: + gtestname = '{}:{}'.format(self.test.workdir, self.test.name) + extra_cmd.append('--gtest_output=xml:{}'.format(gtestname)) + + p = subprocess.Popen(cmd + extra_cmd, stdout=stdout, stderr=stderr, env=self.env, @@ -744,6 +782,8 @@ class SingleTestRunner: else: if self.test.protocol is TestProtocol.EXITCODE: return TestRun.make_exitcode(self.test, self.test_env, p.returncode, starttime, duration, stdo, stde, cmd) + elif self.test.protocol is TestProtocol.GTEST: + return TestRun.make_gtest(self.test, self.test_env, p.returncode, starttime, duration, stdo, stde, cmd) else: if self.options.verbose: print(stdo, end='') diff --git a/run_unittests.py b/run_unittests.py index da898a3..3826762 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -4625,8 +4625,7 @@ recommended as it is not supported on some platforms''') schema = et.XMLSchema(et.parse(str(Path(__file__).parent / 'data' / 'schema.xsd'))) - testdir = os.path.join(self.common_test_dir, case) - self.init(testdir) + self.init(case) self.run_tests() junit = et.parse(str(Path(self.builddir) / 'meson-logs' / 'testlog.junit.xml')) @@ -4636,10 +4635,13 @@ recommended as it is not supported on some platforms''') self.fail(e.error_log) def test_junit_valid_tap(self): - self._test_junit('213 tap tests') + self._test_junit(os.path.join(self.common_test_dir, '213 tap tests')) def test_junit_valid_exitcode(self): - self._test_junit('44 test args') + self._test_junit(os.path.join(self.common_test_dir, '44 test args')) + + def test_junit_valid_gtest(self): + self._test_junit(os.path.join(self.framework_test_dir, '2 gtest')) class FailureTests(BasePlatformTests): diff --git a/test cases/frameworks/2 gtest/meson.build b/test cases/frameworks/2 gtest/meson.build index 2d93b52..ea3ef48 100644 --- a/test cases/frameworks/2 gtest/meson.build +++ b/test cases/frameworks/2 gtest/meson.build @@ -8,7 +8,7 @@ endif gtest_nomain = dependency('gtest', main : false, method : 'system') e = executable('testprog', 'test.cc', dependencies : gtest) -test('gtest test', e) +test('gtest test', e, protocol : 'gtest') e = executable('testprog_nomain', 'test_nomain.cc', dependencies : gtest_nomain) -test('gtest nomain test', e) +test('gtest nomain test', e, protocol : 'gtest') |