mtest: implement TAP parsing

This provides an initial support for parsing TAP output.  It detects failures
and skipped tests without relying on exit code, as well as early termination
of the test due to an error or a crash.

For now, subtests are not recorded in the TestRun object.  However, because the
TAP output goes on stdout, it is printed by --print-errorlogs when a test does
not behave as expected.  Handling subtests as TestRuns, and serializing them
to JSON, can be added later.

The parser was written specifically for Meson, and comes with its own
test suite.

Fixes #2923.

1 files changed, 186 insertions, 1 deletions

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 21e5403..02b728e 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -23,6 +23,9 @@ from mesonbuild.dependencies import ExternalProgram
 from mesonbuild.mesonlib import substring_is_in_list, MesonException
 from mesonbuild import mlog
 
+from collections import namedtuple
+import io
+import re
 import tempfile
 import time, datetime, multiprocessing, json
 import concurrent.futures as conc
@@ -153,6 +156,150 @@ class TestResult(enum.Enum):
     ERROR = 'ERROR'
 
 
+class TAPParser(object):
+    Plan = namedtuple('Plan', ['count', 'late', 'skipped', 'explanation'])
+    Bailout = namedtuple('Bailout', ['message'])
+    Test = namedtuple('Test', ['number', 'name', 'result', 'explanation'])
+    Error = namedtuple('Error', ['message'])
+    Version = namedtuple('Version', ['version'])
+
+    _MAIN = 1
+    _AFTER_TEST = 2
+    _YAML = 3
+
+    _RE_BAILOUT = r'Bail out!\s*(.*)'
+    _RE_DIRECTIVE = r'(?:\s*\#\s*([Ss][Kk][Ii][Pp]\S*|[Tt][Oo][Dd][Oo])\b\s*(.*))?'
+    _RE_PLAN = r'1\.\.([0-9]+)' + _RE_DIRECTIVE
+    _RE_TEST = r'((?:not )?ok)\s*(?:([0-9]+)\s*)?([^#]*)' + _RE_DIRECTIVE
+    _RE_VERSION = r'TAP version ([0-9]+)'
+    _RE_YAML_START = r'(\s+)---.*'
+    _RE_YAML_END = r'\s+\.\.\.\s*'
+
+    def __init__(self, io):
+        self.io = io
+
+    def parse_test(self, ok, num, name, directive, explanation):
+        name = name.strip()
+        explanation = explanation.strip() if explanation else None
+        if directive is not None:
+            directive = directive.upper()
+            if directive == 'SKIP':
+                if ok:
+                    yield self.Test(num, name, TestResult.SKIP, explanation)
+                    return
+            elif directive == 'TODO':
+                yield self.Test(num, name, TestResult.UNEXPECTEDPASS if ok else TestResult.EXPECTEDFAIL, explanation)
+                return
+            else:
+                yield self.Error('invalid directive "%s"' % (directive,))
+
+        yield self.Test(num, name, TestResult.OK if ok else TestResult.FAIL, explanation)
+
+    def parse(self):
+        found_late_test = False
+        bailed_out = False
+        plan = None
+        lineno = 0
+        num_tests = 0
+        yaml_lineno = None
+        yaml_indent = None
+        state = self._MAIN
+        version = 12
+        while True:
+            lineno += 1
+            try:
+                line = next(self.io).rstrip()
+            except StopIteration:
+                break
+
+            # YAML blocks are only accepted after a test
+            if state == self._AFTER_TEST:
+                if version >= 13:
+                    m = re.match(self._RE_YAML_START, line)
+                    if m:
+                        state = self._YAML
+                        yaml_lineno = lineno
+                        yaml_indent = m.group(1)
+                        continue
+                state = self._MAIN
+
+            elif state == self._YAML:
+                if re.match(self._RE_YAML_END, line):
+                    state = self._MAIN
+                    continue
+                if line.startswith(yaml_indent):
+                    continue
+                yield self.Error('YAML block not terminated (started on line %d)' % (yaml_lineno,))
+                state = self._MAIN
+
+            assert state == self._MAIN
+            if line.startswith('#'):
+                continue
+
+            m = re.match(self._RE_TEST, line)
+            if m:
+                if plan and plan.late and not found_late_test:
+                    yield self.Error('unexpected test after late plan')
+                    found_late_test = True
+                num_tests += 1
+                num = num_tests if m.group(2) is None else int(m.group(2))
+                if num != num_tests:
+                    yield self.Error('out of order test numbers')
+                yield from self.parse_test(m.group(1) == 'ok', num,
+                                           m.group(3), m.group(4), m.group(5))
+                state = self._AFTER_TEST
+                continue
+
+            m = re.match(self._RE_PLAN, line)
+            if m:
+                if plan:
+                    yield self.Error('more than one plan found')
+                else:
+                    count = int(m.group(1))
+                    skipped = (count == 0)
+                    if m.group(2):
+                        if m.group(2).upper().startswith('SKIP'):
+                            if count > 0:
+                                yield self.Error('invalid SKIP directive for plan')
+                            skipped = True
+                        else:
+                            yield self.Error('invalid directive for plan')
+                    plan = self.Plan(count=count, late=(num_tests > 0),
+                                     skipped=skipped, explanation=m.group(3))
+                    yield plan
+                continue
+
+            m = re.match(self._RE_BAILOUT, line)
+            if m:
+                yield self.Bailout(m.group(1))
+                bailed_out = True
+                continue
+
+            m = re.match(self._RE_VERSION, line)
+            if m:
+                # The TAP version is only accepted as the first line
+                if lineno != 1:
+                    yield self.Error('version number must be on the first line')
+                    continue
+                version = int(m.group(1))
+                if version < 13:
+                    yield self.Error('version number should be at least 13')
+                else:
+                    yield self.Version(version=version)
+                continue
+
+            yield self.Error('unexpected input at line %d' % (lineno,))
+
+        if state == self._YAML:
+            yield self.Error('YAML block not terminated (started on line %d)' % (yaml_lineno,))
+
+        if not bailed_out and plan and num_tests != plan.count:
+            if num_tests < plan.count:
+                yield self.Error('Too few tests run (expected %d, got %d)' % (plan.count, num_tests))
+            else:
+                yield self.Error('Too many tests run (expected %d, got %d)' % (plan.count, num_tests))
+
+
 class TestRun:
     @staticmethod
     def make_exitcode(test, returncode, duration, stdo, stde, cmd):
@@ -166,6 +313,41 @@ class TestRun:
             res = TestResult.FAIL if bool(returncode) else TestResult.OK
         return TestRun(test, res, returncode, duration, stdo, stde, cmd)
 
+    def make_tap(test, returncode, duration, stdo, stde, cmd):
+        res = None
+        num_tests = 0
+        failed = False
+        num_skipped = 0
+
+        for i in TAPParser(io.StringIO(stdo)).parse():
+            if isinstance(i, TAPParser.Bailout):
+                res = TestResult.ERROR
+            elif isinstance(i, TAPParser.Test):
+                if i.result == TestResult.SKIP:
+                    num_skipped += 1
+                elif i.result in (TestResult.FAIL, TestResult.UNEXPECTEDPASS):
+                    failed = True
+                num_tests += 1
+            elif isinstance(i, TAPParser.Error):
+                res = TestResult.ERROR
+                stde += '\nTAP parsing error: ' + i.message
+
+        if returncode != 0:
+            res = TestResult.ERROR
+            stde += '\n(test program exited with status code %d)' % (returncode,)
+
+        if res is None:
+            # Now determine the overall result of the test based on the outcome of the subcases
+            if num_skipped == num_tests:
+                # This includes the case where num_tests is zero
+                res = TestResult.SKIP
+            elif test.should_fail:
+                res = TestResult.EXPECTEDFAIL if failed else TestResult.UNEXPECTEDPASS
+            else:
+                res = TestResult.FAIL if failed else TestResult.OK
+
+        return TestRun(test, res, returncode, duration, stdo, stde, cmd)
+
     def __init__(self, test, res, returncode, duration, stdo, stde, cmd):
         assert isinstance(res, TestResult)
         self.res = res
@@ -405,7 +587,10 @@ class SingleTestRunner:
         if timed_out:
             return TestRun(self.test, TestResult.TIMEOUT, p.returncode, duration, stdo, stde, cmd)
         else:
-            return TestRun.make_exitcode(self.test, p.returncode, duration, stdo, stde, cmd)
+            if self.test.protocol == 'exitcode':
+                return TestRun.make_exitcode(self.test, p.returncode, duration, stdo, stde, cmd)
+            else:
+                return TestRun.make_tap(self.test, p.returncode, duration, stdo, stde, cmd)
 
 
 class TestHarness: