Report expected failures and unexpected passes separately

This makes it clear in the results that tests marked "should_fail"
exist. We also avoid the all caps output and make the classifications
unambigous compared to pytest or autotools' XFAIL/XPASS.

Before:

  OK:       329
  FAIL:       1
  SKIP:       0
  TIMEOUT:    0

After:

  Ok:                 323
  Expected Fail:        1
  Fail:                 6
  Unexpected Pass:      0
  Skipped:              0
  Timeout:              0

1 files changed, 23 insertions, 11 deletions

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index cee4c5a..883e556 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -144,6 +144,8 @@ class TestResult(enum.Enum):
     TIMEOUT = 'TIMEOUT'
     SKIP = 'SKIP'
     FAIL = 'FAIL'
+    EXPECTEDFAIL = 'EXPECTEDFAIL'
+    UNEXPECTEDPASS = 'UNEXPECTEDPASS'
 
 
 class TestRun:
@@ -389,10 +391,10 @@ class SingleTestRunner:
             res = TestResult.TIMEOUT
         elif p.returncode == GNU_SKIP_RETURNCODE:
             res = TestResult.SKIP
-        elif self.test.should_fail == bool(p.returncode):
-            res = TestResult.OK
+        elif self.test.should_fail:
+            res = TestResult.EXPECTEDFAIL if bool(p.returncode) else TestResult.UNEXPECTEDPASS
         else:
-            res = TestResult.FAIL
+            res = TestResult.FAIL if bool(p.returncode) else TestResult.OK
         return TestRun(res, p.returncode, self.test.should_fail, duration, stdo, stde, cmd, self.test.env)
 
 
@@ -401,6 +403,8 @@ class TestHarness:
         self.options = options
         self.collected_logs = []
         self.fail_count = 0
+        self.expectedfail_count = 0
+        self.unexpectedpass_count = 0
         self.success_count = 0
         self.skip_count = 0
         self.timeout_count = 0
@@ -465,6 +469,10 @@ class TestHarness:
             self.success_count += 1
         elif result.res is TestResult.FAIL:
             self.fail_count += 1
+        elif result.res is TestResult.EXPECTEDFAIL:
+            self.expectedfail_count += 1
+        elif result.res is TestResult.UNEXPECTEDPASS:
+            self.unexpectedpass_count += 1
         else:
             sys.exit('Unknown test result encountered: {}'.format(result.res))
 
@@ -480,9 +488,10 @@ class TestHarness:
         result_str = '%s %s  %s%s%s%5.2f s %s' % \
             (num, name, padding1, result.res.value, padding2, result.duration,
              status)
-        if not self.options.quiet or result.res is not TestResult.OK:
-            if result.res is not TestResult.OK and mlog.colorize_console:
-                if result.res in (TestResult.FAIL, TestResult.TIMEOUT):
+        ok_statuses = (TestResult.OK, TestResult.EXPECTEDFAIL)
+        if not self.options.quiet or result.res not in ok_statuses:
+            if result.res not in ok_statuses and mlog.colorize_console:
+                if result.res in (TestResult.FAIL, TestResult.TIMEOUT, TestResult.UNEXPECTEDPASS):
                     decorator = mlog.red
                 elif result.res is TestResult.SKIP:
                     decorator = mlog.yellow
@@ -503,11 +512,14 @@ class TestHarness:
 
     def print_summary(self):
         msg = '''
-OK:      %4d
-FAIL:    %4d
-SKIP:    %4d
-TIMEOUT: %4d
-''' % (self.success_count, self.fail_count, self.skip_count, self.timeout_count)
+Ok:                 %4d
+Expected Fail:      %4d
+Fail:               %4d
+Unexpected Pass:    %4d
+Skipped:            %4d
+Timeout:            %4d
+''' % (self.success_count, self.expectedfail_count, self.fail_count,
+        self.unexpectedpass_count, self.skip_count, self.timeout_count)
         print(msg)
         if self.logfile:
             self.logfile.write(msg)