Report xpass results as failures.

4 files changed, 25 insertions, 3 deletions

diff --git a/docs/markdown/snippets/xfail.md b/docs/markdown/snippets/xfail.md
new file mode 100644
index 0000000..5392fa9
--- /dev/null
+++ b/docs/markdown/snippets/xfail.md
@@ -0,0 +1,15 @@
+## Tests that should fail but did not are now errors
+
+You can tag a test as needing to fail like this:
+
+```meson
+test('shoulfail', exe, should_fail: true)
+```
+
+If the test passes the problem is reported in the error logs but due
+to a bug it was not reported in the test runner's exit code. Starting
+from this release the unexpected passes are properly reported in the
+test runner's exit code. This means that test runs that were passing
+in earlier versions of Meson will report failures with the current
+version. This is a good thing, though, since it reveals an error in
+your test suite that has, until now, gone unnoticed.
diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 0f15690..17af4df 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -664,7 +664,6 @@ class TestHarness:
     def process_test_result(self, result):
         if result.res is TestResult.TIMEOUT:
             self.timeout_count += 1
-            self.fail_count += 1
         elif result.res is TestResult.SKIP:
             self.skip_count += 1
         elif result.res is TestResult.OK:
@@ -746,6 +745,9 @@ Timeout:            %4d
                         line = line.encode('ascii', errors='replace').decode()
                         print(line)
 
+    def total_failure_count(self):
+        return self.fail_count + self.unexpectedpass_count + self.timeout_count
+
     def doit(self):
         if self.is_run:
             raise RuntimeError('Test harness object can only be used once.')
@@ -754,7 +756,7 @@ Timeout:            %4d
         if not tests:
             return 0
         self.run_tests(tests)
-        return self.fail_count
+        return self.total_failure_count()
 
     @staticmethod
     def split_suite_string(suite):
@@ -939,7 +941,7 @@ Timeout:            %4d
         if not tests:
             return 0
         self.run_tests(tests)
-        return self.fail_count
+        return self.total_failure_count()
 
 
 def list_tests(th):
diff --git a/test cases/failing test/6 xpass/meson.build b/test cases/failing test/6 xpass/meson.build
new file mode 100644
index 0000000..7649dde
--- /dev/null
+++ b/test cases/failing test/6 xpass/meson.build
@@ -0,0 +1,4 @@
+project('unexpected pass', 'c')
+
+test('should_fail_but_does_not', executable('xpass', 'xpass.c'),
+     should_fail: true)
diff --git a/test cases/failing test/6 xpass/xpass.c b/test cases/failing test/6 xpass/xpass.c
new file mode 100644
index 0000000..0314ff1
--- /dev/null
+++ b/test cases/failing test/6 xpass/xpass.c
@@ -0,0 +1 @@
+int main(int argc, char **argv) { return 0; }