mtest: create separate runners for multiple repeats

Reusing the runners for multiple repeats of the test run gets in the
way of the progress report, which stores runners in an OrderedSet.
Instead, create a separate SingleTestRunner object for each repeat.

While at it, fix the "duplicate suite" assertion as it can fire
with TAP tests and --repeat=N.

Fixes: #8405

2 files changed, 27 insertions, 18 deletions

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index d5bcf94..af5ad6d 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -774,7 +774,7 @@ class JunitBuilder(TestLogger):
         # separately
         if test.results:
             suitename = '{}.{}'.format(test.project, test.name)
-            assert suitename not in self.suites, 'duplicate suite'
+            assert suitename not in self.suites or harness.options.repeat > 1, 'duplicate suite'
 
             suite = self.suites[suitename] = et.Element(
                 'testsuite',
@@ -1627,18 +1627,22 @@ class TestHarness:
             # wrapper script.
             sys.exit(125)
 
-        self.test_count = len(tests)
         self.name_max_len = max([uniwidth(self.get_pretty_suite(test)) for test in tests])
         startdir = os.getcwd()
         try:
             if self.options.wd:
                 os.chdir(self.options.wd)
-            runners = [self.get_test_runner(test) for test in tests]
-            self.duration_max_len = max([len(str(int(runner.timeout or 99)))
-                                         for runner in runners])
-            # Disable the progress report if it gets in the way
-            self.need_console = any((runner.console_mode is not ConsoleUser.LOGGER
-                                     for runner in runners))
+            runners = []             # type: T.List[SingleTestRunner]
+            for i in range(self.options.repeat):
+                runners.extend((self.get_test_runner(test) for test in tests))
+                if i == 0:
+                    self.duration_max_len = max([len(str(int(runner.timeout or 99)))
+                                                 for runner in runners])
+                    # Disable the progress report if it gets in the way
+                    self.need_console = any((runner.console_mode is not ConsoleUser.LOGGER
+                                             for runner in runners))
+
+            self.test_count = len(runners)
             self.run_tests(runners)
         finally:
             os.chdir(startdir)
@@ -1860,16 +1864,15 @@ class TestHarness:
             asyncio.get_event_loop().add_signal_handler(signal.SIGINT, sigint_handler)
             asyncio.get_event_loop().add_signal_handler(signal.SIGTERM, sigterm_handler)
         try:
-            for _ in range(self.options.repeat):
-                for runner in runners:
-                    if not runner.is_parallel:
-                        await complete_all(futures)
-                    future = asyncio.ensure_future(run_test(runner))
-                    futures.append(future)
-                    running_tests[future] = runner.visible_name
-                    future.add_done_callback(test_done)
-                    if not runner.is_parallel:
-                        await complete(future)
+            for runner in runners:
+                if not runner.is_parallel:
+                    await complete_all(futures)
+                future = asyncio.ensure_future(run_test(runner))
+                futures.append(future)
+                running_tests[future] = runner.visible_name
+                future.add_done_callback(test_done)
+                if not runner.is_parallel:
+                    await complete(future)
                 if self.options.repeat > 1 and self.fail_count:
                     break
 
diff --git a/run_unittests.py b/run_unittests.py
index acd775c..53642ca 100755
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -2524,6 +2524,12 @@ class AllPlatformTests(BasePlatformTests):
         self.assertNotRegex(out, r'WARNING: Overriding.*TEST_VAR_SET')
         self.run_tests()
 
+    def test_testrepeat(self):
+        testdir = os.path.join(self.common_test_dir, '207 tap tests')
+        self.init(testdir)
+        self.build()
+        self._run(self.mtest_command + ['--repeat=2'])
+
     def test_testsetups(self):
         if not shutil.which('valgrind'):
             raise unittest.SkipTest('Valgrind not installed.')