All testing is now in mesontest.py, which simplifies a lot of stuff.

6 files changed, 294 insertions, 344 deletions

diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index 5ca569b..8f64db9 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -711,7 +711,10 @@ int dummy;
 
     def generate_tests(self, outfile):
         (test_data, benchmark_data) = self.serialise_tests()
-        cmd = [ sys.executable, self.environment.get_build_command(), '--internal', 'test' ]
+        meson_exe = self.environment.get_build_command()
+        (base, ext) = os.path.splitext(meson_exe)
+        test_exe = base + 'test' + ext
+        cmd = [sys.executable, test_exe]
         if not self.environment.coredata.get_builtin_option('stdsplit'):
             cmd += ['--no-stdsplit']
         if self.environment.coredata.get_builtin_option('errorlogs'):
@@ -724,7 +727,8 @@ int dummy;
         elem.write(outfile)
 
         # And then benchmarks.
-        cmd = [sys.executable, self.environment.get_build_command(), '--internal', 'benchmark', benchmark_data]
+        cmd = [sys.executable, test_exe, '--benchmark','--logbase',
+               'benchmarklog', '--num-processes=1', benchmark_data]
         elem = NinjaBuildElement(self.all_outputs, 'benchmark', 'CUSTOM_COMMAND', ['all', 'PHONY'])
         elem.add_item('COMMAND', cmd)
         elem.add_item('DESC', 'Running benchmark suite.')
diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py
index 88826f8..57c814c 100644
--- a/mesonbuild/mesonmain.py
+++ b/mesonbuild/mesonmain.py
@@ -192,12 +192,6 @@ def run_script_command(args):
     if cmdname == 'exe':
         import mesonbuild.scripts.meson_exe as abc
         cmdfunc = abc.run
-    elif cmdname == 'test':
-        import mesonbuild.scripts.meson_test as abc
-        cmdfunc = abc.run
-    elif cmdname == 'benchmark':
-        import mesonbuild.scripts.meson_benchmark as abc
-        cmdfunc = abc.run
     elif cmdname == 'install':
         import mesonbuild.scripts.meson_install as abc
         cmdfunc = abc.run
diff --git a/mesonbuild/scripts/meson_benchmark.py b/mesonbuild/scripts/meson_benchmark.py
deleted file mode 100755
index 553fc89..0000000
--- a/mesonbuild/scripts/meson_benchmark.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright 2015-2016 The Meson development team
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-
-#     http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-from . import meson_test
-
-def run(args):
-    return meson_test.run(args + ['--logbase', 'benchmarklog', '--num-processes=1'])
-
-if __name__ == '__main__':
-    sys.exit(run(sys.argv[1:]))
diff --git a/mesonbuild/scripts/meson_test.py b/mesonbuild/scripts/meson_test.py
deleted file mode 100755
index 42f2c8e..0000000
--- a/mesonbuild/scripts/meson_test.py
+++ /dev/null
@@ -1,300 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright 2013-2016 The Meson development team
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-
-#     http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import mesonbuild
-from .. import build
-import sys, os, subprocess, time, datetime, pickle, multiprocessing, json
-import concurrent.futures as conc
-import argparse
-import platform
-import signal
-
-def is_windows():
-    platname = platform.system().lower()
-    return platname == 'windows' or 'mingw' in platname
-
-def determine_worker_count():
-    varname = 'MESON_TESTTHREADS'
-    if varname in os.environ:
-        try:
-            num_workers = int(os.environ[varname])
-        except ValueError:
-            print('Invalid value in %s, using 1 thread.' % varname)
-            num_workers = 1
-    else:
-        try:
-            # Fails in some weird environments such as Debian
-            # reproducible build.
-            num_workers = multiprocessing.cpu_count()
-        except Exception:
-            num_workers = 1
-    return num_workers
-
-parser = argparse.ArgumentParser()
-parser.add_argument('--wrapper', default=None, dest='wrapper',
-                    help='wrapper to run tests with (e.g. valgrind)')
-parser.add_argument('--wd', default=None, dest='wd',
-                    help='directory to cd into before running')
-parser.add_argument('--suite', default=None, dest='suite',
-                    help='Only run tests belonging to this suite.')
-parser.add_argument('--no-stdsplit', default=True, dest='split', action='store_false',
-                    help='Do not split stderr and stdout in test logs.')
-parser.add_argument('--print-errorlogs', default=False, action='store_true',
-                    help="Whether to print faling tests' logs.")
-parser.add_argument('--logbase', default='testlog',
-                    help="Base name for log file.")
-parser.add_argument('--num-processes', default=determine_worker_count(), type=int,
-                    help='How many parallel processes to use.')
-parser.add_argument('args', nargs='*')
-
-
-class TestRun():
-    def __init__(self, res, returncode, should_fail, duration, stdo, stde, cmd,
-                 env):
-        self.res = res
-        self.returncode = returncode
-        self.duration = duration
-        self.stdo = stdo
-        self.stde = stde
-        self.cmd = cmd
-        self.env = env
-        self.should_fail = should_fail
-
-    def get_log(self):
-        res = '--- command ---\n'
-        if self.cmd is None:
-            res += 'NONE\n'
-        else:
-            res += "\n%s %s\n" %(' '.join(
-                ["%s='%s'" % (k, v) for k, v in self.env.items()]),
-                ' ' .join(self.cmd))
-        if self.stdo:
-            res += '--- stdout ---\n'
-            res += self.stdo
-        if self.stde:
-            if res[-1:] != '\n':
-                res += '\n'
-            res += '--- stderr ---\n'
-            res += self.stde
-        if res[-1:] != '\n':
-            res += '\n'
-        res += '-------\n\n'
-        return res
-
-def decode(stream):
-    try:
-        return stream.decode('utf-8')
-    except UnicodeDecodeError:
-        return stream.decode('iso-8859-1', errors='ignore')
-
-def write_json_log(jsonlogfile, test_name, result):
-    jresult = {'name' : test_name,
-              'stdout' : result.stdo,
-              'result' : result.res,
-              'duration' : result.duration,
-              'returncode' : result.returncode,
-              'command' : result.cmd,
-              'env' : result.env}
-    if result.stde:
-        jresult['stderr'] = result.stde
-    jsonlogfile.write(json.dumps(jresult) + '\n')
-
-def run_with_mono(fname):
-    if fname.endswith('.exe') and not is_windows():
-        return True
-    return False
-
-class TestHarness:
-    def __init__(self, options):
-        self.options = options
-        self.collected_logs = []
-        self.error_count = 0
-        self.datafile = options.args[0]
-
-    def run_single_test(self, wrap, test):
-        if test.fname[0].endswith('.jar'):
-            cmd = ['java', '-jar'] + test.fname
-        elif not test.is_cross and run_with_mono(test.fname[0]):
-            cmd = ['mono'] + test.fname
-        else:
-            if test.is_cross:
-                if test.exe_runner is None:
-                    # Can not run test on cross compiled executable
-                    # because there is no execute wrapper.
-                    cmd = None
-                else:
-                    cmd = [test.exe_runner] + test.fname
-            else:
-                cmd = test.fname
-        if cmd is None:
-            res = 'SKIP'
-            duration = 0.0
-            stdo = 'Not run because can not execute cross compiled binaries.'
-            stde = None
-            returncode = -1
-        else:
-            cmd = wrap + cmd + test.cmd_args
-            starttime = time.time()
-            child_env = os.environ.copy()
-            if isinstance(test.env, build.EnvironmentVariables):
-                test.env = test.env.get_env(child_env)
-
-            child_env.update(test.env)
-            if len(test.extra_paths) > 0:
-                child_env['PATH'] = child_env['PATH'] + os.pathsep.join([''] + test.extra_paths)
-            if is_windows():
-                setsid = None
-            else:
-                setsid = os.setsid
-            p = subprocess.Popen(cmd,
-                                 stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE if self.options and self.options.split else subprocess.STDOUT,
-                                 env=child_env,
-                                 cwd=test.workdir,
-                                 preexec_fn=setsid)
-            timed_out = False
-            try:
-                (stdo, stde) = p.communicate(timeout=test.timeout)
-            except subprocess.TimeoutExpired:
-                timed_out = True
-                # Python does not provide multiplatform support for
-                # killing a process and all its children so we need
-                # to roll our own.
-                if is_windows():
-                    subprocess.call(['taskkill', '/F', '/T', '/PID', str(p.pid)])
-                else:
-                    os.killpg(os.getpgid(p.pid), signal.SIGKILL)
-                (stdo, stde) = p.communicate()
-            endtime = time.time()
-            duration = endtime - starttime
-            stdo = decode(stdo)
-            if stde:
-                stde = decode(stde)
-            if timed_out:
-                res = 'TIMEOUT'
-            elif (not test.should_fail and p.returncode == 0) or \
-                (test.should_fail and p.returncode != 0):
-                res = 'OK'
-            else:
-                res = 'FAIL'
-            returncode = p.returncode
-        return TestRun(res, returncode, test.should_fail, duration, stdo, stde, cmd, test.env)
-
-    def print_stats(self, numlen, tests, name, result, i, logfile, jsonlogfile):
-        startpad = ' '*(numlen - len('%d' % (i+1)))
-        num = '%s%d/%d' % (startpad, i+1, len(tests))
-        padding1 = ' '*(38-len(name))
-        padding2 = ' '*(8-len(result.res))
-        result_str = '%s %s  %s%s%s%5.2f s' % \
-            (num, name, padding1, result.res, padding2, result.duration)
-        print(result_str)
-        result_str += "\n\n" + result.get_log()
-        if (result.returncode != 0) != result.should_fail:
-            self.error_count += 1
-            if self.options.print_errorlogs:
-                self.collected_logs.append(result_str)
-        logfile.write(result_str)
-        write_json_log(jsonlogfile, name, result)
-
-    def doit(self):
-        datafilename = self.options.args[0]
-        logfilename = self.run_tests(datafilename, self.options.logbase)
-        if len(self.collected_logs) > 0:
-            if len(self.collected_logs) > 10:
-                print('\nThe output from 10 first failed tests:\n')
-            else:
-                print('\nThe output from the failed tests:\n')
-            for log in self.collected_logs[:10]:
-                lines = log.splitlines()
-                if len(lines) > 100:
-                    print(lines[0])
-                    print('--- Listing only the last 100 lines from a long log. ---')
-                    lines = lines[-99:]
-                for line in lines:
-                    print(line)
-        print('Full log written to %s.' % logfilename)
-        return self.error_count
-
-    def run_tests(self, datafilename, log_base):
-        logfile_base = os.path.join('meson-logs', log_base)
-        if self.options.wrapper is None:
-            wrap = []
-            logfilename = logfile_base + '.txt'
-            jsonlogfilename = logfile_base+ '.json'
-        else:
-            wrap = self.options.wrapper.split()
-            namebase = wrap[0]
-            logfilename = logfile_base + '-' + namebase.replace(' ', '_') + '.txt'
-            jsonlogfilename = logfile_base + '-' + namebase.replace(' ', '_') + '.json'
-        with open(datafilename, 'rb') as f:
-            tests = pickle.load(f)
-        if len(tests) == 0:
-            print('No tests defined.')
-            return
-        numlen = len('%d' % len(tests))
-        executor = conc.ThreadPoolExecutor(max_workers=self.options.num_processes)
-        futures = []
-        filtered_tests = filter_tests(self.options.suite, tests)
-
-        with open(jsonlogfilename, 'w') as jsonlogfile, \
-             open(logfilename, 'w') as logfile:
-            logfile.write('Log of Meson test suite run on %s.\n\n' %
-                          datetime.datetime.now().isoformat())
-            for i, test in enumerate(filtered_tests):
-                if test.suite[0] == '':
-                    visible_name = test.name
-                else:
-                    if self.options.suite is not None:
-                        visible_name = self.options.suite + ' / ' + test.name
-                    else:
-                        visible_name = test.suite[0] + ' / ' + test.name
-
-                if not test.is_parallel:
-                    self.drain_futures(futures)
-                    futures = []
-                    res = self.run_single_test(wrap, test)
-                    print_stats(numlen, filtered_tests, visible_name, res, i,
-                                logfile, jsonlogfile)
-                else:
-                    f = executor.submit(self.run_single_test, wrap, test)
-                    futures.append((f, numlen, filtered_tests, visible_name, i,
-                                    logfile, jsonlogfile))
-            self.drain_futures(futures)
-        return logfilename
-
-
-    def drain_futures(self, futures):
-        for i in futures:
-            (result, numlen, tests, name, i, logfile, jsonlogfile) = i
-            self.print_stats(numlen, tests, name, result.result(), i, logfile, jsonlogfile)
-
-def filter_tests(suite, tests):
-    if suite is None:
-        return tests
-    return [x for x in tests if suite in x.suite]
-
-def run(args):
-    options = parser.parse_args(args)
-    if len(options.args) != 1:
-        print('Test runner for Meson. Do not run on your own, mmm\'kay?')
-        print('%s [data file]' % sys.argv[0])
-    if options.wd is not None:
-        os.chdir(options.wd)
-    th = TestHarness(options)
-    return th.doit()
-
-if __name__ == '__main__':
-    sys.exit(run(sys.argv[1:]))
diff --git a/mesontest.py b/mesontest.py
index 3a2b0a3..135d463 100755
--- a/mesontest.py
+++ b/mesontest.py
@@ -18,19 +18,291 @@
 
 import subprocess, sys, os, argparse
 import pickle
-from mesonbuild.scripts import meson_test, meson_benchmark
+import mesonbuild
+from mesonbuild import build
 from mesonbuild import environment
 
+import time, datetime, pickle, multiprocessing, json
+import concurrent.futures as conc
+import platform
+import signal
+
+def is_windows():
+    platname = platform.system().lower()
+    return platname == 'windows' or 'mingw' in platname
+
+def determine_worker_count():
+    varname = 'MESON_TESTTHREADS'
+    if varname in os.environ:
+        try:
+            num_workers = int(os.environ[varname])
+        except ValueError:
+            print('Invalid value in %s, using 1 thread.' % varname)
+            num_workers = 1
+    else:
+        try:
+            # Fails in some weird environments such as Debian
+            # reproducible build.
+            num_workers = multiprocessing.cpu_count()
+        except Exception:
+            num_workers = 1
+    return num_workers
+
 parser = argparse.ArgumentParser()
 parser.add_argument('--repeat', default=1, dest='repeat', type=int,
                     help='Number of times to run the tests.')
-parser.add_argument('--wrapper', default='', dest='wrapper',
-                    help='Exe wrapper (such as Valgrind) to use')
 parser.add_argument('--gdb', default=False, dest='gdb', action='store_true',
                     help='Run test under gdb.')
 parser.add_argument('--list', default=False, dest='list', action='store_true',
                     help='List available tests.')
-parser.add_argument('tests', nargs='*')
+parser.add_argument('--wrapper', default=None, dest='wrapper',
+                    help='wrapper to run tests with (e.g. Valgrind)')
+parser.add_argument('--wd', default=None, dest='wd',
+                    help='directory to cd into before running')
+parser.add_argument('--suite', default=None, dest='suite',
+                    help='Only run tests belonging to the given suite.')
+parser.add_argument('--no-stdsplit', default=True, dest='split', action='store_false',
+                    help='Do not split stderr and stdout in test logs.')
+parser.add_argument('--print-errorlogs', default=False, action='store_true',
+                    help="Whether to print faling tests' logs.")
+parser.add_argument('--benchmark', default=False, action='store_true',
+                    help="Run benchmarks instead of tests.")
+parser.add_argument('--logbase', default='testlog',
+                    help="Base name for log file.")
+parser.add_argument('--num-processes', default=determine_worker_count(), type=int,
+                    help='How many parallel processes to use.')
+parser.add_argument('args', nargs='*')
+
+class TestRun():
+    def __init__(self, res, returncode, should_fail, duration, stdo, stde, cmd,
+                 env):
+        self.res = res
+        self.returncode = returncode
+        self.duration = duration
+        self.stdo = stdo
+        self.stde = stde
+        self.cmd = cmd
+        self.env = env
+        self.should_fail = should_fail
+
+    def get_log(self):
+        res = '--- command ---\n'
+        if self.cmd is None:
+            res += 'NONE\n'
+        else:
+            res += "\n%s %s\n" %(' '.join(
+                ["%s='%s'" % (k, v) for k, v in self.env.items()]),
+                ' ' .join(self.cmd))
+        if self.stdo:
+            res += '--- stdout ---\n'
+            res += self.stdo
+        if self.stde:
+            if res[-1:] != '\n':
+                res += '\n'
+            res += '--- stderr ---\n'
+            res += self.stde
+        if res[-1:] != '\n':
+            res += '\n'
+        res += '-------\n\n'
+        return res
+
+def decode(stream):
+    try:
+        return stream.decode('utf-8')
+    except UnicodeDecodeError:
+        return stream.decode('iso-8859-1', errors='ignore')
+
+def write_json_log(jsonlogfile, test_name, result):
+    jresult = {'name' : test_name,
+              'stdout' : result.stdo,
+              'result' : result.res,
+              'duration' : result.duration,
+              'returncode' : result.returncode,
+              'command' : result.cmd,
+              'env' : result.env}
+    if result.stde:
+        jresult['stderr'] = result.stde
+    jsonlogfile.write(json.dumps(jresult) + '\n')
+
+def run_with_mono(fname):
+    if fname.endswith('.exe') and not is_windows():
+        return True
+    return False
+
+class TestHarness:
+    def __init__(self, options):
+        self.options = options
+        self.collected_logs = []
+        self.error_count = 0
+        self.is_run = False
+
+    def run_single_test(self, wrap, test):
+        if test.fname[0].endswith('.jar'):
+            cmd = ['java', '-jar'] + test.fname
+        elif not test.is_cross and run_with_mono(test.fname[0]):
+            cmd = ['mono'] + test.fname
+        else:
+            if test.is_cross:
+                if test.exe_runner is None:
+                    # Can not run test on cross compiled executable
+                    # because there is no execute wrapper.
+                    cmd = None
+                else:
+                    cmd = [test.exe_runner] + test.fname
+            else:
+                cmd = test.fname
+        if cmd is None:
+            res = 'SKIP'
+            duration = 0.0
+            stdo = 'Not run because can not execute cross compiled binaries.'
+            stde = None
+            returncode = -1
+        else:
+            cmd = wrap + cmd + test.cmd_args
+            starttime = time.time()
+            child_env = os.environ.copy()
+            if isinstance(test.env, build.EnvironmentVariables):
+                test.env = test.env.get_env(child_env)
+
+            child_env.update(test.env)
+            if len(test.extra_paths) > 0:
+                child_env['PATH'] = child_env['PATH'] + ';'.join([''] + test.extra_paths)
+            if is_windows():
+                setsid = None
+            else:
+                setsid = os.setsid
+            p = subprocess.Popen(cmd,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE if self.options and self.options.split else subprocess.STDOUT,
+                                 env=child_env,
+                                 cwd=test.workdir,
+                                 preexec_fn=setsid)
+            timed_out = False
+            try:
+                (stdo, stde) = p.communicate(timeout=test.timeout)
+            except subprocess.TimeoutExpired:
+                timed_out = True
+                # Python does not provide multiplatform support for
+                # killing a process and all its children so we need
+                # to roll our own.
+                if is_windows():
+                    subprocess.call(['taskkill', '/F', '/T', '/PID', str(p.pid)])
+                else:
+                    os.killpg(os.getpgid(p.pid), signal.SIGKILL)
+                (stdo, stde) = p.communicate()
+            endtime = time.time()
+            duration = endtime - starttime
+            stdo = decode(stdo)
+            if stde:
+                stde = decode(stde)
+            if timed_out:
+                res = 'TIMEOUT'
+            elif (not test.should_fail and p.returncode == 0) or \
+                (test.should_fail and p.returncode != 0):
+                res = 'OK'
+            else:
+                res = 'FAIL'
+            returncode = p.returncode
+        return TestRun(res, returncode, test.should_fail, duration, stdo, stde, cmd, test.env)
+
+    def print_stats(self, numlen, tests, name, result, i, logfile, jsonlogfile):
+        startpad = ' '*(numlen - len('%d' % (i+1)))
+        num = '%s%d/%d' % (startpad, i+1, len(tests))
+        padding1 = ' '*(38-len(name))
+        padding2 = ' '*(8-len(result.res))
+        result_str = '%s %s  %s%s%s%5.2f s' % \
+            (num, name, padding1, result.res, padding2, result.duration)
+        print(result_str)
+        result_str += "\n\n" + result.get_log()
+        if (result.returncode != 0) != result.should_fail:
+            self.error_count += 1
+            if self.options.print_errorlogs:
+                self.collected_logs.append(result_str)
+        logfile.write(result_str)
+        write_json_log(jsonlogfile, name, result)
+
+    def doit(self):
+        if self.options.benchmark:
+            datafile = 'meson-private/meson_benchmark_setup.dat'
+        else:
+            datafile = 'meson-private/meson_test_setup.dat'
+        if self.is_run:
+            raise RuntimeError('Test harness object can only be used once.')
+        self.is_run = True
+        logfilename = self.run_tests(datafile, self.options.logbase)
+        if len(self.collected_logs) > 0:
+            if len(self.collected_logs) > 10:
+                print('\nThe output from 10 first failed tests:\n')
+            else:
+                print('\nThe output from the failed tests:\n')
+            for log in self.collected_logs[:10]:
+                lines = log.splitlines()
+                if len(lines) > 100:
+                    print(lines[0])
+                    print('--- Listing only the last 100 lines from a long log. ---')
+                    lines = lines[-99:]
+                for line in lines:
+                    print(line)
+        print('Full log written to %s.' % logfilename)
+        return self.error_count
+
+    def run_tests(self, datafilename, log_base):
+        logfile_base = os.path.join('meson-logs', log_base)
+        if self.options.wrapper is None:
+            wrap = []
+            logfilename = logfile_base + '.txt'
+            jsonlogfilename = logfile_base+ '.json'
+        else:
+            wrap = self.options.wrapper.split()
+            namebase = wrap[0]
+            logfilename = logfile_base + '-' + namebase.replace(' ', '_') + '.txt'
+            jsonlogfilename = logfile_base + '-' + namebase.replace(' ', '_') + '.json'
+        with open(datafilename, 'rb') as f:
+            tests = pickle.load(f)
+        if len(tests) == 0:
+            print('No tests defined.')
+            return
+        numlen = len('%d' % len(tests))
+        executor = conc.ThreadPoolExecutor(max_workers=self.options.num_processes)
+        futures = []
+        filtered_tests = filter_tests(self.options.suite, tests)
+
+        with open(jsonlogfilename, 'w') as jsonlogfile, \
+             open(logfilename, 'w') as logfile:
+            logfile.write('Log of Meson test suite run on %s.\n\n' %
+                          datetime.datetime.now().isoformat())
+            for i, test in enumerate(filtered_tests):
+                if test.suite[0] == '':
+                    visible_name = test.name
+                else:
+                    if self.options.suite is not None:
+                        visible_name = self.options.suite + ' / ' + test.name
+                    else:
+                        visible_name = test.suite[0] + ' / ' + test.name
+
+                if not test.is_parallel:
+                    self.drain_futures(futures)
+                    futures = []
+                    res = self.run_single_test(wrap, test)
+                    self.print_stats(numlen, filtered_tests, visible_name, res, i,
+                                     logfile, jsonlogfile)
+                else:
+                    f = executor.submit(self.run_single_test, wrap, test)
+                    futures.append((f, numlen, filtered_tests, visible_name, i,
+                                    logfile, jsonlogfile))
+            self.drain_futures(futures)
+        return logfilename
+
+
+    def drain_futures(self, futures):
+        for i in futures:
+            (result, numlen, tests, name, i, logfile, jsonlogfile) = i
+            self.print_stats(numlen, tests, name, result.result(), i, logfile, jsonlogfile)
+
+def filter_tests(suite, tests):
+    if suite is None:
+        return tests
+    return [x for x in tests if suite in x.suite]
 
 def gdbrun(test):
     child_env = os.environ.copy()
@@ -53,15 +325,19 @@ def gdbrun(test):
     p.communicate()
 
 def run(args):
-    datafile = 'meson-private/meson_test_setup.dat'
+    options = parser.parse_args(args)
+    if options.benchmark:
+        options.num_processes = 1
+    th = TestHarness(options)
+    return th.doit()
     if not os.path.isfile(datafile):
         print('Test data file. Probably this means that you did not run this in the build directory.')
         return 1
+    datafile = os.path.join(os.curdir, datafile)
     if os.path.isfile('build.ninja'):
         subprocess.check_call([environment.detect_ninja(), 'all'])
-    if len(args) > 0 and args[0] == '--benchmark':
-        return meson_benchmark.run(args[1:] + ['meson-private/meson_benchmark_setup.dat'])
-    options = parser.parse_args(args)
+    if options.wd is not None:
+        os.chdir(options.wd)
     if len(options.tests) == 0:
         # Run basic tests.
         return meson_test.run(args + ['meson-private/meson_test_setup.dat'])
@@ -84,7 +360,7 @@ def run(args):
                 if options.gdb:
                     gdbrun(t)
                 else:
-                    res = meson_test.run_single_test(wrap, t)
+                    res = run_single_test(wrap, t)
                     if (res.returncode == 0 and res.should_fail) or \
                         (res.returncode != 0 and not res.should_fail):
                         print(res.stdo)
diff --git a/run_project_tests.py b/run_project_tests.py
index 15d656b..dcc6006 100755
--- a/run_project_tests.py
+++ b/run_project_tests.py
@@ -19,12 +19,12 @@ import os, subprocess, shutil, sys, signal
 from io import StringIO
 from ast import literal_eval
 import sys, tempfile
+import mesontest
 from mesonbuild import environment
 from mesonbuild import mesonlib
 from mesonbuild import mlog
 from mesonbuild import mesonmain
 from mesonbuild.mesonlib import stringlistify
-from mesonbuild.scripts import meson_test, meson_benchmark
 import argparse
 import xml.etree.ElementTree as ET
 import time
@@ -211,8 +211,8 @@ def run_test_inprocess(testdir):
     old_cwd = os.getcwd()
     os.chdir(testdir)
     try:
-        returncode_test = meson_test.run(['meson-private/meson_test_setup.dat'])
-        returncode_benchmark = meson_benchmark.run(['meson-private/meson_benchmark_setup.dat'])
+        returncode_test = mesontest.run([])
+        returncode_benchmark = mesontest.run(['--benchmark', '--logbase', 'benchmarklog'])
     finally:
         sys.stdout = old_stdout
         sys.stderr = old_stderr