diff options
Diffstat (limited to 'mesonbuild/mtest.py')
| -rw-r--r-- | mesonbuild/mtest.py | 624 |
1 files changed, 624 insertions, 0 deletions
diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py new file mode 100644 index 0000000..2520ae8 --- /dev/null +++ b/mesonbuild/mtest.py @@ -0,0 +1,624 @@ +# Copyright 2016-2017 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A tool to run tests in many different ways. + +import shlex +import subprocess, sys, os, argparse +import pickle +from mesonbuild import build +from mesonbuild import environment +from mesonbuild.dependencies import ExternalProgram +from mesonbuild import mesonlib +from mesonbuild import mlog + +import time, datetime, multiprocessing, json +import concurrent.futures as conc +import platform +import signal +import random + +# GNU autotools interprets a return code of 77 from tests it executes to +# mean that the test should be skipped. +GNU_SKIP_RETURNCODE = 77 + +def is_windows(): + platname = platform.system().lower() + return platname == 'windows' or 'mingw' in platname + +def is_cygwin(): + platname = platform.system().lower() + return 'cygwin' in platname + +def determine_worker_count(): + varname = 'MESON_TESTTHREADS' + if varname in os.environ: + try: + num_workers = int(os.environ[varname]) + except ValueError: + print('Invalid value in %s, using 1 thread.' % varname) + num_workers = 1 + else: + try: + # Fails in some weird environments such as Debian + # reproducible build. + num_workers = multiprocessing.cpu_count() + except Exception: + num_workers = 1 + return num_workers + +parser = argparse.ArgumentParser() +parser.add_argument('--repeat', default=1, dest='repeat', type=int, + help='Number of times to run the tests.') +parser.add_argument('--no-rebuild', default=False, action='store_true', + help='Do not rebuild before running tests.') +parser.add_argument('--gdb', default=False, dest='gdb', action='store_true', + help='Run test under gdb.') +parser.add_argument('--list', default=False, dest='list', action='store_true', + help='List available tests.') +parser.add_argument('--wrapper', default=None, dest='wrapper', type=shlex.split, + help='wrapper to run tests with (e.g. Valgrind)') +parser.add_argument('-C', default='.', dest='wd', + help='directory to cd into before running') +parser.add_argument('--suite', default=[], dest='include_suites', action='append', metavar='SUITE', + help='Only run tests belonging to the given suite.') +parser.add_argument('--no-suite', default=[], dest='exclude_suites', action='append', metavar='SUITE', + help='Do not run tests belonging to the given suite.') +parser.add_argument('--no-stdsplit', default=True, dest='split', action='store_false', + help='Do not split stderr and stdout in test logs.') +parser.add_argument('--print-errorlogs', default=False, action='store_true', + help="Whether to print failing tests' logs.") +parser.add_argument('--benchmark', default=False, action='store_true', + help="Run benchmarks instead of tests.") +parser.add_argument('--logbase', default='testlog', + help="Base name for log file.") +parser.add_argument('--num-processes', default=determine_worker_count(), type=int, + help='How many parallel processes to use.') +parser.add_argument('-v', '--verbose', default=False, action='store_true', + help='Do not redirect stdout and stderr') +parser.add_argument('-q', '--quiet', default=False, action='store_true', + help='Produce less output to the terminal.') +parser.add_argument('-t', '--timeout-multiplier', type=float, default=None, + help='Define a multiplier for test timeout, for example ' + ' when running tests in particular conditions they might take' + ' more time to execute.') +parser.add_argument('--setup', default=None, dest='setup', + help='Which test setup to use.') +parser.add_argument('--test-args', default=[], type=shlex.split, + help='Arguments to pass to the specified test(s) or all tests') +parser.add_argument('args', nargs='*', + help='Optional list of tests to run') + + +class TestException(mesonlib.MesonException): + pass + + +class TestRun: + def __init__(self, res, returncode, should_fail, duration, stdo, stde, cmd, + env): + self.res = res + self.returncode = returncode + self.duration = duration + self.stdo = stdo + self.stde = stde + self.cmd = cmd + self.env = env + self.should_fail = should_fail + + def get_log(self): + res = '--- command ---\n' + if self.cmd is None: + res += 'NONE\n' + else: + res += "%s%s\n" % (''.join(["%s='%s' " % (k, v) for k, v in self.env.items()]), ' ' .join(self.cmd)) + if self.stdo: + res += '--- stdout ---\n' + res += self.stdo + if self.stde: + if res[-1:] != '\n': + res += '\n' + res += '--- stderr ---\n' + res += self.stde + if res[-1:] != '\n': + res += '\n' + res += '-------\n\n' + return res + +def decode(stream): + if stream is None: + return '' + try: + return stream.decode('utf-8') + except UnicodeDecodeError: + return stream.decode('iso-8859-1', errors='ignore') + +def write_json_log(jsonlogfile, test_name, result): + jresult = {'name': test_name, + 'stdout': result.stdo, + 'result': result.res, + 'duration': result.duration, + 'returncode': result.returncode, + 'command': result.cmd} + if isinstance(result.env, dict): + jresult['env'] = result.env + else: + jresult['env'] = result.env.get_env(os.environ) + if result.stde: + jresult['stderr'] = result.stde + jsonlogfile.write(json.dumps(jresult) + '\n') + +def run_with_mono(fname): + if fname.endswith('.exe') and not (is_windows() or is_cygwin()): + return True + return False + +class TestHarness: + def __init__(self, options): + self.options = options + self.collected_logs = [] + self.fail_count = 0 + self.success_count = 0 + self.skip_count = 0 + self.timeout_count = 0 + self.is_run = False + self.tests = None + self.suites = None + self.logfilename = None + self.logfile = None + self.jsonlogfile = None + if self.options.benchmark: + datafile = os.path.join(options.wd, 'meson-private', 'meson_benchmark_setup.dat') + else: + datafile = os.path.join(options.wd, 'meson-private', 'meson_test_setup.dat') + if not os.path.isfile(datafile): + raise TestException('Directory %s does not seem to be a Meson build directory.' % options.wd) + self.load_datafile(datafile) + + def __del__(self): + if self.logfile: + self.logfile.close() + if self.jsonlogfile: + self.jsonlogfile.close() + + def run_single_test(self, wrap, test): + if test.fname[0].endswith('.jar'): + cmd = ['java', '-jar'] + test.fname + elif not test.is_cross and run_with_mono(test.fname[0]): + cmd = ['mono'] + test.fname + else: + if test.is_cross: + if test.exe_runner is None: + # Can not run test on cross compiled executable + # because there is no execute wrapper. + cmd = None + else: + cmd = [test.exe_runner] + test.fname + else: + cmd = test.fname + + if cmd is None: + res = 'SKIP' + duration = 0.0 + stdo = 'Not run because can not execute cross compiled binaries.' + stde = None + returncode = GNU_SKIP_RETURNCODE + else: + cmd = wrap + cmd + test.cmd_args + self.options.test_args + starttime = time.time() + child_env = os.environ.copy() + child_env.update(self.options.global_env.get_env(child_env)) + if isinstance(test.env, build.EnvironmentVariables): + test.env = test.env.get_env(child_env) + + child_env.update(test.env) + if len(test.extra_paths) > 0: + child_env['PATH'] += os.pathsep.join([''] + test.extra_paths) + + # If MALLOC_PERTURB_ is not set, or if it is set to an empty value, + # (i.e., the test or the environment don't explicitly set it), set + # it ourselves. We do this unconditionally because it is extremely + # useful to have in tests. + # Setting MALLOC_PERTURB_="0" will completely disable this feature. + if 'MALLOC_PERTURB_' not in child_env or not child_env['MALLOC_PERTURB_']: + child_env['MALLOC_PERTURB_'] = str(random.randint(1, 255)) + + setsid = None + stdout = None + stderr = None + if not self.options.verbose: + stdout = subprocess.PIPE + stderr = subprocess.PIPE if self.options and self.options.split else subprocess.STDOUT + + if not is_windows(): + setsid = os.setsid + + p = subprocess.Popen(cmd, + stdout=stdout, + stderr=stderr, + env=child_env, + cwd=test.workdir, + preexec_fn=setsid) + timed_out = False + if test.timeout is None: + timeout = None + else: + timeout = test.timeout * self.options.timeout_multiplier + try: + (stdo, stde) = p.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + if self.options.verbose: + print("%s time out (After %d seconds)" % (test.name, timeout)) + timed_out = True + # Python does not provide multiplatform support for + # killing a process and all its children so we need + # to roll our own. + if is_windows(): + subprocess.call(['taskkill', '/F', '/T', '/PID', str(p.pid)]) + else: + os.killpg(os.getpgid(p.pid), signal.SIGKILL) + (stdo, stde) = p.communicate() + endtime = time.time() + duration = endtime - starttime + stdo = decode(stdo) + if stde: + stde = decode(stde) + if timed_out: + res = 'TIMEOUT' + self.timeout_count += 1 + self.fail_count += 1 + elif p.returncode == GNU_SKIP_RETURNCODE: + res = 'SKIP' + self.skip_count += 1 + elif test.should_fail == bool(p.returncode): + res = 'OK' + self.success_count += 1 + else: + res = 'FAIL' + self.fail_count += 1 + returncode = p.returncode + result = TestRun(res, returncode, test.should_fail, duration, stdo, stde, cmd, test.env) + + return result + + def print_stats(self, numlen, tests, name, result, i): + startpad = ' ' * (numlen - len('%d' % (i + 1))) + num = '%s%d/%d' % (startpad, i + 1, len(tests)) + padding1 = ' ' * (38 - len(name)) + padding2 = ' ' * (8 - len(result.res)) + result_str = '%s %s %s%s%s%5.2f s' % \ + (num, name, padding1, result.res, padding2, result.duration) + if not self.options.quiet or result.res != 'OK': + if result.res != 'OK' and mlog.colorize_console: + if result.res == 'FAIL' or result.res == 'TIMEOUT': + decorator = mlog.red + elif result.res == 'SKIP': + decorator = mlog.yellow + else: + sys.exit('Unreachable code was ... well ... reached.') + print(decorator(result_str).get_text(True)) + else: + print(result_str) + result_str += "\n\n" + result.get_log() + if (result.returncode != GNU_SKIP_RETURNCODE) \ + and (result.returncode != 0) != result.should_fail: + if self.options.print_errorlogs: + self.collected_logs.append(result_str) + if self.logfile: + self.logfile.write(result_str) + if self.jsonlogfile: + write_json_log(self.jsonlogfile, name, result) + + def print_summary(self): + msg = ''' +OK: %4d +FAIL: %4d +SKIP: %4d +TIMEOUT: %4d +''' % (self.success_count, self.fail_count, self.skip_count, self.timeout_count) + print(msg) + if self.logfile: + self.logfile.write(msg) + + def print_collected_logs(self): + if len(self.collected_logs) > 0: + if len(self.collected_logs) > 10: + print('\nThe output from 10 first failed tests:\n') + else: + print('\nThe output from the failed tests:\n') + for log in self.collected_logs[:10]: + lines = log.splitlines() + if len(lines) > 104: + print('\n'.join(lines[0:4])) + print('--- Listing only the last 100 lines from a long log. ---') + lines = lines[-100:] + for line in lines: + print(line) + + def doit(self): + if self.is_run: + raise RuntimeError('Test harness object can only be used once.') + if not os.path.isfile(self.datafile): + print('Test data file. Probably this means that you did not run this in the build directory.') + return 1 + self.is_run = True + tests = self.get_tests() + if not tests: + return 0 + self.run_tests(tests) + return self.fail_count + + @staticmethod + def split_suite_string(suite): + if ':' in suite: + return suite.split(':', 1) + else: + return suite, "" + + @staticmethod + def test_in_suites(test, suites): + for suite in suites: + (prj_match, st_match) = TestHarness.split_suite_string(suite) + for prjst in test.suite: + (prj, st) = TestHarness.split_suite_string(prjst) + if prj_match and prj != prj_match: + continue + if st_match and st != st_match: + continue + return True + return False + + def test_suitable(self, test): + return (not self.options.include_suites or TestHarness.test_in_suites(test, self.options.include_suites)) \ + and not TestHarness.test_in_suites(test, self.options.exclude_suites) + + def load_suites(self): + ss = set() + for t in self.tests: + for s in t.suite: + ss.add(s) + self.suites = list(ss) + + def load_tests(self): + with open(self.datafile, 'rb') as f: + self.tests = pickle.load(f) + + def load_datafile(self, datafile): + self.datafile = datafile + self.load_tests() + self.load_suites() + + def get_tests(self): + if not self.tests: + print('No tests defined.') + return [] + + if len(self.options.include_suites) or len(self.options.exclude_suites): + tests = [] + for tst in self.tests: + if self.test_suitable(tst): + tests.append(tst) + else: + tests = self.tests + + if self.options.args: + tests = [t for t in tests if t.name in self.options.args] + + if not tests: + print('No suitable tests defined.') + return [] + + for test in tests: + test.rebuilt = False + + return tests + + def open_log_files(self): + if not self.options.logbase or self.options.verbose: + return None, None, None, None + + namebase = None + logfile_base = os.path.join(self.options.wd, 'meson-logs', self.options.logbase) + + if self.options.wrapper: + namebase = os.path.split(self.get_wrapper()[0])[1] + elif self.options.setup: + namebase = self.options.setup + + if namebase: + logfile_base += '-' + namebase.replace(' ', '_') + self.logfilename = logfile_base + '.txt' + self.jsonlogfilename = logfile_base + '.json' + + self.jsonlogfile = open(self.jsonlogfilename, 'w') + self.logfile = open(self.logfilename, 'w') + + self.logfile.write('Log of Meson test suite run on %s\n\n' + % datetime.datetime.now().isoformat()) + + def get_wrapper(self): + wrap = [] + if self.options.gdb: + wrap = ['gdb', '--quiet', '--nh'] + if self.options.repeat > 1: + wrap += ['-ex', 'run', '-ex', 'quit'] + # Signal the end of arguments to gdb + wrap += ['--args'] + if self.options.wrapper: + wrap += self.options.wrapper + assert(isinstance(wrap, list)) + return wrap + + def get_pretty_suite(self, test): + if len(self.suites) > 1: + rv = TestHarness.split_suite_string(test.suite[0])[0] + s = "+".join(TestHarness.split_suite_string(s)[1] for s in test.suite) + if len(s): + rv += ":" + return rv + s + " / " + test.name + else: + return test.name + + def run_tests(self, tests): + executor = None + futures = [] + numlen = len('%d' % len(tests)) + self.open_log_files() + wrap = self.get_wrapper() + + for _ in range(self.options.repeat): + for i, test in enumerate(tests): + visible_name = self.get_pretty_suite(test) + + if self.options.gdb: + test.timeout = None + + if not test.is_parallel or self.options.gdb: + self.drain_futures(futures) + futures = [] + res = self.run_single_test(wrap, test) + self.print_stats(numlen, tests, visible_name, res, i) + else: + if not executor: + executor = conc.ThreadPoolExecutor(max_workers=self.options.num_processes) + f = executor.submit(self.run_single_test, wrap, test) + futures.append((f, numlen, tests, visible_name, i)) + if self.options.repeat > 1 and self.fail_count: + break + if self.options.repeat > 1 and self.fail_count: + break + + self.drain_futures(futures) + self.print_summary() + self.print_collected_logs() + + if self.logfilename: + print('Full log written to %s' % self.logfilename) + + def drain_futures(self, futures): + for i in futures: + (result, numlen, tests, name, i) = i + if self.options.repeat > 1 and self.fail_count: + result.cancel() + if self.options.verbose: + result.result() + self.print_stats(numlen, tests, name, result.result(), i) + + def run_special(self): + 'Tests run by the user, usually something like "under gdb 1000 times".' + if self.is_run: + raise RuntimeError('Can not use run_special after a full run.') + tests = self.get_tests() + if not tests: + return 0 + self.run_tests(tests) + return self.fail_count + + +def list_tests(th): + tests = th.get_tests() + for t in tests: + print(th.get_pretty_suite(t)) + +def merge_suite_options(options): + buildfile = os.path.join(options.wd, 'meson-private/build.dat') + with open(buildfile, 'rb') as f: + build = pickle.load(f) + setups = build.test_setups + if options.setup not in setups: + sys.exit('Unknown test setup: %s' % options.setup) + current = setups[options.setup] + if not options.gdb: + options.gdb = current.gdb + if options.timeout_multiplier is None: + options.timeout_multiplier = current.timeout_multiplier +# if options.env is None: +# options.env = current.env # FIXME, should probably merge options here. + if options.wrapper is not None and current.exe_wrapper is not None: + sys.exit('Conflict: both test setup and command line specify an exe wrapper.') + if options.wrapper is None: + options.wrapper = current.exe_wrapper + return current.env + +def rebuild_all(wd): + if not os.path.isfile(os.path.join(wd, 'build.ninja')): + print("Only ninja backend is supported to rebuild tests before running them.") + return True + + ninja = environment.detect_ninja() + if not ninja: + print("Can't find ninja, can't rebuild test.") + return False + + p = subprocess.Popen([ninja, '-C', wd]) + p.communicate() + + if p.returncode != 0: + print("Could not rebuild") + return False + + return True + +def run(args): + options = parser.parse_args(args) + + if options.benchmark: + options.num_processes = 1 + + if options.setup is not None: + global_env = merge_suite_options(options) + else: + global_env = build.EnvironmentVariables() + if options.timeout_multiplier is None: + options.timeout_multiplier = 1 + + setattr(options, 'global_env', global_env) + + if options.verbose and options.quiet: + print('Can not be both quiet and verbose at the same time.') + return 1 + + check_bin = None + if options.gdb: + options.verbose = True + if options.wrapper: + print('Must not specify both a wrapper and gdb at the same time.') + return 1 + check_bin = 'gdb' + + if options.wrapper: + check_bin = options.wrapper[0] + + if check_bin is not None: + exe = ExternalProgram(check_bin, silent=True) + if not exe.found(): + sys.exit("Could not find requested program: %s" % check_bin) + options.wd = os.path.abspath(options.wd) + + if not options.list and not options.no_rebuild: + if not rebuild_all(options.wd): + sys.exit(-1) + + try: + th = TestHarness(options) + if options.list: + list_tests(th) + return 0 + if not options.args: + return th.doit() + return th.run_special() + except TestException as e: + print('Mesontest encountered an error:\n') + print(e) + return 1 |