From 70695bead40f97825357e620394918d1a248bf1a Mon Sep 17 00:00:00 2001 From: Jussi Pakkanen Date: Wed, 9 Dec 2015 18:00:06 +0200 Subject: Added possibility to have multiple test suites so you can run only a subset of tests. Closes #325. --- backends.py | 5 +++-- coredata.py | 1 + interpreter.py | 12 ++++++++---- meson_test.py | 18 +++++++++++++++--- mesonintrospect.py | 1 + ninjabackend.py | 21 ++++++++++++++++++++- test cases/common/101 suites/exe1.c | 6 ++++++ test cases/common/101 suites/exe2.c | 6 ++++++ test cases/common/101 suites/meson.build | 9 +++++++++ .../common/101 suites/subprojects/sub/meson.build | 7 +++++++ test cases/common/101 suites/subprojects/sub/sub1.c | 6 ++++++ test cases/common/101 suites/subprojects/sub/sub2.c | 6 ++++++ 12 files changed, 88 insertions(+), 10 deletions(-) create mode 100644 test cases/common/101 suites/exe1.c create mode 100644 test cases/common/101 suites/exe2.c create mode 100644 test cases/common/101 suites/meson.build create mode 100644 test cases/common/101 suites/subprojects/sub/meson.build create mode 100644 test cases/common/101 suites/subprojects/sub/sub1.c create mode 100644 test cases/common/101 suites/subprojects/sub/sub2.c diff --git a/backends.py b/backends.py index 7f7c338..e0a9ec4 100644 --- a/backends.py +++ b/backends.py @@ -20,9 +20,10 @@ import json from coredata import MesonException class TestSerialisation: - def __init__(self, name, fname, is_cross, exe_wrapper, is_parallel, cmd_args, env, + def __init__(self, name, suite, fname, is_cross, exe_wrapper, is_parallel, cmd_args, env, should_fail, valgrind_args, timeout, workdir, extra_paths): self.name = name + self.suite = suite self.fname = fname self.is_cross = is_cross self.exe_runner = exe_wrapper @@ -302,7 +303,7 @@ class Backend(): if isinstance(a, mesonlib.File): a = os.path.join(self.environment.get_build_dir(), a.rel_to_builddir(self.build_to_src)) cmd_args.append(a) - ts = TestSerialisation(t.get_name(), fname, is_cross, exe_wrapper, + ts = TestSerialisation(t.get_name(), t.suite, fname, is_cross, exe_wrapper, t.is_parallel, cmd_args, t.env, t.should_fail, t.valgrind_args, t.timeout, t.workdir, extra_paths) arr.append(ts) diff --git a/coredata.py b/coredata.py index e216428..c305407 100644 --- a/coredata.py +++ b/coredata.py @@ -215,6 +215,7 @@ forbidden_target_names = {'clean': None, 'all': None, 'test': None, 'test-valgrind': None, + 'test-': None, 'benchmark': None, 'install': None, 'build.ninja': None, diff --git a/interpreter.py b/interpreter.py index e7f7a3a..8ba2b78 100644 --- a/interpreter.py +++ b/interpreter.py @@ -528,9 +528,10 @@ class RunTargetHolder(InterpreterObject): self.held_object = build.RunTarget(name, command, args, subdir) class Test(InterpreterObject): - def __init__(self, name, exe, is_parallel, cmd_args, env, should_fail, valgrind_args, timeout, workdir): + def __init__(self, name, suite, exe, is_parallel, cmd_args, env, should_fail, valgrind_args, timeout, workdir): InterpreterObject.__init__(self) self.name = name + self.suite = suite self.exe = exe self.is_parallel = is_parallel self.cmd_args = cmd_args @@ -1378,9 +1379,9 @@ class Interpreter(): self.parse_default_options(kwargs['default_options']) self.active_projectname = args[0] self.project_version = kwargs.get('version', 'undefined') - license = mesonlib.stringlistify(kwargs.get('license', 'unknown')) + proj_license = mesonlib.stringlistify(kwargs.get('license', 'unknown')) self.build.dep_manifest[args[0]] = {'version': self.project_version, - 'license': license} + 'license': proj_license} if self.subproject in self.build.projects: raise InvalidCode('Second call to project().') if not self.is_subproject() and 'subproject_dir' in kwargs: @@ -1734,7 +1735,10 @@ class Interpreter(): workdir = None if not isinstance(timeout, int): raise InterpreterException('Timeout must be an integer.') - t = Test(args[0], args[1].held_object, par, cmd_args, env, should_fail, valgrind_args, timeout, workdir) + suite = kwargs.get('suite', '') + if self.is_subproject(): + suite = self.subproject.replace(' ', '_') + ':' + suite + t = Test(args[0], suite, args[1].held_object, par, cmd_args, env, should_fail, valgrind_args, timeout, workdir) if is_base_test: self.build.tests.append(t) mlog.debug('Adding test "', mlog.bold(args[0]), '".', sep='') diff --git a/meson_test.py b/meson_test.py index a7ba7de..5771fa3 100755 --- a/meson_test.py +++ b/meson_test.py @@ -26,6 +26,8 @@ parser.add_argument('--wrapper', default=None, dest='wrapper', help='wrapper to run tests with (e.g. valgrind)') parser.add_argument('--wd', default=None, dest='wd', help='directory to cd into before running') +parser.add_argument('--suite', default=None, dest='suite', + help='Only run tests belonging to this suite.') parser.add_argument('args', nargs='+') @@ -144,6 +146,11 @@ def drain_futures(futures): (result, numlen, tests, name, i, logfile, jsonlogfile) = i print_stats(numlen, tests, name, result.result(), i, logfile, jsonlogfile) +def filter_tests(suite, tests): + if suite is None: + return tests + return [x for x in tests if x.suite == suite] + def run_tests(options, datafilename): logfile_base = 'meson-logs/testlog' if options.wrapper is None: @@ -173,15 +180,20 @@ def run_tests(options, datafilename): num_workers = multiprocessing.cpu_count() executor = conc.ThreadPoolExecutor(max_workers=num_workers) futures = [] - for i, test in enumerate(tests): + filtered_tests = filter_tests(options.suite, tests) + for i, test in enumerate(filtered_tests): + if test.suite == '': + visible_name = test.name + else: + visible_name = test.suite + ' / ' + test.name if not test.is_parallel: drain_futures(futures) futures = [] res = run_single_test(wrap, test) - print_stats(numlen, tests, test.name, res, i, logfile, jsonlogfile) + print_stats(numlen, tests, visible_name, res, i, logfile, jsonlogfile) else: f = executor.submit(run_single_test, wrap, test) - futures.append((f, numlen, tests, test.name, i, logfile, jsonlogfile)) + futures.append((f, numlen, tests, visible_name, i, logfile, jsonlogfile)) drain_futures(futures) print('\nFull log written to %s.' % logfilename) diff --git a/mesonintrospect.py b/mesonintrospect.py index fcc9bfe..9fcd4db 100755 --- a/mesonintrospect.py +++ b/mesonintrospect.py @@ -168,6 +168,7 @@ def list_tests(testdata): to['name'] = t.name to['workdir'] = t.workdir to['timeout'] = t.timeout + to['suite'] = t.suite result.append(to) print(json.dumps(result)) diff --git a/ninjabackend.py b/ninjabackend.py index 7d72e06..f614923 100644 --- a/ninjabackend.py +++ b/ninjabackend.py @@ -528,6 +528,24 @@ class NinjaBackend(backends.Backend): dst_dir = os.path.join(self.environment.get_prefix(), sd.install_dir) d.install_subdirs.append([src_dir, dst_dir]) + def write_test_suite_targets(self, cmd, outfile): + suites = {} + for t in self.build.get_tests(): + suites[t.suite] = True + suites = list(suites.keys()) + suites.sort() + for s in suites: + if s == '': + visible_name = 'for top level tests' + else: + visible_name = s + elem = NinjaBuildElement('test-' + s, 'CUSTOM_COMMAND', ['all', 'PHONY']) + elem.add_item('COMMAND', cmd + ['--suite=' + s]) + elem.add_item('DESC', 'Running test suite %s.' % visible_name) + elem.add_item('pool', 'console') + elem.write(outfile) + self.check_outputs(elem) + def generate_tests(self, outfile): self.serialise_tests() valgrind = environment.find_valgrind() @@ -537,10 +555,11 @@ class NinjaBackend(backends.Backend): cmd = [sys.executable, test_script, test_data] elem = NinjaBuildElement('test', 'CUSTOM_COMMAND', ['all', 'PHONY']) elem.add_item('COMMAND', cmd) - elem.add_item('DESC', 'Running test suite.') + elem.add_item('DESC', 'Running all tests.') elem.add_item('pool', 'console') elem.write(outfile) self.check_outputs(elem) + self.write_test_suite_targets(cmd, outfile) if valgrind: velem = NinjaBuildElement('test-valgrind', 'CUSTOM_COMMAND', ['all', 'PHONY']) diff --git a/test cases/common/101 suites/exe1.c b/test cases/common/101 suites/exe1.c new file mode 100644 index 0000000..23894c0 --- /dev/null +++ b/test cases/common/101 suites/exe1.c @@ -0,0 +1,6 @@ +#include + +int main(int argc, char **argv) { + printf("I am test exe1.\n"); + return 0; +} diff --git a/test cases/common/101 suites/exe2.c b/test cases/common/101 suites/exe2.c new file mode 100644 index 0000000..ec88e20 --- /dev/null +++ b/test cases/common/101 suites/exe2.c @@ -0,0 +1,6 @@ +#include + +int main(int argc, char **argv) { + printf("I am test exe2.\n"); + return 0; +} diff --git a/test cases/common/101 suites/meson.build b/test cases/common/101 suites/meson.build new file mode 100644 index 0000000..098c95f --- /dev/null +++ b/test cases/common/101 suites/meson.build @@ -0,0 +1,9 @@ +project('multiple test suites', 'c') + +subproject('sub') + +exe1 = executable('exe1', 'exe1.c') +exe2 = executable('exe2', 'exe2.c') + +test('exe1', exe1) +test('exe2', exe2, suite : 'suite2') diff --git a/test cases/common/101 suites/subprojects/sub/meson.build b/test cases/common/101 suites/subprojects/sub/meson.build new file mode 100644 index 0000000..697d95f --- /dev/null +++ b/test cases/common/101 suites/subprojects/sub/meson.build @@ -0,0 +1,7 @@ +project('subproject test suites', 'c') + +sub1 = executable('sub1', 'sub1.c') +sub2 = executable('sub2', 'sub2.c') + +test('sub1', sub1) +test('sub2', sub2, suite : 'suite2') diff --git a/test cases/common/101 suites/subprojects/sub/sub1.c b/test cases/common/101 suites/subprojects/sub/sub1.c new file mode 100644 index 0000000..3409e48 --- /dev/null +++ b/test cases/common/101 suites/subprojects/sub/sub1.c @@ -0,0 +1,6 @@ +#include + +int main(int argc, char **argv) { + printf("I am test sub1.\n"); + return 0; +} diff --git a/test cases/common/101 suites/subprojects/sub/sub2.c b/test cases/common/101 suites/subprojects/sub/sub2.c new file mode 100644 index 0000000..1a64a2a --- /dev/null +++ b/test cases/common/101 suites/subprojects/sub/sub2.c @@ -0,0 +1,6 @@ +#include + +int main(int argc, char **argv) { + printf("I am test sub2.\n"); + return 0; +} -- cgit v1.1 From b3d61beb2d8194228bda193f6ae17f4dc4b3e7b4 Mon Sep 17 00:00:00 2001 From: Jussi Pakkanen Date: Thu, 10 Dec 2015 22:22:42 +0200 Subject: A test can be part of multiple suites. --- interpreter.py | 7 +++++-- meson_test.py | 6 +++--- ninjabackend.py | 3 ++- test cases/common/101 suites/meson.build | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/interpreter.py b/interpreter.py index 8ba2b78..1a5c6b2 100644 --- a/interpreter.py +++ b/interpreter.py @@ -1735,9 +1735,12 @@ class Interpreter(): workdir = None if not isinstance(timeout, int): raise InterpreterException('Timeout must be an integer.') - suite = kwargs.get('suite', '') + suite = mesonlib.stringlistify(kwargs.get('suite', '')) if self.is_subproject(): - suite = self.subproject.replace(' ', '_') + ':' + suite + newsuite = [] + for s in suite: + newsuite.append(self.subproject.replace(' ', '_') + ':' + s) + suite = newsuite t = Test(args[0], suite, args[1].held_object, par, cmd_args, env, should_fail, valgrind_args, timeout, workdir) if is_base_test: self.build.tests.append(t) diff --git a/meson_test.py b/meson_test.py index 5771fa3..53dc6ea 100755 --- a/meson_test.py +++ b/meson_test.py @@ -149,7 +149,7 @@ def drain_futures(futures): def filter_tests(suite, tests): if suite is None: return tests - return [x for x in tests if x.suite == suite] + return [x for x in tests if suite in x.suite] def run_tests(options, datafilename): logfile_base = 'meson-logs/testlog' @@ -182,10 +182,10 @@ def run_tests(options, datafilename): futures = [] filtered_tests = filter_tests(options.suite, tests) for i, test in enumerate(filtered_tests): - if test.suite == '': + if test.suite[0] == '': visible_name = test.name else: - visible_name = test.suite + ' / ' + test.name + visible_name = test.suite[0] + ' / ' + test.name if not test.is_parallel: drain_futures(futures) futures = [] diff --git a/ninjabackend.py b/ninjabackend.py index f614923..d837074 100644 --- a/ninjabackend.py +++ b/ninjabackend.py @@ -531,7 +531,8 @@ class NinjaBackend(backends.Backend): def write_test_suite_targets(self, cmd, outfile): suites = {} for t in self.build.get_tests(): - suites[t.suite] = True + for s in t.suite: + suites[s] = True suites = list(suites.keys()) suites.sort() for s in suites: diff --git a/test cases/common/101 suites/meson.build b/test cases/common/101 suites/meson.build index 098c95f..057e059 100644 --- a/test cases/common/101 suites/meson.build +++ b/test cases/common/101 suites/meson.build @@ -6,4 +6,4 @@ exe1 = executable('exe1', 'exe1.c') exe2 = executable('exe2', 'exe2.c') test('exe1', exe1) -test('exe2', exe2, suite : 'suite2') +test('exe2', exe2, suite : ['suite2', 'super-special']) -- cgit v1.1 From 8bb2586ae472ab625b358cde433f553b578ad097 Mon Sep 17 00:00:00 2001 From: Jussi Pakkanen Date: Fri, 11 Dec 2015 01:28:19 +0200 Subject: Some text format fixes. --- meson_test.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/meson_test.py b/meson_test.py index 53dc6ea..e8942e0 100755 --- a/meson_test.py +++ b/meson_test.py @@ -185,15 +185,19 @@ def run_tests(options, datafilename): if test.suite[0] == '': visible_name = test.name else: - visible_name = test.suite[0] + ' / ' + test.name + if options.suite is not None: + visible_name = options.suite + ' / ' + test.name + else: + visible_name = test.suite[0] + ' / ' + test.name + if not test.is_parallel: drain_futures(futures) futures = [] res = run_single_test(wrap, test) - print_stats(numlen, tests, visible_name, res, i, logfile, jsonlogfile) + print_stats(numlen, filtered_tests, visible_name, res, i, logfile, jsonlogfile) else: f = executor.submit(run_single_test, wrap, test) - futures.append((f, numlen, tests, visible_name, i, logfile, jsonlogfile)) + futures.append((f, numlen, filtered_tests, visible_name, i, logfile, jsonlogfile)) drain_futures(futures) print('\nFull log written to %s.' % logfilename) -- cgit v1.1 From fcf6643507fa6db7dee9d844cedf53307f6174a4 Mon Sep 17 00:00:00 2001 From: Jussi Pakkanen Date: Fri, 11 Dec 2015 21:48:47 +0200 Subject: Use period as the separator because colon causes problems on Windows. --- interpreter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interpreter.py b/interpreter.py index 1a5c6b2..a11f753 100644 --- a/interpreter.py +++ b/interpreter.py @@ -1739,7 +1739,7 @@ class Interpreter(): if self.is_subproject(): newsuite = [] for s in suite: - newsuite.append(self.subproject.replace(' ', '_') + ':' + s) + newsuite.append(self.subproject.replace(' ', '_').replace('.', '_') + '.' + s) suite = newsuite t = Test(args[0], suite, args[1].held_object, par, cmd_args, env, should_fail, valgrind_args, timeout, workdir) if is_base_test: -- cgit v1.1