Merge pull request #2311 from trhd/setups

Improve test setup selection.

11 files changed, 134 insertions, 61 deletions

diff --git a/docs/markdown/snippets/test-setups.md b/docs/markdown/snippets/test-setups.md
new file mode 100644
index 0000000..4598e73
--- /dev/null
+++ b/docs/markdown/snippets/test-setups.md
@@ -0,0 +1,16 @@
+## Improve test setup selection
+
+Test setups are now identified (also) by the project they belong to and it
+is possible to select the used test setup from a specific project. E.g.
+to use a test setup `some_setup` from project `some_project` for all
+executed tests one can use
+
+    meson test --setup some_project:some_setup
+
+Should one rather want test setups to be used from the same project as
+where the current test itself has been defined, one can use just
+
+    meson test --setup some_setup
+
+In the latter case every (sub)project must have a test setup `some_setup`
+defined in it.
diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py
index b547bc3..a8e8164 100644
--- a/mesonbuild/backend/backends.py
+++ b/mesonbuild/backend/backends.py
@@ -65,9 +65,10 @@ class ExecutableSerialisation:
         self.capture = capture
 
 class TestSerialisation:
-    def __init__(self, name, suite, fname, is_cross_built, exe_wrapper, is_parallel, cmd_args, env,
-                 should_fail, timeout, workdir, extra_paths):
+    def __init__(self, name, project, suite, fname, is_cross_built, exe_wrapper, is_parallel,
+                 cmd_args, env, should_fail, timeout, workdir, extra_paths):
         self.name = name
+        self.project_name = project
         self.suite = suite
         self.fname = fname
         self.is_cross_built = is_cross_built
@@ -603,9 +604,9 @@ class Backend:
                     cmd_args.append(self.get_target_filename(a))
                 else:
                     raise MesonException('Bad object in test command.')
-            ts = TestSerialisation(t.get_name(), t.suite, cmd, is_cross, exe_wrapper,
-                                   t.is_parallel, cmd_args, t.env, t.should_fail,
-                                   t.timeout, t.workdir, extra_paths)
+            ts = TestSerialisation(t.get_name(), t.project_name, t.suite, cmd, is_cross,
+                                   exe_wrapper, t.is_parallel, cmd_args, t.env,
+                                   t.should_fail, t.timeout, t.workdir, extra_paths)
             arr.append(ts)
         pickle.dump(arr, datafile)
 
diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py
index 5c333a8..e3680ac 100644
--- a/mesonbuild/interpreter.py
+++ b/mesonbuild/interpreter.py
@@ -653,10 +653,11 @@ class RunTargetHolder(InterpreterObject, ObjectHolder):
         return r.format(self.__class__.__name__, h.get_id(), h.command)
 
 class Test(InterpreterObject):
-    def __init__(self, name, suite, exe, is_parallel, cmd_args, env, should_fail, timeout, workdir):
+    def __init__(self, name, project, suite, exe, is_parallel, cmd_args, env, should_fail, timeout, workdir):
         InterpreterObject.__init__(self)
         self.name = name
         self.suite = suite
+        self.project_name = project
         self.exe = exe
         self.is_parallel = is_parallel
         self.cmd_args = cmd_args
@@ -2169,7 +2170,7 @@ to directly access options of other subprojects.''')
         if progobj is None:
             progobj = self.program_from_system(args)
         if required and (progobj is None or not progobj.found()):
-            raise InvalidArguments('Program "%s" not found or not executable' % args[0])
+            raise InvalidArguments('Program(s) {!r} not found or not executable'.format(args))
         if progobj is None:
             return ExternalProgramHolder(dependencies.NonExistingExternalProgram())
         return progobj
@@ -2586,14 +2587,12 @@ root and issuing %s.
         if not isinstance(timeout, int):
             raise InterpreterException('Timeout must be an integer.')
         suite = []
+        prj = self.subproject if self.is_subproject() else self.build.project_name
         for s in mesonlib.stringlistify(kwargs.get('suite', '')):
             if len(s) > 0:
                 s = ':' + s
-            if self.is_subproject():
-                suite.append(self.subproject.replace(' ', '_').replace(':', '_') + s)
-            else:
-                suite.append(self.build.project_name.replace(' ', '_').replace(':', '_') + s)
-        t = Test(args[0], suite, exe.held_object, par, cmd_args, env, should_fail, timeout, workdir)
+            suite.append(prj.replace(' ', '_').replace(':', '_') + s)
+        t = Test(args[0], prj, suite, exe.held_object, par, cmd_args, env, should_fail, timeout, workdir)
         if is_base_test:
             self.build.tests.append(t)
             mlog.debug('Adding test "', mlog.bold(args[0]), '".', sep='')
@@ -2892,8 +2891,10 @@ different subdirectory.
         if len(args) != 1:
             raise InterpreterException('Add_test_setup needs one argument for the setup name.')
         setup_name = args[0]
-        if re.fullmatch('[_a-zA-Z][_0-9a-zA-Z]*', setup_name) is None:
+        if re.fullmatch('([_a-zA-Z][_0-9a-zA-Z]*:)?[_a-zA-Z][_0-9a-zA-Z]*', setup_name) is None:
             raise InterpreterException('Setup name may only contain alphanumeric characters.')
+        if ":" not in setup_name:
+            setup_name = (self.subproject if self.subproject else self.build.project_name) + ":" + setup_name
         try:
             inp = extract_as_list(kwargs, 'exe_wrapper')
             exe_wrapper = []
@@ -2917,14 +2918,10 @@ different subdirectory.
         if not isinstance(timeout_multiplier, int):
             raise InterpreterException('Timeout multiplier must be a number.')
         env = self.unpack_env_kwarg(kwargs)
-        setupobj = build.TestSetup(exe_wrapper=exe_wrapper,
-                                   gdb=gdb,
-                                   timeout_multiplier=timeout_multiplier,
-                                   env=env)
-        if self.subproject == '':
-            # Dunno what we should do with subprojects really. Let's start simple
-            # and just use the master project ones.
-            self.build.test_setups[setup_name] = setupobj
+        self.build.test_setups[setup_name] = build.TestSetup(exe_wrapper=exe_wrapper,
+                                                             gdb=gdb,
+                                                             timeout_multiplier=timeout_multiplier,
+                                                             env=env)
 
     @permittedKwargs(permitted_kwargs['add_global_arguments'])
     @stringArgs
diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index a697106..686ddfc 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -28,6 +28,7 @@ import concurrent.futures as conc
 import platform
 import signal
 import random
+from copy import deepcopy
 
 # GNU autotools interprets a return code of 77 from tests it executes to
 # mean that the test should be skipped.
@@ -89,7 +90,7 @@ parser.add_argument('-v', '--verbose', default=False, action='store_true',
                     help='Do not redirect stdout and stderr')
 parser.add_argument('-q', '--quiet', default=False, action='store_true',
                     help='Produce less output to the terminal.')
-parser.add_argument('-t', '--timeout-multiplier', type=float, default=None,
+parser.add_argument('-t', '--timeout-multiplier', type=float, default=1,
                     help='Define a multiplier for test timeout, for example '
                     ' when running tests in particular conditions they might take'
                     ' more time to execute.')
@@ -192,7 +193,17 @@ class TestHarness:
         if self.jsonlogfile:
             self.jsonlogfile.close()
 
-    def run_single_test(self, wrap, test):
+    def get_test_env(self, options, test):
+        if options.setup:
+            env = merge_suite_options(options, test)
+        else:
+            env = os.environ.copy()
+        if isinstance(test.env, build.EnvironmentVariables):
+            test.env = test.env.get_env(env)
+        env.update(test.env)
+        return env
+
+    def run_single_test(self, test):
         if test.fname[0].endswith('.jar'):
             cmd = ['java', '-jar'] + test.fname
         elif not test.is_cross_built and run_with_mono(test.fname[0]):
@@ -215,24 +226,26 @@ class TestHarness:
             stde = None
             returncode = GNU_SKIP_RETURNCODE
         else:
+            test_opts = deepcopy(self.options)
+            test_env = self.get_test_env(test_opts, test)
+            wrap = self.get_wrapper(test_opts)
+
+            if test_opts.gdb:
+                test.timeout = None
+
             cmd = wrap + cmd + test.cmd_args + self.options.test_args
             starttime = time.time()
-            child_env = os.environ.copy()
-            child_env.update(self.options.global_env.get_env(child_env))
-            if isinstance(test.env, build.EnvironmentVariables):
-                test.env = test.env.get_env(child_env)
 
-            child_env.update(test.env)
             if len(test.extra_paths) > 0:
-                child_env['PATH'] = os.pathsep.join(test.extra_paths + ['']) + child_env['PATH']
+                test_env['PATH'] = os.pathsep.join(test.extra_paths + ['']) + test_env['PATH']
 
             # If MALLOC_PERTURB_ is not set, or if it is set to an empty value,
             # (i.e., the test or the environment don't explicitly set it), set
             # it ourselves. We do this unconditionally for regular tests
             # because it is extremely useful to have.
             # Setting MALLOC_PERTURB_="0" will completely disable this feature.
-            if ('MALLOC_PERTURB_' not in child_env or not child_env['MALLOC_PERTURB_']) and not self.options.benchmark:
-                child_env['MALLOC_PERTURB_'] = str(random.randint(1, 255))
+            if ('MALLOC_PERTURB_' not in test_env or not test_env['MALLOC_PERTURB_']) and not self.options.benchmark:
+                test_env['MALLOC_PERTURB_'] = str(random.randint(1, 255))
 
             setsid = None
             stdout = None
@@ -247,7 +260,7 @@ class TestHarness:
             p = subprocess.Popen(cmd,
                                  stdout=stdout,
                                  stderr=stderr,
-                                 env=child_env,
+                                 env=test_env,
                                  cwd=test.workdir,
                                  preexec_fn=setsid)
             timed_out = False
@@ -255,7 +268,7 @@ class TestHarness:
             if test.timeout is None:
                 timeout = None
             else:
-                timeout = test.timeout * self.options.timeout_multiplier
+                timeout = test.timeout * test_opts.timeout_multiplier
             try:
                 (stdo, stde) = p.communicate(timeout=timeout)
             except subprocess.TimeoutExpired:
@@ -444,9 +457,9 @@ TIMEOUT: %4d
         logfile_base = os.path.join(self.options.wd, 'meson-logs', self.options.logbase)
 
         if self.options.wrapper:
-            namebase = os.path.basename(self.get_wrapper()[0])
+            namebase = os.path.basename(self.get_wrapper(self.options)[0])
         elif self.options.setup:
-            namebase = self.options.setup
+            namebase = self.options.setup.replace(":", "_")
 
         if namebase:
             logfile_base += '-' + namebase.replace(' ', '_')
@@ -459,16 +472,16 @@ TIMEOUT: %4d
         self.logfile.write('Log of Meson test suite run on %s\n\n'
                            % datetime.datetime.now().isoformat())
 
-    def get_wrapper(self):
+    def get_wrapper(self, options):
         wrap = []
-        if self.options.gdb:
+        if options.gdb:
             wrap = ['gdb', '--quiet', '--nh']
-            if self.options.repeat > 1:
+            if options.repeat > 1:
                 wrap += ['-ex', 'run', '-ex', 'quit']
             # Signal the end of arguments to gdb
             wrap += ['--args']
-        if self.options.wrapper:
-            wrap += self.options.wrapper
+        if options.wrapper:
+            wrap += options.wrapper
         assert(isinstance(wrap, list))
         return wrap
 
@@ -487,7 +500,6 @@ TIMEOUT: %4d
         futures = []
         numlen = len('%d' % len(tests))
         self.open_log_files()
-        wrap = self.get_wrapper()
         startdir = os.getcwd()
         if self.options.wd:
             os.chdir(self.options.wd)
@@ -497,18 +509,15 @@ TIMEOUT: %4d
                 for i, test in enumerate(tests):
                     visible_name = self.get_pretty_suite(test)
 
-                    if self.options.gdb:
-                        test.timeout = None
-
                     if not test.is_parallel or self.options.gdb:
                         self.drain_futures(futures)
                         futures = []
-                        res = self.run_single_test(wrap, test)
+                        res = self.run_single_test(test)
                         self.print_stats(numlen, tests, visible_name, res, i)
                     else:
                         if not executor:
                             executor = conc.ThreadPoolExecutor(max_workers=self.options.num_processes)
-                        f = executor.submit(self.run_single_test, wrap, test)
+                        f = executor.submit(self.run_single_test, test)
                         futures.append((f, numlen, tests, visible_name, i))
                     if self.options.repeat > 1 and self.fail_count:
                         break
@@ -549,14 +558,19 @@ def list_tests(th):
     for t in tests:
         print(th.get_pretty_suite(t))
 
-def merge_suite_options(options):
+def merge_suite_options(options, test):
     buildfile = os.path.join(options.wd, 'meson-private/build.dat')
     with open(buildfile, 'rb') as f:
         build = pickle.load(f)
-    setups = build.test_setups
-    if options.setup not in setups:
-        sys.exit('Unknown test setup: %s' % options.setup)
-    current = setups[options.setup]
+    if ":" in options.setup:
+        if options.setup not in build.test_setups:
+            sys.exit("Unknown test setup '%s'." % options.setup)
+        current = build.test_setups[options.setup]
+    else:
+        full_name = test.project_name + ":" + options.setup
+        if full_name not in build.test_setups:
+            sys.exit("Test setup '%s' not found from project '%s'." % (options.setup, test.project_name))
+        current = build.test_setups[full_name]
     if not options.gdb:
         options.gdb = current.gdb
     if options.timeout_multiplier is None:
@@ -567,7 +581,7 @@ def merge_suite_options(options):
         sys.exit('Conflict: both test setup and command line specify an exe wrapper.')
     if options.wrapper is None:
         options.wrapper = current.exe_wrapper
-    return current.env
+    return current.env.get_env(os.environ.copy())
 
 def rebuild_all(wd):
     if not os.path.isfile(os.path.join(wd, 'build.ninja')):
@@ -594,15 +608,6 @@ def run(args):
     if options.benchmark:
         options.num_processes = 1
 
-    if options.setup is not None:
-        global_env = merge_suite_options(options)
-    else:
-        global_env = build.EnvironmentVariables()
-        if options.timeout_multiplier is None:
-            options.timeout_multiplier = 1
-
-    setattr(options, 'global_env', global_env)
-
     if options.verbose and options.quiet:
         print('Can not be both quiet and verbose at the same time.')
         return 1
diff --git a/run_unittests.py b/run_unittests.py
index 41cbf39..b69aaae 100755
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -949,6 +949,31 @@ class AllPlatformTests(BasePlatformTests):
         # Setup with only a timeout works
         self._run(self.mtest_command + ['--setup=timeout'])
 
+    def test_testsetup_selection(self):
+        testdir = os.path.join(self.unit_test_dir, '13 testsetup selection')
+        self.init(testdir)
+        self.build()
+
+        # Run tests without setup
+        self.run_tests()
+
+        self.assertRaises(subprocess.CalledProcessError, self._run, self.mtest_command + ['--setup=missingfromfoo'])
+        self._run(self.mtest_command + ['--setup=missingfromfoo', '--no-suite=foo:'])
+
+        self._run(self.mtest_command + ['--setup=worksforall'])
+        self._run(self.mtest_command + ['--setup=main:worksforall'])
+
+        self.assertRaises(subprocess.CalledProcessError, self._run,
+                          self.mtest_command + ['--setup=onlyinbar'])
+        self.assertRaises(subprocess.CalledProcessError, self._run,
+                          self.mtest_command + ['--setup=onlyinbar', '--no-suite=main:'])
+        self._run(self.mtest_command + ['--setup=onlyinbar', '--no-suite=main:', '--no-suite=foo:'])
+        self._run(self.mtest_command + ['--setup=bar:onlyinbar'])
+        self.assertRaises(subprocess.CalledProcessError, self._run,
+                          self.mtest_command + ['--setup=foo:onlyinbar'])
+        self.assertRaises(subprocess.CalledProcessError, self._run,
+                          self.mtest_command + ['--setup=main:onlyinbar'])
+
     def assertFailedTestCount(self, failure_count, command):
         try:
             self._run(command)
diff --git a/test cases/unit/13 testsetup selection/main.c b/test cases/unit/13 testsetup selection/main.c
new file mode 100644
index 0000000..cb3f748
--- /dev/null
+++ b/test cases/unit/13 testsetup selection/main.c
@@ -0,0 +1,3 @@
+int main() {
+	return 0;
+}
diff --git a/test cases/unit/13 testsetup selection/meson.build b/test cases/unit/13 testsetup selection/meson.build
new file mode 100644
index 0000000..ae996c5
--- /dev/null
+++ b/test cases/unit/13 testsetup selection/meson.build
@@ -0,0 +1,10 @@
+project('main', 'c')
+
+main = executable('main', 'main.c')
+test('Test main', main)
+
+add_test_setup('worksforall')
+add_test_setup('missingfromfoo')
+
+subproject('foo')
+subproject('bar')
diff --git a/test cases/unit/13 testsetup selection/subprojects/bar/bar.c b/test cases/unit/13 testsetup selection/subprojects/bar/bar.c
new file mode 100644
index 0000000..cb3f748
--- /dev/null
+++ b/test cases/unit/13 testsetup selection/subprojects/bar/bar.c
@@ -0,0 +1,3 @@
+int main() {
+	return 0;
+}
diff --git a/test cases/unit/13 testsetup selection/subprojects/bar/meson.build b/test cases/unit/13 testsetup selection/subprojects/bar/meson.build
new file mode 100644
index 0000000..1155a88
--- /dev/null
+++ b/test cases/unit/13 testsetup selection/subprojects/bar/meson.build
@@ -0,0 +1,6 @@
+project('bar', 'c')
+bar = executable('bar', 'bar.c')
+test('Test bar', bar)
+add_test_setup('onlyinbar')
+add_test_setup('worksforall')
+add_test_setup('missingfromfoo')
diff --git a/test cases/unit/13 testsetup selection/subprojects/foo/foo.c b/test cases/unit/13 testsetup selection/subprojects/foo/foo.c
new file mode 100644
index 0000000..cb3f748
--- /dev/null
+++ b/test cases/unit/13 testsetup selection/subprojects/foo/foo.c
@@ -0,0 +1,3 @@
+int main() {
+	return 0;
+}
diff --git a/test cases/unit/13 testsetup selection/subprojects/foo/meson.build b/test cases/unit/13 testsetup selection/subprojects/foo/meson.build
new file mode 100644
index 0000000..2eef840
--- /dev/null
+++ b/test cases/unit/13 testsetup selection/subprojects/foo/meson.build
@@ -0,0 +1,4 @@
+project('foo', 'c')
+foo = executable('foo', 'foo.c')
+test('Test foo', foo)
+add_test_setup('worksforall')