Can specify test setups and run them with mesontest.

8 files changed, 130 insertions, 20 deletions

diff --git a/mesonbuild/build.py b/mesonbuild/build.py
index 449afe7..f895531 100644
--- a/mesonbuild/build.py
+++ b/mesonbuild/build.py
@@ -98,6 +98,7 @@ class Build:
         self.dep_manifest_name = None
         self.dep_manifest = {}
         self.cross_stdlibs = {}
+        self.test_setups = {}
 
     def add_compiler(self, compiler):
         if self.static_linker is None and compiler.needs_static_linker():
@@ -1507,3 +1508,10 @@ class RunScript(dict):
         assert(isinstance(args, list))
         self['exe'] = script
         self['args'] = args
+
+class TestSetup:
+    def __init__(self, *, exe_wrapper=None, gdb=None, timeout_multiplier=None, env=None):
+        self.exe_wrapper = exe_wrapper
+        self.gdb = gdb
+        self.timeout_multiplier = timeout_multiplier
+        self.env = env
diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py
index ac15401..f561062 100644
--- a/mesonbuild/interpreter.py
+++ b/mesonbuild/interpreter.py
@@ -1207,6 +1207,7 @@ class Interpreter(InterpreterBase):
                            'add_project_arguments': self.func_add_project_arguments,
                            'add_global_link_arguments': self.func_add_global_link_arguments,
                            'add_project_link_arguments': self.func_add_project_link_arguments,
+                           'add_test_setup' : self.func_add_test_setup,
                            'add_languages': self.func_add_languages,
                            'find_program': self.func_find_program,
                            'find_library': self.func_find_library,
@@ -1942,22 +1943,7 @@ requirements use the version keyword argument instead.''')
     def func_test(self, node, args, kwargs):
         self.add_test(node, args, kwargs, True)
 
-    def add_test(self, node, args, kwargs, is_base_test):
-        if len(args) != 2:
-            raise InterpreterException('Incorrect number of arguments')
-        if not isinstance(args[0], str):
-            raise InterpreterException('First argument of test must be a string.')
-        if not isinstance(args[1], (ExecutableHolder, JarHolder, ExternalProgramHolder)):
-            raise InterpreterException('Second argument must be executable.')
-        par = kwargs.get('is_parallel', True)
-        if not isinstance(par, bool):
-            raise InterpreterException('Keyword argument is_parallel must be a boolean.')
-        cmd_args = kwargs.get('args', [])
-        if not isinstance(cmd_args, list):
-            cmd_args = [cmd_args]
-        for i in cmd_args:
-            if not isinstance(i, (str, mesonlib.File)):
-                raise InterpreterException('Command line arguments must be strings')
+    def unpack_env_kwarg(self, kwargs):
         envlist = kwargs.get('env', [])
         if isinstance(envlist, EnvironmentVariablesHolder):
             env = envlist.held_object
@@ -1974,8 +1960,25 @@ requirements use the version keyword argument instead.''')
                 if ' ' in k:
                     raise InterpreterException('Env var key must not have spaces in it.')
                 env[k] = val
-        if not isinstance(envlist, list):
-            envlist = [envlist]
+        return env
+
+    def add_test(self, node, args, kwargs, is_base_test):
+        if len(args) != 2:
+            raise InterpreterException('Incorrect number of arguments')
+        if not isinstance(args[0], str):
+            raise InterpreterException('First argument of test must be a string.')
+        if not isinstance(args[1], (ExecutableHolder, JarHolder, ExternalProgramHolder)):
+            raise InterpreterException('Second argument must be executable.')
+        par = kwargs.get('is_parallel', True)
+        if not isinstance(par, bool):
+            raise InterpreterException('Keyword argument is_parallel must be a boolean.')
+        cmd_args = kwargs.get('args', [])
+        if not isinstance(cmd_args, list):
+            cmd_args = [cmd_args]
+        for i in cmd_args:
+            if not isinstance(i, (str, mesonlib.File)):
+                raise InterpreterException('Command line arguments must be strings')
+        env = self.unpack_env_kwarg(kwargs)
         should_fail = kwargs.get('should_fail', False)
         if not isinstance(should_fail, bool):
             raise InterpreterException('Keyword argument should_fail must be a boolean.')
@@ -2139,6 +2142,31 @@ requirements use the version keyword argument instead.''')
         return i
 
     @stringArgs
+    def func_add_test_setup(self, node, args, kwargs):
+        if len(args) != 1:
+            raise InterpreterException('Add_test_setup needs one argument for the setup name.')
+        setup_name = args[0]
+        try:
+            exe_wrapper = mesonlib.stringlistify(kwargs['exe_wrapper'])
+        except KeyError:
+            exe_wrapper = None
+        gdb = kwargs.get('gdb', False)
+        if not isinstance(gdb, bool):
+            raise InterpreterException('Gdb option must be a boolean')
+        timeout_multiplier = kwargs.get('timeout_multiplier', 1)
+        if not isinstance(timeout_multiplier, int):
+            raise InterpreterException('Timeout multiplier must be a number.')
+        env = self.unpack_env_kwarg(kwargs)
+        setupobj = build.TestSetup(exe_wrapper=exe_wrapper,
+                                   gdb=gdb,
+                                   timeout_multiplier=timeout_multiplier,
+                                   env=env)
+        if self.subproject == '':
+            # Dunno what we should do with subprojects really. Let's start simple
+            # and just use the master project ones.
+            self.build.test_setups[setup_name] = setupobj
+
+    @stringArgs
     def func_add_global_arguments(self, node, args, kwargs):
         if self.subproject != '':
             msg = 'Global arguments can not be set in subprojects because ' \
diff --git a/mesontest.py b/mesontest.py
index ecf1b41..01ed056 100755
--- a/mesontest.py
+++ b/mesontest.py
@@ -69,7 +69,7 @@ parser.add_argument('--suite', default=None, dest='suite',
 parser.add_argument('--no-stdsplit', default=True, dest='split', action='store_false',
                     help='Do not split stderr and stdout in test logs.')
 parser.add_argument('--print-errorlogs', default=False, action='store_true',
-                    help="Whether to print faling tests' logs.")
+                    help="Whether to print failing tests' logs.")
 parser.add_argument('--benchmark', default=False, action='store_true',
                     help="Run benchmarks instead of tests.")
 parser.add_argument('--logbase', default='testlog',
@@ -82,6 +82,8 @@ parser.add_argument('-t', '--timeout-multiplier', type=float, default=1.0,
                     help='Define a multiplier for test timeout, for example '
                     ' when running tests in particular conditions they might take'
                     ' more time to execute.')
+parser.add_argument('--setup', default=None, dest='setup',
+                    help='Which test setup to use.')
 parser.add_argument('args', nargs='*')
 
 class TestRun():
@@ -328,7 +330,11 @@ class TestHarness:
             logfilename = logfile_base + '.txt'
             jsonlogfilename = logfile_base + '.json'
         else:
-            wrap = self.options.wrapper.split()
+            if isinstance(self.options.wrapper, str):
+                wrap = self.options.wrapper.split()
+            else:
+                wrap = self.options.wrapper
+            assert(isinstance(wrap, list))
             namebase = wrap[0]
             logfilename = logfile_base + '-' + namebase.replace(' ', '_') + '.txt'
             jsonlogfilename = logfile_base + '-' + namebase.replace(' ', '_') + '.json'
@@ -453,11 +459,33 @@ def filter_tests(suite, tests):
     return [x for x in tests if suite in x.suite]
 
 
+def merge_suite_options(options):
+    buildfile = os.path.join(options.wd, 'meson-private/build.dat')
+    with open(buildfile, 'rb') as f:
+        build = pickle.load(f)
+    setups = build.test_setups
+    if options.setup not in setups:
+        sys.exit('Unknown test setup: %s' % options.setup)
+    current = setups[options.setup]
+    if not options.gdb:
+        options.gdb = current.gdb
+    if options.timeout_multiplier is None:
+        options.timeout_multiplier = current.timeout_multiplier
+#    if options.env is None:
+#        options.env = current.env # FIXME, should probably merge options here.
+    if options.wrapper is not None and current.exe_wrapper is not None:
+        sys.exit('Conflict: both test setup and command line specify an exe wrapper.')
+    if options.wrapper is None:
+        options.wrapper = current.exe_wrapper
+
 def run(args):
     options = parser.parse_args(args)
     if options.benchmark:
         options.num_processes = 1
 
+    if options.setup is not None:
+        merge_suite_options(options)
+
     if options.gdb:
         options.verbose = True
 
diff --git a/run_unittests.py b/run_unittests.py
index 179bed6..b7a393c 100755
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -63,6 +63,7 @@ class LinuxlikeTests(unittest.TestCase):
         self.meson_command = [sys.executable, os.path.join(src_root, 'meson.py')]
         self.mconf_command = [sys.executable, os.path.join(src_root, 'mesonconf.py')]
         self.mintro_command = [sys.executable, os.path.join(src_root, 'mesonintrospect.py')]
+        self.mtest_command = [sys.executable, os.path.join(src_root, 'mesontest.py'), '-C', self.builddir]
         self.ninja_command = [detect_ninja(), '-C', self.builddir]
         self.common_test_dir = os.path.join(src_root, 'test cases/common')
         self.vala_test_dir = os.path.join(src_root, 'test cases/vala')
@@ -89,6 +90,9 @@ class LinuxlikeTests(unittest.TestCase):
     def build(self):
         self._run(self.ninja_command)
 
+    def run_tests(self):
+        self._run(self.ninja_command + ['test'])
+
     def install(self):
         os.environ['DESTDIR'] = self.installdir
         self._run(self.ninja_command + ['install'])
@@ -387,6 +391,13 @@ class LinuxlikeTests(unittest.TestCase):
         meson_exe_dat2 = glob(os.path.join(self.privatedir, 'meson_exe*.dat'))
         self.assertListEqual(meson_exe_dat1, meson_exe_dat2)
 
+    def test_testsetups(self):
+        testdir = os.path.join(self.unit_test_dir, '2 testsetups')
+        self.init(testdir)
+        self.build()
+        self.run_tests()
+        self.assertRaises(subprocess.CalledProcessError,
+                          self._run, self.mtest_command  + ['--setup=valgrind'])
 
 class RewriterTests(unittest.TestCase):
 
@@ -446,5 +457,6 @@ class RewriterTests(unittest.TestCase):
         self.assertEqual(top, self.read_contents('meson.build'))
         self.assertEqual(s2, self.read_contents('sub2/meson.build'))
 
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/test cases/unit/2 testsetups/buggy.c b/test cases/unit/2 testsetups/buggy.c
new file mode 100644
index 0000000..1aa56f9
--- /dev/null
+++ b/test cases/unit/2 testsetups/buggy.c
@@ -0,0 +1,11 @@
+#include<stdio.h>
+#include<stdlib.h>
+
+#include<impl.h>
+
+int main(int argc, char **argv) {
+    char *ten = malloc(10);
+    do_nasty(ten);
+    free(ten);
+    return 0;
+}
diff --git a/test cases/unit/2 testsetups/impl.c b/test cases/unit/2 testsetups/impl.c
new file mode 100644
index 0000000..d87f3de
--- /dev/null
+++ b/test cases/unit/2 testsetups/impl.c
@@ -0,0 +1,5 @@
+/* Write past the end. */
+
+void do_nasty(char *ptr) {
+    ptr[10] = 'n';
+}
diff --git a/test cases/unit/2 testsetups/impl.h b/test cases/unit/2 testsetups/impl.h
new file mode 100644
index 0000000..7a08cb3
--- /dev/null
+++ b/test cases/unit/2 testsetups/impl.h
@@ -0,0 +1,3 @@
+#pragma once
+
+void do_nasty(char *ptr);
diff --git a/test cases/unit/2 testsetups/meson.build b/test cases/unit/2 testsetups/meson.build
new file mode 100644
index 0000000..f2295f5
--- /dev/null
+++ b/test cases/unit/2 testsetups/meson.build
@@ -0,0 +1,15 @@
+project('testsetups', 'c')
+
+vg = find_program('valgrind', required : false)
+
+env = environment()
+env.set('TEST_ENV', '1')
+
+add_test_setup('valgrind',
+  exe_wrapper : ['valgrind', '--error-exitcode=1', '--leak-check=full'],
+  timeout_multiplier : 100,
+  env : env)
+
+buggy = executable('buggy', 'buggy.c', 'impl.c')
+test('Test buggy', buggy)
+