Merge pull request #1356 from centricular/cross-platform-unit-tests

Run unit tests on more platforms, and more

9 files changed, 119 insertions, 73 deletions

diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py
index 46f8563..7372c4c 100644
--- a/mesonbuild/backend/backends.py
+++ b/mesonbuild/backend/backends.py
@@ -215,13 +215,13 @@ class Backend:
         exe_data = os.path.join(self.environment.get_scratch_dir(), scratch_file)
         with open(exe_data, 'wb') as f:
             if isinstance(exe, dependencies.ExternalProgram):
-                exe_fullpath = exe.fullpath
+                exe_cmd = exe.get_command()
                 exe_needs_wrapper = False
             elif isinstance(exe, (build.BuildTarget, build.CustomTarget)):
-                exe_fullpath = [self.get_target_filename_abs(exe)]
+                exe_cmd = [self.get_target_filename_abs(exe)]
                 exe_needs_wrapper = exe.is_cross
             else:
-                exe_fullpath = [exe]
+                exe_cmd = [exe]
                 exe_needs_wrapper = False
             is_cross = exe_needs_wrapper and \
                 self.environment.is_cross_build() and \
@@ -235,7 +235,7 @@ class Backend:
                 extra_paths = self.determine_windows_extra_paths(exe)
             else:
                 extra_paths = []
-            es = ExecutableSerialisation(basename, exe_fullpath, cmd_args, env,
+            es = ExecutableSerialisation(basename, exe_cmd, cmd_args, env,
                                          is_cross, exe_wrapper, workdir,
                                          extra_paths, capture)
             pickle.dump(es, f)
@@ -444,9 +444,9 @@ class Backend:
         for t in tests:
             exe = t.get_exe()
             if isinstance(exe, dependencies.ExternalProgram):
-                fname = exe.fullpath
+                cmd = exe.get_command()
             else:
-                fname = [os.path.join(self.environment.get_build_dir(), self.get_target_filename(t.get_exe()))]
+                cmd = [os.path.join(self.environment.get_build_dir(), self.get_target_filename(t.get_exe()))]
             is_cross = self.environment.is_cross_build() and \
                 self.environment.cross_info.need_cross_compiler() and \
                 self.environment.cross_info.need_exe_wrapper()
@@ -471,7 +471,7 @@ class Backend:
                     cmd_args.append(self.get_target_filename(a))
                 else:
                     raise MesonException('Bad object in test command.')
-            ts = TestSerialisation(t.get_name(), t.suite, fname, is_cross, exe_wrapper,
+            ts = TestSerialisation(t.get_name(), t.suite, cmd, is_cross, exe_wrapper,
                                    t.is_parallel, cmd_args, t.env, t.should_fail,
                                    t.timeout, t.workdir, extra_paths)
             arr.append(ts)
diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index a22e0ab..27e1e9a 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -30,9 +30,11 @@ from collections import OrderedDict
 if mesonlib.is_windows():
     quote_char = '"'
     execute_wrapper = 'cmd /c'
+    rmfile_prefix = 'del /f /s /q {} &&'
 else:
     quote_char = "'"
     execute_wrapper = ''
+    rmfile_prefix = 'rm -f {} &&'
 
 def ninja_quote(text):
     return text.replace(' ', '$ ').replace(':', '$:')
@@ -1238,10 +1240,16 @@ int dummy;
 '''
         else:
             command_template = ' command = {executable} $LINK_ARGS {output_args} $in\n'
+        cmdlist = []
+        if isinstance(static_linker, compilers.ArLinker):
+            # `ar` has no options to overwrite archives. It always appends,
+            # which is never what we want. Delete an existing library first if
+            # it exists. https://github.com/mesonbuild/meson/issues/1355
+            cmdlist = [execute_wrapper, rmfile_prefix.format('$out')]
+        cmdlist += static_linker.get_exelist()
         command = command_template.format(
-            executable=' '.join(static_linker.get_exelist()),
-            output_args=' '.join(static_linker.get_output_args('$out'))
-        )
+            executable=' '.join(cmdlist),
+            output_args=' '.join(static_linker.get_output_args('$out')))
         description = ' description = Static linking library $out\n\n'
         outfile.write(rule)
         outfile.write(command)
diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py
index 666da7d..547889c 100644
--- a/mesonbuild/backend/vs2010backend.py
+++ b/mesonbuild/backend/vs2010backend.py
@@ -395,7 +395,7 @@ class Vs2010Backend(backends.Backend):
             if isinstance(i, build.BuildTarget):
                 cmd.append(os.path.join(self.environment.get_build_dir(), self.get_target_filename(i)))
             elif isinstance(i, dependencies.ExternalProgram):
-                cmd += i.fullpath
+                cmd += i.get_command()
             else:
                 cmd.append(i)
         cmd_templ = '''"%s" ''' * len(cmd)
diff --git a/mesonbuild/dependencies.py b/mesonbuild/dependencies.py
index 9525ffa..e4317f1 100644
--- a/mesonbuild/dependencies.py
+++ b/mesonbuild/dependencies.py
@@ -120,8 +120,8 @@ class PkgConfigDependency(Dependency):
                 if self.required:
                     raise DependencyException('Pkg-config binary missing from cross file')
             else:
-                potential_pkgbin = ExternalProgram(environment.cross_info.config['binaries'].get('pkgconfig', 'non_existing_binary'),
-                                                   silent=True)
+                pkgname = environment.cross_info.config['binaries']['pkgconfig']
+                potential_pkgbin = ExternalProgram(pkgname, silent=True)
                 if potential_pkgbin.found():
                     # FIXME, we should store all pkg-configs in ExternalPrograms.
                     # However that is too destabilizing a change to do just before release.
@@ -402,24 +402,28 @@ class WxDependency(Dependency):
         return self.is_found
 
 class ExternalProgram:
-    windows_exts = ('exe', 'com', 'bat')
+    windows_exts = ('exe', 'msc', 'com', 'bat')
 
-    def __init__(self, name, fullpath=None, silent=False, search_dir=None):
+    def __init__(self, name, command=None, silent=False, search_dir=None):
         self.name = name
-        if fullpath is not None:
-            if not isinstance(fullpath, list):
-                self.fullpath = [fullpath]
+        if command is not None:
+            if not isinstance(command, list):
+                self.command = [command]
             else:
-                self.fullpath = fullpath
+                self.command = command
         else:
-            self.fullpath = self._search(name, search_dir)
+            self.command = self._search(name, search_dir)
         if not silent:
             if self.found():
                 mlog.log('Program', mlog.bold(name), 'found:', mlog.green('YES'),
-                         '(%s)' % ' '.join(self.fullpath))
+                         '(%s)' % ' '.join(self.command))
             else:
                 mlog.log('Program', mlog.bold(name), 'found:', mlog.red('NO'))
 
+    def __repr__(self):
+        r = '<{} {!r} -> {!r}>'
+        return r.format(self.__class__.__name__, self.name, self.command)
+
     @staticmethod
     def _shebang_to_cmd(script):
         """
@@ -473,34 +477,63 @@ class ExternalProgram:
         return self._shebang_to_cmd(trial)
 
     def _search(self, name, search_dir):
+        '''
+        Search in the specified dir for the specified executable by name
+        and if not found search in PATH
+        '''
         commands = self._search_dir(name, search_dir)
         if commands:
             return commands
         # Do a standard search in PATH
-        fullpath = shutil.which(name)
-        if fullpath or not mesonlib.is_windows():
+        command = shutil.which(name)
+        if not mesonlib.is_windows():
             # On UNIX-like platforms, the standard PATH search is enough
-            return [fullpath]
-        # On Windows, if name is an absolute path, we need the extension too
-        for ext in self.windows_exts:
-            fullpath = '{}.{}'.format(name, ext)
-            if os.path.exists(fullpath):
-                return [fullpath]
-        # On Windows, interpreted scripts must have an extension otherwise they
-        # cannot be found by a standard PATH search. So we do a custom search
-        # where we manually search for a script with a shebang in PATH.
-        search_dirs = os.environ.get('PATH', '').split(';')
-        for search_dir in search_dirs:
-            commands = self._search_dir(name, search_dir)
+            return [command]
+        # HERE BEGINS THE TERROR OF WINDOWS
+        if command:
+            # On Windows, even if the PATH search returned a full path, we can't be
+            # sure that it can be run directly if it's not a native executable.
+            # For instance, interpreted scripts sometimes need to be run explicitly
+            # with an interpreter if the file association is not done properly.
+            name_ext = os.path.splitext(command)[1]
+            if name_ext[1:].lower() in self.windows_exts:
+                # Good, it can be directly executed
+                return [command]
+            # Try to extract the interpreter from the shebang
+            commands = self._shebang_to_cmd(command)
             if commands:
                 return commands
+        else:
+            # Maybe the name is an absolute path to a native Windows
+            # executable, but without the extension. This is technically wrong,
+            # but many people do it because it works in the MinGW shell.
+            if os.path.isabs(name):
+                for ext in self.windows_exts:
+                    command = '{}.{}'.format(name, ext)
+                    if os.path.exists(command):
+                        return [command]
+            # On Windows, interpreted scripts must have an extension otherwise they
+            # cannot be found by a standard PATH search. So we do a custom search
+            # where we manually search for a script with a shebang in PATH.
+            search_dirs = os.environ.get('PATH', '').split(';')
+            for search_dir in search_dirs:
+                commands = self._search_dir(name, search_dir)
+                if commands:
+                    return commands
         return [None]
 
     def found(self):
-        return self.fullpath[0] is not None
+        return self.command[0] is not None
 
     def get_command(self):
-        return self.fullpath[:]
+        return self.command[:]
+
+    def get_path(self):
+        # Assume that the last element is the full path to the script
+        # If it's not a script, this will be an array of length 1
+        if self.found():
+            return self.command[-1]
+        return None
 
     def get_name(self):
         return self.name
@@ -531,6 +564,9 @@ class ExternalLibrary(Dependency):
     def found(self):
         return self.is_found
 
+    def get_name(self):
+        return self.name
+
     def get_link_args(self):
         return self.link_args
 
@@ -994,7 +1030,7 @@ class QtBaseDependency(Dependency):
             if not self.qmake.found():
                 continue
             # Check that the qmake is for qt5
-            pc, stdo = Popen_safe(self.qmake.fullpath + ['-v'])[0:2]
+            pc, stdo = Popen_safe(self.qmake.get_command() + ['-v'])[0:2]
             if pc.returncode != 0:
                 continue
             if not 'Qt version ' + self.qtver in stdo:
@@ -1007,7 +1043,7 @@ class QtBaseDependency(Dependency):
             return
         self.version = re.search(self.qtver + '(\.\d+)+', stdo).group(0)
         # Query library path, header path, and binary path
-        stdo = Popen_safe(self.qmake.fullpath + ['-query'])[1]
+        stdo = Popen_safe(self.qmake.get_command() + ['-query'])[1]
         qvars = {}
         for line in stdo.split('\n'):
             line = line.strip()
diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py
index 224f98e..f6065d5 100644
--- a/mesonbuild/interpreter.py
+++ b/mesonbuild/interpreter.py
@@ -285,16 +285,16 @@ class ExternalProgramHolder(InterpreterObject):
         return self.found()
 
     def path_method(self, args, kwargs):
-        return self.get_command()
+        return self.held_object.get_path()
 
     def found(self):
         return self.held_object.found()
 
     def get_command(self):
-        return self.held_object.fullpath
+        return self.held_object.get_command()
 
     def get_name(self):
-        return self.held_object.name
+        return self.held_object.get_name()
 
 class ExternalLibraryHolder(InterpreterObject):
     def __init__(self, el):
@@ -308,9 +308,6 @@ class ExternalLibraryHolder(InterpreterObject):
     def found_method(self, args, kwargs):
         return self.found()
 
-    def get_filename(self):
-        return self.held_object.fullpath
-
     def get_name(self):
         return self.held_object.name
 
@@ -1424,7 +1421,8 @@ class Interpreter(InterpreterBase):
         elif isinstance(cmd, str):
             cmd = [cmd]
         else:
-            raise InterpreterException('First argument is of incorrect type.')
+            raise InterpreterException('First argument should be find_program() '
+                                       'or string, not {!r}'.format(cmd))
         expanded_args = []
         for a in mesonlib.flatten(cargs):
             if isinstance(a, str):
@@ -1759,7 +1757,6 @@ class Interpreter(InterpreterBase):
                     break
             self.coredata.base_options[optname] = oobj
 
-    @stringArgs
     def func_find_program(self, node, args, kwargs):
         if len(args) == 0:
             raise InterpreterException('No program name specified.')
@@ -1769,8 +1766,21 @@ class Interpreter(InterpreterBase):
         # Search for scripts relative to current subdir.
         # Do not cache found programs because find_program('foobar')
         # might give different results when run from different source dirs.
-        search_dir = os.path.join(self.environment.get_source_dir(), self.subdir)
+        source_dir = os.path.join(self.environment.get_source_dir(), self.subdir)
         for exename in args:
+            if isinstance(exename, mesonlib.File):
+                if exename.is_built:
+                    search_dir = os.path.join(self.environment.get_build_dir(),
+                                              exename.subdir)
+                else:
+                    search_dir = os.path.join(self.environment.get_source_dir(),
+                                              exename.subdir)
+                exename = exename.fname
+            elif isinstance(exename, str):
+                search_dir = source_dir
+            else:
+                raise InvalidArguments('find_program only accepts strings and '
+                                       'files, not {!r}'.format(exename))
             extprog = dependencies.ExternalProgram(exename, search_dir=search_dir)
             progobj = ExternalProgramHolder(extprog)
             if progobj.found():
diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py
index e30500f..6eab76e 100644
--- a/mesonbuild/mintro.py
+++ b/mesonbuild/mintro.py
@@ -23,6 +23,7 @@ import json, pickle
 from . import coredata, build
 import argparse
 import sys, os
+import pathlib
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--targets', action='store_true', dest='list_targets', default=False,
@@ -56,7 +57,9 @@ def determine_installed_path(target, installdata):
     fname = i[0]
     outdir = i[1]
     outname = os.path.join(installdata.prefix, outdir, os.path.split(fname)[-1])
-    return outname
+    # Normalize the path by using os.path.sep consistently, etc.
+    # Does not change the effective path.
+    return str(pathlib.PurePath(outname))
 
 
 def list_installed(installdata):
@@ -111,23 +114,11 @@ def list_target_files(target_name, coredata, builddata):
     print(json.dumps(sources))
 
 def list_buildoptions(coredata, builddata):
-    buildtype = {'choices': ['plain', 'debug', 'debugoptimized', 'release', 'minsize'],
-                 'type': 'combo',
-                 'value': coredata.get_builtin_option('buildtype'),
-                 'description': 'Build type',
-                 'name': 'type'}
-    strip = {'value': coredata.get_builtin_option('strip'),
-             'type': 'boolean',
-             'description': 'Strip on install',
-             'name': 'strip'}
-    unity = {'value': coredata.get_builtin_option('unity'),
-             'type': 'boolean',
-             'description': 'Unity build',
-             'name': 'unity'}
-    optlist = [buildtype, strip, unity]
+    optlist = []
     add_keys(optlist, coredata.user_options)
     add_keys(optlist, coredata.compiler_options)
     add_keys(optlist, coredata.base_options)
+    add_keys(optlist, coredata.builtins)
     print(json.dumps(optlist))
 
 def add_keys(optlist, options):
diff --git a/mesonbuild/modules/qt4.py b/mesonbuild/modules/qt4.py
index 33c9f80..7146739 100644
--- a/mesonbuild/modules/qt4.py
+++ b/mesonbuild/modules/qt4.py
@@ -48,7 +48,7 @@ class Qt4Module(ExtensionModule):
                 raise MesonException('Moc preprocessor is not for Qt 4. Output:\n%s\n%s' %
                                      (stdout, stderr))
             mlog.log(' moc:', mlog.green('YES'), '(%s, %s)' %
-                     (' '.join(self.moc.fullpath), moc_ver.split()[-1]))
+                     (self.moc.get_path(), moc_ver.split()[-1]))
         else:
             mlog.log(' moc:', mlog.red('NO'))
         if self.uic.found():
@@ -61,7 +61,7 @@ class Qt4Module(ExtensionModule):
                 raise MesonException('Uic compiler is not for Qt4. Output:\n%s\n%s' %
                                      (stdout, stderr))
             mlog.log(' uic:', mlog.green('YES'), '(%s, %s)' %
-                     (' '.join(self.uic.fullpath), uic_ver.split()[-1]))
+                     (self.uic.get_path(), uic_ver.split()[-1]))
         else:
             mlog.log(' uic:', mlog.red('NO'))
         if self.rcc.found():
@@ -74,7 +74,7 @@ class Qt4Module(ExtensionModule):
                 raise MesonException('Rcc compiler is not for Qt 4. Output:\n%s\n%s' %
                                      (stdout, stderr))
             mlog.log(' rcc:', mlog.green('YES'), '(%s, %s)'
-                     % (' '.join(self.rcc.fullpath), rcc_ver.split()[-1]))
+                     % (self.rcc.get_path(), rcc_ver.split()[-1]))
         else:
             mlog.log(' rcc:', mlog.red('NO'))
         self.tools_detected = True
diff --git a/mesonbuild/modules/qt5.py b/mesonbuild/modules/qt5.py
index b4f1475..2a87a80 100644
--- a/mesonbuild/modules/qt5.py
+++ b/mesonbuild/modules/qt5.py
@@ -50,7 +50,7 @@ class Qt5Module(ExtensionModule):
                 raise MesonException('Moc preprocessor is not for Qt 5. Output:\n%s\n%s' %
                                      (stdout, stderr))
             mlog.log(' moc:', mlog.green('YES'), '(%s, %s)' %
-                     (' '.join(self.moc.fullpath), moc_ver.split()[-1]))
+                     (self.moc.get_path(), moc_ver.split()[-1]))
         else:
             mlog.log(' moc:', mlog.red('NO'))
         if self.uic.found():
@@ -65,7 +65,7 @@ class Qt5Module(ExtensionModule):
                 raise MesonException('Uic compiler is not for Qt 5. Output:\n%s\n%s' %
                                      (stdout, stderr))
             mlog.log(' uic:', mlog.green('YES'), '(%s, %s)' %
-                     (' '.join(self.uic.fullpath), uic_ver.split()[-1]))
+                     (self.uic.get_path(), uic_ver.split()[-1]))
         else:
             mlog.log(' uic:', mlog.red('NO'))
         if self.rcc.found():
@@ -80,7 +80,7 @@ class Qt5Module(ExtensionModule):
                 raise MesonException('Rcc compiler is not for Qt 5. Output:\n%s\n%s' %
                                      (stdout, stderr))
             mlog.log(' rcc:', mlog.green('YES'), '(%s, %s)'
-                     % (' '.join(self.rcc.fullpath), rcc_ver.split()[-1]))
+                     % (self.rcc.get_path(), rcc_ver.split()[-1]))
         else:
             mlog.log(' rcc:', mlog.red('NO'))
         self.tools_detected = True
diff --git a/mesonbuild/modules/rpm.py b/mesonbuild/modules/rpm.py
index bd8a3c4..bbfeaa0 100644
--- a/mesonbuild/modules/rpm.py
+++ b/mesonbuild/modules/rpm.py
@@ -98,17 +98,18 @@ class RPMModule(ExtensionModule):
             for dep in state.environment.coredata.deps:
                 fn.write('BuildRequires: pkgconfig(%s)\n' % dep[0])
             for lib in state.environment.coredata.ext_libs.values():
-                fn.write('BuildRequires: %s # FIXME\n' % lib.fullpath)
-                mlog.warning('replace', mlog.bold(lib.fullpath), 'with real package.',
+                name = lib.get_name()
+                fn.write('BuildRequires: {} # FIXME\n'.format(name))
+                mlog.warning('replace', mlog.bold(name), 'with the real package.',
                              'You can use following command to find package which '
                              'contains this lib:',
-                             mlog.bold('dnf provides %s' % lib.fullpath))
+                             mlog.bold("dnf provides '*/lib{}.so".format(name))
             for prog in state.environment.coredata.ext_progs.values():
                 if not prog.found():
                     fn.write('BuildRequires: %%{_bindir}/%s # FIXME\n' %
                              prog.get_name())
                 else:
-                    fn.write('BuildRequires: %s\n' % ' '.join(prog.fullpath))
+                    fn.write('BuildRequires: {}\n'.format(prog.get_path()))
             fn.write('BuildRequires: meson\n')
             fn.write('\n')
             fn.write('%description\n')