Merge pull request #2697 from mesonbuild/custom-target-depends-serialize

custom target: Consider all build depends while serializing

5 files changed, 110 insertions, 27 deletions

diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py
index 067b719..f7b56bd 100644
--- a/mesonbuild/backend/backends.py
+++ b/mesonbuild/backend/backends.py
@@ -238,8 +238,14 @@ class Backend:
         return obj_list
 
     def serialize_executable(self, exe, cmd_args, workdir, env={},
-                             capture=None):
+                             extra_paths=None, capture=None):
         import hashlib
+        if extra_paths is None:
+            # The callee didn't check if we needed extra paths, so check it here
+            if mesonlib.is_windows() or mesonlib.is_cygwin():
+                extra_paths = self.determine_windows_extra_paths(exe, [])
+            else:
+                extra_paths = []
         # Can't just use exe.name here; it will likely be run more than once
         if isinstance(exe, (dependencies.ExternalProgram,
                             build.BuildTarget, build.CustomTarget)):
@@ -272,10 +278,6 @@ class Backend:
                 exe_wrapper = self.environment.cross_info.config['binaries'].get('exe_wrapper', None)
             else:
                 exe_wrapper = None
-            if mesonlib.is_windows() or mesonlib.is_cygwin():
-                extra_paths = self.determine_windows_extra_paths(exe)
-            else:
-                extra_paths = []
             es = ExecutableSerialisation(basename, exe_cmd, cmd_args, env,
                                          is_cross_built, exe_wrapper, workdir,
                                          extra_paths, capture)
@@ -531,23 +533,27 @@ class Backend:
             args.append(d_arg)
         return args
 
-    def determine_windows_extra_paths(self, target):
+    def determine_windows_extra_paths(self, target, extra_bdeps):
         '''On Windows there is no such thing as an rpath.
         We must determine all locations of DLLs that this exe
         links to and return them so they can be used in unit
         tests.'''
-        if not isinstance(target, build.Executable):
-            return []
-        prospectives = target.get_transitive_link_deps()
         result = []
+        prospectives = []
+        if isinstance(target, build.Executable):
+            prospectives = target.get_transitive_link_deps()
+            # External deps
+            for deppath in self.rpaths_for_bundled_shared_libraries(target):
+                result.append(os.path.normpath(os.path.join(self.environment.get_build_dir(), deppath)))
+        for bdep in extra_bdeps:
+            prospectives += bdep.get_transitive_link_deps()
+        # Internal deps
         for ld in prospectives:
             if ld == '' or ld == '.':
                 continue
             dirseg = os.path.join(self.environment.get_build_dir(), self.get_target_dir(ld))
             if dirseg not in result:
                 result.append(dirseg)
-        for deppath in self.rpaths_for_bundled_shared_libraries(target):
-            result.append(os.path.normpath(os.path.join(self.environment.get_build_dir(), deppath)))
         return result
 
     def write_benchmark_file(self, datafile):
@@ -578,7 +584,7 @@ class Backend:
             else:
                 exe_wrapper = None
             if mesonlib.is_windows() or mesonlib.is_cygwin():
-                extra_paths = self.determine_windows_extra_paths(exe)
+                extra_paths = self.determine_windows_extra_paths(exe, [])
             else:
                 extra_paths = []
             cmd_args = []
diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index bcda603..e35a391 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -504,22 +504,30 @@ int dummy;
             # Add a dependency on all the outputs of this target
             for output in d.get_outputs():
                 elem.add_dep(os.path.join(self.get_target_dir(d), output))
+        serialize = False
+        extra_paths = []
         # If the target requires capturing stdout, then use the serialized
         # executable wrapper to capture that output and save it to a file.
-        #
+        if target.capture:
+            serialize = True
         # If the command line requires a newline, also use the wrapper, as
         # ninja does not support them in its build rule syntax.
-        #
+        if any('\n' in c for c in cmd):
+            serialize = True
         # Windows doesn't have -rpath, so for EXEs that need DLLs built within
         # the project, we need to set PATH so the DLLs are found. We use
         # a serialized executable wrapper for that and check if the
         # CustomTarget command needs extra paths first.
-        if (target.capture or any('\n' in c for c in cmd) or
-                ((mesonlib.is_windows() or mesonlib.is_cygwin()) and
-                 self.determine_windows_extra_paths(target.command[0]))):
+        if mesonlib.is_windows() or mesonlib.is_cygwin():
+            extra_bdeps = target.get_transitive_build_target_deps()
+            extra_paths = self.determine_windows_extra_paths(target.command[0], extra_bdeps)
+            if extra_paths:
+                serialize = True
+        if serialize:
             exe_data = self.serialize_executable(target.command[0], cmd[1:],
                                                  # All targets are built from the build dir
                                                  self.environment.get_build_dir(),
+                                                 extra_paths=extra_paths,
                                                  capture=ofilenames[0] if target.capture else None)
             cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data]
             cmd_type = 'meson_exe.py custom'
diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py
index ea02580..e872a04 100644
--- a/mesonbuild/backend/vs2010backend.py
+++ b/mesonbuild/backend/vs2010backend.py
@@ -429,9 +429,12 @@ class Vs2010Backend(backends.Backend):
         # Always use a wrapper because MSBuild eats random characters when
         # there are many arguments.
         tdir_abs = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
+        extra_bdeps = target.get_transitive_build_target_deps()
+        extra_paths = self.determine_windows_extra_paths(target.command[0], extra_bdeps)
         exe_data = self.serialize_executable(target.command[0], cmd[1:],
                                              # All targets run from the target dir
                                              tdir_abs,
+                                             extra_paths=extra_paths,
                                              capture=ofilenames[0] if target.capture else None)
         wrapper_cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data]
         ET.SubElement(customstep, 'Command').text = ' '.join(self.quote_arguments(wrapper_cmd))
diff --git a/mesonbuild/build.py b/mesonbuild/build.py
index 8eb95dc..7757300 100644
--- a/mesonbuild/build.py
+++ b/mesonbuild/build.py
@@ -1549,6 +1549,24 @@ class CustomTarget(Target):
                 deps.append(c)
         return deps
 
+    def get_transitive_build_target_deps(self):
+        '''
+        Recursively fetch the build targets that this custom target depends on,
+        whether through `command:`, `depends:`, or `sources:` The recursion is
+        only performed on custom targets.
+        This is useful for setting PATH on Windows for finding required DLLs.
+        F.ex, if you have a python script that loads a C module that links to
+        other DLLs in your project.
+        '''
+        bdeps = set()
+        deps = self.get_target_dependencies()
+        for d in deps:
+            if isinstance(d, BuildTarget):
+                bdeps.add(d)
+            elif isinstance(d, CustomTarget):
+                bdeps.update(d.get_transitive_build_target_deps())
+        return bdeps
+
     def flatten_command(self, cmd):
         cmd = listify(cmd, unholder=True)
         final_cmd = []
diff --git a/mesonbuild/dependencies/misc.py b/mesonbuild/dependencies/misc.py
index 61dd953..ce9313e 100644
--- a/mesonbuild/dependencies/misc.py
+++ b/mesonbuild/dependencies/misc.py
@@ -21,6 +21,8 @@ import shlex
 import shutil
 import sysconfig
 
+from pathlib import Path
+
 from .. import mlog
 from .. import mesonlib
 from ..mesonlib import Popen_safe, extract_as_list
@@ -603,6 +605,7 @@ class Python3Dependency(ExternalDependency):
     def __init__(self, environment, kwargs):
         super().__init__('python3', environment, None, kwargs)
         self.name = 'python3'
+        self.static = kwargs.get('static', False)
         # We can only be sure that it is Python 3 at this point
         self.version = '3'
         self.pkgdep = None
@@ -638,12 +641,55 @@ class Python3Dependency(ExternalDependency):
         else:
             mlog.log('Dependency', mlog.bold(self.name), 'found:', mlog.red('NO'))
 
+    @staticmethod
+    def get_windows_python_arch():
+        pyplat = sysconfig.get_platform()
+        if pyplat == 'mingw':
+            pycc = sysconfig.get_config_var('CC')
+            if pycc.startswith('x86_64'):
+                return '64'
+            elif pycc.startswith(('i686', 'i386')):
+                return '32'
+            else:
+                mlog.log('MinGW Python built with unknown CC {!r}, please file'
+                         'a bug'.format(pycc))
+                return None
+        elif pyplat == 'win32':
+            return '32'
+        elif pyplat in ('win64', 'win-amd64'):
+            return '64'
+        mlog.log('Unknown Windows Python platform {!r}'.format(pyplat))
+        return None
+
+    def get_windows_link_args(self):
+        pyplat = sysconfig.get_platform()
+        if pyplat.startswith('win'):
+            vernum = sysconfig.get_config_var('py_version_nodot')
+            if self.static:
+                libname = 'libpython{}.a'.format(vernum)
+            else:
+                libname = 'python{}.lib'.format(vernum)
+            lib = Path(sysconfig.get_config_var('base')) / 'libs' / libname
+        elif pyplat == 'mingw':
+            if self.static:
+                libname = sysconfig.get_config_var('LIBRARY')
+            else:
+                libname = sysconfig.get_config_var('LDLIBRARY')
+            lib = Path(sysconfig.get_config_var('LIBDIR')) / libname
+        if not lib.exists():
+            mlog.log('Could not find Python3 library {!r}'.format(str(lib)))
+            return None
+        return [str(lib)]
+
     def _find_libpy3_windows(self, env):
         '''
         Find python3 libraries on Windows and also verify that the arch matches
         what we are building for.
         '''
-        pyarch = sysconfig.get_platform()
+        pyarch = self.get_windows_python_arch()
+        if pyarch is None:
+            self.is_found = False
+            return
         arch = detect_cpu_family(env.coredata.compilers)
         if arch == 'x86':
             arch = '32'
@@ -656,22 +702,24 @@ class Python3Dependency(ExternalDependency):
             self.is_found = False
             return
         # Pyarch ends in '32' or '64'
-        if arch != pyarch[-2:]:
-            mlog.log('Need', mlog.bold(self.name),
-                     'for {}-bit, but found {}-bit'.format(arch, pyarch[-2:]))
+        if arch != pyarch:
+            mlog.log('Need', mlog.bold(self.name), 'for {}-bit, but '
+                     'found {}-bit'.format(arch, pyarch))
+            self.is_found = False
+            return
+        # This can fail if the library is not found
+        largs = self.get_windows_link_args()
+        if largs is None:
             self.is_found = False
             return
+        self.link_args = largs
+        # Compile args
         inc = sysconfig.get_path('include')
         platinc = sysconfig.get_path('platinclude')
         self.compile_args = ['-I' + inc]
         if inc != platinc:
             self.compile_args.append('-I' + platinc)
-        # Nothing exposes this directly that I coulf find
-        basedir = sysconfig.get_config_var('base')
-        vernum = sysconfig.get_config_var('py_version_nodot')
-        self.link_args = ['-L{}/libs'.format(basedir),
-                          '-lpython{}'.format(vernum)]
-        self.version = sysconfig.get_config_var('py_version_short')
+        self.version = sysconfig.get_config_var('py_version')
         self.is_found = True
 
     def get_methods(self):