Merge pull request #5644 from bonzini/meson-exe-cmdline

Show command line in `ninja -v` for `capture: true` custom targets and generators

4 files changed, 117 insertions, 134 deletions

diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py
index 982b0ee..8c2752a 100644
--- a/mesonbuild/backend/backends.py
+++ b/mesonbuild/backend/backends.py
@@ -69,17 +69,13 @@ class TargetInstallData:
         self.optional = optional
 
 class ExecutableSerialisation:
-    def __init__(self, name, fname, cmd_args, env, is_cross, exe_wrapper,
-                 workdir, extra_paths, capture, needs_exe_wrapper: bool):
-        self.name = name
-        self.fname = fname
+    def __init__(self, cmd_args, env=None, exe_wrapper=None,
+                 workdir=None, extra_paths=None, capture=None):
         self.cmd_args = cmd_args
-        self.env = env
-        self.is_cross = is_cross
+        self.env = env or {}
         if exe_wrapper is not None:
             assert(isinstance(exe_wrapper, dependencies.ExternalProgram))
         self.exe_runner = exe_wrapper
-        self.needs_exe_wrapper = needs_exe_wrapper
         self.workdir = workdir
         self.extra_paths = extra_paths
         self.capture = capture
@@ -323,26 +319,61 @@ class Backend:
                 raise MesonException('Unknown data type in object list.')
         return obj_list
 
-    def serialize_executable(self, tname, exe, cmd_args, workdir, env=None,
-                             extra_paths=None, capture=None):
+    def as_meson_exe_cmdline(self, tname, exe, cmd_args, workdir=None,
+                             for_machine=MachineChoice.BUILD,
+                             extra_bdeps=None, capture=None, force_serialize=False):
         '''
         Serialize an executable for running with a generator or a custom target
         '''
         import hashlib
-        if env is None:
-            env = {}
-        if extra_paths is None:
-            # The callee didn't check if we needed extra paths, so check it here
-            if mesonlib.is_windows() or mesonlib.is_cygwin():
-                extra_paths = self.determine_windows_extra_paths(exe, [])
-            else:
-                extra_paths = []
-        # Can't just use exe.name here; it will likely be run more than once
+        machine = self.environment.machines[for_machine]
+        if machine.is_windows() or machine.is_cygwin():
+            extra_paths = self.determine_windows_extra_paths(exe, extra_bdeps or [])
+        else:
+            extra_paths = []
+
+        if isinstance(exe, dependencies.ExternalProgram):
+            exe_cmd = exe.get_command()
+            exe_for_machine = exe.for_machine
+        elif isinstance(exe, (build.BuildTarget, build.CustomTarget)):
+            exe_cmd = [self.get_target_filename_abs(exe)]
+            exe_for_machine = exe.for_machine
+        else:
+            exe_cmd = [exe]
+            exe_for_machine = MachineChoice.BUILD
+
+        is_cross_built = not self.environment.machines.matches_build_machine(exe_for_machine)
+        if is_cross_built and self.environment.need_exe_wrapper():
+            exe_wrapper = self.environment.get_exe_wrapper()
+            if not exe_wrapper.found():
+                msg = 'The exe_wrapper {!r} defined in the cross file is ' \
+                      'needed by target {!r}, but was not found. Please ' \
+                      'check the command and/or add it to PATH.'
+                raise MesonException(msg.format(exe_wrapper.name, tname))
+        else:
+            if exe_cmd[0].endswith('.jar'):
+                exe_cmd = ['java', '-jar'] + exe_cmd
+            elif exe_cmd[0].endswith('.exe') and not (mesonlib.is_windows() or mesonlib.is_cygwin()):
+                exe_cmd = ['mono'] + exe_cmd
+            exe_wrapper = None
+
+        force_serialize = force_serialize or extra_paths or workdir or \
+            exe_wrapper or any('\n' in c for c in cmd_args)
+        if not force_serialize:
+            if not capture:
+                return None
+            return (self.environment.get_build_command() +
+                    ['--internal', 'exe', '--capture', capture, '--'] + exe_cmd + cmd_args)
+
+        workdir = workdir or self.environment.get_build_dir()
+        env = {}
         if isinstance(exe, (dependencies.ExternalProgram,
                             build.BuildTarget, build.CustomTarget)):
             basename = exe.name
         else:
             basename = os.path.basename(exe)
+
+        # Can't just use exe.name here; it will likely be run more than once
         # Take a digest of the cmd args, env, workdir, and capture. This avoids
         # collisions and also makes the name deterministic over regenerations
         # which avoids a rebuild by Ninja because the cmdline stays the same.
@@ -352,31 +383,11 @@ class Backend:
         scratch_file = 'meson_exe_{0}_{1}.dat'.format(basename, digest)
         exe_data = os.path.join(self.environment.get_scratch_dir(), scratch_file)
         with open(exe_data, 'wb') as f:
-            if isinstance(exe, dependencies.ExternalProgram):
-                exe_cmd = exe.get_command()
-                exe_for_machine = exe.for_machine
-            elif isinstance(exe, (build.BuildTarget, build.CustomTarget)):
-                exe_cmd = [self.get_target_filename_abs(exe)]
-                exe_for_machine = exe.for_machine
-            else:
-                exe_cmd = [exe]
-                exe_for_machine = MachineChoice.BUILD
-            is_cross_built = not self.environment.machines.matches_build_machine(exe_for_machine)
-            if is_cross_built and self.environment.need_exe_wrapper():
-                exe_wrapper = self.environment.get_exe_wrapper()
-                if not exe_wrapper.found():
-                    msg = 'The exe_wrapper {!r} defined in the cross file is ' \
-                          'needed by target {!r}, but was not found. Please ' \
-                          'check the command and/or add it to PATH.'
-                    raise MesonException(msg.format(exe_wrapper.name, tname))
-            else:
-                exe_wrapper = None
-            es = ExecutableSerialisation(basename, exe_cmd, cmd_args, env,
-                                         is_cross_built, exe_wrapper, workdir,
-                                         extra_paths, capture,
-                                         self.environment.need_exe_wrapper())
+            es = ExecutableSerialisation(exe_cmd + cmd_args, env,
+                                         exe_wrapper, workdir,
+                                         extra_paths, capture)
             pickle.dump(es, f)
-        return exe_data
+        return self.environment.get_build_command() + ['--internal', 'exe', '--unpickle', exe_data]
 
     def serialize_tests(self):
         test_data = os.path.join(self.environment.get_scratch_dir(), 'meson_test_setup.dat')
diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index 570a153..b614b41 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -658,33 +658,13 @@ int dummy;
             # Add a dependency on all the outputs of this target
             for output in d.get_outputs():
                 elem.add_dep(os.path.join(self.get_target_dir(d), output))
-        serialize = False
-        extra_paths = []
-        # If the target requires capturing stdout, then use the serialized
-        # executable wrapper to capture that output and save it to a file.
-        if target.capture:
-            serialize = True
-        # If the command line requires a newline, also use the wrapper, as
-        # ninja does not support them in its build rule syntax.
-        if any('\n' in c for c in cmd):
-            serialize = True
-        # Windows doesn't have -rpath, so for EXEs that need DLLs built within
-        # the project, we need to set PATH so the DLLs are found. We use
-        # a serialized executable wrapper for that and check if the
-        # CustomTarget command needs extra paths first.
-        machine = self.environment.machines[target.for_machine]
-        if machine.is_windows() or machine.is_cygwin():
-            extra_bdeps = target.get_transitive_build_target_deps()
-            extra_paths = self.determine_windows_extra_paths(target.command[0], extra_bdeps)
-            if extra_paths:
-                serialize = True
-        if serialize:
-            exe_data = self.serialize_executable(target.name, target.command[0], cmd[1:],
-                                                 # All targets are built from the build dir
-                                                 self.environment.get_build_dir(),
-                                                 extra_paths=extra_paths,
-                                                 capture=ofilenames[0] if target.capture else None)
-            cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data]
+
+        meson_exe_cmd = self.as_meson_exe_cmdline(target.name, target.command[0], cmd[1:],
+                                                  for_machine=target.for_machine,
+                                                  extra_bdeps=target.get_transitive_build_target_deps(),
+                                                  capture=ofilenames[0] if target.capture else None)
+        if meson_exe_cmd:
+            cmd = meson_exe_cmd
             cmd_type = 'meson_exe.py custom'
         else:
             cmd_type = 'custom'
@@ -1786,19 +1766,13 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
                 outfilelist = outfilelist[len(generator.outputs):]
             args = self.replace_paths(target, args, override_subdir=subdir)
             cmdlist = exe_arr + self.replace_extra_args(args, genlist)
-            if generator.capture:
-                exe_data = self.serialize_executable(
-                    'generator ' + cmdlist[0],
-                    cmdlist[0],
-                    cmdlist[1:],
-                    self.environment.get_build_dir(),
-                    capture=outfiles[0]
-                )
-                cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data]
-                abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
-                os.makedirs(abs_pdir, exist_ok=True)
-            else:
-                cmd = cmdlist
+            meson_exe_cmd = self.as_meson_exe_cmdline('generator ' + cmdlist[0],
+                                                      cmdlist[0], cmdlist[1:],
+                                                      capture=outfiles[0] if generator.capture else None)
+            if meson_exe_cmd:
+                cmdlist = meson_exe_cmd
+            abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
+            os.makedirs(abs_pdir, exist_ok=True)
 
             elem = NinjaBuildElement(self.all_outputs, outfiles, rulename, infilename)
             elem.add_dep([self.get_target_filename(x) for x in generator.depends])
@@ -1813,7 +1787,7 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
                 elem.add_item('DESC', 'Generating source from {!r}.'.format(sole_output))
             if isinstance(exe, build.BuildTarget):
                 elem.add_dep(self.get_target_filename(exe))
-            elem.add_item('COMMAND', cmd)
+            elem.add_item('COMMAND', cmdlist)
             self.add_build(elem)
 
     def scan_fortran_module_outputs(self, target):
diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py
index 82fc0cf..a0f6a95 100644
--- a/mesonbuild/backend/vs2010backend.py
+++ b/mesonbuild/backend/vs2010backend.py
@@ -143,22 +143,24 @@ class Vs2010Backend(backends.Backend):
                             for x in args]
                     args = [x.replace('\\', '/') for x in args]
                     cmd = exe_arr + self.replace_extra_args(args, genlist)
-                    if generator.capture:
-                        exe_data = self.serialize_executable(
-                            'generator ' + cmd[0],
-                            cmd[0],
-                            cmd[1:],
-                            self.environment.get_build_dir(),
-                            capture=outfiles[0]
-                        )
-                        cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data]
-                        abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
-                        os.makedirs(abs_pdir, exist_ok=True)
+                    # Always use a wrapper because MSBuild eats random characters when
+                    # there are many arguments.
+                    tdir_abs = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
+                    cmd = self.as_meson_exe_cmdline(
+                        'generator ' + cmd[0],
+                        cmd[0],
+                        cmd[1:],
+                        workdir=tdir_abs,
+                        capture=outfiles[0] if generator.capture else None,
+                        force_serialize=True
+                    )
+                    deps = cmd[-1:] + deps
+                    abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
+                    os.makedirs(abs_pdir, exist_ok=True)
                     cbs = ET.SubElement(idgroup, 'CustomBuild', Include=infilename)
                     ET.SubElement(cbs, 'Command').text = ' '.join(self.quote_arguments(cmd))
                     ET.SubElement(cbs, 'Outputs').text = ';'.join(outfiles)
-                    if deps:
-                        ET.SubElement(cbs, 'AdditionalInputs').text = ';'.join(deps)
+                    ET.SubElement(cbs, 'AdditionalInputs').text = ';'.join(deps)
         return generator_output_files, custom_target_output_files, custom_target_include_dirs
 
     def generate(self, interp):
@@ -558,19 +560,18 @@ class Vs2010Backend(backends.Backend):
         # there are many arguments.
         tdir_abs = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
         extra_bdeps = target.get_transitive_build_target_deps()
-        extra_paths = self.determine_windows_extra_paths(target.command[0], extra_bdeps)
-        exe_data = self.serialize_executable(target.name, target.command[0], cmd[1:],
-                                             # All targets run from the target dir
-                                             tdir_abs,
-                                             extra_paths=extra_paths,
-                                             capture=ofilenames[0] if target.capture else None)
-        wrapper_cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data]
+        wrapper_cmd = self.as_meson_exe_cmdline(target.name, target.command[0], cmd[1:],
+                                                # All targets run from the target dir
+                                                workdir=tdir_abs,
+                                                extra_bdeps=extra_bdeps,
+                                                capture=ofilenames[0] if target.capture else None,
+                                                force_serialize=True)
         if target.build_always_stale:
             # Use a nonexistent file to always consider the target out-of-date.
             ofilenames += [self.nonexistent_file(os.path.join(self.environment.get_scratch_dir(),
                                                  'outofdate.file'))]
         self.add_custom_build(root, 'custom_target', ' '.join(self.quote_arguments(wrapper_cmd)),
-                              deps=[exe_data] + srcs + depend_files, outputs=ofilenames)
+                              deps=wrapper_cmd[-1:] + srcs + depend_files, outputs=ofilenames)
         ET.SubElement(root, 'Import', Project=r'$(VCTargetsPath)\Microsoft.Cpp.targets')
         self.generate_custom_generator_commands(target, root)
         self.add_regen_dependency(root)
diff --git a/mesonbuild/scripts/meson_exe.py b/mesonbuild/scripts/meson_exe.py
index c1d0d64..8b34448 100644
--- a/mesonbuild/scripts/meson_exe.py
+++ b/mesonbuild/scripts/meson_exe.py
@@ -20,12 +20,14 @@ import platform
 import subprocess
 
 from .. import mesonlib
+from ..backend.backends import ExecutableSerialisation
 
 options = None
 
 def buildparser():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('args', nargs='+')
+    parser = argparse.ArgumentParser(description='Custom executable wrapper for Meson. Do not run on your own, mmm\'kay?')
+    parser.add_argument('--unpickle')
+    parser.add_argument('--capture')
     return parser
 
 def is_windows():
@@ -36,28 +38,14 @@ def is_cygwin():
     platname = platform.system().lower()
     return 'cygwin' in platname
 
-def run_with_mono(fname):
-    if fname.endswith('.exe') and not (is_windows() or is_cygwin()):
-        return True
-    return False
-
 def run_exe(exe):
-    if exe.fname[0].endswith('.jar'):
-        cmd = ['java', '-jar'] + exe.fname
-    elif not exe.is_cross and run_with_mono(exe.fname[0]):
-        cmd = ['mono'] + exe.fname
+    if exe.exe_runner:
+        if not exe.exe_runner.found():
+            raise AssertionError('BUG: Can\'t run cross-compiled exe {!r} with not-found '
+                                 'wrapper {!r}'.format(exe.cmd_args[0], exe.exe_runner.get_path()))
+        cmd_args = exe.exe_runner.get_command() + exe.cmd_args
     else:
-        if exe.is_cross and exe.needs_exe_wrapper:
-            if exe.exe_runner is None:
-                raise AssertionError('BUG: Can\'t run cross-compiled exe {!r} '
-                                     'with no wrapper'.format(exe.name))
-            elif not exe.exe_runner.found():
-                raise AssertionError('BUG: Can\'t run cross-compiled exe {!r} with not-found '
-                                     'wrapper {!r}'.format(exe.name, exe.exe_runner.get_path()))
-            else:
-                cmd = exe.exe_runner.get_command() + exe.fname
-        else:
-            cmd = exe.fname
+        cmd_args = exe.cmd_args
     child_env = os.environ.copy()
     child_env.update(exe.env)
     if exe.extra_paths:
@@ -73,7 +61,7 @@ def run_exe(exe):
             else:
                 child_env['WINEPATH'] = wine_path
 
-    p = subprocess.Popen(cmd + exe.cmd_args, env=child_env, cwd=exe.workdir,
+    p = subprocess.Popen(cmd_args, env=child_env, cwd=exe.workdir,
                          close_fds=False,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE)
@@ -101,13 +89,22 @@ def run_exe(exe):
 
 def run(args):
     global options
-    options = buildparser().parse_args(args)
-    if len(options.args) != 1:
-        print('Test runner for Meson. Do not run on your own, mmm\'kay?')
-        print(sys.argv[0] + ' [data file]')
-    exe_data_file = options.args[0]
-    with open(exe_data_file, 'rb') as f:
-        exe = pickle.load(f)
+    parser = buildparser()
+    options, cmd_args = parser.parse_known_args(args)
+    # argparse supports double dash to separate options and positional arguments,
+    # but the user has to remove it manually.
+    if cmd_args and cmd_args[0] == '--':
+        cmd_args = cmd_args[1:]
+    if not options.unpickle and not cmd_args:
+        parser.error('either --unpickle or executable and arguments are required')
+    if options.unpickle:
+        if cmd_args or options.capture:
+            parser.error('no other arguments can be used with --unpickle')
+        with open(options.unpickle, 'rb') as f:
+            exe = pickle.load(f)
+    else:
+        exe = ExecutableSerialisation(cmd_args, capture=options.capture)
+
     return run_exe(exe)
 
 if __name__ == '__main__':