Merge pull request #1403 from centricular/compile_resources

Make configure_file() great again

5 files changed, 248 insertions, 56 deletions

diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py
index 7372c4c..d6f1c38 100644
--- a/mesonbuild/backend/backends.py
+++ b/mesonbuild/backend/backends.py
@@ -603,19 +603,15 @@ class Backend:
         return srcs
 
     def eval_custom_target_command(self, target, absolute_outputs=False):
-        # We only want the outputs to be absolute when using the VS backend
-        if not absolute_outputs:
-            ofilenames = [os.path.join(self.get_target_dir(target), i) for i in target.output]
-        else:
-            ofilenames = [os.path.join(self.environment.get_build_dir(), self.get_target_dir(target), i)
-                          for i in target.output]
-        srcs = self.get_custom_target_sources(target)
+        # We want the outputs to be absolute only when using the VS backend
         outdir = self.get_target_dir(target)
-        # Many external programs fail on empty arguments.
-        if outdir == '':
-            outdir = '.'
-        if target.absolute_paths:
+        if absolute_outputs:
             outdir = os.path.join(self.environment.get_build_dir(), outdir)
+        outputs = []
+        for i in target.output:
+            outputs.append(os.path.join(outdir, i))
+        inputs = self.get_custom_target_sources(target)
+        # Evaluate the command list
         cmd = []
         for i in target.command:
             if isinstance(i, build.Executable):
@@ -631,37 +627,10 @@ class Backend:
                 if target.absolute_paths:
                     i = os.path.join(self.environment.get_build_dir(), i)
             # FIXME: str types are blindly added ignoring 'target.absolute_paths'
+            # because we can't know if they refer to a file or just a string
             elif not isinstance(i, str):
                 err_msg = 'Argument {0} is of unknown type {1}'
                 raise RuntimeError(err_msg.format(str(i), str(type(i))))
-            for (j, src) in enumerate(srcs):
-                i = i.replace('@INPUT%d@' % j, src)
-            for (j, res) in enumerate(ofilenames):
-                i = i.replace('@OUTPUT%d@' % j, res)
-            if '@INPUT@' in i:
-                msg = 'Custom target {} has @INPUT@ in the command, but'.format(target.name)
-                if len(srcs) == 0:
-                    raise MesonException(msg + ' no input files')
-                if i == '@INPUT@':
-                    cmd += srcs
-                    continue
-                else:
-                    if len(srcs) > 1:
-                        raise MesonException(msg + ' more than one input file')
-                    i = i.replace('@INPUT@', srcs[0])
-            elif '@OUTPUT@' in i:
-                msg = 'Custom target {} has @OUTPUT@ in the command, but'.format(target.name)
-                if len(ofilenames) == 0:
-                    raise MesonException(msg + ' no output files')
-                if i == '@OUTPUT@':
-                    cmd += ofilenames
-                    continue
-                else:
-                    if len(ofilenames) > 1:
-                        raise MesonException(msg + ' more than one output file')
-                    i = i.replace('@OUTPUT@', ofilenames[0])
-            elif '@OUTDIR@' in i:
-                i = i.replace('@OUTDIR@', outdir)
             elif '@DEPFILE@' in i:
                 if target.depfile is None:
                     msg = 'Custom target {!r} has @DEPFILE@ but no depfile ' \
@@ -680,10 +649,11 @@ class Backend:
                     lead_dir = ''
                 else:
                     lead_dir = self.environment.get_build_dir()
-                i = i.replace(source,
-                              os.path.join(lead_dir,
-                                           outdir))
+                i = i.replace(source, os.path.join(lead_dir, outdir))
             cmd.append(i)
+        # Substitute the rest of the template strings
+        values = mesonlib.get_filenames_templates_dict(inputs, outputs)
+        cmd = mesonlib.substitute_values(cmd, values)
         # This should not be necessary but removing it breaks
         # building GStreamer on Windows. The underlying issue
         # is problems with quoting backslashes on Windows
@@ -703,7 +673,7 @@ class Backend:
         #
         # https://github.com/mesonbuild/meson/pull/737
         cmd = [i.replace('\\', '/') for i in cmd]
-        return srcs, ofilenames, cmd
+        return inputs, outputs, cmd
 
     def run_postconf_scripts(self):
         env = {'MESON_SOURCE_ROOT': self.environment.get_source_dir(),
diff --git a/mesonbuild/build.py b/mesonbuild/build.py
index 5f2de3b..91a3dd8 100644
--- a/mesonbuild/build.py
+++ b/mesonbuild/build.py
@@ -16,7 +16,9 @@ from . import environment
 from . import dependencies
 from . import mlog
 import copy, os, re
-from .mesonlib import File, flatten, MesonException, stringlistify, classify_unity_sources
+from .mesonlib import File, MesonException
+from .mesonlib import flatten, stringlistify, classify_unity_sources
+from .mesonlib import get_filenames_templates_dict, substitute_values
 from .environment import for_windows, for_darwin
 from .compilers import is_object, clike_langs, lang_suffixes
 
@@ -1331,11 +1333,25 @@ class CustomTarget(Target):
         self.output = kwargs['output']
         if not isinstance(self.output, list):
             self.output = [self.output]
+        # This will substitute values from the input into output and return it.
+        inputs = get_sources_string_names(self.sources)
+        values = get_filenames_templates_dict(inputs, [])
         for i in self.output:
             if not(isinstance(i, str)):
                 raise InvalidArguments('Output argument not a string.')
             if '/' in i:
                 raise InvalidArguments('Output must not contain a path segment.')
+            if '@INPUT@' in i or '@INPUT0@' in i:
+                m = 'Output cannot contain @INPUT@ or @INPUT0@, did you ' \
+                    'mean @PLAINNAME@ or @BASENAME@?'
+                raise InvalidArguments(m)
+            # We already check this during substitution, but the error message
+            # will be unclear/confusing, so check it here.
+            if len(inputs) != 1 and ('@PLAINNAME@' in i or '@BASENAME@' in i):
+                m = "Output cannot contain @PLAINNAME@ or @BASENAME@ when " \
+                    "there is more than one input (we can't know which to use)"
+                raise InvalidArguments(m)
+        self.output = substitute_values(self.output, values)
         self.capture = kwargs.get('capture', False)
         if self.capture and len(self.output) != 1:
             raise InvalidArguments('Capturing can only output to a single file.')
@@ -1530,3 +1546,22 @@ class TestSetup:
         self.gdb = gdb
         self.timeout_multiplier = timeout_multiplier
         self.env = env
+
+def get_sources_string_names(sources):
+    '''
+    For the specified list of @sources which can be strings, Files, or targets,
+    get all the output basenames.
+    '''
+    names = []
+    for s in sources:
+        if hasattr(s, 'held_object'):
+            s = s.held_object
+        if isinstance(s, str):
+            names.append(s)
+        elif isinstance(s, (BuildTarget, CustomTarget, GeneratedList)):
+            names += s.get_outputs()
+        elif isinstance(s, File):
+            names.append(s.fname)
+        else:
+            raise AssertionError('Unknown source type: {!r}'.format(s))
+    return names
diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py
index f6065d5..0d252fd 100644
--- a/mesonbuild/interpreter.py
+++ b/mesonbuild/interpreter.py
@@ -2224,12 +2224,28 @@ requirements use the version keyword argument instead.''')
             raise InterpreterException("configure_file takes only keyword arguments.")
         if 'output' not in kwargs:
             raise InterpreterException('Required keyword argument "output" not defined.')
-        inputfile = kwargs.get('input', None)
+        if 'configuration' in kwargs and 'command' in kwargs:
+            raise InterpreterException('Must not specify both "configuration" '
+                                       'and "command" keyword arguments since '
+                                       'they are mutually exclusive.')
+        # Validate input
+        inputfile = None
+        if 'input' in kwargs:
+            inputfile = kwargs['input']
+            if isinstance(inputfile, list):
+                if len(inputfile) != 1:
+                    m = "Keyword argument 'input' requires exactly one file"
+                    raise InterpreterException(m)
+                inputfile = inputfile[0]
+            if not isinstance(inputfile, (str, mesonlib.File)):
+                raise InterpreterException('Input must be a string or a file')
+            ifile_abs = os.path.join(self.environment.source_dir, self.subdir, inputfile)
+        elif 'command' in kwargs:
+            raise InterpreterException('Required keyword argument \'input\' missing')
+        # Validate output
         output = kwargs['output']
-        if not isinstance(inputfile, (str, type(None))):
-            raise InterpreterException('Input must be a string.')
         if not isinstance(output, str):
-            raise InterpreterException('Output must be a string.')
+            raise InterpreterException('Output file name must be a string')
         if os.path.split(output)[0] != '':
             raise InterpreterException('Output file name must not contain a subdirectory.')
         (ofile_path, ofile_fname) = os.path.split(os.path.join(self.subdir, output))
@@ -2238,6 +2254,7 @@ requirements use the version keyword argument instead.''')
             conf = kwargs['configuration']
             if not isinstance(conf, ConfigurationDataHolder):
                 raise InterpreterException('Argument "configuration" is not of type configuration_data')
+            mlog.log('Configuring', mlog.bold(output), 'using configuration')
             if inputfile is not None:
                 # Normalize the path of the conffile to avoid duplicates
                 # This is especially important to convert '/' to '\' on Windows
@@ -2245,15 +2262,19 @@ requirements use the version keyword argument instead.''')
                 if conffile not in self.build_def_files:
                     self.build_def_files.append(conffile)
                 os.makedirs(os.path.join(self.environment.build_dir, self.subdir), exist_ok=True)
-                ifile_abs = os.path.join(self.environment.source_dir, self.subdir, inputfile)
                 mesonlib.do_conf_file(ifile_abs, ofile_abs, conf.held_object)
             else:
                 mesonlib.dump_conf_header(ofile_abs, conf.held_object)
             conf.mark_used()
         elif 'command' in kwargs:
-            if 'input' not in kwargs:
-                raise InterpreterException('Required keyword input missing.')
-            res = self.func_run_command(node, kwargs['command'], {})
+            # We use absolute paths for input and output here because the cwd
+            # that the command is run from is 'unspecified', so it could change.
+            # Currently it's builddir/subdir for in_builddir else srcdir/subdir.
+            values = mesonlib.get_filenames_templates_dict([ifile_abs], [ofile_abs])
+            # Substitute @INPUT@, @OUTPUT@, etc here.
+            cmd = mesonlib.substitute_values(kwargs['command'], values)
+            mlog.log('Configuring', mlog.bold(output), 'with command')
+            res = self.func_run_command(node, cmd, {'in_builddir': True})
             if res.returncode != 0:
                 raise InterpreterException('Running configure command failed.\n%s\n%s' %
                                            (res.stdout, res.stderr))
diff --git a/mesonbuild/mesonlib.py b/mesonbuild/mesonlib.py
index f0b20e1..c7368d5 100644
--- a/mesonbuild/mesonlib.py
+++ b/mesonbuild/mesonlib.py
@@ -521,3 +521,154 @@ def commonpath(paths):
         new = os.path.join(*new)
         common = pathlib.PurePath(new)
     return str(common)
+
+def iter_regexin_iter(regexiter, initer):
+    '''
+    Takes each regular expression in @regexiter and tries to search for it in
+    every item in @initer. If there is a match, returns that match.
+    Else returns False.
+    '''
+    for regex in regexiter:
+        for ii in initer:
+            if not isinstance(ii, str):
+                continue
+            match = re.search(regex, ii)
+            if match:
+                return match.group()
+    return False
+
+def _substitute_values_check_errors(command, values):
+    # Error checking
+    inregex = ('@INPUT([0-9]+)?@', '@PLAINNAME@', '@BASENAME@')
+    outregex = ('@OUTPUT([0-9]+)?@', '@OUTDIR@')
+    if '@INPUT@' not in values:
+        # Error out if any input-derived templates are present in the command
+        match = iter_regexin_iter(inregex, command)
+        if match:
+            m = 'Command cannot have {!r}, since no input files were specified'
+            raise MesonException(m.format(match))
+    else:
+        if len(values['@INPUT@']) > 1:
+            # Error out if @PLAINNAME@ or @BASENAME@ is present in the command
+            match = iter_regexin_iter(inregex[1:], command)
+            if match:
+                raise MesonException('Command cannot have {!r} when there is '
+                                     'more than one input file'.format(match))
+        # Error out if an invalid @INPUTnn@ template was specified
+        for each in command:
+            if not isinstance(each, str):
+                continue
+            match = re.search(inregex[0], each)
+            if match and match.group() not in values:
+                m = 'Command cannot have {!r} since there are only {!r} inputs'
+                raise MesonException(m.format(match.group(), len(values['@INPUT@'])))
+    if '@OUTPUT@' not in values:
+        # Error out if any output-derived templates are present in the command
+        match = iter_regexin_iter(outregex, command)
+        if match:
+            m = 'Command cannot have {!r} since there are no outputs'
+            raise MesonException(m.format(match))
+    else:
+        # Error out if an invalid @OUTPUTnn@ template was specified
+        for each in command:
+            if not isinstance(each, str):
+                continue
+            match = re.search(outregex[0], each)
+            if match and match.group() not in values:
+                m = 'Command cannot have {!r} since there are only {!r} outputs'
+                raise MesonException(m.format(match.group(), len(values['@OUTPUT@'])))
+
+def substitute_values(command, values):
+    '''
+    Substitute the template strings in the @values dict into the list of
+    strings @command and return a new list. For a full list of the templates,
+    see get_filenames_templates_dict()
+
+    If multiple inputs/outputs are given in the @values dictionary, we
+    substitute @INPUT@ and @OUTPUT@ only if they are the entire string, not
+    just a part of it, and in that case we substitute *all* of them.
+    '''
+    # Error checking
+    _substitute_values_check_errors(command, values)
+    # Substitution
+    outcmd = []
+    for vv in command:
+        if not isinstance(vv, str):
+            outcmd.append(vv)
+        elif '@INPUT@' in vv:
+            inputs = values['@INPUT@']
+            if vv == '@INPUT@':
+                outcmd += inputs
+            elif len(inputs) == 1:
+                outcmd.append(vv.replace('@INPUT@', inputs[0]))
+            else:
+                raise MesonException("Command has '@INPUT@' as part of a "
+                                     "string and more than one input file")
+        elif '@OUTPUT@' in vv:
+            outputs = values['@OUTPUT@']
+            if vv == '@OUTPUT@':
+                outcmd += outputs
+            elif len(outputs) == 1:
+                outcmd.append(vv.replace('@OUTPUT@', outputs[0]))
+            else:
+                raise MesonException("Command has '@OUTPUT@' as part of a "
+                                     "string and more than one output file")
+        # Append values that are exactly a template string.
+        # This is faster than a string replace.
+        elif vv in values:
+            outcmd.append(values[vv])
+        # Substitute everything else with replacement
+        else:
+            for key, value in values.items():
+                if key in ('@INPUT@', '@OUTPUT@'):
+                    # Already done above
+                    continue
+                vv = vv.replace(key, value)
+            outcmd.append(vv)
+    return outcmd
+
+def get_filenames_templates_dict(inputs, outputs):
+    '''
+    Create a dictionary with template strings as keys and values as values for
+    the following templates:
+
+    @INPUT@  - the full path to one or more input files, from @inputs
+    @OUTPUT@ - the full path to one or more output files, from @outputs
+    @OUTDIR@ - the full path to the directory containing the output files
+
+    If there is only one input file, the following keys are also created:
+
+    @PLAINNAME@ - the filename of the input file
+    @BASENAME@ - the filename of the input file with the extension removed
+
+    If there is more than one input file, the following keys are also created:
+
+    @INPUT0@, @INPUT1@, ... one for each input file
+
+    If there is more than one output file, the following keys are also created:
+
+    @OUTPUT0@, @OUTPUT1@, ... one for each output file
+    '''
+    values = {}
+    # Gather values derived from the input
+    if inputs:
+        # We want to substitute all the inputs.
+        values['@INPUT@'] = inputs
+        for (ii, vv) in enumerate(inputs):
+            # Write out @INPUT0@, @INPUT1@, ...
+            values['@INPUT{}@'.format(ii)] = vv
+        if len(inputs) == 1:
+            # Just one value, substitute @PLAINNAME@ and @BASENAME@
+            values['@PLAINNAME@'] = plain = os.path.split(inputs[0])[1]
+            values['@BASENAME@'] = os.path.splitext(plain)[0]
+    if outputs:
+        # Gather values derived from the outputs, similar to above.
+        values['@OUTPUT@'] = outputs
+        for (ii, vv) in enumerate(outputs):
+            values['@OUTPUT{}@'.format(ii)] = vv
+        # Outdir should be the same for all outputs
+        values['@OUTDIR@'] = os.path.split(outputs[0])[0]
+        # Many external programs fail on empty arguments.
+        if values['@OUTDIR@'] == '':
+            values['@OUTDIR@'] = '.'
+    return values
diff --git a/mesonbuild/modules/gnome.py b/mesonbuild/modules/gnome.py
index f288b58..bf4b8be 100644
--- a/mesonbuild/modules/gnome.py
+++ b/mesonbuild/modules/gnome.py
@@ -115,11 +115,24 @@ class GnomeModule(ExtensionModule):
 
         ifile = args[1]
         if isinstance(ifile, mesonlib.File):
-            ifile = os.path.join(ifile.subdir, ifile.fname)
+            # glib-compile-resources will be run inside the source dir,
+            # so we need either 'src_to_build' or the absolute path.
+            # Absolute path is the easiest choice.
+            if ifile.is_built:
+                ifile = os.path.join(state.environment.get_build_dir(), ifile.subdir, ifile.fname)
+            else:
+                ifile = os.path.join(ifile.subdir, ifile.fname)
         elif isinstance(ifile, str):
             ifile = os.path.join(state.subdir, ifile)
+        elif isinstance(ifile, (interpreter.CustomTargetHolder,
+                                interpreter.GeneratedObjectsHolder)):
+            m = 'Resource xml files generated at build-time cannot be used ' \
+                'with gnome.compile_resources() because we need to scan ' \
+                'the xml for dependencies. Use configure_file() instead ' \
+                'to generate it at configure-time.'
+            raise MesonException(m)
         else:
-            raise RuntimeError('Unreachable code.')
+            raise MesonException('Invalid file argument: {!r}'.format(ifile))
 
         depend_files, depends, subdirs = self._get_gresource_dependencies(
             state, ifile, source_dirs, dependencies)
@@ -202,9 +215,10 @@ class GnomeModule(ExtensionModule):
             cmd += ['--sourcedir', os.path.join(state.subdir, source_dir)]
         cmd += ['--sourcedir', state.subdir] # Current dir
 
-        pc, stdout = Popen_safe(cmd, cwd=state.environment.get_source_dir())[0:2]
+        pc, stdout, stderr = Popen_safe(cmd, cwd=state.environment.get_source_dir())
         if pc.returncode != 0:
-            mlog.warning('glib-compile-resources has failed to get the dependencies for {}'.format(cmd[1]))
+            m = 'glib-compile-resources failed to get dependencies for {}:\n{}'
+            mlog.warning(m.format(cmd[1], stderr))
             raise subprocess.CalledProcessError(pc.returncode, cmd)
 
         dep_files = stdout.split('\n')[:-1]
@@ -866,6 +880,7 @@ class GnomeModule(ExtensionModule):
         }
         custom_kwargs.update(kwargs)
         return build.CustomTarget(output, state.subdir, custom_kwargs,
+                                  # https://github.com/mesonbuild/meson/issues/973
                                   absolute_paths=True)
 
     def genmarshal(self, state, args, kwargs):