Merge pull request #908 from centricular/vala-generated-sources

Support all kinds of generated vala and vapi sources

1 files changed, 231 insertions, 134 deletions

diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index f8c8109..fa537ad 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -35,6 +35,10 @@ def ninja_quote(text):
     return text.replace(' ', '$ ').replace(':', '$:')
 
 class RawFilename():
+    """
+    Used when a filename is already relative to the root build directory, so
+    that we know not to add the target's private build directory to it.
+    """
     def __init__(self, fname):
         self.fname = fname
 
@@ -216,26 +220,57 @@ int dummy;
     # we need to add an order dependency to them.
     def get_generated_headers(self, target):
         header_deps = []
-        for gensource in target.get_generated_sources():
-            if isinstance(gensource, build.CustomTarget):
+        # XXX: Why don't we add deps to CustomTarget headers here?
+        for genlist in target.get_generated_sources():
+            if isinstance(genlist, build.CustomTarget):
                 continue
-            for src in gensource.get_outfilelist():
+            for src in genlist.get_outputs():
                 if self.environment.is_header(src):
-                    header_deps.append(os.path.join(self.get_target_private_dir(target), src))
+                    header_deps.append(self.get_target_generated_dir(target, genlist, src))
+        # Recurse and find generated headers
         for dep in target.link_targets:
             if isinstance(dep, (build.StaticLibrary, build.SharedLibrary)):
                 header_deps += self.get_generated_headers(dep)
         return header_deps
 
+    def get_target_generated_sources(self, target):
+        """
+        Returns a dictionary with the keys being the path to the file
+        (relative to the build directory) of that type and the value
+        being the GeneratorList or CustomTarget that generated it.
+        """
+        srcs = {}
+        for gensrc in target.get_generated_sources():
+            for s in gensrc.get_outputs():
+                f = self.get_target_generated_dir(target, gensrc, s)
+                srcs[f] = s
+        return srcs
+
+    def get_target_sources(self, target):
+        srcs = {}
+        for s in target.get_sources():
+            # BuildTarget sources are always mesonlib.File files which are
+            # either in the source root, or generated with configure_file and
+            # in the build root
+            if not isinstance(s, File):
+                raise InvalidArguments('All sources in target {!r} must be of type mesonlib.File'.format(t))
+            f = s.rel_to_builddir(self.build_to_src)
+            srcs[f] = s
+        return srcs
+
     def generate_target(self, target, outfile):
         if isinstance(target, build.CustomTarget):
             self.generate_custom_target(target, outfile)
         if isinstance(target, build.RunTarget):
             self.generate_run_target(target, outfile)
         name = target.get_id()
-        vala_gen_sources = []
         if name in self.processed_targets:
             return
+        self.processed_targets[name] = True
+        # Generate rules for all dependency targets
+        self.process_target_dependencies(target, outfile)
+        # If target uses a language that cannot link to C objects,
+        # just generate for that language and return.
         if isinstance(target, build.Jar):
             self.generate_jar_target(target, outfile)
             return
@@ -248,11 +283,28 @@ int dummy;
         if 'swift' in target.compilers:
             self.generate_swift_target(target, outfile)
             return
+
+        # Pre-existing target C/C++ sources to be built; dict of full path to
+        # source relative to build root and the original File object.
+        target_sources = {}
+        # GeneratedList and CustomTarget sources to be built; dict of the full
+        # path to source relative to build root and the generating target/list
+        generated_sources = {}
+        # Array of sources generated by valac that have to be compiled
+        vala_generated_sources = []
         if 'vala' in target.compilers:
-            vala_gen_sources = self.generate_vala_compile(target, outfile)
+            # Sources consumed by valac are filtered out. These only contain
+            # C/C++ sources, objects, generated libs, and unknown sources now.
+            target_sources, generated_sources, \
+            vala_generated_sources = self.generate_vala_compile(target, outfile)
+        else:
+            target_sources = self.get_target_sources(target)
+            generated_sources = self.get_target_generated_sources(target)
         self.scan_fortran_module_outputs(target)
-        self.process_target_dependencies(target, outfile)
-        self.generate_custom_generator_rules(target, outfile)
+        # Generate rules for GeneratedLists
+        self.generate_generator_list_rules(target, outfile)
+
+        # Generate rules for building the remaining source files in this target
         outname = self.get_target_filename(target)
         obj_list = []
         use_pch = self.environment.coredata.base_options.get('b_pch', False)
@@ -275,41 +327,24 @@ int dummy;
         # This will be set as dependencies of all the target's sources. At the
         # same time, also deal with generated sources that need to be compiled.
         generated_source_files = []
-        for gensource in target.get_generated_sources():
-            if isinstance(gensource, build.CustomTarget):
-                for src in gensource.output:
-                    src = os.path.join(self.get_target_dir(gensource), src)
-                    generated_output_sources.append(src)
-                    if self.environment.is_source(src) and not self.environment.is_header(src):
-                        if is_unity:
-                            unity_deps.append(os.path.join(self.environment.get_build_dir(), RawFilename(src)))
-                        else:
-                            generated_source_files.append(RawFilename(src))
-                    elif self.environment.is_object(src):
-                        obj_list.append(src)
-                    elif self.environment.is_library(src):
-                        pass
-                    else:
-                        # Assume anything not specifically a source file is a header. This is because
-                        # people generate files with weird suffixes (.inc, .fh) that they then include
-                        # in their source files.
-                        header_deps.append(RawFilename(src))
+        for rel_src, gensrc in generated_sources.items():
+            generated_output_sources.append(rel_src)
+            if self.environment.is_source(rel_src) and not self.environment.is_header(rel_src):
+                if is_unity:
+                    unity_deps.append(rel_src)
+                    abs_src = os.path.join(self.environment.get_build_dir(), rel_src)
+                    unity_src.append(abs_src)
+                else:
+                    generated_source_files.append(RawFilename(rel_src))
+            elif self.environment.is_object(rel_src):
+                obj_list.append(rel_src)
+            elif self.environment.is_library(rel_src):
+                pass
             else:
-                for src in gensource.get_outfilelist():
-                    generated_output_sources.append(src)
-                    if self.environment.is_object(src):
-                        obj_list.append(os.path.join(self.get_target_private_dir(target), src))
-                    elif not self.environment.is_header(src):
-                        if is_unity:
-                            if self.has_dir_part(src):
-                                rel_src = src
-                            else:
-                                rel_src = os.path.join(self.get_target_private_dir(target), src)
-                            unity_deps.append(rel_src)
-                            abs_src = os.path.join(self.environment.get_build_dir(), rel_src)
-                            unity_src.append(abs_src)
-                        else:
-                            generated_source_files.append(src)
+                # Assume anything not specifically a source file is a header. This is because
+                # people generate files with weird suffixes (.inc, .fh) that they then include
+                # in their source files.
+                header_deps.append(RawFilename(rel_src))
         # These are the generated source files that need to be built for use by
         # this target. We create the Ninja build file elements for this here
         # because we need `header_deps` to be fully generated in the above loop.
@@ -319,8 +354,8 @@ int dummy;
                                                          header_deps=header_deps))
         # Generate compilation targets for C sources generated from Vala
         # sources. This can be extended to other $LANG->C compilers later if
-        # necessary.
-        for src in vala_gen_sources:
+        # necessary. This needs to be separate for at least Vala
+        for src in vala_generated_sources:
             src_list.append(src)
             if is_unity:
                 unity_src.append(os.path.join(self.environment.get_build_dir(), src))
@@ -334,11 +369,12 @@ int dummy;
                 if self.environment.is_header(src):
                     header_deps.append(src)
                 else:
+                    # Passing 'vala' here signifies that we want the compile
+                    # arguments to be specialized for C code generated by
+                    # valac. For instance, no warnings should be emitted.
                     obj_list.append(self.generate_single_compile(target, outfile, src, 'vala', [], header_deps))
         # Generate compile targets for all the pre-existing sources for this target
-        for src in target.get_sources():
-            if src.endswith('.vala'):
-                continue
+        for f, src in target_sources.items():
             if not self.environment.is_header(src):
                 src_list.append(src)
                 if is_unity:
@@ -355,7 +391,6 @@ int dummy;
         elem = self.generate_link(target, outfile, outname, obj_list, linker, pch_objects)
         self.generate_shlib_aliases(target, self.get_target_dir(target))
         elem.write(outfile)
-        self.processed_targets[name] = True
 
     def process_target_dependencies(self, target, outfile):
         for t in target.get_dependencies():
@@ -376,10 +411,9 @@ int dummy;
             # FIXME, should not grab element at zero but rather expand all.
             if isinstance(i, list):
                 i = i[0]
-            fname = i.get_filename()
-            if isinstance(fname, list):
-                fname = fname[0]
-            deps.append(os.path.join(self.get_target_dir(i), fname))
+            # Add a dependency on all the outputs of this target
+            for output in i.get_outputs():
+                deps.append(os.path.join(self.get_target_dir(i), output))
         return deps
 
     def generate_custom_target(self, target, outfile):
@@ -401,11 +435,9 @@ int dummy;
                 deps.append(os.path.join(self.build_to_src, i))
         elem.add_dep(deps)
         for d in target.extra_depends:
-            tmp = d.get_filename()
-            if not isinstance(tmp, list):
-                tmp = [tmp]
-            for fname in tmp:
-                elem.add_dep(os.path.join(self.get_target_dir(d), fname))
+            # Add a dependency on all the outputs of this target
+            for output in d.get_outputs():
+                elem.add_dep(os.path.join(self.get_target_dir(d), output))
         # If the target requires capturing stdout, then use the serialized
         # executable wrapper to capture that output and save it to a file.
         #
@@ -541,7 +573,8 @@ int dummy;
         should_strip = self.environment.coredata.get_builtin_option('strip')
         for t in self.build.get_targets().values():
             if t.should_install():
-                # Find the installation directory
+                # Find the installation directory. FIXME: Currently only one
+                # installation directory is supported for each target
                 outdir = t.get_custom_install_dir()
                 if outdir is not None:
                     pass
@@ -572,9 +605,14 @@ int dummy;
                         # stripped, and doesn't have an install_rpath
                         i = [self.get_target_debug_filename(t), outdir, [], False, '']
                         d.targets.append(i)
-                i = [self.get_target_filename(t), outdir, t.get_aliaslist(),\
-                    should_strip, t.install_rpath]
-                d.targets.append(i)
+                if isinstance(t, build.BuildTarget):
+                    i = [self.get_target_filename(t), outdir, t.get_aliaslist(),\
+                        should_strip, t.install_rpath]
+                    d.targets.append(i)
+                elif isinstance(t, build.CustomTarget):
+                    for output in t.get_outputs():
+                        f = os.path.join(self.get_target_dir(t), output)
+                        d.targets.append([f, outdir, [], False, None])
 
     def generate_custom_install_script(self, d):
         d.install_scripts = self.build.install_scripts
@@ -838,17 +876,13 @@ int dummy;
         outfile.write(description)
         outfile.write('\n')
 
-    def split_vala_sources(self, sources):
-        other_src = []
-        vapi_src = []
-        for s in sources:
-            if s.endswith('.vapi'):
-                vapi_src.append(s)
-            else:
-                other_src.append(s)
-        return (other_src, vapi_src)
-
     def determine_dep_vapis(self, target):
+        """
+        Peek into the sources of BuildTargets we're linking with, and if any of
+        them was built with Vala, assume that it also generated a .vapi file of
+        the same name as the BuildTarget and return the path to it relative to
+        the build directory.
+        """
         result = []
         for dep in target.link_targets:
             for i in dep.sources:
@@ -861,57 +895,119 @@ int dummy;
                     break
         return result
 
+    def split_vala_sources(self, t):
+        """
+        Splits the target's sources into .vala, .vapi, and other sources.
+        Handles both pre-existing and generated sources.
+
+        Returns a tuple (vala, vapi, others) each of which is a dictionary with
+        the keys being the path to the file (relative to the build directory)
+        and the value being the object that generated or represents the file.
+        """
+        vala = {}
+        vapi = {}
+        others = {}
+        othersgen = {}
+        # Split pre-existing sources
+        for s in t.get_sources():
+            # BuildTarget sources are always mesonlib.File files which are
+            # either in the source root, or generated with configure_file and
+            # in the build root
+            if not isinstance(s, File):
+                msg = 'All sources in target {!r} must be of type ' \
+                      'mesonlib.File, not {!r}'.format(t, s)
+                raise InvalidArguments(msg)
+            f = s.rel_to_builddir(self.build_to_src)
+            if s.endswith('.vala'):
+                srctype = vala
+            elif s.endswith('.vapi'):
+                srctype = vapi
+            else:
+                srctype = others
+            srctype[f] = s
+        # Split generated sources
+        for gensrc in t.get_generated_sources():
+            for s in gensrc.get_outputs():
+                f = self.get_target_generated_dir(t, gensrc, s)
+                if s.endswith('.vala'):
+                    srctype = vala
+                elif s.endswith('.vapi'):
+                    srctype = vapi
+                # Generated non-Vala (C/C++) sources. Won't be used for
+                # generating the Vala compile rule below.
+                else:
+                    srctype = othersgen
+                # Duplicate outputs are disastrous
+                if f in srctype:
+                    msg = 'Duplicate output {0!r} from {1!r} {2!r}; ' \
+                          'conflicts with {0!r} from {4!r} {3!r}' \
+                          ''.format(f, type(gensrc).__name__, gensrc.name,
+                                    srctype[f].name, type(srctype[f]).__name__)
+                    raise InvalidArguments(msg)
+                # Store 'somefile.vala': GeneratedList (or CustomTarget)
+                srctype[f] = gensrc
+        return (vala, vapi, (others, othersgen))
+
     def generate_vala_compile(self, target, outfile):
         """Vala is compiled into C. Set up all necessary build steps here."""
-        valac = target.compilers['vala']
-        (other_src, vapi_src) = self.split_vala_sources(target.get_sources())
-        vapi_src = [x.rel_to_builddir(self.build_to_src) for x in vapi_src]
+        (vala_src, vapi_src, other_src) = self.split_vala_sources(target)
         extra_dep_files = []
-        if len(other_src) == 0:
+        if len(vala_src) == 0:
             raise InvalidArguments('Vala library has no Vala source files.')
-        namebase = target.name
-        base_h = namebase + '.h'
-        base_vapi = namebase + '.vapi'
-        hname = os.path.normpath(os.path.join(self.get_target_dir(target), base_h))
-        vapiname = os.path.normpath(os.path.join(self.get_target_dir(target), base_vapi))
-
-        generated_c_files = []
-        outputs = [vapiname]
-        args = []
-        args += self.build.get_global_args(valac)
-        args += valac.get_buildtype_args(self.environment.coredata.get_builtin_option('buildtype'))
-        args += ['-d', self.get_target_private_dir(target)]
-        args += ['-C']#, '-o', cname]
-        if not isinstance(target, build.Executable):
-            outputs.append(hname)
-            args += ['-H', hname]
-            args += ['--library=' + target.name]
-        args += ['--vapi=' + os.path.join('..', base_vapi)]
-        vala_src = []
-        for s in other_src:
-            if not s.endswith('.vala'):
-                continue
-            vala_file = s.rel_to_builddir(self.build_to_src)
-            vala_src.append(vala_file)
+
+        valac = target.compilers['vala']
+        c_out_dir = self.get_target_private_dir(target)
+        # C files generated by valac
+        vala_c_src = []
+        # Files generated by valac
+        valac_outputs = []
+        # All sources that are passed to valac on the commandline
+        all_files = list(vapi_src.keys())
+        for (vala_file, gensrc) in vala_src.items():
+            all_files.append(vala_file)
             # Figure out where the Vala compiler will write the compiled C file
-            dirname, basename = os.path.split(vala_file)
             # If the Vala file is in a subdir of the build dir (in our case
             # because it was generated/built by something else), the subdir path
             # components will be preserved in the output path. But if the Vala
             # file is outside the build directory, the path components will be
             # stripped and just the basename will be used.
-            c_file = os.path.splitext(basename)[0] + '.c'
-            if s.is_built:
-                c_file = os.path.join(dirname, c_file)
-            full_c = os.path.join(self.get_target_private_dir(target), c_file)
-            generated_c_files.append(full_c)
-            outputs.append(full_c)
+            if isinstance(gensrc, (build.CustomTarget, build.GeneratedList)) or gensrc.is_built:
+                vala_c_file = os.path.splitext(vala_file)[0] + '.c'
+            else:
+                vala_c_file = os.path.splitext(os.path.basename(vala_file))[0] + '.c'
+            # All this will be placed inside the c_out_dir
+            vala_c_file = os.path.join(c_out_dir, vala_c_file)
+            vala_c_src.append(vala_c_file)
+            valac_outputs.append(vala_c_file)
+
+        args = []
+        args += self.build.get_global_args(valac)
+        args += valac.get_buildtype_args(self.environment.coredata.get_builtin_option('buildtype'))
+        # Tell Valac to output everything in our private directory. Sadly this
+        # means it will also preserve the directory components of Vala sources
+        # found inside the build tree (generated sources).
+        args += ['-d', c_out_dir]
+        args += ['-C']
+        if not isinstance(target, build.Executable):
+            # Library name
+            args += ['--library=' + target.name]
+            # Outputted header
+            hname = os.path.join(self.get_target_dir(target), target.name + '.h')
+            args += ['-H', hname]
+            valac_outputs.append(hname)
+            # Outputted vapi file
+            base_vapi = target.name + '.vapi'
+            vapiname = os.path.join(self.get_target_dir(target), base_vapi)
+            # Force valac to write the vapi file in the target build dir.
+            # Without this, it will write it inside c_out_dir
+            args += ['--vapi=../' + base_vapi]
+            valac_outputs.append(vapiname)
         if self.environment.coredata.get_builtin_option('werror'):
             args += valac.get_werror_args()
-        for d in target.external_deps:
+        for d in target.get_external_deps():
             if isinstance(d, dependencies.PkgConfigDependency):
                 if d.name == 'glib-2.0' and d.version_requirement is not None \
-                and d.version_requirement.startswith(('>=', '==')):
+                   and d.version_requirement.startswith(('>=', '==')):
                     args += ['--target-glib', d.version_requirement[2:]]
                 args += ['--pkg', d.name]
         extra_args = []
@@ -926,14 +1022,13 @@ int dummy;
         dependency_vapis = self.determine_dep_vapis(target)
         extra_dep_files += dependency_vapis
         args += extra_args
-        args += dependency_vapis
-        element = NinjaBuildElement(self.all_outputs, outputs,
+        element = NinjaBuildElement(self.all_outputs, valac_outputs,
                                     valac.get_language() + '_COMPILER',
-                                    vala_src + vapi_src)
+                                    all_files + dependency_vapis)
         element.add_item('ARGS', args)
         element.add_dep(extra_dep_files)
         element.write(outfile)
-        return generated_c_files
+        return other_src[0], other_src[1], vala_c_src
 
     def generate_rust_target(self, target, outfile):
         rustc = target.compilers['rust']
@@ -1006,16 +1101,7 @@ int dummy;
         return result
 
     def split_swift_generated_sources(self, target):
-        all_srcs = []
-        for genlist in target.get_generated_sources():
-            if isinstance(genlist, build.CustomTarget):
-                for ifile in genlist.get_filename():
-                    rel = os.path.join(self.get_target_dir(genlist), ifile)
-                    all_srcs.append(rel)
-            else:
-                for ifile in genlist.get_outfilelist():
-                    rel = os.path.join(self.get_target_private_dir(target), ifile)
-                    all_srcs.append(rel)
+        all_srcs = self.get_target_generated_sources(target)
         srcs = []
         others = []
         for i in all_srcs:
@@ -1392,18 +1478,20 @@ rule FORTRAN_DEP_HACK
                 self.generate_pch_rule_for(langname, compiler, qstr, True, outfile)
         outfile.write('\n')
 
-    def generate_custom_generator_rules(self, target, outfile):
+    def generate_generator_list_rules(self, target, outfile):
+        # CustomTargets have already written their rules,
+        # so write rules for GeneratedLists here
         for genlist in target.get_generated_sources():
             if isinstance(genlist, build.CustomTarget):
-                continue # Customtarget has already written its output rules
+                continue
             self.generate_genlist_for_target(genlist, target, outfile)
 
     def generate_genlist_for_target(self, genlist, target, outfile):
         generator = genlist.get_generator()
         exe = generator.get_exe()
         exe_arr = self.exe_object_to_cmd_array(exe)
-        infilelist = genlist.get_infilelist()
-        outfilelist = genlist.get_outfilelist()
+        infilelist = genlist.get_inputs()
+        outfilelist = genlist.get_outputs()
         base_args = generator.get_arglist()
         extra_dependencies = [os.path.join(self.build_to_src, i) for i in genlist.extra_depends]
         for i in range(len(infilelist)):
@@ -1579,6 +1667,9 @@ rule FORTRAN_DEP_HACK
         return linker.get_link_debugfile_args(outname)
 
     def generate_single_compile(self, target, outfile, src, is_generated=False, header_deps=[], order_deps=[]):
+        """
+        Compiles only C/C++ and ObjC/ObjC++ sources
+        """
         if(isinstance(src, str) and src.endswith('.h')):
             raise RuntimeError('Fug')
         if isinstance(src, RawFilename) and src.fname.endswith('.h'):
@@ -1635,7 +1726,7 @@ rule FORTRAN_DEP_HACK
             if isinstance(src, File):
                 rel_src = src.rel_to_builddir(self.build_to_src)
             else:
-                raise build.InvalidArguments('Invalid source type.')
+                raise InvalidArguments('Invalid source type: {!r}'.format(src))
             abs_src = os.path.join(self.environment.get_build_dir(), rel_src)
         if isinstance(src, (RawFilename, File)):
             src_filename = src.fname
@@ -1659,11 +1750,12 @@ rule FORTRAN_DEP_HACK
             arr.append(i)
             pch_dep = arr
         custom_target_include_dirs = []
-        for i in target.generated:
-            if isinstance(i, build.CustomTarget):
-                idir = self.get_target_dir(i)
-                if idir not in custom_target_include_dirs:
-                    custom_target_include_dirs.append(idir)
+        for i in target.get_generated_sources():
+            if not isinstance(i, build.CustomTarget):
+                continue
+            idir = self.get_target_dir(i)
+            if idir not in custom_target_include_dirs:
+                custom_target_include_dirs.append(idir)
         for i in custom_target_include_dirs:
             commands+= compiler.get_include_args(i, False)
         if self.environment.coredata.base_options.get('b_pch', False):
@@ -1761,7 +1853,10 @@ rule FORTRAN_DEP_HACK
             if len(pch) == 0:
                 continue
             if '/' not in pch[0] or '/' not in pch[-1]:
-                raise build.InvalidArguments('Precompiled header of "%s" must not be in the same directory as source, please put it in a subdirectory.' % target.get_basename())
+                msg = 'Precompiled header of {!r} must not be in the same ' \
+                      'directory as source, please put it in a subdirectory.' \
+                      ''.format(target.get_basename())
+                raise InvalidArguments(msg)
             compiler = self.get_compiler_for_lang(lang)
             if compiler.id == 'msvc':
                 src = os.path.join(self.build_to_src, target.get_source_subdir(), pch[-1])
@@ -1941,7 +2036,9 @@ rule FORTRAN_DEP_HACK
             # are used by something else or are meant to be always built
             if isinstance(t, build.CustomTarget) and not (t.install or t.build_always):
                 continue
-            targetlist.append(self.get_target_filename(t))
+            # Add the first output of each target to the 'all' target so that
+            # they are all built
+            targetlist.append(os.path.join(self.get_target_dir(t), t.get_outputs()[0]))
 
         elem = NinjaBuildElement(self.all_outputs, 'all', 'phony', targetlist)
         elem.write(outfile)