Merge pull request #4547 from mensinda/introIncDirs

mintro: Save introspection to disk and --targets modifications

2 files changed, 160 insertions, 19 deletions

diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py
index 22920f4..39aa365 100644
--- a/mesonbuild/backend/backends.py
+++ b/mesonbuild/backend/backends.py
@@ -156,6 +156,8 @@ class Backend:
         self.build = build
         self.environment = build.environment
         self.processed_targets = {}
+        self.build_dir = self.environment.get_build_dir()
+        self.source_dir = self.environment.get_source_dir()
         self.build_to_src = mesonlib.relpath(self.environment.get_source_dir(),
                                              self.environment.get_build_dir())
 
@@ -683,7 +685,7 @@ class Backend:
     def write_test_file(self, datafile):
         self.write_test_serialisation(self.build.get_tests(), datafile)
 
-    def write_test_serialisation(self, tests, datafile):
+    def create_test_serialisation(self, tests):
         arr = []
         for t in tests:
             exe = t.get_exe()
@@ -730,7 +732,10 @@ class Backend:
                                    exe_wrapper, t.is_parallel, cmd_args, t.env,
                                    t.should_fail, t.timeout, t.workdir, extra_paths)
             arr.append(ts)
-        pickle.dump(arr, datafile)
+        return arr
+
+    def write_test_serialisation(self, tests, datafile):
+        pickle.dump(self.create_test_serialisation(tests), datafile)
 
     def generate_depmf_install(self, d):
         if self.build.dep_manifest_name is None:
@@ -974,9 +979,7 @@ class Backend:
             cmd = s['exe'] + s['args']
             subprocess.check_call(cmd, env=child_env)
 
-    def create_install_data_files(self):
-        install_data_file = os.path.join(self.environment.get_scratch_dir(), 'install.dat')
-
+    def create_install_data(self):
         strip_bin = self.environment.binaries.host.lookup_entry('strip')
         if strip_bin is None:
             if self.environment.is_cross_build():
@@ -997,8 +1000,12 @@ class Backend:
         self.generate_data_install(d)
         self.generate_custom_install_script(d)
         self.generate_subdir_install(d)
+        return d
+
+    def create_install_data_files(self):
+        install_data_file = os.path.join(self.environment.get_scratch_dir(), 'install.dat')
         with open(install_data_file, 'wb') as ofile:
-            pickle.dump(d, ofile)
+            pickle.dump(self.create_install_data(), ofile)
 
     def generate_target_install(self, d):
         for t in self.build.get_targets().values():
@@ -1144,3 +1151,53 @@ class Backend:
                 dst_dir = os.path.join(dst_dir, os.path.basename(src_dir))
             d.install_subdirs.append([src_dir, dst_dir, sd.install_mode,
                                       sd.exclude])
+
+    def get_introspection_data(self, target_id, target):
+        '''
+        Returns a list of source dicts with the following format for a given target:
+        [
+            {
+                "language": "<LANG>",
+                "compiler": ["result", "of", "comp.get_exelist()"],
+                "parameters": ["list", "of", "compiler", "parameters],
+                "sources": ["list", "of", "all", "<LANG>", "source", "files"],
+                "generated_sources": ["list", "of", "generated", "source", "files"]
+            }
+        ]
+
+        This is a limited fallback / reference implementation. The backend should override this method.
+        '''
+        if isinstance(target, (build.CustomTarget, build.BuildTarget)):
+            source_list_raw = target.sources + target.extra_files
+            source_list = []
+            for j in source_list_raw:
+                if isinstance(j, mesonlib.File):
+                    source_list += [j.absolute_path(self.source_dir, self.build_dir)]
+                elif isinstance(j, str):
+                    source_list += [os.path.join(self.source_dir, j)]
+            source_list = list(map(lambda x: os.path.normpath(x), source_list))
+
+            compiler = []
+            if isinstance(target, build.CustomTarget):
+                tmp_compiler = target.command
+                if not isinstance(compiler, list):
+                    tmp_compiler = [compiler]
+                for j in tmp_compiler:
+                    if isinstance(j, mesonlib.File):
+                        compiler += [j.absolute_path(self.source_dir, self.build_dir)]
+                    elif isinstance(j, str):
+                        compiler += [j]
+                    elif isinstance(j, (build.BuildTarget, build.CustomTarget)):
+                        compiler += j.get_outputs()
+                    else:
+                        raise RuntimeError('Type "{}" is not supported in get_introspection_data. This is a bug'.format(type(j).__name__))
+
+            return [{
+                'language': 'unknown',
+                'compiler': compiler,
+                'parameters': [],
+                'sources': source_list,
+                'generated_sources': []
+            }]
+
+        return []
diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index 44bdaab..3688f29 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -150,6 +150,7 @@ class NinjaBackend(backends.Backend):
         self.ninja_filename = 'build.ninja'
         self.fortran_deps = {}
         self.all_outputs = {}
+        self.introspection_data = {}
 
     def create_target_alias(self, to_target, outfile):
         # We need to use aliases for targets that might be used as directory
@@ -321,6 +322,57 @@ int dummy;
                 return False
         return True
 
+    def create_target_source_introspection(self, target: build.Target, comp: compilers.Compiler, parameters, sources, generated_sources):
+        '''
+        Adds the source file introspection information for a language of a target
+
+        Internal introspection storage formart:
+        self.introspection_data = {
+            '<target ID>': {
+                <id tuple>: {
+                    'language: 'lang',
+                    'compiler': ['comp', 'exe', 'list'],
+                    'parameters': ['UNIQUE', 'parameter', 'list'],
+                    'sources': [],
+                    'generated_sources': [],
+                }
+            }
+        }
+        '''
+        id = target.get_id()
+        lang = comp.get_language()
+        tgt = self.introspection_data[id]
+        # Find an existing entry or create a new one
+        id_hash = (lang, tuple(parameters))
+        src_block = tgt.get(id_hash, None)
+        if src_block is None:
+            # Convert parameters
+            if isinstance(parameters, CompilerArgs):
+                parameters = parameters.to_native(copy=True)
+            parameters = comp.compute_parameters_with_absolute_paths(parameters, self.build_dir)
+            if target.is_cross:
+                extra_parameters = comp.get_cross_extra_flags(self.environment, False)
+                if isinstance(parameters, CompilerArgs):
+                    extra_parameters = extra_parameters.to_native(copy=True)
+                parameters = extra_parameters + parameters
+            # The new entry
+            src_block = {
+                'language': lang,
+                'compiler': comp.get_exelist(),
+                'parameters': parameters,
+                'sources': [],
+                'generated_sources': [],
+            }
+            tgt[id_hash] = src_block
+        # Make source files absolute
+        sources = [x.absolute_path(self.source_dir, self.build_dir) if isinstance(x, File) else os.path.normpath(os.path.join(self.build_dir, x))
+                   for x in sources]
+        generated_sources = [x.absolute_path(self.source_dir, self.build_dir) if isinstance(x, File) else os.path.normpath(os.path.join(self.build_dir, x))
+                             for x in generated_sources]
+        # Add the source files
+        src_block['sources'] += sources
+        src_block['generated_sources'] += generated_sources
+
     def generate_target(self, target, outfile):
         if isinstance(target, build.CustomTarget):
             self.generate_custom_target(target, outfile)
@@ -330,6 +382,8 @@ int dummy;
         if name in self.processed_targets:
             return
         self.processed_targets[name] = True
+        # Initialize an empty introspection source list
+        self.introspection_data[name] = {}
         # Generate rules for all dependency targets
         self.process_target_dependencies(target, outfile)
         # If target uses a language that cannot link to C objects,
@@ -770,14 +824,16 @@ int dummy;
 
         # Add possible java generated files to src list
         generated_sources = self.get_target_generated_sources(target)
+        gen_src_list = []
         for rel_src, gensrc in generated_sources.items():
             dirpart, fnamepart = os.path.split(rel_src)
             raw_src = File(True, dirpart, fnamepart)
             if rel_src.endswith('.java'):
-                src_list.append(raw_src)
+                gen_src_list.append(raw_src)
 
-        for src in src_list:
-            plain_class_path = self.generate_single_java_compile(src, target, compiler, outfile)
+        compile_args = self.determine_single_java_compile_args(target, compiler)
+        for src in src_list + gen_src_list:
+            plain_class_path = self.generate_single_java_compile(src, target, compiler, compile_args, outfile)
             class_list.append(plain_class_path)
         class_dep_list = [os.path.join(self.get_target_private_dir(target), i) for i in class_list]
         manifest_path = os.path.join(self.get_target_private_dir(target), 'META-INF', 'MANIFEST.MF')
@@ -803,6 +859,8 @@ int dummy;
         elem.add_dep(class_dep_list)
         elem.add_item('ARGS', commands)
         elem.write(outfile)
+        # Create introspection information
+        self.create_target_source_introspection(target, compiler, compile_args, src_list, gen_src_list)
 
     def generate_cs_resource_tasks(self, target, outfile):
         args = []
@@ -856,10 +914,11 @@ int dummy;
         else:
             outputs = [outname_rel]
         generated_sources = self.get_target_generated_sources(target)
+        generated_rel_srcs = []
         for rel_src in generated_sources.keys():
             dirpart, fnamepart = os.path.split(rel_src)
             if rel_src.lower().endswith('.cs'):
-                rel_srcs.append(os.path.normpath(rel_src))
+                generated_rel_srcs.append(os.path.normpath(rel_src))
             deps.append(os.path.normpath(rel_src))
 
         for dep in target.get_external_deps():
@@ -867,19 +926,15 @@ int dummy;
         commands += self.build.get_project_args(compiler, target.subproject, target.is_cross)
         commands += self.build.get_global_args(compiler, target.is_cross)
 
-        elem = NinjaBuildElement(self.all_outputs, outputs, 'cs_COMPILER', rel_srcs)
+        elem = NinjaBuildElement(self.all_outputs, outputs, 'cs_COMPILER', rel_srcs + generated_rel_srcs)
         elem.add_dep(deps)
         elem.add_item('ARGS', commands)
         elem.write(outfile)
 
         self.generate_generator_list_rules(target, outfile)
+        self.create_target_source_introspection(target, compiler, commands, rel_srcs, generated_rel_srcs)
 
-    def generate_single_java_compile(self, src, target, compiler, outfile):
-        deps = [os.path.join(self.get_target_dir(l), l.get_filename()) for l in target.link_targets]
-        generated_sources = self.get_target_generated_sources(target)
-        for rel_src, gensrc in generated_sources.items():
-            if rel_src.endswith('.java'):
-                deps.append(rel_src)
+    def determine_single_java_compile_args(self, target, compiler):
         args = []
         args += compiler.get_buildtype_args(self.get_option_for_target('buildtype', target))
         args += self.build.get_global_args(compiler, target.is_cross)
@@ -894,6 +949,14 @@ int dummy;
             for idir in i.get_incdirs():
                 sourcepath += os.path.join(self.build_to_src, i.curdir, idir) + os.pathsep
         args += ['-sourcepath', sourcepath]
+        return args
+
+    def generate_single_java_compile(self, src, target, compiler, args, outfile):
+        deps = [os.path.join(self.get_target_dir(l), l.get_filename()) for l in target.link_targets]
+        generated_sources = self.get_target_generated_sources(target)
+        for rel_src, gensrc in generated_sources.items():
+            if rel_src.endswith('.java'):
+                deps.append(rel_src)
         rel_src = src.rel_to_builddir(self.build_to_src)
         plain_class_path = src.fname[:-4] + 'class'
         rel_obj = os.path.join(self.get_target_private_dir(target), plain_class_path)
@@ -1102,6 +1165,7 @@ int dummy;
         element.add_item('ARGS', args)
         element.add_dep(extra_dep_files)
         element.write(outfile)
+        self.create_target_source_introspection(target, valac, args, all_files, [])
         return other_src[0], other_src[1], vala_c_src
 
     def generate_rust_target(self, target, outfile):
@@ -1193,6 +1257,7 @@ int dummy;
         element.write(outfile)
         if isinstance(target, build.SharedLibrary):
             self.generate_shsym(outfile, target)
+        self.create_target_source_introspection(target, rustc, args, [main_rust_file], [])
 
     def swift_module_file_name(self, target):
         return os.path.join(self.get_target_private_dir(target),
@@ -1241,12 +1306,14 @@ int dummy;
         module_name = self.target_swift_modulename(target)
         swiftc = target.compilers['swift']
         abssrc = []
+        relsrc = []
         abs_headers = []
         header_imports = []
         for i in target.get_sources():
             if swiftc.can_compile(i):
-                relsrc = i.rel_to_builddir(self.build_to_src)
-                abss = os.path.normpath(os.path.join(self.environment.get_build_dir(), relsrc))
+                rels = i.rel_to_builddir(self.build_to_src)
+                abss = os.path.normpath(os.path.join(self.environment.get_build_dir(), rels))
+                relsrc.append(rels)
                 abssrc.append(abss)
             elif self.environment.is_header(i):
                 relh = i.rel_to_builddir(self.build_to_src)
@@ -1330,6 +1397,8 @@ int dummy;
             elem.write(outfile)
         else:
             raise MesonException('Swift supports only executable and static library targets.')
+        # Introspection information
+        self.create_target_source_introspection(target, swiftc, compile_args + header_imports + module_includes, relsrc, rel_generated)
 
     def generate_static_link_rules(self, is_cross, outfile):
         num_pools = self.environment.coredata.backend_options['backend_max_links'].value
@@ -2049,6 +2118,12 @@ rule FORTRAN_DEP_HACK%s
         commands = self._generate_single_compile(target, compiler, is_generated)
         commands = CompilerArgs(commands.compiler, commands)
 
+        # Create introspection information
+        if is_generated is False:
+            self.create_target_source_introspection(target, compiler, commands, [src], [])
+        else:
+            self.create_target_source_introspection(target, compiler, commands, [], [src])
+
         build_dir = self.environment.get_build_dir()
         if isinstance(src, File):
             rel_src = src.rel_to_builddir(self.build_to_src)
@@ -2663,6 +2738,15 @@ rule FORTRAN_DEP_HACK%s
         elem = NinjaBuildElement(self.all_outputs, deps, 'phony', '')
         elem.write(outfile)
 
+    def get_introspection_data(self, target_id, target):
+        if target_id not in self.introspection_data or len(self.introspection_data[target_id]) == 0:
+            return super().get_introspection_data(target_id, target)
+
+        result = []
+        for _, i in self.introspection_data[target_id].items():
+            result += [i]
+        return result
+
 def load(build_dir):
     filename = os.path.join(build_dir, 'meson-private', 'install.dat')
     with open(filename, 'rb') as f: