Merge pull request #4547 from mensinda/introIncDirs

mintro: Save introspection to disk and --targets modifications

17 files changed, 483 insertions, 128 deletions

diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py
index 22920f4..39aa365 100644
--- a/mesonbuild/backend/backends.py
+++ b/mesonbuild/backend/backends.py
@@ -156,6 +156,8 @@ class Backend:
         self.build = build
         self.environment = build.environment
         self.processed_targets = {}
+        self.build_dir = self.environment.get_build_dir()
+        self.source_dir = self.environment.get_source_dir()
         self.build_to_src = mesonlib.relpath(self.environment.get_source_dir(),
                                              self.environment.get_build_dir())
 
@@ -683,7 +685,7 @@ class Backend:
     def write_test_file(self, datafile):
         self.write_test_serialisation(self.build.get_tests(), datafile)
 
-    def write_test_serialisation(self, tests, datafile):
+    def create_test_serialisation(self, tests):
         arr = []
         for t in tests:
             exe = t.get_exe()
@@ -730,7 +732,10 @@ class Backend:
                                    exe_wrapper, t.is_parallel, cmd_args, t.env,
                                    t.should_fail, t.timeout, t.workdir, extra_paths)
             arr.append(ts)
-        pickle.dump(arr, datafile)
+        return arr
+
+    def write_test_serialisation(self, tests, datafile):
+        pickle.dump(self.create_test_serialisation(tests), datafile)
 
     def generate_depmf_install(self, d):
         if self.build.dep_manifest_name is None:
@@ -974,9 +979,7 @@ class Backend:
             cmd = s['exe'] + s['args']
             subprocess.check_call(cmd, env=child_env)
 
-    def create_install_data_files(self):
-        install_data_file = os.path.join(self.environment.get_scratch_dir(), 'install.dat')
-
+    def create_install_data(self):
         strip_bin = self.environment.binaries.host.lookup_entry('strip')
         if strip_bin is None:
             if self.environment.is_cross_build():
@@ -997,8 +1000,12 @@ class Backend:
         self.generate_data_install(d)
         self.generate_custom_install_script(d)
         self.generate_subdir_install(d)
+        return d
+
+    def create_install_data_files(self):
+        install_data_file = os.path.join(self.environment.get_scratch_dir(), 'install.dat')
         with open(install_data_file, 'wb') as ofile:
-            pickle.dump(d, ofile)
+            pickle.dump(self.create_install_data(), ofile)
 
     def generate_target_install(self, d):
         for t in self.build.get_targets().values():
@@ -1144,3 +1151,53 @@ class Backend:
                 dst_dir = os.path.join(dst_dir, os.path.basename(src_dir))
             d.install_subdirs.append([src_dir, dst_dir, sd.install_mode,
                                       sd.exclude])
+
+    def get_introspection_data(self, target_id, target):
+        '''
+        Returns a list of source dicts with the following format for a given target:
+        [
+            {
+                "language": "<LANG>",
+                "compiler": ["result", "of", "comp.get_exelist()"],
+                "parameters": ["list", "of", "compiler", "parameters],
+                "sources": ["list", "of", "all", "<LANG>", "source", "files"],
+                "generated_sources": ["list", "of", "generated", "source", "files"]
+            }
+        ]
+
+        This is a limited fallback / reference implementation. The backend should override this method.
+        '''
+        if isinstance(target, (build.CustomTarget, build.BuildTarget)):
+            source_list_raw = target.sources + target.extra_files
+            source_list = []
+            for j in source_list_raw:
+                if isinstance(j, mesonlib.File):
+                    source_list += [j.absolute_path(self.source_dir, self.build_dir)]
+                elif isinstance(j, str):
+                    source_list += [os.path.join(self.source_dir, j)]
+            source_list = list(map(lambda x: os.path.normpath(x), source_list))
+
+            compiler = []
+            if isinstance(target, build.CustomTarget):
+                tmp_compiler = target.command
+                if not isinstance(compiler, list):
+                    tmp_compiler = [compiler]
+                for j in tmp_compiler:
+                    if isinstance(j, mesonlib.File):
+                        compiler += [j.absolute_path(self.source_dir, self.build_dir)]
+                    elif isinstance(j, str):
+                        compiler += [j]
+                    elif isinstance(j, (build.BuildTarget, build.CustomTarget)):
+                        compiler += j.get_outputs()
+                    else:
+                        raise RuntimeError('Type "{}" is not supported in get_introspection_data. This is a bug'.format(type(j).__name__))
+
+            return [{
+                'language': 'unknown',
+                'compiler': compiler,
+                'parameters': [],
+                'sources': source_list,
+                'generated_sources': []
+            }]
+
+        return []
diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index 44bdaab..3688f29 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -150,6 +150,7 @@ class NinjaBackend(backends.Backend):
         self.ninja_filename = 'build.ninja'
         self.fortran_deps = {}
         self.all_outputs = {}
+        self.introspection_data = {}
 
     def create_target_alias(self, to_target, outfile):
         # We need to use aliases for targets that might be used as directory
@@ -321,6 +322,57 @@ int dummy;
                 return False
         return True
 
+    def create_target_source_introspection(self, target: build.Target, comp: compilers.Compiler, parameters, sources, generated_sources):
+        '''
+        Adds the source file introspection information for a language of a target
+
+        Internal introspection storage formart:
+        self.introspection_data = {
+            '<target ID>': {
+                <id tuple>: {
+                    'language: 'lang',
+                    'compiler': ['comp', 'exe', 'list'],
+                    'parameters': ['UNIQUE', 'parameter', 'list'],
+                    'sources': [],
+                    'generated_sources': [],
+                }
+            }
+        }
+        '''
+        id = target.get_id()
+        lang = comp.get_language()
+        tgt = self.introspection_data[id]
+        # Find an existing entry or create a new one
+        id_hash = (lang, tuple(parameters))
+        src_block = tgt.get(id_hash, None)
+        if src_block is None:
+            # Convert parameters
+            if isinstance(parameters, CompilerArgs):
+                parameters = parameters.to_native(copy=True)
+            parameters = comp.compute_parameters_with_absolute_paths(parameters, self.build_dir)
+            if target.is_cross:
+                extra_parameters = comp.get_cross_extra_flags(self.environment, False)
+                if isinstance(parameters, CompilerArgs):
+                    extra_parameters = extra_parameters.to_native(copy=True)
+                parameters = extra_parameters + parameters
+            # The new entry
+            src_block = {
+                'language': lang,
+                'compiler': comp.get_exelist(),
+                'parameters': parameters,
+                'sources': [],
+                'generated_sources': [],
+            }
+            tgt[id_hash] = src_block
+        # Make source files absolute
+        sources = [x.absolute_path(self.source_dir, self.build_dir) if isinstance(x, File) else os.path.normpath(os.path.join(self.build_dir, x))
+                   for x in sources]
+        generated_sources = [x.absolute_path(self.source_dir, self.build_dir) if isinstance(x, File) else os.path.normpath(os.path.join(self.build_dir, x))
+                             for x in generated_sources]
+        # Add the source files
+        src_block['sources'] += sources
+        src_block['generated_sources'] += generated_sources
+
     def generate_target(self, target, outfile):
         if isinstance(target, build.CustomTarget):
             self.generate_custom_target(target, outfile)
@@ -330,6 +382,8 @@ int dummy;
         if name in self.processed_targets:
             return
         self.processed_targets[name] = True
+        # Initialize an empty introspection source list
+        self.introspection_data[name] = {}
         # Generate rules for all dependency targets
         self.process_target_dependencies(target, outfile)
         # If target uses a language that cannot link to C objects,
@@ -770,14 +824,16 @@ int dummy;
 
         # Add possible java generated files to src list
         generated_sources = self.get_target_generated_sources(target)
+        gen_src_list = []
         for rel_src, gensrc in generated_sources.items():
             dirpart, fnamepart = os.path.split(rel_src)
             raw_src = File(True, dirpart, fnamepart)
             if rel_src.endswith('.java'):
-                src_list.append(raw_src)
+                gen_src_list.append(raw_src)
 
-        for src in src_list:
-            plain_class_path = self.generate_single_java_compile(src, target, compiler, outfile)
+        compile_args = self.determine_single_java_compile_args(target, compiler)
+        for src in src_list + gen_src_list:
+            plain_class_path = self.generate_single_java_compile(src, target, compiler, compile_args, outfile)
             class_list.append(plain_class_path)
         class_dep_list = [os.path.join(self.get_target_private_dir(target), i) for i in class_list]
         manifest_path = os.path.join(self.get_target_private_dir(target), 'META-INF', 'MANIFEST.MF')
@@ -803,6 +859,8 @@ int dummy;
         elem.add_dep(class_dep_list)
         elem.add_item('ARGS', commands)
         elem.write(outfile)
+        # Create introspection information
+        self.create_target_source_introspection(target, compiler, compile_args, src_list, gen_src_list)
 
     def generate_cs_resource_tasks(self, target, outfile):
         args = []
@@ -856,10 +914,11 @@ int dummy;
         else:
             outputs = [outname_rel]
         generated_sources = self.get_target_generated_sources(target)
+        generated_rel_srcs = []
         for rel_src in generated_sources.keys():
             dirpart, fnamepart = os.path.split(rel_src)
             if rel_src.lower().endswith('.cs'):
-                rel_srcs.append(os.path.normpath(rel_src))
+                generated_rel_srcs.append(os.path.normpath(rel_src))
             deps.append(os.path.normpath(rel_src))
 
         for dep in target.get_external_deps():
@@ -867,19 +926,15 @@ int dummy;
         commands += self.build.get_project_args(compiler, target.subproject, target.is_cross)
         commands += self.build.get_global_args(compiler, target.is_cross)
 
-        elem = NinjaBuildElement(self.all_outputs, outputs, 'cs_COMPILER', rel_srcs)
+        elem = NinjaBuildElement(self.all_outputs, outputs, 'cs_COMPILER', rel_srcs + generated_rel_srcs)
         elem.add_dep(deps)
         elem.add_item('ARGS', commands)
         elem.write(outfile)
 
         self.generate_generator_list_rules(target, outfile)
+        self.create_target_source_introspection(target, compiler, commands, rel_srcs, generated_rel_srcs)
 
-    def generate_single_java_compile(self, src, target, compiler, outfile):
-        deps = [os.path.join(self.get_target_dir(l), l.get_filename()) for l in target.link_targets]
-        generated_sources = self.get_target_generated_sources(target)
-        for rel_src, gensrc in generated_sources.items():
-            if rel_src.endswith('.java'):
-                deps.append(rel_src)
+    def determine_single_java_compile_args(self, target, compiler):
         args = []
         args += compiler.get_buildtype_args(self.get_option_for_target('buildtype', target))
         args += self.build.get_global_args(compiler, target.is_cross)
@@ -894,6 +949,14 @@ int dummy;
             for idir in i.get_incdirs():
                 sourcepath += os.path.join(self.build_to_src, i.curdir, idir) + os.pathsep
         args += ['-sourcepath', sourcepath]
+        return args
+
+    def generate_single_java_compile(self, src, target, compiler, args, outfile):
+        deps = [os.path.join(self.get_target_dir(l), l.get_filename()) for l in target.link_targets]
+        generated_sources = self.get_target_generated_sources(target)
+        for rel_src, gensrc in generated_sources.items():
+            if rel_src.endswith('.java'):
+                deps.append(rel_src)
         rel_src = src.rel_to_builddir(self.build_to_src)
         plain_class_path = src.fname[:-4] + 'class'
         rel_obj = os.path.join(self.get_target_private_dir(target), plain_class_path)
@@ -1102,6 +1165,7 @@ int dummy;
         element.add_item('ARGS', args)
         element.add_dep(extra_dep_files)
         element.write(outfile)
+        self.create_target_source_introspection(target, valac, args, all_files, [])
         return other_src[0], other_src[1], vala_c_src
 
     def generate_rust_target(self, target, outfile):
@@ -1193,6 +1257,7 @@ int dummy;
         element.write(outfile)
         if isinstance(target, build.SharedLibrary):
             self.generate_shsym(outfile, target)
+        self.create_target_source_introspection(target, rustc, args, [main_rust_file], [])
 
     def swift_module_file_name(self, target):
         return os.path.join(self.get_target_private_dir(target),
@@ -1241,12 +1306,14 @@ int dummy;
         module_name = self.target_swift_modulename(target)
         swiftc = target.compilers['swift']
         abssrc = []
+        relsrc = []
         abs_headers = []
         header_imports = []
         for i in target.get_sources():
             if swiftc.can_compile(i):
-                relsrc = i.rel_to_builddir(self.build_to_src)
-                abss = os.path.normpath(os.path.join(self.environment.get_build_dir(), relsrc))
+                rels = i.rel_to_builddir(self.build_to_src)
+                abss = os.path.normpath(os.path.join(self.environment.get_build_dir(), rels))
+                relsrc.append(rels)
                 abssrc.append(abss)
             elif self.environment.is_header(i):
                 relh = i.rel_to_builddir(self.build_to_src)
@@ -1330,6 +1397,8 @@ int dummy;
             elem.write(outfile)
         else:
             raise MesonException('Swift supports only executable and static library targets.')
+        # Introspection information
+        self.create_target_source_introspection(target, swiftc, compile_args + header_imports + module_includes, relsrc, rel_generated)
 
     def generate_static_link_rules(self, is_cross, outfile):
         num_pools = self.environment.coredata.backend_options['backend_max_links'].value
@@ -2049,6 +2118,12 @@ rule FORTRAN_DEP_HACK%s
         commands = self._generate_single_compile(target, compiler, is_generated)
         commands = CompilerArgs(commands.compiler, commands)
 
+        # Create introspection information
+        if is_generated is False:
+            self.create_target_source_introspection(target, compiler, commands, [src], [])
+        else:
+            self.create_target_source_introspection(target, compiler, commands, [], [src])
+
         build_dir = self.environment.get_build_dir()
         if isinstance(src, File):
             rel_src = src.rel_to_builddir(self.build_to_src)
@@ -2663,6 +2738,15 @@ rule FORTRAN_DEP_HACK%s
         elem = NinjaBuildElement(self.all_outputs, deps, 'phony', '')
         elem.write(outfile)
 
+    def get_introspection_data(self, target_id, target):
+        if target_id not in self.introspection_data or len(self.introspection_data[target_id]) == 0:
+            return super().get_introspection_data(target_id, target)
+
+        result = []
+        for _, i in self.introspection_data[target_id].items():
+            result += [i]
+        return result
+
 def load(build_dir):
     filename = os.path.join(build_dir, 'meson-private', 'install.dat')
     with open(filename, 'rb') as f:
diff --git a/mesonbuild/build.py b/mesonbuild/build.py
index 5d0fefa..91edbb8 100644
--- a/mesonbuild/build.py
+++ b/mesonbuild/build.py
@@ -345,6 +345,8 @@ a hard error in the future.''' % name)
         self.install = False
         self.build_always_stale = False
         self.option_overrides = {}
+        if not hasattr(self, 'typename'):
+            raise RuntimeError('Target type is not set for target class "{}". This is a bug'.format(type(self).__name__))
 
     def get_install_dir(self, environment):
         # Find the installation directory.
@@ -366,6 +368,9 @@ a hard error in the future.''' % name)
     def get_subdir(self):
         return self.subdir
 
+    def get_typename(self):
+        return self.typename
+
     @staticmethod
     def _get_id_hash(target_id):
         # We don't really need cryptographic security here.
@@ -1361,6 +1366,7 @@ class Executable(BuildTarget):
     known_kwargs = known_exe_kwargs
 
     def __init__(self, name, subdir, subproject, is_cross, sources, objects, environment, kwargs):
+        self.typename = 'executable'
         if 'pie' not in kwargs and 'b_pie' in environment.coredata.base_options:
             kwargs['pie'] = environment.coredata.base_options['b_pie'].value
         super().__init__(name, subdir, subproject, is_cross, sources, objects, environment, kwargs)
@@ -1450,6 +1456,7 @@ class StaticLibrary(BuildTarget):
     known_kwargs = known_stlib_kwargs
 
     def __init__(self, name, subdir, subproject, is_cross, sources, objects, environment, kwargs):
+        self.typename = 'static library'
         if 'pic' not in kwargs and 'b_staticpic' in environment.coredata.base_options:
             kwargs['pic'] = environment.coredata.base_options['b_staticpic'].value
         super().__init__(name, subdir, subproject, is_cross, sources, objects, environment, kwargs)
@@ -1509,6 +1516,7 @@ class SharedLibrary(BuildTarget):
     known_kwargs = known_shlib_kwargs
 
     def __init__(self, name, subdir, subproject, is_cross, sources, objects, environment, kwargs):
+        self.typename = 'shared library'
         self.soversion = None
         self.ltversion = None
         # Max length 2, first element is compatibility_version, second is current_version
@@ -1817,6 +1825,7 @@ class SharedModule(SharedLibrary):
         if 'soversion' in kwargs:
             raise MesonException('Shared modules must not specify the soversion kwarg.')
         super().__init__(name, subdir, subproject, is_cross, sources, objects, environment, kwargs)
+        self.typename = 'shared module'
 
     def get_default_install_dir(self, environment):
         return environment.get_shared_module_dir()
@@ -1842,6 +1851,7 @@ class CustomTarget(Target):
     ])
 
     def __init__(self, name, subdir, subproject, kwargs, absolute_paths=False):
+        self.typename = 'custom'
         super().__init__(name, subdir, subproject, False)
         self.dependencies = []
         self.extra_depends = []
@@ -2083,6 +2093,7 @@ class CustomTarget(Target):
 
 class RunTarget(Target):
     def __init__(self, name, command, args, dependencies, subdir, subproject):
+        self.typename = 'run'
         super().__init__(name, subdir, subproject, False)
         self.command = command
         self.args = args
@@ -2110,6 +2121,14 @@ class RunTarget(Target):
     def get_filename(self):
         return self.name
 
+    def get_outputs(self):
+        if isinstance(self.name, str):
+            return [self.name]
+        elif isinstance(self.name, list):
+            return self.name
+        else:
+            raise RuntimeError('RunTarget: self.name is neither a list nor a string. This is a bug')
+
     def type_suffix(self):
         return "@run"
 
@@ -2117,6 +2136,7 @@ class Jar(BuildTarget):
     known_kwargs = known_jar_kwargs
 
     def __init__(self, name, subdir, subproject, is_cross, sources, objects, environment, kwargs):
+        self.typename = 'jar'
         super().__init__(name, subdir, subproject, is_cross, sources, objects, environment, kwargs)
         for s in self.sources:
             if not s.endswith('.java'):
@@ -2160,6 +2180,7 @@ class CustomTargetIndex:
     """
 
     def __init__(self, target, output):
+        self.typename = 'custom'
         self.target = target
         self.output = output
 
diff --git a/mesonbuild/compilers/__init__.py b/mesonbuild/compilers/__init__.py
index 31b7b89..b5b2475 100644
--- a/mesonbuild/compilers/__init__.py
+++ b/mesonbuild/compilers/__init__.py
@@ -15,6 +15,7 @@
 # Public symbols for compilers sub-package when using 'from . import compilers'
 __all__ = [
     'CompilerType',
+    'Compiler',
 
     'all_languages',
     'base_options',
@@ -91,6 +92,7 @@ __all__ = [
 # Bring symbols from each module into compilers sub-package namespace
 from .compilers import (
     CompilerType,
+    Compiler,
     all_languages,
     base_options,
     clib_langs,
diff --git a/mesonbuild/compilers/c.py b/mesonbuild/compilers/c.py
index 92a9fa6..6350eee 100644
--- a/mesonbuild/compilers/c.py
+++ b/mesonbuild/compilers/c.py
@@ -1475,6 +1475,15 @@ class VisualStudioCCompiler(CCompiler):
         # msvc does not have a concept of system header dirs.
         return ['-I' + path]
 
+    def compute_parameters_with_absolute_paths(self, parameter_list, build_dir):
+        for idx, i in enumerate(parameter_list):
+            if i[:2] == '-I' or i[:2] == '/I':
+                parameter_list[idx] = i[:2] + os.path.normpath(os.path.join(build_dir, i[2:]))
+            elif i[:9] == '/LIBPATH:':
+                parameter_list[idx] = i[:9] + os.path.normpath(os.path.join(build_dir, i[9:]))
+
+        return parameter_list
+
     # Visual Studio is special. It ignores some arguments it does not
     # understand and you can't tell it to error out on those.
     # http://stackoverflow.com/questions/15259720/how-can-i-make-the-microsoft-c-compiler-treat-unknown-flags-as-errors-rather-t
diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py
index 2be6ef1..3ef4ffc 100644
--- a/mesonbuild/compilers/compilers.py
+++ b/mesonbuild/compilers/compilers.py
@@ -884,6 +884,9 @@ class Compiler:
     def compute_int(self, expression, low, high, guess, prefix, env, extra_args, dependencies):
         raise EnvironmentException('%s does not support compute_int ' % self.get_id())
 
+    def compute_parameters_with_absolute_paths(self, parameter_list, build_dir):
+        raise EnvironmentException('%s does not support compute_parameters_with_absolute_paths ' % self.get_id())
+
     def has_members(self, typename, membernames, prefix, env, *, extra_args=None, dependencies=None):
         raise EnvironmentException('%s does not support has_member(s) ' % self.get_id())
 
@@ -1547,6 +1550,13 @@ class GnuLikeCompiler(abc.ABC):
             return ['-mwindows']
         return []
 
+    def compute_parameters_with_absolute_paths(self, parameter_list, build_dir):
+        for idx, i in enumerate(parameter_list):
+            if i[:2] == '-I' or i[:2] == '-L':
+                parameter_list[idx] = i[:2] + os.path.normpath(os.path.join(build_dir, i[2:]))
+
+        return parameter_list
+
 class GnuCompiler(GnuLikeCompiler):
     """
     GnuCompiler represents an actual GCC in its many incarnations.
@@ -1776,6 +1786,13 @@ class ArmclangCompiler:
         """
         return ['--symdefs=' + implibname]
 
+    def compute_parameters_with_absolute_paths(self, parameter_list, build_dir):
+        for idx, i in enumerate(parameter_list):
+            if i[:2] == '-I' or i[:2] == '-L':
+                parameter_list[idx] = i[:2] + os.path.normpath(os.path.join(build_dir, i[2:]))
+
+        return parameter_list
+
 
 # Tested on linux for ICC 14.0.3, 15.0.6, 16.0.4, 17.0.1, 19.0.0
 class IntelCompiler(GnuLikeCompiler):
@@ -1910,6 +1927,13 @@ class ArmCompiler:
     def get_debug_args(self, is_debug):
         return clike_debug_args[is_debug]
 
+    def compute_parameters_with_absolute_paths(self, parameter_list, build_dir):
+        for idx, i in enumerate(parameter_list):
+            if i[:2] == '-I' or i[:2] == '-L':
+                parameter_list[idx] = i[:2] + os.path.normpath(os.path.join(build_dir, i[2:]))
+
+        return parameter_list
+
 class CcrxCompiler:
     def __init__(self, compiler_type):
         if not self.is_cross:
@@ -2003,3 +2027,10 @@ class CcrxCompiler:
                 continue
             result.append(i)
         return result
+
+    def compute_parameters_with_absolute_paths(self, parameter_list, build_dir):
+        for idx, i in enumerate(parameter_list):
+            if i[:9] == '-include=':
+                parameter_list[idx] = i[:9] + os.path.normpath(os.path.join(build_dir, i[9:]))
+
+        return parameter_list
diff --git a/mesonbuild/compilers/cs.py b/mesonbuild/compilers/cs.py
index a6c74d2..cbfcd9c 100644
--- a/mesonbuild/compilers/cs.py
+++ b/mesonbuild/compilers/cs.py
@@ -88,6 +88,15 @@ class CsCompiler(Compiler):
     def get_pic_args(self):
         return []
 
+    def compute_parameters_with_absolute_paths(self, parameter_list, build_dir):
+        for idx, i in enumerate(parameter_list):
+            if i[:2] == '-L':
+                parameter_list[idx] = i[:2] + os.path.normpath(os.path.join(build_dir, i[2:]))
+            if i[:5] == '-lib:':
+                parameter_list[idx] = i[:5] + os.path.normpath(os.path.join(build_dir, i[5:]))
+
+        return parameter_list
+
     def name_string(self):
         return ' '.join(self.exelist)
 
diff --git a/mesonbuild/compilers/d.py b/mesonbuild/compilers/d.py
index 2cf0fbd..3065ac7 100644
--- a/mesonbuild/compilers/d.py
+++ b/mesonbuild/compilers/d.py
@@ -111,6 +111,19 @@ class DCompiler(Compiler):
     def get_include_args(self, path, is_system):
         return ['-I=' + path]
 
+    def compute_parameters_with_absolute_paths(self, parameter_list, build_dir):
+        for idx, i in enumerate(parameter_list):
+            if i[:3] == '-I=':
+                parameter_list[idx] = i[:3] + os.path.normpath(os.path.join(build_dir, i[3:]))
+            if i[:4] == '-L-L':
+                parameter_list[idx] = i[:4] + os.path.normpath(os.path.join(build_dir, i[4:]))
+            if i[:5] == '-L=-L':
+                parameter_list[idx] = i[:5] + os.path.normpath(os.path.join(build_dir, i[5:]))
+            if i[:6] == '-Wl,-L':
+                parameter_list[idx] = i[:6] + os.path.normpath(os.path.join(build_dir, i[6:]))
+
+        return parameter_list
+
     def get_warn_args(self, level):
         return ['-wi']
 
@@ -511,6 +524,13 @@ class GnuDCompiler(DCompiler):
     def get_buildtype_args(self, buildtype):
         return d_gdc_buildtype_args[buildtype]
 
+    def compute_parameters_with_absolute_paths(self, parameter_list, build_dir):
+        for idx, i in enumerate(parameter_list):
+            if i[:2] == '-I' or i[:2] == '-L':
+                parameter_list[idx] = i[:2] + os.path.normpath(os.path.join(build_dir, i[2:]))
+
+        return parameter_list
+
     def build_rpath_args(self, build_dir, from_dir, rpath_paths, build_rpath, install_rpath):
         return self.build_unix_rpath_args(build_dir, from_dir, rpath_paths, build_rpath, install_rpath)
 
diff --git a/mesonbuild/compilers/fortran.py b/mesonbuild/compilers/fortran.py
index 75db26d..8056969 100644
--- a/mesonbuild/compilers/fortran.py
+++ b/mesonbuild/compilers/fortran.py
@@ -171,6 +171,13 @@ end program prog
     def get_module_outdir_args(self, path):
         return ['-module', path]
 
+    def compute_parameters_with_absolute_paths(self, parameter_list, build_dir):
+        for idx, i in enumerate(parameter_list):
+            if i[:2] == '-I' or i[:2] == '-L':
+                parameter_list[idx] = i[:2] + os.path.normpath(os.path.join(build_dir, i[2:]))
+
+        return parameter_list
+
     def module_name_to_filename(self, module_name):
         return module_name.lower() + '.mod'
 
diff --git a/mesonbuild/compilers/java.py b/mesonbuild/compilers/java.py
index 978562c..03ee382 100644
--- a/mesonbuild/compilers/java.py
+++ b/mesonbuild/compilers/java.py
@@ -81,6 +81,15 @@ class JavaCompiler(Compiler):
     def get_buildtype_args(self, buildtype):
         return java_buildtype_args[buildtype]
 
+    def compute_parameters_with_absolute_paths(self, parameter_list, build_dir):
+        for idx, i in enumerate(parameter_list):
+            if i in ['-cp', '-classpath', '-sourcepath'] and idx + 1 < len(parameter_list):
+                path_list = parameter_list[idx + 1].split(os.pathsep)
+                path_list = [os.path.normpath(os.path.join(build_dir, x)) for x in path_list]
+                parameter_list[idx + 1] = os.pathsep.join(path_list)
+
+        return parameter_list
+
     def sanity_check(self, work_dir, environment):
         src = 'SanityCheck.java'
         obj = 'SanityCheck'
diff --git a/mesonbuild/compilers/rust.py b/mesonbuild/compilers/rust.py
index 93c2917..68da823 100644
--- a/mesonbuild/compilers/rust.py
+++ b/mesonbuild/compilers/rust.py
@@ -82,3 +82,14 @@ class RustCompiler(Compiler):
 
     def get_optimization_args(self, optimization_level):
         return rust_optimization_args[optimization_level]
+
+    def compute_parameters_with_absolute_paths(self, parameter_list, build_dir):
+        for idx, i in enumerate(parameter_list):
+            if i[:2] == '-L':
+                for j in ['dependency', 'crate', 'native', 'framework', 'all']:
+                    combined_len = len(j) + 3
+                    if i[:combined_len] == '-L{}='.format(j):
+                        parameter_list[idx] = i[:combined_len] + os.path.normpath(os.path.join(build_dir, i[combined_len:]))
+                        break
+
+        return parameter_list
diff --git a/mesonbuild/compilers/swift.py b/mesonbuild/compilers/swift.py
index 4d5dd0c..eb58d11 100644
--- a/mesonbuild/compilers/swift.py
+++ b/mesonbuild/compilers/swift.py
@@ -91,6 +91,13 @@ class SwiftCompiler(Compiler):
     def get_compile_only_args(self):
         return ['-c']
 
+    def compute_parameters_with_absolute_paths(self, parameter_list, build_dir):
+        for idx, i in enumerate(parameter_list):
+            if i[:2] == '-I' or i[:2] == '-L':
+                parameter_list[idx] = i[:2] + os.path.normpath(os.path.join(build_dir, i[2:]))
+
+        return parameter_list
+
     def sanity_check(self, work_dir, environment):
         src = 'swifttest.swift'
         source_name = os.path.join(work_dir, src)
diff --git a/mesonbuild/compilers/vala.py b/mesonbuild/compilers/vala.py
index 46bb210..e64d57f 100644
--- a/mesonbuild/compilers/vala.py
+++ b/mesonbuild/compilers/vala.py
@@ -66,6 +66,19 @@ class ValaCompiler(Compiler):
             return ['--color=' + colortype]
         return []
 
+    def compute_parameters_with_absolute_paths(self, parameter_list, build_dir):
+        for idx, i in enumerate(parameter_list):
+            if i[:9] == '--girdir=':
+                parameter_list[idx] = i[:9] + os.path.normpath(os.path.join(build_dir, i[9:]))
+            if i[:10] == '--vapidir=':
+                parameter_list[idx] = i[:10] + os.path.normpath(os.path.join(build_dir, i[10:]))
+            if i[:13] == '--includedir=':
+                parameter_list[idx] = i[:13] + os.path.normpath(os.path.join(build_dir, i[13:]))
+            if i[:14] == '--metadatadir=':
+                parameter_list[idx] = i[:14] + os.path.normpath(os.path.join(build_dir, i[14:]))
+
+        return parameter_list
+
     def sanity_check(self, work_dir, environment):
         code = 'class MesonSanityCheck : Object { }'
         args = self.get_cross_extra_flags(environment, link=False)
diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py
index b10f826..e99174c 100644
--- a/mesonbuild/environment.py
+++ b/mesonbuild/environment.py
@@ -308,6 +308,7 @@ def search_version(text):
 class Environment:
     private_dir = 'meson-private'
     log_dir = 'meson-logs'
+    info_dir = 'meson-info'
 
     def __init__(self, source_dir, build_dir, options):
         self.source_dir = source_dir
@@ -318,8 +319,10 @@ class Environment:
         if build_dir is not None:
             self.scratch_dir = os.path.join(build_dir, Environment.private_dir)
             self.log_dir = os.path.join(build_dir, Environment.log_dir)
+            self.info_dir = os.path.join(build_dir, Environment.info_dir)
             os.makedirs(self.scratch_dir, exist_ok=True)
             os.makedirs(self.log_dir, exist_ok=True)
+            os.makedirs(self.info_dir, exist_ok=True)
             try:
                 self.coredata = coredata.load(self.get_build_dir())
                 self.first_invocation = False
diff --git a/mesonbuild/mconf.py b/mesonbuild/mconf.py
index 1257fd3..61bb74b 100644
--- a/mesonbuild/mconf.py
+++ b/mesonbuild/mconf.py
@@ -14,6 +14,7 @@
 
 import os
 from . import (coredata, mesonlib, build)
+from . import mintro
 
 def add_arguments(parser):
     coredata.register_builtin_arguments(parser)
@@ -162,6 +163,7 @@ def run(options):
             c.print_conf()
         if save:
             c.save()
+            mintro.update_build_options(c.coredata, c.build.environment.info_dir)
     except ConfException as e:
         print('Meson configurator encountered an error:')
         raise e
diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py
index 3896c92..3382e0d 100644
--- a/mesonbuild/mintro.py
+++ b/mesonbuild/mintro.py
@@ -20,7 +20,7 @@ Currently only works for the Ninja backend. Others use generated
 project files and don't need this info."""
 
 import json
-from . import build, mtest, coredata as cdata
+from . import build, coredata as cdata
 from . import environment
 from . import mesonlib
 from . import astinterpreter
@@ -29,7 +29,7 @@ from . import mlog
 from . import compilers
 from . import optinterpreter
 from .interpreterbase import InvalidArguments
-from .backend import ninjabackend, backends
+from .backend import backends
 import sys, os
 import pathlib
 
@@ -54,26 +54,14 @@ def add_arguments(parser):
                         help='Information about projects.')
     parser.add_argument('--backend', choices=cdata.backendlist, dest='backend', default='ninja',
                         help='The backend to use for the --buildoptions introspection.')
+    parser.add_argument('-a', '--all', action='store_true', dest='all', default=False,
+                        help='Print all available information.')
+    parser.add_argument('-i', '--indent', action='store_true', dest='indent', default=False,
+                        help='Enable pretty printed JSON.')
+    parser.add_argument('-f', '--force-object-output', action='store_true', dest='force_dict', default=False,
+                        help='Always use the new JSON format for multiple entries (even for 0 and 1 introspection commands)')
     parser.add_argument('builddir', nargs='?', default='.', help='The build directory')
 
-def determine_installed_path(target, installdata):
-    install_targets = []
-    for i in target.outputs:
-        for j in installdata.targets:
-            if os.path.basename(j.fname) == i: # FIXME, might clash due to subprojects.
-                install_targets += [j]
-                break
-    if len(install_targets) == 0:
-        raise RuntimeError('Something weird happened. File a bug.')
-
-    # Normalize the path by using os.path.sep consistently, etc.
-    # Does not change the effective path.
-    install_targets = list(map(lambda x: os.path.join(installdata.prefix, x.outdir, os.path.basename(x.fname)), install_targets))
-    install_targets = list(map(lambda x: str(pathlib.PurePath(x)), install_targets))
-
-    return install_targets
-
-
 def list_installed(installdata):
     res = {}
     if installdata is not None:
@@ -86,54 +74,43 @@ def list_installed(installdata):
             res[path] = os.path.join(installdata.prefix, installdir, os.path.basename(path))
         for path, installpath, unused_custom_install_mode in installdata.man:
             res[path] = os.path.join(installdata.prefix, installpath)
-    print(json.dumps(res))
-
+    return ('installed', res)
 
-def list_targets(coredata, builddata, installdata):
+def list_targets(builddata: build.Build, installdata, backend: backends.Backend):
     tlist = []
+
+    # Fast lookup table for installation files
+    install_lookuptable = {}
+    for i in installdata.targets:
+        outname = os.path.join(installdata.prefix, i.outdir, os.path.basename(i.fname))
+        install_lookuptable[os.path.basename(i.fname)] = str(pathlib.PurePath(outname))
+
     for (idname, target) in builddata.get_targets().items():
-        t = {'name': target.get_basename(), 'id': idname}
-        fname = target.get_filename()
-        if isinstance(fname, list):
-            fname = [os.path.join(target.subdir, x) for x in fname]
-        else:
-            fname = os.path.join(target.subdir, fname)
-        t['filename'] = fname
-        if isinstance(target, build.Executable):
-            typename = 'executable'
-        elif isinstance(target, build.SharedLibrary):
-            typename = 'shared library'
-        elif isinstance(target, build.StaticLibrary):
-            typename = 'static library'
-        elif isinstance(target, build.CustomTarget):
-            typename = 'custom'
-        elif isinstance(target, build.RunTarget):
-            typename = 'run'
-        else:
-            typename = 'unknown'
-        t['type'] = typename
+        if not isinstance(target, build.Target):
+            raise RuntimeError('The target object in `builddata.get_targets()` is not of type `build.Target`. Please file a bug with this error message.')
+
+        # TODO Change this to the full list in a seperate PR
+        fname = [os.path.join(target.subdir, x) for x in target.get_outputs()]
+        if len(fname) == 1:
+            fname = fname[0]
+
+        t = {
+            'name': target.get_basename(),
+            'id': idname,
+            'type': target.get_typename(),
+            'filename': fname,
+            'build_by_default': target.build_by_default,
+            'target_sources': backend.get_introspection_data(idname, target)
+        }
+
         if installdata and target.should_install():
             t['installed'] = True
-            t['install_filename'] = determine_installed_path(target, installdata)
+            # TODO Change this to the full list in a seperate PR
+            t['install_filename'] = [install_lookuptable.get(x, None) for x in target.get_outputs()][0]
         else:
             t['installed'] = False
-        t['build_by_default'] = target.build_by_default
         tlist.append(t)
-    print(json.dumps(tlist))
-
-def list_target_files(target_name, coredata, builddata):
-    try:
-        t = builddata.targets[target_name]
-        sources = t.sources + t.extra_files
-    except KeyError:
-        print("Unknown target %s." % target_name)
-        sys.exit(1)
-    out = []
-    for i in sources:
-        if isinstance(i, mesonlib.File):
-            i = os.path.join(i.subdir, i.fname)
-        out.append(i)
-    print(json.dumps(out))
+    return ('targets', tlist)
 
 class BuildoptionsOptionHelper:
     # mimic an argparse namespace
@@ -272,7 +249,7 @@ class BuildoptionsInterperter(astinterpreter.AstInterpreter):
         self.parse_project()
         self.run()
 
-def list_buildoptions_from_source(sourcedir, backend):
+def list_buildoptions_from_source(sourcedir, backend, indent):
     # Make sure that log entries in other parts of meson don't interfere with the JSON output
     mlog.disable()
     backend = backends.get_backend_from_name(backend, None)
@@ -280,9 +257,31 @@ def list_buildoptions_from_source(sourcedir, backend):
     intr.analyze()
     # Reenable logging just in case
     mlog.enable()
-    list_buildoptions(intr.coredata)
+    buildoptions = list_buildoptions(intr.coredata)[1]
+    print(json.dumps(buildoptions, indent=indent))
+
+def list_target_files(target_name, targets, builddata: build.Build):
+    result = []
+    tgt = None
+
+    for i in targets:
+        if i['id'] == target_name:
+            tgt = i
+            break
+
+    if tgt is None:
+        print('Target with the ID "{}" could not be found'.format(target_name))
+        sys.exit(1)
+
+    for i in tgt['target_sources']:
+        result += i['sources'] + i['generated_sources']
 
-def list_buildoptions(coredata):
+    # TODO Remove this line in a future PR with other breaking changes
+    result = list(map(lambda x: os.path.relpath(x, builddata.environment.get_source_dir()), result))
+
+    return ('target_files', result)
+
+def list_buildoptions(coredata: cdata.CoreData):
     optlist = []
 
     dir_option_names = ['bindir',
@@ -313,7 +312,7 @@ def list_buildoptions(coredata):
     add_keys(optlist, dir_options, 'directory')
     add_keys(optlist, coredata.user_options, 'user')
     add_keys(optlist, test_options, 'test')
-    print(json.dumps(optlist))
+    return ('buildoptions', optlist)
 
 def add_keys(optlist, options, section):
     keys = list(options.keys())
@@ -347,21 +346,21 @@ def find_buildsystem_files_list(src_dir):
                 filelist.append(os.path.relpath(os.path.join(root, f), src_dir))
     return filelist
 
-def list_buildsystem_files(builddata):
+def list_buildsystem_files(builddata: build.Build):
     src_dir = builddata.environment.get_source_dir()
     filelist = find_buildsystem_files_list(src_dir)
-    print(json.dumps(filelist))
+    return ('buildsystem_files', filelist)
 
-def list_deps(coredata):
+def list_deps(coredata: cdata.CoreData):
     result = []
     for d in coredata.deps.values():
         if d.found():
             result += [{'name': d.name,
                         'compile_args': d.get_compile_args(),
                         'link_args': d.get_link_args()}]
-    print(json.dumps(result))
+    return ('dependencies', result)
 
-def list_tests(testdata):
+def get_test_list(testdata):
     result = []
     for t in testdata:
         to = {}
@@ -380,9 +379,15 @@ def list_tests(testdata):
         to['suite'] = t.suite
         to['is_parallel'] = t.is_parallel
         result.append(to)
-    print(json.dumps(result))
+    return result
+
+def list_tests(testdata):
+    return ('tests', get_test_list(testdata))
+
+def list_benchmarks(benchdata):
+    return ('benchmarks', get_test_list(benchdata))
 
-def list_projinfo(builddata):
+def list_projinfo(builddata: build.Build):
     result = {'version': builddata.project_version,
               'descriptive_name': builddata.project_name}
     subprojects = []
@@ -392,7 +397,7 @@ def list_projinfo(builddata):
              'descriptive_name': builddata.projects.get(k)}
         subprojects.append(c)
     result['subprojects'] = subprojects
-    print(json.dumps(result))
+    return ('projectinfo', result)
 
 class ProjectInfoInterperter(astinterpreter.AstInterpreter):
     def __init__(self, source_root, subdir):
@@ -418,7 +423,7 @@ class ProjectInfoInterperter(astinterpreter.AstInterpreter):
         self.parse_project()
         self.run()
 
-def list_projinfo_from_source(sourcedir):
+def list_projinfo_from_source(sourcedir, indent):
     files = find_buildsystem_files_list(sourcedir)
 
     result = {'buildsystem_files': []}
@@ -447,55 +452,112 @@ def list_projinfo_from_source(sourcedir):
 
     subprojects = [obj for name, obj in subprojects.items()]
     result['subprojects'] = subprojects
-    print(json.dumps(result))
+    print(json.dumps(result, indent=indent))
 
 def run(options):
     datadir = 'meson-private'
+    infodir = 'meson-info'
+    indent = 4 if options.indent else None
     if options.builddir is not None:
         datadir = os.path.join(options.builddir, datadir)
+        infodir = os.path.join(options.builddir, infodir)
     if options.builddir.endswith('/meson.build') or options.builddir.endswith('\\meson.build') or options.builddir == 'meson.build':
         sourcedir = '.' if options.builddir == 'meson.build' else options.builddir[:-11]
         if options.projectinfo:
-            list_projinfo_from_source(sourcedir)
+            list_projinfo_from_source(sourcedir, indent)
             return 0
         if options.buildoptions:
-            list_buildoptions_from_source(sourcedir, options.backend)
+            list_buildoptions_from_source(sourcedir, options.backend, indent)
             return 0
-    if not os.path.isdir(datadir):
-        print('Current directory is not a build dir. Please specify it or '
-              'change the working directory to it.')
+    if not os.path.isdir(datadir) or not os.path.isdir(infodir):
+        print('Current directory is not a meson build directory.'
+              'Please specify a valid build dir or change the working directory to it.'
+              'It is also possible that the build directory was generated with an old'
+              'meson version. Please regenerate it in this case.')
         return 1
 
-    coredata = cdata.load(options.builddir)
-    builddata = build.load(options.builddir)
-    testdata = mtest.load_tests(options.builddir)
-    benchmarkdata = mtest.load_benchmarks(options.builddir)
-
-    # Install data is only available with the Ninja backend
-    try:
-        installdata = ninjabackend.load(options.builddir)
-    except FileNotFoundError:
-        installdata = None
-
-    if options.list_targets:
-        list_targets(coredata, builddata, installdata)
-    elif options.list_installed:
-        list_installed(installdata)
-    elif options.target_files is not None:
-        list_target_files(options.target_files, coredata, builddata)
-    elif options.buildsystem_files:
-        list_buildsystem_files(builddata)
-    elif options.buildoptions:
-        list_buildoptions(coredata)
-    elif options.tests:
-        list_tests(testdata)
-    elif options.benchmarks:
-        list_tests(benchmarkdata)
-    elif options.dependencies:
-        list_deps(coredata)
-    elif options.projectinfo:
-        list_projinfo(builddata)
-    else:
+    # Load build data to make sure that the version matches
+    # TODO Find a better solution for this
+    cdata.load(options.builddir)
+
+    results = []
+    toextract = []
+
+    if options.all or options.benchmarks:
+        toextract += ['benchmarks']
+    if options.all or options.buildoptions:
+        toextract += ['buildoptions']
+    if options.all or options.buildsystem_files:
+        toextract += ['buildsystem_files']
+    if options.all or options.dependencies:
+        toextract += ['dependencies']
+    if options.all or options.list_installed:
+        toextract += ['installed']
+    if options.all or options.projectinfo:
+        toextract += ['projectinfo']
+    if options.all or options.list_targets:
+        toextract += ['targets']
+    if options.target_files is not None:
+        targets_file = os.path.join(infodir, 'intro-targets.json')
+        with open(targets_file, 'r') as fp:
+            targets = json.load(fp)
+        builddata = build.load(options.builddir) # TODO remove this in a breaking changes PR
+        results += [list_target_files(options.target_files, targets, builddata)]
+    if options.all or options.tests:
+        toextract += ['tests']
+
+    for i in toextract:
+        curr = os.path.join(infodir, 'intro-{}.json'.format(i))
+        if not os.path.isfile(curr):
+            print('Introspection file {} does not exist.'.format(curr))
+            return 1
+        with open(curr, 'r') as fp:
+            results += [(i, json.load(fp))]
+
+    if len(results) == 0 and not options.force_dict:
         print('No command specified')
         return 1
+    elif len(results) == 1 and not options.force_dict:
+        # Make to keep the existing output format for a single option
+        print(json.dumps(results[0][1], indent=indent))
+    else:
+        out = {}
+        for i in results:
+            out[i[0]] = i[1]
+        print(json.dumps(out, indent=indent))
     return 0
+
+def write_intro_info(intro_info, info_dir):
+    for i in intro_info:
+        out_file = os.path.join(info_dir, 'intro-{}.json'.format(i[0]))
+        tmp_file = os.path.join(info_dir, 'tmp_dump.json')
+        with open(tmp_file, 'w') as fp:
+            json.dump(i[1], fp)
+            fp.flush() # Not sure if this is needed
+        os.replace(tmp_file, out_file)
+
+def generate_introspection_file(builddata: build.Build, backend: backends.Backend):
+    coredata = builddata.environment.get_coredata()
+    benchmarkdata = backend.create_test_serialisation(builddata.get_benchmarks())
+    testdata = backend.create_test_serialisation(builddata.get_tests())
+    installdata = backend.create_install_data()
+
+    intro_info = [
+        list_benchmarks(benchmarkdata),
+        list_buildoptions(coredata),
+        list_buildsystem_files(builddata),
+        list_deps(coredata),
+        list_installed(installdata),
+        list_projinfo(builddata),
+        list_targets(builddata, installdata, backend),
+        list_tests(testdata)
+    ]
+
+    write_intro_info(intro_info, builddata.environment.info_dir)
+
+def update_build_options(coredata: cdata.CoreData, info_dir):
+    intro_info = [
+        list_buildoptions(coredata)
+    ]
+
+    write_intro_info(intro_info, info_dir)
diff --git a/mesonbuild/msetup.py b/mesonbuild/msetup.py
index 56a0e9a..402f756 100644
--- a/mesonbuild/msetup.py
+++ b/mesonbuild/msetup.py
@@ -23,6 +23,7 @@ import argparse
 from . import environment, interpreter, mesonlib
 from . import build
 from . import mlog, coredata
+from . import mintro
 from .mesonlib import MesonException
 
 def add_arguments(parser):
@@ -215,6 +216,13 @@ class MesonApp:
                 coredata.write_cmd_line_file(self.build_dir, self.options)
             else:
                 coredata.update_cmd_line_file(self.build_dir, self.options)
+
+            # Generate an IDE introspection file with the same syntax as the already existing API
+            if self.options.profile:
+                fname = os.path.join(self.build_dir, 'meson-private', 'profile-introspector.log')
+                profile.runctx('mintro.generate_introspection_file(b, intr.backend)', globals(), locals(), filename=fname)
+            else:
+                mintro.generate_introspection_file(b, intr.backend)
         except:
             if 'cdf' in locals():
                 old_cdf = cdf + '.prev'