From af38722f8976d2527cade370dcd09b3b7ea62e04 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Wed, 16 Jan 2019 22:42:54 +0100 Subject: mintro: Introspection interpreter refactoring (#4733) * Fixed spelling * Merged the Buildoptions and Projectinfo interpreter * Moved detect_compilers to Environment * Added removed test case * Split detect_compilers and moved even more code into Environment * Moved set_default_options to coredata * Small code simplification in mintro.run * Move cmd_line_options back to `environment` We don't actually wish to persist something this unstructured, so we shouldn't make it a field on `coredata`. It would also be data denormalization since the information we already store in coredata depends on the CLI args. --- mesonbuild/coredata.py | 39 +++++++++++++ mesonbuild/environment.py | 19 +++++- mesonbuild/interpreter.py | 51 +--------------- mesonbuild/mintro.py | 144 +++++++++++++--------------------------------- 4 files changed, 100 insertions(+), 153 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index 8c9d513..d5f7d94 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -553,6 +553,45 @@ class CoreData: self.set_options(options, subproject) + def process_new_compilers(self, lang: str, comp, cross_comp, cmd_line_options): + from . import compilers + self.compilers[lang] = comp + # Native compiler always exist so always add its options. + new_options = comp.get_options() + if cross_comp is not None: + self.cross_compilers[lang] = cross_comp + new_options.update(cross_comp.get_options()) + + optprefix = lang + '_' + for k, o in new_options.items(): + if not k.startswith(optprefix): + raise MesonException('Internal error, %s has incorrect prefix.' % k) + if k in cmd_line_options: + o.set_value(cmd_line_options[k]) + self.compiler_options.setdefault(k, o) + + # Unlike compiler and linker flags, preprocessor flags are not in + # compiler_options because they are not visible to user. + preproc_flags = comp.get_preproc_flags() + preproc_flags = shlex.split(preproc_flags) + self.external_preprocess_args.setdefault(lang, preproc_flags) + + enabled_opts = [] + for optname in comp.base_options: + if optname in self.base_options: + continue + oobj = compilers.base_options[optname] + if optname in cmd_line_options: + oobj.set_value(cmd_line_options[optname]) + enabled_opts.append(optname) + self.base_options[optname] = oobj + self.emit_base_options_warnings(enabled_opts) + + def emit_base_options_warnings(self, enabled_opts: list): + if 'b_bitcode' in enabled_opts: + mlog.warning('Base option \'b_bitcode\' is enabled, which is incompatible with many linker options. Incompatible options such as such as \'b_asneeded\' have been disabled.') + mlog.warning('Please see https://mesonbuild.com/Builtin-options.html#Notes_about_Apple_Bitcode_support for more details.') + class CmdLineFileParser(configparser.ConfigParser): def __init__(self): # We don't want ':' as key delimiter, otherwise it would break when diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 277b8d8..0e74851 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -33,6 +33,7 @@ from .compilers import ( is_source, ) from .compilers import ( + Compiler, ArmCCompiler, ArmCPPCompiler, ArmclangCCompiler, @@ -973,7 +974,7 @@ class Environment: return compilers.SwiftCompiler(exelist, version) raise EnvironmentException('Unknown compiler "' + ' '.join(exelist) + '"') - def detect_compilers(self, lang, need_cross_compiler): + def compilers_from_language(self, lang: str, need_cross_compiler: bool): comp = None cross_comp = None if lang == 'c': @@ -1021,7 +1022,23 @@ class Environment: if need_cross_compiler: raise EnvironmentException('Cross compilation with Swift is not working yet.') # cross_comp = self.environment.detect_fortran_compiler(True) + else: + return None, None + + return comp, cross_comp + + def check_compilers(self, lang: str, comp: Compiler, cross_comp: Compiler): + if comp is None: + raise EnvironmentException('Tried to use unknown language "%s".' % lang) + + comp.sanity_check(self.get_scratch_dir(), self) + if cross_comp: + cross_comp.sanity_check(self.get_scratch_dir(), self) + def detect_compilers(self, lang: str, need_cross_compiler: bool): + (comp, cross_comp) = self.compilers_from_language(lang, need_cross_compiler) + if comp is not None: + self.coredata.process_new_compilers(lang, comp, cross_comp, self.cmd_line_options) return comp, cross_comp def detect_static_linker(self, compiler): diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index 7ee9a3f..d2c1ffe 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -2657,36 +2657,6 @@ external dependencies (including libraries) must go to "dependencies".''') self.validate_arguments(args, 0, []) raise Exception() - def detect_compilers(self, lang, need_cross_compiler): - comp, cross_comp = self.environment.detect_compilers(lang, need_cross_compiler) - if comp is None: - raise InvalidCode('Tried to use unknown language "%s".' % lang) - - comp.sanity_check(self.environment.get_scratch_dir(), self.environment) - self.coredata.compilers[lang] = comp - # Native compiler always exist so always add its options. - new_options = comp.get_options() - if cross_comp is not None: - cross_comp.sanity_check(self.environment.get_scratch_dir(), self.environment) - self.coredata.cross_compilers[lang] = cross_comp - new_options.update(cross_comp.get_options()) - - optprefix = lang + '_' - for k, o in new_options.items(): - if not k.startswith(optprefix): - raise InterpreterException('Internal error, %s has incorrect prefix.' % k) - if k in self.environment.cmd_line_options: - o.set_value(self.environment.cmd_line_options[k]) - self.coredata.compiler_options.setdefault(k, o) - - # Unlike compiler and linker flags, preprocessor flags are not in - # compiler_options because they are not visible to user. - preproc_flags = comp.get_preproc_flags() - preproc_flags = shlex.split(preproc_flags) - self.coredata.external_preprocess_args.setdefault(lang, preproc_flags) - - return comp, cross_comp - def add_languages(self, args, required): success = True need_cross_compiler = self.environment.is_cross_build() @@ -2697,7 +2667,8 @@ external dependencies (including libraries) must go to "dependencies".''') cross_comp = self.coredata.cross_compilers.get(lang, None) else: try: - (comp, cross_comp) = self.detect_compilers(lang, need_cross_compiler) + (comp, cross_comp) = self.environment.detect_compilers(lang, need_cross_compiler) + self.environment.check_compilers(lang, comp, cross_comp) except Exception: if not required: mlog.log('Compiler for language', mlog.bold(lang), 'not found.') @@ -2717,26 +2688,8 @@ external dependencies (including libraries) must go to "dependencies".''') mlog.log('Cross', cross_comp.get_display_language(), 'compiler:', mlog.bold(' '.join(cross_comp.get_exelist())), version_string) self.build.ensure_static_cross_linker(comp) - self.add_base_options(comp) return success - def emit_base_options_warnings(self, enabled_opts): - if 'b_bitcode' in enabled_opts: - mlog.warning('Base option \'b_bitcode\' is enabled, which is incompatible with many linker options. Incompatible options such as such as \'b_asneeded\' have been disabled.') - mlog.warning('Please see https://mesonbuild.com/Builtin-options.html#Notes_about_Apple_Bitcode_support for more details.') - - def add_base_options(self, compiler): - enabled_opts = [] - for optname in compiler.base_options: - if optname in self.coredata.base_options: - continue - oobj = compilers.base_options[optname] - if optname in self.environment.cmd_line_options: - oobj.set_value(self.environment.cmd_line_options[optname]) - enabled_opts.append(optname) - self.coredata. base_options[optname] = oobj - self.emit_base_options_warnings(enabled_opts) - def program_from_file_for(self, for_machine, prognames, silent): bins = self.environment.binaries[for_machine] for p in prognames: diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py index 2039553..36368af 100644 --- a/mesonbuild/mintro.py +++ b/mesonbuild/mintro.py @@ -151,20 +151,20 @@ def list_targets(builddata: build.Build, installdata, backend: backends.Backend) tlist.append(t) return tlist -class BuildoptionsOptionHelper: +class IntrospectionHelper: # mimic an argparse namespace def __init__(self, cross_file): self.cross_file = cross_file self.native_file = None self.cmd_line_options = {} -class BuildoptionsInterperter(astinterpreter.AstInterpreter): +class IntrospectionInterpreter(astinterpreter.AstInterpreter): # Interpreter to detect the options without a build directory # Most of the code is stolen from interperter.Interpreter def __init__(self, source_root, subdir, backend, cross_file=None, subproject='', subproject_dir='subprojects', env=None): super().__init__(source_root, subdir) - options = BuildoptionsOptionHelper(cross_file) + options = IntrospectionHelper(cross_file) self.cross_file = cross_file if env is None: self.environment = environment.Environment(source_root, None, options) @@ -176,37 +176,18 @@ class BuildoptionsInterperter(astinterpreter.AstInterpreter): self.option_file = os.path.join(self.source_root, self.subdir, 'meson_options.txt') self.backend = backend self.default_options = {'backend': self.backend} + self.project_data = {} self.funcs.update({ 'project': self.func_project, 'add_languages': self.func_add_languages }) - def detect_compilers(self, lang, need_cross_compiler): - comp, cross_comp = self.environment.detect_compilers(lang, need_cross_compiler) - if comp is None: - return None, None - - self.coredata.compilers[lang] = comp - # Native compiler always exist so always add its options. - new_options = comp.get_options() - if cross_comp is not None: - self.coredata.cross_compilers[lang] = cross_comp - new_options.update(cross_comp.get_options()) - - optprefix = lang + '_' - for k, o in new_options.items(): - if not k.startswith(optprefix): - raise RuntimeError('Internal error, %s has incorrect prefix.' % k) - if k in self.environment.cmd_line_options: - o.set_value(self.environment.cmd_line_options[k]) - self.coredata.compiler_options.setdefault(k, o) - - return comp, cross_comp - def flatten_args(self, args): # Resolve mparser.ArrayNode if needed flattend_args = [] + if isinstance(args, mparser.ArrayNode): + args = [x.value for x in args.args.arguments] for i in args: if isinstance(i, mparser.ArrayNode): flattend_args += [x.value for x in i.args.arguments] @@ -216,35 +197,25 @@ class BuildoptionsInterperter(astinterpreter.AstInterpreter): pass return flattend_args - def add_languages(self, args): - need_cross_compiler = self.environment.is_cross_build() - for lang in sorted(args, key=compilers.sort_clink): - lang = lang.lower() - if lang not in self.coredata.compilers: - (comp, _) = self.detect_compilers(lang, need_cross_compiler) - if comp is None: - return - for optname in comp.base_options: - if optname in self.coredata.base_options: - continue - oobj = compilers.base_options[optname] - self.coredata.base_options[optname] = oobj - def func_project(self, node, args, kwargs): if len(args) < 1: raise InvalidArguments('Not enough arguments to project(). Needs at least the project name.') + proj_name = args[0] + proj_vers = kwargs.get('version', 'undefined') proj_langs = self.flatten_args(args[1:]) + if isinstance(proj_vers, mparser.ElementaryNode): + proj_vers = proj_vers.value + if not isinstance(proj_vers, str): + proj_vers = 'undefined' + self.project_data = {'descriptive_name': proj_name, 'version': proj_vers} if os.path.exists(self.option_file): oi = optinterpreter.OptionInterpreter(self.subproject) oi.process(self.option_file) self.coredata.merge_user_options(oi.options) - def_opts = kwargs.get('default_options', []) - if isinstance(def_opts, mparser.ArrayNode): - def_opts = [x.value for x in def_opts.args.arguments] - + def_opts = self.flatten_args(kwargs.get('default_options', [])) self.project_default_options = mesonlib.stringlistify(def_opts) self.project_default_options = cdata.create_options_dict(self.project_default_options) self.default_options.update(self.project_default_options) @@ -255,6 +226,7 @@ class BuildoptionsInterperter(astinterpreter.AstInterpreter): if isinstance(spdirname, str): self.subproject_dir = spdirname if not self.is_subproject(): + self.project_data['subprojects'] = [] subprojects_dir = os.path.join(self.source_root, self.subproject_dir) if os.path.isdir(subprojects_dir): for i in os.listdir(subprojects_dir): @@ -265,19 +237,26 @@ class BuildoptionsInterperter(astinterpreter.AstInterpreter): options = {k: v for k, v in self.environment.cmd_line_options.items() if k.startswith('backend_')} self.coredata.set_options(options) - self.add_languages(proj_langs) + self.func_add_languages(None, proj_langs, None) def do_subproject(self, dirname): subproject_dir_abs = os.path.join(self.environment.get_source_dir(), self.subproject_dir) subpr = os.path.join(subproject_dir_abs, dirname) try: - subi = BuildoptionsInterperter(subpr, '', self.backend, cross_file=self.cross_file, subproject=dirname, subproject_dir=self.subproject_dir, env=self.environment) + subi = IntrospectionInterpreter(subpr, '', self.backend, cross_file=self.cross_file, subproject=dirname, subproject_dir=self.subproject_dir, env=self.environment) subi.analyze() + subi.project_data['name'] = dirname + self.project_data['subprojects'] += [subi.project_data] except: return def func_add_languages(self, node, args, kwargs): - return self.add_languages(self.flatten_args(args)) + args = self.flatten_args(args) + need_cross_compiler = self.environment.is_cross_build() + for lang in sorted(args, key=compilers.sort_clink): + lang = lang.lower() + if lang not in self.coredata.compilers: + self.environment.detect_compilers(lang, need_cross_compiler) def is_subproject(self): return self.subproject != '' @@ -292,7 +271,7 @@ def list_buildoptions_from_source(sourcedir, backend, indent): # Make sure that log entries in other parts of meson don't interfere with the JSON output mlog.disable() backend = backends.get_backend_from_name(backend, None) - intr = BuildoptionsInterperter(sourcedir, '', backend.name) + intr = IntrospectionInterpreter(sourcedir, '', backend.name) intr.analyze() # Reenable logging just in case mlog.enable() @@ -438,60 +417,22 @@ def list_projinfo(builddata: build.Build): result['subprojects'] = subprojects return result -class ProjectInfoInterperter(astinterpreter.AstInterpreter): - def __init__(self, source_root, subdir): - super().__init__(source_root, subdir) - self.funcs.update({'project': self.func_project}) - self.project_name = None - self.project_version = None - - def func_project(self, node, args, kwargs): - if len(args) < 1: - raise InvalidArguments('Not enough arguments to project(). Needs at least the project name.') - self.project_name = args[0] - self.project_version = kwargs.get('version', 'undefined') - if isinstance(self.project_version, mparser.ElementaryNode): - self.project_version = self.project_version.value - - def set_variable(self, varname, variable): - pass - - def analyze(self): - self.load_root_meson_file() - self.sanity_check_ast() - self.parse_project() - self.run() - def list_projinfo_from_source(sourcedir, indent): files = find_buildsystem_files_list(sourcedir) + files = [os.path.normpath(x) for x in files] - result = {'buildsystem_files': []} - subprojects = {} - - for f in files: - f = f.replace('\\', '/') - if f == 'meson.build': - interpreter = ProjectInfoInterperter(sourcedir, '') - interpreter.analyze() - version = None - if interpreter.project_version is str: - version = interpreter.project_version - result.update({'version': version, 'descriptive_name': interpreter.project_name}) - result['buildsystem_files'].append(f) - elif f.startswith('subprojects/'): - subproject_id = f.split('/')[1] - subproject = subprojects.setdefault(subproject_id, {'buildsystem_files': []}) - subproject['buildsystem_files'].append(f) - if f.count('/') == 2 and f.endswith('meson.build'): - interpreter = ProjectInfoInterperter(os.path.join(sourcedir, 'subprojects', subproject_id), '') - interpreter.analyze() - subproject.update({'name': subproject_id, 'version': interpreter.project_version, 'descriptive_name': interpreter.project_name}) - else: - result['buildsystem_files'].append(f) + mlog.disable() + intr = IntrospectionInterpreter(sourcedir, '', 'ninja') + intr.analyze() + mlog.enable() - subprojects = [obj for name, obj in subprojects.items()] - result['subprojects'] = subprojects - print(json.dumps(result, indent=indent)) + for i in intr.project_data['subprojects']: + basedir = os.path.join(intr.subproject_dir, i['name']) + i['buildsystem_files'] = [x for x in files if x.startswith(basedir)] + files = [x for x in files if not x.startswith(basedir)] + + intr.project_data['buildsystem_files'] = files + print(json.dumps(intr.project_data, indent=indent)) def run(options): datadir = 'meson-private' @@ -532,13 +473,8 @@ def run(options): return 1 results = [] - toextract = [] intro_types = get_meson_introspection_types() - for i in intro_types.keys(): - if options.all or getattr(options, i, False): - toextract += [i] - # Handle the one option that does not have its own JSON file (meybe deprecate / remove this?) if options.target_files is not None: targets_file = os.path.join(infodir, 'intro-targets.json') @@ -547,7 +483,9 @@ def run(options): results += [('target_files', list_target_files(options.target_files, targets, source_dir))] # Extract introspection information from JSON - for i in toextract: + for i in intro_types.keys(): + if not options.all and not getattr(options, i, False): + continue curr = os.path.join(infodir, 'intro-{}.json'.format(i)) if not os.path.isfile(curr): print('Introspection file {} does not exist.'.format(curr)) -- cgit v1.1