diff options
author | Dylan Baker <dylan@pnwbakers.com> | 2020-06-19 17:01:10 -0700 |
---|---|---|
committer | Dylan Baker <dylan@pnwbakers.com> | 2020-08-01 22:00:06 -0700 |
commit | 591e6e94b9fccfc49ee7093cb21735a27fd64005 (patch) | |
tree | 2a1565a9d32ff9934bc43dfa4ab6c9ba29001f5c | |
parent | 5db3860abf6a27b0dd4653fa8c7143f4a70df7a7 (diff) | |
download | meson-591e6e94b9fccfc49ee7093cb21735a27fd64005.zip meson-591e6e94b9fccfc49ee7093cb21735a27fd64005.tar.gz meson-591e6e94b9fccfc49ee7093cb21735a27fd64005.tar.bz2 |
Put machine file and cmd line parsing in Environment
This creates a full set of option in environment that mirror those in
coredata, this mirroring of the coredata structure is convenient because
lookups int env (such as when initializing compilers) becomes a straight
dict lookup, with no list iteration. It also means that all of the
command line and machine files are read and stored in the correct order
before they're ever accessed, simplifying the logic of using them.
-rw-r--r-- | mesonbuild/ast/introspection.py | 2 | ||||
-rw-r--r-- | mesonbuild/coredata.py | 172 | ||||
-rw-r--r-- | mesonbuild/environment.py | 130 | ||||
-rw-r--r-- | mesonbuild/interpreter.py | 8 | ||||
-rwxr-xr-x | run_unittests.py | 61 |
5 files changed, 238 insertions, 135 deletions
diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py index 142c219..6e6927f 100644 --- a/mesonbuild/ast/introspection.py +++ b/mesonbuild/ast/introspection.py @@ -120,7 +120,7 @@ class IntrospectionInterpreter(AstInterpreter): self.do_subproject(i) self.coredata.init_backend_options(self.backend) - options = {k: v for k, v in self.environment.cmd_line_options.items() if k.startswith('backend_')} + options = {k: v for k, v in self.environment.meson_options.host[''].items() if k.startswith('backend_')} self.coredata.set_options(options) self.func_add_languages(None, proj_langs, None) diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index aaf31aa..724e111 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -20,10 +20,8 @@ from pathlib import PurePath from collections import OrderedDict, defaultdict from .mesonlib import ( MesonException, EnvironmentException, MachineChoice, PerMachine, - OrderedSet, default_libdir, default_libexecdir, default_prefix, - split_args + default_libdir, default_libexecdir, default_prefix, split_args ) -from .envconfig import get_env_var_pair from .wrap import WrapMode import ast import argparse @@ -741,117 +739,54 @@ class CoreData: if not self.is_cross_build(): self.copy_build_options_from_regular_ones() - def set_default_options(self, default_options: T.Mapping[str, str], subproject: str, env: 'Environment') -> None: - # Warn if the user is using two different ways of setting build-type - # options that override each other - if 'buildtype' in env.cmd_line_options and \ - ('optimization' in env.cmd_line_options or 'debug' in env.cmd_line_options): - mlog.warning('Recommend using either -Dbuildtype or -Doptimization + -Ddebug. ' - 'Using both is redundant since they override each other. ' - 'See: https://mesonbuild.com/Builtin-options.html#build-type-options') - - cmd_line_options = OrderedDict() - - from . import optinterpreter - from .compilers import all_languages - if not subproject: - # Set default options as if they were passed to the command line. - # Subprojects can only define default for user options and not yielding - # builtin option. - for k, v in chain(default_options.items(), env.meson_options.host.get('', {}).items()): - cmd_line_options[k] = v - - # compiler options are always per-machine, but not per sub-project - if '' in env.meson_options.build: - for lang in all_languages: - prefix = '{}_'.format(lang) - for k in env.meson_options.build['']: - if k.startswith(prefix): - cmd_line_options['build.{}'.format(k)] = env.meson_options.build[subproject][k] - else: - # If the subproject options comes from a machine file, then we need to - # set the option as subproject:option - for k, v in chain(default_options.items(), env.meson_options.host.get('', {}).items(), - env.meson_options.host.get(subproject, {}).items()): - if (k not in builtin_options or builtin_options[k].yielding) \ - and optinterpreter.is_invalid_name(k, log=False): - continue - cmd_line_options['{}:{}'.format(subproject, k)] = v - cmd_line_options.update(env.cmd_line_options) - env.cmd_line_options = cmd_line_options + def set_default_options(self, default_options: 'T.OrderedDict[str, str]', subproject: str, env: 'Environment') -> None: + def make_key(key: str) -> str: + if subproject: + return '{}:{}'.format(subproject, key) + return key options = OrderedDict() - # load the values for user options out of the appropriate machine file, - # then overload the command line - for k, v in env.user_options.get(subproject, {}).items(): - if subproject: - k = '{}:{}'.format(subproject, k) - options[k] = v - - # Report that [properties]c_args - for lang in all_languages: - for args in ['{}_args'.format(lang), '{}_link_args'.format(lang)]: - msg = ('{} in the [properties] section of the machine file is deprecated, ' - 'use the [built-in options] section.') - if args in env.properties.host or args in env.properties.build: - mlog.deprecation(msg.format(args)) - - # Currently we don't support any options that are both per-subproject - # and per-machine, but when we do this will need to account for that. - # For cross builds we need to get the build specifc options - if env.meson_options.host != env.meson_options.build and subproject in env.meson_options.build: - if subproject: - template = '{s}:build.{k}' + # TODO: validate these + from .compilers import all_languages, base_options + lang_prefixes = tuple('{}_'.format(l) for l in all_languages) + # split arguments that can be set now, and those that cannot so they + # can be set later, when they've been initialized. + for k, v in default_options.items(): + if k.startswith(lang_prefixes): + lang, key = k.split('_', 1) + for machine in MachineChoice: + if key not in env.compiler_options[machine][lang]: + env.compiler_options[machine][lang][key] = v + elif k in base_options: + if not subproject and k not in env.base_options: + env.base_options[k] = v else: - template = 'build.{k}' - for k in builtin_options_per_machine.keys(): - if k in env.meson_options.build[subproject]: - options[template.format(s=subproject, k=k)] = env.meson_options.build[subproject][k] - - # Some options default to environment variables if they are - # unset, set those now. These will either be overwritten - # below, or they won't. These should only be set on the first run. - for for_machine in MachineChoice: - p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), 'PKG_CONFIG_PATH') - if p_env_pair is not None: - p_env_var, p_env = p_env_pair - - # PKG_CONFIG_PATH may contain duplicates, which must be - # removed, else a duplicates-in-array-option warning arises. - p_list = list(OrderedSet(p_env.split(':'))) - - key = 'pkg_config_path' - if for_machine == MachineChoice.BUILD: - key = 'build.' + key - - if env.first_invocation: - options[key] = p_list - elif options.get(key, []) != p_list: - mlog.warning( - p_env_var + - ' environment variable has changed ' - 'between configurations, meson ignores this. ' - 'Use -Dpkg_config_path to change pkg-config search ' - 'path instead.' - ) - - def remove_prefix(text, prefix): - if text.startswith(prefix): - return text[len(prefix):] - return text - - for k, v in cmd_line_options.items(): - if subproject: - if not k.startswith(subproject + ':'): - continue - elif k not in builtin_options.keys() \ - and remove_prefix(k, 'build.') not in builtin_options_per_machine.keys(): - if ':' in k: - continue - if optinterpreter.is_invalid_name(k, log=False): + options[make_key(k)] = v + + for k, v in chain(env.meson_options.host.get('', {}).items(), + env.meson_options.host.get(subproject, {}).items()): + options[make_key(k)] = v + + for k, v in chain(env.meson_options.build.get('', {}).items(), + env.meson_options.build.get(subproject, {}).items()): + if k in builtin_options_per_machine: + options[make_key('build.{}'.format(k))] = v + + options.update({make_key(k): v for k, v in env.user_options.get(subproject, {}).items()}) + + # Some options (namely the compiler options) are not preasant in + # coredata until the compiler is fully initialized. As such, we need to + # put those options into env.meson_options, only if they're not already + # in there, as the machine files and command line have precendence. + for k, v in default_options.items(): + if k in builtin_options and not builtin_options[k].yielding: + continue + for machine in MachineChoice: + if machine is MachineChoice.BUILD and not self.is_cross_build(): continue - options[k] = v + if k not in env.meson_options[machine][subproject]: + env.meson_options[machine][subproject][k] = v self.set_options(options, subproject=subproject) @@ -867,24 +802,19 @@ class CoreData: env.is_cross_build(), env.properties[for_machine]).items(): # prefixed compiler options affect just this machine - opt_prefix = for_machine.get_prefix() - user_k = opt_prefix + lang + '_' + k - if user_k in env.cmd_line_options: - o.set_value(env.cmd_line_options[user_k]) + if k in env.compiler_options[for_machine].get(lang, {}): + o.set_value(env.compiler_options[for_machine][lang][k]) self.compiler_options[for_machine][lang].setdefault(k, o) - def process_new_compiler(self, lang: str, comp: T.Type['Compiler'], env: 'Environment') -> None: + def process_new_compiler(self, lang: str, comp: 'Compiler', env: 'Environment') -> None: from . import compilers self.compilers[comp.for_machine][lang] = comp - enabled_opts = [] for k, o in comp.get_options().items(): # prefixed compiler options affect just this machine - opt_prefix = comp.for_machine.get_prefix() - user_k = opt_prefix + lang + '_' + k - if user_k in env.cmd_line_options: - o.set_value(env.cmd_line_options[user_k]) + if k in env.compiler_options[comp.for_machine].get(lang, {}): + o.set_value(env.compiler_options[comp.for_machine][lang][k]) self.compiler_options[comp.for_machine][lang].setdefault(k, o) enabled_opts = [] @@ -892,8 +822,8 @@ class CoreData: if optname in self.base_options: continue oobj = compilers.base_options[optname] - if optname in env.cmd_line_options: - oobj.set_value(env.cmd_line_options[optname]) + if optname in env.base_options: + oobj.set_value(env.base_options[optname]) enabled_opts.append(optname) self.base_options[optname] = oobj self.emit_base_options_warnings(enabled_opts) diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 7dfffa2..9830b45 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -16,6 +16,7 @@ import os, platform, re, sys, shutil, subprocess import tempfile import shlex import typing as T +import collections from . import coredata from .linkers import ArLinker, ArmarLinker, VisualStudioLinker, DLinker, CcrxLinker, Xc16Linker, C2000Linker, IntelVisualStudioLinker @@ -28,11 +29,13 @@ from . import mlog from .envconfig import ( BinaryTable, MachineInfo, - Properties, known_cpu_families, + Properties, known_cpu_families, get_env_var_pair, ) from . import compilers from .compilers import ( Compiler, + all_languages, + base_options, is_assembly, is_header, is_library, @@ -549,10 +552,10 @@ class Environment: properties = PerMachineDefaultable() # We only need one of these as project options are not per machine - user_options = {} + user_options = collections.defaultdict(dict) # type: T.DefaultDict[str, T.Dict[str, object]] # meson builtin options, as passed through cross or native files - meson_options = PerMachineDefaultable() + meson_options = PerMachineDefaultable() # type: PerMachineDefaultable[T.DefaultDict[str, T.Dict[str, object]]] ## Setup build machine defaults @@ -564,6 +567,13 @@ class Environment: binaries.build = BinaryTable() properties.build = Properties() + # meson base options + _base_options = {} # type: T.Dict[str, object] + + # Per language compiler arguments + compiler_options = PerMachineDefaultable() # type: PerMachineDefaultable[T.DefaultDict[str, T.Dict[str, object]]] + compiler_options.build = collections.defaultdict(dict) + ## Read in native file(s) to override build machine configuration def load_options(tag: str, store: T.Dict[str, T.Any]) -> None: @@ -573,7 +583,44 @@ class Environment: project = section.split(':')[0] else: project = '' - store[project] = config.get(section, {}) + store[project].update(config.get(section, {})) + + def split_base_options(mopts: T.DefaultDict[str, T.Dict[str, object]]) -> None: + for k, v in list(mopts.get('', {}).items()): + if k in base_options: + _base_options[k] = v + del mopts[k] + + lang_prefixes = tuple('{}_'.format(l) for l in all_languages) + def split_compiler_options(mopts: T.DefaultDict[str, T.Dict[str, object]], machine: MachineChoice) -> None: + for k, v in list(mopts.get('', {}).items()): + if k.startswith(lang_prefixes): + lang, key = k.split('_', 1) + if compiler_options[machine] is None: + compiler_options[machine] = collections.defaultdict(dict) + if lang not in compiler_options[machine]: + compiler_options[machine][lang] = collections.defaultdict(dict) + compiler_options[machine][lang][key] = v + del mopts[''][k] + + def move_compiler_options(properties: Properties, compopts: T.Dict[str, T.DefaultDict[str, object]]) -> None: + for k, v in properties.properties.copy().items(): + for lang in all_languages: + if k == '{}_args'.format(lang): + if 'args' not in compopts[lang]: + compopts[lang]['args'] = v + else: + mlog.warning('Ignoring {}_args in [properties] section for those in the [built-in options]'.format(lang)) + elif k == '{}_link_args'.format(lang): + if 'link_args' not in compopts[lang]: + compopts[lang]['link_args'] = v + else: + mlog.warning('Ignoring {}_link_args in [properties] section in favor of the [built-in options] section.') + else: + continue + mlog.deprecation('{} in the [properties] section of the machine file is deprecated, use the [built-in options] section.'.format(k)) + del properties.properties[k] + break if self.coredata.config_files is not None: config = coredata.parse_machine_files(self.coredata.config_files) @@ -584,11 +631,15 @@ class Environment: # the native values if we're doing a cross build if not self.coredata.cross_files: load_options('project options', user_options) - meson_options.build = {} + meson_options.build = collections.defaultdict(dict) if config.get('paths') is not None: mlog.deprecation('The [paths] section is deprecated, use the [built-in options] section instead.') load_options('paths', meson_options.build) load_options('built-in options', meson_options.build) + if not self.coredata.cross_files: + split_base_options(meson_options.build) + split_compiler_options(meson_options.build, MachineChoice.BUILD) + move_compiler_options(properties.build, compiler_options.build) ## Read in cross file(s) to override host machine configuration @@ -601,11 +652,15 @@ class Environment: if 'target_machine' in config: machines.target = MachineInfo.from_literal(config['target_machine']) load_options('project options', user_options) - meson_options.host = {} + meson_options.host = collections.defaultdict(dict) + compiler_options.host = collections.defaultdict(dict) if config.get('paths') is not None: mlog.deprecation('The [paths] section is deprecated, use the [built-in options] section instead.') load_options('paths', meson_options.host) load_options('built-in options', meson_options.host) + split_base_options(meson_options.host) + split_compiler_options(meson_options.host, MachineChoice.HOST) + move_compiler_options(properties.host, compiler_options.host) ## "freeze" now initialized configuration, and "save" to the class. @@ -614,6 +669,67 @@ class Environment: self.properties = properties.default_missing() self.user_options = user_options self.meson_options = meson_options.default_missing() + self.base_options = _base_options + self.compiler_options = compiler_options.default_missing() + + # Some options default to environment variables if they are + # unset, set those now. + + for for_machine in MachineChoice: + p_env_pair = get_env_var_pair(for_machine, self.coredata.is_cross_build(), 'PKG_CONFIG_PATH') + if p_env_pair is not None: + p_env_var, p_env = p_env_pair + + # PKG_CONFIG_PATH may contain duplicates, which must be + # removed, else a duplicates-in-array-option warning arises. + p_list = list(mesonlib.OrderedSet(p_env.split(':'))) + + key = 'pkg_config_path' + + if self.first_invocation: + # Environment variables override config + self.meson_options[for_machine][''][key] = p_list + elif self.meson_options[for_machine][''].get(key, []) != p_list: + mlog.warning( + p_env_var, + 'environment variable does not match configured', + 'between configurations, meson ignores this.', + 'Use -Dpkg_config_path to change pkg-config search', + 'path instead.' + ) + + # Read in command line and populate options + # TODO: validate all of this + all_builtins = set(coredata.builtin_options) | set(coredata.builtin_options_per_machine) | set(coredata.builtin_dir_noprefix_options) + for k, v in options.cmd_line_options.items(): + try: + subproject, k = k.split(':') + except ValueError: + subproject = '' + if k in base_options: + self.base_options[k] = v + elif k.startswith(lang_prefixes): + lang, key = k.split('_', 1) + self.compiler_options.host[lang][key] = v + elif k in all_builtins or k.startswith('backend_'): + self.meson_options.host[subproject][k] = v + elif k.startswith('build.'): + k = k.lstrip('build.') + if k in coredata.builtin_options_per_machine: + if self.meson_options.build is None: + self.meson_options.build = collections.defaultdict(dict) + self.meson_options.build[subproject][k] = v + else: + assert not k.startswith('build.') + self.user_options[subproject][k] = v + + # Warn if the user is using two different ways of setting build-type + # options that override each other + if meson_options.build and 'buildtype' in meson_options.build[''] and \ + ('optimization' in meson_options.build[''] or 'debug' in meson_options.build['']): + mlog.warning('Recommend using either -Dbuildtype or -Doptimization + -Ddebug. ' + 'Using both is redundant since they override each other. ' + 'See: https://mesonbuild.com/Builtin-options.html#build-type-options') exe_wrapper = self.lookup_binary_entry(MachineChoice.HOST, 'exe_wrapper') if exe_wrapper is not None: @@ -622,8 +738,6 @@ class Environment: else: self.exe_wrapper = None - self.cmd_line_options = options.cmd_line_options.copy() - # List of potential compilers. if mesonlib.is_windows(): # Intel C and C++ compiler is icl on Windows, but icc and icpc elsewhere. diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index 317793d..cf7f282 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -2946,6 +2946,7 @@ external dependencies (including libraries) must go to "dependencies".''') if self.is_subproject(): optname = self.subproject + ':' + optname + for opts in [ self.coredata.base_options, compilers.base_options, self.coredata.builtins, dict(self.coredata.get_prefixed_options_per_machine(self.coredata.builtins_per_machine)), @@ -3031,8 +3032,9 @@ external dependencies (including libraries) must go to "dependencies".''') if self.environment.first_invocation: self.coredata.init_backend_options(backend) - options = {k: v for k, v in self.environment.cmd_line_options.items() if k.startswith('backend_')} - self.coredata.set_options(options) + if '' in self.environment.meson_options.host: + options = {k: v for k, v in self.environment.meson_options.host[''].items() if k.startswith('backend_')} + self.coredata.set_options(options) @stringArgs @permittedKwargs(permitted_kwargs['project']) @@ -3065,7 +3067,7 @@ external dependencies (including libraries) must go to "dependencies".''') self.project_default_options = mesonlib.stringlistify(kwargs.get('default_options', [])) self.project_default_options = coredata.create_options_dict(self.project_default_options) if self.environment.first_invocation: - default_options = self.project_default_options + default_options = self.project_default_options.copy() default_options.update(self.default_project_options) self.coredata.init_builtins(self.subproject) else: diff --git a/run_unittests.py b/run_unittests.py index 7bab408..21eabde 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -8030,7 +8030,7 @@ class NativeFileTests(BasePlatformTests): for opt, value in [('testoption', 'some other val'), ('other_one', True), ('combo_opt', 'one'), ('array_opt', ['two']), ('integer_opt', 0)]: - config = self.helper_create_native_file({'project options': {'sub:{}'.format(opt): value}}) + config = self.helper_create_native_file({'sub:project options': {opt: value}}) with self.assertRaises(subprocess.CalledProcessError) as cm: self.init(testcase, extra_args=['--native-file', config]) self.assertRegex(cm.exception.stdout, r'Incorrect value to [a-z]+ option') @@ -8076,6 +8076,19 @@ class NativeFileTests(BasePlatformTests): else: self.fail('Did not find werror in build options?') + def test_builtin_options_env_overrides_conf(self): + testcase = os.path.join(self.common_test_dir, '2 cpp') + config = self.helper_create_native_file({'built-in options': {'pkg_config_path': '/foo'}}) + + self.init(testcase, extra_args=['--native-file', config], override_envvars={'PKG_CONFIG_PATH': '/bar'}) + configuration = self.introspect('--buildoptions') + for each in configuration: + if each['name'] == 'pkg_config_path': + self.assertEqual(each['value'], ['/bar']) + break + else: + self.fail('Did not find pkg_config_path in build options?') + def test_builtin_options_subprojects(self): testcase = os.path.join(self.common_test_dir, '102 subproject subdir') config = self.helper_create_native_file({'built-in options': {'default_library': 'both', 'c_args': ['-Dfoo']}, 'sub:built-in options': {'default_library': 'static'}}) @@ -8185,6 +8198,22 @@ class NativeFileTests(BasePlatformTests): else: self.fail('Did not find bindir in build options?') + def test_builtin_options_paths_legacy(self): + testcase = os.path.join(self.common_test_dir, '1 trivial') + config = self.helper_create_native_file({ + 'built-in options': {'default_library': 'static'}, + 'paths': {'bindir': 'bar'}, + }) + + self.init(testcase, extra_args=['--native-file', config]) + configuration = self.introspect('--buildoptions') + for each in configuration: + if each['name'] == 'bindir': + self.assertEqual(each['value'], 'bar') + break + else: + self.fail('Did not find bindir in build options?') + class CrossFileTests(BasePlatformTests): @@ -8431,7 +8460,15 @@ class CrossFileTests(BasePlatformTests): cross = self.helper_create_cross_file({'built-in options': {'pkg_config_path': '/cross/path', 'cpp_std': 'c++17'}}) native = self.helper_create_cross_file({'built-in options': {'pkg_config_path': '/native/path', 'cpp_std': 'c++14'}}) - self.init(testcase, extra_args=['--cross-file', cross, '--native-file', native]) + # Ensure that PKG_CONFIG_PATH is not set in the environment + with mock.patch.dict('os.environ'): + for k in ['PKG_CONFIG_PATH', 'PKG_CONFIG_PATH_FOR_BUILD']: + try: + del os.environ[k] + except KeyError: + pass + self.init(testcase, extra_args=['--cross-file', cross, '--native-file', native]) + configuration = self.introspect('--buildoptions') found = 0 for each in configuration: @@ -8452,6 +8489,26 @@ class CrossFileTests(BasePlatformTests): break self.assertEqual(found, 4, 'Did not find all sections.') + def test_builtin_options_env_overrides_conf(self): + testcase = os.path.join(self.common_test_dir, '2 cpp') + config = self.helper_create_cross_file({'built-in options': {'pkg_config_path': '/foo'}}) + cross = self.helper_create_cross_file({'built-in options': {'pkg_config_path': '/foo'}}) + + self.init(testcase, extra_args=['--native-file', config, '--cross-file', cross], + override_envvars={'PKG_CONFIG_PATH': '/bar', 'PKG_CONFIG_PATH_FOR_BUILD': '/dir'}) + configuration = self.introspect('--buildoptions') + found = 0 + for each in configuration: + if each['name'] == 'pkg_config_path': + self.assertEqual(each['value'], ['/bar']) + found += 1 + elif each['name'] == 'build.pkg_config_path': + self.assertEqual(each['value'], ['/dir']) + found += 1 + if found == 2: + break + self.assertEqual(found, 2, 'Did not find all sections.') + class TAPParserTests(unittest.TestCase): def assert_test(self, events, **kwargs): |