diff options
| -rw-r--r-- | mesonbuild/ast/introspection.py | 2 | ||||
| -rw-r--r-- | mesonbuild/coredata.py | 172 | ||||
| -rw-r--r-- | mesonbuild/environment.py | 130 | ||||
| -rw-r--r-- | mesonbuild/interpreter.py | 8 | ||||
| -rwxr-xr-x | run_unittests.py | 61 |
5 files changed, 238 insertions, 135 deletions
diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py index 142c219..6e6927f 100644 --- a/mesonbuild/ast/introspection.py +++ b/mesonbuild/ast/introspection.py @@ -120,7 +120,7 @@ class IntrospectionInterpreter(AstInterpreter): self.do_subproject(i) self.coredata.init_backend_options(self.backend) - options = {k: v for k, v in self.environment.cmd_line_options.items() if k.startswith('backend_')} + options = {k: v for k, v in self.environment.meson_options.host[''].items() if k.startswith('backend_')} self.coredata.set_options(options) self.func_add_languages(None, proj_langs, None) diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index aaf31aa..724e111 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -20,10 +20,8 @@ from pathlib import PurePath from collections import OrderedDict, defaultdict from .mesonlib import ( MesonException, EnvironmentException, MachineChoice, PerMachine, - OrderedSet, default_libdir, default_libexecdir, default_prefix, - split_args + default_libdir, default_libexecdir, default_prefix, split_args ) -from .envconfig import get_env_var_pair from .wrap import WrapMode import ast import argparse @@ -741,117 +739,54 @@ class CoreData: if not self.is_cross_build(): self.copy_build_options_from_regular_ones() - def set_default_options(self, default_options: T.Mapping[str, str], subproject: str, env: 'Environment') -> None: - # Warn if the user is using two different ways of setting build-type - # options that override each other - if 'buildtype' in env.cmd_line_options and \ - ('optimization' in env.cmd_line_options or 'debug' in env.cmd_line_options): - mlog.warning('Recommend using either -Dbuildtype or -Doptimization + -Ddebug. ' - 'Using both is redundant since they override each other. ' - 'See: https://mesonbuild.com/Builtin-options.html#build-type-options') - - cmd_line_options = OrderedDict() - - from . import optinterpreter - from .compilers import all_languages - if not subproject: - # Set default options as if they were passed to the command line. - # Subprojects can only define default for user options and not yielding - # builtin option. - for k, v in chain(default_options.items(), env.meson_options.host.get('', {}).items()): - cmd_line_options[k] = v - - # compiler options are always per-machine, but not per sub-project - if '' in env.meson_options.build: - for lang in all_languages: - prefix = '{}_'.format(lang) - for k in env.meson_options.build['']: - if k.startswith(prefix): - cmd_line_options['build.{}'.format(k)] = env.meson_options.build[subproject][k] - else: - # If the subproject options comes from a machine file, then we need to - # set the option as subproject:option - for k, v in chain(default_options.items(), env.meson_options.host.get('', {}).items(), - env.meson_options.host.get(subproject, {}).items()): - if (k not in builtin_options or builtin_options[k].yielding) \ - and optinterpreter.is_invalid_name(k, log=False): - continue - cmd_line_options['{}:{}'.format(subproject, k)] = v - cmd_line_options.update(env.cmd_line_options) - env.cmd_line_options = cmd_line_options + def set_default_options(self, default_options: 'T.OrderedDict[str, str]', subproject: str, env: 'Environment') -> None: + def make_key(key: str) -> str: + if subproject: + return '{}:{}'.format(subproject, key) + return key options = OrderedDict() - # load the values for user options out of the appropriate machine file, - # then overload the command line - for k, v in env.user_options.get(subproject, {}).items(): - if subproject: - k = '{}:{}'.format(subproject, k) - options[k] = v - - # Report that [properties]c_args - for lang in all_languages: - for args in ['{}_args'.format(lang), '{}_link_args'.format(lang)]: - msg = ('{} in the [properties] section of the machine file is deprecated, ' - 'use the [built-in options] section.') - if args in env.properties.host or args in env.properties.build: - mlog.deprecation(msg.format(args)) - - # Currently we don't support any options that are both per-subproject - # and per-machine, but when we do this will need to account for that. - # For cross builds we need to get the build specifc options - if env.meson_options.host != env.meson_options.build and subproject in env.meson_options.build: - if subproject: - template = '{s}:build.{k}' + # TODO: validate these + from .compilers import all_languages, base_options + lang_prefixes = tuple('{}_'.format(l) for l in all_languages) + # split arguments that can be set now, and those that cannot so they + # can be set later, when they've been initialized. + for k, v in default_options.items(): + if k.startswith(lang_prefixes): + lang, key = k.split('_', 1) + for machine in MachineChoice: + if key not in env.compiler_options[machine][lang]: + env.compiler_options[machine][lang][key] = v + elif k in base_options: + if not subproject and k not in env.base_options: + env.base_options[k] = v else: - template = 'build.{k}' - for k in builtin_options_per_machine.keys(): - if k in env.meson_options.build[subproject]: - options[template.format(s=subproject, k=k)] = env.meson_options.build[subproject][k] - - # Some options default to environment variables if they are - # unset, set those now. These will either be overwritten - # below, or they won't. These should only be set on the first run. - for for_machine in MachineChoice: - p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), 'PKG_CONFIG_PATH') - if p_env_pair is not None: - p_env_var, p_env = p_env_pair - - # PKG_CONFIG_PATH may contain duplicates, which must be - # removed, else a duplicates-in-array-option warning arises. - p_list = list(OrderedSet(p_env.split(':'))) - - key = 'pkg_config_path' - if for_machine == MachineChoice.BUILD: - key = 'build.' + key - - if env.first_invocation: - options[key] = p_list - elif options.get(key, []) != p_list: - mlog.warning( - p_env_var + - ' environment variable has changed ' - 'between configurations, meson ignores this. ' - 'Use -Dpkg_config_path to change pkg-config search ' - 'path instead.' - ) - - def remove_prefix(text, prefix): - if text.startswith(prefix): - return text[len(prefix):] - return text - - for k, v in cmd_line_options.items(): - if subproject: - if not k.startswith(subproject + ':'): - continue - elif k not in builtin_options.keys() \ - and remove_prefix(k, 'build.') not in builtin_options_per_machine.keys(): - if ':' in k: - continue - if optinterpreter.is_invalid_name(k, log=False): + options[make_key(k)] = v + + for k, v in chain(env.meson_options.host.get('', {}).items(), + env.meson_options.host.get(subproject, {}).items()): + options[make_key(k)] = v + + for k, v in chain(env.meson_options.build.get('', {}).items(), + env.meson_options.build.get(subproject, {}).items()): + if k in builtin_options_per_machine: + options[make_key('build.{}'.format(k))] = v + + options.update({make_key(k): v for k, v in env.user_options.get(subproject, {}).items()}) + + # Some options (namely the compiler options) are not preasant in + # coredata until the compiler is fully initialized. As such, we need to + # put those options into env.meson_options, only if they're not already + # in there, as the machine files and command line have precendence. + for k, v in default_options.items(): + if k in builtin_options and not builtin_options[k].yielding: + continue + for machine in MachineChoice: + if machine is MachineChoice.BUILD and not self.is_cross_build(): continue - options[k] = v + if k not in env.meson_options[machine][subproject]: + env.meson_options[machine][subproject][k] = v self.set_options(options, subproject=subproject) @@ -867,24 +802,19 @@ class CoreData: env.is_cross_build(), env.properties[for_machine]).items(): # prefixed compiler options affect just this machine - opt_prefix = for_machine.get_prefix() - user_k = opt_prefix + lang + '_' + k - if user_k in env.cmd_line_options: - o.set_value(env.cmd_line_options[user_k]) + if k in env.compiler_options[for_machine].get(lang, {}): + o.set_value(env.compiler_options[for_machine][lang][k]) self.compiler_options[for_machine][lang].setdefault(k, o) - def process_new_compiler(self, lang: str, comp: T.Type['Compiler'], env: 'Environment') -> None: + def process_new_compiler(self, lang: str, comp: 'Compiler', env: 'Environment') -> None: from . import compilers self.compilers[comp.for_machine][lang] = comp - enabled_opts = [] for k, o in comp.get_options().items(): # prefixed compiler options affect just this machine - opt_prefix = comp.for_machine.get_prefix() - user_k = opt_prefix + lang + '_' + k - if user_k in env.cmd_line_options: - o.set_value(env.cmd_line_options[user_k]) + if k in env.compiler_options[comp.for_machine].get(lang, {}): + o.set_value(env.compiler_options[comp.for_machine][lang][k]) self.compiler_options[comp.for_machine][lang].setdefault(k, o) enabled_opts = [] @@ -892,8 +822,8 @@ class CoreData: if optname in self.base_options: continue oobj = compilers.base_options[optname] - if optname in env.cmd_line_options: - oobj.set_value(env.cmd_line_options[optname]) + if optname in env.base_options: + oobj.set_value(env.base_options[optname]) enabled_opts.append(optname) self.base_options[optname] = oobj self.emit_base_options_warnings(enabled_opts) diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 7dfffa2..9830b45 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -16,6 +16,7 @@ import os, platform, re, sys, shutil, subprocess import tempfile import shlex import typing as T +import collections from . import coredata from .linkers import ArLinker, ArmarLinker, VisualStudioLinker, DLinker, CcrxLinker, Xc16Linker, C2000Linker, IntelVisualStudioLinker @@ -28,11 +29,13 @@ from . import mlog from .envconfig import ( BinaryTable, MachineInfo, - Properties, known_cpu_families, + Properties, known_cpu_families, get_env_var_pair, ) from . import compilers from .compilers import ( Compiler, + all_languages, + base_options, is_assembly, is_header, is_library, @@ -549,10 +552,10 @@ class Environment: properties = PerMachineDefaultable() # We only need one of these as project options are not per machine - user_options = {} + user_options = collections.defaultdict(dict) # type: T.DefaultDict[str, T.Dict[str, object]] # meson builtin options, as passed through cross or native files - meson_options = PerMachineDefaultable() + meson_options = PerMachineDefaultable() # type: PerMachineDefaultable[T.DefaultDict[str, T.Dict[str, object]]] ## Setup build machine defaults @@ -564,6 +567,13 @@ class Environment: binaries.build = BinaryTable() properties.build = Properties() + # meson base options + _base_options = {} # type: T.Dict[str, object] + + # Per language compiler arguments + compiler_options = PerMachineDefaultable() # type: PerMachineDefaultable[T.DefaultDict[str, T.Dict[str, object]]] + compiler_options.build = collections.defaultdict(dict) + ## Read in native file(s) to override build machine configuration def load_options(tag: str, store: T.Dict[str, T.Any]) -> None: @@ -573,7 +583,44 @@ class Environment: project = section.split(':')[0] else: project = '' - store[project] = config.get(section, {}) + store[project].update(config.get(section, {})) + + def split_base_options(mopts: T.DefaultDict[str, T.Dict[str, object]]) -> None: + for k, v in list(mopts.get('', {}).items()): + if k in base_options: + _base_options[k] = v + del mopts[k] + + lang_prefixes = tuple('{}_'.format(l) for l in all_languages) + def split_compiler_options(mopts: T.DefaultDict[str, T.Dict[str, object]], machine: MachineChoice) -> None: + for k, v in list(mopts.get('', {}).items()): + if k.startswith(lang_prefixes): + lang, key = k.split('_', 1) + if compiler_options[machine] is None: + compiler_options[machine] = collections.defaultdict(dict) + if lang not in compiler_options[machine]: + compiler_options[machine][lang] = collections.defaultdict(dict) + compiler_options[machine][lang][key] = v + del mopts[''][k] + + def move_compiler_options(properties: Properties, compopts: T.Dict[str, T.DefaultDict[str, object]]) -> None: + for k, v in properties.properties.copy().items(): + for lang in all_languages: + if k == '{}_args'.format(lang): + if 'args' not in compopts[lang]: + compopts[lang]['args'] = v + else: + mlog.warning('Ignoring {}_args in [properties] section for those in the [built-in options]'.format(lang)) + elif k == '{}_link_args'.format(lang): + if 'link_args' not in compopts[lang]: + compopts[lang]['link_args'] = v + else: + mlog.warning('Ignoring {}_link_args in [properties] section in favor of the [built-in options] section.') + else: + continue + mlog.deprecation('{} in the [properties] section of the machine file is deprecated, use the [built-in options] section.'.format(k)) + del properties.properties[k] + break if self.coredata.config_files is not None: config = coredata.parse_machine_files(self.coredata.config_files) @@ -584,11 +631,15 @@ class Environment: # the native values if we're doing a cross build if not self.coredata.cross_files: load_options('project options', user_options) - meson_options.build = {} + meson_options.build = collections.defaultdict(dict) if config.get('paths') is not None: mlog.deprecation('The [paths] section is deprecated, use the [built-in options] section instead.') load_options('paths', meson_options.build) load_options('built-in options', meson_options.build) + if not self.coredata.cross_files: + split_base_options(meson_options.build) + split_compiler_options(meson_options.build, MachineChoice.BUILD) + move_compiler_options(properties.build, compiler_options.build) ## Read in cross file(s) to override host machine configuration @@ -601,11 +652,15 @@ class Environment: if 'target_machine' in config: machines.target = MachineInfo.from_literal(config['target_machine']) load_options('project options', user_options) - meson_options.host = {} + meson_options.host = collections.defaultdict(dict) + compiler_options.host = collections.defaultdict(dict) if config.get('paths') is not None: mlog.deprecation('The [paths] section is deprecated, use the [built-in options] section instead.') load_options('paths', meson_options.host) load_options('built-in options', meson_options.host) + split_base_options(meson_options.host) + split_compiler_options(meson_options.host, MachineChoice.HOST) + move_compiler_options(properties.host, compiler_options.host) ## "freeze" now initialized configuration, and "save" to the class. @@ -614,6 +669,67 @@ class Environment: self.properties = properties.default_missing() self.user_options = user_options self.meson_options = meson_options.default_missing() + self.base_options = _base_options + self.compiler_options = compiler_options.default_missing() + + # Some options default to environment variables if they are + # unset, set those now. + + for for_machine in MachineChoice: + p_env_pair = get_env_var_pair(for_machine, self.coredata.is_cross_build(), 'PKG_CONFIG_PATH') + if p_env_pair is not None: + p_env_var, p_env = p_env_pair + + # PKG_CONFIG_PATH may contain duplicates, which must be + # removed, else a duplicates-in-array-option warning arises. + p_list = list(mesonlib.OrderedSet(p_env.split(':'))) + + key = 'pkg_config_path' + + if self.first_invocation: + # Environment variables override config + self.meson_options[for_machine][''][key] = p_list + elif self.meson_options[for_machine][''].get(key, []) != p_list: + mlog.warning( + p_env_var, + 'environment variable does not match configured', + 'between configurations, meson ignores this.', + 'Use -Dpkg_config_path to change pkg-config search', + 'path instead.' + ) + + # Read in command line and populate options + # TODO: validate all of this + all_builtins = set(coredata.builtin_options) | set(coredata.builtin_options_per_machine) | set(coredata.builtin_dir_noprefix_options) + for k, v in options.cmd_line_options.items(): + try: + subproject, k = k.split(':') + except ValueError: + subproject = '' + if k in base_options: + self.base_options[k] = v + elif k.startswith(lang_prefixes): + lang, key = k.split('_', 1) + self.compiler_options.host[lang][key] = v + elif k in all_builtins or k.startswith('backend_'): + self.meson_options.host[subproject][k] = v + elif k.startswith('build.'): + k = k.lstrip('build.') + if k in coredata.builtin_options_per_machine: + if self.meson_options.build is None: + self.meson_options.build = collections.defaultdict(dict) + self.meson_options.build[subproject][k] = v + else: + assert not k.startswith('build.') + self.user_options[subproject][k] = v + + # Warn if the user is using two different ways of setting build-type + # options that override each other + if meson_options.build and 'buildtype' in meson_options.build[''] and \ + ('optimization' in meson_options.build[''] or 'debug' in meson_options.build['']): + mlog.warning('Recommend using either -Dbuildtype or -Doptimization + -Ddebug. ' + 'Using both is redundant since they override each other. ' + 'See: https://mesonbuild.com/Builtin-options.html#build-type-options') exe_wrapper = self.lookup_binary_entry(MachineChoice.HOST, 'exe_wrapper') if exe_wrapper is not None: @@ -622,8 +738,6 @@ class Environment: else: self.exe_wrapper = None - self.cmd_line_options = options.cmd_line_options.copy() - # List of potential compilers. if mesonlib.is_windows(): # Intel C and C++ compiler is icl on Windows, but icc and icpc elsewhere. diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index 317793d..cf7f282 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -2946,6 +2946,7 @@ external dependencies (including libraries) must go to "dependencies".''') if self.is_subproject(): optname = self.subproject + ':' + optname + for opts in [ self.coredata.base_options, compilers.base_options, self.coredata.builtins, dict(self.coredata.get_prefixed_options_per_machine(self.coredata.builtins_per_machine)), @@ -3031,8 +3032,9 @@ external dependencies (including libraries) must go to "dependencies".''') if self.environment.first_invocation: self.coredata.init_backend_options(backend) - options = {k: v for k, v in self.environment.cmd_line_options.items() if k.startswith('backend_')} - self.coredata.set_options(options) + if '' in self.environment.meson_options.host: + options = {k: v for k, v in self.environment.meson_options.host[''].items() if k.startswith('backend_')} + self.coredata.set_options(options) @stringArgs @permittedKwargs(permitted_kwargs['project']) @@ -3065,7 +3067,7 @@ external dependencies (including libraries) must go to "dependencies".''') self.project_default_options = mesonlib.stringlistify(kwargs.get('default_options', [])) self.project_default_options = coredata.create_options_dict(self.project_default_options) if self.environment.first_invocation: - default_options = self.project_default_options + default_options = self.project_default_options.copy() default_options.update(self.default_project_options) self.coredata.init_builtins(self.subproject) else: diff --git a/run_unittests.py b/run_unittests.py index 7bab408..21eabde 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -8030,7 +8030,7 @@ class NativeFileTests(BasePlatformTests): for opt, value in [('testoption', 'some other val'), ('other_one', True), ('combo_opt', 'one'), ('array_opt', ['two']), ('integer_opt', 0)]: - config = self.helper_create_native_file({'project options': {'sub:{}'.format(opt): value}}) + config = self.helper_create_native_file({'sub:project options': {opt: value}}) with self.assertRaises(subprocess.CalledProcessError) as cm: self.init(testcase, extra_args=['--native-file', config]) self.assertRegex(cm.exception.stdout, r'Incorrect value to [a-z]+ option') @@ -8076,6 +8076,19 @@ class NativeFileTests(BasePlatformTests): else: self.fail('Did not find werror in build options?') + def test_builtin_options_env_overrides_conf(self): + testcase = os.path.join(self.common_test_dir, '2 cpp') + config = self.helper_create_native_file({'built-in options': {'pkg_config_path': '/foo'}}) + + self.init(testcase, extra_args=['--native-file', config], override_envvars={'PKG_CONFIG_PATH': '/bar'}) + configuration = self.introspect('--buildoptions') + for each in configuration: + if each['name'] == 'pkg_config_path': + self.assertEqual(each['value'], ['/bar']) + break + else: + self.fail('Did not find pkg_config_path in build options?') + def test_builtin_options_subprojects(self): testcase = os.path.join(self.common_test_dir, '102 subproject subdir') config = self.helper_create_native_file({'built-in options': {'default_library': 'both', 'c_args': ['-Dfoo']}, 'sub:built-in options': {'default_library': 'static'}}) @@ -8185,6 +8198,22 @@ class NativeFileTests(BasePlatformTests): else: self.fail('Did not find bindir in build options?') + def test_builtin_options_paths_legacy(self): + testcase = os.path.join(self.common_test_dir, '1 trivial') + config = self.helper_create_native_file({ + 'built-in options': {'default_library': 'static'}, + 'paths': {'bindir': 'bar'}, + }) + + self.init(testcase, extra_args=['--native-file', config]) + configuration = self.introspect('--buildoptions') + for each in configuration: + if each['name'] == 'bindir': + self.assertEqual(each['value'], 'bar') + break + else: + self.fail('Did not find bindir in build options?') + class CrossFileTests(BasePlatformTests): @@ -8431,7 +8460,15 @@ class CrossFileTests(BasePlatformTests): cross = self.helper_create_cross_file({'built-in options': {'pkg_config_path': '/cross/path', 'cpp_std': 'c++17'}}) native = self.helper_create_cross_file({'built-in options': {'pkg_config_path': '/native/path', 'cpp_std': 'c++14'}}) - self.init(testcase, extra_args=['--cross-file', cross, '--native-file', native]) + # Ensure that PKG_CONFIG_PATH is not set in the environment + with mock.patch.dict('os.environ'): + for k in ['PKG_CONFIG_PATH', 'PKG_CONFIG_PATH_FOR_BUILD']: + try: + del os.environ[k] + except KeyError: + pass + self.init(testcase, extra_args=['--cross-file', cross, '--native-file', native]) + configuration = self.introspect('--buildoptions') found = 0 for each in configuration: @@ -8452,6 +8489,26 @@ class CrossFileTests(BasePlatformTests): break self.assertEqual(found, 4, 'Did not find all sections.') + def test_builtin_options_env_overrides_conf(self): + testcase = os.path.join(self.common_test_dir, '2 cpp') + config = self.helper_create_cross_file({'built-in options': {'pkg_config_path': '/foo'}}) + cross = self.helper_create_cross_file({'built-in options': {'pkg_config_path': '/foo'}}) + + self.init(testcase, extra_args=['--native-file', config, '--cross-file', cross], + override_envvars={'PKG_CONFIG_PATH': '/bar', 'PKG_CONFIG_PATH_FOR_BUILD': '/dir'}) + configuration = self.introspect('--buildoptions') + found = 0 + for each in configuration: + if each['name'] == 'pkg_config_path': + self.assertEqual(each['value'], ['/bar']) + found += 1 + elif each['name'] == 'build.pkg_config_path': + self.assertEqual(each['value'], ['/dir']) + found += 1 + if found == 2: + break + self.assertEqual(found, 2, 'Did not find all sections.') + class TAPParserTests(unittest.TestCase): def assert_test(self, events, **kwargs): |