From 9f343212e071cfee99e0be76de0f61bec9a6d5f2 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Mon, 11 Jan 2021 11:09:16 -0800 Subject: unittests: Clang on windows can use either ld or link.exe it generally uses the ld style linkers with msys2 and link.exe style linkers otherwise, but anything's possible. --- run_unittests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/run_unittests.py b/run_unittests.py index 27c0677..5118b00 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -2606,8 +2606,10 @@ class AllPlatformTests(BasePlatformTests): if is_osx(): self.assertIsInstance(cc.linker, mesonbuild.linkers.AppleDynamicLinker) elif is_windows(): - # This is clang, not clang-cl - self.assertIsInstance(cc.linker, mesonbuild.linkers.MSVCDynamicLinker) + # This is clang, not clang-cl. This can be either an + # ld-like linker of link.exe-like linker (usually the + # former for msys2, the latter otherwise) + self.assertIsInstance(cc.linker, (mesonbuild.linkers.MSVCDynamicLinker, mesonbuild.linkers.GnuLikeDynamicLinkerMixin)) else: self.assertIsInstance(cc.linker, mesonbuild.linkers.GnuLikeDynamicLinkerMixin) if isinstance(cc, intel): -- cgit v1.1 From a2055ad18bba5704c9cbbe211ed4e46549be05f8 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 14:09:56 -0800 Subject: import MachineChoice from mesonlib there are a couple of places importing it from envconfig, which is not correct. It's defined in mesonlib, and then imported into envconfig. --- mesonbuild/dependencies/hdf5.py | 2 +- mesonbuild/linkers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mesonbuild/dependencies/hdf5.py b/mesonbuild/dependencies/hdf5.py index 21f4e71..ad28975 100644 --- a/mesonbuild/dependencies/hdf5.py +++ b/mesonbuild/dependencies/hdf5.py @@ -30,8 +30,8 @@ import typing as T if T.TYPE_CHECKING: from .base import Dependency - from ..envconfig import MachineChoice from ..environment import Environment + from ..mesonlib import MachineChoice class HDF5PkgConfigDependency(PkgConfigDependency): diff --git a/mesonbuild/linkers.py b/mesonbuild/linkers.py index 141c8fd..8dae94c 100644 --- a/mesonbuild/linkers.py +++ b/mesonbuild/linkers.py @@ -22,8 +22,8 @@ from .envconfig import get_env_var if T.TYPE_CHECKING: from .coredata import KeyedOptionDictType - from .envconfig import MachineChoice from .environment import Environment + from .mesonlib import MachineChoice class StaticLinker: -- cgit v1.1 From 284b89c3217688ac09b79463a20e4099d7c158b9 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 14:10:59 -0800 Subject: dependencies/mpi: Add a type annotation Some change in this series causes mypy to notice that this isn't annotated, and it makes a wrong assumption. --- mesonbuild/dependencies/mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesonbuild/dependencies/mpi.py b/mesonbuild/dependencies/mpi.py index 2e1e764..d4b324c 100644 --- a/mesonbuild/dependencies/mpi.py +++ b/mesonbuild/dependencies/mpi.py @@ -106,7 +106,7 @@ class _MPIConfigToolDependency(ConfigToolDependency): Drop -O2 and everything that is not needed. """ result = [] - multi_args = ('-I', ) + multi_args: T.Tuple[str, ...] = ('-I', ) if self.language == 'fortran': fc = self.env.coredata.compilers[self.for_machine]['fortran'] multi_args += fc.get_module_incdir_args() -- cgit v1.1 From 3949c2a0e0a5fbd9ede8e7a4607220e58934637e Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 8 Dec 2020 10:26:03 -0800 Subject: move CMAKE_PREFIX_PATH env var handling to environment This causes the variable to be read up front and stored, rather than be re-read on each invocation of meson. This does have two slight behavioral changes. First is the obvious one that changing the variable between `meson --reconfigure` invocations has no effect. This is the way PKG_CONFIG_PATH already works. The second change is that CMAKE_PREFIX_PATH the env var is no longer appended to the values set in the machine file or on the command line, and is instead replaced by them. CMAKE_PREFIX_PATH is the only env var in meson that works this way, every other one is replaced not appended, so while this is a behavioral change, I also think its a bug fix. --- mesonbuild/cmake/executor.py | 18 ------------------ mesonbuild/environment.py | 26 ++++++++++++++++++++------ 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/mesonbuild/cmake/executor.py b/mesonbuild/cmake/executor.py index 674b854..e4b85de 100644 --- a/mesonbuild/cmake/executor.py +++ b/mesonbuild/cmake/executor.py @@ -24,7 +24,6 @@ import os from .. import mlog from ..mesonlib import PerMachine, Popen_safe, version_compare, MachineChoice, is_windows, OptionKey -from ..envconfig import get_env_var if T.TYPE_CHECKING: from ..environment import Environment @@ -63,23 +62,6 @@ class CMakeExecutor: return self.prefix_paths = self.environment.coredata.options[OptionKey('cmake_prefix_path', machine=self.for_machine)].value - env_pref_path_raw = get_env_var( - self.for_machine, - self.environment.is_cross_build(), - 'CMAKE_PREFIX_PATH') - if env_pref_path_raw is not None: - env_pref_path = [] # type: T.List[str] - if is_windows(): - # Cannot split on ':' on Windows because its in the drive letter - env_pref_path = env_pref_path_raw.split(os.pathsep) - else: - # https://github.com/mesonbuild/meson/issues/7294 - env_pref_path = re.split(r':|;', env_pref_path_raw) - env_pref_path = [x for x in env_pref_path if x] # Filter out empty strings - if not self.prefix_paths: - self.prefix_paths = [] - self.prefix_paths += env_pref_path - if self.prefix_paths: self.extra_cmake_args += ['-DCMAKE_PREFIX_PATH={}'.format(';'.join(self.prefix_paths))] diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 59675ff..7555be5 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -1,4 +1,4 @@ -# Copyright 2012-2016 The Meson development team +# Copyright 2012-2020 The Meson development team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -579,7 +579,7 @@ class Environment: # Stores machine infos, the only *three* machine one because we have a # target machine info on for the user (Meson never cares about the # target machine.) - machines = PerThreeMachineDefaultable() # type: PerMachineDefaultable[MachineInfo] + machines: PerThreeMachineDefaultable[MachineInfo] = PerThreeMachineDefaultable() # Similar to coredata.compilers, but lower level in that there is no # meta data, only names/paths. @@ -751,14 +751,28 @@ class Environment: def set_default_options_from_env(self) -> None: for for_machine in MachineChoice: - for evar, keyname in [('PKG_CONFIG_PATH', 'pkg_config_path')]: + for evar, keyname in [('PKG_CONFIG_PATH', 'pkg_config_path'), + ('CMAKE_PREFIX_PATH', 'cmake_prefix_path')]: p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) if p_env_pair is not None: _, p_env = p_env_pair - # PKG_CONFIG_PATH may contain duplicates, which must be - # removed, else a duplicates-in-array-option warning arises. - p_list = list(mesonlib.OrderedSet(p_env.split(':'))) + # these may contain duplicates, which must be removed, else + # a duplicates-in-array-option warning arises. + if keyname == 'cmake_prefix_path': + if self.machines[for_machine].is_windows(): + # Cannot split on ':' on Windows because its in the drive letter + _p_env = p_env.split(os.pathsep) + else: + # https://github.com/mesonbuild/meson/issues/7294 + _p_env = re.split(r':|;', p_env) + p_list = list(mesonlib.OrderedSet(_p_env)) + elif keyname == 'pkg_config_path': + p_list = list(mesonlib.OrderedSet(p_env.split(':'))) + else: + raise RuntimeError('Should be unreachable') + p_list = [e for e in p_list if e] # filter out any empty eelemnts + # Take env vars only on first invocation, if the env changes when # reconfiguring it gets ignored. # FIXME: We should remember if we took the value from env to warn -- cgit v1.1 From 7248a64750fc0748f7d86b65e03172f00b515690 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Thu, 10 Dec 2020 18:01:02 -0800 Subject: dependencies: Don't read PKG_CONFIG_PATH from the env again We already read this in, don't read it again. Just rely on the value we have stored. --- mesonbuild/dependencies/base.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/mesonbuild/dependencies/base.py b/mesonbuild/dependencies/base.py index 9ddf6db..9330e46 100644 --- a/mesonbuild/dependencies/base.py +++ b/mesonbuild/dependencies/base.py @@ -32,7 +32,6 @@ from pathlib import Path, PurePath from .. import mlog from .. import mesonlib from ..compilers import clib_langs -from ..envconfig import get_env_var from ..environment import Environment, MachineInfo from ..cmake import CMakeExecutor, CMakeTraceParser, CMakeException, CMakeToolchain, CMakeExecScope, check_cmake_args from ..mesonlib import MachineChoice, MesonException, OrderedSet, PerMachine @@ -317,7 +316,7 @@ class HasNativeKwarg: return MachineChoice.BUILD if kwargs.get('native', False) else MachineChoice.HOST class ExternalDependency(Dependency, HasNativeKwarg): - def __init__(self, type_name, environment, kwargs, language: T.Optional[str] = None): + def __init__(self, type_name, environment: Environment, kwargs, language: T.Optional[str] = None): Dependency.__init__(self, type_name, kwargs) self.env = environment self.name = type_name # default @@ -784,14 +783,7 @@ class PkgConfigDependency(ExternalDependency): # # Only prefix_libpaths are reordered here because there should not be # too many system_libpaths to cause library version issues. - pkg_config_path = get_env_var( - self.for_machine, - self.env.is_cross_build(), - 'PKG_CONFIG_PATH') - if pkg_config_path: - pkg_config_path = pkg_config_path.split(os.pathsep) - else: - pkg_config_path = [] + pkg_config_path: T.List[str] = self.env.coredata.options[OptionKey('pkg_config_path', machine=self.for_machine)].value pkg_config_path = self._convert_mingw_paths(pkg_config_path) prefix_libpaths = sort_libpaths(prefix_libpaths, pkg_config_path) system_libpaths = OrderedSet() -- cgit v1.1 From 4580433b8255b9f69dfbd39a28c21bc1ec785bcf Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 8 Dec 2020 10:33:11 -0800 Subject: rename cflags_mapping to CFLAGS_MAPPING This is PEP8 convention for a const variable. Also, make the type Mapping, which doesn't have mutation methods. This means mypy will warn us if someone tries to change this. --- mesonbuild/compilers/compilers.py | 32 ++++++++++++++----------- mesonbuild/modules/unstable_external_project.py | 8 +++---- run_unittests.py | 2 +- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py index 234ce06..41e1036 100644 --- a/mesonbuild/compilers/compilers.py +++ b/mesonbuild/compilers/compilers.py @@ -91,18 +91,22 @@ languages_using_cppflags = {'c', 'cpp', 'objc', 'objcpp'} # type: T.Set[str] soregex = re.compile(r'.*\.so(\.[0-9]+)?(\.[0-9]+)?(\.[0-9]+)?$') # Environment variables that each lang uses. -cflags_mapping = {'c': 'CFLAGS', - 'cpp': 'CXXFLAGS', - 'cuda': 'CUFLAGS', - 'objc': 'OBJCFLAGS', - 'objcpp': 'OBJCXXFLAGS', - 'fortran': 'FFLAGS', - 'd': 'DFLAGS', - 'vala': 'VALAFLAGS', - 'rust': 'RUSTFLAGS'} # type: T.Dict[str, str] - -cexe_mapping = {'c': 'CC', - 'cpp': 'CXX'} +CFLAGS_MAPPING: T.Mapping[str, str] = { + 'c': 'CFLAGS', + 'cpp': 'CXXFLAGS', + 'cuda': 'CUFLAGS', + 'objc': 'OBJCFLAGS', + 'objcpp': 'OBJCXXFLAGS', + 'fortran': 'FFLAGS', + 'd': 'DFLAGS', + 'vala': 'VALAFLAGS', + 'rust': 'RUSTFLAGS', +} + +CEXE_MAPPING: T.Mapping = { + 'c': 'CC', + 'cpp': 'CXX', +} # All these are only for C-linkable languages; see `clink_langs` above. @@ -1207,13 +1211,13 @@ def get_args_from_envvars(lang: str, Returns a tuple of (compile_flags, link_flags) for the specified language from the inherited environment """ - if lang not in cflags_mapping: + if lang not in CFLAGS_MAPPING: return [], [] compile_flags = [] # type: T.List[str] link_flags = [] # type: T.List[str] - env_compile_flags = get_env_var(for_machine, is_cross, cflags_mapping[lang]) + env_compile_flags = get_env_var(for_machine, is_cross, CFLAGS_MAPPING[lang]) if env_compile_flags is not None: compile_flags += split_args(env_compile_flags) diff --git a/mesonbuild/modules/unstable_external_project.py b/mesonbuild/modules/unstable_external_project.py index 7249078..809b590 100644 --- a/mesonbuild/modules/unstable_external_project.py +++ b/mesonbuild/modules/unstable_external_project.py @@ -23,7 +23,7 @@ from ..mesonlib import (MesonException, Popen_safe, MachineChoice, from ..interpreterbase import InterpreterObject, InterpreterException, FeatureNew from ..interpreterbase import stringArgs, permittedKwargs from ..interpreter import Interpreter, DependencyHolder, InstallDir -from ..compilers.compilers import cflags_mapping, cexe_mapping +from ..compilers.compilers import CFLAGS_MAPPING, CEXE_MAPPING from ..dependencies.base import InternalDependency, PkgConfigDependency from ..environment import Environment from ..mesonlib import OptionKey @@ -110,11 +110,11 @@ class ExternalProject(InterpreterObject): link_args = [] self.run_env = os.environ.copy() for lang, compiler in self.env.coredata.compilers[MachineChoice.HOST].items(): - if any(lang not in i for i in (cexe_mapping, cflags_mapping)): + if any(lang not in i for i in (CEXE_MAPPING, CFLAGS_MAPPING)): continue cargs = self.env.coredata.get_external_args(MachineChoice.HOST, lang) - self.run_env[cexe_mapping[lang]] = self._quote_and_join(compiler.get_exelist()) - self.run_env[cflags_mapping[lang]] = self._quote_and_join(cargs) + self.run_env[CEXE_MAPPING[lang]] = self._quote_and_join(compiler.get_exelist()) + self.run_env[CFLAGS_MAPPING[lang]] = self._quote_and_join(cargs) if not link_exelist: link_exelist = compiler.get_linker_exelist() link_args = self.env.coredata.get_external_link_args(MachineChoice.HOST, lang) diff --git a/run_unittests.py b/run_unittests.py index 5118b00..2f102b4 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -9347,7 +9347,7 @@ def unset_envs(): # For unit tests we must fully control all command lines # so that there are no unexpected changes coming from the # environment, for example when doing a package build. - varnames = ['CPPFLAGS', 'LDFLAGS'] + list(mesonbuild.compilers.compilers.cflags_mapping.values()) + varnames = ['CPPFLAGS', 'LDFLAGS'] + list(mesonbuild.compilers.compilers.CFLAGS_MAPPING.values()) for v in varnames: if v in os.environ: del os.environ[v] -- cgit v1.1 From f202da0689795ba4330581e69a879f089e169f6c Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 8 Dec 2020 10:56:44 -0800 Subject: use PEP8 style naming for LANGUAGES_USING_* as well --- mesonbuild/backend/backends.py | 4 ++-- mesonbuild/compilers/__init__.py | 2 +- mesonbuild/compilers/compilers.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index 9bb870c..6dad189 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -29,7 +29,7 @@ from .. import build from .. import dependencies from .. import mesonlib from .. import mlog -from ..compilers import languages_using_ldflags +from ..compilers import LANGUAGES_USING_LDFLAGS from ..mesonlib import ( File, MachineChoice, MesonException, OptionType, OrderedSet, OptionOverrideProxy, classify_unity_sources, unholder, OptionKey @@ -480,7 +480,7 @@ class Backend: def get_external_rpath_dirs(self, target): dirs = set() args = [] - for lang in languages_using_ldflags: + for lang in LANGUAGES_USING_LDFLAGS: try: args.extend(self.environment.coredata.get_external_link_args(target.for_machine, lang)) except Exception: diff --git a/mesonbuild/compilers/__init__.py b/mesonbuild/compilers/__init__.py index ec5d30f..bda6086 100644 --- a/mesonbuild/compilers/__init__.py +++ b/mesonbuild/compilers/__init__.py @@ -127,7 +127,7 @@ from .compilers import ( is_library, is_known_suffix, lang_suffixes, - languages_using_ldflags, + LANGUAGES_USING_LDFLAGS, sort_clink, ) from .c import ( diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py index 41e1036..23a3bc5 100644 --- a/mesonbuild/compilers/compilers.py +++ b/mesonbuild/compilers/compilers.py @@ -85,9 +85,9 @@ clink_suffixes += ('h', 'll', 's') all_suffixes = set(itertools.chain(*lang_suffixes.values(), clink_suffixes)) # type: T.Set[str] # Languages that should use LDFLAGS arguments when linking. -languages_using_ldflags = {'objcpp', 'cpp', 'objc', 'c', 'fortran', 'd', 'cuda'} # type: T.Set[str] +LANGUAGES_USING_LDFLAGS = {'objcpp', 'cpp', 'objc', 'c', 'fortran', 'd', 'cuda'} # type: T.Set[str] # Languages that should use CPPFLAGS arguments when linking. -languages_using_cppflags = {'c', 'cpp', 'objc', 'objcpp'} # type: T.Set[str] +LANGUAGES_USING_CPPFLAGS = {'c', 'cpp', 'objc', 'objcpp'} # type: T.Set[str] soregex = re.compile(r'.*\.so(\.[0-9]+)?(\.[0-9]+)?(\.[0-9]+)?$') # Environment variables that each lang uses. @@ -1222,7 +1222,7 @@ def get_args_from_envvars(lang: str, compile_flags += split_args(env_compile_flags) # Link flags (same for all languages) - if lang in languages_using_ldflags: + if lang in LANGUAGES_USING_LDFLAGS: link_flags += LinkerEnvVarsMixin.get_args_from_envvars(for_machine, is_cross) if use_linker_args: # When the compiler is used as a wrapper around the linker (such as @@ -1232,7 +1232,7 @@ def get_args_from_envvars(lang: str, link_flags = compile_flags + link_flags # Pre-processor flags for certain languages - if lang in languages_using_cppflags: + if lang in LANGUAGES_USING_CPPFLAGS: env_preproc_flags = get_env_var(for_machine, is_cross, 'CPPFLAGS') if env_preproc_flags is not None: compile_flags += split_args(env_preproc_flags) -- cgit v1.1 From 4b0b44aface1cd9ed073733dd80b040b5aaf4c99 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 8 Dec 2020 11:08:37 -0800 Subject: move handling of CFLAGS and friends to environment This has a bunch of nice features. It obviously centralizes everything, which is nice. It also means that env is only re-read at `meson --wipe`, not `meson --reconfigure`. And it's going to allow more cleanups. --- mesonbuild/compilers/c.py | 3 +- mesonbuild/compilers/compilers.py | 75 +++++++---------------------------- mesonbuild/compilers/cpp.py | 3 +- mesonbuild/coredata.py | 2 +- mesonbuild/environment.py | 83 +++++++++++++++++++++++++-------------- mesonbuild/linkers.py | 17 +------- 6 files changed, 72 insertions(+), 111 deletions(-) diff --git a/mesonbuild/compilers/c.py b/mesonbuild/compilers/c.py index 311e65a..4a55b46 100644 --- a/mesonbuild/compilers/c.py +++ b/mesonbuild/compilers/c.py @@ -17,7 +17,6 @@ import typing as T from .. import coredata from ..mesonlib import MachineChoice, MesonException, mlog, version_compare, OptionKey -from ..linkers import LinkerEnvVarsMixin from .c_function_attributes import C_FUNC_ATTRIBUTES from .mixins.clike import CLikeCompiler from .mixins.ccrx import CcrxCompiler @@ -195,7 +194,7 @@ class AppleClangCCompiler(ClangCCompiler): _C2X_VERSION = '>=11.0.0' -class EmscriptenCCompiler(EmscriptenMixin, LinkerEnvVarsMixin, ClangCCompiler): +class EmscriptenCCompiler(EmscriptenMixin, ClangCCompiler): def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoice, is_cross: bool, info: 'MachineInfo', exe_wrapper: T.Optional['ExternalProgram'] = None, linker: T.Optional['DynamicLinker'] = None, diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py index 23a3bc5..cf9f35b 100644 --- a/mesonbuild/compilers/compilers.py +++ b/mesonbuild/compilers/compilers.py @@ -22,13 +22,9 @@ from functools import lru_cache from .. import coredata from .. import mlog from .. import mesonlib -from ..linkers import LinkerEnvVarsMixin from ..mesonlib import ( EnvironmentException, MachineChoice, MesonException, - Popen_safe, split_args, LibType, TemporaryDirectoryWinProof, OptionKey, -) -from ..envconfig import ( - get_env_var + Popen_safe, LibType, TemporaryDirectoryWinProof, OptionKey, ) from ..arglist import CompilerArgs @@ -105,7 +101,7 @@ CFLAGS_MAPPING: T.Mapping[str, str] = { CEXE_MAPPING: T.Mapping = { 'c': 'CC', - 'cpp': 'CXX', + 'cpp': 'CXX', } # All these are only for C-linkable languages; see `clink_langs` above. @@ -591,11 +587,6 @@ class Compiler(metaclass=abc.ABCMeta): """ return [] - def get_linker_args_from_envvars(self, - for_machine: MachineChoice, - is_cross: bool) -> T.List[str]: - return self.linker.get_args_from_envvars(for_machine, is_cross) - def get_options(self) -> 'KeyedOptionDictType': return {} @@ -1203,68 +1194,32 @@ class Compiler(metaclass=abc.ABCMeta): def get_prelink_args(self, prelink_name: str, obj_list: T.List[str]) -> T.List[str]: raise EnvironmentException('{} does not know how to do prelinking.'.format(self.id)) -def get_args_from_envvars(lang: str, - for_machine: MachineChoice, - is_cross: bool, - use_linker_args: bool) -> T.Tuple[T.List[str], T.List[str]]: - """ - Returns a tuple of (compile_flags, link_flags) for the specified language - from the inherited environment - """ - if lang not in CFLAGS_MAPPING: - return [], [] - - compile_flags = [] # type: T.List[str] - link_flags = [] # type: T.List[str] - - env_compile_flags = get_env_var(for_machine, is_cross, CFLAGS_MAPPING[lang]) - if env_compile_flags is not None: - compile_flags += split_args(env_compile_flags) - - # Link flags (same for all languages) - if lang in LANGUAGES_USING_LDFLAGS: - link_flags += LinkerEnvVarsMixin.get_args_from_envvars(for_machine, is_cross) - if use_linker_args: - # When the compiler is used as a wrapper around the linker (such as - # with GCC and Clang), the compile flags can be needed while linking - # too. This is also what Autotools does. However, we don't want to do - # this when the linker is stand-alone such as with MSVC C/C++, etc. - link_flags = compile_flags + link_flags - - # Pre-processor flags for certain languages - if lang in LANGUAGES_USING_CPPFLAGS: - env_preproc_flags = get_env_var(for_machine, is_cross, 'CPPFLAGS') - if env_preproc_flags is not None: - compile_flags += split_args(env_preproc_flags) - - return compile_flags, link_flags - def get_global_options(lang: str, comp: T.Type[Compiler], for_machine: MachineChoice, - is_cross: bool) -> 'KeyedOptionDictType': + env: 'Environment') -> 'KeyedOptionDictType': """Retreive options that apply to all compilers for a given language.""" description = 'Extra arguments passed to the {}'.format(lang) argkey = OptionKey('args', lang=lang, machine=for_machine) largkey = argkey.evolve('link_args') + + # We shouldn't need listify here, but until we have a separate + # linker-driver representation and can have that do the combine we have to + # do it htis way. + compile_args = mesonlib.listify(env.options.get(argkey, [])) + link_args = mesonlib.listify(env.options.get(largkey, [])) + + if comp.INVOKES_LINKER: + link_args = compile_args + link_args + opts: 'KeyedOptionDictType' = { argkey: coredata.UserArrayOption( description + ' compiler', - [], split_args=True, user_input=True, allow_dups=True), + compile_args, split_args=True, user_input=True, allow_dups=True), largkey: coredata.UserArrayOption( description + ' linker', - [], split_args=True, user_input=True, allow_dups=True), + link_args, split_args=True, user_input=True, allow_dups=True), } - # Get from env vars. - compile_args, link_args = get_args_from_envvars( - lang, - for_machine, - is_cross, - comp.INVOKES_LINKER) - - opts[argkey].set_value(compile_args) - opts[largkey].set_value(link_args) - return opts diff --git a/mesonbuild/compilers/cpp.py b/mesonbuild/compilers/cpp.py index 2e94e48..ebe3a4f 100644 --- a/mesonbuild/compilers/cpp.py +++ b/mesonbuild/compilers/cpp.py @@ -21,7 +21,6 @@ from .. import coredata from .. import mlog from ..mesonlib import MesonException, MachineChoice, version_compare, OptionKey -from ..linkers import LinkerEnvVarsMixin from .compilers import ( gnu_winlibs, msvc_winlibs, @@ -256,7 +255,7 @@ class AppleClangCPPCompiler(ClangCPPCompiler): return ['-lc++'] -class EmscriptenCPPCompiler(EmscriptenMixin, LinkerEnvVarsMixin, ClangCPPCompiler): +class EmscriptenCPPCompiler(EmscriptenMixin, ClangCPPCompiler): def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoice, is_cross: bool, info: 'MachineInfo', exe_wrapper: T.Optional['ExternalProgram'] = None, linker: T.Optional['DynamicLinker'] = None, diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index cda0566..f2aba80 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -759,7 +759,7 @@ class CoreData: for_machine: MachineChoice, env: 'Environment') -> None: """Add global language arguments that are needed before compiler/linker detection.""" from .compilers import compilers - options = compilers.get_global_options(lang, comp, for_machine, env.is_cross_build()) + options = compilers.get_global_options(lang, comp, for_machine, env) self.add_compiler_options(options, lang, for_machine, env) def process_new_compiler(self, lang: str, comp: 'Compiler', env: 'Environment') -> None: diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 7555be5..12b9dbb 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import itertools import os, platform, re, sys, shutil, subprocess import tempfile import shlex @@ -605,7 +606,7 @@ class Environment: # # Note that order matters because of 'buildtype', if it is after # 'optimization' and 'debug' keys, it override them. - self.options: T.MutableMapping[OptionKey, str] = collections.OrderedDict() + self.options: T.MutableMapping[OptionKey, T.Union[str, T.List[str]]] = collections.OrderedDict() ## Read in native file(s) to override build machine configuration @@ -750,36 +751,58 @@ class Environment: self.options[key] = v def set_default_options_from_env(self) -> None: - for for_machine in MachineChoice: - for evar, keyname in [('PKG_CONFIG_PATH', 'pkg_config_path'), - ('CMAKE_PREFIX_PATH', 'cmake_prefix_path')]: - p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) - if p_env_pair is not None: - _, p_env = p_env_pair - - # these may contain duplicates, which must be removed, else - # a duplicates-in-array-option warning arises. - if keyname == 'cmake_prefix_path': - if self.machines[for_machine].is_windows(): - # Cannot split on ':' on Windows because its in the drive letter - _p_env = p_env.split(os.pathsep) - else: - # https://github.com/mesonbuild/meson/issues/7294 - _p_env = re.split(r':|;', p_env) - p_list = list(mesonlib.OrderedSet(_p_env)) - elif keyname == 'pkg_config_path': - p_list = list(mesonlib.OrderedSet(p_env.split(':'))) + opts: T.List[T.Tuple[str, str]] = ( + [(v, f'{k}_args') for k, v in compilers.compilers.CFLAGS_MAPPING.items()] + + [ + ('PKG_CONFIG_PATH', 'pkg_config_path'), + ('CMAKE_PREFIX_PATH', 'cmake_prefix_path'), + ('LDFLAGS', 'ldflags'), + ('CPPFLAGS', 'cppflags'), + ] + ) + + for (evar, keyname), for_machine in itertools.product(opts, MachineChoice): + p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) + if p_env_pair is not None: + _, p_env = p_env_pair + + # these may contain duplicates, which must be removed, else + # a duplicates-in-array-option warning arises. + if keyname == 'cmake_prefix_path': + if self.machines[for_machine].is_windows(): + # Cannot split on ':' on Windows because its in the drive letter + _p_env = p_env.split(os.pathsep) else: - raise RuntimeError('Should be unreachable') - p_list = [e for e in p_list if e] # filter out any empty eelemnts - - # Take env vars only on first invocation, if the env changes when - # reconfiguring it gets ignored. - # FIXME: We should remember if we took the value from env to warn - # if it changes on future invocations. - if self.first_invocation: - key = OptionKey(keyname, machine=for_machine) - self.options.setdefault(key, p_list) + # https://github.com/mesonbuild/meson/issues/7294 + _p_env = re.split(r':|;', p_env) + p_list = list(mesonlib.OrderedSet(_p_env)) + elif keyname == 'pkg_config_path': + p_list = list(mesonlib.OrderedSet(p_env.split(':'))) + else: + p_list = split_args(p_env) + p_list = [e for e in p_list if e] # filter out any empty eelemnts + + # Take env vars only on first invocation, if the env changes when + # reconfiguring it gets ignored. + # FIXME: We should remember if we took the value from env to warn + # if it changes on future invocations. + if self.first_invocation: + if keyname == 'ldflags': + key = OptionKey('link_args', machine=for_machine, lang='c') # needs a language to initialize properly + for lang in compilers.compilers.LANGUAGES_USING_LDFLAGS: + key = key.evolve(lang=lang) + v = mesonlib.listify(self.options.get(key, [])) + self.options.setdefault(key, v + p_list) + elif keyname == 'cppflags': + key = OptionKey('args', machine=for_machine, lang='c') + for lang in compilers.compilers.LANGUAGES_USING_CPPFLAGS: + key = key.evolve(lang=lang) + v = mesonlib.listify(self.options.get(key, [])) + self.options.setdefault(key, v + p_list) + else: + key = OptionKey.from_string(keyname).evolve(machine=for_machine) + v = mesonlib.listify(self.options.get(key, [])) + self.options.setdefault(key, v + p_list) def create_new_coredata(self, options: 'argparse.Namespace') -> None: # WARNING: Don't use any values from coredata in __init__. It gets diff --git a/mesonbuild/linkers.py b/mesonbuild/linkers.py index 8dae94c..86e6aac 100644 --- a/mesonbuild/linkers.py +++ b/mesonbuild/linkers.py @@ -18,7 +18,6 @@ import typing as T from . import mesonlib from .arglist import CompilerArgs -from .envconfig import get_env_var if T.TYPE_CHECKING: from .coredata import KeyedOptionDictType @@ -301,21 +300,7 @@ def evaluate_rpath(p: str, build_dir: str, from_dir: str) -> str: else: return os.path.relpath(os.path.join(build_dir, p), os.path.join(build_dir, from_dir)) - -class LinkerEnvVarsMixin(metaclass=abc.ABCMeta): - - """Mixin reading LDFLAGS from the environment.""" - - @staticmethod - def get_args_from_envvars(for_machine: mesonlib.MachineChoice, - is_cross: bool) -> T.List[str]: - raw_value = get_env_var(for_machine, is_cross, 'LDFLAGS') - if raw_value is not None: - return mesonlib.split_args(raw_value) - else: - return [] - -class DynamicLinker(LinkerEnvVarsMixin, metaclass=abc.ABCMeta): +class DynamicLinker(metaclass=abc.ABCMeta): """Base class for dynamic linkers.""" -- cgit v1.1 From 38c7a7590c25426dfd2d480d805d570d7e645096 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 11:29:43 -0800 Subject: pull env to program mappings out of BinaryType class These really aren't pivotal to that class, and they're used outside of it. In a follow up patch they're not going to be used inside it at all. --- mesonbuild/envconfig.py | 90 ++++++++++++++++++++++++------------------------- run_unittests.py | 8 ++--- 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/mesonbuild/envconfig.py b/mesonbuild/envconfig.py index db1041f..642aab3 100644 --- a/mesonbuild/envconfig.py +++ b/mesonbuild/envconfig.py @@ -85,6 +85,49 @@ CPU_FAMILES_64_BIT = [ 'x86_64', ] +# Map from language identifiers to environment variables. +ENV_VAR_PROG_MAP: T.Mapping[str, str] = { + # Compilers + 'c': 'CC', + 'cpp': 'CXX', + 'cs': 'CSC', + 'd': 'DC', + 'fortran': 'FC', + 'objc': 'OBJC', + 'objcpp': 'OBJCXX', + 'rust': 'RUSTC', + 'vala': 'VALAC', + + # Linkers + 'c_ld': 'CC_LD', + 'cpp_ld': 'CXX_LD', + 'd_ld': 'DC_LD', + 'fortran_ld': 'FC_LD', + 'objc_ld': 'OBJC_LD', + 'objcpp_ld': 'OBJCXX_LD', + 'rust_ld': 'RUSTC_LD', + + # Binutils + 'strip': 'STRIP', + 'ar': 'AR', + 'windres': 'WINDRES', + + # Other tools + 'cmake': 'CMAKE', + 'qmake': 'QMAKE', + 'pkgconfig': 'PKG_CONFIG', + 'make': 'MAKE', +} + +# Deprecated environment variables mapped from the new variable to the old one +# Deprecated in 0.54.0 +DEPRECATED_ENV_PROG_MAP: T.Mapping[str, str] = { + 'DC_LD': 'D_LD', + 'FC_LD': 'F_LD', + 'RUSTC_LD': 'RUST_LD', + 'OBJCXX_LD': 'OBJCPP_LD', +} + class CMakeSkipCompilerTest(Enum): ALWAYS = 'always' NEVER = 'never' @@ -363,49 +406,6 @@ class BinaryTable: 'Invalid type {!r} for binary {!r} in cross file' ''.format(command, name)) - # Map from language identifiers to environment variables. - evarMap = { - # Compilers - 'c': 'CC', - 'cpp': 'CXX', - 'cs': 'CSC', - 'd': 'DC', - 'fortran': 'FC', - 'objc': 'OBJC', - 'objcpp': 'OBJCXX', - 'rust': 'RUSTC', - 'vala': 'VALAC', - - # Linkers - 'c_ld': 'CC_LD', - 'cpp_ld': 'CXX_LD', - 'd_ld': 'DC_LD', - 'fortran_ld': 'FC_LD', - 'objc_ld': 'OBJC_LD', - 'objcpp_ld': 'OBJCXX_LD', - 'rust_ld': 'RUSTC_LD', - - # Binutils - 'strip': 'STRIP', - 'ar': 'AR', - 'windres': 'WINDRES', - - # Other tools - 'cmake': 'CMAKE', - 'qmake': 'QMAKE', - 'pkgconfig': 'PKG_CONFIG', - 'make': 'MAKE', - } # type: T.Dict[str, str] - - # Deprecated environment variables mapped from the new variable to the old one - # Deprecated in 0.54.0 - DEPRECATION_MAP = { - 'DC_LD': 'D_LD', - 'FC_LD': 'F_LD', - 'RUSTC_LD': 'RUST_LD', - 'OBJCXX_LD': 'OBJCPP_LD', - } # type: T.Dict[str, str] - @staticmethod def detect_ccache() -> T.List[str]: try: @@ -442,11 +442,11 @@ class BinaryTable: if raw_command is not None: command = mesonlib.stringlistify(raw_command) break # found - evar = self.evarMap.get(name) + evar = ENV_VAR_PROG_MAP.get(name) if evar is not None: raw_command = get_env_var(for_machine, is_cross, evar) if raw_command is None: - deprecated = self.DEPRECATION_MAP.get(evar) + deprecated = DEPRECATED_ENV_PROG_MAP.get(evar) if deprecated is not None: raw_command = get_env_var(for_machine, is_cross, deprecated) if raw_command is not None: diff --git a/run_unittests.py b/run_unittests.py index 2f102b4..cf78e3e 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -5689,12 +5689,12 @@ class WindowsTests(BasePlatformTests): def _check_ld(self, name: str, lang: str, expected: str) -> None: if not shutil.which(name): raise unittest.SkipTest('Could not find {}.'.format(name)) - envvars = [mesonbuild.envconfig.BinaryTable.evarMap['{}_ld'.format(lang)]] + envvars = [mesonbuild.envconfig.ENV_VAR_PROG_MAP['{}_ld'.format(lang)]] # Also test a deprecated variable if there is one. - if envvars[0] in mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP: + if envvars[0] in mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP: envvars.append( - mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP[envvars[0]]) + mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP[envvars[0]]) for envvar in envvars: with mock.patch.dict(os.environ, {envvar: name}): @@ -7293,7 +7293,7 @@ class LinuxlikeTests(BasePlatformTests): raise unittest.SkipTest('Solaris currently cannot override the linker.') if not shutil.which(check): raise unittest.SkipTest('Could not find {}.'.format(check)) - envvars = [mesonbuild.envconfig.BinaryTable.evarMap['{}_ld'.format(lang)]] + envvars = [mesonbuild.envconfig.BinaryTable.ENV_VAR_PROG_MAP['{}_ld'.format(lang)]] # Also test a deprecated variable if there is one. if envvars[0] in mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP: -- cgit v1.1 From e7a5c75285ce63a7197cd82e893450eb9bb68b6c Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 12:30:19 -0800 Subject: Move BinaryTable environment lookups to Environment This means that all the env lookups are done once, at initial configure time. This has all of the expected advantages. --- mesonbuild/envconfig.py | 63 ++++++++++++++++------------------------------- mesonbuild/environment.py | 26 ++++++++++++++----- run_unittests.py | 33 ++++++++++++------------- 3 files changed, 57 insertions(+), 65 deletions(-) diff --git a/mesonbuild/envconfig.py b/mesonbuild/envconfig.py index 642aab3..69b6678 100644 --- a/mesonbuild/envconfig.py +++ b/mesonbuild/envconfig.py @@ -23,6 +23,9 @@ from pathlib import Path _T = T.TypeVar('_T') +if T.TYPE_CHECKING: + from .environment import Environment + # These classes contains all the data pulled from configuration files (native # and cross file currently), and also assists with the reading environment @@ -122,10 +125,10 @@ ENV_VAR_PROG_MAP: T.Mapping[str, str] = { # Deprecated environment variables mapped from the new variable to the old one # Deprecated in 0.54.0 DEPRECATED_ENV_PROG_MAP: T.Mapping[str, str] = { - 'DC_LD': 'D_LD', - 'FC_LD': 'F_LD', - 'RUSTC_LD': 'RUST_LD', - 'OBJCXX_LD': 'OBJCPP_LD', + 'd_ld': 'D_LD', + 'fortran_ld': 'F_LD', + 'rust_ld': 'RUST_LD', + 'objcpp_ld': 'OBJCPP_LD', } class CMakeSkipCompilerTest(Enum): @@ -394,17 +397,18 @@ class MachineInfo: return self.is_windows() or self.is_cygwin() class BinaryTable: + def __init__( self, binaries: T.Optional[T.Dict[str, T.Union[str, T.List[str]]]] = None, ): - self.binaries = binaries or {} # type: T.Dict[str, T.Union[str, T.List[str]]] - for name, command in self.binaries.items(): - if not isinstance(command, (list, str)): - # TODO generalize message - raise mesonlib.MesonException( - 'Invalid type {!r} for binary {!r} in cross file' - ''.format(command, name)) + self.binaries: T.Dict[str, T.List[str]] = {} + if binaries: + for name, command in binaries.items(): + if not isinstance(command, (list, str)): + raise mesonlib.MesonException( + f'Invalid type {command!r} for entry {name!r} in cross file') + self.binaries[name] = mesonlib.listify(command) @staticmethod def detect_ccache() -> T.List[str]: @@ -426,42 +430,17 @@ class BinaryTable: # Return value has to be a list of compiler 'choices' return compiler, ccache - def lookup_entry(self, - for_machine: MachineChoice, - is_cross: bool, - name: str) -> T.Optional[T.List[str]]: + def lookup_entry(self, name: str) -> T.Optional[T.List[str]]: """Lookup binary in cross/native file and fallback to environment. Returns command with args as list if found, Returns `None` if nothing is found. """ - # Try explicit map, don't fall back on env var - # Try explict map, then env vars - for _ in [()]: # a trick to get `break` - raw_command = self.binaries.get(name) - if raw_command is not None: - command = mesonlib.stringlistify(raw_command) - break # found - evar = ENV_VAR_PROG_MAP.get(name) - if evar is not None: - raw_command = get_env_var(for_machine, is_cross, evar) - if raw_command is None: - deprecated = DEPRECATED_ENV_PROG_MAP.get(evar) - if deprecated is not None: - raw_command = get_env_var(for_machine, is_cross, deprecated) - if raw_command is not None: - mlog.deprecation( - 'The', deprecated, 'environment variable is deprecated in favor of', - evar, once=True) - if raw_command is not None: - command = split_args(raw_command) - break # found - command = None - - - # Do not return empty or blank string entries - if command is not None and (len(command) == 0 or len(command[0].strip()) == 0): - command = None + command = self.binaries.get(name) + if not command: + return None + elif not command[0].strip(): + return None return command class CMakeVariables: diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 12b9dbb..6275da0 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -29,7 +29,7 @@ from .mesonlib import ( from . import mlog from .envconfig import ( - BinaryTable, MachineInfo, + BinaryTable, ENV_VAR_PROG_MAP, MachineInfo, Properties, known_cpu_families, get_env_var_pair, CMakeVariables, ) @@ -129,6 +129,7 @@ from .compilers import ( VisualStudioCCompiler, VisualStudioCPPCompiler, ) +from mesonbuild import envconfig if T.TYPE_CHECKING: from configparser import ConfigParser @@ -654,6 +655,7 @@ class Environment: # Take default value from env if not set in cross/native files or command line. self.set_default_options_from_env() + self._set_default_binaries_from_env() # Warn if the user is using two different ways of setting build-type # options that override each other @@ -804,6 +806,21 @@ class Environment: v = mesonlib.listify(self.options.get(key, [])) self.options.setdefault(key, v + p_list) + def _set_default_binaries_from_env(self) -> None: + """Set default binaries from the environment. + + For example, pkg-config can be set via PKG_CONFIG, or in the machine + file. We want to set the default to the env variable. + """ + opts = itertools.chain(envconfig.DEPRECATED_ENV_PROG_MAP.items(), + envconfig.ENV_VAR_PROG_MAP.items()) + + for (name, evar), for_machine in itertools.product(opts, MachineChoice): + p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) + if p_env_pair is not None: + _, p_env = p_env_pair + self.binaries[for_machine].binaries.setdefault(name, mesonlib.split_args(p_env)) + def create_new_coredata(self, options: 'argparse.Namespace') -> None: # WARNING: Don't use any values from coredata in __init__. It gets # re-initialized with project options by the interpreter during @@ -853,11 +870,8 @@ class Environment: def is_library(self, fname): return is_library(fname) - def lookup_binary_entry(self, for_machine: MachineChoice, name: str): - return self.binaries[for_machine].lookup_entry( - for_machine, - self.is_cross_build(), - name) + def lookup_binary_entry(self, for_machine: MachineChoice, name: str) -> T.List[str]: + return self.binaries[for_machine].lookup_entry(name) @staticmethod def get_gnu_compiler_defines(compiler): diff --git a/run_unittests.py b/run_unittests.py index cf78e3e..bbcdb5f 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -2637,17 +2637,16 @@ class AllPlatformTests(BasePlatformTests): # something like `ccache gcc -pipe` or `distcc ccache gcc` works. wrapper = os.path.join(testdir, 'compiler wrapper.py') wrappercc = python_command + [wrapper] + cc.get_exelist() + ['-DSOME_ARG'] - wrappercc_s = '' - for w in wrappercc: - wrappercc_s += quote_arg(w) + ' ' - os.environ[evar] = wrappercc_s - wcc = getattr(env, 'detect_{}_compiler'.format(lang))(MachineChoice.HOST) + os.environ[evar] = ' '.join(quote_arg(w) for w in wrappercc) + # Check static linker too wrapperlinker = python_command + [wrapper] + linker.get_exelist() + linker.get_always_args() - wrapperlinker_s = '' - for w in wrapperlinker: - wrapperlinker_s += quote_arg(w) + ' ' - os.environ['AR'] = wrapperlinker_s + os.environ['AR'] = ' '.join(quote_arg(w) for w in wrapperlinker) + + # Need a new env to re-run environment loading + env = get_fake_env(testdir, self.builddir, self.prefix) + + wcc = getattr(env, 'detect_{}_compiler'.format(lang))(MachineChoice.HOST) wlinker = env.detect_static_linker(wcc) # Pop it so we don't use it for the next detection evalue = os.environ.pop('AR') @@ -5692,9 +5691,9 @@ class WindowsTests(BasePlatformTests): envvars = [mesonbuild.envconfig.ENV_VAR_PROG_MAP['{}_ld'.format(lang)]] # Also test a deprecated variable if there is one. - if envvars[0] in mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP: + if f'{lang}_ld' in mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP: envvars.append( - mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP[envvars[0]]) + mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP[f'{lang}_ld']) for envvar in envvars: with mock.patch.dict(os.environ, {envvar: name}): @@ -7293,12 +7292,12 @@ class LinuxlikeTests(BasePlatformTests): raise unittest.SkipTest('Solaris currently cannot override the linker.') if not shutil.which(check): raise unittest.SkipTest('Could not find {}.'.format(check)) - envvars = [mesonbuild.envconfig.BinaryTable.ENV_VAR_PROG_MAP['{}_ld'.format(lang)]] + envvars = [mesonbuild.envconfig.ENV_VAR_PROG_MAP['{}_ld'.format(lang)]] # Also test a deprecated variable if there is one. - if envvars[0] in mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP: + if f'{lang}_ld' in mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP: envvars.append( - mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP[envvars[0]]) + mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP[f'{lang}_ld']) for envvar in envvars: with mock.patch.dict(os.environ, {envvar: name}): @@ -8274,7 +8273,7 @@ class NativeFileTests(BasePlatformTests): return 'gfortran', 'gcc' self.helper_for_compiler('fortran', cb) - def _single_implementation_compiler(self, lang, binary, version_str, version): + def _single_implementation_compiler(self, lang: str, binary: str, version_str: str, version: str) -> None: """Helper for languages with a single (supported) implementation. Builds a wrapper around the compiler to override the version. @@ -8283,7 +8282,7 @@ class NativeFileTests(BasePlatformTests): env = get_fake_env() getter = getattr(env, 'detect_{}_compiler'.format(lang)) getter = functools.partial(getter, MachineChoice.HOST) - env.binaries.host.binaries[lang] = wrapper + env.binaries.host.binaries[lang] = [wrapper] compiler = getter() self.assertEqual(compiler.version, version) @@ -8310,7 +8309,7 @@ class NativeFileTests(BasePlatformTests): 'swiftc', version='Swift 1.2345', outfile='stderr', extra_args={'Xlinker': 'macosx_version. PROJECT:ld - 1.2.3'}) env = get_fake_env() - env.binaries.host.binaries['swift'] = wrapper + env.binaries.host.binaries['swift'] = [wrapper] compiler = env.detect_swift_compiler(MachineChoice.HOST) self.assertEqual(compiler.version, '1.2345') -- cgit v1.1 From f3fcbba1f89f0b2a323dd596a3caf4fce5a1611e Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 13:27:26 -0800 Subject: boost: default machine file properties to env var values This both moves the env reading to configuration time, which is useful, and also simplifies the implementation of the boost dependency. The simplification comes from being able to delete basically duplicated code since the values will be in the Properties if they exist at all. --- mesonbuild/dependencies/boost.py | 65 ++++++++++------------------------------ mesonbuild/envconfig.py | 8 ----- mesonbuild/environment.py | 21 +++++++++++++ run_unittests.py | 2 +- 4 files changed, 37 insertions(+), 59 deletions(-) diff --git a/mesonbuild/dependencies/boost.py b/mesonbuild/dependencies/boost.py index 622ee37..d12f37c 100644 --- a/mesonbuild/dependencies/boost.py +++ b/mesonbuild/dependencies/boost.py @@ -20,12 +20,14 @@ from pathlib import Path from .. import mlog from .. import mesonlib -from ..envconfig import get_env_var from ..environment import Environment from .base import DependencyException, ExternalDependency, PkgConfigDependency from .misc import threads_factory +if T.TYPE_CHECKING: + from ..environment import Properties + # On windows 3 directory layouts are supported: # * The default layout (versioned) installed: # - $BOOST_ROOT/include/boost-x_x/boost/*.hpp @@ -372,18 +374,11 @@ class BoostDependency(ExternalDependency): # First, look for paths specified in a machine file props = self.env.properties[self.for_machine] - boost_property_env = [props.get('boost_includedir'), props.get('boost_librarydir'), props.get('boost_root')] - if any(boost_property_env): + if any(x in self.env.properties[self.for_machine] for x in + ['boost_includedir', 'boost_librarydir', 'boost_root']): self.detect_boost_machine_file(props) return - # Next, look for paths in the environment - boost_manual_env_list = ['BOOST_INCLUDEDIR', 'BOOST_LIBRARYDIR', 'BOOST_ROOT', 'BOOSTROOT'] - boost_manual_env = [get_env_var(self.for_machine, self.env.is_cross_build, x) for x in boost_manual_env_list] - if any(boost_manual_env): - self.detect_boost_env() - return - # Finally, look for paths from .pc files and from searching the filesystem self.detect_roots() @@ -405,13 +400,20 @@ class BoostDependency(ExternalDependency): self.boost_root = j break - def detect_boost_machine_file(self, props: T.Dict[str, str]) -> None: + def detect_boost_machine_file(self, props: 'Properties') -> None: + """Detect boost with values in the machine file or environment. + + The machine file values are defaulted to the environment values. + """ + # XXX: if we had a TypedDict we woudn't need this incdir = props.get('boost_includedir') + assert incdir is None or isinstance(incdir, str) libdir = props.get('boost_librarydir') + assert libdir is None or isinstance(libdir, str) if incdir and libdir: - inc_dir = Path(props['boost_includedir']) - lib_dir = Path(props['boost_librarydir']) + inc_dir = Path(incdir) + lib_dir = Path(libdir) if not inc_dir.is_absolute() or not lib_dir.is_absolute(): raise DependencyException('Paths given for boost_includedir and boost_librarydir in machine file must be absolute') @@ -436,43 +438,6 @@ class BoostDependency(ExternalDependency): self.check_and_set_roots(paths) - def detect_boost_env(self) -> None: - boost_includedir = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOST_INCLUDEDIR') - boost_librarydir = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOST_LIBRARYDIR') - - boost_manual_env = [boost_includedir, boost_librarydir] - if all(boost_manual_env): - inc_dir = Path(boost_includedir) - lib_dir = Path(boost_librarydir) - - if not inc_dir.is_absolute() or not lib_dir.is_absolute(): - raise DependencyException('Paths given in BOOST_INCLUDEDIR and BOOST_LIBRARYDIR must be absolute') - - mlog.debug('Trying to find boost with:') - mlog.debug(' - BOOST_INCLUDEDIR = {}'.format(inc_dir)) - mlog.debug(' - BOOST_LIBRARYDIR = {}'.format(lib_dir)) - - return self.detect_split_root(inc_dir, lib_dir) - - elif any(boost_manual_env): - raise DependencyException('Both BOOST_INCLUDEDIR *and* BOOST_LIBRARYDIR have to be set (one is not enough). Ignoring.') - - boost_root = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOST_ROOT') - boostroot = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOSTROOT') - - # It shouldn't be possible to get here without something in BOOST_ROOT or BOOSTROOT - assert(boost_root or boostroot) - - for path, name in [(boost_root, 'BOOST_ROOT'), (boostroot, 'BOOSTROOT')]: - if path: - raw_paths = path.split(os.pathsep) - paths = [Path(x) for x in raw_paths] - if paths and any([not x.is_absolute() for x in paths]): - raise DependencyException('Paths in {} must be absolute'.format(name)) - break - - self.check_and_set_roots(paths) - def run_check(self, inc_dirs: T.List[BoostIncludeDir], lib_dirs: T.List[Path]) -> bool: mlog.debug(' - potential library dirs: {}'.format([x.as_posix() for x in lib_dirs])) mlog.debug(' - potential include dirs: {}'.format([x.path.as_posix() for x in inc_dirs])) diff --git a/mesonbuild/envconfig.py b/mesonbuild/envconfig.py index 69b6678..596defe 100644 --- a/mesonbuild/envconfig.py +++ b/mesonbuild/envconfig.py @@ -163,14 +163,6 @@ def get_env_var_pair(for_machine: MachineChoice, mlog.debug('Using {!r} from environment with value: {!r}'.format(var, value)) return var, value -def get_env_var(for_machine: MachineChoice, - is_cross: bool, - var_name: str) -> T.Optional[str]: - ret = get_env_var_pair(for_machine, is_cross, var_name) - if ret is None: - return None - return ret[1] - class Properties: def __init__( self, diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 6275da0..ba24411 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -656,6 +656,7 @@ class Environment: # Take default value from env if not set in cross/native files or command line. self.set_default_options_from_env() self._set_default_binaries_from_env() + self._set_default_properties_from_env() # Warn if the user is using two different ways of setting build-type # options that override each other @@ -821,6 +822,26 @@ class Environment: _, p_env = p_env_pair self.binaries[for_machine].binaries.setdefault(name, mesonlib.split_args(p_env)) + def _set_default_properties_from_env(self) -> None: + """Properties which can alkso be set from the environment.""" + # name, evar, split + opts: T.List[T.Tuple[str, T.List[str], bool]] = [ + ('boost_includedir', ['BOOST_INCLUDEDIR'], False), + ('boost_librarydir', ['BOOST_LIBRARYDIR'], False), + ('boost_root', ['BOOST_ROOT', 'BOOSTROOT'], True), + ] + + for (name, evars, split), for_machine in itertools.product(opts, MachineChoice): + for evar in evars: + p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) + if p_env_pair is not None: + _, p_env = p_env_pair + if split: + self.properties[for_machine].properties.setdefault(name, p_env.split(os.pathsep)) + else: + self.properties[for_machine].properties.setdefault(name, p_env) + break + def create_new_coredata(self, options: 'argparse.Namespace') -> None: # WARNING: Don't use any values from coredata in __init__. It gets # re-initialized with project options by the interpreter during diff --git a/run_unittests.py b/run_unittests.py index bbcdb5f..21b6608 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -5376,7 +5376,7 @@ class FailureTests(BasePlatformTests): def test_boost_BOOST_ROOT_dependency(self): # Test BOOST_ROOT; can be run even if Boost is found or not self.assertMesonRaises("dependency('boost')", - "(BOOST_ROOT.*absolute|{})".format(self.dnf), + "(boost_root.*absolute|{})".format(self.dnf), override_envvars = {'BOOST_ROOT': 'relative/path'}) def test_dependency_invalid_method(self): -- cgit v1.1 From 0076db6ff9309a34e2b86a6ae3fd85ff34b8f9f3 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 13:37:11 -0800 Subject: move get_env_var_pair to environment This is only used in environment, so it should live there too. --- mesonbuild/envconfig.py | 36 ++---------------------------------- mesonbuild/environment.py | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 38 deletions(-) diff --git a/mesonbuild/envconfig.py b/mesonbuild/envconfig.py index 596defe..c133ec9 100644 --- a/mesonbuild/envconfig.py +++ b/mesonbuild/envconfig.py @@ -12,20 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, subprocess +import subprocess import typing as T from enum import Enum from . import mesonlib -from .mesonlib import EnvironmentException, MachineChoice, PerMachine, split_args +from .mesonlib import EnvironmentException from . import mlog from pathlib import Path -_T = T.TypeVar('_T') - -if T.TYPE_CHECKING: - from .environment import Environment - # These classes contains all the data pulled from configuration files (native # and cross file currently), and also assists with the reading environment @@ -136,33 +131,6 @@ class CMakeSkipCompilerTest(Enum): NEVER = 'never' DEP_ONLY = 'dep_only' - -def get_env_var_pair(for_machine: MachineChoice, - is_cross: bool, - var_name: str) -> T.Optional[T.Tuple[str, str]]: - """ - Returns the exact env var and the value. - """ - candidates = PerMachine( - # The prefixed build version takes priority, but if we are native - # compiling we fall back on the unprefixed host version. This - # allows native builds to never need to worry about the 'BUILD_*' - # ones. - ([var_name + '_FOR_BUILD'] if is_cross else [var_name]), - # Always just the unprefixed host verions - [var_name] - )[for_machine] - for var in candidates: - value = os.environ.get(var) - if value is not None: - break - else: - formatted = ', '.join(['{!r}'.format(var) for var in candidates]) - mlog.debug('None of {} are defined in the environment, not changing global flags.'.format(formatted)) - return None - mlog.debug('Using {!r} from environment with value: {!r}'.format(var, value)) - return var, value - class Properties: def __init__( self, diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index ba24411..8ff9574 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -23,15 +23,13 @@ from . import coredata from .linkers import ArLinker, ArmarLinker, VisualStudioLinker, DLinker, CcrxLinker, Xc16Linker, CompCertLinker, C2000Linker, IntelVisualStudioLinker, AIXArLinker from . import mesonlib from .mesonlib import ( - MesonException, EnvironmentException, MachineChoice, Popen_safe, + MesonException, EnvironmentException, MachineChoice, Popen_safe, PerMachine, PerMachineDefaultable, PerThreeMachineDefaultable, split_args, quote_arg, OptionKey ) from . import mlog from .envconfig import ( - BinaryTable, ENV_VAR_PROG_MAP, MachineInfo, - Properties, known_cpu_families, get_env_var_pair, - CMakeVariables, + BinaryTable, MachineInfo, Properties, known_cpu_families, CMakeVariables, ) from . import compilers from .compilers import ( @@ -143,6 +141,34 @@ CompilersDict = T.Dict[str, Compiler] if T.TYPE_CHECKING: import argparse + +def get_env_var_pair(for_machine: MachineChoice, + is_cross: bool, + var_name: str) -> T.Optional[T.Tuple[str, str]]: + """ + Returns the exact env var and the value. + """ + candidates = PerMachine( + # The prefixed build version takes priority, but if we are native + # compiling we fall back on the unprefixed host version. This + # allows native builds to never need to worry about the 'BUILD_*' + # ones. + ([var_name + '_FOR_BUILD'] if is_cross else [var_name]), + # Always just the unprefixed host verions + [var_name] + )[for_machine] + for var in candidates: + value = os.environ.get(var) + if value is not None: + break + else: + formatted = ', '.join(['{!r}'.format(var) for var in candidates]) + mlog.debug('None of {} are defined in the environment, not changing global flags.'.format(formatted)) + return None + mlog.debug('Using {!r} from environment with value: {!r}'.format(var, value)) + return var, value + + def detect_gcovr(min_version='3.3', new_rootdir_version='4.2', log=False): gcovr_exe = 'gcovr' try: -- cgit v1.1 From 1cc7e4c84cc2b411b7adbc58d7a302fa7117c6e0 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 13:39:14 -0800 Subject: clean up get_env_var_pair This function returns both the name and the value, but we never actually use the name, just the value. Also make this module private. We really want to keep all environment variable reading in the Environment class so it's done once up front. This should help with that goal. --- mesonbuild/environment.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 8ff9574..7889911 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -142,9 +142,7 @@ if T.TYPE_CHECKING: import argparse -def get_env_var_pair(for_machine: MachineChoice, - is_cross: bool, - var_name: str) -> T.Optional[T.Tuple[str, str]]: +def _get_env_var(for_machine: MachineChoice, is_cross: bool, var_name: str) -> T.Optional[str]: """ Returns the exact env var and the value. """ @@ -166,7 +164,7 @@ def get_env_var_pair(for_machine: MachineChoice, mlog.debug('None of {} are defined in the environment, not changing global flags.'.format(formatted)) return None mlog.debug('Using {!r} from environment with value: {!r}'.format(var, value)) - return var, value + return value def detect_gcovr(min_version='3.3', new_rootdir_version='4.2', log=False): @@ -791,10 +789,8 @@ class Environment: ) for (evar, keyname), for_machine in itertools.product(opts, MachineChoice): - p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) - if p_env_pair is not None: - _, p_env = p_env_pair - + p_env = _get_env_var(for_machine, self.is_cross_build(), evar) + if p_env is not None: # these may contain duplicates, which must be removed, else # a duplicates-in-array-option warning arises. if keyname == 'cmake_prefix_path': @@ -843,9 +839,8 @@ class Environment: envconfig.ENV_VAR_PROG_MAP.items()) for (name, evar), for_machine in itertools.product(opts, MachineChoice): - p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) - if p_env_pair is not None: - _, p_env = p_env_pair + p_env = _get_env_var(for_machine, self.is_cross_build(), evar) + if p_env is not None: self.binaries[for_machine].binaries.setdefault(name, mesonlib.split_args(p_env)) def _set_default_properties_from_env(self) -> None: @@ -859,9 +854,8 @@ class Environment: for (name, evars, split), for_machine in itertools.product(opts, MachineChoice): for evar in evars: - p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) - if p_env_pair is not None: - _, p_env = p_env_pair + p_env = _get_env_var(for_machine, self.is_cross_build(), evar) + if p_env is not None: if split: self.properties[for_machine].properties.setdefault(name, p_env.split(os.pathsep)) else: -- cgit v1.1 From ff40ca25b776392625275bd7891701e02675e2b7 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 13:40:29 -0800 Subject: make some Environment methods protected they're really not public methods, they'r only meant to be called from the initializer. Let's mark them as such. --- mesonbuild/environment.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 7889911..1a7f3f1 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -640,7 +640,7 @@ class Environment: binaries.build = BinaryTable(config.get('binaries', {})) properties.build = Properties(config.get('properties', {})) cmakevars.build = CMakeVariables(config.get('cmake', {})) - self.load_machine_file_options(config, properties.build, MachineChoice.BUILD) + self._load_machine_file_options(config, properties.build, MachineChoice.BUILD) ## Read in cross file(s) to override host machine configuration @@ -658,7 +658,7 @@ class Environment: for key, value in list(self.options.items()): if self.coredata.is_per_machine_option(key): self.options[key.as_build()] = value - self.load_machine_file_options(config, properties.host, MachineChoice.HOST) + self._load_machine_file_options(config, properties.host, MachineChoice.HOST) else: # IF we aren't cross compiling, but we hav ea native file, the # native file is for the host. This is due to an mismatch between @@ -678,7 +678,7 @@ class Environment: self.options.update(options.cmd_line_options) # Take default value from env if not set in cross/native files or command line. - self.set_default_options_from_env() + self._set_default_options_from_env() self._set_default_binaries_from_env() self._set_default_properties_from_env() @@ -746,7 +746,7 @@ class Environment: self.default_pkgconfig = ['pkg-config'] self.wrap_resolver = None - def load_machine_file_options(self, config: 'ConfigParser', properties: Properties, machine: MachineChoice) -> None: + def _load_machine_file_options(self, config: 'ConfigParser', properties: Properties, machine: MachineChoice) -> None: """Read the contents of a Machine file and put it in the options store.""" paths = config.get('paths') if paths: @@ -777,7 +777,7 @@ class Environment: key = OptionKey.from_string(k).evolve(subproject=subproject) self.options[key] = v - def set_default_options_from_env(self) -> None: + def _set_default_options_from_env(self) -> None: opts: T.List[T.Tuple[str, str]] = ( [(v, f'{k}_args') for k, v in compilers.compilers.CFLAGS_MAPPING.items()] + [ -- cgit v1.1