diff options
author | Jussi Pakkanen <jpakkane@gmail.com> | 2021-01-12 16:19:58 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-01-12 16:19:58 +0000 |
commit | 5ff1a3ab25504563f87ac064ce21826cb0b936aa (patch) | |
tree | 0043efe041af7080ef79f19fec21b21929d0cd43 | |
parent | c659be692896f9f36fa46c4955220dc57653f09b (diff) | |
parent | ff40ca25b776392625275bd7891701e02675e2b7 (diff) | |
download | meson-5ff1a3ab25504563f87ac064ce21826cb0b936aa.zip meson-5ff1a3ab25504563f87ac064ce21826cb0b936aa.tar.gz meson-5ff1a3ab25504563f87ac064ce21826cb0b936aa.tar.bz2 |
Merge pull request #8159 from dcbaker/submit/all-env-variables-up-front
Read and store all environment variables up front
-rw-r--r-- | mesonbuild/backend/backends.py | 4 | ||||
-rw-r--r-- | mesonbuild/cmake/executor.py | 18 | ||||
-rw-r--r-- | mesonbuild/compilers/__init__.py | 2 | ||||
-rw-r--r-- | mesonbuild/compilers/c.py | 3 | ||||
-rw-r--r-- | mesonbuild/compilers/compilers.py | 105 | ||||
-rw-r--r-- | mesonbuild/compilers/cpp.py | 3 | ||||
-rw-r--r-- | mesonbuild/coredata.py | 2 | ||||
-rw-r--r-- | mesonbuild/dependencies/base.py | 12 | ||||
-rw-r--r-- | mesonbuild/dependencies/boost.py | 65 | ||||
-rw-r--r-- | mesonbuild/dependencies/hdf5.py | 2 | ||||
-rw-r--r-- | mesonbuild/dependencies/mpi.py | 2 | ||||
-rw-r--r-- | mesonbuild/envconfig.py | 179 | ||||
-rw-r--r-- | mesonbuild/environment.py | 156 | ||||
-rw-r--r-- | mesonbuild/linkers.py | 19 | ||||
-rw-r--r-- | mesonbuild/modules/unstable_external_project.py | 8 | ||||
-rwxr-xr-x | run_unittests.py | 45 |
16 files changed, 269 insertions, 356 deletions
diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index 9bb870c..6dad189 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -29,7 +29,7 @@ from .. import build from .. import dependencies from .. import mesonlib from .. import mlog -from ..compilers import languages_using_ldflags +from ..compilers import LANGUAGES_USING_LDFLAGS from ..mesonlib import ( File, MachineChoice, MesonException, OptionType, OrderedSet, OptionOverrideProxy, classify_unity_sources, unholder, OptionKey @@ -480,7 +480,7 @@ class Backend: def get_external_rpath_dirs(self, target): dirs = set() args = [] - for lang in languages_using_ldflags: + for lang in LANGUAGES_USING_LDFLAGS: try: args.extend(self.environment.coredata.get_external_link_args(target.for_machine, lang)) except Exception: diff --git a/mesonbuild/cmake/executor.py b/mesonbuild/cmake/executor.py index 674b854..e4b85de 100644 --- a/mesonbuild/cmake/executor.py +++ b/mesonbuild/cmake/executor.py @@ -24,7 +24,6 @@ import os from .. import mlog from ..mesonlib import PerMachine, Popen_safe, version_compare, MachineChoice, is_windows, OptionKey -from ..envconfig import get_env_var if T.TYPE_CHECKING: from ..environment import Environment @@ -63,23 +62,6 @@ class CMakeExecutor: return self.prefix_paths = self.environment.coredata.options[OptionKey('cmake_prefix_path', machine=self.for_machine)].value - env_pref_path_raw = get_env_var( - self.for_machine, - self.environment.is_cross_build(), - 'CMAKE_PREFIX_PATH') - if env_pref_path_raw is not None: - env_pref_path = [] # type: T.List[str] - if is_windows(): - # Cannot split on ':' on Windows because its in the drive letter - env_pref_path = env_pref_path_raw.split(os.pathsep) - else: - # https://github.com/mesonbuild/meson/issues/7294 - env_pref_path = re.split(r':|;', env_pref_path_raw) - env_pref_path = [x for x in env_pref_path if x] # Filter out empty strings - if not self.prefix_paths: - self.prefix_paths = [] - self.prefix_paths += env_pref_path - if self.prefix_paths: self.extra_cmake_args += ['-DCMAKE_PREFIX_PATH={}'.format(';'.join(self.prefix_paths))] diff --git a/mesonbuild/compilers/__init__.py b/mesonbuild/compilers/__init__.py index ec5d30f..bda6086 100644 --- a/mesonbuild/compilers/__init__.py +++ b/mesonbuild/compilers/__init__.py @@ -127,7 +127,7 @@ from .compilers import ( is_library, is_known_suffix, lang_suffixes, - languages_using_ldflags, + LANGUAGES_USING_LDFLAGS, sort_clink, ) from .c import ( diff --git a/mesonbuild/compilers/c.py b/mesonbuild/compilers/c.py index 311e65a..4a55b46 100644 --- a/mesonbuild/compilers/c.py +++ b/mesonbuild/compilers/c.py @@ -17,7 +17,6 @@ import typing as T from .. import coredata from ..mesonlib import MachineChoice, MesonException, mlog, version_compare, OptionKey -from ..linkers import LinkerEnvVarsMixin from .c_function_attributes import C_FUNC_ATTRIBUTES from .mixins.clike import CLikeCompiler from .mixins.ccrx import CcrxCompiler @@ -195,7 +194,7 @@ class AppleClangCCompiler(ClangCCompiler): _C2X_VERSION = '>=11.0.0' -class EmscriptenCCompiler(EmscriptenMixin, LinkerEnvVarsMixin, ClangCCompiler): +class EmscriptenCCompiler(EmscriptenMixin, ClangCCompiler): def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoice, is_cross: bool, info: 'MachineInfo', exe_wrapper: T.Optional['ExternalProgram'] = None, linker: T.Optional['DynamicLinker'] = None, diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py index 234ce06..cf9f35b 100644 --- a/mesonbuild/compilers/compilers.py +++ b/mesonbuild/compilers/compilers.py @@ -22,13 +22,9 @@ from functools import lru_cache from .. import coredata from .. import mlog from .. import mesonlib -from ..linkers import LinkerEnvVarsMixin from ..mesonlib import ( EnvironmentException, MachineChoice, MesonException, - Popen_safe, split_args, LibType, TemporaryDirectoryWinProof, OptionKey, -) -from ..envconfig import ( - get_env_var + Popen_safe, LibType, TemporaryDirectoryWinProof, OptionKey, ) from ..arglist import CompilerArgs @@ -85,24 +81,28 @@ clink_suffixes += ('h', 'll', 's') all_suffixes = set(itertools.chain(*lang_suffixes.values(), clink_suffixes)) # type: T.Set[str] # Languages that should use LDFLAGS arguments when linking. -languages_using_ldflags = {'objcpp', 'cpp', 'objc', 'c', 'fortran', 'd', 'cuda'} # type: T.Set[str] +LANGUAGES_USING_LDFLAGS = {'objcpp', 'cpp', 'objc', 'c', 'fortran', 'd', 'cuda'} # type: T.Set[str] # Languages that should use CPPFLAGS arguments when linking. -languages_using_cppflags = {'c', 'cpp', 'objc', 'objcpp'} # type: T.Set[str] +LANGUAGES_USING_CPPFLAGS = {'c', 'cpp', 'objc', 'objcpp'} # type: T.Set[str] soregex = re.compile(r'.*\.so(\.[0-9]+)?(\.[0-9]+)?(\.[0-9]+)?$') # Environment variables that each lang uses. -cflags_mapping = {'c': 'CFLAGS', - 'cpp': 'CXXFLAGS', - 'cuda': 'CUFLAGS', - 'objc': 'OBJCFLAGS', - 'objcpp': 'OBJCXXFLAGS', - 'fortran': 'FFLAGS', - 'd': 'DFLAGS', - 'vala': 'VALAFLAGS', - 'rust': 'RUSTFLAGS'} # type: T.Dict[str, str] - -cexe_mapping = {'c': 'CC', - 'cpp': 'CXX'} +CFLAGS_MAPPING: T.Mapping[str, str] = { + 'c': 'CFLAGS', + 'cpp': 'CXXFLAGS', + 'cuda': 'CUFLAGS', + 'objc': 'OBJCFLAGS', + 'objcpp': 'OBJCXXFLAGS', + 'fortran': 'FFLAGS', + 'd': 'DFLAGS', + 'vala': 'VALAFLAGS', + 'rust': 'RUSTFLAGS', +} + +CEXE_MAPPING: T.Mapping = { + 'c': 'CC', + 'cpp': 'CXX', +} # All these are only for C-linkable languages; see `clink_langs` above. @@ -587,11 +587,6 @@ class Compiler(metaclass=abc.ABCMeta): """ return [] - def get_linker_args_from_envvars(self, - for_machine: MachineChoice, - is_cross: bool) -> T.List[str]: - return self.linker.get_args_from_envvars(for_machine, is_cross) - def get_options(self) -> 'KeyedOptionDictType': return {} @@ -1199,68 +1194,32 @@ class Compiler(metaclass=abc.ABCMeta): def get_prelink_args(self, prelink_name: str, obj_list: T.List[str]) -> T.List[str]: raise EnvironmentException('{} does not know how to do prelinking.'.format(self.id)) -def get_args_from_envvars(lang: str, - for_machine: MachineChoice, - is_cross: bool, - use_linker_args: bool) -> T.Tuple[T.List[str], T.List[str]]: - """ - Returns a tuple of (compile_flags, link_flags) for the specified language - from the inherited environment - """ - if lang not in cflags_mapping: - return [], [] - - compile_flags = [] # type: T.List[str] - link_flags = [] # type: T.List[str] - - env_compile_flags = get_env_var(for_machine, is_cross, cflags_mapping[lang]) - if env_compile_flags is not None: - compile_flags += split_args(env_compile_flags) - - # Link flags (same for all languages) - if lang in languages_using_ldflags: - link_flags += LinkerEnvVarsMixin.get_args_from_envvars(for_machine, is_cross) - if use_linker_args: - # When the compiler is used as a wrapper around the linker (such as - # with GCC and Clang), the compile flags can be needed while linking - # too. This is also what Autotools does. However, we don't want to do - # this when the linker is stand-alone such as with MSVC C/C++, etc. - link_flags = compile_flags + link_flags - - # Pre-processor flags for certain languages - if lang in languages_using_cppflags: - env_preproc_flags = get_env_var(for_machine, is_cross, 'CPPFLAGS') - if env_preproc_flags is not None: - compile_flags += split_args(env_preproc_flags) - - return compile_flags, link_flags - def get_global_options(lang: str, comp: T.Type[Compiler], for_machine: MachineChoice, - is_cross: bool) -> 'KeyedOptionDictType': + env: 'Environment') -> 'KeyedOptionDictType': """Retreive options that apply to all compilers for a given language.""" description = 'Extra arguments passed to the {}'.format(lang) argkey = OptionKey('args', lang=lang, machine=for_machine) largkey = argkey.evolve('link_args') + + # We shouldn't need listify here, but until we have a separate + # linker-driver representation and can have that do the combine we have to + # do it htis way. + compile_args = mesonlib.listify(env.options.get(argkey, [])) + link_args = mesonlib.listify(env.options.get(largkey, [])) + + if comp.INVOKES_LINKER: + link_args = compile_args + link_args + opts: 'KeyedOptionDictType' = { argkey: coredata.UserArrayOption( description + ' compiler', - [], split_args=True, user_input=True, allow_dups=True), + compile_args, split_args=True, user_input=True, allow_dups=True), largkey: coredata.UserArrayOption( description + ' linker', - [], split_args=True, user_input=True, allow_dups=True), + link_args, split_args=True, user_input=True, allow_dups=True), } - # Get from env vars. - compile_args, link_args = get_args_from_envvars( - lang, - for_machine, - is_cross, - comp.INVOKES_LINKER) - - opts[argkey].set_value(compile_args) - opts[largkey].set_value(link_args) - return opts diff --git a/mesonbuild/compilers/cpp.py b/mesonbuild/compilers/cpp.py index 2e94e48..ebe3a4f 100644 --- a/mesonbuild/compilers/cpp.py +++ b/mesonbuild/compilers/cpp.py @@ -21,7 +21,6 @@ from .. import coredata from .. import mlog from ..mesonlib import MesonException, MachineChoice, version_compare, OptionKey -from ..linkers import LinkerEnvVarsMixin from .compilers import ( gnu_winlibs, msvc_winlibs, @@ -256,7 +255,7 @@ class AppleClangCPPCompiler(ClangCPPCompiler): return ['-lc++'] -class EmscriptenCPPCompiler(EmscriptenMixin, LinkerEnvVarsMixin, ClangCPPCompiler): +class EmscriptenCPPCompiler(EmscriptenMixin, ClangCPPCompiler): def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoice, is_cross: bool, info: 'MachineInfo', exe_wrapper: T.Optional['ExternalProgram'] = None, linker: T.Optional['DynamicLinker'] = None, diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index cda0566..f2aba80 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -759,7 +759,7 @@ class CoreData: for_machine: MachineChoice, env: 'Environment') -> None: """Add global language arguments that are needed before compiler/linker detection.""" from .compilers import compilers - options = compilers.get_global_options(lang, comp, for_machine, env.is_cross_build()) + options = compilers.get_global_options(lang, comp, for_machine, env) self.add_compiler_options(options, lang, for_machine, env) def process_new_compiler(self, lang: str, comp: 'Compiler', env: 'Environment') -> None: diff --git a/mesonbuild/dependencies/base.py b/mesonbuild/dependencies/base.py index 9ddf6db..9330e46 100644 --- a/mesonbuild/dependencies/base.py +++ b/mesonbuild/dependencies/base.py @@ -32,7 +32,6 @@ from pathlib import Path, PurePath from .. import mlog from .. import mesonlib from ..compilers import clib_langs -from ..envconfig import get_env_var from ..environment import Environment, MachineInfo from ..cmake import CMakeExecutor, CMakeTraceParser, CMakeException, CMakeToolchain, CMakeExecScope, check_cmake_args from ..mesonlib import MachineChoice, MesonException, OrderedSet, PerMachine @@ -317,7 +316,7 @@ class HasNativeKwarg: return MachineChoice.BUILD if kwargs.get('native', False) else MachineChoice.HOST class ExternalDependency(Dependency, HasNativeKwarg): - def __init__(self, type_name, environment, kwargs, language: T.Optional[str] = None): + def __init__(self, type_name, environment: Environment, kwargs, language: T.Optional[str] = None): Dependency.__init__(self, type_name, kwargs) self.env = environment self.name = type_name # default @@ -784,14 +783,7 @@ class PkgConfigDependency(ExternalDependency): # # Only prefix_libpaths are reordered here because there should not be # too many system_libpaths to cause library version issues. - pkg_config_path = get_env_var( - self.for_machine, - self.env.is_cross_build(), - 'PKG_CONFIG_PATH') - if pkg_config_path: - pkg_config_path = pkg_config_path.split(os.pathsep) - else: - pkg_config_path = [] + pkg_config_path: T.List[str] = self.env.coredata.options[OptionKey('pkg_config_path', machine=self.for_machine)].value pkg_config_path = self._convert_mingw_paths(pkg_config_path) prefix_libpaths = sort_libpaths(prefix_libpaths, pkg_config_path) system_libpaths = OrderedSet() diff --git a/mesonbuild/dependencies/boost.py b/mesonbuild/dependencies/boost.py index 622ee37..d12f37c 100644 --- a/mesonbuild/dependencies/boost.py +++ b/mesonbuild/dependencies/boost.py @@ -20,12 +20,14 @@ from pathlib import Path from .. import mlog from .. import mesonlib -from ..envconfig import get_env_var from ..environment import Environment from .base import DependencyException, ExternalDependency, PkgConfigDependency from .misc import threads_factory +if T.TYPE_CHECKING: + from ..environment import Properties + # On windows 3 directory layouts are supported: # * The default layout (versioned) installed: # - $BOOST_ROOT/include/boost-x_x/boost/*.hpp @@ -372,18 +374,11 @@ class BoostDependency(ExternalDependency): # First, look for paths specified in a machine file props = self.env.properties[self.for_machine] - boost_property_env = [props.get('boost_includedir'), props.get('boost_librarydir'), props.get('boost_root')] - if any(boost_property_env): + if any(x in self.env.properties[self.for_machine] for x in + ['boost_includedir', 'boost_librarydir', 'boost_root']): self.detect_boost_machine_file(props) return - # Next, look for paths in the environment - boost_manual_env_list = ['BOOST_INCLUDEDIR', 'BOOST_LIBRARYDIR', 'BOOST_ROOT', 'BOOSTROOT'] - boost_manual_env = [get_env_var(self.for_machine, self.env.is_cross_build, x) for x in boost_manual_env_list] - if any(boost_manual_env): - self.detect_boost_env() - return - # Finally, look for paths from .pc files and from searching the filesystem self.detect_roots() @@ -405,13 +400,20 @@ class BoostDependency(ExternalDependency): self.boost_root = j break - def detect_boost_machine_file(self, props: T.Dict[str, str]) -> None: + def detect_boost_machine_file(self, props: 'Properties') -> None: + """Detect boost with values in the machine file or environment. + + The machine file values are defaulted to the environment values. + """ + # XXX: if we had a TypedDict we woudn't need this incdir = props.get('boost_includedir') + assert incdir is None or isinstance(incdir, str) libdir = props.get('boost_librarydir') + assert libdir is None or isinstance(libdir, str) if incdir and libdir: - inc_dir = Path(props['boost_includedir']) - lib_dir = Path(props['boost_librarydir']) + inc_dir = Path(incdir) + lib_dir = Path(libdir) if not inc_dir.is_absolute() or not lib_dir.is_absolute(): raise DependencyException('Paths given for boost_includedir and boost_librarydir in machine file must be absolute') @@ -436,43 +438,6 @@ class BoostDependency(ExternalDependency): self.check_and_set_roots(paths) - def detect_boost_env(self) -> None: - boost_includedir = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOST_INCLUDEDIR') - boost_librarydir = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOST_LIBRARYDIR') - - boost_manual_env = [boost_includedir, boost_librarydir] - if all(boost_manual_env): - inc_dir = Path(boost_includedir) - lib_dir = Path(boost_librarydir) - - if not inc_dir.is_absolute() or not lib_dir.is_absolute(): - raise DependencyException('Paths given in BOOST_INCLUDEDIR and BOOST_LIBRARYDIR must be absolute') - - mlog.debug('Trying to find boost with:') - mlog.debug(' - BOOST_INCLUDEDIR = {}'.format(inc_dir)) - mlog.debug(' - BOOST_LIBRARYDIR = {}'.format(lib_dir)) - - return self.detect_split_root(inc_dir, lib_dir) - - elif any(boost_manual_env): - raise DependencyException('Both BOOST_INCLUDEDIR *and* BOOST_LIBRARYDIR have to be set (one is not enough). Ignoring.') - - boost_root = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOST_ROOT') - boostroot = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOSTROOT') - - # It shouldn't be possible to get here without something in BOOST_ROOT or BOOSTROOT - assert(boost_root or boostroot) - - for path, name in [(boost_root, 'BOOST_ROOT'), (boostroot, 'BOOSTROOT')]: - if path: - raw_paths = path.split(os.pathsep) - paths = [Path(x) for x in raw_paths] - if paths and any([not x.is_absolute() for x in paths]): - raise DependencyException('Paths in {} must be absolute'.format(name)) - break - - self.check_and_set_roots(paths) - def run_check(self, inc_dirs: T.List[BoostIncludeDir], lib_dirs: T.List[Path]) -> bool: mlog.debug(' - potential library dirs: {}'.format([x.as_posix() for x in lib_dirs])) mlog.debug(' - potential include dirs: {}'.format([x.path.as_posix() for x in inc_dirs])) diff --git a/mesonbuild/dependencies/hdf5.py b/mesonbuild/dependencies/hdf5.py index 21f4e71..ad28975 100644 --- a/mesonbuild/dependencies/hdf5.py +++ b/mesonbuild/dependencies/hdf5.py @@ -30,8 +30,8 @@ import typing as T if T.TYPE_CHECKING: from .base import Dependency - from ..envconfig import MachineChoice from ..environment import Environment + from ..mesonlib import MachineChoice class HDF5PkgConfigDependency(PkgConfigDependency): diff --git a/mesonbuild/dependencies/mpi.py b/mesonbuild/dependencies/mpi.py index 2e1e764..d4b324c 100644 --- a/mesonbuild/dependencies/mpi.py +++ b/mesonbuild/dependencies/mpi.py @@ -106,7 +106,7 @@ class _MPIConfigToolDependency(ConfigToolDependency): Drop -O2 and everything that is not needed. """ result = [] - multi_args = ('-I', ) + multi_args: T.Tuple[str, ...] = ('-I', ) if self.language == 'fortran': fc = self.env.coredata.compilers[self.for_machine]['fortran'] multi_args += fc.get_module_incdir_args() diff --git a/mesonbuild/envconfig.py b/mesonbuild/envconfig.py index db1041f..c133ec9 100644 --- a/mesonbuild/envconfig.py +++ b/mesonbuild/envconfig.py @@ -12,17 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, subprocess +import subprocess import typing as T from enum import Enum from . import mesonlib -from .mesonlib import EnvironmentException, MachineChoice, PerMachine, split_args +from .mesonlib import EnvironmentException from . import mlog from pathlib import Path -_T = T.TypeVar('_T') - # These classes contains all the data pulled from configuration files (native # and cross file currently), and also assists with the reading environment @@ -85,46 +83,54 @@ CPU_FAMILES_64_BIT = [ 'x86_64', ] +# Map from language identifiers to environment variables. +ENV_VAR_PROG_MAP: T.Mapping[str, str] = { + # Compilers + 'c': 'CC', + 'cpp': 'CXX', + 'cs': 'CSC', + 'd': 'DC', + 'fortran': 'FC', + 'objc': 'OBJC', + 'objcpp': 'OBJCXX', + 'rust': 'RUSTC', + 'vala': 'VALAC', + + # Linkers + 'c_ld': 'CC_LD', + 'cpp_ld': 'CXX_LD', + 'd_ld': 'DC_LD', + 'fortran_ld': 'FC_LD', + 'objc_ld': 'OBJC_LD', + 'objcpp_ld': 'OBJCXX_LD', + 'rust_ld': 'RUSTC_LD', + + # Binutils + 'strip': 'STRIP', + 'ar': 'AR', + 'windres': 'WINDRES', + + # Other tools + 'cmake': 'CMAKE', + 'qmake': 'QMAKE', + 'pkgconfig': 'PKG_CONFIG', + 'make': 'MAKE', +} + +# Deprecated environment variables mapped from the new variable to the old one +# Deprecated in 0.54.0 +DEPRECATED_ENV_PROG_MAP: T.Mapping[str, str] = { + 'd_ld': 'D_LD', + 'fortran_ld': 'F_LD', + 'rust_ld': 'RUST_LD', + 'objcpp_ld': 'OBJCPP_LD', +} + class CMakeSkipCompilerTest(Enum): ALWAYS = 'always' NEVER = 'never' DEP_ONLY = 'dep_only' - -def get_env_var_pair(for_machine: MachineChoice, - is_cross: bool, - var_name: str) -> T.Optional[T.Tuple[str, str]]: - """ - Returns the exact env var and the value. - """ - candidates = PerMachine( - # The prefixed build version takes priority, but if we are native - # compiling we fall back on the unprefixed host version. This - # allows native builds to never need to worry about the 'BUILD_*' - # ones. - ([var_name + '_FOR_BUILD'] if is_cross else [var_name]), - # Always just the unprefixed host verions - [var_name] - )[for_machine] - for var in candidates: - value = os.environ.get(var) - if value is not None: - break - else: - formatted = ', '.join(['{!r}'.format(var) for var in candidates]) - mlog.debug('None of {} are defined in the environment, not changing global flags.'.format(formatted)) - return None - mlog.debug('Using {!r} from environment with value: {!r}'.format(var, value)) - return var, value - -def get_env_var(for_machine: MachineChoice, - is_cross: bool, - var_name: str) -> T.Optional[str]: - ret = get_env_var_pair(for_machine, is_cross, var_name) - if ret is None: - return None - return ret[1] - class Properties: def __init__( self, @@ -351,60 +357,18 @@ class MachineInfo: return self.is_windows() or self.is_cygwin() class BinaryTable: + def __init__( self, binaries: T.Optional[T.Dict[str, T.Union[str, T.List[str]]]] = None, ): - self.binaries = binaries or {} # type: T.Dict[str, T.Union[str, T.List[str]]] - for name, command in self.binaries.items(): - if not isinstance(command, (list, str)): - # TODO generalize message - raise mesonlib.MesonException( - 'Invalid type {!r} for binary {!r} in cross file' - ''.format(command, name)) - - # Map from language identifiers to environment variables. - evarMap = { - # Compilers - 'c': 'CC', - 'cpp': 'CXX', - 'cs': 'CSC', - 'd': 'DC', - 'fortran': 'FC', - 'objc': 'OBJC', - 'objcpp': 'OBJCXX', - 'rust': 'RUSTC', - 'vala': 'VALAC', - - # Linkers - 'c_ld': 'CC_LD', - 'cpp_ld': 'CXX_LD', - 'd_ld': 'DC_LD', - 'fortran_ld': 'FC_LD', - 'objc_ld': 'OBJC_LD', - 'objcpp_ld': 'OBJCXX_LD', - 'rust_ld': 'RUSTC_LD', - - # Binutils - 'strip': 'STRIP', - 'ar': 'AR', - 'windres': 'WINDRES', - - # Other tools - 'cmake': 'CMAKE', - 'qmake': 'QMAKE', - 'pkgconfig': 'PKG_CONFIG', - 'make': 'MAKE', - } # type: T.Dict[str, str] - - # Deprecated environment variables mapped from the new variable to the old one - # Deprecated in 0.54.0 - DEPRECATION_MAP = { - 'DC_LD': 'D_LD', - 'FC_LD': 'F_LD', - 'RUSTC_LD': 'RUST_LD', - 'OBJCXX_LD': 'OBJCPP_LD', - } # type: T.Dict[str, str] + self.binaries: T.Dict[str, T.List[str]] = {} + if binaries: + for name, command in binaries.items(): + if not isinstance(command, (list, str)): + raise mesonlib.MesonException( + f'Invalid type {command!r} for entry {name!r} in cross file') + self.binaries[name] = mesonlib.listify(command) @staticmethod def detect_ccache() -> T.List[str]: @@ -426,42 +390,17 @@ class BinaryTable: # Return value has to be a list of compiler 'choices' return compiler, ccache - def lookup_entry(self, - for_machine: MachineChoice, - is_cross: bool, - name: str) -> T.Optional[T.List[str]]: + def lookup_entry(self, name: str) -> T.Optional[T.List[str]]: """Lookup binary in cross/native file and fallback to environment. Returns command with args as list if found, Returns `None` if nothing is found. """ - # Try explicit map, don't fall back on env var - # Try explict map, then env vars - for _ in [()]: # a trick to get `break` - raw_command = self.binaries.get(name) - if raw_command is not None: - command = mesonlib.stringlistify(raw_command) - break # found - evar = self.evarMap.get(name) - if evar is not None: - raw_command = get_env_var(for_machine, is_cross, evar) - if raw_command is None: - deprecated = self.DEPRECATION_MAP.get(evar) - if deprecated is not None: - raw_command = get_env_var(for_machine, is_cross, deprecated) - if raw_command is not None: - mlog.deprecation( - 'The', deprecated, 'environment variable is deprecated in favor of', - evar, once=True) - if raw_command is not None: - command = split_args(raw_command) - break # found - command = None - - - # Do not return empty or blank string entries - if command is not None and (len(command) == 0 or len(command[0].strip()) == 0): - command = None + command = self.binaries.get(name) + if not command: + return None + elif not command[0].strip(): + return None return command class CMakeVariables: diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 59675ff..1a7f3f1 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -1,4 +1,4 @@ -# Copyright 2012-2016 The Meson development team +# Copyright 2012-2020 The Meson development team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import itertools import os, platform, re, sys, shutil, subprocess import tempfile import shlex @@ -22,15 +23,13 @@ from . import coredata from .linkers import ArLinker, ArmarLinker, VisualStudioLinker, DLinker, CcrxLinker, Xc16Linker, CompCertLinker, C2000Linker, IntelVisualStudioLinker, AIXArLinker from . import mesonlib from .mesonlib import ( - MesonException, EnvironmentException, MachineChoice, Popen_safe, + MesonException, EnvironmentException, MachineChoice, Popen_safe, PerMachine, PerMachineDefaultable, PerThreeMachineDefaultable, split_args, quote_arg, OptionKey ) from . import mlog from .envconfig import ( - BinaryTable, MachineInfo, - Properties, known_cpu_families, get_env_var_pair, - CMakeVariables, + BinaryTable, MachineInfo, Properties, known_cpu_families, CMakeVariables, ) from . import compilers from .compilers import ( @@ -128,6 +127,7 @@ from .compilers import ( VisualStudioCCompiler, VisualStudioCPPCompiler, ) +from mesonbuild import envconfig if T.TYPE_CHECKING: from configparser import ConfigParser @@ -141,6 +141,32 @@ CompilersDict = T.Dict[str, Compiler] if T.TYPE_CHECKING: import argparse + +def _get_env_var(for_machine: MachineChoice, is_cross: bool, var_name: str) -> T.Optional[str]: + """ + Returns the exact env var and the value. + """ + candidates = PerMachine( + # The prefixed build version takes priority, but if we are native + # compiling we fall back on the unprefixed host version. This + # allows native builds to never need to worry about the 'BUILD_*' + # ones. + ([var_name + '_FOR_BUILD'] if is_cross else [var_name]), + # Always just the unprefixed host verions + [var_name] + )[for_machine] + for var in candidates: + value = os.environ.get(var) + if value is not None: + break + else: + formatted = ', '.join(['{!r}'.format(var) for var in candidates]) + mlog.debug('None of {} are defined in the environment, not changing global flags.'.format(formatted)) + return None + mlog.debug('Using {!r} from environment with value: {!r}'.format(var, value)) + return value + + def detect_gcovr(min_version='3.3', new_rootdir_version='4.2', log=False): gcovr_exe = 'gcovr' try: @@ -579,7 +605,7 @@ class Environment: # Stores machine infos, the only *three* machine one because we have a # target machine info on for the user (Meson never cares about the # target machine.) - machines = PerThreeMachineDefaultable() # type: PerMachineDefaultable[MachineInfo] + machines: PerThreeMachineDefaultable[MachineInfo] = PerThreeMachineDefaultable() # Similar to coredata.compilers, but lower level in that there is no # meta data, only names/paths. @@ -605,7 +631,7 @@ class Environment: # # Note that order matters because of 'buildtype', if it is after # 'optimization' and 'debug' keys, it override them. - self.options: T.MutableMapping[OptionKey, str] = collections.OrderedDict() + self.options: T.MutableMapping[OptionKey, T.Union[str, T.List[str]]] = collections.OrderedDict() ## Read in native file(s) to override build machine configuration @@ -614,7 +640,7 @@ class Environment: binaries.build = BinaryTable(config.get('binaries', {})) properties.build = Properties(config.get('properties', {})) cmakevars.build = CMakeVariables(config.get('cmake', {})) - self.load_machine_file_options(config, properties.build, MachineChoice.BUILD) + self._load_machine_file_options(config, properties.build, MachineChoice.BUILD) ## Read in cross file(s) to override host machine configuration @@ -632,7 +658,7 @@ class Environment: for key, value in list(self.options.items()): if self.coredata.is_per_machine_option(key): self.options[key.as_build()] = value - self.load_machine_file_options(config, properties.host, MachineChoice.HOST) + self._load_machine_file_options(config, properties.host, MachineChoice.HOST) else: # IF we aren't cross compiling, but we hav ea native file, the # native file is for the host. This is due to an mismatch between @@ -652,7 +678,9 @@ class Environment: self.options.update(options.cmd_line_options) # Take default value from env if not set in cross/native files or command line. - self.set_default_options_from_env() + self._set_default_options_from_env() + self._set_default_binaries_from_env() + self._set_default_properties_from_env() # Warn if the user is using two different ways of setting build-type # options that override each other @@ -718,7 +746,7 @@ class Environment: self.default_pkgconfig = ['pkg-config'] self.wrap_resolver = None - def load_machine_file_options(self, config: 'ConfigParser', properties: Properties, machine: MachineChoice) -> None: + def _load_machine_file_options(self, config: 'ConfigParser', properties: Properties, machine: MachineChoice) -> None: """Read the contents of a Machine file and put it in the options store.""" paths = config.get('paths') if paths: @@ -749,23 +777,90 @@ class Environment: key = OptionKey.from_string(k).evolve(subproject=subproject) self.options[key] = v - def set_default_options_from_env(self) -> None: - for for_machine in MachineChoice: - for evar, keyname in [('PKG_CONFIG_PATH', 'pkg_config_path')]: - p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) - if p_env_pair is not None: - _, p_env = p_env_pair - - # PKG_CONFIG_PATH may contain duplicates, which must be - # removed, else a duplicates-in-array-option warning arises. + def _set_default_options_from_env(self) -> None: + opts: T.List[T.Tuple[str, str]] = ( + [(v, f'{k}_args') for k, v in compilers.compilers.CFLAGS_MAPPING.items()] + + [ + ('PKG_CONFIG_PATH', 'pkg_config_path'), + ('CMAKE_PREFIX_PATH', 'cmake_prefix_path'), + ('LDFLAGS', 'ldflags'), + ('CPPFLAGS', 'cppflags'), + ] + ) + + for (evar, keyname), for_machine in itertools.product(opts, MachineChoice): + p_env = _get_env_var(for_machine, self.is_cross_build(), evar) + if p_env is not None: + # these may contain duplicates, which must be removed, else + # a duplicates-in-array-option warning arises. + if keyname == 'cmake_prefix_path': + if self.machines[for_machine].is_windows(): + # Cannot split on ':' on Windows because its in the drive letter + _p_env = p_env.split(os.pathsep) + else: + # https://github.com/mesonbuild/meson/issues/7294 + _p_env = re.split(r':|;', p_env) + p_list = list(mesonlib.OrderedSet(_p_env)) + elif keyname == 'pkg_config_path': p_list = list(mesonlib.OrderedSet(p_env.split(':'))) - # Take env vars only on first invocation, if the env changes when - # reconfiguring it gets ignored. - # FIXME: We should remember if we took the value from env to warn - # if it changes on future invocations. - if self.first_invocation: - key = OptionKey(keyname, machine=for_machine) - self.options.setdefault(key, p_list) + else: + p_list = split_args(p_env) + p_list = [e for e in p_list if e] # filter out any empty eelemnts + + # Take env vars only on first invocation, if the env changes when + # reconfiguring it gets ignored. + # FIXME: We should remember if we took the value from env to warn + # if it changes on future invocations. + if self.first_invocation: + if keyname == 'ldflags': + key = OptionKey('link_args', machine=for_machine, lang='c') # needs a language to initialize properly + for lang in compilers.compilers.LANGUAGES_USING_LDFLAGS: + key = key.evolve(lang=lang) + v = mesonlib.listify(self.options.get(key, [])) + self.options.setdefault(key, v + p_list) + elif keyname == 'cppflags': + key = OptionKey('args', machine=for_machine, lang='c') + for lang in compilers.compilers.LANGUAGES_USING_CPPFLAGS: + key = key.evolve(lang=lang) + v = mesonlib.listify(self.options.get(key, [])) + self.options.setdefault(key, v + p_list) + else: + key = OptionKey.from_string(keyname).evolve(machine=for_machine) + v = mesonlib.listify(self.options.get(key, [])) + self.options.setdefault(key, v + p_list) + + def _set_default_binaries_from_env(self) -> None: + """Set default binaries from the environment. + + For example, pkg-config can be set via PKG_CONFIG, or in the machine + file. We want to set the default to the env variable. + """ + opts = itertools.chain(envconfig.DEPRECATED_ENV_PROG_MAP.items(), + envconfig.ENV_VAR_PROG_MAP.items()) + + for (name, evar), for_machine in itertools.product(opts, MachineChoice): + p_env = _get_env_var(for_machine, self.is_cross_build(), evar) + if p_env is not None: + self.binaries[for_machine].binaries.setdefault(name, mesonlib.split_args(p_env)) + + def _set_default_properties_from_env(self) -> None: + """Properties which can alkso be set from the environment.""" + # name, evar, split + opts: T.List[T.Tuple[str, T.List[str], bool]] = [ + ('boost_includedir', ['BOOST_INCLUDEDIR'], False), + ('boost_librarydir', ['BOOST_LIBRARYDIR'], False), + ('boost_root', ['BOOST_ROOT', 'BOOSTROOT'], True), + ] + + for (name, evars, split), for_machine in itertools.product(opts, MachineChoice): + for evar in evars: + p_env = _get_env_var(for_machine, self.is_cross_build(), evar) + if p_env is not None: + if split: + self.properties[for_machine].properties.setdefault(name, p_env.split(os.pathsep)) + else: + self.properties[for_machine].properties.setdefault(name, p_env) + break def create_new_coredata(self, options: 'argparse.Namespace') -> None: # WARNING: Don't use any values from coredata in __init__. It gets @@ -816,11 +911,8 @@ class Environment: def is_library(self, fname): return is_library(fname) - def lookup_binary_entry(self, for_machine: MachineChoice, name: str): - return self.binaries[for_machine].lookup_entry( - for_machine, - self.is_cross_build(), - name) + def lookup_binary_entry(self, for_machine: MachineChoice, name: str) -> T.List[str]: + return self.binaries[for_machine].lookup_entry(name) @staticmethod def get_gnu_compiler_defines(compiler): diff --git a/mesonbuild/linkers.py b/mesonbuild/linkers.py index 141c8fd..86e6aac 100644 --- a/mesonbuild/linkers.py +++ b/mesonbuild/linkers.py @@ -18,12 +18,11 @@ import typing as T from . import mesonlib from .arglist import CompilerArgs -from .envconfig import get_env_var if T.TYPE_CHECKING: from .coredata import KeyedOptionDictType - from .envconfig import MachineChoice from .environment import Environment + from .mesonlib import MachineChoice class StaticLinker: @@ -301,21 +300,7 @@ def evaluate_rpath(p: str, build_dir: str, from_dir: str) -> str: else: return os.path.relpath(os.path.join(build_dir, p), os.path.join(build_dir, from_dir)) - -class LinkerEnvVarsMixin(metaclass=abc.ABCMeta): - - """Mixin reading LDFLAGS from the environment.""" - - @staticmethod - def get_args_from_envvars(for_machine: mesonlib.MachineChoice, - is_cross: bool) -> T.List[str]: - raw_value = get_env_var(for_machine, is_cross, 'LDFLAGS') - if raw_value is not None: - return mesonlib.split_args(raw_value) - else: - return [] - -class DynamicLinker(LinkerEnvVarsMixin, metaclass=abc.ABCMeta): +class DynamicLinker(metaclass=abc.ABCMeta): """Base class for dynamic linkers.""" diff --git a/mesonbuild/modules/unstable_external_project.py b/mesonbuild/modules/unstable_external_project.py index 7249078..809b590 100644 --- a/mesonbuild/modules/unstable_external_project.py +++ b/mesonbuild/modules/unstable_external_project.py @@ -23,7 +23,7 @@ from ..mesonlib import (MesonException, Popen_safe, MachineChoice, from ..interpreterbase import InterpreterObject, InterpreterException, FeatureNew from ..interpreterbase import stringArgs, permittedKwargs from ..interpreter import Interpreter, DependencyHolder, InstallDir -from ..compilers.compilers import cflags_mapping, cexe_mapping +from ..compilers.compilers import CFLAGS_MAPPING, CEXE_MAPPING from ..dependencies.base import InternalDependency, PkgConfigDependency from ..environment import Environment from ..mesonlib import OptionKey @@ -110,11 +110,11 @@ class ExternalProject(InterpreterObject): link_args = [] self.run_env = os.environ.copy() for lang, compiler in self.env.coredata.compilers[MachineChoice.HOST].items(): - if any(lang not in i for i in (cexe_mapping, cflags_mapping)): + if any(lang not in i for i in (CEXE_MAPPING, CFLAGS_MAPPING)): continue cargs = self.env.coredata.get_external_args(MachineChoice.HOST, lang) - self.run_env[cexe_mapping[lang]] = self._quote_and_join(compiler.get_exelist()) - self.run_env[cflags_mapping[lang]] = self._quote_and_join(cargs) + self.run_env[CEXE_MAPPING[lang]] = self._quote_and_join(compiler.get_exelist()) + self.run_env[CFLAGS_MAPPING[lang]] = self._quote_and_join(cargs) if not link_exelist: link_exelist = compiler.get_linker_exelist() link_args = self.env.coredata.get_external_link_args(MachineChoice.HOST, lang) diff --git a/run_unittests.py b/run_unittests.py index 27c0677..21b6608 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -2606,8 +2606,10 @@ class AllPlatformTests(BasePlatformTests): if is_osx(): self.assertIsInstance(cc.linker, mesonbuild.linkers.AppleDynamicLinker) elif is_windows(): - # This is clang, not clang-cl - self.assertIsInstance(cc.linker, mesonbuild.linkers.MSVCDynamicLinker) + # This is clang, not clang-cl. This can be either an + # ld-like linker of link.exe-like linker (usually the + # former for msys2, the latter otherwise) + self.assertIsInstance(cc.linker, (mesonbuild.linkers.MSVCDynamicLinker, mesonbuild.linkers.GnuLikeDynamicLinkerMixin)) else: self.assertIsInstance(cc.linker, mesonbuild.linkers.GnuLikeDynamicLinkerMixin) if isinstance(cc, intel): @@ -2635,17 +2637,16 @@ class AllPlatformTests(BasePlatformTests): # something like `ccache gcc -pipe` or `distcc ccache gcc` works. wrapper = os.path.join(testdir, 'compiler wrapper.py') wrappercc = python_command + [wrapper] + cc.get_exelist() + ['-DSOME_ARG'] - wrappercc_s = '' - for w in wrappercc: - wrappercc_s += quote_arg(w) + ' ' - os.environ[evar] = wrappercc_s - wcc = getattr(env, 'detect_{}_compiler'.format(lang))(MachineChoice.HOST) + os.environ[evar] = ' '.join(quote_arg(w) for w in wrappercc) + # Check static linker too wrapperlinker = python_command + [wrapper] + linker.get_exelist() + linker.get_always_args() - wrapperlinker_s = '' - for w in wrapperlinker: - wrapperlinker_s += quote_arg(w) + ' ' - os.environ['AR'] = wrapperlinker_s + os.environ['AR'] = ' '.join(quote_arg(w) for w in wrapperlinker) + + # Need a new env to re-run environment loading + env = get_fake_env(testdir, self.builddir, self.prefix) + + wcc = getattr(env, 'detect_{}_compiler'.format(lang))(MachineChoice.HOST) wlinker = env.detect_static_linker(wcc) # Pop it so we don't use it for the next detection evalue = os.environ.pop('AR') @@ -5375,7 +5376,7 @@ class FailureTests(BasePlatformTests): def test_boost_BOOST_ROOT_dependency(self): # Test BOOST_ROOT; can be run even if Boost is found or not self.assertMesonRaises("dependency('boost')", - "(BOOST_ROOT.*absolute|{})".format(self.dnf), + "(boost_root.*absolute|{})".format(self.dnf), override_envvars = {'BOOST_ROOT': 'relative/path'}) def test_dependency_invalid_method(self): @@ -5687,12 +5688,12 @@ class WindowsTests(BasePlatformTests): def _check_ld(self, name: str, lang: str, expected: str) -> None: if not shutil.which(name): raise unittest.SkipTest('Could not find {}.'.format(name)) - envvars = [mesonbuild.envconfig.BinaryTable.evarMap['{}_ld'.format(lang)]] + envvars = [mesonbuild.envconfig.ENV_VAR_PROG_MAP['{}_ld'.format(lang)]] # Also test a deprecated variable if there is one. - if envvars[0] in mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP: + if f'{lang}_ld' in mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP: envvars.append( - mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP[envvars[0]]) + mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP[f'{lang}_ld']) for envvar in envvars: with mock.patch.dict(os.environ, {envvar: name}): @@ -7291,12 +7292,12 @@ class LinuxlikeTests(BasePlatformTests): raise unittest.SkipTest('Solaris currently cannot override the linker.') if not shutil.which(check): raise unittest.SkipTest('Could not find {}.'.format(check)) - envvars = [mesonbuild.envconfig.BinaryTable.evarMap['{}_ld'.format(lang)]] + envvars = [mesonbuild.envconfig.ENV_VAR_PROG_MAP['{}_ld'.format(lang)]] # Also test a deprecated variable if there is one. - if envvars[0] in mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP: + if f'{lang}_ld' in mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP: envvars.append( - mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP[envvars[0]]) + mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP[f'{lang}_ld']) for envvar in envvars: with mock.patch.dict(os.environ, {envvar: name}): @@ -8272,7 +8273,7 @@ class NativeFileTests(BasePlatformTests): return 'gfortran', 'gcc' self.helper_for_compiler('fortran', cb) - def _single_implementation_compiler(self, lang, binary, version_str, version): + def _single_implementation_compiler(self, lang: str, binary: str, version_str: str, version: str) -> None: """Helper for languages with a single (supported) implementation. Builds a wrapper around the compiler to override the version. @@ -8281,7 +8282,7 @@ class NativeFileTests(BasePlatformTests): env = get_fake_env() getter = getattr(env, 'detect_{}_compiler'.format(lang)) getter = functools.partial(getter, MachineChoice.HOST) - env.binaries.host.binaries[lang] = wrapper + env.binaries.host.binaries[lang] = [wrapper] compiler = getter() self.assertEqual(compiler.version, version) @@ -8308,7 +8309,7 @@ class NativeFileTests(BasePlatformTests): 'swiftc', version='Swift 1.2345', outfile='stderr', extra_args={'Xlinker': 'macosx_version. PROJECT:ld - 1.2.3'}) env = get_fake_env() - env.binaries.host.binaries['swift'] = wrapper + env.binaries.host.binaries['swift'] = [wrapper] compiler = env.detect_swift_compiler(MachineChoice.HOST) self.assertEqual(compiler.version, '1.2345') @@ -9345,7 +9346,7 @@ def unset_envs(): # For unit tests we must fully control all command lines # so that there are no unexpected changes coming from the # environment, for example when doing a package build. - varnames = ['CPPFLAGS', 'LDFLAGS'] + list(mesonbuild.compilers.compilers.cflags_mapping.values()) + varnames = ['CPPFLAGS', 'LDFLAGS'] + list(mesonbuild.compilers.compilers.CFLAGS_MAPPING.values()) for v in varnames: if v in os.environ: del os.environ[v] |