Put machine file and cmd line parsing in Environment

This creates a full set of option in environment that mirror those in
coredata, this mirroring of the coredata structure is convenient because
lookups int env (such as when initializing compilers) becomes a straight
dict lookup, with no list iteration. It also means that all of the
command line and machine files are read and stored in the correct order
before they're ever accessed, simplifying the logic of using them.

1 files changed, 122 insertions, 8 deletions

diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py
index 7dfffa2..9830b45 100644
--- a/mesonbuild/environment.py
+++ b/mesonbuild/environment.py
@@ -16,6 +16,7 @@ import os, platform, re, sys, shutil, subprocess
 import tempfile
 import shlex
 import typing as T
+import collections
 
 from . import coredata
 from .linkers import ArLinker, ArmarLinker, VisualStudioLinker, DLinker, CcrxLinker, Xc16Linker, C2000Linker, IntelVisualStudioLinker
@@ -28,11 +29,13 @@ from . import mlog
 
 from .envconfig import (
     BinaryTable, MachineInfo,
-    Properties, known_cpu_families,
+    Properties, known_cpu_families, get_env_var_pair,
 )
 from . import compilers
 from .compilers import (
     Compiler,
+    all_languages,
+    base_options,
     is_assembly,
     is_header,
     is_library,
@@ -549,10 +552,10 @@ class Environment:
         properties = PerMachineDefaultable()
 
         # We only need one of these as project options are not per machine
-        user_options = {}
+        user_options = collections.defaultdict(dict)  # type: T.DefaultDict[str, T.Dict[str, object]]
 
         # meson builtin options, as passed through cross or native files
-        meson_options = PerMachineDefaultable()
+        meson_options = PerMachineDefaultable()  # type: PerMachineDefaultable[T.DefaultDict[str, T.Dict[str, object]]]
 
         ## Setup build machine defaults
 
@@ -564,6 +567,13 @@ class Environment:
         binaries.build = BinaryTable()
         properties.build = Properties()
 
+        # meson base options
+        _base_options = {}  # type: T.Dict[str, object]
+
+        # Per language compiler arguments
+        compiler_options = PerMachineDefaultable()  # type: PerMachineDefaultable[T.DefaultDict[str, T.Dict[str, object]]]
+        compiler_options.build = collections.defaultdict(dict)
+
         ## Read in native file(s) to override build machine configuration
 
         def load_options(tag: str, store: T.Dict[str, T.Any]) -> None:
@@ -573,7 +583,44 @@ class Environment:
                         project = section.split(':')[0]
                     else:
                         project = ''
-                    store[project] = config.get(section, {})
+                    store[project].update(config.get(section, {}))
+
+        def split_base_options(mopts: T.DefaultDict[str, T.Dict[str, object]]) -> None:
+            for k, v in list(mopts.get('', {}).items()):
+                if k in base_options:
+                    _base_options[k] = v
+                    del mopts[k]
+
+        lang_prefixes = tuple('{}_'.format(l) for l in all_languages)
+        def split_compiler_options(mopts: T.DefaultDict[str, T.Dict[str, object]], machine: MachineChoice) -> None:
+            for k, v in list(mopts.get('', {}).items()):
+                if k.startswith(lang_prefixes):
+                    lang, key = k.split('_', 1)
+                    if compiler_options[machine] is None:
+                        compiler_options[machine] = collections.defaultdict(dict)
+                    if lang not in compiler_options[machine]:
+                        compiler_options[machine][lang] = collections.defaultdict(dict)
+                    compiler_options[machine][lang][key] = v
+                    del mopts[''][k]
+
+        def move_compiler_options(properties: Properties, compopts: T.Dict[str, T.DefaultDict[str, object]]) -> None:
+            for k, v in properties.properties.copy().items():
+                for lang in all_languages:
+                    if k == '{}_args'.format(lang):
+                        if 'args' not in compopts[lang]:
+                            compopts[lang]['args'] = v
+                        else:
+                            mlog.warning('Ignoring {}_args in [properties] section for those in the [built-in options]'.format(lang))
+                    elif k == '{}_link_args'.format(lang):
+                        if 'link_args' not in compopts[lang]:
+                            compopts[lang]['link_args'] = v
+                        else:
+                            mlog.warning('Ignoring {}_link_args in [properties] section in favor of the [built-in options] section.')
+                    else:
+                        continue
+                    mlog.deprecation('{} in the [properties] section of the machine file is deprecated, use the [built-in options] section.'.format(k))
+                    del properties.properties[k]
+                    break
 
         if self.coredata.config_files is not None:
             config = coredata.parse_machine_files(self.coredata.config_files)
@@ -584,11 +631,15 @@ class Environment:
             # the native values if we're doing a cross build
             if not self.coredata.cross_files:
                 load_options('project options', user_options)
-            meson_options.build = {}
+            meson_options.build = collections.defaultdict(dict)
             if config.get('paths') is not None:
                 mlog.deprecation('The [paths] section is deprecated, use the [built-in options] section instead.')
                 load_options('paths', meson_options.build)
             load_options('built-in options', meson_options.build)
+            if not self.coredata.cross_files:
+                split_base_options(meson_options.build)
+            split_compiler_options(meson_options.build, MachineChoice.BUILD)
+            move_compiler_options(properties.build, compiler_options.build)
 
         ## Read in cross file(s) to override host machine configuration
 
@@ -601,11 +652,15 @@ class Environment:
             if 'target_machine' in config:
                 machines.target = MachineInfo.from_literal(config['target_machine'])
             load_options('project options', user_options)
-            meson_options.host = {}
+            meson_options.host = collections.defaultdict(dict)
+            compiler_options.host = collections.defaultdict(dict)
             if config.get('paths') is not None:
                 mlog.deprecation('The [paths] section is deprecated, use the [built-in options] section instead.')
                 load_options('paths', meson_options.host)
             load_options('built-in options', meson_options.host)
+            split_base_options(meson_options.host)
+            split_compiler_options(meson_options.host, MachineChoice.HOST)
+            move_compiler_options(properties.host, compiler_options.host)
 
         ## "freeze" now initialized configuration, and "save" to the class.
 
@@ -614,6 +669,67 @@ class Environment:
         self.properties = properties.default_missing()
         self.user_options = user_options
         self.meson_options = meson_options.default_missing()
+        self.base_options = _base_options
+        self.compiler_options = compiler_options.default_missing()
+
+        # Some options default to environment variables if they are
+        # unset, set those now.
+
+        for for_machine in MachineChoice:
+            p_env_pair = get_env_var_pair(for_machine, self.coredata.is_cross_build(), 'PKG_CONFIG_PATH')
+            if p_env_pair is not None:
+                p_env_var, p_env = p_env_pair
+
+                # PKG_CONFIG_PATH may contain duplicates, which must be
+                # removed, else a duplicates-in-array-option warning arises.
+                p_list = list(mesonlib.OrderedSet(p_env.split(':')))
+
+                key = 'pkg_config_path'
+
+                if self.first_invocation:
+                    # Environment variables override config
+                    self.meson_options[for_machine][''][key] = p_list
+                elif self.meson_options[for_machine][''].get(key, []) != p_list:
+                    mlog.warning(
+                        p_env_var,
+                        'environment variable does not match configured',
+                        'between configurations, meson ignores this.',
+                        'Use -Dpkg_config_path to change pkg-config search',
+                        'path instead.'
+                    )
+
+        # Read in command line and populate options
+        # TODO: validate all of this
+        all_builtins = set(coredata.builtin_options) | set(coredata.builtin_options_per_machine) | set(coredata.builtin_dir_noprefix_options)
+        for k, v in options.cmd_line_options.items():
+            try:
+                subproject, k = k.split(':')
+            except ValueError:
+                subproject = ''
+            if k in base_options:
+                self.base_options[k] = v
+            elif k.startswith(lang_prefixes):
+                lang, key = k.split('_', 1)
+                self.compiler_options.host[lang][key] = v
+            elif k in all_builtins or k.startswith('backend_'):
+                self.meson_options.host[subproject][k] = v
+            elif k.startswith('build.'):
+                k = k.lstrip('build.')
+                if k in coredata.builtin_options_per_machine:
+                    if self.meson_options.build is None:
+                        self.meson_options.build = collections.defaultdict(dict)
+                    self.meson_options.build[subproject][k] = v
+            else:
+                assert not k.startswith('build.')
+                self.user_options[subproject][k] = v
+
+        # Warn if the user is using two different ways of setting build-type
+        # options that override each other
+        if meson_options.build and 'buildtype' in meson_options.build[''] and \
+           ('optimization' in meson_options.build[''] or 'debug' in meson_options.build['']):
+            mlog.warning('Recommend using either -Dbuildtype or -Doptimization + -Ddebug. '
+                         'Using both is redundant since they override each other. '
+                         'See: https://mesonbuild.com/Builtin-options.html#build-type-options')
 
         exe_wrapper = self.lookup_binary_entry(MachineChoice.HOST, 'exe_wrapper')
         if exe_wrapper is not None:
@@ -622,8 +738,6 @@ class Environment:
         else:
             self.exe_wrapper = None
 
-        self.cmd_line_options = options.cmd_line_options.copy()
-
         # List of potential compilers.
         if mesonlib.is_windows():
             # Intel C and C++ compiler is icl on Windows, but icc and icpc elsewhere.