envconfig: Add [constants] section in machine files

Machine files already supports `+` operator as an implementation detail,
since it's using eval(). Now make it an officially supported feature and
add a way to define constants that are used while evaluating an entry
value.

6 files changed, 193 insertions, 44 deletions

diff --git a/docs/markdown/Machine-files.md b/docs/markdown/Machine-files.md
index 404c3d2..9011f79 100644
--- a/docs/markdown/Machine-files.md
+++ b/docs/markdown/Machine-files.md
@@ -8,10 +8,83 @@ environments](Native-environments.md).
 ## Sections
 
 The following sections are allowed:
+- constants
 - binaries
 - paths
 - properties
 
+### constants
+
+*Since 0.55.0*
+
+String and list concatenation is supported using the `+` operator, joining paths
+is supported using the `/` operator.
+Entries defined in the `[constants]` section can be used in any other section
+(they are always parsed first), entries in any other section can be used only
+within that same section and only after it has been defined.
+
+```ini
+[constants]
+toolchain = '/toolchain'
+common_flags = ['--sysroot=' + toolchain / 'sysroot']
+
+[properties]
+c_args = common_flags + ['-DSOMETHING']
+cpp_args = c_args + ['-DSOMETHING_ELSE']
+
+[binaries]
+c = toolchain / 'gcc'
+```
+
+This can be useful with cross file composition as well. A generic cross file
+could be composed with a platform specific file where constants are defined:
+```ini
+# aarch64.ini
+[constants]
+arch = 'aarch64-linux-gnu'
+```
+
+```ini
+# cross.ini
+[binaries]
+c = arch + '-gcc'
+cpp = arch + '-g++'
+strip = arch + '-strip'
+pkgconfig = arch + '-pkg-config'
+...
+```
+
+This can be used as `meson setup --cross-file aarch64.ini --cross-file cross.ini builddir`.
+
+Note that file composition happens before the parsing of values. The example
+below results in `b` being `'HelloWorld'`:
+```ini
+# file1.ini:
+[constants]
+a = 'Foo'
+b = a + 'World'
+```
+
+```ini
+#file2.ini:
+[constants]
+a = 'Hello'
+```
+
+The example below results in an error when file1.ini is included before file2.ini
+because `b` would be defined before `a`:
+```ini
+# file1.ini:
+[constants]
+b = a + 'World'
+```
+
+```ini
+#file2.ini:
+[constants]
+a = 'Hello'
+```
+
 ### Binaries
 
 The binaries section contains a list of binaries. These can be used
diff --git a/docs/markdown/snippets/machine_file_constants.md b/docs/markdown/snippets/machine_file_constants.md
new file mode 100644
index 0000000..84b0848
--- /dev/null
+++ b/docs/markdown/snippets/machine_file_constants.md
@@ -0,0 +1,20 @@
+## Machine file constants
+
+Native and cross files now support string and list concatenation using the `+`
+operator, and joining paths using the `/` operator.
+Entries defined in the `[constants]` section can be used in any other section.
+An entry defined in any other section can be used only within that same section and only
+after it has been defined.
+
+```ini
+[constants]
+toolchain = '/toolchain'
+common_flags = ['--sysroot=' + toolchain + '/sysroot']
+
+[properties]
+c_args = common_flags + ['-DSOMETHING']
+cpp_args = c_args + ['-DSOMETHING_ELSE']
+
+[binaries]
+c = toolchain + '/gcc'
+```
diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py
index 94f977f..329c333 100644
--- a/mesonbuild/coredata.py
+++ b/mesonbuild/coredata.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from . import mlog
+from . import mlog, mparser
 import pickle, os, uuid
 import sys
 from itertools import chain
@@ -229,14 +229,6 @@ class UserFeatureOption(UserComboOption):
     def is_auto(self):
         return self.value == 'auto'
 
-
-def load_configs(filenames: T.List[str]) -> configparser.ConfigParser:
-    """Load configuration files from a named subdirectory."""
-    config = configparser.ConfigParser(interpolation=None)
-    config.read(filenames)
-    return config
-
-
 if T.TYPE_CHECKING:
     CacheKeyType = T.Tuple[T.Tuple[T.Any, ...], ...]
     SubCacheKeyType = T.Tuple[T.Any, ...]
@@ -879,6 +871,69 @@ class CmdLineFileParser(configparser.ConfigParser):
         # storing subproject options like "subproject:option=value"
         super().__init__(delimiters=['='], interpolation=None)
 
+class MachineFileParser():
+    def __init__(self, filenames: T.List[str]):
+        self.parser = CmdLineFileParser()
+        self.constants = {'True': True, 'False': False}
+        self.sections = {}
+
+        self.parser.read(filenames)
+
+        # Parse [constants] first so they can be used in other sections
+        if self.parser.has_section('constants'):
+            self.constants.update(self._parse_section('constants'))
+
+        for s in self.parser.sections():
+            if s == 'constants':
+                continue
+            self.sections[s] = self._parse_section(s)
+
+    def _parse_section(self, s):
+        self.scope = self.constants.copy()
+        section = {}
+        for entry, value in self.parser.items(s):
+            if ' ' in entry or '\t' in entry or "'" in entry or '"' in entry:
+                raise EnvironmentException('Malformed variable name {!r} in machine file.'.format(entry))
+            # Windows paths...
+            value = value.replace('\\', '\\\\')
+            try:
+                ast = mparser.Parser(value, 'machinefile').parse()
+                res = self._evaluate_statement(ast.lines[0])
+            except MesonException:
+                raise EnvironmentException('Malformed value in machine file variable {!r}.'.format(entry))
+            except KeyError as e:
+                raise EnvironmentException('Undefined constant {!r} in machine file variable {!r}.'.format(e.args[0], entry))
+            section[entry] = res
+            self.scope[entry] = res
+        return section
+
+    def _evaluate_statement(self, node):
+        if isinstance(node, (mparser.StringNode)):
+            return node.value
+        elif isinstance(node, mparser.BooleanNode):
+            return node.value
+        elif isinstance(node, mparser.NumberNode):
+            return node.value
+        elif isinstance(node, mparser.ArrayNode):
+            return [self._evaluate_statement(arg) for arg in node.args.arguments]
+        elif isinstance(node, mparser.IdNode):
+            return self.scope[node.value]
+        elif isinstance(node, mparser.ArithmeticNode):
+            l = self._evaluate_statement(node.left)
+            r = self._evaluate_statement(node.right)
+            if node.operation == 'add':
+                if (isinstance(l, str) and isinstance(r, str)) or \
+                   (isinstance(l, list) and isinstance(r, list)):
+                    return l + r
+            elif node.operation == 'div':
+                if isinstance(l, str) and isinstance(r, str):
+                    return os.path.join(l, r)
+        raise EnvironmentException('Unsupported node type')
+
+def parse_machine_files(filenames):
+    parser = MachineFileParser(filenames)
+    return parser.sections
+
 def get_cmd_line_file(build_dir):
     return os.path.join(build_dir, 'meson-private', 'cmd_line.txt')
 
diff --git a/mesonbuild/envconfig.py b/mesonbuild/envconfig.py
index 10464a2..219b62e 100644
--- a/mesonbuild/envconfig.py
+++ b/mesonbuild/envconfig.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import configparser, os, subprocess
+import os, subprocess
 import typing as T
 
 from . import mesonlib
@@ -83,33 +83,6 @@ CPU_FAMILES_64_BIT = [
     'x86_64',
 ]
 
-class MesonConfigFile:
-    @classmethod
-    def from_config_parser(cls, parser: configparser.ConfigParser) -> T.Dict[str, T.Dict[str, T.Dict[str, str]]]:
-        out = {}
-        # This is a bit hackish at the moment.
-        for s in parser.sections():
-            section = {}
-            for entry in parser[s]:
-                value = parser[s][entry]
-                # Windows paths...
-                value = value.replace('\\', '\\\\')
-                if ' ' in entry or '\t' in entry or "'" in entry or '"' in entry:
-                    raise EnvironmentException('Malformed variable name {} in cross file..'.format(entry))
-                try:
-                    res = eval(value, {'__builtins__': None}, {'true': True, 'false': False})
-                except Exception:
-                    raise EnvironmentException('Malformed value in cross file variable {}.'.format(entry))
-
-                for i in (res if isinstance(res, list) else [res]):
-                    if not isinstance(i, (str, int, bool)):
-                        raise EnvironmentException('Malformed value in cross file variable {}.'.format(entry))
-
-                section[entry] = res
-
-            out[s] = section
-        return out
-
 def get_env_var_pair(for_machine: MachineChoice,
                      is_cross: bool,
                      var_name: str) -> T.Tuple[T.Optional[str], T.Optional[str]]:
diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py
index afc2a63..d1cbfe7 100644
--- a/mesonbuild/environment.py
+++ b/mesonbuild/environment.py
@@ -27,7 +27,7 @@ from .mesonlib import (
 from . import mlog
 
 from .envconfig import (
-    BinaryTable, Directories, MachineInfo, MesonConfigFile,
+    BinaryTable, Directories, MachineInfo,
     Properties, known_cpu_families,
 )
 from . import compilers
@@ -563,8 +563,7 @@ class Environment:
         ## Read in native file(s) to override build machine configuration
 
         if self.coredata.config_files is not None:
-            config = MesonConfigFile.from_config_parser(
-                coredata.load_configs(self.coredata.config_files))
+            config = coredata.parse_machine_files(self.coredata.config_files)
             binaries.build = BinaryTable(config.get('binaries', {}))
             paths.build = Directories(**config.get('paths', {}))
             properties.build = Properties(config.get('properties', {}))
@@ -572,8 +571,7 @@ class Environment:
         ## Read in cross file(s) to override host machine configuration
 
         if self.coredata.cross_files:
-            config = MesonConfigFile.from_config_parser(
-                coredata.load_configs(self.coredata.cross_files))
+            config = coredata.parse_machine_files(self.coredata.cross_files)
             properties.host = Properties(config.get('properties', {}))
             binaries.host = BinaryTable(config.get('binaries', {}))
             if 'host_machine' in config:
diff --git a/run_unittests.py b/run_unittests.py
index 8a12180..6cc6302 100755
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -3915,7 +3915,7 @@ recommended as it is not supported on some platforms''')
         with tempfile.NamedTemporaryFile(mode='w', delete=False) as crossfile:
             crossfile.write(textwrap.dedent(
                 '''[binaries]
-                pkgconfig = r'{0}'
+                pkgconfig = '{0}'
 
                 [properties]
 
@@ -3945,7 +3945,7 @@ recommended as it is not supported on some platforms''')
                 pkgconfig = 'pkg-config'
 
                 [properties]
-                pkg_config_libdir = [r'{0}']
+                pkg_config_libdir = ['{0}']
 
                 [host_machine]
                 system = 'linux'
@@ -4969,6 +4969,36 @@ recommended as it is not supported on some platforms''')
         self.run_tests()
         self.run_target('coverage-xml')
 
+    def test_cross_file_constants(self):
+        with temp_filename() as crossfile1, temp_filename() as crossfile2:
+            with open(crossfile1, 'w') as f:
+                f.write(textwrap.dedent(
+                    '''
+                    [constants]
+                    compiler = 'gcc'
+                    '''))
+            with open(crossfile2, 'w') as f:
+                f.write(textwrap.dedent(
+                    '''
+                    [constants]
+                    toolchain = '/toolchain/'
+                    common_flags = ['--sysroot=' + toolchain / 'sysroot']
+
+                    [properties]
+                    c_args = common_flags + ['-DSOMETHING']
+                    cpp_args = c_args + ['-DSOMETHING_ELSE']
+
+                    [binaries]
+                    c = toolchain / compiler
+                    '''))
+
+            values = mesonbuild.coredata.parse_machine_files([crossfile1, crossfile2])
+            self.assertEqual(values['binaries']['c'], '/toolchain/gcc')
+            self.assertEqual(values['properties']['c_args'],
+                             ['--sysroot=/toolchain/sysroot', '-DSOMETHING'])
+            self.assertEqual(values['properties']['cpp_args'],
+                             ['--sysroot=/toolchain/sysroot', '-DSOMETHING', '-DSOMETHING_ELSE'])
+
 class FailureTests(BasePlatformTests):
     '''
     Tests that test failure conditions. Build files here should be dynamically