Merge pull request #13286 from mesonbuild/machinerefactor

Extract MachineFile code to its own file

10 files changed, 175 insertions, 135 deletions

diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py
index 76da0b6..782e770 100644
--- a/mesonbuild/coredata.py
+++ b/mesonbuild/coredata.py
@@ -6,7 +6,7 @@ from __future__ import annotations
 
 import copy
 
-from . import mlog, mparser, options
+from . import mlog, options
 import pickle, os, uuid
 import sys
 from itertools import chain
@@ -16,14 +16,16 @@ from dataclasses import dataclass
 
 from .mesonlib import (
     MesonBugException,
-    MesonException, EnvironmentException, MachineChoice, PerMachine,
+    MesonException, MachineChoice, PerMachine,
     PerMachineDefaultable,
     OptionKey, OptionType, stringlistify,
     pickle_load
 )
+
+from .machinefile import CmdLineFileParser
+
 import ast
 import argparse
-import configparser
 import enum
 import shlex
 import typing as T
@@ -760,99 +762,6 @@ class CoreData:
             mlog.warning('Base option \'b_bitcode\' is enabled, which is incompatible with many linker options. Incompatible options such as \'b_asneeded\' have been disabled.', fatal=False)
             mlog.warning('Please see https://mesonbuild.com/Builtin-options.html#Notes_about_Apple_Bitcode_support for more details.', fatal=False)
 
-class CmdLineFileParser(configparser.ConfigParser):
-    def __init__(self) -> None:
-        # We don't want ':' as key delimiter, otherwise it would break when
-        # storing subproject options like "subproject:option=value"
-        super().__init__(delimiters=['='], interpolation=None)
-
-    def read(self, filenames: T.Union['StrOrBytesPath', T.Iterable['StrOrBytesPath']], encoding: T.Optional[str] = 'utf-8') -> T.List[str]:
-        return super().read(filenames, encoding)
-
-    def optionxform(self, optionstr: str) -> str:
-        # Don't call str.lower() on keys
-        return optionstr
-
-class MachineFileParser():
-    def __init__(self, filenames: T.List[str], sourcedir: str) -> None:
-        self.parser = CmdLineFileParser()
-        self.constants: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {'True': True, 'False': False}
-        self.sections: T.Dict[str, T.Dict[str, T.Union[str, bool, int, T.List[str]]]] = {}
-
-        for fname in filenames:
-            try:
-                with open(fname, encoding='utf-8') as f:
-                    content = f.read()
-            except UnicodeDecodeError as e:
-                raise EnvironmentException(f'Malformed machine file {fname!r} failed to parse as unicode: {e}')
-
-            content = content.replace('@GLOBAL_SOURCE_ROOT@', sourcedir)
-            content = content.replace('@DIRNAME@', os.path.dirname(fname))
-            try:
-                self.parser.read_string(content, fname)
-            except configparser.Error as e:
-                raise EnvironmentException(f'Malformed machine file: {e}')
-
-        # Parse [constants] first so they can be used in other sections
-        if self.parser.has_section('constants'):
-            self.constants.update(self._parse_section('constants'))
-
-        for s in self.parser.sections():
-            if s == 'constants':
-                continue
-            self.sections[s] = self._parse_section(s)
-
-    def _parse_section(self, s: str) -> T.Dict[str, T.Union[str, bool, int, T.List[str]]]:
-        self.scope = self.constants.copy()
-        section: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {}
-        for entry, value in self.parser.items(s):
-            if ' ' in entry or '\t' in entry or "'" in entry or '"' in entry:
-                raise EnvironmentException(f'Malformed variable name {entry!r} in machine file.')
-            # Windows paths...
-            value = value.replace('\\', '\\\\')
-            try:
-                ast = mparser.Parser(value, 'machinefile').parse()
-                if not ast.lines:
-                    raise EnvironmentException('value cannot be empty')
-                res = self._evaluate_statement(ast.lines[0])
-            except MesonException as e:
-                raise EnvironmentException(f'Malformed value in machine file variable {entry!r}: {str(e)}.')
-            except KeyError as e:
-                raise EnvironmentException(f'Undefined constant {e.args[0]!r} in machine file variable {entry!r}.')
-            section[entry] = res
-            self.scope[entry] = res
-        return section
-
-    def _evaluate_statement(self, node: mparser.BaseNode) -> T.Union[str, bool, int, T.List[str]]:
-        if isinstance(node, (mparser.StringNode)):
-            return node.value
-        elif isinstance(node, mparser.BooleanNode):
-            return node.value
-        elif isinstance(node, mparser.NumberNode):
-            return node.value
-        elif isinstance(node, mparser.ParenthesizedNode):
-            return self._evaluate_statement(node.inner)
-        elif isinstance(node, mparser.ArrayNode):
-            # TODO: This is where recursive types would come in handy
-            return [self._evaluate_statement(arg) for arg in node.args.arguments]
-        elif isinstance(node, mparser.IdNode):
-            return self.scope[node.value]
-        elif isinstance(node, mparser.ArithmeticNode):
-            l = self._evaluate_statement(node.left)
-            r = self._evaluate_statement(node.right)
-            if node.operation == 'add':
-                if (isinstance(l, str) and isinstance(r, str)) or \
-                   (isinstance(l, list) and isinstance(r, list)):
-                    return l + r
-            elif node.operation == 'div':
-                if isinstance(l, str) and isinstance(r, str):
-                    return os.path.join(l, r)
-        raise EnvironmentException('Unsupported node type')
-
-def parse_machine_files(filenames: T.List[str], sourcedir: str):
-    parser = MachineFileParser(filenames, sourcedir)
-    return parser.sections
-
 def get_cmd_line_file(build_dir: str) -> str:
     return os.path.join(build_dir, 'meson-private', 'cmd_line.txt')
 
diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py
index 19b9e81..1607c32 100644
--- a/mesonbuild/environment.py
+++ b/mesonbuild/environment.py
@@ -11,6 +11,10 @@ import collections
 
 from . import coredata
 from . import mesonlib
+from . import machinefile
+
+CmdLineFileParser = machinefile.CmdLineFileParser
+
 from .mesonlib import (
     MesonException, MachineChoice, Popen_safe, PerMachine,
     PerMachineDefaultable, PerThreeMachineDefaultable, split_args, quote_arg, OptionKey,
@@ -589,7 +593,7 @@ class Environment:
         ## Read in native file(s) to override build machine configuration
 
         if self.coredata.config_files is not None:
-            config = coredata.parse_machine_files(self.coredata.config_files, self.source_dir)
+            config = machinefile.parse_machine_files(self.coredata.config_files, self.source_dir)
             binaries.build = BinaryTable(config.get('binaries', {}))
             properties.build = Properties(config.get('properties', {}))
             cmakevars.build = CMakeVariables(config.get('cmake', {}))
@@ -600,7 +604,7 @@ class Environment:
         ## Read in cross file(s) to override host machine configuration
 
         if self.coredata.cross_files:
-            config = coredata.parse_machine_files(self.coredata.cross_files, self.source_dir)
+            config = machinefile.parse_machine_files(self.coredata.cross_files, self.source_dir)
             properties.host = Properties(config.get('properties', {}))
             binaries.host = BinaryTable(config.get('binaries', {}))
             cmakevars.host = CMakeVariables(config.get('cmake', {}))
diff --git a/mesonbuild/machinefile.py b/mesonbuild/machinefile.py
new file mode 100644
index 0000000..afeb4d0
--- /dev/null
+++ b/mesonbuild/machinefile.py
@@ -0,0 +1,117 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2013-2024 Contributors to the The Meson project
+
+import typing as T
+import configparser
+import os
+
+from . import mparser
+
+from .mesonlib import MesonException
+
+if T.TYPE_CHECKING:
+    from .compilers import Compiler
+    from .coredata import StrOrBytesPath
+
+    CompilersDict = T.Dict[str, Compiler]
+
+
+class CmdLineFileParser(configparser.ConfigParser):
+    def __init__(self) -> None:
+        # We don't want ':' as key delimiter, otherwise it would break when
+        # storing subproject options like "subproject:option=value"
+        super().__init__(delimiters=['='], interpolation=None)
+
+    def read(self, filenames: T.Union['StrOrBytesPath', T.Iterable['StrOrBytesPath']], encoding: T.Optional[str] = 'utf-8') -> T.List[str]:
+        return super().read(filenames, encoding)
+
+    def optionxform(self, optionstr: str) -> str:
+        # Don't call str.lower() on keys
+        return optionstr
+
+
+class MachineFileParser():
+    def __init__(self, filenames: T.List[str], sourcedir: str) -> None:
+        self.parser = CmdLineFileParser()
+        self.constants: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {'True': True, 'False': False}
+        self.sections: T.Dict[str, T.Dict[str, T.Union[str, bool, int, T.List[str]]]] = {}
+
+        for fname in filenames:
+            try:
+                with open(fname, encoding='utf-8') as f:
+                    content = f.read()
+            except UnicodeDecodeError as e:
+                raise MesonException(f'Malformed machine file {fname!r} failed to parse as unicode: {e}')
+
+            content = content.replace('@GLOBAL_SOURCE_ROOT@', sourcedir)
+            content = content.replace('@DIRNAME@', os.path.dirname(fname))
+            try:
+                self.parser.read_string(content, fname)
+            except configparser.Error as e:
+                raise MesonException(f'Malformed machine file: {e}')
+
+        # Parse [constants] first so they can be used in other sections
+        if self.parser.has_section('constants'):
+            self.constants.update(self._parse_section('constants'))
+
+        for s in self.parser.sections():
+            if s == 'constants':
+                continue
+            self.sections[s] = self._parse_section(s)
+
+    def _parse_section(self, s: str) -> T.Dict[str, T.Union[str, bool, int, T.List[str]]]:
+        self.scope = self.constants.copy()
+        section: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {}
+        for entry, value in self.parser.items(s):
+            if ' ' in entry or '\t' in entry or "'" in entry or '"' in entry:
+                raise MesonException(f'Malformed variable name {entry!r} in machine file.')
+            # Windows paths...
+            value = value.replace('\\', '\\\\')
+            try:
+                ast = mparser.Parser(value, 'machinefile').parse()
+                if not ast.lines:
+                    raise MesonException('value cannot be empty')
+                res = self._evaluate_statement(ast.lines[0])
+            except MesonException as e:
+                raise MesonException(f'Malformed value in machine file variable {entry!r}: {str(e)}.')
+            except KeyError as e:
+                raise MesonException(f'Undefined constant {e.args[0]!r} in machine file variable {entry!r}.')
+            section[entry] = res
+            self.scope[entry] = res
+        return section
+
+    def _evaluate_statement(self, node: mparser.BaseNode) -> T.Union[str, bool, int, T.List[str]]:
+        if isinstance(node, (mparser.StringNode)):
+            return node.value
+        elif isinstance(node, mparser.BooleanNode):
+            return node.value
+        elif isinstance(node, mparser.NumberNode):
+            return node.value
+        elif isinstance(node, mparser.ParenthesizedNode):
+            return self._evaluate_statement(node.inner)
+        elif isinstance(node, mparser.ArrayNode):
+            # TODO: This is where recursive types would come in handy
+            return [self._evaluate_statement(arg) for arg in node.args.arguments]
+        elif isinstance(node, mparser.IdNode):
+            return self.scope[node.value]
+        elif isinstance(node, mparser.ArithmeticNode):
+            l = self._evaluate_statement(node.left)
+            r = self._evaluate_statement(node.right)
+            if node.operation == 'add':
+                if (isinstance(l, str) and isinstance(r, str)) or \
+                   (isinstance(l, list) and isinstance(r, list)):
+                    return l + r
+            elif node.operation == 'div':
+                if isinstance(l, str) and isinstance(r, str):
+                    return os.path.join(l, r)
+        raise MesonException('Unsupported node type')
+
+def parse_machine_files(filenames: T.List[str], sourcedir: str):
+    parser = MachineFileParser(filenames, sourcedir)
+    return parser.sections
+
+
+class MachineFileStore:
+    def __init__(self, native_files, cross_files, source_dir):
+        self.native = MachineFileParser(native_files if native_files is not None else [], source_dir).sections
+        self.cross = MachineFileParser(cross_files if cross_files is not None else [], source_dir).sections
diff --git a/mesonbuild/scripts/scanbuild.py b/mesonbuild/scripts/scanbuild.py
index d7fbcf4..b738aee 100644
--- a/mesonbuild/scripts/scanbuild.py
+++ b/mesonbuild/scripts/scanbuild.py
@@ -7,7 +7,8 @@ import subprocess
 import shutil
 import tempfile
 from ..environment import detect_ninja, detect_scanbuild
-from ..coredata import get_cmd_line_file, CmdLineFileParser
+from ..coredata import get_cmd_line_file
+from ..machinefile import CmdLineFileParser
 from ..mesonlib import windows_proof_rmtree
 from pathlib import Path
 import typing as T
diff --git a/test cases/unit/116 empty project/expected_mods.json b/test cases/unit/116 empty project/expected_mods.json
index 19f56a5..fa5e0ec 100644
--- a/test cases/unit/116 empty project/expected_mods.json
+++ b/test cases/unit/116 empty project/expected_mods.json
@@ -217,6 +217,7 @@
       "mesonbuild.linkers",
       "mesonbuild.linkers.base",
       "mesonbuild.linkers.detect",
+      "mesonbuild.machinefile",
       "mesonbuild.mesonlib",
       "mesonbuild.mesonmain",
       "mesonbuild.mintro",
diff --git a/unittests/allplatformstests.py b/unittests/allplatformstests.py
index 9c9f616..63445ec 100644
--- a/unittests/allplatformstests.py
+++ b/unittests/allplatformstests.py
@@ -25,6 +25,7 @@ import mesonbuild.dependencies.factory
 import mesonbuild.envconfig
 import mesonbuild.environment
 import mesonbuild.coredata
+import mesonbuild.machinefile
 import mesonbuild.modules.gnome
 from mesonbuild.mesonlib import (
     BuildDirLock, MachineChoice, is_windows, is_osx, is_cygwin, is_dragonflybsd,
@@ -61,6 +62,8 @@ from run_tests import (
 from .baseplatformtests import BasePlatformTests
 from .helpers import *
 
+UNIT_MACHINEFILE_DIR = Path(__file__).parent / 'machinefiles'
+
 @contextmanager
 def temp_filename():
     '''A context manager which provides a filename to an empty temporary file.
@@ -4111,40 +4114,19 @@ class AllPlatformTests(BasePlatformTests):
         self._check_coverage_files()
 
     def test_cross_file_constants(self):
-        with temp_filename() as crossfile1, temp_filename() as crossfile2:
-            with open(crossfile1, 'w', encoding='utf-8') as f:
-                f.write(textwrap.dedent(
-                    '''
-                    [constants]
-                    compiler = 'gcc'
-                    '''))
-            with open(crossfile2, 'w', encoding='utf-8') as f:
-                f.write(textwrap.dedent(
-                    '''
-                    [constants]
-                    toolchain = '/toolchain/'
-                    common_flags = ['--sysroot=' + toolchain / 'sysroot']
-
-                    [properties]
-                    c_args = common_flags + ['-DSOMETHING']
-                    cpp_args = c_args + ['-DSOMETHING_ELSE']
-                    rel_to_src = '@GLOBAL_SOURCE_ROOT@' / 'tool'
-                    rel_to_file = '@DIRNAME@' / 'tool'
-                    no_escaping = '@@DIRNAME@@' / 'tool'
-
-                    [binaries]
-                    c = toolchain / compiler
-                    '''))
-
-            values = mesonbuild.coredata.parse_machine_files([crossfile1, crossfile2], self.builddir)
-            self.assertEqual(values['binaries']['c'], '/toolchain/gcc')
-            self.assertEqual(values['properties']['c_args'],
-                             ['--sysroot=/toolchain/sysroot', '-DSOMETHING'])
-            self.assertEqual(values['properties']['cpp_args'],
-                             ['--sysroot=/toolchain/sysroot', '-DSOMETHING', '-DSOMETHING_ELSE'])
-            self.assertEqual(values['properties']['rel_to_src'], os.path.join(self.builddir, 'tool'))
-            self.assertEqual(values['properties']['rel_to_file'], os.path.join(os.path.dirname(crossfile2), 'tool'))
-            self.assertEqual(values['properties']['no_escaping'], os.path.join(f'@{os.path.dirname(crossfile2)}@', 'tool'))
+        crossfile1 = UNIT_MACHINEFILE_DIR / 'constant1.txt'
+        crossfile2 = UNIT_MACHINEFILE_DIR / 'constant2.txt'
+        values = mesonbuild.machinefile.parse_machine_files([crossfile1,
+                                                             crossfile2],
+                                                             self.builddir)
+        self.assertEqual(values['binaries']['c'], '/toolchain/gcc')
+        self.assertEqual(values['properties']['c_args'],
+                         ['--sysroot=/toolchain/sysroot', '-DSOMETHING'])
+        self.assertEqual(values['properties']['cpp_args'],
+                         ['--sysroot=/toolchain/sysroot', '-DSOMETHING', '-DSOMETHING_ELSE'])
+        self.assertEqual(values['properties']['rel_to_src'], os.path.join(self.builddir, 'tool'))
+        self.assertEqual(values['properties']['rel_to_file'], os.path.join(os.path.dirname(crossfile2), 'tool'))
+        self.assertEqual(values['properties']['no_escaping'], os.path.join(f'@{os.path.dirname(crossfile2)}@', 'tool'))
 
     @skipIf(is_windows(), 'Directory cleanup fails for some reason')
     def test_wrap_git(self):
diff --git a/unittests/machinefiles/constant1.txt b/unittests/machinefiles/constant1.txt
new file mode 100644
index 0000000..eeba7cb
--- /dev/null
+++ b/unittests/machinefiles/constant1.txt
@@ -0,0 +1,2 @@
+[constants]
+compiler = 'gcc'
diff --git a/unittests/machinefiles/constant2.txt b/unittests/machinefiles/constant2.txt
new file mode 100644
index 0000000..226dcc8
--- /dev/null
+++ b/unittests/machinefiles/constant2.txt
@@ -0,0 +1,13 @@
+[constants]
+toolchain = '/toolchain/'
+common_flags = ['--sysroot=' + toolchain / 'sysroot']
+
+[properties]
+c_args = common_flags + ['-DSOMETHING']
+cpp_args = c_args + ['-DSOMETHING_ELSE']
+rel_to_src = '@GLOBAL_SOURCE_ROOT@' / 'tool'
+rel_to_file = '@DIRNAME@' / 'tool'
+no_escaping = '@@DIRNAME@@' / 'tool'
+
+[binaries]
+c = toolchain / compiler
diff --git a/unittests/machinefiletests.py b/unittests/machinefiletests.py
index 22341cb..5ff862c 100644
--- a/unittests/machinefiletests.py
+++ b/unittests/machinefiletests.py
@@ -12,7 +12,7 @@ import functools
 import threading
 import sys
 from itertools import chain
-from unittest import mock, skipIf, SkipTest
+from unittest import mock, skipIf, SkipTest, TestCase
 from pathlib import Path
 import typing as T
 
@@ -23,6 +23,9 @@ import mesonbuild.envconfig
 import mesonbuild.environment
 import mesonbuild.coredata
 import mesonbuild.modules.gnome
+
+from mesonbuild import machinefile
+
 from mesonbuild.mesonlib import (
     MachineChoice, is_windows, is_osx, is_cygwin, is_haiku, is_sunos
 )
@@ -50,6 +53,14 @@ def is_real_gnu_compiler(path):
     out = subprocess.check_output([path, '--version'], universal_newlines=True, stderr=subprocess.STDOUT)
     return 'Free Software Foundation' in out
 
+cross_dir = Path(__file__).parent.parent / 'cross'
+
+class MachineFileStoreTests(TestCase):
+
+    def test_loading(self):
+        store = machinefile.MachineFileStore([cross_dir / 'ubuntu-armhf.txt'], [], str(cross_dir))
+        self.assertIsNotNone(store)
+
 class NativeFileTests(BasePlatformTests):
 
     def setUp(self):
diff --git a/unittests/platformagnostictests.py b/unittests/platformagnostictests.py
index 33a789b..fe598a7 100644
--- a/unittests/platformagnostictests.py
+++ b/unittests/platformagnostictests.py
@@ -274,7 +274,7 @@ class PlatformAgnosticTests(BasePlatformTests):
             expected = json.load(f)['meson']['modules']
 
         self.assertEqual(data['modules'], expected)
-        self.assertEqual(data['count'], 69)
+        self.assertEqual(data['count'], 70)
 
     def test_meson_package_cache_dir(self):
         # Copy testdir into temporary directory to not pollute meson source tree.