Move classes used by scripts to their own module

Those classes are used by wrapper scripts and we should not have to
import the rest of mesonlib, build.py, and all their dependencies for
that.

This renames mesonlib/ directory to utils/ and add a mesonlib.py module
that imports everything from utils/ to not have to change `import
mesonlib` everywhere. It allows to import utils.core without importing
the rest of mesonlib.

15 files changed, 209 insertions, 142 deletions

diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py
index 8af8285..cd0a738 100644
--- a/mesonbuild/backend/backends.py
+++ b/mesonbuild/backend/backends.py
@@ -34,7 +34,8 @@ from .. import mlog
 from ..compilers import LANGUAGES_USING_LDFLAGS, detect
 from ..mesonlib import (
     File, MachineChoice, MesonException, OrderedSet,
-    classify_unity_sources, OptionKey, join_args
+    classify_unity_sources, OptionKey, join_args,
+    ExecutableSerialisation
 )
 
 if T.TYPE_CHECKING:
@@ -185,27 +186,6 @@ class SubdirInstallData(InstallDataBase):
         super().__init__(path, install_path, install_path_name, install_mode, subproject, tag, data_type)
         self.exclude = exclude
 
-@dataclass(eq=False)
-class ExecutableSerialisation:
-
-    # XXX: should capture and feed default to False, instead of None?
-
-    cmd_args: T.List[str]
-    env: T.Optional[build.EnvironmentVariables] = None
-    exe_wrapper: T.Optional['programs.ExternalProgram'] = None
-    workdir: T.Optional[str] = None
-    extra_paths: T.Optional[T.List] = None
-    capture: T.Optional[bool] = None
-    feed: T.Optional[bool] = None
-    tag: T.Optional[str] = None
-    verbose: bool = False
-
-    def __post_init__(self) -> None:
-        if self.exe_wrapper is not None:
-            assert isinstance(self.exe_wrapper, programs.ExternalProgram)
-        self.pickled = False
-        self.skip_if_destdir = False
-        self.subproject = ''
 
 @dataclass(eq=False)
 class TestSerialisation:
diff --git a/mesonbuild/build.py b/mesonbuild/build.py
index eb72add..c39726c 100644
--- a/mesonbuild/build.py
+++ b/mesonbuild/build.py
@@ -36,7 +36,7 @@ from .mesonlib import (
     extract_as_list, typeslistify, stringlistify, classify_unity_sources,
     get_filenames_templates_dict, substitute_values, has_path_sep,
     OptionKey, PerMachineDefaultable, OptionOverrideProxy,
-    MesonBugException
+    MesonBugException, EnvironmentVariables
 )
 from .compilers import (
     is_object, clink_langs, sort_clink, all_languages,
@@ -502,71 +502,6 @@ class StructuredSources(HoldableObject):
         return False
 
 
-EnvInitValueType = T.Dict[str, T.Union[str, T.List[str]]]
-
-
-class EnvironmentVariables(HoldableObject):
-    def __init__(self, values: T.Optional[EnvInitValueType] = None,
-                 init_method: Literal['set', 'prepend', 'append'] = 'set', separator: str = os.pathsep) -> None:
-        self.envvars: T.List[T.Tuple[T.Callable[[T.Dict[str, str], str, T.List[str], str], str], str, T.List[str], str]] = []
-        # The set of all env vars we have operations for. Only used for self.has_name()
-        self.varnames: T.Set[str] = set()
-
-        if values:
-            init_func = getattr(self, init_method)
-            for name, value in values.items():
-                init_func(name, listify(value), separator)
-
-    def __repr__(self) -> str:
-        repr_str = "<{0}: {1}>"
-        return repr_str.format(self.__class__.__name__, self.envvars)
-
-    def hash(self, hasher: T.Any):
-        myenv = self.get_env({})
-        for key in sorted(myenv.keys()):
-            hasher.update(bytes(key, encoding='utf-8'))
-            hasher.update(b',')
-            hasher.update(bytes(myenv[key], encoding='utf-8'))
-            hasher.update(b';')
-
-    def has_name(self, name: str) -> bool:
-        return name in self.varnames
-
-    def get_names(self) -> T.Set[str]:
-        return self.varnames
-
-    def set(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
-        self.varnames.add(name)
-        self.envvars.append((self._set, name, values, separator))
-
-    def append(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
-        self.varnames.add(name)
-        self.envvars.append((self._append, name, values, separator))
-
-    def prepend(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
-        self.varnames.add(name)
-        self.envvars.append((self._prepend, name, values, separator))
-
-    @staticmethod
-    def _set(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
-        return separator.join(values)
-
-    @staticmethod
-    def _append(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
-        curr = env.get(name)
-        return separator.join(values if curr is None else [curr] + values)
-
-    @staticmethod
-    def _prepend(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
-        curr = env.get(name)
-        return separator.join(values if curr is None else values + [curr])
-
-    def get_env(self, full_env: T.MutableMapping[str, str]) -> T.Dict[str, str]:
-        env = full_env.copy()
-        for method, name, values, separator in self.envvars:
-            env[name] = method(env, name, values, separator)
-        return env
-
 @dataclass(eq=False)
 class Target(HoldableObject):
 
diff --git a/mesonbuild/dependencies/dub.py b/mesonbuild/dependencies/dub.py
index 8821e2c..a4a7676 100644
--- a/mesonbuild/dependencies/dub.py
+++ b/mesonbuild/dependencies/dub.py
@@ -14,8 +14,7 @@
 
 from .base import ExternalDependency, DependencyException, DependencyTypeName
 from .pkgconfig import PkgConfigDependency
-from ..mesonlib import (Popen_safe, OptionKey)
-from ..mesonlib.universal import join_args
+from ..mesonlib import (Popen_safe, OptionKey, join_args)
 from ..programs import ExternalProgram
 from .. import mlog
 import re
diff --git a/mesonbuild/interpreter/type_checking.py b/mesonbuild/interpreter/type_checking.py
index 80be85d..e56c3b6 100644
--- a/mesonbuild/interpreter/type_checking.py
+++ b/mesonbuild/interpreter/type_checking.py
@@ -8,13 +8,15 @@ import os
 import typing as T
 
 from .. import compilers
-from ..build import (EnvironmentVariables, EnvInitValueType, CustomTarget, BuildTarget,
+from ..build import (CustomTarget, BuildTarget,
                      CustomTargetIndex, ExtractedObjects, GeneratedList, IncludeDirs,
                      BothLibraries, SharedLibrary, StaticLibrary, Jar, Executable)
 from ..coredata import UserFeatureOption
 from ..dependencies import Dependency, InternalDependency
 from ..interpreterbase.decorators import KwargInfo, ContainerTypeInfo
-from ..mesonlib import File, FileMode, MachineChoice, listify, has_path_sep, OptionKey
+from ..mesonlib import (
+    File, FileMode, MachineChoice, listify, has_path_sep, OptionKey,
+    EnvInitValueType, EnvironmentVariables)
 from ..programs import ExternalProgram
 
 # Helper definition for type checks that are `Optional[T]`
diff --git a/mesonbuild/mesonlib/__init__.py b/mesonbuild/mesonlib.py
index 9d673fd..be69a12 100644
--- a/mesonbuild/mesonlib/__init__.py
+++ b/mesonbuild/mesonlib.py
@@ -14,19 +14,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# pylint: skip-file
 
 """Helper functions and classes."""
 
 import os
 
-from .universal import *
-from .vsenv import setup_vsenv
+from .utils.core import *
+from .utils.vsenv import *
+
+from .utils.universal import *
 
 # Here we import either the posix implementations, the windows implementations,
 # or a generic no-op implementation
 if os.name == 'posix':
-    from .posix import *
+    from .utils.posix import *
 elif os.name == 'nt':
-    from .win32 import *
+    from .utils.win32 import *
 else:
-    from .platform import *
+    from .utils.platform import *
diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py
index 2a55c03..7eed11d 100644
--- a/mesonbuild/mesonmain.py
+++ b/mesonbuild/mesonmain.py
@@ -18,18 +18,18 @@ from . import _pathlib
 import sys
 sys.modules['pathlib'] = _pathlib
 
+# This file is an entry point for all commands, including scripts. Include the
+# strict minimum python modules for performance reasons.
 import os.path
 import platform
 import importlib
-import traceback
 import argparse
-import shutil
 
-from . import mesonlib
+from .utils.core import MesonException, MesonBugException
 from . import mlog
-from .mesonlib import MesonException, MesonBugException
 
 def errorhandler(e, command):
+    import traceback
     if isinstance(e, MesonException):
         mlog.exception(e)
         logfile = mlog.shutdown()
@@ -72,6 +72,7 @@ class CommandLineParser:
         from . import mconf, mdist, minit, minstall, mintro, msetup, mtest, rewriter, msubprojects, munstable_coredata, mcompile, mdevenv
         from .scripts import env2mfile
         from .wrap import wraptool
+        import shutil
 
         self.term_width = shutil.get_terminal_size().columns
         self.formatter = lambda prog: argparse.HelpFormatter(prog, max_help_position=int(self.term_width / 2), width=self.term_width)
@@ -176,6 +177,7 @@ class CommandLineParser:
             parser = self.parser
             command = None
 
+        from . import mesonlib
         args = mesonlib.expand_arguments(args)
         options = parser.parse_args(args)
 
@@ -228,6 +230,11 @@ def ensure_stdout_accepts_unicode():
     if sys.stdout.encoding and not sys.stdout.encoding.upper().startswith('UTF-'):
         sys.stdout.reconfigure(errors='surrogateescape')
 
+def set_meson_command(mainfile):
+    # Set the meson command that will be used to run scripts and so on
+    from . import mesonlib
+    mesonlib.set_meson_command(mainfile)
+
 def run(original_args, mainfile):
     if sys.version_info >= (3, 10) and os.environ.get('MESON_RUNNING_IN_PROJECT_TESTS'):
         # workaround for https://bugs.python.org/issue34624
@@ -245,15 +252,13 @@ def run(original_args, mainfile):
         mlog.error('Please install and use mingw-w64-x86_64-python3 and/or mingw-w64-x86_64-meson with Pacman')
         return 2
 
-    # Set the meson command that will be used to run scripts and so on
-    mesonlib.set_meson_command(mainfile)
-
     args = original_args[:]
 
     # Special handling of internal commands called from backends, they don't
     # need to go through argparse.
     if len(args) >= 2 and args[0] == '--internal':
         if args[1] == 'regenerate':
+            set_meson_command(mainfile)
             from . import msetup
             try:
                 return msetup.run(['--reconfigure'] + args[2:])
@@ -262,6 +267,7 @@ def run(original_args, mainfile):
         else:
             return run_script_command(args[1], args[2:])
 
+    set_meson_command(mainfile)
     return CommandLineParser().run(args)
 
 def main():
diff --git a/mesonbuild/scripts/meson_exe.py b/mesonbuild/scripts/meson_exe.py
index bb9d940..3dd91c9 100644
--- a/mesonbuild/scripts/meson_exe.py
+++ b/mesonbuild/scripts/meson_exe.py
@@ -20,8 +20,7 @@ import subprocess
 import typing as T
 import locale
 
-from .. import mesonlib
-from ..backend.backends import ExecutableSerialisation
+from ..utils.core import ExecutableSerialisation
 
 def buildparser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(description='Custom executable wrapper for Meson. Do not run on your own, mmm\'kay?')
@@ -46,7 +45,8 @@ def run_exe(exe: ExecutableSerialisation, extra_env: T.Optional[T.Dict[str, str]
     if exe.extra_paths:
         child_env['PATH'] = (os.pathsep.join(exe.extra_paths + ['']) +
                              child_env['PATH'])
-        if exe.exe_wrapper and mesonlib.substring_is_in_list('wine', exe.exe_wrapper.get_command()):
+        if exe.exe_wrapper and any('wine' in i for i in exe.exe_wrapper.get_command()):
+            from .. import mesonlib
             child_env['WINEPATH'] = mesonlib.get_wine_shortpath(
                 exe.exe_wrapper.get_command(),
                 ['Z:' + p for p in exe.extra_paths] + child_env.get('WINEPATH', '').split(';'),
diff --git a/mesonbuild/scripts/test_loaded_modules.py b/mesonbuild/scripts/test_loaded_modules.py
new file mode 100644
index 0000000..b3547be
--- /dev/null
+++ b/mesonbuild/scripts/test_loaded_modules.py
@@ -0,0 +1,11 @@
+import sys
+import json
+import typing as T
+from . import meson_exe
+
+# This script is used by run_unittests.py to verify we don't load too many
+# modules when executing a wrapped command.
+def run(args: T.List[str]) -> int:
+    meson_exe.run(args)
+    print(json.dumps(list(sys.modules.keys())))
+    return 0
diff --git a/mesonbuild/utils/__init__.py b/mesonbuild/utils/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/mesonbuild/utils/__init__.py
diff --git a/mesonbuild/utils/core.py b/mesonbuild/utils/core.py
new file mode 100644
index 0000000..ed413ca
--- /dev/null
+++ b/mesonbuild/utils/core.py
@@ -0,0 +1,160 @@
+# Copyright 2012-2022 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Contains the strict minimum to run scripts.
+
+When the backend needs to call back into Meson during compilation for running
+scripts or wrapping commands, it is important to load as little python modules
+as possible for performance reasons.
+"""
+
+from __future__ import annotations
+from dataclasses import dataclass
+import os
+import abc
+import typing as T
+
+if T.TYPE_CHECKING:
+    from typing_extensions import Literal
+    from ..mparser import BaseNode
+    from . import programs
+
+
+__all__ = [
+    'MesonException',
+    'MesonBugException',
+    'HoldableObject',
+    'EnvInitValueType',
+    'EnvironmentVariables',
+    'ExecutableSerialisation',
+]
+
+
+class MesonException(Exception):
+    '''Exceptions thrown by Meson'''
+
+    def __init__(self, *args: object, file: T.Optional[str] = None,
+                 lineno: T.Optional[int] = None, colno: T.Optional[int] = None):
+        super().__init__(*args)
+        self.file = file
+        self.lineno = lineno
+        self.colno = colno
+
+    @classmethod
+    def from_node(cls, *args: object, node: BaseNode) -> MesonException:
+        """Create a MesonException with location data from a BaseNode
+
+        :param node: A BaseNode to set location data from
+        :return: A Meson Exception instance
+        """
+        return cls(*args, file=node.filename, lineno=node.lineno, colno=node.colno)
+
+class MesonBugException(MesonException):
+    '''Exceptions thrown when there is a clear Meson bug that should be reported'''
+
+    def __init__(self, msg: str, file: T.Optional[str] = None,
+                 lineno: T.Optional[int] = None, colno: T.Optional[int] = None):
+        super().__init__(msg + '\n\n    This is a Meson bug and should be reported!',
+                         file=file, lineno=lineno, colno=colno)
+
+class HoldableObject(metaclass=abc.ABCMeta):
+    ''' Dummy base class for all objects that can be
+        held by an interpreter.baseobjects.ObjectHolder '''
+
+EnvInitValueType = T.Dict[str, T.Union[str, T.List[str]]]
+
+class EnvironmentVariables(HoldableObject):
+    def __init__(self, values: T.Optional[EnvInitValueType] = None,
+                 init_method: Literal['set', 'prepend', 'append'] = 'set', separator: str = os.pathsep) -> None:
+        self.envvars: T.List[T.Tuple[T.Callable[[T.Dict[str, str], str, T.List[str], str], str], str, T.List[str], str]] = []
+        # The set of all env vars we have operations for. Only used for self.has_name()
+        self.varnames: T.Set[str] = set()
+
+        if values:
+            init_func = getattr(self, init_method)
+            for name, value in values.items():
+                v = value if isinstance(value, list) else [value]
+                init_func(name, v, separator)
+
+    def __repr__(self) -> str:
+        repr_str = "<{0}: {1}>"
+        return repr_str.format(self.__class__.__name__, self.envvars)
+
+    def hash(self, hasher: T.Any):
+        myenv = self.get_env({})
+        for key in sorted(myenv.keys()):
+            hasher.update(bytes(key, encoding='utf-8'))
+            hasher.update(b',')
+            hasher.update(bytes(myenv[key], encoding='utf-8'))
+            hasher.update(b';')
+
+    def has_name(self, name: str) -> bool:
+        return name in self.varnames
+
+    def get_names(self) -> T.Set[str]:
+        return self.varnames
+
+    def set(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
+        self.varnames.add(name)
+        self.envvars.append((self._set, name, values, separator))
+
+    def append(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
+        self.varnames.add(name)
+        self.envvars.append((self._append, name, values, separator))
+
+    def prepend(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
+        self.varnames.add(name)
+        self.envvars.append((self._prepend, name, values, separator))
+
+    @staticmethod
+    def _set(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
+        return separator.join(values)
+
+    @staticmethod
+    def _append(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
+        curr = env.get(name)
+        return separator.join(values if curr is None else [curr] + values)
+
+    @staticmethod
+    def _prepend(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
+        curr = env.get(name)
+        return separator.join(values if curr is None else values + [curr])
+
+    def get_env(self, full_env: T.MutableMapping[str, str]) -> T.Dict[str, str]:
+        env = full_env.copy()
+        for method, name, values, separator in self.envvars:
+            env[name] = method(env, name, values, separator)
+        return env
+
+
+@dataclass(eq=False)
+class ExecutableSerialisation:
+
+    # XXX: should capture and feed default to False, instead of None?
+
+    cmd_args: T.List[str]
+    env: T.Optional[EnvironmentVariables] = None
+    exe_wrapper: T.Optional['programs.ExternalProgram'] = None
+    workdir: T.Optional[str] = None
+    extra_paths: T.Optional[T.List] = None
+    capture: T.Optional[bool] = None
+    feed: T.Optional[bool] = None
+    tag: T.Optional[str] = None
+    verbose: bool = False
+
+    def __post_init__(self) -> None:
+        self.pickled = False
+        self.skip_if_destdir = False
+        self.subproject = ''
diff --git a/mesonbuild/mesonlib/platform.py b/mesonbuild/utils/platform.py
index f0676a6..f0676a6 100644
--- a/mesonbuild/mesonlib/platform.py
+++ b/mesonbuild/utils/platform.py
diff --git a/mesonbuild/mesonlib/posix.py b/mesonbuild/utils/posix.py
index 67f9a44..67f9a44 100644
--- a/mesonbuild/mesonlib/posix.py
+++ b/mesonbuild/utils/posix.py
diff --git a/mesonbuild/mesonlib/universal.py b/mesonbuild/utils/universal.py
index bdf3d49..aaefbc5 100644
--- a/mesonbuild/mesonlib/universal.py
+++ b/mesonbuild/utils/universal.py
@@ -33,6 +33,7 @@ import copy
 import pickle
 
 from mesonbuild import mlog
+from .core import MesonException, HoldableObject
 
 if T.TYPE_CHECKING:
     from typing_extensions import Literal
@@ -41,7 +42,6 @@ if T.TYPE_CHECKING:
     from ..build import ConfigurationData
     from ..coredata import KeyedOptionDictType, UserOption
     from ..compilers.compilers import Compiler
-    from ..mparser import BaseNode
 
 FileOrString = T.Union['File', str]
 
@@ -52,15 +52,12 @@ __all__ = [
     'GIT',
     'python_command',
     'project_meson_versions',
-    'HoldableObject',
     'SecondLevelHolder',
     'File',
     'FileMode',
     'GitException',
     'LibType',
     'MachineChoice',
-    'MesonException',
-    'MesonBugException',
     'EnvironmentException',
     'FileOrString',
     'GitException',
@@ -164,33 +161,6 @@ else:
     python_command = [sys.executable]
 _meson_command: T.Optional['ImmutableListProtocol[str]'] = None
 
-class MesonException(Exception):
-    '''Exceptions thrown by Meson'''
-
-    def __init__(self, *args: object, file: T.Optional[str] = None,
-                 lineno: T.Optional[int] = None, colno: T.Optional[int] = None):
-        super().__init__(*args)
-        self.file = file
-        self.lineno = lineno
-        self.colno = colno
-
-    @classmethod
-    def from_node(cls, *args: object, node: BaseNode) -> MesonException:
-        """Create a MesonException with location data from a BaseNode
-
-        :param node: A BaseNode to set location data from
-        :return: A Meson Exception instance
-        """
-        return cls(*args, file=node.filename, lineno=node.lineno, colno=node.colno)
-
-
-class MesonBugException(MesonException):
-    '''Exceptions thrown when there is a clear Meson bug that should be reported'''
-
-    def __init__(self, msg: str, file: T.Optional[str] = None,
-                 lineno: T.Optional[int] = None, colno: T.Optional[int] = None):
-        super().__init__(msg + '\n\n    This is a Meson bug and should be reported!',
-                         file=file, lineno=lineno, colno=colno)
 
 class EnvironmentException(MesonException):
     '''Exceptions thrown while processing and creating the build environment'''
@@ -279,10 +249,6 @@ def check_direntry_issues(direntry_array: T.Union[T.Iterable[T.Union[str, bytes]
                 not pure ASCII. This may cause problems.
                 '''), file=sys.stderr)
 
-class HoldableObject(metaclass=abc.ABCMeta):
-    ''' Dummy base class for all objects that can be
-        held by an interpreter.baseobjects.ObjectHolder '''
-
 class SecondLevelHolder(HoldableObject, metaclass=abc.ABCMeta):
     ''' A second level object holder. The primary purpose
         of such objects is to hold multiple objects with one
diff --git a/mesonbuild/mesonlib/vsenv.py b/mesonbuild/utils/vsenv.py
index 4eab428..5f32990 100644
--- a/mesonbuild/mesonlib/vsenv.py
+++ b/mesonbuild/utils/vsenv.py
@@ -10,6 +10,11 @@ from .. import mlog
 from .universal import MesonException, is_windows
 
 
+__all__ = [
+    'setup_vsenv',
+]
+
+
 bat_template = '''@ECHO OFF
 
 call "{}"
diff --git a/mesonbuild/mesonlib/win32.py b/mesonbuild/utils/win32.py
index bc0caec..bc0caec 100644
--- a/mesonbuild/mesonlib/win32.py
+++ b/mesonbuild/utils/win32.py