holders: Ensure that InterpreterBase is the sole instance for (un)holderifying

1 files changed, 111 insertions, 38 deletions

diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py
index be90049..0acb699 100644
--- a/mesonbuild/interpreterbase/interpreterbase.py
+++ b/mesonbuild/interpreterbase/interpreterbase.py
@@ -20,13 +20,15 @@ from .. import environment, dependencies
 
 from .baseobjects import (
     InterpreterObject,
+    MesonInterpreterObject,
     MutableInterpreterObject,
+    InterpreterObjectTypeVar,
     ObjectHolder,
     RangeHolder,
 
+    TYPE_elementary,
     TYPE_var,
-    TYPE_nvar,
-    TYPE_nkwargs,
+    TYPE_kwargs,
 )
 
 from .exceptions import (
@@ -41,10 +43,24 @@ from .exceptions import (
 from .decorators import FeatureNew, builtinMethodNoKwargs
 from .disabler import Disabler, is_disabled
 from .helpers import check_stringlist, default_resolve_key, flatten
+from ._unholder import _unholder
 
 import os, copy, re
 import typing as T
 
+if T.TYPE_CHECKING:
+    from ..interpreter import Interpreter
+
+HolderMapType = T.Dict[
+    T.Type[mesonlib.HoldableObject],
+    # For some reason, this has to be a callable and can't just be ObjectHolder[InterpreterObjectTypeVar]
+    T.Callable[[InterpreterObjectTypeVar, 'Interpreter'], ObjectHolder[InterpreterObjectTypeVar]]
+]
+
+FunctionType = T.Dict[
+    str,
+    T.Callable[[mparser.BaseNode, T.List[TYPE_var], T.Dict[str, TYPE_var]], TYPE_var]
+]
 
 class MesonVersionString(str):
     pass
@@ -54,12 +70,16 @@ class InterpreterBase:
 
     def __init__(self, source_root: str, subdir: str, subproject: str):
         self.source_root = source_root
-        self.funcs = {}    # type: T.Dict[str, T.Callable[[mparser.BaseNode, T.List[TYPE_nvar], T.Dict[str, TYPE_nvar]], TYPE_var]]
-        self.builtin = {}  # type: T.Dict[str, InterpreterObject]
+        self.funcs: FunctionType = {}
+        self.builtin: T.Dict[str, InterpreterObject] = {}
+        # Holder maps store a mapping from an HoldableObject to a class ObjectHolder
+        self.holder_map: HolderMapType = {}
+        self.bound_holder_map: HolderMapType = {}
         self.subdir = subdir
         self.root_subdir = subdir
         self.subproject = subproject
-        self.variables = {}  # type: T.Dict[str, TYPE_var]
+        # TODO: This should actually be more strict: T.Union[TYPE_elementary, InterpreterObject]
+        self.variables: T.Dict[str, T.Union[TYPE_var, InterpreterObject]] = {}
         self.argument_depth = 0
         self.current_lineno = -1
         # Current node set during a function call. This can be used as location
@@ -137,7 +157,7 @@ class InterpreterBase:
                 raise e
             i += 1 # In THE FUTURE jump over blocks and stuff.
 
-    def evaluate_statement(self, cur: mparser.BaseNode) -> T.Optional[TYPE_var]:
+    def evaluate_statement(self, cur: mparser.BaseNode) -> T.Optional[T.Union[TYPE_var, InterpreterObject]]:
         self.current_node = cur
         if isinstance(cur, mparser.FunctionNode):
             return self.function_call(cur)
@@ -191,14 +211,14 @@ class InterpreterBase:
             raise InvalidCode("Unknown statement.")
         return None
 
-    def evaluate_arraystatement(self, cur: mparser.ArrayNode) -> list:
+    def evaluate_arraystatement(self, cur: mparser.ArrayNode) -> T.List[T.Union[TYPE_var, InterpreterObject]]:
         (arguments, kwargs) = self.reduce_arguments(cur.args)
         if len(kwargs) > 0:
             raise InvalidCode('Keyword arguments are invalid in array construction.')
         return arguments
 
     @FeatureNew('dict', '0.47.0')
-    def evaluate_dictstatement(self, cur: mparser.DictNode) -> TYPE_nkwargs:
+    def evaluate_dictstatement(self, cur: mparser.DictNode) -> T.Union[TYPE_var, InterpreterObject]:
         def resolve_key(key: mparser.BaseNode) -> str:
             if not isinstance(key, mparser.StringNode):
                 FeatureNew.single_use('Dictionary entry using non literal key', '0.53.0', self.subproject)
@@ -387,7 +407,7 @@ The result of this is undefined and will become a hard error in a future Meson r
         else:
             raise InvalidCode('You broke me.')
 
-    def evaluate_ternary(self, node: mparser.TernaryNode) -> TYPE_var:
+    def evaluate_ternary(self, node: mparser.TernaryNode) -> T.Union[TYPE_var, InterpreterObject]:
         assert(isinstance(node, mparser.TernaryNode))
         result = self.evaluate_statement(node.condition)
         if isinstance(result, Disabler):
@@ -479,7 +499,7 @@ The result of this is undefined and will become a hard error in a future Meson r
             raise InvalidArguments('The += operator currently only works with arrays, dicts, strings or ints')
         self.set_variable(varname, new_value)
 
-    def evaluate_indexing(self, node: mparser.IndexNode) -> TYPE_var:
+    def evaluate_indexing(self, node: mparser.IndexNode) -> T.Union[TYPE_elementary, InterpreterObject]:
         assert(isinstance(node, mparser.IndexNode))
         iobject = self.evaluate_statement(node.iobject)
         if isinstance(iobject, Disabler):
@@ -494,7 +514,7 @@ The result of this is undefined and will become a hard error in a future Meson r
                 raise InterpreterException('Key is not a string')
             try:
                 # The cast is required because we don't have recursive types...
-                return T.cast(TYPE_var, iobject[index])
+                return T.cast(T.Union[TYPE_elementary, InterpreterObject], iobject[index])
             except KeyError:
                 raise InterpreterException('Key %s is not in dict' % index)
         else:
@@ -504,35 +524,45 @@ The result of this is undefined and will become a hard error in a future Meson r
                 # Ignore the MyPy error, since we don't know all indexable types here
                 # and we handle non indexable types with an exception
                 # TODO maybe find a better solution
-                return iobject[index]  # type: ignore
+                res = iobject[index]  # type: ignore
+                # Only holderify if we are dealing with `InterpreterObject`, since raw
+                # lists already store ObjectHolders
+                if isinstance(iobject, InterpreterObject):
+                    return self._holderify(res)
+                else:
+                    return res
             except IndexError:
                 # We are already checking for the existence of __getitem__, so this should be save
                 raise InterpreterException('Index %d out of bounds of array of size %d.' % (index, len(iobject)))  # type: ignore
 
-    def function_call(self, node: mparser.FunctionNode) -> T.Optional[TYPE_var]:
+    def function_call(self, node: mparser.FunctionNode) -> T.Optional[T.Union[TYPE_elementary, InterpreterObject]]:
         func_name = node.func_name
-        (posargs, kwargs) = self.reduce_arguments(node.args)
+        (h_posargs, h_kwargs) = self.reduce_arguments(node.args)
+        (posargs, kwargs) = self._unholder_args(h_posargs, h_kwargs)
         if is_disabled(posargs, kwargs) and func_name not in {'get_variable', 'set_variable', 'is_disabler'}:
             return Disabler()
         if func_name in self.funcs:
             func = self.funcs[func_name]
-            func_args = posargs  # type: T.Any
+            func_args = posargs
             if not getattr(func, 'no-args-flattening', False):
                 func_args = flatten(posargs)
-            return func(node, func_args, kwargs)
+            res = func(node, func_args, kwargs)
+            return self._holderify(res)
         else:
             self.unknown_function_called(func_name)
             return None
 
-    def method_call(self, node: mparser.MethodNode) -> TYPE_var:
+    def method_call(self, node: mparser.MethodNode) -> T.Optional[T.Union[TYPE_var, InterpreterObject]]:
         invokable = node.source_object
+        obj: T.Union[TYPE_var, InterpreterObject]
         if isinstance(invokable, mparser.IdNode):
             object_name = invokable.value
             obj = self.get_variable(object_name)
         else:
             obj = self.evaluate_statement(invokable)
         method_name = node.name
-        (args, kwargs) = self.reduce_arguments(node.args)
+        (h_args, h_kwargs) = self.reduce_arguments(node.args)
+        (args, kwargs) = self._unholder_args(h_args, h_kwargs)
         if is_disabled(args, kwargs):
             return Disabler()
         if isinstance(obj, str):
@@ -554,15 +584,48 @@ The result of this is undefined and will become a hard error in a future Meson r
                 return False
             else:
                 return Disabler()
+        # TODO: InterpreterBase **really** shouldn't be in charge of checking this
         if method_name == 'extract_objects':
             if not isinstance(obj, ObjectHolder):
-                raise InvalidArguments(f'Invalid operation "extract_objects" on variable "{object_name}"')
+                raise InvalidArguments(f'Invalid operation "extract_objects" on variable "{object_name}" of type {type(obj).__name__}')
             self.validate_extraction(obj.held_object)
         obj.current_node = node
-        return obj.method_call(method_name, args, kwargs)
+        return self._holderify(obj.method_call(method_name, args, kwargs))
+
+    def _holderify(self, res: T.Optional[TYPE_var]) -> T.Union[TYPE_elementary, InterpreterObject]:
+        if res is None:
+            return None
+        if isinstance(res, (int, bool, str)):
+            return res
+        elif isinstance(res, list):
+            return [self._holderify(x) for x in res]
+        elif isinstance(res, dict):
+            return {k: self._holderify(v) for k, v in res.items()}
+        elif isinstance(res, mesonlib.HoldableObject):
+            # Always check for an exact match first.
+            cls = self.holder_map.get(type(res), None)
+            if cls is not None:
+                # Casts to Interpreter are required here since an assertion would
+                # not work for the `ast` module.
+                return cls(res, T.cast('Interpreter', self))
+            # Try the boundary types next.
+            for typ, cls in self.bound_holder_map.items():
+                if isinstance(res, typ):
+                    return cls(res, T.cast('Interpreter', self))
+            raise mesonlib.MesonBugException(f'Object {res} of type {type(res).__name__} is neither in self.holder_map nor self.bound_holder_map.')
+        elif isinstance(res, ObjectHolder):
+            raise mesonlib.MesonBugException(f'Returned object {res} of type {type(res).__name__} is an object holder.')
+        elif isinstance(res, MesonInterpreterObject):
+            return res
+        raise mesonlib.MesonBugException(f'Unknown returned object {res} of type {type(res).__name__} in the parameters.')
+
+    def _unholder_args(self,
+                       args: T.List[T.Union[TYPE_var, InterpreterObject]],
+                       kwargs: T.Dict[str, T.Union[TYPE_var, InterpreterObject]]) -> T.Tuple[T.List[TYPE_var], TYPE_kwargs]:
+        return [_unholder(x) for x in args], {k: _unholder(v) for k, v in kwargs.items()}
 
     @builtinMethodNoKwargs
-    def bool_method_call(self, obj: bool, method_name: str, posargs: T.List[TYPE_nvar], kwargs: T.Dict[str, T.Any]) -> T.Union[str, int]:
+    def bool_method_call(self, obj: bool, method_name: str, posargs: T.List[TYPE_var], kwargs: TYPE_kwargs) -> T.Union[str, int]:
         if method_name == 'to_string':
             if not posargs:
                 if obj:
@@ -585,7 +648,7 @@ The result of this is undefined and will become a hard error in a future Meson r
             raise InterpreterException('Unknown method "%s" for a boolean.' % method_name)
 
     @builtinMethodNoKwargs
-    def int_method_call(self, obj: int, method_name: str, posargs: T.List[TYPE_nvar], kwargs: T.Dict[str, T.Any]) -> T.Union[str, bool]:
+    def int_method_call(self, obj: int, method_name: str, posargs: T.List[TYPE_var], kwargs: TYPE_kwargs) -> T.Union[str, bool]:
         if method_name == 'is_even':
             if not posargs:
                 return obj % 2 == 0
@@ -605,7 +668,7 @@ The result of this is undefined and will become a hard error in a future Meson r
             raise InterpreterException('Unknown method "%s" for an integer.' % method_name)
 
     @staticmethod
-    def _get_one_string_posarg(posargs: T.List[TYPE_nvar], method_name: str) -> str:
+    def _get_one_string_posarg(posargs: T.List[TYPE_var], method_name: str) -> str:
         if len(posargs) > 1:
             m = '{}() must have zero or one arguments'
             raise InterpreterException(m.format(method_name))
@@ -618,7 +681,7 @@ The result of this is undefined and will become a hard error in a future Meson r
         return None
 
     @builtinMethodNoKwargs
-    def string_method_call(self, obj: str, method_name: str, posargs: T.List[TYPE_nvar], kwargs: T.Dict[str, T.Any]) -> T.Union[str, int, bool, T.List[str]]:
+    def string_method_call(self, obj: str, method_name: str, posargs: T.List[TYPE_var], kwargs: TYPE_kwargs) -> T.Union[str, int, bool, T.List[str]]:
         if method_name == 'strip':
             s1 = self._get_one_string_posarg(posargs, 'strip')
             if s1 is not None:
@@ -690,7 +753,7 @@ The result of this is undefined and will become a hard error in a future Meson r
             return obj.replace(posargs[0], posargs[1])
         raise InterpreterException('Unknown method "%s" for a string.' % method_name)
 
-    def format_string(self, templ: str, args: T.List[TYPE_nvar]) -> str:
+    def format_string(self, templ: str, args: T.List[TYPE_var]) -> str:
         arg_strings = []
         for arg in args:
             if isinstance(arg, mparser.BaseNode):
@@ -711,7 +774,11 @@ The result of this is undefined and will become a hard error in a future Meson r
         raise InvalidCode('Unknown function "%s".' % func_name)
 
     @builtinMethodNoKwargs
-    def array_method_call(self, obj: T.List[TYPE_var], method_name: str, posargs: T.List[TYPE_nvar], kwargs: T.Dict[str, T.Any]) -> TYPE_var:
+    def array_method_call(self,
+                          obj: T.List[T.Union[TYPE_elementary, InterpreterObject]],
+                          method_name: str,
+                          posargs: T.List[TYPE_var],
+                          kwargs: TYPE_kwargs) -> T.Union[TYPE_var, InterpreterObject]:
         if method_name == 'contains':
             def check_contains(el: list) -> bool:
                 if len(posargs) != 1:
@@ -752,7 +819,11 @@ The result of this is undefined and will become a hard error in a future Meson r
         raise InterpreterException(m.format(method_name))
 
     @builtinMethodNoKwargs
-    def dict_method_call(self, obj: T.Dict[str, TYPE_var], method_name: str, posargs: T.List[TYPE_nvar], kwargs: T.Dict[str, T.Any]) -> TYPE_var:
+    def dict_method_call(self,
+                         obj: T.Dict[str, T.Union[TYPE_elementary, InterpreterObject]],
+                         method_name: str,
+                         posargs: T.List[TYPE_var],
+                         kwargs: TYPE_kwargs) -> T.Union[TYPE_var, InterpreterObject]:
         if method_name in ('has_key', 'get'):
             if method_name == 'has_key':
                 if len(posargs) != 1:
@@ -793,18 +864,20 @@ The result of this is undefined and will become a hard error in a future Meson r
                 args: mparser.ArgumentNode,
                 key_resolver: T.Callable[[mparser.BaseNode], str] = default_resolve_key,
                 duplicate_key_error: T.Optional[str] = None,
-            ) -> T.Tuple[T.List[TYPE_nvar], TYPE_nkwargs]:
+            ) -> T.Tuple[
+                T.List[T.Union[TYPE_var, InterpreterObject]],
+                T.Dict[str, T.Union[TYPE_var, InterpreterObject]]
+            ]:
         assert(isinstance(args, mparser.ArgumentNode))
         if args.incorrect_order():
             raise InvalidArguments('All keyword arguments must be after positional arguments.')
         self.argument_depth += 1
-        reduced_pos = [self.evaluate_statement(arg) for arg in args.arguments]  # type: T.List[TYPE_nvar]
-        reduced_kw = {}  # type: TYPE_nkwargs
+        reduced_pos: T.List[T.Union[TYPE_var, InterpreterObject]] = [self.evaluate_statement(arg) for arg in args.arguments]
+        reduced_kw: T.Dict[str, T.Union[TYPE_var, InterpreterObject]] = {}
         for key, val in args.kwargs.items():
             reduced_key = key_resolver(key)
-            reduced_val = val  # type: TYPE_nvar
-            if isinstance(reduced_val, mparser.BaseNode):
-                reduced_val = self.evaluate_statement(reduced_val)
+            assert isinstance(val, mparser.BaseNode)
+            reduced_val = self.evaluate_statement(val)
             if duplicate_key_error and reduced_key in reduced_kw:
                 raise InvalidArguments(duplicate_key_error.format(reduced_key))
             reduced_kw[reduced_key] = reduced_val
@@ -812,7 +885,7 @@ The result of this is undefined and will become a hard error in a future Meson r
         final_kw = self.expand_default_kwargs(reduced_kw)
         return reduced_pos, final_kw
 
-    def expand_default_kwargs(self, kwargs: TYPE_nkwargs) -> TYPE_nkwargs:
+    def expand_default_kwargs(self, kwargs: T.Dict[str, T.Union[TYPE_var, InterpreterObject]]) -> T.Dict[str, T.Union[TYPE_var, InterpreterObject]]:
         if 'kwargs' not in kwargs:
             return kwargs
         to_expand = kwargs.pop('kwargs')
@@ -843,7 +916,7 @@ To specify a keyword argument, use : instead of =.''')
         self.set_variable(var_name, value)
         return None
 
-    def set_variable(self, varname: str, variable: TYPE_var) -> None:
+    def set_variable(self, varname: str, variable: T.Union[TYPE_var, InterpreterObject]) -> None:
         if variable is None:
             raise InvalidCode('Can not assign None to variable.')
         if not isinstance(varname, str):
@@ -856,7 +929,7 @@ To specify a keyword argument, use : instead of =.''')
             raise InvalidCode('Tried to overwrite internal variable "%s"' % varname)
         self.variables[varname] = variable
 
-    def get_variable(self, varname: str) -> TYPE_var:
+    def get_variable(self, varname: str) -> T.Union[TYPE_var, InterpreterObject]:
         if varname in self.builtin:
             return self.builtin[varname]
         if varname in self.variables:
@@ -865,7 +938,7 @@ To specify a keyword argument, use : instead of =.''')
 
     def is_assignable(self, value: T.Any) -> bool:
         return isinstance(value, (InterpreterObject, dependencies.Dependency,
-                                  str, int, list, dict))
+                                  str, int, list, dict, mesonlib.File))
 
-    def validate_extraction(self, buildtarget: InterpreterObject) -> None:
+    def validate_extraction(self, buildtarget: mesonlib.HoldableObject) -> None:
         raise InterpreterException('validate_extraction is not implemented in this context (please file a bug)')