From 79ed2415e9a50e5181b203790ec057c6229609a3 Mon Sep 17 00:00:00 2001 From: JCWasmx86 Date: Mon, 31 Oct 2022 16:47:40 +0100 Subject: Add ParenthesizedNode --- mesonbuild/ast/visitor.py | 4 ++++ mesonbuild/interpreterbase/interpreterbase.py | 2 ++ mesonbuild/mparser.py | 11 ++++++++++- mesonbuild/optinterpreter.py | 2 ++ 4 files changed, 18 insertions(+), 1 deletion(-) (limited to 'mesonbuild') diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index 8a0e77b..51a6620 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -144,3 +144,7 @@ class AstVisitor: for key, val in node.kwargs.items(): key.accept(self) val.accept(self) + + def visit_ParenthesizedNode(self, node: mparser.ParenthesizedNode) -> None: + self.visit_default_func(node) + node.inner.accept(self) diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index 902f84a..f35da33 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -239,6 +239,8 @@ class InterpreterBase: raise ContinueRequest() elif isinstance(cur, mparser.BreakNode): raise BreakRequest() + elif isinstance(cur, mparser.ParenthesizedNode): + return self.evaluate_statement(cur.inner) elif isinstance(cur, mparser.TestCaseClauseNode): return self.evaluate_testcase(cur) else: diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index fb4e433..3697d8a 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -577,6 +577,15 @@ class TernaryNode(BaseNode): self.trueblock = trueblock self.falseblock = falseblock +@dataclass(unsafe_hash=True) +class ParenthesizedNode(BaseNode): + + inner: BaseNode + + def __init__(self, inner: BaseNode, lineno: int, colno: int, end_lineno: int, end_colno: int): + super().__init__(lineno, colno, inner.filename, end_lineno=end_lineno, end_colno=end_colno) + self.inner = inner + if T.TYPE_CHECKING: COMPARISONS = Literal['==', '!=', '<', '<=', '>=', '>', 'in', 'notin'] @@ -778,7 +787,7 @@ class Parser: if self.accept('lparen'): e = self.statement() self.block_expect('rparen', block_start) - return e + return ParenthesizedNode(e, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) elif self.accept('lbracket'): args = self.args() self.block_expect('rbracket', block_start) diff --git a/mesonbuild/optinterpreter.py b/mesonbuild/optinterpreter.py index 8ad84aa..37ef55a 100644 --- a/mesonbuild/optinterpreter.py +++ b/mesonbuild/optinterpreter.py @@ -113,6 +113,8 @@ class OptionInterpreter: def reduce_single(self, arg: T.Union[str, mparser.BaseNode]) -> 'TYPE_var': if isinstance(arg, str): return arg + if isinstance(arg, mparser.ParenthesizedNode): + return self.reduce_single(arg.inner) elif isinstance(arg, (mparser.StringNode, mparser.BooleanNode, mparser.NumberNode)): return arg.value -- cgit v1.1 From 3ff3b8abf6f24f676b6c03250ba96a365ae77b03 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Tue, 22 Aug 2023 22:05:40 -0400 Subject: parser: more specific error for float numbers --- mesonbuild/mparser.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'mesonbuild') diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 3697d8a..9867c71 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -863,6 +863,9 @@ class Parser: def method_call(self, source_object: BaseNode) -> MethodNode: methodname = self.e9() if not isinstance(methodname, IdNode): + if isinstance(source_object, NumberNode) and isinstance(methodname, NumberNode): + raise ParseException('meson does not support float numbers', + self.getline(), source_object.lineno, source_object.colno) raise ParseException('Method name must be plain id', self.getline(), self.current.lineno, self.current.colno) assert isinstance(methodname.value, str) -- cgit v1.1 From 5707d390174b7e8db431bc9cee2257043ea59f84 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 09:32:21 -0400 Subject: parser: preserve number base --- mesonbuild/cargo/builder.py | 2 +- mesonbuild/cmake/interpreter.py | 2 +- mesonbuild/mparser.py | 12 ++++++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/cargo/builder.py b/mesonbuild/cargo/builder.py index fb086d1..2442fbf 100644 --- a/mesonbuild/cargo/builder.py +++ b/mesonbuild/cargo/builder.py @@ -45,7 +45,7 @@ def number(value: int, filename: str) -> mparser.NumberNode: :param filename: the file that the value came from :return: A NumberNode """ - return mparser.NumberNode(_token('number', filename, value)) + return mparser.NumberNode(_token('number', filename, str(value))) def bool(value: builtins.bool, filename: str) -> mparser.BooleanNode: diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py index f88d091..f577ba7 100644 --- a/mesonbuild/cmake/interpreter.py +++ b/mesonbuild/cmake/interpreter.py @@ -966,7 +966,7 @@ class CMakeInterpreter: return IdNode(token(val=value)) def number(value: int) -> NumberNode: - return NumberNode(token(val=value)) + return NumberNode(token(val=str(value))) def nodeify(value: TYPE_mixed_list) -> BaseNode: if isinstance(value, str): diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 9867c71..5ab76e7 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -217,7 +217,7 @@ class Lexer: lineno += len(lines) - 1 line_start = mo.end() - len(lines[-1]) elif tid == 'number': - value = int(match_text, base=0) + value = match_text elif tid == 'eol_cont': lineno += 1 line_start = loc @@ -285,8 +285,16 @@ class IdNode(ElementaryNode[str]): def __str__(self) -> str: return "Id node: '%s' (%d, %d)." % (self.value, self.lineno, self.colno) +@dataclass(unsafe_hash=True) class NumberNode(ElementaryNode[int]): - pass + + raw_value: str = field(hash=False) + + def __init__(self, token: Token[str]): + BaseNode.__init__(self, token.lineno, token.colno, token.filename) + self.raw_value = token.value + self.value = int(token.value, base=0) + self.bytespan = token.bytespan class StringNode(ElementaryNode[str]): def __str__(self) -> str: -- cgit v1.1 From a730a2fe215ae45c928370b5e28d2a844c082f38 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 15:51:58 -0400 Subject: parser: remove useless __str__ methods on nodes --- mesonbuild/mparser.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 5ab76e7..f9b8967 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -282,8 +282,7 @@ class BooleanNode(ElementaryNode[bool]): pass class IdNode(ElementaryNode[str]): - def __str__(self) -> str: - return "Id node: '%s' (%d, %d)." % (self.value, self.lineno, self.colno) + pass @dataclass(unsafe_hash=True) class NumberNode(ElementaryNode[int]): @@ -297,16 +296,13 @@ class NumberNode(ElementaryNode[int]): self.bytespan = token.bytespan class StringNode(ElementaryNode[str]): - def __str__(self) -> str: - return "String node: '%s' (%d, %d)." % (self.value, self.lineno, self.colno) + pass class FormatStringNode(ElementaryNode[str]): - def __str__(self) -> str: - return f"Format string node: '{self.value}' ({self.lineno}, {self.colno})." + pass class MultilineFormatStringNode(FormatStringNode): - def __str__(self) -> str: - return f"Multiline Format string node: '{self.value}' ({self.lineno}, {self.colno})." + pass class ContinueNode(ElementaryNode): pass -- cgit v1.1 From 35936283d24ed5a0aa76b184a7489d637d3e49c4 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 09:24:05 -0400 Subject: parser: preserve escape chars in strings use separate Node for multiline strings --- mesonbuild/ast/interpreter.py | 2 +- mesonbuild/ast/introspection.py | 8 +++---- mesonbuild/ast/printer.py | 12 +++++++++- mesonbuild/ast/visitor.py | 6 +++++ mesonbuild/cmake/interpreter.py | 2 +- mesonbuild/coredata.py | 2 +- mesonbuild/interpreter/interpreter.py | 2 +- mesonbuild/interpreterbase/helpers.py | 4 ++-- mesonbuild/interpreterbase/interpreterbase.py | 20 ++++++++-------- mesonbuild/mintro.py | 4 ++-- mesonbuild/mparser.py | 33 ++++++++++++++++++--------- mesonbuild/optinterpreter.py | 4 ++-- mesonbuild/rewriter.py | 24 +++++++++---------- 13 files changed, 74 insertions(+), 49 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index 9e098d0..501be7d 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -239,7 +239,7 @@ class AstInterpreter(InterpreterBase): def evaluate_dictstatement(self, node: mparser.DictNode) -> TYPE_nkwargs: def resolve_key(node: mparser.BaseNode) -> str: - if isinstance(node, mparser.StringNode): + if isinstance(node, mparser.BaseStringNode): return node.value return '__AST_UNKNOWN__' arguments, kwargs = self.reduce_arguments(node.args, key_resolver=resolve_key) diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py index e8055c5..50aa744 100644 --- a/mesonbuild/ast/introspection.py +++ b/mesonbuild/ast/introspection.py @@ -27,7 +27,7 @@ from ..build import Executable, Jar, SharedLibrary, SharedModule, StaticLibrary from ..compilers import detect_compiler_for from ..interpreterbase import InvalidArguments from ..mesonlib import MachineChoice, OptionKey -from ..mparser import BaseNode, ArithmeticNode, ArrayNode, ElementaryNode, IdNode, FunctionNode, StringNode +from ..mparser import BaseNode, ArithmeticNode, ArrayNode, ElementaryNode, IdNode, FunctionNode, BaseStringNode from .interpreter import AstInterpreter if T.TYPE_CHECKING: @@ -128,7 +128,7 @@ class IntrospectionInterpreter(AstInterpreter): if not self.is_subproject() and 'subproject_dir' in kwargs: spdirname = kwargs['subproject_dir'] - if isinstance(spdirname, StringNode): + if isinstance(spdirname, BaseStringNode): assert isinstance(spdirname.value, str) self.subproject_dir = spdirname.value if not self.is_subproject(): @@ -174,7 +174,7 @@ class IntrospectionInterpreter(AstInterpreter): for l in self.flatten_args(raw_langs): if isinstance(l, str): langs.append(l) - elif isinstance(l, StringNode): + elif isinstance(l, BaseStringNode): langs.append(l.value) for lang in sorted(langs, key=compilers.sort_clink): @@ -263,7 +263,7 @@ class IntrospectionInterpreter(AstInterpreter): # Pop the first element if the function is a build target function if isinstance(curr, FunctionNode) and curr.func_name in BUILD_TARGET_FUNCTIONS: arg_nodes.pop(0) - elementary_nodes = [x for x in arg_nodes if isinstance(x, (str, StringNode))] + elementary_nodes = [x for x in arg_nodes if isinstance(x, (str, BaseStringNode))] inqueue += [x for x in arg_nodes if isinstance(x, (FunctionNode, ArrayNode, IdNode, ArithmeticNode))] if elementary_nodes: res += [curr] diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index ebf63af..4626e5e 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -84,7 +84,17 @@ class AstPrinter(AstVisitor): def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: assert isinstance(node.value, str) - self.append("f'" + node.value + "'", node) + self.append("f'" + self.escape(node.value) + "'", node) + node.lineno = self.curr_line or node.lineno + + def visit_MultilineStringNode(self, node: mparser.StringNode) -> None: + assert isinstance(node.value, str) + self.append("'''" + node.value + "'''", node) + node.lineno = self.curr_line or node.lineno + + def visit_FormatMultilineStringNode(self, node: mparser.FormatStringNode) -> None: + assert isinstance(node.value, str) + self.append("f'''" + node.value + "'''", node) node.lineno = self.curr_line or node.lineno def visit_ContinueNode(self, node: mparser.ContinueNode) -> None: diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index 51a6620..69b93c7 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -43,6 +43,12 @@ class AstVisitor: def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: self.visit_default_func(node) + def visit_MultilineStringNode(self, node: mparser.StringNode) -> None: + self.visit_default_func(node) + + def visit_FormatMultilineStringNode(self, node: mparser.FormatStringNode) -> None: + self.visit_default_func(node) + def visit_ContinueNode(self, node: mparser.ContinueNode) -> None: self.visit_default_func(node) diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py index f577ba7..80dfff5 100644 --- a/mesonbuild/cmake/interpreter.py +++ b/mesonbuild/cmake/interpreter.py @@ -960,7 +960,7 @@ class CMakeInterpreter: return Token(tid, self.subdir.as_posix(), 0, 0, 0, None, val) def string(value: str) -> StringNode: - return StringNode(token(val=value)) + return StringNode(token(val=value), escape=False) def id_node(value: str) -> IdNode: return IdNode(token(val=value)) diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index f151c7b..f91c583 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -1098,7 +1098,7 @@ class MachineFileParser(): return section def _evaluate_statement(self, node: mparser.BaseNode) -> T.Union[str, bool, int, T.List[str]]: - if isinstance(node, (mparser.StringNode)): + if isinstance(node, (mparser.BaseStringNode)): return node.value elif isinstance(node, mparser.BooleanNode): return node.value diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py index 9b005cb..0814739 100644 --- a/mesonbuild/interpreter/interpreter.py +++ b/mesonbuild/interpreter/interpreter.py @@ -536,7 +536,7 @@ class Interpreter(InterpreterBase, HoldableObject): assert isinstance(kw, mparser.IdNode), 'for mypy' if kw.value == 'meson_version': # mypy does not understand "and isinstance" - if isinstance(val, mparser.StringNode): + if isinstance(val, mparser.BaseStringNode): self.handle_meson_version(val.value, val) def get_build_def_files(self) -> mesonlib.OrderedSet[str]: diff --git a/mesonbuild/interpreterbase/helpers.py b/mesonbuild/interpreterbase/helpers.py index f2ee1b1..917969b 100644 --- a/mesonbuild/interpreterbase/helpers.py +++ b/mesonbuild/interpreterbase/helpers.py @@ -25,7 +25,7 @@ if T.TYPE_CHECKING: from .baseobjects import TYPE_var, TYPE_kwargs, SubProject def flatten(args: T.Union['TYPE_var', T.List['TYPE_var']]) -> T.List['TYPE_var']: - if isinstance(args, mparser.StringNode): + if isinstance(args, mparser.BaseStringNode): assert isinstance(args.value, str) return [args.value] if not isinstance(args, collections.abc.Sequence): @@ -35,7 +35,7 @@ def flatten(args: T.Union['TYPE_var', T.List['TYPE_var']]) -> T.List['TYPE_var'] if isinstance(a, list): rest = flatten(a) result = result + rest - elif isinstance(a, mparser.StringNode): + elif isinstance(a, mparser.BaseStringNode): result.append(a.value) else: result.append(a) diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index f35da33..eab96ef 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -196,8 +196,13 @@ class InterpreterBase: self.assignment(cur) elif isinstance(cur, mparser.MethodNode): return self.method_call(cur) - elif isinstance(cur, mparser.StringNode): - return self._holderify(cur.value) + elif isinstance(cur, mparser.BaseStringNode): + if isinstance(cur, mparser.MultilineFormatStringNode): + return self.evaluate_multiline_fstring(cur) + elif isinstance(cur, mparser.FormatStringNode): + return self.evaluate_fstring(cur) + else: + return self._holderify(cur.value) elif isinstance(cur, mparser.BooleanNode): return self._holderify(cur.value) elif isinstance(cur, mparser.IfClauseNode): @@ -230,11 +235,6 @@ class InterpreterBase: return self.evaluate_indexing(cur) elif isinstance(cur, mparser.TernaryNode): return self.evaluate_ternary(cur) - elif isinstance(cur, mparser.FormatStringNode): - if isinstance(cur, mparser.MultilineFormatStringNode): - return self.evaluate_multiline_fstring(cur) - else: - return self.evaluate_fstring(cur) elif isinstance(cur, mparser.ContinueNode): raise ContinueRequest() elif isinstance(cur, mparser.BreakNode): @@ -256,7 +256,7 @@ class InterpreterBase: @FeatureNew('dict', '0.47.0') def evaluate_dictstatement(self, cur: mparser.DictNode) -> InterpreterObject: def resolve_key(key: mparser.BaseNode) -> str: - if not isinstance(key, mparser.StringNode): + if not isinstance(key, mparser.BaseStringNode): FeatureNew.single_use('Dictionary entry using non literal key', '0.53.0', self.subproject) key_holder = self.evaluate_statement(key) if key_holder is None: @@ -428,9 +428,7 @@ class InterpreterBase: return self.evaluate_fstring(node) @FeatureNew('format strings', '0.58.0') - def evaluate_fstring(self, node: mparser.FormatStringNode) -> InterpreterObject: - assert isinstance(node, mparser.FormatStringNode) - + def evaluate_fstring(self, node: T.Union[mparser.FormatStringNode, mparser.MultilineFormatStringNode]) -> InterpreterObject: def replace(match: T.Match[str]) -> str: var = str(match.group(1)) try: diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py index 0391535..cf4a53a 100644 --- a/mesonbuild/mintro.py +++ b/mesonbuild/mintro.py @@ -36,7 +36,7 @@ from .dependencies import Dependency from . import environment from .interpreterbase import ObjectHolder from .mesonlib import OptionKey -from .mparser import FunctionNode, ArrayNode, ArgumentNode, StringNode +from .mparser import FunctionNode, ArrayNode, ArgumentNode, BaseStringNode if T.TYPE_CHECKING: import argparse @@ -194,7 +194,7 @@ def list_targets_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[st elif isinstance(n, ArgumentNode): args = n.arguments for j in args: - if isinstance(j, StringNode): + if isinstance(j, BaseStringNode): assert isinstance(j.value, str) res += [Path(j.value)] elif isinstance(j, str): diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index f9b8967..6144794 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -193,23 +193,15 @@ class Lexer: elif tid == 'dblquote': raise ParseException('Double quotes are not supported. Use single quotes.', self.getline(line_start), lineno, col) elif tid in {'string', 'fstring'}: - # Handle here and not on the regexp to give a better error message. if match_text.find("\n") != -1: msg = ("Newline character in a string detected, use ''' (three single quotes) " "for multiline strings instead.\n" "This will become a hard error in a future Meson release.") mlog.warning(mlog.code_line(msg, self.getline(line_start), col), location=BaseNode(lineno, col, filename)) value = match_text[2 if tid == 'fstring' else 1:-1] - value = ESCAPE_SEQUENCE_SINGLE_RE.sub(decode_match, value) elif tid in {'multiline_string', 'multiline_fstring'}: - # For multiline strings, parse out the value and pass - # through the normal string logic. - # For multiline format strings, we have to emit a - # different AST node so we can add a feature check, - # but otherwise, it follows the normal fstring logic. if tid == 'multiline_string': value = match_text[3:-3] - tid = 'string' else: value = match_text[4:-3] lines = match_text.split('\n') @@ -295,13 +287,30 @@ class NumberNode(ElementaryNode[int]): self.value = int(token.value, base=0) self.bytespan = token.bytespan -class StringNode(ElementaryNode[str]): +class BaseStringNode(ElementaryNode[str]): pass -class FormatStringNode(ElementaryNode[str]): +@dataclass(unsafe_hash=True) +class StringNode(BaseStringNode): + + raw_value: str = field(hash=False) + + def __init__(self, token: Token[str], escape: bool = True): + super().__init__(token) + self.value = ESCAPE_SEQUENCE_SINGLE_RE.sub(decode_match, token.value) if escape else token.value + self.raw_value = token.value + +class FormatStringNode(StringNode): pass -class MultilineFormatStringNode(FormatStringNode): +@dataclass(unsafe_hash=True) +class MultilineStringNode(BaseStringNode): + + def __init__(self, token: Token[str]): + super().__init__(token) + self.value = token.value + +class MultilineFormatStringNode(MultilineStringNode): pass class ContinueNode(ElementaryNode): @@ -819,6 +828,8 @@ class Parser: return StringNode(t) if self.accept('fstring'): return FormatStringNode(t) + if self.accept('multiline_string'): + return MultilineStringNode(t) if self.accept('multiline_fstring'): return MultilineFormatStringNode(t) return EmptyNode(self.current.lineno, self.current.colno, self.current.filename) diff --git a/mesonbuild/optinterpreter.py b/mesonbuild/optinterpreter.py index 37ef55a..9eeb3fd 100644 --- a/mesonbuild/optinterpreter.py +++ b/mesonbuild/optinterpreter.py @@ -115,7 +115,7 @@ class OptionInterpreter: return arg if isinstance(arg, mparser.ParenthesizedNode): return self.reduce_single(arg.inner) - elif isinstance(arg, (mparser.StringNode, mparser.BooleanNode, + elif isinstance(arg, (mparser.BaseStringNode, mparser.BooleanNode, mparser.NumberNode)): return arg.value elif isinstance(arg, mparser.ArrayNode): @@ -123,7 +123,7 @@ class OptionInterpreter: elif isinstance(arg, mparser.DictNode): d = {} for k, v in arg.args.kwargs.items(): - if not isinstance(k, mparser.StringNode): + if not isinstance(k, mparser.BaseStringNode): raise OptionException('Dictionary keys must be a string literal') d[k.value] = self.reduce_single(v) return d diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index a9b2e88..0d7b111 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -28,7 +28,7 @@ from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionL from mesonbuild.mesonlib import MesonException, setup_vsenv from . import mlog, environment from functools import wraps -from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode +from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseStringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode import json, os, re, sys import typing as T @@ -267,12 +267,12 @@ class MTypeStrList(MTypeList): return StringNode(Token('', '', 0, 0, 0, None, str(value))) def _check_is_equal(self, node, value) -> bool: - if isinstance(node, StringNode): + if isinstance(node, BaseStringNode): return node.value == value return False def _check_regex_matches(self, node, regex: str) -> bool: - if isinstance(node, StringNode): + if isinstance(node, BaseStringNode): return re.match(regex, node.value) is not None return False @@ -292,7 +292,7 @@ class MTypeIDList(MTypeList): return False def _check_regex_matches(self, node, regex: str) -> bool: - if isinstance(node, StringNode): + if isinstance(node, BaseStringNode): return re.match(regex, node.value) is not None return False @@ -652,7 +652,7 @@ class Rewriter: src_list = [] for i in target['sources']: for j in arg_list_from_node(i): - if isinstance(j, StringNode): + if isinstance(j, BaseStringNode): src_list += [j.value] # Generate the new String nodes @@ -686,7 +686,7 @@ class Rewriter: def find_node(src): for i in target['sources']: for j in arg_list_from_node(i): - if isinstance(j, StringNode): + if isinstance(j, BaseStringNode): if j.value == src: return i, j return None, None @@ -745,7 +745,7 @@ class Rewriter: extra_files_list = [] for i in target['extra_files']: for j in arg_list_from_node(i): - if isinstance(j, StringNode): + if isinstance(j, BaseStringNode): extra_files_list += [j.value] # Generate the new String nodes @@ -776,7 +776,7 @@ class Rewriter: def find_node(src): for i in target['extra_files']: for j in arg_list_from_node(i): - if isinstance(j, StringNode): + if isinstance(j, BaseStringNode): if j.value == src: return i, j return None, None @@ -845,12 +845,12 @@ class Rewriter: src_list = [] for i in target['sources']: for j in arg_list_from_node(i): - if isinstance(j, StringNode): + if isinstance(j, BaseStringNode): src_list += [j.value] extra_files_list = [] for i in target['extra_files']: for j in arg_list_from_node(i): - if isinstance(j, StringNode): + if isinstance(j, BaseStringNode): extra_files_list += [j.value] test_data = { 'name': target['name'], @@ -865,8 +865,8 @@ class Rewriter: alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)] path_sorter = lambda key: ([(key.count('/') <= idx, alphanum_key(x)) for idx, x in enumerate(key.split('/'))]) - unknown = [x for x in i.arguments if not isinstance(x, StringNode)] - sources = [x for x in i.arguments if isinstance(x, StringNode)] + unknown = [x for x in i.arguments if not isinstance(x, BaseStringNode)] + sources = [x for x in i.arguments if isinstance(x, BaseStringNode)] sources = sorted(sources, key=lambda x: path_sorter(x.value)) i.arguments = unknown + sources -- cgit v1.1 From 306562b4666dea7828af680afbd4d111e667027f Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 14:22:15 -0400 Subject: parser: use IdNode for function name and assignment name --- mesonbuild/ast/interpreter.py | 19 +++++++++--------- mesonbuild/ast/introspection.py | 2 +- mesonbuild/ast/printer.py | 16 +++++++-------- mesonbuild/ast/visitor.py | 4 ++++ mesonbuild/cargo/builder.py | 6 +++--- mesonbuild/cmake/interpreter.py | 6 +++--- mesonbuild/interpreter/interpreter.py | 4 ++-- mesonbuild/interpreterbase/interpreterbase.py | 10 +++++----- mesonbuild/mintro.py | 2 +- mesonbuild/mparser.py | 28 ++++++++++++--------------- mesonbuild/optinterpreter.py | 2 +- mesonbuild/rewriter.py | 14 +++++++------- 12 files changed, 57 insertions(+), 56 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index 501be7d..1dea817 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -254,10 +254,10 @@ class AstInterpreter(InterpreterBase): def evaluate_plusassign(self, node: PlusAssignmentNode) -> None: assert isinstance(node, PlusAssignmentNode) # Cheat by doing a reassignment - self.assignments[node.var_name] = node.value # Save a reference to the value node + self.assignments[node.var_name.value] = node.value # Save a reference to the value node if node.value.ast_id: self.reverse_assignment[node.value.ast_id] = node - self.assign_vals[node.var_name] = self.evaluate_statement(node.value) + self.assign_vals[node.var_name.value] = self.evaluate_statement(node.value) def evaluate_indexing(self, node: IndexNode) -> int: return 0 @@ -319,10 +319,10 @@ class AstInterpreter(InterpreterBase): def assignment(self, node: AssignmentNode) -> None: assert isinstance(node, AssignmentNode) - self.assignments[node.var_name] = node.value # Save a reference to the value node + self.assignments[node.var_name.value] = node.value # Save a reference to the value node if node.value.ast_id: self.reverse_assignment[node.value.ast_id] = node - self.assign_vals[node.var_name] = self.evaluate_statement(node.value) # Evaluate the value just in case + self.assign_vals[node.var_name.value] = self.evaluate_statement(node.value) # Evaluate the value just in case def resolve_node(self, node: BaseNode, include_unknown_args: bool = False, id_loop_detect: T.Optional[T.List[str]] = None) -> T.Optional[T.Any]: def quick_resolve(n: BaseNode, loop_detect: T.Optional[T.List[str]] = None) -> T.Any: @@ -382,17 +382,18 @@ class AstInterpreter(InterpreterBase): src = quick_resolve(node.source_object) margs = self.flatten_args(node.args.arguments, include_unknown_args, id_loop_detect) mkwargs: T.Dict[str, TYPE_nvar] = {} + method_name = node.name.value try: if isinstance(src, str): - result = StringHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) + result = StringHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) elif isinstance(src, bool): - result = BooleanHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) + result = BooleanHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) elif isinstance(src, int): - result = IntegerHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) + result = IntegerHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) elif isinstance(src, list): - result = ArrayHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) + result = ArrayHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) elif isinstance(src, dict): - result = DictHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) + result = DictHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) except mesonlib.MesonException: return None diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py index 50aa744..fa4b8d3 100644 --- a/mesonbuild/ast/introspection.py +++ b/mesonbuild/ast/introspection.py @@ -261,7 +261,7 @@ class IntrospectionInterpreter(AstInterpreter): continue arg_nodes = arg_node.arguments.copy() # Pop the first element if the function is a build target function - if isinstance(curr, FunctionNode) and curr.func_name in BUILD_TARGET_FUNCTIONS: + if isinstance(curr, FunctionNode) and curr.func_name.value in BUILD_TARGET_FUNCTIONS: arg_nodes.pop(0) elementary_nodes = [x for x in arg_nodes if isinstance(x, (str, BaseStringNode))] inqueue += [x for x in arg_nodes if isinstance(x, (FunctionNode, ArrayNode, IdNode, ArithmeticNode))] diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index 4626e5e..020299d 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -162,24 +162,24 @@ class AstPrinter(AstVisitor): def visit_MethodNode(self, node: mparser.MethodNode) -> None: node.lineno = self.curr_line or node.lineno node.source_object.accept(self) - self.append('.' + node.name + '(', node) + self.append('.' + node.name.value + '(', node) node.args.accept(self) self.append(')', node) def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: node.lineno = self.curr_line or node.lineno - self.append(node.func_name + '(', node) + self.append(node.func_name.value + '(', node) node.args.accept(self) self.append(')', node) def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None: node.lineno = self.curr_line or node.lineno - self.append(node.var_name + ' = ', node) + self.append(node.var_name.value + ' = ', node) node.value.accept(self) def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None: node.lineno = self.curr_line or node.lineno - self.append(node.var_name + ' += ', node) + self.append(node.var_name.value + ' += ', node) node.value.accept(self) def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: @@ -352,22 +352,22 @@ class AstJSONPrinter(AstVisitor): def visit_MethodNode(self, node: mparser.MethodNode) -> None: self._accept('object', node.source_object) self._accept('args', node.args) - self.current['name'] = node.name + self.current['name'] = node.name.value self.setbase(node) def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: self._accept('args', node.args) - self.current['name'] = node.func_name + self.current['name'] = node.func_name.value self.setbase(node) def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None: self._accept('value', node.value) - self.current['var_name'] = node.var_name + self.current['var_name'] = node.var_name.value self.setbase(node) def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None: self._accept('value', node.value) - self.current['var_name'] = node.var_name + self.current['var_name'] = node.var_name.value self.setbase(node) def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index 69b93c7..6e40ea0 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -103,18 +103,22 @@ class AstVisitor: def visit_MethodNode(self, node: mparser.MethodNode) -> None: self.visit_default_func(node) node.source_object.accept(self) + node.name.accept(self) node.args.accept(self) def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: self.visit_default_func(node) + node.func_name.accept(self) node.args.accept(self) def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None: self.visit_default_func(node) + node.var_name.accept(self) node.value.accept(self) def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None: self.visit_default_func(node) + node.var_name.accept(self) node.value.accept(self) def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: diff --git a/mesonbuild/cargo/builder.py b/mesonbuild/cargo/builder.py index 2442fbf..6ef66ee 100644 --- a/mesonbuild/cargo/builder.py +++ b/mesonbuild/cargo/builder.py @@ -97,7 +97,7 @@ def method(name: str, id_: mparser.IdNode, args.arguments = pos if kw is not None: args.kwargs = {identifier(k, id_.filename): v for k, v in kw.items()} - return mparser.MethodNode(id_.filename, -1, -1, id_, name, args) + return mparser.MethodNode(id_.filename, -1, -1, id_, identifier(name, id_.filename), args) def function(name: str, filename: str, @@ -117,7 +117,7 @@ def function(name: str, filename: str, args.arguments = pos if kw is not None: args.kwargs = {identifier(k, filename): v for k, v in kw.items()} - return mparser.FunctionNode(filename, -1, -1, -1, -1, name, args) + return mparser.FunctionNode(filename, -1, -1, -1, -1, identifier(name, filename), args) def equal(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.ComparisonNode: @@ -168,7 +168,7 @@ def assign(value: mparser.BaseNode, varname: str, filename: str) -> mparser.Assi :param filename: The filename :return: An AssignmentNode """ - return mparser.AssignmentNode(filename, -1, -1, varname, value) + return mparser.AssignmentNode(filename, -1, -1, identifier(varname, filename), value) def block(filename: str) -> mparser.CodeBlockNode: diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py index 80dfff5..0703c1b 100644 --- a/mesonbuild/cmake/interpreter.py +++ b/mesonbuild/cmake/interpreter.py @@ -1002,7 +1002,7 @@ class CMakeInterpreter: args = [args] args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} - func_n = FunctionNode(self.subdir.as_posix(), 0, 0, 0, 0, name, args_n) + func_n = FunctionNode(self.subdir.as_posix(), 0, 0, 0, 0, id_node(name), args_n) return func_n def method(obj: BaseNode, name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> MethodNode: @@ -1014,10 +1014,10 @@ class CMakeInterpreter: args = [args] args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} - return MethodNode(self.subdir.as_posix(), 0, 0, obj, name, args_n) + return MethodNode(self.subdir.as_posix(), 0, 0, obj, id_node(name), args_n) def assign(var_name: str, value: BaseNode) -> AssignmentNode: - return AssignmentNode(self.subdir.as_posix(), 0, 0, var_name, value) + return AssignmentNode(self.subdir.as_posix(), 0, 0, id_node(var_name), value) # Generate the root code block and the project function call root_cb = CodeBlockNode(token()) diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py index 0814739..c637087 100644 --- a/mesonbuild/interpreter/interpreter.py +++ b/mesonbuild/interpreter/interpreter.py @@ -2942,7 +2942,7 @@ class Interpreter(InterpreterBase, HoldableObject): def _add_global_arguments(self, node: mparser.FunctionNode, argsdict: T.Dict[str, T.List[str]], args: T.List[str], kwargs: 'kwtypes.FuncAddProjectArgs') -> None: if self.is_subproject(): - msg = f'Function \'{node.func_name}\' cannot be used in subprojects because ' \ + msg = f'Function \'{node.func_name.value}\' cannot be used in subprojects because ' \ 'there is no way to make that reliable.\nPlease only call ' \ 'this if is_subproject() returns false. Alternatively, ' \ 'define a variable that\ncontains your language-specific ' \ @@ -2962,7 +2962,7 @@ class Interpreter(InterpreterBase, HoldableObject): def _add_arguments(self, node: mparser.FunctionNode, argsdict: T.Dict[str, T.List[str]], args_frozen: bool, args: T.List[str], kwargs: 'kwtypes.FuncAddProjectArgs') -> None: if args_frozen: - msg = f'Tried to use \'{node.func_name}\' after a build target has been declared.\n' \ + msg = f'Tried to use \'{node.func_name.value}\' after a build target has been declared.\n' \ 'This is not permitted. Please declare all arguments before your targets.' raise InvalidCode(msg) diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index eab96ef..1c09128 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -137,7 +137,7 @@ class InterpreterBase: if not self.ast.lines: raise InvalidCode('No statements in code.') first = self.ast.lines[0] - if not isinstance(first, mparser.FunctionNode) or first.func_name != 'project': + if not isinstance(first, mparser.FunctionNode) or first.func_name.value != 'project': p = pathlib.Path(self.source_root).resolve() found = p for parent in p.parents: @@ -476,7 +476,7 @@ class InterpreterBase: def evaluate_plusassign(self, node: mparser.PlusAssignmentNode) -> None: assert isinstance(node, mparser.PlusAssignmentNode) - varname = node.var_name + varname = node.var_name.value addition = self.evaluate_statement(node.value) if addition is None: raise InvalidCodeOnVoid('plus assign') @@ -504,7 +504,7 @@ class InterpreterBase: return self._holderify(iobject.operator_call(MesonOperator.INDEX, index)) def function_call(self, node: mparser.FunctionNode) -> T.Optional[InterpreterObject]: - func_name = node.func_name + func_name = node.func_name.value (h_posargs, h_kwargs) = self.reduce_arguments(node.args) (posargs, kwargs) = self._unholder_args(h_posargs, h_kwargs) if is_disabled(posargs, kwargs) and func_name not in {'get_variable', 'set_variable', 'unset_variable', 'is_disabler'}: @@ -532,7 +532,7 @@ class InterpreterBase: else: object_display_name = invocable.__class__.__name__ obj = self.evaluate_statement(invocable) - method_name = node.name + method_name = node.name.value (h_args, h_kwargs) = self.reduce_arguments(node.args) (args, kwargs) = self._unholder_args(h_args, h_kwargs) if is_disabled(args, kwargs): @@ -628,7 +628,7 @@ class InterpreterBase: Tried to assign values inside an argument list. To specify a keyword argument, use : instead of =. ''')) - var_name = node.var_name + var_name = node.var_name.value if not isinstance(var_name, str): raise InvalidArguments('Tried to assign value to a non-variable.') value = self.evaluate_statement(node.value) diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py index cf4a53a..7bd4205 100644 --- a/mesonbuild/mintro.py +++ b/mesonbuild/mintro.py @@ -187,7 +187,7 @@ def list_targets_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[st args: T.List[BaseNode] = [] if isinstance(n, FunctionNode): args = list(n.args.arguments) - if n.func_name in BUILD_TARGET_FUNCTIONS: + if n.func_name.value in BUILD_TARGET_FUNCTIONS: args.pop(0) elif isinstance(n, ArrayNode): args = n.args.arguments diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 6144794..ceddf0a 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -470,52 +470,48 @@ class IndexNode(BaseNode): class MethodNode(BaseNode): source_object: BaseNode - name: str + name: IdNode args: ArgumentNode - def __init__(self, filename: str, lineno: int, colno: int, source_object: BaseNode, name: str, args: ArgumentNode): + def __init__(self, filename: str, lineno: int, colno: int, source_object: BaseNode, name: IdNode, args: ArgumentNode): super().__init__(lineno, colno, filename) self.source_object = source_object self.name = name - assert isinstance(self.name, str) self.args = args @dataclass(unsafe_hash=True) class FunctionNode(BaseNode): - func_name: str + func_name: IdNode args: ArgumentNode - def __init__(self, filename: str, lineno: int, colno: int, end_lineno: int, end_colno: int, func_name: str, args: ArgumentNode): + def __init__(self, filename: str, lineno: int, colno: int, end_lineno: int, end_colno: int, func_name: IdNode, args: ArgumentNode): super().__init__(lineno, colno, filename, end_lineno=end_lineno, end_colno=end_colno) self.func_name = func_name - assert isinstance(func_name, str) self.args = args @dataclass(unsafe_hash=True) class AssignmentNode(BaseNode): - var_name: str + var_name: IdNode value: BaseNode - def __init__(self, filename: str, lineno: int, colno: int, var_name: str, value: BaseNode): + def __init__(self, filename: str, lineno: int, colno: int, var_name: IdNode, value: BaseNode): super().__init__(lineno, colno, filename) self.var_name = var_name - assert isinstance(var_name, str) self.value = value @dataclass(unsafe_hash=True) class PlusAssignmentNode(BaseNode): - var_name: str + var_name: IdNode value: BaseNode - def __init__(self, filename: str, lineno: int, colno: int, var_name: str, value: BaseNode): + def __init__(self, filename: str, lineno: int, colno: int, var_name: IdNode, value: BaseNode): super().__init__(lineno, colno, filename) self.var_name = var_name - assert isinstance(var_name, str) self.value = value @@ -687,14 +683,14 @@ class Parser: if not isinstance(left, IdNode): raise ParseException('Plusassignment target must be an id.', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - return PlusAssignmentNode(left.filename, left.lineno, left.colno, left.value, value) + return PlusAssignmentNode(left.filename, left.lineno, left.colno, left, value) elif self.accept('assign'): value = self.e1() if not isinstance(left, IdNode): raise ParseException('Assignment target must be an id.', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - return AssignmentNode(left.filename, left.lineno, left.colno, left.value, value) + return AssignmentNode(left.filename, left.lineno, left.colno, left, value) elif self.accept('questionmark'): if self.in_ternary: raise ParseException('Nested ternary operators are not allowed.', @@ -783,7 +779,7 @@ class Parser: raise ParseException('Function call must be applied to plain id', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - left = FunctionNode(left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left.value, args) + left = FunctionNode(left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left, args) go_again = True while go_again: go_again = False @@ -887,7 +883,7 @@ class Parser: self.expect('lparen') args = self.args() self.expect('rparen') - method = MethodNode(methodname.filename, methodname.lineno, methodname.colno, source_object, methodname.value, args) + method = MethodNode(methodname.filename, methodname.lineno, methodname.colno, source_object, methodname, args) if self.accept('dot'): return self.method_call(method) return method diff --git a/mesonbuild/optinterpreter.py b/mesonbuild/optinterpreter.py index 9eeb3fd..895cada 100644 --- a/mesonbuild/optinterpreter.py +++ b/mesonbuild/optinterpreter.py @@ -164,7 +164,7 @@ class OptionInterpreter: def evaluate_statement(self, node: mparser.BaseNode) -> None: if not isinstance(node, mparser.FunctionNode): raise OptionException('Option file may only contain option definitions') - func_name = node.func_name + func_name = node.func_name.value if func_name != 'option': raise OptionException('Only calls to option() are allowed in option files.') (posargs, kwargs) = self.reduce_arguments(node.args) diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 0d7b111..af3210c 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -420,7 +420,7 @@ class Rewriter: if target in self.interpreter.assignments: node = self.interpreter.assignments[target] if isinstance(node, FunctionNode): - if node.func_name in {'executable', 'jar', 'library', 'shared_library', 'shared_module', 'static_library', 'both_libraries'}: + if node.func_name.value in {'executable', 'jar', 'library', 'shared_library', 'shared_module', 'static_library', 'both_libraries'}: tgt = self.interpreter.assign_vals[target] return tgt @@ -440,7 +440,7 @@ class Rewriter: if dependency in self.interpreter.assignments: node = self.interpreter.assignments[dependency] if isinstance(node, FunctionNode): - if node.func_name == 'dependency': + if node.func_name.value == 'dependency': name = self.interpreter.flatten_args(node.args)[0] dep = check_list(name) @@ -630,7 +630,7 @@ class Rewriter: args = [] if isinstance(n, FunctionNode): args = list(n.args.arguments) - if n.func_name in BUILD_TARGET_FUNCTIONS: + if n.func_name.value in BUILD_TARGET_FUNCTIONS: args.pop(0) elif isinstance(n, ArrayNode): args = n.args.arguments @@ -814,15 +814,15 @@ class Rewriter: src_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) src_arr_node = ArrayNode(src_arg_node, 0, 0, 0, 0) src_far_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - src_fun_node = FunctionNode(filename, 0, 0, 0, 0, 'files', src_far_node) - src_ass_node = AssignmentNode(filename, 0, 0, source_id, src_fun_node) + src_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), src_far_node) + src_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), src_fun_node) src_arg_node.arguments = [StringNode(Token('string', filename, 0, 0, 0, None, x)) for x in cmd['sources']] src_far_node.arguments = [src_arr_node] # Build target tgt_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, cmd['target_type'], tgt_arg_node) - tgt_ass_node = AssignmentNode(filename, 0, 0, target_id, tgt_fun_node) + tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), tgt_arg_node) + tgt_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), tgt_fun_node) tgt_arg_node.arguments = [ StringNode(Token('string', filename, 0, 0, 0, None, cmd['target'])), IdNode(Token('string', filename, 0, 0, 0, None, source_id)) -- cgit v1.1 From 8d357ba62cba0c5ddf3265743454b0ddcb08d83f Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 14:28:40 -0400 Subject: parser: use IdNode for foreach varnames --- mesonbuild/ast/printer.py | 4 ++-- mesonbuild/ast/visitor.py | 2 ++ mesonbuild/interpreterbase/interpreterbase.py | 4 ++-- mesonbuild/mparser.py | 8 ++++---- 4 files changed, 10 insertions(+), 8 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index 020299d..410aabd 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -185,7 +185,7 @@ class AstPrinter(AstVisitor): def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: node.lineno = self.curr_line or node.lineno self.append_padded('foreach', node) - self.append_padded(', '.join(node.varnames), node) + self.append_padded(', '.join(varname.value for varname in node.varnames), node) self.append_padded(':', node) node.items.accept(self) self.newline() @@ -373,7 +373,7 @@ class AstJSONPrinter(AstVisitor): def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: self._accept('items', node.items) self._accept('block', node.block) - self.current['varnames'] = node.varnames + self.current['varnames'] = [varname.value for varname in node.varnames] self.setbase(node) def visit_IfClauseNode(self, node: mparser.IfClauseNode) -> None: diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index 6e40ea0..5b1fb6a 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -123,6 +123,8 @@ class AstVisitor: def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: self.visit_default_func(node) + for varname in node.varnames: + varname.accept(self) node.items.accept(self) node.block.accept(self) diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index 1c09128..7e3f8da 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -459,14 +459,14 @@ class InterpreterBase: if tsize is None: if isinstance(i, tuple): raise mesonlib.MesonBugException(f'Iteration of {items} returned a tuple even though iter_tuple_size() is None') - self.set_variable(node.varnames[0], self._holderify(i)) + self.set_variable(node.varnames[0].value, self._holderify(i)) else: if not isinstance(i, tuple): raise mesonlib.MesonBugException(f'Iteration of {items} did not return a tuple even though iter_tuple_size() is {tsize}') if len(i) != tsize: raise mesonlib.MesonBugException(f'Iteration of {items} did not return a tuple even though iter_tuple_size() is {tsize}') for j in range(tsize): - self.set_variable(node.varnames[j], self._holderify(i[j])) + self.set_variable(node.varnames[j].value, self._holderify(i[j])) try: self.evaluate_codeblock(node.block) except ContinueRequest: diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index ceddf0a..2d7f115 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -518,11 +518,11 @@ class PlusAssignmentNode(BaseNode): @dataclass(unsafe_hash=True) class ForeachClauseNode(BaseNode): - varnames: T.List[str] = field(hash=False) + varnames: T.List[IdNode] = field(hash=False) items: BaseNode block: CodeBlockNode - def __init__(self, token: Token, varnames: T.List[str], items: BaseNode, block: CodeBlockNode): + def __init__(self, token: Token, varnames: T.List[IdNode], items: BaseNode, block: CodeBlockNode): super().__init__(token.lineno, token.colno, token.filename) self.varnames = varnames self.items = items @@ -898,13 +898,13 @@ class Parser: self.expect('id') assert isinstance(t.value, str) varname = t - varnames = [t.value] + varnames = [IdNode(t)] if self.accept('comma'): t = self.current self.expect('id') assert isinstance(t.value, str) - varnames.append(t.value) + varnames.append(IdNode(t)) self.expect('colon') items = self.statement() -- cgit v1.1 From b94167ef50e891355a7d9a0014e6b0cf4ea1302d Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 15:10:07 -0400 Subject: parser: preserve value of all tokens --- mesonbuild/mparser.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 2d7f115..2ff32dc 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -163,7 +163,7 @@ class Lexer: col = 0 while loc < len(self.code): matched = False - value: T.Union[str, bool, int] = None + value: str = '' for (tid, reg) in self.token_specification: mo = reg.match(self.code, loc) if mo: @@ -175,7 +175,7 @@ class Lexer: loc = mo.end() span_end = loc bytespan = (span_start, span_end) - match_text = mo.group() + value = mo.group() if tid in {'ignore', 'comment'}: break elif tid == 'lparen': @@ -193,23 +193,18 @@ class Lexer: elif tid == 'dblquote': raise ParseException('Double quotes are not supported. Use single quotes.', self.getline(line_start), lineno, col) elif tid in {'string', 'fstring'}: - if match_text.find("\n") != -1: + if value.find("\n") != -1: msg = ("Newline character in a string detected, use ''' (three single quotes) " "for multiline strings instead.\n" "This will become a hard error in a future Meson release.") mlog.warning(mlog.code_line(msg, self.getline(line_start), col), location=BaseNode(lineno, col, filename)) - value = match_text[2 if tid == 'fstring' else 1:-1] + value = value[2 if tid == 'fstring' else 1:-1] elif tid in {'multiline_string', 'multiline_fstring'}: - if tid == 'multiline_string': - value = match_text[3:-3] - else: - value = match_text[4:-3] - lines = match_text.split('\n') + value = value[4 if tid == 'multiline_fstring' else 3:-3] + lines = value.split('\n') if len(lines) > 1: lineno += len(lines) - 1 line_start = mo.end() - len(lines[-1]) - elif tid == 'number': - value = match_text elif tid == 'eol_cont': lineno += 1 line_start = loc @@ -220,13 +215,12 @@ class Lexer: if par_count > 0 or bracket_count > 0 or curl_count > 0: break elif tid == 'id': - if match_text in self.keywords: - tid = match_text + if value in self.keywords: + tid = value else: - if match_text in self.future_keywords: - mlog.warning(f"Identifier '{match_text}' will become a reserved keyword in a future release. Please rename it.", + if value in self.future_keywords: + mlog.warning(f"Identifier '{value}' will become a reserved keyword in a future release. Please rename it.", location=BaseNode(lineno, col, filename)) - value = match_text yield Token(tid, filename, curline_start, curline, col, bytespan, value) break if not matched: -- cgit v1.1 From 4b7a56caa265c54a9b77381e2db1819ed87f21de Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 08:58:23 -0400 Subject: parser: remember previous Token --- mesonbuild/mparser.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 2ff32dc..ce83cc2 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -623,10 +623,12 @@ class Parser: self.lexer = Lexer(code) self.stream = self.lexer.lex(filename) self.current: Token = Token('eof', '', 0, 0, 0, (0, 0), None) + self.previous = self.current self.getsym() self.in_ternary = False def getsym(self) -> None: + self.previous = self.current try: self.current = next(self.stream) except StopIteration: @@ -831,10 +833,9 @@ class Parser: while not isinstance(s, EmptyNode): if self.accept('colon'): a.set_kwarg_no_check(s, self.statement()) - potential = self.current if not self.accept('comma'): return a - a.commas.append(potential) + a.commas.append(self.previous) else: raise ParseException('Only key:value pairs are valid in dict construction.', self.getline(), s.lineno, s.colno) @@ -846,19 +847,17 @@ class Parser: a = ArgumentNode(self.current) while not isinstance(s, EmptyNode): - potential = self.current if self.accept('comma'): - a.commas.append(potential) + a.commas.append(self.previous) a.append(s) elif self.accept('colon'): if not isinstance(s, IdNode): raise ParseException('Dictionary key must be a plain identifier.', self.getline(), s.lineno, s.colno) a.set_kwarg(s, self.statement()) - potential = self.current if not self.accept('comma'): return a - a.commas.append(potential) + a.commas.append(self.previous) else: a.append(s) return a @@ -888,17 +887,15 @@ class Parser: return IndexNode(source_object, index_statement) def foreachblock(self) -> ForeachClauseNode: - t = self.current self.expect('id') - assert isinstance(t.value, str) - varname = t - varnames = [IdNode(t)] + assert isinstance(self.previous.value, str) + varname = self.previous + varnames = [IdNode(self.previous)] if self.accept('comma'): - t = self.current self.expect('id') - assert isinstance(t.value, str) - varnames.append(IdNode(t)) + assert isinstance(self.previous.value, str) + varnames.append(IdNode(self.previous)) self.expect('colon') items = self.statement() -- cgit v1.1 From f13260dd43771bd7e9594cbad942b7d9aa4292fc Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 09:18:21 -0400 Subject: parser: add ElseNode --- mesonbuild/ast/interpreter.py | 2 +- mesonbuild/ast/visitor.py | 4 ++++ mesonbuild/interpreterbase/interpreterbase.py | 2 +- mesonbuild/mparser.py | 17 +++++++++++++---- 4 files changed, 19 insertions(+), 6 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index 1dea817..da2119c 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -312,7 +312,7 @@ class AstInterpreter(InterpreterBase): for i in node.ifs: self.evaluate_codeblock(i.block) if not isinstance(node.elseblock, EmptyNode): - self.evaluate_codeblock(node.elseblock) + self.evaluate_codeblock(node.elseblock.block) def get_variable(self, varname: str) -> int: return 0 diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index 5b1fb6a..a837826 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -143,6 +143,10 @@ class AstVisitor: node.condition.accept(self) node.block.accept(self) + def visit_ElseNode(self, node: mparser.IfNode) -> None: + self.visit_default_func(node) + node.block.accept(self) + def visit_TernaryNode(self, node: mparser.TernaryNode) -> None: self.visit_default_func(node) node.condition.accept(self) diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index 7e3f8da..5b07dc0 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -303,7 +303,7 @@ class InterpreterBase: mesonlib.project_meson_versions[self.subproject] = prev_meson_version return None if not isinstance(node.elseblock, mparser.EmptyNode): - self.evaluate_codeblock(node.elseblock) + self.evaluate_codeblock(node.elseblock.block) return None def evaluate_testcase(self, node: mparser.TestCaseClauseNode) -> T.Optional[Disabler]: diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index ce83cc2..6b578b5 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -534,17 +534,25 @@ class IfNode(BaseNode): self.condition = condition self.block = block +@dataclass(unsafe_hash=True) +class ElseNode(BaseNode): + + block: CodeBlockNode + + def __init__(self, block: CodeBlockNode): + super().__init__(block.lineno, block.colno, block.filename) + self.block = block @dataclass(unsafe_hash=True) class IfClauseNode(BaseNode): ifs: T.List[IfNode] = field(hash=False) - elseblock: T.Union[EmptyNode, CodeBlockNode] + elseblock: T.Union[EmptyNode, ElseNode] def __init__(self, linenode: BaseNode): super().__init__(linenode.lineno, linenode.colno, linenode.filename) self.ifs = [] - self.elseblock = None + self.elseblock = EmptyNode(linenode.lineno, linenode.colno, linenode.filename) @dataclass(unsafe_hash=True) class TestCaseClauseNode(BaseNode): @@ -919,10 +927,11 @@ class Parser: b = self.codeblock() clause.ifs.append(IfNode(s, s, b)) - def elseblock(self) -> T.Union[CodeBlockNode, EmptyNode]: + def elseblock(self) -> T.Union[ElseNode, EmptyNode]: if self.accept('else'): self.expect('eol') - return self.codeblock() + block = self.codeblock() + return ElseNode(block) return EmptyNode(self.current.lineno, self.current.colno, self.current.filename) def testcaseblock(self) -> TestCaseClauseNode: -- cgit v1.1 From 02ff9553dbe09f3f2b7a93221dfd28fc53926d0e Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 09:24:17 -0400 Subject: parser: add SymbolNode to preserve operators --- mesonbuild/ast/visitor.py | 6 + mesonbuild/cargo/builder.py | 20 +-- mesonbuild/cmake/interpreter.py | 14 ++- mesonbuild/mparser.py | 266 ++++++++++++++++++++++++++++++---------- mesonbuild/rewriter.py | 19 +-- 5 files changed, 239 insertions(+), 86 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index a837826..d05d3ff 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -55,6 +55,12 @@ class AstVisitor: def visit_BreakNode(self, node: mparser.BreakNode) -> None: self.visit_default_func(node) + def visit_SymbolNode(self, node: mparser.SymbolNode) -> None: + self.visit_default_func(node) + + def visit_WhitespaceNode(self, node: mparser.WhitespaceNode) -> None: + self.visit_default_func(node) + def visit_ArrayNode(self, node: mparser.ArrayNode) -> None: self.visit_default_func(node) node.args.accept(self) diff --git a/mesonbuild/cargo/builder.py b/mesonbuild/cargo/builder.py index 6ef66ee..9c650b9 100644 --- a/mesonbuild/cargo/builder.py +++ b/mesonbuild/cargo/builder.py @@ -28,6 +28,10 @@ def _token(tid: str, filename: str, value: mparser.TV_TokenTypes) -> mparser.Tok return mparser.Token(tid, filename, -1, -1, -1, (-1, -1), value) +def _symbol(filename: str, val: str) -> mparser.SymbolNode: + return mparser.SymbolNode(_token('', filename, val)) + + def string(value: str, filename: str) -> mparser.StringNode: """Build A StringNode @@ -67,7 +71,7 @@ def array(value: T.List[mparser.BaseNode], filename: str) -> mparser.ArrayNode: """ args = mparser.ArgumentNode(_token('array', filename, 'unused')) args.arguments = value - return mparser.ArrayNode(args, -1, -1, -1, -1) + return mparser.ArrayNode(_symbol(filename, '['), args, _symbol(filename, ']'), -1, -1, -1, -1) def identifier(value: str, filename: str) -> mparser.IdNode: @@ -97,7 +101,7 @@ def method(name: str, id_: mparser.IdNode, args.arguments = pos if kw is not None: args.kwargs = {identifier(k, id_.filename): v for k, v in kw.items()} - return mparser.MethodNode(id_.filename, -1, -1, id_, identifier(name, id_.filename), args) + return mparser.MethodNode(id_.filename, -1, -1, id_, _symbol(id_.filename, '.'), identifier(name, id_.filename), _symbol(id_.filename, '('), args, _symbol(id_.filename, ')')) def function(name: str, filename: str, @@ -117,7 +121,7 @@ def function(name: str, filename: str, args.arguments = pos if kw is not None: args.kwargs = {identifier(k, filename): v for k, v in kw.items()} - return mparser.FunctionNode(filename, -1, -1, -1, -1, identifier(name, filename), args) + return mparser.FunctionNode(filename, -1, -1, -1, -1, identifier(name, filename), _symbol(filename, '('), args, _symbol(filename, ')')) def equal(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.ComparisonNode: @@ -127,7 +131,7 @@ def equal(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.ComparisonNod :param rhs: the right hand side of the equal :return: A compraison node """ - return mparser.ComparisonNode('==', lhs, rhs) + return mparser.ComparisonNode('==', lhs, _symbol(lhs.filename, '=='), rhs) def or_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.OrNode: @@ -137,7 +141,7 @@ def or_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.OrNode: :param rhs: The Right of the Node :return: The OrNode """ - return mparser.OrNode(lhs, rhs) + return mparser.OrNode(lhs, _symbol(lhs.filename, 'or'), rhs) def and_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.AndNode: @@ -147,7 +151,7 @@ def and_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.AndNode: :param rhs: The right of the And :return: The AndNode """ - return mparser.AndNode(lhs, rhs) + return mparser.AndNode(lhs, _symbol(lhs.filename, 'and'), rhs) def not_(value: mparser.BaseNode, filename: str) -> mparser.NotNode: @@ -157,7 +161,7 @@ def not_(value: mparser.BaseNode, filename: str) -> mparser.NotNode: :param filename: the string filename :return: The NotNode """ - return mparser.NotNode(_token('not', filename, ''), value) + return mparser.NotNode(_token('not', filename, ''), _symbol(filename, 'not'), value) def assign(value: mparser.BaseNode, varname: str, filename: str) -> mparser.AssignmentNode: @@ -168,7 +172,7 @@ def assign(value: mparser.BaseNode, varname: str, filename: str) -> mparser.Assi :param filename: The filename :return: An AssignmentNode """ - return mparser.AssignmentNode(filename, -1, -1, identifier(varname, filename), value) + return mparser.AssignmentNode(filename, -1, -1, identifier(varname, filename), _symbol(filename, '='), value) def block(filename: str) -> mparser.CodeBlockNode: diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py index 0703c1b..4f3b31e 100644 --- a/mesonbuild/cmake/interpreter.py +++ b/mesonbuild/cmake/interpreter.py @@ -48,6 +48,7 @@ from ..mparser import ( IndexNode, MethodNode, NumberNode, + SymbolNode, ) @@ -959,6 +960,9 @@ class CMakeInterpreter: def token(tid: str = 'string', val: TYPE_mixed = '') -> Token: return Token(tid, self.subdir.as_posix(), 0, 0, 0, None, val) + def symbol(val: str) -> SymbolNode: + return SymbolNode(token('', val)) + def string(value: str) -> StringNode: return StringNode(token(val=value), escape=False) @@ -984,14 +988,14 @@ class CMakeInterpreter: raise RuntimeError('invalid type of value: {} ({})'.format(type(value).__name__, str(value))) def indexed(node: BaseNode, index: int) -> IndexNode: - return IndexNode(node, nodeify(index)) + return IndexNode(node, symbol('['), nodeify(index), symbol(']')) def array(elements: TYPE_mixed_list) -> ArrayNode: args = ArgumentNode(token()) if not isinstance(elements, list): elements = [args] args.arguments += [nodeify(x) for x in elements if x is not None] - return ArrayNode(args, 0, 0, 0, 0) + return ArrayNode(symbol('['), args, symbol(']'), 0, 0, 0, 0) def function(name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> FunctionNode: args = [] if args is None else args @@ -1002,7 +1006,7 @@ class CMakeInterpreter: args = [args] args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} - func_n = FunctionNode(self.subdir.as_posix(), 0, 0, 0, 0, id_node(name), args_n) + func_n = FunctionNode(self.subdir.as_posix(), 0, 0, 0, 0, id_node(name), symbol('('), args_n, symbol(')')) return func_n def method(obj: BaseNode, name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> MethodNode: @@ -1014,10 +1018,10 @@ class CMakeInterpreter: args = [args] args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} - return MethodNode(self.subdir.as_posix(), 0, 0, obj, id_node(name), args_n) + return MethodNode(self.subdir.as_posix(), 0, 0, obj, symbol('.'), id_node(name), symbol('('), args_n, symbol(')')) def assign(var_name: str, value: BaseNode) -> AssignmentNode: - return AssignmentNode(self.subdir.as_posix(), 0, 0, id_node(var_name), value) + return AssignmentNode(self.subdir.as_posix(), 0, 0, id_node(var_name), symbol('='), value) # Generate the root code block and the project function call root_cb = CodeBlockNode(token()) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 6b578b5..7f329dd 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -26,6 +26,8 @@ if T.TYPE_CHECKING: from .ast import AstVisitor + BaseNodeT = T.TypeVar('BaseNodeT', bound='BaseNode') + # This is the regex for the supported escape sequences of a regular string # literal, like 'abc\x00' ESCAPE_SEQUENCE_SINGLE_RE = re.compile(r''' @@ -234,12 +236,14 @@ class BaseNode: end_lineno: int = field(hash=False) end_colno: int = field(hash=False) - def __init__(self, lineno: int, colno: int, filename: str, end_lineno: T.Optional[int] = None, end_colno: T.Optional[int] = None) -> None: + def __init__(self, lineno: int, colno: int, filename: str, + end_lineno: T.Optional[int] = None, end_colno: T.Optional[int] = None) -> None: self.lineno = lineno self.colno = colno self.filename = filename self.end_lineno = end_lineno if end_lineno is not None else lineno self.end_colno = end_colno if end_colno is not None else colno + self.whitespaces = None # Attributes for the visitors self.level = 0 @@ -313,17 +317,22 @@ class ContinueNode(ElementaryNode): class BreakNode(ElementaryNode): pass +class SymbolNode(ElementaryNode[str]): + pass + @dataclass(unsafe_hash=True) class ArgumentNode(BaseNode): arguments: T.List[BaseNode] = field(hash=False) - commas: T.List[Token] = field(hash=False) + commas: T.List[SymbolNode] = field(hash=False) + columns: T.List[SymbolNode] = field(hash=False) kwargs: T.Dict[BaseNode, BaseNode] = field(hash=False) def __init__(self, token: Token[TV_TokenTypes]): super().__init__(token.lineno, token.colno, token.filename) self.arguments = [] self.commas = [] + self.columns = [] self.kwargs = {} self.order_error = False @@ -363,20 +372,30 @@ class ArgumentNode(BaseNode): @dataclass(unsafe_hash=True) class ArrayNode(BaseNode): + lbracket: SymbolNode args: ArgumentNode + rbracket: SymbolNode - def __init__(self, args: ArgumentNode, lineno: int, colno: int, end_lineno: int, end_colno: int): + def __init__(self, lbracket: SymbolNode, args: ArgumentNode, rbracket: SymbolNode, + lineno: int, colno: int, end_lineno: int, end_colno: int): super().__init__(lineno, colno, args.filename, end_lineno=end_lineno, end_colno=end_colno) + self.lbracket = lbracket self.args = args + self.rbracket = rbracket @dataclass(unsafe_hash=True) class DictNode(BaseNode): + lcurl: SymbolNode args: ArgumentNode + rcurl: SymbolNode - def __init__(self, args: ArgumentNode, lineno: int, colno: int, end_lineno: int, end_colno: int): + def __init__(self, lcurl: SymbolNode, args: ArgumentNode, rcurl: SymbolNode, + lineno: int, colno: int, end_lineno: int, end_colno: int): super().__init__(lineno, colno, args.filename, end_lineno=end_lineno, end_colno=end_colno) + self.lcurl = lcurl self.args = args + self.rcurl = rcurl class EmptyNode(BaseNode): pass @@ -385,34 +404,40 @@ class EmptyNode(BaseNode): class OrNode(BaseNode): left: BaseNode + operator: SymbolNode right: BaseNode - def __init__(self, left: BaseNode, right: BaseNode): + def __init__(self, left: BaseNode, operator: SymbolNode, right: BaseNode): super().__init__(left.lineno, left.colno, left.filename) self.left = left + self.operator = operator self.right = right @dataclass(unsafe_hash=True) class AndNode(BaseNode): left: BaseNode + operator: SymbolNode right: BaseNode - def __init__(self, left: BaseNode, right: BaseNode): + def __init__(self, left: BaseNode, operator: SymbolNode, right: BaseNode): super().__init__(left.lineno, left.colno, left.filename) self.left = left + self.operator = operator self.right = right @dataclass(unsafe_hash=True) class ComparisonNode(BaseNode): left: BaseNode + operator: SymbolNode right: BaseNode ctype: COMPARISONS - def __init__(self, ctype: COMPARISONS, left: BaseNode, right: BaseNode): + def __init__(self, ctype: COMPARISONS, left: BaseNode, operator: SymbolNode, right: BaseNode): super().__init__(left.lineno, left.colno, left.filename) self.left = left + self.operator = operator self.right = right self.ctype = ctype @@ -423,21 +448,25 @@ class ArithmeticNode(BaseNode): right: BaseNode # TODO: use a Literal for operation operation: str + operator: SymbolNode - def __init__(self, operation: str, left: BaseNode, right: BaseNode): + def __init__(self, operation: str, left: BaseNode, operator: SymbolNode, right: BaseNode): super().__init__(left.lineno, left.colno, left.filename) self.left = left self.right = right self.operation = operation + self.operator = operator @dataclass(unsafe_hash=True) class NotNode(BaseNode): + operator: SymbolNode value: BaseNode - def __init__(self, token: Token[TV_TokenTypes], value: BaseNode): + def __init__(self, token: Token[TV_TokenTypes], operator: SymbolNode, value: BaseNode): super().__init__(token.lineno, token.colno, token.filename) + self.operator = operator self.value = value @dataclass(unsafe_hash=True) @@ -447,53 +476,73 @@ class CodeBlockNode(BaseNode): def __init__(self, token: Token[TV_TokenTypes]): super().__init__(token.lineno, token.colno, token.filename) + self.pre_whitespaces = None self.lines = [] @dataclass(unsafe_hash=True) class IndexNode(BaseNode): iobject: BaseNode + lbracket: SymbolNode index: BaseNode + rbracket: SymbolNode - def __init__(self, iobject: BaseNode, index: BaseNode): + def __init__(self, iobject: BaseNode, lbracket: SymbolNode, index: BaseNode, rbracket: SymbolNode): super().__init__(iobject.lineno, iobject.colno, iobject.filename) self.iobject = iobject + self.lbracket = lbracket self.index = index + self.rbracket = rbracket @dataclass(unsafe_hash=True) class MethodNode(BaseNode): source_object: BaseNode + dot: SymbolNode name: IdNode + lpar: SymbolNode args: ArgumentNode + rpar: SymbolNode - def __init__(self, filename: str, lineno: int, colno: int, source_object: BaseNode, name: IdNode, args: ArgumentNode): + def __init__(self, filename: str, lineno: int, colno: int, + source_object: BaseNode, dot: SymbolNode, name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode): super().__init__(lineno, colno, filename) self.source_object = source_object + self.dot = dot self.name = name + self.lpar = lpar self.args = args + self.rpar = rpar @dataclass(unsafe_hash=True) class FunctionNode(BaseNode): func_name: IdNode + lpar: SymbolNode args: ArgumentNode + rpar: SymbolNode - def __init__(self, filename: str, lineno: int, colno: int, end_lineno: int, end_colno: int, func_name: IdNode, args: ArgumentNode): + def __init__(self, filename: str, lineno: int, colno: int, end_lineno: int, end_colno: int, + func_name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode): super().__init__(lineno, colno, filename, end_lineno=end_lineno, end_colno=end_colno) self.func_name = func_name + self.lpar = lpar self.args = args + self.rpar = rpar @dataclass(unsafe_hash=True) class AssignmentNode(BaseNode): var_name: IdNode + operator: SymbolNode value: BaseNode - def __init__(self, filename: str, lineno: int, colno: int, var_name: IdNode, value: BaseNode): + def __init__(self, filename: str, lineno: int, colno: int, + var_name: IdNode, operator: SymbolNode, value: BaseNode): super().__init__(lineno, colno, filename) self.var_name = var_name + self.operator = operator self.value = value @@ -501,46 +550,61 @@ class AssignmentNode(BaseNode): class PlusAssignmentNode(BaseNode): var_name: IdNode + operator: SymbolNode value: BaseNode - def __init__(self, filename: str, lineno: int, colno: int, var_name: IdNode, value: BaseNode): + def __init__(self, filename: str, lineno: int, colno: int, + var_name: IdNode, operator: SymbolNode, value: BaseNode): super().__init__(lineno, colno, filename) self.var_name = var_name + self.operator = operator self.value = value @dataclass(unsafe_hash=True) class ForeachClauseNode(BaseNode): + foreach_: SymbolNode = field(hash=False) varnames: T.List[IdNode] = field(hash=False) + commas: T.List[SymbolNode] = field(hash=False) + column: SymbolNode = field(hash=False) items: BaseNode block: CodeBlockNode + endforeach: SymbolNode = field(hash=False) - def __init__(self, token: Token, varnames: T.List[IdNode], items: BaseNode, block: CodeBlockNode): - super().__init__(token.lineno, token.colno, token.filename) + def __init__(self, foreach_: SymbolNode, varnames: T.List[IdNode], commas: T.List[SymbolNode], column: SymbolNode, items: BaseNode, block: CodeBlockNode, endforeach: SymbolNode): + super().__init__(foreach_.lineno, foreach_.colno, foreach_.filename) + self.foreach_ = foreach_ self.varnames = varnames + self.commas = commas + self.column = column self.items = items self.block = block + self.endforeach = endforeach @dataclass(unsafe_hash=True) class IfNode(BaseNode): + if_: SymbolNode condition: BaseNode block: CodeBlockNode - def __init__(self, linenode: BaseNode, condition: BaseNode, block: CodeBlockNode): + def __init__(self, linenode: BaseNode, if_node: SymbolNode, condition: BaseNode, block: CodeBlockNode): super().__init__(linenode.lineno, linenode.colno, linenode.filename) + self.if_ = if_node self.condition = condition self.block = block @dataclass(unsafe_hash=True) class ElseNode(BaseNode): + else_: SymbolNode block: CodeBlockNode - def __init__(self, block: CodeBlockNode): + def __init__(self, else_: SymbolNode, block: CodeBlockNode): super().__init__(block.lineno, block.colno, block.filename) + self.else_ = else_ self.block = block @dataclass(unsafe_hash=True) @@ -548,30 +612,38 @@ class IfClauseNode(BaseNode): ifs: T.List[IfNode] = field(hash=False) elseblock: T.Union[EmptyNode, ElseNode] + endif: SymbolNode def __init__(self, linenode: BaseNode): super().__init__(linenode.lineno, linenode.colno, linenode.filename) self.ifs = [] self.elseblock = EmptyNode(linenode.lineno, linenode.colno, linenode.filename) + self.endif = None @dataclass(unsafe_hash=True) class TestCaseClauseNode(BaseNode): + testcase: SymbolNode condition: BaseNode block: CodeBlockNode + endtestcase: SymbolNode - def __init__(self, condition: BaseNode, block: CodeBlockNode): + def __init__(self, testcase: SymbolNode, condition: BaseNode, block: CodeBlockNode, endtestcase: SymbolNode): super().__init__(condition.lineno, condition.colno, condition.filename) + self.testcase = testcase self.condition = condition self.block = block + self.endtestcase = endtestcase @dataclass(unsafe_hash=True) class UMinusNode(BaseNode): + operator: SymbolNode value: BaseNode - def __init__(self, current_location: Token, value: BaseNode): + def __init__(self, current_location: Token, operator: SymbolNode, value: BaseNode): super().__init__(current_location.lineno, current_location.colno, current_location.filename) + self.operator = operator self.value = value @@ -579,23 +651,32 @@ class UMinusNode(BaseNode): class TernaryNode(BaseNode): condition: BaseNode + questionmark: SymbolNode trueblock: BaseNode + column: SymbolNode falseblock: BaseNode - def __init__(self, condition: BaseNode, trueblock: BaseNode, falseblock: BaseNode): + def __init__(self, condition: BaseNode, questionmark: SymbolNode, trueblock: BaseNode, column: SymbolNode, falseblock: BaseNode): super().__init__(condition.lineno, condition.colno, condition.filename) self.condition = condition + self.questionmark = questionmark self.trueblock = trueblock + self.column = column self.falseblock = falseblock + @dataclass(unsafe_hash=True) class ParenthesizedNode(BaseNode): + lpar: SymbolNode = field(hash=False) inner: BaseNode + rpar: SymbolNode = field(hash=False) - def __init__(self, inner: BaseNode, lineno: int, colno: int, end_lineno: int, end_colno: int): + def __init__(self, lpar: SymbolNode, inner: BaseNode, rpar: SymbolNode, lineno: int, colno: int, end_lineno: int, end_colno: int): super().__init__(lineno, colno, inner.filename, end_lineno=end_lineno, end_colno=end_colno) + self.lpar = lpar self.inner = inner + self.rpar = rpar if T.TYPE_CHECKING: COMPARISONS = Literal['==', '!=', '<', '<=', '>=', '>', 'in', 'notin'] @@ -632,13 +713,19 @@ class Parser: self.stream = self.lexer.lex(filename) self.current: Token = Token('eof', '', 0, 0, 0, (0, 0), None) self.previous = self.current + self.getsym() self.in_ternary = False + def create_node(self, node_type: T.Type[BaseNodeT], *args: T.Any, **kwargs: T.Any) -> BaseNodeT: + node = node_type(*args, **kwargs) + return node + def getsym(self) -> None: self.previous = self.current try: self.current = next(self.stream) + except StopIteration: self.current = Token('eof', '', self.current.line_start, self.current.lineno, self.current.colno + self.current.bytespan[1] - self.current.bytespan[0], (0, 0), None) @@ -683,55 +770,69 @@ class Parser: def e1(self) -> BaseNode: left = self.e2() if self.accept('plusassign'): + operator = self.create_node(SymbolNode, self.previous) value = self.e1() if not isinstance(left, IdNode): raise ParseException('Plusassignment target must be an id.', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - return PlusAssignmentNode(left.filename, left.lineno, left.colno, left, value) + return self.create_node(PlusAssignmentNode, left.filename, left.lineno, left.colno, left, operator, value) elif self.accept('assign'): + operator = self.create_node(SymbolNode, self.previous) value = self.e1() if not isinstance(left, IdNode): raise ParseException('Assignment target must be an id.', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - return AssignmentNode(left.filename, left.lineno, left.colno, left, value) + return self.create_node(AssignmentNode, left.filename, left.lineno, left.colno, left, operator, value) elif self.accept('questionmark'): if self.in_ternary: raise ParseException('Nested ternary operators are not allowed.', self.getline(), left.lineno, left.colno) + + qm_node = self.create_node(SymbolNode, self.previous) self.in_ternary = True trueblock = self.e1() self.expect('colon') + column_node = self.create_node(SymbolNode, self.previous) falseblock = self.e1() self.in_ternary = False - return TernaryNode(left, trueblock, falseblock) + return self.create_node(TernaryNode, left, qm_node, trueblock, column_node, falseblock) return left def e2(self) -> BaseNode: left = self.e3() while self.accept('or'): + operator = self.create_node(SymbolNode, self.previous) if isinstance(left, EmptyNode): raise ParseException('Invalid or clause.', self.getline(), left.lineno, left.colno) - left = OrNode(left, self.e3()) + left = self.create_node(OrNode, left, operator, self.e3()) return left def e3(self) -> BaseNode: left = self.e4() while self.accept('and'): + operator = self.create_node(SymbolNode, self.previous) if isinstance(left, EmptyNode): raise ParseException('Invalid and clause.', self.getline(), left.lineno, left.colno) - left = AndNode(left, self.e4()) + left = self.create_node(AndNode, left, operator, self.e4()) return left def e4(self) -> BaseNode: left = self.e5() for nodename, operator_type in comparison_map.items(): if self.accept(nodename): - return ComparisonNode(operator_type, left, self.e5()) - if self.accept('not') and self.accept('in'): - return ComparisonNode('notin', left, self.e5()) + operator = self.create_node(SymbolNode, self.previous) + return self.create_node(ComparisonNode, operator_type, left, operator, self.e5()) + if self.accept('not'): + not_token = self.previous + if self.accept('in'): + in_token = self.previous + not_token.bytespan = (not_token.bytespan[0], in_token.bytespan[1]) + not_token.value += in_token.value + operator = self.create_node(SymbolNode, not_token) + return self.create_node(ComparisonNode, 'notin', left, operator, self.e5()) return left def e5(self) -> BaseNode: @@ -746,7 +847,8 @@ class Parser: while True: op = self.accept_any(tuple(op_map.keys())) if op: - left = ArithmeticNode(op_map[op], left, self.e5muldiv()) + operator = self.create_node(SymbolNode, self.previous) + left = self.create_node(ArithmeticNode, op_map[op], left, operator, self.e5muldiv()) else: break return left @@ -761,29 +863,34 @@ class Parser: while True: op = self.accept_any(tuple(op_map.keys())) if op: - left = ArithmeticNode(op_map[op], left, self.e6()) + operator = self.create_node(SymbolNode, self.previous) + left = self.create_node(ArithmeticNode, op_map[op], left, operator, self.e6()) else: break return left def e6(self) -> BaseNode: if self.accept('not'): - return NotNode(self.current, self.e7()) + operator = self.create_node(SymbolNode, self.previous) + return self.create_node(NotNode, self.current, operator, self.e7()) if self.accept('dash'): - return UMinusNode(self.current, self.e7()) + operator = self.create_node(SymbolNode, self.previous) + return self.create_node(UMinusNode, self.current, operator, self.e7()) return self.e7() def e7(self) -> BaseNode: left = self.e8() block_start = self.current if self.accept('lparen'): + lpar = self.create_node(SymbolNode, block_start) args = self.args() self.block_expect('rparen', block_start) + rpar = self.create_node(SymbolNode, self.previous) if not isinstance(left, IdNode): raise ParseException('Function call must be applied to plain id', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - left = FunctionNode(left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left, args) + left = self.create_node(FunctionNode, left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left, lpar, args, rpar) go_again = True while go_again: go_again = False @@ -798,17 +905,23 @@ class Parser: def e8(self) -> BaseNode: block_start = self.current if self.accept('lparen'): + lpar = self.create_node(SymbolNode, block_start) e = self.statement() self.block_expect('rparen', block_start) - return ParenthesizedNode(e, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + rpar = self.create_node(SymbolNode, self.previous) + return ParenthesizedNode(lpar, e, rpar, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) elif self.accept('lbracket'): + lbracket = self.create_node(SymbolNode, block_start) args = self.args() self.block_expect('rbracket', block_start) - return ArrayNode(args, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + rbracket = self.create_node(SymbolNode, self.previous) + return self.create_node(ArrayNode, lbracket, args, rbracket, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) elif self.accept('lcurl'): + lcurl = self.create_node(SymbolNode, block_start) key_values = self.key_values() self.block_expect('rcurl', block_start) - return DictNode(key_values, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + rcurl = self.create_node(SymbolNode, self.previous) + return self.create_node(DictNode, lcurl, key_values, rcurl, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) else: return self.e9() @@ -816,34 +929,35 @@ class Parser: t = self.current if self.accept('true'): t.value = True - return BooleanNode(t) + return self.create_node(BooleanNode, t) if self.accept('false'): t.value = False - return BooleanNode(t) + return self.create_node(BooleanNode, t) if self.accept('id'): - return IdNode(t) + return self.create_node(IdNode, t) if self.accept('number'): - return NumberNode(t) + return self.create_node(NumberNode, t) if self.accept('string'): - return StringNode(t) + return self.create_node(StringNode, t) if self.accept('fstring'): - return FormatStringNode(t) + return self.create_node(FormatStringNode, t) if self.accept('multiline_string'): - return MultilineStringNode(t) + return self.create_node(MultilineStringNode, t) if self.accept('multiline_fstring'): - return MultilineFormatStringNode(t) + return self.create_node(MultilineFormatStringNode, t) return EmptyNode(self.current.lineno, self.current.colno, self.current.filename) def key_values(self) -> ArgumentNode: s = self.statement() - a = ArgumentNode(self.current) + a = self.create_node(ArgumentNode, self.current) while not isinstance(s, EmptyNode): if self.accept('colon'): + a.columns.append(self.create_node(SymbolNode, self.previous)) a.set_kwarg_no_check(s, self.statement()) if not self.accept('comma'): return a - a.commas.append(self.previous) + a.commas.append(self.create_node(SymbolNode, self.previous)) else: raise ParseException('Only key:value pairs are valid in dict construction.', self.getline(), s.lineno, s.colno) @@ -852,20 +966,21 @@ class Parser: def args(self) -> ArgumentNode: s = self.statement() - a = ArgumentNode(self.current) + a = self.create_node(ArgumentNode, self.current) while not isinstance(s, EmptyNode): if self.accept('comma'): - a.commas.append(self.previous) + a.commas.append(self.create_node(SymbolNode, self.previous)) a.append(s) elif self.accept('colon'): + a.columns.append(self.create_node(SymbolNode, self.previous)) if not isinstance(s, IdNode): raise ParseException('Dictionary key must be a plain identifier.', self.getline(), s.lineno, s.colno) a.set_kwarg(s, self.statement()) if not self.accept('comma'): return a - a.commas.append(self.previous) + a.commas.append(self.create_node(SymbolNode, self.previous)) else: a.append(s) return a @@ -873,6 +988,7 @@ class Parser: return a def method_call(self, source_object: BaseNode) -> MethodNode: + dot = self.create_node(SymbolNode, self.previous) methodname = self.e9() if not isinstance(methodname, IdNode): if isinstance(source_object, NumberNode) and isinstance(methodname, NumberNode): @@ -882,63 +998,78 @@ class Parser: self.getline(), self.current.lineno, self.current.colno) assert isinstance(methodname.value, str) self.expect('lparen') + lpar = self.create_node(SymbolNode, self.previous) args = self.args() + rpar = self.create_node(SymbolNode, self.current) self.expect('rparen') - method = MethodNode(methodname.filename, methodname.lineno, methodname.colno, source_object, methodname, args) + method = self.create_node(MethodNode, methodname.filename, methodname.lineno, methodname.colno, + source_object, dot, methodname, lpar, args, rpar) if self.accept('dot'): return self.method_call(method) return method def index_call(self, source_object: BaseNode) -> IndexNode: + lbracket = self.create_node(SymbolNode, self.previous) index_statement = self.statement() self.expect('rbracket') - return IndexNode(source_object, index_statement) + rbracket = self.create_node(SymbolNode, self.previous) + return self.create_node(IndexNode, source_object, lbracket, index_statement, rbracket) def foreachblock(self) -> ForeachClauseNode: + foreach_ = self.create_node(SymbolNode, self.previous) self.expect('id') assert isinstance(self.previous.value, str) - varname = self.previous - varnames = [IdNode(self.previous)] + varnames = [self.create_node(IdNode, self.previous)] + commas = [] if self.accept('comma'): + commas.append(self.create_node(SymbolNode, self.previous)) self.expect('id') assert isinstance(self.previous.value, str) - varnames.append(IdNode(self.previous)) + varnames.append(self.create_node(IdNode, self.previous)) self.expect('colon') + column = self.create_node(SymbolNode, self.previous) items = self.statement() block = self.codeblock() - return ForeachClauseNode(varname, varnames, items, block) + endforeach = self.create_node(SymbolNode, self.current) + return self.create_node(ForeachClauseNode, foreach_, varnames, commas, column, items, block, endforeach) def ifblock(self) -> IfClauseNode: + if_node = self.create_node(SymbolNode, self.previous) condition = self.statement() - clause = IfClauseNode(condition) + clause = self.create_node(IfClauseNode, condition) self.expect('eol') block = self.codeblock() - clause.ifs.append(IfNode(clause, condition, block)) + clause.ifs.append(self.create_node(IfNode, clause, if_node, condition, block)) self.elseifblock(clause) clause.elseblock = self.elseblock() + clause.endif = self.create_node(SymbolNode, self.current) return clause def elseifblock(self, clause: IfClauseNode) -> None: while self.accept('elif'): + elif_ = self.create_node(SymbolNode, self.previous) s = self.statement() self.expect('eol') b = self.codeblock() - clause.ifs.append(IfNode(s, s, b)) + clause.ifs.append(self.create_node(IfNode, s, elif_, s, b)) def elseblock(self) -> T.Union[ElseNode, EmptyNode]: if self.accept('else'): + else_ = self.create_node(SymbolNode, self.previous) self.expect('eol') block = self.codeblock() - return ElseNode(block) + return ElseNode(else_, block) return EmptyNode(self.current.lineno, self.current.colno, self.current.filename) def testcaseblock(self) -> TestCaseClauseNode: + testcase = self.create_node(SymbolNode, self.previous) condition = self.statement() self.expect('eol') block = self.codeblock() - return TestCaseClauseNode(condition, block) + endtestcase = SymbolNode(self.current) + return self.create_node(TestCaseClauseNode, testcase, condition, block, endtestcase) def line(self) -> BaseNode: block_start = self.current @@ -953,9 +1084,9 @@ class Parser: self.block_expect('endforeach', block_start) return forblock if self.accept('continue'): - return ContinueNode(self.current) + return self.create_node(ContinueNode, self.current) if self.accept('break'): - return BreakNode(self.current) + return self.create_node(BreakNode, self.current) if self.lexer.in_unit_test and self.accept('testcase'): block = self.testcaseblock() self.block_expect('endtestcase', block_start) @@ -963,15 +1094,20 @@ class Parser: return self.statement() def codeblock(self) -> CodeBlockNode: - block = CodeBlockNode(self.current) + block = self.create_node(CodeBlockNode, self.current) cond = True + try: while cond: curline = self.line() + if not isinstance(curline, EmptyNode): block.lines.append(curline) + cond = self.accept('eol') + except ParseException as e: e.ast = block raise + return block diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index af3210c..64eb2f2 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -28,7 +28,7 @@ from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionL from mesonbuild.mesonlib import MesonException, setup_vsenv from . import mlog, environment from functools import wraps -from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseStringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode +from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseStringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode, SymbolNode import json, os, re, sys import typing as T @@ -104,6 +104,9 @@ class RequiredKeys: return wrapped +def _symbol(val: str) -> SymbolNode: + return SymbolNode(Token('', '', 0, 0, 0, (0, 0), val)) + class MTypeBase: def __init__(self, node: T.Optional[BaseNode] = None): if node is None: @@ -189,7 +192,7 @@ class MTypeList(MTypeBase): super().__init__(node) def _new_node(self): - return ArrayNode(ArgumentNode(Token('', '', 0, 0, 0, None, '')), 0, 0, 0, 0) + return ArrayNode(_symbol('['), ArgumentNode(Token('', '', 0, 0, 0, None, '')), _symbol(']'), 0, 0, 0, 0) def _new_element_node(self, value): # Overwrite in derived class @@ -728,7 +731,7 @@ class Rewriter: node = tgt_function.args.kwargs[extra_files_key] except StopIteration: # Target has no extra_files kwarg, create one - node = ArrayNode(ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), tgt_function.end_lineno, tgt_function.end_colno, tgt_function.end_lineno, tgt_function.end_colno) + node = ArrayNode(_symbol('['), ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), _symbol(']'), tgt_function.end_lineno, tgt_function.end_colno, tgt_function.end_lineno, tgt_function.end_colno) tgt_function.args.kwargs[IdNode(Token('string', tgt_function.filename, 0, 0, 0, None, 'extra_files'))] = node mark_array = False if tgt_function not in self.modified_nodes: @@ -812,17 +815,17 @@ class Rewriter: # Build src list src_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - src_arr_node = ArrayNode(src_arg_node, 0, 0, 0, 0) + src_arr_node = ArrayNode(_symbol('['), src_arg_node, _symbol(']'), 0, 0, 0, 0) src_far_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - src_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), src_far_node) - src_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), src_fun_node) + src_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), _symbol('('), src_far_node, _symbol(')')) + src_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), _symbol('='), src_fun_node) src_arg_node.arguments = [StringNode(Token('string', filename, 0, 0, 0, None, x)) for x in cmd['sources']] src_far_node.arguments = [src_arr_node] # Build target tgt_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), tgt_arg_node) - tgt_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), tgt_fun_node) + tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), _symbol('('), tgt_arg_node, _symbol(')')) + tgt_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), _symbol('='), tgt_fun_node) tgt_arg_node.arguments = [ StringNode(Token('string', filename, 0, 0, 0, None, cmd['target'])), IdNode(Token('string', filename, 0, 0, 0, None, source_id)) -- cgit v1.1 From 0f4891cdf46e28d900cc35d57999fecff3ba8598 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 09:58:32 -0400 Subject: parser: simplify Assignment and PlusAssignment nodes --- mesonbuild/cargo/builder.py | 2 +- mesonbuild/cmake/interpreter.py | 2 +- mesonbuild/interpreterbase/interpreterbase.py | 4 ++-- mesonbuild/mparser.py | 27 ++++++--------------------- mesonbuild/rewriter.py | 4 ++-- 5 files changed, 12 insertions(+), 27 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/cargo/builder.py b/mesonbuild/cargo/builder.py index 9c650b9..76d2e8b 100644 --- a/mesonbuild/cargo/builder.py +++ b/mesonbuild/cargo/builder.py @@ -172,7 +172,7 @@ def assign(value: mparser.BaseNode, varname: str, filename: str) -> mparser.Assi :param filename: The filename :return: An AssignmentNode """ - return mparser.AssignmentNode(filename, -1, -1, identifier(varname, filename), _symbol(filename, '='), value) + return mparser.AssignmentNode(identifier(varname, filename), _symbol(filename, '='), value) def block(filename: str) -> mparser.CodeBlockNode: diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py index 4f3b31e..88e5183 100644 --- a/mesonbuild/cmake/interpreter.py +++ b/mesonbuild/cmake/interpreter.py @@ -1021,7 +1021,7 @@ class CMakeInterpreter: return MethodNode(self.subdir.as_posix(), 0, 0, obj, symbol('.'), id_node(name), symbol('('), args_n, symbol(')')) def assign(var_name: str, value: BaseNode) -> AssignmentNode: - return AssignmentNode(self.subdir.as_posix(), 0, 0, id_node(var_name), symbol('='), value) + return AssignmentNode(id_node(var_name), symbol('='), value) # Generate the root code block and the project function call root_cb = CodeBlockNode(token()) diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index 5b07dc0..ea6e37c 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -192,6 +192,8 @@ class InterpreterBase: self.current_node = cur if isinstance(cur, mparser.FunctionNode): return self.function_call(cur) + elif isinstance(cur, mparser.PlusAssignmentNode): + self.evaluate_plusassign(cur) elif isinstance(cur, mparser.AssignmentNode): self.assignment(cur) elif isinstance(cur, mparser.MethodNode): @@ -229,8 +231,6 @@ class InterpreterBase: return self.evaluate_arithmeticstatement(cur) elif isinstance(cur, mparser.ForeachClauseNode): self.evaluate_foreach(cur) - elif isinstance(cur, mparser.PlusAssignmentNode): - self.evaluate_plusassign(cur) elif isinstance(cur, mparser.IndexNode): return self.evaluate_indexing(cur) elif isinstance(cur, mparser.TernaryNode): diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 7f329dd..ffe7750 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -530,7 +530,6 @@ class FunctionNode(BaseNode): self.args = args self.rpar = rpar - @dataclass(unsafe_hash=True) class AssignmentNode(BaseNode): @@ -538,28 +537,14 @@ class AssignmentNode(BaseNode): operator: SymbolNode value: BaseNode - def __init__(self, filename: str, lineno: int, colno: int, - var_name: IdNode, operator: SymbolNode, value: BaseNode): - super().__init__(lineno, colno, filename) - self.var_name = var_name - self.operator = operator - self.value = value - - -@dataclass(unsafe_hash=True) -class PlusAssignmentNode(BaseNode): - - var_name: IdNode - operator: SymbolNode - value: BaseNode - - def __init__(self, filename: str, lineno: int, colno: int, - var_name: IdNode, operator: SymbolNode, value: BaseNode): - super().__init__(lineno, colno, filename) + def __init__(self, var_name: IdNode, operator: SymbolNode, value: BaseNode): + super().__init__(var_name.lineno, var_name.colno, var_name.filename) self.var_name = var_name self.operator = operator self.value = value +class PlusAssignmentNode(AssignmentNode): + pass @dataclass(unsafe_hash=True) class ForeachClauseNode(BaseNode): @@ -775,7 +760,7 @@ class Parser: if not isinstance(left, IdNode): raise ParseException('Plusassignment target must be an id.', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - return self.create_node(PlusAssignmentNode, left.filename, left.lineno, left.colno, left, operator, value) + return self.create_node(PlusAssignmentNode, left, operator, value) elif self.accept('assign'): operator = self.create_node(SymbolNode, self.previous) value = self.e1() @@ -783,7 +768,7 @@ class Parser: raise ParseException('Assignment target must be an id.', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - return self.create_node(AssignmentNode, left.filename, left.lineno, left.colno, left, operator, value) + return self.create_node(AssignmentNode, left, operator, value) elif self.accept('questionmark'): if self.in_ternary: raise ParseException('Nested ternary operators are not allowed.', diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 64eb2f2..7527dab 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -818,14 +818,14 @@ class Rewriter: src_arr_node = ArrayNode(_symbol('['), src_arg_node, _symbol(']'), 0, 0, 0, 0) src_far_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) src_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), _symbol('('), src_far_node, _symbol(')')) - src_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), _symbol('='), src_fun_node) + src_ass_node = AssignmentNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), _symbol('='), src_fun_node) src_arg_node.arguments = [StringNode(Token('string', filename, 0, 0, 0, None, x)) for x in cmd['sources']] src_far_node.arguments = [src_arr_node] # Build target tgt_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), _symbol('('), tgt_arg_node, _symbol(')')) - tgt_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), _symbol('='), tgt_fun_node) + tgt_ass_node = AssignmentNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), _symbol('='), tgt_fun_node) tgt_arg_node.arguments = [ StringNode(Token('string', filename, 0, 0, 0, None, cmd['target'])), IdNode(Token('string', filename, 0, 0, 0, None, source_id)) -- cgit v1.1 From 13ddf8bd025c3ff6257049cab6f3df1072ac0daf Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 09:41:47 -0400 Subject: parser: simplify by using Unary and Binary Operator Node --- mesonbuild/mparser.py | 59 +++++++++++++++------------------------------------ 1 file changed, 17 insertions(+), 42 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index ffe7750..00b8731 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -401,7 +401,7 @@ class EmptyNode(BaseNode): pass @dataclass(unsafe_hash=True) -class OrNode(BaseNode): +class BinaryOperatorNode(BaseNode): left: BaseNode operator: SymbolNode @@ -413,53 +413,33 @@ class OrNode(BaseNode): self.operator = operator self.right = right -@dataclass(unsafe_hash=True) -class AndNode(BaseNode): - - left: BaseNode - operator: SymbolNode - right: BaseNode +class OrNode(BinaryOperatorNode): + pass - def __init__(self, left: BaseNode, operator: SymbolNode, right: BaseNode): - super().__init__(left.lineno, left.colno, left.filename) - self.left = left - self.operator = operator - self.right = right +class AndNode(BinaryOperatorNode): + pass @dataclass(unsafe_hash=True) -class ComparisonNode(BaseNode): +class ComparisonNode(BinaryOperatorNode): - left: BaseNode - operator: SymbolNode - right: BaseNode ctype: COMPARISONS def __init__(self, ctype: COMPARISONS, left: BaseNode, operator: SymbolNode, right: BaseNode): - super().__init__(left.lineno, left.colno, left.filename) - self.left = left - self.operator = operator - self.right = right + super().__init__(left, operator, right) self.ctype = ctype @dataclass(unsafe_hash=True) -class ArithmeticNode(BaseNode): +class ArithmeticNode(BinaryOperatorNode): - left: BaseNode - right: BaseNode # TODO: use a Literal for operation operation: str - operator: SymbolNode def __init__(self, operation: str, left: BaseNode, operator: SymbolNode, right: BaseNode): - super().__init__(left.lineno, left.colno, left.filename) - self.left = left - self.right = right + super().__init__(left, operator, right) self.operation = operation - self.operator = operator - @dataclass(unsafe_hash=True) -class NotNode(BaseNode): +class UnaryOperatorNode(BaseNode): operator: SymbolNode value: BaseNode @@ -469,6 +449,12 @@ class NotNode(BaseNode): self.operator = operator self.value = value +class NotNode(UnaryOperatorNode): + pass + +class UMinusNode(UnaryOperatorNode): + pass + @dataclass(unsafe_hash=True) class CodeBlockNode(BaseNode): @@ -621,18 +607,6 @@ class TestCaseClauseNode(BaseNode): self.endtestcase = endtestcase @dataclass(unsafe_hash=True) -class UMinusNode(BaseNode): - - operator: SymbolNode - value: BaseNode - - def __init__(self, current_location: Token, operator: SymbolNode, value: BaseNode): - super().__init__(current_location.lineno, current_location.colno, current_location.filename) - self.operator = operator - self.value = value - - -@dataclass(unsafe_hash=True) class TernaryNode(BaseNode): condition: BaseNode @@ -663,6 +637,7 @@ class ParenthesizedNode(BaseNode): self.inner = inner self.rpar = rpar + if T.TYPE_CHECKING: COMPARISONS = Literal['==', '!=', '<', '<=', '>=', '>', 'in', 'notin'] -- cgit v1.1 From 5b29eff8ad02348ff0495cd307b29259567e11df Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 10:21:53 -0400 Subject: parser: simplify other node constructors --- mesonbuild/cargo/builder.py | 6 +++--- mesonbuild/cmake/interpreter.py | 6 +++--- mesonbuild/mparser.py | 35 +++++++++++++++-------------------- mesonbuild/rewriter.py | 10 +++++----- 4 files changed, 26 insertions(+), 31 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/cargo/builder.py b/mesonbuild/cargo/builder.py index 76d2e8b..3f7f688 100644 --- a/mesonbuild/cargo/builder.py +++ b/mesonbuild/cargo/builder.py @@ -71,7 +71,7 @@ def array(value: T.List[mparser.BaseNode], filename: str) -> mparser.ArrayNode: """ args = mparser.ArgumentNode(_token('array', filename, 'unused')) args.arguments = value - return mparser.ArrayNode(_symbol(filename, '['), args, _symbol(filename, ']'), -1, -1, -1, -1) + return mparser.ArrayNode(_symbol(filename, '['), args, _symbol(filename, ']')) def identifier(value: str, filename: str) -> mparser.IdNode: @@ -101,7 +101,7 @@ def method(name: str, id_: mparser.IdNode, args.arguments = pos if kw is not None: args.kwargs = {identifier(k, id_.filename): v for k, v in kw.items()} - return mparser.MethodNode(id_.filename, -1, -1, id_, _symbol(id_.filename, '.'), identifier(name, id_.filename), _symbol(id_.filename, '('), args, _symbol(id_.filename, ')')) + return mparser.MethodNode(id_, _symbol(id_.filename, '.'), identifier(name, id_.filename), _symbol(id_.filename, '('), args, _symbol(id_.filename, ')')) def function(name: str, filename: str, @@ -121,7 +121,7 @@ def function(name: str, filename: str, args.arguments = pos if kw is not None: args.kwargs = {identifier(k, filename): v for k, v in kw.items()} - return mparser.FunctionNode(filename, -1, -1, -1, -1, identifier(name, filename), _symbol(filename, '('), args, _symbol(filename, ')')) + return mparser.FunctionNode(identifier(name, filename), _symbol(filename, '('), args, _symbol(filename, ')')) def equal(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.ComparisonNode: diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py index 88e5183..2f7cb69 100644 --- a/mesonbuild/cmake/interpreter.py +++ b/mesonbuild/cmake/interpreter.py @@ -995,7 +995,7 @@ class CMakeInterpreter: if not isinstance(elements, list): elements = [args] args.arguments += [nodeify(x) for x in elements if x is not None] - return ArrayNode(symbol('['), args, symbol(']'), 0, 0, 0, 0) + return ArrayNode(symbol('['), args, symbol(']')) def function(name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> FunctionNode: args = [] if args is None else args @@ -1006,7 +1006,7 @@ class CMakeInterpreter: args = [args] args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} - func_n = FunctionNode(self.subdir.as_posix(), 0, 0, 0, 0, id_node(name), symbol('('), args_n, symbol(')')) + func_n = FunctionNode(id_node(name), symbol('('), args_n, symbol(')')) return func_n def method(obj: BaseNode, name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> MethodNode: @@ -1018,7 +1018,7 @@ class CMakeInterpreter: args = [args] args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} - return MethodNode(self.subdir.as_posix(), 0, 0, obj, symbol('.'), id_node(name), symbol('('), args_n, symbol(')')) + return MethodNode(obj, symbol('.'), id_node(name), symbol('('), args_n, symbol(')')) def assign(var_name: str, value: BaseNode) -> AssignmentNode: return AssignmentNode(id_node(var_name), symbol('='), value) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 00b8731..75a12aa 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -376,9 +376,8 @@ class ArrayNode(BaseNode): args: ArgumentNode rbracket: SymbolNode - def __init__(self, lbracket: SymbolNode, args: ArgumentNode, rbracket: SymbolNode, - lineno: int, colno: int, end_lineno: int, end_colno: int): - super().__init__(lineno, colno, args.filename, end_lineno=end_lineno, end_colno=end_colno) + def __init__(self, lbracket: SymbolNode, args: ArgumentNode, rbracket: SymbolNode): + super().__init__(lbracket.lineno, lbracket.colno, args.filename, end_lineno=rbracket.lineno, end_colno=rbracket.colno+1) self.lbracket = lbracket self.args = args self.rbracket = rbracket @@ -390,9 +389,8 @@ class DictNode(BaseNode): args: ArgumentNode rcurl: SymbolNode - def __init__(self, lcurl: SymbolNode, args: ArgumentNode, rcurl: SymbolNode, - lineno: int, colno: int, end_lineno: int, end_colno: int): - super().__init__(lineno, colno, args.filename, end_lineno=end_lineno, end_colno=end_colno) + def __init__(self, lcurl: SymbolNode, args: ArgumentNode, rcurl: SymbolNode): + super().__init__(lcurl.lineno, lcurl.colno, args.filename, end_lineno=rcurl.lineno, end_colno=rcurl.colno+1) self.lcurl = lcurl self.args = args self.rcurl = rcurl @@ -490,9 +488,8 @@ class MethodNode(BaseNode): args: ArgumentNode rpar: SymbolNode - def __init__(self, filename: str, lineno: int, colno: int, - source_object: BaseNode, dot: SymbolNode, name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode): - super().__init__(lineno, colno, filename) + def __init__(self, source_object: BaseNode, dot: SymbolNode, name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode): + super().__init__(name.lineno, name.colno, name.filename, end_lineno=rpar.lineno, end_colno=rpar.colno+1) self.source_object = source_object self.dot = dot self.name = name @@ -508,9 +505,8 @@ class FunctionNode(BaseNode): args: ArgumentNode rpar: SymbolNode - def __init__(self, filename: str, lineno: int, colno: int, end_lineno: int, end_colno: int, - func_name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode): - super().__init__(lineno, colno, filename, end_lineno=end_lineno, end_colno=end_colno) + def __init__(self, func_name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode): + super().__init__(func_name.lineno, func_name.colno, func_name.filename, end_lineno=rpar.end_lineno, end_colno=rpar.end_colno+1) self.func_name = func_name self.lpar = lpar self.args = args @@ -631,8 +627,8 @@ class ParenthesizedNode(BaseNode): inner: BaseNode rpar: SymbolNode = field(hash=False) - def __init__(self, lpar: SymbolNode, inner: BaseNode, rpar: SymbolNode, lineno: int, colno: int, end_lineno: int, end_colno: int): - super().__init__(lineno, colno, inner.filename, end_lineno=end_lineno, end_colno=end_colno) + def __init__(self, lpar: SymbolNode, inner: BaseNode, rpar: SymbolNode): + super().__init__(lpar.lineno, lpar.colno, inner.filename, end_lineno=rpar.lineno, end_colno=rpar.colno+1) self.lpar = lpar self.inner = inner self.rpar = rpar @@ -850,7 +846,7 @@ class Parser: raise ParseException('Function call must be applied to plain id', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - left = self.create_node(FunctionNode, left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left, lpar, args, rpar) + left = self.create_node(FunctionNode, left, lpar, args, rpar) go_again = True while go_again: go_again = False @@ -869,19 +865,19 @@ class Parser: e = self.statement() self.block_expect('rparen', block_start) rpar = self.create_node(SymbolNode, self.previous) - return ParenthesizedNode(lpar, e, rpar, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + return ParenthesizedNode(lpar, e, rpar) elif self.accept('lbracket'): lbracket = self.create_node(SymbolNode, block_start) args = self.args() self.block_expect('rbracket', block_start) rbracket = self.create_node(SymbolNode, self.previous) - return self.create_node(ArrayNode, lbracket, args, rbracket, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + return self.create_node(ArrayNode, lbracket, args, rbracket) elif self.accept('lcurl'): lcurl = self.create_node(SymbolNode, block_start) key_values = self.key_values() self.block_expect('rcurl', block_start) rcurl = self.create_node(SymbolNode, self.previous) - return self.create_node(DictNode, lcurl, key_values, rcurl, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + return self.create_node(DictNode, lcurl, key_values, rcurl) else: return self.e9() @@ -962,8 +958,7 @@ class Parser: args = self.args() rpar = self.create_node(SymbolNode, self.current) self.expect('rparen') - method = self.create_node(MethodNode, methodname.filename, methodname.lineno, methodname.colno, - source_object, dot, methodname, lpar, args, rpar) + method = self.create_node(MethodNode, source_object, dot, methodname, lpar, args, rpar) if self.accept('dot'): return self.method_call(method) return method diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 7527dab..a86b97e 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -192,7 +192,7 @@ class MTypeList(MTypeBase): super().__init__(node) def _new_node(self): - return ArrayNode(_symbol('['), ArgumentNode(Token('', '', 0, 0, 0, None, '')), _symbol(']'), 0, 0, 0, 0) + return ArrayNode(_symbol('['), ArgumentNode(Token('', '', 0, 0, 0, None, '')), _symbol(']')) def _new_element_node(self, value): # Overwrite in derived class @@ -731,7 +731,7 @@ class Rewriter: node = tgt_function.args.kwargs[extra_files_key] except StopIteration: # Target has no extra_files kwarg, create one - node = ArrayNode(_symbol('['), ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), _symbol(']'), tgt_function.end_lineno, tgt_function.end_colno, tgt_function.end_lineno, tgt_function.end_colno) + node = ArrayNode(_symbol('['), ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), _symbol(']')) tgt_function.args.kwargs[IdNode(Token('string', tgt_function.filename, 0, 0, 0, None, 'extra_files'))] = node mark_array = False if tgt_function not in self.modified_nodes: @@ -815,16 +815,16 @@ class Rewriter: # Build src list src_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - src_arr_node = ArrayNode(_symbol('['), src_arg_node, _symbol(']'), 0, 0, 0, 0) + src_arr_node = ArrayNode(_symbol('['), src_arg_node, _symbol(']')) src_far_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - src_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), _symbol('('), src_far_node, _symbol(')')) + src_fun_node = FunctionNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), _symbol('('), src_far_node, _symbol(')')) src_ass_node = AssignmentNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), _symbol('='), src_fun_node) src_arg_node.arguments = [StringNode(Token('string', filename, 0, 0, 0, None, x)) for x in cmd['sources']] src_far_node.arguments = [src_arr_node] # Build target tgt_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), _symbol('('), tgt_arg_node, _symbol(')')) + tgt_fun_node = FunctionNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), _symbol('('), tgt_arg_node, _symbol(')')) tgt_ass_node = AssignmentNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), _symbol('='), tgt_fun_node) tgt_arg_node.arguments = [ StringNode(Token('string', filename, 0, 0, 0, None, cmd['target'])), -- cgit v1.1 From 11ef2a536c6c9fec0b048da74b36a0231ef2199a Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 10:08:24 -0400 Subject: parser: preserve whitespaces and comments --- mesonbuild/mparser.py | 67 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 7 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 75a12aa..a161842 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -116,7 +116,7 @@ class Lexer: self.keywords.update({'testcase', 'endtestcase'}) self.token_specification = [ # Need to be sorted longest to shortest. - ('ignore', re.compile(r'[ \t]')), + ('whitespace', re.compile(r'[ \t]+')), ('multiline_fstring', re.compile(r"f'''(.|\n)*?'''", re.M)), ('fstring', re.compile(r"f'([^'\\]|(\\.))*'")), ('id', re.compile('[_a-zA-Z][_0-9a-zA-Z]*')), @@ -178,9 +178,7 @@ class Lexer: span_end = loc bytespan = (span_start, span_end) value = mo.group() - if tid in {'ignore', 'comment'}: - break - elif tid == 'lparen': + if tid == 'lparen': par_count += 1 elif tid == 'rparen': par_count -= 1 @@ -210,12 +208,12 @@ class Lexer: elif tid == 'eol_cont': lineno += 1 line_start = loc - break + tid = 'whitespace' elif tid == 'eol': lineno += 1 line_start = loc if par_count > 0 or bracket_count > 0 or curl_count > 0: - break + tid = 'whitespace' elif tid == 'id': if value in self.keywords: tid = value @@ -235,6 +233,7 @@ class BaseNode: filename: str = field(hash=False) end_lineno: int = field(hash=False) end_colno: int = field(hash=False) + whitespaces: T.Optional[WhitespaceNode] = field(hash=False) def __init__(self, lineno: int, colno: int, filename: str, end_lineno: T.Optional[int] = None, end_colno: T.Optional[int] = None) -> None: @@ -257,6 +256,26 @@ class BaseNode: if callable(func): func(self) + def append_whitespaces(self, token: Token) -> None: + if self.whitespaces is None: + self.whitespaces = WhitespaceNode(token) + else: + self.whitespaces.append(token) + + +@dataclass(unsafe_hash=True) +class WhitespaceNode(BaseNode): + + value: str + + def __init__(self, token: Token[str]): + super().__init__(token.lineno, token.colno, token.filename) + self.value = '' + self.append(token) + + def append(self, token: Token[str]) -> None: + self.value += token.value + @dataclass(unsafe_hash=True) class ElementaryNode(T.Generic[TV_TokenTypes], BaseNode): @@ -456,6 +475,7 @@ class UMinusNode(UnaryOperatorNode): @dataclass(unsafe_hash=True) class CodeBlockNode(BaseNode): + pre_whitespaces: T.Optional[WhitespaceNode] = field(hash=False) lines: T.List[BaseNode] = field(hash=False) def __init__(self, token: Token[TV_TokenTypes]): @@ -463,6 +483,14 @@ class CodeBlockNode(BaseNode): self.pre_whitespaces = None self.lines = [] + def append_whitespaces(self, token: Token) -> None: + if self.lines: + self.lines[-1].append_whitespaces(token) + elif self.pre_whitespaces is None: + self.pre_whitespaces = WhitespaceNode(token) + else: + self.pre_whitespaces.append(token) + @dataclass(unsafe_hash=True) class IndexNode(BaseNode): @@ -669,12 +697,16 @@ class Parser: self.stream = self.lexer.lex(filename) self.current: Token = Token('eof', '', 0, 0, 0, (0, 0), None) self.previous = self.current + self.current_ws: T.List[Token] = [] self.getsym() self.in_ternary = False def create_node(self, node_type: T.Type[BaseNodeT], *args: T.Any, **kwargs: T.Any) -> BaseNodeT: node = node_type(*args, **kwargs) + for ws_token in self.current_ws: + node.append_whitespaces(ws_token) + self.current_ws = [] return node def getsym(self) -> None: @@ -682,6 +714,12 @@ class Parser: try: self.current = next(self.stream) + while self.current.tid in {'eol', 'comment', 'whitespace'}: + self.current_ws.append(self.current) + if self.current.tid == 'eol': + break + self.current = next(self.stream) + except StopIteration: self.current = Token('eof', '', self.current.line_start, self.current.lineno, self.current.colno + self.current.bytespan[1] - self.current.bytespan[0], (0, 0), None) @@ -782,11 +820,17 @@ class Parser: operator = self.create_node(SymbolNode, self.previous) return self.create_node(ComparisonNode, operator_type, left, operator, self.e5()) if self.accept('not'): + ws = self.current_ws.copy() not_token = self.previous if self.accept('in'): in_token = self.previous + self.current_ws = self.current_ws[len(ws):] # remove whitespaces between not and in + temp_node = EmptyNode(in_token.lineno, in_token.colno, in_token.filename) + for w in ws: + temp_node.append_whitespaces(w) + not_token.bytespan = (not_token.bytespan[0], in_token.bytespan[1]) - not_token.value += in_token.value + not_token.value += temp_node.whitespaces.value + in_token.value operator = self.create_node(SymbolNode, not_token) return self.create_node(ComparisonNode, 'notin', left, operator, self.e5()) return left @@ -1054,6 +1098,10 @@ class Parser: try: while cond: + for ws_token in self.current_ws: + block.append_whitespaces(ws_token) + self.current_ws = [] + curline = self.line() if not isinstance(curline, EmptyNode): @@ -1065,4 +1113,9 @@ class Parser: e.ast = block raise + # Remaining whitespaces will not be catched since there are no more nodes + for ws_token in self.current_ws: + block.append_whitespaces(ws_token) + self.current_ws = [] + return block -- cgit v1.1 From 6a18ae48b33c345185a8eda49e93adb1fb4594a9 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Mon, 28 Aug 2023 11:42:55 -0400 Subject: ast: fully resolve nodes for add operation Otherwise, string + stringmethod results in a list of two strings instead of the concatenation of the strings --- mesonbuild/ast/interpreter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mesonbuild') diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index da2119c..c51af09 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -371,8 +371,8 @@ class AstInterpreter(InterpreterBase): elif isinstance(node, ArithmeticNode): if node.operation != 'add': return None # Only handle string and array concats - l = quick_resolve(node.left) - r = quick_resolve(node.right) + l = self.resolve_node(node.left, include_unknown_args, id_loop_detect) + r = self.resolve_node(node.right, include_unknown_args, id_loop_detect) if isinstance(l, str) and isinstance(r, str): result = l + r # String concatenation detected else: -- cgit v1.1 From d3a26d158e4607ce677404920ad72508eb6f9de2 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 12:46:18 -0400 Subject: raw printer this printer preserves all whitespaces and comments in original meson.build file. It will be useful for rewrite and potential auto-formatter --- mesonbuild/ast/printer.py | 219 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 219 insertions(+) (limited to 'mesonbuild') diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index 410aabd..155b5fc 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -18,6 +18,8 @@ from __future__ import annotations from .. import mparser from .visitor import AstVisitor + +from itertools import zip_longest import re import typing as T @@ -248,6 +250,223 @@ class AstPrinter(AstVisitor): else: self.result = re.sub(r', $', '', self.result) +class RawPrinter(AstVisitor): + + def __init__(self): + self.result = '' + + def visit_default_func(self, node: mparser.BaseNode): + self.result += node.value + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_unary_operator(self, node: mparser.UnaryOperatorNode): + node.operator.accept(self) + node.value.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_binary_operator(self, node: mparser.BinaryOperatorNode): + node.left.accept(self) + node.operator.accept(self) + node.right.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_BooleanNode(self, node: mparser.BooleanNode) -> None: + self.result += 'true' if node.value else 'false' + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_NumberNode(self, node: mparser.NumberNode) -> None: + self.result += node.raw_value + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_StringNode(self, node: mparser.StringNode) -> None: + self.result += f"'{node.raw_value}'" + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_MultilineStringNode(self, node: mparser.MultilineStringNode) -> None: + self.result += f"'''{node.value}'''" + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: + self.result += 'f' + self.visit_StringNode(node) + + def visit_MultilineFormatStringNode(self, node: mparser.MultilineFormatStringNode) -> None: + self.result += 'f' + self.visit_MultilineStringNode(node) + + def visit_ContinueNode(self, node: mparser.ContinueNode) -> None: + self.result += 'continue' + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_BreakNode(self, node: mparser.BreakNode) -> None: + self.result += 'break' + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_ArrayNode(self, node: mparser.ArrayNode) -> None: + node.lbracket.accept(self) + node.args.accept(self) + node.rbracket.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_DictNode(self, node: mparser.DictNode) -> None: + node.lcurl.accept(self) + node.args.accept(self) + node.rcurl.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_ParenthesizedNode(self, node: mparser.ParenthesizedNode) -> None: + node.lpar.accept(self) + node.inner.accept(self) + node.rpar.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_OrNode(self, node: mparser.OrNode) -> None: + self.visit_binary_operator(node) + + def visit_AndNode(self, node: mparser.AndNode) -> None: + self.visit_binary_operator(node) + + def visit_ComparisonNode(self, node: mparser.ComparisonNode) -> None: + self.visit_binary_operator(node) + + def visit_ArithmeticNode(self, node: mparser.ArithmeticNode) -> None: + self.visit_binary_operator(node) + + def visit_NotNode(self, node: mparser.NotNode) -> None: + self.visit_unary_operator(node) + + def visit_CodeBlockNode(self, node: mparser.CodeBlockNode) -> None: + if node.pre_whitespaces: + node.pre_whitespaces.accept(self) + for i in node.lines: + i.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_IndexNode(self, node: mparser.IndexNode) -> None: + node.iobject.accept(self) + node.lbracket.accept(self) + node.index.accept(self) + node.rbracket.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_MethodNode(self, node: mparser.MethodNode) -> None: + node.source_object.accept(self) + node.dot.accept(self) + node.name.accept(self) + node.lpar.accept(self) + node.args.accept(self) + node.rpar.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: + node.func_name.accept(self) + node.lpar.accept(self) + node.args.accept(self) + node.rpar.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None: + node.var_name.accept(self) + node.operator.accept(self) + node.value.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None: + node.var_name.accept(self) + node.operator.accept(self) + node.value.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: + node.foreach_.accept(self) + for varname, comma in zip_longest(node.varnames, node.commas): + varname.accept(self) + if comma is not None: + comma.accept(self) + node.column.accept(self) + node.items.accept(self) + node.block.accept(self) + node.endforeach.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_IfClauseNode(self, node: mparser.IfClauseNode) -> None: + for i in node.ifs: + i.accept(self) + if not isinstance(node.elseblock, mparser.EmptyNode): + node.elseblock.accept(self) + node.endif.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_UMinusNode(self, node: mparser.UMinusNode) -> None: + self.visit_unary_operator(node) + + def visit_IfNode(self, node: mparser.IfNode) -> None: + node.if_.accept(self) + node.condition.accept(self) + node.block.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_ElseNode(self, node: mparser.ElseNode) -> None: + node.else_.accept(self) + node.block.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_TernaryNode(self, node: mparser.TernaryNode) -> None: + node.condition.accept(self) + node.questionmark.accept(self) + node.trueblock.accept(self) + node.column.accept(self) + node.falseblock.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None: + commas_iter = iter(node.commas) + + for arg in node.arguments: + arg.accept(self) + try: + comma = next(commas_iter) + comma.accept(self) + except StopIteration: + pass + + assert len(node.columns) == len(node.kwargs) + for (key, val), column in zip(node.kwargs.items(), node.columns): + key.accept(self) + column.accept(self) + val.accept(self) + try: + comma = next(commas_iter) + comma.accept(self) + except StopIteration: + pass + + if node.whitespaces: + node.whitespaces.accept(self) + class AstJSONPrinter(AstVisitor): def __init__(self) -> None: self.result: T.Dict[str, T.Any] = {} -- cgit v1.1 From 14e35b63c02a16f69dd1ad5bde775e6868965eb2 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 15:39:43 -0400 Subject: parser: allow whitespaces and comments in cont_eol FIXME: another approach would be to consider cont_eol as comment (i.e. add backslash and whitespaces to the comment regex). In both cases it works until we want to parse comments separately. TODO?: handle eol_cont inside a string (to split long string without breaking lines). Probably a bad idea and better to simply join a multiline string. --- mesonbuild/mparser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mesonbuild') diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index a161842..0f63c9e 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -121,7 +121,7 @@ class Lexer: ('fstring', re.compile(r"f'([^'\\]|(\\.))*'")), ('id', re.compile('[_a-zA-Z][_0-9a-zA-Z]*')), ('number', re.compile(r'0[bB][01]+|0[oO][0-7]+|0[xX][0-9a-fA-F]+|0|[1-9]\d*')), - ('eol_cont', re.compile(r'\\\n')), + ('eol_cont', re.compile(r'\\[ \t]*(#.*)?\n')), ('eol', re.compile(r'\n')), ('multiline_string', re.compile(r"'''(.|\n)*?'''", re.M)), ('comment', re.compile(r'#.*')), -- cgit v1.1