From 2502beccc9ac17d408f32b51cd1cb9a4a91f1ec1 Mon Sep 17 00:00:00 2001 From: Jussi Pakkanen Date: Mon, 17 Mar 2014 00:37:47 +0200 Subject: The final renaming. --- backends.py | 6 +- interpreter.py | 86 +++++----- mparser.py | 468 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ optinterpreter.py | 20 +-- parsertest.py | 468 ------------------------------------------------------ 5 files changed, 524 insertions(+), 524 deletions(-) create mode 100644 mparser.py delete mode 100755 parsertest.py diff --git a/backends.py b/backends.py index 75ef3d2..8e3e07a 100644 --- a/backends.py +++ b/backends.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import parsertest as mparser2 +import mparser import os, sys, re, pickle import environment, mlog from meson_install import InstallData @@ -39,7 +39,7 @@ def do_replacement(regex, line, confdata): var = confdata.get(varname) if isinstance(var, str): pass - elif isinstance(var, mparser2.StringNode): + elif isinstance(var, mparser.StringNode): var = var.value elif isinstance(var, int): var = str(var) @@ -60,7 +60,7 @@ def do_mesondefine(line, confdata): v = confdata.get(varname) except KeyError: return '/* undef %s */\n' % varname - if isinstance(v, mparser2.BooleanNode): + if isinstance(v, mparser.BooleanNode): v = v.value if isinstance(v, bool): if v: diff --git a/interpreter.py b/interpreter.py index 842621c..bde38c1 100644 --- a/interpreter.py +++ b/interpreter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import parsertest as mparser2 +import mparser import environment import coredata import dependencies @@ -601,7 +601,7 @@ class Interpreter(): raise InvalidCode('Builder file is empty.') assert(isinstance(code, str)) try: - self.ast = mparser2.Parser(code).parse() + self.ast = mparser.Parser(code).parse() except coredata.MesonException as me: me.file = environment.build_filename raise me @@ -658,12 +658,12 @@ class Interpreter(): return self.variables def sanity_check_ast(self): - if not isinstance(self.ast, mparser2.CodeBlockNode): + if not isinstance(self.ast, mparser.CodeBlockNode): raise InvalidCode('AST is of invalid type. Possibly a bug in the parser.') if len(self.ast.lines) == 0: raise InvalidCode('No statements in code.') first = self.ast.lines[0] - if not isinstance(first, mparser2.FunctionNode) or first.func_name != 'project': + if not isinstance(first, mparser.FunctionNode) or first.func_name != 'project': raise InvalidCode('First statement must be a call to project') def run(self): @@ -672,7 +672,7 @@ class Interpreter(): def evaluate_codeblock(self, node): if node is None: return - if not isinstance(node, mparser2.CodeBlockNode): + if not isinstance(node, mparser.CodeBlockNode): e = InvalidCode('Tried to execute a non-codeblock. Possibly a bug in the parser.') e.lineno = node.lineno e.colno = node.colno @@ -703,31 +703,31 @@ class Interpreter(): self.variables[varname] = variable def evaluate_statement(self, cur): - if isinstance(cur, mparser2.FunctionNode): + if isinstance(cur, mparser.FunctionNode): return self.function_call(cur) - elif isinstance(cur, mparser2.AssignmentNode): + elif isinstance(cur, mparser.AssignmentNode): return self.assignment(cur) - elif isinstance(cur, mparser2.MethodNode): + elif isinstance(cur, mparser.MethodNode): return self.method_call(cur) - elif isinstance(cur, mparser2.StringNode): + elif isinstance(cur, mparser.StringNode): return cur.value - elif isinstance(cur, mparser2.BooleanNode): + elif isinstance(cur, mparser.BooleanNode): return cur.value - elif isinstance(cur, mparser2.IfClauseNode): + elif isinstance(cur, mparser.IfClauseNode): return self.evaluate_if(cur) - elif isinstance(cur, mparser2.IdNode): + elif isinstance(cur, mparser.IdNode): return self.get_variable(cur.value) - elif isinstance(cur, mparser2.ComparisonNode): + elif isinstance(cur, mparser.ComparisonNode): return self.evaluate_comparison(cur) - elif isinstance(cur, mparser2.ArrayNode): + elif isinstance(cur, mparser.ArrayNode): return self.evaluate_arraystatement(cur) - elif isinstance(cur, mparser2.NumberNode): + elif isinstance(cur, mparser.NumberNode): return cur - elif isinstance(cur, mparser2.AndNode): + elif isinstance(cur, mparser.AndNode): return self.evaluate_andstatement(cur) - elif isinstance(cur, mparser2.OrNode): + elif isinstance(cur, mparser.OrNode): return self.evaluate_orstatement(cur) - elif isinstance(cur, mparser2.NotNode): + elif isinstance(cur, mparser.NotNode): return self.evaluate_notstatement(cur) else: raise InvalidCode("Unknown statement.") @@ -1046,7 +1046,7 @@ class Interpreter(): code = open(absname).read() assert(isinstance(code, str)) try: - codeblock = mparser2.Parser(code).parse() + codeblock = mparser.Parser(code).parse() except coredata.MesonException as me: me.file = buildfilename raise me @@ -1113,7 +1113,7 @@ class Interpreter(): self.build.global_args[lang] = args def flatten(self, args): - if isinstance(args, mparser2.StringNode): + if isinstance(args, mparser.StringNode): return args.value if isinstance(args, str): return args @@ -1124,7 +1124,7 @@ class Interpreter(): if isinstance(a, list): rest = self.flatten(a) result = result + rest - elif isinstance(a, mparser2.StringNode): + elif isinstance(a, mparser.StringNode): result.append(a.value) else: result.append(a) @@ -1193,7 +1193,7 @@ class Interpreter(): return False def assignment(self, node): - assert(isinstance(node, mparser2.AssignmentNode)) + assert(isinstance(node, mparser.AssignmentNode)) var_name = node.var_name if not isinstance(var_name, str): raise InvalidArguments('Tried to assign value to a non-variable.') @@ -1207,27 +1207,27 @@ class Interpreter(): return value def reduce_single(self, arg): - if isinstance(arg, mparser2.IdNode): + if isinstance(arg, mparser.IdNode): return self.get_variable(arg.value) elif isinstance(arg, str): return arg - elif isinstance(arg, mparser2.StringNode): + elif isinstance(arg, mparser.StringNode): return arg.value - elif isinstance(arg, mparser2.FunctionNode): + elif isinstance(arg, mparser.FunctionNode): return self.function_call(arg) - elif isinstance(arg, mparser2.MethodNode): + elif isinstance(arg, mparser.MethodNode): return self.method_call(arg) - elif isinstance(arg, mparser2.BooleanNode): + elif isinstance(arg, mparser.BooleanNode): return arg.value - elif isinstance(arg, mparser2.ArrayNode): + elif isinstance(arg, mparser.ArrayNode): return [self.reduce_single(curarg) for curarg in arg.args.arguments] - elif isinstance(arg, mparser2.NumberNode): + elif isinstance(arg, mparser.NumberNode): return arg.value else: raise InvalidCode('Irreducible argument.') def reduce_arguments(self, args): - assert(isinstance(args, mparser2.ArgumentNode)) + assert(isinstance(args, mparser.ArgumentNode)) if args.incorrect_order(): raise InvalidArguments('All keyword arguments must be after positional arguments.') reduced_pos = [self.reduce_single(arg) for arg in args.arguments] @@ -1247,15 +1247,15 @@ class Interpreter(): raise InterpreterException('Unknown method "%s" for a string.' % method_name) def to_native(self, arg): - if isinstance(arg, mparser2.StringNode) or \ - isinstance(arg, mparser2.NumberNode) or \ - isinstance(arg, mparser2.BooleanNode): + if isinstance(arg, mparser.StringNode) or \ + isinstance(arg, mparser.NumberNode) or \ + isinstance(arg, mparser.BooleanNode): return arg.value return arg def format_string(self, templ, args): templ = self.to_native(templ) - if isinstance(args, mparser2.ArgumentNode): + if isinstance(args, mparser.ArgumentNode): args = args.arguments for (i, arg) in enumerate(args): arg = self.to_native(self.reduce_single(arg)) @@ -1266,7 +1266,7 @@ class Interpreter(): def method_call(self, node): invokable = node.source_object - if isinstance(invokable, mparser2.IdNode): + if isinstance(invokable, mparser.IdNode): object_name = invokable.value obj = self.get_variable(object_name) else: @@ -1275,7 +1275,7 @@ class Interpreter(): if method_name == 'extract_objects' and self.environment.coredata.unity: raise InterpreterException('Single object files can not be extracted in Unity builds.') args = node.args - if isinstance(obj, mparser2.StringNode): + if isinstance(obj, mparser.StringNode): obj = obj.get_value() if isinstance(obj, str): return self.string_method_call(obj, method_name, args) @@ -1285,7 +1285,7 @@ class Interpreter(): return obj.method_call(method_name, args, kwargs) def evaluate_if(self, node): - assert(isinstance(node, mparser2.IfClauseNode)) + assert(isinstance(node, mparser.IfClauseNode)) for i in node.ifs: result = self.evaluate_statement(i.condition) if not(isinstance(result, bool)): @@ -1293,7 +1293,7 @@ class Interpreter(): if result: self.evaluate_codeblock(i.block) return - if not isinstance(node.elseblock, mparser2.EmptyNode): + if not isinstance(node.elseblock, mparser.EmptyNode): self.evaluate_codeblock(node.elseblock) def is_elementary_type(self, v): @@ -1324,14 +1324,14 @@ class Interpreter(): def evaluate_andstatement(self, cur): l = self.evaluate_statement(cur.left) - if isinstance(l, mparser2.BooleanNode): + if isinstance(l, mparser.BooleanNode): l = l.value if not isinstance(l, bool): raise InterpreterException('First argument to "and" is not a boolean.') if not l: return False r = self.evaluate_statement(cur.right) - if isinstance(r, mparser2.BooleanNode): + if isinstance(r, mparser.BooleanNode): r = r.value if not isinstance(r, bool): raise InterpreterException('Second argument to "and" is not a boolean.') @@ -1339,14 +1339,14 @@ class Interpreter(): def evaluate_orstatement(self, cur): l = self.evaluate_statement(cur.left) - if isinstance(l, mparser2.BooleanNode): + if isinstance(l, mparser.BooleanNode): l = l.get_value() if not isinstance(l, bool): raise InterpreterException('First argument to "or" is not a boolean.') if l: return True r = self.evaluate_statement(cur.right) - if isinstance(r, mparser2.BooleanNode): + if isinstance(r, mparser.BooleanNode): r = r.get_value() if not isinstance(r, bool): raise InterpreterException('Second argument to "or" is not a boolean.') @@ -1354,7 +1354,7 @@ class Interpreter(): def evaluate_notstatement(self, cur): v = self.evaluate_statement(cur.value) - if isinstance(v, mparser2.BooleanNode): + if isinstance(v, mparser.BooleanNode): v = v.value if not isinstance(v, bool): raise InterpreterException('Argument to "not" is not a boolean.') diff --git a/mparser.py b/mparser.py new file mode 100644 index 0000000..1f7de21 --- /dev/null +++ b/mparser.py @@ -0,0 +1,468 @@ +#!/usr/bin/python3 + +# Copyright 2014 Jussi Pakkanen + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import sys + +class ParseException(Exception): + def __init__(self, text, lineno, colno): + super().__init__() + self.text = text + self.lineno = lineno + self.colno = colno + +class Token: + def __init__(self, tid, lineno, colno, value): + self.tid = tid + self.lineno = lineno + self.colno = colno + self.value = value + + def __eq__(self, other): + if isinstance(other, str): + return self.tid == other + return self.tid == other.tid + +class Lexer: + def __init__(self): + self.keywords = {'true', 'false', 'if', 'else', 'elif', + 'endif', 'and', 'or', 'not'} + self.token_specification = [ + # Need to be sorted longest to shortest. + ('ignore', re.compile(r'[ \t]')), + ('id', re.compile('[_a-zA-Z][_0-9a-zA-Z]*')), + ('number', re.compile(r'\d+')), + ('eol_cont', re.compile(r'\\\n')), + ('eol', re.compile(r'\n')), + ('multiline_string', re.compile(r"'''(.|\n)*?'''", re.M)), + ('comment', re.compile(r'\#.*')), + ('lparen', re.compile(r'\(')), + ('rparen', re.compile(r'\)')), + ('lbracket', re.compile(r'\[')), + ('rbracket', re.compile(r'\]')), + ('string', re.compile("'[^']*?'")), + ('comma', re.compile(r',')), + ('dot', re.compile(r'\.')), + ('colon', re.compile(r':')), + ('equal', re.compile(r'==')), + ('nequal', re.compile(r'\!=')), + ('assign', re.compile(r'=')), + ] + + def lex(self, code): + lineno = 1 + line_start = 0 + loc = 0; + par_count = 0 + bracket_count = 0 + col = 0 + while(loc < len(code)): + matched = False + value = None + for (tid, reg) in self.token_specification: + mo = reg.match(code, loc) + if mo: + curline = lineno + col = mo.start()-line_start + matched = True + loc = mo.end() + match_text = mo.group() + if tid == 'ignore' or tid == 'comment': + break + elif tid == 'lparen': + par_count += 1 + elif tid == 'rparen': + par_count -= 1 + elif tid == 'lbracket': + bracket_count += 1 + elif tid == 'rbracket': + bracket_count -= 1 + elif tid == 'string': + value = match_text[1:-1] + elif tid == 'multiline_string': + tid = 'string' + value = match_text[3:-3] + lines = match_text.split('\n') + if len(lines) > 1: + lineno += len(lines) - 1 + line_start = mo.end() - len(lines[-1]) + elif tid == 'number': + value = int(match_text) + elif tid == 'eol' or tid == 'eol_cont': + lineno += 1 + line_start = loc + if par_count > 0 or bracket_count > 0: + break + elif tid == 'id': + if match_text in self.keywords: + tid = match_text + else: + value = match_text + yield Token(tid, curline, col, value) + if not matched: + raise ParseException('lexer', lineno, col) + +class BooleanNode: + def __init__(self, token, value): + self.lineno = token.lineno + self.colno = token.colno + self.value = value + assert(isinstance(self.value, bool)) + +class IdNode: + def __init__(self, token): + self.lineno = token.lineno + self.colno = token.colno + self.value = token.value + assert(isinstance(self.value, str)) + +class NumberNode: + def __init__(self, token): + self.lineno = token.lineno + self.colno = token.colno + self.value = token.value + assert(isinstance(self.value, int)) + +class StringNode: + def __init__(self, token): + self.lineno = token.lineno + self.colno = token.colno + self.value = token.value + assert(isinstance(self.value, str)) + +class ArrayNode: + def __init__(self, args): + self.lineno = args.lineno + self.colno = args.colno + self.args = args + +class EmptyNode: + def __init__(self): + self.lineno = 0 + self.colno = 0 + self.value = None + +class OrNode: + def __init__(self, lineno, colno, left, right): + self.lineno = lineno + self.colno = colno + self.left = left + self.right = right + +class AndNode: + def __init__(self, lineno, colno, left, right): + self.lineno = lineno + self.colno = colno + self.left = left + self.right = right + +class ComparisonNode: + def __init__(self, lineno, colno, ctype, left, right): + self.lineno = lineno + self.colno = colno + self.left = left + self.right = right + self.ctype = ctype + +class NotNode: + def __init__(self, lineno, colno, value): + self.lineno = lineno + self.colno = colno + self.value = value + +class CodeBlockNode: + def __init__(self, lineno, colno): + self.lineno = lineno + self.colno = colno + self.lines = [] + +class MethodNode: + def __init__(self, lineno, colno, source_object, name, args): + self.lineno = lineno + self.colno = colno + self.source_object = source_object + self.name = name + assert(isinstance(self.name, str)) + self.args = args + +class FunctionNode: + def __init__(self, lineno, colno, func_name, args): + self.lineno = lineno + self.colno = colno + self.func_name = func_name + assert(isinstance(func_name, str)) + self.args = args + +class AssignmentNode: + def __init__(self, lineno, colno, var_name, value): + self.lineno = lineno + self.colno = colno + self.var_name = var_name + assert(isinstance(var_name, str)) + self.value = value + +class IfClauseNode(): + def __init__(self, lineno, colno): + self.lineno = lineno + self.colno = colno + self.ifs = [] + self.elseblock = EmptyNode() + +class IfNode(): + def __init__(self, lineno, colno, condition, block): + self.lineno = lineno + self.colno = colno + self.condition = condition + self.block = block + +class ArgumentNode(): + def __init__(self, token): + self.lineno = token.lineno + self.colno = token.colno + self.arguments = [] + self.kwargs = {} + self.order_error = False + + def prepend(self, statement): + if not isinstance(statement, EmptyNode): + self.arguments = [statement] + self.arguments + + def append(self, statement): + if not isinstance(statement, EmptyNode): + self.arguments = self.arguments + [statement] + + def set_kwarg(self, name, value): + if self.num_args() > 0: + self.order_error = True + self.kwargs[name] = value + + def num_args(self): + return len(self.arguments) + + def num_kwargs(self): + return len(self.kwargs) + + def incorrect_order(self): + return self.order_error + + def __len__(self): + return self.num_args() # Fixme + +# Recursive descent parser for Meson's definition language. +# Very basic apart from the fact that we have many precedence +# levels so there are not enough words to describe them all. +# Enter numbering: +# +# 1 assignment +# 2 or +# 3 and +# 4 equality +# comparison, plus and multiplication would go here +# 5 negation +# 6 funcall, method call +# 7 parentheses +# 8 plain token + +class Parser: + def __init__(self, code): + self.stream = Lexer().lex(code) + self.getsym() + + def getsym(self): + try: + self.current = next(self.stream) + except StopIteration: + self.current = Token('eof', 0, 0, None) + + def accept(self, s): + if self.current.tid == s: + self.getsym() + return True + return False + + def expect(self, s): + if self.accept(s): + return True + raise ParseException('Expecting %s got %s.' % (s, self.current.tid), self.current.lineno, self.current.colno) + + def parse(self): + block = self.codeblock() + self.expect('eof') + return block + + def statement(self): + return self.e1() + + def e1(self): + left = self.e2() + if self.accept('assign'): + value = self.e1() + if not isinstance(left, IdNode): + raise ParseException('Assignment target must be an id.', + left.lineno, left.colno) + return AssignmentNode(left.lineno, left.colno, left.value, value) + return left + + def e2(self): + left = self.e3() + if self.accept('or'): + return OrNode(left.lineno, left.colno, left, self.e3()) + return left + + def e3(self): + left = self.e4() + if self.accept('and'): + return AndNode(left.lineno, left.colno, left, self.e4()) + return left + + def e4(self): + left = self.e5() + if self.accept('equal'): + return ComparisonNode(left.lineno, left.colno, '==', left, self.e5()) + if self.accept('nequal'): + return ComparisonNode(left.lineno, left.colno, '!=', left, self.e5()) + return left + + def e5(self): + if self.accept('not'): + return NotNode(self.current.lineno, self.current.colno, self.e6()) + return self.e6() + + def e6(self): + left = self.e7() + if self.accept('dot'): + return self.method_call(left) + elif self.accept('lparen'): + args = self.args() + self.expect('rparen') + if not isinstance(left, IdNode): + raise ParseException('Function call must be applied to plain id', + left.lineno, left.colno) + return FunctionNode(left.lineno, left.colno, left.value, args) + return left + + def e7(self): + if self.accept('lparen'): + e = self.statement() + self.expect('rparen') + return e + elif self.accept('lbracket'): + args = self.args() + self.expect('rbracket') + return ArrayNode(args) + else: + return self.e8() + + def e8(self): + t = self.current + if self.accept('true'): + return BooleanNode(t, True); + if self.accept('false'): + return BooleanNode(t, False) + if self.accept('id'): + return IdNode(t) + if self.accept('number'): + return NumberNode(t) + if self.accept('string'): + return StringNode(t) + return EmptyNode() + + def args(self): + s = self.statement() + if isinstance(s, EmptyNode): + ArgumentNode(s) + + if self.accept('comma'): + rest = self.args() + rest.prepend(s) + return rest + if self.accept('colon'): + if not isinstance(s, IdNode): + raise ParseException('Keyword argument must be a plain identifier.', + s.lineno, s.colno) + value = self.statement() + if self.accept('comma'): + a = self.args() + else: + a = ArgumentNode(self.current) + a.set_kwarg(s.value, value) + return a + a = ArgumentNode(self.current) + a.append(s) + return a + + def method_call(self, source_object): + methodname = self.e8() + if not(isinstance(methodname, IdNode)): + raise ParseException('Method name must be plain id', + self.current.lineno, self.current.colno) + self.expect('lparen') + args = self.args() + self.expect('rparen') + method = MethodNode(methodname.lineno, methodname.colno, source_object, methodname.value, args) + if self.accept('dot'): + return self.method_call(method) + return method + + def ifblock(self): + condition = self.statement() + clause = IfClauseNode(condition.lineno, condition.colno) + block = self.codeblock() + clause.ifs.append(IfNode(clause.lineno, clause.colno, condition, block)) + self.elseifblock(clause) + clause.elseblock = self.elseblock() + return clause + + def elseifblock(self, clause): + while self.accept('elif'): + s = self.statement() + self.expect('eol') + b = self.codeblock() + clause.ifs.append(IfNode(s.lineno, s.colno, s, b)) + + def elseblock(self): + if self.accept('else'): + self.expect('eol') + return self.codeblock() + + def line(self): + if self.current == 'eol': + return EmptyNode() + if self.accept('if'): + block = self.ifblock() + self.expect('endif') + return block + return self.statement() + + def codeblock(self): + block = CodeBlockNode(self.current.lineno, self.current.colno) + cond = True + while cond: + curline = self.line() + if not isinstance(curline, EmptyNode): + block.lines.append(curline) + cond = self.accept('eol') + return block + +if __name__ == '__main__': + for code in sys.argv[1:]: + parser = Parser(open(code).read()) + try: + print(code) + parser.parse() + except ParseException as e: + print('Error', e.text, 'line', e.lineno, 'column', e.colno) + diff --git a/optinterpreter.py b/optinterpreter.py index 7c9be4c..75926a6 100644 --- a/optinterpreter.py +++ b/optinterpreter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import parsertest as mparser2 +import mparser import coredata import os @@ -88,11 +88,11 @@ class OptionInterpreter: def process(self, option_file): try: - ast = mparser2.Parser(open(option_file, 'r').read()).parse() + ast = mparser.Parser(open(option_file, 'r').read()).parse() except coredata.MesonException as me: me.file = option_file raise me - if not isinstance(ast, mparser2.CodeBlockNode): + if not isinstance(ast, mparser.CodeBlockNode): e = OptionException('Option file is malformed.') e.lineno = ast.lineno() raise e @@ -106,23 +106,23 @@ class OptionInterpreter: raise e def reduce_single(self, arg): - if isinstance(arg, mparser2.IdNode): + if isinstance(arg, mparser.IdNode): return self.get_variable(arg.value) elif isinstance(arg, str): return arg - elif isinstance(arg, mparser2.StringNode): + elif isinstance(arg, mparser.StringNode): return arg.value - elif isinstance(arg, mparser2.BooleanNode): + elif isinstance(arg, mparser.BooleanNode): return arg.value - elif isinstance(arg, mparser2.ArrayNode): + elif isinstance(arg, mparser.ArrayNode): return [self.reduce_single(curarg) for curarg in arg.args.arguments] - elif isinstance(arg, mparser2.NumberNode): + elif isinstance(arg, mparser.NumberNode): return arg.get_value() else: raise OptionException('Arguments may only be string, int, bool, or array of those.') def reduce_arguments(self, args): - assert(isinstance(args, mparser2.ArgumentNode)) + assert(isinstance(args, mparser.ArgumentNode)) if args.incorrect_order(): raise OptionException('All keyword arguments must be after positional arguments.') reduced_pos = [self.reduce_single(arg) for arg in args.arguments] @@ -135,7 +135,7 @@ class OptionInterpreter: return (reduced_pos, reduced_kw) def evaluate_statement(self, node): - if not isinstance(node, mparser2.FunctionNode): + if not isinstance(node, mparser.FunctionNode): raise OptionException('Option file may only contain option definitions') func_name = node.func_name if func_name != 'option': diff --git a/parsertest.py b/parsertest.py deleted file mode 100755 index 1f7de21..0000000 --- a/parsertest.py +++ /dev/null @@ -1,468 +0,0 @@ -#!/usr/bin/python3 - -# Copyright 2014 Jussi Pakkanen - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import sys - -class ParseException(Exception): - def __init__(self, text, lineno, colno): - super().__init__() - self.text = text - self.lineno = lineno - self.colno = colno - -class Token: - def __init__(self, tid, lineno, colno, value): - self.tid = tid - self.lineno = lineno - self.colno = colno - self.value = value - - def __eq__(self, other): - if isinstance(other, str): - return self.tid == other - return self.tid == other.tid - -class Lexer: - def __init__(self): - self.keywords = {'true', 'false', 'if', 'else', 'elif', - 'endif', 'and', 'or', 'not'} - self.token_specification = [ - # Need to be sorted longest to shortest. - ('ignore', re.compile(r'[ \t]')), - ('id', re.compile('[_a-zA-Z][_0-9a-zA-Z]*')), - ('number', re.compile(r'\d+')), - ('eol_cont', re.compile(r'\\\n')), - ('eol', re.compile(r'\n')), - ('multiline_string', re.compile(r"'''(.|\n)*?'''", re.M)), - ('comment', re.compile(r'\#.*')), - ('lparen', re.compile(r'\(')), - ('rparen', re.compile(r'\)')), - ('lbracket', re.compile(r'\[')), - ('rbracket', re.compile(r'\]')), - ('string', re.compile("'[^']*?'")), - ('comma', re.compile(r',')), - ('dot', re.compile(r'\.')), - ('colon', re.compile(r':')), - ('equal', re.compile(r'==')), - ('nequal', re.compile(r'\!=')), - ('assign', re.compile(r'=')), - ] - - def lex(self, code): - lineno = 1 - line_start = 0 - loc = 0; - par_count = 0 - bracket_count = 0 - col = 0 - while(loc < len(code)): - matched = False - value = None - for (tid, reg) in self.token_specification: - mo = reg.match(code, loc) - if mo: - curline = lineno - col = mo.start()-line_start - matched = True - loc = mo.end() - match_text = mo.group() - if tid == 'ignore' or tid == 'comment': - break - elif tid == 'lparen': - par_count += 1 - elif tid == 'rparen': - par_count -= 1 - elif tid == 'lbracket': - bracket_count += 1 - elif tid == 'rbracket': - bracket_count -= 1 - elif tid == 'string': - value = match_text[1:-1] - elif tid == 'multiline_string': - tid = 'string' - value = match_text[3:-3] - lines = match_text.split('\n') - if len(lines) > 1: - lineno += len(lines) - 1 - line_start = mo.end() - len(lines[-1]) - elif tid == 'number': - value = int(match_text) - elif tid == 'eol' or tid == 'eol_cont': - lineno += 1 - line_start = loc - if par_count > 0 or bracket_count > 0: - break - elif tid == 'id': - if match_text in self.keywords: - tid = match_text - else: - value = match_text - yield Token(tid, curline, col, value) - if not matched: - raise ParseException('lexer', lineno, col) - -class BooleanNode: - def __init__(self, token, value): - self.lineno = token.lineno - self.colno = token.colno - self.value = value - assert(isinstance(self.value, bool)) - -class IdNode: - def __init__(self, token): - self.lineno = token.lineno - self.colno = token.colno - self.value = token.value - assert(isinstance(self.value, str)) - -class NumberNode: - def __init__(self, token): - self.lineno = token.lineno - self.colno = token.colno - self.value = token.value - assert(isinstance(self.value, int)) - -class StringNode: - def __init__(self, token): - self.lineno = token.lineno - self.colno = token.colno - self.value = token.value - assert(isinstance(self.value, str)) - -class ArrayNode: - def __init__(self, args): - self.lineno = args.lineno - self.colno = args.colno - self.args = args - -class EmptyNode: - def __init__(self): - self.lineno = 0 - self.colno = 0 - self.value = None - -class OrNode: - def __init__(self, lineno, colno, left, right): - self.lineno = lineno - self.colno = colno - self.left = left - self.right = right - -class AndNode: - def __init__(self, lineno, colno, left, right): - self.lineno = lineno - self.colno = colno - self.left = left - self.right = right - -class ComparisonNode: - def __init__(self, lineno, colno, ctype, left, right): - self.lineno = lineno - self.colno = colno - self.left = left - self.right = right - self.ctype = ctype - -class NotNode: - def __init__(self, lineno, colno, value): - self.lineno = lineno - self.colno = colno - self.value = value - -class CodeBlockNode: - def __init__(self, lineno, colno): - self.lineno = lineno - self.colno = colno - self.lines = [] - -class MethodNode: - def __init__(self, lineno, colno, source_object, name, args): - self.lineno = lineno - self.colno = colno - self.source_object = source_object - self.name = name - assert(isinstance(self.name, str)) - self.args = args - -class FunctionNode: - def __init__(self, lineno, colno, func_name, args): - self.lineno = lineno - self.colno = colno - self.func_name = func_name - assert(isinstance(func_name, str)) - self.args = args - -class AssignmentNode: - def __init__(self, lineno, colno, var_name, value): - self.lineno = lineno - self.colno = colno - self.var_name = var_name - assert(isinstance(var_name, str)) - self.value = value - -class IfClauseNode(): - def __init__(self, lineno, colno): - self.lineno = lineno - self.colno = colno - self.ifs = [] - self.elseblock = EmptyNode() - -class IfNode(): - def __init__(self, lineno, colno, condition, block): - self.lineno = lineno - self.colno = colno - self.condition = condition - self.block = block - -class ArgumentNode(): - def __init__(self, token): - self.lineno = token.lineno - self.colno = token.colno - self.arguments = [] - self.kwargs = {} - self.order_error = False - - def prepend(self, statement): - if not isinstance(statement, EmptyNode): - self.arguments = [statement] + self.arguments - - def append(self, statement): - if not isinstance(statement, EmptyNode): - self.arguments = self.arguments + [statement] - - def set_kwarg(self, name, value): - if self.num_args() > 0: - self.order_error = True - self.kwargs[name] = value - - def num_args(self): - return len(self.arguments) - - def num_kwargs(self): - return len(self.kwargs) - - def incorrect_order(self): - return self.order_error - - def __len__(self): - return self.num_args() # Fixme - -# Recursive descent parser for Meson's definition language. -# Very basic apart from the fact that we have many precedence -# levels so there are not enough words to describe them all. -# Enter numbering: -# -# 1 assignment -# 2 or -# 3 and -# 4 equality -# comparison, plus and multiplication would go here -# 5 negation -# 6 funcall, method call -# 7 parentheses -# 8 plain token - -class Parser: - def __init__(self, code): - self.stream = Lexer().lex(code) - self.getsym() - - def getsym(self): - try: - self.current = next(self.stream) - except StopIteration: - self.current = Token('eof', 0, 0, None) - - def accept(self, s): - if self.current.tid == s: - self.getsym() - return True - return False - - def expect(self, s): - if self.accept(s): - return True - raise ParseException('Expecting %s got %s.' % (s, self.current.tid), self.current.lineno, self.current.colno) - - def parse(self): - block = self.codeblock() - self.expect('eof') - return block - - def statement(self): - return self.e1() - - def e1(self): - left = self.e2() - if self.accept('assign'): - value = self.e1() - if not isinstance(left, IdNode): - raise ParseException('Assignment target must be an id.', - left.lineno, left.colno) - return AssignmentNode(left.lineno, left.colno, left.value, value) - return left - - def e2(self): - left = self.e3() - if self.accept('or'): - return OrNode(left.lineno, left.colno, left, self.e3()) - return left - - def e3(self): - left = self.e4() - if self.accept('and'): - return AndNode(left.lineno, left.colno, left, self.e4()) - return left - - def e4(self): - left = self.e5() - if self.accept('equal'): - return ComparisonNode(left.lineno, left.colno, '==', left, self.e5()) - if self.accept('nequal'): - return ComparisonNode(left.lineno, left.colno, '!=', left, self.e5()) - return left - - def e5(self): - if self.accept('not'): - return NotNode(self.current.lineno, self.current.colno, self.e6()) - return self.e6() - - def e6(self): - left = self.e7() - if self.accept('dot'): - return self.method_call(left) - elif self.accept('lparen'): - args = self.args() - self.expect('rparen') - if not isinstance(left, IdNode): - raise ParseException('Function call must be applied to plain id', - left.lineno, left.colno) - return FunctionNode(left.lineno, left.colno, left.value, args) - return left - - def e7(self): - if self.accept('lparen'): - e = self.statement() - self.expect('rparen') - return e - elif self.accept('lbracket'): - args = self.args() - self.expect('rbracket') - return ArrayNode(args) - else: - return self.e8() - - def e8(self): - t = self.current - if self.accept('true'): - return BooleanNode(t, True); - if self.accept('false'): - return BooleanNode(t, False) - if self.accept('id'): - return IdNode(t) - if self.accept('number'): - return NumberNode(t) - if self.accept('string'): - return StringNode(t) - return EmptyNode() - - def args(self): - s = self.statement() - if isinstance(s, EmptyNode): - ArgumentNode(s) - - if self.accept('comma'): - rest = self.args() - rest.prepend(s) - return rest - if self.accept('colon'): - if not isinstance(s, IdNode): - raise ParseException('Keyword argument must be a plain identifier.', - s.lineno, s.colno) - value = self.statement() - if self.accept('comma'): - a = self.args() - else: - a = ArgumentNode(self.current) - a.set_kwarg(s.value, value) - return a - a = ArgumentNode(self.current) - a.append(s) - return a - - def method_call(self, source_object): - methodname = self.e8() - if not(isinstance(methodname, IdNode)): - raise ParseException('Method name must be plain id', - self.current.lineno, self.current.colno) - self.expect('lparen') - args = self.args() - self.expect('rparen') - method = MethodNode(methodname.lineno, methodname.colno, source_object, methodname.value, args) - if self.accept('dot'): - return self.method_call(method) - return method - - def ifblock(self): - condition = self.statement() - clause = IfClauseNode(condition.lineno, condition.colno) - block = self.codeblock() - clause.ifs.append(IfNode(clause.lineno, clause.colno, condition, block)) - self.elseifblock(clause) - clause.elseblock = self.elseblock() - return clause - - def elseifblock(self, clause): - while self.accept('elif'): - s = self.statement() - self.expect('eol') - b = self.codeblock() - clause.ifs.append(IfNode(s.lineno, s.colno, s, b)) - - def elseblock(self): - if self.accept('else'): - self.expect('eol') - return self.codeblock() - - def line(self): - if self.current == 'eol': - return EmptyNode() - if self.accept('if'): - block = self.ifblock() - self.expect('endif') - return block - return self.statement() - - def codeblock(self): - block = CodeBlockNode(self.current.lineno, self.current.colno) - cond = True - while cond: - curline = self.line() - if not isinstance(curline, EmptyNode): - block.lines.append(curline) - cond = self.accept('eol') - return block - -if __name__ == '__main__': - for code in sys.argv[1:]: - parser = Parser(open(code).read()) - try: - print(code) - parser.parse() - except ParseException as e: - print('Error', e.text, 'line', e.lineno, 'column', e.colno) - -- cgit v1.1