From ccad493e85e46bf0e78cfac8b77b03b7be75a396 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Tue, 15 Jan 2019 18:45:25 +0100 Subject: Basic AST visitor pattern --- mesonbuild/rewriter.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) (limited to 'mesonbuild/rewriter.py') diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 37ed7ef..1495d23 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -23,7 +23,7 @@ # - move targets # - reindent? -import mesonbuild.astinterpreter +from .ast import (AstInterpreter, AstVisitor) from mesonbuild.mesonlib import MesonException from mesonbuild import mlog import sys, traceback @@ -31,24 +31,20 @@ import sys, traceback def add_arguments(parser): parser.add_argument('--sourcedir', default='.', help='Path to source directory.') - parser.add_argument('--target', default=None, - help='Name of target to edit.') - parser.add_argument('--filename', default=None, - help='Name of source file to add or remove to target.') - parser.add_argument('commands', nargs='+') + parser.add_argument('-p', '--print', action='store_true', default=False, dest='print', + help='Print the parsed AST.') def run(options): - if options.target is None or options.filename is None: - sys.exit("Must specify both target and filename.") print('This tool is highly experimental, use with care.') - rewriter = mesonbuild.astinterpreter.RewriterInterpreter(options.sourcedir, '') + rewriter = AstInterpreter(options.sourcedir, '') try: - if options.commands[0] == 'add': - rewriter.add_source(options.target, options.filename) - elif options.commands[0] == 'remove': - rewriter.remove_source(options.target, options.filename) - else: - sys.exit('Unknown command: ' + options.commands[0]) + rewriter.load_root_meson_file() + rewriter.sanity_check_ast() + rewriter.parse_project() + rewriter.run() + + visitor = AstVisitor() + rewriter.ast.accept(visitor) except Exception as e: if isinstance(e, MesonException): mlog.exception(e) -- cgit v1.1 From 46320bfba89bf088dfe760ee1ff39658b5eb559f Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Tue, 15 Jan 2019 21:59:49 +0100 Subject: Added Ast printer --- mesonbuild/rewriter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'mesonbuild/rewriter.py') diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 1495d23..f025f23 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -23,7 +23,7 @@ # - move targets # - reindent? -from .ast import (AstInterpreter, AstVisitor) +from .ast import (AstInterpreter, AstVisitor, AstPrinter) from mesonbuild.mesonlib import MesonException from mesonbuild import mlog import sys, traceback @@ -35,7 +35,6 @@ def add_arguments(parser): help='Print the parsed AST.') def run(options): - print('This tool is highly experimental, use with care.') rewriter = AstInterpreter(options.sourcedir, '') try: rewriter.load_root_meson_file() @@ -43,8 +42,9 @@ def run(options): rewriter.parse_project() rewriter.run() - visitor = AstVisitor() + visitor = AstPrinter() rewriter.ast.accept(visitor) + print(visitor.result) except Exception as e: if isinstance(e, MesonException): mlog.exception(e) -- cgit v1.1 From 750af9c853423138e0d2812ba538402eedea80f2 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Sat, 19 Jan 2019 14:03:34 +0100 Subject: Moved the introspection interpreter --- mesonbuild/rewriter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mesonbuild/rewriter.py') diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index f025f23..45504be 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -23,10 +23,10 @@ # - move targets # - reindent? -from .ast import (AstInterpreter, AstVisitor, AstPrinter) +from .ast import (AstInterpreter, AstPrinter) from mesonbuild.mesonlib import MesonException from mesonbuild import mlog -import sys, traceback +import traceback def add_arguments(parser): parser.add_argument('--sourcedir', default='.', -- cgit v1.1 From 277dc10a5d21eaed884ef8674ca6cc1bec547ec0 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Sun, 20 Jan 2019 17:43:09 +0100 Subject: AST post processing --- mesonbuild/rewriter.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'mesonbuild/rewriter.py') diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 45504be..cc5d2ab 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -23,7 +23,7 @@ # - move targets # - reindent? -from .ast import (AstInterpreter, AstPrinter) +from .ast import AstInterpreter, AstVisitor, AstIDGenerator, AstIndentationGenerator, AstPrinter from mesonbuild.mesonlib import MesonException from mesonbuild import mlog import traceback @@ -42,9 +42,13 @@ def run(options): rewriter.parse_project() rewriter.run() - visitor = AstPrinter() - rewriter.ast.accept(visitor) - print(visitor.result) + indentor = AstIndentationGenerator() + idgen = AstIDGenerator() + printer = AstPrinter() + rewriter.ast.accept(indentor) + rewriter.ast.accept(idgen) + rewriter.ast.accept(printer) + print(printer.result) except Exception as e: if isinstance(e, MesonException): mlog.exception(e) -- cgit v1.1 From 86d5799bc4d945927e26fdcb6e239905e0aa8146 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Mon, 21 Jan 2019 21:52:18 +0100 Subject: First rewriter test case --- mesonbuild/rewriter.py | 154 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 132 insertions(+), 22 deletions(-) (limited to 'mesonbuild/rewriter.py') diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index cc5d2ab..37099ce 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -23,36 +23,146 @@ # - move targets # - reindent? -from .ast import AstInterpreter, AstVisitor, AstIDGenerator, AstIndentationGenerator, AstPrinter +from .ast import IntrospectionInterpreter, build_target_functions, AstVisitor, AstIDGenerator, AstIndentationGenerator, AstPrinter from mesonbuild.mesonlib import MesonException -from mesonbuild import mlog +from . import mlog, mparser, environment import traceback +from functools import wraps +from pprint import pprint +import json, os + +class RewriterException(MesonException): + pass def add_arguments(parser): parser.add_argument('--sourcedir', default='.', help='Path to source directory.') parser.add_argument('-p', '--print', action='store_true', default=False, dest='print', help='Print the parsed AST.') + parser.add_argument('command', type=str) + +class RequiredKeys: + def __init__(self, keys): + self.keys = keys + + def __call__(self, f): + @wraps(f) + def wrapped(*wrapped_args, **wrapped_kwargs): + assert(len(wrapped_args) >= 2) + cmd = wrapped_args[1] + for key, val in self.keys.items(): + typ = val[0] # The type of the value + default = val[1] # The default value -- None is required + choices = val[2] # Valid choices -- None is for everything + if key not in cmd: + if default is not None: + cmd[key] = default + else: + raise RewriterException('Key "{}" is missing in object for {}' + .format(key, f.__name__)) + if not isinstance(cmd[key], typ): + raise RewriterException('Invalid type of "{}". Required is {} but provided was {}' + .format(key, typ.__name__, type(cmd[key]).__name__)) + if choices is not None: + assert(isinstance(choices, list)) + if cmd[key] not in choices: + raise RewriterException('Invalid value of "{}": Possible values are {} but provided was "{}"' + .format(key, choices, cmd[key])) + return f(*wrapped_args, **wrapped_kwargs) + + return wrapped + +rewriter_keys = { + 'target': { + 'target': (str, None, None), + 'operation': (str, None, ['src_add', 'src_rm', 'test']), + 'sources': (list, [], None), + 'debug': (bool, False, None) + } +} + +class Rewriter: + def __init__(self, sourcedir: str, generator: str = 'ninja'): + self.sourcedir = sourcedir + self.interpreter = IntrospectionInterpreter(sourcedir, '', generator) + self.id_generator = AstIDGenerator() + self.functions = { + 'target': self.process_target, + } + + def analyze_meson(self): + mlog.log('Analyzing meson file:', mlog.bold(os.path.join(self.sourcedir, environment.build_filename))) + self.interpreter.analyze() + mlog.log(' -- Project:', mlog.bold(self.interpreter.project_data['descriptive_name'])) + mlog.log(' -- Version:', mlog.cyan(self.interpreter.project_data['version'])) + self.interpreter.ast.accept(AstIndentationGenerator()) + self.interpreter.ast.accept(self.id_generator) + + def find_target(self, target: str): + for i in self.interpreter.targets: + if target == i['name'] or target == i['id']: + return i + return None + + @RequiredKeys(rewriter_keys['target']) + def process_target(self, cmd): + mlog.log('Processing target', mlog.bold(cmd['target']), 'operation', mlog.cyan(cmd['operation'])) + target = self.find_target(cmd['target']) + if target is None: + mlog.error('Unknown target "{}" --> skipping'.format(cmd['target'])) + if cmd['debug']: + pprint(self.interpreter.targets) + return + if cmd['debug']: + pprint(target) + + if cmd['operation'] == 'src_add': + mlog.warning('TODO') + elif cmd['operation'] == 'src_rm': + mlog.warning('TODO') + elif cmd['operation'] == 'test': + src_list = [] + for i in target['sources']: + args = [] + if isinstance(i, mparser.FunctionNode): + args = list(i.args.arguments) + if i.func_name in build_target_functions: + args.pop(0) + elif isinstance(i, mparser.ArrayNode): + args = i.args.arguments + elif isinstance(i, mparser.ArgumentNode): + args = i.arguments + for j in args: + if isinstance(j, mparser.StringNode): + src_list += [j.value] + test_data = { + 'name': target['name'], + 'sources': src_list + } + mlog.log(' !! target {}={}'.format(target['id'], json.dumps(test_data))) + + def process(self, cmd): + if 'type' not in cmd: + raise RewriterException('Command has no key "type"') + if cmd['type'] not in self.functions: + raise RewriterException('Unknown command "{}". Supported commands are: {}' + .format(cmd['type'], list(self.functions.keys()))) + self.functions[cmd['type']](cmd) def run(options): - rewriter = AstInterpreter(options.sourcedir, '') - try: - rewriter.load_root_meson_file() - rewriter.sanity_check_ast() - rewriter.parse_project() - rewriter.run() - - indentor = AstIndentationGenerator() - idgen = AstIDGenerator() - printer = AstPrinter() - rewriter.ast.accept(indentor) - rewriter.ast.accept(idgen) - rewriter.ast.accept(printer) - print(printer.result) - except Exception as e: - if isinstance(e, MesonException): - mlog.exception(e) - else: - traceback.print_exc() - return 1 + rewriter = Rewriter(options.sourcedir) + rewriter.analyze_meson() + if os.path.exists(options.command): + with open(options.command, 'r') as fp: + commands = json.load(fp) + else: + commands = json.loads(options.command) + + if not isinstance(commands, list): + raise TypeError('Command is not a list') + + for i in commands: + if not isinstance(i, object): + raise TypeError('Command is not an object') + rewriter.process(i) return 0 -- cgit v1.1 From 0ce663239371576fcf4ad751d548cf2424525053 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Mon, 21 Jan 2019 22:52:27 +0100 Subject: Added suport for adding sources to a target --- mesonbuild/rewriter.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) (limited to 'mesonbuild/rewriter.py') diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 37099ce..9650d89 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -86,6 +86,7 @@ class Rewriter: self.sourcedir = sourcedir self.interpreter = IntrospectionInterpreter(sourcedir, '', generator) self.id_generator = AstIDGenerator() + self.modefied_nodes = [] self.functions = { 'target': self.process_target, } @@ -117,7 +118,31 @@ class Rewriter: pprint(target) if cmd['operation'] == 'src_add': - mlog.warning('TODO') + node = None + if target['sources']: + node = target['sources'][0] + else: + node = target['node'] + assert(node is not None) + + # Generate the new String nodes + to_append = [] + for i in cmd['sources']: + mlog.log(' -- Adding source', mlog.green(i), 'at', + mlog.yellow('{}:{}'.format(os.path.join(node.subdir, environment.build_filename), node.lineno))) + token = mparser.Token('string', node.subdir, 0, 0, 0, None, i) + to_append += [mparser.StringNode(token)] + + # Append to the AST at the right place + if isinstance(node, mparser.FunctionNode): + node.args.arguments += to_append + elif isinstance(node, mparser.ArrayNode): + node.args.arguments += to_append + elif isinstance(node, mparser.ArgumentNode): + node.arguments += to_append + + # Mark the node as modified + self.modefied_nodes += [node] elif cmd['operation'] == 'src_rm': mlog.warning('TODO') elif cmd['operation'] == 'test': -- cgit v1.1 From 8dd9b44831ec6b8de1c633ad5d366f3dea2df2cd Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Tue, 22 Jan 2019 11:12:36 +0100 Subject: Added support for removing sources from a target --- mesonbuild/rewriter.py | 69 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 12 deletions(-) (limited to 'mesonbuild/rewriter.py') diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 9650d89..6a11767 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -117,7 +117,23 @@ class Rewriter: if cmd['debug']: pprint(target) + # Utility function to get a list of the sources from a node + def arg_list_from_node(n): + args = [] + if isinstance(n, mparser.FunctionNode): + args = list(n.args.arguments) + if n.func_name in build_target_functions: + args.pop(0) + elif isinstance(n, mparser.ArrayNode): + args = n.args.arguments + elif isinstance(n, mparser.ArgumentNode): + args = n.arguments + return args + if cmd['operation'] == 'src_add': + ################################# + ### Add sources to the target ### + ################################# node = None if target['sources']: node = target['sources'][0] @@ -142,22 +158,51 @@ class Rewriter: node.arguments += to_append # Mark the node as modified - self.modefied_nodes += [node] + if node not in self.modefied_nodes: + self.modefied_nodes += [node] elif cmd['operation'] == 'src_rm': - mlog.warning('TODO') + ###################################### + ### Remove sources from the target ### + ###################################### + # Helper to find the exact string node and its parent + def find_node(src): + for i in target['sources']: + for j in arg_list_from_node(i): + if isinstance(j, mparser.StringNode): + if j.value == src: + return i, j + return None, None + + for i in cmd['sources']: + # Try to find the node with the source string + root, string_node = find_node(i) + if root is None: + mlog.warning(' -- Unable to find source', mlog.green(i), 'in the target') + continue + + # Remove the found string node from the argument list + arg_node = None + if isinstance(root, mparser.FunctionNode): + arg_node = root.args + if isinstance(root, mparser.ArrayNode): + arg_node = root.args + if isinstance(root, mparser.ArgumentNode): + arg_node = root + assert(arg_node is not None) + mlog.log(' -- Removing source', mlog.green(i), 'from', + mlog.yellow('{}:{}'.format(os.path.join(string_node.subdir, environment.build_filename), string_node.lineno))) + arg_node.arguments.remove(string_node) + + # Mark the node as modified + if root not in self.modefied_nodes: + self.modefied_nodes += [root] elif cmd['operation'] == 'test': + ###################################### + ### List all sources in the target ### + ###################################### src_list = [] for i in target['sources']: - args = [] - if isinstance(i, mparser.FunctionNode): - args = list(i.args.arguments) - if i.func_name in build_target_functions: - args.pop(0) - elif isinstance(i, mparser.ArrayNode): - args = i.args.arguments - elif isinstance(i, mparser.ArgumentNode): - args = i.arguments - for j in args: + for j in arg_list_from_node(i): if isinstance(j, mparser.StringNode): src_list += [j.value] test_data = { -- cgit v1.1 From b7c6f3ec72c831c2af20eb5320d5f51b52d79227 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Tue, 22 Jan 2019 16:00:10 +0100 Subject: Can now rewrite files --- mesonbuild/rewriter.py | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) (limited to 'mesonbuild/rewriter.py') diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 6a11767..dfed1c6 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -219,6 +219,90 @@ class Rewriter: .format(cmd['type'], list(self.functions.keys()))) self.functions[cmd['type']](cmd) + def apply_changes(self): + assert(all(hasattr(x, 'lineno') and hasattr(x, 'colno') and hasattr(x, 'subdir') for x in self.modefied_nodes)) + assert(all(isinstance(x, (mparser.ArrayNode, mparser.FunctionNode)) for x in self.modefied_nodes)) + # Sort based on line and column in reversed order + work_nodes = list(sorted(self.modefied_nodes, key=lambda x: x.lineno * 1000 + x.colno, reverse=True)) + + # Generating the new replacement string + str_list = [] + for i in work_nodes: + printer = AstPrinter() + i.accept(printer) + printer.post_process() + data = { + 'file': os.path.join(i.subdir, environment.build_filename), + 'str': printer.result.strip(), + 'node': i + } + str_list += [data] + + # Load build files + files = {} + for i in str_list: + if i['file'] in files: + continue + fpath = os.path.realpath(os.path.join(self.sourcedir, i['file'])) + fdata = '' + with open(fpath, 'r') as fp: + fdata = fp.read() + + # Generate line offsets numbers + m_lines = fdata.splitlines(True) + offset = 0 + line_offsets = [] + for j in m_lines: + line_offsets += [offset] + offset += len(j) + + files[i['file']] = { + 'path': fpath, + 'raw': fdata, + 'offsets': line_offsets + } + + # Replace in source code + for i in str_list: + offsets = files[i['file']]['offsets'] + raw = files[i['file']]['raw'] + node = i['node'] + line = node.lineno - 1 + col = node.colno + start = offsets[line]+col + end = start + if isinstance(node, mparser.ArrayNode): + if raw[end] != '[': + mlog.warning('Internal error: expected "[" at {}:{} but got "{}"'.format(line, col, raw[end])) + continue + counter = 1 + while counter > 0: + end += 1 + if raw[end] == '[': + counter += 1 + elif raw[end] == ']': + counter -= 1 + end += 1 + elif isinstance(node, mparser.FunctionNode): + while raw[end] != '(': + end += 1 + end += 1 + counter = 1 + while counter > 0: + end += 1 + if raw[end] == '(': + counter += 1 + elif raw[end] == ')': + counter -= 1 + end += 1 + raw = files[i['file']]['raw'] = raw[:start] + i['str'] + raw[end:] + + # Write the files back + for key, val in files.items(): + mlog.log('Rewriting', mlog.yellow(key)) + with open(val['path'], 'w') as fp: + fp.write(val['raw']) + def run(options): rewriter = Rewriter(options.sourcedir) rewriter.analyze_meson() @@ -235,4 +319,6 @@ def run(options): if not isinstance(i, object): raise TypeError('Command is not an object') rewriter.process(i) + + rewriter.apply_changes() return 0 -- cgit v1.1 From 6fe2c2b209814d7fe94f60d54c38ae75b1dc67af Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Tue, 22 Jan 2019 17:31:15 +0100 Subject: Fixed flake8 issues --- mesonbuild/rewriter.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'mesonbuild/rewriter.py') diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index dfed1c6..f66da33 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -23,10 +23,9 @@ # - move targets # - reindent? -from .ast import IntrospectionInterpreter, build_target_functions, AstVisitor, AstIDGenerator, AstIndentationGenerator, AstPrinter +from .ast import IntrospectionInterpreter, build_target_functions, AstIDGenerator, AstIndentationGenerator, AstPrinter from mesonbuild.mesonlib import MesonException from . import mlog, mparser, environment -import traceback from functools import wraps from pprint import pprint import json, os @@ -269,7 +268,7 @@ class Rewriter: node = i['node'] line = node.lineno - 1 col = node.colno - start = offsets[line]+col + start = offsets[line] + col end = start if isinstance(node, mparser.ArrayNode): if raw[end] != '[': -- cgit v1.1 From dbb94f122ddeb5a37ff2603acbcc701b996958bb Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Thu, 24 Jan 2019 21:38:29 +0100 Subject: Fixed style issues --- mesonbuild/rewriter.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'mesonbuild/rewriter.py') diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index f66da33..277835c 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -130,9 +130,6 @@ class Rewriter: return args if cmd['operation'] == 'src_add': - ################################# - ### Add sources to the target ### - ################################# node = None if target['sources']: node = target['sources'][0] @@ -159,10 +156,8 @@ class Rewriter: # Mark the node as modified if node not in self.modefied_nodes: self.modefied_nodes += [node] + elif cmd['operation'] == 'src_rm': - ###################################### - ### Remove sources from the target ### - ###################################### # Helper to find the exact string node and its parent def find_node(src): for i in target['sources']: @@ -195,10 +190,9 @@ class Rewriter: # Mark the node as modified if root not in self.modefied_nodes: self.modefied_nodes += [root] + elif cmd['operation'] == 'test': - ###################################### - ### List all sources in the target ### - ###################################### + # List all sources in the target src_list = [] for i in target['sources']: for j in arg_list_from_node(i): -- cgit v1.1