Merge pull request #6765 from mensinda/astDump2

mintro: AST JSON printer

10 files changed, 344 insertions, 32 deletions

diff --git a/docs/markdown/IDE-integration.md b/docs/markdown/IDE-integration.md
index 73737e8..f51075e 100644
--- a/docs/markdown/IDE-integration.md
+++ b/docs/markdown/IDE-integration.md
@@ -29,16 +29,16 @@ watch for changes in this directory to know when something changed.
 
 The `meson-info` directory should contain the following files:
 
-| File                            | Description                                                         |
-| ----                            | -----------                                                         |
-| `intro-benchmarks.json`         | Lists all benchmarks                                                |
-| `intro-buildoptions.json`       | Contains a full list of meson configuration options for the project |
-| `intro-buildsystem_files.json`  | Full list of all meson build files                                  |
-| `intro-dependencies.json`       | Lists all dependencies used in the project                          |
-| `intro-installed.json`          | Contains mapping of files to their installed location               |
-| `intro-projectinfo.json`        | Stores basic information about the project (name, version, etc.)    |
-| `intro-targets.json`            | Full list of all build targets                                      |
-| `intro-tests.json`              | Lists all tests with instructions how to run them                   |
+| File                           | Description                                                         |
+| ------------------------------ | ------------------------------------------------------------------- |
+| `intro-benchmarks.json`        | Lists all benchmarks                                                |
+| `intro-buildoptions.json`      | Contains a full list of meson configuration options for the project |
+| `intro-buildsystem_files.json` | Full list of all meson build files                                  |
+| `intro-dependencies.json`      | Lists all dependencies used in the project                          |
+| `intro-installed.json`         | Contains mapping of files to their installed location               |
+| `intro-projectinfo.json`       | Stores basic information about the project (name, version, etc.)    |
+| `intro-targets.json`           | Full list of all build targets                                      |
+| `intro-tests.json`             | Lists all tests with instructions how to run them                   |
 
 The content of the JSON files is further specified in the remainder of this document.
 
@@ -99,15 +99,15 @@ for actual compilation.
 
 The following table shows all valid types for a target.
 
-| value of `type`  | Description                                  |
-| ---------------  | -----------                                  |
-| `executable`     | This target will generate an executable file |
-| `static library` | Target for a static library                  |
-| `shared library` | Target for a shared library                  |
+| value of `type`  | Description                                                                                   |
+| ---------------- | --------------------------------------------------------------------------------------------- |
+| `executable`     | This target will generate an executable file                                                  |
+| `static library` | Target for a static library                                                                   |
+| `shared library` | Target for a shared library                                                                   |
 | `shared module`  | A shared library that is meant to be used with dlopen rather than linking into something else |
-| `custom`         | A custom target                              |
-| `run`            | A Meson run target                           |
-| `jar`            | A Java JAR target                            |
+| `custom`         | A custom target                                                                               |
+| `run`            | A Meson run target                                                                            |
+| `jar`            | A Java JAR target                                                                             |
 
 ### Using `--targets` without a build directory
 
@@ -275,6 +275,57 @@ command line. Use `meson introspect -h` to see all available options.
 
 This API can also work without a build directory for the `--projectinfo` command.
 
+# AST of a `meson.build`
+
+Since meson *0.55.0* it is possible to dump the AST of a `meson.build` as a JSON
+object. The interface for this is `meson introspect --ast /path/to/meson.build`.
+
+Each node of the AST has at least the following entries:
+
+| Key          | Description                                             |
+| ------------ | ------------------------------------------------------- |
+| `node`       | Type of the node (see following table)                  |
+| `lineno`     | Line number of the node in the file                     |
+| `colno`      | Column number of the node in the file                   |
+| `end_lineno` | Marks the end of the node (may be the same as `lineno`) |
+| `end_colno`  | Marks the end of the node (may be the same as `colno`)  |
+
+Possible values for `node` with additional keys:
+
+| Node type            | Additional keys                                  |
+| -------------------- | ------------------------------------------------ |
+| `BooleanNode`        | `value`: bool                                    |
+| `IdNode`             | `value`: str                                     |
+| `NumberNode`         | `value`: int                                     |
+| `StringNode`         | `value`: str                                     |
+| `ContinueNode`       |                                                  |
+| `BreakNode`          |                                                  |
+| `ArgumentNode`       | `positional`: node list; `kwargs`: accept_kwargs |
+| `ArrayNode`          | `args`: node                                     |
+| `DictNode`           | `args`: node                                     |
+| `EmptyNode`          |                                                  |
+| `OrNode`             | `left`: node; `right`: node                      |
+| `AndNode`            | `left`: node; `right`: node                      |
+| `ComparisonNode`     | `left`: node; `right`: node; `ctype`: str        |
+| `ArithmeticNode`     | `left`: node; `right`: node; `op`: str           |
+| `NotNode`            | `right`: node                                    |
+| `CodeBlockNode`      | `lines`: node list                               |
+| `IndexNode`          | `object`: node; `index`: node                    |
+| `MethodNode`         | `object`: node; `args`: node; `name`: str        |
+| `FunctionNode`       | `args`: node; `name`: str                        |
+| `AssignmentNode`     | `value`: node; `var_name`: str                   |
+| `PlusAssignmentNode` | `value`: node; `var_name`: str                   |
+| `ForeachClauseNode`  | `items`: node; `block`: node; `varnames`: list   |
+| `IfClauseNode`       | `ifs`: node list; `else`: node                   |
+| `IfNode`             | `condition`: node; `block`: node                 |
+| `UMinusNode`         | `right`: node                                    |
+| `TernaryNode`        | `condition`: node; `true`: node; `false`: node   |
+
+We do not guarantee the stability of this format since it is heavily linked to
+the internal Meson AST. However, breaking changes (removal of a node type or the
+removal of a key) are unlikely and will be announced in the release notes.
+
+
 # Existing integrations
 
 - [Gnome Builder](https://wiki.gnome.org/Apps/Builder)
diff --git a/docs/markdown/snippets/introspect.md b/docs/markdown/snippets/introspect.md
new file mode 100644
index 0000000..8eab486
--- /dev/null
+++ b/docs/markdown/snippets/introspect.md
@@ -0,0 +1,4 @@
+## Introspection API changes
+
+dumping the AST (--ast): **new in 0.55.0**
+- prints the AST of a meson.build as JSON
diff --git a/mesonbuild/ast/__init__.py b/mesonbuild/ast/__init__.py
index 48de523..4fb56cb 100644
--- a/mesonbuild/ast/__init__.py
+++ b/mesonbuild/ast/__init__.py
@@ -20,6 +20,7 @@ __all__ = [
     'AstInterpreter',
     'AstIDGenerator',
     'AstIndentationGenerator',
+    'AstJSONPrinter',
     'AstVisitor',
     'AstPrinter',
     'IntrospectionInterpreter',
@@ -30,4 +31,4 @@ from .interpreter import AstInterpreter
 from .introspection import IntrospectionInterpreter, build_target_functions
 from .visitor import AstVisitor
 from .postprocess import AstConditionLevel, AstIDGenerator, AstIndentationGenerator
-from .printer import AstPrinter
+from .printer import AstPrinter, AstJSONPrinter
diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py
index cc5c94c..6a826ef 100644
--- a/mesonbuild/ast/interpreter.py
+++ b/mesonbuild/ast/interpreter.py
@@ -297,6 +297,11 @@ class AstInterpreter(interpreterbase.InterpreterBase):
         elif isinstance(node, ElementaryNode):
             result = node.value
 
+        elif isinstance(node, NotNode):
+            result = self.resolve_node(node.value, include_unknown_args, id_loop_detect)
+            if isinstance(result, bool):
+                result = not result
+
         elif isinstance(node, ArrayNode):
             result = [x for x in node.args.arguments]
 
diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py
index 39e2cca..a57ba20 100644
--- a/mesonbuild/ast/printer.py
+++ b/mesonbuild/ast/printer.py
@@ -18,6 +18,7 @@
 from .. import mparser
 from . import AstVisitor
 import re
+import typing as T
 
 arithmic_map = {
     'add': '+',
@@ -155,7 +156,7 @@ class AstPrinter(AstVisitor):
             self.append_padded(prefix + 'if', node)
             prefix = 'el'
             i.accept(self)
-        if node.elseblock:
+        if not isinstance(node.elseblock, mparser.EmptyNode):
             self.append('else', node)
             node.elseblock.accept(self)
         self.append('endif', node)
@@ -199,3 +200,160 @@ class AstPrinter(AstVisitor):
             self.result = re.sub(r', \n$', '\n', self.result)
         else:
             self.result = re.sub(r', $', '', self.result)
+
+class AstJSONPrinter(AstVisitor):
+    def __init__(self) -> None:
+        self.result = {}  # type: T.Dict[str, T.Any]
+        self.current = self.result
+
+    def _accept(self, key: str, node: mparser.BaseNode) -> None:
+        old = self.current
+        data = {}  # type: T.Dict[str, T.Any]
+        self.current = data
+        node.accept(self)
+        self.current = old
+        self.current[key] = data
+
+    def _accept_list(self, key: str, nodes: T.Sequence[mparser.BaseNode]) -> None:
+        old = self.current
+        datalist = []  # type: T.List[T.Dict[str, T.Any]]
+        for i in nodes:
+            self.current = {}
+            i.accept(self)
+            datalist += [self.current]
+        self.current = old
+        self.current[key] = datalist
+
+    def _raw_accept(self, node: mparser.BaseNode, data: T.Dict[str, T.Any]) -> None:
+        old = self.current
+        self.current = data
+        node.accept(self)
+        self.current = old
+
+    def setbase(self, node: mparser.BaseNode) -> None:
+        self.current['node'] = type(node).__name__
+        self.current['lineno'] = node.lineno
+        self.current['colno'] = node.colno
+        self.current['end_lineno'] = node.end_lineno
+        self.current['end_colno'] = node.end_colno
+
+    def visit_default_func(self, node: mparser.BaseNode) -> None:
+        self.setbase(node)
+
+    def gen_ElementaryNode(self, node: mparser.ElementaryNode) -> None:
+        self.current['value'] = node.value
+        self.setbase(node)
+
+    def visit_BooleanNode(self, node: mparser.BooleanNode) -> None:
+        self.gen_ElementaryNode(node)
+
+    def visit_IdNode(self, node: mparser.IdNode) -> None:
+        self.gen_ElementaryNode(node)
+
+    def visit_NumberNode(self, node: mparser.NumberNode) -> None:
+        self.gen_ElementaryNode(node)
+
+    def visit_StringNode(self, node: mparser.StringNode) -> None:
+        self.gen_ElementaryNode(node)
+
+    def visit_ArrayNode(self, node: mparser.ArrayNode) -> None:
+        self._accept('args', node.args)
+        self.setbase(node)
+
+    def visit_DictNode(self, node: mparser.DictNode) -> None:
+        self._accept('args', node.args)
+        self.setbase(node)
+
+    def visit_OrNode(self, node: mparser.OrNode) -> None:
+        self._accept('left', node.left)
+        self._accept('right', node.right)
+        self.setbase(node)
+
+    def visit_AndNode(self, node: mparser.AndNode) -> None:
+        self._accept('left', node.left)
+        self._accept('right', node.right)
+        self.setbase(node)
+
+    def visit_ComparisonNode(self, node: mparser.ComparisonNode) -> None:
+        self._accept('left', node.left)
+        self._accept('right', node.right)
+        self.current['ctype'] = node.ctype
+        self.setbase(node)
+
+    def visit_ArithmeticNode(self, node: mparser.ArithmeticNode) -> None:
+        self._accept('left', node.left)
+        self._accept('right', node.right)
+        self.current['op'] = arithmic_map[node.operation]
+        self.setbase(node)
+
+    def visit_NotNode(self, node: mparser.NotNode) -> None:
+        self._accept('right', node.value)
+        self.setbase(node)
+
+    def visit_CodeBlockNode(self, node: mparser.CodeBlockNode) -> None:
+        self._accept_list('lines', node.lines)
+        self.setbase(node)
+
+    def visit_IndexNode(self, node: mparser.IndexNode) -> None:
+        self._accept('object', node.iobject)
+        self._accept('index', node.index)
+        self.setbase(node)
+
+    def visit_MethodNode(self, node: mparser.MethodNode) -> None:
+        self._accept('object', node.source_object)
+        self._accept('args', node.args)
+        self.current['name'] = node.name
+        self.setbase(node)
+
+    def visit_FunctionNode(self, node: mparser.FunctionNode) -> None:
+        self._accept('args', node.args)
+        self.current['name'] = node.func_name
+        self.setbase(node)
+
+    def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None:
+        self._accept('value', node.value)
+        self.current['var_name'] = node.var_name
+        self.setbase(node)
+
+    def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None:
+        self._accept('value', node.value)
+        self.current['var_name'] = node.var_name
+        self.setbase(node)
+
+    def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None:
+        self._accept('items', node.items)
+        self._accept('block', node.block)
+        self.current['varnames'] = node.varnames
+        self.setbase(node)
+
+    def visit_IfClauseNode(self, node: mparser.IfClauseNode) -> None:
+        self._accept_list('ifs', node.ifs)
+        self._accept('else', node.elseblock)
+        self.setbase(node)
+
+    def visit_UMinusNode(self, node: mparser.UMinusNode) -> None:
+        self._accept('right', node.value)
+        self.setbase(node)
+
+    def visit_IfNode(self, node: mparser.IfNode) -> None:
+        self._accept('condition', node.condition)
+        self._accept('block', node.block)
+        self.setbase(node)
+
+    def visit_TernaryNode(self, node: mparser.TernaryNode) -> None:
+        self._accept('condition', node.condition)
+        self._accept('true', node.trueblock)
+        self._accept('false', node.falseblock)
+        self.setbase(node)
+
+    def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None:
+        self._accept_list('positional', node.arguments)
+        kwargs_list = []  # type: T.List[T.Dict[str, T.Dict[str, T.Any]]]
+        for key, val in node.kwargs.items():
+            key_res = {}  # type: T.Dict[str, T.Any]
+            val_res = {}  # type: T.Dict[str, T.Any]
+            self._raw_accept(key, key_res)
+            self._raw_accept(val, val_res)
+            kwargs_list += [{'key': key_res, 'val': val_res}]
+        self.current['kwargs'] = kwargs_list
+        self.setbase(node)
diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py
index 37be463..451020d 100644
--- a/mesonbuild/ast/visitor.py
+++ b/mesonbuild/ast/visitor.py
@@ -113,8 +113,7 @@ class AstVisitor:
         self.visit_default_func(node)
         for i in node.ifs:
             i.accept(self)
-        if node.elseblock:
-            node.elseblock.accept(self)
+        node.elseblock.accept(self)
 
     def visit_UMinusNode(self, node: mparser.UMinusNode) -> None:
         self.visit_default_func(node)
diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py
index 54e302b..8eb659b 100644
--- a/mesonbuild/mintro.py
+++ b/mesonbuild/mintro.py
@@ -22,7 +22,7 @@ project files and don't need this info."""
 import json
 from . import build, coredata as cdata
 from . import mesonlib
-from .ast import IntrospectionInterpreter, build_target_functions, AstConditionLevel, AstIDGenerator, AstIndentationGenerator
+from .ast import IntrospectionInterpreter, build_target_functions, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstJSONPrinter
 from . import mlog
 from .backend import backends
 from .mparser import BaseNode, FunctionNode, ArrayNode, ArgumentNode, StringNode
@@ -62,6 +62,7 @@ def get_meson_introspection_types(coredata: T.Optional[cdata.CoreData] = None,
         benchmarkdata = testdata = installdata = None
 
     return {
+        'ast': IntroCommand('Dump the AST of the meson file', no_bd=dump_ast),
         'benchmarks': IntroCommand('List all benchmarks', func=lambda: list_benchmarks(benchmarkdata)),
         'buildoptions': IntroCommand('List all build options', func=lambda: list_buildoptions(coredata), no_bd=list_buildoptions_from_source),
         'buildsystem_files': IntroCommand('List files that make up the build system', func=lambda: list_buildsystem_files(builddata, interpreter)),
@@ -89,6 +90,11 @@ def add_arguments(parser):
                         help='Always use the new JSON format for multiple entries (even for 0 and 1 introspection commands)')
     parser.add_argument('builddir', nargs='?', default='.', help='The build directory')
 
+def dump_ast(intr: IntrospectionInterpreter) -> T.Dict[str, T.Any]:
+    printer = AstJSONPrinter()
+    intr.ast.accept(printer)
+    return printer.result
+
 def list_installed(installdata):
     res = {}
     if installdata is not None:
diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py
index 2cffc47..b9e381e 100644
--- a/mesonbuild/mparser.py
+++ b/mesonbuild/mparser.py
@@ -426,8 +426,8 @@ class IfNode(BaseNode):
 class IfClauseNode(BaseNode):
     def __init__(self, linenode: BaseNode):
         super().__init__(linenode.lineno, linenode.colno, linenode.filename)
-        self.ifs = []  # type: T.List[IfNode]
-        self.elseblock = EmptyNode(linenode.lineno, linenode.colno, linenode.filename)  # type: T.Union[EmptyNode, CodeBlockNode]
+        self.ifs = []          # type: T.List[IfNode]
+        self.elseblock = None  # type: T.Union[EmptyNode, CodeBlockNode]
 
 class UMinusNode(BaseNode):
     def __init__(self, current_location: Token, value: BaseNode):
@@ -747,9 +747,7 @@ class Parser:
         block = self.codeblock()
         clause.ifs.append(IfNode(clause, condition, block))
         self.elseifblock(clause)
-        elseblock = self.elseblock()
-        if elseblock:
-            clause.elseblock = elseblock
+        clause.elseblock = self.elseblock()
         return clause
 
     def elseifblock(self, clause) -> None:
@@ -759,11 +757,11 @@ class Parser:
             b = self.codeblock()
             clause.ifs.append(IfNode(s, s, b))
 
-    def elseblock(self) -> T.Optional[CodeBlockNode]:
+    def elseblock(self) -> T.Union[CodeBlockNode, EmptyNode]:
         if self.accept('else'):
             self.expect('eol')
             return self.codeblock()
-        return None
+        return EmptyNode(self.current.lineno, self.current.colno, self.current.filename)
 
     def line(self) -> BaseNode:
         block_start = self.current
diff --git a/run_unittests.py b/run_unittests.py
index 651e366..7e0c403 100755
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -4400,6 +4400,83 @@ recommended as it is not supported on some platforms''')
         self.maxDiff = None
         self.assertListEqual(res_nb, res_wb)
 
+    def test_introspect_ast_source(self):
+        testdir = os.path.join(self.unit_test_dir, '57 introspection')
+        testfile = os.path.join(testdir, 'meson.build')
+        res_nb = self.introspect_directory(testfile, ['--ast'] + self.meson_args)
+
+        node_counter = {}
+
+        def accept_node(json_node):
+            self.assertIsInstance(json_node, dict)
+            for i in ['lineno', 'colno', 'end_lineno', 'end_colno']:
+                self.assertIn(i, json_node)
+                self.assertIsInstance(json_node[i], int)
+            self.assertIn('node', json_node)
+            n = json_node['node']
+            self.assertIsInstance(n, str)
+            self.assertIn(n, nodes)
+            if n not in node_counter:
+                node_counter[n] = 0
+            node_counter[n] = node_counter[n] + 1
+            for nodeDesc in nodes[n]:
+                key = nodeDesc[0]
+                func = nodeDesc[1]
+                self.assertIn(key, json_node)
+                if func is None:
+                    tp = nodeDesc[2]
+                    self.assertIsInstance(json_node[key], tp)
+                    continue
+                func(json_node[key])
+
+        def accept_node_list(node_list):
+            self.assertIsInstance(node_list, list)
+            for i in node_list:
+                accept_node(i)
+
+        def accept_kwargs(kwargs):
+            self.assertIsInstance(kwargs, list)
+            for i in kwargs:
+                self.assertIn('key', i)
+                self.assertIn('val', i)
+                accept_node(i['key'])
+                accept_node(i['val'])
+
+        nodes = {
+            'BooleanNode': [('value', None, bool)],
+            'IdNode': [('value', None, str)],
+            'NumberNode': [('value', None, int)],
+            'StringNode': [('value', None, str)],
+            'ContinueNode': [],
+            'BreakNode': [],
+            'ArgumentNode': [('positional', accept_node_list), ('kwargs', accept_kwargs)],
+            'ArrayNode': [('args', accept_node)],
+            'DictNode': [('args', accept_node)],
+            'EmptyNode': [],
+            'OrNode': [('left', accept_node), ('right', accept_node)],
+            'AndNode': [('left', accept_node), ('right', accept_node)],
+            'ComparisonNode': [('left', accept_node), ('right', accept_node), ('ctype', None, str)],
+            'ArithmeticNode': [('left', accept_node), ('right', accept_node), ('op', None, str)],
+            'NotNode': [('right', accept_node)],
+            'CodeBlockNode': [('lines', accept_node_list)],
+            'IndexNode': [('object', accept_node), ('index', accept_node)],
+            'MethodNode': [('object', accept_node), ('args', accept_node), ('name', None, str)],
+            'FunctionNode': [('args', accept_node), ('name', None, str)],
+            'AssignmentNode': [('value', accept_node), ('var_name', None, str)],
+            'PlusAssignmentNode': [('value', accept_node), ('var_name', None, str)],
+            'ForeachClauseNode': [('items', accept_node), ('block', accept_node), ('varnames', None, list)],
+            'IfClauseNode': [('ifs', accept_node_list), ('else', accept_node)],
+            'IfNode': [('condition', accept_node), ('block', accept_node)],
+            'UMinusNode': [('right', accept_node)],
+            'TernaryNode': [('condition', accept_node), ('true', accept_node), ('false', accept_node)],
+        }
+
+        accept_node(res_nb)
+
+        for n, c in [('ContinueNode', 2), ('BreakNode', 1), ('NotNode', 3)]:
+            self.assertIn(n, node_counter)
+            self.assertEqual(node_counter[n], c)
+
     def test_introspect_dependencies_from_source(self):
         testdir = os.path.join(self.unit_test_dir, '57 introspection')
         testfile = os.path.join(testdir, 'meson.build')
diff --git a/test cases/unit/57 introspection/meson.build b/test cases/unit/57 introspection/meson.build
index 9716eae..5d4dd02 100644
--- a/test cases/unit/57 introspection/meson.build
+++ b/test cases/unit/57 introspection/meson.build
@@ -13,7 +13,7 @@ test_bool = not test_bool
 set_variable('list_test_plusassign', [])
 list_test_plusassign += ['bugs everywhere']
 
-if false
+if not true
   vers_str = '<=99.9.9'
   dependency('somethingthatdoesnotexist', required: true, version: '>=1.2.3')
   dependency('look_i_have_a_fallback', version: ['>=1.0.0', vers_str], fallback: ['oh_no', 'the_subproject_does_not_exist'])
@@ -26,7 +26,7 @@ var1 = '1'
 var2 = 2.to_string()
 var3 = 'test3'
 
-t1 = executable('test' + var1, ['t1.cpp'], link_with: [sharedlib], install: true, build_by_default: get_option('test_opt2'))
+t1 = executable('test' + var1, ['t1.cpp'], link_with: [sharedlib], install: not false, build_by_default: get_option('test_opt2'))
 t2 = executable('test@0@'.format('@0@'.format(var2)), sources: ['t2.cpp'], link_with: [staticlib])
 t3 = executable(var3, 't3.cpp', link_with: [sharedlib, staticlib], dependencies: [dep1])
 
@@ -46,3 +46,16 @@ message(osmesa_lib_name) # Infinite recursion gets triggered here when the param
 test('test case 1', t1)
 test('test case 2', t2)
 benchmark('benchmark 1', t3)
+
+### Stuff to test the AST JSON printer
+foreach x : ['a', 'b', 'c']
+  if x == 'a'
+    message('a')
+  elif x == 'b'
+    message('a')
+  else
+    continue
+  endif
+  break
+  continue
+endforeach