Merge pull request #9377 from mensinda/jsonDocs

docs: Add a JSON documentation backend

6 files changed, 249 insertions, 9 deletions

diff --git a/docs/refman/generatorjson.py b/docs/refman/generatorjson.py
new file mode 100644
index 0000000..f5164d4
--- /dev/null
+++ b/docs/refman/generatorjson.py
@@ -0,0 +1,120 @@
+# SPDX-License-Identifer: Apache-2.0
+# Copyright 2021 The Meson development team
+
+from pathlib import Path
+import json
+import re
+
+from .generatorbase import GeneratorBase
+from . import jsonschema as J
+from .model import (
+    ReferenceManual,
+    Function,
+    Object,
+    Type,
+
+    PosArg,
+    VarArgs,
+    Kwarg,
+)
+
+import typing as T
+
+class GeneratorJSON(GeneratorBase):
+    def __init__(self, manual: ReferenceManual, out: Path, enable_modules: bool) -> None:
+        super().__init__(manual)
+        self.out = out
+        self.enable_modules = enable_modules
+
+    def _generate_type(self, typ: Type) -> T.List[J.Type]:
+        return [
+            {
+                'obj': x.data_type.name,
+                'holds': self._generate_type(x.holds) if x.holds else [],
+            }
+            for x in typ.resolved
+        ]
+
+    def _generate_type_str(self, typ: Type) -> str:
+        # Remove all whitespaces
+        return re.sub(r'[ \n\r\t]', '', typ.raw)
+
+    def _generate_arg(self, arg: T.Union[PosArg, VarArgs, Kwarg], isOptarg: bool = False) -> J.Argument:
+        return {
+            'name': arg.name,
+            'description': arg.description,
+            'since': arg.since if arg.since else None,
+            'deprecated': arg.deprecated if arg.deprecated else None,
+            'type': self._generate_type(arg.type),
+            'type_str': self._generate_type_str(arg.type),
+            'required': arg.required if isinstance(arg, Kwarg) else not isOptarg and not isinstance(arg, VarArgs),
+            'default': arg.default if isinstance(arg, (PosArg, Kwarg)) else None,
+            'min_varargs': arg.min_varargs if isinstance(arg, VarArgs) and arg.min_varargs > 0 else None,
+            'max_varargs': arg.max_varargs if isinstance(arg, VarArgs) and arg.max_varargs > 0 else None,
+
+            # Not yet supported
+            'notes': [],
+            'warnings': [],
+        }
+
+    def _generate_function(self, func: Function) -> J.Function:
+        return {
+            'name': func.name,
+            'description': func.description,
+            'since': func.since if func.since else None,
+            'deprecated': func.deprecated if func.deprecated else None,
+            'notes': func.notes,
+            'warnings': func.warnings,
+            'example': func.example if func.example else None,
+            'returns': self._generate_type(func.returns),
+            'returns_str': self._generate_type_str(func.returns),
+            'posargs': {x.name: self._generate_arg(x) for x in func.posargs},
+            'optargs': {x.name: self._generate_arg(x, True) for x in func.optargs},
+            'kwargs': {x.name: self._generate_arg(x) for x in self.sorted_and_filtered(list(func.kwargs.values()))},
+            'varargs': self._generate_arg(func.varargs) if func.varargs else None,
+        }
+
+    def _generate_objects(self, obj: Object) -> J.Object:
+        return {
+            'name': obj.name,
+            'description': obj.description,
+            'since': obj.since if obj.since else None,
+            'deprecated': obj.deprecated if obj.deprecated else None,
+            'notes': obj.notes,
+            'warnings': obj.warnings,
+            'defined_by_module': obj.defined_by_module.name if obj.defined_by_module else None,
+            'object_type': obj.obj_type.name,
+            'is_container': obj.is_container,
+            'example': obj.example if obj.example else None,
+            'extends': obj.extends if obj.extends else None,
+            'returned_by': [x.name for x in self.sorted_and_filtered(obj.returned_by)],
+            'extended_by': [x.name for x in self.sorted_and_filtered(obj.extended_by)],
+            'methods': {x.name: self._generate_function(x) for x in self.sorted_and_filtered(obj.methods)},
+        }
+
+    def _extract_meson_version(self) -> str:
+        # Hack around python relative imports to get to the Meson version
+        import sys
+        sys.path.append(Path(__file__).resolve().parents[2].as_posix())
+        from mesonbuild.coredata import version
+        return version
+
+    def generate(self) -> None:
+        data: J.Root = {
+            'version_major': J.VERSION_MAJOR,
+            'version_minor': J.VERSION_MINOR,
+            'meson_version': self._extract_meson_version(),
+            'functions': {x.name: self._generate_function(x) for x in self.sorted_and_filtered(self.functions)},
+            'objects': {x.name: self._generate_objects(x) for x in self.sorted_and_filtered(self.objects)},
+            'objects_by_type': {
+                'elementary': [x.name for x in self.elementary],
+                'builtins': [x.name for x in self.builtins],
+                'returned': [x.name for x in self.returned],
+                'modules': {
+                    x.name: [y.name for y in self.sorted_and_filtered(self.extract_returned_by_module(x))]
+                    for x in self.modules
+                },
+            },
+        }
+
+        self.out.write_text(json.dumps(data), encoding='utf-8')
diff --git a/docs/refman/jsonschema.py b/docs/refman/jsonschema.py
new file mode 100644
index 0000000..6d6f98a
--- /dev/null
+++ b/docs/refman/jsonschema.py
@@ -0,0 +1,87 @@
+# SPDX-License-Identifer: Apache-2.0
+# Copyright 2021 The Meson development team
+
+import typing as T
+
+# The following variables define the current version of
+# the JSON documentation format. This is different from
+# the Meson version
+
+VERSION_MAJOR = 1  # Changes here indicate breaking format changes (changes to existing keys)
+VERSION_MINOR = 0  # Changes here indicate non-breaking changes (only new keys are added to the existing structure)
+
+class BaseObject(T.TypedDict):
+    '''
+        Base object for most dicts in the JSON doc.
+
+        All objects inheriting from BaseObject will support
+        the keys specified here:
+    '''
+    name:        str
+    description: str
+    since:       T.Optional[str]
+    deprecated:  T.Optional[str]
+    notes:       T.List[str]
+    warnings:    T.List[str]
+
+class Type(T.TypedDict):
+    obj:   str                 # References an object from `root.objects`
+    holds: T.Sequence[object]  # Mypy does not support recusive dicts, but this should be T.List[Type]...
+
+class Argument(BaseObject):
+    '''
+        Object that represents any type of a single function or method argumet.
+    '''
+    type:        T.List[Type]  # A non-empty list of types that are supported.
+    type_str:    str           # Formated version of `type`. Is guranteed to not contain any whitespaces.
+    required:    bool
+    default:     T.Optional[str]
+    min_varargs: T.Optional[int]  # Only relevant for varargs, must be `null` for all other types of arguments
+    max_varargs: T.Optional[int]  # Only relevant for varargs, must be `null` for all other types of arguments
+
+class Function(BaseObject):
+    '''
+        Represents a function or method.
+    '''
+    returns:     T.List[Type]  # A non-empty list of types that are supported.
+    returns_str: str           # Formated version of `returns`. Is guranteed to not contain any whitespaces.
+    example:     T.Optional[str]
+    posargs:     T.Dict[str, Argument]
+    optargs:     T.Dict[str, Argument]
+    kwargs:      T.Dict[str, Argument]
+    varargs:     T.Optional[Argument]
+
+class Object(BaseObject):
+    '''
+        Represents all types of Meson objects. The specific object type is stored in the `object_type` field.
+    '''
+    example:           T.Optional[str]
+    object_type:       str                    # Defines the object type: Must be one of: ELEMENTARY, BUILTIN, MODULE, RETURNED
+    methods:           T.Dict[str, Function]
+    is_container:      bool
+    extends:           T.Optional[str]
+    returned_by:       T.List[str]
+    extended_by:       T.List[str]
+    defined_by_module: T.Optional[str]
+
+class ObjectsByType(T.TypedDict):
+    '''
+        References to other objects are stored here for ease of navigation / filtering
+    '''
+    elementary: T.List[str]
+    builtins:   T.List[str]
+    returned:   T.List[str]
+    modules:    T.Dict[str, T.List[str]]
+
+
+
+class Root(T.TypedDict):
+    '''
+        The root object of the JSON reference manual
+    '''
+    version_major:   int # See the description above for
+    version_minor:   int # VERSION_MAJOR and VERSION_MINOR
+    meson_version:   str
+    functions:       T.Dict[str, Function]  # A mapping of <name> to a `Function` object for *all* Meson functions
+    objects:         T.Dict[str, Object]    # A mapping of <name> to a `Object`   object for *all* Meson objects (including modules, elementary, etc.)
+    objects_by_type: ObjectsByType
diff --git a/docs/refman/loaderpickle.py b/docs/refman/loaderpickle.py
new file mode 100644
index 0000000..722fedf
--- /dev/null
+++ b/docs/refman/loaderpickle.py
@@ -0,0 +1,22 @@
+# SPDX-License-Identifer: Apache-2.0
+# Copyright 2021 The Meson development team
+
+from pathlib import Path
+import pickle
+
+from .loaderbase import LoaderBase
+from .model import ReferenceManual
+
+class LoaderPickle(LoaderBase):
+    def __init__(self, in_file: Path) -> None:
+        super().__init__()
+        self.in_file = in_file
+
+    def load_impl(self) -> ReferenceManual:
+        res = pickle.loads(self.in_file.read_bytes())
+        assert isinstance(res, ReferenceManual)
+        return res
+
+    # Assume that the pickled data is OK and skip validation
+    def load(self) -> ReferenceManual:
+        return self.load_impl()
diff --git a/docs/refman/main.py b/docs/refman/main.py
index 913a0f3..5bc40f7 100644
--- a/docs/refman/main.py
+++ b/docs/refman/main.py
@@ -19,9 +19,11 @@ import typing as T
 from mesonbuild import mlog
 
 from .loaderbase import LoaderBase
+from .loaderpickle import LoaderPickle
 from .loaderyaml import LoaderYAML
 
 from .generatorbase import GeneratorBase
+from .generatorjson import GeneratorJSON
 from .generatorprint import GeneratorPrint
 from .generatorpickle import GeneratorPickle
 from .generatormd import GeneratorMD
@@ -30,10 +32,11 @@ meson_root = Path(__file__).absolute().parents[2]
 
 def main() -> int:
     parser = argparse.ArgumentParser(description='Meson reference manual generator')
-    parser.add_argument('-l', '--loader', type=str, default='yaml', choices=['yaml'], help='Information loader backend')
-    parser.add_argument('-g', '--generator', type=str, choices=['print', 'pickle', 'md'], required=True, help='Generator backend')
+    parser.add_argument('-l', '--loader', type=str, default='yaml', choices=['yaml', 'pickle'], help='Information loader backend')
+    parser.add_argument('-g', '--generator', type=str, choices=['print', 'pickle', 'md', 'json'], required=True, help='Generator backend')
     parser.add_argument('-s', '--sitemap', type=Path, default=meson_root / 'docs' / 'sitemap.txt', help='Path to the input sitemap.txt')
     parser.add_argument('-o', '--out', type=Path, required=True, help='Output directory for generated files')
+    parser.add_argument('-i', '--input', type=Path, default=meson_root / 'docs' / 'yaml', help='Input path for the selected loader')
     parser.add_argument('--link-defs', type=Path, help='Output file for the MD generator link definition file')
     parser.add_argument('--depfile', type=Path, default=None, help='Set to generate a depfile')
     parser.add_argument('--force-color', action='store_true', help='Force enable colors')
@@ -44,7 +47,8 @@ def main() -> int:
         mlog.colorize_console = lambda: True
 
     loaders: T.Dict[str, T.Callable[[], LoaderBase]] = {
-        'yaml': lambda: LoaderYAML(meson_root / 'docs' / 'yaml'),
+        'yaml': lambda: LoaderYAML(args.input),
+        'pickle': lambda: LoaderPickle(args.input),
     }
 
     loader = loaders[args.loader]()
@@ -54,6 +58,7 @@ def main() -> int:
         'print': lambda: GeneratorPrint(refMan),
         'pickle': lambda: GeneratorPickle(refMan, args.out),
         'md': lambda: GeneratorMD(refMan, args.out, args.sitemap, args.link_defs, not args.no_modules),
+        'json': lambda: GeneratorJSON(refMan, args.out, not args.no_modules),
     }
     generator = generators[args.generator]()
 
diff --git a/docs/refman/model.py b/docs/refman/model.py
index 23515a2..8a051c9 100644
--- a/docs/refman/model.py
+++ b/docs/refman/model.py
@@ -44,20 +44,20 @@ class Type:
 
 # Arguments
 @dataclass
-class ArgBase(NamedObject):
+class ArgBase(NamedObject, FetureCheck):
     type: Type
 
 @dataclass
-class PosArg(ArgBase, FetureCheck):
+class PosArg(ArgBase):
     default: str
 
 @dataclass
-class VarArgs(ArgBase, FetureCheck):
+class VarArgs(ArgBase):
     min_varargs: int
     max_varargs: int
 
 @dataclass
-class Kwarg(ArgBase, FetureCheck):
+class Kwarg(ArgBase):
     required: bool
     default: str
 
diff --git a/docs/refman/templates/root.mustache b/docs/refman/templates/root.mustache
index 3752a22..c793b04 100644
--- a/docs/refman/templates/root.mustache
+++ b/docs/refman/templates/root.mustache
@@ -5,8 +5,14 @@ render-subpages: false
 
 # Reference manual
 
-This is the root page of the Meson reference manual. All functions
-and methods are documented in detail in the following subpages:
+This is the root page of the online Meson reference manual. This
+manual is also available in a more machine readable format as a
+JSON documented attached to every release since *0.60.0*. See our
+[IDE integration](IDE-integration.md) documentation for more
+information.
+
+All functions and methods are documented in detail in the
+following subpages:
 
 ## Elementary types