# Copyright 2021 The Meson development team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from .generatorbase import GeneratorBase import re import json from .model import ( ReferenceManual, Function, Method, Object, ObjectType, Type, DataTypeInfo, ArgBase, PosArg, VarArgs, Kwarg, ) from pathlib import Path from textwrap import dedent import typing as T from mesonbuild import mlog PlaceholderTypes = T.Union[None, str, bool] FunctionDictType = T.Dict[ str, T.Union[ PlaceholderTypes, T.Dict[str, PlaceholderTypes], T.Dict[str, T.Dict[str, PlaceholderTypes]], T.Dict[str, T.List[T.Dict[str, PlaceholderTypes]]], T.List[T.Dict[str, PlaceholderTypes]], T.List[str], ] ] _ROOT_BASENAME = 'Reference-manual' _OBJ_ID_MAP = { ObjectType.ELEMENTARY: 'elementary', ObjectType.BUILTIN: 'builtin', ObjectType.MODULE: 'module', ObjectType.RETURNED: 'returned', } # Indent all but the first line with 4*depth spaces. # This function is designed to be used with `dedent` # and fstrings where multiline strings are used during # the string interpolation. def smart_indent(raw: str, depth: int = 3) -> str: lines = raw.split('\n') first_line = lines[0] lines = [' ' * (4 * depth) + x for x in lines] lines[0] = first_line # Do not indent the first line return '\n'.join(lines) def code_block(code: str) -> str: code = dedent(code) return f'
{code}
'
class GeneratorMD(GeneratorBase):
def __init__(self, manual: ReferenceManual, sitemap_out: Path, sitemap_in: Path, link_def_out: Path, enable_modules: bool) -> None:
super().__init__(manual)
self.sitemap_out = sitemap_out.resolve()
self.sitemap_in = sitemap_in.resolve()
self.link_def_out = link_def_out.resolve()
self.out_dir = self.sitemap_out.parent
self.enable_modules = enable_modules
self.generated_files: T.Dict[str, str] = {}
# Utility functions
def _gen_filename(self, file_id: str, *, extension: str = 'md') -> str:
parts = file_id.split('.')
assert parts[0] == 'root'
assert all([x for x in parts])
parts[0] = _ROOT_BASENAME
parts = [re.sub(r'[0-9]+_', '', x) for x in parts]
return f'{"_".join(parts)}.{extension}'
def _gen_object_file_id(self, obj: Object) -> str:
'''
Deterministically generate a unique file ID for the Object.
This ID determines where the object will be inserted in the sitemap.
'''
if obj.obj_type == ObjectType.RETURNED and obj.defined_by_module is not None:
base = self._gen_object_file_id(obj.defined_by_module)
return f'{base}.{obj.name}'
return f'root.{_OBJ_ID_MAP[obj.obj_type]}.{obj.name}'
def _link_to_object(self, obj: T.Union[Function, Object], in_code_block: bool = False) -> str:
'''
Generate a palaceholder tag for the function/method/object documentation.
This tag is then replaced in the custom hotdoc plugin.
'''
prefix = '#' if in_code_block else ''
if isinstance(obj, Object):
return f'[[{prefix}@{obj.name}]]'
elif isinstance(obj, Method):
return f'[[{prefix}{obj.obj.name}.{obj.name}]]'
elif isinstance(obj, Function):
return f'[[{prefix}{obj.name}]]'
else:
raise RuntimeError(f'Invalid argument {obj}')
def _write_file(self, data: str, file_id: str) -> None:#
''' Write the data to disk and store the id for the generated data '''
self.generated_files[file_id] = self._gen_filename(file_id)
out_file = self.out_dir / self.generated_files[file_id]
out_file.write_text(data, encoding='ascii')
mlog.log('Generated', mlog.bold(out_file.name))
def _write_template(self, data: T.Dict[str, T.Any], file_id: str, template_name: T.Optional[str] = None) -> None:
''' Render the template mustache files and write the result '''
template_dir = Path(__file__).resolve().parent / 'templates'
template_name = template_name or file_id
template_name = f'{template_name}.mustache'
template_file = template_dir / template_name
# Import here, so that other generators don't also depend on it
import chevron
result = chevron.render(
template=template_file.read_text(encoding='utf-8'),
data=data,
partials_path=template_dir.as_posix(),
warn=True,
)
self._write_file(result, file_id)
# Actual generator functions
def _gen_func_or_method(self, func: Function) -> FunctionDictType:
def render_type(typ: Type, in_code_block: bool = False) -> str:
def data_type_to_str(dt: DataTypeInfo) -> str:
base = self._link_to_object(dt.data_type, in_code_block)
if dt.holds:
return f'{base}[{render_type(dt.holds, in_code_block)}]'
return base
assert typ.resolved
return ' | '.join([data_type_to_str(x) for x in typ.resolved])
def len_stripped(s: str) -> int:
s = s.replace(']]', '')
# I know, this regex is ugly but it works.
return len(re.sub(r'\[\[(#|@)*([^\[])', r'\2', s))
def render_signature() -> str:
# Skip a lot of computations if the function does not take any arguments
if not any([func.posargs, func.optargs, func.kwargs, func.varargs]):
return f'{render_type(func.returns, True)} {func.name}()'
signature = dedent(f'''\
# {self.brief(func)}
{render_type(func.returns, True)} {func.name}(
''')
# Calculate maximum lengths of the type and name
all_args: T.List[ArgBase] = []
all_args += func.posargs
all_args += func.optargs
all_args += [func.varargs] if func.varargs else []
max_type_len = 0
max_name_len = 0
if all_args:
max_type_len = max([len_stripped(render_type(x.type)) for x in all_args])
max_name_len = max([len(x.name) for x in all_args])
# Generate some common strings
def prepare(arg: ArgBase) -> T.Tuple[str, str, str, str]:
type_str = render_type(arg.type, True)
type_len = len_stripped(type_str)
type_space = ' ' * (max_type_len - type_len)
name_space = ' ' * (max_name_len - len(arg.name))
name_str = f'{arg.name.replace("<", "<").replace(">", ">")}'
return type_str, type_space, name_str, name_space
for i in func.posargs:
type_str, type_space, name_str, name_space = prepare(i)
signature += f' {type_str}{type_space} {name_str},{name_space} # {self.brief(i)}\n'
for i in func.optargs:
type_str, type_space, name_str, name_space = prepare(i)
signature += f' {type_str}{type_space} [{name_str}],{name_space} # {self.brief(i)}\n'
if func.varargs:
type_str, type_space, name_str, name_space = prepare(func.varargs)
signature += f' {type_str}{type_space} {name_str}...,{name_space} # {self.brief(func.varargs)}\n'
# Abort if there are no kwargs
if not func.kwargs:
return signature + ')'
# Only add this separator if there are any posargs
if all_args:
signature += '\n # Keyword arguments:\n'
# Recalculate lengths for kwargs
all_args = list(func.kwargs.values())
max_type_len = max([len_stripped(render_type(x.type)) for x in all_args])
max_name_len = max([len(x.name) for x in all_args])
for kwarg in self.sorted_and_filtered(list(func.kwargs.values())):
type_str, type_space, name_str, name_space = prepare(kwarg)
required = ' [required] ' if kwarg.required else ' '
required = required if any([x.required for x in func.kwargs.values()]) else ''
signature += f' {name_str}{name_space} : {type_str}{type_space} {required} # {self.brief(kwarg)}\n'
return signature + ')'
def gen_arg_data(arg: T.Union[PosArg, Kwarg, VarArgs], *, optional: bool = False) -> T.Dict[str, PlaceholderTypes]:
data: T.Dict[str, PlaceholderTypes] = {
'name': arg.name,
'type': render_type(arg.type),
'description': arg.description,
'since': arg.since or None,
'deprecated': arg.deprecated or None,
'optional': optional,
'default': None,
}
if isinstance(arg, VarArgs):
data.update({
'min': str(arg.min_varargs) if arg.min_varargs > 0 else '0',
'max': str(arg.max_varargs) if arg.max_varargs > 0 else 'infinity',
})
if isinstance(arg, (Kwarg, PosArg)):
data.update({'default': arg.default or None})
if isinstance(arg, Kwarg):
data.update({'required': arg.required})
return data
mname = f'\\{func.name}' if func.name == '[index]' else func.name
data: FunctionDictType = {
'name': f'{func.obj.name}.{mname}' if isinstance(func, Method) else func.name,
'base_level': '##' if isinstance(func, Method) else '#',
'type_name_upper': 'Method' if isinstance(func, Method) else 'Function',
'type_name': 'method' if isinstance(func, Method) else 'function',
'description': func.description,
'notes': func.notes,
'warnings': func.warnings,
'example': func.example or None,
'signature_level': 'h4' if isinstance(func, Method) else 'h3',
'signature': render_signature(),
'has_args': bool(func.posargs or func.optargs or func.kwargs or func.varargs),
# Merge posargs and optargs by generating the *[optional]* tag for optargs
'posargs': {
'args': [gen_arg_data(x) for x in func.posargs] + [gen_arg_data(x, optional=True) for x in func.optargs]
} if func.posargs or func.optargs else None,
'kwargs': {'args': [gen_arg_data(x) for x in self.sorted_and_filtered(list(func.kwargs.values()))]} if func.kwargs else None,
'varargs': gen_arg_data(func.varargs) if func.varargs else None,
'arg_flattening': func.arg_flattening,
# For the feature taggs template
'since': func.since or None,
'deprecated': func.deprecated or None,
'optional': False,
'default': None
}
return data
def _write_object(self, obj: Object) -> None:
data = {
'name': obj.name,
'title': obj.long_name if obj.obj_type == ObjectType.RETURNED else obj.name,
'description': obj.description,
'notes': obj.notes,
'warnings': obj.warnings,
'long_name': obj.long_name,
'obj_type_name': _OBJ_ID_MAP[obj.obj_type].capitalize(),
'example': obj.example or None,
'has_methods': bool(obj.methods),
'has_inherited_methods': bool(obj.inherited_methods),
'has_subclasses': bool(obj.extended_by),
'is_returned': bool(obj.returned_by),
'extends': obj.extends_obj.name if obj.extends_obj else None,
'returned_by': [self._link_to_object(x) for x in self.sorted_and_filtered(obj.returned_by)],
'extended_by': [self._link_to_object(x) for x in self.sorted_and_filtered(obj.extended_by)],
'methods': [self._gen_func_or_method(m) for m in self.sorted_and_filtered(obj.methods)],
'inherited_methods': [self._gen_func_or_method(m) for m in self.sorted_and_filtered(obj.inherited_methods)],
}
self._write_template(data, self._gen_object_file_id(obj), 'object')
def _write_functions(self) -> None:
data = {'functions': [self._gen_func_or_method(x) for x in self.functions]}
self._write_template(data, 'root.functions')
def _root_refman_docs(self) -> None:
def gen_obj_links(objs: T.List[Object]) -> T.List[T.Dict[str, str]]:
ret: T.List[T.Dict[str, str]] = []
for o in objs:
ret += [{'indent': '', 'link': self._link_to_object(o), 'brief': self.brief(o)}]
for m in self.sorted_and_filtered(o.methods):
ret += [{'indent': ' ', 'link': self._link_to_object(m), 'brief': self.brief(m)}]
if o.obj_type == ObjectType.MODULE and self.extract_returned_by_module(o):
tmp = gen_obj_links(self.extract_returned_by_module(o))
tmp = [{**x, 'indent': ' ' + x['indent']} for x in tmp]
ret += [{'indent': ' ', 'link': '**New objects:**', 'brief': ''}]
ret += [*tmp]
return ret
data = {
'root': self._gen_filename('root'),
'elementary': gen_obj_links(self.elementary),
'returned': gen_obj_links(self.returned),
'builtins': gen_obj_links(self.builtins),
'modules': gen_obj_links(self.modules),
'functions': [{'indent': '', 'link': self._link_to_object(x), 'brief': self.brief(x)} for x in self.functions],
'enable_modules': self.enable_modules,
}
dummy = {'root': self._gen_filename('root')}
self._write_template(data, 'root')
self._write_template({**dummy, 'name': 'Elementary types'}, f'root.{_OBJ_ID_MAP[ObjectType.ELEMENTARY]}', 'dummy')
self._write_template({**dummy, 'name': 'Builtin objects'}, f'root.{_OBJ_ID_MAP[ObjectType.BUILTIN]}', 'dummy')
self._write_template({**dummy, 'name': 'Returned objects'}, f'root.{_OBJ_ID_MAP[ObjectType.RETURNED]}', 'dummy')
if self.enable_modules:
self._write_template({**dummy, 'name': 'Modules'}, f'root.{_OBJ_ID_MAP[ObjectType.MODULE]}', 'dummy')
def generate(self) -> None:
mlog.log('Generating markdown files...')
with mlog.nested():
self._write_functions()
for obj in self.objects:
if not self.enable_modules and (obj.obj_type == ObjectType.MODULE or obj.defined_by_module is not None):
continue
self._write_object(obj)
self._root_refman_docs()
self._configure_sitemap()
self._generate_link_def()
def _configure_sitemap(self) -> None:
'''
Replaces the `@REFMAN_PLACEHOLDER@` placeholder with the reference
manual sitemap. The structure of the sitemap is derived from the
file IDs.
'''
raw = self.sitemap_in.read_text(encoding='utf-8')
out = ''
for l in raw.split('\n'):
if '@REFMAN_PLACEHOLDER@' not in l:
out += f'{l}\n'
continue
mlog.log('Generating', mlog.bold(self.sitemap_out.as_posix()))
base_indent = l.replace('@REFMAN_PLACEHOLDER@', '')
for k in sorted(self.generated_files.keys()):
indent = base_indent + '\t' * k.count('.')
out += f'{indent}{self.generated_files[k]}\n'
self.sitemap_out.write_text(out, encoding='utf-8')
def _generate_link_def(self) -> None:
'''
Generate the link definition file for the refman_links hotdoc
plugin. The plugin is then responsible for replacing the [[tag]]
tags with custom HTML elements.
'''
data: T.Dict[str, str] = {}
# Objects and methods
for obj in self.objects:
obj_file = self._gen_filename(self._gen_object_file_id(obj), extension='html')
data[f'@{obj.name}'] = obj_file
for m in obj.methods:
data[f'{obj.name}.{m.name}'] = f'{obj_file}#{obj.name}{m.name}'
# Functions
funcs_file = self._gen_filename('root.functions', extension='html')
for fn in self.functions:
data[fn.name] = f'{funcs_file}#{fn.name}'
self.link_def_out.write_text(json.dumps(data, indent=2), encoding='utf-8')