From 99dff36d3a5fb38bd3d079cd16b811bbfcb3ad07 Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Fri, 25 Sep 2020 17:22:59 +0100 Subject: scripts/qapi: Move doc-comment whitespace stripping to doc.py As we accumulate lines from doc comments when parsing the JSON, the QAPIDoc class generally strips leading and trailing whitespace using line.strip() when it calls _append_freeform(). This is fine for Texinfo, but for rST leading whitespace is significant. We'd like to move to having the text in doc comments be rST format rather than a custom syntax, so move the removal of leading whitespace from the QAPIDoc class to the texinfo-specific processing code in texi_format() in qapi/doc.py. (Trailing whitespace will always be stripped by the rstrip() in Section::append regardless.) In a followup commit we will make the whitespace in the lines of doc comment sections more consistently follow the input source. There is no change to the generated .texi files before and after this commit. Because the qapi-schema test checks the exact values of the documentation comments against a reference, we need to update that reference to match the new whitespace. In the first four places this is now correctly checking that we did put in the amount of whitespace to pass a rST-formatted list to the backend; in the last two places the extra whitespace is 'wrong' and will go away again in the following commit. Reviewed-by: Richard Henderson Reviewed-by: Markus Armbruster Signed-off-by: Peter Maydell Message-Id: <20200925162316.21205-5-peter.maydell@linaro.org> Signed-off-by: Markus Armbruster --- scripts/qapi/doc.py | 1 + scripts/qapi/parser.py | 12 ++++-------- 2 files changed, 5 insertions(+), 8 deletions(-) (limited to 'scripts/qapi') diff --git a/scripts/qapi/doc.py b/scripts/qapi/doc.py index 92f584e..7764de1 100644 --- a/scripts/qapi/doc.py +++ b/scripts/qapi/doc.py @@ -79,6 +79,7 @@ def texi_format(doc): inlist = '' lastempty = False for line in doc.split('\n'): + line = line.strip() empty = line == '' # FIXME: Doing this in a single if / elif chain is diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py index 165925c..04bf10d 100644 --- a/scripts/qapi/parser.py +++ b/scripts/qapi/parser.py @@ -427,10 +427,10 @@ class QAPIDoc: self._append_line = self._append_various_line self._append_various_line(line) else: - self._append_freeform(line.strip()) + self._append_freeform(line) else: # This is a free-form documentation block - self._append_freeform(line.strip()) + self._append_freeform(line) def _append_args_line(self, line): """ @@ -463,7 +463,7 @@ class QAPIDoc: self._append_various_line(line) return - self._append_freeform(line.strip()) + self._append_freeform(line) def _append_features_line(self, line): name = line.split(' ', 1)[0] @@ -482,7 +482,7 @@ class QAPIDoc: self._append_various_line(line) return - self._append_freeform(line.strip()) + self._append_freeform(line) def _append_various_line(self, line): """ @@ -505,10 +505,6 @@ class QAPIDoc: line = line[len(name)+1:] self._start_section(name[:-1]) - if (not self._section.name or - not self._section.name.startswith('Example')): - line = line.strip() - self._append_freeform(line) def _start_symbol_section(self, symbols_dict, name): -- cgit v1.1 From a69a6d4b4d4fae2e3d2506241e22a78ff1732283 Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Fri, 25 Sep 2020 17:23:00 +0100 Subject: scripts/qapi/parser.py: improve doc comment indent handling Make the handling of indentation in doc comments more sophisticated, so that when we see a section like: Notes: some text some more text indented line 3 we save it for the doc-comment processing code as: some text some more text indented line 3 and when we see a section with the heading on its own line: Notes: some text some more text indented text we also accept that and save it in the same form. If we detect that the comment document text is not indented as much as we expect it to be, we throw a parse error. (We don't complain about over-indented sections, because for rST this can be legitimate markup.) The golden reference for the doc comment text is updated to remove the two 'wrong' indents; these now form a test case that we correctly stripped leading whitespace from an indented multi-line argument definition. We update the documentation in docs/devel/qapi-code-gen.txt to describe the new indentation rules. Signed-off-by: Peter Maydell Message-Id: <20200925162316.21205-6-peter.maydell@linaro.org> Reviewed-by: Markus Armbruster [Whitespace between sentences tweaked] Signed-off-by: Markus Armbruster --- scripts/qapi/parser.py | 93 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 74 insertions(+), 19 deletions(-) (limited to 'scripts/qapi') diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py index 04bf10d..9d1a3e2 100644 --- a/scripts/qapi/parser.py +++ b/scripts/qapi/parser.py @@ -319,17 +319,32 @@ class QAPIDoc: """ class Section: - def __init__(self, name=None): + def __init__(self, parser, name=None, indent=0): + # parser, for error messages about indentation + self._parser = parser # optional section name (argument/member or section name) self.name = name self.text = '' + # the expected indent level of the text of this section + self._indent = indent def append(self, line): + # Strip leading spaces corresponding to the expected indent level + # Blank lines are always OK. + if line: + indent = re.match(r'\s*', line).end() + if indent < self._indent: + raise QAPIParseError( + self._parser, + "unexpected de-indent (expected at least %d spaces)" % + self._indent) + line = line[self._indent:] + self.text += line.rstrip() + '\n' class ArgSection(Section): - def __init__(self, name): - super().__init__(name) + def __init__(self, parser, name, indent=0): + super().__init__(parser, name, indent) self.member = None def connect(self, member): @@ -343,7 +358,7 @@ class QAPIDoc: self._parser = parser self.info = info self.symbol = None - self.body = QAPIDoc.Section() + self.body = QAPIDoc.Section(parser) # dict mapping parameter name to ArgSection self.args = OrderedDict() self.features = OrderedDict() @@ -447,8 +462,21 @@ class QAPIDoc: name = line.split(' ', 1)[0] if name.startswith('@') and name.endswith(':'): - line = line[len(name)+1:] - self._start_args_section(name[1:-1]) + # If line is "@arg: first line of description", find + # the index of 'f', which is the indent we expect for any + # following lines. We then remove the leading "@arg:" + # from line and replace it with spaces so that 'f' has the + # same index as it did in the original line and can be + # handled the same way we will handle following lines. + indent = re.match(r'@\S*:\s*', line).end() + line = line[indent:] + if not line: + # Line was just the "@arg:" header; following lines + # are not indented + indent = 0 + else: + line = ' ' * indent + line + self._start_args_section(name[1:-1], indent) elif self._is_section_tag(name): self._append_line = self._append_various_line self._append_various_line(line) @@ -469,8 +497,21 @@ class QAPIDoc: name = line.split(' ', 1)[0] if name.startswith('@') and name.endswith(':'): - line = line[len(name)+1:] - self._start_features_section(name[1:-1]) + # If line is "@arg: first line of description", find + # the index of 'f', which is the indent we expect for any + # following lines. We then remove the leading "@arg:" + # from line and replace it with spaces so that 'f' has the + # same index as it did in the original line and can be + # handled the same way we will handle following lines. + indent = re.match(r'@\S*:\s*', line).end() + line = line[indent:] + if not line: + # Line was just the "@arg:" header; following lines + # are not indented + indent = 0 + else: + line = ' ' * indent + line + self._start_features_section(name[1:-1], indent) elif self._is_section_tag(name): self._append_line = self._append_various_line self._append_various_line(line) @@ -502,12 +543,25 @@ class QAPIDoc: "'%s' can't follow '%s' section" % (name, self.sections[0].name)) if self._is_section_tag(name): - line = line[len(name)+1:] - self._start_section(name[:-1]) + # If line is "Section: first line of description", find + # the index of 'f', which is the indent we expect for any + # following lines. We then remove the leading "Section:" + # from line and replace it with spaces so that 'f' has the + # same index as it did in the original line and can be + # handled the same way we will handle following lines. + indent = re.match(r'\S*:\s*', line).end() + line = line[indent:] + if not line: + # Line was just the "Section:" header; following lines + # are not indented + indent = 0 + else: + line = ' ' * indent + line + self._start_section(name[:-1], indent) self._append_freeform(line) - def _start_symbol_section(self, symbols_dict, name): + def _start_symbol_section(self, symbols_dict, name, indent): # FIXME invalid names other than the empty string aren't flagged if not name: raise QAPIParseError(self._parser, "invalid parameter name") @@ -516,21 +570,21 @@ class QAPIDoc: "'%s' parameter name duplicated" % name) assert not self.sections self._end_section() - self._section = QAPIDoc.ArgSection(name) + self._section = QAPIDoc.ArgSection(self._parser, name, indent) symbols_dict[name] = self._section - def _start_args_section(self, name): - self._start_symbol_section(self.args, name) + def _start_args_section(self, name, indent): + self._start_symbol_section(self.args, name, indent) - def _start_features_section(self, name): - self._start_symbol_section(self.features, name) + def _start_features_section(self, name, indent): + self._start_symbol_section(self.features, name, indent) - def _start_section(self, name=None): + def _start_section(self, name=None, indent=0): if name in ('Returns', 'Since') and self.has_section(name): raise QAPIParseError(self._parser, "duplicated '%s' section" % name) self._end_section() - self._section = QAPIDoc.Section(name) + self._section = QAPIDoc.Section(self._parser, name, indent) self.sections.append(self._section) def _end_section(self): @@ -553,7 +607,8 @@ class QAPIDoc: def connect_member(self, member): if member.name not in self.args: # Undocumented TODO outlaw - self.args[member.name] = QAPIDoc.ArgSection(member.name) + self.args[member.name] = QAPIDoc.ArgSection(self._parser, + member.name) self.args[member.name].connect(member) def connect_feature(self, feature): -- cgit v1.1 From a27ff0a24916e7a6cdc90e4a984423069d8dfea6 Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Fri, 25 Sep 2020 17:23:11 +0100 Subject: scripts/qapi: Remove texinfo generation support We no longer use the generated texinfo format documentation, so delete the code that generates it, and the test case for the generation. Reviewed-by: Richard Henderson Signed-off-by: Peter Maydell Message-Id: <20200925162316.21205-17-peter.maydell@linaro.org> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/doc.py | 302 ---------------------------------------------------- scripts/qapi/gen.py | 7 -- 2 files changed, 309 deletions(-) delete mode 100644 scripts/qapi/doc.py (limited to 'scripts/qapi') diff --git a/scripts/qapi/doc.py b/scripts/qapi/doc.py deleted file mode 100644 index 7764de1..0000000 --- a/scripts/qapi/doc.py +++ /dev/null @@ -1,302 +0,0 @@ -# QAPI texi generator -# -# This work is licensed under the terms of the GNU LGPL, version 2+. -# See the COPYING file in the top-level directory. -"""This script produces the documentation of a qapi schema in texinfo format""" - -import re -from qapi.gen import QAPIGenDoc, QAPISchemaVisitor - - -MSG_FMT = """ -@deftypefn {type} {{}} {name} - -{body}{members}{features}{sections} -@end deftypefn - -""".format - -TYPE_FMT = """ -@deftp {{{type}}} {name} - -{body}{members}{features}{sections} -@end deftp - -""".format - -EXAMPLE_FMT = """@example -{code} -@end example -""".format - - -def subst_strong(doc): - """Replaces *foo* by @strong{foo}""" - return re.sub(r'\*([^*\n]+)\*', r'@strong{\1}', doc) - - -def subst_emph(doc): - """Replaces _foo_ by @emph{foo}""" - return re.sub(r'\b_([^_\n]+)_\b', r'@emph{\1}', doc) - - -def subst_vars(doc): - """Replaces @var by @code{var}""" - return re.sub(r'@([\w-]+)', r'@code{\1}', doc) - - -def subst_braces(doc): - """Replaces {} with @{ @}""" - return doc.replace('{', '@{').replace('}', '@}') - - -def texi_example(doc): - """Format @example""" - # TODO: Neglects to escape @ characters. - # We should probably escape them in subst_braces(), and rename the - # function to subst_special() or subs_texi_special(). If we do that, we - # need to delay it until after subst_vars() in texi_format(). - doc = subst_braces(doc).strip('\n') - return EXAMPLE_FMT(code=doc) - - -def texi_format(doc): - """ - Format documentation - - Lines starting with: - - |: generates an @example - - =: generates @section - - ==: generates @subsection - - 1. or 1): generates an @enumerate @item - - */-: generates an @itemize list - """ - ret = '' - doc = subst_braces(doc) - doc = subst_vars(doc) - doc = subst_emph(doc) - doc = subst_strong(doc) - inlist = '' - lastempty = False - for line in doc.split('\n'): - line = line.strip() - empty = line == '' - - # FIXME: Doing this in a single if / elif chain is - # problematic. For instance, a line without markup terminates - # a list if it follows a blank line (reaches the final elif), - # but a line with some *other* markup, such as a = title - # doesn't. - # - # Make sure to update section "Documentation markup" in - # docs/devel/qapi-code-gen.txt when fixing this. - if line.startswith('| '): - line = EXAMPLE_FMT(code=line[2:]) - elif line.startswith('= '): - line = '@section ' + line[2:] - elif line.startswith('== '): - line = '@subsection ' + line[3:] - elif re.match(r'^([0-9]*\.) ', line): - if not inlist: - ret += '@enumerate\n' - inlist = 'enumerate' - ret += '@item\n' - line = line[line.find(' ')+1:] - elif re.match(r'^[*-] ', line): - if not inlist: - ret += '@itemize %s\n' % {'*': '@bullet', - '-': '@minus'}[line[0]] - inlist = 'itemize' - ret += '@item\n' - line = line[2:] - elif lastempty and inlist: - ret += '@end %s\n\n' % inlist - inlist = '' - - lastempty = empty - ret += line + '\n' - - if inlist: - ret += '@end %s\n\n' % inlist - return ret - - -def texi_body(doc): - """Format the main documentation body""" - return texi_format(doc.body.text) - - -def texi_if(ifcond, prefix='\n', suffix='\n'): - """Format the #if condition""" - if not ifcond: - return '' - return '%s@b{If:} @code{%s}%s' % (prefix, ', '.join(ifcond), suffix) - - -def texi_enum_value(value, desc, suffix): - """Format a table of members item for an enumeration value""" - return '@item @code{%s}\n%s%s' % ( - value.name, desc, texi_if(value.ifcond, prefix='@*')) - - -def texi_member(member, desc, suffix): - """Format a table of members item for an object type member""" - typ = member.type.doc_type() - membertype = ': ' + typ if typ else '' - return '@item @code{%s%s}%s%s\n%s%s' % ( - member.name, membertype, - ' (optional)' if member.optional else '', - suffix, desc, texi_if(member.ifcond, prefix='@*')) - - -def texi_members(doc, what, base=None, variants=None, - member_func=texi_member): - """Format the table of members""" - items = '' - for section in doc.args.values(): - # TODO Drop fallbacks when undocumented members are outlawed - if section.text: - desc = texi_format(section.text) - elif (variants and variants.tag_member == section.member - and not section.member.type.doc_type()): - values = section.member.type.member_names() - members_text = ', '.join(['@t{"%s"}' % v for v in values]) - desc = 'One of ' + members_text + '\n' - else: - desc = 'Not documented\n' - items += member_func(section.member, desc, suffix='') - if base: - items += '@item The members of @code{%s}\n' % base.doc_type() - if variants: - for v in variants.variants: - when = ' when @code{%s} is @t{"%s"}%s' % ( - variants.tag_member.name, v.name, texi_if(v.ifcond, " (", ")")) - if v.type.is_implicit(): - assert not v.type.base and not v.type.variants - for m in v.type.local_members: - items += member_func(m, desc='', suffix=when) - else: - items += '@item The members of @code{%s}%s\n' % ( - v.type.doc_type(), when) - if not items: - return '' - return '\n@b{%s:}\n@table @asis\n%s@end table\n' % (what, items) - - -def texi_arguments(doc, boxed_arg_type): - if boxed_arg_type: - assert not doc.args - return ('\n@b{Arguments:} the members of @code{%s}\n' - % boxed_arg_type.name) - return texi_members(doc, 'Arguments') - - -def texi_features(doc): - """Format the table of features""" - items = '' - for section in doc.features.values(): - desc = texi_format(section.text) - items += '@item @code{%s}\n%s' % (section.name, desc) - if not items: - return '' - return '\n@b{Features:}\n@table @asis\n%s@end table\n' % (items) - - -def texi_sections(doc, ifcond): - """Format additional sections following arguments""" - body = '' - for section in doc.sections: - if section.name: - # prefer @b over @strong, so txt doesn't translate it to *Foo:* - body += '\n@b{%s:}\n' % section.name - if section.name and section.name.startswith('Example'): - body += texi_example(section.text) - else: - body += texi_format(section.text) - body += texi_if(ifcond, suffix='') - return body - - -def texi_type(typ, doc, ifcond, members): - return TYPE_FMT(type=typ, - name=doc.symbol, - body=texi_body(doc), - members=members, - features=texi_features(doc), - sections=texi_sections(doc, ifcond)) - - -def texi_msg(typ, doc, ifcond, members): - return MSG_FMT(type=typ, - name=doc.symbol, - body=texi_body(doc), - members=members, - features=texi_features(doc), - sections=texi_sections(doc, ifcond)) - - -class QAPISchemaGenDocVisitor(QAPISchemaVisitor): - def __init__(self, prefix): - self._prefix = prefix - self._gen = QAPIGenDoc(self._prefix + 'qapi-doc.texi') - self.cur_doc = None - - def write(self, output_dir): - self._gen.write(output_dir) - - def visit_enum_type(self, name, info, ifcond, features, members, prefix): - doc = self.cur_doc - self._gen.add(texi_type('Enum', doc, ifcond, - texi_members(doc, 'Values', - member_func=texi_enum_value))) - - def visit_object_type(self, name, info, ifcond, features, - base, members, variants): - doc = self.cur_doc - if base and base.is_implicit(): - base = None - self._gen.add(texi_type('Object', doc, ifcond, - texi_members(doc, 'Members', base, variants))) - - def visit_alternate_type(self, name, info, ifcond, features, variants): - doc = self.cur_doc - self._gen.add(texi_type('Alternate', doc, ifcond, - texi_members(doc, 'Members'))) - - def visit_command(self, name, info, ifcond, features, - arg_type, ret_type, gen, success_response, boxed, - allow_oob, allow_preconfig): - doc = self.cur_doc - self._gen.add(texi_msg('Command', doc, ifcond, - texi_arguments(doc, - arg_type if boxed else None))) - - def visit_event(self, name, info, ifcond, features, arg_type, boxed): - doc = self.cur_doc - self._gen.add(texi_msg('Event', doc, ifcond, - texi_arguments(doc, - arg_type if boxed else None))) - - def symbol(self, doc, entity): - if self._gen._body: - self._gen.add('\n') - self.cur_doc = doc - entity.visit(self) - self.cur_doc = None - - def freeform(self, doc): - assert not doc.args - if self._gen._body: - self._gen.add('\n') - self._gen.add(texi_body(doc) + texi_sections(doc, None)) - - -def gen_doc(schema, output_dir, prefix): - vis = QAPISchemaGenDocVisitor(prefix) - vis.visit_begin(schema) - for doc in schema.docs: - if doc.symbol: - vis.symbol(doc, schema.lookup_entity(doc.symbol)) - else: - vis.freeform(doc) - vis.write(output_dir) diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py index bf5552a..ca66c82 100644 --- a/scripts/qapi/gen.py +++ b/scripts/qapi/gen.py @@ -178,13 +178,6 @@ def ifcontext(ifcond, *args): arg.end_if() -class QAPIGenDoc(QAPIGen): - - def _top(self): - return (super()._top() - + '@c AUTOMATICALLY GENERATED, DO NOT MODIFY\n\n') - - class QAPISchemaMonolithicCVisitor(QAPISchemaVisitor): def __init__(self, prefix, what, blurb, pydoc): -- cgit v1.1