From d0a263cdd019116565682896d115ecd662515f78 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:16 -0400 Subject: qapi/expr: Comment cleanup The linter yaps after 0825f62c842. Fix this trivial issue to restore the linter baseline. (Yes, ideally -- and soon -- the linter will be part of CI so we don't clutter up the log with fixups. For now, though, the baseline is useful for testing intermediate commits as types are added to the QAPI library.) Signed-off-by: John Snow Message-Id: <20210421182032.3521476-2-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 540b398..c207481 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -241,7 +241,7 @@ def check_enum(expr, info): source = "%s '%s'" % (source, member_name) # Enum members may start with a digit if member_name[0].isdigit(): - member_name = 'd' + member_name # Hack: hide the digit + member_name = 'd' + member_name # Hack: hide the digit check_name_lower(member_name, info, source, permit_upper=permissive, permit_underscore=permissive) -- cgit v1.1 From b7341b89c9c0212fef7b38b04ffaf39ea73bfca9 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:17 -0400 Subject: qapi/expr.py: Remove 'info' argument from nested check_if_str The function can just use the argument from the scope above. Otherwise, we get shadowed argument errors because the parameter name clashes with the name of a variable already in-scope. Signed-off-by: John Snow Reviewed-by: Eduardo Habkost Reviewed-by: Cleber Rosa Message-Id: <20210421182032.3521476-3-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index c207481..3fda5d5 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -122,7 +122,7 @@ def check_flags(expr, info): def check_if(expr, info, source): - def check_if_str(ifcond, info): + def check_if_str(ifcond): if not isinstance(ifcond, str): raise QAPISemError( info, @@ -142,9 +142,9 @@ def check_if(expr, info, source): raise QAPISemError( info, "'if' condition [] of %s is useless" % source) for elt in ifcond: - check_if_str(elt, info) + check_if_str(elt) else: - check_if_str(ifcond, info) + check_if_str(ifcond) expr['if'] = [ifcond] -- cgit v1.1 From 0f231dcf2921fa8bc475d222a8ef81e67d4019e8 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:18 -0400 Subject: qapi/expr.py: Check for dict instead of OrderedDict OrderedDict is a subtype of dict, so we can check for a more general form. These functions do not themselves depend on it being any particular type. Signed-off-by: John Snow Reviewed-by: Eduardo Habkost Reviewed-by: Cleber Rosa Message-Id: <20210421182032.3521476-4-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 3fda5d5..b4bbcd5 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -14,7 +14,6 @@ # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. -from collections import OrderedDict import re from .common import c_name @@ -149,7 +148,7 @@ def check_if(expr, info, source): def normalize_members(members): - if isinstance(members, OrderedDict): + if isinstance(members, dict): for key, arg in members.items(): if isinstance(arg, dict): continue @@ -180,7 +179,7 @@ def check_type(value, info, source, if not allow_dict: raise QAPISemError(info, "%s should be a type name" % source) - if not isinstance(value, OrderedDict): + if not isinstance(value, dict): raise QAPISemError(info, "%s should be an object or type name" % source) -- cgit v1.1 From 59b5556ce8c1d3dc1f2c6445ca32f2e515114a8e Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:19 -0400 Subject: qapi/expr.py: constrain incoming expression types mypy does not know the types of values stored in Dicts that masquerade as objects. Help the type checker out by constraining the type. Signed-off-by: John Snow Message-Id: <20210421182032.3521476-5-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index b4bbcd5..06a0081 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -15,9 +15,20 @@ # See the COPYING file in the top-level directory. import re +from typing import Dict, Optional from .common import c_name from .error import QAPISemError +from .parser import QAPIDoc +from .source import QAPISourceInfo + + +# Deserialized JSON objects as returned by the parser. +# The values of this mapping are not necessary to exhaustively type +# here (and also not practical as long as mypy lacks recursive +# types), because the purpose of this module is to interrogate that +# type. +_JSONObject = Dict[str, object] # Names consist of letters, digits, -, and _, starting with a letter. @@ -315,9 +326,20 @@ def check_event(expr, info): def check_exprs(exprs): for expr_elem in exprs: - expr = expr_elem['expr'] - info = expr_elem['info'] - doc = expr_elem.get('doc') + # Expression + assert isinstance(expr_elem['expr'], dict) + for key in expr_elem['expr'].keys(): + assert isinstance(key, str) + expr: _JSONObject = expr_elem['expr'] + + # QAPISourceInfo + assert isinstance(expr_elem['info'], QAPISourceInfo) + info: QAPISourceInfo = expr_elem['info'] + + # Optional[QAPIDoc] + tmp = expr_elem.get('doc') + assert tmp is None or isinstance(tmp, QAPIDoc) + doc: Optional[QAPIDoc] = tmp if 'include' in expr: continue -- cgit v1.1 From b66c62a2d3318c5d968d5b95428efb74ca5d5702 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:20 -0400 Subject: qapi/expr.py: Add assertion for union type 'check_dict' mypy isn't fond of allowing you to check for bool membership in a collection of str elements. Guard this lookup for precisely when we were given a name. Signed-off-by: John Snow Reviewed-by: Eduardo Habkost Reviewed-by: Cleber Rosa Message-Id: <20210421182032.3521476-6-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 06a0081..3ab78a5 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -194,7 +194,9 @@ def check_type(value, info, source, raise QAPISemError(info, "%s should be an object or type name" % source) - permissive = allow_dict in info.pragma.member_name_exceptions + permissive = False + if isinstance(allow_dict, str): + permissive = allow_dict in info.pragma.member_name_exceptions # value is a dictionary, check that each member is okay for (key, arg) in value.items(): -- cgit v1.1 From 926bb8add7c549496c612fcd4a32f3cf37883c2a Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:21 -0400 Subject: qapi/expr.py: move string check upwards in check_type For readability purposes only, shimmy the early return upwards to the top of the function, so cases proceed in order from least to most complex. Signed-off-by: John Snow Reviewed-by: Eduardo Habkost Reviewed-by: Cleber Rosa Message-Id: <20210421182032.3521476-7-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 3ab78a5..c0d18dc 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -171,6 +171,10 @@ def check_type(value, info, source, if value is None: return + # Type name + if isinstance(value, str): + return + # Array type if isinstance(value, list): if not allow_array: @@ -181,10 +185,6 @@ def check_type(value, info, source, source) return - # Type name - if isinstance(value, str): - return - # Anonymous type if not allow_dict: -- cgit v1.1 From 4918bb7defbdcb1e27cc2adf4e1604486d778ece Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:22 -0400 Subject: qapi/expr.py: Check type of union and alternate 'data' member Prior to this commit, specifying a non-object value here causes the QAPI parser to crash in expr.py with a stack trace with (likely) an AttributeError when we attempt to call that value's items() method. This member needs to be an object (Dict), and not anything else. Add a check for this with a nicer error message, and formalize that check with new test cases that exercise that error. Signed-off-by: John Snow Message-Id: <20210421182032.3521476-8-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index c0d18dc..03624bd 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -283,6 +283,9 @@ def check_union(expr, info): raise QAPISemError(info, "'discriminator' requires 'base'") check_name_is_str(discriminator, info, "'discriminator'") + if not isinstance(members, dict): + raise QAPISemError(info, "'data' must be an object") + for (key, value) in members.items(): source = "'data' member '%s'" % key if discriminator is None: @@ -298,6 +301,10 @@ def check_alternate(expr, info): if not members: raise QAPISemError(info, "'data' must not be empty") + + if not isinstance(members, dict): + raise QAPISemError(info, "'data' must be an object") + for (key, value) in members.items(): source = "'data' member '%s'" % key check_name_lower(key, info, source) -- cgit v1.1 From 7a783ce5b5a3ac4762b866e22370dd4fb30b91bf Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:23 -0400 Subject: qapi/expr.py: Add casts in a few select cases Casts are instructions to the type checker only, they aren't "safe" and should probably be avoided in general. In this case, when we perform type checking on a nested structure, the type of each field does not "stick". (See PEP 647 for an example of "type narrowing" that does "stick". It is available in Python 3.10, so we can't use it yet.) We don't need to assert that something is a str if we've already checked or asserted that it is -- use a cast instead for these cases. Signed-off-by: John Snow Message-Id: <20210421182032.3521476-9-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 03624bd..f3a4a85 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -15,7 +15,7 @@ # See the COPYING file in the top-level directory. import re -from typing import Dict, Optional +from typing import Dict, Optional, cast from .common import c_name from .error import QAPISemError @@ -261,7 +261,7 @@ def check_enum(expr, info): def check_struct(expr, info): - name = expr['struct'] + name = cast(str, expr['struct']) # Checked in check_exprs members = expr['data'] check_type(members, info, "'data'", allow_dict=name) @@ -269,7 +269,7 @@ def check_struct(expr, info): def check_union(expr, info): - name = expr['union'] + name = cast(str, expr['union']) # Checked in check_exprs base = expr.get('base') discriminator = expr.get('discriminator') members = expr['data'] @@ -368,8 +368,8 @@ def check_exprs(exprs): else: raise QAPISemError(info, "expression is missing metatype") - name = expr[meta] - check_name_is_str(name, info, "'%s'" % meta) + check_name_is_str(expr[meta], info, "'%s'" % meta) + name = cast(str, expr[meta]) info.set_defn(meta, name) check_defn_name_str(name, info, meta) -- cgit v1.1 From 538cd41065ae5e506a1a07e866b1fd40b4b53d07 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:24 -0400 Subject: qapi/expr.py: Modify check_keys to accept any Collection This is a minor adjustment that lets parameters @required and @optional take tuple arguments, in particular (). Later patches will make use of that. (Iterable would also have worked, but Iterable also includes things like generator expressions which are consumed upon iteration, which would require a rewrite to make sure that each input was only traversed once. Collection implies the "can re-iterate" property.) Signed-off-by: John Snow Message-Id: <20210421182032.3521476-10-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index f3a4a85..396c812 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -102,7 +102,7 @@ def check_keys(value, info, source, required, optional): "%s misses key%s %s" % (source, 's' if len(missing) > 1 else '', pprint(missing))) - allowed = set(required + optional) + allowed = set(required) | set(optional) unknown = set(value) - allowed if unknown: raise QAPISemError( -- cgit v1.1 From b9ad358aa057e83f8039a1e222d6941d2bf1f70a Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:25 -0400 Subject: qapi/expr.py: add type hint annotations Annotations do not change runtime behavior. This commit *only* adds annotations. Signed-off-by: John Snow Message-Id: <20210421182032.3521476-11-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 68 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 25 deletions(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 396c812..4ebed4c 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -15,7 +15,15 @@ # See the COPYING file in the top-level directory. import re -from typing import Dict, Optional, cast +from typing import ( + Collection, + Dict, + Iterable, + List, + Optional, + Union, + cast, +) from .common import c_name from .error import QAPISemError @@ -39,12 +47,14 @@ valid_name = re.compile(r'(__[a-z0-9.-]+_)?' r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) -def check_name_is_str(name, info, source): +def check_name_is_str(name: object, + info: QAPISourceInfo, + source: str) -> None: if not isinstance(name, str): raise QAPISemError(info, "%s requires a string name" % source) -def check_name_str(name, info, source): +def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' # and 'q_obj_*' implicit type names. match = valid_name.match(name) @@ -53,16 +63,16 @@ def check_name_str(name, info, source): return match.group(3) -def check_name_upper(name, info, source): +def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: stem = check_name_str(name, info, source) if re.search(r'[a-z-]', stem): raise QAPISemError( info, "name of %s must not use lowercase or '-'" % source) -def check_name_lower(name, info, source, - permit_upper=False, - permit_underscore=False): +def check_name_lower(name: str, info: QAPISourceInfo, source: str, + permit_upper: bool = False, + permit_underscore: bool = False) -> None: stem = check_name_str(name, info, source) if ((not permit_upper and re.search(r'[A-Z]', stem)) or (not permit_underscore and '_' in stem)): @@ -70,13 +80,13 @@ def check_name_lower(name, info, source, info, "name of %s must not use uppercase or '_'" % source) -def check_name_camel(name, info, source): +def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: stem = check_name_str(name, info, source) if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): raise QAPISemError(info, "name of %s must use CamelCase" % source) -def check_defn_name_str(name, info, meta): +def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: if meta == 'event': check_name_upper(name, info, meta) elif meta == 'command': @@ -90,9 +100,13 @@ def check_defn_name_str(name, info, meta): info, "%s name should not end in '%s'" % (meta, name[-4:])) -def check_keys(value, info, source, required, optional): +def check_keys(value: _JSONObject, + info: QAPISourceInfo, + source: str, + required: Collection[str], + optional: Collection[str]) -> None: - def pprint(elems): + def pprint(elems: Iterable[str]) -> str: return ', '.join("'" + e + "'" for e in sorted(elems)) missing = set(required) - set(value) @@ -112,7 +126,7 @@ def check_keys(value, info, source, required, optional): pprint(unknown), pprint(allowed))) -def check_flags(expr, info): +def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: for key in ['gen', 'success-response']: if key in expr and expr[key] is not False: raise QAPISemError( @@ -130,9 +144,9 @@ def check_flags(expr, info): "are incompatible") -def check_if(expr, info, source): +def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: - def check_if_str(ifcond): + def check_if_str(ifcond: object) -> None: if not isinstance(ifcond, str): raise QAPISemError( info, @@ -158,7 +172,7 @@ def check_if(expr, info, source): expr['if'] = [ifcond] -def normalize_members(members): +def normalize_members(members: object) -> None: if isinstance(members, dict): for key, arg in members.items(): if isinstance(arg, dict): @@ -166,8 +180,11 @@ def normalize_members(members): members[key] = {'type': arg} -def check_type(value, info, source, - allow_array=False, allow_dict=False): +def check_type(value: Optional[object], + info: QAPISourceInfo, + source: str, + allow_array: bool = False, + allow_dict: Union[bool, str] = False) -> None: if value is None: return @@ -214,7 +231,8 @@ def check_type(value, info, source, check_type(arg['type'], info, key_source, allow_array=True) -def check_features(features, info): +def check_features(features: Optional[object], + info: QAPISourceInfo) -> None: if features is None: return if not isinstance(features, list): @@ -231,7 +249,7 @@ def check_features(features, info): check_if(f, info, source) -def check_enum(expr, info): +def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: name = expr['enum'] members = expr['data'] prefix = expr.get('prefix') @@ -260,7 +278,7 @@ def check_enum(expr, info): check_if(member, info, source) -def check_struct(expr, info): +def check_struct(expr: _JSONObject, info: QAPISourceInfo) -> None: name = cast(str, expr['struct']) # Checked in check_exprs members = expr['data'] @@ -268,7 +286,7 @@ def check_struct(expr, info): check_type(expr.get('base'), info, "'base'") -def check_union(expr, info): +def check_union(expr: _JSONObject, info: QAPISourceInfo) -> None: name = cast(str, expr['union']) # Checked in check_exprs base = expr.get('base') discriminator = expr.get('discriminator') @@ -296,7 +314,7 @@ def check_union(expr, info): check_type(value['type'], info, source, allow_array=not base) -def check_alternate(expr, info): +def check_alternate(expr: _JSONObject, info: QAPISourceInfo) -> None: members = expr['data'] if not members: @@ -313,7 +331,7 @@ def check_alternate(expr, info): check_type(value['type'], info, source) -def check_command(expr, info): +def check_command(expr: _JSONObject, info: QAPISourceInfo) -> None: args = expr.get('data') rets = expr.get('returns') boxed = expr.get('boxed', False) @@ -324,7 +342,7 @@ def check_command(expr, info): check_type(rets, info, "'returns'", allow_array=True) -def check_event(expr, info): +def check_event(expr: _JSONObject, info: QAPISourceInfo) -> None: args = expr.get('data') boxed = expr.get('boxed', False) @@ -333,7 +351,7 @@ def check_event(expr, info): check_type(args, info, "'data'", allow_dict=not boxed) -def check_exprs(exprs): +def check_exprs(exprs: List[_JSONObject]) -> List[_JSONObject]: for expr_elem in exprs: # Expression assert isinstance(expr_elem['expr'], dict) -- cgit v1.1 From 210fd63104525b6e3154de529565258f19146f1a Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:26 -0400 Subject: qapi/expr.py: Consolidate check_if_str calls in check_if This is a small rewrite to address some minor style nits. Don't compare against the empty list to check for the empty condition, and move the normalization forward to unify the check on the now-normalized structure. With the check unified, the local nested function isn't needed anymore and can be brought down into the normal flow of the function. With the nesting level changed, shuffle the error strings around a bit to get them to fit in 79 columns. Note: although ifcond is typed as Sequence[str] elsewhere, we *know* that the parser will produce real, bona-fide lists. It's okay to check isinstance(ifcond, list) here. Signed-off-by: John Snow Message-Id: <20210421182032.3521476-12-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 4ebed4c..de7fc16 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -146,30 +146,29 @@ def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: - def check_if_str(ifcond: object) -> None: - if not isinstance(ifcond, str): - raise QAPISemError( - info, - "'if' condition of %s must be a string or a list of strings" - % source) - if ifcond.strip() == '': - raise QAPISemError( - info, - "'if' condition '%s' of %s makes no sense" - % (ifcond, source)) - ifcond = expr.get('if') if ifcond is None: return + if isinstance(ifcond, list): - if ifcond == []: + if not ifcond: raise QAPISemError( info, "'if' condition [] of %s is useless" % source) - for elt in ifcond: - check_if_str(elt) else: - check_if_str(ifcond) - expr['if'] = [ifcond] + # Normalize to a list + ifcond = expr['if'] = [ifcond] + + for elt in ifcond: + if not isinstance(elt, str): + raise QAPISemError( + info, + "'if' condition of %s must be a string or a list of strings" + % source) + if not elt.strip(): + raise QAPISemError( + info, + "'if' condition '%s' of %s makes no sense" + % (elt, source)) def normalize_members(members: object) -> None: -- cgit v1.1 From e42648dccdd1defe8f35f247966cd7283f865cd6 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:27 -0400 Subject: qapi/expr.py: Remove single-letter variable Signed-off-by: John Snow Message-Id: <20210421182032.3521476-13-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index de7fc16..5e4d5f8 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -238,14 +238,14 @@ def check_features(features: Optional[object], raise QAPISemError(info, "'features' must be an array") features[:] = [f if isinstance(f, dict) else {'name': f} for f in features] - for f in features: + for feat in features: source = "'features' member" - assert isinstance(f, dict) - check_keys(f, info, source, ['name'], ['if']) - check_name_is_str(f['name'], info, source) - source = "%s '%s'" % (source, f['name']) - check_name_lower(f['name'], info, source) - check_if(f, info, source) + assert isinstance(feat, dict) + check_keys(feat, info, source, ['name'], ['if']) + check_name_is_str(feat['name'], info, source) + source = "%s '%s'" % (source, feat['name']) + check_name_str(feat['name'], info, source) + check_if(feat, info, source) def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: -- cgit v1.1 From 79e4fd14fb0f9e145413c6813dc541fdb50e3923 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:29 -0400 Subject: qapi/expr: Only explicitly prohibit 'Kind' nor 'List' for type names Per list review: qapi-code-gen.txt reserves suffixes Kind and List only for type names, but the code rejects them for events and commands, too. It turns out we reject them earlier anyway: In check_name_upper() for event names, and in check_name_lower() for command names. Still, adjust the code for clarity over what precisely we are guarding against. Signed-off-by: John Snow Message-Id: <20210421182032.3521476-15-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 5e4d5f8..c33caf0 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -95,9 +95,9 @@ def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: permit_underscore=name in info.pragma.command_name_exceptions) else: check_name_camel(name, info, meta) - if name.endswith('Kind') or name.endswith('List'): - raise QAPISemError( - info, "%s name should not end in '%s'" % (meta, name[-4:])) + if name.endswith('Kind') or name.endswith('List'): + raise QAPISemError( + info, "%s name should not end in '%s'" % (meta, name[-4:])) def check_keys(value: _JSONObject, -- cgit v1.1 From a48653638fab9c9a9356b41d6e11544b2a7b330f Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:30 -0400 Subject: qapi/expr.py: Add docstrings Now with more :words:! Signed-off-by: John Snow Message-Id: <20210421182032.3521476-16-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 256 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 251 insertions(+), 5 deletions(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index c33caf0..0b66d80 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- # -# Check (context-free) QAPI schema expression structure -# # Copyright IBM, Corp. 2011 # Copyright (c) 2013-2019 Red Hat Inc. # @@ -14,6 +12,24 @@ # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. +""" +Normalize and validate (context-free) QAPI schema expression structures. + +`QAPISchemaParser` parses a QAPI schema into abstract syntax trees +consisting of dict, list, str, bool, and int nodes. This module ensures +that these nested structures have the correct type(s) and key(s) where +appropriate for the QAPI context-free grammar. + +The QAPI schema expression language allows for certain syntactic sugar; +this module also handles the normalization process of these nested +structures. + +See `check_exprs` for the main entry point. + +See `schema.QAPISchema` for processing into native Python data +structures and contextual semantic validation. +""" + import re from typing import ( Collection, @@ -39,9 +55,7 @@ from .source import QAPISourceInfo _JSONObject = Dict[str, object] -# Names consist of letters, digits, -, and _, starting with a letter. -# An experimental name is prefixed with x-. A name of a downstream -# extension is prefixed with __RFQDN_. The latter prefix goes first. +# See check_name_str(), below. valid_name = re.compile(r'(__[a-z0-9.-]+_)?' r'(x-)?' r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) @@ -50,11 +64,33 @@ valid_name = re.compile(r'(__[a-z0-9.-]+_)?' def check_name_is_str(name: object, info: QAPISourceInfo, source: str) -> None: + """ + Ensure that ``name`` is a ``str``. + + :raise QAPISemError: When ``name`` fails validation. + """ if not isinstance(name, str): raise QAPISemError(info, "%s requires a string name" % source) def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: + """ + Ensure that ``name`` is a valid QAPI name. + + A valid name consists of ASCII letters, digits, ``-``, and ``_``, + starting with a letter. It may be prefixed by a downstream prefix + of the form __RFQDN_, or the experimental prefix ``x-``. If both + prefixes are present, the __RFDQN_ prefix goes first. + + A valid name cannot start with ``q_``, which is reserved. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param source: Error string describing what ``name`` belongs to. + + :raise QAPISemError: When ``name`` fails validation. + :return: The stem of the valid name, with no prefixes. + """ # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' # and 'q_obj_*' implicit type names. match = valid_name.match(name) @@ -64,6 +100,19 @@ def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: + """ + Ensure that ``name`` is a valid event name. + + This means it must be a valid QAPI name as checked by + `check_name_str()`, but where the stem prohibits lowercase + characters and ``-``. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param source: Error string describing what ``name`` belongs to. + + :raise QAPISemError: When ``name`` fails validation. + """ stem = check_name_str(name, info, source) if re.search(r'[a-z-]', stem): raise QAPISemError( @@ -73,6 +122,21 @@ def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: def check_name_lower(name: str, info: QAPISourceInfo, source: str, permit_upper: bool = False, permit_underscore: bool = False) -> None: + """ + Ensure that ``name`` is a valid command or member name. + + This means it must be a valid QAPI name as checked by + `check_name_str()`, but where the stem prohibits uppercase + characters and ``_``. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param source: Error string describing what ``name`` belongs to. + :param permit_upper: Additionally permit uppercase. + :param permit_underscore: Additionally permit ``_``. + + :raise QAPISemError: When ``name`` fails validation. + """ stem = check_name_str(name, info, source) if ((not permit_upper and re.search(r'[A-Z]', stem)) or (not permit_underscore and '_' in stem)): @@ -81,12 +145,39 @@ def check_name_lower(name: str, info: QAPISourceInfo, source: str, def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: + """ + Ensure that ``name`` is a valid user-defined type name. + + This means it must be a valid QAPI name as checked by + `check_name_str()`, but where the stem must be in CamelCase. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param source: Error string describing what ``name`` belongs to. + + :raise QAPISemError: When ``name`` fails validation. + """ stem = check_name_str(name, info, source) if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): raise QAPISemError(info, "name of %s must use CamelCase" % source) def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: + """ + Ensure that ``name`` is a valid definition name. + + Based on the value of ``meta``, this means that: + - 'event' names adhere to `check_name_upper()`. + - 'command' names adhere to `check_name_lower()`. + - Else, meta is a type, and must pass `check_name_camel()`. + These names must not end with ``Kind`` nor ``List``. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param meta: Meta-type name of the QAPI expression. + + :raise QAPISemError: When ``name`` fails validation. + """ if meta == 'event': check_name_upper(name, info, meta) elif meta == 'command': @@ -105,6 +196,17 @@ def check_keys(value: _JSONObject, source: str, required: Collection[str], optional: Collection[str]) -> None: + """ + Ensure that a dict has a specific set of keys. + + :param value: The dict to check. + :param info: QAPI schema source file information. + :param source: Error string describing this ``value``. + :param required: Keys that *must* be present. + :param optional: Keys that *may* be present. + + :raise QAPISemError: When unknown keys are present. + """ def pprint(elems: Iterable[str]) -> str: return ', '.join("'" + e + "'" for e in sorted(elems)) @@ -127,6 +229,16 @@ def check_keys(value: _JSONObject, def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Ensure flag members (if present) have valid values. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: + When certain flags have an invalid value, or when + incompatible flags are present. + """ for key in ['gen', 'success-response']: if key in expr and expr[key] is not False: raise QAPISemError( @@ -145,7 +257,25 @@ def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: + """ + Normalize and validate the ``if`` member of an object. + The ``if`` member may be either a ``str`` or a ``List[str]``. + A ``str`` value will be normalized to ``List[str]``. + + :forms: + :sugared: ``Union[str, List[str]]`` + :canonical: ``List[str]`` + + :param expr: The expression containing the ``if`` member to validate. + :param info: QAPI schema source file information. + :param source: Error string describing ``expr``. + + :raise QAPISemError: + When the "if" member fails validation, or when there are no + non-empty conditions. + :return: None, ``expr`` is normalized in-place as needed. + """ ifcond = expr.get('if') if ifcond is None: return @@ -172,6 +302,21 @@ def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: def normalize_members(members: object) -> None: + """ + Normalize a "members" value. + + If ``members`` is a dict, for every value in that dict, if that + value is not itself already a dict, normalize it to + ``{'type': value}``. + + :forms: + :sugared: ``Dict[str, Union[str, TypeRef]]`` + :canonical: ``Dict[str, TypeRef]`` + + :param members: The members value to normalize. + + :return: None, ``members`` is normalized in-place as needed. + """ if isinstance(members, dict): for key, arg in members.items(): if isinstance(arg, dict): @@ -184,6 +329,26 @@ def check_type(value: Optional[object], source: str, allow_array: bool = False, allow_dict: Union[bool, str] = False) -> None: + """ + Normalize and validate the QAPI type of ``value``. + + Python types of ``str`` or ``None`` are always allowed. + + :param value: The value to check. + :param info: QAPI schema source file information. + :param source: Error string describing this ``value``. + :param allow_array: + Allow a ``List[str]`` of length 1, which indicates an array of + the type named by the list element. + :param allow_dict: + Allow a dict. Its members can be struct type members or union + branches. When the value of ``allow_dict`` is in pragma + ``member-name-exceptions``, the dict's keys may violate the + member naming rules. The dict members are normalized in place. + + :raise QAPISemError: When ``value`` fails validation. + :return: None, ``value`` is normalized in-place as needed. + """ if value is None: return @@ -232,6 +397,22 @@ def check_type(value: Optional[object], def check_features(features: Optional[object], info: QAPISourceInfo) -> None: + """ + Normalize and validate the ``features`` member. + + ``features`` may be a ``list`` of either ``str`` or ``dict``. + Any ``str`` element will be normalized to ``{'name': element}``. + + :forms: + :sugared: ``List[Union[str, Feature]]`` + :canonical: ``List[Feature]`` + + :param features: The features member value to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``features`` fails validation. + :return: None, ``features`` is normalized in-place as needed. + """ if features is None: return if not isinstance(features, list): @@ -249,6 +430,15 @@ def check_features(features: Optional[object], def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as an ``enum`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``expr`` is not a valid ``enum``. + :return: None, ``expr`` is normalized in-place as needed. + """ name = expr['enum'] members = expr['data'] prefix = expr.get('prefix') @@ -278,6 +468,15 @@ def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_struct(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as a ``struct`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``expr`` is not a valid ``struct``. + :return: None, ``expr`` is normalized in-place as needed. + """ name = cast(str, expr['struct']) # Checked in check_exprs members = expr['data'] @@ -286,6 +485,15 @@ def check_struct(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_union(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as a ``union`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: when ``expr`` is not a valid ``union``. + :return: None, ``expr`` is normalized in-place as needed. + """ name = cast(str, expr['union']) # Checked in check_exprs base = expr.get('base') discriminator = expr.get('discriminator') @@ -314,6 +522,15 @@ def check_union(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_alternate(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as an ``alternate`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``expr`` is not a valid ``alternate``. + :return: None, ``expr`` is normalized in-place as needed. + """ members = expr['data'] if not members: @@ -331,6 +548,15 @@ def check_alternate(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_command(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as a ``command`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``expr`` is not a valid ``command``. + :return: None, ``expr`` is normalized in-place as needed. + """ args = expr.get('data') rets = expr.get('returns') boxed = expr.get('boxed', False) @@ -342,6 +568,15 @@ def check_command(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_event(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as an ``event`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``expr`` is not a valid ``event``. + :return: None, ``expr`` is normalized in-place as needed. + """ args = expr.get('data') boxed = expr.get('boxed', False) @@ -351,6 +586,17 @@ def check_event(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_exprs(exprs: List[_JSONObject]) -> List[_JSONObject]: + """ + Validate and normalize a list of parsed QAPI schema expressions. + + This function accepts a list of expressions and metadata as returned + by the parser. It destructively normalizes the expressions in-place. + + :param exprs: The list of expressions to normalize and validate. + + :raise QAPISemError: When any expression fails validation. + :return: The same list of expressions (now modified). + """ for expr_elem in exprs: # Expression assert isinstance(expr_elem['expr'], dict) -- cgit v1.1 From eab99939a7cd289a8b50c2e7ef6a38ca2706a46c Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:31 -0400 Subject: qapi/expr.py: Use tuples instead of lists for static data It is -- maybe -- possibly -- three nanoseconds faster. Signed-off-by: John Snow Reviewed-by: Eduardo Habkost Reviewed-by: Cleber Rosa Message-Id: <20210421182032.3521476-17-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 0b66d80..225a82e 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -239,11 +239,11 @@ def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: When certain flags have an invalid value, or when incompatible flags are present. """ - for key in ['gen', 'success-response']: + for key in ('gen', 'success-response'): if key in expr and expr[key] is not False: raise QAPISemError( info, "flag '%s' may only use false value" % key) - for key in ['boxed', 'allow-oob', 'allow-preconfig', 'coroutine']: + for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): if key in expr and expr[key] is not True: raise QAPISemError( info, "flag '%s' may only use true value" % key) -- cgit v1.1 From e81718c698a9f1a1d98edd605f508dadbffe0d4d Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:32 -0400 Subject: qapi/expr: Update authorship and copyright information Signed-off-by: John Snow Message-Id: <20210421182032.3521476-18-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts/qapi/expr.py') diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 225a82e..496f7e0 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -1,13 +1,14 @@ # -*- coding: utf-8 -*- # # Copyright IBM, Corp. 2011 -# Copyright (c) 2013-2019 Red Hat Inc. +# Copyright (c) 2013-2021 Red Hat Inc. # # Authors: # Anthony Liguori # Markus Armbruster # Eric Blake # Marc-André Lureau +# John Snow # # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. -- cgit v1.1