diff options
27 files changed, 573 insertions, 386 deletions
diff --git a/docs/markdown/Reference-manual.md b/docs/markdown/Reference-manual.md index 8ef36de..31ed77e 100644 --- a/docs/markdown/Reference-manual.md +++ b/docs/markdown/Reference-manual.md @@ -2268,6 +2268,7 @@ are immutable, all operations return their results as a new string. - `join(list_of_strings)`: the opposite of split, for example `'.'.join(['a', 'b', 'c']` yields `'a.b.c'`. + *(Since 0.60.0)* more than one argument is supported and lists will be flattened. - `replace('old_substr', 'new_str')` *(since 0.58.0)*: replaces instances of `old_substr` in the string with `new_str` and returns a new string diff --git a/docs/markdown/Syntax.md b/docs/markdown/Syntax.md index e3a70c7..33b06cb 100644 --- a/docs/markdown/Syntax.md +++ b/docs/markdown/Syntax.md @@ -178,6 +178,18 @@ These are raw strings that do not support the escape sequences listed above. These strings can also be combined with the string formatting functionality described below. +### String index + +Stings support the indexing (`[<num>]`) operator. This operator allows (read +only) acessing a specific character. The returned value is guaranteed to be +a string of length 1. + +```meson +foo = 'abcd' +message(foo[1]) # Will print 'b' +foo[2] = 'C' # ERROR: Meson objects are immutable! +``` + ### String formatting #### .format() diff --git a/docs/markdown/snippets/str_join.md b/docs/markdown/snippets/str_join.md new file mode 100644 index 0000000..b430d66 --- /dev/null +++ b/docs/markdown/snippets/str_join.md @@ -0,0 +1,5 @@ +## Relax restrictions of `str.join()` + +Since 0.60.0, the [[str.join]] method can take an arbitrary number of arguments +instead of just one list. Additionally, all lists past to [[str.join]] will now +be flattened. diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index 4fd1378..5998e5b 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -364,7 +364,8 @@ class AstInterpreter(InterpreterBase): mkwargs = {} # type: T.Dict[str, TYPE_nvar] try: if isinstance(src, str): - result = self.string_method_call(src, node.name, margs, mkwargs) + from ..interpreter import Interpreter, StringHolder + result = StringHolder(src, T.cast(Interpreter, self)).method_call(node.name, margs, mkwargs) elif isinstance(src, bool): from ..interpreter import Interpreter, BooleanHolder result = BooleanHolder(src, T.cast(Interpreter, self)).method_call(node.name, margs, mkwargs) diff --git a/mesonbuild/build.py b/mesonbuild/build.py index 2ee2d4a..24eff8c 100644 --- a/mesonbuild/build.py +++ b/mesonbuild/build.py @@ -1470,7 +1470,7 @@ You probably should put it in link_with instead.''') # If the user set the link_language, just return that. if self.link_language: comp = all_compilers[self.link_language] - return comp, comp.language_stdlib_only_link_flags() + return comp, comp.language_stdlib_only_link_flags(self.environment) # Languages used by dependencies dep_langs = self.get_langs_used_by_deps() @@ -1488,7 +1488,7 @@ You probably should put it in link_with instead.''') added_languages: T.Set[str] = set() for dl in itertools.chain(self.compilers, dep_langs): if dl != linker.language: - stdlib_args += all_compilers[dl].language_stdlib_only_link_flags() + stdlib_args += all_compilers[dl].language_stdlib_only_link_flags(self.environment) added_languages.add(dl) # Type of var 'linker' is Compiler. # Pretty hard to fix because the return value is passed everywhere @@ -2524,6 +2524,9 @@ class CustomTarget(Target, CommandBase): for i in self.outputs: yield CustomTargetIndex(self, i) + def __len__(self) -> int: + return len(self.outputs) + class RunTarget(Target, CommandBase): def __init__(self, name: str, command, dependencies, subdir: str, subproject: str, env: T.Optional['EnvironmentVariables'] = None): diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py index de5e472..6896b76 100644 --- a/mesonbuild/compilers/compilers.py +++ b/mesonbuild/compilers/compilers.py @@ -893,7 +893,7 @@ class Compiler(HoldableObject, metaclass=abc.ABCMeta): def openmp_link_flags(self) -> T.List[str]: return self.openmp_flags() - def language_stdlib_only_link_flags(self) -> T.List[str]: + def language_stdlib_only_link_flags(self, env: 'Environment') -> T.List[str]: return [] def gnu_symbol_visibility_args(self, vistype: str) -> T.List[str]: diff --git a/mesonbuild/compilers/cpp.py b/mesonbuild/compilers/cpp.py index ecc911d..6cbc265 100644 --- a/mesonbuild/compilers/cpp.py +++ b/mesonbuild/compilers/cpp.py @@ -246,13 +246,31 @@ class ClangCPPCompiler(ClangCompiler, CPPCompiler): return libs return [] - def language_stdlib_only_link_flags(self) -> T.List[str]: - return ['-lstdc++'] + def language_stdlib_only_link_flags(self, env: 'Environment') -> T.List[str]: + # We need to apply the search prefix here, as these link arguments may + # be passed to a differen compiler with a different set of default + # search paths, such as when using Clang for C/C++ and gfortran for + # fortran, + search_dir = self._get_search_dirs(env) + search_dirs: T.List[str] = [] + if search_dir is not None: + for d in search_dir.split()[-1][len('libraries: ='):].split(':'): + search_dirs.append(f'-L{d}') + return search_dirs + ['-lstdc++'] class AppleClangCPPCompiler(ClangCPPCompiler): - def language_stdlib_only_link_flags(self) -> T.List[str]: - return ['-lc++'] + def language_stdlib_only_link_flags(self, env: 'Environment') -> T.List[str]: + # We need to apply the search prefix here, as these link arguments may + # be passed to a differen compiler with a different set of default + # search paths, such as when using Clang for C/C++ and gfortran for + # fortran, + search_dir = self._get_search_dirs(env) + search_dirs: T.List[str] = [] + if search_dir is not None: + for d in search_dir.split()[-1][len('libraries: ='):].split(':'): + search_dirs.append(f'-L{d}') + return search_dirs + ['-lc++'] class EmscriptenCPPCompiler(EmscriptenMixin, ClangCPPCompiler): @@ -396,7 +414,16 @@ class GnuCPPCompiler(GnuCompiler, CPPCompiler): def get_pch_use_args(self, pch_dir: str, header: str) -> T.List[str]: return ['-fpch-preprocess', '-include', os.path.basename(header)] - def language_stdlib_only_link_flags(self) -> T.List[str]: + def language_stdlib_only_link_flags(self, env: 'Environment') -> T.List[str]: + # We need to apply the search prefix here, as these link arguments may + # be passed to a differen compiler with a different set of default + # search paths, such as when using Clang for C/C++ and gfortran for + # fortran, + search_dir = self._get_search_dirs(env) + search_dirs: T.List[str] = [] + if search_dir is not None: + for d in search_dir.split()[-1][len('libraries: ='):].split(':'): + search_dirs.append(f'-L{d}') return ['-lstdc++'] diff --git a/mesonbuild/compilers/fortran.py b/mesonbuild/compilers/fortran.py index 639c40f..5d72f4b 100644 --- a/mesonbuild/compilers/fortran.py +++ b/mesonbuild/compilers/fortran.py @@ -211,8 +211,17 @@ class GnuFortranCompiler(GnuCompiler, FortranCompiler): def get_module_outdir_args(self, path: str) -> T.List[str]: return ['-J' + path] - def language_stdlib_only_link_flags(self) -> T.List[str]: - return ['-lgfortran', '-lm'] + def language_stdlib_only_link_flags(self, env: 'Environment') -> T.List[str]: + # We need to apply the search prefix here, as these link arguments may + # be passed to a differen compiler with a different set of default + # search paths, such as when using Clang for C/C++ and gfortran for + # fortran, + search_dir = self._get_search_dirs(env) + search_dirs: T.List[str] = [] + if search_dir is not None: + for d in search_dir.split()[-1][len('libraries: ='):].split(':'): + search_dirs.append(f'-L{d}') + return search_dirs + ['-lgfortran', '-lm'] def has_header(self, hname: str, prefix: str, env: 'Environment', *, extra_args: T.Union[None, T.List[str], T.Callable[['CompileCheckMode'], T.List[str]]] = None, @@ -336,7 +345,8 @@ class IntelFortranCompiler(IntelGnuLikeCompiler, FortranCompiler): def get_preprocess_only_args(self) -> T.List[str]: return ['-cpp', '-EP'] - def language_stdlib_only_link_flags(self) -> T.List[str]: + def language_stdlib_only_link_flags(self, env: 'Environment') -> T.List[str]: + # TODO: needs default search path added return ['-lifcore', '-limf'] def get_dependency_gen_args(self, outtarget: str, outfile: str) -> T.List[str]: @@ -420,7 +430,8 @@ class PGIFortranCompiler(PGICompiler, FortranCompiler): '2': default_warn_args, '3': default_warn_args + ['-Mdclchk']} - def language_stdlib_only_link_flags(self) -> T.List[str]: + def language_stdlib_only_link_flags(self, env: 'Environment') -> T.List[str]: + # TODO: needs default search path added return ['-lpgf90rtl', '-lpgf90', '-lpgf90_rpm1', '-lpgf902', '-lpgf90rtl', '-lpgftnrtl', '-lrt'] @@ -461,8 +472,18 @@ class FlangFortranCompiler(ClangCompiler, FortranCompiler): '2': default_warn_args, '3': default_warn_args} - def language_stdlib_only_link_flags(self) -> T.List[str]: - return ['-lflang', '-lpgmath'] + def language_stdlib_only_link_flags(self, env: 'Environment') -> T.List[str]: + # We need to apply the search prefix here, as these link arguments may + # be passed to a differen compiler with a different set of default + # search paths, such as when using Clang for C/C++ and gfortran for + # fortran, + # XXX: Untested.... + search_dir = self._get_search_dirs(env) + search_dirs: T.List[str] = [] + if search_dir is not None: + for d in search_dir.split()[-1][len('libraries: ='):].split(':'): + search_dirs.append(f'-L{d}') + return search_dirs + ['-lflang', '-lpgmath'] class Open64FortranCompiler(FortranCompiler): diff --git a/mesonbuild/interpreter/__init__.py b/mesonbuild/interpreter/__init__.py index 90d7faf..c93dbc9 100644 --- a/mesonbuild/interpreter/__init__.py +++ b/mesonbuild/interpreter/__init__.py @@ -37,6 +37,7 @@ __all__ = [ 'BooleanHolder', 'IntegerHolder', + 'StringHolder', ] from .interpreter import Interpreter, permitted_dependency_kwargs @@ -50,4 +51,5 @@ from .interpreterobjects import (ExecutableHolder, BuildTargetHolder, CustomTarg from .primitives import ( BooleanHolder, IntegerHolder, + StringHolder, ) diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py index f94ed2d..7a935da 100644 --- a/mesonbuild/interpreter/interpreter.py +++ b/mesonbuild/interpreter/interpreter.py @@ -376,6 +376,8 @@ class Interpreter(InterpreterBase, HoldableObject): # Primitives int: P_OBJ.IntegerHolder, bool: P_OBJ.BooleanHolder, + str: P_OBJ.StringHolder, + P_OBJ.MesonVersionString: P_OBJ.MesonVersionStringHolder, # Meson types mesonlib.File: OBJ.FileHolder, @@ -2399,7 +2401,7 @@ This will become a hard error in the future.''' % kwargs['input'], location=self @typed_pos_args('join_paths', varargs=str, min_varargs=1) @noKwargs def func_join_paths(self, node: mparser.BaseNode, args: T.Tuple[T.List[str]], kwargs: 'TYPE_kwargs') -> str: - return self.join_path_strings(args[0]) + return os.path.join(*args[0]).replace('\\', '/') def run(self) -> None: super().run() diff --git a/mesonbuild/interpreter/interpreterobjects.py b/mesonbuild/interpreter/interpreterobjects.py index 78c7fb9..fca371c 100644 --- a/mesonbuild/interpreter/interpreterobjects.py +++ b/mesonbuild/interpreter/interpreterobjects.py @@ -14,10 +14,10 @@ from .. import mlog from ..modules import ModuleReturnValue, ModuleObject, ModuleState, ExtensionModule from ..backend.backends import TestProtocol from ..interpreterbase import ( - ContainerTypeInfo, KwargInfo, + ContainerTypeInfo, KwargInfo, MesonOperator, InterpreterObject, MesonInterpreterObject, ObjectHolder, MutableInterpreterObject, FeatureCheckBase, FeatureNewKwargs, FeatureNew, FeatureDeprecated, - typed_pos_args, typed_kwargs, permittedKwargs, + typed_pos_args, typed_kwargs, typed_operator, permittedKwargs, noArgsFlattening, noPosargs, noKwargs, unholder_return, TYPE_var, TYPE_kwargs, TYPE_nvar, TYPE_nkwargs, flatten, resolve_second_level_holders, InterpreterException, InvalidArguments, InvalidCode) from ..interpreter.type_checking import NoneType @@ -912,6 +912,10 @@ class CustomTargetHolder(ObjectHolder[build.CustomTarget]): 'to_list': self.to_list_method, }) + self.operators.update({ + MesonOperator.INDEX: self.op_index, + }) + def __repr__(self) -> str: r = '<{} {}: {}>' h = self.held_object @@ -931,14 +935,13 @@ class CustomTargetHolder(ObjectHolder[build.CustomTarget]): result.append(i) return result - def __getitem__(self, index: int) -> build.CustomTargetIndex: - return self.held_object[index] - - def __setitem__(self, index: int, value: T.Any) -> None: # lgtm[py/unexpected-raise-in-special-method] - raise InterpreterException('Cannot set a member of a CustomTarget') - - def __delitem__(self, index: int) -> None: # lgtm[py/unexpected-raise-in-special-method] - raise InterpreterException('Cannot delete a member of a CustomTarget') + @noKwargs + @typed_operator(MesonOperator.INDEX, int) + def op_index(self, other: int) -> build.CustomTargetIndex: + try: + return self.held_object[other] + except IndexError: + raise InvalidArguments(f'Index {other} out of bounds of custom target {self.held_object.name} output of size {len(self.held_object)}.') class RunTargetHolder(ObjectHolder[build.RunTarget]): pass diff --git a/mesonbuild/interpreter/mesonmain.py b/mesonbuild/interpreter/mesonmain.py index 637ca72..15c1082 100644 --- a/mesonbuild/interpreter/mesonmain.py +++ b/mesonbuild/interpreter/mesonmain.py @@ -14,8 +14,9 @@ from ..mesonlib import MachineChoice, OptionKey from ..programs import OverrideProgram, ExternalProgram from ..interpreter.type_checking import ENV_KW from ..interpreterbase import (MesonInterpreterObject, FeatureNew, FeatureDeprecated, - typed_pos_args, noArgsFlattening, noPosargs, noKwargs, - typed_kwargs, KwargInfo, MesonVersionString, InterpreterException) + typed_pos_args, noArgsFlattening, noPosargs, noKwargs, + typed_kwargs, KwargInfo, InterpreterException) +from .primitives import MesonVersionString from .type_checking import NATIVE_KW, NoneType if T.TYPE_CHECKING: diff --git a/mesonbuild/interpreter/primitives/__init__.py b/mesonbuild/interpreter/primitives/__init__.py index 5d16744..d6c0795 100644 --- a/mesonbuild/interpreter/primitives/__init__.py +++ b/mesonbuild/interpreter/primitives/__init__.py @@ -4,7 +4,11 @@ __all__ = [ 'BooleanHolder', 'IntegerHolder', + 'StringHolder', + 'MesonVersionString', + 'MesonVersionStringHolder', ] from .boolean import BooleanHolder from .integer import IntegerHolder +from .string import StringHolder, MesonVersionString, MesonVersionStringHolder diff --git a/mesonbuild/interpreter/primitives/string.py b/mesonbuild/interpreter/primitives/string.py new file mode 100644 index 0000000..d9c441a --- /dev/null +++ b/mesonbuild/interpreter/primitives/string.py @@ -0,0 +1,194 @@ +# Copyright 2021 The Meson development team +# SPDX-license-identifier: Apache-2.0 + +import re +from pathlib import PurePath + +import typing as T + +from ...mesonlib import version_compare +from ...interpreterbase import ( + ObjectHolder, + MesonOperator, + FeatureNew, + typed_operator, + noKwargs, + noPosargs, + typed_pos_args, + + TYPE_var, + TYPE_kwargs, + + InvalidArguments, +) +from ...mparser import ( + MethodNode, + StringNode, + ArrayNode, +) + + +if T.TYPE_CHECKING: + # Object holders need the actual interpreter + from ...interpreter import Interpreter + +class StringHolder(ObjectHolder[str]): + def __init__(self, obj: str, interpreter: 'Interpreter') -> None: + super().__init__(obj, interpreter) + self.methods.update({ + 'contains': self.contains_method, + 'startswith': self.startswith_method, + 'endswith': self.endswith_method, + 'format': self.format_method, + 'join': self.join_method, + 'replace': self.replace_method, + 'split': self.split_method, + 'strip': self.strip_method, + 'substring': self.substring_method, + 'to_int': self.to_int_method, + 'to_lower': self.to_lower_method, + 'to_upper': self.to_upper_method, + 'underscorify': self.underscorify_method, + 'version_compare': self.version_compare_method, + }) + + + self.trivial_operators.update({ + # Arithmetic + MesonOperator.PLUS: (str, lambda x: self.held_object + x), + + # Comparison + MesonOperator.EQUALS: (str, lambda x: self.held_object == x), + MesonOperator.NOT_EQUALS: (str, lambda x: self.held_object != x), + MesonOperator.GREATER: (str, lambda x: self.held_object > x), + MesonOperator.LESS: (str, lambda x: self.held_object < x), + MesonOperator.GREATER_EQUALS: (str, lambda x: self.held_object >= x), + MesonOperator.LESS_EQUALS: (str, lambda x: self.held_object <= x), + }) + + # Use actual methods for functions that require additional checks + self.operators.update({ + MesonOperator.DIV: self.op_div, + MesonOperator.INDEX: self.op_index, + }) + + def display_name(self) -> str: + return 'str' + + @noKwargs + @typed_pos_args('str.contains', str) + def contains_method(self, args: T.Tuple[str], kwargs: TYPE_kwargs) -> bool: + return self.held_object.find(args[0]) >= 0 + + @noKwargs + @typed_pos_args('str.startswith', str) + def startswith_method(self, args: T.Tuple[str], kwargs: TYPE_kwargs) -> bool: + return self.held_object.startswith(args[0]) + + @noKwargs + @typed_pos_args('str.endswith', str) + def endswith_method(self, args: T.Tuple[str], kwargs: TYPE_kwargs) -> bool: + return self.held_object.endswith(args[0]) + + @noKwargs + @typed_pos_args('str.format', varargs=object) + def format_method(self, args: T.Tuple[T.List[object]], kwargs: TYPE_kwargs) -> str: + arg_strings: T.List[str] = [] + for arg in args[0]: + if isinstance(arg, bool): # Python boolean is upper case. + arg = str(arg).lower() + arg_strings.append(str(arg)) + + def arg_replace(match: T.Match[str]) -> str: + idx = int(match.group(1)) + if idx >= len(arg_strings): + raise InvalidArguments(f'Format placeholder @{idx}@ out of range.') + return arg_strings[idx] + + return re.sub(r'@(\d+)@', arg_replace, self.held_object) + + @noKwargs + @typed_pos_args('str.join', varargs=str) + def join_method(self, args: T.Tuple[T.List[str]], kwargs: TYPE_kwargs) -> str: + # Implement some basic FeatureNew check on the AST level + assert isinstance(self.current_node, MethodNode) + n_args = self.current_node.args.arguments + if len(n_args) != 1 or not isinstance(n_args[0], ArrayNode) or not all(isinstance(x, StringNode) for x in n_args[0].args.arguments): + FeatureNew.single_use('str.join (varargs)', '0.60.0', self.subproject, 'List-flattening and variadic arguments') + + # Actual implementation + return self.held_object.join(args[0]) + + @noKwargs + @typed_pos_args('str.replace', str, str) + def replace_method(self, args: T.Tuple[str, str], kwargs: TYPE_kwargs) -> str: + return self.held_object.replace(args[0], args[1]) + + @noKwargs + @typed_pos_args('str.split', optargs=[str]) + def split_method(self, args: T.Tuple[T.Optional[str]], kwargs: TYPE_kwargs) -> T.List[str]: + return self.held_object.split(args[0]) + + @noKwargs + @typed_pos_args('str.strip', optargs=[str]) + def strip_method(self, args: T.Tuple[T.Optional[str]], kwargs: TYPE_kwargs) -> str: + return self.held_object.strip(args[0]) + + @noKwargs + @typed_pos_args('str.substring', optargs=[int, int]) + def substring_method(self, args: T.Tuple[T.Optional[int], T.Optional[int]], kwargs: TYPE_kwargs) -> str: + start = args[0] if args[0] is not None else 0 + end = args[1] if args[1] is not None else len(self.held_object) + return self.held_object[start:end] + + @noKwargs + @noPosargs + def to_int_method(self, args: T.List[TYPE_var], kwargs: TYPE_kwargs) -> int: + try: + return int(self.held_object) + except ValueError: + raise InvalidArguments(f'String {self.held_object!r} cannot be converted to int') + + @noKwargs + @noPosargs + def to_lower_method(self, args: T.List[TYPE_var], kwargs: TYPE_kwargs) -> str: + return self.held_object.lower() + + @noKwargs + @noPosargs + def to_upper_method(self, args: T.List[TYPE_var], kwargs: TYPE_kwargs) -> str: + return self.held_object.upper() + + @noKwargs + @noPosargs + def underscorify_method(self, args: T.List[TYPE_var], kwargs: TYPE_kwargs) -> str: + return re.sub(r'[^a-zA-Z0-9]', '_', self.held_object) + + @noKwargs + @typed_pos_args('str.version_compare', str) + def version_compare_method(self, args: T.Tuple[str], kwargs: TYPE_kwargs) -> bool: + return version_compare(self.held_object, args[0]) + + + @FeatureNew('/ with string arguments', '0.49.0') + @typed_operator(MesonOperator.DIV, str) + def op_div(self, other: str) -> str: + return (PurePath(self.held_object) / other).as_posix() + + @typed_operator(MesonOperator.INDEX, int) + def op_index(self, other: int) -> str: + try: + return self.held_object[other] + except IndexError: + raise InvalidArguments(f'Index {other} out of bounds of string of size {len(self.held_object)}.') + + +class MesonVersionString(str): + pass + +class MesonVersionStringHolder(StringHolder): + @noKwargs + @typed_pos_args('str.version_compare', str) + def version_compare_method(self, args: T.Tuple[str], kwargs: TYPE_kwargs) -> bool: + self.interpreter.tmp_meson_version = args[0] + return version_compare(self.held_object, args[0]) diff --git a/mesonbuild/interpreterbase/__init__.py b/mesonbuild/interpreterbase/__init__.py index f237c2f..0375430 100644 --- a/mesonbuild/interpreterbase/__init__.py +++ b/mesonbuild/interpreterbase/__init__.py @@ -17,7 +17,6 @@ __all__ = [ 'MesonInterpreterObject', 'ObjectHolder', 'RangeHolder', - 'MesonVersionString', 'MutableInterpreterObject', 'MesonOperator', @@ -32,7 +31,6 @@ __all__ = [ 'ContinueRequest', 'BreakRequest', - 'check_stringlist', 'default_resolve_key', 'flatten', 'resolve_second_level_holders', @@ -128,6 +126,6 @@ from .exceptions import ( ) from .disabler import Disabler, is_disabled -from .helpers import check_stringlist, default_resolve_key, flatten, resolve_second_level_holders -from .interpreterbase import MesonVersionString, InterpreterBase +from .helpers import default_resolve_key, flatten, resolve_second_level_holders +from .interpreterbase import InterpreterBase from .operator import MesonOperator diff --git a/mesonbuild/interpreterbase/_unholder.py b/mesonbuild/interpreterbase/_unholder.py index 202f53b..221c52c 100644 --- a/mesonbuild/interpreterbase/_unholder.py +++ b/mesonbuild/interpreterbase/_unholder.py @@ -19,9 +19,7 @@ from ..mesonlib import HoldableObject, MesonBugException import typing as T def _unholder(obj: T.Union[TYPE_var, InterpreterObject]) -> TYPE_var: - if isinstance(obj, str): - return obj - elif isinstance(obj, list): + if isinstance(obj, list): return [_unholder(x) for x in obj] elif isinstance(obj, dict): return {k: _unholder(v) for k, v in obj.items()} diff --git a/mesonbuild/interpreterbase/baseobjects.py b/mesonbuild/interpreterbase/baseobjects.py index 80cf0b5..62a2381 100644 --- a/mesonbuild/interpreterbase/baseobjects.py +++ b/mesonbuild/interpreterbase/baseobjects.py @@ -129,8 +129,8 @@ class MesonInterpreterObject(InterpreterObject): class MutableInterpreterObject: ''' Dummy class to mark the object type as mutable ''' -HoldableTypes = (HoldableObject, int, bool) -TYPE_HoldableTypes = T.Union[HoldableObject, int, bool] +HoldableTypes = (HoldableObject, int, bool, str) +TYPE_HoldableTypes = T.Union[HoldableObject, int, bool, str] InterpreterObjectTypeVar = T.TypeVar('InterpreterObjectTypeVar', bound=TYPE_HoldableTypes) class ObjectHolder(InterpreterObject, T.Generic[InterpreterObjectTypeVar]): @@ -168,12 +168,18 @@ class RangeHolder(MesonInterpreterObject): def __init__(self, start: int, stop: int, step: int, *, subproject: str) -> None: super().__init__(subproject=subproject) self.range = range(start, stop, step) + self.operators.update({ + MesonOperator.INDEX: self.op_index, + }) + + def op_index(self, other: int) -> int: + try: + return self.range[other] + except: + raise InvalidArguments(f'Index {other} out of bounds of range.') def __iter__(self) -> T.Iterator[int]: return iter(self.range) - def __getitem__(self, key: int) -> int: - return self.range[key] - def __len__(self) -> int: return len(self.range) diff --git a/mesonbuild/interpreterbase/decorators.py b/mesonbuild/interpreterbase/decorators.py index b9c4a1f..54f4be3 100644 --- a/mesonbuild/interpreterbase/decorators.py +++ b/mesonbuild/interpreterbase/decorators.py @@ -16,7 +16,6 @@ from .. import mesonlib, mlog from .baseobjects import TV_func, TYPE_var, TYPE_kwargs from .disabler import Disabler from .exceptions import InterpreterException, InvalidArguments -from .helpers import check_stringlist from .operator import MesonOperator from ._unholder import _unholder @@ -64,8 +63,12 @@ def stringArgs(f: TV_func) -> TV_func: @wraps(f) def wrapped(*wrapped_args: T.Any, **wrapped_kwargs: T.Any) -> T.Any: args = get_callee_args(wrapped_args)[1] - assert isinstance(args, list) - check_stringlist(args) + if not isinstance(args, list): + mlog.debug('Not a list:', str(args)) + raise InvalidArguments('Argument not a list.') + if not all(isinstance(s, str) for s in args): + mlog.debug('Element not a string:', str(args)) + raise InvalidArguments('Arguments must be strings.') return f(*wrapped_args, **wrapped_kwargs) return T.cast(TV_func, wrapped) diff --git a/mesonbuild/interpreterbase/helpers.py b/mesonbuild/interpreterbase/helpers.py index 3d45e1f..12fa813 100644 --- a/mesonbuild/interpreterbase/helpers.py +++ b/mesonbuild/interpreterbase/helpers.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .. import mesonlib, mparser, mlog -from .exceptions import InvalidArguments, InterpreterException +from .. import mesonlib, mparser +from .exceptions import InterpreterException import collections.abc import typing as T @@ -49,14 +49,6 @@ def resolve_second_level_holders(args: T.List['TYPE_var'], kwargs: 'TYPE_kwargs' return arg return [resolver(x) for x in args], {k: resolver(v) for k, v in kwargs.items()} -def check_stringlist(a: T.Any, msg: str = 'Arguments must be strings.') -> None: - if not isinstance(a, list): - mlog.debug('Not a list:', str(a)) - raise InvalidArguments('Argument not a list.') - if not all(isinstance(s, str) for s in a): - mlog.debug('Element not a string:', str(a)) - raise InvalidArguments(msg) - def default_resolve_key(key: mparser.BaseNode) -> str: if not isinstance(key, mparser.IdNode): raise InterpreterException('Invalid kwargs format.') diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index 6e942b7..4b4b3c0 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -44,7 +44,7 @@ from .exceptions import ( from .decorators import FeatureNew, noKwargs from .disabler import Disabler, is_disabled -from .helpers import check_stringlist, default_resolve_key, flatten, resolve_second_level_holders +from .helpers import default_resolve_key, flatten, resolve_second_level_holders from .operator import MesonOperator from ._unholder import _unholder @@ -62,6 +62,8 @@ HolderMapType = T.Dict[ T.Union[ T.Type[mesonlib.HoldableObject], T.Type[int], + T.Type[bool], + T.Type[str], ], # For some reason, this has to be a callable and can't just be ObjectHolder[InterpreterObjectTypeVar] T.Callable[[InterpreterObjectTypeVar, 'Interpreter'], ObjectHolder[InterpreterObjectTypeVar]] @@ -84,11 +86,8 @@ def _holderify_result(types: T.Union[None, T.Type, T.Tuple[T.Type, ...]] = None) return T.cast(__FN, wrapper) return inner -class MesonVersionString(str): - pass - class InterpreterBase: - elementary_types = (str, list) + elementary_types = (list, ) def __init__(self, source_root: str, subdir: str, subproject: str): self.source_root = source_root @@ -128,9 +127,6 @@ class InterpreterBase: me.file = mesonfile raise me - def join_path_strings(self, args: T.Sequence[str]) -> str: - return os.path.join(*args).replace('\\', '/') - def parse_project(self) -> None: """ Parses project() and initializes languages, compilers etc. Do this @@ -203,7 +199,7 @@ class InterpreterBase: elif isinstance(cur, mparser.MethodNode): return self.method_call(cur) elif isinstance(cur, mparser.StringNode): - return cur.value + return self._holderify(cur.value) elif isinstance(cur, mparser.BooleanNode): return self._holderify(cur.value) elif isinstance(cur, mparser.IfClauseNode): @@ -259,7 +255,7 @@ class InterpreterBase: def resolve_key(key: mparser.BaseNode) -> str: if not isinstance(key, mparser.StringNode): FeatureNew.single_use('Dictionary entry using non literal key', '0.53.0', self.subproject) - str_key = self.evaluate_statement(key) + str_key = _unholder(self.evaluate_statement(key)) if not isinstance(str_key, str): raise InvalidArguments('Key must be a string') return str_key @@ -382,13 +378,13 @@ class InterpreterBase: # Use type: ignore because mypy will complain that we are comparing two Unions, # but we actually guarantee earlier that both types are the same elif node.ctype == '<': - return val1 < val2 # type: ignore + return val1 < val2 elif node.ctype == '<=': - return val1 <= val2 # type: ignore + return val1 <= val2 elif node.ctype == '>': - return val1 > val2 # type: ignore + return val1 > val2 elif node.ctype == '>=': - return val1 >= val2 # type: ignore + return val1 >= val2 else: raise InvalidCode('You broke my compare eval.') @@ -436,14 +432,6 @@ class InterpreterBase: raise InterpreterException(f'Argument to negation ({v}) is not an InterpreterObject but {type(v).__name__}.') return v.operator_call(MesonOperator.UMINUS, None) - @FeatureNew('/ with string arguments', '0.49.0') - def evaluate_path_join(self, l: str, r: str) -> str: - if not isinstance(l, str): - raise InvalidCode('The division operator can only append to a string.') - if not isinstance(r, str): - raise InvalidCode('The division operator can only append a string.') - return self.join_path_strings((l, r)) - def evaluate_arithmeticstatement(self, cur: mparser.ArithmeticNode) -> T.Union[TYPE_var, InterpreterObject]: l = self.evaluate_statement(cur.left) if isinstance(l, Disabler): @@ -484,9 +472,7 @@ class InterpreterBase: raise InvalidCode('Multiplication works only with integers.') raise mesonlib.MesonBugException('The integer was not held by an ObjectHolder!') elif cur.operation == 'div': - if isinstance(l, str) and isinstance(r, str): - return self.evaluate_path_join(l, r) - raise InvalidCode('Division works only with strings or integers.') + raise mesonlib.MesonBugException('The integer or string was not held by an ObjectHolder!') elif cur.operation == 'mod': if not isinstance(l, int) or not isinstance(r, int): raise InvalidCode('Modulo works only with integers.') @@ -508,7 +494,8 @@ class InterpreterBase: return self.evaluate_statement(node.falseblock) @FeatureNew('format strings', '0.58.0') - def evaluate_fstring(self, node: mparser.FormatStringNode) -> TYPE_var: + @_holderify_result(str) + def evaluate_fstring(self, node: mparser.FormatStringNode) -> str: assert isinstance(node, mparser.FormatStringNode) def replace(match: T.Match[str]) -> str: @@ -545,8 +532,8 @@ class InterpreterBase: if len(node.varnames) != 2: raise InvalidArguments('Foreach on dict unpacks key and value') for key, value in sorted(items.items()): - self.set_variable(node.varnames[0], key) - self.set_variable(node.varnames[1], value) + self.set_variable(node.varnames[0], self._holderify(key)) + self.set_variable(node.varnames[1], self._holderify(value, permissive=True)) try: self.evaluate_codeblock(node.block) except ContinueRequest: @@ -592,11 +579,13 @@ class InterpreterBase: iobject = self.evaluate_statement(node.iobject) if isinstance(iobject, Disabler): return iobject + index = _unholder(self.evaluate_statement(node.index)) + + if isinstance(iobject, InterpreterObject): + return self._holderify(iobject.operator_call(MesonOperator.INDEX, index)) if not hasattr(iobject, '__getitem__'): raise InterpreterException( 'Tried to index an object that doesn\'t support indexing.') - index = _unholder(self.evaluate_statement(node.index)) - if isinstance(iobject, dict): if not isinstance(index, str): raise InterpreterException('Key is not a string') @@ -656,7 +645,7 @@ class InterpreterBase: if is_disabled(args, kwargs): return Disabler() if isinstance(obj, str): - return self._holderify(self.string_method_call(obj, method_name, args, kwargs)) + raise mesonlib.MesonBugException('Strings are now wrapped in object holders!') if isinstance(obj, bool): raise mesonlib.MesonBugException('Booleans are now wrapped in object holders!') if isinstance(obj, int): @@ -680,8 +669,6 @@ class InterpreterBase: # TODO: remove `permissive` once all primitives are ObjectHolders if res is None: return None - if isinstance(res, str): - return res elif isinstance(res, list): return [self._holderify(x, permissive=permissive) for x in res] elif isinstance(res, dict): @@ -722,96 +709,6 @@ class InterpreterBase: return s return None - @noKwargs - def string_method_call(self, obj: str, method_name: str, posargs: T.List[TYPE_var], kwargs: TYPE_kwargs) -> T.Union[str, int, bool, T.List[str]]: - if method_name == 'strip': - s1 = self._get_one_string_posarg(posargs, 'strip') - if s1 is not None: - return obj.strip(s1) - return obj.strip() - elif method_name == 'format': - return self.format_string(obj, posargs) - elif method_name == 'to_upper': - return obj.upper() - elif method_name == 'to_lower': - return obj.lower() - elif method_name == 'underscorify': - return re.sub(r'[^a-zA-Z0-9]', '_', obj) - elif method_name == 'split': - s2 = self._get_one_string_posarg(posargs, 'split') - if s2 is not None: - return obj.split(s2) - return obj.split() - elif method_name == 'startswith' or method_name == 'contains' or method_name == 'endswith': - s3 = posargs[0] - if not isinstance(s3, str): - raise InterpreterException('Argument must be a string.') - if method_name == 'startswith': - return obj.startswith(s3) - elif method_name == 'contains': - return obj.find(s3) >= 0 - return obj.endswith(s3) - elif method_name == 'to_int': - try: - return int(obj) - except Exception: - raise InterpreterException(f'String {obj!r} cannot be converted to int') - elif method_name == 'join': - if len(posargs) != 1: - raise InterpreterException('Join() takes exactly one argument.') - strlist = posargs[0] - check_stringlist(strlist) - assert isinstance(strlist, list) # Required for mypy - return obj.join(strlist) - elif method_name == 'version_compare': - if len(posargs) != 1: - raise InterpreterException('Version_compare() takes exactly one argument.') - cmpr = posargs[0] - if not isinstance(cmpr, str): - raise InterpreterException('Version_compare() argument must be a string.') - if isinstance(obj, MesonVersionString): - self.tmp_meson_version = cmpr - return mesonlib.version_compare(obj, cmpr) - elif method_name == 'substring': - if len(posargs) > 2: - raise InterpreterException('substring() takes maximum two arguments.') - start = 0 - end = len(obj) - if len (posargs) > 0: - if not isinstance(posargs[0], int): - raise InterpreterException('substring() argument must be an int') - start = posargs[0] - if len (posargs) > 1: - if not isinstance(posargs[1], int): - raise InterpreterException('substring() argument must be an int') - end = posargs[1] - return obj[start:end] - elif method_name == 'replace': - FeatureNew.single_use('str.replace', '0.58.0', self.subproject) - if len(posargs) != 2: - raise InterpreterException('replace() takes exactly two arguments.') - if not isinstance(posargs[0], str) or not isinstance(posargs[1], str): - raise InterpreterException('replace() requires that both arguments be strings') - return obj.replace(posargs[0], posargs[1]) - raise InterpreterException('Unknown method "%s" for a string.' % method_name) - - def format_string(self, templ: str, args: T.List[TYPE_var]) -> str: - arg_strings = [] - for arg in args: - if isinstance(arg, mparser.BaseNode): - arg = self.evaluate_statement(arg) - if isinstance(arg, bool): # Python boolean is upper case. - arg = str(arg).lower() - arg_strings.append(str(arg)) - - def arg_replace(match: T.Match[str]) -> str: - idx = int(match.group(1)) - if idx >= len(arg_strings): - raise InterpreterException(f'Format placeholder @{idx}@ out of range.') - return arg_strings[idx] - - return re.sub(r'@(\d+)@', arg_replace, templ) - def unknown_function_called(self, func_name: str) -> None: raise InvalidCode('Unknown function "%s".' % func_name) @@ -992,7 +889,7 @@ To specify a keyword argument, use : instead of =.''') raise InvalidCode('Unknown variable "%s".' % varname) def is_assignable(self, value: T.Any) -> bool: - return isinstance(value, (InterpreterObject, str, int, list, dict)) + return isinstance(value, (InterpreterObject, list, dict)) def validate_extraction(self, buildtarget: mesonlib.HoldableObject) -> None: raise InterpreterException('validate_extraction is not implemented in this context (please file a bug)') diff --git a/mesonbuild/mlog.py b/mesonbuild/mlog.py index 18cbc48..0385e0b 100644 --- a/mesonbuild/mlog.py +++ b/mesonbuild/mlog.py @@ -158,6 +158,9 @@ class AnsiText: def bold(text: str, quoted: bool = False) -> AnsiDecorator: return AnsiDecorator(text, "\033[1m", quoted=quoted) +def italic(text: str, quoted: bool = False) -> AnsiDecorator: + return AnsiDecorator(text, "\033[3m", quoted=quoted) + def plain(text: str) -> AnsiDecorator: return AnsiDecorator(text, "") diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index fd175ba..acb03a5 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -17,6 +17,7 @@ from pathlib import Path from collections import deque from copy import deepcopy +from itertools import islice import argparse import asyncio import datetime @@ -263,9 +264,6 @@ class TestResult(enum.Enum): result_str = '{res:{reslen}}'.format(res=self.value, reslen=self.maxlen()) return self.colorize(result_str).get_text(colorize) - def get_command_marker(self) -> str: - return str(self.colorize('>>> ')) - TYPE_TAPResult = T.Union['TAPParser.Test', 'TAPParser.Error', 'TAPParser.Version', 'TAPParser.Plan', 'TAPParser.Bailout'] @@ -319,6 +317,8 @@ class TAPParser: def parse_test(self, ok: bool, num: int, name: str, directive: T.Optional[str], explanation: T.Optional[str]) -> \ T.Generator[T.Union['TAPParser.Test', 'TAPParser.Error'], None, None]: name = name.strip() + if name[0:2] == '- ': + name = name[2:] explanation = explanation.strip() if explanation else None if directive is not None: directive = directive.upper() @@ -452,8 +452,8 @@ class TestLogger: def start_test(self, harness: 'TestHarness', test: 'TestRun') -> None: pass - def log_subtest(self, harness: 'TestHarness', test: 'TestRun', s: str, res: TestResult) -> None: - pass + def log_subtest(self, harness: 'TestHarness', test: 'TestRun', s: str, res: TestResult) -> str: + return '' def log(self, harness: 'TestHarness', result: 'TestRun') -> None: pass @@ -477,25 +477,15 @@ class TestFileLogger(TestLogger): class ConsoleLogger(TestLogger): - SPINNER = "\U0001f311\U0001f312\U0001f313\U0001f314" + \ - "\U0001f315\U0001f316\U0001f317\U0001f318" - - SCISSORS = "\u2700 " - HLINE = "\u2015" - RTRI = "\u25B6 " - def __init__(self) -> None: self.update = asyncio.Event() self.running_tests = OrderedSet() # type: OrderedSet['TestRun'] - self.progress_test = None # type: T.Optional['TestRun'] self.progress_task = None # type: T.Optional[asyncio.Future] self.max_left_width = 0 # type: int self.stop = False - self.update = asyncio.Event() self.should_erase_line = '' self.test_count = 0 self.started_tests = 0 - self.spinner_index = 0 try: self.cols, _ = os.get_terminal_size(1) self.is_tty = True @@ -503,59 +493,46 @@ class ConsoleLogger(TestLogger): self.cols = 80 self.is_tty = False - self.output_start = dashes(self.SCISSORS, self.HLINE, self.cols - 2) - self.output_end = dashes('', self.HLINE, self.cols - 2) - self.sub = self.RTRI - try: - self.output_start.encode(sys.stdout.encoding or 'ascii') - except UnicodeEncodeError: - self.output_start = dashes('8<', '-', self.cols - 2) - self.output_end = dashes('', '-', self.cols - 2) - self.sub = '| ' - def flush(self) -> None: if self.should_erase_line: print(self.should_erase_line, end='') self.should_erase_line = '' - def print_progress(self, line: str) -> None: - print(self.should_erase_line, line, sep='', end='\r') - self.should_erase_line = '\x1b[K' + def print_progress(self, lines: T.List[str]) -> None: + line_count = len(lines) + if line_count > 0: + self.flush() + for line in lines: + print(line) + print(f'\x1b[{line_count}A', end='') + self.should_erase_line = '\x1b[K' + '\x1b[1B\x1b[K' * (line_count - 1) + if line_count > 1: + self.should_erase_line += f'\x1b[{line_count - 1}A' def request_update(self) -> None: self.update.set() def emit_progress(self, harness: 'TestHarness') -> None: - if self.progress_test is None: - self.flush() - return - - if len(self.running_tests) == 1: - count = f'{self.started_tests}/{self.test_count}' - else: - count = '{}-{}/{}'.format(self.started_tests - len(self.running_tests) + 1, - self.started_tests, self.test_count) - - left = '[{}] {} '.format(count, self.SPINNER[self.spinner_index]) - self.spinner_index = (self.spinner_index + 1) % len(self.SPINNER) - - right = '{spaces} {dur:{durlen}}'.format( - spaces=' ' * TestResult.maxlen(), - dur=int(time.time() - self.progress_test.starttime), - durlen=harness.duration_max_len) - if self.progress_test.timeout: - right += '/{timeout:{durlen}}'.format( - timeout=self.progress_test.timeout, + lines: T.List[str] = [] + for test in islice(reversed(self.running_tests), 10): + left = ' ' * (len(str(self.test_count)) * 2 + 2) + right = '{spaces} {dur:{durlen}}'.format( + spaces=' ' * TestResult.maxlen(), + dur=int(time.time() - test.starttime), durlen=harness.duration_max_len) - right += 's' - detail = self.progress_test.detail - if detail: - right += ' ' + detail - - line = harness.format(self.progress_test, colorize=True, - max_left_width=self.max_left_width, - left=left, right=right) - self.print_progress(line) + if test.timeout: + right += '/{timeout:{durlen}}'.format( + timeout=test.timeout, + durlen=harness.duration_max_len) + right += 's' + lines = [harness.format(test, colorize=True, + max_left_width=self.max_left_width, + left=left, + right=right)] + lines + if len(self.running_tests) > 10: + lines += [' ' * len(harness.get_test_num_prefix(0)) + + f'[{len(self.running_tests) - 10} more tests running]'] + self.print_progress(lines) def start(self, harness: 'TestHarness') -> None: async def report_progress() -> None: @@ -565,26 +542,12 @@ class ConsoleLogger(TestLogger): while not self.stop: await self.update.wait() self.update.clear() - # We may get here simply because the progress line has been # overwritten, so do not always switch. Only do so every # second, or if the printed test has finished if loop.time() >= next_update: - self.progress_test = None next_update = loop.time() + 1 loop.call_at(next_update, self.request_update) - - if (self.progress_test and - self.progress_test.res is not TestResult.RUNNING): - self.progress_test = None - - if not self.progress_test: - if not self.running_tests: - continue - # Pick a test in round robin order - self.progress_test = self.running_tests.pop(last=False) - self.running_tests.add(self.progress_test) - self.emit_progress(harness) self.flush() @@ -602,77 +565,92 @@ class ConsoleLogger(TestLogger): print(harness.format(test, mlog.colorize_console(), max_left_width=self.max_left_width, right=test.res.get_text(mlog.colorize_console()))) - print(test.res.get_command_marker() + test.cmdline) - if test.needs_parsing: - pass - elif not test.is_parallel: - print(self.output_start, flush=True) - else: - print(flush=True) - self.started_tests += 1 self.running_tests.add(test) self.running_tests.move_to_end(test, last=False) self.request_update() - def shorten_log(self, harness: 'TestHarness', result: 'TestRun') -> str: - if not harness.options.verbose and not harness.options.print_errorlogs: - return '' - - log = result.get_log(mlog.colorize_console(), - stderr_only=result.needs_parsing) - if harness.options.verbose: - return log - - lines = log.splitlines() - if len(lines) < 100: - return log - else: - return str(mlog.bold('Listing only the last 100 lines from a long log.\n')) + '\n'.join(lines[-100:]) - - def print_log(self, harness: 'TestHarness', result: 'TestRun') -> None: - if not harness.options.verbose: - cmdline = result.cmdline - if not cmdline: - print(result.res.get_command_marker() + result.stdo) - return - print(result.res.get_command_marker() + cmdline) - - log = self.shorten_log(harness, result) - if log: - print(self.output_start) - print_safe(log) - print(self.output_end) + @staticmethod + def print_test_details_header(prefix: str, header: str) -> None: + header += ':' + print(prefix + mlog.italic(f'{header:<9}').get_text(mlog.colorize_console())) - def log_subtest(self, harness: 'TestHarness', test: 'TestRun', s: str, result: TestResult) -> None: - if harness.options.verbose or (harness.options.print_errorlogs and result.is_bad()): - self.flush() - print(harness.format(test, mlog.colorize_console(), max_left_width=self.max_left_width, - prefix=self.sub, - middle=s, - right=result.get_text(mlog.colorize_console())), flush=True) + @staticmethod + def print_test_details_line(prefix: str, + line: str, + end: str = '\n', + flush: bool = False) -> None: + print(prefix + ' ' + line, flush=flush, end=end) - self.request_update() + @staticmethod + def print_test_details(prefix: str, + header: str, + lines: T.Union[T.List[str], str], + clip: T.Optional[bool] = False) -> None: + offset = 0 + if not isinstance(lines, list): + lines = [lines] + if clip and len(lines) > 100: + offset = -100 + header += ' (only the last 100 lines from a long output included)' + ConsoleLogger.print_test_details_header(prefix, header) + for line in lines[offset:]: + ConsoleLogger.print_test_details_line(prefix, line) + + def print_log(self, + harness: 'TestHarness', + result: 'TestRun', + no_output: bool = False) -> None: + assert result.cmdline + prefix = harness.get_test_num_prefix(result.num) + self.print_test_details(prefix, "command", result.cmdline) + self.print_test_details(prefix, + "exit details", + returncode_to_status(result.returncode)) + if not no_output: + if result.stdo: + if harness.options.split or result.stde: + name = 'stdout' + else: + name = 'output' + self.print_test_details(prefix, + name, + result.stdo.splitlines(), + not harness.options.verbose) + if result.stde: + self.print_test_details(prefix, + "stderr", + result.stde.splitlines(), + not harness.options.verbose) + if result.additional_out: + self.print_test_details(prefix, + "additional output", + result.additional_out.splitlines(), + not harness.options.verbose) + if result.additional_err: + self.print_test_details(prefix, + "additional error", + result.additional_err.splitlines(), + not harness.options.verbose) + + def log_subtest(self, harness: 'TestHarness', test: 'TestRun', s: str, result: TestResult) -> str: + return 'subtest %s %s' % (s, result.get_text(mlog.colorize_console())) def log(self, harness: 'TestHarness', result: 'TestRun') -> None: self.running_tests.remove(result) - if result.res is TestResult.TIMEOUT and harness.options.verbose: - self.flush() - print(f'{result.name} time out (After {result.timeout} seconds)') + if result.res is TestResult.TIMEOUT and (harness.options.verbose or + harness.options.print_errorlogs): + result.additional_err += f'timed out (after {result.timeout} seconds)\n' if not harness.options.quiet or not result.res.is_ok(): self.flush() - if harness.options.verbose and not result.is_parallel and result.cmdline: - if not result.needs_parsing: - print(self.output_end) - print(harness.format(result, mlog.colorize_console(), max_left_width=self.max_left_width)) - else: - print(harness.format(result, mlog.colorize_console(), max_left_width=self.max_left_width), - flush=True) - if harness.options.verbose or result.res.is_bad(): - self.print_log(harness, result) - if harness.options.verbose or result.res.is_bad(): - print(flush=True) + print(harness.format(result, mlog.colorize_console(), max_left_width=self.max_left_width)) + if harness.options.verbose and not result.is_parallel and result.cmdline and not result.needs_parsing: + # output already printed during execution + self.print_log(harness, result, no_output=True) + elif harness.options.verbose or (result.res.is_bad() and harness.options.print_errorlogs): + # verbose or fail + print_errorlogs -> print + self.print_log(harness, result) self.request_update() @@ -703,9 +681,14 @@ class TextLogfileBuilder(TestFileLogger): if cmdline: starttime_str = time.strftime("%H:%M:%S", time.gmtime(result.starttime)) self.file.write(starttime_str + ' ' + cmdline + '\n') - self.file.write(dashes('output', '-', 78) + '\n') - self.file.write(result.get_log()) - self.file.write(dashes('', '-', 78) + '\n\n') + if result.stdo: + self.file.write(dashes('stdout', '-', 78) + '\n') + self.file.write(result.stdo + '\n') + self.file.write(dashes('', '-', 78) + '\n\n') + if result.stde: + self.file.write(dashes('stderr', '-', 78) + '\n') + self.file.write(result.stde + '\n') + self.file.write(dashes('', '-', 78) + '\n\n') async def finish(self, harness: 'TestHarness') -> None: if harness.collected_failures: @@ -895,7 +878,6 @@ class TestRun: self._num = TestRun.TEST_NUM return self._num - @property def detail(self) -> str: if self.res is TestResult.PENDING: return '' @@ -912,7 +894,8 @@ class TestRun: return '' def _complete(self, returncode: int, res: TestResult, - stdo: T.Optional[str], stde: T.Optional[str]) -> None: + stdo: T.Optional[str], stde: T.Optional[str], + additional_out: T.Optional[str], additional_err: T.Optional[str]) -> None: assert isinstance(res, TestResult) if self.should_fail and res in (TestResult.OK, TestResult.FAIL): res = TestResult.UNEXPECTEDPASS if res.is_ok() else TestResult.EXPECTEDFAIL @@ -922,6 +905,8 @@ class TestRun: self.duration = time.time() - self.starttime self.stdo = stdo self.stde = stde + self.additional_out = additional_out + self.additional_err = additional_err @property def cmdline(self) -> T.Optional[str]: @@ -933,43 +918,28 @@ class TestRun: def complete_skip(self, message: str) -> None: self.starttime = time.time() - self._complete(GNU_SKIP_RETURNCODE, TestResult.SKIP, message, None) + self._complete(GNU_SKIP_RETURNCODE, TestResult.SKIP, message, None, None, None) def complete(self, returncode: int, res: TestResult, - stdo: T.Optional[str], stde: T.Optional[str]) -> None: - self._complete(returncode, res, stdo, stde) - - def get_log(self, colorize: bool = False, stderr_only: bool = False) -> str: - stdo = '' if stderr_only else self.stdo - if self.stde: - res = '' - if stdo: - res += mlog.cyan('stdout:').get_text(colorize) + '\n' - res += stdo - if res[-1:] != '\n': - res += '\n' - res += mlog.cyan('stderr:').get_text(colorize) + '\n' - res += self.stde - else: - res = stdo - if res and res[-1:] != '\n': - res += '\n' - return res + stdo: T.Optional[str], stde: T.Optional[str], + additional_out: T.Optional[str], additional_err: T.Optional[str]) -> None: + self._complete(returncode, res, stdo, stde, additional_out, additional_err) @property def needs_parsing(self) -> bool: return False - async def parse(self, harness: 'TestHarness', lines: T.AsyncIterator[str]) -> T.Tuple[TestResult, str]: + async def parse(self, harness: 'TestHarness', lines: T.AsyncIterator[str]) -> T.Tuple[TestResult, str, str]: async for l in lines: pass - return TestResult.OK, '' + return TestResult.OK, '', '' class TestRunExitCode(TestRun): def complete(self, returncode: int, res: TestResult, - stdo: T.Optional[str], stde: T.Optional[str]) -> None: + stdo: T.Optional[str], stde: T.Optional[str], + additional_out: T.Optional[str], additional_err: T.Optional[str]) -> None: if res: pass elif returncode == GNU_SKIP_RETURNCODE: @@ -978,14 +948,15 @@ class TestRunExitCode(TestRun): res = TestResult.ERROR else: res = TestResult.FAIL if bool(returncode) else TestResult.OK - super().complete(returncode, res, stdo, stde) + super().complete(returncode, res, stdo, stde, additional_out, additional_err) TestRun.PROTOCOL_TO_CLASS[TestProtocol.EXITCODE] = TestRunExitCode class TestRunGTest(TestRunExitCode): def complete(self, returncode: int, res: TestResult, - stdo: T.Optional[str], stde: T.Optional[str]) -> None: + stdo: T.Optional[str], stde: T.Optional[str], + additional_out: T.Optional[str], additional_err: T.Optional[str]) -> None: filename = f'{self.test.name}.xml' if self.test.workdir: filename = os.path.join(self.test.workdir, filename) @@ -998,7 +969,7 @@ class TestRunGTest(TestRunExitCode): # will handle the failure, don't generate a stacktrace. pass - super().complete(returncode, res, stdo, stde) + super().complete(returncode, res, stdo, stde, additional_out, additional_err) TestRun.PROTOCOL_TO_CLASS[TestProtocol.GTEST] = TestRunGTest @@ -1009,35 +980,39 @@ class TestRunTAP(TestRun): return True def complete(self, returncode: int, res: TestResult, - stdo: str, stde: str) -> None: + stdo: T.Optional[str], stde: T.Optional[str], + additional_out: T.Optional[str], additional_err: T.Optional[str]) -> None: if returncode != 0 and not res.was_killed(): res = TestResult.ERROR stde = stde or '' stde += f'\n(test program exited with status code {returncode})' - super().complete(returncode, res, stdo, stde) + super().complete(returncode, res, stdo, stde, additional_out, additional_err) - async def parse(self, harness: 'TestHarness', lines: T.AsyncIterator[str]) -> T.Tuple[TestResult, str]: + async def parse(self, + harness: 'TestHarness', + lines: T.AsyncIterator[str]) -> T.Tuple[TestResult, str, str]: res = TestResult.OK + output = '' error = '' async for i in TAPParser().parse_async(lines): if isinstance(i, TAPParser.Bailout): res = TestResult.ERROR - harness.log_subtest(self, i.message, res) + output += '\n' + harness.log_subtest(self, i.message, res) elif isinstance(i, TAPParser.Test): self.results.append(i) if i.result.is_bad(): res = TestResult.FAIL - harness.log_subtest(self, i.name or f'subtest {i.number}', i.result) + output += '\n' + harness.log_subtest(self, i.name or f'subtest {i.number}', i.result) elif isinstance(i, TAPParser.Error): - error = '\nTAP parsing error: ' + i.message + error += '\nTAP parsing error: ' + i.message res = TestResult.ERROR if all(t.result is TestResult.SKIP for t in self.results): # This includes the case where self.results is empty res = TestResult.SKIP - return res, error + return res, output, error TestRun.PROTOCOL_TO_CLASS[TestProtocol.TAP] = TestRunTAP @@ -1047,7 +1022,9 @@ class TestRunRust(TestRun): def needs_parsing(self) -> bool: return True - async def parse(self, harness: 'TestHarness', lines: T.AsyncIterator[str]) -> T.Tuple[TestResult, str]: + async def parse(self, + harness: 'TestHarness', + lines: T.AsyncIterator[str]) -> T.Tuple[TestResult, str, str]: def parse_res(n: int, name: str, result: str) -> TAPParser.Test: if result == 'ok': return TAPParser.Test(n, name, TestResult.OK, None) @@ -1058,6 +1035,7 @@ class TestRunRust(TestRun): return TAPParser.Test(n, name, TestResult.ERROR, f'Unsupported output from rust test: {result}') + output = '' n = 1 async for line in lines: if line.startswith('test ') and not line.startswith('test result'): @@ -1065,17 +1043,17 @@ class TestRunRust(TestRun): name = name.replace('::', '.') t = parse_res(n, name, result) self.results.append(t) - harness.log_subtest(self, name, t.result) + output += '\n' + harness.log_subtest(self, name, t.result) n += 1 if all(t.result is TestResult.SKIP for t in self.results): # This includes the case where self.results is empty - return TestResult.SKIP, '' + return TestResult.SKIP, output, '' elif any(t.result is TestResult.ERROR for t in self.results): - return TestResult.ERROR, '' + return TestResult.ERROR, output, '' elif any(t.result is TestResult.FAIL for t in self.results): - return TestResult.FAIL, '' - return TestResult.OK, '' + return TestResult.FAIL, output, '' + return TestResult.OK, output, '' TestRun.PROTOCOL_TO_CLASS[TestProtocol.RUST] = TestRunRust @@ -1088,14 +1066,17 @@ def decode(stream: T.Union[None, bytes]) -> str: except UnicodeDecodeError: return stream.decode('iso-8859-1', errors='ignore') -async def read_decode(reader: asyncio.StreamReader, console_mode: ConsoleUser) -> str: +async def read_decode(reader: asyncio.StreamReader, + line_handler: T.Callable[[str], None]) -> str: stdo_lines = [] try: while not reader.at_eof(): line = decode(await reader.readline()) + if len(line) == 0: + continue stdo_lines.append(line) - if console_mode is ConsoleUser.STDOUT: - print(line, end='', flush=True) + if line_handler: + line_handler(line) return ''.join(stdo_lines) except asyncio.CancelledError: return ''.join(stdo_lines) @@ -1206,16 +1187,17 @@ class TestSubprocess: self.stdo_task = asyncio.ensure_future(decode_coro) return queue_iter(q) - def communicate(self, console_mode: ConsoleUser) -> T.Tuple[T.Optional[T.Awaitable[str]], - T.Optional[T.Awaitable[str]]]: + def communicate(self, + console_mode: ConsoleUser, + line_handler: T.Callable[[str], None] = None) -> T.Tuple[T.Optional[T.Awaitable[str]], T.Optional[T.Awaitable[str]]]: # asyncio.ensure_future ensures that printing can # run in the background, even before it is awaited if self.stdo_task is None and self.stdout is not None: - decode_coro = read_decode(self._process.stdout, console_mode) + decode_coro = read_decode(self._process.stdout, line_handler) self.stdo_task = asyncio.ensure_future(decode_coro) self.all_futures.append(self.stdo_task) if self.stderr is not None and self.stderr != asyncio.subprocess.STDOUT: - decode_coro = read_decode(self._process.stderr, console_mode) + decode_coro = read_decode(self._process.stderr, line_handler) self.stde_task = asyncio.ensure_future(decode_coro) self.all_futures.append(self.stde_task) @@ -1285,7 +1267,9 @@ class TestSubprocess: if self.postwait_fn: self.postwait_fn() - return p.returncode or 0, result, additional_error + return p.returncode or 0, \ + result, \ + additional_error + '\n' if additional_error else '' class SingleTestRunner: @@ -1443,22 +1427,39 @@ class SingleTestRunner: parse_task = None if self.runobj.needs_parsing: - parse_coro = self.runobj.parse(harness, p.stdout_lines(self.console_mode)) + parse_coro = self.runobj.parse(harness, + p.stdout_lines(self.console_mode)) parse_task = asyncio.ensure_future(parse_coro) - stdo_task, stde_task = p.communicate(self.console_mode) + if self.console_mode == ConsoleUser.STDOUT: + prefix = harness.get_test_num_prefix(self.runobj.num) + + def printer(line: str) -> None: + ConsoleLogger.print_test_details_line(prefix, + line, + flush=True, + end='') + ConsoleLogger.print_test_details_header(prefix, 'output') + stdo_task, stde_task = p.communicate(self.console_mode, printer) + else: + stdo_task, stde_task = p.communicate(self.console_mode) + additional_output = '' returncode, result, additional_error = await p.wait(self.runobj.timeout) if parse_task is not None: - res, error = await parse_task + res, additional_output, error = await parse_task if error: additional_error = join_lines(additional_error, error) result = result or res stdo = await stdo_task if stdo_task else '' stde = await stde_task if stde_task else '' - stde = join_lines(stde, additional_error) - self.runobj.complete(returncode, result, stdo, stde) + self.runobj.complete(returncode, + result, + stdo.strip(), + stde.strip(), + additional_output.strip(), + additional_error.strip()) class TestHarness: @@ -1598,18 +1599,18 @@ class TestHarness: def max_left_width(self) -> int: return 2 * self.numlen + 2 + def get_test_num_prefix(self, num: int) -> str: + return '{num:{numlen}}/{testcount} '.format(numlen=self.numlen, + num=num, + testcount=self.test_count) + def format(self, result: TestRun, colorize: bool, max_left_width: int = 0, - prefix: str = '', left: T.Optional[str] = None, middle: T.Optional[str] = None, right: T.Optional[str] = None) -> str: - if left is None: - left = '{num:{numlen}}/{testcount} '.format( - numlen=self.numlen, - num=result.num, - testcount=self.test_count) + left = self.get_test_num_prefix(result.num) # A non-default max_left_width lets the logger print more stuff before the # name, while ensuring that the rightmost columns remain aligned. @@ -1617,7 +1618,7 @@ class TestHarness: if middle is None: middle = result.name - extra_mid_width = max_left_width + self.name_max_len + 1 - uniwidth(middle) - uniwidth(left) - uniwidth(prefix) + extra_mid_width = max_left_width + self.name_max_len + 1 - uniwidth(middle) - uniwidth(left) middle += ' ' * max(1, extra_mid_width) if right is None: @@ -1625,13 +1626,16 @@ class TestHarness: res=result.res.get_text(colorize), dur=result.duration, durlen=self.duration_max_len + 3) - detail = result.detail - if detail: - right += ' ' + detail - return prefix + left + middle + right + if not (result.res.is_bad() and self.options.print_errorlogs) \ + and not self.options.verbose \ + and (result.res.is_bad() or result.needs_parsing): + detail = result.detail() + if detail: + right += ' ' + detail + return left + middle + right def summary(self) -> str: - return textwrap.dedent(''' + return textwrap.dedent('''\ Ok: {:<4} Expected Fail: {:<4} @@ -1672,7 +1676,7 @@ class TestHarness: for runner in runners]) # Disable the progress report if it gets in the way self.need_console = any(runner.console_mode is not ConsoleUser.LOGGER - for runner in runners) + for runner in runners) self.test_count = len(runners) self.run_tests(runners) @@ -1818,9 +1822,13 @@ class TestHarness: finally: self.close_logfiles() - def log_subtest(self, test: TestRun, s: str, res: TestResult) -> None: + def log_subtest(self, test: TestRun, s: str, res: TestResult) -> str: + rv = '' for l in self.loggers: - l.log_subtest(self, test, s, res) + tmp = l.log_subtest(self, test, s, res) + if tmp: + rv += tmp + return rv def log_start_test(self, test: TestRun) -> None: for l in self.loggers: diff --git a/test cases/common/35 string operations/meson.build b/test cases/common/35 string operations/meson.build index ca0342d..54eab88 100644 --- a/test cases/common/35 string operations/meson.build +++ b/test cases/common/35 string operations/meson.build @@ -1,4 +1,4 @@ -project('string formatting', 'c') +project('string formatting', 'c', meson_version: '>=0.59.0') templ = '@0@bar@1@' @@ -19,6 +19,9 @@ long = 'abcde' prefix = 'abc' suffix = 'cde' +assert(long[0] == 'a') +assert(long[2] == 'c') + assert(long.replace('b', 'd') == 'adcde') assert(long.replace('z', 'x') == long) assert(long.replace(prefix, suffix) == 'cdede') @@ -61,6 +64,7 @@ assert('@0@'.format(true) == 'true', 'bool string formatting failed') assert(' '.join(['a', 'b', 'c']) == 'a b c', 'join() array broken') assert(''.join(['a', 'b', 'c']) == 'abc', 'empty join() broken') assert(' '.join(['a']) == 'a', 'single join broken') +assert(' '.join(['a'], ['b', ['c']], 'd') == 'a b c d', 'varargs join broken') version_number = '1.2.8' diff --git a/test cases/common/35 string operations/test.json b/test cases/common/35 string operations/test.json new file mode 100644 index 0000000..96f9659 --- /dev/null +++ b/test cases/common/35 string operations/test.json @@ -0,0 +1,7 @@ +{ + "stdout": [ + { + "line": "WARNING: Project targeting '>=0.59.0' but tried to use feature introduced in '0.60.0': str.join (varargs). List-flattening and variadic arguments" + } + ] +} diff --git a/test cases/failing/11 object arithmetic/test.json b/test cases/failing/11 object arithmetic/test.json index 5339fac..84f5c46 100644 --- a/test cases/failing/11 object arithmetic/test.json +++ b/test cases/failing/11 object arithmetic/test.json @@ -2,7 +2,7 @@ "stdout": [ { "match": "re", - "line": "test cases/failing/11 object arithmetic/meson\\.build:3:0: ERROR: Invalid use of addition: .*" + "line": "test cases/failing/11 object arithmetic/meson\\.build:3:0: ERROR: The `\\+` of str does not accept objects of type MesonMain .*" } ] } diff --git a/test cases/failing/12 string arithmetic/test.json b/test cases/failing/12 string arithmetic/test.json index 476f9bb..c762229 100644 --- a/test cases/failing/12 string arithmetic/test.json +++ b/test cases/failing/12 string arithmetic/test.json @@ -1,8 +1,7 @@ { "stdout": [ { - "match": "re", - "line": "test cases/failing/12 string arithmetic/meson\\.build:3:0: ERROR: Invalid use of addition: .*" + "line": "test cases/failing/12 string arithmetic/meson.build:3:0: ERROR: The `+` of str does not accept objects of type int (3)" } ] } diff --git a/test cases/fortran/9 cpp/meson.build b/test cases/fortran/9 cpp/meson.build index f96944b..270fae5 100644 --- a/test cases/fortran/9 cpp/meson.build +++ b/test cases/fortran/9 cpp/meson.build @@ -3,10 +3,6 @@ project('C, C++ and Fortran', 'c', 'cpp', 'fortran') cpp = meson.get_compiler('cpp') fc = meson.get_compiler('fortran') -if cpp.get_id() == 'clang' - error('MESON_SKIP_TEST Clang C++ does not find -lgfortran for some reason.') -endif - if build_machine.system() == 'windows' and cpp.get_id() != fc.get_id() error('MESON_SKIP_TEST mixing gfortran with non-GNU C++ does not work.') endif |