From f107f9b3962167e1ff5014951aa89fdc176ea683 Mon Sep 17 00:00:00 2001 From: Jon Turney Date: Wed, 12 Feb 2020 00:03:51 +0000 Subject: Make colourize_console() a function Currently, colourize_console is a constant, set at process initialization. To allow the actual stdout to be easily compared with the expected when running tests, we want to allow colourization to be on for the test driver, but not for the in-process configure done by run_configure, which has stdout redirected from a tty to a pipe. v2: Cache _colorize_console per file object v3: Reset cache on setup_console() --- run_project_tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'run_project_tests.py') diff --git a/run_project_tests.py b/run_project_tests.py index 8cbf989..f636d63 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -341,19 +341,19 @@ def log_text_file(logfile, testdir, stdo, stde): def bold(text): - return mlog.bold(text).get_text(mlog.colorize_console) + return mlog.bold(text).get_text(mlog.colorize_console()) def green(text): - return mlog.green(text).get_text(mlog.colorize_console) + return mlog.green(text).get_text(mlog.colorize_console()) def red(text): - return mlog.red(text).get_text(mlog.colorize_console) + return mlog.red(text).get_text(mlog.colorize_console()) def yellow(text): - return mlog.yellow(text).get_text(mlog.colorize_console) + return mlog.yellow(text).get_text(mlog.colorize_console()) def _run_ci_include(args: T.List[str]) -> str: -- cgit v1.1 From f867bfbce07aae6ed7a5b38c583490af3ea13af9 Mon Sep 17 00:00:00 2001 From: Jon Turney Date: Tue, 21 Jan 2020 00:37:29 +0000 Subject: Add a mechanism for validating meson output in tests Expected stdout lines must match lines from the actual stdout, in the same order. Lines with match type 're' are regex matched. v2: Ignore comment lines in expected_stdout v3: Automatically adjust path separators for location in expected output v4: Put expected stdout in test.json, rather than a separate file --- run_project_tests.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) (limited to 'run_project_tests.py') diff --git a/run_project_tests.py b/run_project_tests.py index f636d63..3abe88c 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -191,6 +191,7 @@ class TestDef: self.env = os.environ.copy() self.installed_files = [] # type: T.List[InstalledFile] self.do_not_set_opts = [] # type: T.List[str] + self.stdout = [] # type: T.List[T.Dict[str, str]] def __repr__(self) -> str: return '<{}: {:<48} [{}: {}] -- {}>'.format(type(self).__name__, str(self.path), self.name, self.args, self.skip) @@ -381,6 +382,54 @@ def run_ci_commands(raw_log: str) -> T.List[str]: res += ['CI COMMAND {}:\n{}\n'.format(cmd[0], ci_commands[cmd[0]](cmd[1:]))] return res +def _compare_output(expected: T.List[T.Dict[str, str]], output: str, desc: str) -> str: + if expected: + i = iter(expected) + + def next_expected(i): + # Get the next expected line + item = next(i) + how = item.get('match', 'literal') + expected = item.get('line') + + # Simple heuristic to automatically convert path separators for + # Windows: + # + # Any '/' appearing before 'WARNING' or 'ERROR' (i.e. a path in a + # filename part of a location) is replaced with '\' (in a re: '\\' + # which matches a literal '\') + # + # (There should probably be a way to turn this off for more complex + # cases which don't fit this) + if mesonlib.is_windows(): + if how != "re": + sub = r'\\' + else: + sub = r'\\\\' + expected = re.sub(r'/(?=.*(WARNING|ERROR))', sub, expected) + + return how, expected + + try: + how, expected = next_expected(i) + for actual in output.splitlines(): + if how == "re": + match = bool(re.match(expected, actual)) + else: + match = (expected == actual) + if match: + how, expected = next_expected(i) + + # reached the end of output without finding expected + return 'expected "{}" not found in {}'.format(expected, desc) + except StopIteration: + # matched all expected lines + pass + + return '' + +def validate_output(test: TestDef, stdo: str, stde: str) -> str: + return _compare_output(test.stdout, stdo, 'stdout') def run_test_inprocess(testdir): old_stdout = sys.stdout @@ -452,6 +501,11 @@ def _run_test(test: TestDef, test_build_dir: str, install_dir: str, extra_args, cicmds = run_ci_commands(mesonlog) testresult = TestResult(cicmds) testresult.add_step(BuildStep.configure, stdo, stde, mesonlog, time.time() - gen_start) + output_msg = validate_output(test, stdo, stde) + testresult.mlog += output_msg + if output_msg: + testresult.fail('Unexpected output while configuring.') + return testresult if should_fail == 'meson': if returncode == 1: return testresult @@ -566,6 +620,9 @@ def gather_tests(testdir: Path) -> T.List[TestDef]: if 'installed' in test_def: installed = [InstalledFile(x) for x in test_def['installed']] + # Handle expected output + stdout = test_def.get('stdout', []) + # Handle the do_not_set_opts list do_not_set_opts = test_def.get('do_not_set_opts', []) # type: T.List[str] @@ -583,6 +640,7 @@ def gather_tests(testdir: Path) -> T.List[TestDef]: t.env.update(env) t.installed_files = installed t.do_not_set_opts = do_not_set_opts + t.stdout = stdout all_tests += [t] continue @@ -653,6 +711,7 @@ def gather_tests(testdir: Path) -> T.List[TestDef]: test.env.update(env) test.installed_files = installed test.do_not_set_opts = do_not_set_opts + test.stdout = stdout all_tests += [test] return sorted(all_tests) -- cgit v1.1