aboutsummaryrefslogtreecommitdiff
path: root/.ci/generate_test_report_lib.py
blob: 36c95852452acfb8fb2d1c693d5d0fce11c9285f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Library to parse JUnit XML files and return a markdown report."""

from junitparser import JUnitXml, Failure

SEE_BUILD_FILE_STR = "Download the build's log file to see the details."
UNRELATED_FAILURES_STR = (
    "If these failures are unrelated to your changes (for example "
    "tests are broken or flaky at HEAD), please open an issue at "
    "https://github.com/llvm/llvm-project/issues and add the "
    "`infrastructure` label."
)
# The maximum number of lines to pull from a ninja failure.
NINJA_LOG_SIZE_THRESHOLD = 500


def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
    """Parses an individual ninja log."""
    failures = []
    index = 0
    while index < len(ninja_log):
        while index < len(ninja_log) and not ninja_log[index].startswith("FAILED:"):
            index += 1
        if index == len(ninja_log):
            # We hit the end of the log without finding a build failure, go to
            # the next log.
            return failures
        # If we are doing a build with LLVM_ENABLE_RUNTIMES, we can have nested
        # ninja invocations. The sub-ninja will print that a subcommand failed,
        # and then the outer ninja will list the command that failed. We should
        # ignore the outer failure.
        if ninja_log[index - 1].startswith("ninja: build stopped:"):
            index += 1
            continue
        # We are trying to parse cases like the following:
        #
        # [4/5] test/4.stamp
        # FAILED: touch test/4.stamp
        # touch test/4.stamp
        #
        # index will point to the line that starts with Failed:. The progress
        # indicator is the line before this ([4/5] test/4.stamp) and contains a pretty
        # printed version of the target being built (test/4.stamp). We use this line
        # and remove the progress information to get a succinct name for the target.
        failing_action = ninja_log[index - 1].split("] ")[1]
        failure_log = []
        while (
            index < len(ninja_log)
            and not ninja_log[index].startswith("[")
            and not ninja_log[index].startswith("ninja: build stopped:")
            and len(failure_log) < NINJA_LOG_SIZE_THRESHOLD
        ):
            failure_log.append(ninja_log[index])
            index += 1
        failures.append((failing_action, "\n".join(failure_log)))
    return failures


def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, str]]:
    """Extracts failure messages from ninja output.

    This function takes stdout/stderr from ninja in the form of a list of files
    represented as a list of lines. This function then returns tuples containing
    the name of the target and the error message.

    Args:
      ninja_logs: A list of files in the form of a list of lines representing the log
        files captured from ninja.

    Returns:
      A list of tuples. The first string is the name of the target that failed. The
      second string is the error message.
    """
    failures = []
    for ninja_log in ninja_logs:
        log_failures = _parse_ninja_log(ninja_log)
        failures.extend(log_failures)
    return failures


def _format_ninja_failures(ninja_failures: list[tuple[str, str]]) -> list[str]:
    """Formats ninja failures into summary views for the report."""
    output = []
    for build_failure in ninja_failures:
        failed_action, failure_message = build_failure
        output.extend(
            [
                "<details>",
                f"<summary>{failed_action}</summary>",
                "",
                "```",
                failure_message,
                "```",
                "</details>",
            ]
        )
    return output

def get_failures(junit_objects) -> dict[str, list[tuple[str, str]]]:
    failures = {}
    for results in junit_objects:
        for testsuite in results:
            for test in testsuite:
                if (
                    not test.is_passed
                    and test.result
                    and isinstance(test.result[0], Failure)
                ):
                    if failures.get(testsuite.name) is None:
                        failures[testsuite.name] = []
                    failures[testsuite.name].append(
                        (test.classname + "/" + test.name, test.result[0].text)
                    )
    return failures


# Set size_limit to limit the byte size of the report. The default is 1MB as this
# is the most that can be put into an annotation. If the generated report exceeds
# this limit and failures are listed, it will be generated again without failures
# listed. This minimal report will always fit into an annotation.
# If include failures is False, total number of test will be reported but their names
# and output will not be.
def generate_report(
    title,
    return_code,
    junit_objects,
    ninja_logs: list[list[str]],
    size_limit=1024 * 1024,
    list_failures=True,
):
    failures = get_failures(junit_objects)
    tests_run = 0
    tests_skipped = 0
    tests_failed = 0

    for results in junit_objects:
        for testsuite in results:
            tests_run += testsuite.tests
            tests_skipped += testsuite.skipped
            tests_failed += testsuite.failures

    report = [f"# {title}", ""]

    if tests_run == 0:
        if return_code == 0:
            report.extend(
                [
                    "The build succeeded and no tests ran. This is expected in some "
                    "build configurations."
                ]
            )
        else:
            ninja_failures = find_failure_in_ninja_logs(ninja_logs)
            if not ninja_failures:
                report.extend(
                    [
                        "The build failed before running any tests. Detailed "
                        "information about the build failure could not be "
                        "automatically obtained.",
                        "",
                        SEE_BUILD_FILE_STR,
                        "",
                        UNRELATED_FAILURES_STR,
                    ]
                )
            else:
                report.extend(
                    [
                        "The build failed before running any tests. Click on a "
                        "failure below to see the details.",
                        "",
                    ]
                )
                report.extend(_format_ninja_failures(ninja_failures))
                report.extend(
                    [
                        "",
                        UNRELATED_FAILURES_STR,
                    ]
                )
        return "\n".join(report)

    tests_passed = tests_run - tests_skipped - tests_failed

    def plural(num_tests):
        return "test" if num_tests == 1 else "tests"

    if tests_passed:
        report.append(f"* {tests_passed} {plural(tests_passed)} passed")
    if tests_skipped:
        report.append(f"* {tests_skipped} {plural(tests_skipped)} skipped")
    if tests_failed:
        report.append(f"* {tests_failed} {plural(tests_failed)} failed")

    if not list_failures:
        report.extend(
            [
                "",
                "Failed tests and their output was too large to report. "
                + SEE_BUILD_FILE_STR,
            ]
        )
    elif failures:
        report.extend(
            ["", "## Failed Tests", "(click on a test name to see its output)"]
        )

        for testsuite_name, failures in failures.items():
            report.extend(["", f"### {testsuite_name}"])
            for name, output in failures:
                report.extend(
                    [
                        "<details>",
                        f"<summary>{name}</summary>",
                        "",
                        "```",
                        output,
                        "```",
                        "</details>",
                    ]
                )
    elif return_code != 0:
        # No tests failed but the build was in a failed state. Bring this to the user's
        # attention.
        ninja_failures = find_failure_in_ninja_logs(ninja_logs)
        if not ninja_failures:
            report.extend(
                [
                    "",
                    "All tests passed but another part of the build **failed**. "
                    "Information about the build failure could not be automatically "
                    "obtained.",
                    "",
                    SEE_BUILD_FILE_STR,
                ]
            )
        else:
            report.extend(
                [
                    "",
                    "All tests passed but another part of the build **failed**. Click on "
                    "a failure below to see the details.",
                    "",
                ]
            )
            report.extend(_format_ninja_failures(ninja_failures))

    if failures or return_code != 0:
        report.extend(["", UNRELATED_FAILURES_STR])

    report = "\n".join(report)
    if len(report.encode("utf-8")) > size_limit:
        return generate_report(
            title,
            return_code,
            junit_objects,
            size_limit,
            list_failures=False,
        )

    return report


def load_info_from_files(build_log_files):
    junit_files = [
        junit_file for junit_file in build_log_files if junit_file.endswith(".xml")
    ]
    ninja_log_files = [
        ninja_log for ninja_log in build_log_files if ninja_log.endswith(".log")
    ]
    ninja_logs = []
    for ninja_log_file in ninja_log_files:
        with open(ninja_log_file, "r") as ninja_log_file_handle:
            ninja_logs.append(
                [log_line.strip() for log_line in ninja_log_file_handle.readlines()]
            )
    return [JUnitXml.fromfile(p) for p in junit_files], ninja_logs


def generate_report_from_files(title, return_code, build_log_files):
    junit_objects, ninja_logs = load_info_from_files(build_log_files)
    return generate_report(title, return_code, junit_objects, ninja_logs)