[CI] Add Support for Parsing Ninja Logs to generate_test_report_lib

This patch adds in support for taking the CLI output from ninja and
parsing it for failures. This is intended to be used in the cases where
all tests pass (or none have run), but the build fails to easily surface
where exactly the build failed.

The actual integration will happen in a future patch.

Reviewers: gburgessiv, dschuff, lnihlen, DavidSpickett, Keenuts, cmtice

Reviewed By: DavidSpickett, cmtice

Pull Request: https://github.com/llvm/llvm-project/pull/152620

2 files changed, 164 insertions, 0 deletions

diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index 25d810f..df95db6 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -12,6 +12,65 @@ UNRELATED_FAILURES_STR = (
     "https://github.com/llvm/llvm-project/issues and add the "
     "`infrastructure` label."
 )
+# The maximum number of lines to pull from a ninja failure.
+NINJA_LOG_SIZE_THRESHOLD = 500
+
+
+def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
+    """Parses an individual ninja log."""
+    failures = []
+    index = 0
+    while index < len(ninja_log):
+        while index < len(ninja_log) and not ninja_log[index].startswith("FAILED:"):
+            index += 1
+        if index == len(ninja_log):
+            # We hit the end of the log without finding a build failure, go to
+            # the next log.
+            return failures
+        # We are trying to parse cases like the following:
+        #
+        # [4/5] test/4.stamp
+        # FAILED: touch test/4.stamp
+        # touch test/4.stamp
+        #
+        # index will point to the line that starts with Failed:. The progress
+        # indicator is the line before this ([4/5] test/4.stamp) and contains a pretty
+        # printed version of the target being built (test/4.stamp). We use this line
+        # and remove the progress information to get a succinct name for the target.
+        failing_action = ninja_log[index - 1].split("] ")[1]
+        failure_log = []
+        while (
+            index < len(ninja_log)
+            and not ninja_log[index].startswith("[")
+            and not ninja_log[index].startswith("ninja: build stopped:")
+            and len(failure_log) < NINJA_LOG_SIZE_THRESHOLD
+        ):
+            failure_log.append(ninja_log[index])
+            index += 1
+        failures.append((failing_action, "\n".join(failure_log)))
+    return failures
+
+
+def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, str]]:
+    """Extracts failure messages from ninja output.
+
+    This function takes stdout/stderr from ninja in the form of a list of files
+    represented as a list of lines. This function then returns tuples containing
+    the name of the target and the error message.
+
+    Args:
+      ninja_logs: A list of files in the form of a list of lines representing the log
+        files captured from ninja.
+
+    Returns:
+      A list of tuples. The first string is the name of the target that failed. The
+      second string is the error message.
+    """
+    failures = []
+    for ninja_log in ninja_logs:
+        log_failures = _parse_ninja_log(ninja_log)
+        failures.extend(log_failures)
+    return failures
 
 
 # Set size_limit to limit the byte size of the report. The default is 1MB as this
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index eda76ead..41f3eae 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -19,6 +19,111 @@ def junit_from_xml(xml):
 
 
 class TestReports(unittest.TestCase):
+    def test_find_failure_ninja_logs(self):
+        failures = generate_test_report_lib.find_failure_in_ninja_logs(
+            [
+                [
+                    "[1/5] test/1.stamp",
+                    "[2/5] test/2.stamp",
+                    "[3/5] test/3.stamp",
+                    "[4/5] test/4.stamp",
+                    "FAILED: touch test/4.stamp",
+                    "Wow! This system is really broken!",
+                    "[5/5] test/5.stamp",
+                ],
+            ]
+        )
+        self.assertEqual(len(failures), 1)
+        self.assertEqual(
+            failures[0],
+            (
+                "test/4.stamp",
+                dedent(
+                    """\
+                    FAILED: touch test/4.stamp
+                    Wow! This system is really broken!"""
+                ),
+            ),
+        )
+
+    def test_no_failure_ninja_log(self):
+        failures = generate_test_report_lib.find_failure_in_ninja_logs(
+            [
+                [
+                    "[1/3] test/1.stamp",
+                    "[2/3] test/2.stamp",
+                    "[3/3] test/3.stamp",
+                ]
+            ]
+        )
+        self.assertEqual(failures, [])
+
+    def test_ninja_log_end(self):
+        failures = generate_test_report_lib.find_failure_in_ninja_logs(
+            [
+                [
+                    "[1/3] test/1.stamp",
+                    "[2/3] test/2.stamp",
+                    "[3/3] test/3.stamp",
+                    "FAILED: touch test/3.stamp",
+                    "Wow! This system is really broken!",
+                    "ninja: build stopped: subcommand failed.",
+                ]
+            ]
+        )
+        self.assertEqual(len(failures), 1)
+        self.assertEqual(
+            failures[0],
+            (
+                "test/3.stamp",
+                dedent(
+                    """\
+                    FAILED: touch test/3.stamp
+                    Wow! This system is really broken!"""
+                ),
+            ),
+        )
+
+    def test_ninja_log_multiple_failures(self):
+        failures = generate_test_report_lib.find_failure_in_ninja_logs(
+            [
+                [
+                    "[1/5] test/1.stamp",
+                    "[2/5] test/2.stamp",
+                    "FAILED: touch test/2.stamp",
+                    "Wow! This system is really broken!",
+                    "[3/5] test/3.stamp",
+                    "[4/5] test/4.stamp",
+                    "FAILED: touch test/4.stamp",
+                    "Wow! This system is maybe broken!",
+                    "[5/5] test/5.stamp",
+                ]
+            ]
+        )
+        self.assertEqual(len(failures), 2)
+        self.assertEqual(
+            failures[0],
+            (
+                "test/2.stamp",
+                dedent(
+                    """\
+                    FAILED: touch test/2.stamp
+                    Wow! This system is really broken!"""
+                ),
+            ),
+        )
+        self.assertEqual(
+            failures[1],
+            (
+                "test/4.stamp",
+                dedent(
+                    """\
+                    FAILED: touch test/4.stamp
+                    Wow! This system is maybe broken!"""
+                ),
+            ),
+        )
+
     def test_title_only(self):
         self.assertEqual(
             generate_test_report_lib.generate_report("Foo", 0, []),