[CI] Move CI over to new project computation script

This patch migrates the CI over to the new compute_projects.py script
for calculating what projects need to be tested based on a change to
LLVM.

Reviewers: lnihlen, ldionne, tstellar, Endilll, joker-eph, Keenuts

Reviewed By: Keenuts, tstellar

Pull Request: https://github.com/llvm/llvm-project/pull/132642

3 files changed, 27 insertions, 279 deletions

diff --git a/.ci/compute-projects.sh b/.ci/compute-projects.sh
deleted file mode 100644
index 32baf26..0000000
--- a/.ci/compute-projects.sh
+++ /dev/null
@@ -1,194 +0,0 @@
-#!/usr/bin/env bash
-#===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===----------------------------------------------------------------------===##
-
-#
-# This file contains functions to compute which projects should be built by CI
-# systems and is intended to provide common functionality applicable across
-# multiple systems during a transition period.
-#
-
-function compute-projects-to-test() {
-  isForWindows=$1
-  shift
-  projects=${@}
-  for project in ${projects}; do
-    echo "${project}"
-    case ${project} in
-    lld)
-      for p in bolt cross-project-tests; do
-        echo $p
-      done
-    ;;
-    llvm)
-      for p in bolt clang clang-tools-extra lld lldb mlir polly; do
-        echo $p
-      done
-      # Flang is not stable in Windows CI at the moment
-      if [[ $isForWindows == 0 ]]; then
-        echo flang
-      fi
-    ;;
-    clang)
-      # lldb is temporarily removed to alleviate Linux pre-commit CI waiting times
-      for p in clang-tools-extra compiler-rt cross-project-tests; do
-        echo $p
-      done
-    ;;
-    clang-tools-extra)
-      echo libc
-    ;;
-    mlir)
-      # Flang is not stable in Windows CI at the moment
-      if [[ $isForWindows == 0 ]]; then
-        echo flang
-      fi
-    ;;
-    *)
-      # Nothing to do
-    ;;
-    esac
-  done
-}
-
-function compute-runtimes-to-test() {
-  projects=${@}
-  for project in ${projects}; do
-    case ${project} in
-    clang)
-      for p in libcxx libcxxabi libunwind; do
-        echo $p
-      done
-    ;;
-    *)
-      # Nothing to do
-    ;;
-    esac
-  done
-}
-
-function add-dependencies() {
-  projects=${@}
-  for project in ${projects}; do
-    echo "${project}"
-    case ${project} in
-    bolt)
-      for p in clang lld llvm; do
-        echo $p
-      done
-    ;;
-    cross-project-tests)
-      for p in lld clang; do
-        echo $p
-      done
-    ;;
-    clang-tools-extra)
-      for p in llvm clang; do
-        echo $p
-      done
-    ;;
-    compiler-rt|libc|openmp)
-      echo clang lld
-    ;;
-    flang|lldb|libclc)
-      for p in llvm clang; do
-        echo $p
-      done
-    ;;
-    lld|mlir|polly)
-      echo llvm
-    ;;
-    *)
-      # Nothing to do
-    ;;
-    esac
-  done
-}
-
-function exclude-linux() {
-  projects=${@}
-  for project in ${projects}; do
-    case ${project} in
-    cross-project-tests) ;; # tests failing
-    openmp)              ;; # https://github.com/google/llvm-premerge-checks/issues/410
-    *)
-      echo "${project}"
-    ;;
-    esac
-  done
-}
-
-function exclude-windows() {
-  projects=${@}
-  for project in ${projects}; do
-    case ${project} in
-    cross-project-tests) ;; # tests failing
-    compiler-rt)         ;; # tests taking too long
-    openmp)              ;; # TODO: having trouble with the Perl installation
-    libc)                ;; # no Windows support
-    lldb)                ;; # custom environment requirements (https://github.com/llvm/llvm-project/pull/94208#issuecomment-2146256857)
-    bolt)                ;; # tests are not supported yet
-    *)
-      echo "${project}"
-    ;;
-    esac
-  done
-}
-
-# Prints only projects that are both present in $modified_dirs and the passed
-# list.
-function keep-modified-projects() {
-  projects=${@}
-  for project in ${projects}; do
-    if echo "$modified_dirs" | grep -q -E "^${project}$"; then
-      echo "${project}"
-    fi
-  done
-}
-
-function check-targets() {
-  # Do not use "check-all" here because if there is "check-all" plus a
-  # project specific target like "check-clang", that project's tests
-  # will be run twice.
-  projects=${@}
-  for project in ${projects}; do
-    case ${project} in
-    clang-tools-extra)
-      echo "check-clang-tools"
-    ;;
-    compiler-rt)
-      echo "check-compiler-rt"
-    ;;
-    cross-project-tests)
-      echo "check-cross-project"
-    ;;
-    libcxx)
-      echo "check-cxx"
-    ;;
-    libcxxabi)
-      echo "check-cxxabi"
-    ;;
-    libunwind)
-      echo "check-unwind"
-    ;;
-    lldb)
-      echo "check-lldb"
-    ;;
-    pstl)
-      # Currently we do not run pstl tests in CI.
-    ;;
-    libclc)
-      # Currently there is no testing for libclc.
-    ;;
-    *)
-      echo "check-${project}"
-    ;;
-    esac
-  done
-}
-
diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index e547afa..5e5f916 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -52,8 +52,6 @@ modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
 echo "Directories modified:" >&2
 echo "$modified_dirs" >&2
 
-. ./.ci/compute-projects.sh
-
 # Project specific pipelines.
 
 # If libc++ or one of the runtimes directories changed.
@@ -73,20 +71,16 @@ fi
 # needs while letting them run on the infrastructure provided by LLVM.
 
 # Figure out which projects need to be built on each platform
-all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
-modified_projects="$(keep-modified-projects ${all_projects})"
-
-linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
-linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
-linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
+source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Linux)
+linux_projects=${projects_to_build}
+linux_check_targets=${project_check_targets}
+linux_runtimes=${runtimes_to_build}
+linux_runtime_check_targets=${runtimes_check_targets}
 
-linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
-linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
-linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
+source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Windows)
+windows_projects=${projects_to_build}
+windows_check_targets=${project_check_targets}
 
-windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
-windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
-windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
 
 # Generate the appropriate pipeline
 if [[ "${linux_projects}" != "" ]]; then
diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml
index c488421..b7d2c71 100644
--- a/.github/workflows/premerge.yaml
+++ b/.github/workflows/premerge.yaml
@@ -49,39 +49,22 @@ jobs:
         run: |
           git config --global --add safe.directory '*'
 
-          modified_files=$(git diff --name-only HEAD~1...HEAD)
-          modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort -u)
+          source <(git diff --name-only HEAD~1...HEAD | python3 .ci/compute_projects.py)
 
-          echo $modified_files
-          echo $modified_dirs
-
-          . ./.ci/compute-projects.sh
-
-          all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
-          modified_projects="$(keep-modified-projects ${all_projects})"
-
-          linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
-          linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
-          linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
-
-          linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
-          linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
-          linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
-
-          if [[ "${linux_projects}" == "" ]]; then
+          if [[ "${projects_to_build}" == "" ]]; then
             echo "No projects to build"
             exit 0
           fi
 
-          echo "Building projects: ${linux_projects}"
-          echo "Running project checks targets: ${linux_check_targets}"
-          echo "Building runtimes: ${linux_runtimes}"
-          echo "Running runtimes checks targets: ${linux_runtime_check_targets}"
+          echo "Building projects: ${projects_to_build}"
+          echo "Running project checks targets: ${project_check_targets}"
+          echo "Building runtimes: ${runtimes_to_build}"
+          echo "Running runtimes checks targets: ${runtimes_check_targets}"
 
           export CC=/opt/llvm/bin/clang
           export CXX=/opt/llvm/bin/clang++
 
-          ./.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})"
+          ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}"
 
   premerge-checks-windows:
     name: Windows Premerge Checks (Test Only - Please Ignore Results)
@@ -105,30 +88,17 @@ jobs:
       - name: Compute Projects
         id: vars
         run: |
-          modified_files=$(git diff --name-only HEAD~1...HEAD)
-          modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort | uniq)
-
-          echo $modified_files
-          echo $modified_dirs
-
-          . ./.ci/compute-projects.sh
+          source <(git diff --name-only HEAD~1...HEAD | python .ci/compute_projects.py)
 
-          all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
-          modified_projects="$(keep-modified-projects ${all_projects})"
-
-          windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
-          windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq | tr -d '\r' | tr '\n' ' ')
-          windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq | tr -d '\r' | tr '\n' ';')
-
-          if [[ "${windows_projects}" == "" ]]; then
+          if [[ "${projects_to_build}" == "" ]]; then
             echo "No projects to build"
           fi
 
-          echo "Building projects: ${windows_projects}"
-          echo "Running project checks targets: ${windows_check_targets}"
+          echo "Building projects: ${projects_to_build}"
+          echo "Running project checks targets: ${project_check_targets}"
 
-          echo "windows-projects=${windows_projects}" >> $GITHUB_OUTPUT
-          echo "windows-check-targets=${windows_check_targets}" >> $GITHUB_OUTPUT
+          echo "windows-projects=${projects_to_build}" >> $GITHUB_OUTPUT
+          echo "windows-check-targets=${project_check_targets}" >> $GITHUB_OUTPUT
       - name: Build and Test
         # Mark the job as a success even if the step fails so that people do
         # not get notified while the new premerge pipeline is in an
@@ -165,37 +135,15 @@ jobs:
         uses: llvm/actions/install-ninja@main
       - name: Build and Test
         run: |
-          modified_files=$(git diff --name-only HEAD~1...HEAD)
-          modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort -u)
-
-          echo $modified_files
-          echo $modified_dirs
-
-          . ./.ci/compute-projects.sh
-
-          all_projects="clang clang-tools-extra lld lldb llvm mlir"
-          modified_projects="$(keep-modified-projects ${all_projects})"
-
-          # We have to disable the runtimes builds due to https://github.com/llvm/llvm-project/issues/90568
-          # and the lldb tests depend on libcxx, so we need to skip them.
-          mac_check_targets=$(check-targets ${modified_projects} | sort | uniq | tr '\n' ' ' | sed -e 's/check-lldb //g')
-          mac_projects=$(add-dependencies ${modified_projects} | sort | uniq | tr '\n' ' ')
-
-          mac_runtimes_to_test=$(compute-runtimes-to-test ${modified_projects})
-          mac_runtime_check_targets=$(check-targets ${mac_runtimes_to_test} | sort | uniq | tr '\n' ' ')
-          mac_runtimes=$(echo ${mac_runtimes_to_test} | tr ' ' '\n' | sort | uniq | tr '\n' ' ')
+          source <(git diff --name-only HEAD~2..HEAD | python3 .ci/compute_projects.py)
 
-          if [[ "${mac_projects}" == "" ]]; then
+          if [[ "${projects_to_build}" == "" ]]; then
             echo "No projects to build"
             exit 0
           fi
 
-          echo "Projects to test: ${modified_projects}"
-          echo "Runtimes to test: ${mac_runtimes_to_test}"
-          echo "Building projects: ${mac_projects}"
-          echo "Running project checks targets: ${mac_check_targets}"
-          echo "Building runtimes: ${mac_runtimes}"
-          echo "Running runtimes checks targets: ${mac_runtime_check_targets}"
+          echo "Building projects: ${projects_to_build}"
+          echo "Running project checks targets: ${project_check_targets}"
 
           # -DLLVM_DISABLE_ASSEMBLY_FILES=ON is for
           # https://github.com/llvm/llvm-project/issues/81967
@@ -203,7 +151,7 @@ jobs:
           cmake -G Ninja \
                 -B build \
                 -S llvm \
-                -DLLVM_ENABLE_PROJECTS="$(echo ${mac_projects} | tr ' ' ';')" \
+                -DLLVM_ENABLE_PROJECTS="${projects_to_build}" \
                 -DLLVM_DISABLE_ASSEMBLY_FILES=ON \
                 -DCMAKE_BUILD_TYPE=Release \
                 -DLLDB_INCLUDE_TESTS=OFF \
@@ -212,4 +160,4 @@ jobs:
                 -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
 
           # The libcxx tests fail, so we are skipping the runtime targets.
-          ninja -C build  $mac_check_targets
+          ninja -C build ${project_check_targets}