diff options
Diffstat (limited to '.ci')
-rwxr-xr-x | .ci/generate-buildkite-pipeline-premerge | 149 |
1 files changed, 101 insertions, 48 deletions
diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge index 0488597..728ec04 100755 --- a/.ci/generate-buildkite-pipeline-premerge +++ b/.ci/generate-buildkite-pipeline-premerge @@ -13,6 +13,25 @@ # # See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format. # +# As this outputs a yaml file, it's possible to log messages to stderr or +# prefix with "#". + + +set -eu +set -o pipefail + +# Environment variables script works with: +# List of files affected by this commit +: ${MODIFIED_FILES:=$(git diff --name-only HEAD~1)} +# Filter rules for generic windows tests +: ${WINDOWS_AGENTS:='{"queue": "windows"}'} +# Filter rules for generic linux tests +: ${LINUX_AGENTS:='{"queue": "linux"}'} +# Service agents, for interacting with Phabricator. +: ${SERVICE_AGENTS:='{"queue": "service"}'} +# Set by buildkite +: ${BUILDKITE_COMMIT:=} +: ${BUILDKITE_BRANCH:=} reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')" if [[ "${reviewID}" != "" ]]; then @@ -25,36 +44,12 @@ cat <<EOF steps: EOF -# If libc++ or one of the runtimes directories changed -if git diff --name-only HEAD~1 | grep -q -E "^libcxx/|^libcxxabi/|^libunwind/|^runtimes/|^cmake/"; then - cat <<EOF - - trigger: "libcxx-ci" - build: - message: "${buildMessage}" - commit: "${BUILDKITE_COMMIT}" - branch: "${BUILDKITE_BRANCH}" -EOF -fi - -# If clang changed -if git diff --name-only HEAD~1 | grep -q -E "^clang/"; then - cat <<EOF - - trigger: "clang-ci" - build: - message: "${buildMessage}" - commit: "${BUILDKITE_COMMIT}" - branch: "${BUILDKITE_BRANCH}" -EOF -fi +echo "Files modified:" >&2 +echo "$MODIFIED_FILES" >&2 +modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u) +echo "Directories modified:" >&2 +echo "$modified_dirs" >&2 -# -# If we're not running a more specific pipeline, generate a legacy monolithic pipeline -# based on which subdirectories have been modified. We have some heuristics for this -# to be reasonable. -# -# Individual projects should instead define the pre-commit CI tests that suits their -# needs while letting them run on the infrastructure provided by LLVM. -# function compute-projects-to-test() { projects=${@} for project in ${projects}; do @@ -157,11 +152,12 @@ function exclude-windows() { done } +# Prints only projects that are both present in $modified_dirs and the passed +# list. function keep-modified-projects() { projects=${@} - git_diff="$(git diff --name-only HEAD~1)" for project in ${projects}; do - if echo "${git_diff}" | grep -q -E "^${project}/"; then + if echo "$modified_dirs" | grep -q -E "^${project}$"; then echo "${project}" fi done @@ -196,6 +192,35 @@ function check-targets() { done } +# Project specific pipelines. + +# If libc++ or one of the runtimes directories changed. +if echo "$modified_dirs" | grep -q -E "^(libcxx|libcxxabi|libunwind|runtimes|cmake)$"; then + cat <<EOF +- trigger: "libcxx-ci" + build: + message: "${buildMessage}" + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" +EOF +fi + +# If clang changed. +if echo "$modified_dirs" | grep -q -E "^(clang)$"; then + cat <<EOF +- trigger: "clang-ci" + build: + message: "${buildMessage}" + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" +EOF +fi + +# Generic pipeline for projects that have not defined custom steps. +# +# Individual projects should instead define the pre-commit CI tests that suits their +# needs while letting them run on the infrastructure provided by LLVM. + # Figure out which projects need to be built on each platform all_projects="bolt clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl" modified_projects="$(keep-modified-projects ${all_projects})" @@ -211,48 +236,76 @@ windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq) # Generate the appropriate pipeline if [[ "${linux_projects}" != "" ]]; then cat <<EOF -- label: ':linux: x64 Debian' +- label: ':linux: Linux x64' artifact_paths: - - '*_result.json' - - 'build/monolithic-linux/test-results.xml' - agents: - queue: 'linux' + - artifacts/**/* + - '*_result.json' + - 'build/monolithic-linux/test-results.xml' + agents: ${LINUX_AGENTS} retry: automatic: - exit_status: -1 # Agent was lost limit: 2 - - exit_status: 255 - limit: 2 # Forced agent shutdown + - exit_status: 255 # Forced agent shutdown + limit: 2 timeout_in_minutes: 120 env: CC: 'clang' CXX: 'clang++' commands: - - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})"' + - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})"' EOF fi if [[ "${windows_projects}" != "" ]]; then cat <<EOF -- label: ':windows: x64 Windows' +- label: ':windows: Windows x64' artifact_paths: - - '*_result.json' - - 'build/monolithic-windows/test-results.xml' - agents: - queue: 'windows' + - artifacts/**/* + - '*_result.json' + - 'build/monolithic-windows/test-results.xml' + agents: ${WINDOWS_AGENTS} retry: automatic: - exit_status: -1 # Agent was lost limit: 2 - - exit_status: 255 - limit: 2 # Forced agent shutdown + - exit_status: 255 # Forced agent shutdown + limit: 2 timeout_in_minutes: 150 env: CC: 'cl' CXX: 'cl' LD: 'link' commands: - - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64' - - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"' + - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64' + - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"' +EOF +fi + +# If build was triggered from a Phabricator review - send an update back. +if [[ -n "${ph_target_phid:-}" ]]; then + cat << EOF +- continue_on_failure: true + wait: '~' +- label: ':phabricator: update build status on Phabricator' + agents: ${SERVICE_AGENTS} + artifact_paths: + - artifacts/**/* + env: + # Temporary disabled before switching pipelines. + ph_dry_run_report: '1' + commands: + - export SRC=\$\${BUILDKITE_BUILD_PATH}/llvm-premerge-checks + - rm -rf \$\${SRC} + - git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "\$\${SRC}" + - cd \$\${SRC} + - git fetch origin "main":x + - git checkout x + - echo "llvm-premerge-checks commit" + - git rev-parse HEAD + - pip install -q -r \$\${SRC}/scripts/requirements.txt + - cd "\$\$BUILDKITE_BUILD_CHECKOUT_PATH" + - \$\${SRC}/scripts/summary.py + timeout_in_minutes: 10 EOF fi |