gitlab: provide way to run with upstream CI environment when pushing to forks

Allow users to avoid wasting cycles re-updating the throwaway
environment for a CI run by using RUN_PIPELINE_UPSTREAM_ENV=1 instead of
RUN_PIPELINE=1

This shows how the CI run will look upstream including avoiding
temporary breakage when the update is broken.

Signed-off-by: Peter Krempa <pkrempa@redhat.com>

4 files changed, 69 insertions, 6 deletions

diff --git a/lcitool/gitlab.py b/lcitool/gitlab.py
index 3ea820e..85ffb42 100644
--- a/lcitool/gitlab.py
+++ b/lcitool/gitlab.py
@@ -35,6 +35,12 @@ def docs(namespace):
         #    created/updated. Setting this variable to a non-empty
         #    value allows CI testing prior to opening a merge request.
         #
+        #  - RUN_PIPELINE_UPSTREAM_ENV - same semantics as RUN_PIPELINE,
+        #    but uses the CI environment (containers) from the upstream project
+        #    rather than creating and updating a throwaway environment
+        #    Should not be used if the pushed branch includes CI container
+        #    changes.
+        #
         #  - RUN_CONTAINER_BUILDS - CI pipelines in upstream only
         #    publish containers if CI file changes are detected.
         #    Setting this variable to a non-empty value will force
@@ -204,6 +210,13 @@ def _build_template(template, image, project, cidir):
             - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
               when: on_success
 
+            # forks: pushes to a branch when a pipeline run in upstream env is explicitly requested
+            - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV && $JOB_OPTIONAL'
+              when: manual
+              allow_failure: true
+            - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV'
+              when: on_success
+
             # upstream: other web/api/scheduled pipelines targeting the default branch
             - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL'
               when: manual
@@ -242,6 +255,10 @@ def _build_template(template, image, project, cidir):
             - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
               when: on_success
 
+            # forks: avoid build in local env when job requests run in upstream containers
+            - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV'
+              when: never
+
             # forks: pushes to branches with pipeline requested
             - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE && $JOB_OPTIONAL'
               when: manual
@@ -344,12 +361,17 @@ def cirrus_template(cidir):
             - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push"'
               when: on_success
 
-            # forks: pushes to branches with pipeline requested
+            # forks: pushes to branches with pipeline requested (including pipeline in upstream environment)
             - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE && $JOB_OPTIONAL'
               when: manual
               allow_failure: true
             - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE'
               when: on_success
+            - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV && $JOB_OPTIONAL'
+              when: manual
+              allow_failure: true
+            - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV'
+              when: on_success
 
             # upstream+forks: Run pipelines on MR, web, api & scheduled
             - if: '$CI_PIPELINE_SOURCE =~ /(web|api|schedule|merge_request_event)/ && $JOB_OPTIONAL'
@@ -381,9 +403,11 @@ def check_dco_job():
             - if: '$CI_PIPELINE_SOURCE =~ "merge_request_event"'
               when: on_success
 
-            # forks: pushes to branches with pipeline requested
+            # forks: pushes to branches with pipeline requested (including upstream env pipelines)
             - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $RUN_PIPELINE'
               when: on_success
+            - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $RUN_PIPELINE_UPSTREAM_ENV'
+              when: on_success
 
             # upstream+forks: that's all folks
             - when: never
@@ -405,9 +429,11 @@ def code_fmt_template():
             - if: '$CI_PIPELINE_SOURCE =~ /(web|api|schedule|merge_request_event)/'
               when: on_success
 
-            # forks: pushes to branches with pipeline requested
+            # forks: pushes to branches with pipeline requested (including upstream env pipelines)
             - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $RUN_PIPELINE'
               when: on_success
+            - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $RUN_PIPELINE_UPSTREAM_ENV'
+              when: on_success
 
             # upstream+forks: that's all folks
             - when: never
diff --git a/tests/data/manifest/out/ci/gitlab.yml b/tests/data/manifest/out/ci/gitlab.yml
index 6a4b167..7bebe1e 100644
--- a/tests/data/manifest/out/ci/gitlab.yml
+++ b/tests/data/manifest/out/ci/gitlab.yml
@@ -7,6 +7,12 @@
 #    created/updated. Setting this variable to a non-empty
 #    value allows CI testing prior to opening a merge request.
 #
+#  - RUN_PIPELINE_UPSTREAM_ENV - same semantics as RUN_PIPELINE,
+#    but uses the CI environment (containers) from the upstream project
+#    rather than creating and updating a throwaway environment
+#    Should not be used if the pushed branch includes CI container
+#    changes.
+#
 #  - RUN_CONTAINER_BUILDS - CI pipelines in upstream only
 #    publish containers if CI file changes are detected.
 #    Setting this variable to a non-empty value will force
diff --git a/tests/data/manifest/out/ci/gitlab/build-templates.yml b/tests/data/manifest/out/ci/gitlab/build-templates.yml
index a843c9e..6f9e8e0 100644
--- a/tests/data/manifest/out/ci/gitlab/build-templates.yml
+++ b/tests/data/manifest/out/ci/gitlab/build-templates.yml
@@ -30,6 +30,13 @@
     - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       when: on_success
 
+    # forks: pushes to a branch when a pipeline run in upstream env is explicitly requested
+    - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV && $JOB_OPTIONAL'
+      when: manual
+      allow_failure: true
+    - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV'
+      when: on_success
+
     # upstream: other web/api/scheduled pipelines targeting the default branch
     - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL'
       when: manual
@@ -68,6 +75,10 @@
     - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
       when: on_success
 
+    # forks: avoid build in local env when job requests run in upstream containers
+    - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV'
+      when: never
+
     # forks: pushes to branches with pipeline requested
     - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE && $JOB_OPTIONAL'
       when: manual
@@ -145,6 +156,13 @@
     - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       when: on_success
 
+    # forks: pushes to a branch when a pipeline run in upstream env is explicitly requested
+    - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV && $JOB_OPTIONAL'
+      when: manual
+      allow_failure: true
+    - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV'
+      when: on_success
+
     # upstream: other web/api/scheduled pipelines targeting the default branch
     - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL'
       when: manual
@@ -183,6 +201,10 @@
     - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
       when: on_success
 
+    # forks: avoid build in local env when job requests run in upstream containers
+    - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV'
+      when: never
+
     # forks: pushes to branches with pipeline requested
     - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE && $JOB_OPTIONAL'
       when: manual
@@ -267,12 +289,17 @@
     - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push"'
       when: on_success
 
-    # forks: pushes to branches with pipeline requested
+    # forks: pushes to branches with pipeline requested (including pipeline in upstream environment)
     - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE && $JOB_OPTIONAL'
       when: manual
       allow_failure: true
     - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE'
       when: on_success
+    - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV && $JOB_OPTIONAL'
+      when: manual
+      allow_failure: true
+    - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV'
+      when: on_success
 
     # upstream+forks: Run pipelines on MR, web, api & scheduled
     - if: '$CI_PIPELINE_SOURCE =~ /(web|api|schedule|merge_request_event)/ && $JOB_OPTIONAL'
diff --git a/tests/data/manifest/out/ci/gitlab/sanity-checks.yml b/tests/data/manifest/out/ci/gitlab/sanity-checks.yml
index b26d4b1..07cc27e 100644
--- a/tests/data/manifest/out/ci/gitlab/sanity-checks.yml
+++ b/tests/data/manifest/out/ci/gitlab/sanity-checks.yml
@@ -10,9 +10,11 @@ check-dco:
     - if: '$CI_PIPELINE_SOURCE =~ "merge_request_event"'
       when: on_success
 
-    # forks: pushes to branches with pipeline requested
+    # forks: pushes to branches with pipeline requested (including upstream env pipelines)
     - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $RUN_PIPELINE'
       when: on_success
+    - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $RUN_PIPELINE_UPSTREAM_ENV'
+      when: on_success
 
     # upstream+forks: that's all folks
     - when: never
@@ -32,9 +34,11 @@ check-dco:
     - if: '$CI_PIPELINE_SOURCE =~ /(web|api|schedule|merge_request_event)/'
       when: on_success
 
-    # forks: pushes to branches with pipeline requested
+    # forks: pushes to branches with pipeline requested (including upstream env pipelines)
     - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $RUN_PIPELINE'
       when: on_success
+    - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $RUN_PIPELINE_UPSTREAM_ENV'
+      when: on_success
 
     # upstream+forks: that's all folks
     - when: never