lcitool/manifest: split gitlab container template jobs into separate file

Currently ci/gitlab.yml contains all the generated build logic.

It is desired to be able to only rebuild containers when either their
dockerfile changes, or the gitlab build logic changes. While we could
write a rule matching ci/gitlab.yml that will trigger redundant
container rebuilds when simply changing the set of targets, or
build jobs.

This change splits the container template jobs into a separate file
called 'ci/gitlab/container-templates.yml'

With this split it is possible to declare:

   rules:
     - changes:
         ci/gitlab/container-templates.yml
	 ci/containers/$NAME.Dockerfile

to optimize exactly when a specific container is rebuilt.

Signed-off-by: Daniel P. Berrangé <berrange@redhat.com>

6 files changed, 34 insertions, 22 deletions

diff --git a/guests/lcitool/lcitool/gitlab.py b/guests/lcitool/lcitool/gitlab.py
index cc81b3e..cc4353e 100644
--- a/guests/lcitool/lcitool/gitlab.py
+++ b/guests/lcitool/lcitool/gitlab.py
@@ -7,6 +7,11 @@
 import textwrap
 
 
+def includes(paths):
+    lines = [f"  - local: '{path}'" for path in paths]
+    return "include:\n" + "\n".join(lines)
+
+
 def container_template(namespace, project, cidir):
     return textwrap.dedent(
         f"""
diff --git a/guests/lcitool/lcitool/manifest.py b/guests/lcitool/lcitool/manifest.py
index 964ea94..944cb8f 100644
--- a/guests/lcitool/lcitool/manifest.py
+++ b/guests/lcitool/lcitool/manifest.py
@@ -247,6 +247,7 @@ class Manifest:
         header = util.generate_file_header(["manifest", self.configpath])
 
         lines = header + "\n".join(content)
+        lines = lines.strip() + "\n"
         if not dryrun:
             util.atomic_write(path, lines)
 
@@ -275,8 +276,13 @@ class Manifest:
         jobinfo = gitlabinfo["jobs"]
 
         gitlabcontent = []
+        includes = []
         if gitlabinfo["containers"]:
-            gitlabcontent.append(gitlab.container_template(namespace, project, self.cidir))
+            path = Path(gitlabdir, "container-templates.yml")
+            content = [gitlab.container_template(namespace, project, self.cidir)]
+            self._replace_file(content, path, dryrun)
+            includes.append(path)
+
         if gitlabinfo["builds"]:
             if have_native:
                 gitlabcontent.append(gitlab.native_build_template())
@@ -303,7 +309,8 @@ class Manifest:
             gitlabcontent.extend(self._generate_gitlab_cirrus_build_jobs())
 
         path = Path(self.cidir, "gitlab.yml")
-        self._replace_file(gitlabcontent, path, dryrun)
+        content = [gitlab.includes(includes)] + gitlabcontent
+        self._replace_file(content, path, dryrun)
 
     def _generate_gitlab_container_jobs(self, cross):
         jobs = []
diff --git a/guests/lcitool/tests/.gitignore b/guests/lcitool/tests/.gitignore
deleted file mode 100644
index 93b6cee..0000000
--- a/guests/lcitool/tests/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-ci/
diff --git a/guests/lcitool/tests/data/manifest/out/ci/gitlab.yml b/guests/lcitool/tests/data/manifest/out/ci/gitlab.yml
index ae922bf..77aded2 100644
--- a/guests/lcitool/tests/data/manifest/out/ci/gitlab.yml
+++ b/guests/lcitool/tests/data/manifest/out/ci/gitlab.yml
@@ -1,22 +1,5 @@
-
-.container_job:
-  image: docker:stable
-  stage: containers
-  needs: []
-  services:
-    - docker:dind
-  before_script:
-    - export TAG="$CI_REGISTRY_IMAGE/ci-$NAME:latest"
-    - export COMMON_TAG="$CI_REGISTRY/test-group/test-minimal/ci-$NAME:latest"
-    - docker info
-    - docker login registry.gitlab.com -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD"
-  script:
-    - docker pull "$TAG" || docker pull "$COMMON_TAG" || true
-    - docker build --cache-from "$TAG" --cache-from "$COMMON_TAG" --tag "$TAG" -f "ci/containers/$NAME.Dockerfile" ci/containers
-    - docker push "$TAG"
-  after_script:
-    - docker logout
-
+include:
+  - local: 'ci/gitlab/container-templates.yml'
 
 .gitlab_native_build_job:
   image: $CI_REGISTRY_IMAGE/ci-$NAME:latest
diff --git a/guests/lcitool/tests/data/manifest/out/ci/gitlab/container-templates.yml b/guests/lcitool/tests/data/manifest/out/ci/gitlab/container-templates.yml
new file mode 100644
index 0000000..15d1072
--- /dev/null
+++ b/guests/lcitool/tests/data/manifest/out/ci/gitlab/container-templates.yml
@@ -0,0 +1,17 @@
+.container_job:
+  image: docker:stable
+  stage: containers
+  needs: []
+  services:
+    - docker:dind
+  before_script:
+    - export TAG="$CI_REGISTRY_IMAGE/ci-$NAME:latest"
+    - export COMMON_TAG="$CI_REGISTRY/test-group/test-minimal/ci-$NAME:latest"
+    - docker info
+    - docker login registry.gitlab.com -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD"
+  script:
+    - docker pull "$TAG" || docker pull "$COMMON_TAG" || true
+    - docker build --cache-from "$TAG" --cache-from "$COMMON_TAG" --tag "$TAG" -f "ci/containers/$NAME.Dockerfile" ci/containers
+    - docker push "$TAG"
+  after_script:
+    - docker logout
diff --git a/guests/lcitool/tests/test_manifest.py b/guests/lcitool/tests/test_manifest.py
index b12af84..cbdd60d 100644
--- a/guests/lcitool/tests/test_manifest.py
+++ b/guests/lcitool/tests/test_manifest.py
@@ -114,6 +114,7 @@ def test_generate(monkeypatch, custom_projects):
 
         # Verify content of files we expect to be created
         assert_write(Path("ci", "gitlab.yml"))
+        assert_write(Path("ci", "gitlab", "container-templates.yml"))
         assert_write(Path("ci", "cirrus", "freebsd-current.vars"))
         assert_write(Path("ci", "cirrus", "macos-11.vars"))
         assert_write(Path("ci", "containers", "centos-stream-9.Dockerfile"))