packages: support overriding package mappings

Allow specifying individual projects to override the version they require
of a required version of a PyPI or CPAN package.  Projects that want
to test with an old version of Meson could therefore disable the distro
version and install the old version directly from PyPI.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

6 files changed, 49 insertions, 7 deletions

diff --git a/docs/platforms_and_mappings.rst b/docs/platforms_and_mappings.rst
index 4351da3..6b45b98 100644
--- a/docs/platforms_and_mappings.rst
+++ b/docs/platforms_and_mappings.rst
@@ -95,3 +95,14 @@ In order to simply add a new package mapping:
    repo  and from the root of the original project's git run::
 
    $ lcitool manifest
+
+Project-specific mappings
+-------------------------
+
+Some projects may need different mappings, for example if they want the
+tests to use specific versions of packages from PyPI or CPAN.
+For this reason the possibility to use YAML files stored outside
+the libvirt-ci repository, located using the ``--data-dir DIR``
+argument to ``lcitool``, is extended to the following paths::
+
+  $DIR/mappings.yml
diff --git a/lcitool/application.py b/lcitool/application.py
index 7505fe2..e70b391 100644
--- a/lcitool/application.py
+++ b/lcitool/application.py
@@ -74,7 +74,7 @@ class Application:
         config = Config()
         targets = Targets()
         inventory = Inventory(targets, config)
-        packages = Packages()
+        packages = Packages(data_dir)
         projects = Projects(data_dir)
 
         hosts_expanded = inventory.expand_hosts(hosts_pattern)
@@ -225,7 +225,7 @@ class Application:
         self._entrypoint_debug(args)
 
         targets = Targets()
-        packages = Packages()
+        packages = Packages(args.data_dir)
         projects = Projects(args.data_dir)
         projects_expanded = projects.expand_names(args.projects)
 
@@ -254,7 +254,7 @@ class Application:
         self._entrypoint_debug(args)
 
         targets = Targets()
-        packages = Packages()
+        packages = Packages(args.data_dir)
         projects = Projects(args.data_dir)
         projects_expanded = projects.expand_names(args.projects)
         target = BuildTarget(targets, packages, args.target, args.cross_arch)
@@ -279,7 +279,7 @@ class Application:
         self._entrypoint_debug(args)
 
         targets = Targets()
-        packages = Packages()
+        packages = Packages(args.data_dir)
         projects = Projects(args.data_dir)
         projects_expanded = projects.expand_names(args.projects)
         target = BuildTarget(targets, packages, args.target, args.cross_arch)
@@ -301,7 +301,7 @@ class Application:
             base_path = Path(args.base_dir)
         ci_path = Path(args.ci_dir)
         targets = Targets()
-        packages = Packages()
+        packages = Packages(args.data_dir)
         projects = Projects(args.data_dir)
         manifest = Manifest(targets, packages, projects, args.manifest, args.quiet, ci_path, base_path)
         manifest.generate(args.dry_run)
diff --git a/lcitool/packages.py b/lcitool/packages.py
index efa7319..227d37b 100644
--- a/lcitool/packages.py
+++ b/lcitool/packages.py
@@ -315,7 +315,7 @@ class Packages:
 
     def _load_mappings(self):
         try:
-            mappings = self._data_dir.load_yaml("facts", "mappings")
+            mappings = self._data_dir.merge_facts("facts", "mappings")
             self._mappings = mappings["mappings"]
             self._pypi_mappings = mappings["pypi_mappings"]
             self._cpan_mappings = mappings["cpan_mappings"]
diff --git a/lcitool/util.py b/lcitool/util.py
index 56a6831..9a899ba 100644
--- a/lcitool/util.py
+++ b/lcitool/util.py
@@ -262,6 +262,14 @@ class DataDir:
         with open(file, "r") as infile:
             return yaml.safe_load(infile)
 
+    def merge_facts(self, resource_path, name):
+        result = {}
+        for file in self._search(resource_path, name + ".yml"):
+            log.debug(f"Loading facts from '{file}'")
+            with open(file, "r") as infile:
+                merge_dict(yaml.safe_load(infile), result)
+        return result
+
 
 def validate_cross_platform(cross_arch, osname):
     if osname not in ["Debian", "Fedora"]:
diff --git a/tests/data/packages/override/mappings.yml b/tests/data/packages/override/mappings.yml
new file mode 100644
index 0000000..7b11bfb
--- /dev/null
+++ b/tests/data/packages/override/mappings.yml
@@ -0,0 +1,10 @@
+mappings:
+  meson:
+    CentOSStream8:
+
+  python3:
+    CentOSStream8: python38
+
+pypi_mappings:
+  meson:
+    default: meson==0.63.2
diff --git a/tests/test_packages.py b/tests/test_packages.py
index ec4492e..8947531 100644
--- a/tests/test_packages.py
+++ b/tests/test_packages.py
@@ -14,8 +14,9 @@ from functools import total_ordering
 from pathlib import Path
 from lcitool import util
 from lcitool.projects import Project, ProjectError
-from lcitool.packages import NativePackage, CrossPackage, PyPIPackage, CPANPackage
+from lcitool.packages import NativePackage, CrossPackage, PyPIPackage, CPANPackage, Packages
 from lcitool.targets import BuildTarget
+from lcitool.util import DataDir
 
 from conftest import ALL_TARGETS
 
@@ -78,6 +79,18 @@ def test_package_resolution(targets, packages, test_project, target, arch):
     test_utils.assert_yaml_matches_file(actual, expected_path)
 
 
+def test_resolution_override(targets, test_project):
+    datadir = DataDir(Path(test_utils.test_data_dir(__file__), 'override'))
+    packages = Packages(datadir)
+    target_obj = BuildTarget(targets, packages, "centos-stream-8")
+    pkgs = test_project.get_packages(target_obj)
+    assert isinstance(pkgs['meson'], PyPIPackage)
+
+    actual = packages_as_dict(pkgs)
+    assert 'meson==0.63.2' in actual['pypi']
+    assert 'python38' in actual['native']
+
+
 @pytest.mark.parametrize(
     "target",
     [pytest.param(target, id=target) for target in get_non_cross_targets()],