Merge pull request #8941 from xclaesse/wrapdb

Automatically use WrapDB fallback

3 files changed, 81 insertions, 23 deletions

diff --git a/mesonbuild/interpreter/dependencyfallbacks.py b/mesonbuild/interpreter/dependencyfallbacks.py
index 3fbce42..54be990 100644
--- a/mesonbuild/interpreter/dependencyfallbacks.py
+++ b/mesonbuild/interpreter/dependencyfallbacks.py
@@ -66,14 +66,6 @@ class DependencyFallbacksHolder(MesonInterpreterObject):
         self._subproject_impl(subp_name, varname)
 
     def _subproject_impl(self, subp_name: str, varname: str) -> None:
-        if not varname:
-            # If no variable name is specified, check if the wrap file has one.
-            # If the wrap file has a variable name, better use it because the
-            # subproject most probably is not using meson.override_dependency().
-            for name in self.names:
-                varname = self.wrap_resolver.get_varname(subp_name, name)
-                if varname:
-                    break
         assert self.subproject_name is None
         self.subproject_name = subp_name
         self.subproject_varname = varname
@@ -175,6 +167,14 @@ class DependencyFallbacksHolder(MesonInterpreterObject):
         # Legacy: Use the variable name if provided instead of relying on the
         # subproject to override one of our dependency names
         if not varname:
+            # If no variable name is specified, check if the wrap file has one.
+            # If the wrap file has a variable name, better use it because the
+            # subproject most probably is not using meson.override_dependency().
+            for name in self.names:
+                varname = self.wrap_resolver.get_varname(subp_name, name)
+                if varname:
+                    break
+        if not varname:
             mlog.warning(f'Subproject {subp_name!r} did not override {self._display_name!r} dependency and no variable name specified')
             mlog.log('Dependency', mlog.bold(self._display_name), 'from subproject',
                      mlog.bold(subproject.subdir), 'found:', mlog.red('NO'))
diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py
index 10aa1b8..1cc55ee 100644
--- a/mesonbuild/wrap/wrap.py
+++ b/mesonbuild/wrap/wrap.py
@@ -29,10 +29,12 @@ import configparser
 import time
 import typing as T
 import textwrap
+import json
 
 from base64 import b64encode
 from netrc import netrc
 from pathlib import Path
+
 from . import WrapMode
 from .. import coredata
 from ..mesonlib import quiet_git, GIT, ProgressBar, MesonException, windows_proof_rmtree, Popen_safe
@@ -262,8 +264,12 @@ class Resolver:
         self.netrc: T.Optional[netrc] = None
         self.provided_deps = {} # type: T.Dict[str, PackageDefinition]
         self.provided_programs = {} # type: T.Dict[str, PackageDefinition]
+        self.wrapdb: T.Dict[str, T.Any] = {}
+        self.wrapdb_provided_deps: T.Dict[str, str] = {}
+        self.wrapdb_provided_programs: T.Dict[str, str] = {}
         self.load_wraps()
         self.load_netrc()
+        self.load_wrapdb()
 
     def load_netrc(self) -> None:
         try:
@@ -294,18 +300,48 @@ class Resolver:
             self.wraps[wrap.name] = wrap
 
         for wrap in self.wraps.values():
-            for k in wrap.provided_deps.keys():
-                if k in self.provided_deps:
-                    prev_wrap = self.provided_deps[k]
-                    m = f'Multiple wrap files provide {k!r} dependency: {wrap.basename} and {prev_wrap.basename}'
-                    raise WrapException(m)
-                self.provided_deps[k] = wrap
-            for k in wrap.provided_programs:
-                if k in self.provided_programs:
-                    prev_wrap = self.provided_programs[k]
-                    m = f'Multiple wrap files provide {k!r} program: {wrap.basename} and {prev_wrap.basename}'
-                    raise WrapException(m)
-                self.provided_programs[k] = wrap
+            self.add_wrap(wrap)
+
+    def add_wrap(self, wrap: PackageDefinition) -> None:
+        for k in wrap.provided_deps.keys():
+            if k in self.provided_deps:
+                prev_wrap = self.provided_deps[k]
+                m = f'Multiple wrap files provide {k!r} dependency: {wrap.basename} and {prev_wrap.basename}'
+                raise WrapException(m)
+            self.provided_deps[k] = wrap
+        for k in wrap.provided_programs:
+            if k in self.provided_programs:
+                prev_wrap = self.provided_programs[k]
+                m = f'Multiple wrap files provide {k!r} program: {wrap.basename} and {prev_wrap.basename}'
+                raise WrapException(m)
+            self.provided_programs[k] = wrap
+
+    def load_wrapdb(self) -> None:
+        try:
+            with Path(self.subdir_root, 'wrapdb.json').open('r', encoding='utf-8') as f:
+                self.wrapdb = json.load(f)
+        except FileNotFoundError:
+            return
+        for name, info in self.wrapdb.items():
+            self.wrapdb_provided_deps.update({i: name for i in info.get('dependency_names', [])})
+            self.wrapdb_provided_programs.update({i: name for i in info.get('program_names', [])})
+
+    def get_from_wrapdb(self, subp_name: str) -> PackageDefinition:
+        info = self.wrapdb.get(subp_name)
+        if not info:
+            return None
+        self.check_can_download()
+        latest_version = info['versions'][0]
+        version, revision = latest_version.rsplit('-', 1)
+        url = urllib.request.urlopen(f'https://wrapdb.mesonbuild.com/v2/{subp_name}_{version}-{revision}/{subp_name}.wrap')
+        fname = Path(self.subdir_root, f'{subp_name}.wrap')
+        with fname.open('wb') as f:
+            f.write(url.read())
+        mlog.log(f'Installed {subp_name} version {version} revision {revision}')
+        wrap = PackageDefinition(str(fname))
+        self.wraps[wrap.name] = wrap
+        self.add_wrap(wrap)
+        return wrap
 
     def merge_wraps(self, other_resolver: 'Resolver') -> None:
         for k, v in other_resolver.wraps.items():
@@ -323,7 +359,8 @@ class Resolver:
         if wrap:
             dep_var = wrap.provided_deps.get(packagename)
             return wrap.name, dep_var
-        return None, None
+        wrap_name = self.wrapdb_provided_deps.get(packagename)
+        return wrap_name, None
 
     def get_varname(self, subp_name: str, depname: str) -> T.Optional[str]:
         wrap = self.wraps.get(subp_name)
@@ -334,6 +371,9 @@ class Resolver:
             wrap = self.provided_programs.get(name)
             if wrap:
                 return wrap.name
+            wrap_name = self.wrapdb_provided_programs.get(name)
+            if wrap_name:
+                return wrap_name
         return None
 
     def resolve(self, packagename: str, method: str) -> str:
@@ -341,6 +381,8 @@ class Resolver:
         self.directory = packagename
         self.wrap = self.wraps.get(packagename)
         if not self.wrap:
+            self.wrap = self.get_from_wrapdb(packagename)
+        if not self.wrap:
             m = f'Neither a subproject directory nor a {self.packagename}.wrap file was found.'
             raise WrapNotFoundException(m)
         self.directory = self.wrap.directory
diff --git a/mesonbuild/wrap/wraptool.py b/mesonbuild/wrap/wraptool.py
index ec2ac3e..80a58ab 100644
--- a/mesonbuild/wrap/wraptool.py
+++ b/mesonbuild/wrap/wraptool.py
@@ -21,6 +21,7 @@ import typing as T
 from glob import glob
 from urllib.parse import urlparse
 from .wrap import open_wrapdburl, WrapException
+from pathlib import Path
 
 from .. import mesonlib
 
@@ -69,9 +70,18 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None:
     p.add_argument('project_path')
     p.set_defaults(wrap_func=promote)
 
-def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]:
+    p = subparsers.add_parser('update-db', help='Update list of projects available in WrapDB (Since 0.61.0)')
+    p.add_argument('--allow-insecure', default=False, action='store_true',
+                   help='Allow insecure server connections.')
+    p.set_defaults(wrap_func=update_db)
+
+def get_releases_data(allow_insecure: bool) -> bytes:
     url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True)
-    return T.cast('T.Dict[str, T.Any]', json.loads(url.read().decode()))
+    return url.read()
+
+def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]:
+    data = get_releases_data(allow_insecure)
+    return T.cast('T.Dict[str, T.Any]', json.loads(data.decode()))
 
 def list_projects(options: 'argparse.Namespace') -> None:
     releases = get_releases(options.allow_insecure)
@@ -244,6 +254,12 @@ def status(options: 'argparse.Namespace') -> None:
         else:
             print('', name, f'not up to date. Have {current_branch} {current_revision}, but {latest_branch} {latest_revision} is available.')
 
+def update_db(options: 'argparse.Namespace') -> None:
+    data = get_releases_data(options.allow_insecure)
+    Path('subprojects').mkdir(exist_ok=True)
+    with Path('subprojects/wrapdb.json').open('wb') as f:
+        f.write(data)
+
 def run(options: 'argparse.Namespace') -> int:
     options.wrap_func(options)
     return 0