Merge pull request #10357 from xclaesse/update-wrapdb

Make `meson wrap update` command update all wraps in parallel

2 files changed, 38 insertions, 43 deletions

diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py
index 1cc55ee..b1c2205 100644
--- a/mesonbuild/wrap/wrap.py
+++ b/mesonbuild/wrap/wrap.py
@@ -107,6 +107,36 @@ def open_wrapdburl(urlstring: str, allow_insecure: bool = False, have_opt: bool
     except urllib.error.URLError as excp:
         raise WrapException(f'WrapDB connection failed to {urlstring} with error {excp}')
 
+def get_releases_data(allow_insecure: bool) -> bytes:
+    url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True)
+    return url.read()
+
+def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]:
+    data = get_releases_data(allow_insecure)
+    return T.cast('T.Dict[str, T.Any]', json.loads(data.decode()))
+
+def update_wrap_file(wrapfile: str, name: str, new_version: str, new_revision: str, allow_insecure: bool) -> None:
+    url = open_wrapdburl(f'https://wrapdb.mesonbuild.com/v2/{name}_{new_version}-{new_revision}/{name}.wrap',
+                         allow_insecure, True)
+    with open(wrapfile, 'wb') as f:
+        f.write(url.read())
+
+def parse_patch_url(patch_url: str) -> T.Tuple[str, str]:
+    u = urllib.parse.urlparse(patch_url)
+    if u.netloc != 'wrapdb.mesonbuild.com':
+        raise WrapException(f'URL {patch_url} does not seems to be a wrapdb patch')
+    arr = u.path.strip('/').split('/')
+    if arr[0] == 'v1':
+        # e.g. https://wrapdb.mesonbuild.com/v1/projects/zlib/1.2.11/5/get_zip
+        return arr[-3], arr[-2]
+    elif arr[0] == 'v2':
+        # e.g. https://wrapdb.mesonbuild.com/v2/zlib_1.2.11-5/get_patch
+        tag = arr[-2]
+        _, version = tag.rsplit('_', 1)
+        version, revision = version.rsplit('-', 1)
+        return version, revision
+    else:
+        raise WrapException(f'Invalid wrapdb URL {patch_url}')
 
 class WrapException(MesonException):
     pass
@@ -283,17 +313,17 @@ class Resolver:
         if not os.path.isdir(self.subdir_root):
             return
         root, dirs, files = next(os.walk(self.subdir_root))
+        ignore_dirs = {'packagecache', 'packagefiles'}
         for i in files:
             if not i.endswith('.wrap'):
                 continue
             fname = os.path.join(self.subdir_root, i)
             wrap = PackageDefinition(fname, self.subproject)
             self.wraps[wrap.name] = wrap
-            if wrap.directory in dirs:
-                dirs.remove(wrap.directory)
+            ignore_dirs |= {wrap.directory, wrap.name}
         # Add dummy package definition for directories not associated with a wrap file.
         for i in dirs:
-            if i in ['packagecache', 'packagefiles']:
+            if i in ignore_dirs:
                 continue
             fname = os.path.join(self.subdir_root, i)
             wrap = PackageDefinition(fname, self.subproject)
diff --git a/mesonbuild/wrap/wraptool.py b/mesonbuild/wrap/wraptool.py
index 80a58ab..bcf0e67 100644
--- a/mesonbuild/wrap/wraptool.py
+++ b/mesonbuild/wrap/wraptool.py
@@ -12,18 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import sys, os
 import configparser
 import shutil
 import typing as T
 
 from glob import glob
-from urllib.parse import urlparse
-from .wrap import open_wrapdburl, WrapException
+from .wrap import (open_wrapdburl, WrapException, get_releases, get_releases_data,
+                   update_wrap_file, parse_patch_url)
 from pathlib import Path
 
-from .. import mesonlib
+from .. import mesonlib, msubprojects
 
 if T.TYPE_CHECKING:
     import argparse
@@ -49,11 +48,8 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None:
     p.add_argument('name')
     p.set_defaults(wrap_func=install)
 
-    p = subparsers.add_parser('update', help='update the project to its newest available release')
-    p.add_argument('--allow-insecure', default=False, action='store_true',
-                   help='Allow insecure server connections.')
-    p.add_argument('name')
-    p.set_defaults(wrap_func=update)
+    p = msubprojects.add_wrap_update_parser(subparsers)
+    p.set_defaults(wrap_func=msubprojects.run)
 
     p = subparsers.add_parser('info', help='show available versions of a project')
     p.add_argument('--allow-insecure', default=False, action='store_true',
@@ -75,14 +71,6 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None:
                    help='Allow insecure server connections.')
     p.set_defaults(wrap_func=update_db)
 
-def get_releases_data(allow_insecure: bool) -> bytes:
-    url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True)
-    return url.read()
-
-def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]:
-    data = get_releases_data(allow_insecure)
-    return T.cast('T.Dict[str, T.Any]', json.loads(data.decode()))
-
 def list_projects(options: 'argparse.Namespace') -> None:
     releases = get_releases(options.allow_insecure)
     for p in releases.keys():
@@ -123,23 +111,6 @@ def install(options: 'argparse.Namespace') -> None:
         f.write(url.read())
     print(f'Installed {name} version {version} revision {revision}')
 
-def parse_patch_url(patch_url: str) -> T.Tuple[str, str]:
-    u = urlparse(patch_url)
-    if u.netloc != 'wrapdb.mesonbuild.com':
-        raise WrapException(f'URL {patch_url} does not seems to be a wrapdb patch')
-    arr = u.path.strip('/').split('/')
-    if arr[0] == 'v1':
-        # e.g. https://wrapdb.mesonbuild.com/v1/projects/zlib/1.2.11/5/get_zip
-        return arr[-3], arr[-2]
-    elif arr[0] == 'v2':
-        # e.g. https://wrapdb.mesonbuild.com/v2/zlib_1.2.11-5/get_patch
-        tag = arr[-2]
-        _, version = tag.rsplit('_', 1)
-        version, revision = version.rsplit('-', 1)
-        return version, revision
-    else:
-        raise WrapException(f'Invalid wrapdb URL {patch_url}')
-
 def get_current_version(wrapfile: str) -> T.Tuple[str, str, str, str, T.Optional[str]]:
     cp = configparser.ConfigParser(interpolation=None)
     cp.read(wrapfile)
@@ -160,12 +131,6 @@ def get_current_version(wrapfile: str) -> T.Tuple[str, str, str, str, T.Optional
         patch_filename = wrap_data['patch_filename']
     return branch, revision, wrap_data['directory'], wrap_data['source_filename'], patch_filename
 
-def update_wrap_file(wrapfile: str, name: str, new_version: str, new_revision: str, allow_insecure: bool) -> None:
-    url = open_wrapdburl(f'https://wrapdb.mesonbuild.com/v2/{name}_{new_version}-{new_revision}/{name}.wrap',
-                         allow_insecure, True)
-    with open(wrapfile, 'wb') as f:
-        f.write(url.read())
-
 def update(options: 'argparse.Namespace') -> None:
     name = options.name
     if not os.path.isdir('subprojects'):