From 312bede4967efd531614fe0dbe51f1b94d653e86 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Fri, 6 May 2022 12:11:54 -0400 Subject: wrap: Ignore directories that have a .wrap with the same name If we update e.g. glib.wrap from wrap-git with directory=glib to wrap-file with directory=glib-2.70 we could still have the glib directory that is not referenced by any .wrap file any more. We should still ignore that directory otherwise it overrides the glib.wrap we parsed. --- mesonbuild/wrap/wrap.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index 1cc55ee..e62614f 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -283,17 +283,17 @@ class Resolver: if not os.path.isdir(self.subdir_root): return root, dirs, files = next(os.walk(self.subdir_root)) + ignore_dirs = {'packagecache', 'packagefiles'} for i in files: if not i.endswith('.wrap'): continue fname = os.path.join(self.subdir_root, i) wrap = PackageDefinition(fname, self.subproject) self.wraps[wrap.name] = wrap - if wrap.directory in dirs: - dirs.remove(wrap.directory) + ignore_dirs |= {wrap.directory, wrap.name} # Add dummy package definition for directories not associated with a wrap file. for i in dirs: - if i in ['packagecache', 'packagefiles']: + if i in ignore_dirs: continue fname = os.path.join(self.subdir_root, i) wrap = PackageDefinition(fname, self.subproject) -- cgit v1.1 From cd82804533d2b858564d960576f126c044849cb3 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Fri, 6 May 2022 18:30:18 -0400 Subject: Move some code from wraptool.py to wrap.py --- mesonbuild/wrap/wrap.py | 30 ++++++++++++++++++++++++++++++ mesonbuild/wrap/wraptool.py | 36 ++---------------------------------- 2 files changed, 32 insertions(+), 34 deletions(-) diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index e62614f..b1c2205 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -107,6 +107,36 @@ def open_wrapdburl(urlstring: str, allow_insecure: bool = False, have_opt: bool except urllib.error.URLError as excp: raise WrapException(f'WrapDB connection failed to {urlstring} with error {excp}') +def get_releases_data(allow_insecure: bool) -> bytes: + url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True) + return url.read() + +def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]: + data = get_releases_data(allow_insecure) + return T.cast('T.Dict[str, T.Any]', json.loads(data.decode())) + +def update_wrap_file(wrapfile: str, name: str, new_version: str, new_revision: str, allow_insecure: bool) -> None: + url = open_wrapdburl(f'https://wrapdb.mesonbuild.com/v2/{name}_{new_version}-{new_revision}/{name}.wrap', + allow_insecure, True) + with open(wrapfile, 'wb') as f: + f.write(url.read()) + +def parse_patch_url(patch_url: str) -> T.Tuple[str, str]: + u = urllib.parse.urlparse(patch_url) + if u.netloc != 'wrapdb.mesonbuild.com': + raise WrapException(f'URL {patch_url} does not seems to be a wrapdb patch') + arr = u.path.strip('/').split('/') + if arr[0] == 'v1': + # e.g. https://wrapdb.mesonbuild.com/v1/projects/zlib/1.2.11/5/get_zip + return arr[-3], arr[-2] + elif arr[0] == 'v2': + # e.g. https://wrapdb.mesonbuild.com/v2/zlib_1.2.11-5/get_patch + tag = arr[-2] + _, version = tag.rsplit('_', 1) + version, revision = version.rsplit('-', 1) + return version, revision + else: + raise WrapException(f'Invalid wrapdb URL {patch_url}') class WrapException(MesonException): pass diff --git a/mesonbuild/wrap/wraptool.py b/mesonbuild/wrap/wraptool.py index 80a58ab..be6ceb7 100644 --- a/mesonbuild/wrap/wraptool.py +++ b/mesonbuild/wrap/wraptool.py @@ -12,15 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import sys, os import configparser import shutil import typing as T from glob import glob -from urllib.parse import urlparse -from .wrap import open_wrapdburl, WrapException +from .wrap import (open_wrapdburl, WrapException, get_releases, get_releases_data, + update_wrap_file, parse_patch_url) from pathlib import Path from .. import mesonlib @@ -75,14 +74,6 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None: help='Allow insecure server connections.') p.set_defaults(wrap_func=update_db) -def get_releases_data(allow_insecure: bool) -> bytes: - url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True) - return url.read() - -def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]: - data = get_releases_data(allow_insecure) - return T.cast('T.Dict[str, T.Any]', json.loads(data.decode())) - def list_projects(options: 'argparse.Namespace') -> None: releases = get_releases(options.allow_insecure) for p in releases.keys(): @@ -123,23 +114,6 @@ def install(options: 'argparse.Namespace') -> None: f.write(url.read()) print(f'Installed {name} version {version} revision {revision}') -def parse_patch_url(patch_url: str) -> T.Tuple[str, str]: - u = urlparse(patch_url) - if u.netloc != 'wrapdb.mesonbuild.com': - raise WrapException(f'URL {patch_url} does not seems to be a wrapdb patch') - arr = u.path.strip('/').split('/') - if arr[0] == 'v1': - # e.g. https://wrapdb.mesonbuild.com/v1/projects/zlib/1.2.11/5/get_zip - return arr[-3], arr[-2] - elif arr[0] == 'v2': - # e.g. https://wrapdb.mesonbuild.com/v2/zlib_1.2.11-5/get_patch - tag = arr[-2] - _, version = tag.rsplit('_', 1) - version, revision = version.rsplit('-', 1) - return version, revision - else: - raise WrapException(f'Invalid wrapdb URL {patch_url}') - def get_current_version(wrapfile: str) -> T.Tuple[str, str, str, str, T.Optional[str]]: cp = configparser.ConfigParser(interpolation=None) cp.read(wrapfile) @@ -160,12 +134,6 @@ def get_current_version(wrapfile: str) -> T.Tuple[str, str, str, str, T.Optional patch_filename = wrap_data['patch_filename'] return branch, revision, wrap_data['directory'], wrap_data['source_filename'], patch_filename -def update_wrap_file(wrapfile: str, name: str, new_version: str, new_revision: str, allow_insecure: bool) -> None: - url = open_wrapdburl(f'https://wrapdb.mesonbuild.com/v2/{name}_{new_version}-{new_revision}/{name}.wrap', - allow_insecure, True) - with open(wrapfile, 'wb') as f: - f.write(url.read()) - def update(options: 'argparse.Namespace') -> None: name = options.name if not os.path.isdir('subprojects'): -- cgit v1.1 From aaabd6224f8fbba80840e598e67d4d16a19ae3ec Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Fri, 6 May 2022 11:06:02 -0400 Subject: Make `meson wrap update` command update all wraps in parallel This moves the implementation into msubprojects because it has all the infrastructure to update wraps in parallel while keeping "meson wrap" UX. --- docs/markdown/Commands.md | 3 ++ docs/markdown/snippets/update_wrapdb.md | 8 ++++ mesonbuild/msubprojects.py | 71 ++++++++++++++++++++++++++++----- mesonbuild/wrap/wraptool.py | 9 ++--- 4 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 docs/markdown/snippets/update_wrapdb.md diff --git a/docs/markdown/Commands.md b/docs/markdown/Commands.md index 11ca6e0..c328461 100644 --- a/docs/markdown/Commands.md +++ b/docs/markdown/Commands.md @@ -266,6 +266,9 @@ Manages subprojects of the Meson project. *Since 0.59.0* commands are run on multiple subprojects in parallel by default, use `--num-processes=1` if it is not desired. +Since *0.64.0* the `update` subcommand will not download new wrap files +from WrapDB any more. Use `meson wrap update` command for that instead. + {{ subprojects_arguments.inc }} ### test diff --git a/docs/markdown/snippets/update_wrapdb.md b/docs/markdown/snippets/update_wrapdb.md new file mode 100644 index 0000000..a4e4f27 --- /dev/null +++ b/docs/markdown/snippets/update_wrapdb.md @@ -0,0 +1,8 @@ +## Update all wraps from WrapDB with `meson wrap update` command + +The command `meson wrap update`, with no extra argument, will now update all wraps +that comes from WrapDB to the latest version. The extra `--force` argument will +also replace wraps that do not come from WrapDB if one is available. + +The command `meson subprojects update` will not download new wrap files from +WrapDB any more. diff --git a/mesonbuild/msubprojects.py b/mesonbuild/msubprojects.py index ab527bb..247690a 100755 --- a/mesonbuild/msubprojects.py +++ b/mesonbuild/msubprojects.py @@ -15,13 +15,13 @@ import zipfile from . import mlog from .mesonlib import quiet_git, GitException, Popen_safe, MesonException, windows_proof_rmtree -from .wrap.wrap import Resolver, WrapException, ALL_TYPES -from .wrap import wraptool +from .wrap.wrap import (Resolver, WrapException, ALL_TYPES, PackageDefinition, + parse_patch_url, update_wrap_file, get_releases) if T.TYPE_CHECKING: from typing_extensions import Protocol - from .wrap.wrap import PackageDefinition + SubParsers = argparse._SubParsersAction[argparse.ArgumentParser] class Arguments(Protocol): sourcedir: str @@ -35,6 +35,10 @@ if T.TYPE_CHECKING: rebase: bool reset: bool + class UpdateWrapDBArguments(Arguments): + force: bool + releases: T.Dict[str, T.Any] + class CheckoutArguments(Arguments): b: bool branch_name: str @@ -130,21 +134,53 @@ class Runner: self.logger.done(self.wrap.name, self.log_queue) return result - def update_wrapdb_file(self) -> None: + @staticmethod + def pre_update_wrapdb(options: 'UpdateWrapDBArguments') -> None: + options.releases = get_releases(options.allow_insecure) + + def update_wrapdb(self) -> bool: + self.log(f'Checking latest WrapDB version for {self.wrap.name}...') + options = T.cast('UpdateWrapDBArguments', self.options) + + # Check if this wrap is in WrapDB + info = options.releases.get(self.wrap.name) + if not info: + self.log(' -> Wrap not found in wrapdb') + return True + + # Determine current version try: - patch_url = self.wrap.get('patch_url') - branch, revision = wraptool.parse_patch_url(patch_url) + wrapdb_version = self.wrap.get('wrapdb_version') + branch, revision = wrapdb_version.split('-', 1) except WrapException: - return - new_branch, new_revision = wraptool.get_latest_version(self.wrap.name, self.options.allow_insecure) + # Fallback to parsing the patch URL to determine current version. + # This won't work for projects that have upstream Meson support. + try: + patch_url = self.wrap.get('patch_url') + branch, revision = parse_patch_url(patch_url) + except WrapException: + if not options.force: + self.log(' ->', mlog.red('Could not determine current version, use --force to update any way')) + return False + branch = revision = None + + # Download latest wrap if version differs + latest_version = info['versions'][0] + new_branch, new_revision = latest_version.rsplit('-', 1) if new_branch != branch or new_revision != revision: - wraptool.update_wrap_file(self.wrap.filename, self.wrap.name, new_branch, new_revision, self.options.allow_insecure) - self.log(' -> New wrap file downloaded.') + filename = self.wrap.filename if self.wrap.has_wrap else f'{self.wrap.filename}.wrap' + update_wrap_file(filename, self.wrap.name, + new_branch, new_revision, + options.allow_insecure) + self.log(' -> New version downloaded:', mlog.blue(latest_version)) + else: + self.log(' -> Already at latest version:', mlog.blue(latest_version)) + + return True def update_file(self) -> bool: options = T.cast('UpdateArguments', self.options) - self.update_wrapdb_file() if not os.path.isdir(self.repo_dir): # The subproject is not needed, or it is a tarball extracted in # 'libfoo-1.0' directory and the version has been bumped and the new @@ -591,6 +627,16 @@ def add_subprojects_argument(p: argparse.ArgumentParser) -> None: p.add_argument('subprojects', nargs='*', help='List of subprojects (default: all)') +def add_wrap_update_parser(subparsers: 'SubParsers') -> argparse.ArgumentParser: + p = subparsers.add_parser('update', help='Update wrap files from WrapDB (Since 0.63.0)') + p.add_argument('--force', default=False, action='store_true', + help='Update wraps that does not seems to come from WrapDB') + add_common_arguments(p) + add_subprojects_argument(p) + p.set_defaults(subprojects_func=Runner.update_wrapdb) + p.set_defaults(pre_func=Runner.pre_update_wrapdb) + return p + def add_arguments(parser: argparse.ArgumentParser) -> None: subparsers = parser.add_subparsers(title='Commands', dest='command') subparsers.required = True @@ -669,6 +715,9 @@ def run(options: 'Arguments') -> int: executor = ThreadPoolExecutor(options.num_processes) if types: wraps = [wrap for wrap in wraps if wrap.type in types] + pre_func = getattr(options, 'pre_func', None) + if pre_func: + pre_func(options) logger = Logger(len(wraps)) for wrap in wraps: dirname = Path(subprojects_dir, wrap.directory).as_posix() diff --git a/mesonbuild/wrap/wraptool.py b/mesonbuild/wrap/wraptool.py index be6ceb7..bcf0e67 100644 --- a/mesonbuild/wrap/wraptool.py +++ b/mesonbuild/wrap/wraptool.py @@ -22,7 +22,7 @@ from .wrap import (open_wrapdburl, WrapException, get_releases, get_releases_dat update_wrap_file, parse_patch_url) from pathlib import Path -from .. import mesonlib +from .. import mesonlib, msubprojects if T.TYPE_CHECKING: import argparse @@ -48,11 +48,8 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None: p.add_argument('name') p.set_defaults(wrap_func=install) - p = subparsers.add_parser('update', help='update the project to its newest available release') - p.add_argument('--allow-insecure', default=False, action='store_true', - help='Allow insecure server connections.') - p.add_argument('name') - p.set_defaults(wrap_func=update) + p = msubprojects.add_wrap_update_parser(subparsers) + p.set_defaults(wrap_func=msubprojects.run) p = subparsers.add_parser('info', help='show available versions of a project') p.add_argument('--allow-insecure', default=False, action='store_true', -- cgit v1.1