aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJussi Pakkanen <jpakkane@gmail.com>2022-10-12 22:52:25 +0300
committerGitHub <noreply@github.com>2022-10-12 22:52:25 +0300
commit9a57608cac4a422ffa450bcf75c0609639035e11 (patch)
tree055814a99331863a371abcf2a8e14f50d26da2d7
parent9b33885c2d00bb3737d3f35c278d599c50cfd609 (diff)
parentaaabd6224f8fbba80840e598e67d4d16a19ae3ec (diff)
downloadmeson-9a57608cac4a422ffa450bcf75c0609639035e11.zip
meson-9a57608cac4a422ffa450bcf75c0609639035e11.tar.gz
meson-9a57608cac4a422ffa450bcf75c0609639035e11.tar.bz2
Merge pull request #10357 from xclaesse/update-wrapdb
Make `meson wrap update` command update all wraps in parallel
-rw-r--r--docs/markdown/Commands.md3
-rw-r--r--docs/markdown/snippets/update_wrapdb.md8
-rwxr-xr-xmesonbuild/msubprojects.py71
-rw-r--r--mesonbuild/wrap/wrap.py36
-rw-r--r--mesonbuild/wrap/wraptool.py45
5 files changed, 109 insertions, 54 deletions
diff --git a/docs/markdown/Commands.md b/docs/markdown/Commands.md
index 11ca6e0..c328461 100644
--- a/docs/markdown/Commands.md
+++ b/docs/markdown/Commands.md
@@ -266,6 +266,9 @@ Manages subprojects of the Meson project. *Since 0.59.0* commands are run on
multiple subprojects in parallel by default, use `--num-processes=1` if it is
not desired.
+Since *0.64.0* the `update` subcommand will not download new wrap files
+from WrapDB any more. Use `meson wrap update` command for that instead.
+
{{ subprojects_arguments.inc }}
### test
diff --git a/docs/markdown/snippets/update_wrapdb.md b/docs/markdown/snippets/update_wrapdb.md
new file mode 100644
index 0000000..a4e4f27
--- /dev/null
+++ b/docs/markdown/snippets/update_wrapdb.md
@@ -0,0 +1,8 @@
+## Update all wraps from WrapDB with `meson wrap update` command
+
+The command `meson wrap update`, with no extra argument, will now update all wraps
+that comes from WrapDB to the latest version. The extra `--force` argument will
+also replace wraps that do not come from WrapDB if one is available.
+
+The command `meson subprojects update` will not download new wrap files from
+WrapDB any more.
diff --git a/mesonbuild/msubprojects.py b/mesonbuild/msubprojects.py
index ab527bb..247690a 100755
--- a/mesonbuild/msubprojects.py
+++ b/mesonbuild/msubprojects.py
@@ -15,13 +15,13 @@ import zipfile
from . import mlog
from .mesonlib import quiet_git, GitException, Popen_safe, MesonException, windows_proof_rmtree
-from .wrap.wrap import Resolver, WrapException, ALL_TYPES
-from .wrap import wraptool
+from .wrap.wrap import (Resolver, WrapException, ALL_TYPES, PackageDefinition,
+ parse_patch_url, update_wrap_file, get_releases)
if T.TYPE_CHECKING:
from typing_extensions import Protocol
- from .wrap.wrap import PackageDefinition
+ SubParsers = argparse._SubParsersAction[argparse.ArgumentParser]
class Arguments(Protocol):
sourcedir: str
@@ -35,6 +35,10 @@ if T.TYPE_CHECKING:
rebase: bool
reset: bool
+ class UpdateWrapDBArguments(Arguments):
+ force: bool
+ releases: T.Dict[str, T.Any]
+
class CheckoutArguments(Arguments):
b: bool
branch_name: str
@@ -130,21 +134,53 @@ class Runner:
self.logger.done(self.wrap.name, self.log_queue)
return result
- def update_wrapdb_file(self) -> None:
+ @staticmethod
+ def pre_update_wrapdb(options: 'UpdateWrapDBArguments') -> None:
+ options.releases = get_releases(options.allow_insecure)
+
+ def update_wrapdb(self) -> bool:
+ self.log(f'Checking latest WrapDB version for {self.wrap.name}...')
+ options = T.cast('UpdateWrapDBArguments', self.options)
+
+ # Check if this wrap is in WrapDB
+ info = options.releases.get(self.wrap.name)
+ if not info:
+ self.log(' -> Wrap not found in wrapdb')
+ return True
+
+ # Determine current version
try:
- patch_url = self.wrap.get('patch_url')
- branch, revision = wraptool.parse_patch_url(patch_url)
+ wrapdb_version = self.wrap.get('wrapdb_version')
+ branch, revision = wrapdb_version.split('-', 1)
except WrapException:
- return
- new_branch, new_revision = wraptool.get_latest_version(self.wrap.name, self.options.allow_insecure)
+ # Fallback to parsing the patch URL to determine current version.
+ # This won't work for projects that have upstream Meson support.
+ try:
+ patch_url = self.wrap.get('patch_url')
+ branch, revision = parse_patch_url(patch_url)
+ except WrapException:
+ if not options.force:
+ self.log(' ->', mlog.red('Could not determine current version, use --force to update any way'))
+ return False
+ branch = revision = None
+
+ # Download latest wrap if version differs
+ latest_version = info['versions'][0]
+ new_branch, new_revision = latest_version.rsplit('-', 1)
if new_branch != branch or new_revision != revision:
- wraptool.update_wrap_file(self.wrap.filename, self.wrap.name, new_branch, new_revision, self.options.allow_insecure)
- self.log(' -> New wrap file downloaded.')
+ filename = self.wrap.filename if self.wrap.has_wrap else f'{self.wrap.filename}.wrap'
+ update_wrap_file(filename, self.wrap.name,
+ new_branch, new_revision,
+ options.allow_insecure)
+ self.log(' -> New version downloaded:', mlog.blue(latest_version))
+ else:
+ self.log(' -> Already at latest version:', mlog.blue(latest_version))
+
+ return True
def update_file(self) -> bool:
options = T.cast('UpdateArguments', self.options)
- self.update_wrapdb_file()
if not os.path.isdir(self.repo_dir):
# The subproject is not needed, or it is a tarball extracted in
# 'libfoo-1.0' directory and the version has been bumped and the new
@@ -591,6 +627,16 @@ def add_subprojects_argument(p: argparse.ArgumentParser) -> None:
p.add_argument('subprojects', nargs='*',
help='List of subprojects (default: all)')
+def add_wrap_update_parser(subparsers: 'SubParsers') -> argparse.ArgumentParser:
+ p = subparsers.add_parser('update', help='Update wrap files from WrapDB (Since 0.63.0)')
+ p.add_argument('--force', default=False, action='store_true',
+ help='Update wraps that does not seems to come from WrapDB')
+ add_common_arguments(p)
+ add_subprojects_argument(p)
+ p.set_defaults(subprojects_func=Runner.update_wrapdb)
+ p.set_defaults(pre_func=Runner.pre_update_wrapdb)
+ return p
+
def add_arguments(parser: argparse.ArgumentParser) -> None:
subparsers = parser.add_subparsers(title='Commands', dest='command')
subparsers.required = True
@@ -669,6 +715,9 @@ def run(options: 'Arguments') -> int:
executor = ThreadPoolExecutor(options.num_processes)
if types:
wraps = [wrap for wrap in wraps if wrap.type in types]
+ pre_func = getattr(options, 'pre_func', None)
+ if pre_func:
+ pre_func(options)
logger = Logger(len(wraps))
for wrap in wraps:
dirname = Path(subprojects_dir, wrap.directory).as_posix()
diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py
index 1cc55ee..b1c2205 100644
--- a/mesonbuild/wrap/wrap.py
+++ b/mesonbuild/wrap/wrap.py
@@ -107,6 +107,36 @@ def open_wrapdburl(urlstring: str, allow_insecure: bool = False, have_opt: bool
except urllib.error.URLError as excp:
raise WrapException(f'WrapDB connection failed to {urlstring} with error {excp}')
+def get_releases_data(allow_insecure: bool) -> bytes:
+ url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True)
+ return url.read()
+
+def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]:
+ data = get_releases_data(allow_insecure)
+ return T.cast('T.Dict[str, T.Any]', json.loads(data.decode()))
+
+def update_wrap_file(wrapfile: str, name: str, new_version: str, new_revision: str, allow_insecure: bool) -> None:
+ url = open_wrapdburl(f'https://wrapdb.mesonbuild.com/v2/{name}_{new_version}-{new_revision}/{name}.wrap',
+ allow_insecure, True)
+ with open(wrapfile, 'wb') as f:
+ f.write(url.read())
+
+def parse_patch_url(patch_url: str) -> T.Tuple[str, str]:
+ u = urllib.parse.urlparse(patch_url)
+ if u.netloc != 'wrapdb.mesonbuild.com':
+ raise WrapException(f'URL {patch_url} does not seems to be a wrapdb patch')
+ arr = u.path.strip('/').split('/')
+ if arr[0] == 'v1':
+ # e.g. https://wrapdb.mesonbuild.com/v1/projects/zlib/1.2.11/5/get_zip
+ return arr[-3], arr[-2]
+ elif arr[0] == 'v2':
+ # e.g. https://wrapdb.mesonbuild.com/v2/zlib_1.2.11-5/get_patch
+ tag = arr[-2]
+ _, version = tag.rsplit('_', 1)
+ version, revision = version.rsplit('-', 1)
+ return version, revision
+ else:
+ raise WrapException(f'Invalid wrapdb URL {patch_url}')
class WrapException(MesonException):
pass
@@ -283,17 +313,17 @@ class Resolver:
if not os.path.isdir(self.subdir_root):
return
root, dirs, files = next(os.walk(self.subdir_root))
+ ignore_dirs = {'packagecache', 'packagefiles'}
for i in files:
if not i.endswith('.wrap'):
continue
fname = os.path.join(self.subdir_root, i)
wrap = PackageDefinition(fname, self.subproject)
self.wraps[wrap.name] = wrap
- if wrap.directory in dirs:
- dirs.remove(wrap.directory)
+ ignore_dirs |= {wrap.directory, wrap.name}
# Add dummy package definition for directories not associated with a wrap file.
for i in dirs:
- if i in ['packagecache', 'packagefiles']:
+ if i in ignore_dirs:
continue
fname = os.path.join(self.subdir_root, i)
wrap = PackageDefinition(fname, self.subproject)
diff --git a/mesonbuild/wrap/wraptool.py b/mesonbuild/wrap/wraptool.py
index 80a58ab..bcf0e67 100644
--- a/mesonbuild/wrap/wraptool.py
+++ b/mesonbuild/wrap/wraptool.py
@@ -12,18 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import json
import sys, os
import configparser
import shutil
import typing as T
from glob import glob
-from urllib.parse import urlparse
-from .wrap import open_wrapdburl, WrapException
+from .wrap import (open_wrapdburl, WrapException, get_releases, get_releases_data,
+ update_wrap_file, parse_patch_url)
from pathlib import Path
-from .. import mesonlib
+from .. import mesonlib, msubprojects
if T.TYPE_CHECKING:
import argparse
@@ -49,11 +48,8 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None:
p.add_argument('name')
p.set_defaults(wrap_func=install)
- p = subparsers.add_parser('update', help='update the project to its newest available release')
- p.add_argument('--allow-insecure', default=False, action='store_true',
- help='Allow insecure server connections.')
- p.add_argument('name')
- p.set_defaults(wrap_func=update)
+ p = msubprojects.add_wrap_update_parser(subparsers)
+ p.set_defaults(wrap_func=msubprojects.run)
p = subparsers.add_parser('info', help='show available versions of a project')
p.add_argument('--allow-insecure', default=False, action='store_true',
@@ -75,14 +71,6 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None:
help='Allow insecure server connections.')
p.set_defaults(wrap_func=update_db)
-def get_releases_data(allow_insecure: bool) -> bytes:
- url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True)
- return url.read()
-
-def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]:
- data = get_releases_data(allow_insecure)
- return T.cast('T.Dict[str, T.Any]', json.loads(data.decode()))
-
def list_projects(options: 'argparse.Namespace') -> None:
releases = get_releases(options.allow_insecure)
for p in releases.keys():
@@ -123,23 +111,6 @@ def install(options: 'argparse.Namespace') -> None:
f.write(url.read())
print(f'Installed {name} version {version} revision {revision}')
-def parse_patch_url(patch_url: str) -> T.Tuple[str, str]:
- u = urlparse(patch_url)
- if u.netloc != 'wrapdb.mesonbuild.com':
- raise WrapException(f'URL {patch_url} does not seems to be a wrapdb patch')
- arr = u.path.strip('/').split('/')
- if arr[0] == 'v1':
- # e.g. https://wrapdb.mesonbuild.com/v1/projects/zlib/1.2.11/5/get_zip
- return arr[-3], arr[-2]
- elif arr[0] == 'v2':
- # e.g. https://wrapdb.mesonbuild.com/v2/zlib_1.2.11-5/get_patch
- tag = arr[-2]
- _, version = tag.rsplit('_', 1)
- version, revision = version.rsplit('-', 1)
- return version, revision
- else:
- raise WrapException(f'Invalid wrapdb URL {patch_url}')
-
def get_current_version(wrapfile: str) -> T.Tuple[str, str, str, str, T.Optional[str]]:
cp = configparser.ConfigParser(interpolation=None)
cp.read(wrapfile)
@@ -160,12 +131,6 @@ def get_current_version(wrapfile: str) -> T.Tuple[str, str, str, str, T.Optional
patch_filename = wrap_data['patch_filename']
return branch, revision, wrap_data['directory'], wrap_data['source_filename'], patch_filename
-def update_wrap_file(wrapfile: str, name: str, new_version: str, new_revision: str, allow_insecure: bool) -> None:
- url = open_wrapdburl(f'https://wrapdb.mesonbuild.com/v2/{name}_{new_version}-{new_revision}/{name}.wrap',
- allow_insecure, True)
- with open(wrapfile, 'wb') as f:
- f.write(url.read())
-
def update(options: 'argparse.Namespace') -> None:
name = options.name
if not os.path.isdir('subprojects'):