diff options
-rw-r--r-- | docs/markdown/Using-wraptool.md | 14 | ||||
-rw-r--r-- | docs/markdown/snippets/wrapdb.md | 6 | ||||
-rw-r--r-- | docs/yaml/functions/dependency.yaml | 5 | ||||
-rw-r--r-- | mesonbuild/interpreter/dependencyfallbacks.py | 16 | ||||
-rw-r--r-- | mesonbuild/wrap/wrap.py | 68 | ||||
-rw-r--r-- | mesonbuild/wrap/wraptool.py | 20 | ||||
-rw-r--r-- | unittests/platformagnostictests.py | 19 |
7 files changed, 125 insertions, 23 deletions
diff --git a/docs/markdown/Using-wraptool.md b/docs/markdown/Using-wraptool.md index cabdc0e..edbceaa 100644 --- a/docs/markdown/Using-wraptool.md +++ b/docs/markdown/Using-wraptool.md @@ -82,3 +82,17 @@ straightforward: Wraptool can do other things besides these. Documentation for these can be found in the command line help, which can be accessed by `meson wrap --help`. + +## Automatic dependency fallback + +Since *0.64.0* Meson can use WrapDB to automatically find missing dependencies. + +The user simply needs to download latest database, the following command stores +it in `subprojects/wrapdb.json`: + $ meson wrap update-db + +Once the database is available locally, any dependency not found on the system +but available in WrapDB will automatically be downloaded. + +Automatic fetch of WrapDB subprojects can be disabled by removing the file +`subprojects/wrapdb.json`, or by using `--wrap-mode=nodownload`. diff --git a/docs/markdown/snippets/wrapdb.md b/docs/markdown/snippets/wrapdb.md new file mode 100644 index 0000000..d5caf4f --- /dev/null +++ b/docs/markdown/snippets/wrapdb.md @@ -0,0 +1,6 @@ +## Automatic fallback using WrapDB + +A new command has been added: `meson wrap update-db`. It downloads the list of +wraps available in [WrapDB](wrapdb.mesonbuild.com) and stores it locally in +`subprojects/wrapdb.json`. When that file exists and a dependency is not found +on the system but is available in WrapDB, Meson will automatically download it. diff --git a/docs/yaml/functions/dependency.yaml b/docs/yaml/functions/dependency.yaml index dcb696d..3a4d2e8 100644 --- a/docs/yaml/functions/dependency.yaml +++ b/docs/yaml/functions/dependency.yaml @@ -15,6 +15,11 @@ description: | of those name will return the same value. This is useful in case a dependency could have different names, such as `png` and `libpng`. + * Since *0.64.0* a dependency fallback can be provided by WrapDB. Simply download + the database locally using `meson wrap update-db` command and Meson will + automatically fallback to subprojects provided by WrapDB if the dependency is + not found on the system and the project does not ship their own `.wrap` file. + Dependencies can also be resolved in two other ways: * if the same name was used in a `meson.override_dependency` prior to diff --git a/mesonbuild/interpreter/dependencyfallbacks.py b/mesonbuild/interpreter/dependencyfallbacks.py index 3fbce42..54be990 100644 --- a/mesonbuild/interpreter/dependencyfallbacks.py +++ b/mesonbuild/interpreter/dependencyfallbacks.py @@ -66,14 +66,6 @@ class DependencyFallbacksHolder(MesonInterpreterObject): self._subproject_impl(subp_name, varname) def _subproject_impl(self, subp_name: str, varname: str) -> None: - if not varname: - # If no variable name is specified, check if the wrap file has one. - # If the wrap file has a variable name, better use it because the - # subproject most probably is not using meson.override_dependency(). - for name in self.names: - varname = self.wrap_resolver.get_varname(subp_name, name) - if varname: - break assert self.subproject_name is None self.subproject_name = subp_name self.subproject_varname = varname @@ -175,6 +167,14 @@ class DependencyFallbacksHolder(MesonInterpreterObject): # Legacy: Use the variable name if provided instead of relying on the # subproject to override one of our dependency names if not varname: + # If no variable name is specified, check if the wrap file has one. + # If the wrap file has a variable name, better use it because the + # subproject most probably is not using meson.override_dependency(). + for name in self.names: + varname = self.wrap_resolver.get_varname(subp_name, name) + if varname: + break + if not varname: mlog.warning(f'Subproject {subp_name!r} did not override {self._display_name!r} dependency and no variable name specified') mlog.log('Dependency', mlog.bold(self._display_name), 'from subproject', mlog.bold(subproject.subdir), 'found:', mlog.red('NO')) diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index 10aa1b8..1cc55ee 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -29,10 +29,12 @@ import configparser import time import typing as T import textwrap +import json from base64 import b64encode from netrc import netrc from pathlib import Path + from . import WrapMode from .. import coredata from ..mesonlib import quiet_git, GIT, ProgressBar, MesonException, windows_proof_rmtree, Popen_safe @@ -262,8 +264,12 @@ class Resolver: self.netrc: T.Optional[netrc] = None self.provided_deps = {} # type: T.Dict[str, PackageDefinition] self.provided_programs = {} # type: T.Dict[str, PackageDefinition] + self.wrapdb: T.Dict[str, T.Any] = {} + self.wrapdb_provided_deps: T.Dict[str, str] = {} + self.wrapdb_provided_programs: T.Dict[str, str] = {} self.load_wraps() self.load_netrc() + self.load_wrapdb() def load_netrc(self) -> None: try: @@ -294,18 +300,48 @@ class Resolver: self.wraps[wrap.name] = wrap for wrap in self.wraps.values(): - for k in wrap.provided_deps.keys(): - if k in self.provided_deps: - prev_wrap = self.provided_deps[k] - m = f'Multiple wrap files provide {k!r} dependency: {wrap.basename} and {prev_wrap.basename}' - raise WrapException(m) - self.provided_deps[k] = wrap - for k in wrap.provided_programs: - if k in self.provided_programs: - prev_wrap = self.provided_programs[k] - m = f'Multiple wrap files provide {k!r} program: {wrap.basename} and {prev_wrap.basename}' - raise WrapException(m) - self.provided_programs[k] = wrap + self.add_wrap(wrap) + + def add_wrap(self, wrap: PackageDefinition) -> None: + for k in wrap.provided_deps.keys(): + if k in self.provided_deps: + prev_wrap = self.provided_deps[k] + m = f'Multiple wrap files provide {k!r} dependency: {wrap.basename} and {prev_wrap.basename}' + raise WrapException(m) + self.provided_deps[k] = wrap + for k in wrap.provided_programs: + if k in self.provided_programs: + prev_wrap = self.provided_programs[k] + m = f'Multiple wrap files provide {k!r} program: {wrap.basename} and {prev_wrap.basename}' + raise WrapException(m) + self.provided_programs[k] = wrap + + def load_wrapdb(self) -> None: + try: + with Path(self.subdir_root, 'wrapdb.json').open('r', encoding='utf-8') as f: + self.wrapdb = json.load(f) + except FileNotFoundError: + return + for name, info in self.wrapdb.items(): + self.wrapdb_provided_deps.update({i: name for i in info.get('dependency_names', [])}) + self.wrapdb_provided_programs.update({i: name for i in info.get('program_names', [])}) + + def get_from_wrapdb(self, subp_name: str) -> PackageDefinition: + info = self.wrapdb.get(subp_name) + if not info: + return None + self.check_can_download() + latest_version = info['versions'][0] + version, revision = latest_version.rsplit('-', 1) + url = urllib.request.urlopen(f'https://wrapdb.mesonbuild.com/v2/{subp_name}_{version}-{revision}/{subp_name}.wrap') + fname = Path(self.subdir_root, f'{subp_name}.wrap') + with fname.open('wb') as f: + f.write(url.read()) + mlog.log(f'Installed {subp_name} version {version} revision {revision}') + wrap = PackageDefinition(str(fname)) + self.wraps[wrap.name] = wrap + self.add_wrap(wrap) + return wrap def merge_wraps(self, other_resolver: 'Resolver') -> None: for k, v in other_resolver.wraps.items(): @@ -323,7 +359,8 @@ class Resolver: if wrap: dep_var = wrap.provided_deps.get(packagename) return wrap.name, dep_var - return None, None + wrap_name = self.wrapdb_provided_deps.get(packagename) + return wrap_name, None def get_varname(self, subp_name: str, depname: str) -> T.Optional[str]: wrap = self.wraps.get(subp_name) @@ -334,6 +371,9 @@ class Resolver: wrap = self.provided_programs.get(name) if wrap: return wrap.name + wrap_name = self.wrapdb_provided_programs.get(name) + if wrap_name: + return wrap_name return None def resolve(self, packagename: str, method: str) -> str: @@ -341,6 +381,8 @@ class Resolver: self.directory = packagename self.wrap = self.wraps.get(packagename) if not self.wrap: + self.wrap = self.get_from_wrapdb(packagename) + if not self.wrap: m = f'Neither a subproject directory nor a {self.packagename}.wrap file was found.' raise WrapNotFoundException(m) self.directory = self.wrap.directory diff --git a/mesonbuild/wrap/wraptool.py b/mesonbuild/wrap/wraptool.py index ec2ac3e..80a58ab 100644 --- a/mesonbuild/wrap/wraptool.py +++ b/mesonbuild/wrap/wraptool.py @@ -21,6 +21,7 @@ import typing as T from glob import glob from urllib.parse import urlparse from .wrap import open_wrapdburl, WrapException +from pathlib import Path from .. import mesonlib @@ -69,9 +70,18 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None: p.add_argument('project_path') p.set_defaults(wrap_func=promote) -def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]: + p = subparsers.add_parser('update-db', help='Update list of projects available in WrapDB (Since 0.61.0)') + p.add_argument('--allow-insecure', default=False, action='store_true', + help='Allow insecure server connections.') + p.set_defaults(wrap_func=update_db) + +def get_releases_data(allow_insecure: bool) -> bytes: url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True) - return T.cast('T.Dict[str, T.Any]', json.loads(url.read().decode())) + return url.read() + +def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]: + data = get_releases_data(allow_insecure) + return T.cast('T.Dict[str, T.Any]', json.loads(data.decode())) def list_projects(options: 'argparse.Namespace') -> None: releases = get_releases(options.allow_insecure) @@ -244,6 +254,12 @@ def status(options: 'argparse.Namespace') -> None: else: print('', name, f'not up to date. Have {current_branch} {current_revision}, but {latest_branch} {latest_revision} is available.') +def update_db(options: 'argparse.Namespace') -> None: + data = get_releases_data(options.allow_insecure) + Path('subprojects').mkdir(exist_ok=True) + with Path('subprojects/wrapdb.json').open('wb') as f: + f.write(data) + def run(options: 'argparse.Namespace') -> int: options.wrap_func(options) return 0 diff --git a/unittests/platformagnostictests.py b/unittests/platformagnostictests.py index a2d6640..ed5d96b 100644 --- a/unittests/platformagnostictests.py +++ b/unittests/platformagnostictests.py @@ -14,7 +14,10 @@ import os import tempfile +import subprocess +import textwrap from unittest import skipIf +from pathlib import Path from .baseplatformtests import BasePlatformTests from .helpers import is_ci @@ -94,3 +97,19 @@ class PlatformAgnosticTests(BasePlatformTests): # https://github.com/mesonbuild/meson/issues/10225. self.setconf('-Dfoo=enabled') self.build('reconfigure') + + def test_update_wrapdb(self): + # Write the project into a temporary directory because it will add files + # into subprojects/ and we don't want to pollute meson source tree. + with tempfile.TemporaryDirectory() as testdir: + with Path(testdir, 'meson.build').open('w', encoding='utf-8') as f: + f.write(textwrap.dedent( + ''' + project('wrap update-db', + default_options: ['wrap_mode=forcefallback']) + + zlib_dep = dependency('zlib') + assert(zlib_dep.type_name() == 'internal') + ''')) + subprocess.check_call(self.wrap_command + ['update-db'], cwd=testdir) + self.init(testdir, workdir=testdir) |