From 9b8378985dbdc0112d11893dd42b33b7bc8d1e62 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 7 Mar 2024 12:23:13 -0500 Subject: cargo: Load Cargo.lock Cargo.lock is essentially identical to subprojects/*.wrap files. When a (sub)project has a Cargo.lock file this allows automatic fallback for its cargo dependencies. --- docs/markdown/Wrap-dependency-system-manual.md | 4 ++ docs/markdown/snippets/cargo_lock.md | 6 +++ mesonbuild/cargo/__init__.py | 5 ++- mesonbuild/cargo/interpreter.py | 46 ++++++++++++++++++++- mesonbuild/cargo/manifest.py | 17 ++++++++ mesonbuild/wrap/wrap.py | 45 +++++++++++--------- run_unittests.py | 2 +- test cases/rust/25 cargo lock/Cargo.lock | 7 ++++ test cases/rust/25 cargo lock/meson.build | 3 ++ .../subprojects/packagecache/bar-0.1.tar.gz | Bin 0 -> 288 bytes unittests/cargotests.py | 36 +++++++++++++++- 11 files changed, 147 insertions(+), 24 deletions(-) create mode 100644 docs/markdown/snippets/cargo_lock.md create mode 100644 test cases/rust/25 cargo lock/Cargo.lock create mode 100644 test cases/rust/25 cargo lock/meson.build create mode 100644 test cases/rust/25 cargo lock/subprojects/packagecache/bar-0.1.tar.gz diff --git a/docs/markdown/Wrap-dependency-system-manual.md b/docs/markdown/Wrap-dependency-system-manual.md index 5f0b473..3983d28 100644 --- a/docs/markdown/Wrap-dependency-system-manual.md +++ b/docs/markdown/Wrap-dependency-system-manual.md @@ -377,6 +377,10 @@ Some naming conventions need to be respected: - The `extra_deps` variable is pre-defined and can be used to add extra dependencies. This is typically used as `extra_deps += dependency('foo')`. +Since *1.5.0* Cargo wraps can also be provided with `Cargo.lock` file at the root +of (sub)project source tree. Meson will automatically load that file and convert +it into a serie of wraps definitions. + ## Using wrapped projects Wraps provide a convenient way of obtaining a project into your diff --git a/docs/markdown/snippets/cargo_lock.md b/docs/markdown/snippets/cargo_lock.md new file mode 100644 index 0000000..e38c5ed --- /dev/null +++ b/docs/markdown/snippets/cargo_lock.md @@ -0,0 +1,6 @@ +## Added support `Cargo.lock` file + +When a (sub)project has a `Cargo.lock` file at its root, it is loaded to provide +an automatic fallback for dependencies it defines, fetching code from +https://crates.io or git. This is identical as providing `subprojects/*.wrap`, +see [cargo wraps](Wrap-dependency-system-manual.md#cargo-wraps) dependency naming convention. diff --git a/mesonbuild/cargo/__init__.py b/mesonbuild/cargo/__init__.py index 0007b9d..10cb0be 100644 --- a/mesonbuild/cargo/__init__.py +++ b/mesonbuild/cargo/__init__.py @@ -1,5 +1,6 @@ __all__ = [ - 'interpret' + 'interpret', + 'load_wraps', ] -from .interpreter import interpret +from .interpreter import interpret, load_wraps diff --git a/mesonbuild/cargo/interpreter.py b/mesonbuild/cargo/interpreter.py index 1d06474..13568cd 100644 --- a/mesonbuild/cargo/interpreter.py +++ b/mesonbuild/cargo/interpreter.py @@ -18,12 +18,14 @@ import json import os import shutil import collections +import urllib.parse import typing as T from . import builder from . import version from ..mesonlib import MesonException, Popen_safe, OptionKey -from .. import coredata, options +from .. import coredata, options, mlog +from ..wrap.wrap import PackageDefinition if T.TYPE_CHECKING: from types import ModuleType @@ -731,3 +733,45 @@ def interpret(subp_name: str, subdir: str, env: Environment) -> T.Tuple[mparser. ast.extend(_create_lib(cargo, build, crate_type)) return build.block(ast), project_options + + +def load_wraps(source_dir: str, subproject_dir: str) -> T.List[PackageDefinition]: + """ Convert Cargo.lock into a list of wraps """ + + wraps: T.List[PackageDefinition] = [] + filename = os.path.join(source_dir, 'Cargo.lock') + if os.path.exists(filename): + cargolock = T.cast('manifest.CargoLock', load_toml(filename)) + for package in cargolock['package']: + name = package['name'] + version = package['version'] + subp_name = _dependency_name(name, _version_to_api(version)) + source = package.get('source') + if source is None: + # This is project's package, or one of its workspace members. + pass + elif source == 'registry+https://github.com/rust-lang/crates.io-index': + url = f'https://crates.io/api/v1/crates/{name}/{version}/download' + directory = f'{name}-{version}' + wraps.append(PackageDefinition.from_values(subp_name, subproject_dir, 'file', { + 'directory': directory, + 'source_url': url, + 'source_filename': f'{directory}.tar.gz', + 'source_hash': package['checksum'], + 'method': 'cargo', + })) + elif source.startswith('git+'): + parts = urllib.parse.urlparse(source[4:]) + query = urllib.parse.parse_qs(parts.query) + branch = query['branch'][0] if 'branch' in query else '' + revision = parts.fragment or branch + url = urllib.parse.urlunparse(parts._replace(params='', query='', fragment='')) + wraps.append(PackageDefinition.from_values(subp_name, subproject_dir, 'git', { + 'directory': name, + 'url': url, + 'revision': revision, + 'method': 'cargo', + })) + else: + mlog.warning(f'Unsupported source URL in {filename}: {source}') + return wraps diff --git a/mesonbuild/cargo/manifest.py b/mesonbuild/cargo/manifest.py index e6192d0..183d91e 100644 --- a/mesonbuild/cargo/manifest.py +++ b/mesonbuild/cargo/manifest.py @@ -225,3 +225,20 @@ class VirtualManifest(TypedDict): """ workspace: Workspace + +class CargoLockPackage(TypedDict, total=False): + + """A description of a package in the Cargo.lock file format.""" + + name: str + version: str + source: str + checksum: str + + +class CargoLock(TypedDict, total=False): + + """A description of the Cargo.lock file format.""" + + version: str + package: T.List[CargoLockPackage] diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index 96b0ef3..4e98c60 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -324,25 +324,32 @@ class Resolver: mlog.warning(f'failed to process netrc file: {e}.', fatal=False) def load_wraps(self) -> None: - if not os.path.isdir(self.subdir_root): - return - root, dirs, files = next(os.walk(self.subdir_root)) - ignore_dirs = {'packagecache', 'packagefiles'} - for i in files: - if not i.endswith('.wrap'): - continue - fname = os.path.join(self.subdir_root, i) - wrap = PackageDefinition.from_wrap_file(fname, self.subproject) - self.wraps[wrap.name] = wrap - ignore_dirs |= {wrap.directory, wrap.name} - # Add dummy package definition for directories not associated with a wrap file. - for i in dirs: - if i in ignore_dirs: - continue - fname = os.path.join(self.subdir_root, i) - wrap = PackageDefinition.from_directory(fname) - self.wraps[wrap.name] = wrap - + # Load Cargo.lock at the root of source tree + source_dir = os.path.dirname(self.subdir_root) + if os.path.exists(os.path.join(source_dir, 'Cargo.lock')): + from .. import cargo + for wrap in cargo.load_wraps(source_dir, self.subdir_root): + self.wraps[wrap.name] = wrap + # Load subprojects/*.wrap + if os.path.isdir(self.subdir_root): + root, dirs, files = next(os.walk(self.subdir_root)) + for i in files: + if not i.endswith('.wrap'): + continue + fname = os.path.join(self.subdir_root, i) + wrap = PackageDefinition.from_wrap_file(fname, self.subproject) + self.wraps[wrap.name] = wrap + # Add dummy package definition for directories not associated with a wrap file. + ignore_dirs = {'packagecache', 'packagefiles'} + for wrap in self.wraps.values(): + ignore_dirs |= {wrap.directory, wrap.name} + for i in dirs: + if i in ignore_dirs: + continue + fname = os.path.join(self.subdir_root, i) + wrap = PackageDefinition.from_directory(fname) + self.wraps[wrap.name] = wrap + # Add provided deps and programs into our lookup tables for wrap in self.wraps.values(): self.add_wrap(wrap) diff --git a/run_unittests.py b/run_unittests.py index 33b0e09..84edb34 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -25,7 +25,7 @@ from mesonbuild.mesonlib import python_command, setup_vsenv import mesonbuild.modules.pkgconfig from unittests.allplatformstests import AllPlatformTests -from unittests.cargotests import CargoVersionTest, CargoCfgTest +from unittests.cargotests import CargoVersionTest, CargoCfgTest, CargoLockTest from unittests.darwintests import DarwinTests from unittests.failuretests import FailureTests from unittests.linuxcrosstests import LinuxCrossArmTests, LinuxCrossMingwTests diff --git a/test cases/rust/25 cargo lock/Cargo.lock b/test cases/rust/25 cargo lock/Cargo.lock new file mode 100644 index 0000000..9bc9814 --- /dev/null +++ b/test cases/rust/25 cargo lock/Cargo.lock @@ -0,0 +1,7 @@ +version = 3 + +[[package]] +name = "bar" +version = "0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2f34e570dcd5f9fe32e6863ee16ee73a356d3b77bce0d8c78501b8bc81a860" diff --git a/test cases/rust/25 cargo lock/meson.build b/test cases/rust/25 cargo lock/meson.build new file mode 100644 index 0000000..b359f7b --- /dev/null +++ b/test cases/rust/25 cargo lock/meson.build @@ -0,0 +1,3 @@ +project('cargo lock') + +dependency('bar-0.1-rs') diff --git a/test cases/rust/25 cargo lock/subprojects/packagecache/bar-0.1.tar.gz b/test cases/rust/25 cargo lock/subprojects/packagecache/bar-0.1.tar.gz new file mode 100644 index 0000000..f4c2ec6 Binary files /dev/null and b/test cases/rust/25 cargo lock/subprojects/packagecache/bar-0.1.tar.gz differ diff --git a/unittests/cargotests.py b/unittests/cargotests.py index f0aedd0..d1ac838 100644 --- a/unittests/cargotests.py +++ b/unittests/cargotests.py @@ -3,9 +3,12 @@ from __future__ import annotations import unittest +import os +import tempfile +import textwrap import typing as T -from mesonbuild.cargo import builder, cfg +from mesonbuild.cargo import builder, cfg, load_wraps from mesonbuild.cargo.cfg import TokenType from mesonbuild.cargo.version import convert @@ -185,3 +188,34 @@ class CargoCfgTest(unittest.TestCase): with self.subTest(): value = cfg.ir_to_meson(cfg.parse(iter(cfg.lexer(data))), build) self.assertEqual(value, expected) + +class CargoLockTest(unittest.TestCase): + def test_cargo_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + with open(os.path.join(tmpdir, 'Cargo.lock'), 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + version = 3 + [[package]] + name = "foo" + version = "0.1" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" + [[package]] + name = "bar" + version = "0.1" + source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=0.19#23c5599424cc75ec66618891c915d9f490f6e4c2" + ''')) + wraps = load_wraps(tmpdir, 'subprojects') + self.assertEqual(len(wraps), 2) + self.assertEqual(wraps[0].name, 'foo-0.1-rs') + self.assertEqual(wraps[0].directory, 'foo-0.1') + self.assertEqual(wraps[0].type, 'file') + self.assertEqual(wraps[0].get('method'), 'cargo') + self.assertEqual(wraps[0].get('source_url'), 'https://crates.io/api/v1/crates/foo/0.1/download') + self.assertEqual(wraps[0].get('source_hash'), '8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb') + self.assertEqual(wraps[1].name, 'bar-0.1-rs') + self.assertEqual(wraps[1].directory, 'bar') + self.assertEqual(wraps[1].type, 'git') + self.assertEqual(wraps[1].get('method'), 'cargo') + self.assertEqual(wraps[1].get('url'), 'https://github.com/gtk-rs/gtk-rs-core') + self.assertEqual(wraps[1].get('revision'), '23c5599424cc75ec66618891c915d9f490f6e4c2') -- cgit v1.1