diff options
Diffstat (limited to 'mesonbuild/cargo')
| -rw-r--r-- | mesonbuild/cargo/cfg.py | 167 | ||||
| -rw-r--r-- | mesonbuild/cargo/interpreter.py | 530 | ||||
| -rw-r--r-- | mesonbuild/cargo/manifest.py | 645 | ||||
| -rw-r--r-- | mesonbuild/cargo/raw.py | 192 | ||||
| -rw-r--r-- | mesonbuild/cargo/toml.py | 49 | ||||
| -rw-r--r-- | mesonbuild/cargo/version.py | 14 |
6 files changed, 848 insertions, 749 deletions
diff --git a/mesonbuild/cargo/cfg.py b/mesonbuild/cargo/cfg.py index 0d49527..a0ee6e2 100644 --- a/mesonbuild/cargo/cfg.py +++ b/mesonbuild/cargo/cfg.py @@ -4,6 +4,7 @@ """Rust CFG parser. Rust uses its `cfg()` format in cargo. +https://doc.rust-lang.org/reference/conditional-compilation.html This may have the following functions: - all() @@ -22,18 +23,15 @@ so you could have examples like: from __future__ import annotations import dataclasses import enum -import functools import typing as T -from . import builder -from .. import mparser from ..mesonlib import MesonBugException if T.TYPE_CHECKING: _T = T.TypeVar('_T') _LEX_TOKEN = T.Tuple['TokenType', T.Optional[str]] - _LEX_STREAM = T.Iterable[_LEX_TOKEN] + _LEX_STREAM = T.Iterator[_LEX_TOKEN] _LEX_STREAM_AH = T.Iterator[T.Tuple[_LEX_TOKEN, T.Optional[_LEX_TOKEN]]] @@ -48,6 +46,7 @@ class TokenType(enum.Enum): NOT = enum.auto() COMMA = enum.auto() EQUAL = enum.auto() + CFG = enum.auto() def lexer(raw: str) -> _LEX_STREAM: @@ -56,45 +55,41 @@ def lexer(raw: str) -> _LEX_STREAM: :param raw: The raw cfg() expression :return: An iterable of tokens """ - buffer: T.List[str] = [] + start: int = 0 is_string: bool = False - for s in raw: - if s.isspace() or s in {')', '(', ',', '='} or (s == '"' and buffer): - val = ''.join(buffer) - buffer.clear() - if is_string: + for i, s in enumerate(raw): + if s.isspace() or s in {')', '(', ',', '=', '"'}: + val = raw[start:i] + start = i + 1 + if s == '"' and is_string: yield (TokenType.STRING, val) + is_string = False + continue elif val == 'any': yield (TokenType.ANY, None) elif val == 'all': yield (TokenType.ALL, None) elif val == 'not': yield (TokenType.NOT, None) + elif val == 'cfg': + yield (TokenType.CFG, None) elif val: yield (TokenType.IDENTIFIER, val) if s == '(': yield (TokenType.LPAREN, None) - continue elif s == ')': yield (TokenType.RPAREN, None) - continue elif s == ',': yield (TokenType.COMMA, None) - continue elif s == '=': yield (TokenType.EQUAL, None) - continue - elif s.isspace(): - continue - - if s == '"': - is_string = not is_string - else: - buffer.append(s) - if buffer: + elif s == '"': + is_string = True + val = raw[start:] + if val: # This should always be an identifier - yield (TokenType.IDENTIFIER, ''.join(buffer)) + yield (TokenType.IDENTIFIER, val) def lookahead(iter: T.Iterator[_T]) -> T.Iterator[T.Tuple[_T, T.Optional[_T]]]: @@ -146,8 +141,8 @@ class Identifier(IR): @dataclasses.dataclass class Equal(IR): - lhs: IR - rhs: IR + lhs: Identifier + rhs: String @dataclasses.dataclass @@ -175,41 +170,40 @@ def _parse(ast: _LEX_STREAM_AH) -> IR: else: ntoken, _ = (None, None) - stream: T.List[_LEX_TOKEN] if token is TokenType.IDENTIFIER: + assert value + id_ = Identifier(value) if ntoken is TokenType.EQUAL: - return Equal(Identifier(value), _parse(ast)) - if token is TokenType.STRING: - return String(value) - if token is TokenType.EQUAL: - # In this case the previous caller already has handled the equal - return _parse(ast) - if token in {TokenType.ANY, TokenType.ALL}: + next(ast) + (token, value), _ = next(ast) + assert token is TokenType.STRING + assert value is not None + return Equal(id_, String(value)) + return id_ + elif token in {TokenType.ANY, TokenType.ALL}: type_ = All if token is TokenType.ALL else Any - assert ntoken is TokenType.LPAREN - next(ast) # advance the iterator to get rid of the LPAREN - stream = [] args: T.List[IR] = [] - while token is not TokenType.RPAREN: + (token, value), n_stream = next(ast) + assert token is TokenType.LPAREN + if n_stream and n_stream[0] == TokenType.RPAREN: + return type_(args) + while True: + args.append(_parse(ast)) (token, value), _ = next(ast) - if token is TokenType.COMMA: - args.append(_parse(lookahead(iter(stream)))) - stream.clear() - else: - stream.append((token, value)) - if stream: - args.append(_parse(lookahead(iter(stream)))) + if token is TokenType.RPAREN: + break + assert token is TokenType.COMMA return type_(args) - if token is TokenType.NOT: - next(ast) # advance the iterator to get rid of the LPAREN - stream = [] - # Mypy can't figure out that token is overridden inside the while loop - while token is not TokenType.RPAREN: # type: ignore - (token, value), _ = next(ast) - stream.append((token, value)) - return Not(_parse(lookahead(iter(stream)))) - - raise MesonBugException(f'Unhandled Cargo token: {token}') + elif token in {TokenType.NOT, TokenType.CFG}: + is_not = token is TokenType.NOT + (token, value), _ = next(ast) + assert token is TokenType.LPAREN + arg = _parse(ast) + (token, value), _ = next(ast) + assert token is TokenType.RPAREN + return Not(arg) if is_not else arg + else: + raise MesonBugException(f'Unhandled Cargo token:{token} {value}') def parse(ast: _LEX_STREAM) -> IR: @@ -218,57 +212,24 @@ def parse(ast: _LEX_STREAM) -> IR: :param ast: An iterable of Tokens :return: An mparser Node to be used as a conditional """ - ast_i: _LEX_STREAM_AH = lookahead(iter(ast)) + ast_i: _LEX_STREAM_AH = lookahead(ast) return _parse(ast_i) -@functools.singledispatch -def ir_to_meson(ir: T.Any, build: builder.Builder) -> mparser.BaseNode: - raise NotImplementedError - - -@ir_to_meson.register -def _(ir: String, build: builder.Builder) -> mparser.BaseNode: - return build.string(ir.value) - - -@ir_to_meson.register -def _(ir: Identifier, build: builder.Builder) -> mparser.BaseNode: - host_machine = build.identifier('host_machine') - if ir.value == "target_arch": - return build.method('cpu_family', host_machine) - elif ir.value in {"target_os", "target_family"}: - return build.method('system', host_machine) - elif ir.value == "target_endian": - return build.method('endian', host_machine) - raise MesonBugException(f"Unhandled Cargo identifier: {ir.value}") - - -@ir_to_meson.register -def _(ir: Equal, build: builder.Builder) -> mparser.BaseNode: - return build.equal(ir_to_meson(ir.lhs, build), ir_to_meson(ir.rhs, build)) - - -@ir_to_meson.register -def _(ir: Not, build: builder.Builder) -> mparser.BaseNode: - return build.not_(ir_to_meson(ir.value, build)) - - -@ir_to_meson.register -def _(ir: Any, build: builder.Builder) -> mparser.BaseNode: - args = iter(reversed(ir.args)) - last = next(args) - cur = build.or_(ir_to_meson(next(args), build), ir_to_meson(last, build)) - for a in args: - cur = build.or_(ir_to_meson(a, build), cur) - return cur +def _eval_cfg(ir: IR, cfgs: T.Dict[str, str]) -> bool: + if isinstance(ir, Identifier): + return ir.value in cfgs + elif isinstance(ir, Equal): + return cfgs.get(ir.lhs.value) == ir.rhs.value + elif isinstance(ir, Not): + return not _eval_cfg(ir.value, cfgs) + elif isinstance(ir, Any): + return any(_eval_cfg(i, cfgs) for i in ir.args) + elif isinstance(ir, All): + return all(_eval_cfg(i, cfgs) for i in ir.args) + else: + raise MesonBugException(f'Unhandled Cargo cfg IR: {ir}') -@ir_to_meson.register -def _(ir: All, build: builder.Builder) -> mparser.BaseNode: - args = iter(reversed(ir.args)) - last = next(args) - cur = build.and_(ir_to_meson(next(args), build), ir_to_meson(last, build)) - for a in args: - cur = build.and_(ir_to_meson(a, build), cur) - return cur +def eval_cfg(raw: str, cfgs: T.Dict[str, str]) -> bool: + return _eval_cfg(parse(lexer(raw)), cfgs) diff --git a/mesonbuild/cargo/interpreter.py b/mesonbuild/cargo/interpreter.py index af272a8..a0d4371 100644 --- a/mesonbuild/cargo/interpreter.py +++ b/mesonbuild/cargo/interpreter.py @@ -11,439 +11,30 @@ port will be required. from __future__ import annotations import dataclasses -import importlib -import json import os -import shutil import collections import urllib.parse import itertools import typing as T -from . import builder -from . import version -from ..mesonlib import MesonException, Popen_safe +from . import builder, version, cfg +from .toml import load_toml, TomlImplementationMissing +from .manifest import Manifest, CargoLock, fixup_meson_varname +from ..mesonlib import MesonException, MachineChoice from .. import coredata, mlog from ..wrap.wrap import PackageDefinition if T.TYPE_CHECKING: - from types import ModuleType - - from typing_extensions import Protocol, Self - - from . import manifest + from . import raw from .. import mparser + from .manifest import Dependency, SystemDependency from ..environment import Environment from ..interpreterbase import SubProject + from ..compilers.rust import RustCompiler - # Copied from typeshed. Blarg that they don't expose this - class DataclassInstance(Protocol): - __dataclass_fields__: T.ClassVar[dict[str, dataclasses.Field[T.Any]]] - - _UnknownKeysT = T.TypeVar('_UnknownKeysT', manifest.FixedPackage, - manifest.FixedDependency, manifest.FixedLibTarget, - manifest.FixedBuildTarget) - - -# tomllib is present in python 3.11, before that it is a pypi module called tomli, -# we try to import tomllib, then tomli, -# TODO: add a fallback to toml2json? -tomllib: T.Optional[ModuleType] = None -toml2json: T.Optional[str] = None -for t in ['tomllib', 'tomli']: - try: - tomllib = importlib.import_module(t) - break - except ImportError: - pass -else: - # TODO: it would be better to use an Executable here, which could be looked - # up in the cross file or provided by a wrap. However, that will have to be - # passed in externally, since we don't have (and I don't think we should), - # have access to the `Environment` for that in this module. - toml2json = shutil.which('toml2json') - - -_EXTRA_KEYS_WARNING = ( - "This may (unlikely) be an error in the cargo manifest, or may be a missing " - "implementation in Meson. If this issue can be reproduced with the latest " - "version of Meson, please help us by opening an issue at " - "https://github.com/mesonbuild/meson/issues. Please include the crate and " - "version that is generating this warning if possible." -) - -class TomlImplementationMissing(MesonException): - pass - - -def load_toml(filename: str) -> T.Dict[object, object]: - if tomllib: - with open(filename, 'rb') as f: - raw = tomllib.load(f) - else: - if toml2json is None: - raise TomlImplementationMissing('Could not find an implementation of tomllib, nor toml2json') - - p, out, err = Popen_safe([toml2json, filename]) - if p.returncode != 0: - raise MesonException('toml2json failed to decode output\n', err) - - raw = json.loads(out) - - if not isinstance(raw, dict): - raise MesonException("Cargo.toml isn't a dictionary? How did that happen?") - - return raw - - -def fixup_meson_varname(name: str) -> str: - """Fixup a meson variable name - - :param name: The name to fix - :return: the fixed name - """ - return name.replace('-', '_') - - -# Pylance can figure out that these do not, in fact, overlap, but mypy can't -@T.overload -def _fixup_raw_mappings(d: manifest.BuildTarget) -> manifest.FixedBuildTarget: ... # type: ignore - -@T.overload -def _fixup_raw_mappings(d: manifest.LibTarget) -> manifest.FixedLibTarget: ... # type: ignore - -@T.overload -def _fixup_raw_mappings(d: manifest.Dependency) -> manifest.FixedDependency: ... - -def _fixup_raw_mappings(d: T.Union[manifest.BuildTarget, manifest.LibTarget, manifest.Dependency] - ) -> T.Union[manifest.FixedBuildTarget, manifest.FixedLibTarget, - manifest.FixedDependency]: - """Fixup raw cargo mappings to ones more suitable for python to consume. - - This does the following: - * replaces any `-` with `_`, cargo likes the former, but python dicts make - keys with `-` in them awkward to work with - * Convert Dependency versions from the cargo format to something meson - understands - - :param d: The mapping to fix - :return: the fixed string - """ - raw = {fixup_meson_varname(k): v for k, v in d.items()} - if 'version' in raw: - assert isinstance(raw['version'], str), 'for mypy' - raw['version'] = version.convert(raw['version']) - return T.cast('T.Union[manifest.FixedBuildTarget, manifest.FixedLibTarget, manifest.FixedDependency]', raw) - - -def _handle_unknown_keys(data: _UnknownKeysT, cls: T.Union[DataclassInstance, T.Type[DataclassInstance]], - msg: str) -> _UnknownKeysT: - """Remove and warn on keys that are coming from cargo, but are unknown to - our representations. - - This is intended to give users the possibility of things proceeding when a - new key is added to Cargo.toml that we don't yet handle, but to still warn - them that things might not work. - - :param data: The raw data to look at - :param cls: The Dataclass derived type that will be created - :param msg: the header for the error message. Usually something like "In N structure". - :return: The original data structure, but with all unknown keys removed. - """ - unexpected = set(data) - {x.name for x in dataclasses.fields(cls)} - if unexpected: - mlog.warning(msg, 'has unexpected keys', '"{}".'.format(', '.join(sorted(unexpected))), - _EXTRA_KEYS_WARNING) - for k in unexpected: - # Mypy and Pyright can't prove that this is okay - del data[k] # type: ignore[misc] - return data - - -@dataclasses.dataclass -class Package: - - """Representation of a Cargo Package entry, with defaults filled in.""" - - name: str - version: str - description: T.Optional[str] = None - resolver: T.Optional[str] = None - authors: T.List[str] = dataclasses.field(default_factory=list) - edition: manifest.EDITION = '2015' - rust_version: T.Optional[str] = None - documentation: T.Optional[str] = None - readme: T.Optional[str] = None - homepage: T.Optional[str] = None - repository: T.Optional[str] = None - license: T.Optional[str] = None - license_file: T.Optional[str] = None - keywords: T.List[str] = dataclasses.field(default_factory=list) - categories: T.List[str] = dataclasses.field(default_factory=list) - workspace: T.Optional[str] = None - build: T.Optional[str] = None - links: T.Optional[str] = None - exclude: T.List[str] = dataclasses.field(default_factory=list) - include: T.List[str] = dataclasses.field(default_factory=list) - publish: bool = True - metadata: T.Dict[str, T.Any] = dataclasses.field(default_factory=dict) - default_run: T.Optional[str] = None - autolib: bool = True - autobins: bool = True - autoexamples: bool = True - autotests: bool = True - autobenches: bool = True - api: str = dataclasses.field(init=False) - - def __post_init__(self) -> None: - self.api = _version_to_api(self.version) - - @classmethod - def from_raw(cls, raw: manifest.Package) -> Self: - pkg = T.cast('manifest.FixedPackage', - {fixup_meson_varname(k): v for k, v in raw.items()}) - pkg = _handle_unknown_keys(pkg, cls, f'Package entry {pkg["name"]}') - return cls(**pkg) - -@dataclasses.dataclass -class SystemDependency: - - """ Representation of a Cargo system-deps entry - https://docs.rs/system-deps/latest/system_deps - """ - - name: str - version: T.List[str] - optional: bool = False - feature: T.Optional[str] = None - feature_overrides: T.Dict[str, T.Dict[str, str]] = dataclasses.field(default_factory=dict) - - @classmethod - def from_raw(cls, name: str, raw: T.Any) -> SystemDependency: - if isinstance(raw, str): - return cls(name, SystemDependency.convert_version(raw)) - name = raw.get('name', name) - version = SystemDependency.convert_version(raw.get('version')) - optional = raw.get('optional', False) - feature = raw.get('feature') - # Everything else are overrides when certain features are enabled. - feature_overrides = {k: v for k, v in raw.items() if k not in {'name', 'version', 'optional', 'feature'}} - return cls(name, version, optional, feature, feature_overrides) - - @staticmethod - def convert_version(version: T.Optional[str]) -> T.List[str]: - vers = version.split(',') if version is not None else [] - result: T.List[str] = [] - for v in vers: - v = v.strip() - if v[0] not in '><=': - v = f'>={v}' - result.append(v) - return result - - def enabled(self, features: T.Set[str]) -> bool: - return self.feature is None or self.feature in features - -@dataclasses.dataclass -class Dependency: - - """Representation of a Cargo Dependency Entry.""" - - name: dataclasses.InitVar[str] - version: T.List[str] - registry: T.Optional[str] = None - git: T.Optional[str] = None - branch: T.Optional[str] = None - rev: T.Optional[str] = None - path: T.Optional[str] = None - optional: bool = False - package: str = '' - default_features: bool = True - features: T.List[str] = dataclasses.field(default_factory=list) - api: str = dataclasses.field(init=False) - - def __post_init__(self, name: str) -> None: - self.package = self.package or name - # Extract wanted API version from version constraints. - api = set() - for v in self.version: - if v.startswith(('>=', '==')): - api.add(_version_to_api(v[2:].strip())) - elif v.startswith('='): - api.add(_version_to_api(v[1:].strip())) - if not api: - self.api = '0' - elif len(api) == 1: - self.api = api.pop() - else: - raise MesonException(f'Cannot determine minimum API version from {self.version}.') - - @classmethod - def from_raw(cls, name: str, raw: manifest.DependencyV) -> Dependency: - """Create a dependency from a raw cargo dictionary""" - if isinstance(raw, str): - return cls(name, version.convert(raw)) - fixed = _handle_unknown_keys(_fixup_raw_mappings(raw), cls, f'Dependency entry {name}') - return cls(name, **fixed) - - -@dataclasses.dataclass -class BuildTarget: - - name: str - crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib']) - path: dataclasses.InitVar[T.Optional[str]] = None - - # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-test-field - # True for lib, bin, test - test: bool = True - - # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-doctest-field - # True for lib - doctest: bool = False - - # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-bench-field - # True for lib, bin, benchmark - bench: bool = True - - # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-doc-field - # True for libraries and binaries - doc: bool = False - - harness: bool = True - edition: manifest.EDITION = '2015' - required_features: T.List[str] = dataclasses.field(default_factory=list) - plugin: bool = False - - @classmethod - def from_raw(cls, raw: manifest.BuildTarget) -> Self: - name = raw.get('name', '<anonymous>') - build = _handle_unknown_keys(_fixup_raw_mappings(raw), cls, f'Binary entry {name}') - return cls(**build) - -@dataclasses.dataclass -class Library(BuildTarget): - - """Representation of a Cargo Library Entry.""" - - doctest: bool = True - doc: bool = True - path: str = os.path.join('src', 'lib.rs') - proc_macro: bool = False - crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib']) - doc_scrape_examples: bool = True - - @classmethod - def from_raw(cls, raw: manifest.LibTarget, fallback_name: str) -> Self: # type: ignore[override] - fixed = _fixup_raw_mappings(raw) - - # We need to set the name field if it's not set manually, including if - # other fields are set in the lib section - if 'name' not in fixed: - fixed['name'] = fallback_name - fixed = _handle_unknown_keys(fixed, cls, f'Library entry {fixed["name"]}') - - return cls(**fixed) - - -@dataclasses.dataclass -class Binary(BuildTarget): - - """Representation of a Cargo Bin Entry.""" - - doc: bool = True - - -@dataclasses.dataclass -class Test(BuildTarget): - - """Representation of a Cargo Test Entry.""" - - bench: bool = True - - -@dataclasses.dataclass -class Benchmark(BuildTarget): - - """Representation of a Cargo Benchmark Entry.""" - - test: bool = True - - -@dataclasses.dataclass -class Example(BuildTarget): - - """Representation of a Cargo Example Entry.""" - - crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['bin']) - - -@dataclasses.dataclass -class Manifest: - - """Cargo Manifest definition. - - Most of these values map up to the Cargo Manifest, but with default values - if not provided. - - Cargo subprojects can contain what Meson wants to treat as multiple, - interdependent, subprojects. - - :param path: the path within the cargo subproject. - """ - - package: Package - dependencies: T.Dict[str, Dependency] - dev_dependencies: T.Dict[str, Dependency] - build_dependencies: T.Dict[str, Dependency] - system_dependencies: T.Dict[str, SystemDependency] = dataclasses.field(init=False) - lib: Library - bin: T.List[Binary] - test: T.List[Test] - bench: T.List[Benchmark] - example: T.List[Example] - features: T.Dict[str, T.List[str]] - target: T.Dict[str, T.Dict[str, Dependency]] - path: str = '' - - def __post_init__(self) -> None: - self.features.setdefault('default', []) - self.system_dependencies = {k: SystemDependency.from_raw(k, v) for k, v in self.package.metadata.get('system-deps', {}).items()} - - -def _convert_manifest(raw_manifest: manifest.Manifest, subdir: str, path: str = '') -> Manifest: - return Manifest( - Package.from_raw(raw_manifest['package']), - {k: Dependency.from_raw(k, v) for k, v in raw_manifest.get('dependencies', {}).items()}, - {k: Dependency.from_raw(k, v) for k, v in raw_manifest.get('dev-dependencies', {}).items()}, - {k: Dependency.from_raw(k, v) for k, v in raw_manifest.get('build-dependencies', {}).items()}, - Library.from_raw(raw_manifest.get('lib', {}), raw_manifest['package']['name']), - [Binary.from_raw(b) for b in raw_manifest.get('bin', {})], - [Test.from_raw(b) for b in raw_manifest.get('test', {})], - [Benchmark.from_raw(b) for b in raw_manifest.get('bench', {})], - [Example.from_raw(b) for b in raw_manifest.get('example', {})], - raw_manifest.get('features', {}), - {k: {k2: Dependency.from_raw(k2, v2) for k2, v2 in v.get('dependencies', {}).items()} - for k, v in raw_manifest.get('target', {}).items()}, - path, - ) - - -def _version_to_api(version: str) -> str: - # x.y.z -> x - # 0.x.y -> 0.x - # 0.0.x -> 0 - vers = version.split('.') - if int(vers[0]) != 0: - return vers[0] - elif len(vers) >= 2 and int(vers[1]) != 0: - return f'0.{vers[1]}' - return '0' - - -def _dependency_name(package_name: str, api: str) -> str: - basename = package_name[:-3] if package_name.endswith('-rs') else package_name - return f'{basename}-{api}-rs' +def _dependency_name(package_name: str, api: str, suffix: str = '-rs') -> str: + basename = package_name[:-len(suffix)] if package_name.endswith(suffix) else package_name + return f'{basename}-{api}{suffix}' def _dependency_varname(package_name: str) -> str: @@ -458,13 +49,13 @@ def _extra_deps_varname() -> str: return 'extra_deps' +@dataclasses.dataclass class PackageState: - def __init__(self, manifest: Manifest, downloaded: bool) -> None: - self.manifest = manifest - self.downloaded = downloaded - self.features: T.Set[str] = set() - self.required_deps: T.Set[str] = set() - self.optional_deps_features: T.Dict[str, T.Set[str]] = collections.defaultdict(set) + manifest: Manifest + downloaded: bool = False + features: T.Set[str] = dataclasses.field(default_factory=set) + required_deps: T.Set[str] = dataclasses.field(default_factory=set) + optional_deps_features: T.Dict[str, T.Set[str]] = dataclasses.field(default_factory=lambda: collections.defaultdict(set)) @dataclasses.dataclass(frozen=True) @@ -476,10 +67,16 @@ class PackageKey: class Interpreter: def __init__(self, env: Environment) -> None: self.environment = env + self.host_rustc = T.cast('RustCompiler', self.environment.coredata.compilers[MachineChoice.HOST]['rust']) # Map Cargo.toml's subdir to loaded manifest. self.manifests: T.Dict[str, Manifest] = {} # Map of cargo package (name + api) to its state self.packages: T.Dict[PackageKey, PackageState] = {} + # Rustc's config + self.cfgs = self._get_cfgs() + + def get_build_def_files(self) -> T.List[str]: + return [os.path.join(subdir, 'Cargo.toml') for subdir in self.manifests] def interpret(self, subdir: str) -> mparser.CodeBlockNode: manifest = self._load_manifest(subdir) @@ -503,9 +100,7 @@ class Interpreter: ast += self._create_dependencies(pkg, build) ast += self._create_meson_subdir(build) - # Libs are always auto-discovered and there's no other way to handle them, - # which is unfortunate for reproducability - if os.path.exists(os.path.join(self.environment.source_dir, subdir, pkg.manifest.path, pkg.manifest.lib.path)): + if pkg.manifest.lib: for crate_type in pkg.manifest.lib.crate_type: ast.extend(self._create_lib(pkg, build, crate_type)) @@ -526,6 +121,10 @@ class Interpreter: self.environment.wrap_resolver.wraps[meson_depname].type is not None pkg = PackageState(manifest, downloaded) self.packages[key] = pkg + # Merge target specific dependencies that are enabled + for condition, dependencies in manifest.target.items(): + if cfg.eval_cfg(condition, self.cfgs): + manifest.dependencies.update(dependencies) # Fetch required dependencies recursively. for depname, dep in manifest.dependencies.items(): if not dep.optional: @@ -538,11 +137,12 @@ class Interpreter: def _load_manifest(self, subdir: str) -> Manifest: manifest_ = self.manifests.get(subdir) if not manifest_: - filename = os.path.join(self.environment.source_dir, subdir, 'Cargo.toml') - raw = load_toml(filename) - if 'package' in raw: - raw_manifest = T.cast('manifest.Manifest', raw) - manifest_ = _convert_manifest(raw_manifest, subdir) + path = os.path.join(self.environment.source_dir, subdir) + filename = os.path.join(path, 'Cargo.toml') + toml = load_toml(filename) + if 'package' in toml: + raw_manifest = T.cast('raw.Manifest', toml) + manifest_ = Manifest.from_raw(raw_manifest, path) self.manifests[subdir] = manifest_ else: raise MesonException(f'{subdir}/Cargo.toml does not have [package] section') @@ -599,6 +199,23 @@ class Interpreter: else: self._enable_feature(pkg, f) + def _get_cfgs(self) -> T.Dict[str, str]: + cfgs = self.host_rustc.get_cfgs().copy() + rustflags = self.environment.coredata.get_external_args(MachineChoice.HOST, 'rust') + rustflags_i = iter(rustflags) + for i in rustflags_i: + if i == '--cfg': + cfgs.append(next(rustflags_i)) + return dict(self._split_cfg(i) for i in cfgs) + + @staticmethod + def _split_cfg(cfg: str) -> T.Tuple[str, str]: + pair = cfg.split('=', maxsplit=1) + value = pair[1] if len(pair) > 1 else '' + if value and value[0] == '"': + value = value[1:-1] + return pair[0], value + def _create_project(self, pkg: PackageState, build: builder.Builder) -> T.List[mparser.BaseNode]: """Create the project() function call @@ -608,6 +225,7 @@ class Interpreter: """ default_options: T.List[mparser.BaseNode] = [] default_options.append(build.string(f'rust_std={pkg.manifest.package.edition}')) + default_options.append(build.string(f'build.rust_std={pkg.manifest.package.edition}')) if pkg.downloaded: default_options.append(build.string('warning_level=0')) @@ -643,8 +261,9 @@ class Interpreter: return ast def _create_system_dependency(self, name: str, dep: SystemDependency, build: builder.Builder) -> T.List[mparser.BaseNode]: + # TODO: handle feature_overrides kw = { - 'version': build.array([build.string(s) for s in dep.version]), + 'version': build.array([build.string(s) for s in dep.meson_version]), 'required': build.bool(not dep.optional), } varname = f'{fixup_meson_varname(name)}_system_dep' @@ -671,7 +290,7 @@ class Interpreter: def _create_dependency(self, dep: Dependency, build: builder.Builder) -> T.List[mparser.BaseNode]: pkg = self._dep_package(dep) kw = { - 'version': build.array([build.string(s) for s in dep.version]), + 'version': build.array([build.string(s) for s in dep.meson_version]), } # Lookup for this dependency with the features we want in default_options kwarg. # @@ -747,7 +366,7 @@ class Interpreter: build.block([build.function('subdir', [build.string('meson')])])) ] - def _create_lib(self, pkg: PackageState, build: builder.Builder, crate_type: manifest.CRATE_TYPE) -> T.List[mparser.BaseNode]: + def _create_lib(self, pkg: PackageState, build: builder.Builder, crate_type: raw.CRATE_TYPE) -> T.List[mparser.BaseNode]: dependencies: T.List[mparser.BaseNode] = [] dependency_map: T.Dict[mparser.BaseNode, mparser.BaseNode] = {} for name in pkg.required_deps: @@ -780,6 +399,9 @@ class Interpreter: 'rust_args': build.array(rust_args), } + depname_suffix = '-rs' if crate_type in {'lib', 'rlib', 'proc-macro'} else f'-{crate_type}' + depname = _dependency_name(pkg.manifest.package.name, pkg.manifest.package.api, depname_suffix) + lib: mparser.BaseNode if pkg.manifest.lib.proc_macro or crate_type == 'proc-macro': lib = build.method('proc_macro', build.identifier('rust'), posargs, kwargs) @@ -812,7 +434,8 @@ class Interpreter: 'link_with': build.identifier('lib'), 'variables': build.dict({ build.string('features'): build.string(','.join(pkg.features)), - }) + }), + 'version': build.string(pkg.manifest.package.version), }, ), 'dep' @@ -821,7 +444,7 @@ class Interpreter: 'override_dependency', build.identifier('meson'), [ - build.string(_dependency_name(pkg.manifest.package.name, pkg.manifest.package.api)), + build.string(depname), build.identifier('dep'), ], ), @@ -835,24 +458,23 @@ def load_wraps(source_dir: str, subproject_dir: str) -> T.List[PackageDefinition filename = os.path.join(source_dir, 'Cargo.lock') if os.path.exists(filename): try: - cargolock = T.cast('manifest.CargoLock', load_toml(filename)) + toml = load_toml(filename) except TomlImplementationMissing as e: mlog.warning('Failed to load Cargo.lock:', str(e), fatal=False) return wraps - for package in cargolock['package']: - name = package['name'] - version = package['version'] - subp_name = _dependency_name(name, _version_to_api(version)) - source = package.get('source') - if source is None: + raw_cargolock = T.cast('raw.CargoLock', toml) + cargolock = CargoLock.from_raw(raw_cargolock) + for package in cargolock.package: + subp_name = _dependency_name(package.name, version.api(package.version)) + if package.source is None: # This is project's package, or one of its workspace members. pass - elif source == 'registry+https://github.com/rust-lang/crates.io-index': - checksum = package.get('checksum') + elif package.source == 'registry+https://github.com/rust-lang/crates.io-index': + checksum = package.checksum if checksum is None: - checksum = cargolock['metadata'][f'checksum {name} {version} ({source})'] - url = f'https://crates.io/api/v1/crates/{name}/{version}/download' - directory = f'{name}-{version}' + checksum = cargolock.metadata[f'checksum {package.name} {package.version} ({package.source})'] + url = f'https://crates.io/api/v1/crates/{package.name}/{package.version}/download' + directory = f'{package.name}-{package.version}' wraps.append(PackageDefinition.from_values(subp_name, subproject_dir, 'file', { 'directory': directory, 'source_url': url, @@ -860,18 +482,18 @@ def load_wraps(source_dir: str, subproject_dir: str) -> T.List[PackageDefinition 'source_hash': checksum, 'method': 'cargo', })) - elif source.startswith('git+'): - parts = urllib.parse.urlparse(source[4:]) + elif package.source.startswith('git+'): + parts = urllib.parse.urlparse(package.source[4:]) query = urllib.parse.parse_qs(parts.query) branch = query['branch'][0] if 'branch' in query else '' revision = parts.fragment or branch url = urllib.parse.urlunparse(parts._replace(params='', query='', fragment='')) wraps.append(PackageDefinition.from_values(subp_name, subproject_dir, 'git', { - 'directory': name, + 'directory': package.name, 'url': url, 'revision': revision, 'method': 'cargo', })) else: - mlog.warning(f'Unsupported source URL in {filename}: {source}') + mlog.warning(f'Unsupported source URL in {filename}: {package.source}') return wraps diff --git a/mesonbuild/cargo/manifest.py b/mesonbuild/cargo/manifest.py index d95df7f..ab059b0 100644 --- a/mesonbuild/cargo/manifest.py +++ b/mesonbuild/cargo/manifest.py @@ -4,244 +4,505 @@ """Type definitions for cargo manifest files.""" from __future__ import annotations + +import dataclasses +import os import typing as T -from typing_extensions import Literal, TypedDict, Required - -EDITION = Literal['2015', '2018', '2021'] -CRATE_TYPE = Literal['bin', 'lib', 'dylib', 'staticlib', 'cdylib', 'rlib', 'proc-macro'] - -Package = TypedDict( - 'Package', - { - 'name': Required[str], - 'version': Required[str], - 'authors': T.List[str], - 'edition': EDITION, - 'rust-version': str, - 'description': str, - 'readme': str, - 'license': str, - 'license-file': str, - 'keywords': T.List[str], - 'categories': T.List[str], - 'workspace': str, - 'build': str, - 'links': str, - 'include': T.List[str], - 'exclude': T.List[str], - 'publish': bool, - 'metadata': T.Dict[str, T.Dict[str, str]], - 'default-run': str, - 'autolib': bool, - 'autobins': bool, - 'autoexamples': bool, - 'autotests': bool, - 'autobenches': bool, - }, - total=False, -) -"""A description of the Package Dictionary.""" - -class FixedPackage(TypedDict, total=False): - - """A description of the Package Dictionary, fixed up.""" - - name: Required[str] - version: Required[str] - authors: T.List[str] - edition: EDITION - rust_version: str - description: str - readme: str - license: str - license_file: str - keywords: T.List[str] - categories: T.List[str] - workspace: str - build: str - links: str - include: T.List[str] - exclude: T.List[str] - publish: bool - metadata: T.Dict[str, T.Dict[str, str]] - default_run: str - autolib: bool - autobins: bool - autoexamples: bool - autotests: bool - autobenches: bool - - -class Badge(TypedDict): - - """An entry in the badge section.""" - - status: Literal['actively-developed', 'passively-developed', 'as-is', 'experimental', 'deprecated', 'none'] - - -Dependency = TypedDict( - 'Dependency', - { - 'version': str, - 'registry': str, - 'git': str, - 'branch': str, - 'rev': str, - 'path': str, - 'optional': bool, - 'package': str, - 'default-features': bool, - 'features': T.List[str], - }, - total=False, -) -"""An entry in the *dependencies sections.""" +from . import version +from ..mesonlib import MesonException, lazy_property +from .. import mlog +if T.TYPE_CHECKING: + from typing_extensions import Protocol, Self -class FixedDependency(TypedDict, total=False): + from . import raw + from .raw import EDITION, CRATE_TYPE - """An entry in the *dependencies sections, fixed up.""" + # Copied from typeshed. Blarg that they don't expose this + class DataclassInstance(Protocol): + __dataclass_fields__: T.ClassVar[dict[str, dataclasses.Field[T.Any]]] - version: T.List[str] - registry: str - git: str - branch: str - rev: str - path: str - optional: bool - package: str - default_features: bool - features: T.List[str] - - -DependencyV = T.Union[Dependency, str] -"""A Dependency entry, either a string or a Dependency Dict.""" - - -_BaseBuildTarget = TypedDict( - '_BaseBuildTarget', - { - 'path': str, - 'test': bool, - 'doctest': bool, - 'bench': bool, - 'doc': bool, - 'plugin': bool, - 'proc-macro': bool, - 'harness': bool, - 'edition': EDITION, - 'crate-type': T.List[CRATE_TYPE], - 'required-features': T.List[str], - }, - total=False, +_DI = T.TypeVar('_DI', bound='DataclassInstance') +_R = T.TypeVar('_R', bound='raw._BaseBuildTarget') + +_EXTRA_KEYS_WARNING = ( + "This may (unlikely) be an error in the cargo manifest, or may be a missing " + "implementation in Meson. If this issue can be reproduced with the latest " + "version of Meson, please help us by opening an issue at " + "https://github.com/mesonbuild/meson/issues. Please include the crate and " + "version that is generating this warning if possible." ) -class BuildTarget(_BaseBuildTarget, total=False): +def fixup_meson_varname(name: str) -> str: + """Fixup a meson variable name + + :param name: The name to fix + :return: the fixed name + """ + return name.replace('-', '_') + + +@T.overload +def _depv_to_dep(depv: raw.FromWorkspace) -> raw.FromWorkspace: ... + +@T.overload +def _depv_to_dep(depv: raw.DependencyV) -> raw.Dependency: ... + +def _depv_to_dep(depv: T.Union[raw.FromWorkspace, raw.DependencyV]) -> T.Union[raw.FromWorkspace, raw.Dependency]: + return {'version': depv} if isinstance(depv, str) else depv + - name: Required[str] +def _raw_to_dataclass(raw: T.Mapping[str, object], cls: T.Type[_DI], + msg: str, **kwargs: T.Callable[[T.Any], object]) -> _DI: + """Fixup raw cargo mappings to ones more suitable for python to consume as dataclass. -class LibTarget(_BaseBuildTarget, total=False): + * Replaces any `-` with `_` in the keys. + * Optionally pass values through the functions in kwargs, in order to do + recursive conversions. + * Remove and warn on keys that are coming from cargo, but are unknown to + our representations. + + This is intended to give users the possibility of things proceeding when a + new key is added to Cargo.toml that we don't yet handle, but to still warn + them that things might not work. + + :param data: The raw data to look at + :param cls: The Dataclass derived type that will be created + :param msg: the header for the error message. Usually something like "In N structure". + :return: The original data structure, but with all unknown keys removed. + """ + new_dict = {} + unexpected = set() + fields = {x.name for x in dataclasses.fields(cls)} + for orig_k, v in raw.items(): + k = fixup_meson_varname(orig_k) + if k not in fields: + unexpected.add(orig_k) + continue + if k in kwargs: + v = kwargs[k](v) + new_dict[k] = v + + if unexpected: + mlog.warning(msg, 'has unexpected keys', '"{}".'.format(', '.join(sorted(unexpected))), + _EXTRA_KEYS_WARNING) + return cls(**new_dict) + + +@T.overload +def _inherit_from_workspace(raw: raw.Package, + raw_from_workspace: T.Optional[T.Mapping[str, object]], + msg: str, + **kwargs: T.Callable[[T.Any, T.Any], object]) -> raw.Package: ... + +@T.overload +def _inherit_from_workspace(raw: T.Union[raw.FromWorkspace, raw.Dependency], + raw_from_workspace: T.Optional[T.Mapping[str, object]], + msg: str, + **kwargs: T.Callable[[T.Any, T.Any], object]) -> raw.Dependency: ... + +def _inherit_from_workspace(raw_: T.Union[raw.FromWorkspace, raw.Package, raw.Dependency], # type: ignore[misc] + raw_from_workspace: T.Optional[T.Mapping[str, object]], + msg: str, + **kwargs: T.Callable[[T.Any, T.Any], object]) -> T.Mapping[str, object]: + # allow accesses by non-literal key below + raw = T.cast('T.Mapping[str, object]', raw_) + + if not raw_from_workspace: + if raw.get('workspace', False) or \ + any(isinstance(v, dict) and v.get('workspace', False) for v in raw): + raise MesonException(f'Cargo.toml file requests {msg} from workspace') + + return raw + + result = {k: v for k, v in raw.items() if k != 'workspace'} + for k, v in raw.items(): + if isinstance(v, dict) and v.get('workspace', False): + if k in raw_from_workspace: + result[k] = raw_from_workspace[k] + if k in kwargs: + result[k] = kwargs[k](v, result[k]) + else: + del result[k] + + if raw.get('workspace', False): + for k, v in raw_from_workspace.items(): + if k not in result or k in kwargs: + if k in kwargs: + v = kwargs[k](raw.get(k), v) + result[k] = v + return result + + +@dataclasses.dataclass +class Package: + + """Representation of a Cargo Package entry, with defaults filled in.""" + + name: str + version: str + description: T.Optional[str] = None + resolver: T.Optional[str] = None + authors: T.List[str] = dataclasses.field(default_factory=list) + edition: EDITION = '2015' + rust_version: T.Optional[str] = None + documentation: T.Optional[str] = None + readme: T.Optional[str] = None + homepage: T.Optional[str] = None + repository: T.Optional[str] = None + license: T.Optional[str] = None + license_file: T.Optional[str] = None + keywords: T.List[str] = dataclasses.field(default_factory=list) + categories: T.List[str] = dataclasses.field(default_factory=list) + workspace: T.Optional[str] = None + build: T.Optional[str] = None + links: T.Optional[str] = None + exclude: T.List[str] = dataclasses.field(default_factory=list) + include: T.List[str] = dataclasses.field(default_factory=list) + publish: bool = True + metadata: T.Dict[str, T.Any] = dataclasses.field(default_factory=dict) + default_run: T.Optional[str] = None + autolib: bool = True + autobins: bool = True + autoexamples: bool = True + autotests: bool = True + autobenches: bool = True + + @lazy_property + def api(self) -> str: + return version.api(self.version) + + @classmethod + def from_raw(cls, raw_pkg: raw.Package, workspace: T.Optional[Workspace] = None) -> Self: + raw_ws_pkg = None + if workspace is not None: + raw_ws_pkg = workspace.package + + raw_pkg = _inherit_from_workspace(raw_pkg, raw_ws_pkg, f'Package entry {raw_pkg["name"]}') + return _raw_to_dataclass(raw_pkg, cls, f'Package entry {raw_pkg["name"]}') + +@dataclasses.dataclass +class SystemDependency: + + """ Representation of a Cargo system-deps entry + https://docs.rs/system-deps/latest/system_deps + """ name: str + version: str = '' + optional: bool = False + feature: T.Optional[str] = None + # TODO: convert values to dataclass + feature_overrides: T.Dict[str, T.Dict[str, str]] = dataclasses.field(default_factory=dict) + + @classmethod + def from_raw(cls, name: str, raw: T.Union[T.Dict[str, T.Any], str]) -> SystemDependency: + if isinstance(raw, str): + raw = {'version': raw} + name = raw.get('name', name) + version = raw.get('version', '') + optional = raw.get('optional', False) + feature = raw.get('feature') + # Everything else are overrides when certain features are enabled. + feature_overrides = {k: v for k, v in raw.items() if k not in {'name', 'version', 'optional', 'feature'}} + return cls(name, version, optional, feature, feature_overrides) + + @lazy_property + def meson_version(self) -> T.List[str]: + vers = self.version.split(',') if self.version else [] + result: T.List[str] = [] + for v in vers: + v = v.strip() + if v[0] not in '><=': + v = f'>={v}' + result.append(v) + return result + + def enabled(self, features: T.Set[str]) -> bool: + return self.feature is None or self.feature in features + +@dataclasses.dataclass +class Dependency: + + """Representation of a Cargo Dependency Entry.""" + package: str + version: str = '' + registry: T.Optional[str] = None + git: T.Optional[str] = None + branch: T.Optional[str] = None + rev: T.Optional[str] = None + path: T.Optional[str] = None + optional: bool = False + default_features: bool = True + features: T.List[str] = dataclasses.field(default_factory=list) + + @lazy_property + def meson_version(self) -> T.List[str]: + return version.convert(self.version) + + @lazy_property + def api(self) -> str: + # Extract wanted API version from version constraints. + api = set() + for v in self.meson_version: + if v.startswith(('>=', '==')): + api.add(version.api(v[2:].strip())) + elif v.startswith('='): + api.add(version.api(v[1:].strip())) + if not api: + return '0' + elif len(api) == 1: + return api.pop() + else: + raise MesonException(f'Cannot determine minimum API version from {self.version}.') + + @classmethod + def from_raw_dict(cls, name: str, raw_dep: T.Union[raw.FromWorkspace, raw.Dependency], member_path: str = '', raw_ws_dep: T.Optional[raw.Dependency] = None) -> Dependency: + raw_dep = _inherit_from_workspace(raw_dep, raw_ws_dep, + f'Dependency entry {name}', + path=lambda pkg_path, ws_path: os.path.relpath(ws_path, member_path), + features=lambda pkg_path, ws_path: (pkg_path or []) + (ws_path or [])) + raw_dep.setdefault('package', name) + return _raw_to_dataclass(raw_dep, cls, f'Dependency entry {name}') + + @classmethod + def from_raw(cls, name: str, raw_depv: T.Union[raw.FromWorkspace, raw.DependencyV], member_path: str = '', workspace: T.Optional[Workspace] = None) -> Dependency: + """Create a dependency from a raw cargo dictionary or string""" + raw_ws_dep: T.Optional[raw.Dependency] = None + if workspace is not None: + raw_ws_depv = workspace.dependencies.get(name, {}) + raw_ws_dep = _depv_to_dep(raw_ws_depv) + + raw_dep = _depv_to_dep(raw_depv) + return cls.from_raw_dict(name, raw_dep, member_path, raw_ws_dep) + + +@dataclasses.dataclass +class BuildTarget(T.Generic[_R]): -class _BaseFixedBuildTarget(TypedDict, total=False): + name: str path: str - test: bool - doctest: bool - bench: bool - doc: bool - plugin: bool - harness: bool - edition: EDITION crate_type: T.List[CRATE_TYPE] - required_features: T.List[str] + # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-test-field + # True for lib, bin, test + test: bool = True -class FixedBuildTarget(_BaseFixedBuildTarget, total=False): + # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-doctest-field + # True for lib + doctest: bool = False - name: str + # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-bench-field + # True for lib, bin, benchmark + bench: bool = True -class FixedLibTarget(_BaseFixedBuildTarget, total=False): + # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-doc-field + # True for libraries and binaries + doc: bool = False - name: Required[str] - proc_macro: bool + harness: bool = True + edition: EDITION = '2015' + required_features: T.List[str] = dataclasses.field(default_factory=list) + plugin: bool = False + @classmethod + def from_raw(cls, raw: _R) -> Self: + name = raw.get('name', '<anonymous>') + return _raw_to_dataclass(raw, cls, f'Binary entry {name}') -class Target(TypedDict): +@dataclasses.dataclass +class Library(BuildTarget['raw.LibTarget']): - """Target entry in the Manifest File.""" + """Representation of a Cargo Library Entry.""" - dependencies: T.Dict[str, DependencyV] + doctest: bool = True + doc: bool = True + path: str = os.path.join('src', 'lib.rs') + proc_macro: bool = False + crate_type: T.List[CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib']) + doc_scrape_examples: bool = True + @classmethod + def from_raw(cls, raw: raw.LibTarget, fallback_name: str) -> Self: # type: ignore[override] + # We need to set the name field if it's not set manually, including if + # other fields are set in the lib section + raw.setdefault('name', fallback_name) + return _raw_to_dataclass(raw, cls, f'Library entry {raw["name"]}') -class Workspace(TypedDict): - """The representation of a workspace. +@dataclasses.dataclass +class Binary(BuildTarget['raw.BuildTarget']): - In a vritual manifest the :attribute:`members` is always present, but in a - project manifest, an empty workspace may be provided, in which case the - workspace is implicitly filled in by values from the path based dependencies. + """Representation of a Cargo Bin Entry.""" - the :attribute:`exclude` is always optional - """ + doc: bool = True + crate_type: T.List[CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['bin']) - members: T.List[str] - exclude: T.List[str] - - -Manifest = TypedDict( - 'Manifest', - { - 'package': Required[Package], - 'badges': T.Dict[str, Badge], - 'dependencies': T.Dict[str, DependencyV], - 'dev-dependencies': T.Dict[str, DependencyV], - 'build-dependencies': T.Dict[str, DependencyV], - 'lib': LibTarget, - 'bin': T.List[BuildTarget], - 'test': T.List[BuildTarget], - 'bench': T.List[BuildTarget], - 'example': T.List[BuildTarget], - 'features': T.Dict[str, T.List[str]], - 'target': T.Dict[str, Target], - 'workspace': Workspace, - - # TODO: patch? - # TODO: replace? - }, - total=False, -) -"""The Cargo Manifest format.""" + @classmethod + def from_raw(cls, raw: raw.BuildTarget) -> Self: + if 'path' not in raw: + raw['path'] = os.path.join('bin', raw['name'] + '.rs') + return super().from_raw(raw) -class VirtualManifest(TypedDict): +@dataclasses.dataclass +class Test(BuildTarget['raw.BuildTarget']): - """The Representation of a virtual manifest. + """Representation of a Cargo Test Entry.""" - Cargo allows a root manifest that contains only a workspace, this is called - a virtual manifest. This doesn't really map 1:1 with any meson concept, - except perhaps the proposed "meta project". + bench: bool = True + crate_type: T.List[CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['bin']) + + @classmethod + def from_raw(cls, raw: raw.BuildTarget) -> Self: + if 'path' not in raw: + raw['path'] = os.path.join('tests', raw['name'] + '.rs') + return super().from_raw(raw) + +@dataclasses.dataclass +class Benchmark(BuildTarget['raw.BuildTarget']): + + """Representation of a Cargo Benchmark Entry.""" + + test: bool = True + crate_type: T.List[CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['bin']) + + @classmethod + def from_raw(cls, raw: raw.BuildTarget) -> Self: + if 'path' not in raw: + raw['path'] = os.path.join('benches', raw['name'] + '.rs') + return super().from_raw(raw) + + +@dataclasses.dataclass +class Example(BuildTarget['raw.BuildTarget']): + + """Representation of a Cargo Example Entry.""" + + crate_type: T.List[CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['bin']) + + @classmethod + def from_raw(cls, raw: raw.BuildTarget) -> Self: + if 'path' not in raw: + raw['path'] = os.path.join('examples', raw['name'] + '.rs') + return super().from_raw(raw) + + +@dataclasses.dataclass +class Manifest: + + """Cargo Manifest definition. + + Most of these values map up to the Cargo Manifest, but with default values + if not provided. + + Cargo subprojects can contain what Meson wants to treat as multiple, + interdependent, subprojects. + + :param path: the path within the cargo subproject. """ - workspace: Workspace + package: Package + dependencies: T.Dict[str, Dependency] = dataclasses.field(default_factory=dict) + dev_dependencies: T.Dict[str, Dependency] = dataclasses.field(default_factory=dict) + build_dependencies: T.Dict[str, Dependency] = dataclasses.field(default_factory=dict) + lib: T.Optional[Library] = None + bin: T.List[Binary] = dataclasses.field(default_factory=list) + test: T.List[Test] = dataclasses.field(default_factory=list) + bench: T.List[Benchmark] = dataclasses.field(default_factory=list) + example: T.List[Example] = dataclasses.field(default_factory=list) + features: T.Dict[str, T.List[str]] = dataclasses.field(default_factory=dict) + target: T.Dict[str, T.Dict[str, Dependency]] = dataclasses.field(default_factory=dict) + + path: str = '' + + def __post_init__(self) -> None: + self.features.setdefault('default', []) + + @lazy_property + def system_dependencies(self) -> T.Dict[str, SystemDependency]: + return {k: SystemDependency.from_raw(k, v) for k, v in self.package.metadata.get('system-deps', {}).items()} + + @classmethod + def from_raw(cls, raw: raw.Manifest, path: str = '', workspace: T.Optional[Workspace] = None, member_path: str = '') -> Self: + # Libs are always auto-discovered and there's no other way to handle them, + # which is unfortunate for reproducability + pkg = Package.from_raw(raw['package'], workspace) + if pkg.autolib and 'lib' not in raw and \ + os.path.exists(os.path.join(path, 'src/lib.rs')): + raw['lib'] = {} + fixed = _raw_to_dataclass(raw, cls, f'Cargo.toml package {raw["package"]["name"]}', + package=lambda x: pkg, + dependencies=lambda x: {k: Dependency.from_raw(k, v, member_path, workspace) for k, v in x.items()}, + dev_dependencies=lambda x: {k: Dependency.from_raw(k, v, member_path, workspace) for k, v in x.items()}, + build_dependencies=lambda x: {k: Dependency.from_raw(k, v, member_path, workspace) for k, v in x.items()}, + lib=lambda x: Library.from_raw(x, raw['package']['name']), + bin=lambda x: [Binary.from_raw(b) for b in x], + test=lambda x: [Test.from_raw(b) for b in x], + bench=lambda x: [Benchmark.from_raw(b) for b in x], + example=lambda x: [Example.from_raw(b) for b in x], + target=lambda x: {k: {k2: Dependency.from_raw(k2, v2, member_path, workspace) for k2, v2 in v.get('dependencies', {}).items()} + for k, v in x.items()}) + fixed.path = path + return fixed + + +@dataclasses.dataclass +class Workspace: + + """Cargo Workspace definition. + """ + + resolver: str = dataclasses.field(default_factory=lambda: '2') + members: T.List[str] = dataclasses.field(default_factory=list) + exclude: T.List[str] = dataclasses.field(default_factory=list) + default_members: T.List[str] = dataclasses.field(default_factory=list) + + # inheritable settings are kept in raw format, for use with _inherit_from_workspace + package: T.Optional[raw.Package] = None + dependencies: T.Dict[str, raw.Dependency] = dataclasses.field(default_factory=dict) + lints: T.Dict[str, T.Any] = dataclasses.field(default_factory=dict) + metadata: T.Dict[str, T.Any] = dataclasses.field(default_factory=dict) -class CargoLockPackage(TypedDict, total=False): + # A workspace can also have a root package. + root_package: T.Optional[Manifest] = dataclasses.field(init=False) + + @classmethod + def from_raw(cls, raw: raw.VirtualManifest) -> Workspace: + ws_raw = raw['workspace'] + fixed = _raw_to_dataclass(ws_raw, cls, 'Workspace') + return fixed + + +@dataclasses.dataclass +class CargoLockPackage: """A description of a package in the Cargo.lock file format.""" name: str version: str - source: str - checksum: str + source: T.Optional[str] = None + checksum: T.Optional[str] = None + dependencies: T.List[str] = dataclasses.field(default_factory=list) + @classmethod + def from_raw(cls, raw: raw.CargoLockPackage) -> CargoLockPackage: + return _raw_to_dataclass(raw, cls, 'Cargo.lock package') -class CargoLock(TypedDict, total=False): + +@dataclasses.dataclass +class CargoLock: """A description of the Cargo.lock file format.""" - version: str - package: T.List[CargoLockPackage] - metadata: T.Dict[str, str] + version: int = 1 + package: T.List[CargoLockPackage] = dataclasses.field(default_factory=list) + metadata: T.Dict[str, str] = dataclasses.field(default_factory=dict) + + @classmethod + def from_raw(cls, raw: raw.CargoLock) -> CargoLock: + return _raw_to_dataclass(raw, cls, 'Cargo.lock', + package=lambda x: [CargoLockPackage.from_raw(p) for p in x]) diff --git a/mesonbuild/cargo/raw.py b/mesonbuild/cargo/raw.py new file mode 100644 index 0000000..67dd58a --- /dev/null +++ b/mesonbuild/cargo/raw.py @@ -0,0 +1,192 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright © 2022-2024 Intel Corporation + +"""Type definitions for cargo manifest files.""" + +from __future__ import annotations +import typing as T + +from typing_extensions import Literal, TypedDict, Required + +EDITION = Literal['2015', '2018', '2021'] +CRATE_TYPE = Literal['bin', 'lib', 'dylib', 'staticlib', 'cdylib', 'rlib', 'proc-macro'] + + +class FromWorkspace(TypedDict): + + """An entry or section that is copied from the workspace.""" + + workspace: bool + + +Package = TypedDict( + 'Package', + { + 'name': Required[str], + 'version': Required[T.Union[FromWorkspace, str]], + 'authors': T.Union[FromWorkspace, T.List[str]], + 'edition': T.Union[FromWorkspace, EDITION], + 'rust-version': T.Union[FromWorkspace, str], + 'description': T.Union[FromWorkspace, str], + 'readme': T.Union[FromWorkspace, str], + 'license': T.Union[FromWorkspace, str], + 'license-file': T.Union[FromWorkspace, str], + 'keywords': T.Union[FromWorkspace, T.List[str]], + 'categories': T.Union[FromWorkspace, T.List[str]], + 'homepage': T.Union[FromWorkspace, str], + 'repository': T.Union[FromWorkspace, str], + 'documentation': T.Union[FromWorkspace, str], + 'workspace': str, + 'build': str, + 'links': str, + 'include': T.Union[FromWorkspace, T.List[str]], + 'exclude': T.Union[FromWorkspace, T.List[str]], + 'publish': T.Union[FromWorkspace, bool], + 'metadata': T.Dict[str, T.Dict[str, str]], + 'default-run': str, + 'autolib': bool, + 'autobins': bool, + 'autoexamples': bool, + 'autotests': bool, + 'autobenches': bool, + }, + total=False, +) +"""A description of the Package Dictionary.""" + +class Badge(TypedDict): + + """An entry in the badge section.""" + + status: Literal['actively-developed', 'passively-developed', 'as-is', 'experimental', 'deprecated', 'none'] + + +Dependency = TypedDict( + 'Dependency', + { + 'version': str, + 'registry': str, + 'git': str, + 'branch': str, + 'rev': str, + 'path': str, + 'optional': bool, + 'package': str, + 'default-features': bool, + 'features': T.List[str], + }, + total=False, +) +"""An entry in the *dependencies sections.""" + + +DependencyV = T.Union[Dependency, str] +"""A Dependency entry, either a string or a Dependency Dict.""" + + +_BaseBuildTarget = TypedDict( + '_BaseBuildTarget', + { + 'path': str, + 'test': bool, + 'doctest': bool, + 'bench': bool, + 'doc': bool, + 'plugin': bool, + 'proc-macro': bool, + 'harness': bool, + 'edition': EDITION, + 'crate-type': T.List[CRATE_TYPE], + 'required-features': T.List[str], + }, + total=False, +) + + +class BuildTarget(_BaseBuildTarget, total=False): + + name: Required[str] + + +class LibTarget(_BaseBuildTarget, total=False): + + name: str + + +class Target(TypedDict): + + """Target entry in the Manifest File.""" + + dependencies: T.Dict[str, T.Union[FromWorkspace, DependencyV]] + + +class Workspace(TypedDict): + + """The representation of a workspace. + + In a vritual manifest the :attribute:`members` is always present, but in a + project manifest, an empty workspace may be provided, in which case the + workspace is implicitly filled in by values from the path based dependencies. + + the :attribute:`exclude` is always optional + """ + + members: T.List[str] + exclude: T.List[str] + package: Package + dependencies: T.Dict[str, DependencyV] + + +Manifest = TypedDict( + 'Manifest', + { + 'package': Required[Package], + 'badges': T.Dict[str, Badge], + 'dependencies': T.Dict[str, T.Union[FromWorkspace, DependencyV]], + 'dev-dependencies': T.Dict[str, T.Union[FromWorkspace, DependencyV]], + 'build-dependencies': T.Dict[str, T.Union[FromWorkspace, DependencyV]], + 'lib': LibTarget, + 'bin': T.List[BuildTarget], + 'test': T.List[BuildTarget], + 'bench': T.List[BuildTarget], + 'example': T.List[BuildTarget], + 'features': T.Dict[str, T.List[str]], + 'target': T.Dict[str, Target], + 'workspace': Workspace, + + # TODO: patch? + # TODO: replace? + }, + total=False, +) +"""The Cargo Manifest format.""" + + +class VirtualManifest(TypedDict, total=False): + + """The Representation of a virtual manifest. + + Cargo allows a root manifest that contains only a workspace, this is called + a virtual manifest. This doesn't really map 1:1 with any meson concept, + except perhaps the proposed "meta project". + """ + + workspace: Workspace + +class CargoLockPackage(TypedDict, total=False): + + """A description of a package in the Cargo.lock file format.""" + + name: str + version: str + source: str + checksum: str + + +class CargoLock(TypedDict, total=False): + + """A description of the Cargo.lock file format.""" + + version: int + package: T.List[CargoLockPackage] + metadata: T.Dict[str, str] diff --git a/mesonbuild/cargo/toml.py b/mesonbuild/cargo/toml.py new file mode 100644 index 0000000..601510e --- /dev/null +++ b/mesonbuild/cargo/toml.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import importlib +import shutil +import json +import typing as T + +from ..mesonlib import MesonException, Popen_safe +if T.TYPE_CHECKING: + from types import ModuleType + + +# tomllib is present in python 3.11, before that it is a pypi module called tomli, +# we try to import tomllib, then tomli, +tomllib: T.Optional[ModuleType] = None +toml2json: T.Optional[str] = None +for t in ['tomllib', 'tomli']: + try: + tomllib = importlib.import_module(t) + break + except ImportError: + pass +else: + # TODO: it would be better to use an Executable here, which could be looked + # up in the cross file or provided by a wrap. However, that will have to be + # passed in externally, since we don't have (and I don't think we should), + # have access to the `Environment` for that in this module. + toml2json = shutil.which('toml2json') + +class TomlImplementationMissing(MesonException): + pass + + +def load_toml(filename: str) -> T.Dict[str, object]: + if tomllib: + with open(filename, 'rb') as f: + raw = tomllib.load(f) + else: + if toml2json is None: + raise TomlImplementationMissing('Could not find an implementation of tomllib, nor toml2json') + + p, out, err = Popen_safe([toml2json, filename]) + if p.returncode != 0: + raise MesonException('toml2json failed to decode output\n', err) + + raw = json.loads(out) + + # tomllib.load() returns T.Dict[str, T.Any] but not other implementations. + return T.cast('T.Dict[str, object]', raw) diff --git a/mesonbuild/cargo/version.py b/mesonbuild/cargo/version.py index cde7a83..ce58945 100644 --- a/mesonbuild/cargo/version.py +++ b/mesonbuild/cargo/version.py @@ -7,6 +7,18 @@ from __future__ import annotations import typing as T +def api(version: str) -> str: + # x.y.z -> x + # 0.x.y -> 0.x + # 0.0.x -> 0 + vers = version.split('.') + if int(vers[0]) != 0: + return vers[0] + elif len(vers) >= 2 and int(vers[1]) != 0: + return f'0.{vers[1]}' + return '0' + + def convert(cargo_ver: str) -> T.List[str]: """Convert a Cargo compatible version into a Meson compatible one. @@ -15,6 +27,8 @@ def convert(cargo_ver: str) -> T.List[str]: """ # Cleanup, just for safety cargo_ver = cargo_ver.strip() + if not cargo_ver: + return [] cargo_vers = [c.strip() for c in cargo_ver.split(',')] out: T.List[str] = [] |