aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDylan Baker <dylan@pnwbakers.com>2022-02-25 15:35:16 -0800
committerDylan Baker <dylan@pnwbakers.com>2023-06-07 19:20:30 -0700
commitf02e26877de7a38fce672db9709666ed85706d43 (patch)
tree28fee57ea1d38d01165a70b521176647858e0ec1
parent4017dab4847da392f7eb1dcdc2cb07bd69eb7863 (diff)
downloadmeson-f02e26877de7a38fce672db9709666ed85706d43.zip
meson-f02e26877de7a38fce672db9709666ed85706d43.tar.gz
meson-f02e26877de7a38fce672db9709666ed85706d43.tar.bz2
cargo/interpreter: Implement an interpreter for Cargo TOML
This converts a Cargo TOML file into Meson AST Co-Authored-By: Thibault Saunier <tsaunier@igalia.com>
-rw-r--r--mesonbuild/cargo/interpreter.py451
1 files changed, 451 insertions, 0 deletions
diff --git a/mesonbuild/cargo/interpreter.py b/mesonbuild/cargo/interpreter.py
new file mode 100644
index 0000000..59e1a1f
--- /dev/null
+++ b/mesonbuild/cargo/interpreter.py
@@ -0,0 +1,451 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright © 2022-2023 Intel Corporation
+
+"""Interpreter for converting Cargo Toml definitions to Meson AST
+
+There are some notable limits here. We don't even try to convert something with
+a build.rs: there's so few limits on what Cargo allows a build.rs (basically
+none), and no good way for us to convert them. In that case, an actual meson
+port will be required.
+"""
+
+from __future__ import annotations
+import dataclasses
+import glob
+import importlib
+import itertools
+import json
+import os
+import shutil
+import typing as T
+
+from . import builder
+from . import version
+from .. import mparser
+from .._pathlib import Path
+from ..mesonlib import MesonException, Popen_safe
+
+if T.TYPE_CHECKING:
+ from types import ModuleType
+
+ from . import manifest
+ from ..environment import Environment
+
+# tomllib is present in python 3.11, before that it is a pypi module called tomli,
+# we try to import tomllib, then tomli,
+# TODO: add a fallback to toml2json?
+tomllib: T.Optional[ModuleType] = None
+toml2json: T.Optional[str] = None
+for t in ['tomllib', 'tomli']:
+ try:
+ tomllib = importlib.import_module(t)
+ break
+ except ImportError:
+ pass
+else:
+ # TODO: it would be better to use an Executable here, which could be looked
+ # up in the cross file or provided by a wrap. However, that will have to be
+ # passed in externally, since we don't have (and I don't think we should),
+ # have access to the `Environment` for that in this module.
+ toml2json = shutil.which('toml2json')
+
+
+def load_toml(filename: str) -> T.Dict[object, object]:
+ if tomllib:
+ with open(filename, 'rb') as f:
+ raw = tomllib.load(f)
+ else:
+ if toml2json is None:
+ raise MesonException('Could not find an implementation of tomllib, nor toml2json')
+
+ p, out, err = Popen_safe([toml2json, filename])
+ if p.returncode != 0:
+ raise MesonException('toml2json failed to decode output\n', err)
+
+ raw = json.loads(out)
+
+ if not isinstance(raw, dict):
+ raise MesonException("Cargo.toml isn't a dictionary? How did that happen?")
+
+ return raw
+
+
+def fixup_meson_varname(name: str) -> str:
+ """Fixup a meson variable name
+
+ :param name: The name to fix
+ :return: the fixed name
+ """
+ return name.replace('-', '_')
+
+# Pylance can figure out that these do not, in fact, overlap, but mypy can't
+@T.overload
+def _fixup_raw_mappings(d: manifest.BuildTarget) -> manifest.FixedBuildTarget: ... # type: ignore
+
+@T.overload
+def _fixup_raw_mappings(d: manifest.LibTarget) -> manifest.FixedLibTarget: ... # type: ignore
+
+@T.overload
+def _fixup_raw_mappings(d: manifest.Dependency) -> manifest.FixedDependency: ...
+
+def _fixup_raw_mappings(d: T.Union[manifest.BuildTarget, manifest.LibTarget, manifest.Dependency]
+ ) -> T.Union[manifest.FixedBuildTarget, manifest.FixedLibTarget,
+ manifest.FixedDependency]:
+ """Fixup raw cargo mappings to ones more suitable for python to consume.
+
+ This does the following:
+ * replaces any `-` with `_`, cargo likes the former, but python dicts make
+ keys with `-` in them awkward to work with
+ * Convert Dependndency versions from the cargo format to something meson
+ understands
+
+ :param d: The mapping to fix
+ :return: the fixed string
+ """
+ raw = {fixup_meson_varname(k): v for k, v in d.items()}
+ if 'version' in raw:
+ assert isinstance(raw['version'], str), 'for mypy'
+ raw['version'] = version.convert(raw['version'])
+ return T.cast('T.Union[manifest.FixedBuildTarget, manifest.FixedLibTarget, manifest.FixedDependency]', raw)
+
+
+@dataclasses.dataclass
+class Package:
+
+ """Representation of a Cargo Package entry, with defaults filled in."""
+
+ name: str
+ version: str
+ description: str
+ resolver: T.Optional[str] = None
+ authors: T.List[str] = dataclasses.field(default_factory=list)
+ edition: manifest.EDITION = '2015'
+ rust_version: T.Optional[str] = None
+ documentation: T.Optional[str] = None
+ readme: T.Optional[str] = None
+ homepage: T.Optional[str] = None
+ repository: T.Optional[str] = None
+ license: T.Optional[str] = None
+ license_file: T.Optional[str] = None
+ keywords: T.List[str] = dataclasses.field(default_factory=list)
+ categories: T.List[str] = dataclasses.field(default_factory=list)
+ workspace: T.Optional[str] = None
+ build: T.Optional[str] = None
+ links: T.Optional[str] = None
+ exclude: T.List[str] = dataclasses.field(default_factory=list)
+ include: T.List[str] = dataclasses.field(default_factory=list)
+ publish: bool = True
+ metadata: T.Dict[str, T.Dict[str, str]] = dataclasses.field(default_factory=dict)
+ default_run: T.Optional[str] = None
+ autobins: bool = True
+ autoexamples: bool = True
+ autotests: bool = True
+ autobenches: bool = True
+
+
+@dataclasses.dataclass
+class Dependency:
+
+ """Representation of a Cargo Dependency Entry."""
+
+ version: T.List[str]
+ registry: T.Optional[str] = None
+ git: T.Optional[str] = None
+ branch: T.Optional[str] = None
+ rev: T.Optional[str] = None
+ path: T.Optional[str] = None
+ optional: bool = False
+ package: T.Optional[str] = None
+ default_features: bool = False
+ features: T.List[str] = dataclasses.field(default_factory=list)
+
+ @classmethod
+ def from_raw(cls, raw: manifest.DependencyV) -> Dependency:
+ """Create a dependency from a raw cargo dictionary"""
+ if isinstance(raw, str):
+ return cls(version.convert(raw))
+ return cls(**_fixup_raw_mappings(raw))
+
+
+@dataclasses.dataclass
+class BuildTarget:
+
+ name: str
+ crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib'])
+ path: dataclasses.InitVar[T.Optional[str]] = None
+
+ # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-test-field
+ # True for lib, bin, test
+ test: bool = True
+
+ # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-doctest-field
+ # True for lib
+ doctest: bool = False
+
+ # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-bench-field
+ # True for lib, bin, benchmark
+ bench: bool = True
+
+ # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-doc-field
+ # True for libraries and binaries
+ doc: bool = False
+
+ harness: bool = True
+ edition: manifest.EDITION = '2015'
+ required_features: T.List[str] = dataclasses.field(default_factory=list)
+ plugin: bool = False
+
+
+@dataclasses.dataclass
+class Library(BuildTarget):
+
+ """Representation of a Cargo Library Entry."""
+
+ doctest: bool = True
+ doc: bool = True
+ proc_macro: bool = False
+ crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib'])
+ doc_scrape_examples: bool = True
+
+
+@dataclasses.dataclass
+class Binary(BuildTarget):
+
+ """Representation of a Cargo Bin Entry."""
+
+ doc: bool = True
+
+
+@dataclasses.dataclass
+class Test(BuildTarget):
+
+ """Representation of a Cargo Test Entry."""
+
+ bench: bool = True
+
+
+@dataclasses.dataclass
+class Benchmark(BuildTarget):
+
+ """Representation of a Cargo Benchmark Entry."""
+
+ test: bool = True
+
+
+@dataclasses.dataclass
+class Example(BuildTarget):
+
+ """Representation of a Cargo Example Entry."""
+
+ crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['bin'])
+
+
+@dataclasses.dataclass
+class Manifest:
+
+ """Cargo Manifest definition.
+
+ Most of these values map up to the Cargo Manifest, but with default values
+ if not provided.
+
+ Cargo subprojects can contain what Meson wants to treat as multiple,
+ interdependent, subprojects.
+
+ :param subdir: the subdirectory that this cargo project is in
+ :param path: the path within the cargo subproject.
+ """
+
+ package: Package
+ dependencies: T.Dict[str, Dependency]
+ dev_dependencies: T.Dict[str, Dependency]
+ build_dependencies: T.Dict[str, Dependency]
+ lib: Library
+ bin: T.List[Binary]
+ test: T.List[Test]
+ bench: T.List[Benchmark]
+ example: T.List[Example]
+ features: T.Dict[str, T.List[str]]
+ target: T.Dict[str, T.Dict[str, Dependency]]
+ subdir: str
+ path: str = ''
+
+
+def _create_project(package: Package, build: builder.Builder, env: Environment) -> mparser.FunctionNode:
+ """Create a function call
+
+ :param package: The Cargo package to generate from
+ :param filename: The full path to the file
+ :param meson_version: The generating meson version
+ :return: a FunctionNode
+ """
+ args: T.List[mparser.BaseNode] = []
+ args.extend([
+ build.string(package.name),
+ build.string('rust'),
+ ])
+ kwargs: T.Dict[str, mparser.BaseNode] = {
+ 'version': build.string(package.version),
+ # Always assume that the generated meson is using the latest features
+ # This will warn when when we generate deprecated code, which is helpful
+ # for the upkeep of the module
+ 'meson_version': build.string(f'>= {env.coredata.version}'),
+ 'default_options': build.array([build.string(f'rust_std={package.edition}')]),
+ }
+ if package.license:
+ kwargs['license'] = build.string(package.license)
+ elif package.license_file:
+ kwargs['license_files'] = build.string(package.license_file)
+
+ return build.function('project', args, kwargs)
+
+
+def _convert_manifest(raw_manifest: manifest.Manifest, subdir: str, path: str = '') -> Manifest:
+ # This cast is a bit of a hack to deal with proc-macro
+ lib = _fixup_raw_mappings(raw_manifest.get('lib', {}))
+
+ # We need to set the name field if it's not set manually,
+ # including if other fields are set in the lib section
+ lib.setdefault('name', raw_manifest['package']['name'])
+
+ pkg = T.cast('manifest.FixedPackage',
+ {fixup_meson_varname(k): v for k, v in raw_manifest['package'].items()})
+
+ return Manifest(
+ Package(**pkg),
+ {k: Dependency.from_raw(v) for k, v in raw_manifest.get('dependencies', {}).items()},
+ {k: Dependency.from_raw(v) for k, v in raw_manifest.get('dev-dependencies', {}).items()},
+ {k: Dependency.from_raw(v) for k, v in raw_manifest.get('build-dependencies', {}).items()},
+ Library(**lib),
+ [Binary(**_fixup_raw_mappings(b)) for b in raw_manifest.get('bin', {})],
+ [Test(**_fixup_raw_mappings(b)) for b in raw_manifest.get('test', {})],
+ [Benchmark(**_fixup_raw_mappings(b)) for b in raw_manifest.get('bench', {})],
+ [Example(**_fixup_raw_mappings(b)) for b in raw_manifest.get('example', {})],
+ raw_manifest.get('features', {}),
+ {k: {k2: Dependency.from_raw(v2) for k2, v2 in v['dependencies'].items()}
+ for k, v in raw_manifest.get('target', {}).items()},
+ subdir,
+ path,
+ )
+
+
+def _load_manifests(subdir: str) -> T.Dict[str, Manifest]:
+ filename = os.path.join(subdir, 'Cargo.toml')
+ raw = load_toml(filename)
+
+ manifests: T.Dict[str, Manifest] = {}
+
+ raw_manifest: T.Union[manifest.Manifest, manifest.VirtualManifest]
+ if 'package' in raw:
+ raw_manifest = T.cast('manifest.Manifest', raw)
+ manifest_ = _convert_manifest(raw_manifest, subdir)
+ manifests[manifest_.package.name] = manifest_
+ else:
+ raw_manifest = T.cast('manifest.VirtualManifest', raw)
+
+ if 'workspace' in raw_manifest:
+ # XXX: need to verify that python glob and cargo globbing are the
+ # same and probably write a glob implementation. Blarg
+
+ # We need to chdir here to make the glob work correctly
+ pwd = os.getcwd()
+ os.chdir(subdir)
+ members: T.Iterable[str]
+ try:
+ members = itertools.chain.from_iterable(
+ glob.glob(m) for m in raw_manifest['workspace']['members'])
+ finally:
+ os.chdir(pwd)
+ if 'exclude' in raw_manifest['workspace']:
+ members = (x for x in members if x not in raw_manifest['workspace']['exclude'])
+
+ for m in members:
+ filename = os.path.join(subdir, m, 'Cargo.toml')
+ raw = load_toml(filename)
+
+ raw_manifest = T.cast('manifest.Manifest', raw)
+ man = _convert_manifest(raw_manifest, subdir, m)
+ manifests[man.package.name] = man
+
+ return manifests
+
+
+def load_all_manifests(subproject_dir: str) -> T.Dict[str, Manifest]:
+ """Find all cargo subprojects, and load them
+
+ :param subproject_dir: Directory to look for subprojects in
+ :return: A dictionary of rust project names to Manifests
+ """
+ manifests: T.Dict[str, Manifest] = {}
+ for p in Path(subproject_dir).iterdir():
+ if p.is_dir() and (p / 'Cargo.toml').exists():
+ manifests.update(_load_manifests(str(p)))
+ return manifests
+
+
+def _create_lib(cargo: Manifest, build: builder.Builder) -> T.List[mparser.BaseNode]:
+ kw: T.Dict[str, mparser.BaseNode] = {}
+ if cargo.dependencies:
+ ids = [build.identifier(f'dep_{n}') for n in cargo.dependencies]
+ kw['dependencies'] = build.array(
+ [build.method('get_variable', i, [build.string('dep')]) for i in ids])
+
+ # FIXME: currently assuming that an rlib is being generated, which is
+ # the most common.
+ return [
+ build.assign(
+ build.function(
+ 'static_library',
+ [
+ build.string(fixup_meson_varname(cargo.package.name)),
+ build.string(os.path.join('src', 'lib.rs')),
+ ],
+ kw,
+ ),
+ 'lib'
+ ),
+
+ build.assign(
+ build.function(
+ 'declare_dependency',
+ kw={'link_with': build.identifier('lib'), **kw},
+ ),
+ 'dep'
+ )
+ ]
+
+
+def interpret(cargo: Manifest, env: Environment) -> mparser.CodeBlockNode:
+ filename = os.path.join(cargo.subdir, cargo.path, 'Cargo.toml')
+ build = builder.Builder(filename)
+
+ ast: T.List[mparser.BaseNode] = [
+ _create_project(cargo.package, build, env),
+ build.assign(build.function('import', [build.string('rust')]), 'rust'),
+ ]
+
+ if cargo.dependencies:
+ for name, dep in cargo.dependencies.items():
+ kw = {
+ 'version': build.array([build.string(s) for s in dep.version]),
+ }
+ ast.extend([
+ build.assign(
+ build.method(
+ 'cargo',
+ build.identifier('rust'),
+ [build.string(name)],
+ kw,
+ ),
+ f'dep_{fixup_meson_varname(name)}',
+ ),
+ ])
+
+ # Libs are always auto-discovered and there's no other way to handle them,
+ # which is unfortunate for reproducability
+ if os.path.exists(os.path.join(env.source_dir, cargo.subdir, cargo.path, 'src', 'lib.rs')):
+ ast.extend(_create_lib(cargo, build))
+
+ # XXX: make this not awful
+ block = builder.block(filename)
+ block.lines = ast
+ return block