From f9445300b3015308fd6a3e0305cf8e5b7f002211 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 18 Mar 2022 10:28:38 -0700 Subject: structured_sources: fix subdir handling We currently don't handle subdirectories correctly in structured_sources, which is problematic. To make this easier to handle correctly, I've simply changed `structured_sources` to only use Files and not strings as an implementation detail. --- mesonbuild/backend/ninjabackend.py | 8 ++------ mesonbuild/build.py | 14 +++++--------- mesonbuild/interpreter/interpreter.py | 6 +++++- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index fdd5cd3..691f844 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -1669,9 +1669,7 @@ class NinjaBackend(backends.Backend): root = Path(self.get_target_private_dir(target)) / 'structured' for path, files in target.structured_sources.sources.items(): for file in files: - if isinstance(file, (str, File)): - if isinstance(file, str): - file = File.from_absolute_file(file) + if isinstance(file, File): out = root / path / Path(file.fname).name orderdeps.append(str(out)) self._generate_copy_target(file, out) @@ -1707,13 +1705,11 @@ class NinjaBackend(backends.Backend): main_rust_file = None if target.structured_sources: - if target.structured_sources.needs_copy(target): + if target.structured_sources.needs_copy(): _ods, main_rust_file = self.__generate_compile_structure(target) orderdeps.extend(_ods) else: g = target.structured_sources.first_file() - if isinstance(g, str): - g = File.from_source_file(self.environment.source_dir, target.subdir, g) if isinstance(g, File): main_rust_file = g.rel_to_builddir(self.build_to_src) diff --git a/mesonbuild/build.py b/mesonbuild/build.py index fbc1618..4f569a6 100644 --- a/mesonbuild/build.py +++ b/mesonbuild/build.py @@ -463,7 +463,7 @@ class StructuredSources(HoldableObject): represent the required filesystem layout. """ - sources: T.DefaultDict[str, T.List[T.Union[str, File, CustomTarget, CustomTargetIndex, GeneratedList]]] = field( + sources: T.DefaultDict[str, T.List[T.Union[File, CustomTarget, CustomTargetIndex, GeneratedList]]] = field( default_factory=lambda: defaultdict(list)) def __add__(self, other: StructuredSources) -> StructuredSources: @@ -475,30 +475,26 @@ class StructuredSources(HoldableObject): def __bool__(self) -> bool: return bool(self.sources) - def first_file(self) -> T.Union[str, File, CustomTarget, CustomTargetIndex, GeneratedList]: + def first_file(self) -> T.Union[File, CustomTarget, CustomTargetIndex, GeneratedList]: """Get the first source in the root :return: The first source in the root """ return self.sources[''][0] - def as_list(self) -> T.List[T.Union[str, File, CustomTarget, CustomTargetIndex, GeneratedList]]: + def as_list(self) -> T.List[T.Union[File, CustomTarget, CustomTargetIndex, GeneratedList]]: return list(itertools.chain.from_iterable(self.sources.values())) - def needs_copy(self, target: BuildTarget) -> bool: + def needs_copy(self) -> bool: """Do we need to create a structure in the build directory. This allows us to avoid making copies if the structures exists in the source dir. Which could happen in situations where a generated source only exists in some configurations """ - p = pathlib.Path(target.subdir) for files in self.sources.values(): for f in files: - if isinstance(f, str): - if not (target.environment.source_dir / p / f).exists(): - return True - elif isinstance(f, File): + if isinstance(f, File): if f.is_built: return True else: diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py index b5e990a..ecbfd7a 100644 --- a/mesonbuild/interpreter/interpreter.py +++ b/mesonbuild/interpreter/interpreter.py @@ -2122,11 +2122,13 @@ external dependencies (including libraries) must go to "dependencies".''') args: T.Tuple[object, T.Optional[T.Dict[str, object]]], kwargs: 'TYPE_kwargs') -> build.StructuredSources: valid_types = (str, mesonlib.File, build.GeneratedList, build.CustomTarget, build.CustomTargetIndex, build.GeneratedList) - sources: T.Dict[str, T.List[T.Union['mesonlib.FileOrString', 'build.GeneratedTypes']]] = collections.defaultdict(list) + sources: T.Dict[str, T.List[T.Union[mesonlib.File, 'build.GeneratedTypes']]] = collections.defaultdict(list) for arg in mesonlib.listify(args[0]): if not isinstance(arg, valid_types): raise InvalidArguments(f'structured_sources: type "{type(arg)}" is not valid') + if isinstance(arg, str): + arg = mesonlib.File.from_source_file(self.environment.source_dir, self.subdir, arg) sources[''].append(arg) if args[1]: if '' in args[1]: @@ -2135,6 +2137,8 @@ external dependencies (including libraries) must go to "dependencies".''') for arg in mesonlib.listify(v): if not isinstance(arg, valid_types): raise InvalidArguments(f'structured_sources: type "{type(arg)}" is not valid') + if isinstance(arg, str): + arg = mesonlib.File.from_source_file(self.environment.source_dir, self.subdir, arg) sources[k].append(arg) return build.StructuredSources(sources) -- cgit v1.1 From 9b83fc5ece8932661ea0255e5362417fef117b15 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 18 Mar 2022 19:46:47 -0700 Subject: ninja: fix handling of rust structured_sources in rare case In the even that all of the inputs are generated, and they're all generated into the same folder, and there are no subfolders, we would fail to correctly handle all of the files after the main file. Let's fix that.t --- mesonbuild/backend/ninjabackend.py | 15 +++++++++++---- .../rust/18 structured sources/main-gen-copy.rs | 5 +++++ test cases/rust/18 structured sources/meson.build | 19 +++++++++++++++++++ test cases/rust/18 structured sources/priv.rs | 3 +++ .../rust/18 structured sources/src2/meson.build | 4 ++++ 5 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 test cases/rust/18 structured sources/main-gen-copy.rs create mode 100644 test cases/rust/18 structured sources/priv.rs create mode 100644 test cases/rust/18 structured sources/src2/meson.build diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 691f844..614e864 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -1709,17 +1709,24 @@ class NinjaBackend(backends.Backend): _ods, main_rust_file = self.__generate_compile_structure(target) orderdeps.extend(_ods) else: + # The only way to get here is to have only files in the "root" + # positional argument, which are all generated into the same + # directory g = target.structured_sources.first_file() if isinstance(g, File): main_rust_file = g.rel_to_builddir(self.build_to_src) elif isinstance(g, GeneratedList): - main_rust_file = os.path.join(self.get_target_private_dir(target), i) + main_rust_file = os.path.join(self.get_target_private_dir(target), g.get_outputs()[0]) else: - main_rust_file = os.path.join(g.get_subdir(), i) - orderdeps.extend([os.path.join(self.build_to_src, target.subdir, s) - for s in target.structured_sources.as_list()]) + main_rust_file = os.path.join(g.get_subdir(), g.get_outputs()[0]) + for f in target.structured_sources.as_list(): + if isinstance(f, File): + orderdeps.append(f.rel_to_builddir(self.build_to_src)) + else: + orderdeps.extend([os.path.join(self.build_to_src, f.subdir, s) + for s in f.get_outputs()]) for i in target.get_sources(): if not rustc.can_compile(i): diff --git a/test cases/rust/18 structured sources/main-gen-copy.rs b/test cases/rust/18 structured sources/main-gen-copy.rs new file mode 100644 index 0000000..db66a51 --- /dev/null +++ b/test cases/rust/18 structured sources/main-gen-copy.rs @@ -0,0 +1,5 @@ +include!(r#"@dir@/include.rs"#); + +pub fn main() { + priv_func(); +} diff --git a/test cases/rust/18 structured sources/meson.build b/test cases/rust/18 structured sources/meson.build index 8fa0443..d84e83f 100644 --- a/test cases/rust/18 structured sources/meson.build +++ b/test cases/rust/18 structured sources/meson.build @@ -37,3 +37,22 @@ executable( ) test('no-copy', find_program('no_copy_test.py'), args : meson.current_build_dir()) + +subdir('src2') + +executable('copy-no-gen', srcs2) + +m_src = configure_file( + input : 'main-gen-copy.rs', + output : 'main-gen-copy.rs', + configuration : {'dir' : meson.current_build_dir().replace('\\', '/')}, +) + +m_src2 = configure_file( + input : 'priv.rs', + output : 'include.rs', + copy : true +) + +executable('gen-no-copy', structured_sources([m_src, m_src2])) + diff --git a/test cases/rust/18 structured sources/priv.rs b/test cases/rust/18 structured sources/priv.rs new file mode 100644 index 0000000..aad196b --- /dev/null +++ b/test cases/rust/18 structured sources/priv.rs @@ -0,0 +1,3 @@ +fn priv_func() { + std::process::exit(0); +} diff --git a/test cases/rust/18 structured sources/src2/meson.build b/test cases/rust/18 structured sources/src2/meson.build new file mode 100644 index 0000000..b4844d2 --- /dev/null +++ b/test cases/rust/18 structured sources/src2/meson.build @@ -0,0 +1,4 @@ +srcs2 = structured_sources( + ['main-unique.rs'], + {'foo': 'foo/mod.rs'}, +) -- cgit v1.1 From cf4d02d82a2e9d455a3d40ced6f7a07d6467fd98 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Mon, 21 Mar 2022 11:05:51 -0700 Subject: tests/rust: renumber structred sources test --- test cases/rust/18 structured sources/gen.py | 20 -------- .../rust/18 structured sources/main-gen-copy.rs | 5 -- test cases/rust/18 structured sources/meson.build | 58 ---------------------- .../rust/18 structured sources/no_copy_test.py | 18 ------- test cases/rust/18 structured sources/priv.rs | 3 -- .../rust/18 structured sources/src/foo.rs.in | 4 -- test cases/rust/18 structured sources/src/main.rs | 5 -- .../rust/18 structured sources/src2/foo/mod.rs | 4 -- .../rust/18 structured sources/src2/main-unique.rs | 5 -- .../rust/18 structured sources/src2/meson.build | 4 -- test cases/rust/19 structured sources/gen.py | 20 ++++++++ .../rust/19 structured sources/main-gen-copy.rs | 5 ++ test cases/rust/19 structured sources/meson.build | 58 ++++++++++++++++++++++ .../rust/19 structured sources/no_copy_test.py | 18 +++++++ test cases/rust/19 structured sources/priv.rs | 3 ++ .../rust/19 structured sources/src/foo.rs.in | 4 ++ test cases/rust/19 structured sources/src/main.rs | 5 ++ .../rust/19 structured sources/src2/foo/mod.rs | 4 ++ .../rust/19 structured sources/src2/main-unique.rs | 5 ++ .../rust/19 structured sources/src2/meson.build | 4 ++ 20 files changed, 126 insertions(+), 126 deletions(-) delete mode 100755 test cases/rust/18 structured sources/gen.py delete mode 100644 test cases/rust/18 structured sources/main-gen-copy.rs delete mode 100644 test cases/rust/18 structured sources/meson.build delete mode 100755 test cases/rust/18 structured sources/no_copy_test.py delete mode 100644 test cases/rust/18 structured sources/priv.rs delete mode 100644 test cases/rust/18 structured sources/src/foo.rs.in delete mode 100644 test cases/rust/18 structured sources/src/main.rs delete mode 100644 test cases/rust/18 structured sources/src2/foo/mod.rs delete mode 100644 test cases/rust/18 structured sources/src2/main-unique.rs delete mode 100644 test cases/rust/18 structured sources/src2/meson.build create mode 100755 test cases/rust/19 structured sources/gen.py create mode 100644 test cases/rust/19 structured sources/main-gen-copy.rs create mode 100644 test cases/rust/19 structured sources/meson.build create mode 100755 test cases/rust/19 structured sources/no_copy_test.py create mode 100644 test cases/rust/19 structured sources/priv.rs create mode 100644 test cases/rust/19 structured sources/src/foo.rs.in create mode 100644 test cases/rust/19 structured sources/src/main.rs create mode 100644 test cases/rust/19 structured sources/src2/foo/mod.rs create mode 100644 test cases/rust/19 structured sources/src2/main-unique.rs create mode 100644 test cases/rust/19 structured sources/src2/meson.build diff --git a/test cases/rust/18 structured sources/gen.py b/test cases/rust/18 structured sources/gen.py deleted file mode 100755 index 16e5c04..0000000 --- a/test cases/rust/18 structured sources/gen.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import textwrap - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('output') - args = parser.parse_args() - - with open(args.output, 'w') as f: - f.write(textwrap.dedent('''\ - pub fn bar() -> () { - println!("Hello, World!"); - }''')) - - -if __name__ == "__main__": - main() diff --git a/test cases/rust/18 structured sources/main-gen-copy.rs b/test cases/rust/18 structured sources/main-gen-copy.rs deleted file mode 100644 index db66a51..0000000 --- a/test cases/rust/18 structured sources/main-gen-copy.rs +++ /dev/null @@ -1,5 +0,0 @@ -include!(r#"@dir@/include.rs"#); - -pub fn main() { - priv_func(); -} diff --git a/test cases/rust/18 structured sources/meson.build b/test cases/rust/18 structured sources/meson.build deleted file mode 100644 index d84e83f..0000000 --- a/test cases/rust/18 structured sources/meson.build +++ /dev/null @@ -1,58 +0,0 @@ -project('structured input', 'rust') - -foo_mod_rs = configure_file( - input : 'src/foo.rs.in', - output : 'mod.rs', - configuration : {'message' : 'Hello, World!'}, -) - -conf_file = executable( - 'main_conf_file', - structured_sources( - 'src/main.rs', - {'foo' : [foo_mod_rs]}, - ), -) - -ct = custom_target( - 'foo.rs', - output : 'foo.rs', - command : ['gen.py', '@OUTPUT@'], -) - -target = executable( - 'main_custom_target', - structured_sources( - ['src/main.rs', ct], - ), -) - -# Should not be coppied -executable( - 'no_copy_target', - structured_sources( - ['src2/main-unique.rs'], - {'foo': 'src2/foo/mod.rs'}, - ), -) - -test('no-copy', find_program('no_copy_test.py'), args : meson.current_build_dir()) - -subdir('src2') - -executable('copy-no-gen', srcs2) - -m_src = configure_file( - input : 'main-gen-copy.rs', - output : 'main-gen-copy.rs', - configuration : {'dir' : meson.current_build_dir().replace('\\', '/')}, -) - -m_src2 = configure_file( - input : 'priv.rs', - output : 'include.rs', - copy : true -) - -executable('gen-no-copy', structured_sources([m_src, m_src2])) - diff --git a/test cases/rust/18 structured sources/no_copy_test.py b/test cases/rust/18 structured sources/no_copy_test.py deleted file mode 100755 index 91506b2..0000000 --- a/test cases/rust/18 structured sources/no_copy_test.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('builddir') - args = parser.parse_args() - - for _, _, files in os.walk(args.builddir): - if 'main-unique.rs' in files: - exit(1) - - -if __name__ == "__main__": - main() diff --git a/test cases/rust/18 structured sources/priv.rs b/test cases/rust/18 structured sources/priv.rs deleted file mode 100644 index aad196b..0000000 --- a/test cases/rust/18 structured sources/priv.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn priv_func() { - std::process::exit(0); -} diff --git a/test cases/rust/18 structured sources/src/foo.rs.in b/test cases/rust/18 structured sources/src/foo.rs.in deleted file mode 100644 index 4f3fc42..0000000 --- a/test cases/rust/18 structured sources/src/foo.rs.in +++ /dev/null @@ -1,4 +0,0 @@ - -pub fn bar() -> () { - println!("@message@"); -} diff --git a/test cases/rust/18 structured sources/src/main.rs b/test cases/rust/18 structured sources/src/main.rs deleted file mode 100644 index 3ffeee2..0000000 --- a/test cases/rust/18 structured sources/src/main.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod foo; - -fn main() { - foo::bar(); -} diff --git a/test cases/rust/18 structured sources/src2/foo/mod.rs b/test cases/rust/18 structured sources/src2/foo/mod.rs deleted file mode 100644 index 9463d95..0000000 --- a/test cases/rust/18 structured sources/src2/foo/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ - -pub fn bar() -> () { - println!("Hello, World!"); -} diff --git a/test cases/rust/18 structured sources/src2/main-unique.rs b/test cases/rust/18 structured sources/src2/main-unique.rs deleted file mode 100644 index 3ffeee2..0000000 --- a/test cases/rust/18 structured sources/src2/main-unique.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod foo; - -fn main() { - foo::bar(); -} diff --git a/test cases/rust/18 structured sources/src2/meson.build b/test cases/rust/18 structured sources/src2/meson.build deleted file mode 100644 index b4844d2..0000000 --- a/test cases/rust/18 structured sources/src2/meson.build +++ /dev/null @@ -1,4 +0,0 @@ -srcs2 = structured_sources( - ['main-unique.rs'], - {'foo': 'foo/mod.rs'}, -) diff --git a/test cases/rust/19 structured sources/gen.py b/test cases/rust/19 structured sources/gen.py new file mode 100755 index 0000000..16e5c04 --- /dev/null +++ b/test cases/rust/19 structured sources/gen.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +import argparse +import textwrap + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('output') + args = parser.parse_args() + + with open(args.output, 'w') as f: + f.write(textwrap.dedent('''\ + pub fn bar() -> () { + println!("Hello, World!"); + }''')) + + +if __name__ == "__main__": + main() diff --git a/test cases/rust/19 structured sources/main-gen-copy.rs b/test cases/rust/19 structured sources/main-gen-copy.rs new file mode 100644 index 0000000..db66a51 --- /dev/null +++ b/test cases/rust/19 structured sources/main-gen-copy.rs @@ -0,0 +1,5 @@ +include!(r#"@dir@/include.rs"#); + +pub fn main() { + priv_func(); +} diff --git a/test cases/rust/19 structured sources/meson.build b/test cases/rust/19 structured sources/meson.build new file mode 100644 index 0000000..d84e83f --- /dev/null +++ b/test cases/rust/19 structured sources/meson.build @@ -0,0 +1,58 @@ +project('structured input', 'rust') + +foo_mod_rs = configure_file( + input : 'src/foo.rs.in', + output : 'mod.rs', + configuration : {'message' : 'Hello, World!'}, +) + +conf_file = executable( + 'main_conf_file', + structured_sources( + 'src/main.rs', + {'foo' : [foo_mod_rs]}, + ), +) + +ct = custom_target( + 'foo.rs', + output : 'foo.rs', + command : ['gen.py', '@OUTPUT@'], +) + +target = executable( + 'main_custom_target', + structured_sources( + ['src/main.rs', ct], + ), +) + +# Should not be coppied +executable( + 'no_copy_target', + structured_sources( + ['src2/main-unique.rs'], + {'foo': 'src2/foo/mod.rs'}, + ), +) + +test('no-copy', find_program('no_copy_test.py'), args : meson.current_build_dir()) + +subdir('src2') + +executable('copy-no-gen', srcs2) + +m_src = configure_file( + input : 'main-gen-copy.rs', + output : 'main-gen-copy.rs', + configuration : {'dir' : meson.current_build_dir().replace('\\', '/')}, +) + +m_src2 = configure_file( + input : 'priv.rs', + output : 'include.rs', + copy : true +) + +executable('gen-no-copy', structured_sources([m_src, m_src2])) + diff --git a/test cases/rust/19 structured sources/no_copy_test.py b/test cases/rust/19 structured sources/no_copy_test.py new file mode 100755 index 0000000..91506b2 --- /dev/null +++ b/test cases/rust/19 structured sources/no_copy_test.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import argparse +import os + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('builddir') + args = parser.parse_args() + + for _, _, files in os.walk(args.builddir): + if 'main-unique.rs' in files: + exit(1) + + +if __name__ == "__main__": + main() diff --git a/test cases/rust/19 structured sources/priv.rs b/test cases/rust/19 structured sources/priv.rs new file mode 100644 index 0000000..aad196b --- /dev/null +++ b/test cases/rust/19 structured sources/priv.rs @@ -0,0 +1,3 @@ +fn priv_func() { + std::process::exit(0); +} diff --git a/test cases/rust/19 structured sources/src/foo.rs.in b/test cases/rust/19 structured sources/src/foo.rs.in new file mode 100644 index 0000000..4f3fc42 --- /dev/null +++ b/test cases/rust/19 structured sources/src/foo.rs.in @@ -0,0 +1,4 @@ + +pub fn bar() -> () { + println!("@message@"); +} diff --git a/test cases/rust/19 structured sources/src/main.rs b/test cases/rust/19 structured sources/src/main.rs new file mode 100644 index 0000000..3ffeee2 --- /dev/null +++ b/test cases/rust/19 structured sources/src/main.rs @@ -0,0 +1,5 @@ +mod foo; + +fn main() { + foo::bar(); +} diff --git a/test cases/rust/19 structured sources/src2/foo/mod.rs b/test cases/rust/19 structured sources/src2/foo/mod.rs new file mode 100644 index 0000000..9463d95 --- /dev/null +++ b/test cases/rust/19 structured sources/src2/foo/mod.rs @@ -0,0 +1,4 @@ + +pub fn bar() -> () { + println!("Hello, World!"); +} diff --git a/test cases/rust/19 structured sources/src2/main-unique.rs b/test cases/rust/19 structured sources/src2/main-unique.rs new file mode 100644 index 0000000..3ffeee2 --- /dev/null +++ b/test cases/rust/19 structured sources/src2/main-unique.rs @@ -0,0 +1,5 @@ +mod foo; + +fn main() { + foo::bar(); +} diff --git a/test cases/rust/19 structured sources/src2/meson.build b/test cases/rust/19 structured sources/src2/meson.build new file mode 100644 index 0000000..b4844d2 --- /dev/null +++ b/test cases/rust/19 structured sources/src2/meson.build @@ -0,0 +1,4 @@ +srcs2 = structured_sources( + ['main-unique.rs'], + {'foo': 'foo/mod.rs'}, +) -- cgit v1.1