From 6333ee88c1a243f28b3a7a9bce2dd003b541280a Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Mon, 13 Apr 2020 14:35:06 -0400 Subject: Merge wraps from subprojects into wraps from main project wraps from subprojects are now merged into the list of wraps from main project, so they can be used to download dependencies of dependencies instead of having to promote wraps manually. If multiple projects provides the same wrap file, the first one to be configured wins. This also fix usage of sub-subproject that don't have wrap files. We can now configure B when its source tree is at `subprojects/A/subprojects/B/`. This has the implication that we cannot assume that subproject "foo" is at `self.subproject_dir / 'foo'` any more. --- docs/markdown/Using-wraptool.md | 21 ----- docs/markdown/snippets/subsubproject.md | 11 +++ mesonbuild/interpreter.py | 94 ++++++++------------- mesonbuild/msubprojects.py | 2 +- mesonbuild/wrap/wrap.py | 47 ++++++++--- run_unittests.py | 18 +++- .../common/102 subproject subdir/meson.build | 11 +++ .../subprojects/packagefiles/subsubsub-1.0.zip | Bin 0 -> 455 bytes .../sub_implicit/subprojects/subsub/meson.build | 3 + .../subprojects/subsub/subprojects/subsubsub.wrap | 4 + .../failing/16 extract from subproject/test.json | 2 +- 11 files changed, 116 insertions(+), 97 deletions(-) create mode 100644 docs/markdown/snippets/subsubproject.md create mode 100644 test cases/common/102 subproject subdir/subprojects/packagefiles/subsubsub-1.0.zip create mode 100644 test cases/common/102 subproject subdir/subprojects/sub_implicit/subprojects/subsub/meson.build create mode 100644 test cases/common/102 subproject subdir/subprojects/sub_implicit/subprojects/subsub/subprojects/subsubsub.wrap diff --git a/docs/markdown/Using-wraptool.md b/docs/markdown/Using-wraptool.md index f6023e8..ffa8309 100644 --- a/docs/markdown/Using-wraptool.md +++ b/docs/markdown/Using-wraptool.md @@ -76,24 +76,3 @@ straightforward: Wraptool can do other things besides these. Documentation for these can be found in the command line help, which can be accessed by `meson wrap --help`. - -## Promoting dependencies - -Meson will only search for subprojects from the top level -`subprojects` directory. If you have subprojects that themselves have -subprojects, you must transfer them to the top level. This can be done -by going to your source root and issuing a promotion command. - - meson wrap promote projname - -This will cause Meson to go through your entire project tree, find an -embedded subproject and copy it to the top level. - -If there are multiple embedded copies of a subproject, Meson will not -try to guess which one you want. Instead it will print all the -possibilities. You can then manually select which one to promote by -writing it out fully. - - meson wrap promote subprojects/s1/subprojects/projname - -This functionality was added in Meson release 0.45.0. diff --git a/docs/markdown/snippets/subsubproject.md b/docs/markdown/snippets/subsubproject.md new file mode 100644 index 0000000..fdba3e0 --- /dev/null +++ b/docs/markdown/snippets/subsubproject.md @@ -0,0 +1,11 @@ +## Sub-subprojects + +It is not required to promote wrap files for subprojects into the main project +any more. When configuring a subproject, meson will look for any wrap file or +directory in the subproject's `subprojects/` directory and add them into the +global list of available subprojects, to be used by any future `subproject()` +call or `dependency()` fallback. If a subproject with the same name already exists, +the new wrap file or directory is ignored. That means that the main project can +always override any subproject's wrap files by providing their own, it also means +the ordering in which subprojects are configured matters, if 2 subprojects provide +foo.wrap only the one from the first subproject to be configured will be used. diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index 713372d..5104af6 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -1010,15 +1010,14 @@ class Test(InterpreterObject): class SubprojectHolder(InterpreterObject, ObjectHolder): - def __init__(self, subinterpreter, subproject_dir, name, warnings=0, disabled_feature=None, + def __init__(self, subinterpreter, subdir, warnings=0, disabled_feature=None, exception=None): InterpreterObject.__init__(self) ObjectHolder.__init__(self, subinterpreter) - self.name = name self.warnings = warnings self.disabled_feature = disabled_feature self.exception = exception - self.subproject_dir = subproject_dir + self.subdir = PurePath(subdir).as_posix() self.methods.update({'get_variable': self.get_variable_method, 'found': self.found_method, }) @@ -1037,8 +1036,7 @@ class SubprojectHolder(InterpreterObject, ObjectHolder): if len(args) < 1 or len(args) > 2: raise InterpreterException('Get_variable takes one or two arguments.') if not self.found(): - raise InterpreterException('Subproject "%s/%s" disabled can\'t get_variable on it.' % ( - self.subproject_dir, self.name)) + raise InterpreterException('Subproject "%s" disabled can\'t get_variable on it.' % (self.subdir)) varname = args[0] if not isinstance(varname, str): raise InterpreterException('Get_variable first argument must be a string.') @@ -2844,7 +2842,7 @@ external dependencies (including libraries) must go to "dependencies".''') return self.do_subproject(subp_name, 'meson', kwargs) def disabled_subproject(self, subp_name, disabled_feature=None, exception=None): - sub = SubprojectHolder(None, self.subproject_dir, subp_name, + sub = SubprojectHolder(None, os.path.join(self.subproject_dir, subp_name), disabled_feature=disabled_feature, exception=exception) self.subprojects[subp_name] = sub return sub @@ -2882,28 +2880,19 @@ external dependencies (including libraries) must go to "dependencies".''') if subp_name in self.subprojects: subproject = self.subprojects[subp_name] if required and not subproject.found(): - raise InterpreterException('Subproject "%s/%s" required but not found.' % ( - self.subproject_dir, subp_name)) + raise InterpreterException('Subproject "%s" required but not found.' % (subproject.subdir)) return subproject r = self.environment.wrap_resolver try: - resolved = r.resolve(subp_name, method, self.subproject) + subdir = r.resolve(subp_name, method, self.subproject) except wrap.WrapException as e: - subprojdir = os.path.join(self.subproject_dir, r.directory) - if isinstance(e, wrap.WrapNotFoundException): - # if the reason subproject execution failed was because - # the directory doesn't exist, try to give some helpful - # advice if it's a nested subproject that needs - # promotion... - self.print_nested_info(subp_name) if not required: mlog.log(e) - mlog.log('Subproject ', mlog.bold(subprojdir), 'is buildable:', mlog.red('NO'), '(disabling)') + mlog.log('Subproject ', mlog.bold(subp_name), 'is buildable:', mlog.red('NO'), '(disabling)') return self.disabled_subproject(subp_name, exception=e) raise e - subdir = os.path.join(self.subproject_dir, resolved) subdir_abs = os.path.join(self.environment.get_source_dir(), subdir) os.makedirs(os.path.join(self.build.environment.get_build_dir(), subdir), exist_ok=True) self.global_args_frozen = True @@ -2927,7 +2916,7 @@ external dependencies (including libraries) must go to "dependencies".''') # Suppress the 'ERROR:' prefix because this exception is not # fatal and VS CI treat any logs with "ERROR:" as fatal. mlog.exception(e, prefix=mlog.yellow('Exception:')) - mlog.log('\nSubproject', mlog.bold(subp_name), 'is buildable:', mlog.red('NO'), '(disabling)') + mlog.log('\nSubproject', mlog.bold(subdir), 'is buildable:', mlog.red('NO'), '(disabling)') return self.disabled_subproject(subp_name, exception=e) raise e @@ -2957,8 +2946,7 @@ external dependencies (including libraries) must go to "dependencies".''') raise InterpreterException('Subproject %s version is %s but %s required.' % (subp_name, pv, wanted)) self.active_projectname = current_active self.subprojects.update(subi.subprojects) - self.subprojects[subp_name] = SubprojectHolder(subi, self.subproject_dir, subp_name, - warnings=subi_warnings) + self.subprojects[subp_name] = SubprojectHolder(subi, subdir, warnings=subi_warnings) # Duplicates are possible when subproject uses files from project root if build_def_files: self.build_def_files = list(set(self.build_def_files + build_def_files)) @@ -3157,8 +3145,11 @@ external dependencies (including libraries) must go to "dependencies".''') 'license': proj_license} if self.subproject in self.build.projects: raise InvalidCode('Second call to project().') - if not self.is_subproject() and 'subproject_dir' in kwargs: - spdirname = kwargs['subproject_dir'] + + # spdirname is the subproject_dir for this project, relative to self.subdir. + # self.subproject_dir is the subproject_dir for the main project, relative to top source dir. + spdirname = kwargs.get('subproject_dir') + if spdirname: if not isinstance(spdirname, str): raise InterpreterException('Subproject_dir must be a string') if os.path.isabs(spdirname): @@ -3167,13 +3158,20 @@ external dependencies (including libraries) must go to "dependencies".''') raise InterpreterException('Subproject_dir must not begin with a period.') if '..' in spdirname: raise InterpreterException('Subproject_dir must not contain a ".." segment.') - self.subproject_dir = spdirname - + if not self.is_subproject(): + self.subproject_dir = spdirname + else: + spdirname = 'subprojects' self.build.subproject_dir = self.subproject_dir - if not self.is_subproject(): - wrap_mode = self.coredata.get_builtin_option('wrap_mode') - subproject_dir_abs = os.path.join(self.environment.get_source_dir(), self.subproject_dir) - self.environment.wrap_resolver = wrap.Resolver(subproject_dir_abs, wrap_mode) + + # Load wrap files from this (sub)project. + wrap_mode = self.coredata.get_builtin_option('wrap_mode') + subdir = os.path.join(self.subdir, spdirname) + r = wrap.Resolver(self.environment.get_source_dir(), subdir, wrap_mode) + if self.is_subproject(): + self.environment.wrap_resolver.merge_wraps(r) + else: + self.environment.wrap_resolver = r self.build.projects[self.subproject] = proj_name mlog.log('Project name:', mlog.bold(proj_name)) @@ -3594,7 +3592,6 @@ external dependencies (including libraries) must go to "dependencies".''') def get_subproject_dep(self, name, display_name, subp_name, varname, kwargs): required = kwargs.get('required', True) wanted = mesonlib.stringlistify(kwargs.get('version', [])) - subproj_path = os.path.join(self.subproject_dir, subp_name) dep = self.notfound_dependency() try: subproject = self.subprojects[subp_name] @@ -3609,9 +3606,9 @@ external dependencies (including libraries) must go to "dependencies".''') else: if required: m = 'Subproject {} did not override dependency {}' - raise DependencyException(m.format(subproj_path, display_name)) + raise DependencyException(m.format(subproject.subdir, display_name)) mlog.log('Dependency', mlog.bold(display_name), 'from subproject', - mlog.bold(subproj_path), 'found:', mlog.red('NO')) + mlog.bold(subproject.subdir), 'found:', mlog.red('NO')) return self.notfound_dependency() if subproject.found(): self.verify_fallback_consistency(subp_name, varname, cached_dep) @@ -3629,7 +3626,7 @@ external dependencies (including libraries) must go to "dependencies".''') ''.format(varname, subp_name)) # If the dependency is not required, don't raise an exception mlog.log('Dependency', mlog.bold(display_name), 'from subproject', - mlog.bold(subproj_path), 'found:', mlog.red('NO')) + mlog.bold(subproject.subdir), 'found:', mlog.red('NO')) return dep found = dep.held_object.get_version() @@ -3640,14 +3637,14 @@ external dependencies (including libraries) must go to "dependencies".''') 'dep {}'.format(found, subp_name, wanted, display_name)) mlog.log('Dependency', mlog.bold(display_name), 'from subproject', - mlog.bold(subproj_path), 'found:', mlog.red('NO'), + mlog.bold(subproject.subdir), 'found:', mlog.red('NO'), 'found', mlog.normal_cyan(found), 'but need:', mlog.bold(', '.join(["'{}'".format(e) for e in wanted]))) return self.notfound_dependency() found = mlog.normal_cyan(found) if found else None mlog.log('Dependency', mlog.bold(display_name), 'from subproject', - mlog.bold(subproj_path), 'found:', mlog.green('YES'), found) + mlog.bold(subproject.subdir), 'found:', mlog.green('YES'), found) return dep def _handle_featurenew_dependencies(self, name): @@ -3798,23 +3795,6 @@ external dependencies (including libraries) must go to "dependencies".''') def func_disabler(self, node, args, kwargs): return Disabler() - def print_nested_info(self, dependency_name): - message = ['Dependency', mlog.bold(dependency_name), 'not found but it is available in a sub-subproject.\n' + - 'To use it in the current project, promote it by going in the project source\n' - 'root and issuing'] - sprojs = mesonlib.detect_subprojects('subprojects', self.source_root) - if dependency_name not in sprojs: - return - found = sprojs[dependency_name] - if len(found) > 1: - message.append('one of the following commands:') - else: - message.append('the following command:') - command_templ = '\nmeson wrap promote {}' - for l in found: - message.append(mlog.bold(command_templ.format(l[len(self.source_root) + 1:]))) - mlog.warning(*message, location=self.current_node) - def get_subproject_infos(self, fbinfo): fbinfo = mesonlib.stringlistify(fbinfo) if len(fbinfo) == 1: @@ -4926,14 +4906,8 @@ This will become a hard error in the future.''', location=self.current_node) # Only permit object extraction from the same subproject def validate_extraction(self, buildtarget: InterpreterObject) -> None: - if not self.subdir.startswith(self.subproject_dir): - if buildtarget.subdir.startswith(self.subproject_dir): - raise InterpreterException('Tried to extract objects from a subproject target.') - else: - if not buildtarget.subdir.startswith(self.subproject_dir): - raise InterpreterException('Tried to extract objects from the main project from a subproject.') - if self.subdir.split('/')[1] != buildtarget.subdir.split('/')[1]: - raise InterpreterException('Tried to extract objects from a different subproject.') + if self.subproject != buildtarget.subproject: + raise InterpreterException('Tried to extract objects from a different subproject.') def is_subproject(self): return self.subproject != '' diff --git a/mesonbuild/msubprojects.py b/mesonbuild/msubprojects.py index b628a47..20639cb 100755 --- a/mesonbuild/msubprojects.py +++ b/mesonbuild/msubprojects.py @@ -364,7 +364,7 @@ def run(options): if not os.path.isdir(subprojects_dir): mlog.log('Directory', mlog.bold(src_dir), 'does not seem to have subprojects.') return 0 - r = Resolver(subprojects_dir) + r = Resolver(src_dir, 'subprojects') if options.subprojects: wraps = [wrap for name, wrap in r.wraps.items() if name in options.subprojects] else: diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index a0a4801..8380661 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -97,11 +97,11 @@ class PackageDefinition: self.provided_deps = {} # type: T.Dict[str, T.Optional[str]] self.provided_programs = [] # type: T.List[str] self.basename = os.path.basename(fname) - self.name = self.basename - if self.name.endswith('.wrap'): - self.name = self.name[:-5] + self.has_wrap = self.basename.endswith('.wrap') + self.name = self.basename[:-5] if self.has_wrap else self.basename + self.directory = self.name self.provided_deps[self.name] = None - if fname.endswith('.wrap'): + if self.has_wrap: self.parse_wrap(fname) self.directory = self.values.get('directory', self.name) if os.path.dirname(self.directory): @@ -164,9 +164,11 @@ def get_directory(subdir_root: str, packagename: str) -> str: return packagename class Resolver: - def __init__(self, subdir_root: str, wrap_mode: WrapMode = WrapMode.default) -> None: + def __init__(self, source_dir: str, subdir: str, wrap_mode: WrapMode = WrapMode.default) -> None: + self.source_dir = source_dir + self.subdir = subdir self.wrap_mode = wrap_mode - self.subdir_root = subdir_root + self.subdir_root = os.path.join(source_dir, subdir) self.cachedir = os.path.join(self.subdir_root, 'packagecache') self.filesdir = os.path.join(self.subdir_root, 'packagefiles') self.wraps = {} # type: T.Dict[str, PackageDefinition] @@ -208,6 +210,14 @@ class Resolver: raise WrapException(m.format(k, wrap.basename, prev_wrap.basename)) self.provided_programs[k] = wrap + def merge_wraps(self, other_resolver: 'Resolver') -> None: + for k, v in other_resolver.wraps.items(): + self.wraps.setdefault(k, v) + for k, v in other_resolver.provided_deps.items(): + self.provided_deps.setdefault(k, v) + for k, v in other_resolver.provided_programs.items(): + self.provided_programs.setdefault(k, v) + def find_dep_provider(self, packagename: str) -> T.Optional[T.Union[str, T.List[str]]]: # Return value is in the same format as fallback kwarg: # ['subproject_name', 'variable_name'], or 'subproject_name'. @@ -235,7 +245,24 @@ class Resolver: m = 'Subproject directory not found and {}.wrap file not found' raise WrapNotFoundException(m.format(self.packagename)) self.directory = self.wrap.directory - self.dirname = os.path.join(self.subdir_root, self.directory) + + if self.wrap.has_wrap: + # We have a .wrap file, source code will be placed into main + # project's subproject_dir even if the wrap file comes from another + # subproject. + self.dirname = os.path.join(self.subdir_root, self.directory) + # Copy .wrap file into main project's subproject_dir + wrap_dir = os.path.normpath(os.path.dirname(self.wrap.filename)) + main_dir = os.path.normpath(self.subdir_root) + if wrap_dir != main_dir: + rel = os.path.relpath(self.wrap.filename, self.source_dir) + mlog.log('Using', mlog.bold(rel)) + shutil.copy2(self.wrap.filename, self.subdir_root) + else: + # No wrap file, it's a dummy package definition for an existing + # directory. Use the source code in place. + self.dirname = self.wrap.filename + rel_path = os.path.relpath(self.dirname, self.source_dir) meson_file = os.path.join(self.dirname, 'meson.build') cmake_file = os.path.join(self.dirname, 'CMakeLists.txt') @@ -245,9 +272,9 @@ class Resolver: # The directory is there and has meson.build? Great, use it. if method == 'meson' and os.path.exists(meson_file): - return self.directory + return rel_path if method == 'cmake' and os.path.exists(cmake_file): - return self.directory + return rel_path # Check if the subproject is a git submodule self.resolve_git_submodule() @@ -276,7 +303,7 @@ class Resolver: if method == 'cmake' and not os.path.exists(cmake_file): raise WrapException('Subproject exists but has no CMakeLists.txt file') - return self.directory + return rel_path def check_can_download(self) -> None: # Don't download subproject data based on wrap file if requested. diff --git a/run_unittests.py b/run_unittests.py index 22e7cdc..cfab11b 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -4183,6 +4183,16 @@ recommended as it is not supported on some platforms''') 'name': 'sub_novar', 'version': '1.0', }, + { + 'descriptive_name': 'subsub', + 'name': 'subsub', + 'version': 'undefined' + }, + { + 'descriptive_name': 'subsubsub', + 'name': 'subsubsub', + 'version': 'undefined' + }, ] } res['subprojects'] = sorted(res['subprojects'], key=lambda i: i['name']) @@ -5365,16 +5375,16 @@ class FailureTests(BasePlatformTests): correct message when the fallback subproject is found but the variable inside it is not. 4. A fallback dependency is found from the subproject parsed in (3) - 5. The correct message is outputted when the .wrap file is missing for - a sub-subproject. + 5. A wrap file from a subproject is used but fails because it does not + contain required keys. ''' tdir = os.path.join(self.unit_test_dir, '20 subproj dep variables') out = self.init(tdir, inprocess=True) self.assertRegex(out, r"Subproject directory not found and .*nosubproj.wrap.* file not found") self.assertRegex(out, r'Function does not take positional arguments.') - self.assertRegex(out, r'WARNING:.* Dependency .*subsubproject.* not found but it is available in a sub-subproject.') - self.assertRegex(out, r'Subproject directory not found and .*subsubproject.wrap.* file not found') + self.assertRegex(out, r'Dependency .*somenotfounddep.* from subproject .*subprojects/somesubproj.* found: .*NO.*') self.assertRegex(out, r'Dependency .*zlibproxy.* from subproject .*subprojects.*somesubproj.* found: .*YES.*') + self.assertRegex(out, r'Missing key .*source_filename.* in subsubproject.wrap') def test_exception_exit_status(self): ''' diff --git a/test cases/common/102 subproject subdir/meson.build b/test cases/common/102 subproject subdir/meson.build index a891ca9..36e48a7 100644 --- a/test cases/common/102 subproject subdir/meson.build +++ b/test cases/common/102 subproject subdir/meson.build @@ -54,3 +54,14 @@ assert(d.type_name() == 'internal') # Using gobject-2.0 here because some CI runners have it installed. d = dependency('gobject-2.0', required : false) assert(not d.found()) + +# Verify that implicit fallback works because subprojects/sub_implicit/subprojects/subsub +# directory exists. +d = dependency('subsub') +assert(d.found(), 'Should be able to fallback to sub-subproject') + +# Verify that implicit fallback works because +# subprojects/sub_implicit/subprojects/subsub/subprojects/subsubsub.wrap +# file exists. +d = dependency('subsubsub') +assert(d.found(), 'Should be able to fallback to sub-sub-subproject') diff --git a/test cases/common/102 subproject subdir/subprojects/packagefiles/subsubsub-1.0.zip b/test cases/common/102 subproject subdir/subprojects/packagefiles/subsubsub-1.0.zip new file mode 100644 index 0000000..dfb7576 Binary files /dev/null and b/test cases/common/102 subproject subdir/subprojects/packagefiles/subsubsub-1.0.zip differ diff --git a/test cases/common/102 subproject subdir/subprojects/sub_implicit/subprojects/subsub/meson.build b/test cases/common/102 subproject subdir/subprojects/sub_implicit/subprojects/subsub/meson.build new file mode 100644 index 0000000..18e2cea --- /dev/null +++ b/test cases/common/102 subproject subdir/subprojects/sub_implicit/subprojects/subsub/meson.build @@ -0,0 +1,3 @@ +project('subsub') + +meson.override_dependency('subsub', declare_dependency()) diff --git a/test cases/common/102 subproject subdir/subprojects/sub_implicit/subprojects/subsub/subprojects/subsubsub.wrap b/test cases/common/102 subproject subdir/subprojects/sub_implicit/subprojects/subsub/subprojects/subsubsub.wrap new file mode 100644 index 0000000..6567ed0 --- /dev/null +++ b/test cases/common/102 subproject subdir/subprojects/sub_implicit/subprojects/subsub/subprojects/subsubsub.wrap @@ -0,0 +1,4 @@ +[wrap-file] +directory = subsubsub-1.0 +source_filename = subsubsub-1.0.zip +source_hash = c073a96b7251937e53216578f6f03d91b84816618a0f1ce3ecfb867beddf1498 diff --git a/test cases/failing/16 extract from subproject/test.json b/test cases/failing/16 extract from subproject/test.json index 78d45a5..2e32904 100644 --- a/test cases/failing/16 extract from subproject/test.json +++ b/test cases/failing/16 extract from subproject/test.json @@ -1,7 +1,7 @@ { "stdout": [ { - "line": "test cases/failing/16 extract from subproject/meson.build:6:0: ERROR: Tried to extract objects from a subproject target." + "line": "test cases/failing/16 extract from subproject/meson.build:6:0: ERROR: Tried to extract objects from a different subproject." } ] } -- cgit v1.1