Merge pull request #6902 from xclaesse/auto-fallback

Implicit dependency fallback when a subproject wrap or dir exists

3 files changed, 194 insertions, 57 deletions

diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py
index 487bdd6..12d6cde 100644
--- a/mesonbuild/interpreter.py
+++ b/mesonbuild/interpreter.py
@@ -2779,10 +2779,9 @@ external dependencies (including libraries) must go to "dependencies".''')
                                            self.subproject_dir, dirname))
             return subproject
 
-        subproject_dir_abs = os.path.join(self.environment.get_source_dir(), self.subproject_dir)
-        r = wrap.Resolver(subproject_dir_abs, self.coredata.get_builtin_option('wrap_mode'), current_subproject=self.subproject)
+        r = self.environment.wrap_resolver
         try:
-            resolved = r.resolve(dirname, method)
+            resolved = r.resolve(dirname, method, self.subproject)
         except wrap.WrapException as e:
             subprojdir = os.path.join(self.subproject_dir, r.directory)
             if isinstance(e, wrap.WrapNotFoundException):
@@ -2798,7 +2797,7 @@ external dependencies (including libraries) must go to "dependencies".''')
             raise e
 
         subdir = os.path.join(self.subproject_dir, resolved)
-        subdir_abs = os.path.join(subproject_dir_abs, resolved)
+        subdir_abs = os.path.join(self.environment.get_source_dir(), subdir)
         os.makedirs(os.path.join(self.build.environment.get_build_dir(), subdir), exist_ok=True)
         self.global_args_frozen = True
 
@@ -3062,6 +3061,10 @@ external dependencies (including libraries) must go to "dependencies".''')
             self.subproject_dir = spdirname
 
         self.build.subproject_dir = self.subproject_dir
+        if not self.is_subproject():
+            wrap_mode = self.coredata.get_builtin_option('wrap_mode')
+            subproject_dir_abs = os.path.join(self.environment.get_source_dir(), self.subproject_dir)
+            self.environment.wrap_resolver = wrap.Resolver(subproject_dir_abs, wrap_mode)
 
         self.build.projects[self.subproject] = proj_name
         mlog.log('Project name:', mlog.bold(proj_name))
@@ -3248,7 +3251,7 @@ external dependencies (including libraries) must go to "dependencies".''')
 
         return success
 
-    def program_from_file_for(self, for_machine, prognames, silent):
+    def program_from_file_for(self, for_machine, prognames):
         for p in unholder(prognames):
             if isinstance(p, mesonlib.File):
                 continue # Always points to a local (i.e. self generated) file.
@@ -3287,15 +3290,13 @@ external dependencies (including libraries) must go to "dependencies".''')
             if progobj.found():
                 return progobj
 
-    def program_from_overrides(self, command_names, silent=False):
+    def program_from_overrides(self, command_names, extra_info):
         for name in command_names:
             if not isinstance(name, str):
                 continue
             if name in self.build.find_overrides:
                 exe = self.build.find_overrides[name]
-                if not silent:
-                    mlog.log('Program', mlog.bold(name), 'found:', mlog.green('YES'),
-                             '(overridden: %s)' % exe.description())
+                extra_info.append(mlog.blue('(overriden)'))
                 return ExternalProgramHolder(exe, self.subproject, self.backend)
         return None
 
@@ -3313,40 +3314,75 @@ external dependencies (including libraries) must go to "dependencies".''')
                                        % name)
         self.build.find_overrides[name] = exe
 
+    def notfound_program(self, args):
+        return ExternalProgramHolder(dependencies.NonExistingExternalProgram(' '.join(args)), self.subproject)
+
     # TODO update modules to always pass `for_machine`. It is bad-form to assume
     # the host machine.
     def find_program_impl(self, args, for_machine: MachineChoice = MachineChoice.HOST,
                           required=True, silent=True, wanted='', search_dirs=None):
-        if not isinstance(args, list):
-            args = [args]
+        args = mesonlib.listify(args)
 
-        progobj = self.program_from_overrides(args, silent=silent)
+        extra_info = []
+        progobj = self.program_lookup(args, for_machine, required, search_dirs, extra_info)
         if progobj is None:
-            progobj = self.program_from_file_for(for_machine, args, silent=silent)
-        if progobj is None:
-            progobj = self.program_from_system(args, search_dirs, silent=silent)
-        if progobj is None and args[0].endswith('python3'):
-            prog = dependencies.ExternalProgram('python3', mesonlib.python_command, silent=True)
-            progobj = ExternalProgramHolder(prog, self.subproject)
-        if required and (progobj is None or not progobj.found()):
-            raise InvalidArguments('Program(s) {!r} not found or not executable'.format(args))
-        if progobj is None:
-            return ExternalProgramHolder(dependencies.NonExistingExternalProgram(' '.join(args)), self.subproject)
-        # Only store successful lookups
-        self.store_name_lookups(args)
+            progobj = self.notfound_program(args)
+
+        if not progobj.found():
+            mlog.log('Program', mlog.bold(progobj.get_name()), 'found:', mlog.red('NO'))
+            if required:
+                m = 'Program {!r} not found'
+                raise InterpreterException(m.format(progobj.get_name()))
+            return progobj
+
         if wanted:
             version = progobj.get_version(self)
             is_found, not_found, found = mesonlib.version_compare_many(version, wanted)
             if not is_found:
                 mlog.log('Program', mlog.bold(progobj.get_name()), 'found:', mlog.red('NO'),
-                         'found {!r} but need:'.format(version),
-                         ', '.join(["'{}'".format(e) for e in not_found]))
+                         'found', mlog.normal_cyan(version), 'but need:',
+                         mlog.bold(', '.join(["'{}'".format(e) for e in not_found])))
                 if required:
                     m = 'Invalid version of program, need {!r} {!r} found {!r}.'
-                    raise InvalidArguments(m.format(progobj.get_name(), not_found, version))
-                return ExternalProgramHolder(dependencies.NonExistingExternalProgram(' '.join(args)), self.subproject)
+                    raise InterpreterException(m.format(progobj.get_name(), not_found, version))
+                return self.notfound_program(args)
+            extra_info.insert(0, mlog.normal_cyan(version))
+
+        # Only store successful lookups
+        self.store_name_lookups(args)
+        mlog.log('Program', mlog.bold(progobj.get_name()), 'found:', mlog.green('YES'), *extra_info)
+        return progobj
+
+    def program_lookup(self, args, for_machine, required, search_dirs, extra_info):
+        progobj = self.program_from_overrides(args, extra_info)
+        if progobj:
+            return progobj
+
+        fallback = None
+        wrap_mode = self.coredata.get_builtin_option('wrap_mode')
+        if wrap_mode != WrapMode.nofallback:
+            fallback = self.environment.wrap_resolver.find_program_provider(args)
+        if fallback and wrap_mode == WrapMode.forcefallback:
+            return self.find_program_fallback(fallback, args, required, extra_info)
+
+        progobj = self.program_from_file_for(for_machine, args)
+        if progobj is None:
+            progobj = self.program_from_system(args, search_dirs, silent=True)
+        if progobj is None and args[0].endswith('python3'):
+            prog = dependencies.ExternalProgram('python3', mesonlib.python_command, silent=True)
+            progobj = ExternalProgramHolder(prog, self.subproject) if prog.found() else None
+        if progobj is None and fallback and required:
+            progobj = self.find_program_fallback(fallback, args, required, extra_info)
+
         return progobj
 
+    def find_program_fallback(self, fallback, args, required, extra_info):
+        mlog.log('Fallback to subproject', mlog.bold(fallback), 'which provides program',
+                 mlog.bold(' '.join(args)))
+        sp_kwargs = { 'required': required }
+        self.do_subproject(fallback, 'meson', sp_kwargs)
+        return self.program_from_overrides(args, extra_info)
+
     @FeatureNewKwargs('find_program', '0.53.0', ['dirs'])
     @FeatureNewKwargs('find_program', '0.52.0', ['version'])
     @FeatureNewKwargs('find_program', '0.49.0', ['disabler'])
@@ -3359,7 +3395,7 @@ external dependencies (including libraries) must go to "dependencies".''')
         disabled, required, feature = extract_required_kwarg(kwargs, self.subproject)
         if disabled:
             mlog.log('Program', mlog.bold(' '.join(args)), 'skipped: feature', mlog.bold(feature), 'disabled')
-            return ExternalProgramHolder(dependencies.NonExistingExternalProgram(' '.join(args)), self.subproject)
+            return self.notfound_program(args)
 
         search_dirs = extract_search_dirs(kwargs)
         wanted = mesonlib.stringlistify(kwargs.get('version', []))
@@ -3453,8 +3489,12 @@ external dependencies (including libraries) must go to "dependencies".''')
                         raise DependencyException(m.format(display_name))
                     return DependencyHolder(cached_dep, self.subproject)
                 else:
-                    m = 'Subproject {} did not override dependency {}'
-                    raise DependencyException(m.format(subproj_path, display_name))
+                    if required:
+                        m = 'Subproject {} did not override dependency {}'
+                        raise DependencyException(m.format(subproj_path, display_name))
+                    mlog.log('Dependency', mlog.bold(display_name), 'from subproject',
+                             mlog.bold(subproj_path), 'found:', mlog.red('NO'))
+                    return self.notfound_dependency()
             if subproject.found():
                 self.verify_fallback_consistency(dirname, varname, cached_dep)
                 dep = self.subprojects[dirname].get_variable_method([varname], {})
@@ -3549,6 +3589,18 @@ external dependencies (including libraries) must go to "dependencies".''')
             return self.notfound_dependency()
 
         has_fallback = 'fallback' in kwargs
+        if not has_fallback and name:
+            # Add an implicit fallback if we have a wrap file or a directory with the same name,
+            # but only if this dependency is required. It is common to first check for a pkg-config,
+            # then fallback to use find_library() and only afterward check again the dependency
+            # with a fallback. If the fallback has already been configured then we have to use it
+            # even if the dependency is not required.
+            provider = self.environment.wrap_resolver.find_dep_provider(name)
+            dirname = mesonlib.listify(provider)[0]
+            if provider and (required or dirname in self.subprojects):
+                kwargs['fallback'] = provider
+                has_fallback = True
+
         if 'default_options' in kwargs and not has_fallback:
             mlog.warning('The "default_options" keyworg argument does nothing without a "fallback" keyword argument.',
                          location=self.current_node)
diff --git a/mesonbuild/mdist.py b/mesonbuild/mdist.py
index 5ab0ad4..9d94ace 100644
--- a/mesonbuild/mdist.py
+++ b/mesonbuild/mdist.py
@@ -259,7 +259,7 @@ def run(options):
     if options.include_subprojects:
         subproject_dir = os.path.join(src_root, b.subproject_dir)
         for sub in b.subprojects:
-            _, directory = wrap.get_directory(subproject_dir, sub)
+            directory = wrap.get_directory(subproject_dir, sub)
             subprojects.append(os.path.join(b.subproject_dir, directory))
         extra_meson_args.append('-Dwrap_mode=nodownload')
 
diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py
index 689fb4f..e0c7a04 100644
--- a/mesonbuild/wrap/wrap.py
+++ b/mesonbuild/wrap/wrap.py
@@ -103,14 +103,31 @@ class WrapNotFoundException(WrapException):
 
 class PackageDefinition:
     def __init__(self, fname: str):
-        self.filename = fname
+        self.type = None
+        self.values = {} # type: T.Dict[str, str]
+        self.provided_deps = {} # type: T.Dict[str, T.Optional[str]]
+        self.provided_programs = [] # type: T.List[str]
         self.basename = os.path.basename(fname)
-        self.name = self.basename[:-5]
+        self.name = self.basename
+        if self.name.endswith('.wrap'):
+            self.name = self.name[:-5]
+        self.provided_deps[self.name] = None
+        if fname.endswith('.wrap'):
+            self.parse_wrap(fname)
+        self.directory = self.values.get('directory', self.name)
+        if os.path.dirname(self.directory):
+            raise WrapException('Directory key must be a name and not a path')
+
+    def parse_wrap(self, fname: str):
         try:
             self.config = configparser.ConfigParser(interpolation=None)
             self.config.read(fname)
         except configparser.Error:
             raise WrapException('Failed to parse {}'.format(self.basename))
+        self.parse_wrap_section()
+        self.parse_provide_section()
+
+    def parse_wrap_section(self):
         if len(self.config.sections()) < 1:
             raise WrapException('Missing sections in {}'.format(self.basename))
         self.wrap_section = self.config.sections()[0]
@@ -120,6 +137,27 @@ class PackageDefinition:
         self.type = self.wrap_section[5:]
         self.values = dict(self.config[self.wrap_section])
 
+    def parse_provide_section(self):
+        if self.config.has_section('provide'):
+            for k, v in self.config['provide'].items():
+                if k == 'dependency_names':
+                    # A comma separated list of dependency names that does not
+                    # need a variable name
+                    names = {n.strip(): None for n in v.split(',')}
+                    self.provided_deps.update(names)
+                    continue
+                if k == 'program_names':
+                    # A comma separated list of program names
+                    names = [n.strip() for n in v.split(',')]
+                    self.provided_programs += names
+                    continue
+                if not v:
+                    m = ('Empty dependency variable name for {!r} in {}. '
+                         'If the subproject uses meson.override_dependency() '
+                         'it can be added in the "dependency_names" special key.')
+                    raise WrapException(m.format(k, self.basename))
+                self.provided_deps[k] = v
+
     def get(self, key: str) -> str:
         try:
             return self.values[key]
@@ -127,35 +165,87 @@ class PackageDefinition:
             m = 'Missing key {!r} in {}'
             raise WrapException(m.format(key, self.basename))
 
-def load_wrap(subdir_root: str, packagename: str) -> PackageDefinition:
+def get_directory(subdir_root: str, packagename: str) -> str:
     fname = os.path.join(subdir_root, packagename + '.wrap')
     if os.path.isfile(fname):
-        return PackageDefinition(fname)
-    return None
-
-def get_directory(subdir_root: str, packagename: str):
-    directory = packagename
-    # We always have to load the wrap file, if it exists, because it could
-    # override the default directory name.
-    wrap = load_wrap(subdir_root, packagename)
-    if wrap and 'directory' in wrap.values:
-        directory = wrap.get('directory')
-        if os.path.dirname(directory):
-            raise WrapException('Directory key must be a name and not a path')
-    return wrap, directory
+        wrap = PackageDefinition(fname)
+        return wrap.directory
+    return packagename
 
 class Resolver:
-    def __init__(self, subdir_root: str, wrap_mode=WrapMode.default, current_subproject: str = ''):
+    def __init__(self, subdir_root: str, wrap_mode=WrapMode.default):
         self.wrap_mode = wrap_mode
         self.subdir_root = subdir_root
-        self.current_subproject = current_subproject
         self.cachedir = os.path.join(self.subdir_root, 'packagecache')
         self.filesdir = os.path.join(self.subdir_root, 'packagefiles')
+        self.wraps = {} # type: T.Dict[str, PackageDefinition]
+        self.provided_deps = {} # type: T.Dict[str, PackageDefinition]
+        self.provided_programs = {} # type: T.Dict[str, PackageDefinition]
+        self.load_wraps()
 
-    def resolve(self, packagename: str, method: str) -> str:
+    def load_wraps(self):
+        if not os.path.isdir(self.subdir_root):
+            return
+        root, dirs, files = next(os.walk(self.subdir_root))
+        for i in files:
+            if not i.endswith('.wrap'):
+                continue
+            fname = os.path.join(self.subdir_root, i)
+            wrap = PackageDefinition(fname)
+            self.wraps[wrap.name] = wrap
+            if wrap.directory in dirs:
+                dirs.remove(wrap.directory)
+        # Add dummy package definition for directories not associated with a wrap file.
+        for i in dirs:
+            if i in ['packagecache', 'packagefiles']:
+                continue
+            fname = os.path.join(self.subdir_root, i)
+            wrap = PackageDefinition(fname)
+            self.wraps[wrap.name] = wrap
+
+        for wrap in self.wraps.values():
+            for k in wrap.provided_deps.keys():
+                if k in self.provided_deps:
+                    prev_wrap = self.provided_deps[k]
+                    m = 'Multiple wrap files provide {!r} dependency: {} and {}'
+                    raise WrapException(m.format(k, wrap.basename, prev_wrap.basename))
+                self.provided_deps[k] = wrap
+            for k in wrap.provided_programs:
+                if k in self.provided_programs:
+                    prev_wrap = self.provided_programs[k]
+                    m = 'Multiple wrap files provide {!r} program: {} and {}'
+                    raise WrapException(m.format(k, wrap.basename, prev_wrap.basename))
+                self.provided_programs[k] = wrap
+
+    def find_dep_provider(self, packagename: str):
+        # Return value is in the same format as fallback kwarg:
+        # ['subproject_name', 'variable_name'], or 'subproject_name'.
+        wrap = self.provided_deps.get(packagename)
+        if wrap:
+            dep_var = wrap.provided_deps.get(packagename)
+            if dep_var:
+                return [wrap.name, dep_var]
+            return wrap.name
+        return None
+
+    def find_program_provider(self, names: T.List[str]):
+        for name in names:
+            wrap = self.provided_programs.get(name)
+            if wrap:
+                return wrap.name
+        return None
+
+    def resolve(self, packagename: str, method: str, current_subproject: str = '') -> str:
+        self.current_subproject = current_subproject
         self.packagename = packagename
-        self.wrap, self.directory = get_directory(self.subdir_root, self.packagename)
+        self.directory = packagename
+        self.wrap = self.wraps.get(packagename)
+        if not self.wrap:
+            m = 'Subproject directory not found and {}.wrap file not found'
+            raise WrapNotFoundException(m.format(self.packagename))
+        self.directory = self.wrap.directory
         self.dirname = os.path.join(self.subdir_root, self.directory)
+
         meson_file = os.path.join(self.dirname, 'meson.build')
         cmake_file = os.path.join(self.dirname, 'CMakeLists.txt')
 
@@ -175,11 +265,6 @@ class Resolver:
             if not os.path.isdir(self.dirname):
                 raise WrapException('Path already exists but is not a directory')
         else:
-            # A wrap file is required to download
-            if not self.wrap:
-                m = 'Subproject directory not found and {}.wrap file not found'
-                raise WrapNotFoundException(m.format(self.packagename))
-
             if self.wrap.type == 'file':
                 self.get_file()
             else: