diff options
| -rw-r--r-- | docs/markdown/Dependencies.md | 8 | ||||
| -rw-r--r-- | mesonbuild/dependencies/__init__.py | 4 | ||||
| -rw-r--r-- | mesonbuild/dependencies/base.py | 23 | ||||
| -rw-r--r-- | mesonbuild/dependencies/mpi.py | 344 | ||||
| -rw-r--r-- | mesonbuild/dependencies/ui.py | 4 | ||||
| -rw-r--r-- | test cases/frameworks/17 mpi/meson.build | 10 | ||||
| -rw-r--r-- | test cases/frameworks/17 mpi/meson_options.txt | 6 |
7 files changed, 189 insertions, 210 deletions
diff --git a/docs/markdown/Dependencies.md b/docs/markdown/Dependencies.md index 6937448..05c97e5 100644 --- a/docs/markdown/Dependencies.md +++ b/docs/markdown/Dependencies.md @@ -411,7 +411,13 @@ not provide them, it will search for the standard wrapper executables, `mpic`, `mpicxx`, `mpic++`, `mpifort`, `mpif90`, `mpif77`. If these are not in your path, they can be specified by setting the standard environment variables `MPICC`, `MPICXX`, `MPIFC`, `MPIF90`, or -`MPIF77`, during configuration. +`MPIF77`, during configuration. It will also try to use the Microsoft +implementation on windows via the `system` method. + +`method` may be `auto`, `config-tool`, `pkg-config` or `system`. + +*New in 0.54.0* The `config-tool` and `system` method values. Previous +versions would always try `pkg-config`, then `config-tool`, then `system`. ## NetCDF diff --git a/mesonbuild/dependencies/__init__.py b/mesonbuild/dependencies/__init__.py index 7c666be..d75d226 100644 --- a/mesonbuild/dependencies/__init__.py +++ b/mesonbuild/dependencies/__init__.py @@ -22,7 +22,7 @@ from .base import ( # noqa: F401 DependencyFactory) from .dev import ValgrindDependency, gmock_factory, gtest_factory, llvm_factory, zlib_factory from .coarrays import coarray_factory -from .mpi import MPIDependency +from .mpi import mpi_factory from .scalapack import scalapack_factory from .misc import ( BlocksDependency, OpenMPDependency, cups_factory, curses_factory, gpgme_factory, @@ -52,7 +52,7 @@ packages.update({ # per-file 'coarray': coarray_factory, 'hdf5': HDF5Dependency, - 'mpi': MPIDependency, + 'mpi': mpi_factory, 'scalapack': scalapack_factory, # From misc: diff --git a/mesonbuild/dependencies/base.py b/mesonbuild/dependencies/base.py index b347a6a..f03af12 100644 --- a/mesonbuild/dependencies/base.py +++ b/mesonbuild/dependencies/base.py @@ -384,19 +384,32 @@ class NotFoundDependency(Dependency): class ConfigToolDependency(ExternalDependency): - """Class representing dependencies found using a config tool.""" + """Class representing dependencies found using a config tool. + + Takes the following extra keys in kwargs that it uses internally: + :tools List[str]: A list of tool names to use + :version_arg str: The argument to pass to the tool to get it's version + :returncode_value int: The value of the correct returncode + Because some tools are stupid and don't return 0 + """ tools = None tool_name = None + version_arg = '--version' __strip_version = re.compile(r'^[0-9][0-9.]+') def __init__(self, name, environment, kwargs, language: T.Optional[str] = None): super().__init__('config-tool', environment, kwargs, language=language) self.name = name + # You may want to overwrite the class version in some cases self.tools = listify(kwargs.get('tools', self.tools)) + if not self.tool_name: + self.tool_name = self.tools[0] + if 'version_arg' in kwargs: + self.version_arg = kwargs['version_arg'] req_version = kwargs.get('version', None) - tool, version = self.find_config(req_version) + tool, version = self.find_config(req_version, kwargs.get('returncode_value', 0)) self.config = tool self.is_found = self.report_config(version, req_version) if not self.is_found: @@ -415,7 +428,7 @@ class ConfigToolDependency(ExternalDependency): return m.group(0).rstrip('.') return version - def find_config(self, versions=None): + def find_config(self, versions=None, returncode: int = 0): """Helper method that searches for config tool binaries in PATH and returns the one that best matches the given version requirements. """ @@ -444,10 +457,10 @@ class ConfigToolDependency(ExternalDependency): continue tool = potential_bin.get_command() try: - p, out = Popen_safe(tool + ['--version'])[:2] + p, out = Popen_safe(tool + [self.version_arg])[:2] except (FileNotFoundError, PermissionError): continue - if p.returncode != 0: + if p.returncode != returncode: continue out = self._sanitize_version(out.strip()) diff --git a/mesonbuild/dependencies/mpi.py b/mesonbuild/dependencies/mpi.py index 59ba0cd..a20fb9c 100644 --- a/mesonbuild/dependencies/mpi.py +++ b/mesonbuild/dependencies/mpi.py @@ -12,111 +12,90 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools import typing as T import os import re -import subprocess -from .. import mlog -from .. import mesonlib -from ..mesonlib import split_args, listify +from .base import (DependencyMethods, PkgConfigDependency, factory_methods, + ConfigToolDependency, detect_compiler, ExternalDependency) from ..environment import detect_cpu_family -from .base import (DependencyException, DependencyMethods, ExternalDependency, ExternalProgram, - PkgConfigDependency) +if T.TYPE_CHECKING: + from .base import DependencyType + from ..compilers import Compiler + from ..environment import Environment, MachineChoice -class MPIDependency(ExternalDependency): - def __init__(self, environment, kwargs: dict): - language = kwargs.get('language', 'c') - super().__init__('mpi', environment, kwargs, language=language) - kwargs['required'] = False - kwargs['silent'] = True - self.is_found = False - methods = listify(self.methods) +@factory_methods({DependencyMethods.PKGCONFIG, DependencyMethods.CONFIG_TOOL, DependencyMethods.SYSTEM}) +def mpi_factory(env: 'Environment', for_machine: 'MachineChoice', + kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> T.List['DependencyType']: + language = kwargs.get('language', 'c') + if language not in {'c', 'cpp', 'fortran'}: + # OpenMPI doesn't work without any other languages + return [] - env_vars = [] - default_wrappers = [] - pkgconfig_files = [] + candidates = [] + compiler = detect_compiler('mpi', env, for_machine, language) + compiler_is_intel = compiler.get_id() in {'intel', 'intel-cl'} + + # Only OpenMPI has pkg-config, and it doesn't work with the intel compilers + if DependencyMethods.PKGCONFIG in methods and not compiler_is_intel: + pkg_name = None if language == 'c': - cid = environment.detect_c_compiler(self.for_machine).get_id() - if cid in ('intel', 'intel-cl'): - env_vars.append('I_MPI_CC') - # IntelMPI doesn't have .pc files - default_wrappers.append('mpiicc') - else: - env_vars.append('MPICC') - pkgconfig_files.append('ompi-c') - default_wrappers.append('mpicc') + pkg_name = 'ompi-c' elif language == 'cpp': - cid = environment.detect_cpp_compiler(self.for_machine).get_id() - if cid in ('intel', 'intel-cl'): - env_vars.append('I_MPI_CXX') - # IntelMPI doesn't have .pc files - default_wrappers.append('mpiicpc') - else: - env_vars.append('MPICXX') - pkgconfig_files.append('ompi-cxx') - default_wrappers += ['mpic++', 'mpicxx', 'mpiCC'] # these are not for intelmpi + pkg_name = 'ompi-cxx' elif language == 'fortran': - cid = environment.detect_fortran_compiler(self.for_machine).get_id() - if cid in ('intel', 'intel-cl'): - env_vars.append('I_MPI_F90') - # IntelMPI doesn't have .pc files - default_wrappers.append('mpiifort') - else: - env_vars += ['MPIFC', 'MPIF90', 'MPIF77'] - pkgconfig_files.append('ompi-fort') - default_wrappers += ['mpifort', 'mpif90', 'mpif77'] - else: - raise DependencyException('Language {} is not supported with MPI.'.format(language)) - - if set([DependencyMethods.AUTO, DependencyMethods.PKGCONFIG]).intersection(methods): - for pkg in pkgconfig_files: - pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language) - if pkgdep.found(): - self.compile_args = pkgdep.get_compile_args() - self.link_args = pkgdep.get_link_args() - self.version = pkgdep.get_version() - self.is_found = True - self.pcdep = pkgdep - return - - if DependencyMethods.AUTO in methods: - for var in env_vars: - if var in os.environ: - wrappers = [os.environ[var]] - break - else: - # Or search for default wrappers. - wrappers = default_wrappers - - for prog in wrappers: - # Note: Some use OpenMPI with Intel compilers on Linux - result = self._try_openmpi_wrapper(prog, cid) - if result is not None: - self.is_found = True - self.version = result[0] - self.compile_args = self._filter_compile_args(result[1]) - self.link_args = self._filter_link_args(result[2], cid) - break - result = self._try_other_wrapper(prog, cid) - if result is not None: - self.is_found = True - self.version = result[0] - self.compile_args = self._filter_compile_args(result[1]) - self.link_args = self._filter_link_args(result[2], cid) - break - - if not self.is_found and mesonlib.is_windows(): - # only Intel Fortran compiler is compatible with Microsoft MPI at this time. - if language == 'fortran' and cid != 'intel-cl': - return - result = self._try_msmpi() - if result is not None: - self.is_found = True - self.version, self.compile_args, self.link_args = result - return + pkg_name = 'ompi-fort' + candidates.append(functools.partial( + PkgConfigDependency, pkg_name, env, kwargs, language=language)) + + if DependencyMethods.CONFIG_TOOL in methods: + nwargs = kwargs.copy() + + if compiler_is_intel: + if env.machines[for_machine].is_windows(): + nwargs['version_arg'] = '-v' + nwargs['returncode_value'] = 3 + + if language == 'c': + tool_names = [os.environ.get('I_MPI_CC'), 'mpiicc'] + elif language == 'cpp': + tool_names = [os.environ.get('I_MPI_CXX'), 'mpiicpc'] + elif language == 'fortran': + tool_names = [os.environ.get('I_MPI_F90'), 'mpiifort'] + + cls = IntelMPIConfigToolDependency # type: T.Type[ConfigToolDependency] + else: # OpenMPI, which doesn't work with intel + # + # We try the environment variables for the tools first, but then + # fall back to the hardcoded names + if language == 'c': + tool_names = [os.environ.get('MPICC'), 'mpicc'] + elif language == 'cpp': + tool_names = [os.environ.get('MPICXX'), 'mpic++', 'mpicxx', 'mpiCC'] + elif language == 'fortran': + tool_names = [os.environ.get(e) for e in ['MPIFC', 'MPIF90', 'MPIF77']] + tool_names.extend(['mpifort', 'mpif90', 'mpif77']) + + cls = OpenMPIConfigToolDependency + + tool_names = [t for t in tool_names if t] # remove empty environment variables + assert tool_names + + nwargs['tools'] = tool_names + candidates.append(functools.partial( + cls, tool_names[0], env, nwargs, language=language)) + + if DependencyMethods.SYSTEM in methods: + candidates.append(functools.partial( + MSMPIDependency, 'msmpi', env, kwargs, language=language)) + + return candidates + + +class _MPIConfigToolDependency(ConfigToolDependency): def _filter_compile_args(self, args: T.Sequence[str]) -> T.List[str]: """ @@ -142,7 +121,7 @@ class MPIDependency(ExternalDependency): result.append(f) return result - def _filter_link_args(self, args: T.Sequence[str], cid: str) -> T.List[str]: + def _filter_link_args(self, args: T.Sequence[str]) -> T.List[str]: """ MPI wrappers return a bunch of garbage args. Drop -O2 and everything that is not needed. @@ -150,7 +129,7 @@ class MPIDependency(ExternalDependency): result = [] include_next = False for f in args: - if self._is_link_arg(f, cid): + if self._is_link_arg(f): result.append(f) if f in ('-L', '-Xlinker'): include_next = True @@ -159,121 +138,94 @@ class MPIDependency(ExternalDependency): result.append(f) return result - @staticmethod - def _is_link_arg(f: str, cid: str) -> bool: - if cid == 'intel-cl': + def _is_link_arg(self, f: str) -> bool: + if self.clib_compiler.id == 'intel-cl': return f == '/link' or f.startswith('/LIBPATH') or f.endswith('.lib') # always .lib whether static or dynamic else: return (f.startswith(('-L', '-l', '-Xlinker')) or f == '-pthread' or (f.startswith('-W') and f != '-Wall' and not f.startswith('-Werror'))) - def _try_openmpi_wrapper(self, prog, cid: str): - # https://www.open-mpi.org/doc/v4.0/man1/mpifort.1.php - if cid == 'intel-cl': # IntelCl doesn't support OpenMPI - return None - prog = ExternalProgram(prog, silent=True) - if not prog.found(): - return None - - # compiler args - cmd = prog.get_command() + ['--showme:compile'] - p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15) - if p.returncode != 0: - mlog.debug('Command', mlog.bold(cmd), 'failed to run:') - mlog.debug(mlog.bold('Standard output\n'), p.stdout) - mlog.debug(mlog.bold('Standard error\n'), p.stderr) - return None - cargs = split_args(p.stdout) - # link args - cmd = prog.get_command() + ['--showme:link'] - p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15) - if p.returncode != 0: - mlog.debug('Command', mlog.bold(cmd), 'failed to run:') - mlog.debug(mlog.bold('Standard output\n'), p.stdout) - mlog.debug(mlog.bold('Standard error\n'), p.stderr) - return None - libs = split_args(p.stdout) - # version - cmd = prog.get_command() + ['--showme:version'] - p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15) - if p.returncode != 0: - mlog.debug('Command', mlog.bold(cmd), 'failed to run:') - mlog.debug(mlog.bold('Standard output\n'), p.stdout) - mlog.debug(mlog.bold('Standard error\n'), p.stderr) - return None - v = re.search(r'\d+.\d+.\d+', p.stdout) + +class IntelMPIConfigToolDependency(_MPIConfigToolDependency): + + """Wrapper around Intel's mpiicc and friends.""" + + version_arg = '-v' # --version is not the same as -v + + def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any], + language: T.Optional[str] = None): + super().__init__(name, env, kwargs, language=language) + if not self.is_found: + return + + args = self.get_config_value(['-show'], 'link and compile args') + self.compile_args = self._filter_compile_args(args) + self.link_args = self._filter_link_args(args) + + def _sanitize_version(self, out: str) -> str: + v = re.search(r'(\d{4}) Update (\d)', out) if v: - version = v.group(0) - else: - version = None + return '{}.{}'.format(v.group(1), v.group(2)) + return out - return version, cargs, libs - def _try_other_wrapper(self, prog, cid: str) -> T.Tuple[str, T.List[str], T.List[str]]: - prog = ExternalProgram(prog, silent=True) - if not prog.found(): - return None +class OpenMPIConfigToolDependency(_MPIConfigToolDependency): - cmd = prog.get_command() - if cid == 'intel-cl': - cmd.append('/show') - else: - cmd.append('-show') - p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15) - if p.returncode != 0: - mlog.debug('Command', mlog.bold(cmd), 'failed to run:') - mlog.debug(mlog.bold('Standard output\n'), p.stdout) - mlog.debug(mlog.bold('Standard error\n'), p.stderr) - return None - - version = None - stdout = p.stdout - if 'Intel(R) MPI Library' in p.stdout: # intel-cl: remove messy compiler logo - out = stdout.split('\n', 2) - version = out[0] - stdout = out[2] - - if version is None: - p = subprocess.run(cmd + ['-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15) - if p.returncode == 0: - version = p.stdout.split('\n', 1)[0] - - args = split_args(stdout) - - return version, args, args - - def _try_msmpi(self) -> T.Tuple[str, T.List[str], T.List[str]]: - if self.language == 'cpp': - # MS-MPI does not support the C++ version of MPI, only the standard C API. - return None - if 'MSMPI_INC' not in os.environ: - return None - - incdir = os.environ['MSMPI_INC'] + """Wrapper around OpenMPI mpicc and friends.""" + + version_arg = '--showme:version' + + def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any], + language: T.Optional[str] = None): + super().__init__(name, env, kwargs, language=language) + if not self.is_found: + return + + c_args = self.get_config_value(['--showme:compile'], 'compile_args') + self.compile_args = self._filter_compile_args(c_args) + + l_args = self.get_config_value(['--showme:link'], 'link_args') + self.link_args = self._filter_link_args(l_args) + + def _sanitize_version(self, out: str) -> str: + v = re.search(r'\d+.\d+.\d+', out) + if v: + return v.group(0) + return out + + +class MSMPIDependency(ExternalDependency): + + """The Microsoft MPI.""" + + def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any], + language: T.Optional[str] = None): + super().__init__(name, env, kwargs, language=language) + # MSMPI only supports the C API + if language not in {'c', 'fortran', None}: + self.is_found = False + return + # MSMPI is only for windows, obviously + if not self.env.machines[self.for_machine].is_windows(): + return + + incdir = os.environ.get('MSMPI_INC') arch = detect_cpu_family(self.env.coredata.compilers.host) + libdir = None if arch == 'x86': - if 'MSMPI_LIB32' not in os.environ: - return None - libdir = os.environ['MSMPI_LIB32'] + libdir = os.environ.get('MSMPI_LIB32') post = 'x86' elif arch == 'x86_64': - if 'MSMPI_LIB64' not in os.environ: - return None - libdir = os.environ['MSMPI_LIB64'] + libdir = os.environ.get('MSMPI_LIB64') post = 'x64' - else: - return None - if self.language == 'fortran': - return (None, - ['-I' + incdir, '-I' + os.path.join(incdir, post)], - [os.path.join(libdir, 'msmpi.lib'), os.path.join(libdir, 'msmpifec.lib')]) - else: - return (None, - ['-I' + incdir, '-I' + os.path.join(incdir, post)], - [os.path.join(libdir, 'msmpi.lib')]) + if libdir is None or incdir is None: + self.is_found = False + return - @staticmethod - def get_methods(): - return [DependencyMethods.AUTO, DependencyMethods.PKGCONFIG] + self.is_found = True + self.link_args = ['-l' + os.path.join(libdir, 'msmpi')] + self.compile_args = ['-I' + incdir, '-I' + os.path.join(incdir, post)] + if self.language == 'fortran': + self.link_args.append('-l' + os.path.join(libdir, 'msmpifec')) diff --git a/mesonbuild/dependencies/ui.py b/mesonbuild/dependencies/ui.py index 31f02e5..1cadeb0 100644 --- a/mesonbuild/dependencies/ui.py +++ b/mesonbuild/dependencies/ui.py @@ -79,13 +79,13 @@ class GnuStepDependency(ConfigToolDependency): ['--gui-libs' if 'gui' in self.modules else '--base-libs'], 'link_args')) - def find_config(self, versions=None): + def find_config(self, versions=None, returncode: int = 0): tool = [self.tools[0]] try: p, out = Popen_safe(tool + ['--help'])[:2] except (FileNotFoundError, PermissionError): return (None, None) - if p.returncode != 0: + if p.returncode != returncode: return (None, None) self.config = tool found_version = self.detect_version() diff --git a/test cases/frameworks/17 mpi/meson.build b/test cases/frameworks/17 mpi/meson.build index b6a44cf..75b463c 100644 --- a/test cases/frameworks/17 mpi/meson.build +++ b/test cases/frameworks/17 mpi/meson.build @@ -1,7 +1,9 @@ project('mpi', 'c', 'cpp', default_options: ['b_asneeded=false']) +method = get_option('method') + cc = meson.get_compiler('c') -mpic = dependency('mpi', language : 'c', required : false) +mpic = dependency('mpi', language : 'c', required : false, method : method) if not mpic.found() error('MESON_SKIP_TEST: MPI not found, skipping.') endif @@ -14,7 +16,7 @@ test('MPI C', exec, timeout: 10) # C++ MPI not supported by MS-MPI cpp = meson.get_compiler('cpp') -mpicpp = dependency('mpi', language : 'cpp', required: false) +mpicpp = dependency('mpi', language : 'cpp', required: false, method : method) if not cpp.links(''' #include <mpi.h> #include <stdio.h> @@ -31,7 +33,7 @@ test('MPI C++', execpp, timeout: 10) if add_languages('fortran', required : false) fc = meson.get_compiler('fortran') - mpif = dependency('mpi', language : 'fortran', required: false) + mpif = dependency('mpi', language : 'fortran', required: false, method : method) if not fc.links('use mpi; end', dependencies: mpif, name: 'Fortran MPI') mpif = disabler() endif @@ -46,5 +48,5 @@ endif # Check we can apply a version constraint if mpic.version() != 'unknown' - dependency('mpi', version: '>=@0@'.format(mpic.version())) + dependency('mpi', version: '>=@0@'.format(mpic.version()), method : method) endif diff --git a/test cases/frameworks/17 mpi/meson_options.txt b/test cases/frameworks/17 mpi/meson_options.txt new file mode 100644 index 0000000..7e9363e --- /dev/null +++ b/test cases/frameworks/17 mpi/meson_options.txt @@ -0,0 +1,6 @@ +option( + 'method', + type : 'combo', + choices : ['auto', 'pkg-config', 'config-tool', 'system'], + value : 'auto', +) |