diff options
31 files changed, 338 insertions, 166 deletions
diff --git a/docs/markdown/Commands.md b/docs/markdown/Commands.md index 0751aed..fc8cdd2 100644 --- a/docs/markdown/Commands.md +++ b/docs/markdown/Commands.md @@ -295,5 +295,12 @@ These variables are set in environment in addition to those set using `meson.add - `PATH` includes every directory where there is an executable that would be installed into `bindir`. On windows it also includes every directory where there is a DLL needed to run those executables. +- `LD_LIBRARY_PATH` includes every directory where there is a shared library that + would be installed into `libdir`. This allows to run system application using + custom build of some libraries. For example running system GEdit when building + GTK from git. On OSX the environment variable is `DYLD_LIBRARY_PATH` and + `PATH` on Windows. +- `GI_TYPELIB_PATH` includes every directory where a GObject Introspection + typelib is built. This is automatically set when using `gnome.generate_gir()`. {{ devenv_arguments.inc }} diff --git a/docs/markdown/Reference-manual.md b/docs/markdown/Reference-manual.md index 6e18e68..74f3324 100644 --- a/docs/markdown/Reference-manual.md +++ b/docs/markdown/Reference-manual.md @@ -2752,8 +2752,19 @@ tests and other functions. It has the following methods. joined by the separator, e.g. `env.set('FOO', 'BAR'),` sets envvar `FOO` to value `BAR`. See `append()` above for how separators work. -**Note:** All these methods overwrite the previously-defined value(s) -if called twice with the same `varname`. +*Since 0.58.0* `append()` and `prepend()` methods can be called multiple times +on the same `varname`. Earlier Meson versions would warn and only the last +operation took effect. + +```meson +env = environment() + +# MY_PATH will be '0:1:2:3' +env.set('MY_PATH', '1') +env.append('MY_PATH', '2') +env.append('MY_PATH', '3') +env.prepend('MY_PATH', '0') +``` ### `external library` object diff --git a/docs/markdown/snippets/devenv.md b/docs/markdown/snippets/devenv.md index c3bac10..1160945 100644 --- a/docs/markdown/snippets/devenv.md +++ b/docs/markdown/snippets/devenv.md @@ -26,4 +26,10 @@ These variables are set in environment in addition to those set using `meson.add - `PATH` includes every directory where there is an executable that would be installed into `bindir`. On windows it also includes every directory where there is a DLL needed to run those executables. - +- `LD_LIBRARY_PATH` includes every directory where there is a shared library that + would be installed into `libdir`. This allows to run system application using + custom build of some libraries. For example running system GEdit when building + GTK from git. On OSX the environment variable is `DYLD_LIBRARY_PATH` and + `PATH` on Windows. +- `GI_TYPELIB_PATH` includes every directory where a GObject Introspection + typelib is built. This is automatically set when using `gnome.generate_gir()`. diff --git a/docs/markdown/snippets/environment.md b/docs/markdown/snippets/environment.md new file mode 100644 index 0000000..1ed102d --- /dev/null +++ b/docs/markdown/snippets/environment.md @@ -0,0 +1,16 @@ +## Multiple append() and prepend() in `environment()` object + +`append()` and `prepend()` methods can now be called multiple times +on the same `varname`. Earlier Meson versions would warn and only the last +opperation was taking effect. + +```meson +env = environment() + +# MY_PATH will be '0:1:2:3' +env.set('MY_PATH', '1') +env.append('MY_PATH', '2') +env.append('MY_PATH', '3') +env.prepend('MY_PATH', '0') +``` + diff --git a/docs/markdown/snippets/nopipe.md b/docs/markdown/snippets/nopipe.md new file mode 100644 index 0000000..431205b --- /dev/null +++ b/docs/markdown/snippets/nopipe.md @@ -0,0 +1,10 @@ +## `-pipe` no longer used by default + +Meson used to add the `-pipe` command line argument to all compilers +that supported it, but no longer does. If you need this, then you can +add it manually. However note that you should not do this unless you +have actually measured that it provides performance improvements. In +our tests we could not find a case where adding `-pipe` made +compilation faster and using `-pipe` [can cause sporadic build +failures in certain +cases](https://github.com/mesonbuild/meson/issues/8508). diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index 9624ed6..5ce27c3 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -1496,18 +1496,33 @@ class Backend: def get_devenv(self) -> build.EnvironmentVariables: env = build.EnvironmentVariables() extra_paths = set() + library_paths = set() for t in self.build.get_targets().values(): cross_built = not self.environment.machines.matches_build_machine(t.for_machine) can_run = not cross_built or not self.environment.need_exe_wrapper() - in_bindir = t.should_install() and not t.get_install_dir(self.environment)[1] - if isinstance(t, build.Executable) and can_run and in_bindir: + in_default_dir = t.should_install() and not t.get_install_dir(self.environment)[1] + if not can_run or not in_default_dir: + continue + tdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(t)) + if isinstance(t, build.Executable): # Add binaries that are going to be installed in bindir into PATH # so they get used by default instead of searching on system when # in developer environment. - extra_paths.add(os.path.join(self.environment.get_build_dir(), self.get_target_dir(t))) + extra_paths.add(tdir) if mesonlib.is_windows() or mesonlib.is_cygwin(): # On windows we cannot rely on rpath to run executables from build # directory. We have to add in PATH the location of every DLL needed. extra_paths.update(self.determine_windows_extra_paths(t, [])) + elif isinstance(t, build.SharedLibrary): + # Add libraries that are going to be installed in libdir into + # LD_LIBRARY_PATH. This allows running system applications using + # that library. + library_paths.add(tdir) + if mesonlib.is_windows() or mesonlib.is_cygwin(): + extra_paths.update(library_paths) + elif mesonlib.is_osx(): + env.prepend('DYLD_LIBRARY_PATH', list(library_paths)) + else: + env.prepend('LD_LIBRARY_PATH', list(library_paths)) env.prepend('PATH', list(extra_paths)) return env diff --git a/mesonbuild/build.py b/mesonbuild/build.py index 547394f..adba208 100644 --- a/mesonbuild/build.py +++ b/mesonbuild/build.py @@ -433,7 +433,7 @@ class EnvironmentVariables: def get_env(self, full_env: T.Dict[str, str]) -> T.Dict[str, str]: env = full_env.copy() for method, name, values, separator in self.envvars: - env[name] = method(full_env, name, values, separator) + env[name] = method(env, name, values, separator) return env class Target: diff --git a/mesonbuild/compilers/fortran.py b/mesonbuild/compilers/fortran.py index 0cff60a..8264638 100644 --- a/mesonbuild/compilers/fortran.py +++ b/mesonbuild/compilers/fortran.py @@ -336,12 +336,6 @@ class IntelFortranCompiler(IntelGnuLikeCompiler, FortranCompiler): def get_preprocess_only_args(self) -> T.List[str]: return ['-cpp', '-EP'] - def get_always_args(self) -> T.List[str]: - """Ifort doesn't have -pipe.""" - val = super().get_always_args() - val.remove('-pipe') - return val - def language_stdlib_only_link_flags(self) -> T.List[str]: return ['-lifcore', '-limf'] diff --git a/mesonbuild/compilers/mixins/clike.py b/mesonbuild/compilers/mixins/clike.py index 3932244..70dde60 100644 --- a/mesonbuild/compilers/mixins/clike.py +++ b/mesonbuild/compilers/mixins/clike.py @@ -155,7 +155,7 @@ class CLikeCompiler(Compiler): ''' Args that are always-on for all C compilers other than MSVC ''' - return ['-pipe'] + self.get_largefile_args() + return self.get_largefile_args() def get_no_stdinc_args(self) -> T.List[str]: return ['-nostdinc'] diff --git a/mesonbuild/compilers/mixins/visualstudio.py b/mesonbuild/compilers/mixins/visualstudio.py index 2b173eb..763d030 100644 --- a/mesonbuild/compilers/mixins/visualstudio.py +++ b/mesonbuild/compilers/mixins/visualstudio.py @@ -412,6 +412,9 @@ class ClangClCompiler(VisualStudioLikeCompiler): super().__init__(target) self.id = 'clang-cl' + # Assembly + self.can_compile_suffixes.add('s') + def has_arguments(self, args: T.List[str], env: 'Environment', code: str, mode: str) -> T.Tuple[bool, bool]: if mode != 'link': args = args + ['-Werror=unknown-argument'] diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index aa83794..d4179d2 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -905,9 +905,9 @@ def write_cmd_line_file(build_dir: str, options: argparse.Namespace) -> None: properties = OrderedDict() if options.cross_file: - properties['cross_file'] = [os.path.abspath(f) for f in options.cross_file] + properties['cross_file'] = options.cross_file if options.native_file: - properties['native_file'] = [os.path.abspath(f) for f in options.native_file] + properties['native_file'] = options.native_file config['options'] = {str(k): str(v) for k, v in options.cmd_line_options.items()} config['properties'] = properties diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index 5cf3dde..7c02631 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -241,9 +241,9 @@ class ConfigureFileHolder(InterpreterObject, ObjectHolder[build.ConfigureFile]): class EnvironmentVariablesHolder(MutableInterpreterObject, ObjectHolder[build.EnvironmentVariables]): - def __init__(self, initial_values=None): + def __init__(self, initial_values=None, subproject: str = ''): MutableInterpreterObject.__init__(self) - ObjectHolder.__init__(self, build.EnvironmentVariables()) + ObjectHolder.__init__(self, build.EnvironmentVariables(), subproject) self.methods.update({'set': self.set_method, 'append': self.append_method, 'prepend': self.prepend_method, @@ -274,12 +274,10 @@ class EnvironmentVariablesHolder(MutableInterpreterObject, ObjectHolder[build.En return separator def warn_if_has_name(self, name: str) -> None: - # Warn when someone tries to use append() or prepend() on an env var - # which already has an operation set on it. People seem to think that - # multiple append/prepend operations stack, but they don't. + # Multiple append/prepend operations was not supported until 0.58.0. if self.held_object.has_name(name): - mlog.warning(f'Overriding previous value of environment variable {name!r} with a new one', - location=self.current_node) + m = f'Overriding previous value of environment variable {name!r} with a new one' + FeatureNew('0.58.0', m).use(self.subproject) @stringArgs @permittedKwargs({'separator'}) @@ -1886,7 +1884,6 @@ class MesonMain(InterpreterObject): InterpreterObject.__init__(self) self.build = build self.interpreter = interpreter - self._found_source_scripts = {} self.methods.update({'get_compiler': self.get_compiler_method, 'is_cross_build': self.is_cross_build_method, 'has_exe_wrapper': self.has_exe_wrapper_method, @@ -1917,25 +1914,10 @@ class MesonMain(InterpreterObject): }) def _find_source_script(self, prog: T.Union[str, mesonlib.File, ExecutableHolder], args): + if isinstance(prog, (ExecutableHolder, ExternalProgramHolder)): return self.interpreter.backend.get_executable_serialisation([unholder(prog)] + args) - # Prefer scripts in the current source directory - search_dir = os.path.join(self.interpreter.environment.source_dir, - self.interpreter.subdir) - key = (prog, search_dir) - if key in self._found_source_scripts: - found = self._found_source_scripts[key] - elif isinstance(prog, mesonlib.File): - prog = prog.rel_to_builddir(self.interpreter.environment.source_dir) - found = ExternalProgram(prog, search_dir=self.interpreter.environment.build_dir) - else: - found = ExternalProgram(prog, search_dir=search_dir) - - if found.found(): - self._found_source_scripts[key] = found - else: - m = 'Script or command {!r} not found or not executable' - raise InterpreterException(m.format(prog)) + found = self.interpreter.func_find_program({}, prog, {}).held_object es = self.interpreter.backend.get_executable_serialisation([found] + args) es.subproject = self.interpreter.subproject return es @@ -2911,9 +2893,12 @@ external dependencies (including libraries) must go to "dependencies".''') os.makedirs(os.path.join(self.build.environment.get_build_dir(), subdir), exist_ok=True) self.global_args_frozen = True - mlog.log() - with mlog.nested(): - mlog.log('Executing subproject', mlog.bold(subp_name), 'method', mlog.bold(method), '\n') + stack = ':'.join(self.subproject_stack + [subp_name]) + m = ['\nExecuting subproject', mlog.bold(stack)] + if method != 'meson': + m += ['method', mlog.bold(method)] + mlog.log(*m,'\n', nested=False) + try: if method == 'meson': return self._do_subproject_meson(subp_name, subdir, default_options, kwargs) @@ -2926,7 +2911,7 @@ external dependencies (including libraries) must go to "dependencies".''') raise except Exception as e: if not required: - with mlog.nested(): + with mlog.nested(subp_name): # Suppress the 'ERROR:' prefix because this exception is not # fatal and VS CI treat any logs with "ERROR:" as fatal. mlog.exception(e, prefix=mlog.yellow('Exception:')) @@ -2938,7 +2923,7 @@ external dependencies (including libraries) must go to "dependencies".''') ast: T.Optional[mparser.CodeBlockNode] = None, build_def_files: T.Optional[T.List[str]] = None, is_translated: bool = False) -> SubprojectHolder: - with mlog.nested(): + with mlog.nested(subp_name): new_build = self.build.copy() subi = Interpreter(new_build, self.backend, subp_name, subdir, self.subproject_dir, self.modules, default_options, ast=ast, is_translated=is_translated) @@ -2975,7 +2960,7 @@ external dependencies (including libraries) must go to "dependencies".''') return self.subprojects[subp_name] def _do_subproject_cmake(self, subp_name, subdir, subdir_abs, default_options, kwargs): - with mlog.nested(): + with mlog.nested(subp_name): new_build = self.build.copy() prefix = self.coredata.options[OptionKey('prefix')].value @@ -2995,7 +2980,7 @@ external dependencies (including libraries) must go to "dependencies".''') ast = cm_int.pretend_to_be_meson(options.target_options) mlog.log() - with mlog.nested(): + with mlog.nested('cmake-ast'): mlog.log('Processing generated meson AST') # Debug print the generated meson file @@ -4009,6 +3994,9 @@ external dependencies (including libraries) must go to "dependencies".''') mlog.warning('''Custom target input \'%s\' can\'t be converted to File object(s). This will become a hard error in the future.''' % kwargs['input'], location=self.current_node) kwargs['env'] = self.unpack_env_kwarg(kwargs) + if 'command' in kwargs and isinstance(kwargs['command'], list) and kwargs['command']: + if isinstance(kwargs['command'][0], str): + kwargs['command'][0] = self.func_find_program(node, kwargs['command'][0], {}) tg = CustomTargetHolder(build.CustomTarget(name, self.subdir, self.subproject, kwargs, backend=self.backend), self) self.add_target(name, tg.held_object) return tg @@ -4600,7 +4588,7 @@ different subdirectory. ''') else: try: - self.validate_within_subproject(a, '') + self.validate_within_subproject(self.subdir, a) except InterpreterException: mlog.warning('include_directories sandbox violation!') print(f'''The project is trying to access the directory {a} which belongs to a different @@ -4716,9 +4704,6 @@ This warning will become a hard error in a future Meson release. elif arg == '-g': mlog.warning(f'Consider using the built-in debug option instead of using "{arg}".', location=self.current_node) - elif arg == '-pipe': - mlog.warning("You don't need to add -pipe, Meson will use it automatically when it is available.", - location=self.current_node) elif arg.startswith('-fsanitize'): mlog.warning(f'Consider using the built-in option for sanitizers instead of using "{arg}".', location=self.current_node) @@ -4772,7 +4757,7 @@ This warning will become a hard error in a future Meson release. raise InterpreterException('environment first argument must be a dictionary or a list') else: initial_values = {} - return EnvironmentVariablesHolder(initial_values) + return EnvironmentVariablesHolder(initial_values, self.subproject) @stringArgs @noKwargs @@ -4832,6 +4817,8 @@ Try setting b_lundef to false instead.'''.format(self.coredata.options[OptionKey # /opt/vendorsdk/src/file_with_license_restrictions.c return project_root = Path(srcdir, self.root_subdir) + if norm == project_root: + return if project_root not in norm.parents: raise InterpreterException(f'Sandbox violation: Tried to grab {inputtype} {norm.name} outside current (sub)project.') if project_root / self.subproject_dir in norm.parents: diff --git a/mesonbuild/mlog.py b/mesonbuild/mlog.py index 15fdb8d..38a4805 100644 --- a/mesonbuild/mlog.py +++ b/mesonbuild/mlog.py @@ -69,7 +69,7 @@ def setup_console() -> None: log_dir = None # type: T.Optional[str] log_file = None # type: T.Optional[T.TextIO] log_fname = 'meson-log.txt' # type: str -log_depth = 0 # type: int +log_depth = [] # type: T.List[str] log_timestamp_start = None # type: T.Optional[float] log_fatal_warnings = False # type: bool log_disable_stdout = False # type: bool @@ -201,7 +201,7 @@ def process_markup(args: T.Sequence[T.Union[AnsiDecorator, str]], keep: bool) -> arr.append(str(arg)) return arr -def force_print(*args: str, **kwargs: T.Any) -> None: +def force_print(*args: str, nested: str, **kwargs: T.Any) -> None: if log_disable_stdout: return iostr = io.StringIO() @@ -209,9 +209,13 @@ def force_print(*args: str, **kwargs: T.Any) -> None: print(*args, **kwargs) raw = iostr.getvalue() - if log_depth > 0: - prepend = '|' * log_depth - raw = prepend + raw.replace('\n', '\n' + prepend, raw.count('\n') - 1) + if log_depth: + prepend = log_depth[-1] + '| ' if nested else '' + lines = [] + for l in raw.split('\n'): + l = l.strip() + lines.append(prepend + l if l else '') + raw = '\n'.join(lines) # _Something_ is going to get printed. try: @@ -246,6 +250,7 @@ def log(*args: T.Union[str, AnsiDecorator], is_error: bool = False, def _log(*args: T.Union[str, AnsiDecorator], is_error: bool = False, **kwargs: T.Any) -> None: + nested = kwargs.pop('nested', True) arr = process_markup(args, False) if log_file is not None: print(*arr, file=log_file, **kwargs) @@ -253,7 +258,7 @@ def _log(*args: T.Union[str, AnsiDecorator], is_error: bool = False, if colorize_console(): arr = process_markup(args, True) if not log_errors_only or is_error: - force_print(*arr, **kwargs) + force_print(*arr, nested=nested, **kwargs) def log_once(*args: T.Union[str, AnsiDecorator], is_error: bool = False, **kwargs: T.Any) -> None: @@ -370,10 +375,10 @@ def format_list(input_list: T.List[str]) -> str: return '' @contextmanager -def nested() -> T.Generator[None, None, None]: +def nested(name: str = '') -> T.Generator[None, None, None]: global log_depth - log_depth += 1 + log_depth.append(name) try: yield finally: - log_depth -= 1 + log_depth.pop() diff --git a/mesonbuild/modules/gnome.py b/mesonbuild/modules/gnome.py index 8a48ca8..dc2979e 100644 --- a/mesonbuild/modules/gnome.py +++ b/mesonbuild/modules/gnome.py @@ -50,11 +50,13 @@ gresource_dep_needed_version = '>= 2.51.1' native_glib_version = None class GnomeModule(ExtensionModule): - gir_dep = None - - install_glib_compile_schemas = False - install_gio_querymodules = [] - install_gtk_update_icon_cache = False + def __init__(self, interpreter: 'Interpreter') -> None: + super().__init__(interpreter) + self.gir_dep = None + self.install_glib_compile_schemas = False + self.install_gio_querymodules = [] + self.install_gtk_update_icon_cache = False + self.devenv = None @staticmethod def _get_native_glib_version(state): @@ -480,6 +482,12 @@ class GnomeModule(ExtensionModule): return girtarget + def _devenv_append(self, varname: str, value: str) -> None: + if self.devenv is None: + self.devenv = build.EnvironmentVariables() + self.interpreter.build.devenv.append(self.devenv) + self.devenv.append(varname, [value]) + def _get_gir_dep(self, state): if not self.gir_dep: self.gir_dep = self._get_native_dep(state, 'gobject-introspection-1.0') @@ -884,6 +892,7 @@ class GnomeModule(ExtensionModule): typelib_cmd += ["--includedir=" + incdir] typelib_target = self._make_typelib_target(state, typelib_output, typelib_cmd, kwargs) + self._devenv_append('GI_TYPELIB_PATH', os.path.join(state.environment.get_build_dir(), state.subdir)) rv = [scan_target, typelib_target] diff --git a/mesonbuild/msetup.py b/mesonbuild/msetup.py index 139b476..5c7bb17 100644 --- a/mesonbuild/msetup.py +++ b/mesonbuild/msetup.py @@ -246,6 +246,10 @@ class MesonApp: b.devenv.append(intr.backend.get_devenv()) build.save(b, dumpfile) if env.first_invocation: + # Use path resolved by coredata because they could have been + # read from a pipe and wrote into a private file. + self.options.cross_file = env.coredata.cross_files + self.options.native_file = env.coredata.config_files coredata.write_cmd_line_file(self.build_dir, self.options) else: coredata.update_cmd_line_file(self.build_dir, self.options) diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index 42963ff..e54740e 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -1129,11 +1129,9 @@ def check_testdata(objs: T.List[TestSerialisation]) -> T.List[TestSerialisation] # Custom waiting primitives for asyncio async def try_wait_one(*awaitables: T.Any, timeout: T.Optional[T.Union[int, float]]) -> None: - try: - await asyncio.wait(awaitables, - timeout=timeout, return_when=asyncio.FIRST_COMPLETED) - except asyncio.TimeoutError: - pass + """Wait for completion of one of the given futures, ignoring timeouts.""" + await asyncio.wait(awaitables, + timeout=timeout, return_when=asyncio.FIRST_COMPLETED) async def queue_iter(q: 'asyncio.Queue[T.Optional[str]]') -> T.AsyncIterator[str]: while True: @@ -1150,14 +1148,37 @@ async def complete(future: asyncio.Future) -> None: except asyncio.CancelledError: pass -async def complete_all(futures: T.Iterable[asyncio.Future]) -> None: - """Wait for completion of all the given futures, ignoring cancellation.""" - while futures: - done, futures = await asyncio.wait(futures, return_when=asyncio.FIRST_EXCEPTION) - # Raise exceptions if needed for all the "done" futures - for f in done: - if not f.cancelled(): +async def complete_all(futures: T.Iterable[asyncio.Future], + timeout: T.Optional[T.Union[int, float]] = None) -> None: + """Wait for completion of all the given futures, ignoring cancellation. + If timeout is not None, raise an asyncio.TimeoutError after the given + time has passed. asyncio.TimeoutError is only raised if some futures + have not completed and none have raised exceptions, even if timeout + is zero.""" + + def check_futures(futures: T.Iterable[asyncio.Future]) -> None: + # Raise exceptions if needed + left = False + for f in futures: + if not f.done(): + left = True + elif not f.cancelled(): f.result() + if left: + raise asyncio.TimeoutError + + # Python is silly and does not have a variant of asyncio.wait with an + # absolute time as deadline. + deadline = None if timeout is None else asyncio.get_event_loop().time() + timeout + while futures and (timeout is None or timeout > 0): + done, futures = await asyncio.wait(futures, timeout=timeout, + return_when=asyncio.FIRST_EXCEPTION) + check_futures(done) + if deadline: + timeout = deadline - asyncio.get_event_loop().time() + + check_futures(futures) + class TestSubprocess: def __init__(self, p: asyncio.subprocess.Process, @@ -1169,6 +1190,7 @@ class TestSubprocess: self.stdo_task = None # type: T.Optional[asyncio.Future[str]] self.stde_task = None # type: T.Optional[asyncio.Future[str]] self.postwait_fn = postwait_fn # type: T.Callable[[], None] + self.all_futures = [] # type: T.List[asyncio.Future] def stdout_lines(self, console_mode: ConsoleUser) -> T.AsyncIterator[str]: q = asyncio.Queue() # type: asyncio.Queue[T.Optional[str]] @@ -1183,9 +1205,11 @@ class TestSubprocess: if self.stdo_task is None and self.stdout is not None: decode_coro = read_decode(self._process.stdout, console_mode) self.stdo_task = asyncio.ensure_future(decode_coro) + self.all_futures.append(self.stdo_task) if self.stderr is not None and self.stderr != asyncio.subprocess.STDOUT: decode_coro = read_decode(self._process.stderr, console_mode) self.stde_task = asyncio.ensure_future(decode_coro) + self.all_futures.append(self.stde_task) return self.stdo_task, self.stde_task @@ -1238,11 +1262,13 @@ class TestSubprocess: p = self._process result = None additional_error = None + + self.all_futures.append(asyncio.ensure_future(p.wait())) try: - await try_wait_one(p.wait(), timeout=timeout) - if p.returncode is None: - additional_error = await self._kill() - result = TestResult.TIMEOUT + await complete_all(self.all_futures, timeout=timeout) + except asyncio.TimeoutError: + additional_error = await self._kill() + result = TestResult.TIMEOUT except asyncio.CancelledError: # The main loop must have seen Ctrl-C. additional_error = await self._kill() diff --git a/mesonbuild/scripts/depfixer.py b/mesonbuild/scripts/depfixer.py index 8ce74ee..c215749 100644 --- a/mesonbuild/scripts/depfixer.py +++ b/mesonbuild/scripts/depfixer.py @@ -13,8 +13,12 @@ # limitations under the License. -import sys, struct -import shutil, subprocess +import sys +import os +import stat +import struct +import shutil +import subprocess import typing as T from ..mesonlib import OrderedSet @@ -120,9 +124,9 @@ class Elf(DataSizes): def __init__(self, bfile: str, verbose: bool = True) -> None: self.bfile = bfile self.verbose = verbose - self.bf = open(bfile, 'r+b') self.sections = [] # type: T.List[SectionHeader] self.dynamic = [] # type: T.List[DynamicEntry] + self.open_bf(bfile) try: (self.ptrsize, self.is_le) = self.detect_elf_type() super().__init__(self.ptrsize, self.is_le) @@ -130,19 +134,40 @@ class Elf(DataSizes): self.parse_sections() self.parse_dynamic() except (struct.error, RuntimeError): - self.bf.close() + self.close_bf() raise + def open_bf(self, bfile: str) -> None: + self.bf = None + self.bf_perms = None + try: + self.bf = open(bfile, 'r+b') + except PermissionError as e: + self.bf_perms = stat.S_IMODE(os.lstat(bfile).st_mode) + os.chmod(bfile, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + try: + self.bf = open(bfile, 'r+b') + except Exception: + os.chmod(bfile, self.bf_perms) + self.bf_perms = None + raise e + + def close_bf(self) -> None: + if self.bf is not None: + if self.bf_perms is not None: + os.fchmod(self.bf.fileno(), self.bf_perms) + self.bf_perms = None + self.bf.close() + self.bf = None + def __enter__(self) -> 'Elf': return self def __del__(self) -> None: - if self.bf: - self.bf.close() + self.close_bf() def __exit__(self, exc_type: T.Any, exc_value: T.Any, traceback: T.Any) -> None: - self.bf.close() - self.bf = None + self.close_bf() def detect_elf_type(self) -> T.Tuple[int, bool]: data = self.bf.read(6) diff --git a/run_project_tests.py b/run_project_tests.py index 6cc89f6..2883e7e 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -34,6 +34,7 @@ import tempfile import time import typing as T import xml.etree.ElementTree as ET +import collections from mesonbuild import build from mesonbuild import environment @@ -370,15 +371,25 @@ def run_ci_commands(raw_log: str) -> T.List[str]: res += ['CI COMMAND {}:\n{}\n'.format(cmd[0], ci_commands[cmd[0]](cmd[1:]))] return res +class OutputMatch: + def __init__(self, how: str, expected: str, count: int) -> None: + self.how = how + self.expected = expected + self.count = count + + def match(self, actual: str) -> bool: + if self.how == "re": + return bool(re.match(self.expected, actual)) + return self.expected == actual + def _compare_output(expected: T.List[T.Dict[str, str]], output: str, desc: str) -> str: if expected: - i = iter(expected) - - def next_expected(i): - # Get the next expected line - item = next(i) + matches = [] + nomatches = [] + for item in expected: how = item.get('match', 'literal') expected = item.get('line') + count = int(item.get('count', -1)) # Simple heuristic to automatically convert path separators for # Windows: @@ -396,23 +407,46 @@ def _compare_output(expected: T.List[T.Dict[str, str]], output: str, desc: str) sub = r'\\\\' expected = re.sub(r'/(?=.*(WARNING|ERROR))', sub, expected) - return how, expected - - try: - how, expected = next_expected(i) - for actual in output.splitlines(): - if how == "re": - match = bool(re.match(expected, actual)) + m = OutputMatch(how, expected, count) + if count == 0: + nomatches.append(m) + else: + matches.append(m) + + + i = 0 + for actual in output.splitlines(): + # Verify this line does not match any unexpected lines (item.count == 0) + for item in nomatches: + if item.match(actual): + return f'unexpected "{item.expected}" found in {desc}' + # If we matched all expected lines, continue to verify there are + # no unexpected line. If nomatches is empty then we are done already. + if i >= len(matches): + if not nomatches: + break + continue + # Check if this line match current expected line + item = matches[i] + if item.match(actual): + if item.count < 0: + # count was not specified, continue with next expected line, + # it does not matter if this line will be matched again or + # not. + i += 1 else: - match = (expected == actual) - if match: - how, expected = next_expected(i) - + # count was specified (must be >0), continue expecting this + # same line. If count reached 0 we continue with next + # expected line but remember that this one must not match + # anymore. + item.count -= 1 + if item.count == 0: + nomatches.append(item) + i += 1 + + if i < len(matches): # reached the end of output without finding expected - return f'expected "{expected}" not found in {desc}' - except StopIteration: - # matched all expected lines - pass + return f'expected "{matches[i].expected}" not found in {desc}' return '' @@ -750,14 +784,16 @@ def load_test_json(t: TestDef, stdout_mandatory: bool) -> T.List[TestDef]: return all_tests -def gather_tests(testdir: Path, stdout_mandatory: bool) -> T.List[TestDef]: - tests = [t.name for t in testdir.iterdir() if t.is_dir()] - tests = [t for t in tests if not t.startswith('.')] # Filter non-tests files (dot files, etc) - test_defs = [TestDef(testdir / t, None, []) for t in tests] +def gather_tests(testdir: Path, stdout_mandatory: bool, only: T.List[str]) -> T.List[TestDef]: all_tests: T.List[TestDef] = [] - for t in test_defs: - all_tests.extend(load_test_json(t, stdout_mandatory)) - + for t in testdir.iterdir(): + # Filter non-tests files (dot files, etc) + if not t.is_dir() or t.name.startswith('.'): + continue + if only and not any(t.name.startswith(prefix) for prefix in only): + continue + test_def = TestDef(t, None, []) + all_tests.extend(load_test_json(test_def, stdout_mandatory)) return sorted(all_tests) @@ -940,11 +976,11 @@ def should_skip_rust(backend: Backend) -> bool: return True return False -def detect_tests_to_run(only: T.List[str], use_tmp: bool) -> T.List[T.Tuple[str, T.List[TestDef], bool]]: +def detect_tests_to_run(only: T.Dict[str, T.List[str]], use_tmp: bool) -> T.List[T.Tuple[str, T.List[TestDef], bool]]: """ Parameters ---------- - only: list of str, optional + only: dict of categories and list of test cases, optional specify names of tests to run Returns @@ -1000,9 +1036,9 @@ def detect_tests_to_run(only: T.List[str], use_tmp: bool) -> T.List[T.Tuple[str, assert categories == ALL_TESTS, 'argparse("--only", choices=ALL_TESTS) need to be updated to match all_tests categories' if only: - all_tests = [t for t in all_tests if t.category in only] + all_tests = [t for t in all_tests if t.category in only.keys()] - gathered_tests = [(t.category, gather_tests(Path('test cases', t.subdir), t.stdout_mandatory), t.skip) for t in all_tests] + gathered_tests = [(t.category, gather_tests(Path('test cases', t.subdir), t.stdout_mandatory, only[t.category]), t.skip) for t in all_tests] return gathered_tests def run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]], @@ -1326,7 +1362,8 @@ if __name__ == '__main__': help='Stop running if test case fails') parser.add_argument('--no-unittests', action='store_true', help='Not used, only here to simplify run_tests.py') - parser.add_argument('--only', help='name of test(s) to run', nargs='+', choices=ALL_TESTS) + parser.add_argument('--only', default=[], + help='name of test(s) to run, in format "category[/name]" where category is one of: ' + ', '.join(ALL_TESTS), nargs='+') parser.add_argument('--cross-file', action='store', help='File describing cross compilation environment.') parser.add_argument('--native-file', action='store', help='File describing native compilation environment.') parser.add_argument('--use-tmpdir', action='store_true', help='Use tmp directory for temporary files.') @@ -1351,8 +1388,15 @@ if __name__ == '__main__': os.chdir(script_dir) check_format() check_meson_commands_work(options) + only = collections.defaultdict(list) + for i in options.only: + try: + cat, case = i.split('/') + only[cat].append(case) + except ValueError: + only[i].append('') try: - all_tests = detect_tests_to_run(options.only, options.use_tmpdir) + all_tests = detect_tests_to_run(only, options.use_tmpdir) (passing_tests, failing_tests, skipped_tests) = run_tests(all_tests, 'meson-test-run', options.failfast, options.extra_args, options.use_tmpdir) except StopException: pass diff --git a/run_unittests.py b/run_unittests.py index 8ef907e..5362616 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -2525,17 +2525,6 @@ class AllPlatformTests(BasePlatformTests): self.build() self.run_tests() - def test_env_ops_dont_stack(self): - ''' - Test that env ops prepend/append do not stack, and that this usage issues a warning - ''' - testdir = os.path.join(self.unit_test_dir, '63 test env does not stack') - out = self.init(testdir) - self.assertRegex(out, r'WARNING: Overriding.*TEST_VAR_APPEND') - self.assertRegex(out, r'WARNING: Overriding.*TEST_VAR_PREPEND') - self.assertNotRegex(out, r'WARNING: Overriding.*TEST_VAR_SET') - self.run_tests() - def test_testrepeat(self): testdir = os.path.join(self.common_test_dir, '207 tap tests') self.init(testdir) @@ -4180,12 +4169,12 @@ class AllPlatformTests(BasePlatformTests): # Parent project warns correctly self.assertRegex(out, "WARNING: Project targeting '>=0.45'.*'0.47.0': dict") # Subprojects warn correctly - self.assertRegex(out, r"\|WARNING: Project targeting '>=0.40'.*'0.44.0': disabler") - self.assertRegex(out, r"\|WARNING: Project targeting '!=0.40'.*'0.44.0': disabler") + self.assertRegex(out, r"\| WARNING: Project targeting '>=0.40'.*'0.44.0': disabler") + self.assertRegex(out, r"\| WARNING: Project targeting '!=0.40'.*'0.44.0': disabler") # Subproject has a new-enough meson_version, no warning self.assertNotRegex(out, "WARNING: Project targeting.*Python") # Ensure a summary is printed in the subproject and the outer project - self.assertRegex(out, r"\|WARNING: Project specifies a minimum meson_version '>=0.40'") + self.assertRegex(out, r"\| WARNING: Project specifies a minimum meson_version '>=0.40'") self.assertRegex(out, r"\| \* 0.44.0: {'disabler'}") self.assertRegex(out, "WARNING: Project specifies a minimum meson_version '>=0.45'") self.assertRegex(out, " * 0.47.0: {'dict'}") diff --git a/test cases/common/128 generated assembly/empty.c b/test cases/common/128 generated assembly/empty.c new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test cases/common/128 generated assembly/empty.c diff --git a/test cases/common/128 generated assembly/meson.build b/test cases/common/128 generated assembly/meson.build index 89b680d..b256630 100644 --- a/test cases/common/128 generated assembly/meson.build +++ b/test cases/common/128 generated assembly/meson.build @@ -6,10 +6,25 @@ if build_machine.system() == 'cygwin' error('MESON_SKIP_TEST: Cygwin is broken and nobody knows how to fix it. Patches welcome.') endif -if ['msvc', 'clang-cl', 'intel-cl'].contains(cc.get_id()) +if ['msvc', 'intel-cl'].contains(cc.get_id()) error('MESON_SKIP_TEST: assembly files cannot be compiled directly by the compiler') endif +crt_workaround = [] +if cc.get_linker_id() == 'lld-link' + # It seems that when building without a .c file, lld-link.exe + # misses the fact that it needs to include the c runtime to + # make a working .dll. So here we add an empty .c file to easily + # pull in crt. + crt_workaround += 'empty.c' + if host_machine.cpu_family() == 'x86' + # x86 assembly needs manual annotation to be compatible with + # Safe Exception Handlers (?) This assembly doesn't have such + # annotation, so just disable the feature. + add_project_link_arguments('/SAFESEH:NO', language : 'c') + endif +endif + cpu = host_machine.cpu_family() supported_cpus = ['arm', 'x86', 'x86_64'] @@ -17,6 +32,11 @@ if not supported_cpus.contains(cpu) error('MESON_SKIP_TEST: unsupported cpu family: ' + cpu) endif +if cc.get_id() == 'clang-cl' and cc.version().version_compare('< 12.0.0') and cpu == 'arm' + # https://reviews.llvm.org/D89622 + error('MESON_SKIP_TEST: arm debug symbols not supported in clang-cl < 12.0.0') +endif + if cc.symbols_have_underscore_prefix() add_project_arguments('-DMESON_TEST__UNDERSCORE_SYMBOL', language : 'c') endif @@ -29,7 +49,8 @@ copygen = generator(copy, arguments : ['@INPUT@', '@OUTPUT@'], output : '@BASENAME@') -l = shared_library('square-gen', copygen.process(input)) +l = shared_library('square-gen', crt_workaround + [copygen.process(input)], + vs_module_defs: 'square.def') test('square-gen-test', executable('square-gen-test', 'main.c', link_with : l)) @@ -38,6 +59,7 @@ copyct = custom_target('square', output : output, command : [copy, '@INPUT@', '@OUTPUT@']) -l = shared_library('square-ct', copyct) +l = shared_library('square-ct', crt_workaround + [copyct], + vs_module_defs: 'square.def') test('square-ct-test', executable('square-ct-test', 'main.c', link_with : l)) diff --git a/test cases/common/128 generated assembly/square-x86.S.in b/test cases/common/128 generated assembly/square-x86.S.in index ee77b81..1a48fc4 100644 --- a/test cases/common/128 generated assembly/square-x86.S.in +++ b/test cases/common/128 generated assembly/square-x86.S.in @@ -1,6 +1,6 @@ #include "symbol-underscore.h" -#ifdef _MSC_VER +#if defined(_MSC_VER) && !defined(__clang__) .386 .MODEL FLAT, C diff --git a/test cases/common/128 generated assembly/square-x86_64.S.in b/test cases/common/128 generated assembly/square-x86_64.S.in index b2cf3eb..d504341 100644 --- a/test cases/common/128 generated assembly/square-x86_64.S.in +++ b/test cases/common/128 generated assembly/square-x86_64.S.in @@ -1,6 +1,6 @@ #include "symbol-underscore.h" -#ifdef _MSC_VER /* MSVC on Windows */ +#if defined(_MSC_VER) && !defined(__clang__) /* MSVC on Windows */ PUBLIC SYMBOL_NAME(square_unsigned) _TEXT SEGMENT diff --git a/test cases/common/128 generated assembly/square.def b/test cases/common/128 generated assembly/square.def new file mode 100644 index 0000000..79f3d65 --- /dev/null +++ b/test cases/common/128 generated assembly/square.def @@ -0,0 +1,2 @@ +EXPORTS + square_unsigned diff --git a/test cases/common/240 includedir violation/meson.build b/test cases/common/240 includedir violation/meson.build index a82069e..0216be6 100644 --- a/test cases/common/240 includedir violation/meson.build +++ b/test cases/common/240 includedir violation/meson.build @@ -1,5 +1,9 @@ project('foo', 'c') +# It is fine to include the root source dir +include_directories('.') +subproject('sub') + # This is here rather than in failing because this needs a # transition period to avoid breaking existing projects. # Once this becomes an error, move this under failing tests. diff --git a/test cases/common/240 includedir violation/subprojects/sub/meson.build b/test cases/common/240 includedir violation/subprojects/sub/meson.build index 7211018..352f4a2 100644 --- a/test cases/common/240 includedir violation/subprojects/sub/meson.build +++ b/test cases/common/240 includedir violation/subprojects/sub/meson.build @@ -1,3 +1,3 @@ project('subproj', 'c') -# This is never actually executed, just here for completeness. +include_directories('.') diff --git a/test cases/common/240 includedir violation/test.json b/test cases/common/240 includedir violation/test.json index d6e56a3..fea19a1 100644 --- a/test cases/common/240 includedir violation/test.json +++ b/test cases/common/240 includedir violation/test.json @@ -1,7 +1,9 @@ { "stdout": [ { - "line": "WARNING: include_directories sandbox violation!" + "line": ".*WARNING: include_directories sandbox violation!", + "match": "re", + "count": 1 } ] } diff --git a/test cases/common/34 run program/check-env.py b/test cases/common/34 run program/check-env.py new file mode 100644 index 0000000..7f106c6 --- /dev/null +++ b/test cases/common/34 run program/check-env.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +import os + +assert os.environ['MY_PATH'] == os.pathsep.join(['0', '1', '2']) + diff --git a/test cases/common/34 run program/meson.build b/test cases/common/34 run program/meson.build index 93897e3..8e472fd 100644 --- a/test cases/common/34 run program/meson.build +++ b/test cases/common/34 run program/meson.build @@ -77,3 +77,9 @@ if dd.found() assert(ret.returncode() == 0, 'failed to run dd: ' + ret.stderr()) assert(ret.stdout() == '', 'stdout is "@0@" instead of empty'.format(ret.stdout())) endif + +env = environment() +env.append('MY_PATH', '1') +env.append('MY_PATH', '2') +env.prepend('MY_PATH', '0') +run_command('check-env.py', env: env, check: true) diff --git a/test cases/unit/63 test env does not stack/meson.build b/test cases/unit/63 test env does not stack/meson.build deleted file mode 100644 index 01f2637..0000000 --- a/test cases/unit/63 test env does not stack/meson.build +++ /dev/null @@ -1,12 +0,0 @@ -project('test env var stacking') - -testenv = environment() -testenv.set('TEST_VAR_SET', 'some-value') -testenv.set('TEST_VAR_APPEND', 'some-value') -testenv.set('TEST_VAR_PREPEND', 'some-value') - -testenv.append('TEST_VAR_APPEND', 'another-value-append', separator: ':') -testenv.prepend('TEST_VAR_PREPEND', 'another-value-prepend', separator: ':') -testenv.set('TEST_VAR_SET', 'another-value-set') - -test('check env', find_program('script.py'), env: testenv) diff --git a/test cases/unit/63 test env does not stack/script.py b/test cases/unit/63 test env does not stack/script.py deleted file mode 100755 index 2a76673..0000000 --- a/test cases/unit/63 test env does not stack/script.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python3 - -import os - -for name in ('append', 'prepend', 'set'): - envname = 'TEST_VAR_' + name.upper() - value = 'another-value-' + name - envvalue = os.environ[envname] - assert (envvalue == value), (name, envvalue) |