diff options
57 files changed, 1269 insertions, 485 deletions
@@ -0,0 +1,107 @@ +#!/usr/bin/env python3 + +# Copyright 2016 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import urllib.request, json, sys, os, shutil, subprocess +import configparser, hashlib + +private_repos = {'meson', 'wrapweb', 'meson-ci'} + +def gh_get(url): + r = urllib.request.urlopen(url) + jd = json.loads(r.read().decode('utf-8')) + return jd + +def list_projects(): + jd = gh_get('https://api.github.com/orgs/mesonbuild/repos') + entries = [entry['name'] for entry in jd] + entries = [e for e in entries if e not in private_repos] + entries.sort() + for i in entries: + print(i) + return 0 + +def unpack(sproj, branch, outdir): + subprocess.check_call(['git', 'clone', '-b', branch, 'https://github.com/mesonbuild/%s.git' % sproj, outdir]) + usfile = os.path.join(outdir, 'upstream.wrap') + assert(os.path.isfile(usfile)) + config = configparser.ConfigParser() + config.read(usfile) + us_url = config['wrap-file']['source_url'] + us = urllib.request.urlopen(us_url).read() + h = hashlib.sha256() + h.update(us) + dig = h.hexdigest() + should = config['wrap-file']['source_hash'] + if dig != should: + print('Incorrect hash on download.') + print(' expected:', dig) + print(' obtained:', should) + return 1 + spdir = os.path.split(outdir)[0] + ofilename = os.path.join(spdir, config['wrap-file']['source_filename']) + ofile = open(ofilename, 'wb') + ofile.write(us) + if 'lead_directory_missing' in config['wrap-file']: + os.mkdir(outdir) + shutil.unpack_archive(ofilename, outdir) + else: + shutil.unpack_archive(ofilename, spdir) + extdir = os.path.join(spdir, config['wrap-file']['directory']) + assert(os.path.isdir(extdir)) + shutil.move(os.path.join(outdir, '.git'), extdir) + subprocess.check_call(['git', 'reset', '--hard'], cwd=extdir) + shutil.rmtree(outdir) + shutil.move(extdir, outdir) + shutil.rmtree(os.path.join(outdir, '.git')) + os.unlink(ofilename) + +def install(sproj): + sproj_dir = os.path.join('subprojects', sproj) + if not os.path.isdir('subprojects'): + print('Run this in your source root and make sure there is a subprojects directory in it.') + return 1 + if os.path.isdir(sproj_dir): + print('Subproject is already there. To update, nuke the dir and reinstall.') + return 1 + blist = gh_get('https://api.github.com/repos/mesonbuild/%s/branches' % sproj) + blist = [b['name'] for b in blist] + blist = [b for b in blist if b != 'master'] + blist.sort() + branch = blist[-1] + print('Using branch', branch) + return unpack(sproj, branch, sproj_dir) + +def run(args): + if len(args) == 0 or args[0] == '-h' or args[0] == '--help': + print(sys.argv[0], 'list/install', 'package_name') + return 1 + command = args[0] + args = args[1:] + if command == 'list': + list_projects() + return 0 + elif command == 'install': + if len(args) != 1: + print('Install requires exactly one argument.') + return 1 + return install(args[0]) + else: + print('Unknown command') + return 1 + +if __name__ == '__main__': + print('This is an emergency wrap downloader. Use only when wrapdb is down.') + sys.exit(run(sys.argv[1:])) diff --git a/man/meson.1 b/man/meson.1 index 5596eeb..31ab52b 100644 --- a/man/meson.1 +++ b/man/meson.1 @@ -1,4 +1,4 @@ -.TH MESON "1" "March 2016" "meson 0.30.0" "User Commands" +.TH MESON "1" "April 2016" "meson 0.31.0" "User Commands" .SH NAME meson - a high productivity build system .SH DESCRIPTION diff --git a/man/mesonconf.1 b/man/mesonconf.1 index be690ae..fe868e7 100644 --- a/man/mesonconf.1 +++ b/man/mesonconf.1 @@ -1,4 +1,4 @@ -.TH MESONCONF "1" "January 2016" "mesonconf 0.29.0" "User Commands" +.TH MESONCONF "1" "April 2016" "mesonconf 0.31.0" "User Commands" .SH NAME mesonconf - a tool to configure Meson builds .SH DESCRIPTION diff --git a/man/mesongui.1 b/man/mesongui.1 index 73d11a4..976c3f7 100644 --- a/man/mesongui.1 +++ b/man/mesongui.1 @@ -1,4 +1,4 @@ -.TH MESONGUI "1" "March 2016" "mesongui 0.30.0" "User Commands" +.TH MESONGUI "1" "April 2016" "mesongui 0.31.0" "User Commands" .SH NAME mesongui - a gui for the Meson build system .SH DESCRIPTION diff --git a/man/mesonintrospect.1 b/man/mesonintrospect.1 index 9fa629c..974a1da 100644 --- a/man/mesonintrospect.1 +++ b/man/mesonintrospect.1 @@ -1,4 +1,4 @@ -.TH MESONCONF "1" "January 2016" "mesonintrospect 0.29.0" "User Commands" +.TH MESONCONF "1" "April 2016" "mesonintrospect 0.31.0" "User Commands" .SH NAME mesonintrospect - a tool to extract information about a Meson build .SH DESCRIPTION diff --git a/man/wraptool.1 b/man/wraptool.1 index f5b7a69..baec06a 100644 --- a/man/wraptool.1 +++ b/man/wraptool.1 @@ -1,4 +1,4 @@ -.TH WRAPTOOL "1" "March 2016" "meson 0.30.0" "User Commands" +.TH WRAPTOOL "1" "April 2016" "meson 0.31.0" "User Commands" .SH NAME wraptool - source dependency downloader .SH DESCRIPTION diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index 8d0b0f6..7c6caa6 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -18,15 +18,14 @@ from .. import dependencies from .. import mesonlib import json import subprocess -from ..coredata import MesonException +from ..mesonlib import MesonException class InstallData(): - def __init__(self, source_dir, build_dir, prefix, depfixer): + def __init__(self, source_dir, build_dir, prefix): self.source_dir = source_dir self.build_dir= build_dir self.prefix = prefix self.targets = [] - self.depfixer = depfixer self.headers = [] self.man = [] self.data = [] @@ -35,6 +34,18 @@ class InstallData(): self.install_scripts = [] self.install_subdirs = [] +class ExecutableSerialisation(): + def __init__(self, name, fname, cmd_args, env, is_cross, exe_wrapper, + workdir, extra_paths): + self.name = name + self.fname = fname + self.cmd_args = cmd_args + self.env = env + self.is_cross = is_cross + self.exe_runner = exe_wrapper + self.workdir = workdir + self.extra_paths = extra_paths + class TestSerialisation: def __init__(self, name, suite, fname, is_cross, exe_wrapper, is_parallel, cmd_args, env, should_fail, valgrind_args, timeout, workdir, extra_paths): @@ -142,7 +153,7 @@ class Backend(): return os.path.relpath(os.path.join('dummyprefixdir', todir),\ os.path.join('dummyprefixdir', fromdir)) - def flatten_object_list(self, target, proj_dir_to_build_root='', include_dir_names=True): + def flatten_object_list(self, target, proj_dir_to_build_root=''): obj_list = [] for obj in target.get_objects(): if isinstance(obj, str): @@ -150,11 +161,40 @@ class Backend(): self.build_to_src, target.get_subdir(), obj) obj_list.append(o) elif isinstance(obj, build.ExtractedObjects): - obj_list += self.determine_ext_objs(obj, proj_dir_to_build_root, include_dir_names) + obj_list += self.determine_ext_objs(obj, proj_dir_to_build_root) else: raise MesonException('Unknown data type in object list.') return obj_list + def serialise_executable(self, exe, cmd_args, workdir, env={}): + import uuid + # Can't just use exe.name here; it will likely be run more than once + scratch_file = 'meson_exe_{0}_{1}.dat'.format(exe.name, + str(uuid.uuid4())[:8]) + exe_data = os.path.join(self.environment.get_scratch_dir(), scratch_file) + with open(exe_data, 'wb') as f: + if isinstance(exe, dependencies.ExternalProgram): + exe_fullpath = exe.fullpath + else: + exe_fullpath = [os.path.join(self.environment.get_build_dir(), + self.get_target_filename(exe))] + is_cross = self.environment.is_cross_build() and \ + self.environment.cross_info.need_cross_compiler() and \ + self.environment.cross_info.need_exe_wrapper() + if is_cross: + exe_wrapper = self.environment.cross_info.config['binaries'].get('exe_wrapper', None) + else: + exe_wrapper = None + if mesonlib.is_windows(): + extra_paths = self.determine_windows_extra_paths(exe) + else: + extra_paths = [] + es = ExecutableSerialisation(exe.name, exe_fullpath, cmd_args, env, + is_cross, exe_wrapper, workdir, + extra_paths) + pickle.dump(es, f) + return exe_data + def serialise_tests(self): test_data = os.path.join(self.environment.get_scratch_dir(), 'meson_test_setup.dat') datafile = open(test_data, 'wb') @@ -164,6 +204,7 @@ class Backend(): datafile = open(benchmark_data, 'wb') self.write_benchmark_file(datafile) datafile.close() + return (test_data, benchmark_data) def has_source_suffix(self, target, suffix): for s in target.get_sources(): @@ -210,28 +251,21 @@ class Backend(): return c raise RuntimeError('Unreachable code') - def determine_ext_objs(self, extobj, proj_dir_to_build_root='', include_dir_names=True): + def object_filename_from_source(self, target, source): + return source.fname.replace('/', '_').replace('\\', '_') + '.' + self.environment.get_object_suffix() + + def determine_ext_objs(self, extobj, proj_dir_to_build_root=''): result = [] targetdir = self.get_target_private_dir(extobj.target) - suffix = '.' + self.environment.get_object_suffix() for osrc in extobj.srclist: - osrc_base = osrc.fname - if not self.source_suffix_in_objs: - osrc_base = '.'.join(osrc.split('.')[:-1]) # If extracting in a subproject, the subproject # name gets duplicated in the file name. pathsegs = osrc.subdir.split(os.sep) if pathsegs[0] == 'subprojects': pathsegs = pathsegs[2:] fixedpath = os.sep.join(pathsegs) - if include_dir_names: - objbase = osrc_base.replace('/', '_').replace('\\', '_') - else: - # vs2010 backend puts all obj files without directory prefixes into build dir, so just - # use the file name without a directory (will be stripped by os.path.basename() below). - objbase = osrc_base - objname = os.path.join(proj_dir_to_build_root, - targetdir, os.path.basename(objbase) + suffix) + objname = os.path.join(proj_dir_to_build_root, targetdir, + self.object_filename_from_source(extobj.target, osrc)) result.append(objname) return result @@ -259,8 +293,6 @@ class Backend(): commands += self.environment.coredata.external_args[compiler.get_language()] commands += target.get_extra_args(compiler.get_language()) commands += compiler.get_buildtype_args(self.environment.coredata.get_builtin_option('buildtype')) - if self.environment.coredata.base_options.get('b_coverage', False): - commands += compiler.get_coverage_args() if self.environment.coredata.get_builtin_option('werror'): commands += compiler.get_werror_args() if isinstance(target, build.SharedLibrary): @@ -325,7 +357,9 @@ class Backend(): fname = exe.fullpath else: fname = [os.path.join(self.environment.get_build_dir(), self.get_target_filename(t.get_exe()))] - is_cross = self.environment.is_cross_build() and self.environment.cross_info.need_cross_compiler() + is_cross = self.environment.is_cross_build() and \ + self.environment.cross_info.need_cross_compiler() and \ + self.environment.cross_info.need_exe_wrapper() if is_cross: exe_wrapper = self.environment.cross_info.config['binaries'].get('exe_wrapper', None) else: @@ -376,8 +410,9 @@ class Backend(): def exe_object_to_cmd_array(self, exe): if self.environment.is_cross_build() and \ - isinstance(exe, build.BuildTarget) and exe.is_cross: - if 'exe_wrapper' not in self.environment.cross_info: + self.environment.cross_info.need_exe_wrapper() and \ + isinstance(exe, build.BuildTarget) and exe.is_cross: + if 'exe_wrapper' not in self.environment.cross_info.config: s = 'Can not use target %s as a generator because it is cross-built\n' s += 'and no exe wrapper is defined. You might want to set it to native instead.' s = s % exe.name @@ -388,6 +423,25 @@ class Backend(): exe_arr = exe.get_command() return exe_arr + def replace_extra_args(self, args, genlist): + final_args = [] + for a in args: + if a == '@EXTRA_ARGS@': + final_args += genlist.get_extra_args() + else: + final_args.append(a) + return final_args + + def get_custom_target_provided_libraries(self, target): + libs = [] + for t in target.get_generated_sources(): + if not isinstance(t, build.CustomTarget): + continue + for f in t.output: + if self.environment.is_library(f): + libs.append(os.path.join(self.get_target_dir(t), f)) + return libs + def eval_custom_target_command(self, target, absolute_paths=False): if not absolute_paths: ofilenames = [os.path.join(self.get_target_dir(target), i) for i in target.output] diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index b6ce421..bd6f4db 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -18,10 +18,9 @@ from .. import build from .. import mlog from .. import dependencies from .. import compilers -from ..mesonlib import File +from ..mesonlib import File, MesonException from .backends import InstallData from ..build import InvalidArguments -from ..coredata import MesonException import os, sys, pickle, re import subprocess, shutil @@ -129,7 +128,6 @@ class NinjaBackend(backends.Backend): def __init__(self, build): super().__init__(build) - self.source_suffix_in_objs = True self.ninja_filename = 'build.ninja' self.fortran_deps = {} self.all_outputs = {} @@ -137,7 +135,10 @@ class NinjaBackend(backends.Backend): def detect_vs_dep_prefix(self, outfile, tempfilename): '''VS writes its dependency in a locale dependent format. Detect the search prefix to use.''' - if shutil.which('cl') is None: + # Of course there is another program called 'cl' on + # some platforms. Let's just require that on Windows + # cl points to msvc. + if not mesonlib.is_windows() or shutil.which('cl') is None: return outfile outfile.close() open(os.path.join(self.environment.get_scratch_dir(), 'incdetect.c'), @@ -179,10 +180,12 @@ int dummy; self.generate_tests(outfile) outfile.write('# Install rules\n\n') self.generate_install(outfile) - if self.environment.coredata.base_options.get('b_coverage', False): + if 'b_coverage' in self.environment.coredata.base_options and \ + self.environment.coredata.base_options['b_coverage'].value: outfile.write('# Coverage rules\n\n') self.generate_coverage_rules(outfile) outfile.write('# Suffix\n\n') + self.generate_utils(outfile) self.generate_ending(outfile) # Only ovewrite the old build file after the new one has been # fully created. @@ -194,7 +197,10 @@ int dummy; def generate_compdb(self): ninja_exe = environment.detect_ninja() builddir = self.environment.get_build_dir() - jsondb = subprocess.check_output([ninja_exe, '-t', 'compdb', 'c_COMPILER', 'cpp_COMPILER'], cwd=builddir) + try: + jsondb = subprocess.check_output([ninja_exe, '-t', 'compdb', 'c_COMPILER', 'cpp_COMPILER'], cwd=builddir) + except Exception: + raise MesonException('Could not create compilation database.') open(os.path.join(builddir, 'compile_commands.json'), 'wb').write(jsondb) # Get all generated headers. Any source file might need them so @@ -231,7 +237,8 @@ int dummy; self.generate_cs_target(target, outfile) return if 'vala' in self.environment.coredata.compilers.keys() and self.has_vala(target): - vala_output_files = self.generate_vala_compile(target, outfile) + vc = self.environment.coredata.compilers['vala'] + vala_output_files = self.generate_vala_compile(vc, target, outfile) gen_src_deps += vala_output_files if 'swift' in self.environment.coredata.compilers.keys() and self.has_swift(target): self.generate_swift_target(target, outfile) @@ -335,6 +342,7 @@ int dummy; def generate_custom_target(self, target, outfile): (srcs, ofilenames, cmd) = self.eval_custom_target_command(target) deps = [] + desc = 'Generating {0} with a {1} command.' for i in target.get_dependencies(): # FIXME, should not grab element at zero but rather expand all. if isinstance(i, list): @@ -358,9 +366,23 @@ int dummy; tmp = [tmp] for fname in tmp: elem.add_dep(os.path.join(self.get_target_dir(d), fname)) + # Windows doesn't have -rpath, so for EXEs that need DLLs built within + # the project, we need to set PATH so the DLLs are found. We use + # a serialized executable wrapper for that and check if the + # CustomTarget command needs extra paths first. + if mesonlib.is_windows() and \ + self.determine_windows_extra_paths(target.command[0]): + exe_data = self.serialise_executable(target.command[0], cmd[1:], + # All targets are built from the build dir + self.environment.get_build_dir()) + cmd = [sys.executable, self.environment.get_build_command(), + '--internal', 'exe', exe_data] + cmd_type = 'meson_exe.py custom' + else: + cmd_type = 'custom' elem.add_item('COMMAND', cmd) - elem.add_item('description', 'Generating %s with a custom command.' % target.name) + elem.add_item('description', desc.format(target.name, cmd_type)) elem.write(outfile) self.processed_targets[target.name + target.type_suffix()] = True @@ -388,9 +410,11 @@ int dummy; if isinstance(texe, build.Executable): abs_exe = os.path.join(self.environment.get_build_dir(), self.get_target_filename(texe)) deps.append(self.get_target_filename(texe)) - if self.environment.is_cross_build() \ - and self.environment.cross_info.config['binaries'].get('exe_wrapper', None) is not None: - cmd += [self.environment.cross_info.config['binaries']['exe_wrapper']] + if self.environment.is_cross_build() and \ + self.environment.cross_info.need_exe_wrapper(): + exe_wrap = self.environment.cross_info.config['binaries'].get('exe_wrapper', None) + if exe_wrap is not None: + cmd += [exe_wrap] cmd.append(abs_exe) else: cmd.append(target.command) @@ -407,26 +431,27 @@ int dummy; if gcovr_exe: added_rule = True elem = NinjaBuildElement(self.all_outputs, 'coverage-xml', 'CUSTOM_COMMAND', '') - elem.add_item('COMMAND', [gcovr_exe, '-x', '-r', self.environment.get_build_dir(),\ + elem.add_item('COMMAND', [gcovr_exe, '-x', '-r', self.environment.get_source_dir(),\ '-o', os.path.join(self.environment.get_log_dir(), 'coverage.xml')]) elem.add_item('DESC', 'Generating XML coverage report.') elem.write(outfile) elem = NinjaBuildElement(self.all_outputs, 'coverage-text', 'CUSTOM_COMMAND', '') - elem.add_item('COMMAND', [gcovr_exe, '-r', self.environment.get_build_dir(),\ + elem.add_item('COMMAND', [gcovr_exe, '-r', self.environment.get_source_dir(),\ '-o', os.path.join(self.environment.get_log_dir(), 'coverage.txt')]) elem.add_item('DESC', 'Generating text coverage report.') elem.write(outfile) if lcov_exe and genhtml_exe: added_rule = True - phony_elem = NinjaBuildElement(self.all_outputs, 'coverage-html', 'phony', 'coveragereport/index.html') + htmloutdir = os.path.join(self.environment.get_log_dir(), 'coveragereport') + covinfo = os.path.join(self.environment.get_log_dir(), 'coverage.info') + phony_elem = NinjaBuildElement(self.all_outputs, 'coverage-html', 'phony', os.path.join(htmloutdir, 'index.html')) phony_elem.write(outfile) - - elem = NinjaBuildElement(self.all_outputs, 'coveragereport/index.html', 'CUSTOM_COMMAND', '') + elem = NinjaBuildElement(self.all_outputs, os.path.join(htmloutdir, 'index.html'), 'CUSTOM_COMMAND', '') command = [lcov_exe, '--directory', self.environment.get_build_dir(),\ - '--capture', '--output-file', 'coverage.info', '--no-checksum',\ + '--capture', '--output-file', covinfo, '--no-checksum',\ '&&', genhtml_exe, '--prefix', self.environment.get_build_dir(),\ - '--output-directory', self.environment.get_log_dir(), '--title', 'Code coverage',\ - '--legend', '--show-details', 'coverage.info'] + '--output-directory', htmloutdir, '--title', 'Code coverage',\ + '--legend', '--show-details', covinfo] elem.add_item('COMMAND', command) elem.add_item('DESC', 'Generating HTML coverage report.') elem.write(outfile) @@ -437,10 +462,9 @@ int dummy; script_root = self.environment.get_script_dir() install_script = os.path.join(script_root, 'meson_install.py') install_data_file = os.path.join(self.environment.get_scratch_dir(), 'install.dat') - depfixer = [sys.executable, self.environment.get_build_command(), '--internal', 'depfixer'] d = InstallData(self.environment.get_source_dir(), self.environment.get_build_dir(), - self.environment.get_prefix(), depfixer) + self.environment.get_prefix()) elem = NinjaBuildElement(self.all_outputs, 'install', 'CUSTOM_COMMAND', 'PHONY') elem.add_dep('all') elem.add_item('DESC', 'Installing files.') @@ -511,12 +535,13 @@ int dummy; assert(isinstance(de, build.Data)) subdir = de.install_dir for f in de.sources: + plain_f = os.path.split(f)[1] if de.in_sourcetree: srcprefix = self.environment.get_source_dir() else: srcprefix = self.environment.get_build_dir() srcabs = os.path.join(srcprefix, de.source_subdir, f) - dstabs = os.path.join(subdir, f) + dstabs = os.path.join(subdir, plain_f) i = [srcabs, dstabs] d.data.append(i) @@ -545,11 +570,15 @@ int dummy; elem.write(outfile) def generate_tests(self, outfile): - self.serialise_tests() + (test_data, benchmark_data) = self.serialise_tests() valgrind = environment.find_valgrind() script_root = self.environment.get_script_dir() - test_data = os.path.join(self.environment.get_scratch_dir(), 'meson_test_setup.dat') - cmd = [sys.executable, self.environment.get_build_command(), '--internal', 'test', test_data] + cmd = [ sys.executable, self.environment.get_build_command(), '--internal', 'test' ] + if not self.environment.coredata.get_builtin_option('stdsplit'): + cmd += ['--no-stdsplit'] + if self.environment.coredata.get_builtin_option('errorlogs'): + cmd += ['--print-errorlogs'] + cmd += [ test_data ] elem = NinjaBuildElement(self.all_outputs, 'test', 'CUSTOM_COMMAND', ['all', 'PHONY']) elem.add_item('COMMAND', cmd) elem.add_item('DESC', 'Running all tests.') @@ -566,7 +595,6 @@ int dummy; # And then benchmarks. benchmark_script = os.path.join(script_root, 'meson_benchmark.py') - benchmark_data = os.path.join(self.environment.get_scratch_dir(), 'meson_benchmark_setup.dat') cmd = [sys.executable, self.environment.get_build_command(), '--internal', 'benchmark', benchmark_data] elem = NinjaBuildElement(self.all_outputs, 'benchmark', 'CUSTOM_COMMAND', ['all', 'PHONY']) elem.add_item('COMMAND', cmd) @@ -737,7 +765,7 @@ int dummy; break return result - def generate_vala_compile(self, target, outfile): + def generate_vala_compile(self, compiler, target, outfile): """Vala is compiled into C. Set up all necessary build steps here.""" valac = self.environment.coredata.compilers['vala'] (src, vapi_src) = self.split_vala_sources(target.get_sources()) @@ -757,7 +785,10 @@ int dummy; generated_c_files = [] outputs = [vapiname] - args = ['-d', self.get_target_private_dir(target)] + args = [] + args += self.build.get_global_args(compiler) + args += compiler.get_buildtype_args(self.environment.coredata.get_builtin_option('buildtype')) + args += ['-d', self.get_target_private_dir(target)] args += ['-C']#, '-o', cname] if not isinstance(target, build.Executable): outputs.append(hname) @@ -1297,13 +1328,7 @@ rule FORTRAN_DEP_HACK relout = self.get_target_private_dir(target) args = [x.replace("@SOURCE_DIR@", self.build_to_src).replace("@BUILD_DIR@", relout) for x in args] - final_args = [] - for a in args: - if a == '@EXTRA_ARGS@': - final_args += genlist.get_extra_args() - else: - final_args.append(a) - cmdlist = exe_arr + final_args + cmdlist = exe_arr + self.replace_extra_args(args, genlist) elem = NinjaBuildElement(self.all_outputs, outfiles, 'CUSTOM_COMMAND', infilename) if len(extra_dependencies) > 0: elem.add_dep(extra_dependencies) @@ -1605,6 +1630,8 @@ rule FORTRAN_DEP_HACK else: soversion = None commands += linker.get_soname_args(target.name, abspath, soversion) + if target.vs_module_defs and hasattr(linker, 'gen_vs_module_defs_args'): + commands += linker.gen_vs_module_defs_args(target.vs_module_defs.rel_to_builddir(self.build_to_src)) elif isinstance(target, build.StaticLibrary): commands += linker.get_std_link_args() else: @@ -1632,8 +1659,6 @@ rule FORTRAN_DEP_HACK commands += dep.get_link_args() commands += linker.build_rpath_args(self.environment.get_build_dir(),\ self.determine_rpath_dirs(target), target.install_rpath) - if self.environment.coredata.base_options.get('b_coverage', False): - commands += linker.get_coverage_link_args() custom_target_libraries = self.get_custom_target_provided_libraries(target) commands += extra_args commands += custom_target_libraries @@ -1646,16 +1671,6 @@ rule FORTRAN_DEP_HACK elem.add_item('LINK_ARGS', commands) return elem - def get_custom_target_provided_libraries(self, target): - libs = [] - for t in target.get_generated_sources(): - if not isinstance(t, build.CustomTarget): - continue - for f in t.output: - if self.environment.is_library(f): - libs.append(os.path.join(self.get_target_dir(t), f)) - return libs - def determine_rpath_dirs(self, target): link_deps = target.get_all_link_deps() result = [] @@ -1731,6 +1746,16 @@ rule FORTRAN_DEP_HACK other_deps.append(outfilename) return (src_deps, other_deps) + # For things like scan-build and other helper tools we might have. + def generate_utils(self, outfile): + cmd = [sys.executable, self.environment.get_build_command(), + '--internal', 'scanbuild', self.environment.source_dir, self.environment.build_dir, + sys.executable, self.environment.get_build_command()] + elem = NinjaBuildElement(self.all_outputs, 'scan-build', 'CUSTOM_COMMAND', 'PHONY') + elem.add_item('COMMAND', cmd) + elem.add_item('pool', 'console') + elem.write(outfile) + def generate_ending(self, outfile): targetlist = [self.get_target_filename(t) for t in self.build.get_targets().values()\ if not isinstance(t, build.RunTarget)] @@ -1747,7 +1772,8 @@ rule FORTRAN_DEP_HACK elem = NinjaBuildElement(self.all_outputs, 'clean', 'CUSTOM_COMMAND', 'PHONY') elem.add_item('COMMAND', [ninja_command, '-t', 'clean']) elem.add_item('description', 'Cleaning') - if self.environment.coredata.base_options.get('b_coverage', False): + if 'b_coverage' in self.environment.coredata.base_options and \ + self.environment.coredata.base_options['b_coverage'].value: self.generate_gcov_clean(outfile) elem.add_dep('clean-gcda') elem.add_dep('clean-gcno') diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py index ec10d4c..82d0dc9 100644 --- a/mesonbuild/backend/vs2010backend.py +++ b/mesonbuild/backend/vs2010backend.py @@ -14,15 +14,18 @@ import os, sys import pickle +import re from mesonbuild import compilers +from mesonbuild.build import BuildTarget +from mesonbuild.mesonlib import File from . import backends from .. import build from .. import dependencies from .. import mlog import xml.etree.ElementTree as ET import xml.dom.minidom -from ..coredata import MesonException +from ..mesonlib import MesonException from ..environment import Environment class RegenInfo(): @@ -35,42 +38,72 @@ class Vs2010Backend(backends.Backend): def __init__(self, build): super().__init__(build) self.project_file_version = '10.0.30319.1' - # foo.c compiles to foo.obj, not foo.c.obj - self.source_suffix_in_objs = False + self.sources_conflicts = {} + + def object_filename_from_source(self, target, source): + basename = os.path.basename(source.fname) + filename_without_extension = '.'.join(basename.split('.')[:-1]) + if basename in self.sources_conflicts[target.get_id()]: + # If there are multiple source files with the same basename, we must resolve the conflict + # by giving each a unique object output file. + filename_without_extension = '.'.join(source.fname.split('.')[:-1]).replace('/', '_').replace('\\', '_') + return filename_without_extension + '.' + self.environment.get_object_suffix() + + def resolve_source_conflicts(self): + for name, target in self.build.targets.items(): + if not isinstance(target, BuildTarget): + continue + conflicts = {} + for s in target.get_sources(): + if hasattr(s, 'held_object'): + s = s.held_object + if not isinstance(s, File): + continue + basename = os.path.basename(s.fname) + conflicting_sources = conflicts.get(basename, None) + if conflicting_sources is None: + conflicting_sources = [] + conflicts[basename] = conflicting_sources + conflicting_sources.append(s) + self.sources_conflicts[target.get_id()] = {name: src_conflicts for name, src_conflicts in conflicts.items() + if len(src_conflicts) > 1} def generate_custom_generator_commands(self, target, parent_node): - all_output_files = [] + generator_output_files = [] commands = [] inputs = [] outputs = [] + custom_target_include_dirs = [] + custom_target_output_files = [] for genlist in target.get_generated_sources(): if isinstance(genlist, build.CustomTarget): - all_output_files += [os.path.join(self.get_target_dir(genlist), i) for i in genlist.output] + custom_target_output_files += [os.path.join(self.get_target_dir(genlist), i) for i in genlist.output] + idir = self.relpath(self.get_target_dir(genlist), self.get_target_dir(target)) + if idir not in custom_target_include_dirs: + custom_target_include_dirs.append(idir) else: generator = genlist.get_generator() exe = generator.get_exe() infilelist = genlist.get_infilelist() outfilelist = genlist.get_outfilelist() - if isinstance(exe, build.BuildTarget): - exe_file = os.path.join(self.environment.get_build_dir(), self.get_target_filename(exe)) - else: - exe_file = exe.get_command()[0] + exe_arr = self.exe_object_to_cmd_array(exe) base_args = generator.get_arglist() + target_private_dir = self.relpath(self.get_target_private_dir(target), self.get_target_dir(target)) for i in range(len(infilelist)): if len(infilelist) == len(outfilelist): - sole_output = os.path.join(self.get_target_private_dir(target), outfilelist[i]) + sole_output = os.path.join(target_private_dir, outfilelist[i]) else: sole_output = '' curfile = infilelist[i] infilename = os.path.join(self.environment.get_source_dir(), curfile) outfiles = genlist.get_outputs_for(curfile) - outfiles = [os.path.join(self.get_target_private_dir(target), of) for of in outfiles] - all_output_files += outfiles + outfiles = [os.path.join(target_private_dir, of) for of in outfiles] + generator_output_files += outfiles args = [x.replace("@INPUT@", infilename).replace('@OUTPUT@', sole_output)\ for x in base_args] - args = [x.replace("@SOURCE_DIR@", self.environment.get_source_dir()).replace("@BUILD_DIR@", self.get_target_private_dir(target)) + args = [x.replace("@SOURCE_DIR@", self.environment.get_source_dir()).replace("@BUILD_DIR@", target_private_dir) for x in args] - fullcmd = [exe_file] + args + fullcmd = exe_arr + self.replace_extra_args(args, genlist) commands.append(' '.join(self.special_quote(fullcmd))) inputs.append(infilename) outputs.extend(outfiles) @@ -83,9 +116,10 @@ class Vs2010Backend(backends.Backend): ET.SubElement(cbs, 'Message').text = 'Generating custom sources.' pg = ET.SubElement(parent_node, 'PropertyGroup') ET.SubElement(pg, 'CustomBuildBeforeTargets').text = 'ClCompile' - return all_output_files + return generator_output_files, custom_target_output_files, custom_target_include_dirs def generate(self, interp): + self.resolve_source_conflicts() self.interpreter = interp self.platform = 'Win32' self.buildtype = self.environment.coredata.get_builtin_option('buildtype') @@ -231,6 +265,8 @@ class Vs2010Backend(backends.Backend): lang = self.lang_from_source_file(i) if lang not in languages: languages.append(lang) + elif self.environment.is_library(i): + pass else: # Everything that is not an object or source file is considered a header. headers.append(i) @@ -348,6 +384,10 @@ class Vs2010Backend(backends.Backend): lang = Vs2010Backend.lang_from_source_file(source_file) ET.SubElement(parent_node, "AdditionalOptions").text = ' '.join(extra_args[lang]) + ' %(AdditionalOptions)' + @classmethod + def quote_define_cmdline(cls, arg): + return re.sub(r'^([-/])D(.*?)="(.*)"$', r'\1D\2=\"\3\"', arg) + def gen_vcxproj(self, target, ofname, guid, compiler): mlog.debug('Generating vcxproj %s.' % target.name) entrypoint = 'WinMainCRTStartup' @@ -403,8 +443,12 @@ class Vs2010Backend(backends.Backend): ET.SubElement(type_config, 'WholeProgramOptimization').text = 'false' ET.SubElement(type_config, 'UseDebugLibraries').text = 'true' ET.SubElement(root, 'Import', Project='$(VCTargetsPath)\Microsoft.Cpp.props') - generated_files = self.generate_custom_generator_commands(target, root) + generated_files, custom_target_output_files, generated_files_include_dirs = self.generate_custom_generator_commands(target, root) (gen_src, gen_hdrs, gen_objs, gen_langs) = self.split_sources(generated_files) + (custom_src, custom_hdrs, custom_objs, custom_langs) = self.split_sources(custom_target_output_files) + gen_src += custom_src + gen_hdrs += custom_hdrs + gen_langs += custom_langs direlem = ET.SubElement(root, 'PropertyGroup') fver = ET.SubElement(direlem, '_ProjectFileVersion') fver.text = self.project_file_version @@ -421,11 +465,8 @@ class Vs2010Backend(backends.Backend): clconf = ET.SubElement(compiles, 'ClCompile') opt = ET.SubElement(clconf, 'Optimization') opt.text = 'disabled' - inc_dirs = [proj_to_src_dir, self.get_target_private_dir(target)] - cur_dir = target.subdir - if cur_dir == '': - cur_dir= '.' - inc_dirs.append(cur_dir) + inc_dirs = ['.', self.relpath(self.get_target_private_dir(target), self.get_target_dir(target)), + proj_to_src_dir] + generated_files_include_dirs extra_args = {'c': [], 'cpp': []} for l, args in self.environment.coredata.external_args.items(): @@ -450,6 +491,9 @@ class Vs2010Backend(backends.Backend): except AttributeError: pass + for l, args in extra_args.items(): + extra_args[l] = [Vs2010Backend.quote_define_cmdline(x) for x in args] + languages += gen_langs has_language_specific_args = any(l != extra_args['c'] for l in extra_args.values()) additional_options_set = False @@ -469,6 +513,10 @@ class Vs2010Backend(backends.Backend): curdir = os.path.join(d.curdir, i) inc_dirs.append(self.relpath(curdir, target.subdir)) # build dir inc_dirs.append(os.path.join(proj_to_src_root, curdir)) # src dir + for i in d.get_extra_build_dirs(): + curdir = os.path.join(d.curdir, i) + inc_dirs.append(self.relpath(curdir, target.subdir)) # build dir + inc_dirs.append('%(AdditionalIncludeDirectories)') ET.SubElement(clconf, 'AdditionalIncludeDirectories').text = ';'.join(inc_dirs) preproc = ET.SubElement(clconf, 'PreprocessorDefinitions') @@ -519,15 +567,19 @@ class Vs2010Backend(backends.Backend): ET.SubElement(link, "AdditionalOptions").text = ' '.join(extra_link_args) additional_links = [] - for t in target.link_targets: + for t in target.get_dependencies(): lobj = self.build.targets[t.get_id()] rel_path = self.relpath(lobj.subdir, target.subdir) linkname = os.path.join(rel_path, lobj.get_import_filename()) additional_links.append(linkname) + for lib in self.get_custom_target_provided_libraries(target): + additional_links.append(self.relpath(lib, self.get_target_dir(target))) additional_objects = [] - for o in self.flatten_object_list(target, down, include_dir_names=False): + for o in self.flatten_object_list(target, down): assert(isinstance(o, str)) additional_objects.append(o) + for o in custom_objs: + additional_objects.append(self.relpath(o, self.get_target_dir(target))) if len(additional_links) > 0: additional_links.append('%(AdditionalDependencies)') ET.SubElement(link, 'AdditionalDependencies').text = ';'.join(additional_links) @@ -548,17 +600,18 @@ class Vs2010Backend(backends.Backend): targetmachine = ET.SubElement(link, 'TargetMachine') targetmachine.text = 'MachineX86' - if len(headers) + len(gen_hdrs) > 0: + extra_files = target.extra_files + if len(headers) + len(gen_hdrs) + len(extra_files) > 0: inc_hdrs = ET.SubElement(root, 'ItemGroup') for h in headers: relpath = h.rel_to_builddir(proj_to_src_root) ET.SubElement(inc_hdrs, 'CLInclude', Include=relpath) for h in gen_hdrs: - if isinstance(h, str): - relpath = h - else: - relpath = h.rel_to_builddir(proj_to_src_root) - ET.SubElement(inc_hdrs, 'CLInclude', Include = relpath) + ET.SubElement(inc_hdrs, 'CLInclude', Include=h) + for h in target.extra_files: + relpath = os.path.join(proj_to_src_dir, h) + ET.SubElement(inc_hdrs, 'CLInclude', Include=relpath) + if len(sources) + len(gen_src) + len(pch_sources) > 0: inc_src = ET.SubElement(root, 'ItemGroup') for s in sources: @@ -566,9 +619,11 @@ class Vs2010Backend(backends.Backend): inc_cl = ET.SubElement(inc_src, 'CLCompile', Include=relpath) self.add_pch(inc_cl, proj_to_src_dir, pch_sources, s) self.add_additional_options(s, inc_cl, extra_args, additional_options_set) + basename = os.path.basename(s.fname) + if basename in self.sources_conflicts[target.get_id()]: + ET.SubElement(inc_cl, 'ObjectFileName').text = "$(IntDir)" + self.object_filename_from_source(target, s) for s in gen_src: - relpath = self.relpath(s, target.subdir) - inc_cl = ET.SubElement(inc_src, 'CLCompile', Include=relpath) + inc_cl = ET.SubElement(inc_src, 'CLCompile', Include=s) self.add_pch(inc_cl, proj_to_src_dir, pch_sources, s) self.add_additional_options(s, inc_cl, extra_args, additional_options_set) for lang in pch_sources: @@ -733,7 +788,6 @@ if %%errorlevel%% neq 0 goto :VCEnd''' ET.SubElement(midl, 'ProxyFileName').text = '%(Filename)_p.c' postbuild = ET.SubElement(action, 'PostBuildEvent') ET.SubElement(postbuild, 'Message') - test_data = os.path.join(self.environment.get_scratch_dir(), 'meson_test_setup.dat') test_command = [sys.executable, self.environment.get_build_command(), '--internal', @@ -747,14 +801,12 @@ endlocal & call :cmErrorLevel %%errorlevel%% & goto :cmDone exit /b %%1 :cmDone if %%errorlevel%% neq 0 goto :VCEnd''' + test_data = self.serialise_tests()[0] ET.SubElement(postbuild, 'Command').text =\ cmd_templ % ('" "'.join(test_command), test_data) ET.SubElement(root, 'Import', Project='$(VCTargetsPath)\Microsoft.Cpp.targets') tree = ET.ElementTree(root) tree.write(ofname, encoding='utf-8', xml_declaration=True) - datafile = open(test_data, 'wb') - self.serialise_tests() - datafile.close() # ElementTree can not do prettyprinting so do it manually #doc = xml.dom.minidom.parse(ofname) #open(ofname, 'w').write(doc.toprettyxml()) diff --git a/mesonbuild/backend/xcodebackend.py b/mesonbuild/backend/xcodebackend.py index eb8b0b9..0ce90ce 100644 --- a/mesonbuild/backend/xcodebackend.py +++ b/mesonbuild/backend/xcodebackend.py @@ -17,7 +17,7 @@ from .. import build from .. import mesonlib import uuid, os, sys -from ..coredata import MesonException +from ..mesonlib import MesonException class XCodeBackend(backends.Backend): def __init__(self, build): @@ -64,7 +64,7 @@ class XCodeBackend(backends.Backend): def generate(self, interp): self.interpreter = interp - self.serialise_tests() + test_data = self.serialise_tests()[0] self.generate_filemap() self.generate_buildmap() self.generate_buildstylemap() @@ -92,7 +92,7 @@ class XCodeBackend(backends.Backend): self.generate_pbx_group() self.generate_pbx_native_target() self.generate_pbx_project() - self.generate_pbx_shell_build_phase() + self.generate_pbx_shell_build_phase(test_data) self.generate_pbx_sources_build_phase() self.generate_pbx_target_dependency() self.generate_xc_build_configuration() @@ -480,7 +480,7 @@ class XCodeBackend(backends.Backend): self.write_line('};') self.ofile.write('/* End PBXProject section */\n') - def generate_pbx_shell_build_phase(self): + def generate_pbx_shell_build_phase(self, test_data): self.ofile.write('\n/* Begin PBXShellScriptBuildPhase section */\n') self.write_line('%s = {' % self.test_command_id) self.indent_level += 1 @@ -496,7 +496,6 @@ class XCodeBackend(backends.Backend): self.write_line('shellPath = /bin/sh;') script_root = self.environment.get_script_dir() test_script = os.path.join(script_root, 'meson_test.py') - test_data = os.path.join(self.environment.get_scratch_dir(), 'meson_test_setup.dat') cmd = [sys.executable, test_script, test_data, '--wd', self.environment.get_build_dir()] cmdstr = ' '.join(["'%s'" % i for i in cmd]) self.write_line('shellScript = "%s";' % cmdstr) diff --git a/mesonbuild/build.py b/mesonbuild/build.py index 93e20c8..1e9a1bb 100644 --- a/mesonbuild/build.py +++ b/mesonbuild/build.py @@ -17,7 +17,7 @@ from . import environment from . import dependencies from . import mlog import copy, os -from .mesonlib import File, flatten +from .mesonlib import File, flatten, MesonException known_basic_kwargs = {'install' : True, 'c_pch' : True, @@ -47,7 +47,7 @@ known_shlib_kwargs.update({'version' : True, 'soversion' : True, 'name_prefix' : True, 'name_suffix' : True, - }) + 'vs_module_defs' : True}) backslash_explanation = \ '''Compiler arguments have a backslash "\\" character. This is unfortunately not @@ -71,7 +71,7 @@ We are fully aware that these are not really usable or pleasant ways to do this but it's the best we can do given the way shell quoting works. ''' -class InvalidArguments(coredata.MesonException): +class InvalidArguments(MesonException): pass class Build: @@ -298,10 +298,10 @@ class BuildTarget(): srclist = [srclist] for src in srclist: if not isinstance(src, str): - raise coredata.MesonException('Extraction arguments must be strings.') + raise MesonException('Extraction arguments must be strings.') src = File(False, self.subdir, src) if src not in self.sources: - raise coredata.MesonException('Tried to extract unknown source %s.' % src) + raise MesonException('Tried to extract unknown source %s.' % src) obj_src.append(src) return ExtractedObjects(self, obj_src) @@ -702,6 +702,7 @@ class SharedLibrary(BuildTarget): def __init__(self, name, subdir, subproject, is_cross, sources, objects, environment, kwargs): self.version = None self.soversion = None + self.vs_module_defs = None super().__init__(name, subdir, subproject, is_cross, sources, objects, environment, kwargs); if len(self.sources) > 0 and self.sources[0].endswith('.cs'): prefix = 'lib' @@ -725,6 +726,12 @@ class SharedLibrary(BuildTarget): self.set_version(kwargs['version']) if 'soversion' in kwargs: self.set_soversion(kwargs['soversion']) + if 'vs_module_defs' in kwargs: + path = kwargs['vs_module_defs'] + if (os.path.isabs(path)): + self.vs_module_defs = File.from_absolute_file(path) + else: + self.vs_module_defs = File.from_source_file(environment.source_dir, self.subdir, path) def check_unknown_kwargs(self, kwargs): self.check_unknown_kwargs_int(kwargs, known_shlib_kwargs) @@ -863,11 +870,12 @@ class CustomTarget: self.install = kwargs['install'] if not isinstance(self.install, bool): raise InvalidArguments('"install" must be boolean.') - if 'install_dir' not in kwargs: - raise InvalidArguments('"install_dir" not specified.') - self.install_dir = kwargs['install_dir'] - if not(isinstance(self.install_dir, str)): - raise InvalidArguments('"install_dir" must be a string.') + if self.install: + if 'install_dir' not in kwargs: + raise InvalidArguments('"install_dir" not specified.') + self.install_dir = kwargs['install_dir'] + if not(isinstance(self.install_dir, str)): + raise InvalidArguments('"install_dir" must be a string.') else: self.install = False self.build_always = kwargs.get('build_always', False) diff --git a/mesonbuild/compilers.py b/mesonbuild/compilers.py index 3079c5e..25eefc6 100644 --- a/mesonbuild/compilers.py +++ b/mesonbuild/compilers.py @@ -16,7 +16,7 @@ import subprocess, os.path import tempfile from .import mesonlib from . import mlog -from .coredata import MesonException +from .mesonlib import MesonException from . import coredata """This file contains the data files of all compilers Meson knows @@ -55,7 +55,9 @@ def is_library(fname): return suffix in lib_suffixes gnulike_buildtype_args = {'plain' : [], - 'debug' : ['-g'], + # -O0 is passed for improved debugging information with gcc + # See https://github.com/mesonbuild/meson/pull/509 + 'debug' : ['-O0', '-g'], 'debugoptimized' : ['-O2', '-g'], 'release' : ['-O3']} @@ -126,7 +128,7 @@ base_options = { 'off'), 'b_coverage': coredata.UserBooleanOption('b_coverage', 'Enable coverage tracking.', - True), + False), } def sanitizer_compile_args(value): @@ -258,6 +260,9 @@ class Compiler(): def has_header(self, *args, **kwargs): raise EnvironmentException('Language %s does not support header checks.' % self.language) + def has_header_symbol(self, *args, **kwargs): + raise EnvironmentException('Language %s does not support header symbol checks.' % self.language) + def compiles(self, *args, **kwargs): raise EnvironmentException('Language %s does not support compile checks.' % self.language) @@ -285,6 +290,9 @@ class Compiler(): def find_library(self, libname, extra_dirs): raise EnvironmentException('Language {} does not support library finding.'.format(self.language)) + def get_library_dirs(self): + return [] + class CCompiler(Compiler): def __init__(self, exelist, version, is_cross, exe_wrapper=None): super().__init__(exelist, version) @@ -344,6 +352,9 @@ class CCompiler(Compiler): def get_compile_only_args(self): return ['-c'] + def get_no_optimization_args(self): + return ['-O0'] + def get_output_args(self, target): return ['-o', target] @@ -372,6 +383,16 @@ class CCompiler(Compiler): def get_std_shared_lib_link_args(self): return ['-shared'] + def get_library_dirs(self): + output = subprocess.Popen(self.exelist + ['--print-search-dirs'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + (stdo, _) = output.communicate() + stdo = stdo.decode('utf-8') + for line in stdo.split('\n'): + if line.startswith('libraries:'): + libstr = line.split('=', 1)[1] + return libstr.split(':') + return [] + def can_compile(self, filename): suffix = filename.split('.')[-1] if suffix == 'c' or suffix == 'h': @@ -447,6 +468,14 @@ int someSymbolHereJustForFun; ''' return self.compiles(templ % hname, extra_args) + def has_header_symbol(self, hname, symbol, prefix, extra_args=[]): + templ = '''{2} +#include <{0}> +int main () {{ {1}; }}''' + # Pass -O0 to ensure that the symbol isn't optimized away + extra_args += self.get_no_optimization_args() + return self.compiles(templ.format(hname, symbol, prefix), extra_args) + def compile(self, code, srcname, extra_args=[]): commands = self.get_exelist() commands.append(srcname) @@ -484,7 +513,6 @@ int someSymbolHereJustForFun; return p.returncode == 0 def links(self, code, extra_args = []): - suflen = len(self.default_suffix) (fd, srcname) = tempfile.mkstemp(suffix='.'+self.default_suffix) os.close(fd) (fd, dstname) = tempfile.mkstemp() @@ -492,7 +520,8 @@ int someSymbolHereJustForFun; ofile = open(srcname, 'w') ofile.write(code) ofile.close() - extra_args = extra_args + self.get_output_args(dstname) + extra_args = self.unix_link_flags_to_native(extra_args) + \ + self.get_output_args(dstname) p = self.compile(code, srcname, extra_args) try: os.remove(dstname) @@ -511,7 +540,7 @@ int someSymbolHereJustForFun; ofile.close() exename = srcname + '.exe' # Is guaranteed to be executable on every platform. commands = self.get_exelist() - commands += extra_args + commands += self.unix_link_flags_to_native(extra_args) commands.append(srcname) commands += self.get_output_args(exename) p = subprocess.Popen(commands, cwd=os.path.split(srcname)[0], stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -548,16 +577,30 @@ int someSymbolHereJustForFun; return RunResult(True, pe.returncode, so, se) def cross_sizeof(self, element, prefix, env, extra_args=[]): - templ = '''%s + element_exists_templ = '''#include <stdio.h> +{0} +int main(int argc, char **argv) {{ + {1} something; +}} +''' + templ = '''#include <stdio.h> +%s int temparray[%d-sizeof(%s)]; ''' try: extra_args += env.cross_info.config['properties'][self.language + '_args'] except KeyError: pass + extra_args += self.get_no_optimization_args() + if not self.compiles(element_exists_templ.format(prefix, element)): + return -1 for i in range(1, 1024): code = templ % (prefix, i, element) if self.compiles(code, extra_args): + if self.id == 'msvc': + # MSVC refuses to construct an array of zero size, so + # the test only succeeds when i is sizeof(element) + 1 + return i - 1 return i raise EnvironmentException('Cross checking sizeof overflowed.') @@ -580,6 +623,11 @@ int main(int argc, char **argv) { return int(res.stdout) def cross_alignment(self, typename, env, extra_args=[]): + type_exists_templ = '''#include <stdio.h> +int main(int argc, char **argv) {{ + {0} something; +}} +''' templ = '''#include<stddef.h> struct tmp { char c; @@ -592,9 +640,16 @@ int testarray[%d-offsetof(struct tmp, target)]; extra_args += env.cross_info.config['properties'][self.language + '_args'] except KeyError: pass + extra_args += self.get_no_optimization_args() + if not self.compiles(type_exists_templ.format(typename)): + return -1 for i in range(1, 1024): code = templ % (typename, i) if self.compiles(code, extra_args): + if self.id == 'msvc': + # MSVC refuses to construct an array of zero size, so + # the test only succeeds when i is sizeof(element) + 1 + return i - 1 return i raise EnvironmentException('Cross checking offsetof overflowed.') @@ -625,15 +680,49 @@ int main(int argc, char **argv) { return align def has_function(self, funcname, prefix, env, extra_args=[]): - # This fails (returns true) if funcname is a ptr or a variable. - # The correct check is a lot more difficult. - # Fix this to do that eventually. - templ = '''%s -int main(int argc, char **argv) { - void *ptr = (void*)(%s); - return 0; -}; -''' + """ + First, this function looks for the symbol in the default libraries + provided by the compiler (stdlib + a few others usually). If that + fails, it checks if any of the headers specified in the prefix provide + an implementation of the function, and if that fails, it checks if it's + implemented as a compiler-builtin. + """ + # Define the symbol to something else in case it is defined by the + # includes or defines listed by the user `{0}` or by the compiler. + # Then, undef the symbol to get rid of it completely. + templ = ''' + #define {1} meson_disable_define_of_{1} + {0} + #undef {1} + ''' + + # Override any GCC internal prototype and declare our own definition for + # the symbol. Use char because that's unlikely to be an actual return + # value for a function which ensures that we override the definition. + templ += ''' + #ifdef __cplusplus + extern "C" + #endif + char {1} (); + ''' + + # glibc defines functions that are not available on Linux as stubs that + # fail with ENOSYS (such as e.g. lchmod). In this case we want to fail + # instead of detecting the stub as a valid symbol. + stubs_fail = ''' + #if defined __stub_{1} || defined __stub___{1} + fail fail fail this function is not going to work + #endif + ''' + templ += stubs_fail + + # And finally the actual function call + templ += ''' + int + main () + {{ + return {1} (); + }}''' varname = 'has function ' + funcname varname = varname.replace(' ', '_') if self.is_cross: @@ -642,7 +731,21 @@ int main(int argc, char **argv) { if isinstance(val, bool): return val raise EnvironmentException('Cross variable {0} is not a boolean.'.format(varname)) - return self.compiles(templ % (prefix, funcname), extra_args) + if self.links(templ.format(prefix, funcname), extra_args): + return True + # Add -O0 to ensure that the symbol isn't optimized away by the compiler + extra_args += self.get_no_optimization_args() + # Sometimes the implementation is provided by the header, or the header + # redefines the symbol to be something else. In that case, we want to + # still detect the function. We still want to fail if __stub_foo or + # _stub_foo are defined, of course. + if self.links('{0}\n' + stubs_fail + '\nint main() {{ {1}; }}'.format(prefix, funcname), extra_args): + return True + # Some functions like alloca() are defined as compiler built-ins which + # are inlined by the compiler, so test for that instead. Built-ins are + # special functions that ignore all includes and defines, so we just + # directly try to link via main(). + return self.links('int main() {{ {0}; }}'.format('__builtin_' + funcname), extra_args) def has_member(self, typename, membername, prefix, extra_args=[]): templ = '''%s @@ -662,16 +765,28 @@ void bar() { return self.compiles(templ % (prefix, typename), extra_args) def find_library(self, libname, extra_dirs): + # First try if we can just add the library as -l. code = '''int main(int argc, char **argv) { return 0; } ''' - args = [] - for i in extra_dirs: - args += self.get_linker_search_args(i) - args.append('-l' + libname) - if self.links(code, extra_args=args): - return args + # Gcc + co seem to prefer builtin lib dirs to -L dirs. + # Only try to find std libs if no extra dirs specified. + if len(extra_dirs) == 0: + args = ['-l' + libname] + if self.links(code, extra_args=args): + return args + # Not found? Try to find the library file itself. + extra_dirs += self.get_library_dirs() + suffixes = ['so', 'dylib', 'lib', 'dll', 'a'] + for d in extra_dirs: + for suffix in suffixes: + trial = os.path.join(d, 'lib' + libname + '.' + suffix) + if os.path.isfile(trial): + return trial + trial2 = os.path.join(d, libname + '.' + suffix) + if os.path.isfile(trial2): + return trial2 return None def thread_flags(self): @@ -1044,6 +1159,11 @@ class ValaCompiler(Compiler): suffix = filename.split('.')[-1] return suffix in ('vala', 'vapi') + def get_buildtype_args(self, buildtype): + if buildtype == 'debug' or buildtype == 'debugoptimized': + return ['--debug'] + return [] + class RustCompiler(Compiler): def __init__(self, exelist, version): super().__init__(exelist, version) @@ -1226,6 +1346,9 @@ class VisualStudioCCompiler(CCompiler): def get_compile_only_args(self): return ['/c'] + def get_no_optimization_args(self): + return ['/Od'] + def get_output_args(self, target): if target.endswith('.exe'): return ['/Fe' + target] @@ -1252,6 +1375,13 @@ class VisualStudioCCompiler(CCompiler): def get_std_shared_lib_link_args(self): return ['/DLL'] + def gen_vs_module_defs_args(self, defsfile): + if not isinstance(defsfile, str): + raise RuntimeError('Module definitions file should be str') + # With MSVC, DLLs only export symbols that are explicitly exported, + # so if a module defs file is specified, we use that to export symbols + return ['/DEF:' + defsfile] + def gen_pch_args(self, header, source, pchname): objname = os.path.splitext(pchname)[0] + '.obj' return (objname, ['/Yc' + header, '/Fp' + pchname, '/Fo' + objname ]) @@ -1277,19 +1407,6 @@ class VisualStudioCCompiler(CCompiler): def build_rpath_args(self, build_dir, rpath_paths, install_rpath): return [] - def find_library(self, libname, extra_dirs): - code = '''int main(int argc, char **argv) { - return 0; -} - ''' - args = [] - for i in extra_dirs: - args += self.get_linker_search_args(i) - args.append(libname + '.lib') - if self.links(code, extra_args=args): - return args - return None - # FIXME, no idea what these should be. def thread_flags(self): return [] @@ -1420,7 +1537,7 @@ class GnuCCompiler(CCompiler): self.warn_args = {'1': ['-Wall', '-Winvalid-pch'], '2': ['-Wall', '-Wextra', '-Winvalid-pch'], '3' : ['-Wall', '-Wpedantic', '-Wextra', '-Winvalid-pch']} - self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize'] + self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize', 'b_coverage'] if self.gcc_type != GCC_OSX: self.base_options.append('b_lundef') @@ -1485,7 +1602,7 @@ class GnuObjCCompiler(ObjCCompiler): self.warn_args = {'1': ['-Wall', '-Winvalid-pch'], '2': ['-Wall', '-Wextra', '-Winvalid-pch'], '3' : ['-Wall', '-Wpedantic', '-Wextra', '-Winvalid-pch']} - self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize'] + self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize', 'b_coverage'] if self.gcc_type != GCC_OSX: self.base_options.append('b_lundef') @@ -1513,7 +1630,7 @@ class GnuObjCPPCompiler(ObjCPPCompiler): self.warn_args = {'1': ['-Wall', '-Winvalid-pch', '-Wnon-virtual-dtor'], '2': ['-Wall', '-Wextra', '-Winvalid-pch', '-Wnon-virtual-dtor'], '3' : ['-Wall', '-Wpedantic', '-Wextra', '-Winvalid-pch', '-Wnon-virtual-dtor']} - self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize'] + self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize', 'b_coverage'] if self.gcc_type != GCC_OSX: self.base_options.append('b_lundef') @@ -1533,7 +1650,7 @@ class ClangObjCCompiler(GnuObjCCompiler): def __init__(self, exelist, version, cltype, is_cross, exe_wrapper=None): super().__init__(exelist, version, is_cross, exe_wrapper) self.id = 'clang' - self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize'] + self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize', 'b_coverage'] self.clang_type = cltype if self.clang_type != CLANG_OSX: self.base_options.append('b_lundef') @@ -1543,7 +1660,7 @@ class ClangObjCPPCompiler(GnuObjCPPCompiler): super().__init__(exelist, version, is_cross, exe_wrapper) self.id = 'clang' self.clang_type = cltype - self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize'] + self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize', 'b_coverage'] if self.clang_type != CLANG_OSX: self.base_options.append('b_lundef') @@ -1555,7 +1672,7 @@ class ClangCCompiler(CCompiler): self.warn_args = {'1': ['-Wall', '-Winvalid-pch'], '2': ['-Wall', '-Wextra', '-Winvalid-pch'], '3' : ['-Wall', '-Wpedantic', '-Wextra', '-Winvalid-pch']} - self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize'] + self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize', 'b_coverage'] if self.clang_type != CLANG_OSX: self.base_options.append('b_lundef') @@ -1603,7 +1720,7 @@ class GnuCPPCompiler(CPPCompiler): self.warn_args = {'1': ['-Wall', '-Winvalid-pch', '-Wnon-virtual-dtor'], '2': ['-Wall', '-Wextra', '-Winvalid-pch', '-Wnon-virtual-dtor'], '3': ['-Wall', '-Wpedantic', '-Wextra', '-Winvalid-pch', '-Wnon-virtual-dtor']} - self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize'] + self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize', 'b_coverage'] if self.gcc_type != GCC_OSX: self.base_options.append('b_lundef') @@ -1625,7 +1742,10 @@ class GnuCPPCompiler(CPPCompiler): def get_options(self): opts = {'cpp_std' : coredata.UserComboOption('cpp_std', 'C++ language standard to use', ['none', 'c++03', 'c++11', 'c++14'], - 'none')} + 'none'), + 'cpp_debugstl': coredata.UserBooleanOption('cpp_debugstl', + 'STL debug mode', + False)} if self.gcc_type == GCC_MINGW: opts.update({ 'cpp_winlibs': coredata.UserStringArrayOption('c_winlibs', 'Standard Win libraries to link against', @@ -1638,6 +1758,8 @@ class GnuCPPCompiler(CPPCompiler): std = options['cpp_std'] if std.value != 'none': args.append('-std=' + std.value) + if options['cpp_debugstl'].value: + args.append('-D_GLIBCXX_DEBUG=1') return args def get_option_link_args(self, options): @@ -1653,7 +1775,7 @@ class ClangCPPCompiler(CPPCompiler): '2': ['-Wall', '-Wextra', '-Winvalid-pch', '-Wnon-virtual-dtor'], '3': ['-Wall', '-Wpedantic', '-Wextra', '-Winvalid-pch', '-Wnon-virtual-dtor']} self.clang_type = cltype - self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize'] + self.base_options = ['b_pch', 'b_lto', 'b_pgo', 'b_sanitize', 'b_coverage'] if self.clang_type != CLANG_OSX: self.base_options.append('b_lundef') diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index d085532..8227340 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -13,35 +13,9 @@ # limitations under the License. import pickle, os, uuid +from .mesonlib import MesonException, default_libdir, default_libexecdir, default_prefix -version = '0.31.0.dev1' - -build_types = ['plain', 'debug', 'debugoptimized', 'release'] -layouts = ['mirror', 'flat'] -warning_levels = ['1', '2', '3'] -libtypelist = ['shared', 'static'] - -builtin_options = {'buildtype': True, - 'strip': True, - 'coverage': True, - 'unity': True, - 'prefix': True, - 'libdir' : True, - 'libexecdir' : True, - 'bindir' : True, - 'includedir' : True, - 'datadir' : True, - 'mandir' : True, - 'localedir' : True, - 'werror' : True, - 'warning_level': True, - 'layout' : True, - 'default_library': True, - } - -class MesonException(Exception): - def __init__(self, *args, **kwargs): - Exception.__init__(self, *args, **kwargs) +version = '0.32.0.dev1' class UserOption: def __init__(self, name, description, choices): @@ -73,7 +47,7 @@ class UserStringOption(UserOption): class UserBooleanOption(UserOption): def __init__(self, name, description, value): - super().__init__(name, description, '[true, false]') + super().__init__(name, description, [ True, False ]) self.set_value(value) def tobool(self, thing): @@ -140,7 +114,6 @@ class CoreData(): self.regen_guid = str(uuid.uuid4()).upper() self.target_guids = {} self.version = version - self.builtin_options = {} self.init_builtins(options) self.user_options = {} self.compiler_options = {} @@ -158,36 +131,21 @@ class CoreData(): self.modules = {} def init_builtins(self, options): - self.builtin_options['prefix'] = UserStringOption('prefix', 'Installation prefix', options.prefix) - self.builtin_options['libdir'] = UserStringOption('libdir', 'Library dir', options.libdir) - self.builtin_options['libexecdir'] = UserStringOption('libexecdir', 'Library executables dir', options.libexecdir) - self.builtin_options['bindir'] = UserStringOption('bindir', 'Executable dir', options.bindir) - self.builtin_options['includedir'] = UserStringOption('includedir', 'Include dir', options.includedir) - self.builtin_options['datadir'] = UserStringOption('datadir', 'Data directory', options.datadir) - self.builtin_options['mandir'] = UserStringOption('mandir', 'Man page dir', options.mandir) - self.builtin_options['localedir'] = UserStringOption('localedir', 'Locale dir', options.localedir) - self.builtin_options['backend'] = UserStringOption('backend', 'Backend to use', options.backend) - self.builtin_options['buildtype'] = UserComboOption('buildtype', 'Build type', build_types, options.buildtype) - self.builtin_options['strip'] = UserBooleanOption('strip', 'Strip on install', options.strip) - self.builtin_options['unity'] = UserBooleanOption('unity', 'Unity build', options.unity) - self.builtin_options['warning_level'] = UserComboOption('warning_level', 'Warning level', warning_levels, options.warning_level) - self.builtin_options['werror'] = UserBooleanOption('werror', 'Warnings are errors', options.werror) - self.builtin_options['layout'] = UserComboOption('layout', 'Build dir layout', layouts, options.layout) - self.builtin_options['default_library'] = UserComboOption('default_library', 'Default_library type', libtypelist, options.default_library) + self.builtins = {} + for key in get_builtin_options(): + args = [key] + builtin_options[key][1:-1] + [ getattr(options, key, get_builtin_option_default(key)) ] + self.builtins[key] = builtin_options[key][0](*args) def get_builtin_option(self, optname): - if optname in self.builtin_options: - return self.builtin_options[optname].value - raise RuntimeError('Tried to get unknown builtin option %s' % optname) + if optname in self.builtins: + return self.builtins[optname].value + raise RuntimeError('Tried to get unknown builtin option %s.' % optname) def set_builtin_option(self, optname, value): - if optname in self.builtin_options: - self.builtin_options[optname].set_value(value) + if optname in self.builtins: + self.builtins[optname].set_value(value) else: - raise RuntimeError('Tried to set unknown builtin option %s' % optname) - - def is_builtin_option(self, optname): - return optname in self.builtin_options + raise RuntimeError('Tried to set unknown builtin option %s.' % optname) def load(filename): obj = pickle.load(open(filename, 'rb')) @@ -203,6 +161,59 @@ def save(obj, filename): raise RuntimeError('Fatal version mismatch corruption.') pickle.dump(obj, open(filename, 'wb')) +def get_builtin_options(): + return list(builtin_options.keys()) + +def is_builtin_option(optname): + return optname in get_builtin_options() + +def get_builtin_option_choices(optname): + if is_builtin_option(optname): + if builtin_options[optname][0] == UserStringOption: + return None + elif builtin_options[optname][0] == UserBooleanOption: + return [ True, False ] + else: + return builtin_options[optname][2] + else: + raise RuntimeError('Tried to get the supported values for an unknown builtin option \'%s\'.' % optname) + +def get_builtin_option_description(optname): + if is_builtin_option(optname): + return builtin_options[optname][1] + else: + raise RuntimeError('Tried to get the description for an unknown builtin option \'%s\'.' % optname) + +def get_builtin_option_default(optname): + if is_builtin_option(optname): + o = builtin_options[optname] + if o[0] == UserComboOption: + return o[3] + return o[2] + else: + raise RuntimeError('Tried to get the default value for an unknown builtin option \'%s\'.' % optname) + +builtin_options = { + 'buildtype' : [ UserComboOption, 'Build type to use.', [ 'plain', 'debug', 'debugoptimized', 'release' ], 'debug' ], + 'strip' : [ UserBooleanOption, 'Strip targets on install.', False ], + 'unity' : [ UserBooleanOption, 'Unity build.', False ], + 'prefix' : [ UserStringOption, 'Installation prefix.', default_prefix() ], + 'libdir' : [ UserStringOption, 'Library directory.', default_libdir() ], + 'libexecdir' : [ UserStringOption, 'Library executable directory.', default_libexecdir() ], + 'bindir' : [ UserStringOption, 'Executable directory.', 'bin' ], + 'includedir' : [ UserStringOption, 'Header file directory.', 'include' ], + 'datadir' : [ UserStringOption, 'Data file directory.', 'share' ], + 'mandir' : [ UserStringOption, 'Manual page directory.', 'share/man' ], + 'localedir' : [ UserStringOption, 'Locale data directory.', 'share/locale' ], + 'werror' : [ UserBooleanOption, 'Treat warnings as errors.', False ], + 'warning_level' : [ UserComboOption, 'Compiler warning level to use.', [ '1', '2', '3' ], '1'], + 'layout' : [ UserComboOption, 'Build directory layout.', ['mirror', 'flat' ], 'mirror' ], + 'default_library' : [ UserComboOption, 'Default library type.', [ 'shared', 'static' ], 'shared' ], + 'backend' : [ UserComboOption, 'Backend to use.', [ 'ninja', 'vs2010', 'xcode' ], 'ninja' ], + 'stdsplit' : [ UserBooleanOption, 'Split stdout and stderr in test logs.', True ], + 'errorlogs' : [ UserBooleanOption, "Whether to print the logs from failing tests.", False ], + } + forbidden_target_names = {'clean': None, 'clean-gcno': None, 'clean-gcda': None, @@ -218,4 +229,5 @@ forbidden_target_names = {'clean': None, 'benchmark': None, 'install': None, 'build.ninja': None, + 'scan-build': None, } diff --git a/mesonbuild/dependencies.py b/mesonbuild/dependencies.py index b9f5c89..d8081b0 100644 --- a/mesonbuild/dependencies.py +++ b/mesonbuild/dependencies.py @@ -22,7 +22,7 @@ import re import os, stat, glob, subprocess, shutil import sysconfig -from . coredata import MesonException +from . mesonlib import MesonException from . import mlog from . import mesonlib diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index c381ab5..41e8531 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, re, subprocess +import os, re, subprocess, platform from . import coredata, mesonlib from .compilers import * import configparser build_filename = 'meson.build' -class EnvironmentException(coredata.MesonException): +class EnvironmentException(mesonlib.MesonException): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -52,6 +52,33 @@ def detect_ninja(): if p.returncode == 0: return n +def detect_cpu_family(): + """ + Python is inconsistent in its platform module. + It returns different values for the same cpu. + For x86 it might return 'x86', 'i686' or somesuch. + Do some canonicalization. + """ + trial = platform.machine().lower() + if trial.startswith('i') and trial.endswith('86'): + return 'x86' + if trial.startswith('arm'): + return 'arm' + if trial == 'amd64': + return 'x86_64' + # Add fixes here as bugs are reported. + return trial + +def detect_cpu(): + trial = platform.machine().lower() + if trial == 'amd64': + return 'x86_64' + # Add fixes here as bugs are reported. + return trial + +def detect_system(): + return platform.system().lower() + class Environment(): private_dir = 'meson-private' @@ -131,7 +158,8 @@ class Environment(): coredata.save(self.coredata, cdf) def get_script_dir(self): - return os.path.join(os.path.dirname(self.meson_script_file), '../scripts') + import mesonbuild.scripts + return os.path.dirname(mesonbuild.scripts.__file__) def get_log_dir(self): return self.log_dir @@ -181,7 +209,10 @@ class Environment(): compilers = [self.cross_info.config['binaries']['c']] ccache = [] is_cross = True - exe_wrap = self.cross_info.config['binaries'].get('exe_wrapper', None) + if self.cross_info.need_exe_wrapper(): + exe_wrap = self.cross_info.config['binaries'].get('exe_wrapper', None) + else: + exe_wrap = [] elif evar in os.environ: compilers = os.environ[evar].split() ccache = [] @@ -192,6 +223,7 @@ class Environment(): ccache = self.detect_ccache() is_cross = False exe_wrap = None + popen_exceptions = {} for compiler in compilers: try: basename = os.path.basename(compiler).lower() @@ -199,9 +231,10 @@ class Environment(): arg = '/?' else: arg = '--version' - p = subprocess.Popen([compiler] + [arg], stdout=subprocess.PIPE, + p = subprocess.Popen([compiler, arg], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - except OSError: + except OSError as e: + popen_exceptions[' '.join([compiler, arg])] = e continue (out, err) = p.communicate() out = out.decode(errors='ignore') @@ -232,14 +265,22 @@ class Environment(): # everything else to stdout. Why? Lord only knows. version = re.search(Environment.version_regex, err).group() return VisualStudioCCompiler([compiler], version, is_cross, exe_wrap) - raise EnvironmentException('Unknown compiler(s): "' + ', '.join(compilers) + '"') + errmsg = 'Unknown compiler(s): "' + ', '.join(compilers) + '"' + if popen_exceptions: + errmsg += '\nThe follow exceptions were encountered:' + for (c, e) in popen_exceptions.items(): + errmsg += '\nRunning "{0}" gave "{1}"'.format(c, e) + raise EnvironmentException(errmsg) def detect_fortran_compiler(self, want_cross): evar = 'FC' if self.is_cross_build() and want_cross: compilers = [self.cross_info['fortran']] is_cross = True - exe_wrap = self.cross_info.get('exe_wrapper', None) + if self.cross_info.need_exe_wrapper(): + exe_wrap = self.cross_info.get('exe_wrapper', None) + else: + exe_wrap = [] elif evar in os.environ: compilers = os.environ[evar].split() is_cross = False @@ -248,14 +289,16 @@ class Environment(): compilers = self.default_fortran is_cross = False exe_wrap = None + popen_exceptions = {} for compiler in compilers: for arg in ['--version', '-V']: try: - p = subprocess.Popen([compiler] + [arg], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except OSError: - continue + p = subprocess.Popen([compiler, arg], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except OSError as e: + popen_exceptions[' '.join([compiler, arg])] = e + continue (out, err) = p.communicate() out = out.decode(errors='ignore') err = err.decode(errors='ignore') @@ -266,34 +309,38 @@ class Environment(): version = vmatch.group(0) if 'GNU Fortran' in out: - return GnuFortranCompiler([compiler], version, GCC_STANDARD, is_cross, exe_wrap) + return GnuFortranCompiler([compiler], version, GCC_STANDARD, is_cross, exe_wrap) if 'G95' in out: - return G95FortranCompiler([compiler], version, is_cross, exe_wrap) + return G95FortranCompiler([compiler], version, is_cross, exe_wrap) if 'Sun Fortran' in err: - version = 'unknown version' - vmatch = re.search(Environment.version_regex, err) - if vmatch: - version = vmatch.group(0) - return SunFortranCompiler([compiler], version, is_cross, exe_wrap) + version = 'unknown version' + vmatch = re.search(Environment.version_regex, err) + if vmatch: + version = vmatch.group(0) + return SunFortranCompiler([compiler], version, is_cross, exe_wrap) if 'ifort (IFORT)' in out: - return IntelFortranCompiler([compiler], version, is_cross, exe_wrap) - + return IntelFortranCompiler([compiler], version, is_cross, exe_wrap) + if 'PathScale EKOPath(tm)' in err: - return PathScaleFortranCompiler([compiler], version, is_cross, exe_wrap) + return PathScaleFortranCompiler([compiler], version, is_cross, exe_wrap) if 'pgf90' in out: - return PGIFortranCompiler([compiler], version, is_cross, exe_wrap) + return PGIFortranCompiler([compiler], version, is_cross, exe_wrap) if 'Open64 Compiler Suite' in err: - return Open64FortranCompiler([compiler], version, is_cross, exe_wrap) + return Open64FortranCompiler([compiler], version, is_cross, exe_wrap) if 'NAG Fortran' in err: - return NAGFortranCompiler([compiler], version, is_cross, exe_wrap) - - raise EnvironmentException('Unknown compiler(s): "' + ', '.join(compilers) + '"') + return NAGFortranCompiler([compiler], version, is_cross, exe_wrap) + errmsg = 'Unknown compiler(s): "' + ', '.join(compilers) + '"' + if popen_exceptions: + errmsg += '\nThe follow exceptions were encountered:' + for (c, e) in popen_exceptions.items(): + errmsg += '\nRunning "{0}" gave "{1}"'.format(c, e) + raise EnvironmentException(errmsg) def get_scratch_dir(self): return self.scratch_dir @@ -308,7 +355,10 @@ class Environment(): compilers = [self.cross_info.config['binaries']['cpp']] ccache = [] is_cross = True - exe_wrap = self.cross_info.config['binaries'].get('exe_wrapper', None) + if self.cross_info.need_exe_wrapper(): + exe_wrap = self.cross_info.config['binaries'].get('exe_wrapper', None) + else: + exe_wrap = [] elif evar in os.environ: compilers = os.environ[evar].split() ccache = [] @@ -319,6 +369,7 @@ class Environment(): ccache = self.detect_ccache() is_cross = False exe_wrap = None + popen_exceptions = {} for compiler in compilers: basename = os.path.basename(compiler).lower() if basename == 'cl' or basename == 'cl.exe': @@ -329,7 +380,8 @@ class Environment(): p = subprocess.Popen([compiler, arg], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - except OSError: + except OSError as e: + popen_exceptions[' '.join([compiler, arg])] = e continue (out, err) = p.communicate() out = out.decode(errors='ignore') @@ -358,13 +410,21 @@ class Environment(): if 'Microsoft' in out or 'Microsoft' in err: version = re.search(Environment.version_regex, err).group() return VisualStudioCPPCompiler([compiler], version, is_cross, exe_wrap) - raise EnvironmentException('Unknown compiler(s) "' + ', '.join(compilers) + '"') + errmsg = 'Unknown compiler(s): "' + ', '.join(compilers) + '"' + if popen_exceptions: + errmsg += '\nThe follow exceptions were encountered:' + for (c, e) in popen_exceptions.items(): + errmsg += '\nRunning "{0}" gave "{1}"'.format(c, e) + raise EnvironmentException(errmsg) def detect_objc_compiler(self, want_cross): if self.is_cross_build() and want_cross: exelist = [self.cross_info['objc']] is_cross = True - exe_wrap = self.cross_info.get('exe_wrapper', None) + if self.cross_info.need_exe_wrapper(): + exe_wrap = self.cross_info.get('exe_wrapper', None) + else: + exe_wrap = [] else: exelist = self.get_objc_compiler_exelist() is_cross = False @@ -394,7 +454,10 @@ class Environment(): if self.is_cross_build() and want_cross: exelist = [self.cross_info['objcpp']] is_cross = True - exe_wrap = self.cross_info.get('exe_wrapper', None) + if self.cross_info.need_exe_wrapper(): + exe_wrap = self.cross_info.get('exe_wrapper', None) + else: + exe_wrap = [] else: exelist = self.get_objcpp_compiler_exelist() is_cross = False @@ -514,7 +577,7 @@ class Environment(): evar = 'AR' if evar in os.environ: linker = os.environ[evar].strip() - if isinstance(compiler, VisualStudioCCompiler): + elif isinstance(compiler, VisualStudioCCompiler): linker= self.vs_static_linker else: linker = self.default_static_linker @@ -657,11 +720,11 @@ class CrossBuildInfo(): if 'target_machine' in self.config: return if not 'host_machine' in self.config: - raise coredata.MesonException('Cross info file must have either host or a target machine.') + raise mesonlib.MesonException('Cross info file must have either host or a target machine.') if not 'properties' in self.config: - raise coredata.MesonException('Cross file is missing "properties".') + raise mesonlib.MesonException('Cross file is missing "properties".') if not 'binaries' in self.config: - raise coredata.MesonException('Cross file is missing "binaries".') + raise mesonlib.MesonException('Cross file is missing "binaries".') def ok_type(self, i): return isinstance(i, str) or isinstance(i, int) or isinstance(i, bool) @@ -700,3 +763,12 @@ class CrossBuildInfo(): # But not when cross compiling a cross compiler. def need_cross_compiler(self): return 'host_machine' in self.config + + def need_exe_wrapper(self): + if self.has_host() and detect_cpu_family() == 'x86_64' and \ + self.config['host_machine']['cpu_family'] == 'x86' and \ + self.config['host_machine']['system'] == detect_system(): + # Can almost always run 32-bit binaries on 64-bit natively if the + # host and build systems are the same + return False + return True diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index b860ec3..da90814 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -28,7 +28,7 @@ from functools import wraps import importlib -class InterpreterException(coredata.MesonException): +class InterpreterException(mesonlib.MesonException): pass class InvalidCode(InterpreterException): @@ -322,24 +322,14 @@ class BuildMachine(InterpreterObject): 'endian' : self.endian_method, }) - # Python is inconsistent in its platform module. - # It returns different values for the same cpu. - # For x86 it might return 'x86', 'i686' or somesuch. - # Do some canonicalization. def cpu_family_method(self, args, kwargs): - trial = platform.machine().lower() - if trial.startswith('i') and trial.endswith('86'): - return 'x86' - if trial.startswith('arm'): - return 'arm' - # Add fixes here as bugs are reported. - return trial + return environment.detect_cpu_family() def cpu_method(self, args, kwargs): - return platform.machine().lower() + return environment.detect_cpu() def system_method(self, args, kwargs): - return platform.system().lower() + return environment.detect_system() def endian_method(self, args, kwargs): return sys.byteorder @@ -575,6 +565,7 @@ class CompilerHolder(InterpreterObject): 'get_id': self.get_id_method, 'sizeof': self.sizeof_method, 'has_header': self.has_header_method, + 'has_header_symbol': self.has_header_symbol_method, 'run' : self.run_method, 'has_function' : self.has_function_method, 'has_member' : self.has_member_method, @@ -752,6 +743,24 @@ class CompilerHolder(InterpreterObject): mlog.log('Has header "%s":' % string, h) return haz + def has_header_symbol_method(self, args, kwargs): + if len(args) != 2: + raise InterpreterException('has_header_symbol method takes exactly two arguments.') + check_stringlist(args) + hname = args[0] + symbol = args[1] + prefix = kwargs.get('prefix', '') + if not isinstance(prefix, str): + raise InterpreterException('Prefix argument of has_function must be a string.') + extra_args = self.determine_args(kwargs) + haz = self.compiler.has_header_symbol(hname, symbol, prefix, extra_args) + if haz: + h = mlog.green('YES') + else: + h = mlog.red('NO') + mlog.log('Header <{0}> has symbol "{1}":'.format(hname, symbol), h) + return haz + def find_library_method(self, args, kwargs): if len(args) != 1: raise InterpreterException('find_library method takes one argument.') @@ -761,13 +770,13 @@ class CompilerHolder(InterpreterObject): required = kwargs.get('required', True) if not isinstance(required, bool): raise InterpreterException('required must be boolean.') - search_dirs = kwargs.get('dirs', []) + search_dirs = mesonlib.stringlistify(kwargs.get('dirs', [])) for i in search_dirs: if not os.path.isabs(i): raise InvalidCode('Search directory %s is not an absolute path.' % i) linkargs = self.compiler.find_library(libname, search_dirs) if required and linkargs is None: - raise InterpreterException('Library %s not found'.format(libname)) + raise InterpreterException('Library {} not found'.format(libname)) lib = dependencies.ExternalLibrary(libname, linkargs) return ExternalLibraryHolder(lib) @@ -864,9 +873,16 @@ class MesonMain(InterpreterObject): return self.interpreter.environment.build_dir def has_exe_wrapper_method(self, args, kwargs): - if self.is_cross_build_method(None, None) and 'binaries' in self.build.environment.cross_info.config: - return 'exe_wrap' in self.build.environment.cross_info.config['binaries'] - return True # This is semantically confusing. + if self.is_cross_build_method(None, None) and \ + 'binaries' in self.build.environment.cross_info.config and \ + self.build.environment.cross_info.need_exe_wrapper(): + exe_wrap = self.build.environment.cross_info.config['binaries'].get('exe_wrap', None) + if exe_wrap is None: + return False + # We return True when exe_wrap is defined, when it's not needed, and + # when we're compiling natively. The last two are semantically confusing. + # Need to revisit this. + return True def is_cross_build_method(self, args, kwargs): return self.build.environment.is_cross_build() @@ -932,7 +948,7 @@ class Interpreter(): assert(isinstance(code, str)) try: self.ast = mparser.Parser(code).parse() - except coredata.MesonException as me: + except mesonlib.MesonException as me: me.file = environment.build_filename raise me self.sanity_check_ast() @@ -1334,7 +1350,7 @@ class Interpreter(): return self.environment.coredata.compiler_options[optname].value except KeyError: pass - if optname not in coredata.builtin_options and self.is_subproject(): + if not coredata.is_builtin_option(optname) and self.is_subproject(): optname = self.subproject + ':' + optname try: return self.environment.coredata.user_options[optname].value @@ -1357,8 +1373,7 @@ class Interpreter(): if '=' not in option: raise InterpreterException('All default options must be of type key=value.') key, value = option.split('=', 1) - builtin_options = self.coredata.builtin_options - if key in builtin_options: + if coredata.is_builtin_option(key): if not self.environment.had_argument_for(key): self.coredata.set_builtin_option(key, value) # If this was set on the command line, do not override. @@ -1745,7 +1760,9 @@ class Interpreter(): if self.is_subproject(): newsuite = [] for s in suite: - newsuite.append(self.subproject.replace(' ', '_').replace('.', '_') + '.' + s) + if len(s) > 0: + s = '.' + s + newsuite.append(self.subproject.replace(' ', '_').replace('.', '_') + s) suite = newsuite t = Test(args[0], suite, args[1].held_object, par, cmd_args, env, should_fail, valgrind_args, timeout, workdir) if is_base_test: @@ -1794,7 +1811,7 @@ class Interpreter(): assert(isinstance(code, str)) try: codeblock = mparser.Parser(code).parse() - except coredata.MesonException as me: + except mesonlib.MesonException as me: me.file = buildfilename raise me self.evaluate_codeblock(codeblock) @@ -2085,6 +2102,12 @@ class Interpreter(): return int(obj) except Exception: raise InterpreterException('String can not be converted to int: ' + obj) + elif method_name == 'join': + if len(posargs) != 1: + raise InterpreterException('Join() takes exactly one argument.') + strlist = posargs[0] + check_stringlist(strlist) + return obj.join(strlist) raise InterpreterException('Unknown method "%s" for a string.' % method_name) def to_native(self, arg): diff --git a/mesonbuild/mconf.py b/mesonbuild/mconf.py index c8ea494..3d38712 100644 --- a/mesonbuild/mconf.py +++ b/mesonbuild/mconf.py @@ -18,7 +18,6 @@ import sys, os import pickle import argparse from . import coredata, mesonlib -from .coredata import build_types, warning_levels, libtypelist parser = argparse.ArgumentParser() @@ -26,7 +25,7 @@ parser.add_argument('-D', action='append', default=[], dest='sets', help='Set an option to the given value.') parser.add_argument('directory', nargs='*') -class ConfException(coredata.MesonException): +class ConfException(mesonlib.MesonException): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -62,7 +61,8 @@ class Conf: longest_name = max(longest_name, len(x[0])) longest_descr = max(longest_descr, len(x[1])) longest_value = max(longest_value, len(str(x[2]))) - longest_possible_value = max(longest_possible_value, len(x[3])) + if x[3]: + longest_possible_value = max(longest_possible_value, len(x[3])) if longest_possible_value > 0: titles[3] = 'Possible Values' @@ -71,10 +71,14 @@ class Conf: for i in arr: name = i[0] descr = i[1] - value = i[2] - if isinstance(value, bool): - value = 'true' if value else 'false' - possible_values = i[3] + value = i[2] if isinstance(i[2], str) else str(i[2]).lower() + possible_values = '' + if isinstance(i[3], list): + if len(i[3]) > 0: + i[3] = [s if isinstance(s, str) else str(s).lower() for s in i[3]] + possible_values = '[%s]' % ', '.join(map(str, i[3])) + elif i[3]: + possible_values = i[3] if isinstance(i[3], str) else str(i[3]).lower() namepad = ' '*(longest_name - len(name)) descrpad = ' '*(longest_descr - len(descr)) valuepad = ' '*(longest_value - len(str(value))) @@ -86,7 +90,7 @@ class Conf: if '=' not in o: raise ConfException('Value "%s" not of type "a=b".' % o) (k, v) = o.split('=', 1) - if self.coredata.is_builtin_option(k): + if coredata.is_builtin_option(k): self.coredata.set_builtin_option(k, v) elif k in self.coredata.user_options: tgt = self.coredata.user_options[k] @@ -123,13 +127,9 @@ class Conf: print('') print('Core options:') carr = [] - booleans = '[true, false]' - carr.append(['buildtype', 'Build type', self.coredata.get_builtin_option('buildtype'), build_types]) - carr.append(['warning_level', 'Warning level', self.coredata.get_builtin_option('warning_level'), warning_levels]) - carr.append(['werror', 'Treat warnings as errors', self.coredata.get_builtin_option('werror'), booleans]) - carr.append(['strip', 'Strip on install', self.coredata.get_builtin_option('strip'), booleans]) - carr.append(['unity', 'Unity build', self.coredata.get_builtin_option('unity'), booleans]) - carr.append(['default_library', 'Default library type', self.coredata.get_builtin_option('default_library'), libtypelist]) + for key in [ 'buildtype', 'warning_level', 'werror', 'strip', 'unity', 'default_library' ]: + carr.append([key, coredata.get_builtin_option_description(key), + self.coredata.get_builtin_option(key), coredata.get_builtin_option_choices(key)]) self.print_aligned(carr) print('') print('Base options:') @@ -164,14 +164,9 @@ class Conf: print('') print('Directories:') parr = [] - parr.append(['prefix', 'Install prefix', self.coredata.get_builtin_option('prefix'), '']) - parr.append(['libdir', 'Library directory', self.coredata.get_builtin_option('libdir'), '']) - parr.append(['libexecdir', 'Library executables directory', self.coredata.get_builtin_option('libexecdir'), '']) - parr.append(['bindir', 'Binary directory', self.coredata.get_builtin_option('bindir'), '']) - parr.append(['includedir', 'Header directory', self.coredata.get_builtin_option('includedir'), '']) - parr.append(['datadir', 'Data directory', self.coredata.get_builtin_option('datadir'), '']) - parr.append(['mandir', 'Man page directory', self.coredata.get_builtin_option('mandir'), '']) - parr.append(['localedir', 'Locale file directory', self.coredata.get_builtin_option('localedir'), '']) + for key in [ 'prefix', 'libdir', 'libexecdir', 'bindir', 'includedir', 'datadir', 'mandir', 'localedir' ]: + parr.append([key, coredata.get_builtin_option_description(key), + self.coredata.get_builtin_option(key), coredata.get_builtin_option_choices(key)]) self.print_aligned(parr) print('') print('Project options:') @@ -192,6 +187,13 @@ class Conf: choices = str(opt.choices); optarr.append([key, opt.description, opt.value, choices]) self.print_aligned(optarr) + print('') + print('Testing options:') + tarr = [] + for key in [ 'stdsplit', 'errorlogs' ]: + tarr.append([key, coredata.get_builtin_option_description(key), + self.coredata.get_builtin_option(key), coredata.get_builtin_option_choices(key)]) + self.print_aligned(tarr) def run(args): args = mesonlib.expand_arguments(args) diff --git a/mesonbuild/mesonlib.py b/mesonbuild/mesonlib.py index 2087eee..fe831bd 100644 --- a/mesonbuild/mesonlib.py +++ b/mesonbuild/mesonlib.py @@ -18,7 +18,9 @@ import platform, subprocess, operator, os, shutil, re, sys from glob import glob -from .coredata import MesonException +class MesonException(Exception): + def __init__(self, *args, **kwargs): + Exception.__init__(self, *args, **kwargs) class File: def __init__(self, is_built, subdir, fname): @@ -177,6 +179,9 @@ def default_libexecdir(): # There is no way to auto-detect this, so it must be set at build time return 'libexec' +def default_prefix(): + return 'c:/' if is_windows() else '/usr/local' + def get_library_dirs(): if is_windows(): return ['C:/mingw/lib'] # Fixme @@ -247,7 +252,9 @@ def do_mesondefine(line, confdata): def do_conf_file(src, dst, confdata): data = open(src).readlines() - regex = re.compile('@(.*?)@') + # Only allow (a-z, A-Z, 0-9, _, -) as valid characters for a define + # Also allow escaping '@' with '\@' + regex = re.compile(r'[^\\]?@([-a-zA-Z0-9_]+)@') result = [] for line in data: if line.startswith('#mesondefine'): diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py index c53c9d1..4f8314c 100644 --- a/mesonbuild/mesonmain.py +++ b/mesonbuild/mesonmain.py @@ -21,8 +21,7 @@ from . import environment, interpreter, mesonlib from . import build import platform from . import mlog, coredata - -from .coredata import MesonException, build_types, layouts, warning_levels, libtypelist +from .mesonlib import MesonException backendlist = ['ninja', 'vs2010', 'xcode'] @@ -30,49 +29,40 @@ parser = argparse.ArgumentParser() default_warning = '1' -if mesonlib.is_windows(): - def_prefix = 'c:/' -else: - def_prefix = '/usr/local' +def add_builtin_argument(name, **kwargs): + k = kwargs.get('dest', name.replace('-', '_')) + c = coredata.get_builtin_option_choices(k) + b = True if kwargs.get('action', None) in [ 'store_true', 'store_false' ] else False + h = coredata.get_builtin_option_description(k) + if not b: + h = h.rstrip('.') + ' (default: %s).' % coredata.get_builtin_option_default(k) + if c and not b: + kwargs['choices'] = c + parser.add_argument('--' + name, default=coredata.get_builtin_option_default(k), help=h, **kwargs) + +add_builtin_argument('prefix') +add_builtin_argument('libdir') +add_builtin_argument('libexecdir') +add_builtin_argument('bindir') +add_builtin_argument('includedir') +add_builtin_argument('datadir') +add_builtin_argument('mandir') +add_builtin_argument('localedir') +add_builtin_argument('backend') +add_builtin_argument('buildtype') +add_builtin_argument('strip', action='store_true') +add_builtin_argument('unity', action='store_true') +add_builtin_argument('werror', action='store_true') +add_builtin_argument('layout') +add_builtin_argument('default-library') +add_builtin_argument('warnlevel', dest='warning_level') -parser.add_argument('--prefix', default=def_prefix, dest='prefix', - help='the installation prefix (default: %(default)s)') -parser.add_argument('--libdir', default=mesonlib.default_libdir(), dest='libdir', - help='the installation subdir of libraries (default: %(default)s)') -parser.add_argument('--libexecdir', default=mesonlib.default_libexecdir(), dest='libexecdir', - help='the installation subdir of library executables (default: %(default)s)') -parser.add_argument('--bindir', default='bin', dest='bindir', - help='the installation subdir of executables (default: %(default)s)') -parser.add_argument('--includedir', default='include', dest='includedir', - help='relative path of installed headers (default: %(default)s)') -parser.add_argument('--datadir', default='share', dest='datadir', - help='relative path to the top of data file subdirectory (default: %(default)s)') -parser.add_argument('--mandir', default='share/man', dest='mandir', - help='relative path of man files (default: %(default)s)') -parser.add_argument('--localedir', default='share/locale', dest='localedir', - help='relative path of locale data (default: %(default)s)') -parser.add_argument('--backend', default='ninja', dest='backend', choices=backendlist, - help='backend to use (default: %(default)s)') -parser.add_argument('--buildtype', default='debug', choices=build_types, dest='buildtype', - help='build type go use (default: %(default)s)') -parser.add_argument('--strip', action='store_true', dest='strip', default=False,\ - help='strip targets on install (default: %(default)s)') -parser.add_argument('--unity', action='store_true', dest='unity', default=False,\ - help='unity build') -parser.add_argument('--werror', action='store_true', dest='werror', default=False,\ - help='Treat warnings as errors') -parser.add_argument('--layout', choices=layouts, dest='layout', default='mirror',\ - help='Build directory layout.') -parser.add_argument('--default-library', choices=libtypelist, dest='default_library', - default='shared', help='Default library type.') -parser.add_argument('--warnlevel', default=default_warning, dest='warning_level', choices=warning_levels,\ - help='Level of compiler warnings to use (larger is more, default is %(default)s)') -parser.add_argument('--cross-file', default=None, dest='cross_file', - help='file describing cross compilation environment') +parser.add_argument('--cross-file', default=None, + help='File describing cross compilation environment.') parser.add_argument('-D', action='append', dest='projectoptions', default=[], help='Set project options.') parser.add_argument('-v', '--version', action='store_true', dest='print_version', default=False, - help='Print version.') + help='Print version information.') parser.add_argument('directories', nargs='*') class MesonApp(): @@ -173,7 +163,10 @@ itself as required.''' def run_script_command(args): cmdname = args[0] cmdargs = args[1:] - if cmdname == 'test': + if cmdname == 'exe': + import mesonbuild.scripts.meson_exe as abc + cmdfunc = abc.run + elif cmdname == 'test': import mesonbuild.scripts.meson_test as abc cmdfunc = abc.run elif cmdname == 'benchmark': @@ -203,6 +196,9 @@ def run_script_command(args): elif cmdname == 'symbolextractor': import mesonbuild.scripts.symbolextractor as abc cmdfunc = abc.run + elif cmdname == 'scanbuild': + import mesonbuild.scripts.scanbuild as abc + cmdfunc = abc.run elif cmdname == 'vcstagger': import mesonbuild.scripts.vcstagger as abc cmdfunc = abc.run diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py index 0881f69..955d12b 100644 --- a/mesonbuild/mintro.py +++ b/mesonbuild/mintro.py @@ -89,14 +89,14 @@ def list_target_files(target_name, coredata, builddata): def list_buildoptions(coredata, builddata): buildtype= {'choices': ['plain', 'debug', 'debugoptimized', 'release'], 'type' : 'combo', - 'value' : coredata.builtin_options['buildtype'].value, + 'value' : coredata.get_builtin_option('buildtype'), 'description' : 'Build type', 'name' : 'type'} - strip = {'value' : coredata.builtin_options['strip'].value, + strip = {'value' : coredata.get_builtin_option('strip'), 'type' : 'boolean', 'description' : 'Strip on install', 'name' : 'strip'} - unity = {'value' : coredata.builtin_options['unity'].value, + unity = {'value' : coredata.get_builtin_option('unity'), 'type' : 'boolean', 'description' : 'Unity build', 'name' : 'unity'} diff --git a/mesonbuild/modules/gnome.py b/mesonbuild/modules/gnome.py index 55bb321..39a6ff7 100644 --- a/mesonbuild/modules/gnome.py +++ b/mesonbuild/modules/gnome.py @@ -18,15 +18,26 @@ functionality such as gobject-introspection and gresources.''' from .. import build import os, sys import subprocess -from ..coredata import MesonException +from ..mesonlib import MesonException from .. import mlog from .. import mesonlib girwarning_printed = False +gresource_warning_printed = False class GnomeModule: + def __print_gresources_warning(self): + global gresource_warning_printed + if not gresource_warning_printed: + mlog.log('Warning, glib compiled dependencies will not work reliably until this upstream issue is fixed:', + mlog.bold('https://bugzilla.gnome.org/show_bug.cgi?id=745754')) + gresource_warning_printed = True + return [] + def compile_resources(self, state, args, kwargs): + self.__print_gresources_warning() + cmd = ['glib-compile-resources', '@INPUT@'] source_dirs = kwargs.pop('source_dir', []) @@ -59,6 +70,8 @@ class GnomeModule: return [target_c, target_h] def get_gresource_dependencies(self, state, input_file, source_dirs): + self.__print_gresources_warning() + cmd = ['glib-compile-resources', input_file, '--generate-dependencies'] @@ -314,8 +327,6 @@ class GnomeModule: return build.CustomTarget(namebase + '-gdbus', state.subdir, custom_kwargs) def initialize(): - mlog.log('Warning, glib compiled dependencies will not work reliably until this upstream issue is fixed:', - mlog.bold('https://bugzilla.gnome.org/show_bug.cgi?id=745754')) return GnomeModule() class GirTarget(build.CustomTarget): diff --git a/mesonbuild/modules/pkgconfig.py b/mesonbuild/modules/pkgconfig.py index f18decf..fe5ca45 100644 --- a/mesonbuild/modules/pkgconfig.py +++ b/mesonbuild/modules/pkgconfig.py @@ -21,7 +21,8 @@ class PkgConfigModule: def print_hello(self, state, args, kwargs): print('Hello from a Meson module') - def generate_pkgconfig_file(self, state, libraries, subdirs, name, description, version, filebase): + def generate_pkgconfig_file(self, state, libraries, subdirs, name, description, version, filebase, + pub_reqs, priv_reqs, priv_libs): outdir = state.environment.scratch_dir fname = os.path.join(outdir, filebase + '.pc') ofile = open(fname, 'w') @@ -34,6 +35,12 @@ class PkgConfigModule: ofile.write('Description: %s\n' % description) if len(version) > 0: ofile.write('Version: %s\n' % version) + if len(pub_reqs) > 0: + ofile.write('Requires: {}\n'.format(' '.join(pub_reqs))) + if len(priv_reqs) > 0: + ofile.write('Requires.private: {}\n'.format(' '.join(priv_reqs))) + if len(priv_libs) > 0: + ofile.write('Libraries.private: {}\n'.format(' '.join(priv_libs))) ofile.write('Libs: -L${libdir} ') for l in libraries: ofile.write('-l%s ' % l.name) @@ -48,7 +55,7 @@ class PkgConfigModule: def generate(self, state, args, kwargs): if len(args) > 0: - raise coredata.MesonException('Pkgconfig_gen takes no positional arguments.') + raise mesonlib.MesonException('Pkgconfig_gen takes no positional arguments.') libs = kwargs.get('libraries', []) if not isinstance(libs, list): libs = [libs] @@ -57,25 +64,29 @@ class PkgConfigModule: if hasattr(l, 'held_object'): l = l.held_object if not (isinstance(l, build.SharedLibrary) or isinstance(l, build.StaticLibrary)): - raise coredata.MesonException('Library argument not a library object.') + raise mesonlib.MesonException('Library argument not a library object.') processed_libs.append(l) libs = processed_libs subdirs = mesonlib.stringlistify(kwargs.get('subdirs', ['.'])) version = kwargs.get('version', '') if not isinstance(version, str): - raise coredata.MesonException('Version must be a string.') + raise mesonlib.MesonException('Version must be a string.') name = kwargs.get('name', None) if not isinstance(name, str): - raise coredata.MesonException('Name not specified.') + raise mesonlib.MesonException('Name not specified.') filebase = kwargs.get('filebase', name) if not isinstance(filebase, str): - raise coredata.MesonException('Filebase must be a string.') + raise mesonlib.MesonException('Filebase must be a string.') description = kwargs.get('description', None) if not isinstance(description, str): - raise coredata.MesonException('Description is not a string.') + raise mesonlib.MesonException('Description is not a string.') + pub_reqs = mesonlib.stringlistify(kwargs.get('requires', [])) + priv_reqs = mesonlib.stringlistify(kwargs.get('requires_private', [])) + priv_libs = mesonlib.stringlistify(kwargs.get('libraries_private', [])) pcfile = filebase + '.pc' pkgroot = os.path.join(state.environment.coredata.get_builtin_option('libdir'), 'pkgconfig') - self.generate_pkgconfig_file(state, libs, subdirs, name, description, version, filebase) + self.generate_pkgconfig_file(state, libs, subdirs, name, description, version, filebase, + pub_reqs, priv_reqs, priv_libs) return build.Data(False, state.environment.get_scratch_dir(), [pcfile], pkgroot) def initialize(): diff --git a/mesonbuild/modules/qt4.py b/mesonbuild/modules/qt4.py index 162b553..81a70fc 100644 --- a/mesonbuild/modules/qt4.py +++ b/mesonbuild/modules/qt4.py @@ -15,7 +15,7 @@ from .. import dependencies, mlog import os, subprocess from .. import build -from ..coredata import MesonException +from ..mesonlib import MesonException import xml.etree.ElementTree as ET class Qt4Module(): diff --git a/mesonbuild/modules/qt5.py b/mesonbuild/modules/qt5.py index cb743a6..f669d77 100644 --- a/mesonbuild/modules/qt5.py +++ b/mesonbuild/modules/qt5.py @@ -15,7 +15,7 @@ from .. import dependencies, mlog import os, subprocess from .. import build -from ..coredata import MesonException +from ..mesonlib import MesonException import xml.etree.ElementTree as ET class Qt5Module(): diff --git a/mesonbuild/modules/rpm.py b/mesonbuild/modules/rpm.py index c8035ec..acad204 100644 --- a/mesonbuild/modules/rpm.py +++ b/mesonbuild/modules/rpm.py @@ -65,9 +65,9 @@ class RPMModule: to_delete.add('%%{buildroot}%%{_libdir}/%s' % target.get_filename()) mlog.log('Warning, removing', mlog.bold(target.get_filename()), 'from package because packaging static libs not recommended') - elif isinstance(target, modules.gnome.GirTarget) and target.should_install(): + elif isinstance(target, gnome.GirTarget) and target.should_install(): files_devel.add('%%{_datadir}/gir-1.0/%s' % target.get_filename()[0]) - elif isinstance(target, modules.gnome.TypelibTarget) and target.should_install(): + elif isinstance(target, gnome.TypelibTarget) and target.should_install(): files.add('%%{_libdir}/girepository-1.0/%s' % target.get_filename()[0]) for header in state.headers: if len(header.get_install_subdir()) > 0: @@ -78,8 +78,6 @@ class RPMModule: for man in state.man: for man_file in man.get_sources(): files.add('%%{_mandir}/man%u/%s.*' % (int(man_file.split('.')[-1]), man_file)) - for pkgconfig in state.pkgconfig_gens: - files_devel.add('%%{_libdir}/pkgconfig/%s.pc' % pkgconfig.filebase) if len(files_devel) > 0: devel_subpkg = True fn = open('%s.spec' % os.path.join(state.environment.get_build_dir(), proj), 'w+') diff --git a/mesonbuild/modules/windows.py b/mesonbuild/modules/windows.py index a785250..29d6236 100644 --- a/mesonbuild/modules/windows.py +++ b/mesonbuild/modules/windows.py @@ -13,7 +13,7 @@ # limitations under the License. from .. import mesonlib, dependencies, build -from ..coredata import MesonException +from ..mesonlib import MesonException import os class WindowsModule: diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 090684c..fd720fb 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -13,7 +13,7 @@ # limitations under the License. import re -from .coredata import MesonException +from .mesonlib import MesonException class ParseException(MesonException): def __init__(self, text, lineno, colno): diff --git a/mesonbuild/optinterpreter.py b/mesonbuild/optinterpreter.py index e2f7ca5..409f9dc 100644 --- a/mesonbuild/optinterpreter.py +++ b/mesonbuild/optinterpreter.py @@ -14,9 +14,10 @@ from . import mparser from . import coredata +from . import mesonlib import os, re -forbidden_option_names = coredata.builtin_options +forbidden_option_names = coredata.get_builtin_options() forbidden_prefixes = {'c_': True, 'cpp_': True, 'rust_': True, @@ -37,7 +38,7 @@ def is_invalid_name(name): return True return False -class OptionException(coredata.MesonException): +class OptionException(mesonlib.MesonException): pass optname_regex = re.compile('[^a-zA-Z0-9_-]') @@ -77,7 +78,7 @@ class OptionInterpreter: def process(self, option_file): try: ast = mparser.Parser(open(option_file, 'r', encoding='utf8').read()).parse() - except coredata.MesonException as me: + except mesonlib.MesonException as me: me.file = option_file raise me if not isinstance(ast, mparser.CodeBlockNode): diff --git a/mesonbuild/scripts/delwithsuffix.py b/mesonbuild/scripts/delwithsuffix.py index 38ab406..e112101 100644 --- a/mesonbuild/scripts/delwithsuffix.py +++ b/mesonbuild/scripts/delwithsuffix.py @@ -17,7 +17,7 @@ import os, sys def run(args): - if len(sys.argv) != 2: + if len(sys.argv) != 3: print('delwithsuffix.py <root of subdir to process> <suffix to delete>') sys.exit(1) diff --git a/mesonbuild/scripts/depfixer.py b/mesonbuild/scripts/depfixer.py index 1ab83b6..8ff0dd1 100644 --- a/mesonbuild/scripts/depfixer.py +++ b/mesonbuild/scripts/depfixer.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2013-2014 The Meson development team +# Copyright 2013-2016 The Meson development team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -110,8 +110,9 @@ class SectionHeader(DataSizes): self.sh_entsize = struct.unpack(self.Word, ifile.read(self.WordSize))[0] class Elf(DataSizes): - def __init__(self, bfile): + def __init__(self, bfile, verbose=True): self.bfile = bfile + self.verbose = verbose self.bf = open(bfile, 'r+b') (self.ptrsize, self.is_le) = self.detect_elf_type() super().__init__(self.ptrsize, self.is_le) @@ -124,22 +125,21 @@ class Elf(DataSizes): if data[1:4] != b'ELF': # This script gets called to non-elf targets too # so just ignore them. - print('File "%s" is not an ELF file.' % self.bfile) + if self.verbose: + print('File "%s" is not an ELF file.' % self.bfile) sys.exit(0) if data[4] == 1: ptrsize = 32 elif data[4] == 2: ptrsize = 64 else: - print('File "%s" has unknown ELF class.' % self.bfile) - sys.exit(1) + sys.exit('File "%s" has unknown ELF class.' % self.bfile) if data[5] == 1: is_le = True elif data[5] == 2: is_le = False else: - print('File "%s" has unknown ELF endianness.' % self.bfile) - sys.exit(1) + sys.exit('File "%s" has unknown ELF endianness.' % self.bfile) return (ptrsize, is_le) def parse_header(self): @@ -257,14 +257,17 @@ class Elf(DataSizes): self.bf.write(newname) def fix_rpath(self, new_rpath): + if isinstance(new_rpath, str): + new_rpath = new_rpath.encode('utf8') rp_off = self.get_rpath_offset() if rp_off is None: - print('File does not have rpath. It should be a fully static executable.') + if self.verbose: + print('File does not have rpath. It should be a fully static executable.') return self.bf.seek(rp_off) old_rpath = self.read_str() if len(old_rpath) < len(new_rpath): - print("New rpath must not be longer than the old one.") + sys.exit("New rpath must not be longer than the old one.") self.bf.seek(rp_off) self.bf.write(new_rpath) self.bf.write(b'\0'*(len(old_rpath) - len(new_rpath) + 1)) @@ -295,7 +298,7 @@ def run(args): e.print_rpath() else: new_rpath = args[1] - e.fix_rpath(new_rpath.encode('utf8')) + e.fix_rpath(new_rpath) return 0 if __name__ == '__main__': diff --git a/mesonbuild/scripts/meson_benchmark.py b/mesonbuild/scripts/meson_benchmark.py index 26f1f95..d1107b6 100644 --- a/mesonbuild/scripts/meson_benchmark.py +++ b/mesonbuild/scripts/meson_benchmark.py @@ -39,9 +39,10 @@ def print_json_log(jsonlogfile, rawruns, test_name, i): for r in rawruns: runobj = {'duration': r.duration, 'stdout': r.stdo, - 'stderr': r.stde, 'returncode' : r.returncode, 'duration' : r.duration} + if r.stde: + runobj['stderr'] = r.stde runs.append(runobj) jsonobj['runs'] = runs jsonlogfile.write(json.dumps(jsonobj) + '\n') diff --git a/mesonbuild/scripts/meson_exe.py b/mesonbuild/scripts/meson_exe.py new file mode 100644 index 0000000..f075fa0 --- /dev/null +++ b/mesonbuild/scripts/meson_exe.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +# Copyright 2013-2016 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import argparse +import pickle +import platform +import subprocess + +import mesonbuild + +options = None + +parser = argparse.ArgumentParser() +parser.add_argument('args', nargs='+') + +def is_windows(): + platname = platform.system().lower() + return platname == 'windows' or 'mingw' in platname + +def run_with_mono(fname): + if fname.endswith('.exe') and not is_windows(): + return True + return False + +def run_exe(exe): + if exe.fname[0].endswith('.jar'): + cmd = ['java', '-jar'] + exe.fname + elif not exe.is_cross and run_with_mono(exe.fname[0]): + cmd = ['mono'] + exe.fname + else: + if exe.is_cross: + if exe.exe_runner is None: + raise Exception('BUG: Trying to run cross-compiled exes with no wrapper') + else: + cmd = [exe.exe_runner] + exe.fname + else: + cmd = exe.fname + child_env = os.environ.copy() + child_env.update(exe.env) + if len(exe.extra_paths) > 0: + child_env['PATH'] = ';'.join(exe.extra_paths + ['']) + child_env['PATH'] + p = subprocess.Popen(cmd + exe.cmd_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=child_env, + cwd=exe.workdir) + +def run(args): + global options + options = parser.parse_args(args) + if len(options.args) != 1: + print('Test runner for Meson. Do not run on your own, mmm\'kay?') + print(sys.argv[0] + ' [data file]') + exe_data_file = options.args[0] + exe = pickle.load(open(exe_data_file, 'rb')) + run_exe(exe) + +if __name__ == '__main__': + sys.exit(run(sys.argv[1:])) diff --git a/mesonbuild/scripts/meson_install.py b/mesonbuild/scripts/meson_install.py index 765969d..20a50f7 100644 --- a/mesonbuild/scripts/meson_install.py +++ b/mesonbuild/scripts/meson_install.py @@ -16,6 +16,7 @@ import sys, pickle, os, shutil, subprocess, gzip, platform from glob import glob +from mesonbuild.scripts import depfixer def do_install(datafilename): ifile = open(datafilename, 'rb') @@ -189,15 +190,14 @@ def install_targets(d): print("Symlink creation does not work on this platform.") printed_symlink_error = True if is_elf_platform(): - p = subprocess.Popen(d.depfixer + [outname, install_rpath], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - (stdo, stde) = p.communicate() - if p.returncode != 0: - print('Could not fix dependency info.\n') - print('Stdout:\n%s\n' % stdo.decode()) - print('Stderr:\n%s\n' % stde.decode()) - sys.exit(1) + try: + e = depfixer.Elf(outname, False) + e.fix_rpath(install_rpath) + except SystemExit as e: + if isinstance(e.code, int) and e.code == 0: + pass + else: + raise def run(args): if len(args) != 1: diff --git a/mesonbuild/scripts/meson_test.py b/mesonbuild/scripts/meson_test.py index 453ea61..524dc7a 100644 --- a/mesonbuild/scripts/meson_test.py +++ b/mesonbuild/scripts/meson_test.py @@ -25,7 +25,9 @@ def is_windows(): platname = platform.system().lower() return platname == 'windows' or 'mingw' in platname -tests_failed = [] +collected_logs = [] +error_count = 0 +options = None parser = argparse.ArgumentParser() parser.add_argument('--wrapper', default=None, dest='wrapper', @@ -34,17 +36,41 @@ parser.add_argument('--wd', default=None, dest='wd', help='directory to cd into before running') parser.add_argument('--suite', default=None, dest='suite', help='Only run tests belonging to this suite.') +parser.add_argument('--no-stdsplit', default=True, dest='split', action='store_false', + help='Do not split stderr and stdout in test logs.') +parser.add_argument('--print-errorlogs', default=False, action='store_true', + help="Whether to print faling tests' logs.") parser.add_argument('args', nargs='+') class TestRun(): - def __init__(self, res, returncode, duration, stdo, stde, cmd): + def __init__(self, res, returncode, should_fail, duration, stdo, stde, cmd): self.res = res self.returncode = returncode self.duration = duration self.stdo = stdo self.stde = stde self.cmd = cmd + self.should_fail = should_fail + + def get_log(self): + res = '--- command ---\n' + if self.cmd is None: + res += 'NONE\n' + else: + res += ' '.join(self.cmd) + '\n' + if self.stdo: + res += '--- stdout ---\n' + res += self.stdo + if self.stde: + if res[-1:] != '\n': + res += '\n' + res += '--- stderr ---\n' + res += self.stde + if res[-1:] != '\n': + res += '\n' + res += '-------\n\n' + return res def decode(stream): try: @@ -52,28 +78,16 @@ def decode(stream): except UnicodeDecodeError: return stream.decode('iso-8859-1', errors='ignore') -def write_log(logfile, test_name, result_str, result): - logfile.write(result_str + '\n\n') - logfile.write('--- command ---\n') - if result.cmd is None: - logfile.write('NONE') - else: - logfile.write(' '.join(result.cmd)) - logfile.write('\n--- "%s" stdout ---\n' % test_name) - logfile.write(result.stdo) - logfile.write('\n--- "%s" stderr ---\n' % test_name) - logfile.write(result.stde) - logfile.write('\n-------\n\n') - def write_json_log(jsonlogfile, test_name, result): - result = {'name' : test_name, + jresult = {'name' : test_name, 'stdout' : result.stdo, - 'stderr' : result.stde, 'result' : result.res, 'duration' : result.duration, 'returncode' : result.returncode, 'command' : result.cmd} - jsonlogfile.write(json.dumps(result) + '\n') + if result.stde: + jresult['stderr'] = result.stde + jsonlogfile.write(json.dumps(jresult) + '\n') def run_with_mono(fname): if fname.endswith('.exe') and not is_windows(): @@ -81,7 +95,7 @@ def run_with_mono(fname): return False def run_single_test(wrap, test): - global tests_failed + global options if test.fname[0].endswith('.jar'): cmd = ['java', '-jar'] + test.fname elif not test.is_cross and run_with_mono(test.fname[0]): @@ -102,7 +116,7 @@ def run_single_test(wrap, test): res = 'SKIP' duration = 0.0 stdo = 'Not run because can not execute cross compiled binaries.' - stde = '' + stde = None returncode = -1 else: cmd = wrap + cmd + test.cmd_args @@ -117,7 +131,7 @@ def run_single_test(wrap, test): setsid = os.setsid p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stderr=subprocess.PIPE if options and options.split else subprocess.STDOUT, env=child_env, cwd=test.workdir, preexec_fn=setsid) @@ -137,20 +151,20 @@ def run_single_test(wrap, test): endtime = time.time() duration = endtime - starttime stdo = decode(stdo) - stde = decode(stde) + if stde: + stde = decode(stde) if timed_out: res = 'TIMEOUT' - tests_failed.append((test.name, stdo, stde)) elif (not test.should_fail and p.returncode == 0) or \ (test.should_fail and p.returncode != 0): res = 'OK' else: res = 'FAIL' - tests_failed.append((test.name, stdo, stde)) returncode = p.returncode - return TestRun(res, returncode, duration, stdo, stde, cmd) + return TestRun(res, returncode, test.should_fail, duration, stdo, stde, cmd) def print_stats(numlen, tests, name, result, i, logfile, jsonlogfile): + global collected_logs, error_count, options startpad = ' '*(numlen - len('%d' % (i+1))) num = '%s%d/%d' % (startpad, i+1, len(tests)) padding1 = ' '*(38-len(name)) @@ -158,7 +172,12 @@ def print_stats(numlen, tests, name, result, i, logfile, jsonlogfile): result_str = '%s %s %s%s%s%5.2f s' % \ (num, name, padding1, result.res, padding2, result.duration) print(result_str) - write_log(logfile, name, result_str, result) + result_str += "\n\n" + result.get_log() + if (result.returncode != 0) != result.should_fail: + error_count += 1 + if options.print_errorlogs: + collected_logs.append(result_str) + logfile.write(result_str) write_json_log(jsonlogfile, name, result) def drain_futures(futures): @@ -171,7 +190,8 @@ def filter_tests(suite, tests): return tests return [x for x in tests if suite in x.suite] -def run_tests(options, datafilename): +def run_tests(datafilename): + global options logfile_base = 'meson-logs/testlog' if options.wrapper is None: wrap = [] @@ -222,8 +242,9 @@ def run_tests(options, datafilename): return logfilename def run(args): - global tests_failed - tests_failed = [] # To avoid state leaks when invoked multiple times (running tests in-process) + global collected_logs, error_count, options + collected_logs = [] # To avoid state leaks when invoked multiple times (running tests in-process) + error_count = 0 options = parser.parse_args(args) if len(options.args) != 1: print('Test runner for Meson. Do not run on your own, mmm\'kay?') @@ -231,19 +252,22 @@ def run(args): if options.wd is not None: os.chdir(options.wd) datafile = options.args[0] - logfilename = run_tests(options, datafile) - returncode = 0 - if len(tests_failed) > 0: - print('\nOutput of failed tests (max 10):') - for (name, stdo, stde) in tests_failed[:10]: - print("{} stdout:\n".format(name)) - print(stdo) - print('\n{} stderr:\n'.format(name)) - print(stde) - print('\n') - returncode = 1 - print('\nFull log written to %s.' % logfilename) - return returncode + logfilename = run_tests(datafile) + if len(collected_logs) > 0: + if len(collected_logs) > 10: + print('\nThe output from 10 first failed tests:\n') + else: + print('\nThe output from the failed tests:\n') + for log in collected_logs[:10]: + lines = log.splitlines() + if len(lines) > 100: + print(lines[0]) + print('--- Listing only the last 100 lines from a long log. ---') + lines = lines[-99:] + for line in lines: + print(line) + print('Full log written to %s.' % logfilename) + return error_count if __name__ == '__main__': sys.exit(run(sys.argv[1:])) diff --git a/mesonbuild/scripts/scanbuild.py b/mesonbuild/scripts/scanbuild.py new file mode 100644 index 0000000..f90c3c7 --- /dev/null +++ b/mesonbuild/scripts/scanbuild.py @@ -0,0 +1,39 @@ +# Copyright 2016 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys, os +import subprocess +import shutil +import tempfile + +def scanbuild(srcdir, blddir, privdir, logdir, args): + with tempfile.TemporaryDirectory(dir=privdir) as scandir: + meson_cmd = ['scan-build'] + args + build_cmd = ['scan-build', '-o', logdir, 'ninja'] + rc = subprocess.call(meson_cmd + [srcdir, scandir]) + if rc != 0: + return rc + return subprocess.call(build_cmd) + +def run(args): + srcdir = args[0] + blddir = args[1] + meson_cmd = args[2:] + privdir = os.path.join(blddir, 'meson-private') + logdir = os.path.join(blddir, 'meson-logs/scanbuild') + shutil.rmtree(logdir, ignore_errors=True) + if not shutil.which('scan-build'): + print('Scan-build not installed') + return 1 + return scanbuild(srcdir, blddir, privdir, logdir, meson_cmd) diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index ad8f106..6e3383c 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -90,10 +90,11 @@ class Resolver: def resolve(self, packagename): fname = os.path.join(self.subdir_root, packagename + '.wrap') dirname = os.path.join(self.subdir_root, packagename) + if os.path.isdir(dirname): + # The directory is there? Great, use it. + return packagename if not os.path.isfile(fname): - if os.path.isdir(dirname): - # No wrap file but dir exists -> user put it there manually. - return packagename + # No wrap file with this name? Give up. return None p = PackageDefinition(fname) if p.type == 'file': diff --git a/run_tests.py b/run_tests.py index c4f14ec..ad2450e 100755 --- a/run_tests.py +++ b/run_tests.py @@ -18,7 +18,7 @@ from glob import glob import os, subprocess, shutil, sys, signal from io import StringIO from ast import literal_eval -import sys +import sys, tempfile from mesonbuild import environment from mesonbuild import mesonlib from mesonbuild import mlog @@ -28,6 +28,8 @@ from mesonbuild.scripts import meson_test, meson_benchmark import argparse import xml.etree.ElementTree as ET import time +import multiprocessing +import concurrent.futures as conc from mesonbuild.mesonmain import backendlist @@ -40,13 +42,31 @@ class TestResult: self.buildtime = buildtime self.testtime = testtime +class AutoDeletedDir(): + def __init__(self, dir): + self.dir = dir + def __enter__(self): + os.makedirs(self.dir, exist_ok=True) + return self.dir + def __exit__(self, type, value, traceback): + # On Windows, shutil.rmtree fails sometimes, because 'the directory is not empty'. + # Retrying fixes this. + # That's why we don't use tempfile.TemporaryDirectory, but wrap the deletion in the AutoDeletedDir class. + retries = 5 + for i in range(0, retries): + try: + shutil.rmtree(self.dir) + return + except OSError: + if i == retries-1: + raise + time.sleep(0.1 * (2**i)) + passing_tests = 0 failing_tests = 0 skipped_tests = 0 print_debug = 'MESON_PRINT_TEST_OUTPUT' in os.environ -test_build_dir = 'work area' -install_dir = os.path.join(os.path.split(os.path.abspath(__file__))[0], 'install dir') meson_command = os.path.join(os.getcwd(), 'meson') if not os.path.exists(meson_command): meson_command += '.py' @@ -136,14 +156,8 @@ def validate_install(srcdir, installdir): return 'Found extra file %s.' % fname return '' -def log_text_file(logfile, testdir, msg, stdo, stde): - global passing_tests, failing_tests, stop - if msg != '': - print('Fail:', msg) - failing_tests += 1 - else: - print('Success') - passing_tests += 1 +def log_text_file(logfile, testdir, stdo, stde): + global stop logfile.write('%s\nstdout\n\n---\n' % testdir) logfile.write(stdo) logfile.write('\n\n---\n\nstderr\n\n---\n') @@ -197,18 +211,21 @@ def parse_test_args(testdir): pass return args -def run_test(testdir, extra_args, should_succeed): - global compile_commands - mlog.shutdown() # Close the log file because otherwise Windows wets itself. - shutil.rmtree(test_build_dir) - shutil.rmtree(install_dir) - os.mkdir(test_build_dir) - os.mkdir(install_dir) - print('Running test: ' + testdir) +def run_test(skipped, testdir, extra_args, flags, compile_commands, install_commands, should_succeed): + if skipped: + return None + with AutoDeletedDir(tempfile.mkdtemp(prefix='b ', dir='.')) as build_dir: + with AutoDeletedDir(tempfile.mkdtemp(prefix='i ', dir=os.getcwd())) as install_dir: + try: + return _run_test(testdir, build_dir, install_dir, extra_args, flags, compile_commands, install_commands, should_succeed) + finally: + mlog.shutdown() # Close the log file because otherwise Windows wets itself. + +def _run_test(testdir, test_build_dir, install_dir, extra_args, flags, compile_commands, install_commands, should_succeed): test_args = parse_test_args(testdir) gen_start = time.time() gen_command = [meson_command, '--prefix', '/usr', '--libdir', 'lib', testdir, test_build_dir]\ - + unity_flags + backend_flags + test_args + extra_args + + flags + test_args + extra_args (returncode, stdo, stde) = run_configure_inprocess(gen_command) gen_time = time.time() - gen_start if not should_succeed: @@ -242,7 +259,6 @@ def run_test(testdir, extra_args, should_succeed): if returncode != 0: return TestResult('Running unit tests failed.', stdo, stde, gen_time, build_time, test_time) if len(install_commands) == 0: - print("Skipping install test") return TestResult('', '', '', gen_time, build_time, test_time) else: env = os.environ.copy() @@ -284,20 +300,15 @@ def detect_tests_to_run(): return all_tests def run_tests(extra_args): + global passing_tests, failing_tests, stop all_tests = detect_tests_to_run() logfile = open('meson-test-run.txt', 'w', encoding="utf_8") junit_root = ET.Element('testsuites') conf_time = 0 build_time = 0 test_time = 0 - try: - os.mkdir(test_build_dir) - except OSError: - pass - try: - os.mkdir(install_dir) - except OSError: - pass + + executor = conc.ProcessPoolExecutor(max_workers=multiprocessing.cpu_count()) for name, test_cases, skipped in all_tests: current_suite = ET.SubElement(junit_root, 'testsuite', {'name' : name, 'tests' : str(len(test_cases))}) @@ -305,28 +316,40 @@ def run_tests(extra_args): print('\nNot running %s tests.\n' % name) else: print('\nRunning %s tests.\n' % name) + futures = [] for t in test_cases: # Jenkins screws us over by automatically sorting test cases by name # and getting it wrong by not doing logical number sorting. (testnum, testbase) = os.path.split(t)[-1].split(' ', 1) testname = '%.3d %s' % (int(testnum), testbase) - if skipped: + result = executor.submit(run_test, skipped, t, extra_args, unity_flags + backend_flags, compile_commands, install_commands, name != 'failing') + futures.append((testname, t, result)) + for (testname, t, result) in futures: + result = result.result() + if result is None: + print('Skipping:', t) current_test = ET.SubElement(current_suite, 'testcase', {'name' : testname, 'classname' : name}) ET.SubElement(current_test, 'skipped', {}) global skipped_tests skipped_tests += 1 else: - ts = time.time() - result = run_test(t, extra_args, name != 'failing') - te = time.time() + without_install = "" if len(install_commands) > 0 else " (without install)" + if result.msg != '': + print('Failed test%s: %s' % (without_install, t)) + print('Reason:', result.msg) + failing_tests += 1 + else: + print('Succeeded test%s: %s' % (without_install, t)) + passing_tests += 1 conf_time += result.conftime build_time += result.buildtime test_time += result.testtime - log_text_file(logfile, t, result.msg, result.stdo, result.stde) + total_time = conf_time + build_time + test_time + log_text_file(logfile, t, result.stdo, result.stde) current_test = ET.SubElement(current_suite, 'testcase', {'name' : testname, 'classname' : name, - 'time' : '%.3f' % (te - ts)}) + 'time' : '%.3f' % total_time}) if result.msg != '': ET.SubElement(current_test, 'failure', {'message' : result.msg}) stdoel = ET.SubElement(current_test, 'system-out') diff --git a/test cases/common/110 extract same name/lib.c b/test cases/common/110 extract same name/lib.c new file mode 100644 index 0000000..6bdeda7 --- /dev/null +++ b/test cases/common/110 extract same name/lib.c @@ -0,0 +1,3 @@ +int func1() { + return 23; +} diff --git a/test cases/common/110 extract same name/main.c b/test cases/common/110 extract same name/main.c new file mode 100644 index 0000000..dc57dd5 --- /dev/null +++ b/test cases/common/110 extract same name/main.c @@ -0,0 +1,6 @@ +int func1(); +int func2(); + +int main(int argc, char **argv) { + return !(func1() == 23 && func2() == 42); +} diff --git a/test cases/common/110 extract same name/meson.build b/test cases/common/110 extract same name/meson.build new file mode 100644 index 0000000..9384c47 --- /dev/null +++ b/test cases/common/110 extract same name/meson.build @@ -0,0 +1,6 @@ +project('object extraction', 'c') + +lib = shared_library('somelib', ['lib.c', 'src/lib.c']) +obj = lib.extract_objects(['lib.c', 'src/lib.c']) +exe = executable('main', 'main.c', objects: obj) +test('extraction', exe) diff --git a/test cases/common/110 extract same name/src/lib.c b/test cases/common/110 extract same name/src/lib.c new file mode 100644 index 0000000..68e6ab9 --- /dev/null +++ b/test cases/common/110 extract same name/src/lib.c @@ -0,0 +1,3 @@ +int func2() { + return 42; +} diff --git a/test cases/common/111 has header symbol/meson.build b/test cases/common/111 has header symbol/meson.build new file mode 100644 index 0000000..e0afb42 --- /dev/null +++ b/test cases/common/111 has header symbol/meson.build @@ -0,0 +1,18 @@ +project('has header symbol', 'c') + +cc = meson.get_compiler('c') + +assert (cc.has_header_symbol('stdio.h', 'int'), 'base types should always be available') +assert (cc.has_header_symbol('stdio.h', 'printf'), 'printf function not found') +assert (cc.has_header_symbol('stdio.h', 'FILE'), 'FILE structure not found') +assert (cc.has_header_symbol('limits.h', 'INT_MAX'), 'INT_MAX define not found') +assert (not cc.has_header_symbol('limits.h', 'guint64'), 'guint64 is not defined in limits.h') +assert (not cc.has_header_symbol('stdlib.h', 'FILE'), 'FILE structure is defined in stdio.h, not stdlib.h') +assert (not cc.has_header_symbol('stdlol.h', 'printf'), 'stdlol.h shouldn\'t exist') +assert (not cc.has_header_symbol('stdlol.h', 'int'), 'shouldn\'t be able to find "int" with invalid header') + +# This is likely only available on Glibc, so just test for it +if cc.has_function('ppoll') + assert (not cc.has_header_symbol('poll.h', 'ppoll'), 'ppoll should not be accessible without _GNU_SOURCE') + assert (cc.has_header_symbol('poll.h', 'ppoll', prefix : '#define _GNU_SOURCE'), 'ppoll should be accessible with _GNU_SOURCE') +endif diff --git a/test cases/common/12 data/installed_files.txt b/test cases/common/12 data/installed_files.txt index 1c58623..3d4b12c 100644 --- a/test cases/common/12 data/installed_files.txt +++ b/test cases/common/12 data/installed_files.txt @@ -1,3 +1,4 @@ usr/share/progname/datafile.dat usr/share/progname/vanishing.dat +usr/share/progname/vanishing2.dat etc/etcfile.dat diff --git a/test cases/common/12 data/meson.build b/test cases/common/12 data/meson.build index 2193f94..80f3835 100644 --- a/test cases/common/12 data/meson.build +++ b/test cases/common/12 data/meson.build @@ -1,4 +1,7 @@ project('data install test', 'c') install_data(sources : 'datafile.dat', install_dir : 'share/progname') install_data(sources : 'etcfile.dat', install_dir : '/etc') + subdir('vanishing') + +install_data(sources : 'vanishing/vanishing2.dat', install_dir : 'share/progname') diff --git a/test cases/common/12 data/vanishing/vanishing2.dat b/test cases/common/12 data/vanishing/vanishing2.dat new file mode 100644 index 0000000..99c923b --- /dev/null +++ b/test cases/common/12 data/vanishing/vanishing2.dat @@ -0,0 +1,4 @@ +This is a data file to be installed in a subdirectory. + +It is installed from a different subdir to test that the +installer strips the source tree dir prefix. diff --git a/test cases/common/42 string formatting/meson.build b/test cases/common/42 string formatting/meson.build index c2ee151..0d17448 100644 --- a/test cases/common/42 string formatting/meson.build +++ b/test cases/common/42 string formatting/meson.build @@ -51,3 +51,7 @@ assert(false.to_string() == 'false', 'bool string conversion failed') assert(true.to_string('yes', 'no') == 'yes', 'bool string conversion with args failed') assert(false.to_string('yes', 'no') == 'no', 'bool string conversion with args failed') assert('@0@'.format(true) == 'true', 'bool string formatting failed') + +assert(' '.join(['a', 'b', 'c']) == 'a b c', 'join() array broken') +assert(''.join(['a', 'b', 'c']) == 'abc', 'empty join() broken') +assert(' '.join(['a']) == 'a', 'single join broken') diff --git a/test cases/common/43 has function/meson.build b/test cases/common/43 has function/meson.build index 8fccaef..c7fe353 100644 --- a/test cases/common/43 has function/meson.build +++ b/test cases/common/43 has function/meson.build @@ -3,9 +3,33 @@ project('has function', 'c') cc = meson.get_compiler('c') if not cc.has_function('printf', prefix : '#include<stdio.h>') - error('Existing function not found.') + error('"printf" function not found (should always exist).') endif +# Should also be able to detect it without specifying the header +# We check for a different function here to make sure the result is +# not taken from a cache (ie. the check above) +assert(cc.has_function('fprintf'), '"fprintf" function not found without include (should always exist).') + if cc.has_function('hfkerhisadf', prefix : '#include<stdio.h>') - error('Found non-existant function.') + error('Found non-existent function "hfkerhisadf".') +endif + +# With glibc on Linux lchmod is a stub that will always return an error, +# we want to detect that and declare that the function is not available. +# We can't check for the C library used here of course, but if it's not +# implemented in glibc it's probably not implemented in any other 'slimmer' +# C library variants either, so the check should be safe either way hopefully. +if host_machine.system() == 'linux' and cc.get_id() == 'gcc' + assert (cc.has_function('poll', prefix : '#include <poll.h>'), 'couldn\'t detect "poll" when defined by a header') + assert (not cc.has_function('lchmod', prefix : '''#include <sys/stat.h> + #include <unistd.h>'''), '"lchmod" check should have failed') +endif + +# For some functions one needs to define _GNU_SOURCE before including the +# right headers to get them picked up. Make sure we can detect these functions +# as well without any prefix +if cc.has_header_symbol('sys/socket.h', 'recvmmsg', prefix : '#define _GNU_SOURCE') + # We assume that if recvmmsg exists sendmmsg does too + assert (cc.has_function('sendmmsg'), 'Failed to detect function "sendmmsg" (should always exist).') endif diff --git a/test cases/common/51 pkgconfig-gen/meson.build b/test cases/common/51 pkgconfig-gen/meson.build index a54fd66..4044b3d 100644 --- a/test cases/common/51 pkgconfig-gen/meson.build +++ b/test cases/common/51 pkgconfig-gen/meson.build @@ -12,5 +12,8 @@ pkgg.generate( version : libver, name : 'libsimple', filebase : 'simple', - description : 'A simple demo library.' + description : 'A simple demo library.', + requires : 'glib-2.0', # Not really, but only here to test that this works. + requires_private : ['gio-2.0', 'gobject-2.0'], + libraries_private : '-lz', ) diff --git a/test cases/failing/28 no vs module defs/meson.build b/test cases/failing/28 no vs module defs/meson.build new file mode 100644 index 0000000..7864daa --- /dev/null +++ b/test cases/failing/28 no vs module defs/meson.build @@ -0,0 +1,9 @@ +project('dll_no_module_def', 'c') + +if meson.get_compiler('c').get_id() != 'msvc' + error('Need to use the Visual Studio compiler') +endif + +subdir('subdir') +exe = executable('prog', 'prog.c', link_with : shlib) +test('runtest', exe) diff --git a/test cases/failing/28 no vs module defs/prog.c b/test cases/failing/28 no vs module defs/prog.c new file mode 100644 index 0000000..f35f4a0 --- /dev/null +++ b/test cases/failing/28 no vs module defs/prog.c @@ -0,0 +1,5 @@ +int somedllfunc(); + +int main(int argc, char **argv) { + return somedllfunc() == 42 ? 0 : 1; +} diff --git a/test cases/failing/28 no vs module defs/subdir/meson.build b/test cases/failing/28 no vs module defs/subdir/meson.build new file mode 100644 index 0000000..8395d59 --- /dev/null +++ b/test cases/failing/28 no vs module defs/subdir/meson.build @@ -0,0 +1 @@ +shlib = shared_library('somedll', 'somedll.c') diff --git a/test cases/failing/28 no vs module defs/subdir/somedll.c b/test cases/failing/28 no vs module defs/subdir/somedll.c new file mode 100644 index 0000000..5c469b1 --- /dev/null +++ b/test cases/failing/28 no vs module defs/subdir/somedll.c @@ -0,0 +1,7 @@ +/* With MSVC, the DLL created from this will not export any symbols + * without a module definitions file specified while linking */ +#ifdef _MSC_VER +int somedllfunc() { + return 42; +} +#endif diff --git a/test cases/windows/6 vs module defs/meson.build b/test cases/windows/6 vs module defs/meson.build new file mode 100644 index 0000000..4b9e735 --- /dev/null +++ b/test cases/windows/6 vs module defs/meson.build @@ -0,0 +1,7 @@ +project('dll_module_defs', 'c') + +if meson.get_compiler('c').get_id() == 'msvc' + subdir('subdir') + exe = executable('prog', 'prog.c', link_with : shlib) + test('runtest', exe) +endif diff --git a/test cases/windows/6 vs module defs/prog.c b/test cases/windows/6 vs module defs/prog.c new file mode 100644 index 0000000..f35f4a0 --- /dev/null +++ b/test cases/windows/6 vs module defs/prog.c @@ -0,0 +1,5 @@ +int somedllfunc(); + +int main(int argc, char **argv) { + return somedllfunc() == 42 ? 0 : 1; +} diff --git a/test cases/windows/6 vs module defs/subdir/meson.build b/test cases/windows/6 vs module defs/subdir/meson.build new file mode 100644 index 0000000..60633c3 --- /dev/null +++ b/test cases/windows/6 vs module defs/subdir/meson.build @@ -0,0 +1 @@ +shlib = shared_library('somedll', 'somedll.c', vs_module_defs : 'somedll.def') diff --git a/test cases/windows/6 vs module defs/subdir/somedll.c b/test cases/windows/6 vs module defs/subdir/somedll.c new file mode 100644 index 0000000..df255e3 --- /dev/null +++ b/test cases/windows/6 vs module defs/subdir/somedll.c @@ -0,0 +1,5 @@ +#ifdef _MSC_VER +int somedllfunc() { + return 42; +} +#endif diff --git a/test cases/windows/6 vs module defs/subdir/somedll.def b/test cases/windows/6 vs module defs/subdir/somedll.def new file mode 100644 index 0000000..217801b --- /dev/null +++ b/test cases/windows/6 vs module defs/subdir/somedll.def @@ -0,0 +1,3 @@ +EXPORTS + somedllfunc + |