From dbcbf19ecea9d07d264dbbc1cd87ab22393be8a7 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 21 Jan 2017 11:15:14 +0530 Subject: compilers: New class CompilerArgs derived from list() The purpose of this class is to make it possible to sanely generate compiler command-lines by ensuring that new arguments appended or added to a list of arguments properly override previous arguments. For instance: >>> a = CompilerArgs(['-Lfoo', '-DBAR']) >>> a += ['-Lgah', '-DTAZ'] >>> print(a) ['-Lgah', '-Lfoo', '-DBAR', '-DTAZ'] Arguments will be de-duped if it is safe to do so. Currently, this is only done for -I and -L arguments (previous occurances are removed when a new one is added) and arguments that once added cannot be overriden such as -pipe are removed completely. --- mesonbuild/backend/backends.py | 2 +- mesonbuild/backend/ninjabackend.py | 2 +- mesonbuild/backend/vs2010backend.py | 6 +- mesonbuild/compilers.py | 301 ++++++++++++++++++++++++++---------- run_unittests.py | 59 ++++++- 5 files changed, 281 insertions(+), 89 deletions(-) diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index 6f8a50e..eadc8cc 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -360,7 +360,7 @@ class Backend: for dep in target.get_external_deps(): # Cflags required by external deps might have UNIX-specific flags, # so filter them out if needed - commands += compiler.unix_compile_flags_to_native(dep.get_compile_args()) + commands += compiler.unix_args_to_native(dep.get_compile_args()) if isinstance(target, build.Executable): commands += dep.get_exe_args() diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 628718f..5bd660c 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -2071,7 +2071,7 @@ rule FORTRAN_DEP_HACK custom_target_libraries = self.get_custom_target_provided_libraries(target) commands += extra_args commands += custom_target_libraries - commands = linker.unix_link_flags_to_native(self.dedup_arguments(commands)) + commands = linker.unix_args_to_native(self.dedup_arguments(commands)) dep_targets = [self.get_dependency_filename(t) for t in dependencies] dep_targets += [os.path.join(self.environment.source_dir, target.subdir, t) for t in target.link_depends] diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py index 61f755b..31a4c0e 100644 --- a/mesonbuild/backend/vs2010backend.py +++ b/mesonbuild/backend/vs2010backend.py @@ -687,14 +687,14 @@ class Vs2010Backend(backends.Backend): file_args[l] += args for l, args in target.extra_args.items(): if l in file_args: - file_args[l] += compiler.unix_compile_flags_to_native(args) + file_args[l] += compiler.unix_args_to_native(args) for l, comp in target.compilers.items(): if l in file_args: file_args[l] += comp.get_option_compile_args(self.environment.coredata.compiler_options) for d in target.get_external_deps(): # Cflags required by external deps might have UNIX-specific flags, # so filter them out if needed - d_compile_args = compiler.unix_compile_flags_to_native(d.get_compile_args()) + d_compile_args = compiler.unix_args_to_native(d.get_compile_args()) for arg in d_compile_args: if arg.startswith(('-D', '/D')): define = arg[2:] @@ -793,7 +793,7 @@ class Vs2010Backend(backends.Backend): if isinstance(d, build.StaticLibrary): for dep in d.get_external_deps(): extra_link_args += dep.get_link_args() - extra_link_args = compiler.unix_link_flags_to_native(extra_link_args) + extra_link_args = compiler.unix_args_to_native(extra_link_args) (additional_libpaths, additional_links, extra_link_args) = self.split_link_args(extra_link_args) if len(extra_link_args) > 0: extra_link_args.append('%(AdditionalOptions)') diff --git a/mesonbuild/compilers.py b/mesonbuild/compilers.py index 0a88d6b..afe267a 100644 --- a/mesonbuild/compilers.py +++ b/mesonbuild/compilers.py @@ -323,6 +323,175 @@ class RunResult: self.stdout = stdout self.stderr = stderr +class CompilerArgs(list): + ''' + Class derived from list() that manages a list of compiler arguments. Should + be used while constructing compiler arguments from various sources. Can be + operated with ordinary lists, so this does not need to be used everywhere. + + All arguments must be inserted and stored in GCC-style (-lfoo, -Idir, etc) + and can converted to the native type of each compiler by using the + .to_native() method to which you must pass an instance of the compiler or + the compiler class. + + New arguments added to this class (either with .append(), .extend(), or +=) + are added in a way that ensures that they override previous arguments. + For example: + + >>> a = ['-Lfoo', '-lbar'] + >>> a += ['-Lpho', '-lbaz'] + >>> print(a) + ['-Lpho', '-Lfoo', '-lbar', '-lbaz'] + + Arguments will also be de-duped if they can be de-duped safely. + + Note that because of all this, this class is not commutative and does not + preserve the order of arguments if it is safe to not. For example: + >>> ['-Ifoo', '-Ibar'] + ['-Ifez', '-Ibaz', '-Werror'] + ['-Ifez', '-Ibaz', '-Ifoo', '-Ibar', '-Werror'] + >>> ['-Ifez', '-Ibaz', '-Werror'] + ['-Ifoo', '-Ibar'] + ['-Ifoo', '-Ibar', '-Ifez', '-Ibaz', '-Werror'] + + ''' + # NOTE: currently this class is only for C-like compilers, but it can be + # extended to other languages easily. Just move the following to the + # compiler class and initialize when self.compiler is set. + + # Arg prefixes that override by prepending instead of appending + prepend_prefixes = ('-I', '-L') + # Arg prefixes and args that must be de-duped by returning 2 + dedup2_prefixes = ('-I', '-L', '-D') + dedup2_args = () + # Arg prefixes and args that must be de-duped by returning 1 + dedup1_prefixes = () + dedup1_args = ('-c', '-S', '-E', '-pipe') + compiler = None + + def _check_args(self, args): + cargs = [] + if len(args) > 2: + raise TypeError("CompilerArgs() only accepts at most 2 arguments: " + "The compiler, and optionally an initial list") + elif len(args) == 0: + return cargs + elif len(args) == 1: + if isinstance(args[0], (Compiler, StaticLinker)): + self.compiler = args[0] + else: + raise TypeError("you must pass a Compiler instance as one of " + "the arguments") + elif len(args) == 2: + if isinstance(args[0], (Compiler, StaticLinker)): + self.compiler = args[0] + cargs = args[1] + elif isinstance(args[1], (Compiler, StaticLinker)): + cargs = args[0] + self.compiler = args[1] + else: + raise TypeError("you must pass a Compiler instance as one of " + "the two arguments") + else: + raise AssertionError('Not reached') + return cargs + + def __init__(self, *args): + super().__init__(self._check_args(args)) + + @classmethod + def _can_dedup(cls, arg): + ''' + Returns whether the argument can be safely de-duped. This is dependent + on two things: + + a) Whether an argument can be 'overriden' by a later argument. For + example, -DFOO defines FOO and -UFOO undefines FOO. In this case, we + can safely remove the previous occurance and add a new one. The same + is true for include paths and library paths with -I and -L. For + these we return `2`. See `dedup2_prefixes` and `dedup2_args`. + b) Arguments that once specifie cannot be undone, such as `-c` or + `-pipe`. New instances of these can be completely skipped. For these + we return `1`. See `dedup1_prefixes` and `dedup1_args`. + c) Whether it matters where or how many times on the command-line + a particular argument is present. This can matter for symbol + resolution in static or shared libraries, so we cannot de-dup or + reorder them. For these we return `0`. This is the default. + ''' + if arg.startswith(cls.dedup2_prefixes) or arg in cls.dedup2_args: + return 2 + if arg.startswith(cls.dedup1_prefixes) or arg in cls.dedup1_args: + return 1 + return 0 + + @classmethod + def _should_prepend(cls, arg): + if arg.startswith(cls.prepend_prefixes): + return True + return False + + def to_native(self): + return self.compiler.unix_args_to_native(self) + + def __add__(self, args): + new = CompilerArgs(self, self.compiler) + new += args + return new + + def __iadd__(self, args): + ''' + Add two CompilerArgs while taking into account overriding of arguments + and while preserving the order of arguments as much as possible + ''' + pre = [] + post = [] + if not isinstance(args, list): + raise TypeError('can only concatenate list (not "{}") to list'.format(args)) + for arg in args: + # If the argument can be de-duped, do it either by removing the + # previous occurance of it and adding a new one, or not adding the + # new occurance. + dedup = self._can_dedup(arg) + if dedup == 1: + # Argument already exists and adding a new instance is useless + if arg in self or arg in pre or arg in post: + continue + if dedup == 2: + # Remove all previous occurances of the arg and add it anew + if arg in self: + self.remove(arg) + if arg in pre: + pre.remove(arg) + if arg in post: + post.remove(arg) + if self._should_prepend(arg): + pre.append(arg) + else: + post.append(arg) + # Insert at the beginning + self[:0] = pre + # Append to the end + super().__iadd__(post) + return self + + def __radd__(self, args): + new = CompilerArgs(args, self.compiler) + new += self + return new + + def __mul__(self, args): + raise TypeError("can't multiply compiler arguments") + + def __imul__(self, args): + raise TypeError("can't multiply compiler arguments") + + def __rmul__(self, args): + raise TypeError("can't multiply compiler arguments") + + def append(self, arg): + self.__iadd__([arg]) + + def extend(self, args): + self.__iadd__(args) + class Compiler: def __init__(self, exelist, version): if isinstance(exelist, str): @@ -412,11 +581,8 @@ class Compiler: def has_function(self, *args, **kwargs): raise EnvironmentException('Language %s does not support function checks.' % self.language) - def unix_link_flags_to_native(self, args): - "Always returns a copy that can be independently mutated" - return args[:] - - def unix_compile_flags_to_native(self, args): + @classmethod + def unix_args_to_native(cls, args): "Always returns a copy that can be independently mutated" return args[:] @@ -435,7 +601,7 @@ class Compiler: self.language)) def get_cross_extra_flags(self, environment, *, compile, link): - extra_flags = [] + extra_flags = CompilerArgs(self) if self.is_cross and environment: if 'properties' in environment.cross_info.config: lang_args_key = self.language + '_args' @@ -474,7 +640,7 @@ class Compiler: output = self._get_compile_output(tmpdirname, mode) # Construct the compiler command-line - commands = self.get_exelist() + commands = CompilerArgs(self) commands.append(srcname) commands += extra_args commands += self.get_always_args() @@ -485,6 +651,8 @@ class Compiler: commands += self.get_preprocess_only_args() else: commands += self.get_output_args(output) + # Generate full command-line with the exelist + commands = self.get_exelist() + commands.to_native() mlog.debug('Running compile:') mlog.debug('Working directory: ', tmpdirname) mlog.debug('Command line: ', ' '.join(commands), '\n') @@ -663,7 +831,7 @@ class CCompiler(Compiler): mlog.debug('Sanity testing ' + self.language + ' compiler:', ' '.join(self.exelist)) mlog.debug('Is cross compiler: %s.' % str(self.is_cross)) - extra_flags = [] + extra_flags = CompilerArgs(self) source_name = os.path.join(work_dir, sname) binname = sname.rsplit('.', 1)[0] if self.is_cross: @@ -742,51 +910,29 @@ class CCompiler(Compiler): }}''' return self.compiles(t.format(**fargs), env, extra_args, dependencies) - @staticmethod - def _override_args(args, override): - ''' - Add @override to @args in such a way that arguments are overriden - correctly. - - We want the include directories to be added first (since they are - chosen left-to-right) and all other arguments later (since they - override previous arguments or add to a list that's chosen - right-to-left). - ''' - before_args = [] - after_args = [] - for arg in override: - if arg.startswith(('-I', '/I')): - before_args.append(arg) - else: - after_args.append(arg) - return before_args + args + after_args - def compiles(self, code, env, extra_args=None, dependencies=None, mode='compile'): if extra_args is None: extra_args = [] - if isinstance(extra_args, str): + elif isinstance(extra_args, str): extra_args = [extra_args] if dependencies is None: dependencies = [] elif not isinstance(dependencies, list): dependencies = [dependencies] - # Add compile flags needed by dependencies after converting to the - # native type of the selected compiler - cargs = [a for d in dependencies for a in d.get_compile_args()] - args = self.unix_link_flags_to_native(cargs) + # Add compile flags needed by dependencies + args = CompilerArgs(self) + for d in dependencies: + args += d.get_compile_args() # Read c_args/cpp_args/etc from the cross-info file (if needed) args += self.get_cross_extra_flags(env, compile=True, link=False) # Add CFLAGS/CXXFLAGS/OBJCFLAGS/OBJCXXFLAGS from the env # We assume that the user has ensured these are compiler-specific args += env.coredata.external_args[self.language] - # Append extra_args to the compiler check args such that it overrides - extra_args = self._override_args(self.get_compiler_check_args(), extra_args) - extra_args = self.unix_link_flags_to_native(extra_args) - # Append both to the compiler args such that they override them - args = self._override_args(args, extra_args) + args += self.get_compiler_check_args() + # extra_args must override all other arguments, so we add them last + args += extra_args # We only want to compile; not link - with self.compile(code, args, mode) as p: + with self.compile(code, args.to_native(), mode) as p: return p.returncode == 0 def _links_wrapper(self, code, env, extra_args, dependencies): @@ -799,11 +945,11 @@ class CCompiler(Compiler): dependencies = [] elif not isinstance(dependencies, list): dependencies = [dependencies] - # Add compile and link flags needed by dependencies after converting to - # the native type of the selected compiler - cargs = [a for d in dependencies for a in d.get_compile_args()] - link_args = [a for d in dependencies for a in d.get_link_args()] - args = self.unix_link_flags_to_native(cargs + link_args) + # Add compile and link flags needed by dependencies + args = CompilerArgs(self) + for d in dependencies: + args += d.get_compile_args() + args += d.get_link_args() # Select a CRT if needed since we're linking args += self.get_linker_debug_crt_args() # Read c_args/c_link_args/cpp_args/cpp_link_args/etc from the @@ -812,12 +958,11 @@ class CCompiler(Compiler): # Add LDFLAGS from the env. We assume that the user has ensured these # are compiler-specific args += env.coredata.external_link_args[self.language] - # Append extra_args to the compiler check args such that it overrides - extra_args = self._override_args(self.get_compiler_check_args(), extra_args) - extra_args = self.unix_link_flags_to_native(extra_args) - # Append both to the compiler args such that they override them - args = self._override_args(args, extra_args) - return self.compile(code, args) + # Add compiler check args such that they override + args += self.get_compiler_check_args() + # extra_args must override all other arguments, so we add them last + args += extra_args + return self.compile(code, args.to_native()) def links(self, code, env, extra_args=None, dependencies=None): with self._links_wrapper(code, env, extra_args, dependencies) as p: @@ -1686,7 +1831,8 @@ class DCompiler(Compiler): paths = paths + ':' + padding return ['-L-rpath={}'.format(paths)] - def translate_args_to_nongnu(self, args): + @classmethod + def translate_args_to_nongnu(cls, args): dcargs = [] # Translate common arguments to flags the LDC/DMD compilers # can understand. @@ -1802,11 +1948,9 @@ class LLVMDCompiler(DCompiler): # -L is for the compiler, telling it to pass the second -L to the linker. return ['-L-L' + dirname] - def unix_link_flags_to_native(self, args): - return self.translate_args_to_nongnu(args) - - def unix_compile_flags_to_native(self, args): - return self.translate_args_to_nongnu(args) + @classmethod + def unix_args_to_native(cls, args): + return cls.translate_args_to_nongnu(args) class DmdDCompiler(DCompiler): def __init__(self, exelist, version, is_cross): @@ -1854,11 +1998,9 @@ class DmdDCompiler(DCompiler): def get_std_shared_lib_link_args(self): return ['-shared', '-defaultlib=libphobos2.so'] - def unix_link_flags_to_native(self, args): - return self.translate_args_to_nongnu(args) - - def unix_compile_flags_to_native(self, args): - return self.translate_args_to_nongnu(args) + @classmethod + def unix_args_to_native(cls, args): + return cls.translate_args_to_nongnu(args) class VisualStudioCCompiler(CCompiler): std_warn_args = ['/W3'] @@ -1978,9 +2120,14 @@ class VisualStudioCCompiler(CCompiler): def get_option_link_args(self, options): return options['c_winlibs'].value[:] - def unix_link_flags_to_native(self, args): + @classmethod + def unix_args_to_native(cls, args): result = [] for i in args: + # -mms-bitfields is specific to MinGW-GCC + # -pthread is only valid for GCC + if i in ('-mms-bitfields', '-pthread'): + continue if i.startswith('-L'): i = '/LIBPATH:' + i[2:] # Translate GNU-style -lfoo library name to the import library @@ -1998,16 +2145,6 @@ class VisualStudioCCompiler(CCompiler): result.append(i) return result - def unix_compile_flags_to_native(self, args): - result = [] - for i in args: - # -mms-bitfields is specific to MinGW-GCC - # -pthread is only valid for GCC - if i in ('-mms-bitfields', '-pthread'): - continue - result.append(i) - return result - def get_werror_args(self): return ['/WX'] @@ -2795,8 +2932,10 @@ class NAGFortranCompiler(FortranCompiler): def get_warn_args(self, level): return NAGFortranCompiler.std_warn_args +class StaticLinker: + pass -class VisualStudioLinker: +class VisualStudioLinker(StaticLinker): always_args = ['/NOLOGO'] def __init__(self, exelist): @@ -2832,18 +2971,16 @@ class VisualStudioLinker: def get_option_link_args(self, options): return [] - def unix_link_flags_to_native(self, args): - return args[:] - - def unix_compile_flags_to_native(self, args): - return args[:] + @classmethod + def unix_args_to_native(cls, args): + return VisualStudioCCompiler.unix_args_to_native(args) def get_link_debugfile_args(self, targetfile): pdbarr = targetfile.split('.')[:-1] pdbarr += ['pdb'] return ['/DEBUG', '/PDB:' + '.'.join(pdbarr)] -class ArLinker: +class ArLinker(StaticLinker): def __init__(self, exelist): self.exelist = exelist @@ -2885,10 +3022,8 @@ class ArLinker: def get_option_link_args(self, options): return [] - def unix_link_flags_to_native(self, args): - return args[:] - - def unix_compile_flags_to_native(self, args): + @classmethod + def unix_args_to_native(cls, args): return args[:] def get_link_debugfile_args(self, targetfile): diff --git a/run_unittests.py b/run_unittests.py index 5eba222..a90e46b 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -19,6 +19,7 @@ import subprocess import re, json import tempfile from glob import glob +import mesonbuild.compilers import mesonbuild.environment import mesonbuild.mesonlib from mesonbuild.environment import detect_ninja, Environment @@ -95,6 +96,62 @@ class InternalTests(unittest.TestCase): stat.S_IRWXU | stat.S_ISUID | stat.S_IRGRP | stat.S_IXGRP) + def test_compiler_args_class(self): + cargsfunc = mesonbuild.compilers.CompilerArgs + c = mesonbuild.environment.CCompiler([], 'fake', False) + # Test that bad initialization fails + self.assertRaises(TypeError, cargsfunc, []) + self.assertRaises(TypeError, cargsfunc, [], []) + self.assertRaises(TypeError, cargsfunc, c, [], []) + # Test that empty initialization works + a = cargsfunc(c) + self.assertEqual(a, []) + # Test that list initialization works + a = cargsfunc(['-I.', '-I..'], c) + self.assertEqual(a, ['-I.', '-I..']) + # Test that there is no de-dup on initialization + self.assertEqual(cargsfunc(['-I.', '-I.'], c), ['-I.', '-I.']) + + ## Test that appending works + a.append('-I..') + self.assertEqual(a, ['-I..', '-I.']) + a.append('-O3') + self.assertEqual(a, ['-I..', '-I.', '-O3']) + + ## Test that in-place addition works + a += ['-O2', '-O2'] + self.assertEqual(a, ['-I..', '-I.', '-O3', '-O2', '-O2']) + # Test that removal works + a.remove('-O2') + self.assertEqual(a, ['-I..', '-I.', '-O3', '-O2']) + # Test that de-dup happens on addition + a += ['-Ifoo', '-Ifoo'] + self.assertEqual(a, ['-Ifoo', '-I..', '-I.', '-O3', '-O2']) + + # .extend() is just +=, so we don't test it + + ## Test that addition works + # Test that adding a list with just one old arg works and yields the same array + a = a + ['-Ifoo'] + self.assertEqual(a, ['-Ifoo', '-I..', '-I.', '-O3', '-O2']) + # Test that adding a list with one arg new and one old works + a = a + ['-Ifoo', '-Ibaz'] + self.assertEqual(a, ['-Ifoo', '-Ibaz', '-I..', '-I.', '-O3', '-O2']) + # Test that adding args that must be prepended and appended works + a = a + ['-Ibar', '-Wall'] + self.assertEqual(a, ['-Ibar', '-Ifoo', '-Ibaz', '-I..', '-I.', '-O3', '-O2', '-Wall']) + + ## Test that reflected addition works + # Test that adding to a list with just one old arg works and DOES NOT yield the same array + a = ['-Ifoo'] + a + self.assertEqual(a, ['-Ibar', '-Ifoo', '-Ibaz', '-I..', '-I.', '-O3', '-O2', '-Wall']) + # Test that adding to a list with just one new arg that is not pre-pended works + a = ['-Werror'] + a + self.assertEqual(a, ['-Ibar', '-Ifoo', '-Ibaz', '-I..', '-I.', '-Werror', '-O3', '-O2', '-Wall']) + # Test that adding to a list with two new args preserves the order + a = ['-Ldir', '-Lbah'] + a + self.assertEqual(a, ['-Ibar', '-Ifoo', '-Ibaz', '-I..', '-I.', '-Ldir', '-Lbah', '-Werror', '-O3', '-O2', '-Wall']) + class LinuxlikeTests(unittest.TestCase): def setUp(self): @@ -416,7 +473,7 @@ class LinuxlikeTests(unittest.TestCase): cmd = cmd[1:] # Verify that -I flags from the `args` kwarg are first # This is set in the '43 has function' test case - self.assertEqual(cmd[2], '-I/tmp') + self.assertEqual(cmd[1], '-I/tmp') # Verify that -O3 set via the environment is overriden by -O0 Oargs = [arg for arg in cmd if arg.startswith('-O')] self.assertEqual(Oargs, [Oflag, '-O0']) -- cgit v1.1 From 2bb58c909fd80cc8ce053b5f1b6565bd28a416d2 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 21 Jan 2017 12:05:38 +0530 Subject: Use CompilerArgs for generation of compile commands At the same time, also fix the order in which compile arguments are added. Detailed comments have been added concerning the priority and order of the arguments. Also adds a unit test and an integration test for the same. --- mesonbuild/backend/backends.py | 50 ++++++-- mesonbuild/backend/ninjabackend.py | 136 ++++++++++++++------- mesonbuild/build.py | 10 +- mesonbuild/coredata.py | 4 +- mesonbuild/dependencies.py | 8 +- run_unittests.py | 33 +++++ test cases/common/138 include order/meson.build | 22 ++++ test cases/common/138 include order/sub1/main.h | 1 + .../common/138 include order/sub1/meson.build | 4 + test cases/common/138 include order/sub1/some.c | 6 + test cases/common/138 include order/sub1/some.h | 10 ++ test cases/common/138 include order/sub2/main.h | 1 + .../common/138 include order/sub2/meson.build | 1 + test cases/common/138 include order/sub3/main.h | 1 + .../common/138 include order/sub3/meson.build | 1 + test cases/common/138 include order/sub4/main.c | 8 ++ test cases/common/138 include order/sub4/main.h | 3 + .../common/138 include order/sub4/meson.build | 4 + 18 files changed, 238 insertions(+), 65 deletions(-) create mode 100644 test cases/common/138 include order/meson.build create mode 100644 test cases/common/138 include order/sub1/main.h create mode 100644 test cases/common/138 include order/sub1/meson.build create mode 100644 test cases/common/138 include order/sub1/some.c create mode 100644 test cases/common/138 include order/sub1/some.h create mode 100644 test cases/common/138 include order/sub2/main.h create mode 100644 test cases/common/138 include order/sub2/meson.build create mode 100644 test cases/common/138 include order/sub3/main.h create mode 100644 test cases/common/138 include order/sub3/meson.build create mode 100644 test cases/common/138 include order/sub4/main.c create mode 100644 test cases/common/138 include order/sub4/main.h create mode 100644 test cases/common/138 include order/sub4/meson.build diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index eadc8cc..a83d95f 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -21,6 +21,7 @@ from .. import compilers import json import subprocess from ..mesonlib import MesonException, get_compiler_for_source, classify_unity_sources +from ..compilers import CompilerArgs class CleanTrees: ''' @@ -338,32 +339,59 @@ class Backend: return extra_args def generate_basic_compiler_args(self, target, compiler, no_warn_args=False): - commands = [] + # Create an empty commands list, and start adding arguments from + # various sources in the order in which they must override each other + # starting from hard-coded defaults followed by build options and so on. + commands = CompilerArgs(compiler) + # First, the trivial ones that are impossible to override. + # + # Add -nostdinc/-nostdinc++ if needed; can't be overriden commands += self.get_cross_stdlib_args(target, compiler) + # Add things like /NOLOGO or -pipe; usually can't be overriden commands += compiler.get_always_args() + # Only add warning-flags by default if the buildtype enables it, and if + # we weren't explicitly asked to not emit warnings (for Vala, f.ex) if no_warn_args: commands += compiler.get_no_warn_args() elif self.environment.coredata.get_builtin_option('buildtype') != 'plain': commands += compiler.get_warn_args(self.environment.coredata.get_builtin_option('warning_level')) + # Add -Werror if werror=true is set in the build options set on the + # command-line or default_options inside project(). This only sets the + # action to be done for warnings if/when they are emitted, so it's ok + # to set it after get_no_warn_args() or get_warn_args(). + if self.environment.coredata.get_builtin_option('werror'): + commands += compiler.get_werror_args() + # Add compile args for c_* or cpp_* build options set on the + # command-line or default_options inside project(). commands += compiler.get_option_compile_args(self.environment.coredata.compiler_options) - commands += self.build.get_global_args(compiler) + # Add buildtype args: optimization level, debugging, etc. + commands += compiler.get_buildtype_args(self.environment.coredata.get_builtin_option('buildtype')) + # Add compile args added using add_project_arguments() commands += self.build.get_project_args(compiler, target.subproject) + # Add compile args added using add_global_arguments() + # These override per-project arguments + commands += self.build.get_global_args(compiler) + # Compile args added from the env: CFLAGS/CXXFLAGS, etc. We want these + # to override all the defaults, but not the per-target compile args. commands += self.environment.coredata.external_args[compiler.get_language()] - commands += self.escape_extra_args(compiler, target.get_extra_args(compiler.get_language())) - commands += compiler.get_buildtype_args(self.environment.coredata.get_builtin_option('buildtype')) - if self.environment.coredata.get_builtin_option('werror'): - commands += compiler.get_werror_args() + # Always set -fPIC for shared libraries if isinstance(target, build.SharedLibrary): commands += compiler.get_pic_args() + # Set -fPIC for static libraries by default unless explicitly disabled if isinstance(target, build.StaticLibrary) and target.pic: commands += compiler.get_pic_args() + # Add compile args needed to find external dependencies + # Link args are added while generating the link command for dep in target.get_external_deps(): - # Cflags required by external deps might have UNIX-specific flags, - # so filter them out if needed - commands += compiler.unix_args_to_native(dep.get_compile_args()) + commands += dep.get_compile_args() + # Qt needs -fPIC for executables + # XXX: We should move to -fPIC for all executables if isinstance(target, build.Executable): - commands += dep.get_exe_args() - + commands += dep.get_exe_args(compiler) + # For 'automagic' deps: Boost and GTest. Also dependency('threads'). + # pkg-config puts the thread flags itself via `Cflags:` + if dep.need_threads(): + commands += compiler.thread_flags() # Fortran requires extra include directives. if compiler.language == 'fortran': for lt in target.link_targets: diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 5bd660c..98740a4 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -19,6 +19,7 @@ from .. import build from .. import mlog from .. import dependencies from .. import compilers +from ..compilers import CompilerArgs from ..mesonlib import File, MesonException, get_compiler_for_source, Popen_safe from .backends import CleanTrees, InstallData from ..build import InvalidArguments @@ -1725,7 +1726,7 @@ rule FORTRAN_DEP_HACK def generate_llvm_ir_compile(self, target, outfile, src): compiler = get_compiler_for_source(target.compilers.values(), src) - commands = [] + commands = CompilerArgs(compiler) # Compiler args for compiling this target commands += compilers.get_base_compile_args(self.environment.coredata.base_options, compiler) @@ -1748,11 +1749,40 @@ rule FORTRAN_DEP_HACK # Write the Ninja build command compiler_name = 'llvm_ir{}_COMPILER'.format('_CROSS' if target.is_cross else '') element = NinjaBuildElement(self.all_outputs, rel_obj, compiler_name, rel_src) - commands = self.dedup_arguments(commands) + # Convert from GCC-style link argument naming to the naming used by the + # current compiler. + commands = commands.to_native() element.add_item('ARGS', commands) element.write(outfile) return rel_obj + def get_source_dir_include_args(self, target, compiler): + curdir = target.get_subdir() + tmppath = os.path.normpath(os.path.join(self.build_to_src, curdir)) + return compiler.get_include_args(tmppath, False) + + def get_build_dir_include_args(self, target, compiler): + curdir = target.get_subdir() + if curdir == '': + curdir = '.' + return compiler.get_include_args(curdir, False) + + def get_custom_target_dir_include_args(self, target, compiler): + custom_target_include_dirs = [] + for i in target.get_generated_sources(): + # Generator output goes into the target private dir which is + # already in the include paths list. Only custom targets have their + # own target build dir. + if not isinstance(i, build.CustomTarget): + continue + idir = self.get_target_dir(i) + if idir not in custom_target_include_dirs: + custom_target_include_dirs.append(idir) + incs = [] + for i in custom_target_include_dirs: + incs += compiler.get_include_args(i, False) + return incs + def generate_single_compile(self, target, outfile, src, is_generated=False, header_deps=[], order_deps=[]): """ Compiles C/C++, ObjC/ObjC++, Fortran, and D sources @@ -1763,30 +1793,40 @@ rule FORTRAN_DEP_HACK raise AssertionError('BUG: sources should not contain headers {!r}'.format(src.fname)) extra_orderdeps = [] compiler = get_compiler_for_source(target.compilers.values(), src) - commands = [] - # The first thing is implicit include directories: source, build and private. - commands += compiler.get_include_args(self.get_target_private_dir(target), False) - # Compiler args for compiling this target + + # Create an empty commands list, and start adding arguments from + # various sources in the order in which they must override each other + commands = CompilerArgs(compiler) + # Add compiler args for compiling this target derived from 'base' build + # options passed on the command-line, in default_options, etc. + # These have the lowest priority. commands += compilers.get_base_compile_args(self.environment.coredata.base_options, compiler) - # Add the root source and build directories as include dirs - curdir = target.get_subdir() - tmppath = os.path.normpath(os.path.join(self.build_to_src, curdir)) - src_inc = compiler.get_include_args(tmppath, False) - if curdir == '': - curdir = '.' - build_inc = compiler.get_include_args(curdir, False) - commands += build_inc + src_inc - # -I args work differently than other ones. In them the first found - # directory is used whereas for other flags (such as -ffoo -fno-foo) the - # latest one is used. Therefore put the internal include directories - # here before generating the "basic compiler args" so they override args - # coming from e.g. pkg-config. + # The code generated by valac is usually crap and has tons of unused + # variables and such, so disable warnings for Vala C sources. + no_warn_args = (is_generated == 'vala') + # Add compiler args and include paths from several sources; defaults, + # build options, external dependencies, etc. + commands += self.generate_basic_compiler_args(target, compiler, no_warn_args) + # Add include dirs from the `include_directories:` kwarg on the target + # and from `include_directories:` of internal deps of the target. + # + # Target include dirs should override internal deps include dirs. + # + # Include dirs from internal deps should override include dirs from + # external deps. for i in target.get_include_dirs(): basedir = i.get_curdir() for d in i.get_incdirs(): - expdir = os.path.join(basedir, d) + # Avoid superfluous '/.' at the end of paths when d is '.' + if d not in ('', '.'): + expdir = os.path.join(basedir, d) + else: + expdir = basedir srctreedir = os.path.join(self.build_to_src, expdir) + # Add source subdir first so that the build subdir overrides it + sargs = compiler.get_include_args(srctreedir, i.is_system) + commands += sargs # There may be include dirs where a build directory has not been # created for some source dir. For example if someone does this: # @@ -1797,20 +1837,32 @@ rule FORTRAN_DEP_HACK bargs = compiler.get_include_args(expdir, i.is_system) else: bargs = [] - sargs = compiler.get_include_args(srctreedir, i.is_system) commands += bargs - commands += sargs for d in i.get_extra_build_dirs(): commands += compiler.get_include_args(d, i.is_system) - commands += self.generate_basic_compiler_args(target, compiler, - # The code generated by valac is usually crap - # and has tons of unused variables and such, - # so disable warnings for Vala C sources. - no_warn_args=(is_generated == 'vala')) - for d in target.external_deps: - if d.need_threads(): - commands += compiler.thread_flags() - break + # Add per-target compile args, f.ex, `c_args : ['-DFOO']`. We set these + # near the end since these are supposed to override everything else. + commands += self.escape_extra_args(compiler, + target.get_extra_args(compiler.get_language())) + # Add source dir and build dir. Project-specific and target-specific + # include paths must override per-target compile args, include paths + # from external dependencies, internal dependencies, and from + # per-target `include_directories:` + # + # We prefer headers in the build dir and the custom target dir over the + # source dir since, for instance, the user might have an + # srcdir == builddir Autotools build in their source tree. Many + # projects that are moving to Meson have both Meson and Autotools in + # parallel as part of the transition. + commands += self.get_source_dir_include_args(target, compiler) + commands += self.get_custom_target_dir_include_args(target, compiler) + commands += self.get_build_dir_include_args(target, compiler) + # Finally add the private dir for the target to the include path. This + # must override everything else and must be the final path added. + commands += compiler.get_include_args(self.get_target_private_dir(target), False) + + # FIXME: This file handling is atrocious and broken. We need to + # replace it with File objects used consistently everywhere. if isinstance(src, RawFilename): rel_src = src.fname if os.path.isabs(src.fname): @@ -1835,7 +1887,13 @@ rule FORTRAN_DEP_HACK rel_obj = os.path.join(self.get_target_private_dir(target), obj_basename) rel_obj += '.' + self.environment.get_object_suffix() dep_file = compiler.depfile_for_object(rel_obj) + + # Add MSVC debug file generation compile flags: /Fd /FS + commands += self.get_compile_debugfile_args(compiler, target, rel_obj) + + # PCH handling if self.environment.coredata.base_options.get('b_pch', False): + commands += self.get_pch_include_args(compiler, target) pchlist = target.get_pch(compiler.language) else: pchlist = [] @@ -1848,19 +1906,7 @@ rule FORTRAN_DEP_HACK i = os.path.join(self.get_target_private_dir(target), compiler.get_pch_name(pchlist[0])) arr.append(i) pch_dep = arr - custom_target_include_dirs = [] - for i in target.get_generated_sources(): - if not isinstance(i, build.CustomTarget): - continue - idir = self.get_target_dir(i) - if idir not in custom_target_include_dirs: - custom_target_include_dirs.append(idir) - for i in custom_target_include_dirs: - commands += compiler.get_include_args(i, False) - if self.environment.coredata.base_options.get('b_pch', False): - commands += self.get_pch_include_args(compiler, target) - commands += self.get_compile_debugfile_args(compiler, target, rel_obj) crstr = '' if target.is_cross: crstr = '_CROSS' @@ -1895,7 +1941,9 @@ rule FORTRAN_DEP_HACK element.add_orderdep(d) element.add_orderdep(pch_dep) element.add_orderdep(extra_orderdeps) - commands = self.dedup_arguments(commands) + # Convert from GCC-style link argument naming to the naming used by the + # current compiler. + commands = commands.to_native() for i in self.get_fortran_orderdeps(target, compiler): element.add_orderdep(i) element.add_item('DEPFILE', dep_file) diff --git a/mesonbuild/build.py b/mesonbuild/build.py index a9f08c3..e5d2284 100644 --- a/mesonbuild/build.py +++ b/mesonbuild/build.py @@ -585,14 +585,16 @@ class BuildTarget(Target): for i in self.link_depends: if not isinstance(i, str): raise InvalidArguments('Link_depends arguments must be strings.') - inclist = kwargs.get('include_directories', []) - if not isinstance(inclist, list): - inclist = [inclist] - self.add_include_dirs(inclist) deplist = kwargs.get('dependencies', []) if not isinstance(deplist, list): deplist = [deplist] self.add_deps(deplist) + # Target-specific include dirs must be added after include dirs from + # internal deps (added inside self.add_deps()) to override correctly. + inclist = kwargs.get('include_directories', []) + if not isinstance(inclist, list): + inclist = [inclist] + self.add_include_dirs(inclist) self.custom_install_dir = kwargs.get('install_dir', None) if self.custom_install_dir is not None: if not isinstance(self.custom_install_dir, str): diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index 60bb10b..1ec769a 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -117,7 +117,9 @@ class CoreData: self.user_options = {} self.compiler_options = {} self.base_options = {} - self.external_args = {} # These are set from "the outside" with e.g. mesonconf + # These two, external_*args, are set via env vars CFLAGS, LDFLAGS, etc + # but only when not cross-compiling. + self.external_args = {} self.external_link_args = {} if options.cross_file is not None: self.cross_file = os.path.join(os.getcwd(), options.cross_file) diff --git a/mesonbuild/dependencies.py b/mesonbuild/dependencies.py index 6ae91d4..b01e0a8 100644 --- a/mesonbuild/dependencies.py +++ b/mesonbuild/dependencies.py @@ -59,7 +59,7 @@ class Dependency: def get_name(self): return self.name - def get_exe_args(self): + def get_exe_args(self, compiler): return [] def need_threads(self): @@ -1045,16 +1045,14 @@ class QtBaseDependency(Dependency): def found(self): return self.is_found - def get_exe_args(self): + def get_exe_args(self, compiler): # Originally this was -fPIE but nowadays the default # for upstream and distros seems to be -reduce-relocations # which requires -fPIC. This may cause a performance # penalty when using self-built Qt or on platforms # where -fPIC is not required. If this is an issue # for you, patches are welcome. - if mesonlib.is_linux(): - return ['-fPIC'] - return [] + return compiler.get_pic_args() class Qt5Dependency(QtBaseDependency): def __init__(self, env, kwargs): diff --git a/run_unittests.py b/run_unittests.py index a90e46b..b6ea073 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -14,6 +14,7 @@ # limitations under the License. import stat +import shlex import unittest, os, sys, shutil, time import subprocess import re, json @@ -698,6 +699,38 @@ class LinuxlikeTests(unittest.TestCase): # The chown failed nonfatally if we're not root self.assertEqual(0, statf.st_uid) + def test_internal_include_order(self): + testdir = os.path.join(self.common_test_dir, '138 include order') + self.init(testdir) + for cmd in self.get_compdb(): + if cmd['file'].endswith('/main.c'): + cmd = cmd['command'] + break + else: + raise Exception('Could not find main.c command') + incs = [a for a in shlex.split(cmd) if a.startswith("-I")] + self.assertEqual(len(incs), 8) + # target private dir + self.assertEqual(incs[0], "-Isub4/someexe@exe") + # target build subdir + self.assertEqual(incs[1], "-Isub4") + # target source subdir + msg = "{!r} does not end with '/sub4'".format(incs[2]) + self.assertTrue(incs[2].endswith("/sub4"), msg) + # include paths added via per-target c_args: ['-I'...] + msg = "{!r} does not end with '/sub3'".format(incs[3]) + self.assertTrue(incs[3].endswith("/sub3"), msg) + # target include_directories: build dir + self.assertEqual(incs[4], "-Isub2") + # target include_directories: source dir + msg = "{!r} does not end with '/sub2'".format(incs[5]) + self.assertTrue(incs[5].endswith("/sub2"), msg) + # target internal dependency include_directories: build dir + self.assertEqual(incs[6], "-Isub1") + # target internal dependency include_directories: source dir + msg = "{!r} does not end with '/sub1'".format(incs[7]) + self.assertTrue(incs[7].endswith("/sub1"), msg) + class RewriterTests(unittest.TestCase): diff --git a/test cases/common/138 include order/meson.build b/test cases/common/138 include order/meson.build new file mode 100644 index 0000000..f744ae7 --- /dev/null +++ b/test cases/common/138 include order/meson.build @@ -0,0 +1,22 @@ +project('include order', 'c') + +# Test that the order of priority of include paths (from first to last) is: +# +# 1. Target's current build directory +# 2. Target's current source directory +# 3. Include paths added with the `c_args:` kwarg +# 4. Include paths added with the `include_directories`: kwarg +# Within this, the build dir takes precedence over the source dir +# 5. Include paths added via `include_directories:` of internal deps +# Within this, the build dir takes precedence over the source dir + +# Defines an internal dep +subdir('sub1') +# Defines a per-target include path +subdir('sub2') +# Directory for `c_args:` include path +subdir('sub3') +# The directory where the target resides +subdir('sub4') + +test('eh', e) diff --git a/test cases/common/138 include order/sub1/main.h b/test cases/common/138 include order/sub1/main.h new file mode 100644 index 0000000..acf4a35 --- /dev/null +++ b/test cases/common/138 include order/sub1/main.h @@ -0,0 +1 @@ +#error "sub1/main.h included" diff --git a/test cases/common/138 include order/sub1/meson.build b/test cases/common/138 include order/sub1/meson.build new file mode 100644 index 0000000..9672945 --- /dev/null +++ b/test cases/common/138 include order/sub1/meson.build @@ -0,0 +1,4 @@ +i = include_directories('.') +l = shared_library('somelib', 'some.c') +dep = declare_dependency(link_with : l, + include_directories : i) diff --git a/test cases/common/138 include order/sub1/some.c b/test cases/common/138 include order/sub1/some.c new file mode 100644 index 0000000..1ab0db4 --- /dev/null +++ b/test cases/common/138 include order/sub1/some.c @@ -0,0 +1,6 @@ +#if defined _WIN32 || defined __CYGWIN__ + __declspec(dllexport) +#endif +int somefunc(void) { + return 1984; +} diff --git a/test cases/common/138 include order/sub1/some.h b/test cases/common/138 include order/sub1/some.h new file mode 100644 index 0000000..6479492 --- /dev/null +++ b/test cases/common/138 include order/sub1/some.h @@ -0,0 +1,10 @@ +#pragma once + +#if defined _WIN32 || defined __CYGWIN__ + #define DLL_PUBLIC __declspec(dllimport) +#else + #define DLL_PUBLIC +#endif + +DLL_PUBLIC +int somefunc(void); diff --git a/test cases/common/138 include order/sub2/main.h b/test cases/common/138 include order/sub2/main.h new file mode 100644 index 0000000..b9c0da9 --- /dev/null +++ b/test cases/common/138 include order/sub2/main.h @@ -0,0 +1 @@ +#error "sub2/main.h included" diff --git a/test cases/common/138 include order/sub2/meson.build b/test cases/common/138 include order/sub2/meson.build new file mode 100644 index 0000000..7b49d6a --- /dev/null +++ b/test cases/common/138 include order/sub2/meson.build @@ -0,0 +1 @@ +j = include_directories('.') diff --git a/test cases/common/138 include order/sub3/main.h b/test cases/common/138 include order/sub3/main.h new file mode 100644 index 0000000..1ab7231 --- /dev/null +++ b/test cases/common/138 include order/sub3/main.h @@ -0,0 +1 @@ +#error "sub3/main.h included" diff --git a/test cases/common/138 include order/sub3/meson.build b/test cases/common/138 include order/sub3/meson.build new file mode 100644 index 0000000..0bd3906 --- /dev/null +++ b/test cases/common/138 include order/sub3/meson.build @@ -0,0 +1 @@ +sub3 = meson.current_source_dir() diff --git a/test cases/common/138 include order/sub4/main.c b/test cases/common/138 include order/sub4/main.c new file mode 100644 index 0000000..0b25eed --- /dev/null +++ b/test cases/common/138 include order/sub4/main.c @@ -0,0 +1,8 @@ +/* Use the <> include notation to force searching in include directories */ +#include + +int main(int argc, char *argv[]) { + if (somefunc() == 1984) + return 0; + return 1; +} diff --git a/test cases/common/138 include order/sub4/main.h b/test cases/common/138 include order/sub4/main.h new file mode 100644 index 0000000..194d7fe --- /dev/null +++ b/test cases/common/138 include order/sub4/main.h @@ -0,0 +1,3 @@ +#pragma once + +#include "some.h" diff --git a/test cases/common/138 include order/sub4/meson.build b/test cases/common/138 include order/sub4/meson.build new file mode 100644 index 0000000..538899a --- /dev/null +++ b/test cases/common/138 include order/sub4/meson.build @@ -0,0 +1,4 @@ +e = executable('someexe', 'main.c', + c_args : ['-I' + sub3], + include_directories : j, + dependencies : dep) -- cgit v1.1 From bc8c0730f3389d979569fc1f4c99fb9b8308a923 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 21 Jan 2017 12:42:26 +0530 Subject: Add __repr__ for IncludeDirs Helps debug issues with include directory handling. --- mesonbuild/build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mesonbuild/build.py b/mesonbuild/build.py index e5d2284..5466431 100644 --- a/mesonbuild/build.py +++ b/mesonbuild/build.py @@ -170,6 +170,10 @@ class IncludeDirs: else: self.extra_build_dirs = extra_build_dirs + def __repr__(self): + r = '<{} {}/{}>' + return r.format(self.__class__.__name__, self.curdir, self.incdirs) + def get_curdir(self): return self.curdir -- cgit v1.1 From a83c7b9ad5756330f5ce51ef150f612d3ab2b3ab Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 21 Jan 2017 12:42:51 +0530 Subject: Add get_no_stdinc_args() for C++ too --- mesonbuild/compilers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mesonbuild/compilers.py b/mesonbuild/compilers.py index afe267a..bb9b04a 100644 --- a/mesonbuild/compilers.py +++ b/mesonbuild/compilers.py @@ -1324,6 +1324,9 @@ class CPPCompiler(CCompiler): self.language = 'cpp' CCompiler.__init__(self, exelist, version, is_cross, exe_wrap) + def get_no_stdinc_args(self): + return ['-nostdinc++'] + def sanity_check(self, work_dir, environment): code = 'class breakCCompiler;int main(int argc, char **argv) { return 0; }\n' return self.sanity_check_impl(work_dir, environment, 'sanitycheckcpp.cc', code) -- cgit v1.1 From 50c4851daa79919f475e2885653eafd572bd6dfb Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 21 Jan 2017 12:05:38 +0530 Subject: Use CompilerArgs for generation of link commands Also, now the linker options are added from various sources in the same order as compiler arguments for compile commands. As a result, all libraries and library paths from external and internal sources are added after all the linker options have been added. As a result option_link_args() are added when libraries are added to the list since currently the only thing they add are the libraries specific in cpp_winlibs/c_winlibs. This fixes an issue where compilation with the MinGW toolchain (which uses static libraries for winlibs) would result in undefined symbol errors because the static libraries would be added in the very beginning and hence would not be scanned for symbols. Detailed comments have been added that explain where each option is coming from and why it's been added at that specific point. More improvements are necessary here because we currently still unnecessarily repeat libraries from dependencies over and over, which is a major problem in gst-build because inter-subproject dependencies cause linker command-lines to almost exceed the argument list length limit imposed by the kernel. It is also causing us to unnecessarily add static libraries which have already been linked into a shared library. See: self.build_target_link_arguments() --- mesonbuild/backend/backends.py | 20 ------- mesonbuild/backend/ninjabackend.py | 116 ++++++++++++++++++++++++++----------- 2 files changed, 81 insertions(+), 55 deletions(-) diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index a83d95f..4988f28 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -692,23 +692,3 @@ class Backend: for s in self.build.postconf_scripts: cmd = s['exe'] + s['args'] subprocess.check_call(cmd, env=child_env) - - # Subprojects of subprojects may cause the same dep args to be used - # multiple times. Remove duplicates here. Note that we can't dedup - # libraries based on name alone, because "-lfoo -lbar -lfoo" is - # a completely valid (though pathological) sequence and removing the - # latter may fail. Usually only applies to static libs, though. - def dedup_arguments(self, commands): - includes = {} - final_commands = [] - previous = '-fsuch_arguments=woof' - for c in commands: - if c.startswith(('-I', '-L', '/LIBPATH')): - if c in includes: - continue - includes[c] = True - if previous == c: - continue - previous = c - final_commands.append(c) - return final_commands diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 98740a4..9444087 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -2043,47 +2043,23 @@ rule FORTRAN_DEP_HACK return [] return linker.get_no_stdlib_link_args() - def generate_link(self, target, outfile, outname, obj_list, linker, extra_args=[]): - if isinstance(target, build.StaticLibrary): - linker_base = 'STATIC' - else: - linker_base = linker.get_language() # Fixme. - if isinstance(target, build.SharedLibrary): - self.generate_shsym(outfile, target) - crstr = '' - if target.is_cross: - crstr = '_CROSS' - linker_rule = linker_base + crstr + '_LINKER' + def get_target_type_link_args(self, target, linker): abspath = os.path.join(self.environment.get_build_dir(), target.subdir) commands = [] - if not isinstance(target, build.StaticLibrary): - commands += self.build.get_project_link_args(linker, target.subproject) - commands += self.build.get_global_link_args(linker) - commands += self.get_cross_stdlib_link_args(target, linker) - commands += linker.get_linker_always_args() - if not isinstance(target, build.StaticLibrary): - commands += compilers.get_base_link_args(self.environment.coredata.base_options, - linker, - isinstance(target, build.SharedModule)) - commands += linker.get_buildtype_linker_args(self.environment.coredata.get_builtin_option('buildtype')) - commands += linker.get_option_link_args(self.environment.coredata.compiler_options) - commands += self.get_link_debugfile_args(linker, target, outname) - if not(isinstance(target, build.StaticLibrary)): - commands += self.environment.coredata.external_link_args[linker.get_language()] if isinstance(target, build.Executable): + # Currently only used with the Swift compiler to add '-emit-executable' commands += linker.get_std_exe_link_args() elif isinstance(target, build.SharedLibrary): if isinstance(target, build.SharedModule): commands += linker.get_std_shared_module_link_args() else: commands += linker.get_std_shared_lib_link_args() + # All shared libraries are PIC commands += linker.get_pic_args() - if hasattr(target, 'soversion'): - soversion = target.soversion - else: - soversion = None + # Add -Wl,-soname arguments on Linux, -install_name on OS X commands += linker.get_soname_args(target.prefix, target.name, target.suffix, - abspath, soversion, isinstance(target, build.SharedModule)) + abspath, target.soversion, + isinstance(target, build.SharedModule)) # This is only visited when using the Visual Studio toolchain if target.vs_module_defs and hasattr(linker, 'gen_vs_module_defs_args'): commands += linker.gen_vs_module_defs_args(target.vs_module_defs.rel_to_builddir(self.build_to_src)) @@ -2094,17 +2070,75 @@ rule FORTRAN_DEP_HACK commands += linker.get_std_link_args() else: raise RuntimeError('Unknown build target type.') - # Link arguments of static libraries are not put in the command line of - # the library. They are instead appended to the command line where - # the static library is used. + return commands + + def generate_link(self, target, outfile, outname, obj_list, linker, extra_args=[]): + if isinstance(target, build.StaticLibrary): + linker_base = 'STATIC' + else: + linker_base = linker.get_language() # Fixme. + if isinstance(target, build.SharedLibrary): + self.generate_shsym(outfile, target) + crstr = '' + if target.is_cross: + crstr = '_CROSS' + linker_rule = linker_base + crstr + '_LINKER' + + # Create an empty commands list, and start adding link arguments from + # various sources in the order in which they must override each other + # starting from hard-coded defaults followed by build options and so on. + # + # Once all the linker options have been passed, we will start passing + # libraries and library paths from internal and external sources. + commands = CompilerArgs(linker) + # First, the trivial ones that are impossible to override. + # + # Add linker args for linking this target derived from 'base' build + # options passed on the command-line, in default_options, etc. + # These have the lowest priority. + if not isinstance(target, build.StaticLibrary): + commands += compilers.get_base_link_args(self.environment.coredata.base_options, + linker, + isinstance(target, build.SharedModule)) + # Add -nostdlib if needed; can't be overriden + commands += self.get_cross_stdlib_link_args(target, linker) + # Add things like /NOLOGO; usually can't be overriden + commands += linker.get_linker_always_args() + # Add buildtype linker args: optimization level, etc. + commands += linker.get_buildtype_linker_args(self.environment.coredata.get_builtin_option('buildtype')) + # Add /DEBUG and the pdb filename when using MSVC + commands += self.get_link_debugfile_args(linker, target, outname) + # Add link args specific to this BuildTarget type, such as soname args, + # PIC, import library generation, etc. + commands += self.get_target_type_link_args(target, linker) + if not isinstance(target, build.StaticLibrary): + # Add link args added using add_project_link_arguments() + commands += self.build.get_project_link_args(linker, target.subproject) + # Add link args added using add_global_link_arguments() + # These override per-project link arguments + commands += self.build.get_global_link_args(linker) + # Link args added from the env: LDFLAGS. We want these to + # override all the defaults but not the per-target link args. + commands += self.environment.coredata.external_link_args[linker.get_language()] + + # Now we will add libraries and library paths from various sources + + # Add link args to link to all internal libraries (link_with:) and + # internal dependencies needed by this target. if linker_base == 'STATIC': + # Link arguments of static libraries are not put in the command + # line of the library. They are instead appended to the command + # line where the static library is used. dependencies = [] else: dependencies = target.get_dependencies() commands += self.build_target_link_arguments(linker, dependencies) + # For 'automagic' deps: Boost and GTest. Also dependency('threads'). + # pkg-config puts the thread flags itself via `Cflags:` for d in target.external_deps: if d.need_threads(): commands += linker.thread_link_flags() + # Only non-static built targets need link args and link dependencies if not isinstance(target, build.StaticLibrary): commands += target.link_args # External deps must be last because target link libraries may depend on them. @@ -2114,12 +2148,24 @@ rule FORTRAN_DEP_HACK if isinstance(d, build.StaticLibrary): for dep in d.get_external_deps(): commands += dep.get_link_args() + # Add link args for c_* or cpp_* build options. Currently this only + # adds c_winlibs and cpp_winlibs when building for Windows. This needs + # to be after all internal and external libraries so that unresolved + # symbols from those can be found here. This is needed when the + # *_winlibs that we want to link to are static mingw64 libraries. + commands += linker.get_option_link_args(self.environment.coredata.compiler_options) + # Set runtime-paths so we can run executables without needing to set + # LD_LIBRARY_PATH, etc in the environment. Doesn't work on Windows. commands += linker.build_rpath_args(self.environment.get_build_dir(), - self.determine_rpath_dirs(target), target.install_rpath) + self.determine_rpath_dirs(target), + target.install_rpath) + # Add libraries generated by custom targets custom_target_libraries = self.get_custom_target_provided_libraries(target) commands += extra_args commands += custom_target_libraries - commands = linker.unix_args_to_native(self.dedup_arguments(commands)) + # Convert from GCC-style link argument naming to the naming used by the + # current compiler. + commands = commands.to_native() dep_targets = [self.get_dependency_filename(t) for t in dependencies] dep_targets += [os.path.join(self.environment.source_dir, target.subdir, t) for t in target.link_depends] -- cgit v1.1 From 991ace04afdd2eb010f8c94c00de7c3e1f197f60 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 21 Jan 2017 13:01:05 +0530 Subject: Disable E266 in flake8 Don't tell me how to format my comments! --- setup.cfg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 9374560..823c84d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,5 +13,7 @@ ignore = # E305: expected 2 blank lines after class or function definition, found 1 E305, # E401: multiple imports on one line - E401 + E401, + # too many leading '#' for block comment + E266 max-line-length = 120 -- cgit v1.1 From 57ec097b5fb08081e458b2b4946ee5943ba8c032 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 28 Jan 2017 00:19:48 +0530 Subject: vs: Use CompilerArgs() for compile and link args At the same time also fix the order in which they are added. They now match the order used in the Ninja backend. --- mesonbuild/backend/vs2010backend.py | 178 ++++++++++++++++++++++++------------ setup.cfg | 4 +- 2 files changed, 120 insertions(+), 62 deletions(-) diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py index 31a4c0e..f0d125d 100644 --- a/mesonbuild/backend/vs2010backend.py +++ b/mesonbuild/backend/vs2010backend.py @@ -23,6 +23,7 @@ from .. import dependencies from .. import mlog from .. import compilers from ..build import BuildTarget +from ..compilers import CompilerArgs from ..mesonlib import MesonException, File from ..environment import Environment @@ -426,24 +427,25 @@ class Vs2010Backend(backends.Backend): pch_out.text = '$(IntDir)$(TargetName)-%s.pch' % lang def add_additional_options(self, lang, parent_node, file_args): - if len(file_args[lang]) == 0: - # We only need per file options if they were not set per project. - return - args = file_args[lang] + ['%(AdditionalOptions)'] + args = [] + for arg in file_args[lang].to_native(): + if arg == '%(AdditionalOptions)': + args.append(arg) + else: + args.append(self.escape_additional_option(arg)) ET.SubElement(parent_node, "AdditionalOptions").text = ' '.join(args) def add_preprocessor_defines(self, lang, parent_node, file_defines): - if len(file_defines[lang]) == 0: - # We only need per file options if they were not set per project. - return - defines = file_defines[lang] + ['%(PreprocessorDefinitions)'] + defines = [] + for define in file_defines[lang]: + if define == '%(PreprocessorDefinitions)': + defines.append(define) + else: + defines.append(self.escape_preprocessor_define(define)) ET.SubElement(parent_node, "PreprocessorDefinitions").text = ';'.join(defines) def add_include_dirs(self, lang, parent_node, file_inc_dirs): - if len(file_inc_dirs[lang]) == 0: - # We only need per file options if they were not set per project. - return - dirs = file_inc_dirs[lang] + ['%(AdditionalIncludeDirectories)'] + dirs = file_inc_dirs[lang] ET.SubElement(parent_node, "AdditionalIncludeDirectories").text = ';'.join(dirs) @staticmethod @@ -668,29 +670,98 @@ class Vs2010Backend(backends.Backend): # Arguments, include dirs, defines for all files in the current target target_args = [] target_defines = [] - target_inc_dirs = ['.', self.relpath(self.get_target_private_dir(target), - self.get_target_dir(target)), - proj_to_src_dir] + generated_files_include_dirs + target_inc_dirs = [] # Arguments, include dirs, defines passed to individual files in # a target; perhaps because the args are language-specific - file_args = dict((lang, []) for lang in target.compilers) + # + # file_args is also later split out into defines and include_dirs in + # case someone passed those in there + file_args = dict((lang, CompilerArgs(comp)) for lang, comp in target.compilers.items()) file_defines = dict((lang, []) for lang in target.compilers) file_inc_dirs = dict((lang, []) for lang in target.compilers) - for l, args in self.environment.coredata.external_args.items(): + # The order in which these compile args are added must match + # generate_single_compile() and generate_basic_compiler_args() + for l, comp in target.compilers.items(): + if l in file_args: + file_args[l] += compilers.get_base_compile_args(self.environment.coredata.base_options, comp) + file_args[l] += comp.get_option_compile_args(self.environment.coredata.compiler_options) + # Add compile args added using add_project_arguments() + for l, args in self.build.projects_args.get(target.subproject, {}).items(): if l in file_args: file_args[l] += args + # Add compile args added using add_global_arguments() + # These override per-project arguments for l, args in self.build.global_args.items(): if l in file_args: file_args[l] += args - for l, args in self.build.projects_args.get(target.subproject, {}).items(): + # Compile args added from the env: CFLAGS/CXXFLAGS, etc. We want these + # to override all the defaults, but not the per-target compile args. + for l, args in self.environment.coredata.external_args.items(): if l in file_args: file_args[l] += args + for args in file_args.values(): + # This is where Visual Studio will insert target_args, target_defines, + # etc, which are added later from external deps (see below). + args += ['%(AdditionalOptions)', '%(PreprocessorDefinitions)', '%(AdditionalIncludeDirectories)'] + # Add include dirs from the `include_directories:` kwarg on the target + # and from `include_directories:` of internal deps of the target. + # + # Target include dirs should override internal deps include dirs. + # + # Include dirs from internal deps should override include dirs from + # external deps. + # These are per-target, but we still add them as per-file because we + # need them to be looked in first. + for d in target.get_include_dirs(): + for i in d.get_incdirs(): + curdir = os.path.join(d.get_curdir(), i) + args.append('-I' + self.relpath(curdir, target.subdir)) # build dir + args.append('-I' + os.path.join(proj_to_src_root, curdir)) # src dir + for i in d.get_extra_build_dirs(): + curdir = os.path.join(d.get_curdir(), i) + args.append('-I' + self.relpath(curdir, target.subdir)) # build dir + # Add per-target compile args, f.ex, `c_args : ['/DFOO']`. We set these + # near the end since these are supposed to override everything else. for l, args in target.extra_args.items(): if l in file_args: - file_args[l] += compiler.unix_args_to_native(args) - for l, comp in target.compilers.items(): - if l in file_args: - file_args[l] += comp.get_option_compile_args(self.environment.coredata.compiler_options) + file_args[l] += args + # The highest priority includes. In order of directory search: + # target private dir, target build dir, generated sources include dirs, + # target source dir + for args in file_args.values(): + t_inc_dirs = ['.', self.relpath(self.get_target_private_dir(target), + self.get_target_dir(target))] + t_inc_dirs += generated_files_include_dirs + [proj_to_src_dir] + args += ['-I' + arg for arg in t_inc_dirs] + + # Split preprocessor defines and include directories out of the list of + # all extra arguments. The rest go into %(AdditionalOptions). + for l, args in file_args.items(): + for arg in args[:]: + if arg.startswith(('-D', '/D')) or arg == '%(PreprocessorDefinitions)': + file_args[l].remove(arg) + # Don't escape the marker + if arg == '%(PreprocessorDefinitions)': + define = arg + else: + define = arg[2:] + # De-dup + if define in file_defines[l]: + file_defines[l].remove(define) + file_defines[l].append(define) + elif arg.startswith(('-I', '/I')) or arg == '%(AdditionalIncludeDirectories)': + file_args[l].remove(arg) + # Don't escape the marker + if arg == '%(AdditionalIncludeDirectories)': + inc_dir = arg + else: + inc_dir = arg[2:] + # De-dup + if inc_dir not in file_inc_dirs[l]: + file_inc_dirs[l].append(inc_dir) + + # Split compile args needed to find external dependencies + # Link args are added while generating the link command for d in target.get_external_deps(): # Cflags required by external deps might have UNIX-specific flags, # so filter them out if needed @@ -699,50 +770,22 @@ class Vs2010Backend(backends.Backend): if arg.startswith(('-D', '/D')): define = arg[2:] # De-dup - if define not in target_defines: - target_defines.append(define) + if define in target_defines: + target_defines.remove(define) + target_defines.append(define) elif arg.startswith(('-I', '/I')): inc_dir = arg[2:] # De-dup if inc_dir not in target_inc_dirs: target_inc_dirs.append(inc_dir) else: - # De-dup - if arg not in target_args: - target_args.append(arg) - - # Split preprocessor defines and include directories out of the list of - # all extra arguments. The rest go into %(AdditionalOptions). - for l, args in file_args.items(): - file_args[l] = [] - for arg in args: - if arg.startswith(('-D', '/D')): - define = self.escape_preprocessor_define(arg[2:]) - # De-dup - if define not in file_defines[l]: - file_defines[l].append(define) - elif arg.startswith(('-I', '/I')): - inc_dir = arg[2:] - # De-dup - if inc_dir not in file_inc_dirs[l]: - file_inc_dirs[l].append(inc_dir) - else: - file_args[l].append(self.escape_additional_option(arg)) + target_args.append(arg) languages += gen_langs if len(target_args) > 0: target_args.append('%(AdditionalOptions)') ET.SubElement(clconf, "AdditionalOptions").text = ' '.join(target_args) - for d in target.include_dirs: - for i in d.incdirs: - curdir = os.path.join(d.curdir, i) - target_inc_dirs.append(self.relpath(curdir, target.subdir)) # build dir - target_inc_dirs.append(os.path.join(proj_to_src_root, curdir)) # src dir - for i in d.get_extra_build_dirs(): - curdir = os.path.join(d.curdir, i) - target_inc_dirs.append(self.relpath(curdir, target.subdir)) # build dir - target_inc_dirs.append('%(AdditionalIncludeDirectories)') ET.SubElement(clconf, 'AdditionalIncludeDirectories').text = ';'.join(target_inc_dirs) target_defines.append('%(PreprocessorDefinitions)') @@ -773,19 +816,27 @@ class Vs2010Backend(backends.Backend): resourcecompile = ET.SubElement(compiles, 'ResourceCompile') ET.SubElement(resourcecompile, 'PreprocessorDefinitions') + + # Linker options link = ET.SubElement(compiles, 'Link') - # Put all language args here, too. - extra_link_args = compiler.get_option_link_args(self.environment.coredata.compiler_options) + extra_link_args = CompilerArgs(compiler) # FIXME: Can these buildtype linker args be added as tags in the # vcxproj file (similar to buildtype compiler args) instead of in # AdditionalOptions? extra_link_args += compiler.get_buildtype_linker_args(self.buildtype) - for l in self.environment.coredata.external_link_args.values(): - extra_link_args += l if not isinstance(target, build.StaticLibrary): - extra_link_args += target.link_args if isinstance(target, build.SharedModule): extra_link_args += compiler.get_std_shared_module_link_args() + # Add link args added using add_project_link_arguments() + extra_link_args += self.build.get_project_link_args(compiler, target.subproject) + # Add link args added using add_global_link_arguments() + # These override per-project link arguments + extra_link_args += self.build.get_global_link_args(compiler) + # Link args added from the env: LDFLAGS. We want these to + # override all the defaults but not the per-target link args. + extra_link_args += self.environment.coredata.external_link_args[compiler.get_language()] + # Only non-static built targets need link args and link dependencies + extra_link_args += target.link_args # External deps must be last because target link libraries may depend on them. for dep in target.get_external_deps(): extra_link_args += dep.get_link_args() @@ -793,8 +844,13 @@ class Vs2010Backend(backends.Backend): if isinstance(d, build.StaticLibrary): for dep in d.get_external_deps(): extra_link_args += dep.get_link_args() - extra_link_args = compiler.unix_args_to_native(extra_link_args) - (additional_libpaths, additional_links, extra_link_args) = self.split_link_args(extra_link_args) + # Add link args for c_* or cpp_* build options. Currently this only + # adds c_winlibs and cpp_winlibs when building for Windows. This needs + # to be after all internal and external libraries so that unresolved + # symbols from those can be found here. This is needed when the + # *_winlibs that we want to link to are static mingw64 libraries. + extra_link_args += compiler.get_option_link_args(self.environment.coredata.compiler_options) + (additional_libpaths, additional_links, extra_link_args) = self.split_link_args(extra_link_args.to_native()) if len(extra_link_args) > 0: extra_link_args.append('%(AdditionalOptions)') ET.SubElement(link, "AdditionalOptions").text = ' '.join(extra_link_args) diff --git a/setup.cfg b/setup.cfg index 823c84d..b3adc59 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,5 +15,7 @@ ignore = # E401: multiple imports on one line E401, # too many leading '#' for block comment - E266 + E266, + # module level import not at top of file + E402 max-line-length = 120 -- cgit v1.1 From 471904f0c50e9646d644efaf27c0ff4d5fce6634 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 28 Jan 2017 04:45:16 +0530 Subject: vs: Add support for the 'werror' builtin option --- mesonbuild/backend/vs2010backend.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py index f0d125d..3b79a9c 100644 --- a/mesonbuild/backend/vs2010backend.py +++ b/mesonbuild/backend/vs2010backend.py @@ -790,11 +790,12 @@ class Vs2010Backend(backends.Backend): ET.SubElement(clconf, 'AdditionalIncludeDirectories').text = ';'.join(target_inc_dirs) target_defines.append('%(PreprocessorDefinitions)') ET.SubElement(clconf, 'PreprocessorDefinitions').text = ';'.join(target_defines) - rebuild = ET.SubElement(clconf, 'MinimalRebuild') - rebuild.text = 'true' - funclink = ET.SubElement(clconf, 'FunctionLevelLinking') - funclink.text = 'true' + ET.SubElement(clconf, 'MinimalRebuild').text = 'true' + ET.SubElement(clconf, 'FunctionLevelLinking').text = 'true' pch_node = ET.SubElement(clconf, 'PrecompiledHeader') + if self.environment.coredata.get_builtin_option('werror'): + ET.SubElement(clconf, 'TreatWarningAsError').text = 'true' + # Note: SuppressStartupBanner is /NOLOGO and is 'true' by default pch_sources = {} for lang in ['c', 'cpp']: pch = target.get_pch(lang) -- cgit v1.1