aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDylan Baker <dylan@pnwbakers.com>2020-06-11 12:06:29 -0700
committerDylan Baker <dylan@pnwbakers.com>2020-06-22 12:06:10 -0700
commit9d0ad66c29fccd2ff72c2b40da02cdb2b03ccba6 (patch)
treee19e88210892ada6d44282f9ce5a37fa33957a82
parent386721f7fd3261ba15004a7f527e9db83e216ace (diff)
downloadmeson-9d0ad66c29fccd2ff72c2b40da02cdb2b03ccba6.zip
meson-9d0ad66c29fccd2ff72c2b40da02cdb2b03ccba6.tar.gz
meson-9d0ad66c29fccd2ff72c2b40da02cdb2b03ccba6.tar.bz2
compilers: Split CompilerArgs into a separate module
I've also moved this out of the compilers pacakge because we're soon going to need it in linkers, and that creates some serious spagetti
-rw-r--r--mesonbuild/arglist.py400
-rw-r--r--mesonbuild/backend/backends.py2
-rw-r--r--mesonbuild/backend/ninjabackend.py3
-rw-r--r--mesonbuild/backend/vs2010backend.py2
-rw-r--r--mesonbuild/compilers/__init__.py2
-rw-r--r--mesonbuild/compilers/compilers.py372
-rw-r--r--mesonbuild/compilers/d.py2
-rw-r--r--mesonbuild/compilers/mixins/clike.py7
-rwxr-xr-xrun_unittests.py10
9 files changed, 416 insertions, 384 deletions
diff --git a/mesonbuild/arglist.py b/mesonbuild/arglist.py
new file mode 100644
index 0000000..ac97a41
--- /dev/null
+++ b/mesonbuild/arglist.py
@@ -0,0 +1,400 @@
+# Copyright 2012-2020 The Meson development team
+# Copyright © 2020 Intel Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import lru_cache
+import collections
+import enum
+import os
+import re
+import typing as T
+
+from . import mesonlib
+from .linkers import (
+ GnuLikeDynamicLinkerMixin, LinkerEnvVarsMixin, SolarisDynamicLinker,
+)
+
+if T.TYPE_CHECKING:
+ from .linkers import StaticLinker
+ from .compilers import Compiler
+
+
+UNIXY_COMPILER_INTERNAL_LIBS = ['m', 'c', 'pthread', 'dl', 'rt'] # type: T.List[str]
+# execinfo is a compiler lib on FreeBSD and NetBSD
+if mesonlib.is_freebsd() or mesonlib.is_netbsd():
+ UNIXY_COMPILER_INTERNAL_LIBS.append('execinfo')
+SOREGEX = re.compile(r'.*\.so(\.[0-9]+)?(\.[0-9]+)?(\.[0-9]+)?$')
+
+class Dedup(enum.Enum):
+
+ """What kind of deduplication can be done to compiler args.
+
+ OVERRIDEN - Whether an argument can be 'overridden' by a later argument.
+ For example, -DFOO defines FOO and -UFOO undefines FOO. In this case,
+ we can safely remove the previous occurrence and add a new one. The
+ same is true for include paths and library paths with -I and -L.
+ UNIQUE - Arguments that once specified cannot be undone, such as `-c` or
+ `-pipe`. New instances of these can be completely skipped.
+ NO_DEDUP - Whether it matters where or how many times on the command-line
+ a particular argument is present. This can matter for symbol
+ resolution in static or shared libraries, so we cannot de-dup or
+ reorder them.
+ """
+
+ NO_DEDUP = 0
+ UNIQUE = 1
+ OVERRIDEN = 2
+
+
+class CompilerArgs(collections.abc.MutableSequence):
+ '''
+ List-like class that manages a list of compiler arguments. Should be used
+ while constructing compiler arguments from various sources. Can be
+ operated with ordinary lists, so this does not need to be used
+ everywhere.
+
+ All arguments must be inserted and stored in GCC-style (-lfoo, -Idir, etc)
+ and can converted to the native type of each compiler by using the
+ .to_native() method to which you must pass an instance of the compiler or
+ the compiler class.
+
+ New arguments added to this class (either with .append(), .extend(), or +=)
+ are added in a way that ensures that they override previous arguments.
+ For example:
+
+ >>> a = ['-Lfoo', '-lbar']
+ >>> a += ['-Lpho', '-lbaz']
+ >>> print(a)
+ ['-Lpho', '-Lfoo', '-lbar', '-lbaz']
+
+ Arguments will also be de-duped if they can be de-duped safely.
+
+ Note that because of all this, this class is not commutative and does not
+ preserve the order of arguments if it is safe to not. For example:
+ >>> ['-Ifoo', '-Ibar'] + ['-Ifez', '-Ibaz', '-Werror']
+ ['-Ifez', '-Ibaz', '-Ifoo', '-Ibar', '-Werror']
+ >>> ['-Ifez', '-Ibaz', '-Werror'] + ['-Ifoo', '-Ibar']
+ ['-Ifoo', '-Ibar', '-Ifez', '-Ibaz', '-Werror']
+
+ '''
+ # NOTE: currently this class is only for C-like compilers, but it can be
+ # extended to other languages easily. Just move the following to the
+ # compiler class and initialize when self.compiler is set.
+
+ # Arg prefixes that override by prepending instead of appending
+ prepend_prefixes = ('-I', '-L')
+ # Arg prefixes and args that must be de-duped by returning 2
+ dedup2_prefixes = ('-I', '-isystem', '-L', '-D', '-U')
+ dedup2_suffixes = ()
+ dedup2_args = ()
+ # Arg prefixes and args that must be de-duped by returning 1
+ #
+ # NOTE: not thorough. A list of potential corner cases can be found in
+ # https://github.com/mesonbuild/meson/pull/4593#pullrequestreview-182016038
+ dedup1_prefixes = ('-l', '-Wl,-l', '-Wl,--export-dynamic')
+ dedup1_suffixes = ('.lib', '.dll', '.so', '.dylib', '.a')
+ # Match a .so of the form path/to/libfoo.so.0.1.0
+ # Only UNIX shared libraries require this. Others have a fixed extension.
+ dedup1_regex = re.compile(r'([\/\\]|\A)lib.*\.so(\.[0-9]+)?(\.[0-9]+)?(\.[0-9]+)?$')
+ dedup1_args = ('-c', '-S', '-E', '-pipe', '-pthread')
+ # In generate_link() we add external libs without de-dup, but we must
+ # *always* de-dup these because they're special arguments to the linker
+ always_dedup_args = tuple('-l' + lib for lib in UNIXY_COMPILER_INTERNAL_LIBS)
+
+ def __init__(self, compiler: T.Union['Compiler', 'StaticLinker'],
+ iterable: T.Optional[T.Iterable[str]] = None):
+ self.compiler = compiler
+ self.__container = list(iterable) if iterable is not None else [] # type: T.List[str]
+ self.pre = collections.deque() # type: T.Deque[str]
+ self.post = collections.deque() # type: T.Deque[str]
+
+ # Flush the saved pre and post list into the __container list
+ #
+ # This correctly deduplicates the entries after _can_dedup definition
+ # Note: This function is designed to work without delete operations, as deletions are worsening the performance a lot.
+ def flush_pre_post(self) -> None:
+ pre_flush = collections.deque() # type: T.Deque[str]
+ pre_flush_set = set() # type: T.Set[str]
+ post_flush = collections.deque() # type: T.Deque[str]
+ post_flush_set = set() # type: T.Set[str]
+
+ #The two lists are here walked from the front to the back, in order to not need removals for deduplication
+ for a in self.pre:
+ dedup = self._can_dedup(a)
+ if a not in pre_flush_set:
+ pre_flush.append(a)
+ if dedup is Dedup.OVERRIDEN:
+ pre_flush_set.add(a)
+ for a in reversed(self.post):
+ dedup = self._can_dedup(a)
+ if a not in post_flush_set:
+ post_flush.appendleft(a)
+ if dedup is Dedup.OVERRIDEN:
+ post_flush_set.add(a)
+
+ #pre and post will overwrite every element that is in the container
+ #only copy over args that are in __container but not in the post flush or pre flush set
+
+ for a in self.__container:
+ if a not in post_flush_set and a not in pre_flush_set:
+ pre_flush.append(a)
+
+ self.__container = list(pre_flush) + list(post_flush)
+ self.pre.clear()
+ self.post.clear()
+
+ def __iter__(self) -> T.Iterator[str]:
+ self.flush_pre_post()
+ return iter(self.__container)
+
+ @T.overload # noqa: F811
+ def __getitem__(self, index: int) -> str: # noqa: F811
+ pass
+
+ @T.overload # noqa: F811
+ def __getitem__(self, index: slice) -> T.List[str]: # noqa: F811
+ pass
+
+ def __getitem__(self, index): # noqa: F811
+ self.flush_pre_post()
+ return self.__container[index]
+
+ @T.overload # noqa: F811
+ def __setitem__(self, index: int, value: str) -> None: # noqa: F811
+ pass
+
+ @T.overload # noqa: F811
+ def __setitem__(self, index: slice, value: T.List[str]) -> None: # noqa: F811
+ pass
+
+ def __setitem__(self, index, value) -> None: # noqa: F811
+ self.flush_pre_post()
+ self.__container[index] = value
+
+ def __delitem__(self, index: T.Union[int, slice]) -> None:
+ self.flush_pre_post()
+ del self.__container[index]
+
+ def __len__(self) -> int:
+ return len(self.__container) + len(self.pre) + len(self.post)
+
+ def insert(self, index: int, value: str) -> None:
+ self.flush_pre_post()
+ self.__container.insert(index, value)
+
+ def copy(self) -> 'CompilerArgs':
+ self.flush_pre_post()
+ return CompilerArgs(self.compiler, self.__container.copy())
+
+ @classmethod
+ @lru_cache(maxsize=None)
+ def _can_dedup(cls, arg: str) -> Dedup:
+ """Returns whether the argument can be safely de-duped.
+
+ In addition to these, we handle library arguments specially.
+ With GNU ld, we surround library arguments with -Wl,--start/end-gr -> Dedupoup
+ to recursively search for symbols in the libraries. This is not needed
+ with other linkers.
+ """
+
+ # A standalone argument must never be deduplicated because it is
+ # defined by what comes _after_ it. Thus dedupping this:
+ # -D FOO -D BAR
+ # would yield either
+ # -D FOO BAR
+ # or
+ # FOO -D BAR
+ # both of which are invalid.
+ if arg in cls.dedup2_prefixes:
+ return Dedup.NO_DEDUP
+ if arg.startswith('-L='):
+ # DMD and LDC proxy all linker arguments using -L=; in conjunction
+ # with ld64 on macOS this can lead to command line arguments such
+ # as: `-L=-compatibility_version -L=0 -L=current_version -L=0`.
+ # These cannot be combined, ld64 insists they must be passed with
+ # spaces and quoting does not work. if we deduplicate these then
+ # one of the -L=0 arguments will be removed and the version
+ # argument will consume the next argument instead.
+ return Dedup.NO_DEDUP
+ if arg in cls.dedup2_args or \
+ arg.startswith(cls.dedup2_prefixes) or \
+ arg.endswith(cls.dedup2_suffixes):
+ return Dedup.OVERRIDEN
+ if arg in cls.dedup1_args or \
+ arg.startswith(cls.dedup1_prefixes) or \
+ arg.endswith(cls.dedup1_suffixes) or \
+ re.search(cls.dedup1_regex, arg):
+ return Dedup.UNIQUE
+ return Dedup.NO_DEDUP
+
+ @classmethod
+ @lru_cache(maxsize=None)
+ def _should_prepend(cls, arg: str) -> bool:
+ return arg.startswith(cls.prepend_prefixes)
+
+ def need_to_split_linker_args(self) -> bool:
+ # XXX: gross
+ from .compilers import Compiler
+ return isinstance(self.compiler, Compiler) and self.compiler.get_language() == 'd'
+
+ def to_native(self, copy: bool = False) -> T.List[str]:
+ # XXX: gross
+ from .compilers import Compiler
+
+ # Check if we need to add --start/end-group for circular dependencies
+ # between static libraries, and for recursively searching for symbols
+ # needed by static libraries that are provided by object files or
+ # shared libraries.
+ self.flush_pre_post()
+ if copy:
+ new = self.copy()
+ else:
+ new = self
+ # To proxy these arguments with D you need to split the
+ # arguments, thus you get `-L=-soname -L=lib.so` we don't
+ # want to put the lib in a link -roup
+ split_linker_args = self.need_to_split_linker_args()
+ # This covers all ld.bfd, ld.gold, ld.gold, and xild on Linux, which
+ # all act like (or are) gnu ld
+ # TODO: this could probably be added to the DynamicLinker instead
+ if (isinstance(self.compiler, Compiler) and
+ self.compiler.linker is not None and
+ isinstance(self.compiler.linker, (GnuLikeDynamicLinkerMixin, SolarisDynamicLinker))):
+ group_start = -1
+ group_end = -1
+ is_soname = False
+ for i, each in enumerate(new):
+ if is_soname:
+ is_soname = False
+ continue
+ elif split_linker_args and '-soname' in each:
+ is_soname = True
+ continue
+ if not each.startswith(('-Wl,-l', '-l')) and not each.endswith('.a') and \
+ not SOREGEX.match(each):
+ continue
+ group_end = i
+ if group_start < 0:
+ # First occurrence of a library
+ group_start = i
+ if group_start >= 0:
+ # Last occurrence of a library
+ new.insert(group_end + 1, '-Wl,--end-group')
+ new.insert(group_start, '-Wl,--start-group')
+ # Remove system/default include paths added with -isystem
+ if hasattr(self.compiler, 'get_default_include_dirs'):
+ default_dirs = self.compiler.get_default_include_dirs()
+ bad_idx_list = [] # type: T.List[int]
+ for i, each in enumerate(new):
+ # Remove the -isystem and the path if the path is a default path
+ if (each == '-isystem' and
+ i < (len(new) - 1) and
+ new[i + 1] in default_dirs):
+ bad_idx_list += [i, i + 1]
+ elif each.startswith('-isystem=') and each[9:] in default_dirs:
+ bad_idx_list += [i]
+ elif each.startswith('-isystem') and each[8:] in default_dirs:
+ bad_idx_list += [i]
+ for i in reversed(bad_idx_list):
+ new.pop(i)
+ return self.compiler.unix_args_to_native(new.__container)
+
+ def append_direct(self, arg: str) -> None:
+ '''
+ Append the specified argument without any reordering or de-dup except
+ for absolute paths to libraries, etc, which can always be de-duped
+ safely.
+ '''
+ self.flush_pre_post()
+ if os.path.isabs(arg):
+ self.append(arg)
+ else:
+ self.__container.append(arg)
+
+ def extend_direct(self, iterable: T.Iterable[str]) -> None:
+ '''
+ Extend using the elements in the specified iterable without any
+ reordering or de-dup except for absolute paths where the order of
+ include search directories is not relevant
+ '''
+ self.flush_pre_post()
+ for elem in iterable:
+ self.append_direct(elem)
+
+ def extend_preserving_lflags(self, iterable: T.Iterable[str]) -> None:
+ normal_flags = []
+ lflags = []
+ for i in iterable:
+ if i not in self.always_dedup_args and (i.startswith('-l') or i.startswith('-L')):
+ lflags.append(i)
+ else:
+ normal_flags.append(i)
+ self.extend(normal_flags)
+ self.extend_direct(lflags)
+
+ def __add__(self, args: T.Iterable[str]) -> 'CompilerArgs':
+ self.flush_pre_post()
+ new = self.copy()
+ new += args
+ return new
+
+ def __iadd__(self, args: T.Iterable[str]) -> 'CompilerArgs':
+ '''
+ Add two CompilerArgs while taking into account overriding of arguments
+ and while preserving the order of arguments as much as possible
+ '''
+ tmp_pre = collections.deque() # type: T.Deque[str]
+ if not isinstance(args, collections.abc.Iterable):
+ raise TypeError('can only concatenate Iterable[str] (not "{}") to CompilerArgs'.format(args))
+ for arg in args:
+ # If the argument can be de-duped, do it either by removing the
+ # previous occurrence of it and adding a new one, or not adding the
+ # new occurrence.
+ dedup = self._can_dedup(arg)
+ if dedup is Dedup.UNIQUE:
+ # Argument already exists and adding a new instance is useless
+ if arg in self.__container or arg in self.pre or arg in self.post:
+ continue
+ if self._should_prepend(arg):
+ tmp_pre.appendleft(arg)
+ else:
+ self.post.append(arg)
+ self.pre.extendleft(tmp_pre)
+ #pre and post is going to be merged later before a iter call
+ return self
+
+ def __radd__(self, args: T.Iterable[str]) -> 'CompilerArgs':
+ self.flush_pre_post()
+ new = CompilerArgs(self.compiler, args)
+ new += self
+ return new
+
+ def __eq__(self, other: T.Any) -> T.Union[bool, type(NotImplemented)]:
+ self.flush_pre_post()
+ # Only allow equality checks against other CompilerArgs and lists instances
+ if isinstance(other, CompilerArgs):
+ return self.compiler == other.compiler and self.__container == other.__container
+ elif isinstance(other, list):
+ return self.__container == other
+ return NotImplemented
+
+ def append(self, arg: str) -> None:
+ self.__iadd__([arg])
+
+ def extend(self, args: T.Iterable[str]) -> None:
+ self.__iadd__(args)
+
+ def __repr__(self) -> str:
+ self.flush_pre_post()
+ return 'CompilerArgs({!r}, {!r})'.format(self.compiler, self.__container)
diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py
index 68ae1a7..bca0304 100644
--- a/mesonbuild/backend/backends.py
+++ b/mesonbuild/backend/backends.py
@@ -28,7 +28,7 @@ from .. import build
from .. import dependencies
from .. import mesonlib
from .. import mlog
-from ..compilers import CompilerArgs
+from ..arglist import CompilerArgs
from ..mesonlib import (
File, MachineChoice, MesonException, OrderedSet, OptionOverrideProxy,
classify_unity_sources, unholder
diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index 252f646..b326e3b 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -30,8 +30,9 @@ from .. import build
from .. import mlog
from .. import dependencies
from .. import compilers
+from ..arglist import CompilerArgs
from ..compilers import (
- Compiler, CompilerArgs, CCompiler,
+ Compiler, CCompiler,
DmdDCompiler,
FortranCompiler, PGICCompiler,
VisualStudioCsCompiler,
diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py
index 87514c6..bd77132 100644
--- a/mesonbuild/backend/vs2010backend.py
+++ b/mesonbuild/backend/vs2010backend.py
@@ -26,7 +26,7 @@ from .. import build
from .. import dependencies
from .. import mlog
from .. import compilers
-from ..compilers import CompilerArgs
+from ..arglist import CompilerArgs
from ..interpreter import Interpreter
from ..mesonlib import (
MesonException, File, python_command, replace_if_different
diff --git a/mesonbuild/compilers/__init__.py b/mesonbuild/compilers/__init__.py
index af7e519..fd47545 100644
--- a/mesonbuild/compilers/__init__.py
+++ b/mesonbuild/compilers/__init__.py
@@ -48,7 +48,6 @@ __all__ = [
'ClangObjCPPCompiler',
'ClangClCCompiler',
'ClangClCPPCompiler',
- 'CompilerArgs',
'CPPCompiler',
'DCompiler',
'DmdDCompiler',
@@ -123,7 +122,6 @@ from .compilers import (
is_known_suffix,
lang_suffixes,
sort_clink,
- CompilerArgs,
)
from .c import (
CCompiler,
diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py
index c34071c..bf66982 100644
--- a/mesonbuild/compilers/compilers.py
+++ b/mesonbuild/compilers/compilers.py
@@ -13,20 +13,14 @@
# limitations under the License.
import contextlib, os.path, re, tempfile
-import collections.abc
-from collections import deque
import itertools
import typing as T
from functools import lru_cache
-import enum
-from ..linkers import (
- GnuLikeDynamicLinkerMixin, LinkerEnvVarsMixin, SolarisDynamicLinker,
- StaticLinker,
-)
from .. import coredata
from .. import mlog
from .. import mesonlib
+from ..linkers import LinkerEnvVarsMixin
from ..mesonlib import (
EnvironmentException, MachineChoice, MesonException,
Popen_safe, split_args
@@ -34,6 +28,7 @@ from ..mesonlib import (
from ..envconfig import (
Properties, get_env_var
)
+from ..arglist import CompilerArgs
if T.TYPE_CHECKING:
from ..coredata import OptionDictType
@@ -100,11 +95,6 @@ cflags_mapping = {'c': 'CFLAGS',
'vala': 'VALAFLAGS',
'rust': 'RUSTFLAGS'}
-unixy_compiler_internal_libs = ('m', 'c', 'pthread', 'dl', 'rt')
-# execinfo is a compiler lib on FreeBSD and NetBSD
-if mesonlib.is_freebsd() or mesonlib.is_netbsd():
- unixy_compiler_internal_libs += ('execinfo',)
-
# All these are only for C-linkable languages; see `clink_langs` above.
def sort_clink(lang):
@@ -411,364 +401,6 @@ class RunResult:
self.stderr = stderr
-class Dedup(enum.Enum):
-
- """What kind of deduplication can be done to compiler args.
-
- OVERRIDEN - Whether an argument can be 'overridden' by a later argument.
- For example, -DFOO defines FOO and -UFOO undefines FOO. In this case,
- we can safely remove the previous occurrence and add a new one. The
- same is true for include paths and library paths with -I and -L.
- UNIQUE - Arguments that once specified cannot be undone, such as `-c` or
- `-pipe`. New instances of these can be completely skipped.
- NO_DEDUP - Whether it matters where or how many times on the command-line
- a particular argument is present. This can matter for symbol
- resolution in static or shared libraries, so we cannot de-dup or
- reorder them.
- """
-
- NO_DEDUP = 0
- UNIQUE = 1
- OVERRIDEN = 2
-
-
-class CompilerArgs(collections.abc.MutableSequence):
- '''
- List-like class that manages a list of compiler arguments. Should be used
- while constructing compiler arguments from various sources. Can be
- operated with ordinary lists, so this does not need to be used
- everywhere.
-
- All arguments must be inserted and stored in GCC-style (-lfoo, -Idir, etc)
- and can converted to the native type of each compiler by using the
- .to_native() method to which you must pass an instance of the compiler or
- the compiler class.
-
- New arguments added to this class (either with .append(), .extend(), or +=)
- are added in a way that ensures that they override previous arguments.
- For example:
-
- >>> a = ['-Lfoo', '-lbar']
- >>> a += ['-Lpho', '-lbaz']
- >>> print(a)
- ['-Lpho', '-Lfoo', '-lbar', '-lbaz']
-
- Arguments will also be de-duped if they can be de-duped safely.
-
- Note that because of all this, this class is not commutative and does not
- preserve the order of arguments if it is safe to not. For example:
- >>> ['-Ifoo', '-Ibar'] + ['-Ifez', '-Ibaz', '-Werror']
- ['-Ifez', '-Ibaz', '-Ifoo', '-Ibar', '-Werror']
- >>> ['-Ifez', '-Ibaz', '-Werror'] + ['-Ifoo', '-Ibar']
- ['-Ifoo', '-Ibar', '-Ifez', '-Ibaz', '-Werror']
-
- '''
- # NOTE: currently this class is only for C-like compilers, but it can be
- # extended to other languages easily. Just move the following to the
- # compiler class and initialize when self.compiler is set.
-
- # Arg prefixes that override by prepending instead of appending
- prepend_prefixes = ('-I', '-L')
- # Arg prefixes and args that must be de-duped by returning 2
- dedup2_prefixes = ('-I', '-isystem', '-L', '-D', '-U')
- dedup2_suffixes = ()
- dedup2_args = ()
- # Arg prefixes and args that must be de-duped by returning 1
- #
- # NOTE: not thorough. A list of potential corner cases can be found in
- # https://github.com/mesonbuild/meson/pull/4593#pullrequestreview-182016038
- dedup1_prefixes = ('-l', '-Wl,-l', '-Wl,--export-dynamic')
- dedup1_suffixes = ('.lib', '.dll', '.so', '.dylib', '.a')
- # Match a .so of the form path/to/libfoo.so.0.1.0
- # Only UNIX shared libraries require this. Others have a fixed extension.
- dedup1_regex = re.compile(r'([\/\\]|\A)lib.*\.so(\.[0-9]+)?(\.[0-9]+)?(\.[0-9]+)?$')
- dedup1_args = ('-c', '-S', '-E', '-pipe', '-pthread')
- # In generate_link() we add external libs without de-dup, but we must
- # *always* de-dup these because they're special arguments to the linker
- always_dedup_args = tuple('-l' + lib for lib in unixy_compiler_internal_libs)
-
- def __init__(self, compiler: T.Union['Compiler', StaticLinker],
- iterable: T.Optional[T.Iterable[str]] = None):
- self.compiler = compiler
- self.__container = list(iterable) if iterable is not None else [] # type: T.List[str]
- self.pre = deque() # type: T.Deque[str]
- self.post = deque() # type: T.Deque[str]
-
- # Flush the saved pre and post list into the __container list
- #
- # This correctly deduplicates the entries after _can_dedup definition
- # Note: This function is designed to work without delete operations, as deletions are worsening the performance a lot.
- def flush_pre_post(self) -> None:
- pre_flush = deque() # type: T.Deque[str]
- pre_flush_set = set() # type: T.Set[str]
- post_flush = deque() # type: T.Deque[str]
- post_flush_set = set() # type: T.Set[str]
-
- #The two lists are here walked from the front to the back, in order to not need removals for deduplication
- for a in self.pre:
- dedup = self._can_dedup(a)
- if a not in pre_flush_set:
- pre_flush.append(a)
- if dedup is Dedup.OVERRIDEN:
- pre_flush_set.add(a)
- for a in reversed(self.post):
- dedup = self._can_dedup(a)
- if a not in post_flush_set:
- post_flush.appendleft(a)
- if dedup is Dedup.OVERRIDEN:
- post_flush_set.add(a)
-
- #pre and post will overwrite every element that is in the container
- #only copy over args that are in __container but not in the post flush or pre flush set
-
- for a in self.__container:
- if a not in post_flush_set and a not in pre_flush_set:
- pre_flush.append(a)
-
- self.__container = list(pre_flush) + list(post_flush)
- self.pre.clear()
- self.post.clear()
-
- def __iter__(self) -> T.Iterator[str]:
- self.flush_pre_post()
- return iter(self.__container)
-
- @T.overload # noqa: F811
- def __getitem__(self, index: int) -> str: # noqa: F811
- pass
-
- @T.overload # noqa: F811
- def __getitem__(self, index: slice) -> T.List[str]: # noqa: F811
- pass
-
- def __getitem__(self, index): # noqa: F811
- self.flush_pre_post()
- return self.__container[index]
-
- @T.overload # noqa: F811
- def __setitem__(self, index: int, value: str) -> None: # noqa: F811
- pass
-
- @T.overload # noqa: F811
- def __setitem__(self, index: slice, value: T.List[str]) -> None: # noqa: F811
- pass
-
- def __setitem__(self, index, value) -> None: # noqa: F811
- self.flush_pre_post()
- self.__container[index] = value
-
- def __delitem__(self, index: T.Union[int, slice]) -> None:
- self.flush_pre_post()
- del self.__container[index]
-
- def __len__(self) -> int:
- return len(self.__container) + len(self.pre) + len(self.post)
-
- def insert(self, index: int, value: str) -> None:
- self.flush_pre_post()
- self.__container.insert(index, value)
-
- def copy(self) -> 'CompilerArgs':
- self.flush_pre_post()
- return CompilerArgs(self.compiler, self.__container.copy())
-
- @classmethod
- @lru_cache(maxsize=None)
- def _can_dedup(cls, arg: str) -> Dedup:
- """Returns whether the argument can be safely de-duped.
-
- In addition to these, we handle library arguments specially.
- With GNU ld, we surround library arguments with -Wl,--start/end-gr -> Dedupoup
- to recursively search for symbols in the libraries. This is not needed
- with other linkers.
- """
-
- # A standalone argument must never be deduplicated because it is
- # defined by what comes _after_ it. Thus dedupping this:
- # -D FOO -D BAR
- # would yield either
- # -D FOO BAR
- # or
- # FOO -D BAR
- # both of which are invalid.
- if arg in cls.dedup2_prefixes:
- return Dedup.NO_DEDUP
- if arg.startswith('-L='):
- # DMD and LDC proxy all linker arguments using -L=; in conjunction
- # with ld64 on macOS this can lead to command line arguments such
- # as: `-L=-compatibility_version -L=0 -L=current_version -L=0`.
- # These cannot be combined, ld64 insists they must be passed with
- # spaces and quoting does not work. if we deduplicate these then
- # one of the -L=0 arguments will be removed and the version
- # argument will consume the next argument instead.
- return Dedup.NO_DEDUP
- if arg in cls.dedup2_args or \
- arg.startswith(cls.dedup2_prefixes) or \
- arg.endswith(cls.dedup2_suffixes):
- return Dedup.OVERRIDEN
- if arg in cls.dedup1_args or \
- arg.startswith(cls.dedup1_prefixes) or \
- arg.endswith(cls.dedup1_suffixes) or \
- re.search(cls.dedup1_regex, arg):
- return Dedup.UNIQUE
- return Dedup.NO_DEDUP
-
- @classmethod
- @lru_cache(maxsize=None)
- def _should_prepend(cls, arg: str) -> bool:
- return arg.startswith(cls.prepend_prefixes)
-
- def need_to_split_linker_args(self) -> bool:
- return isinstance(self.compiler, Compiler) and self.compiler.get_language() == 'd'
-
- def to_native(self, copy: bool = False) -> T.List[str]:
- # Check if we need to add --start/end-group for circular dependencies
- # between static libraries, and for recursively searching for symbols
- # needed by static libraries that are provided by object files or
- # shared libraries.
- self.flush_pre_post()
- if copy:
- new = self.copy()
- else:
- new = self
- # To proxy these arguments with D you need to split the
- # arguments, thus you get `-L=-soname -L=lib.so` we don't
- # want to put the lib in a link -roup
- split_linker_args = self.need_to_split_linker_args()
- # This covers all ld.bfd, ld.gold, ld.gold, and xild on Linux, which
- # all act like (or are) gnu ld
- # TODO: this could probably be added to the DynamicLinker instead
- if (isinstance(self.compiler, Compiler) and
- self.compiler.linker is not None and
- isinstance(self.compiler.linker, (GnuLikeDynamicLinkerMixin, SolarisDynamicLinker))):
- group_start = -1
- group_end = -1
- is_soname = False
- for i, each in enumerate(new):
- if is_soname:
- is_soname = False
- continue
- elif split_linker_args and '-soname' in each:
- is_soname = True
- continue
- if not each.startswith(('-Wl,-l', '-l')) and not each.endswith('.a') and \
- not soregex.match(each):
- continue
- group_end = i
- if group_start < 0:
- # First occurrence of a library
- group_start = i
- if group_start >= 0:
- # Last occurrence of a library
- new.insert(group_end + 1, '-Wl,--end-group')
- new.insert(group_start, '-Wl,--start-group')
- # Remove system/default include paths added with -isystem
- if hasattr(self.compiler, 'get_default_include_dirs'):
- default_dirs = self.compiler.get_default_include_dirs()
- bad_idx_list = [] # type: T.List[int]
- for i, each in enumerate(new):
- # Remove the -isystem and the path if the path is a default path
- if (each == '-isystem' and
- i < (len(new) - 1) and
- new[i + 1] in default_dirs):
- bad_idx_list += [i, i + 1]
- elif each.startswith('-isystem=') and each[9:] in default_dirs:
- bad_idx_list += [i]
- elif each.startswith('-isystem') and each[8:] in default_dirs:
- bad_idx_list += [i]
- for i in reversed(bad_idx_list):
- new.pop(i)
- return self.compiler.unix_args_to_native(new.__container)
-
- def append_direct(self, arg: str) -> None:
- '''
- Append the specified argument without any reordering or de-dup except
- for absolute paths to libraries, etc, which can always be de-duped
- safely.
- '''
- self.flush_pre_post()
- if os.path.isabs(arg):
- self.append(arg)
- else:
- self.__container.append(arg)
-
- def extend_direct(self, iterable: T.Iterable[str]) -> None:
- '''
- Extend using the elements in the specified iterable without any
- reordering or de-dup except for absolute paths where the order of
- include search directories is not relevant
- '''
- self.flush_pre_post()
- for elem in iterable:
- self.append_direct(elem)
-
- def extend_preserving_lflags(self, iterable: T.Iterable[str]) -> None:
- normal_flags = []
- lflags = []
- for i in iterable:
- if i not in self.always_dedup_args and (i.startswith('-l') or i.startswith('-L')):
- lflags.append(i)
- else:
- normal_flags.append(i)
- self.extend(normal_flags)
- self.extend_direct(lflags)
-
- def __add__(self, args: T.Iterable[str]) -> 'CompilerArgs':
- self.flush_pre_post()
- new = self.copy()
- new += args
- return new
-
- def __iadd__(self, args: T.Iterable[str]) -> 'CompilerArgs':
- '''
- Add two CompilerArgs while taking into account overriding of arguments
- and while preserving the order of arguments as much as possible
- '''
- tmp_pre = deque() # type: T.Deque[str]
- if not isinstance(args, collections.abc.Iterable):
- raise TypeError('can only concatenate Iterable[str] (not "{}") to CompilerArgs'.format(args))
- for arg in args:
- # If the argument can be de-duped, do it either by removing the
- # previous occurrence of it and adding a new one, or not adding the
- # new occurrence.
- dedup = self._can_dedup(arg)
- if dedup is Dedup.UNIQUE:
- # Argument already exists and adding a new instance is useless
- if arg in self.__container or arg in self.pre or arg in self.post:
- continue
- if self._should_prepend(arg):
- tmp_pre.appendleft(arg)
- else:
- self.post.append(arg)
- self.pre.extendleft(tmp_pre)
- #pre and post is going to be merged later before a iter call
- return self
-
- def __radd__(self, args: T.Iterable[str]) -> 'CompilerArgs':
- self.flush_pre_post()
- new = CompilerArgs(self.compiler, args)
- new += self
- return new
-
- def __eq__(self, other: T.Any) -> T.Union[bool, type(NotImplemented)]:
- self.flush_pre_post()
- # Only allow equality checks against other CompilerArgs and lists instances
- if isinstance(other, CompilerArgs):
- return self.compiler == other.compiler and self.__container == other.__container
- elif isinstance(other, list):
- return self.__container == other
- return NotImplemented
-
- def append(self, arg: str) -> None:
- self.__iadd__([arg])
-
- def extend(self, args: T.Iterable[str]) -> None:
- self.__iadd__(args)
-
- def __repr__(self) -> str:
- self.flush_pre_post()
- return 'CompilerArgs({!r}, {!r})'.format(self.compiler, self.__container)
-
class Compiler:
# Libraries to ignore in find_library() since they are provided by the
# compiler or the C library. Currently only used for MSVC.
diff --git a/mesonbuild/compilers/d.py b/mesonbuild/compilers/d.py
index 777fa19..924ac90 100644
--- a/mesonbuild/compilers/d.py
+++ b/mesonbuild/compilers/d.py
@@ -19,13 +19,13 @@ from ..mesonlib import (
EnvironmentException, MachineChoice, version_compare,
)
+from ..arglist import CompilerArgs
from .compilers import (
d_dmd_buildtype_args,
d_gdc_buildtype_args,
d_ldc_buildtype_args,
clike_debug_args,
Compiler,
- CompilerArgs,
)
from .mixins.gnu import GnuCompiler
diff --git a/mesonbuild/compilers/mixins/clike.py b/mesonbuild/compilers/mixins/clike.py
index b088fde..b32ac29 100644
--- a/mesonbuild/compilers/mixins/clike.py
+++ b/mesonbuild/compilers/mixins/clike.py
@@ -29,9 +29,10 @@ import subprocess
import typing as T
from pathlib import Path
+from ... import arglist
from ... import mesonlib
-from ...mesonlib import LibType
from ... import mlog
+from ...mesonlib import LibType
from .. import compilers
from .visualstudio import VisualStudioLikeCompiler
@@ -48,7 +49,7 @@ class CLikeCompiler:
program_dirs_cache = {}
find_library_cache = {}
find_framework_cache = {}
- internal_libs = compilers.unixy_compiler_internal_libs
+ internal_libs = arglist.UNIXY_COMPILER_INTERNAL_LIBS
def __init__(self, is_cross: bool, exe_wrapper: T.Optional[str] = None):
# If a child ObjC or CPP class has already set it, don't set it ourselves
@@ -338,7 +339,7 @@ class CLikeCompiler:
elif not isinstance(dependencies, list):
dependencies = [dependencies]
# Collect compiler arguments
- cargs = compilers.CompilerArgs(self)
+ cargs = arglist.CompilerArgs(self)
largs = []
for d in dependencies:
# Add compile flags needed by dependencies
diff --git a/run_unittests.py b/run_unittests.py
index b663e83..b1ad965 100755
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -358,7 +358,7 @@ class InternalTests(unittest.TestCase):
stat.S_IRGRP | stat.S_IXGRP)
def test_compiler_args_class_none_flush(self):
- cargsfunc = mesonbuild.compilers.CompilerArgs
+ cargsfunc = mesonbuild.arglist.CompilerArgs
cc = mesonbuild.compilers.CCompiler([], 'fake', False, MachineChoice.HOST, mock.Mock())
a = cargsfunc(cc, ['-I.'])
#first we are checking if the tree construction deduplicates the correct -I argument
@@ -377,7 +377,7 @@ class InternalTests(unittest.TestCase):
def test_compiler_args_class(self):
- cargsfunc = mesonbuild.compilers.CompilerArgs
+ cargsfunc = mesonbuild.arglist.CompilerArgs
cc = mesonbuild.compilers.CCompiler([], 'fake', False, MachineChoice.HOST, mock.Mock())
# Test that empty initialization works
a = cargsfunc(cc)
@@ -458,7 +458,7 @@ class InternalTests(unittest.TestCase):
self.assertEqual(l, ['-Lfoodir', '-lfoo', '-Lbardir', '-lbar', '-lbar', '/libbaz.a'])
def test_compiler_args_class_gnuld(self):
- cargsfunc = mesonbuild.compilers.CompilerArgs
+ cargsfunc = mesonbuild.arglist.CompilerArgs
## Test --start/end-group
linker = mesonbuild.linkers.GnuDynamicLinker([], MachineChoice.HOST, 'fake', '-Wl,', [])
gcc = mesonbuild.compilers.GnuCCompiler([], 'fake', False, MachineChoice.HOST, mock.Mock(), linker=linker)
@@ -487,7 +487,7 @@ class InternalTests(unittest.TestCase):
self.assertEqual(l.to_native(copy=True), ['-Lfoo', '-Lfoodir', '-Wl,--start-group', '-lfoo', '-Lbardir', '-lbar', '-lbar', '/libbaz.a', '-Wl,--export-dynamic', '-Wl,-ldl', '-Wl,--end-group'])
def test_compiler_args_remove_system(self):
- cargsfunc = mesonbuild.compilers.CompilerArgs
+ cargsfunc = mesonbuild.arglist.CompilerArgs
## Test --start/end-group
linker = mesonbuild.linkers.GnuDynamicLinker([], MachineChoice.HOST, 'fake', '-Wl,', [])
gcc = mesonbuild.compilers.GnuCCompiler([], 'fake', False, MachineChoice.HOST, mock.Mock(), linker=linker)
@@ -4764,7 +4764,7 @@ recommended as it is not supported on some platforms''')
out = re.sub(r'(^ +| +$)', '', out, flags=re.MULTILINE) # strip lines
out = re.sub(r'(^\n)', '', out, flags=re.MULTILINE) # remove empty lines
return out
-
+
def clean_dir_arguments(text):
# Remove platform specific defaults
args = [