diff options
author | Aleksey Gurtovoy <agurtovoy@acm.org> | 2019-08-09 16:06:47 -0500 |
---|---|---|
committer | Jussi Pakkanen <jpakkane@gmail.com> | 2019-09-05 23:42:47 +0300 |
commit | 75daed27bc4e363696157617c7461414fc4e707b (patch) | |
tree | 2863934de82e0a7cc6a3dcd9ee23b4c4e378c550 /mesonbuild | |
parent | caec875fe1922b40037e1fd9229433ede64f9f25 (diff) | |
download | meson-75daed27bc4e363696157617c7461414fc4e707b.zip meson-75daed27bc4e363696157617c7461414fc4e707b.tar.gz meson-75daed27bc4e363696157617c7461414fc4e707b.tar.bz2 |
mesonlib.split_args/quote_arg/join_args
Diffstat (limited to 'mesonbuild')
-rw-r--r-- | mesonbuild/backend/ninjabackend.py | 8 | ||||
-rw-r--r-- | mesonbuild/compilers/compilers.py | 14 | ||||
-rw-r--r-- | mesonbuild/compilers/mixins/islinker.py | 3 | ||||
-rw-r--r-- | mesonbuild/coredata.py | 12 | ||||
-rw-r--r-- | mesonbuild/dependencies/base.py | 18 | ||||
-rw-r--r-- | mesonbuild/dependencies/misc.py | 8 | ||||
-rw-r--r-- | mesonbuild/envconfig.py | 6 | ||||
-rw-r--r-- | mesonbuild/environment.py | 10 | ||||
-rw-r--r-- | mesonbuild/linkers.py | 3 | ||||
-rw-r--r-- | mesonbuild/mesonlib.py | 80 | ||||
-rw-r--r-- | mesonbuild/modules/gnome.py | 11 | ||||
-rw-r--r-- | mesonbuild/modules/pkgconfig.py | 2 | ||||
-rw-r--r-- | mesonbuild/mtest.py | 7 | ||||
-rw-r--r-- | mesonbuild/scripts/gtkdochelper.py | 5 | ||||
-rw-r--r-- | mesonbuild/scripts/scanbuild.py | 5 |
15 files changed, 134 insertions, 58 deletions
diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index b948e25..98f244d 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -14,7 +14,6 @@ from typing import List import os import re -import shlex import pickle import subprocess from collections import OrderedDict @@ -32,7 +31,7 @@ from .. import compilers from ..compilers import Compiler, CompilerArgs, CCompiler, VisualStudioLikeCompiler, FortranCompiler from ..linkers import ArLinker from ..mesonlib import ( - File, LibType, MachineChoice, MesonException, OrderedSet, PerMachine, ProgressBar + File, LibType, MachineChoice, MesonException, OrderedSet, PerMachine, ProgressBar, quote_arg ) from ..mesonlib import get_compiler_for_source, has_path_sep from .backends import CleanTrees @@ -44,11 +43,14 @@ FORTRAN_SUBMOD_PAT = r"^\s*\bsubmodule\b\s*\((\w+:?\w+)\)\s*(\w+)" FORTRAN_USE_PAT = r"^\s*use,?\s*(?:non_intrinsic)?\s*(?:::)?\s*(\w+)" if mesonlib.is_windows(): + # FIXME: can't use quote_arg on Windows just yet; there are a number of existing workarounds + # throughout the codebase that cumulatively make the current code work (see, e.g. Backend.escape_extra_args + # and NinjaBuildElement.write below) and need to be properly untangled before attempting this quote_func = lambda s: '"{}"'.format(s) execute_wrapper = ['cmd', '/c'] rmfile_prefix = ['del', '/f', '/s', '/q', '{}', '&&'] else: - quote_func = shlex.quote + quote_func = quote_arg execute_wrapper = [] rmfile_prefix = ['rm', '-f', '{}', '&&'] diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py index 4218775..38bf240 100644 --- a/mesonbuild/compilers/compilers.py +++ b/mesonbuild/compilers/compilers.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import contextlib, enum, os.path, re, tempfile, shlex +import contextlib, enum, os.path, re, tempfile import typing -from typing import List, Optional, Tuple +from typing import Optional, Tuple, List from ..linkers import StaticLinker, GnuLikeDynamicLinkerMixin from .. import coredata @@ -22,7 +22,7 @@ from .. import mlog from .. import mesonlib from ..mesonlib import ( EnvironmentException, MachineChoice, MesonException, OrderedSet, - Popen_safe + Popen_safe, split_args ) from ..envconfig import ( Properties, @@ -799,7 +799,7 @@ class Compiler: env_compile_flags = os.environ.get(cflags_mapping[lang]) log_var(cflags_mapping[lang], env_compile_flags) if env_compile_flags is not None: - compile_flags += shlex.split(env_compile_flags) + compile_flags += split_args(env_compile_flags) # Link flags (same for all languages) if self.use_ldflags(): @@ -820,7 +820,7 @@ class Compiler: env_preproc_flags = os.environ.get('CPPFLAGS') log_var('CPPFLAGS', env_preproc_flags) if env_preproc_flags is not None: - compile_flags += shlex.split(env_preproc_flags) + compile_flags += split_args(env_preproc_flags) return compile_flags, link_flags @@ -830,10 +830,10 @@ class Compiler: opts.update({ self.language + '_args': coredata.UserArrayOption( description + ' compiler', - [], shlex_split=True, user_input=True, allow_dups=True), + [], split_args=True, user_input=True, allow_dups=True), self.language + '_link_args': coredata.UserArrayOption( description + ' linker', - [], shlex_split=True, user_input=True, allow_dups=True), + [], split_args=True, user_input=True, allow_dups=True), }) return opts diff --git a/mesonbuild/compilers/mixins/islinker.py b/mesonbuild/compilers/mixins/islinker.py index 4c1a476..dca20d0 100644 --- a/mesonbuild/compilers/mixins/islinker.py +++ b/mesonbuild/compilers/mixins/islinker.py @@ -21,7 +21,6 @@ classes for those cases. """ import os -import shlex import typing from ... import mesonlib @@ -39,7 +38,7 @@ class LinkerEnvVarsMixin: flags = os.environ.get('LDFLAGS') if not flags: return [] - return shlex.split(flags) + return mesonlib.split_args(flags) class BasicLinkerIsCompilerMixin: diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index fe14c35..5796377 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -13,14 +13,14 @@ # limitations under the License. from . import mlog -import pickle, os, uuid, shlex +import pickle, os, uuid import sys from itertools import chain from pathlib import PurePath from collections import OrderedDict from .mesonlib import ( MesonException, MachineChoice, PerMachine, - default_libdir, default_libexecdir, default_prefix + default_libdir, default_libexecdir, default_prefix, split_args ) from .wrap import WrapMode import ast @@ -163,9 +163,9 @@ class UserComboOption(UserOption[str]): return value class UserArrayOption(UserOption[List[str]]): - def __init__(self, description, value, shlex_split=False, user_input=False, allow_dups=False, **kwargs): + def __init__(self, description, value, split_args=False, user_input=False, allow_dups=False, **kwargs): super().__init__(description, kwargs.get('choices', []), yielding=kwargs.get('yielding', None)) - self.shlex_split = shlex_split + self.split_args = split_args self.allow_dups = allow_dups self.value = self.validate_value(value, user_input=user_input) @@ -183,8 +183,8 @@ class UserArrayOption(UserOption[List[str]]): elif value == '': newvalue = [] else: - if self.shlex_split: - newvalue = shlex.split(value) + if self.split_args: + newvalue = split_args(value) else: newvalue = [v.strip() for v in value.split(',')] elif isinstance(value, list): diff --git a/mesonbuild/dependencies/base.py b/mesonbuild/dependencies/base.py index e517fea..b0f24c8 100644 --- a/mesonbuild/dependencies/base.py +++ b/mesonbuild/dependencies/base.py @@ -34,7 +34,7 @@ from ..compilers import clib_langs from ..environment import BinaryTable, Environment, MachineInfo from ..cmake import CMakeExecutor, CMakeTraceParser, CMakeException from ..mesonlib import MachineChoice, MesonException, OrderedSet, PerMachine -from ..mesonlib import Popen_safe, version_compare_many, version_compare, listify, stringlistify, extract_as_list +from ..mesonlib import Popen_safe, version_compare_many, version_compare, listify, stringlistify, extract_as_list, split_args from ..mesonlib import Version, LibType # These must be defined in this file to avoid cyclical references. @@ -490,16 +490,13 @@ class ConfigToolDependency(ExternalDependency): def get_config_value(self, args, stage): p, out, err = Popen_safe(self.config + args) - # This is required to keep shlex from stripping path separators on - # Windows. Also, don't put escape sequences in config values, okay? - out = out.replace('\\', '\\\\') if p.returncode != 0: if self.required: raise DependencyException( 'Could not generate {} for {}.\n{}'.format( stage, self.name, err)) return [] - return shlex.split(out) + return split_args(out) @staticmethod def get_methods(): @@ -697,6 +694,11 @@ class PkgConfigDependency(ExternalDependency): converted.append(arg) return converted + def _split_args(self, cmd): + # pkg-config paths follow Unix conventions, even on Windows; split the + # output using shlex.split rather than mesonlib.split_args + return shlex.split(cmd) + def _set_cargs(self): env = None if self.language == 'fortran': @@ -708,7 +710,7 @@ class PkgConfigDependency(ExternalDependency): if ret != 0: raise DependencyException('Could not generate cargs for %s:\n\n%s' % (self.name, out)) - self.compile_args = self._convert_mingw_paths(shlex.split(out)) + self.compile_args = self._convert_mingw_paths(self._split_args(out)) def _search_libs(self, out, out_raw): ''' @@ -737,7 +739,7 @@ class PkgConfigDependency(ExternalDependency): # always searched first. prefix_libpaths = OrderedSet() # We also store this raw_link_args on the object later - raw_link_args = self._convert_mingw_paths(shlex.split(out_raw)) + raw_link_args = self._convert_mingw_paths(self._split_args(out_raw)) for arg in raw_link_args: if arg.startswith('-L') and not arg.startswith(('-L-l', '-L-L')): path = arg[2:] @@ -746,7 +748,7 @@ class PkgConfigDependency(ExternalDependency): path = os.path.join(self.env.get_build_dir(), path) prefix_libpaths.add(path) system_libpaths = OrderedSet() - full_args = self._convert_mingw_paths(shlex.split(out)) + full_args = self._convert_mingw_paths(self._split_args(out)) for arg in full_args: if arg.startswith(('-L-l', '-L-L')): # These are D language arguments, not library paths diff --git a/mesonbuild/dependencies/misc.py b/mesonbuild/dependencies/misc.py index 53c3747..23a283f 100644 --- a/mesonbuild/dependencies/misc.py +++ b/mesonbuild/dependencies/misc.py @@ -18,11 +18,11 @@ from pathlib import Path import functools import os import re -import shlex import sysconfig from .. import mlog from .. import mesonlib +from ..mesonlib import split_args from ..environment import detect_cpu_family from .base import ( @@ -277,7 +277,7 @@ class MPIDependency(ExternalDependency): mlog.debug(mlog.bold('Standard output\n'), o) mlog.debug(mlog.bold('Standard error\n'), e) return - cargs = shlex.split(o) + cargs = split_args(o) cmd = prog.get_command() + ['--showme:link'] p, o, e = mesonlib.Popen_safe(cmd) @@ -287,7 +287,7 @@ class MPIDependency(ExternalDependency): mlog.debug(mlog.bold('Standard output\n'), o) mlog.debug(mlog.bold('Standard error\n'), e) return - libs = shlex.split(o) + libs = split_args(o) cmd = prog.get_command() + ['--showme:version'] p, o, e = mesonlib.Popen_safe(cmd) @@ -316,7 +316,7 @@ class MPIDependency(ExternalDependency): mlog.debug(mlog.bold('Standard output\n'), o) mlog.debug(mlog.bold('Standard error\n'), e) return - args = shlex.split(o) + args = split_args(o) version = None diff --git a/mesonbuild/envconfig.py b/mesonbuild/envconfig.py index f7a43a0..0c9f03f 100644 --- a/mesonbuild/envconfig.py +++ b/mesonbuild/envconfig.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import configparser, os, shlex, subprocess +import configparser, os, subprocess import typing from . import mesonlib -from .mesonlib import EnvironmentException +from .mesonlib import EnvironmentException, split_args from . import mlog _T = typing.TypeVar('_T') @@ -361,7 +361,7 @@ This is probably wrong, it should always point to the native compiler.''' % evar evar = self.evarMap.get(name, "") command = os.environ.get(evar) if command is not None: - command = shlex.split(command) + command = split_args(command) # Do not return empty or blank string entries if command is not None and (len(command) == 0 or len(command[0].strip()) == 0): diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 7e1ca9d..cf386da 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, platform, re, sys, shlex, shutil, subprocess, typing +import os, platform, re, sys, shutil, subprocess, typing import tempfile from . import coredata @@ -20,7 +20,7 @@ from .linkers import ArLinker, ArmarLinker, VisualStudioLinker, DLinker, CcrxLin from . import mesonlib from .mesonlib import ( MesonException, EnvironmentException, MachineChoice, Popen_safe, - PerMachineDefaultable, PerThreeMachineDefaultable + PerMachineDefaultable, PerThreeMachineDefaultable, split_args, quote_arg ) from . import mlog @@ -120,7 +120,7 @@ def detect_gcovr(min_version='3.3', new_rootdir_version='4.2', log=False): found = search_version(found) if p.returncode == 0 and mesonlib.version_compare(found, '>=' + min_version): if log: - mlog.log('Found gcovr-{} at {}'.format(found, shlex.quote(shutil.which(gcovr_exe)))) + mlog.log('Found gcovr-{} at {}'.format(found, quote_arg(shutil.which(gcovr_exe)))) return gcovr_exe, mesonlib.version_compare(found, '>=' + new_rootdir_version) return None, None @@ -158,7 +158,7 @@ def detect_ninja(version: str = '1.5', log: bool = False) -> str: name = 'ninja' if name == 'samu': name = 'samurai' - mlog.log('Found {}-{} at {}'.format(name, found, shlex.quote(n))) + mlog.log('Found {}-{} at {}'.format(name, found, quote_arg(n))) return n def detect_native_windows_arch(): @@ -1322,7 +1322,7 @@ class Environment: if isinstance(compiler, compilers.CudaCompiler): linkers = [self.cuda_static_linker, self.default_static_linker] elif evar in os.environ: - linkers = [shlex.split(os.environ[evar])] + linkers = [split_args(os.environ[evar])] elif isinstance(compiler, compilers.VisualStudioLikeCompiler): linkers = [self.vs_static_linker, self.clang_cl_static_linker] elif isinstance(compiler, compilers.GnuCompiler): diff --git a/mesonbuild/linkers.py b/mesonbuild/linkers.py index 6cce78b..8de254b 100644 --- a/mesonbuild/linkers.py +++ b/mesonbuild/linkers.py @@ -14,7 +14,6 @@ import abc import os -import shlex import typing from . import mesonlib @@ -279,7 +278,7 @@ class DynamicLinker(metaclass=abc.ABCMeta): flags = os.environ.get('LDFLAGS') if not flags: return [] - return shlex.split(flags) + return mesonlib.split_args(flags) def get_option_args(self, options: 'OptionDictType') -> typing.List[str]: return [] diff --git a/mesonbuild/mesonlib.py b/mesonbuild/mesonlib.py index 1333027..d5646ed 100644 --- a/mesonbuild/mesonlib.py +++ b/mesonbuild/mesonlib.py @@ -17,7 +17,7 @@ from pathlib import Path import sys import stat import time -import platform, subprocess, operator, os, shutil, re +import platform, subprocess, operator, os, shlex, shutil, re import collections from enum import Enum from functools import lru_cache @@ -729,6 +729,84 @@ def has_path_sep(name, sep='/\\'): return True return False + +if is_windows(): + # shlex.split is not suitable for splitting command line on Window (https://bugs.python.org/issue1724822); + # shlex.quote is similarly problematic. Below are "proper" implementations of these functions according to + # https://docs.microsoft.com/en-us/cpp/c-language/parsing-c-command-line-arguments and + # https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ + + _whitespace = ' \t\n\r' + _find_unsafe_char = re.compile(r'[{}"]'.format(_whitespace)).search + + def quote_arg(arg): + if arg and not _find_unsafe_char(arg): + return arg + + result = '"' + num_backslashes = 0 + for c in arg: + if c == '\\': + num_backslashes += 1 + else: + if c == '"': + # Escape all backslashes and the following double quotation mark + num_backslashes = num_backslashes * 2 + 1 + + result += num_backslashes * '\\' + c + num_backslashes = 0 + + # Escape all backslashes, but let the terminating double quotation + # mark we add below be interpreted as a metacharacter + result += (num_backslashes * 2) * '\\' + '"' + return result + + def split_args(cmd): + result = [] + arg = '' + num_backslashes = 0 + num_quotes = 0 + in_quotes = False + for c in cmd: + if c == '\\': + num_backslashes += 1 + else: + if c == '"' and not (num_backslashes % 2): + # unescaped quote, eat it + arg += (num_backslashes // 2) * '\\' + num_quotes += 1 + in_quotes = not in_quotes + elif c in _whitespace and not in_quotes: + if arg or num_quotes: + # reached the end of the argument + result.append(arg) + arg = '' + num_quotes = 0 + else: + if c == '"': + # escaped quote + num_backslashes = (num_backslashes - 1) // 2 + + arg += num_backslashes * '\\' + c + + num_backslashes = 0 + + if arg or num_quotes: + result.append(arg) + + return result +else: + def quote_arg(arg): + return shlex.quote(arg) + + def split_args(cmd): + return shlex.split(cmd) + + +def join_args(args): + return ' '.join([quote_arg(x) for x in args]) + + def do_replacement(regex, line, variable_format, confdata): missing_variables = set() start_tag = '@' diff --git a/mesonbuild/modules/gnome.py b/mesonbuild/modules/gnome.py index 4e97d3a..5c9d3dd 100644 --- a/mesonbuild/modules/gnome.py +++ b/mesonbuild/modules/gnome.py @@ -17,7 +17,6 @@ functionality such as gobject-introspection, gresources and gtk-doc''' import os import copy -import shlex import subprocess from .. import build @@ -29,7 +28,7 @@ from . import get_include_args from . import ExtensionModule from . import ModuleReturnValue from ..mesonlib import ( - MachineChoice, MesonException, OrderedSet, Popen_safe, extract_as_list + MachineChoice, MesonException, OrderedSet, Popen_safe, extract_as_list, join_args ) from ..dependencies import Dependency, PkgConfigDependency, InternalDependency from ..interpreterbase import noKwargs, permittedKwargs, FeatureNew, FeatureNewKwargs @@ -1079,12 +1078,12 @@ This will become a hard error in the future.''') ldflags.extend(compiler_flags[1]) ldflags.extend(compiler_flags[2]) if compiler: - args += ['--cc=%s' % ' '.join([shlex.quote(x) for x in compiler.get_exelist()])] - args += ['--ld=%s' % ' '.join([shlex.quote(x) for x in compiler.get_linker_exelist()])] + args += ['--cc=%s' % join_args(compiler.get_exelist())] + args += ['--ld=%s' % join_args(compiler.get_linker_exelist())] if cflags: - args += ['--cflags=%s' % ' '.join([shlex.quote(x) for x in cflags])] + args += ['--cflags=%s' % join_args(cflags)] if ldflags: - args += ['--ldflags=%s' % ' '.join([shlex.quote(x) for x in ldflags])] + args += ['--ldflags=%s' % join_args(ldflags)] return args diff --git a/mesonbuild/modules/pkgconfig.py b/mesonbuild/modules/pkgconfig.py index 78fffb1..60d4b3f 100644 --- a/mesonbuild/modules/pkgconfig.py +++ b/mesonbuild/modules/pkgconfig.py @@ -240,7 +240,7 @@ class PkgConfigModule(ExtensionModule): def _escape(self, value): ''' - We cannot use shlex.quote because it quotes with ' and " which does not + We cannot use quote_arg because it quotes with ' and " which does not work with pkg-config and pkgconf at all. ''' # We should always write out paths with / because pkg-config requires diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index e11c8e4..70585f4 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -29,7 +29,6 @@ import pickle import platform import random import re -import shlex import signal import subprocess import sys @@ -41,7 +40,7 @@ from . import build from . import environment from . import mlog from .dependencies import ExternalProgram -from .mesonlib import MesonException, get_wine_shortpath +from .mesonlib import MesonException, get_wine_shortpath, split_args if typing.TYPE_CHECKING: from .backend.backends import TestSerialisation @@ -88,7 +87,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: help='Run test under gdb.') parser.add_argument('--list', default=False, dest='list', action='store_true', help='List available tests.') - parser.add_argument('--wrapper', default=None, dest='wrapper', type=shlex.split, + parser.add_argument('--wrapper', default=None, dest='wrapper', type=split_args, help='wrapper to run tests with (e.g. Valgrind)') parser.add_argument('-C', default='.', dest='wd', help='directory to cd into before running') @@ -116,7 +115,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: ' more time to execute.') parser.add_argument('--setup', default=None, dest='setup', help='Which test setup to use.') - parser.add_argument('--test-args', default=[], type=shlex.split, + parser.add_argument('--test-args', default=[], type=split_args, help='Arguments to pass to the specified test(s) or all tests') parser.add_argument('args', nargs='*', help='Optional list of tests to run') diff --git a/mesonbuild/scripts/gtkdochelper.py b/mesonbuild/scripts/gtkdochelper.py index 4998e17..ddcc8c0 100644 --- a/mesonbuild/scripts/gtkdochelper.py +++ b/mesonbuild/scripts/gtkdochelper.py @@ -14,10 +14,9 @@ import sys, os import subprocess -import shlex import shutil import argparse -from ..mesonlib import MesonException, Popen_safe, is_windows +from ..mesonlib import MesonException, Popen_safe, is_windows, split_args from . import destdir_join parser = argparse.ArgumentParser() @@ -149,7 +148,7 @@ def build_gtkdoc(source_root, build_root, doc_subdir, src_subdirs, '--output-dir=' + abs_out] library_paths = [] - for ldflag in shlex.split(ldflags): + for ldflag in split_args(ldflags): if ldflag.startswith('-Wl,-rpath,'): library_paths.append(ldflag[11:]) diff --git a/mesonbuild/scripts/scanbuild.py b/mesonbuild/scripts/scanbuild.py index 51f70f0..8c0f423 100644 --- a/mesonbuild/scripts/scanbuild.py +++ b/mesonbuild/scripts/scanbuild.py @@ -13,12 +13,11 @@ # limitations under the License. import os -import shlex import subprocess import shutil import tempfile from ..environment import detect_ninja -from ..mesonlib import Popen_safe +from ..mesonlib import Popen_safe, split_args def scanbuild(exelist, srcdir, blddir, privdir, logdir, args): with tempfile.TemporaryDirectory(dir=privdir) as scandir: @@ -63,7 +62,7 @@ def run(args): break if 'SCANBUILD' in os.environ: - exelist = shlex.split(os.environ['SCANBUILD']) + exelist = split_args(os.environ['SCANBUILD']) else: exelist = [toolname] |