From 75daed27bc4e363696157617c7461414fc4e707b Mon Sep 17 00:00:00 2001 From: Aleksey Gurtovoy Date: Fri, 9 Aug 2019 16:06:47 -0500 Subject: mesonlib.split_args/quote_arg/join_args --- mesonbuild/backend/ninjabackend.py | 8 ++- mesonbuild/compilers/compilers.py | 14 ++-- mesonbuild/compilers/mixins/islinker.py | 3 +- mesonbuild/coredata.py | 12 ++-- mesonbuild/dependencies/base.py | 18 ++--- mesonbuild/dependencies/misc.py | 8 +-- mesonbuild/envconfig.py | 6 +- mesonbuild/environment.py | 10 +-- mesonbuild/linkers.py | 3 +- mesonbuild/mesonlib.py | 80 +++++++++++++++++++++- mesonbuild/modules/gnome.py | 11 ++- mesonbuild/modules/pkgconfig.py | 2 +- mesonbuild/mtest.py | 7 +- mesonbuild/scripts/gtkdochelper.py | 5 +- mesonbuild/scripts/scanbuild.py | 5 +- run_tests.py | 2 +- run_unittests.py | 116 +++++++++++++++++++++++++++++--- 17 files changed, 241 insertions(+), 69 deletions(-) diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index b948e25..98f244d 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -14,7 +14,6 @@ from typing import List import os import re -import shlex import pickle import subprocess from collections import OrderedDict @@ -32,7 +31,7 @@ from .. import compilers from ..compilers import Compiler, CompilerArgs, CCompiler, VisualStudioLikeCompiler, FortranCompiler from ..linkers import ArLinker from ..mesonlib import ( - File, LibType, MachineChoice, MesonException, OrderedSet, PerMachine, ProgressBar + File, LibType, MachineChoice, MesonException, OrderedSet, PerMachine, ProgressBar, quote_arg ) from ..mesonlib import get_compiler_for_source, has_path_sep from .backends import CleanTrees @@ -44,11 +43,14 @@ FORTRAN_SUBMOD_PAT = r"^\s*\bsubmodule\b\s*\((\w+:?\w+)\)\s*(\w+)" FORTRAN_USE_PAT = r"^\s*use,?\s*(?:non_intrinsic)?\s*(?:::)?\s*(\w+)" if mesonlib.is_windows(): + # FIXME: can't use quote_arg on Windows just yet; there are a number of existing workarounds + # throughout the codebase that cumulatively make the current code work (see, e.g. Backend.escape_extra_args + # and NinjaBuildElement.write below) and need to be properly untangled before attempting this quote_func = lambda s: '"{}"'.format(s) execute_wrapper = ['cmd', '/c'] rmfile_prefix = ['del', '/f', '/s', '/q', '{}', '&&'] else: - quote_func = shlex.quote + quote_func = quote_arg execute_wrapper = [] rmfile_prefix = ['rm', '-f', '{}', '&&'] diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py index 4218775..38bf240 100644 --- a/mesonbuild/compilers/compilers.py +++ b/mesonbuild/compilers/compilers.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import contextlib, enum, os.path, re, tempfile, shlex +import contextlib, enum, os.path, re, tempfile import typing -from typing import List, Optional, Tuple +from typing import Optional, Tuple, List from ..linkers import StaticLinker, GnuLikeDynamicLinkerMixin from .. import coredata @@ -22,7 +22,7 @@ from .. import mlog from .. import mesonlib from ..mesonlib import ( EnvironmentException, MachineChoice, MesonException, OrderedSet, - Popen_safe + Popen_safe, split_args ) from ..envconfig import ( Properties, @@ -799,7 +799,7 @@ class Compiler: env_compile_flags = os.environ.get(cflags_mapping[lang]) log_var(cflags_mapping[lang], env_compile_flags) if env_compile_flags is not None: - compile_flags += shlex.split(env_compile_flags) + compile_flags += split_args(env_compile_flags) # Link flags (same for all languages) if self.use_ldflags(): @@ -820,7 +820,7 @@ class Compiler: env_preproc_flags = os.environ.get('CPPFLAGS') log_var('CPPFLAGS', env_preproc_flags) if env_preproc_flags is not None: - compile_flags += shlex.split(env_preproc_flags) + compile_flags += split_args(env_preproc_flags) return compile_flags, link_flags @@ -830,10 +830,10 @@ class Compiler: opts.update({ self.language + '_args': coredata.UserArrayOption( description + ' compiler', - [], shlex_split=True, user_input=True, allow_dups=True), + [], split_args=True, user_input=True, allow_dups=True), self.language + '_link_args': coredata.UserArrayOption( description + ' linker', - [], shlex_split=True, user_input=True, allow_dups=True), + [], split_args=True, user_input=True, allow_dups=True), }) return opts diff --git a/mesonbuild/compilers/mixins/islinker.py b/mesonbuild/compilers/mixins/islinker.py index 4c1a476..dca20d0 100644 --- a/mesonbuild/compilers/mixins/islinker.py +++ b/mesonbuild/compilers/mixins/islinker.py @@ -21,7 +21,6 @@ classes for those cases. """ import os -import shlex import typing from ... import mesonlib @@ -39,7 +38,7 @@ class LinkerEnvVarsMixin: flags = os.environ.get('LDFLAGS') if not flags: return [] - return shlex.split(flags) + return mesonlib.split_args(flags) class BasicLinkerIsCompilerMixin: diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index fe14c35..5796377 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -13,14 +13,14 @@ # limitations under the License. from . import mlog -import pickle, os, uuid, shlex +import pickle, os, uuid import sys from itertools import chain from pathlib import PurePath from collections import OrderedDict from .mesonlib import ( MesonException, MachineChoice, PerMachine, - default_libdir, default_libexecdir, default_prefix + default_libdir, default_libexecdir, default_prefix, split_args ) from .wrap import WrapMode import ast @@ -163,9 +163,9 @@ class UserComboOption(UserOption[str]): return value class UserArrayOption(UserOption[List[str]]): - def __init__(self, description, value, shlex_split=False, user_input=False, allow_dups=False, **kwargs): + def __init__(self, description, value, split_args=False, user_input=False, allow_dups=False, **kwargs): super().__init__(description, kwargs.get('choices', []), yielding=kwargs.get('yielding', None)) - self.shlex_split = shlex_split + self.split_args = split_args self.allow_dups = allow_dups self.value = self.validate_value(value, user_input=user_input) @@ -183,8 +183,8 @@ class UserArrayOption(UserOption[List[str]]): elif value == '': newvalue = [] else: - if self.shlex_split: - newvalue = shlex.split(value) + if self.split_args: + newvalue = split_args(value) else: newvalue = [v.strip() for v in value.split(',')] elif isinstance(value, list): diff --git a/mesonbuild/dependencies/base.py b/mesonbuild/dependencies/base.py index e517fea..b0f24c8 100644 --- a/mesonbuild/dependencies/base.py +++ b/mesonbuild/dependencies/base.py @@ -34,7 +34,7 @@ from ..compilers import clib_langs from ..environment import BinaryTable, Environment, MachineInfo from ..cmake import CMakeExecutor, CMakeTraceParser, CMakeException from ..mesonlib import MachineChoice, MesonException, OrderedSet, PerMachine -from ..mesonlib import Popen_safe, version_compare_many, version_compare, listify, stringlistify, extract_as_list +from ..mesonlib import Popen_safe, version_compare_many, version_compare, listify, stringlistify, extract_as_list, split_args from ..mesonlib import Version, LibType # These must be defined in this file to avoid cyclical references. @@ -490,16 +490,13 @@ class ConfigToolDependency(ExternalDependency): def get_config_value(self, args, stage): p, out, err = Popen_safe(self.config + args) - # This is required to keep shlex from stripping path separators on - # Windows. Also, don't put escape sequences in config values, okay? - out = out.replace('\\', '\\\\') if p.returncode != 0: if self.required: raise DependencyException( 'Could not generate {} for {}.\n{}'.format( stage, self.name, err)) return [] - return shlex.split(out) + return split_args(out) @staticmethod def get_methods(): @@ -697,6 +694,11 @@ class PkgConfigDependency(ExternalDependency): converted.append(arg) return converted + def _split_args(self, cmd): + # pkg-config paths follow Unix conventions, even on Windows; split the + # output using shlex.split rather than mesonlib.split_args + return shlex.split(cmd) + def _set_cargs(self): env = None if self.language == 'fortran': @@ -708,7 +710,7 @@ class PkgConfigDependency(ExternalDependency): if ret != 0: raise DependencyException('Could not generate cargs for %s:\n\n%s' % (self.name, out)) - self.compile_args = self._convert_mingw_paths(shlex.split(out)) + self.compile_args = self._convert_mingw_paths(self._split_args(out)) def _search_libs(self, out, out_raw): ''' @@ -737,7 +739,7 @@ class PkgConfigDependency(ExternalDependency): # always searched first. prefix_libpaths = OrderedSet() # We also store this raw_link_args on the object later - raw_link_args = self._convert_mingw_paths(shlex.split(out_raw)) + raw_link_args = self._convert_mingw_paths(self._split_args(out_raw)) for arg in raw_link_args: if arg.startswith('-L') and not arg.startswith(('-L-l', '-L-L')): path = arg[2:] @@ -746,7 +748,7 @@ class PkgConfigDependency(ExternalDependency): path = os.path.join(self.env.get_build_dir(), path) prefix_libpaths.add(path) system_libpaths = OrderedSet() - full_args = self._convert_mingw_paths(shlex.split(out)) + full_args = self._convert_mingw_paths(self._split_args(out)) for arg in full_args: if arg.startswith(('-L-l', '-L-L')): # These are D language arguments, not library paths diff --git a/mesonbuild/dependencies/misc.py b/mesonbuild/dependencies/misc.py index 53c3747..23a283f 100644 --- a/mesonbuild/dependencies/misc.py +++ b/mesonbuild/dependencies/misc.py @@ -18,11 +18,11 @@ from pathlib import Path import functools import os import re -import shlex import sysconfig from .. import mlog from .. import mesonlib +from ..mesonlib import split_args from ..environment import detect_cpu_family from .base import ( @@ -277,7 +277,7 @@ class MPIDependency(ExternalDependency): mlog.debug(mlog.bold('Standard output\n'), o) mlog.debug(mlog.bold('Standard error\n'), e) return - cargs = shlex.split(o) + cargs = split_args(o) cmd = prog.get_command() + ['--showme:link'] p, o, e = mesonlib.Popen_safe(cmd) @@ -287,7 +287,7 @@ class MPIDependency(ExternalDependency): mlog.debug(mlog.bold('Standard output\n'), o) mlog.debug(mlog.bold('Standard error\n'), e) return - libs = shlex.split(o) + libs = split_args(o) cmd = prog.get_command() + ['--showme:version'] p, o, e = mesonlib.Popen_safe(cmd) @@ -316,7 +316,7 @@ class MPIDependency(ExternalDependency): mlog.debug(mlog.bold('Standard output\n'), o) mlog.debug(mlog.bold('Standard error\n'), e) return - args = shlex.split(o) + args = split_args(o) version = None diff --git a/mesonbuild/envconfig.py b/mesonbuild/envconfig.py index f7a43a0..0c9f03f 100644 --- a/mesonbuild/envconfig.py +++ b/mesonbuild/envconfig.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import configparser, os, shlex, subprocess +import configparser, os, subprocess import typing from . import mesonlib -from .mesonlib import EnvironmentException +from .mesonlib import EnvironmentException, split_args from . import mlog _T = typing.TypeVar('_T') @@ -361,7 +361,7 @@ This is probably wrong, it should always point to the native compiler.''' % evar evar = self.evarMap.get(name, "") command = os.environ.get(evar) if command is not None: - command = shlex.split(command) + command = split_args(command) # Do not return empty or blank string entries if command is not None and (len(command) == 0 or len(command[0].strip()) == 0): diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 7e1ca9d..cf386da 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, platform, re, sys, shlex, shutil, subprocess, typing +import os, platform, re, sys, shutil, subprocess, typing import tempfile from . import coredata @@ -20,7 +20,7 @@ from .linkers import ArLinker, ArmarLinker, VisualStudioLinker, DLinker, CcrxLin from . import mesonlib from .mesonlib import ( MesonException, EnvironmentException, MachineChoice, Popen_safe, - PerMachineDefaultable, PerThreeMachineDefaultable + PerMachineDefaultable, PerThreeMachineDefaultable, split_args, quote_arg ) from . import mlog @@ -120,7 +120,7 @@ def detect_gcovr(min_version='3.3', new_rootdir_version='4.2', log=False): found = search_version(found) if p.returncode == 0 and mesonlib.version_compare(found, '>=' + min_version): if log: - mlog.log('Found gcovr-{} at {}'.format(found, shlex.quote(shutil.which(gcovr_exe)))) + mlog.log('Found gcovr-{} at {}'.format(found, quote_arg(shutil.which(gcovr_exe)))) return gcovr_exe, mesonlib.version_compare(found, '>=' + new_rootdir_version) return None, None @@ -158,7 +158,7 @@ def detect_ninja(version: str = '1.5', log: bool = False) -> str: name = 'ninja' if name == 'samu': name = 'samurai' - mlog.log('Found {}-{} at {}'.format(name, found, shlex.quote(n))) + mlog.log('Found {}-{} at {}'.format(name, found, quote_arg(n))) return n def detect_native_windows_arch(): @@ -1322,7 +1322,7 @@ class Environment: if isinstance(compiler, compilers.CudaCompiler): linkers = [self.cuda_static_linker, self.default_static_linker] elif evar in os.environ: - linkers = [shlex.split(os.environ[evar])] + linkers = [split_args(os.environ[evar])] elif isinstance(compiler, compilers.VisualStudioLikeCompiler): linkers = [self.vs_static_linker, self.clang_cl_static_linker] elif isinstance(compiler, compilers.GnuCompiler): diff --git a/mesonbuild/linkers.py b/mesonbuild/linkers.py index 6cce78b..8de254b 100644 --- a/mesonbuild/linkers.py +++ b/mesonbuild/linkers.py @@ -14,7 +14,6 @@ import abc import os -import shlex import typing from . import mesonlib @@ -279,7 +278,7 @@ class DynamicLinker(metaclass=abc.ABCMeta): flags = os.environ.get('LDFLAGS') if not flags: return [] - return shlex.split(flags) + return mesonlib.split_args(flags) def get_option_args(self, options: 'OptionDictType') -> typing.List[str]: return [] diff --git a/mesonbuild/mesonlib.py b/mesonbuild/mesonlib.py index 1333027..d5646ed 100644 --- a/mesonbuild/mesonlib.py +++ b/mesonbuild/mesonlib.py @@ -17,7 +17,7 @@ from pathlib import Path import sys import stat import time -import platform, subprocess, operator, os, shutil, re +import platform, subprocess, operator, os, shlex, shutil, re import collections from enum import Enum from functools import lru_cache @@ -729,6 +729,84 @@ def has_path_sep(name, sep='/\\'): return True return False + +if is_windows(): + # shlex.split is not suitable for splitting command line on Window (https://bugs.python.org/issue1724822); + # shlex.quote is similarly problematic. Below are "proper" implementations of these functions according to + # https://docs.microsoft.com/en-us/cpp/c-language/parsing-c-command-line-arguments and + # https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ + + _whitespace = ' \t\n\r' + _find_unsafe_char = re.compile(r'[{}"]'.format(_whitespace)).search + + def quote_arg(arg): + if arg and not _find_unsafe_char(arg): + return arg + + result = '"' + num_backslashes = 0 + for c in arg: + if c == '\\': + num_backslashes += 1 + else: + if c == '"': + # Escape all backslashes and the following double quotation mark + num_backslashes = num_backslashes * 2 + 1 + + result += num_backslashes * '\\' + c + num_backslashes = 0 + + # Escape all backslashes, but let the terminating double quotation + # mark we add below be interpreted as a metacharacter + result += (num_backslashes * 2) * '\\' + '"' + return result + + def split_args(cmd): + result = [] + arg = '' + num_backslashes = 0 + num_quotes = 0 + in_quotes = False + for c in cmd: + if c == '\\': + num_backslashes += 1 + else: + if c == '"' and not (num_backslashes % 2): + # unescaped quote, eat it + arg += (num_backslashes // 2) * '\\' + num_quotes += 1 + in_quotes = not in_quotes + elif c in _whitespace and not in_quotes: + if arg or num_quotes: + # reached the end of the argument + result.append(arg) + arg = '' + num_quotes = 0 + else: + if c == '"': + # escaped quote + num_backslashes = (num_backslashes - 1) // 2 + + arg += num_backslashes * '\\' + c + + num_backslashes = 0 + + if arg or num_quotes: + result.append(arg) + + return result +else: + def quote_arg(arg): + return shlex.quote(arg) + + def split_args(cmd): + return shlex.split(cmd) + + +def join_args(args): + return ' '.join([quote_arg(x) for x in args]) + + def do_replacement(regex, line, variable_format, confdata): missing_variables = set() start_tag = '@' diff --git a/mesonbuild/modules/gnome.py b/mesonbuild/modules/gnome.py index 4e97d3a..5c9d3dd 100644 --- a/mesonbuild/modules/gnome.py +++ b/mesonbuild/modules/gnome.py @@ -17,7 +17,6 @@ functionality such as gobject-introspection, gresources and gtk-doc''' import os import copy -import shlex import subprocess from .. import build @@ -29,7 +28,7 @@ from . import get_include_args from . import ExtensionModule from . import ModuleReturnValue from ..mesonlib import ( - MachineChoice, MesonException, OrderedSet, Popen_safe, extract_as_list + MachineChoice, MesonException, OrderedSet, Popen_safe, extract_as_list, join_args ) from ..dependencies import Dependency, PkgConfigDependency, InternalDependency from ..interpreterbase import noKwargs, permittedKwargs, FeatureNew, FeatureNewKwargs @@ -1079,12 +1078,12 @@ This will become a hard error in the future.''') ldflags.extend(compiler_flags[1]) ldflags.extend(compiler_flags[2]) if compiler: - args += ['--cc=%s' % ' '.join([shlex.quote(x) for x in compiler.get_exelist()])] - args += ['--ld=%s' % ' '.join([shlex.quote(x) for x in compiler.get_linker_exelist()])] + args += ['--cc=%s' % join_args(compiler.get_exelist())] + args += ['--ld=%s' % join_args(compiler.get_linker_exelist())] if cflags: - args += ['--cflags=%s' % ' '.join([shlex.quote(x) for x in cflags])] + args += ['--cflags=%s' % join_args(cflags)] if ldflags: - args += ['--ldflags=%s' % ' '.join([shlex.quote(x) for x in ldflags])] + args += ['--ldflags=%s' % join_args(ldflags)] return args diff --git a/mesonbuild/modules/pkgconfig.py b/mesonbuild/modules/pkgconfig.py index 78fffb1..60d4b3f 100644 --- a/mesonbuild/modules/pkgconfig.py +++ b/mesonbuild/modules/pkgconfig.py @@ -240,7 +240,7 @@ class PkgConfigModule(ExtensionModule): def _escape(self, value): ''' - We cannot use shlex.quote because it quotes with ' and " which does not + We cannot use quote_arg because it quotes with ' and " which does not work with pkg-config and pkgconf at all. ''' # We should always write out paths with / because pkg-config requires diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index e11c8e4..70585f4 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -29,7 +29,6 @@ import pickle import platform import random import re -import shlex import signal import subprocess import sys @@ -41,7 +40,7 @@ from . import build from . import environment from . import mlog from .dependencies import ExternalProgram -from .mesonlib import MesonException, get_wine_shortpath +from .mesonlib import MesonException, get_wine_shortpath, split_args if typing.TYPE_CHECKING: from .backend.backends import TestSerialisation @@ -88,7 +87,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: help='Run test under gdb.') parser.add_argument('--list', default=False, dest='list', action='store_true', help='List available tests.') - parser.add_argument('--wrapper', default=None, dest='wrapper', type=shlex.split, + parser.add_argument('--wrapper', default=None, dest='wrapper', type=split_args, help='wrapper to run tests with (e.g. Valgrind)') parser.add_argument('-C', default='.', dest='wd', help='directory to cd into before running') @@ -116,7 +115,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: ' more time to execute.') parser.add_argument('--setup', default=None, dest='setup', help='Which test setup to use.') - parser.add_argument('--test-args', default=[], type=shlex.split, + parser.add_argument('--test-args', default=[], type=split_args, help='Arguments to pass to the specified test(s) or all tests') parser.add_argument('args', nargs='*', help='Optional list of tests to run') diff --git a/mesonbuild/scripts/gtkdochelper.py b/mesonbuild/scripts/gtkdochelper.py index 4998e17..ddcc8c0 100644 --- a/mesonbuild/scripts/gtkdochelper.py +++ b/mesonbuild/scripts/gtkdochelper.py @@ -14,10 +14,9 @@ import sys, os import subprocess -import shlex import shutil import argparse -from ..mesonlib import MesonException, Popen_safe, is_windows +from ..mesonlib import MesonException, Popen_safe, is_windows, split_args from . import destdir_join parser = argparse.ArgumentParser() @@ -149,7 +148,7 @@ def build_gtkdoc(source_root, build_root, doc_subdir, src_subdirs, '--output-dir=' + abs_out] library_paths = [] - for ldflag in shlex.split(ldflags): + for ldflag in split_args(ldflags): if ldflag.startswith('-Wl,-rpath,'): library_paths.append(ldflag[11:]) diff --git a/mesonbuild/scripts/scanbuild.py b/mesonbuild/scripts/scanbuild.py index 51f70f0..8c0f423 100644 --- a/mesonbuild/scripts/scanbuild.py +++ b/mesonbuild/scripts/scanbuild.py @@ -13,12 +13,11 @@ # limitations under the License. import os -import shlex import subprocess import shutil import tempfile from ..environment import detect_ninja -from ..mesonlib import Popen_safe +from ..mesonlib import Popen_safe, split_args def scanbuild(exelist, srcdir, blddir, privdir, logdir, args): with tempfile.TemporaryDirectory(dir=privdir) as scandir: @@ -63,7 +62,7 @@ def run(args): break if 'SCANBUILD' in os.environ: - exelist = shlex.split(os.environ['SCANBUILD']) + exelist = split_args(os.environ['SCANBUILD']) else: exelist = [toolname] diff --git a/run_tests.py b/run_tests.py index 06a0727..85a55c3 100755 --- a/run_tests.py +++ b/run_tests.py @@ -117,7 +117,7 @@ Backend = Enum('Backend', 'ninja vs xcode') if 'MESON_EXE' in os.environ: import shlex - meson_exe = shlex.split(os.environ['MESON_EXE']) + meson_exe = mesonlib.split_args(os.environ['MESON_EXE']) else: meson_exe = None diff --git a/run_unittests.py b/run_unittests.py index dd5b434..f9ba017 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -14,7 +14,6 @@ # limitations under the License. import stat -import shlex import subprocess import re import json @@ -51,7 +50,7 @@ from mesonbuild.ast import AstInterpreter from mesonbuild.mesonlib import ( BuildDirLock, LibType, MachineChoice, PerMachine, Version, is_windows, is_osx, is_cygwin, is_dragonflybsd, is_openbsd, is_haiku, - windows_proof_rmtree, python_command, version_compare, + windows_proof_rmtree, python_command, version_compare, split_args, quote_arg ) from mesonbuild.environment import detect_ninja from mesonbuild.mesonlib import MesonException, EnvironmentException @@ -1022,6 +1021,103 @@ class InternalTests(unittest.TestCase): self.assertTrue(vctools_ver.startswith(toolset_ver), msg='{!r} does not start with {!r}'.format(vctools_ver, toolset_ver)) + def test_split_args(self): + split_args = mesonbuild.mesonlib.split_args + join_args = mesonbuild.mesonlib.join_args + if is_windows(): + test_data = [ + # examples from https://docs.microsoft.com/en-us/cpp/c-language/parsing-c-command-line-arguments + (r'"a b c" d e', ['a b c', 'd', 'e'], True), + (r'"ab\"c" "\\" d', ['ab"c', '\\', 'd'], False), + (r'a\\\b d"e f"g h', [r'a\\\b', 'de fg', 'h'], False), + (r'a\\\"b c d', [r'a\"b', 'c', 'd'], False), + (r'a\\\\"b c" d e', [r'a\\b c', 'd', 'e'], False), + # other basics + (r'""', [''], True), + (r'a b c d "" e', ['a', 'b', 'c', 'd', '', 'e'], True), + (r"'a b c' d e", ["'a", 'b', "c'", 'd', 'e'], True), + (r"'a&b&c' d e", ["'a&b&c'", 'd', 'e'], True), + (r"a & b & c d e", ['a', '&', 'b', '&', 'c', 'd', 'e'], True), + (r"'a & b & c d e'", ["'a", '&', 'b', '&', 'c', 'd', "e'"], True), + ('a b\nc\rd \n\re', ['a', 'b', 'c', 'd', 'e'], False), + # more illustrative tests + (r'cl test.cpp /O1 /Fe:test.exe', ['cl', 'test.cpp', '/O1', '/Fe:test.exe'], True), + (r'cl "test.cpp /O1 /Fe:test.exe"', ['cl', 'test.cpp /O1 /Fe:test.exe'], True), + (r'cl /DNAME=\"Bob\" test.cpp', ['cl', '/DNAME="Bob"', 'test.cpp'], False), + (r'cl "/DNAME=\"Bob\"" test.cpp', ['cl', '/DNAME="Bob"', 'test.cpp'], True), + (r'cl /DNAME=\"Bob, Alice\" test.cpp', ['cl', '/DNAME="Bob,', 'Alice"', 'test.cpp'], False), + (r'cl "/DNAME=\"Bob, Alice\"" test.cpp', ['cl', '/DNAME="Bob, Alice"', 'test.cpp'], True), + (r'cl C:\path\with\backslashes.cpp', ['cl', r'C:\path\with\backslashes.cpp'], True), + (r'cl C:\\path\\with\\double\\backslashes.cpp', ['cl', r'C:\\path\\with\\double\\backslashes.cpp'], True), + (r'cl "C:\\path\\with\\double\\backslashes.cpp"', ['cl', r'C:\\path\\with\\double\\backslashes.cpp'], False), + (r'cl C:\path with spaces\test.cpp', ['cl', r'C:\path', 'with', r'spaces\test.cpp'], False), + (r'cl "C:\path with spaces\test.cpp"', ['cl', r'C:\path with spaces\test.cpp'], True), + (r'cl /DPATH="C:\path\with\backslashes test.cpp', ['cl', r'/DPATH=C:\path\with\backslashes test.cpp'], False), + (r'cl /DPATH=\"C:\\ends\\with\\backslashes\\\" test.cpp', ['cl', r'/DPATH="C:\\ends\\with\\backslashes\"', 'test.cpp'], False), + (r'cl /DPATH="C:\\ends\\with\\backslashes\\" test.cpp', ['cl', '/DPATH=C:\\\\ends\\\\with\\\\backslashes\\', 'test.cpp'], False), + (r'cl "/DNAME=\"C:\\ends\\with\\backslashes\\\"" test.cpp', ['cl', r'/DNAME="C:\\ends\\with\\backslashes\"', 'test.cpp'], True), + (r'cl "/DNAME=\"C:\\ends\\with\\backslashes\\\\"" test.cpp', ['cl', r'/DNAME="C:\\ends\\with\\backslashes\\ test.cpp'], False), + (r'cl "/DNAME=\"C:\\ends\\with\\backslashes\\\\\"" test.cpp', ['cl', r'/DNAME="C:\\ends\\with\\backslashes\\"', 'test.cpp'], True), + ] + else: + test_data = [ + (r"'a b c' d e", ['a b c', 'd', 'e'], True), + (r"a/b/c d e", ['a/b/c', 'd', 'e'], True), + (r"a\b\c d e", [r'abc', 'd', 'e'], False), + (r"a\\b\\c d e", [r'a\b\c', 'd', 'e'], False), + (r'"a b c" d e', ['a b c', 'd', 'e'], False), + (r'"a\\b\\c\\" d e', ['a\\b\\c\\', 'd', 'e'], False), + (r"'a\b\c\' d e", ['a\\b\\c\\', 'd', 'e'], True), + (r"'a&b&c' d e", ['a&b&c', 'd', 'e'], True), + (r"a & b & c d e", ['a', '&', 'b', '&', 'c', 'd', 'e'], False), + (r"'a & b & c d e'", ['a & b & c d e'], True), + (r"abd'e f'g h", [r'abde fg', 'h'], False), + ('a b\nc\rd \n\re', ['a', 'b', 'c', 'd', 'e'], False), + + ('g++ -DNAME="Bob" test.cpp', ['g++', '-DNAME=Bob', 'test.cpp'], False), + ("g++ '-DNAME=\"Bob\"' test.cpp", ['g++', '-DNAME="Bob"', 'test.cpp'], True), + ('g++ -DNAME="Bob, Alice" test.cpp', ['g++', '-DNAME=Bob, Alice', 'test.cpp'], False), + ("g++ '-DNAME=\"Bob, Alice\"' test.cpp", ['g++', '-DNAME="Bob, Alice"', 'test.cpp'], True), + ] + + for (cmd, expected, roundtrip) in test_data: + self.assertEqual(split_args(cmd), expected) + if roundtrip: + self.assertEqual(join_args(expected), cmd) + + def test_quote_arg(self): + split_args = mesonbuild.mesonlib.split_args + quote_arg = mesonbuild.mesonlib.quote_arg + if is_windows(): + test_data = [ + ('', '""'), + ('arg1', 'arg1'), + ('/option1', '/option1'), + ('/Ovalue', '/Ovalue'), + ('/OBob&Alice', '/OBob&Alice'), + ('/Ovalue with spaces', r'"/Ovalue with spaces"'), + (r'/O"value with spaces"', r'"/O\"value with spaces\""'), + (r'/OC:\path with spaces\test.exe', r'"/OC:\path with spaces\test.exe"'), + ('/LIBPATH:C:\\path with spaces\\ends\\with\\backslashes\\', r'"/LIBPATH:C:\path with spaces\ends\with\backslashes\\"'), + ('/LIBPATH:"C:\\path with spaces\\ends\\with\\backslashes\\\\"', r'"/LIBPATH:\"C:\path with spaces\ends\with\backslashes\\\\\""'), + (r'/DMSG="Alice said: \"Let\'s go\""', r'"/DMSG=\"Alice said: \\\"Let\'s go\\\"\""'), + ] + else: + test_data = [ + ('arg1', 'arg1'), + ('--option1', '--option1'), + ('-O=value', '-O=value'), + ('-O=Bob&Alice', "'-O=Bob&Alice'"), + ('-O=value with spaces', "'-O=value with spaces'"), + ('-O="value with spaces"', '\'-O=\"value with spaces\"\''), + ('-O=/path with spaces/test', '\'-O=/path with spaces/test\''), + ('-DMSG="Alice said: \\"Let\'s go\\""', "'-DMSG=\"Alice said: \\\"Let'\"'\"'s go\\\"\"'"), + ] + + for (arg, expected) in test_data: + self.assertEqual(quote_arg(arg), expected) + self.assertEqual(split_args(expected)[0], arg) + @unittest.skipIf(is_tarball(), 'Skipping because this is a tarball release') class DataTests(unittest.TestCase): @@ -2033,7 +2129,7 @@ class AllPlatformTests(BasePlatformTests): if not execmd or not fxecmd: raise Exception('Could not find someexe and somfxe commands') # Check include order for 'someexe' - incs = [a for a in shlex.split(execmd) if a.startswith("-I")] + incs = [a for a in split_args(execmd) if a.startswith("-I")] self.assertEqual(len(incs), 9) # target private dir someexe_id = Target.construct_id_from_path("sub4", "someexe", "@exe") @@ -2055,7 +2151,7 @@ class AllPlatformTests(BasePlatformTests): # custom target include dir self.assertPathEqual(incs[8], '-Ictsub') # Check include order for 'somefxe' - incs = [a for a in shlex.split(fxecmd) if a.startswith('-I')] + incs = [a for a in split_args(fxecmd) if a.startswith('-I')] self.assertEqual(len(incs), 9) # target private dir self.assertPathEqual(incs[0], '-Isomefxe@exe') @@ -2123,7 +2219,7 @@ class AllPlatformTests(BasePlatformTests): else: raise AssertionError('Unknown compiler {!r}'.format(evalue)) # Check that we actually used the evalue correctly as the compiler - self.assertEqual(ecc.get_exelist(), shlex.split(evalue)) + self.assertEqual(ecc.get_exelist(), split_args(evalue)) # Do auto-detection of compiler based on platform, PATH, etc. cc = getattr(env, 'detect_{}_compiler'.format(lang))(MachineChoice.HOST) self.assertTrue(cc.version) @@ -2174,14 +2270,14 @@ class AllPlatformTests(BasePlatformTests): wrappercc = python_command + [wrapper] + cc.get_exelist() + ['-DSOME_ARG'] wrappercc_s = '' for w in wrappercc: - wrappercc_s += shlex.quote(w) + ' ' + wrappercc_s += quote_arg(w) + ' ' os.environ[evar] = wrappercc_s wcc = getattr(env, 'detect_{}_compiler'.format(lang))(MachineChoice.HOST) # Check static linker too wrapperlinker = python_command + [wrapper] + linker.get_exelist() + linker.get_always_args() wrapperlinker_s = '' for w in wrapperlinker: - wrapperlinker_s += shlex.quote(w) + ' ' + wrapperlinker_s += quote_arg(w) + ' ' os.environ['AR'] = wrapperlinker_s wlinker = env.detect_static_linker(wcc) # Pop it so we don't use it for the next detection @@ -2207,7 +2303,7 @@ class AllPlatformTests(BasePlatformTests): commands = {'c-asm': {}, 'cpp-asm': {}, 'cpp-c-asm': {}, 'c-cpp-asm': {}} for cmd in self.get_compdb(): # Get compiler - split = shlex.split(cmd['command']) + split = split_args(cmd['command']) if split[0] == 'ccache': compiler = split[1] else: @@ -2272,7 +2368,7 @@ class AllPlatformTests(BasePlatformTests): define = 'MESON_TEST_DEFINE_VALUE' # NOTE: this list can't have \n, ' or " # \n is never substituted by the GNU pre-processor via a -D define - # ' and " confuse shlex.split() even when they are escaped + # ' and " confuse split_args() even when they are escaped # % and # confuse the MSVC preprocessor # !, ^, *, and < confuse lcc preprocessor value = 'spaces and fun@$&()-=_+{}[]:;>?,./~`' @@ -3154,7 +3250,7 @@ recommended as it is not supported on some platforms''') self.assertEqual(obj.user_options['subp:subp_opt'].value, 'foo') self.wipe() - # c_args value should be parsed with shlex + # c_args value should be parsed with split_args self.init(testdir, extra_args=['-Dc_args=-Dfoo -Dbar "-Dthird=one two"']) obj = mesonbuild.coredata.load(self.builddir) self.assertEqual(obj.compiler_options.host['c_args'].value, ['-Dfoo', '-Dbar', '-Dthird=one two']) -- cgit v1.1