Merge pull request #7245 from dankegel/response-files-when-needed-tidied

Make ninja backend only use response files when needed, on linux too

16 files changed, 444 insertions, 110 deletions

diff --git a/.travis.yml b/.travis.yml
index f5a32a6..22d76e7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,9 +31,10 @@ matrix:
     compiler: gcc
   include:
   # Test cross builds separately, they do not use the global compiler
+  # Also hijack one cross build to test long commandline handling codepath (and avoid overloading Travis)
   - os: linux
     compiler: gcc
-    env: RUN_TESTS_ARGS="--cross ubuntu-armhf.txt --cross linux-mingw-w64-64bit.txt"
+    env: RUN_TESTS_ARGS="--cross ubuntu-armhf.txt --cross linux-mingw-w64-64bit.txt" MESON_RSP_THRESHOLD=0
   - os: linux
     compiler: gcc
     env: RUN_TESTS_ARGS="--cross ubuntu-armhf.txt --cross linux-mingw-w64-64bit.txt" MESON_ARGS="--unity=on"
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 066f1a5..de956c8 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -22,6 +22,7 @@ jobs:
           arch: x86
           compiler: msvc2017
           backend: ninja
+          MESON_RSP_THRESHOLD: 0
         vc2017x64vs:
           arch: x64
           compiler: msvc2017
@@ -138,6 +139,7 @@ jobs:
       gccx64ninja:
         MSYSTEM: MINGW64
         MSYS2_ARCH: x86_64
+        MESON_RSP_THRESHOLD: 0
         compiler: gcc
       clangx64ninja:
         MSYSTEM: MINGW64
diff --git a/ci/travis_script.sh b/ci/travis_script.sh
index a91a5dd..bdfd4c2 100755
--- a/ci/travis_script.sh
+++ b/ci/travis_script.sh
@@ -23,6 +23,10 @@ export CXX=$CXX
 export OBJC=$CC
 export OBJCXX=$CXX
 export PATH=/root/tools:$PATH
+if test "$MESON_RSP_THRESHOLD" != ""
+then
+  export MESON_RSP_THRESHOLD=$MESON_RSP_THRESHOLD
+fi
 
 source /ci/env_vars.sh
 cd /root
@@ -55,5 +59,9 @@ elif [[ "$TRAVIS_OS_NAME" == "osx"   ]]; then
   export OBJC=$CC
   export OBJCXX=$CXX
   export PATH=$HOME/tools:/usr/local/opt/qt/bin:$PATH:$(brew --prefix llvm)/bin
+  if test "$MESON_RSP_THRESHOLD" != ""
+  then
+    export MESON_RSP_THRESHOLD=$MESON_RSP_THRESHOLD
+  fi
   ./run_tests.py $RUN_TESTS_ARGS --backend=ninja -- $MESON_ARGS
 fi
diff --git a/docs/markdown/snippets/response-files.md b/docs/markdown/snippets/response-files.md
new file mode 100644
index 0000000..624b664
--- /dev/null
+++ b/docs/markdown/snippets/response-files.md
@@ -0,0 +1,7 @@
+## Response files enabled on Linux, reined in on Windows
+
+Meson used to always use response files on Windows,
+but never on Linux.
+
+It now strikes a happier balance, using them on both platforms,
+but only when needed to avoid command line length limits.
diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py
index 840c9a3..3573d94 100644
--- a/mesonbuild/backend/backends.py
+++ b/mesonbuild/backend/backends.py
@@ -28,7 +28,7 @@ from .. import build
 from .. import dependencies
 from .. import mesonlib
 from .. import mlog
-from ..compilers import CompilerArgs, VisualStudioLikeCompiler
+from ..compilers import CompilerArgs
 from ..mesonlib import (
     File, MachineChoice, MesonException, OrderedSet, OptionOverrideProxy,
     classify_unity_sources, unholder
@@ -607,29 +607,13 @@ class Backend:
 
     @staticmethod
     def escape_extra_args(compiler, args):
-        # No extra escaping/quoting needed when not running on Windows
-        if not mesonlib.is_windows():
-            return args
+        # all backslashes in defines are doubly-escaped
         extra_args = []
-        # Compiler-specific escaping is needed for -D args but not for any others
-        if isinstance(compiler, VisualStudioLikeCompiler):
-            # MSVC needs escaping when a -D argument ends in \ or \"
-            for arg in args:
-                if arg.startswith('-D') or arg.startswith('/D'):
-                    # Without extra escaping for these two, the next character
-                    # gets eaten
-                    if arg.endswith('\\'):
-                        arg += '\\'
-                    elif arg.endswith('\\"'):
-                        arg = arg[:-2] + '\\\\"'
-                extra_args.append(arg)
-        else:
-            # MinGW GCC needs all backslashes in defines to be doubly-escaped
-            # FIXME: Not sure about Cygwin or Clang
-            for arg in args:
-                if arg.startswith('-D') or arg.startswith('/D'):
-                    arg = arg.replace('\\', '\\\\')
-                extra_args.append(arg)
+        for arg in args:
+            if arg.startswith('-D') or arg.startswith('/D'):
+                arg = arg.replace('\\', '\\\\')
+            extra_args.append(arg)
+
         return extra_args
 
     def generate_basic_compiler_args(self, target, compiler, no_warn_args=False):
diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index 6cf8026..cff35ee 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -15,8 +15,10 @@ import typing as T
 import os
 import re
 import pickle
+import shlex
 import subprocess
 from collections import OrderedDict
+from enum import Enum, unique
 import itertools
 from pathlib import PurePath, Path
 from functools import lru_cache
@@ -28,9 +30,14 @@ from .. import build
 from .. import mlog
 from .. import dependencies
 from .. import compilers
-from ..compilers import (Compiler, CompilerArgs, CCompiler, FortranCompiler,
-                         PGICCompiler, VisualStudioLikeCompiler)
-from ..linkers import ArLinker
+from ..compilers import (
+    Compiler, CompilerArgs, CCompiler,
+    DmdDCompiler,
+    FortranCompiler, PGICCompiler,
+    VisualStudioCsCompiler,
+    VisualStudioLikeCompiler,
+)
+from ..linkers import ArLinker, VisualStudioLinker
 from ..mesonlib import (
     File, LibType, MachineChoice, MesonException, OrderedSet, PerMachine,
     ProgressBar, quote_arg, unholder,
@@ -45,18 +52,67 @@ FORTRAN_MODULE_PAT = r"^\s*\bmodule\b\s+(\w+)\s*(?:!+.*)*$"
 FORTRAN_SUBMOD_PAT = r"^\s*\bsubmodule\b\s*\((\w+:?\w+)\)\s*(\w+)"
 FORTRAN_USE_PAT = r"^\s*use,?\s*(?:non_intrinsic)?\s*(?:::)?\s*(\w+)"
 
+def cmd_quote(s):
+    # see: https://docs.microsoft.com/en-us/windows/desktop/api/shellapi/nf-shellapi-commandlinetoargvw#remarks
+
+    # backslash escape any existing double quotes
+    # any existing backslashes preceding a quote are doubled
+    s = re.sub(r'(\\*)"', lambda m: '\\' * (len(m.group(1)) * 2 + 1) + '"', s)
+    # any terminal backslashes likewise need doubling
+    s = re.sub(r'(\\*)$', lambda m: '\\' * (len(m.group(1)) * 2), s)
+    # and double quote
+    s = '"{}"'.format(s)
+
+    return s
+
+def gcc_rsp_quote(s):
+    # see: the function buildargv() in libiberty
+    #
+    # this differs from sh-quoting in that a backslash *always* escapes the
+    # following character, even inside single quotes.
+
+    s = s.replace('\\', '\\\\')
+
+    return shlex.quote(s)
+
+# How ninja executes command lines differs between Unix and Windows
+# (see https://ninja-build.org/manual.html#ref_rule_command)
 if mesonlib.is_windows():
-    # FIXME: can't use quote_arg on Windows just yet; there are a number of existing workarounds
-    # throughout the codebase that cumulatively make the current code work (see, e.g. Backend.escape_extra_args
-    # and NinjaBuildElement.write below) and need to be properly untangled before attempting this
-    quote_func = lambda s: '"{}"'.format(s)
-    execute_wrapper = ['cmd', '/c']
+    quote_func = cmd_quote
+    execute_wrapper = ['cmd', '/c']  # unused
     rmfile_prefix = ['del', '/f', '/s', '/q', '{}', '&&']
 else:
     quote_func = quote_arg
     execute_wrapper = []
     rmfile_prefix = ['rm', '-f', '{}', '&&']
 
+def get_rsp_threshold():
+    '''Return a conservative estimate of the commandline size in bytes
+    above which a response file should be used.  May be overridden for
+    debugging by setting environment variable MESON_RSP_THRESHOLD.'''
+
+    if mesonlib.is_windows():
+        # Usually 32k, but some projects might use cmd.exe,
+        # and that has a limit of 8k.
+        limit = 8192
+    else:
+        # On Linux, ninja always passes the commandline as a single
+        # big string to /bin/sh, and the kernel limits the size of a
+        # single argument; see MAX_ARG_STRLEN
+        limit = 131072
+    # Be conservative
+    limit = limit / 2
+    return int(os.environ.get('MESON_RSP_THRESHOLD', limit))
+
+# a conservative estimate of the command-line length limit
+rsp_threshold = get_rsp_threshold()
+
+# ninja variables whose value should remain unquoted. The value of these ninja
+# variables (or variables we use them in) is interpreted directly by ninja
+# (e.g. the value of the depfile variable is a pathname that ninja will read
+# from, etc.), so it must not be shell quoted.
+raw_names = {'DEPFILE_UNQUOTED', 'DESC', 'pool', 'description', 'targetdep'}
+
 def ninja_quote(text, is_build_line=False):
     if is_build_line:
         qcs = ('$', ' ', ':')
@@ -73,6 +129,25 @@ Please report this error with a test case to the Meson bug tracker.'''.format(te
         raise MesonException(errmsg)
     return text
 
+@unique
+class Quoting(Enum):
+    both = 0
+    notShell = 1
+    notNinja = 2
+    none = 3
+
+class NinjaCommandArg:
+    def __init__(self, s, quoting = Quoting.both):
+        self.s = s
+        self.quoting = quoting
+
+    def __str__(self):
+        return self.s
+
+    @staticmethod
+    def list(l, q):
+        return [NinjaCommandArg(i, q) for i in l]
+
 class NinjaComment:
     def __init__(self, comment):
         self.comment = comment
@@ -86,49 +161,127 @@ class NinjaComment:
 
 class NinjaRule:
     def __init__(self, rule, command, args, description,
-                 rspable = False, deps = None, depfile = None, extra = None):
+                 rspable = False, deps = None, depfile = None, extra = None,
+                 rspfile_quote_style = 'gcc'):
+
+        def strToCommandArg(c):
+            if isinstance(c, NinjaCommandArg):
+                return c
+
+            # deal with common cases here, so we don't have to explicitly
+            # annotate the required quoting everywhere
+            if c == '&&':
+                # shell constructs shouldn't be shell quoted
+                return NinjaCommandArg(c, Quoting.notShell)
+            if c.startswith('$'):
+                var = re.search(r'\$\{?(\w*)\}?', c).group(1)
+                if var not in raw_names:
+                    # ninja variables shouldn't be ninja quoted, and their value
+                    # is already shell quoted
+                    return NinjaCommandArg(c, Quoting.none)
+                else:
+                    # shell quote the use of ninja variables whose value must
+                    # not be shell quoted (as it also used by ninja)
+                    return NinjaCommandArg(c, Quoting.notNinja)
+
+            return NinjaCommandArg(c)
+
         self.name = rule
-        self.command = command  # includes args which never go into a rspfile
-        self.args = args  # args which will go into a rspfile, if used
+        self.command = list(map(strToCommandArg, command))  # includes args which never go into a rspfile
+        self.args = list(map(strToCommandArg, args))  # args which will go into a rspfile, if used
         self.description = description
         self.deps = deps  # depstyle 'gcc' or 'msvc'
         self.depfile = depfile
         self.extra = extra
         self.rspable = rspable  # if a rspfile can be used
         self.refcount = 0
+        self.rsprefcount = 0
+        self.rspfile_quote_style = rspfile_quote_style  # rspfile quoting style is 'gcc' or 'cl'
 
-    def write(self, outfile):
-        if not self.refcount:
-            return
+        if self.depfile == '$DEPFILE':
+            self.depfile += '_UNQUOTED'
+
+    @staticmethod
+    def _quoter(x, qf = quote_func):
+        if isinstance(x, NinjaCommandArg):
+            if x.quoting == Quoting.none:
+                return x.s
+            elif x.quoting == Quoting.notNinja:
+                return qf(x.s)
+            elif x.quoting == Quoting.notShell:
+                return ninja_quote(x.s)
+            # fallthrough
+        return ninja_quote(qf(str(x)))
 
-        outfile.write('rule {}\n'.format(self.name))
-        if self.rspable:
-            outfile.write(' command = {} @$out.rsp\n'.format(' '.join(self.command)))
-            outfile.write(' rspfile = $out.rsp\n')
-            outfile.write(' rspfile_content = {}\n'.format(' '.join(self.args)))
+    def write(self, outfile):
+        if self.rspfile_quote_style == 'cl':
+            rspfile_quote_func = cmd_quote
         else:
-            outfile.write(' command = {}\n'.format(' '.join(self.command + self.args)))
-        if self.deps:
-            outfile.write(' deps = {}\n'.format(self.deps))
-        if self.depfile:
-            outfile.write(' depfile = {}\n'.format(self.depfile))
-        outfile.write(' description = {}\n'.format(self.description))
-        if self.extra:
-            for l in self.extra.split('\n'):
-                outfile.write(' ')
-                outfile.write(l)
-                outfile.write('\n')
-        outfile.write('\n')
+            rspfile_quote_func = gcc_rsp_quote
+
+        def rule_iter():
+            if self.refcount:
+                yield ''
+            if self.rsprefcount:
+                yield '_RSP'
+
+        for rsp in rule_iter():
+            outfile.write('rule {}{}\n'.format(self.name, rsp))
+            if rsp == '_RSP':
+                outfile.write(' command = {} @$out.rsp\n'.format(' '.join([self._quoter(x) for x in self.command])))
+                outfile.write(' rspfile = $out.rsp\n')
+                outfile.write(' rspfile_content = {}\n'.format(' '.join([self._quoter(x, rspfile_quote_func) for x in self.args])))
+            else:
+                outfile.write(' command = {}\n'.format(' '.join([self._quoter(x) for x in (self.command + self.args)])))
+            if self.deps:
+                outfile.write(' deps = {}\n'.format(self.deps))
+            if self.depfile:
+                outfile.write(' depfile = {}\n'.format(self.depfile))
+            outfile.write(' description = {}\n'.format(self.description))
+            if self.extra:
+                for l in self.extra.split('\n'):
+                    outfile.write(' ')
+                    outfile.write(l)
+                    outfile.write('\n')
+            outfile.write('\n')
+
+    def length_estimate(self, infiles, outfiles, elems):
+        # determine variables
+        # this order of actions only approximates ninja's scoping rules, as
+        # documented at: https://ninja-build.org/manual.html#ref_scope
+        ninja_vars = {}
+        for e in elems:
+            (name, value) = e
+            ninja_vars[name] = value
+        ninja_vars['deps'] = self.deps
+        ninja_vars['depfile'] = self.depfile
+        ninja_vars['in'] = infiles
+        ninja_vars['out'] = outfiles
+
+        # expand variables in command
+        command = ' '.join([self._quoter(x) for x in self.command + self.args])
+        expanded_command = ''
+        for m in re.finditer(r'(\${\w*})|(\$\w*)|([^$]*)', command):
+            chunk = m.group()
+            if chunk.startswith('$'):
+                chunk = chunk[1:]
+                chunk = re.sub(r'{(.*)}', r'\1', chunk)
+                chunk = ninja_vars.get(chunk, [])  # undefined ninja variables are empty
+                chunk = ' '.join(chunk)
+            expanded_command += chunk
+
+        # determine command length
+        return len(expanded_command)
 
 class NinjaBuildElement:
-    def __init__(self, all_outputs, outfilenames, rule, infilenames, implicit_outs=None):
+    def __init__(self, all_outputs, outfilenames, rulename, infilenames, implicit_outs=None):
         self.implicit_outfilenames = implicit_outs or []
         if isinstance(outfilenames, str):
             self.outfilenames = [outfilenames]
         else:
             self.outfilenames = outfilenames
-        assert(isinstance(rule, str))
-        self.rule = rule
+        assert(isinstance(rulename, str))
+        self.rulename = rulename
         if isinstance(infilenames, str):
             self.infilenames = [infilenames]
         else:
@@ -159,6 +312,31 @@ class NinjaBuildElement:
             elems = [elems]
         self.elems.append((name, elems))
 
+        if name == 'DEPFILE':
+            self.elems.append((name + '_UNQUOTED', elems))
+
+    def _should_use_rspfile(self):
+        # 'phony' is a rule built-in to ninja
+        if self.rulename == 'phony':
+            return False
+
+        if not self.rule.rspable:
+            return False
+
+        infilenames = ' '.join([ninja_quote(i, True) for i in self.infilenames])
+        outfilenames = ' '.join([ninja_quote(i, True) for i in self.outfilenames])
+
+        return self.rule.length_estimate(infilenames,
+                                         outfilenames,
+                                         self.elems) >= rsp_threshold
+
+    def count_rule_references(self):
+        if self.rulename != 'phony':
+            if self._should_use_rspfile():
+                self.rule.rsprefcount += 1
+            else:
+                self.rule.refcount += 1
+
     def write(self, outfile):
         self.check_outputs()
         ins = ' '.join([ninja_quote(i, True) for i in self.infilenames])
@@ -166,7 +344,13 @@ class NinjaBuildElement:
         implicit_outs = ' '.join([ninja_quote(i, True) for i in self.implicit_outfilenames])
         if implicit_outs:
             implicit_outs = ' | ' + implicit_outs
-        line = 'build {}{}: {} {}'.format(outs, implicit_outs, self.rule, ins)
+        use_rspfile = self._should_use_rspfile()
+        if use_rspfile:
+            rulename = self.rulename + '_RSP'
+            mlog.log("Command line for building %s is long, using a response file" % self.outfilenames)
+        else:
+            rulename = self.rulename
+        line = 'build {}{}: {} {}'.format(outs, implicit_outs, rulename, ins)
         if len(self.deps) > 0:
             line += ' | ' + ' '.join([ninja_quote(x, True) for x in self.deps])
         if len(self.orderdeps) > 0:
@@ -180,11 +364,13 @@ class NinjaBuildElement:
         line = line.replace('\\', '/')
         outfile.write(line)
 
-        # ninja variables whose value should remain unquoted. The value of these
-        # ninja variables (or variables we use them in) is interpreted directly
-        # by ninja (e.g. the value of the depfile variable is a pathname that
-        # ninja will read from, etc.), so it must not be shell quoted.
-        raw_names = {'DEPFILE', 'DESC', 'pool', 'description', 'targetdep'}
+        if use_rspfile:
+            if self.rule.rspfile_quote_style == 'cl':
+                qf = cmd_quote
+            else:
+                qf = gcc_rsp_quote
+        else:
+            qf = quote_func
 
         for e in self.elems:
             (name, elems) = e
@@ -195,10 +381,7 @@ class NinjaBuildElement:
                 if not should_quote or i == '&&': # Hackety hack hack
                     quoter = ninja_quote
                 else:
-                    quoter = lambda x: ninja_quote(quote_func(x))
-                i = i.replace('\\', '\\\\')
-                if quote_func('') == '""':
-                    i = i.replace('"', '\\"')
+                    quoter = lambda x: ninja_quote(qf(x))
                 newelems.append(quoter(i))
             line += ' '.join(newelems)
             line += '\n'
@@ -350,10 +533,14 @@ int dummy;
     # http://clang.llvm.org/docs/JSONCompilationDatabase.html
     def generate_compdb(self):
         rules = []
+        # TODO: Rather than an explicit list here, rules could be marked in the
+        # rule store as being wanted in compdb
         for for_machine in MachineChoice:
             for lang in self.environment.coredata.compilers[for_machine]:
-                rules += [self.get_compiler_rule_name(lang, for_machine)]
-                rules += [self.get_pch_rule_name(lang, for_machine)]
+                rules += [ "%s%s" % (rule, ext) for rule in [self.get_compiler_rule_name(lang, for_machine)]
+                                                for ext in ['', '_RSP']]
+                rules += [ "%s%s" % (rule, ext) for rule in [self.get_pch_rule_name(lang, for_machine)]
+                                                for ext in ['', '_RSP']]
         compdb_options = ['-x'] if mesonlib.version_compare(self.ninja_version, '>=1.9') else []
         ninja_compdb = [self.ninja_command, '-t', 'compdb'] + compdb_options + rules
         builddir = self.environment.get_build_dir()
@@ -877,13 +1064,15 @@ int dummy;
                                 deps='gcc', depfile='$DEPFILE',
                                 extra='restat = 1'))
 
-        c = [ninja_quote(quote_func(x)) for x in self.environment.get_build_command()] + \
+        c = self.environment.get_build_command() + \
             ['--internal',
              'regenerate',
-             ninja_quote(quote_func(self.environment.get_source_dir())),
-             ninja_quote(quote_func(self.environment.get_build_dir()))]
+             self.environment.get_source_dir(),
+             self.environment.get_build_dir(),
+             '--backend',
+             'ninja']
         self.add_rule(NinjaRule('REGENERATE_BUILD',
-                                c + ['--backend', 'ninja'], [],
+                                c, [],
                                 'Regenerating build files.',
                                 extra='generator = 1'))
 
@@ -900,11 +1089,15 @@ int dummy;
     def add_build(self, build):
         self.build_elements.append(build)
 
-        # increment rule refcount
-        if build.rule != 'phony':
-            self.ruledict[build.rule].refcount += 1
+        if build.rulename != 'phony':
+            # reference rule
+            build.rule = self.ruledict[build.rulename]
 
     def write_rules(self, outfile):
+        for b in self.build_elements:
+            if isinstance(b, NinjaBuildElement):
+                b.count_rule_references()
+
         for r in self.rules:
             r.write(outfile)
 
@@ -1558,7 +1751,7 @@ int dummy;
                 cmdlist = execute_wrapper + [c.format('$out') for c in rmfile_prefix]
             cmdlist += static_linker.get_exelist()
             cmdlist += ['$LINK_ARGS']
-            cmdlist += static_linker.get_output_args('$out')
+            cmdlist += NinjaCommandArg.list(static_linker.get_output_args('$out'), Quoting.none)
             description = 'Linking static target $out'
             if num_pools > 0:
                 pool = 'pool = link_pool'
@@ -1566,6 +1759,7 @@ int dummy;
                 pool = None
             self.add_rule(NinjaRule(rule, cmdlist, args, description,
                                     rspable=static_linker.can_linker_accept_rsp(),
+                                    rspfile_quote_style='cl' if isinstance(static_linker, VisualStudioLinker) else 'gcc',
                                     extra=pool))
 
     def generate_dynamic_link_rules(self):
@@ -1580,7 +1774,7 @@ int dummy;
                     continue
                 rule = '{}_LINKER{}'.format(langname, self.get_rule_suffix(for_machine))
                 command = compiler.get_linker_exelist()
-                args = ['$ARGS'] + compiler.get_linker_output_args('$out') + ['$in', '$LINK_ARGS']
+                args = ['$ARGS'] + NinjaCommandArg.list(compiler.get_linker_output_args('$out'), Quoting.none) + ['$in', '$LINK_ARGS']
                 description = 'Linking target $out'
                 if num_pools > 0:
                     pool = 'pool = link_pool'
@@ -1588,12 +1782,14 @@ int dummy;
                     pool = None
                 self.add_rule(NinjaRule(rule, command, args, description,
                                         rspable=compiler.can_linker_accept_rsp(),
+                                        rspfile_quote_style='cl' if (compiler.get_argument_syntax() == 'msvc' or
+                                                                     isinstance(compiler, DmdDCompiler)) else 'gcc',
                                         extra=pool))
 
-        args = [ninja_quote(quote_func(x)) for x in self.environment.get_build_command()] + \
+        args = self.environment.get_build_command() + \
             ['--internal',
              'symbolextractor',
-             ninja_quote(quote_func(self.environment.get_build_dir())),
+             self.environment.get_build_dir(),
              '$in',
              '$IMPLIB',
              '$out']
@@ -1605,31 +1801,28 @@ int dummy;
 
     def generate_java_compile_rule(self, compiler):
         rule = self.compiler_to_rule_name(compiler)
-        invoc = [ninja_quote(i) for i in compiler.get_exelist()]
-        command = invoc + ['$ARGS', '$in']
+        command = compiler.get_exelist() + ['$ARGS', '$in']
         description = 'Compiling Java object $in'
         self.add_rule(NinjaRule(rule, command, [], description))
 
     def generate_cs_compile_rule(self, compiler):
         rule = self.compiler_to_rule_name(compiler)
-        invoc = [ninja_quote(i) for i in compiler.get_exelist()]
-        command = invoc
+        command = compiler.get_exelist()
         args = ['$ARGS', '$in']
         description = 'Compiling C Sharp target $out'
         self.add_rule(NinjaRule(rule, command, args, description,
-                                rspable=mesonlib.is_windows()))
+                                rspable=mesonlib.is_windows(),
+                                rspfile_quote_style='cl' if isinstance(compiler, VisualStudioCsCompiler) else 'gcc'))
 
     def generate_vala_compile_rules(self, compiler):
         rule = self.compiler_to_rule_name(compiler)
-        invoc = [ninja_quote(i) for i in compiler.get_exelist()]
-        command = invoc + ['$ARGS', '$in']
+        command = compiler.get_exelist() + ['$ARGS', '$in']
         description = 'Compiling Vala source $in'
         self.add_rule(NinjaRule(rule, command, [], description, extra='restat = 1'))
 
     def generate_rust_compile_rules(self, compiler):
         rule = self.compiler_to_rule_name(compiler)
-        invoc = [ninja_quote(i) for i in compiler.get_exelist()]
-        command = invoc + ['$ARGS', '$in']
+        command = compiler.get_exelist() + ['$ARGS', '$in']
         description = 'Compiling Rust source $in'
         depfile = '$targetdep'
         depstyle = 'gcc'
@@ -1638,12 +1831,12 @@ int dummy;
 
     def generate_swift_compile_rules(self, compiler):
         rule = self.compiler_to_rule_name(compiler)
-        full_exe = [ninja_quote(x) for x in self.environment.get_build_command()] + [
+        full_exe = self.environment.get_build_command() + [
             '--internal',
             'dirchanger',
             '$RUNDIR',
         ]
-        invoc = full_exe + [ninja_quote(i) for i in compiler.get_exelist()]
+        invoc = full_exe + compiler.get_exelist()
         command = invoc + ['$ARGS', '$in']
         description = 'Compiling Swift source $in'
         self.add_rule(NinjaRule(rule, command, [], description))
@@ -1663,8 +1856,8 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
         if self.created_llvm_ir_rule[compiler.for_machine]:
             return
         rule = self.get_compiler_rule_name('llvm_ir', compiler.for_machine)
-        command = [ninja_quote(i) for i in compiler.get_exelist()]
-        args = ['$ARGS'] + compiler.get_output_args('$out') + compiler.get_compile_only_args() + ['$in']
+        command = compiler.get_exelist()
+        args = ['$ARGS'] + NinjaCommandArg.list(compiler.get_output_args('$out'), Quoting.none) + compiler.get_compile_only_args() + ['$in']
         description = 'Compiling LLVM IR object $in'
         self.add_rule(NinjaRule(rule, command, args, description,
                                 rspable=compiler.can_linker_accept_rsp()))
@@ -1693,15 +1886,9 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
         if langname == 'fortran':
             self.generate_fortran_dep_hack(crstr)
         rule = self.get_compiler_rule_name(langname, compiler.for_machine)
-        depargs = compiler.get_dependency_gen_args('$out', '$DEPFILE')
-        quoted_depargs = []
-        for d in depargs:
-            if d != '$out' and d != '$in':
-                d = quote_func(d)
-            quoted_depargs.append(d)
-
-        command = [ninja_quote(i) for i in compiler.get_exelist()]
-        args = ['$ARGS'] + quoted_depargs + compiler.get_output_args('$out') + compiler.get_compile_only_args() + ['$in']
+        depargs = NinjaCommandArg.list(compiler.get_dependency_gen_args('$out', '$DEPFILE'), Quoting.none)
+        command = compiler.get_exelist()
+        args = ['$ARGS'] + depargs + NinjaCommandArg.list(compiler.get_output_args('$out'), Quoting.none) + compiler.get_compile_only_args() + ['$in']
         description = 'Compiling {} object $out'.format(compiler.get_display_language())
         if isinstance(compiler, VisualStudioLikeCompiler):
             deps = 'msvc'
@@ -1711,6 +1898,8 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
             depfile = '$DEPFILE'
         self.add_rule(NinjaRule(rule, command, args, description,
                                 rspable=compiler.can_linker_accept_rsp(),
+                                rspfile_quote_style='cl' if (compiler.get_argument_syntax() == 'msvc' or
+                                                             isinstance(compiler, DmdDCompiler)) else 'gcc',
                                 deps=deps, depfile=depfile))
 
     def generate_pch_rule_for(self, langname, compiler):
@@ -1719,16 +1908,11 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
         rule = self.compiler_to_pch_rule_name(compiler)
         depargs = compiler.get_dependency_gen_args('$out', '$DEPFILE')
 
-        quoted_depargs = []
-        for d in depargs:
-            if d != '$out' and d != '$in':
-                d = quote_func(d)
-            quoted_depargs.append(d)
         if isinstance(compiler, VisualStudioLikeCompiler):
             output = []
         else:
-            output = compiler.get_output_args('$out')
-        command = compiler.get_exelist() + ['$ARGS'] + quoted_depargs + output + compiler.get_compile_only_args() + ['$in']
+            output = NinjaCommandArg.list(compiler.get_output_args('$out'), Quoting.none)
+        command = compiler.get_exelist() + ['$ARGS'] + depargs + output + compiler.get_compile_only_args() + ['$in']
         description = 'Precompiling header $in'
         if isinstance(compiler, VisualStudioLikeCompiler):
             deps = 'msvc'
diff --git a/mesonbuild/linkers.py b/mesonbuild/linkers.py
index f02c297..25a8c9c 100644
--- a/mesonbuild/linkers.py
+++ b/mesonbuild/linkers.py
@@ -149,6 +149,10 @@ class ArLinker(StaticLinker):
             self.std_args = ['csrD']
         else:
             self.std_args = ['csr']
+        self.can_rsp = '@<' in stdo
+
+    def can_linker_accept_rsp(self) -> bool:
+        return self.can_rsp
 
     def get_std_link_args(self) -> T.List[str]:
         return self.std_args
@@ -704,6 +708,9 @@ class GnuDynamicLinker(GnuLikeDynamicLinkerMixin, PosixDynamicLinkerMixin, Dynam
 
     """Representation of GNU ld.bfd and ld.gold."""
 
+    def get_accepts_rsp(self) -> bool:
+        return True;
+
 
 class GnuGoldDynamicLinker(GnuDynamicLinker):
 
diff --git a/run_unittests.py b/run_unittests.py
index 7e0c403..a6817c3 100755
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -2432,6 +2432,9 @@ class AllPlatformTests(BasePlatformTests):
         self.assertPathExists(exe2)
 
     def test_internal_include_order(self):
+        if mesonbuild.environment.detect_msys2_arch() and ('MESON_RSP_THRESHOLD' in os.environ):
+            raise unittest.SkipTest('Test does not yet support gcc rsp files on msys2')
+
         testdir = os.path.join(self.common_test_dir, '134 include order')
         self.init(testdir)
         execmd = fxecmd = None
diff --git a/test cases/common/145 special characters/arg-char-test.c b/test cases/common/145 special characters/arg-char-test.c
new file mode 100644
index 0000000..04e02f8
--- /dev/null
+++ b/test cases/common/145 special characters/arg-char-test.c
@@ -0,0 +1,10 @@
+#include <assert.h>
+#include <stdio.h>
+
+int main(int argc, char **argv) {
+  char c = CHAR;
+  assert(argc == 2);
+  if (c != argv[1][0])
+    fprintf(stderr, "Expected %x, got %x\n", (unsigned int) c, (unsigned int) argv[1][0]);
+  assert(c == argv[1][0]);
+}
diff --git a/test cases/common/145 special characters/arg-string-test.c b/test cases/common/145 special characters/arg-string-test.c
new file mode 100644
index 0000000..199fd79
--- /dev/null
+++ b/test cases/common/145 special characters/arg-string-test.c
@@ -0,0 +1,12 @@
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+int main(int argc, char **argv) {
+  const char *s = CHAR;
+  assert(argc == 2);
+  assert(strlen(s) == 1);
+  if (s[0] != argv[1][0])
+    fprintf(stderr, "Expected %x, got %x\n", (unsigned int) s[0], (unsigned int) argv[1][0]);
+  assert(s[0] == argv[1][0]);
+}
diff --git a/test cases/common/145 special characters/arg-unquoted-test.c b/test cases/common/145 special characters/arg-unquoted-test.c
new file mode 100644
index 0000000..7f679ca
--- /dev/null
+++ b/test cases/common/145 special characters/arg-unquoted-test.c
@@ -0,0 +1,17 @@
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+#define Q(x) #x
+#define QUOTE(x) Q(x)
+
+int main(int argc, char **argv) {
+  const char *s = QUOTE(CHAR);
+  assert(argc == 2);
+  assert(strlen(s) == 1);
+  if (s[0] != argv[1][0])
+    fprintf(stderr, "Expected %x, got %x\n", (unsigned int) s[0], (unsigned int) argv[1][0]);
+  assert(s[0] == argv[1][0]);
+  // There is no way to convert a macro argument into a character constant.
+  // Otherwise we'd test that as well
+}
diff --git a/test cases/common/145 special characters/meson.build b/test cases/common/145 special characters/meson.build
index ecba650..579601e 100644
--- a/test cases/common/145 special characters/meson.build
+++ b/test cases/common/145 special characters/meson.build
@@ -35,3 +35,41 @@ gen2 = custom_target('gen2',
   output : 'result2',
   install : true,
   install_dir : get_option('datadir'))
+
+# Test that we can pass these special characters in compiler arguments
+#
+# (this part of the test is crafted so we don't try to use these special
+# characters in filenames or target names)
+#
+# TODO: similar tests needed for languages other than C
+# TODO: add similar test for quote, doublequote, and hash, carefully
+# Re hash, see
+# https://docs.microsoft.com/en-us/cpp/build/reference/d-preprocessor-definitions
+
+special = [
+  ['amp', '&'],
+  ['at', '@'],
+  ['backslash', '\\'],
+  ['dollar', '$'],
+  ['gt', '>'],
+  ['lt', '<'],
+  ['slash', '/'],
+]
+
+cc = meson.get_compiler('c')
+
+foreach s : special
+  args = '-DCHAR="@0@"'.format(s[1])
+  e = executable('arg-string-' + s[0], 'arg-string-test.c', c_args: args)
+  test('arg-string-' + s[0], e, args: s[1])
+
+  args = '-DCHAR=@0@'.format(s[1])
+  e = executable('arg-unquoted-' + s[0], 'arg-unquoted-test.c', c_args: args)
+  test('arg-unquoted-' + s[0], e, args: s[1])
+endforeach
+
+foreach s : special
+  args = '-DCHAR=\'@0@\''.format(s[1])
+  e = executable('arg-char-' + s[0], 'arg-char-test.c', c_args: args)
+  test('arg-char-' + s[0], e, args: s[1])
+endforeach
diff --git a/test cases/common/234 very long commmand line/codegen.py b/test cases/common/234 very long commmand line/codegen.py
new file mode 100755
index 0000000..4de78ce
--- /dev/null
+++ b/test cases/common/234 very long commmand line/codegen.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+import sys
+
+with open(sys.argv[2], 'w') as f:
+    print('int func{n}(void) {{ return {n}; }}'.format(n=sys.argv[1]), file=f)
diff --git a/test cases/common/234 very long commmand line/main.c b/test cases/common/234 very long commmand line/main.c
new file mode 100644
index 0000000..dbb64a8
--- /dev/null
+++ b/test cases/common/234 very long commmand line/main.c
@@ -0,0 +1,5 @@
+int main(int argc, char **argv) {
+  (void) argc;
+  (void) argv;
+  return 0;
+}
diff --git a/test cases/common/234 very long commmand line/meson.build b/test cases/common/234 very long commmand line/meson.build
new file mode 100644
index 0000000..fe47b5e
--- /dev/null
+++ b/test cases/common/234 very long commmand line/meson.build
@@ -0,0 +1,44 @@
+project('very long command lines', 'c')
+
+# Get the current system's commandline length limit.
+if build_machine.system() == 'windows'
+  # Various limits on windows:
+  # cmd.exe: 8kb
+  # CreateProcess: 32kb
+  limit = 32767
+elif build_machine.system() == 'cygwin'
+  # cygwin-to-win32: see above
+  # cygwin-to-cygwin: no limit?
+  # Cygwin is slow, so only test it lightly here.
+  limit = 8192
+else
+  # ninja passes whole line as a single argument, for which
+  # the limit is 128k as of Linux 2.6.23.  See MAX_ARG_STRLEN.
+  # BSD seems similar, see https://www.in-ulm.de/~mascheck/various/argmax
+  limit = 131072
+endif
+# Now exceed that limit, but not so far that the test takes too long.
+name = 'ALongFilenameMuchLongerThanIsNormallySeenAndReallyHardToReadThroughToTheEndAMooseOnceBitMySisterSheNowWorksAtLLamaFreshFarmsThisHasToBeSoLongThatWeExceed128KBWithoutCompilingTooManyFiles'
+namelen = 187
+nfiles = 50 + limit / namelen
+message('Expected link commandline length is approximately ' + '@0@'.format((nfiles * (namelen+28))))
+
+seq = run_command('seq.py', '1', '@0@'.format(nfiles)).stdout().strip().split('\n')
+
+sources = []
+codegen = find_program('codegen.py')
+
+foreach i : seq
+  sources += custom_target('codegen' + i,
+                           command: [codegen, i, '@OUTPUT@'],
+                           output: name + i + '.c')
+endforeach
+
+shared_library('sharedlib', sources)
+static_library('staticlib', sources)
+executable('app', 'main.c', sources)
+
+# Also test short commandlines to make sure that doesn't regress
+shared_library('sharedlib0', sources[0])
+static_library('staticlib0', sources[0])
+executable('app0', 'main.c', sources[0])
diff --git a/test cases/common/234 very long commmand line/seq.py b/test cases/common/234 very long commmand line/seq.py
new file mode 100755
index 0000000..637bf57
--- /dev/null
+++ b/test cases/common/234 very long commmand line/seq.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+import sys
+
+for i in range(int(sys.argv[1]), int(sys.argv[2])):
+    print(i)