ninja: Quoting in rspfile depends on the compiler, not the shell

In certain exotic configurations, the style of quoting expected in the
response file may not match that expected by the shell.

e.g. under MSYS2, ninja invokes commands via CreateProcess (which
results in cmd-style quoting processed by parse_cmdline or
CommandLineToArgvW), but gcc will use sh-style quoting in any response
file it reads.

Future work: The rspfile quoting style should be a method of the
compiler or linker object, rather than hardcoded in ninjabackend.

(In fact, can_linker_accept_rsp() should be extended to do this, since
if we can accept rsp, we should know the quoting style)

1 files changed, 48 insertions, 17 deletions

diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index 9af079e..5743d48 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -15,6 +15,7 @@ import typing as T
 import os
 import re
 import pickle
+import shlex
 import subprocess
 from collections import OrderedDict
 from enum import Enum, unique
@@ -29,9 +30,14 @@ from .. import build
 from .. import mlog
 from .. import dependencies
 from .. import compilers
-from ..compilers import (Compiler, CompilerArgs, CCompiler, FortranCompiler,
-                         PGICCompiler, VisualStudioLikeCompiler)
-from ..linkers import ArLinker
+from ..compilers import (
+    Compiler, CompilerArgs, CCompiler,
+    DmdDCompiler,
+    FortranCompiler, PGICCompiler,
+    VisualStudioCsCompiler,
+    VisualStudioLikeCompiler,
+)
+from ..linkers import ArLinker, VisualStudioLinker
 from ..mesonlib import (
     File, LibType, MachineChoice, MesonException, OrderedSet, PerMachine,
     ProgressBar, quote_arg, unholder,
@@ -46,12 +52,15 @@ FORTRAN_MODULE_PAT = r"^\s*\bmodule\b\s+(\w+)\s*(?:!+.*)*$"
 FORTRAN_SUBMOD_PAT = r"^\s*\bsubmodule\b\s*\((\w+:?\w+)\)\s*(\w+)"
 FORTRAN_USE_PAT = r"^\s*use,?\s*(?:non_intrinsic)?\s*(?:::)?\s*(\w+)"
 
+def cmd_quote(s):
+    # XXX: this needs to understand how to escape any existing double quotes(")
+    return '"{}"'.format(s)
+
+# How ninja executes command lines differs between Unix and Windows
+# (see https://ninja-build.org/manual.html#ref_rule_command)
 if mesonlib.is_windows():
-    # FIXME: can't use quote_arg on Windows just yet; there are a number of existing workarounds
-    # throughout the codebase that cumulatively make the current code work (see, e.g. Backend.escape_extra_args
-    # and NinjaBuildElement.write below) and need to be properly untangled before attempting this
-    quote_func = lambda s: '"{}"'.format(s)
-    execute_wrapper = ['cmd', '/c']
+    quote_func = cmd_quote
+    execute_wrapper = ['cmd', '/c']  # unused
     rmfile_prefix = ['del', '/f', '/s', '/q', '{}', '&&']
 else:
     quote_func = quote_arg
@@ -115,7 +124,8 @@ class NinjaComment:
 
 class NinjaRule:
     def __init__(self, rule, command, args, description,
-                 rspable = False, deps = None, depfile = None, extra = None):
+                 rspable = False, deps = None, depfile = None, extra = None,
+                 rspfile_quote_style = 'sh'):
 
         def strToCommandArg(c):
             if isinstance(c, NinjaCommandArg):
@@ -149,20 +159,26 @@ class NinjaRule:
         self.rspable = rspable  # if a rspfile can be used
         self.refcount = 0
         self.rsprefcount = 0
+        self.rspfile_quote_style = rspfile_quote_style  # rspfile quoting style is 'sh' or 'cl'
 
     @staticmethod
-    def _quoter(x):
+    def _quoter(x, qf = quote_func):
         if isinstance(x, NinjaCommandArg):
             if x.quoting == Quoting.none:
                 return x.s
             elif x.quoting == Quoting.notNinja:
-                return quote_func(x.s)
+                return qf(x.s)
             elif x.quoting == Quoting.notShell:
                 return ninja_quote(x.s)
             # fallthrough
-        return ninja_quote(quote_func(str(x)))
+        return ninja_quote(qf(str(x)))
 
     def write(self, outfile):
+        if self.rspfile_quote_style == 'cl':
+            rspfile_quote_func = cmd_quote
+        else:
+            rspfile_quote_func = shlex.quote
+
         def rule_iter():
             if self.refcount:
                 yield ''
@@ -174,7 +190,7 @@ class NinjaRule:
             if rsp == '_RSP':
                 outfile.write(' command = {} @$out.rsp\n'.format(' '.join([self._quoter(x) for x in self.command])))
                 outfile.write(' rspfile = $out.rsp\n')
-                outfile.write(' rspfile_content = {}\n'.format(' '.join([self._quoter(x) for x in self.args])))
+                outfile.write(' rspfile_content = {}\n'.format(' '.join([self._quoter(x, rspfile_quote_func) for x in self.args])))
             else:
                 outfile.write(' command = {}\n'.format(' '.join([self._quoter(x) for x in (self.command + self.args)])))
             if self.deps:
@@ -285,7 +301,8 @@ class NinjaBuildElement:
         implicit_outs = ' '.join([ninja_quote(i, True) for i in self.implicit_outfilenames])
         if implicit_outs:
             implicit_outs = ' | ' + implicit_outs
-        if self._should_use_rspfile():
+        use_rspfile = self._should_use_rspfile()
+        if use_rspfile:
             rulename = self.rulename + '_RSP'
             mlog.log("Command line for building %s is long, using a response file" % self.outfilenames)
         else:
@@ -304,6 +321,14 @@ class NinjaBuildElement:
         line = line.replace('\\', '/')
         outfile.write(line)
 
+        if use_rspfile:
+            if self.rule.rspfile_quote_style == 'cl':
+                qf = cmd_quote
+            else:
+                qf = shlex.quote
+        else:
+            qf = quote_func
+
         for e in self.elems:
             (name, elems) = e
             should_quote = name not in raw_names
@@ -313,9 +338,9 @@ class NinjaBuildElement:
                 if not should_quote or i == '&&': # Hackety hack hack
                     quoter = ninja_quote
                 else:
-                    quoter = lambda x: ninja_quote(quote_func(x))
+                    quoter = lambda x: ninja_quote(qf(x))
                 i = i.replace('\\', '\\\\')
-                if quote_func('') == '""':
+                if qf('') == '""':
                     i = i.replace('"', '\\"')
                 newelems.append(quoter(i))
             line += ' '.join(newelems)
@@ -1694,6 +1719,7 @@ int dummy;
                 pool = None
             self.add_rule(NinjaRule(rule, cmdlist, args, description,
                                     rspable=static_linker.can_linker_accept_rsp(),
+                                    rspfile_quote_style='cl' if isinstance(static_linker, VisualStudioLinker) else 'sh',
                                     extra=pool))
 
     def generate_dynamic_link_rules(self):
@@ -1716,6 +1742,8 @@ int dummy;
                     pool = None
                 self.add_rule(NinjaRule(rule, command, args, description,
                                         rspable=compiler.can_linker_accept_rsp(),
+                                        rspfile_quote_style='cl' if (compiler.get_argument_syntax() == 'msvc' or
+                                                                     isinstance(compiler, DmdDCompiler)) else 'sh',
                                         extra=pool))
 
         args = self.environment.get_build_command() + \
@@ -1743,7 +1771,8 @@ int dummy;
         args = ['$ARGS', '$in']
         description = 'Compiling C Sharp target $out'
         self.add_rule(NinjaRule(rule, command, args, description,
-                                rspable=mesonlib.is_windows()))
+                                rspable=mesonlib.is_windows(),
+                                rspfile_quote_style='cl' if isinstance(compiler, VisualStudioCsCompiler) else 'sh'))
 
     def generate_vala_compile_rules(self, compiler):
         rule = self.compiler_to_rule_name(compiler)
@@ -1829,6 +1858,8 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
             depfile = '$DEPFILE'
         self.add_rule(NinjaRule(rule, command, args, description,
                                 rspable=compiler.can_linker_accept_rsp(),
+                                rspfile_quote_style='cl' if (compiler.get_argument_syntax() == 'msvc' or
+                                                             isinstance(compiler, DmdDCompiler)) else 'sh',
                                 deps=deps, depfile=depfile))
 
     def generate_pch_rule_for(self, langname, compiler):