ninja: Push ninja and shell quoting down into NinjaRule

Rather than ad-hoc avoiding quoting where harmful, identify arguments
which contain shell constructs and ninja variables, and don't apply
quoting to those arguments.

This is made more complex by some arguments which might contain ninja
variables anywhere, not just at start, e.g. '/Fo$out'

(This implementation would fall down if there was an argument which
contained both a literal $ or shell metacharacter and a ninja variable,
but there are no instances of such a thing and it seems unlikely)

$DEPFILE needs special treatment.  It's used in the special variable
depfile, so it's value can't be shell quoted (as it used as a filename
to read by ninja).  So instead that variable needs to be shell quoted
when it appears in a command.

(Test common/129, which uses a depfile with a space in it's name,
exercises that)

If 'targetdep' is not in raw_names, test cases/rust all fail.

1 files changed, 91 insertions, 50 deletions

diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index ef3fc45..9af079e 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -17,6 +17,7 @@ import re
 import pickle
 import subprocess
 from collections import OrderedDict
+from enum import Enum, unique
 import itertools
 from pathlib import PurePath, Path
 from functools import lru_cache
@@ -60,6 +61,12 @@ else:
 # a conservative estimate of the command-line length limit on windows
 rsp_threshold = 4096
 
+# ninja variables whose value should remain unquoted. The value of these ninja
+# variables (or variables we use them in) is interpreted directly by ninja
+# (e.g. the value of the depfile variable is a pathname that ninja will read
+# from, etc.), so it must not be shell quoted.
+raw_names = {'DEPFILE', 'DESC', 'pool', 'description', 'targetdep'}
+
 def ninja_quote(text, is_build_line=False):
     if is_build_line:
         qcs = ('$', ' ', ':')
@@ -76,6 +83,25 @@ Please report this error with a test case to the Meson bug tracker.'''.format(te
         raise MesonException(errmsg)
     return text
 
+@unique
+class Quoting(Enum):
+    both = 0
+    notShell = 1
+    notNinja = 2
+    none = 3
+
+class NinjaCommandArg:
+    def __init__(self, s, quoting = Quoting.both):
+        self.s = s
+        self.quoting = quoting
+
+    def __str__(self):
+        return self.s
+
+    @staticmethod
+    def list(l, q):
+        return [NinjaCommandArg(i, q) for i in l]
+
 class NinjaComment:
     def __init__(self, comment):
         self.comment = comment
@@ -90,9 +116,32 @@ class NinjaComment:
 class NinjaRule:
     def __init__(self, rule, command, args, description,
                  rspable = False, deps = None, depfile = None, extra = None):
+
+        def strToCommandArg(c):
+            if isinstance(c, NinjaCommandArg):
+                return c
+
+            # deal with common cases here, so we don't have to explicitly
+            # annotate the required quoting everywhere
+            if c == '&&':
+                # shell constructs shouldn't be shell quoted
+                return NinjaCommandArg(c, Quoting.notShell)
+            if c.startswith('$'):
+                var = re.search(r'\$\{?(\w*)\}?', c).group(1)
+                if var not in raw_names:
+                    # ninja variables shouldn't be ninja quoted, and their value
+                    # is already shell quoted
+                    return NinjaCommandArg(c, Quoting.none)
+                else:
+                    # shell quote the use of ninja variables whose value must
+                    # not be shell quoted (as it also used by ninja)
+                    return NinjaCommandArg(c, Quoting.notNinja)
+
+            return NinjaCommandArg(c)
+
         self.name = rule
-        self.command = command  # includes args which never go into a rspfile
-        self.args = args  # args which will go into a rspfile, if used
+        self.command = list(map(strToCommandArg, command))  # includes args which never go into a rspfile
+        self.args = list(map(strToCommandArg, args))  # args which will go into a rspfile, if used
         self.description = description
         self.deps = deps  # depstyle 'gcc' or 'msvc'
         self.depfile = depfile
@@ -101,6 +150,18 @@ class NinjaRule:
         self.refcount = 0
         self.rsprefcount = 0
 
+    @staticmethod
+    def _quoter(x):
+        if isinstance(x, NinjaCommandArg):
+            if x.quoting == Quoting.none:
+                return x.s
+            elif x.quoting == Quoting.notNinja:
+                return quote_func(x.s)
+            elif x.quoting == Quoting.notShell:
+                return ninja_quote(x.s)
+            # fallthrough
+        return ninja_quote(quote_func(str(x)))
+
     def write(self, outfile):
         def rule_iter():
             if self.refcount:
@@ -111,11 +172,11 @@ class NinjaRule:
         for rsp in rule_iter():
             outfile.write('rule {}{}\n'.format(self.name, rsp))
             if rsp == '_RSP':
-                outfile.write(' command = {} @$out.rsp\n'.format(' '.join(self.command)))
+                outfile.write(' command = {} @$out.rsp\n'.format(' '.join([self._quoter(x) for x in self.command])))
                 outfile.write(' rspfile = $out.rsp\n')
-                outfile.write(' rspfile_content = {}\n'.format(' '.join(self.args)))
+                outfile.write(' rspfile_content = {}\n'.format(' '.join([self._quoter(x) for x in self.args])))
             else:
-                outfile.write(' command = {}\n'.format(' '.join(self.command + self.args)))
+                outfile.write(' command = {}\n'.format(' '.join([self._quoter(x) for x in (self.command + self.args)])))
             if self.deps:
                 outfile.write(' deps = {}\n'.format(self.deps))
             if self.depfile:
@@ -141,9 +202,8 @@ class NinjaRule:
         ninja_vars['in'] = infiles
         ninja_vars['out'] = outfiles
 
-        # expand variables in command (XXX: this ignores any escaping/quoting
-        # that NinjaBuildElement.write() might do)
-        command = ' '.join(self.command + self.args)
+        # expand variables in command
+        command = ' '.join([self._quoter(x) for x in self.command + self.args])
         expanded_command = ''
         for m in re.finditer(r'(\${\w*})|(\$\w*)|([^$]*)', command):
             chunk = m.group()
@@ -244,12 +304,6 @@ class NinjaBuildElement:
         line = line.replace('\\', '/')
         outfile.write(line)
 
-        # ninja variables whose value should remain unquoted. The value of these
-        # ninja variables (or variables we use them in) is interpreted directly
-        # by ninja (e.g. the value of the depfile variable is a pathname that
-        # ninja will read from, etc.), so it must not be shell quoted.
-        raw_names = {'DEPFILE', 'DESC', 'pool', 'description', 'targetdep'}
-
         for e in self.elems:
             (name, elems) = e
             should_quote = name not in raw_names
@@ -945,13 +999,15 @@ int dummy;
                                 deps='gcc', depfile='$DEPFILE',
                                 extra='restat = 1'))
 
-        c = [ninja_quote(quote_func(x)) for x in self.environment.get_build_command()] + \
+        c = self.environment.get_build_command() + \
             ['--internal',
              'regenerate',
-             ninja_quote(quote_func(self.environment.get_source_dir())),
-             ninja_quote(quote_func(self.environment.get_build_dir()))]
+             self.environment.get_source_dir(),
+             self.environment.get_build_dir(),
+             '--backend',
+             'ninja']
         self.add_rule(NinjaRule('REGENERATE_BUILD',
-                                c + ['--backend', 'ninja'], [],
+                                c, [],
                                 'Regenerating build files.',
                                 extra='generator = 1'))
 
@@ -1630,7 +1686,7 @@ int dummy;
                 cmdlist = execute_wrapper + [c.format('$out') for c in rmfile_prefix]
             cmdlist += static_linker.get_exelist()
             cmdlist += ['$LINK_ARGS']
-            cmdlist += static_linker.get_output_args('$out')
+            cmdlist += NinjaCommandArg.list(static_linker.get_output_args('$out'), Quoting.none)
             description = 'Linking static target $out'
             if num_pools > 0:
                 pool = 'pool = link_pool'
@@ -1652,7 +1708,7 @@ int dummy;
                     continue
                 rule = '{}_LINKER{}'.format(langname, self.get_rule_suffix(for_machine))
                 command = compiler.get_linker_exelist()
-                args = ['$ARGS'] + compiler.get_linker_output_args('$out') + ['$in', '$LINK_ARGS']
+                args = ['$ARGS'] + NinjaCommandArg.list(compiler.get_linker_output_args('$out'), Quoting.none) + ['$in', '$LINK_ARGS']
                 description = 'Linking target $out'
                 if num_pools > 0:
                     pool = 'pool = link_pool'
@@ -1662,10 +1718,10 @@ int dummy;
                                         rspable=compiler.can_linker_accept_rsp(),
                                         extra=pool))
 
-        args = [ninja_quote(quote_func(x)) for x in self.environment.get_build_command()] + \
+        args = self.environment.get_build_command() + \
             ['--internal',
              'symbolextractor',
-             ninja_quote(quote_func(self.environment.get_build_dir())),
+             self.environment.get_build_dir(),
              '$in',
              '$IMPLIB',
              '$out']
@@ -1677,15 +1733,13 @@ int dummy;
 
     def generate_java_compile_rule(self, compiler):
         rule = self.compiler_to_rule_name(compiler)
-        invoc = [ninja_quote(i) for i in compiler.get_exelist()]
-        command = invoc + ['$ARGS', '$in']
+        command = compiler.get_exelist() + ['$ARGS', '$in']
         description = 'Compiling Java object $in'
         self.add_rule(NinjaRule(rule, command, [], description))
 
     def generate_cs_compile_rule(self, compiler):
         rule = self.compiler_to_rule_name(compiler)
-        invoc = [ninja_quote(i) for i in compiler.get_exelist()]
-        command = invoc
+        command = compiler.get_exelist()
         args = ['$ARGS', '$in']
         description = 'Compiling C Sharp target $out'
         self.add_rule(NinjaRule(rule, command, args, description,
@@ -1693,15 +1747,13 @@ int dummy;
 
     def generate_vala_compile_rules(self, compiler):
         rule = self.compiler_to_rule_name(compiler)
-        invoc = [ninja_quote(i) for i in compiler.get_exelist()]
-        command = invoc + ['$ARGS', '$in']
+        command = compiler.get_exelist() + ['$ARGS', '$in']
         description = 'Compiling Vala source $in'
         self.add_rule(NinjaRule(rule, command, [], description, extra='restat = 1'))
 
     def generate_rust_compile_rules(self, compiler):
         rule = self.compiler_to_rule_name(compiler)
-        invoc = [ninja_quote(i) for i in compiler.get_exelist()]
-        command = invoc + ['$ARGS', '$in']
+        command = compiler.get_exelist() + ['$ARGS', '$in']
         description = 'Compiling Rust source $in'
         depfile = '$targetdep'
         depstyle = 'gcc'
@@ -1710,12 +1762,12 @@ int dummy;
 
     def generate_swift_compile_rules(self, compiler):
         rule = self.compiler_to_rule_name(compiler)
-        full_exe = [ninja_quote(x) for x in self.environment.get_build_command()] + [
+        full_exe = self.environment.get_build_command() + [
             '--internal',
             'dirchanger',
             '$RUNDIR',
         ]
-        invoc = full_exe + [ninja_quote(i) for i in compiler.get_exelist()]
+        invoc = full_exe + compiler.get_exelist()
         command = invoc + ['$ARGS', '$in']
         description = 'Compiling Swift source $in'
         self.add_rule(NinjaRule(rule, command, [], description))
@@ -1735,8 +1787,8 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
         if self.created_llvm_ir_rule[compiler.for_machine]:
             return
         rule = self.get_compiler_rule_name('llvm_ir', compiler.for_machine)
-        command = [ninja_quote(i) for i in compiler.get_exelist()]
-        args = ['$ARGS'] + compiler.get_output_args('$out') + compiler.get_compile_only_args() + ['$in']
+        command = compiler.get_exelist()
+        args = ['$ARGS'] + NinjaCommandArg.list(compiler.get_output_args('$out'), Quoting.none) + compiler.get_compile_only_args() + ['$in']
         description = 'Compiling LLVM IR object $in'
         self.add_rule(NinjaRule(rule, command, args, description,
                                 rspable=compiler.can_linker_accept_rsp()))
@@ -1765,15 +1817,9 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
         if langname == 'fortran':
             self.generate_fortran_dep_hack(crstr)
         rule = self.get_compiler_rule_name(langname, compiler.for_machine)
-        depargs = compiler.get_dependency_gen_args('$out', '$DEPFILE')
-        quoted_depargs = []
-        for d in depargs:
-            if d != '$out' and d != '$in':
-                d = quote_func(d)
-            quoted_depargs.append(d)
-
-        command = [ninja_quote(i) for i in compiler.get_exelist()]
-        args = ['$ARGS'] + quoted_depargs + compiler.get_output_args('$out') + compiler.get_compile_only_args() + ['$in']
+        depargs = NinjaCommandArg.list(compiler.get_dependency_gen_args('$out', '$DEPFILE'), Quoting.none)
+        command = compiler.get_exelist()
+        args = ['$ARGS'] + depargs + NinjaCommandArg.list(compiler.get_output_args('$out'), Quoting.none) + compiler.get_compile_only_args() + ['$in']
         description = 'Compiling {} object $out'.format(compiler.get_display_language())
         if isinstance(compiler, VisualStudioLikeCompiler):
             deps = 'msvc'
@@ -1791,16 +1837,11 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
         rule = self.compiler_to_pch_rule_name(compiler)
         depargs = compiler.get_dependency_gen_args('$out', '$DEPFILE')
 
-        quoted_depargs = []
-        for d in depargs:
-            if d != '$out' and d != '$in':
-                d = quote_func(d)
-            quoted_depargs.append(d)
         if isinstance(compiler, VisualStudioLikeCompiler):
             output = []
         else:
-            output = compiler.get_output_args('$out')
-        command = compiler.get_exelist() + ['$ARGS'] + quoted_depargs + output + compiler.get_compile_only_args() + ['$in']
+            output = NinjaCommandArg.list(compiler.get_output_args('$out'), Quoting.none)
+        command = compiler.get_exelist() + ['$ARGS'] + depargs + output + compiler.get_compile_only_args() + ['$in']
         description = 'Precompiling header $in'
         if isinstance(compiler, VisualStudioLikeCompiler):
             deps = 'msvc'