[UpdateTestChecks] Re-add --filter and --filter-out options

Re-add filtering options with fixes for failed tests.  We were not passing the
is_filtered argument in all check generator calls in update_cc_test_checks.py

Enhance the various update_*_test_checks.py tools to allow filtering the tool
output with regular expressions.  The --filter option will emit only tool output
lines matching the given regular expression while the --filter-out option will
emit only tools output lines not matching the given regular expression.  Filters
are applied in order of appearance on the command line (or in UTC_ARGS) and the
first matching filter terminates the search.

This allows test authors to create more focused tests by removing irrelevant
tool output and checking only the pieces of output necessary to test the desired
functionality.

Differential Revision: https://reviews.llvm.org/D117694

5 files changed, 178 insertions, 24 deletions

diff --git a/llvm/utils/UpdateTestChecks/asm.py b/llvm/utils/UpdateTestChecks/asm.py
index c24849b..95d17ba 100644
--- a/llvm/utils/UpdateTestChecks/asm.py
+++ b/llvm/utils/UpdateTestChecks/asm.py
@@ -456,8 +456,11 @@ def get_run_handler(triple):
 
 ##### Generator of assembly CHECK lines
 
-def add_asm_checks(output_lines, comment_marker, prefix_list, func_dict, func_name):
+def add_asm_checks(output_lines, comment_marker, prefix_list, func_dict,
+                   func_name, is_filtered):
   # Label format is based on ASM string.
   check_label_format = '{} %s-LABEL: %s%s:'.format(comment_marker)
   global_vars_seen_dict = {}
-  common.add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name, check_label_format, True, False, global_vars_seen_dict)
+  common.add_checks(output_lines, comment_marker, prefix_list, func_dict,
+                    func_name, check_label_format, True, False,
+                    global_vars_seen_dict, is_filtered = is_filtered)
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index 269b429..dbcff53 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -1,5 +1,6 @@
 from __future__ import print_function
 
+import argparse
 import copy
 import glob
 import os
@@ -13,7 +14,99 @@ import sys
 _verbose = False
 _prefix_filecheck_ir_name = ''
 
+class Regex(object):
+  """Wrap a compiled regular expression object to allow deep copy of a regexp.
+  This is required for the deep copy done in do_scrub.
+
+  """
+  def __init__(self, regex):
+    self.regex = regex
+
+  def __deepcopy__(self, memo):
+    result = copy.copy(self)
+    result.regex = self.regex
+    return result
+
+  def search(self, line):
+    return self.regex.search(line)
+
+  def sub(self, repl, line):
+    return self.regex.sub(repl, line)
+
+  def pattern(self):
+    return self.regex.pattern
+
+  def flags(self):
+    return self.regex.flags
+
+class Filter(Regex):
+  """Augment a Regex object with a flag indicating whether a match should be
+    added (!is_filter_out) or removed (is_filter_out) from the generated checks.
+
+  """
+  def __init__(self, regex, is_filter_out):
+    super(Filter, self).__init__(regex)
+    self.is_filter_out = is_filter_out
+
+  def __deepcopy__(self, memo):
+    result = copy.deepcopy(super(Filter, self), memo)
+    result.is_filter_out = copy.deepcopy(self.is_filter_out, memo)
+    return result
+
 def parse_commandline_args(parser):
+  class RegexAction(argparse.Action):
+    """Add a regular expression option value to a list of regular expressions.
+    This compiles the expression, wraps it in a Regex and adds it to the option
+    value list."""
+    def __init__(self, option_strings, dest, nargs=None, **kwargs):
+      if nargs is not None:
+        raise ValueError('nargs not allowed')
+      super(RegexAction, self).__init__(option_strings, dest, **kwargs)
+
+    def do_call(self, namespace, values, flags):
+      value_list = getattr(namespace, self.dest)
+      if value_list is None:
+        value_list = []
+
+      try:
+        value_list.append(Regex(re.compile(values, flags)))
+      except re.error as error:
+        raise ValueError('{}: Invalid regular expression \'{}\' ({})'.format(
+          option_string, error.pattern, error.msg))
+
+      setattr(namespace, self.dest, value_list)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+      self.do_call(namespace, values, 0)
+
+  class FilterAction(RegexAction):
+    """Add a filter to a list of filter option values."""
+    def __init__(self, option_strings, dest, nargs=None, **kwargs):
+      super(FilterAction, self).__init__(option_strings, dest, nargs, **kwargs)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+      super(FilterAction, self).__call__(parser, namespace, values, option_string)
+
+      value_list = getattr(namespace, self.dest)
+
+      is_filter_out = ( option_string == '--filter-out' )
+
+      value_list[-1] = Filter(value_list[-1].regex, is_filter_out)
+
+      setattr(namespace, self.dest, value_list)
+
+  filter_group = parser.add_argument_group(
+    'filtering',
+    """Filters are applied to each output line according to the order given. The
+    first matching filter terminates filter processing for that current line.""")
+
+  filter_group.add_argument('--filter', action=FilterAction, dest='filters',
+                            metavar='REGEX',
+                            help='Only include lines matching REGEX (may be specified multiple times)')
+  filter_group.add_argument('--filter-out', action=FilterAction, dest='filters',
+                            metavar='REGEX',
+                            help='Exclude lines matching REGEX')
+
   parser.add_argument('--include-generated-funcs', action='store_true',
                       help='Output checks for functions not in source')
   parser.add_argument('-v', '--verbose', action='store_true',
@@ -258,6 +351,21 @@ def find_run_lines(test, lines):
     debug('  RUN: {}'.format(l))
   return run_lines
 
+def apply_filters(line, filters):
+  has_filter = False
+  for f in filters:
+    if not f.is_filter_out:
+      has_filter = True
+    if f.search(line):
+      return False if f.is_filter_out else True
+  # If we only used filter-out, keep the line, otherwise discard it since no
+  # filter matched.
+  return False if has_filter else True
+
+def do_filter(body, filters):
+  return body if not filters else '\n'.join(filter(
+    lambda line: apply_filters(line, filters), body.splitlines()))
+
 def scrub_body(body):
   # Scrub runs of whitespace out of the assembly, but leave the leading
   # whitespace in place.
@@ -320,6 +428,11 @@ class FunctionTestBuilder:
     self._verbose = flags.verbose
     self._record_args = flags.function_signature
     self._check_attributes = flags.check_attributes
+    # Strip double-quotes if input was read by UTC_ARGS
+    self._filters = list(map(lambda f: Filter(re.compile(f.pattern().strip('"'),
+                                                         f.flags()),
+                                              f.is_filter_out),
+                             flags.filters)) if flags.filters else []
     self._scrubber_args = scrubber_args
     self._path = path
     # Strip double-quotes if input was read by UTC_ARGS
@@ -344,6 +457,9 @@ class FunctionTestBuilder:
   def global_var_dict(self):
     return self._global_var_dict
 
+  def is_filtered(self):
+    return bool(self._filters)
+
   def process_run_line(self, function_re, scrubber, raw_tool_output, prefixes, is_asm):
     build_global_values_dictionary(self._global_var_dict, raw_tool_output, prefixes)
     for m in function_re.finditer(raw_tool_output):
@@ -360,9 +476,10 @@ class FunctionTestBuilder:
           args_and_sig = '('
       else:
           args_and_sig = ''
-      scrubbed_body = do_scrub(body, scrubber, self._scrubber_args,
+      filtered_body = do_filter(body, self._filters)
+      scrubbed_body = do_scrub(filtered_body, scrubber, self._scrubber_args,
                                extra=False)
-      scrubbed_extra = do_scrub(body, scrubber, self._scrubber_args,
+      scrubbed_extra = do_scrub(filtered_body, scrubber, self._scrubber_args,
                                 extra=True)
       if 'analysis' in m.groupdict():
         analysis = m.group('analysis')
@@ -649,7 +766,7 @@ def generalize_check_lines(lines, is_analyze, vars_seen, global_vars_seen):
   return lines
 
 
-def add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name, check_label_format, is_asm, is_analyze, global_vars_seen_dict):
+def add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name, check_label_format, is_asm, is_analyze, global_vars_seen_dict, is_filtered):
   # prefix_exclusions are prefixes we cannot use to print the function because it doesn't exist in run lines that use these prefixes as well.
   prefix_exclusions = set()
   printed_prefixes = []
@@ -707,15 +824,26 @@ def add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
       else:
         output_lines.append(check_label_format % (checkprefix, func_name, args_and_sig))
       func_body = str(func_dict[checkprefix][func_name]).splitlines()
+      if not func_body:
+        # We have filtered everything.
+        continue
 
       # For ASM output, just emit the check lines.
       if is_asm:
-        output_lines.append('%s %s:       %s' % (comment_marker, checkprefix, func_body[0]))
+        body_start = 1
+        if is_filtered:
+          # For filtered output we don't add "-NEXT" so don't add extra spaces
+          # before the first line.
+          body_start = 0
+        else:
+          output_lines.append('%s %s:       %s' % (comment_marker, checkprefix, func_body[0]))
         for func_line in func_body[1:]:
           if func_line.strip() == '':
             output_lines.append('%s %s-EMPTY:' % (comment_marker, checkprefix))
           else:
-            output_lines.append('%s %s-NEXT:  %s' % (comment_marker, checkprefix, func_line))
+            check_suffix = '-NEXT' if not is_filtered else ''
+            output_lines.append('%s %s%s:  %s' % (comment_marker, checkprefix,
+                                                  check_suffix, func_line))
         break
 
       # For IR output, change all defs to FileCheck variables, so we're immune
@@ -747,8 +875,9 @@ def add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
           output_lines.append('{} {}:       {}'.format(
               comment_marker, checkprefix, func_line))
         else:
-          output_lines.append('{} {}-NEXT:  {}'.format(
-              comment_marker, checkprefix, func_line))
+          check_suffix = '-NEXT' if not is_filtered else ''
+          output_lines.append('{} {}{}:  {}'.format(
+              comment_marker, checkprefix, check_suffix, func_line))
         is_blank_line = False
 
       # Add space between different check prefixes and also before the first
@@ -762,18 +891,21 @@ def add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
       break
 
 def add_ir_checks(output_lines, comment_marker, prefix_list, func_dict,
-                  func_name, preserve_names, function_sig, global_vars_seen_dict):
+                  func_name, preserve_names, function_sig,
+                  global_vars_seen_dict, is_filtered):
   # Label format is based on IR string.
   function_def_regex = 'define {{[^@]+}}' if function_sig else ''
   check_label_format = '{} %s-LABEL: {}@%s%s'.format(comment_marker, function_def_regex)
   add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
-             check_label_format, False, preserve_names, global_vars_seen_dict)
+             check_label_format, False, preserve_names, global_vars_seen_dict,
+             is_filtered)
 
 def add_analyze_checks(output_lines, comment_marker, prefix_list, func_dict, func_name):
   check_label_format = '{} %s-LABEL: \'%s%s\''.format(comment_marker)
   global_vars_seen_dict = {}
   add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
-             check_label_format, False, True, global_vars_seen_dict)
+             check_label_format, False, True, global_vars_seen_dict,
+             is_filtered = False)
 
 def build_global_values_dictionary(glob_val_dict, raw_tool_output, prefixes):
   for nameless_value in nameless_values:
@@ -901,11 +1033,21 @@ def get_autogennote_suffix(parser, args):
         continue
     if parser.get_default(action.dest) == value:
       continue  # Don't add default values
-    autogenerated_note_args += action.option_strings[0] + ' '
-    if action.const is None:  # action takes a parameter
-      if action.nargs == '+':
-        value = ' '.join(map(lambda v: '"' + v.strip('"') + '"', value))
-      autogenerated_note_args += '%s ' % value
+    if action.dest == 'filters':
+      # Create a separate option for each filter element.  The value is a list
+      # of Filter objects.
+      for elem in value:
+        opt_name = 'filter-out' if elem.is_filter_out else 'filter'
+        opt_value = elem.pattern()
+        new_arg = '--%s "%s" ' % (opt_name, opt_value.strip('"'))
+        if new_arg not in autogenerated_note_args:
+          autogenerated_note_args += new_arg
+    else:
+      autogenerated_note_args += action.option_strings[0] + ' '
+      if action.const is None:  # action takes a parameter
+        if action.nargs == '+':
+          value = ' '.join(map(lambda v: '"' + v.strip('"') + '"', value))
+        autogenerated_note_args += '%s ' % value
   if autogenerated_note_args:
     autogenerated_note_args = ' %s %s' % (UTC_ARGS_KEY, autogenerated_note_args[:-1])
   return autogenerated_note_args
diff --git a/llvm/utils/update_cc_test_checks.py b/llvm/utils/update_cc_test_checks.py
index 15318e1..d9c53c3 100755
--- a/llvm/utils/update_cc_test_checks.py
+++ b/llvm/utils/update_cc_test_checks.py
@@ -344,11 +344,13 @@ def main():
                                prefixes,
                                func_dict, func, False,
                                ti.args.function_signature,
-                               global_vars_seen_dict)
+                               global_vars_seen_dict,
+                               is_filtered=builder.is_filtered())
         else:
           asm.add_asm_checks(my_output_lines, '//',
                              prefixes,
-                             func_dict, func)
+                             func_dict, func,
+                             is_filtered=builder.is_filtered())
 
       if ti.args.check_globals:
         common.add_global_checks(builder.global_var_dict(), '//', run_list,
@@ -398,7 +400,8 @@ def main():
                 output_lines.append('//')
               added.add(mangled)
               common.add_ir_checks(output_lines, '//', filecheck_run_list, func_dict, mangled,
-                                   False, args.function_signature, global_vars_seen_dict)
+                                   False, args.function_signature, global_vars_seen_dict,
+                                   is_filtered=builder.is_filtered())
               if line.rstrip('\n') == '//':
                 include_line = False
 
diff --git a/llvm/utils/update_llc_test_checks.py b/llvm/utils/update_llc_test_checks.py
index a9e86ea..cf68d23 100755
--- a/llvm/utils/update_llc_test_checks.py
+++ b/llvm/utils/update_llc_test_checks.py
@@ -111,6 +111,7 @@ def main():
         run_list=run_list,
         flags=type('', (object,), {
             'verbose': ti.args.verbose,
+            'filters': ti.args.filters,
             'function_signature': False,
             'check_attributes': False,
             'replace_value_regex': []}),
@@ -161,7 +162,8 @@ def main():
                                lambda my_output_lines, prefixes, func:
                                asm.add_asm_checks(my_output_lines,
                                                   check_indent + ';',
-                                                  prefixes, func_dict, func))
+                                                  prefixes, func_dict, func,
+                                                  is_filtered=builder.is_filtered()))
     else:
       for input_info in ti.iterlines(output_lines):
         input_line = input_info.line
@@ -176,7 +178,9 @@ def main():
               continue
 
           # Print out the various check lines here.
-          asm.add_asm_checks(output_lines, check_indent + ';', run_list, func_dict, func_name)
+          asm.add_asm_checks(output_lines, check_indent + ';', run_list,
+                             func_dict, func_name,
+                             is_filtered=builder.is_filtered())
           is_in_function_start = False
 
         if is_in_function:
diff --git a/llvm/utils/update_test_checks.py b/llvm/utils/update_test_checks.py
index d7fce8d..dee3a51 100755
--- a/llvm/utils/update_test_checks.py
+++ b/llvm/utils/update_test_checks.py
@@ -160,7 +160,8 @@ def main():
                                                     prefixes,
                                                     func_dict, func, False,
                                                     args.function_signature,
-                                                    global_vars_seen_dict))
+                                                    global_vars_seen_dict,
+                                                    is_filtered=builder.is_filtered()))
     else:
       # "Normal" mode.
       for input_line_info in ti.iterlines(output_lines):
@@ -178,7 +179,8 @@ def main():
           # Print out the various check lines here.
           common.add_ir_checks(output_lines, ';', prefix_list, func_dict,
                                func_name, args.preserve_names, args.function_signature,
-                               global_vars_seen_dict)
+                               global_vars_seen_dict,
+                               is_filtered=builder.is_filtered())
           is_in_function_start = False
 
         m = common.IR_FUNCTION_RE.match(input_line)