external/fwts: Add parser to extract olog pattern definitions

The fwts project has a facility to scan system logs for interesting
error messages, by matching on patterns in a JSON file. Recently, Deb
has added support for the OPAL msglog to fwts, called 'olog':

  http://kernel.ubuntu.com/git/hwe/fwts.git/commit/?id=652b79b

However, we don't yet have any patterns for OPAL.

Rather than generate a separate set of patterns that may go stale, Anton
suggested that we may want to pull these directly from the OPAL source.

This change implements a parser to generate olog pattern definitions
from annotations in OPAL itself. For example, a check in the flash code
might look like:

  if (!ffs) {
  	/**
  	 * @fwts-label SystemFlashNoPartitionTable
  	 * @fwts-advice OPAL Could not read a partition table on
  	 *    system flash. Since we've still booted the machine (which
  	 *    requires flash), check that we're registering the proper
  	 *    system flash device.
  	 */
  	prlog(PR_WARNING, "FLASH: attempted to register system flash "
  			"%s, wwhich has no partition info\n", name);
  	return;
  }

By running generate-fwts-olog on the codebase, we get:

  {
   "olog_error_warning_patterns": [
    {
     "advice": "OPAL Could not read a partition table on system flash. Since we've still booted the machine (which requires flash), check that we're registering the proper system flash device.",
     "compare_mode": "regex",
     "label": "SystemFlashNoPartitionTable",
     "level": "LOG_LEVEL_HIGH",
     "pattern": "FLASH: attempted to register system flash .*, wwhich has no partition info"
    }
   ]
  }

- which is suitable for input to the fwts pattern definitions.

Signed-off-by: Jeremy Kerr <jk@ozlabs.org>
[stewart@linux.vnet.ibm.com: squash trailing whitespace]
Signed-off-by: Stewart Smith <stewart@linux.vnet.ibm.com>

1 files changed, 229 insertions, 0 deletions

diff --git a/external/fwts/generate-fwts-olog b/external/fwts/generate-fwts-olog
new file mode 100755
index 0000000..c9d8259
--- /dev/null
+++ b/external/fwts/generate-fwts-olog
@@ -0,0 +1,229 @@
+#!/usr/bin/env python2
+#
+# Copyright 2016 Jeremy Kerr <jk@ozlabs.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path
+import re
+import sys
+import string
+import json
+import argparse
+from pyparsing import Regex, Literal, Word, Combine, OneOrMore, QuotedString, \
+         lineno
+
+json_params = {
+    'indent': 1,
+    'sort_keys': True,
+}
+
+def create_parser():
+    # Match a C-style comment starting with two *s
+    comment = Regex(r'/\*\*(?P<content>.*?)\*/', re.DOTALL)
+
+    # Match an @fwts-<tag> annotation (within the comment), plus the proceeding
+    # text
+    annotation = Regex(r'@fwts-(?P<tag>\w+)\W+(?P<text>.*?)(?=@fwts-|\Z)',
+                re.DOTALL)
+
+    # Match the following prlog() call
+    log_call = (Literal("prlog") +
+                Literal('(').suppress() +
+                Word(string.letters + string.digits + '_') +
+                Literal(',').suppress() +
+                Combine(OneOrMore(QuotedString('"')), adjacent=False) +
+                (Literal(')') | Literal(',')).suppress()
+               )
+
+    pattern = comment + log_call
+    pattern.setWhitespaceChars(string.whitespace + '\n')
+
+    def comment_action(tok):
+        patterns = {}
+        for result in annotation.scanString(tok['content']):
+            patterns.update(result[0][0])
+        return patterns
+
+    def annotation_action(tok):
+        return {
+            tok['tag']: cleanup_content(tok['text'])
+        }
+
+    comment.setParseAction(comment_action)
+    annotation.setParseAction(annotation_action)
+    pattern.parseWithTabs()
+
+    return pattern
+
+def find_sources(dirname):
+    sources = []
+
+    def is_source(fname):
+        return fname.endswith('.c')
+
+    def add_fn(s, dname, fnames):
+        s.extend([ os.path.join(dname, fname)
+                         for fname in fnames if is_source(fname) ])
+
+    os.path.walk(dirname, add_fn, sources)
+    return sources
+
+def cleanup_content(content):
+    comment_prefix_re = re.compile(r'^\s*\*\s*', re.MULTILINE)
+    whitespace_re = re.compile(r'\s+')
+
+    content = comment_prefix_re.sub(' ', content)
+    content = whitespace_re.sub(' ', content)
+    return content.strip()
+
+def warn(loc, message):
+    print >>sys.stderr, 'WARNING:%s:%d: %s' % (loc[0], loc[1], message)
+
+def log_level_to_fwts(level):
+    level_map = {
+        'PR_EMERG':     'LOG_LEVEL_CRITICAL',
+        'PR_ALERT':     'LOG_LEVEL_CRITICAL',
+        'PR_CRIT':      'LOG_LEVEL_CRITICAL',
+        'PR_ERR':       'LOG_LEVEL_CRITICAL',
+        'PR_WARNING':   'LOG_LEVEL_HIGH',
+        'PR_NOTICE':    'LOG_LEVEL_MEDIUM',
+        'PR_PRINTF':    'LOG_LEVEL_MEDIUM',
+    }
+    return level_map.get(level, 'LOG_LEVEL_LOW')
+
+def message_to_pattern(loc, msg):
+    """ Convert a C printf()-style template to a pattern suitable for fwts """
+
+    # Somewhat-simplified match for a %-template
+    template_re = re.compile(
+            '%(?P<flag>[-#0 +]*)'
+            '(?P<width>(?:[1-9][0-9]*|\*))?'
+            '(?P<precision>\.*(?:[1-9][0-9]*|\*))?'
+            '(?:hh|h|l|ll|L|j|z|t)?'
+            '(?P<conversion>[a-zA-Z%])')
+    global is_regex
+    is_regex = False
+
+    def expand_template(match):
+        global is_regex
+        c = match.group('conversion').lower()
+        if c == '%':
+            return '%'
+        is_regex = True
+        if c in ['d', 'i', 'u']:
+            return '[0-9]+'
+        elif c == 'o':
+            return '[0-7]+'
+        elif c == 'x':
+            return '[0-9a-f]+'
+        elif c == 'p':
+            return '(?:0x[0-9a-f]+|nil)'
+        elif c == 's':
+            return '.*'
+        else:
+            warn(loc, "Unknown template conversion '%s'" % match.group(0))
+            return '.*'
+
+    escape_re = re.compile(r'\\(?P<char>.)', re.DOTALL)
+    def expand_escape(match):
+        global is_regex
+        c = match.group('char')
+        if c == 'n':
+            return '\n'
+        elif c in ['\\', '"']:
+            return c
+        else:
+            warn(loc, "Unhandled escape sequence '%s'" % match.group(0))
+            is_regex = True
+            return '.'
+
+    pattern = template_re.sub(expand_template, msg)
+    pattern = escape_re.sub(expand_escape, pattern)
+    pattern = pattern.strip()
+
+    compare_mode = "string"
+    if is_regex:
+        compare_mode = "regex"
+
+    return (compare_mode, pattern)
+
+def parse_patterns(parser, fname):
+    patterns = []
+    data = open(fname).read()
+    i = 1
+    for result in parser.scanString(data):
+        (token, loc, _) = result
+        (annotations, logfn, level, msg) = token
+
+        loc = (fname, lineno(loc, data))
+
+        if logfn != 'prlog':
+            warn(loc, "unknown log output function '%s'" % logfn)
+
+        compare_mode, pattern_str = message_to_pattern(loc, msg)
+
+        pattern = {
+            'log_level': log_level_to_fwts(level),
+            'compare_mode': compare_mode,
+            'pattern': pattern_str,
+        }
+
+        pattern.update(annotations)
+
+        if not 'label' in pattern:
+            warn(loc, "missing label")
+            pattern['label'] = '%s:%d' % (fname, i)
+            i += 1
+
+        if not 'advice' in pattern:
+            warn(loc, "missing advice")
+
+        allowed_data = ['compare_mode', 'log_level',
+                        'pattern', 'advice', 'label']
+        extras = set(pattern.keys()) - set(allowed_data)
+        if extras:
+            warn(loc, "unknown pattern annotation: %s" %
+                    ','.join([ "'%s'" % e for e in extras]))
+            for e in extras:
+                del pattern[e]
+
+        patterns.append(pattern)
+
+    return patterns
+
+if __name__ == '__main__':
+    argparser = argparse.ArgumentParser(
+            description='Generate FWTS olog definitions from the skiboot '
+                        'source tree')
+    argparser.add_argument('directories', metavar='DIR', nargs='*',
+            help='path to source files (default .)', default=['.'])
+    argparser.add_argument('--output', '-o', metavar='FILE',
+            type=argparse.FileType('w'), default=sys.stdout,
+            help='output to FILE (default to stdout)', nargs='?')
+    args = argparser.parse_args()
+
+    sources = []
+    for directory in args.directories:
+        sources.extend(find_sources(directory))
+
+    parser = create_parser()
+    patterns = []
+    for source in sources:
+        patterns.extend(parse_patterns(parser, source))
+
+    data = {'olog_error_warning_patterns': patterns}
+
+    args.output.write(json.dumps(data, **json_params) + '\n')
+