diff options
author | Jeremy Kerr <jk@ozlabs.org> | 2016-04-28 08:41:53 +0800 |
---|---|---|
committer | Stewart Smith <stewart@linux.vnet.ibm.com> | 2016-04-28 14:08:03 +1000 |
commit | 7eeb1cc1f4d9b4a183ef8a16ad20930ed9dba6d6 (patch) | |
tree | 73b6f751e30736eb0304be2826296941346ba627 /external | |
parent | 895d8238eb54a2d09388efb7ed9e29094dddd9b0 (diff) | |
download | skiboot-7eeb1cc1f4d9b4a183ef8a16ad20930ed9dba6d6.zip skiboot-7eeb1cc1f4d9b4a183ef8a16ad20930ed9dba6d6.tar.gz skiboot-7eeb1cc1f4d9b4a183ef8a16ad20930ed9dba6d6.tar.bz2 |
external/fwts: Add parser to extract olog pattern definitions
The fwts project has a facility to scan system logs for interesting
error messages, by matching on patterns in a JSON file. Recently, Deb
has added support for the OPAL msglog to fwts, called 'olog':
http://kernel.ubuntu.com/git/hwe/fwts.git/commit/?id=652b79b
However, we don't yet have any patterns for OPAL.
Rather than generate a separate set of patterns that may go stale, Anton
suggested that we may want to pull these directly from the OPAL source.
This change implements a parser to generate olog pattern definitions
from annotations in OPAL itself. For example, a check in the flash code
might look like:
if (!ffs) {
/**
* @fwts-label SystemFlashNoPartitionTable
* @fwts-advice OPAL Could not read a partition table on
* system flash. Since we've still booted the machine (which
* requires flash), check that we're registering the proper
* system flash device.
*/
prlog(PR_WARNING, "FLASH: attempted to register system flash "
"%s, wwhich has no partition info\n", name);
return;
}
By running generate-fwts-olog on the codebase, we get:
{
"olog_error_warning_patterns": [
{
"advice": "OPAL Could not read a partition table on system flash. Since we've still booted the machine (which requires flash), check that we're registering the proper system flash device.",
"compare_mode": "regex",
"label": "SystemFlashNoPartitionTable",
"level": "LOG_LEVEL_HIGH",
"pattern": "FLASH: attempted to register system flash .*, wwhich has no partition info"
}
]
}
- which is suitable for input to the fwts pattern definitions.
Signed-off-by: Jeremy Kerr <jk@ozlabs.org>
[stewart@linux.vnet.ibm.com: squash trailing whitespace]
Signed-off-by: Stewart Smith <stewart@linux.vnet.ibm.com>
Diffstat (limited to 'external')
-rwxr-xr-x | external/fwts/generate-fwts-olog | 229 |
1 files changed, 229 insertions, 0 deletions
diff --git a/external/fwts/generate-fwts-olog b/external/fwts/generate-fwts-olog new file mode 100755 index 0000000..c9d8259 --- /dev/null +++ b/external/fwts/generate-fwts-olog @@ -0,0 +1,229 @@ +#!/usr/bin/env python2 +# +# Copyright 2016 Jeremy Kerr <jk@ozlabs.org> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os.path +import re +import sys +import string +import json +import argparse +from pyparsing import Regex, Literal, Word, Combine, OneOrMore, QuotedString, \ + lineno + +json_params = { + 'indent': 1, + 'sort_keys': True, +} + +def create_parser(): + # Match a C-style comment starting with two *s + comment = Regex(r'/\*\*(?P<content>.*?)\*/', re.DOTALL) + + # Match an @fwts-<tag> annotation (within the comment), plus the proceeding + # text + annotation = Regex(r'@fwts-(?P<tag>\w+)\W+(?P<text>.*?)(?=@fwts-|\Z)', + re.DOTALL) + + # Match the following prlog() call + log_call = (Literal("prlog") + + Literal('(').suppress() + + Word(string.letters + string.digits + '_') + + Literal(',').suppress() + + Combine(OneOrMore(QuotedString('"')), adjacent=False) + + (Literal(')') | Literal(',')).suppress() + ) + + pattern = comment + log_call + pattern.setWhitespaceChars(string.whitespace + '\n') + + def comment_action(tok): + patterns = {} + for result in annotation.scanString(tok['content']): + patterns.update(result[0][0]) + return patterns + + def annotation_action(tok): + return { + tok['tag']: cleanup_content(tok['text']) + } + + comment.setParseAction(comment_action) + annotation.setParseAction(annotation_action) + pattern.parseWithTabs() + + return pattern + +def find_sources(dirname): + sources = [] + + def is_source(fname): + return fname.endswith('.c') + + def add_fn(s, dname, fnames): + s.extend([ os.path.join(dname, fname) + for fname in fnames if is_source(fname) ]) + + os.path.walk(dirname, add_fn, sources) + return sources + +def cleanup_content(content): + comment_prefix_re = re.compile(r'^\s*\*\s*', re.MULTILINE) + whitespace_re = re.compile(r'\s+') + + content = comment_prefix_re.sub(' ', content) + content = whitespace_re.sub(' ', content) + return content.strip() + +def warn(loc, message): + print >>sys.stderr, 'WARNING:%s:%d: %s' % (loc[0], loc[1], message) + +def log_level_to_fwts(level): + level_map = { + 'PR_EMERG': 'LOG_LEVEL_CRITICAL', + 'PR_ALERT': 'LOG_LEVEL_CRITICAL', + 'PR_CRIT': 'LOG_LEVEL_CRITICAL', + 'PR_ERR': 'LOG_LEVEL_CRITICAL', + 'PR_WARNING': 'LOG_LEVEL_HIGH', + 'PR_NOTICE': 'LOG_LEVEL_MEDIUM', + 'PR_PRINTF': 'LOG_LEVEL_MEDIUM', + } + return level_map.get(level, 'LOG_LEVEL_LOW') + +def message_to_pattern(loc, msg): + """ Convert a C printf()-style template to a pattern suitable for fwts """ + + # Somewhat-simplified match for a %-template + template_re = re.compile( + '%(?P<flag>[-#0 +]*)' + '(?P<width>(?:[1-9][0-9]*|\*))?' + '(?P<precision>\.*(?:[1-9][0-9]*|\*))?' + '(?:hh|h|l|ll|L|j|z|t)?' + '(?P<conversion>[a-zA-Z%])') + global is_regex + is_regex = False + + def expand_template(match): + global is_regex + c = match.group('conversion').lower() + if c == '%': + return '%' + is_regex = True + if c in ['d', 'i', 'u']: + return '[0-9]+' + elif c == 'o': + return '[0-7]+' + elif c == 'x': + return '[0-9a-f]+' + elif c == 'p': + return '(?:0x[0-9a-f]+|nil)' + elif c == 's': + return '.*' + else: + warn(loc, "Unknown template conversion '%s'" % match.group(0)) + return '.*' + + escape_re = re.compile(r'\\(?P<char>.)', re.DOTALL) + def expand_escape(match): + global is_regex + c = match.group('char') + if c == 'n': + return '\n' + elif c in ['\\', '"']: + return c + else: + warn(loc, "Unhandled escape sequence '%s'" % match.group(0)) + is_regex = True + return '.' + + pattern = template_re.sub(expand_template, msg) + pattern = escape_re.sub(expand_escape, pattern) + pattern = pattern.strip() + + compare_mode = "string" + if is_regex: + compare_mode = "regex" + + return (compare_mode, pattern) + +def parse_patterns(parser, fname): + patterns = [] + data = open(fname).read() + i = 1 + for result in parser.scanString(data): + (token, loc, _) = result + (annotations, logfn, level, msg) = token + + loc = (fname, lineno(loc, data)) + + if logfn != 'prlog': + warn(loc, "unknown log output function '%s'" % logfn) + + compare_mode, pattern_str = message_to_pattern(loc, msg) + + pattern = { + 'log_level': log_level_to_fwts(level), + 'compare_mode': compare_mode, + 'pattern': pattern_str, + } + + pattern.update(annotations) + + if not 'label' in pattern: + warn(loc, "missing label") + pattern['label'] = '%s:%d' % (fname, i) + i += 1 + + if not 'advice' in pattern: + warn(loc, "missing advice") + + allowed_data = ['compare_mode', 'log_level', + 'pattern', 'advice', 'label'] + extras = set(pattern.keys()) - set(allowed_data) + if extras: + warn(loc, "unknown pattern annotation: %s" % + ','.join([ "'%s'" % e for e in extras])) + for e in extras: + del pattern[e] + + patterns.append(pattern) + + return patterns + +if __name__ == '__main__': + argparser = argparse.ArgumentParser( + description='Generate FWTS olog definitions from the skiboot ' + 'source tree') + argparser.add_argument('directories', metavar='DIR', nargs='*', + help='path to source files (default .)', default=['.']) + argparser.add_argument('--output', '-o', metavar='FILE', + type=argparse.FileType('w'), default=sys.stdout, + help='output to FILE (default to stdout)', nargs='?') + args = argparser.parse_args() + + sources = [] + for directory in args.directories: + sources.extend(find_sources(directory)) + + parser = create_parser() + patterns = [] + for source in sources: + patterns.extend(parse_patterns(parser, source)) + + data = {'olog_error_warning_patterns': patterns} + + args.output.write(json.dumps(data, **json_params) + '\n') + |