KDC Audit infrastructure and plugin implementation

Per project http://k5wiki.kerberos.org/wiki/Projects/Audit

The purpose of this project is to create an Audit infrastructure to monitor
security related events on the KDC.

The following events are targeted in the initial version:
- startup and shutdown of the KDC;
- AS_REQ and TGS_REQ exchanges.  This includes client address and port, KDC
  request and request ID, KDC reply, primary and derived ticket and their
  ticket IDs, second ticket ID, cross-realm referral, was ticket renewed and
  validated, local policy violation and protocol constraints, and KDC status
  message.

Ticket ID is introduced to allow to link tickets to their initial TGT at any
stage of the Kerberos exchange. For the purpose of this project it is a private
to KDC ticket ID: each successfully created ticket is hashed and recorded
into audit log. The administrators can correlate the primary and derived
ticket IDs after the fact.

Request ID is a randomly generated alpha-numeric string. Using this ID an
administrator can easily correlate multiple audit events related to a single
request. It should be informative both in cases when the request is sent to
multiple KDCs, or to the same KDC multiple times.

For the purpose of testing and demo of the Audit, the JSON based modules are
implemented: "test" and "simple" audit modules respectively.
The file plugins/audit/j_dict.h is a dictionary used in this implememtations.

The new Audit system is build-time enabled and run-time pluggable.

[kaduk@mit.edu: remove potential KDC crashes, minor reordering]

ticket: 7712
target_version: 1.12

4 files changed, 213 insertions, 0 deletions

diff --git a/src/tests/Makefile.in b/src/tests/Makefile.in
index 3525b48..d50d513 100644
--- a/src/tests/Makefile.in
+++ b/src/tests/Makefile.in
@@ -118,8 +118,13 @@ check-pytests:: t_init_creds t_localauth
 	$(RUNPYTEST) $(srcdir)/t_cve-2012-1015.py $(PYTESTFLAGS)
 	$(RUNPYTEST) $(srcdir)/t_cve-2013-1416.py $(PYTESTFLAGS)
 	$(RUNPYTEST) $(srcdir)/t_cve-2013-1417.py $(PYTESTFLAGS)
+	$(RM) au.log
+	$(RUNPYTEST) $(srcdir)/t_audit.py $(PYTESTFLAGS)
+	$(RUNPYTEST) $(srcdir)/jsonwalker.py -d $(srcdir)/au_dict.json \
+			-i au.log
 
 clean::
 	$(RM) gcred hist hrealm kdbtest plugorder responder
 	$(RM) t_init_creds t_localauth krb5.conf kdc.conf
 	$(RM) -rf kdc_realm/sandbox ldap
+	$(RM) au.log
diff --git a/src/tests/au_dict.json b/src/tests/au_dict.json
new file mode 100644
index 0000000..c0a6e64
--- /dev/null
+++ b/src/tests/au_dict.json
@@ -0,0 +1,64 @@
+{
+"event_name":"",
+"event_success":0,
+"evidence_tkt_id":"",
+"fromport":0,
+"fromaddr":{
+	"type":0,
+	"length":0,
+	"ipv4":[]},
+"kdc_status":"",
+"rep_etype":0,
+"rep.ticket":{
+	"authtime":0,
+	"cname":{
+		"components":[],
+		"realm":"",
+		"length":0,
+		"type":0},
+	"end":0,
+	"flags":0,
+	"sess_etype":0,
+	"srv_etype":0,
+	"sname":{
+		"components":[],
+		"realm":"",
+		"length":0,
+		"type":0}},
+"req.avail_etypes":[],
+"req.client":{
+	"components":[],
+	"realm":"",
+	"length":0,
+	"type":0},
+"req_id":"",
+"req.kdc_options":0,
+"req.pa_type":[]
+"req.server":{
+	"components":[],
+	"realm":"",
+	"length":0,
+	"type":0},
+"req.tkt_end":0,
+"s4u2proxy_user":{
+	"components":[],
+	"realm":"",
+	"length":0,
+	"type":0},
+"s4u2self_user":{
+	"components":[],
+	"realm":"",
+	"length":0,
+	"type":0},
+"stage":1,
+"tkt_in_id":"",
+"tkt_renewed":0,
+"tkt_out_id":"",
+"tkt_validated":0,
+"u2u_user":{
+	"components":[],
+	"realm":"",
+	"length":0,
+	"type":0},
+"violation":0
+}
diff --git a/src/tests/jsonwalker.py b/src/tests/jsonwalker.py
new file mode 100644
index 0000000..265c69c
--- /dev/null
+++ b/src/tests/jsonwalker.py
@@ -0,0 +1,113 @@
+#!/usr/bin/python
+
+import sys
+try:
+    import cjson
+except ImportError:
+    print "Warning: skipping audit log verification because the cjson module" \
+          " is unavailable"
+    sys.exit(0)
+from collections import defaultdict
+from optparse import OptionParser
+
+class Parser(object):
+    DEFAULTS = {int:0,
+                str:'',
+                list:[]}
+
+    def __init__(self, defconf=None):
+        self.defaults = None
+        if defconf is not None:
+            self.defaults = self.flatten(defconf)
+
+    def run(self, logs, verbose=None):
+        result = self.parse(logs)
+        if len(result) != len(self.defaults):
+            diff = set(self.defaults.keys()).difference(result.keys())
+            print 'Test failed.'
+            print 'The following attributes were not set:'
+            for it in diff:
+                print it
+            sys.exit(1)
+
+    def flatten(self, defaults):
+        """
+        Flattens pathes to attributes.
+
+        Parameters
+        ----------
+        defaults : a dictionaries populated with default values
+
+        Returns :
+        dict : with flattened attributes
+        """
+        result = dict()
+        for path,value in self._walk(defaults):
+            if path in result:
+                print 'Warning: attribute path %s already exists' % path
+            result[path] = value
+
+        return result
+
+    def parse(self, logs):
+        result = defaultdict(list)
+        for msg in logs:
+            # each message is treated as a dictionary of dictionaries
+            for a,v in self._walk(msg):
+                # see if path is registered in defaults
+                if a in self.defaults:
+                    dv = self.defaults.get(a)
+                    if dv is None:
+                        # determine default value by type
+                        if v is not None:
+                            dv = self.DEFAULTS[type(v)]
+                        else:
+                            print 'Warning: attribute %s is set to None' % a
+                            continue
+                    # by now we have default value
+                    if v != dv:
+                        # test passed
+                        result[a].append(v)
+        return result
+
+    def _walk(self, adict):
+        """
+        Generator that works through dictionary.
+        """
+        for a,v in adict.iteritems():
+            if isinstance(v,dict):
+                for (attrpath,u) in self._walk(v):
+                    yield (a+'.'+attrpath,u)
+            else:
+                yield (a,v)
+
+
+if __name__ == '__main__':
+
+    parser = OptionParser()
+    parser.add_option("-i", "--logfile", dest="filename",
+                  help="input log file in json fmt", metavar="FILE")
+    parser.add_option("-d", "--defaults", dest="defaults",
+                  help="dictionary with defaults", metavar="FILE")
+
+    (options, args) = parser.parse_args()
+    if options.filename is not None:
+        with open(options.filename, 'r') as f:
+            content = list()
+            for l in f:
+                content.append(cjson.decode(l.rstrip()))
+        f.close()
+    else:
+        print 'Input file in jason format is required'
+        exit()
+
+    defaults = None
+    if options.defaults is not None:
+        with open(options.defaults, 'r') as f:
+            defaults = cjson.decode(f.read())
+        f.close()
+
+    # run test
+    p = Parser(defaults)
+    p.run(content)
+    exit()
diff --git a/src/tests/t_audit.py b/src/tests/t_audit.py
new file mode 100644
index 0000000..0cf5254
--- /dev/null
+++ b/src/tests/t_audit.py
@@ -0,0 +1,31 @@
+#!/usr/bin/python
+from k5test import *
+
+conf = {'plugins': {'audit': {
+            'module': 'test:$plugins/audit/test/k5audit_test.so'}}}
+
+realm = K5Realm(krb5_conf=conf, get_creds=False)
+realm.addprinc('target')
+realm.run_kadminl('modprinc +ok_to_auth_as_delegate ' + realm.host_princ)
+
+# Make normal AS and TGS requests so they will be audited.
+realm.kinit(realm.host_princ, flags=['-k', '-f'])
+realm.run([kvno, 'target'])
+
+# Make S4U2Self and S4U2Proxy requests so they will be audited.  The
+# S4U2Proxy request is expected to fail.
+out = realm.run([kvno, '-k', realm.keytab, '-U', 'user', '-P', 'target'],
+                expected_code=1)
+if 'NOT_ALLOWED_TO_DELEGATE' not in out:
+    fail('Unexpected error for S4U2Proxy')
+
+# Make a U2U request so it will be audited.
+uuserver = os.path.join(buildtop, 'appl', 'user_user', 'uuserver')
+uuclient = os.path.join(buildtop, 'appl', 'user_user', 'uuclient')
+port_arg = str(realm.server_port())
+realm.start_server([uuserver, port_arg], 'Server started')
+output = realm.run([uuclient, hostname, 'testing message', port_arg])
+if 'Hello' not in output:
+    fail('U2U request failed unexpectedly')
+
+success('Audit tests')