Improve parsing, insert into datatbase

1 files changed, 62 insertions, 45 deletions

diff --git a/contrib/database/manifest.py b/contrib/database/manifest.py
index 0d18082..1454175 100644
--- a/contrib/database/manifest.py
+++ b/contrib/database/manifest.py
@@ -30,60 +30,75 @@ class AbeManifest(object):
         # Read manifest file
         #
         file = open(filespec, 'r')
-        line = "#"
-        data = dict()
+        done = False
         tool = None
         oldtool = None
         key = None
         value = None
-        while len(line) > 0:
-            line = file.readline().rstrip().lstrip('"')
-            if len(line) == 0 or line == '\n' or line.find('#') >= 0:
-                line = file.readline()
+        data = dict()
+        patterns = ("^target",
+                    ".*_branch",
+                    ".*_filespec",
+                    ".*_revision",
+                    ".*_url",
+                    ".*_md5sum")
+        lines = file.readlines()
+        for line in lines:
+            # skip comments or blank lines
+            if len(line) <= 1 and line[0] == '\n':
+                self.manifest[tool] = data
+                data = dict()
                 continue
-            nodes = line.split('=')
-            if len(nodes) < 2:
-                nodes = line.split(':')
-                if len(nodes) == 2:
-                    key = nodes[0]
-                    value = nodes[1]
-                else:
-                    key = None
-                    value = None
-            else:
-                key = nodes[0]
-                value = nodes[1]
-
-            if not key:
+            if line.find('=') <= 0:
                 continue
-            patterns = ("^target$", "^host$",  "^host_gcc$", ".*_branch", ".*_filespec", ".*_revision", ".*_md5sum", ".*_configure", ".*_url")
+            # We only want a subset of the fields.
             for pat in patterns:
-                m = re.match(pat, key, re.IGNORECASE)
+                m = re.match(pat + "=.*$", line.strip(), re.IGNORECASE)
                 if m is not None:
-                    tool = key.split('_')[0]
-                    if tool == 'target':
-                        entry = line.split('=')[1]
-                    elif key == 'host' or key == 'host_gcc':
-                        entry = line.split(':')[1]
+                    tmp = line.split('=')
+                    tool = tmp[0].split('_')[0]
+                    key = tmp[0].split('_')
+                    if len(key) == 1:
+                        key = key[0]
                     else:
-                        entry = key.split('_')[1]
-                        # print("FIXME: %r, %r, %r" % (tool, entry, value))
-                    data[entry] = value
-            if tool is not None and oldtool != tool:
-                oldtool = tool
-                self.manifest[tool] = data
-                data = dict()
+                        key = key[1]
+                    value = tmp[1].strip()
+                    data[key] = value
 
-    def insert(self):
+    def insert(self, testrun):
         for tool,entry in self.manifest.items():
-            if len(entry) <= 0:
+#            if not tool:
+#                continue
+            query = "INSERT INTO manifest(testrun, tool) VALUES('%d', %r)" % (testrun, tool)
+            # try:
+            #     self.post.execute(query)
+            # except Exception as e:
+            #     if e.pgcode != None:
+            #         print("ERROR: Query failed to fetch! %r" % e.pgerror)
+            #         print("ERROR: Query that failed: %r" % query)
+            #         quit()
+
+            if len(entry) ==0:
+                continue
+
+            if 'branch' in entry and 'revision' in entry:
+                query = """INSERT INTO manifest(testrun, tool, branch, revision) VALUES(%r, %r, %r, %r);""" % (testrun,  tool, entry['branch'], entry['revision'])
+            elif 'filespec' in entry and 'md5sum' in entry:
+                query = """INSERT INTO manifest(testrun, tool, filespec, md5sum) VALUES(%r, %r, %r, %r);"""% (testrun, tool, entry['filespec'], entry['md5sum'])
+            elif 'filespec' in entry and 'md5sum' not in entry:
+                query = """INSERT INTO manifest(testrun, tool, filespec) VALUES(%r, %r, %r);"""% (testrun, tool, self.manifest[tool]['filespec'])
+            elif not 'filespec' in entry and 'md5sum' in entry:
+                continue
+            elif not 'branch' in entry and 'revision' in entry:
                 continue
-            # print("FIXME: %r" % entry)
-            # If filespec is present, it's from a tarball
-            if 'filespec' in entry and 'md5sum' in entry:
-                query = """INSERT INTO manifest(testrun, tool, filespec, md5sum) VALUES(%r, %r, %r, %r);"""% (testrun, tool, manifest[tool]['filespec'], manifest[tool]['md5sum'])
-            elif 'branch' in entry and 'revision' in entry:
-                query = """INSERT INTO manifest(testrun, tool, branch, revision) VALUES(%r, %r, %r, %r);""" % (testrun,  tool, manifest[tool]['branch'], manifest[tool]['revision'])
+
+            try:
+                self.post.execute(query)
+            except Exception as e:
+                if e.pgcode != None:
+                    print("ERROR: Query failed to fetch! %r" % e.pgerror)
+                    print("ERROR: Query that failed: %r" % query)
+                    quit()
 
     def queryManifest(self):
         pass
@@ -104,6 +119,8 @@ class AbeManifest(object):
 
     def dump(self):
         for tool,entry in self.manifest.items():
-            print("Details for %s" % tool)
-            for key,value in entry.items():
-                print("\t%s %s, %s" % (tool, key, value))
+            if tool:
+                if len(entry) > 1:
+                    print("Details for %s" % tool)
+                    for key,value in entry.items():
+                        print("\t%s_%s = %s" % (tool, key, value))