count-headers: Initial file.

2015-11-11  Andrew MacLeod  <amacleod@redhat.com>

	* count-headers: Initial file.
	* gcc-order-headers: Initial file.
	* graph-header-logs: Initial file.
	* graph-include-web: Initial file.
	* headerutils.py: Initial file.
	* included-by: Initial file.
	* README: Initial file.
	* reduce-headers: Initial file.
	* replace-header: Initial file.
	* show-headers: Initial file.

From-SVN: r230171

1 files changed, 554 insertions, 0 deletions

diff --git a/contrib/header-tools/headerutils.py b/contrib/header-tools/headerutils.py
new file mode 100755
index 0000000..95c47fb
--- /dev/null
+++ b/contrib/header-tools/headerutils.py
@@ -0,0 +1,554 @@
+#! /usr/bin/python2
+import os.path
+import sys
+import shlex
+import re
+import subprocess
+import shutil
+import pickle
+
+import multiprocessing 
+
+def find_pound_include (line, use_outside, use_slash):
+  inc = re.findall (ur"^\s*#\s*include\s*\"(.+?)\"", line)
+  if len(inc) == 1:
+    nm = inc[0]
+    if use_outside or os.path.exists (nm):
+      if use_slash or '/' not in nm:
+        return nm
+  return ""
+
+def find_system_include (line):
+  inc = re.findall (ur"^\s*#\s*include\s*<(.+?)>", line)
+  if len(inc) == 1:
+    return inc[0]
+  return ""
+  
+def find_pound_define (line):
+  inc = re.findall (ur"^\s*#\s*define ([A-Za-z0-9_]+)", line)
+  if len(inc) != 0:
+    if len(inc) > 1:
+      print "What? more than 1 match in #define??"
+      print inc
+      sys.exit(5)
+    return inc[0];
+  return ""
+
+def is_pound_if (line):
+  inc = re.findall ("^\s*#\s*if\s", line)
+  if not inc:
+    inc = re.findall ("^\s*#\s*if[n]?def\s", line)
+  if inc:
+    return True
+  return False
+
+def is_pound_endif (line):
+  inc = re.findall ("^\s*#\s*endif", line)
+  if inc:
+    return True
+  return False
+
+def find_pound_if (line):
+  inc = re.findall (ur"^\s*#\s*if\s+(.*)", line)
+  if len(inc) == 0:
+    inc = re.findall (ur"^\s*#\s*elif\s+(.*)", line)
+  if len(inc) > 0:
+    inc2 = re.findall (ur"defined\s*\((.+?)\)", inc[0])
+    inc3 = re.findall (ur"defined\s+([a-zA-Z0-9_]+)", inc[0])
+    for yy in inc3:
+      inc2.append (yy)
+    return inc2
+  else:
+    inc = re.findall (ur"^\s*#\s*ifdef\s(.*)", line)
+    if len(inc) == 0:
+      inc = re.findall (ur"^\s*#\s*ifndef\s(.*)", line)
+    if len(inc) > 0:
+      inc2 = re.findall ("[A-Za-z_][A-Za-z_0-9]*", inc[0])
+      return inc2
+  if len(inc) == 0:
+    return list ()
+  print "WTF. more than one line returned for find_pound_if"
+  print inc
+  sys.exit(5)
+
+
+# IINFO - this is a vector of include information. It consists of 7 elements.
+# [0] - base name of the file
+# [1] - path leading to this file.
+# [2] - orderd list of all headers directly included by this file.
+# [3] - Ordered list of any headers included within condionally compiled code.
+#       headers files are expected to have all includes one level deep due to
+#       the omnipresent guards at the top of the file.  
+# [4] - List of all macros which are consumed (used) within this file.
+# [5] - list of all macros which may be defined in this file.
+# [6] - The source code for this file, if cached.
+# [7] - line number info for any headers in the source file.  Indexed by base
+#       name, returning the line the include is on.
+
+empty_iinfo =  ("", "", list(), list(), list(), list(), list())
+
+# This function will process a file and extract interesting information.
+# DO_MACROS indicates whether macros defined and used should be recorded.
+# KEEP_SRC indicates the source for the file should be cached.
+def process_include_info (filen, do_macros, keep_src):
+  header = False
+  if not os.path.exists (filen):
+    return empty_iinfo
+
+  sfile = open (filen, "r");
+  data = sfile.readlines()
+  sfile.close()
+
+  # Ignore the initial #ifdef HEADER_H in header files
+  if filen[-2:] == ".h":
+    nest = -1
+    header = True
+  else:
+    nest = 0
+
+  macout = list ()
+  macin = list()
+  incl = list()
+  cond_incl = list()
+  src_line = { }
+  guard = ""
+
+  for line in (data):
+    if is_pound_if (line):
+      nest += 1
+    elif is_pound_endif (line):
+      nest -= 1
+
+    nm = find_pound_include (line, True, True)
+    if nm != "" and nm not in incl and nm[-2:] == ".h":
+      incl.append (nm)
+      if nest > 0:
+        cond_incl.append (nm)
+      if keep_src:
+        src_line[nm] = line
+      continue
+
+    if do_macros:
+      d = find_pound_define (line)
+      if d:
+        if d not in macout:
+          macout.append (d);
+          continue
+
+      d = find_pound_if (line)
+      if d:
+        # The first #if in a header file should be the guard
+        if header and len (d) == 1 and guard == "":
+          if d[0][-2:] == "_H":
+            guard = d
+          else:
+            guard = "Guess there was no guard..."
+        else:
+          for mac in d:
+            if mac != "defined" and mac not in macin:
+              macin.append (mac);
+
+  if not keep_src:
+    data = list()
+
+  return (os.path.basename (filen), os.path.dirname (filen), incl, cond_incl,
+          macin, macout, data, src_line)
+
+# Extract header info, but no macros or source code.
+def process_ii (filen):
+  return process_include_info (filen, False, False)
+
+# Extract header information, and collect macro information.
+def process_ii_macro (filen):
+  return process_include_info (filen, True, False)
+
+# Extract header information, cache the source lines.
+def process_ii_src (filen):
+  return process_include_info (filen, False, True)
+
+# Extract header information, coolewc macro info and cache the source lines.
+def process_ii_macro_src (filen):
+  return process_include_info (filen, True, True)
+
+
+def ii_base (iinfo):
+  return iinfo[0]
+
+def ii_path (iinfo):
+  return iinfo[1]
+
+def ii_include_list (iinfo):
+  return iinfo[2]
+
+def ii_include_list_cond (iinfo):
+  return iinfo[3]
+
+def ii_include_list_non_cond (iinfo):
+  l = ii_include_list (iinfo)
+  for n in ii_include_list_cond (iinfo):
+    l.remove (n)
+  return l
+
+def ii_macro_consume (iinfo):
+  return iinfo[4]
+  
+def ii_macro_define (iinfo):
+  return iinfo[5]
+
+def ii_src (iinfo):
+  return iinfo[6]
+
+def ii_src_line (iinfo):
+  return iinfo[7]
+
+def ii_read (fname):
+  f = open (fname, 'rb')
+  incl = pickle.load (f)
+  consumes = pickle.load (f)
+  defines = pickle.load (f)
+  obj = (fname,fname,incl,list(), list(), consumes, defines, list(), list())
+  return obj
+
+def ii_write (fname, obj):
+  f = open (fname, 'wb')
+  pickle.dump (obj[2], f)
+  pickle.dump (obj[4], f)
+  pickle.dump (obj[5], f)
+  f.close ()
+
+# execute a system command which returns file names
+def execute_command (command):
+  files = list()
+  f = os.popen (command)
+  for x in f:
+    if x[0:2] == "./":
+      fn = x.rstrip()[2:]
+    else:
+      fn = x.rstrip()
+    files.append(fn)
+  return files
+
+# Try to locate a build directory from PATH
+def find_gcc_bld_dir (path):
+  blddir = ""
+  # Look for blddir/gcc/tm.h
+  command = "find " + path + " -mindepth 2 -maxdepth 3 -name tm.h"
+  files = execute_command (command)
+  for y in files:
+    p = os.path.dirname (y)
+    if os.path.basename (p) == "gcc":
+      blddir = p
+      break
+  # If not found, try looking a bit deeper
+  # Dont look this deep initially because a lot of cross target builds may show
+  # up in the list before a native build... but those are better than nothing.
+  if not blddir:
+    command = "find " + path + " -mindepth 3 -maxdepth 5 -name tm.h"
+    files = execute_command (command)
+    for y in files:
+      p = os.path.dirname (y)
+      if os.path.basename (p) == "gcc":
+	blddir = p
+	break
+
+  return blddir
+
+
+# Find files matching pattern NAME, return in a list.
+# CURRENT is True if you want to include the current directory
+# DEEPER is True if you want to search 3 levels below the current directory
+# any files with testsuite diurectories are ignored
+
+def find_gcc_files (name, current, deeper):
+  files = list()
+  command = ""
+  if current:
+    if not deeper:
+      command = "find -maxdepth 1 -name " + name + " -not -path \"./testsuite/*\""
+    else:
+      command = "find -maxdepth 4 -name " + name + " -not -path \"./testsuite/*\""
+  else:
+    if deeper:
+      command = "find -maxdepth 4 -mindepth 2 -name " + name + " -not -path \"./testsuite/*\""
+
+  if command != "":
+    files = execute_command (command)
+
+  return files
+
+# find the list of unique include names found in a file.
+def find_unique_include_list_src (data):
+  found = list ()
+  for line in data:
+    d = find_pound_include (line, True, True)
+    if d and d not in found and d[-2:] == ".h":
+      found.append (d)
+  return found
+
+# find the list of unique include names found in a file.
+def find_unique_include_list (filen):
+  data = open (filen).read().splitlines()
+  return find_unique_include_list_src (data)
+
+
+# Create the macin, macout, and incl vectors for a file FILEN.
+# macin are the macros that are used in #if* conditional expressions
+# macout are the macros which are #defined
+# incl is the list of incluide files encountered
+# returned as a tuple of the filename followed by the triplet of lists
+# (filen, macin, macout, incl)
+
+def create_macro_in_out (filen):
+  sfile = open (filen, "r");
+  data = sfile.readlines()
+  sfile.close()
+
+  macout = list ()
+  macin = list()
+  incl = list()
+
+  for line in (data):
+    d = find_pound_define (line)
+    if d != "":
+      if d not in macout:
+        macout.append (d);
+      continue
+
+    d = find_pound_if (line)
+    if len(d) != 0:
+      for mac in d:
+        if mac != "defined" and mac not in macin:
+          macin.append (mac);
+      continue
+
+    nm = find_pound_include (line, True, True)
+    if nm != "" and nm not in incl:
+      incl.append (nm)
+
+  return (filen, macin, macout, incl)
+
+# create the macro information for filen, and create .macin, .macout, and .incl
+# files.  Return the created macro tuple.
+def create_include_data_files (filen):
+
+  macros = create_macro_in_out (filen)
+  depends = macros[1]
+  defines = macros[2]
+  incls = macros[3]
+  
+  disp_message = filen
+  if len (defines) > 0:
+    disp_message = disp_message + " " + str(len (defines)) + " #defines"
+  dfile = open (filen + ".macout", "w")
+  for x in defines:
+    dfile.write (x + "\n")
+  dfile.close ()
+
+  if len (depends) > 0:
+    disp_message = disp_message + " " + str(len (depends)) + " #if dependencies"
+  dfile = open (filen + ".macin", "w")
+  for x in depends:
+    dfile.write (x + "\n")
+  dfile.close ()
+
+  if len (incls) > 0:
+    disp_message = disp_message + " " + str(len (incls)) + " #includes"
+  dfile = open (filen + ".incl", "w")
+  for x in incls:
+    dfile.write (x + "\n")
+  dfile.close ()
+
+  return macros
+
+
+
+# extract data for include file name_h and enter it into the dictionary.
+# this does not change once read in.  use_requires is True if you want to 
+# prime the values with already created .requires and .provides files.
+def get_include_data (name_h, use_requires):
+  macin = list()
+  macout = list()
+  incl = list ()
+  if use_requires and os.path.exists (name_h + ".requires"):
+    macin = open (name_h + ".requires").read().splitlines()
+  elif os.path.exists (name_h + ".macin"):
+    macin = open (name_h + ".macin").read().splitlines()
+
+  if use_requires and os.path.exists (name_h + ".provides"):
+    macout  = open (name_h + ".provides").read().splitlines()
+  elif os.path.exists (name_h + ".macout"):
+    macout  = open (name_h + ".macout").read().splitlines()
+
+  if os.path.exists (name_h + ".incl"):
+    incl = open (name_h + ".incl").read().splitlines()
+
+  if len(macin) == 0 and len(macout) == 0 and len(incl) == 0:
+    return ()
+  data = ( name_h, macin, macout, incl )
+  return data
+  
+# find FIND in src, and replace it with the list of headers in REPLACE.
+# Remove any duplicates of FIND in REPLACE, and if some of the REPLACE
+# headers occur earlier in the include chain, leave them.
+# Return the new SRC only if anything changed.
+def find_replace_include (find, replace, src):
+  res = list()
+  seen = { }
+  anything = False
+  for line in src:
+    inc = find_pound_include (line, True, True)
+    if inc == find:
+      for y in replace:
+        if seen.get(y) == None:
+          res.append("#include \""+y+"\"\n")
+          seen[y] = True
+          if y != find:
+            anything = True
+# if find isnt in the replacement list, then we are deleting FIND, so changes.
+      if find not in replace:
+        anything = True
+    else:
+      if inc in replace:
+        if seen.get(inc) == None:
+          res.append (line)
+          seen[inc] = True
+      else:
+        res.append (line)
+
+  if (anything):
+    return res
+  else:
+    return list()
+      
+
+# pass in a require and provide dictionary to be read in.
+def read_require_provides (require, provide):
+  if not os.path.exists ("require-provide.master"):
+    print "require-provide.master file is not available. please run data collection."
+    sys.exit(1)
+  incl_list = open("require-provide.master").read().splitlines()
+  for f in incl_list:
+    if os.path.exists (f+".requires"):
+      require[os.path.basename (f)] = open (f + ".requires").read().splitlines()
+    else:
+      require[os.path.basename (f)] = list ()
+    if os.path.exists (f+".provides"):
+      provide[os.path.basename (f)] = open (f + ".provides").read().splitlines()
+    else:
+      provide [os.path.basename (f)] = list ()
+
+   
+def build_include_list (filen):
+  include_files = list()
+  sfile = open (filen, "r")
+  data = sfile.readlines()
+  sfile.close()
+  for line in data:
+    nm = find_pound_include (line, False, False)
+    if nm != "" and nm[-2:] == ".h":
+      if nm not in include_files:
+        include_files.append(nm)
+  return include_files
+ 
+def build_reverse_include_list (filen):
+  include_files = list()
+  sfile = open (filen, "r")
+  data = sfile.readlines()
+  sfile.close()
+  for line in reversed(data):
+    nm = find_pound_include (line, False, False)
+    if nm != "":
+      if nm not in include_files:
+        include_files.append(nm)
+  return include_files
+     
+# Get compilation return code, and compensate for a warning that we want to 
+# consider an error when it comes to inlined templates.
+def get_make_rc (rc, output):
+  rc = rc % 1280
+  if rc == 0:
+    # This is not considered an error during compilation of an individual file,
+    # but it will cause an error during link if it isn't defined.  If this
+    # warning is seen during compiling a file, make it a build error so we 
+    # don't remove the header.
+    h = re.findall ("warning: inline function.*used but never defined", output)
+    if len(h) != 0:
+      rc = 1
+  return rc;
+
+def get_make_output (build_dir, make_opt):
+  devnull = open('/dev/null', 'w')
+  at_a_time = multiprocessing.cpu_count() * 2
+  make = "make -j"+str(at_a_time)+ " "
+  if build_dir != "":
+    command = "cd " + build_dir +"; " + make + make_opt
+  else:
+    command = make + make_opt
+  process = subprocess.Popen(command, stdout=devnull, stderr=subprocess.PIPE, shell=True)
+  output = process.communicate();
+  rc = get_make_rc (process.returncode, output[1])
+  return (rc , output[1])
+
+def spawn_makes (command_list):
+  devnull = open('/dev/null', 'w')
+  rc = (0,"", "")
+  proc_res = list()
+  text = "  Trying target builds : "
+  for command_pair in command_list:
+    tname = command_pair[0]
+    command = command_pair[1]
+    text += tname + ", "
+    c = subprocess.Popen(command, bufsize=-1, stdout=devnull, stderr=subprocess.PIPE, shell=True)
+    proc_res.append ((c, tname))
+
+  print text[:-2]
+
+  for p in proc_res:
+    output = p[0].communicate()
+    ret = (get_make_rc (p[0].returncode, output[1]), output[1], p[1])
+    if (ret[0] != 0):
+      # Just record the first one.
+      if rc[0] == 0:
+        rc = ret;
+  return rc
+
+def get_make_output_parallel (targ_list, make_opt, at_a_time):
+  command = list()
+  targname = list()
+  if at_a_time == 0:
+    at_a_time = multiprocessing.cpu_count() * 2
+  proc_res = [0] * at_a_time
+  for x in targ_list:
+    if make_opt[-2:] == ".o":
+      s = "cd " + x[1] + "/gcc/; make " + make_opt
+    else:
+      s = "cd " + x[1] +"; make " + make_opt
+    command.append ((x[0],s))
+
+  num = len(command) 
+  rc = (0,"", "")
+  loops = num // at_a_time
+  
+  if (loops > 0):
+    for idx in range (loops):
+      ret = spawn_makes (command[idx*at_a_time:(idx+1)*at_a_time])
+      if ret[0] != 0:
+        rc = ret
+        break
+
+  if (rc[0] == 0):
+    leftover = num % at_a_time
+    if (leftover > 0):
+      ret = spawn_makes (command[-leftover:])
+      if ret[0] != 0:
+        rc = ret
+
+  return rc
+
+
+def readwholefile (src_file):
+  sfile = open (src_file, "r")
+  src_data = sfile.readlines()
+  sfile.close()
+  return src_data
+