#! /usr/bin/python3 import os.path import sys import shlex import re import tempfile import copy from headerutils import * requires = { } provides = { } no_remove = [ "system.h", "coretypes.h", "config.h" , "bconfig.h", "backend.h" ] # These targets are the ones which provide "coverage". Typically, if any # target is going to fail compilation, it's one of these. This was determined # during the initial runs of reduce-headers... On a full set of target builds, # every failure which occured was triggered by one of these. # This list is used during target-list construction simply to put any of these # *first* in the candidate list, increasing the probability that a failure is # found quickly. target_priority = [ "aarch64-linux-gnu", "arm-netbsdelf", "c6x-elf", "epiphany-elf", "i686-mingw32crt", "i686-pc-msdosdjgpp", "mipsel-elf", "powerpc-eabisimaltivec", "rs6000-ibm-aix5.1.0", "sh-superh-elf", "sparc64-elf" ] target_dir = "" build_dir = "" ignore_list = list() target_builds = list() target_dict = { } header_dict = { } search_path = [ ".", "../include", "../libcpp/include" ] remove_count = { } # Given a header name, normalize it. ie. cp/cp-tree.h could be in gcc, while # the same header could be referenced from within the cp subdirectory as # just cp-tree.h # for now, just assume basenames are unique def normalize_header (header): return os.path.basename (header) # Adds a header file and its sub-includes to the global dictionary if they # aren't already there. Specify s_path since different build directories may # append themselves on demand to the global list. # return entry for the specified header, knowing all sub entries are completed def get_header_info (header, s_path): global header_dict global empty_iinfo process_list = list () location = "" bname = "" bname_iinfo = empty_iinfo for path in s_path: if os.path.exists (path + "/" + header): location = path + "/" + header break if location: bname = normalize_header (location) if header_dict.get (bname): bname_iinfo = header_dict[bname] loc2 = ii_path (bname_iinfo)+ "/" + bname if loc2[:2] == "./": loc2 = loc2[2:] if location[:2] == "./": location = location[2:] if loc2 != location: # Don't use the cache if it isnt the right one. bname_iinfo = process_ii_macro (location) return bname_iinfo bname_iinfo = process_ii_macro (location) header_dict[bname] = bname_iinfo # now decend into the include tree for i in ii_include_list (bname_iinfo): get_header_info (i, s_path) else: # if the file isnt in the source directories, look in the build and target # directories. If it is here, then aggregate all the versions. location = build_dir + "/gcc/" + header build_inc = target_inc = False if os.path.exists (location): build_inc = True for x in target_dict: location = target_dict[x] + "/gcc/" + header if os.path.exists (location): target_inc = True break if (build_inc or target_inc): bname = normalize_header(header) defines = set() consumes = set() incl = set() if build_inc: iinfo = process_ii_macro (build_dir + "/gcc/" + header) defines = set (ii_macro_define (iinfo)) consumes = set (ii_macro_consume (iinfo)) incl = set (ii_include_list (iinfo)) if (target_inc): for x in target_dict: location = target_dict[x] + "/gcc/" + header if os.path.exists (location): iinfo = process_ii_macro (location) defines.update (ii_macro_define (iinfo)) consumes.update (ii_macro_consume (iinfo)) incl.update (ii_include_list (iinfo)) bname_iinfo = (header, "build", list(incl), list(), list(consumes), list(defines), list(), list()) header_dict[bname] = bname_iinfo for i in incl: get_header_info (i, s_path) return bname_iinfo # return a list of all headers brought in by this header def all_headers (fname): global header_dict headers_stack = list() headers_list = list() if header_dict.get (fname) == None: return list () for y in ii_include_list (header_dict[fname]): headers_stack.append (y) while headers_stack: h = headers_stack.pop () hn = normalize_header (h) if hn not in headers_list: headers_list.append (hn) if header_dict.get(hn): for y in ii_include_list (header_dict[hn]): if normalize_header (y) not in headers_list: headers_stack.append (y) return headers_list # Search bld_dir for all target tuples, confirm that they have a build path with # bld_dir/target-tuple/gcc, and build a dictionary of build paths indexed by # target tuple.. def build_target_dict (bld_dir, just_these): global target_dict target_doct = { } error = False if os.path.exists (bld_dir): if just_these: ls = just_these else: ls = os.listdir(bld_dir) for t in ls: if t.find("-") != -1: target = t.strip() tpath = bld_dir + "/" + target if not os.path.exists (tpath + "/gcc"): print ("Error: gcc build directory for target " + t + " Does not exist: " + tpath + "/gcc") error = True else: target_dict[target] = tpath if error: target_dict = { } def get_obj_name (src_file): if src_file[-2:] == ".c": return src_file.replace (".c", ".o") elif src_file[-3:] == ".cc": return src_file.replace (".cc", ".o") return "" def target_obj_exists (target, obj_name): global target_dict # look in a subdir if src has a subdir, then check gcc base directory. if target_dict.get(target): obj = target_dict[target] + "/gcc/" + obj_name if not os.path.exists (obj): obj = target_dict[target] + "/gcc/" + os.path.basename(obj_name) if os.path.exists (obj): return True return False # Given a src file, return a list of targets which may build this file. def find_targets (src_file): global target_dict targ_list = list() obj_name = get_obj_name (src_file) if not obj_name: print ("Error: " + src_file + " - Cannot determine object name.") return list() # Put the high priority targets which tend to trigger failures first for target in target_priority: if target_obj_exists (target, obj_name): targ_list.append ((target, target_dict[target])) for target in target_dict: if target not in target_priority and target_obj_exists (target, obj_name): targ_list.append ((target, target_dict[target])) return targ_list def try_to_remove (src_file, h_list, verbose): global target_dict global header_dict global build_dir # build from scratch each time header_dict = { } summary = "" rmcount = 0 because = { } src_info = process_ii_macro_src (src_file) src_data = ii_src (src_info) if src_data: inclist = ii_include_list_non_cond (src_info) # work is done if there are no includes to check if not inclist: return src_file + ": No include files to attempt to remove" # work on the include list in reverse. inclist.reverse() # Get the target list targ_list = list() targ_list = find_targets (src_file) spath = search_path if os.path.dirname (src_file): spath.append (os.path.dirname (src_file)) hostbuild = True if src_file.find("config/") != -1: # config files dont usually build on the host hostbuild = False obn = get_obj_name (os.path.basename (src_file)) if obn and os.path.exists (build_dir + "/gcc/" + obn): hostbuild = True if not target_dict: summary = src_file + ": Target builds are required for config files. None found." print (summary) return summary if not targ_list: summary =src_file + ": Cannot find any targets which build this file." print (summary) return summary if hostbuild: # confirm it actually builds before we do anything print ("Confirming source file builds") res = get_make_output (build_dir + "/gcc", "all") if res[0] != 0: message = "Error: " + src_file + " does not build currently." summary = src_file + " does not build on host." print (message) print (res[1]) if verbose: verbose.write (message + "\n") verbose.write (res[1]+ "\n") return summary src_requires = set (ii_macro_consume (src_info)) for macro in src_requires: because[macro] = src_file header_seen = list () os.rename (src_file, src_file + ".bak") src_orig = copy.deepcopy (src_data) src_tmp = copy.deepcopy (src_data) try: # process the includes from bottom to top. This is because we know that # later includes have are known to be needed, so any dependency from this # header is a true dependency for inc_file in inclist: inc_file_norm = normalize_header (inc_file) if inc_file in no_remove: continue if len (h_list) != 0 and inc_file_norm not in h_list: continue if inc_file_norm[0:3] == "gt-": continue if inc_file_norm[0:6] == "gtype-": continue if inc_file_norm.replace(".h",".c") == os.path.basename(src_file): continue lookfor = ii_src_line(src_info)[inc_file] src_tmp.remove (lookfor) message = "Trying " + src_file + " without " + inc_file print (message) if verbose: verbose.write (message + "\n") out = open(src_file, "w") for line in src_tmp: out.write (line) out.close() keep = False if hostbuild: res = get_make_output (build_dir + "/gcc", "all") else: res = (0, "") rc = res[0] message = "Passed Host build" if (rc != 0): # host build failed message = "Compilation failed:\n"; keep = True else: if targ_list: objfile = get_obj_name (src_file) t1 = targ_list[0] if objfile and os.path.exists(t1[1] +"/gcc/"+objfile): res = get_make_output_parallel (targ_list, objfile, 0) else: res = get_make_output_parallel (targ_list, "all-gcc", 0) rc = res[0] if rc != 0: message = "Compilation failed on TARGET : " + res[2] keep = True else: message = "Passed host and target builds" if keep: print (message + "\n") if (rc != 0): if verbose: verbose.write (message + "\n"); verbose.write (res[1]) verbose.write ("\n"); if os.path.exists (inc_file): ilog = open(inc_file+".log","a") ilog.write (message + " for " + src_file + ":\n\n"); ilog.write ("============================================\n"); ilog.write (res[1]) ilog.write ("\n"); ilog.close() if os.path.exists (src_file): ilog = open(src_file+".log","a") ilog.write (message + " for " +inc_file + ":\n\n"); ilog.write ("============================================\n"); ilog.write (res[1]) ilog.write ("\n"); ilog.close() # Given a sequence where : # #include "tm.h" # #include "target.h" // includes tm.h # target.h was required, and when attempting to remove tm.h we'd see that # all the macro defintions are "required" since they all look like: # #ifndef HAVE_blah # #define HAVE_blah # endif # when target.h was found to be required, tm.h will be tagged as included. # so when we get this far, we know we dont have to check the macros for # tm.h since we know it is already been included. if inc_file_norm not in header_seen: iinfo = get_header_info (inc_file, spath) newlist = all_headers (inc_file_norm) if ii_path(iinfo) == "build" and not target_dict: keep = True text = message + " : Will not remove a build file without some targets." print (text) ilog = open(src_file+".log","a") ilog.write (text +"\n") ilog.write ("============================================\n"); ilog.close() ilog = open("reduce-headers-kept.log","a") ilog.write (src_file + " " + text +"\n") ilog.close() else: newlist = list() if not keep and inc_file_norm not in header_seen: # now look for any macro requirements. for h in newlist: if not h in header_seen: if header_dict.get(h): defined = ii_macro_define (header_dict[h]) for dep in defined: if dep in src_requires and dep not in ignore_list: keep = True; text = message + ", but must keep " + inc_file + " because it provides " + dep if because.get(dep) != None: text = text + " Possibly required by " + because[dep] print (text) ilog = open(inc_file+".log","a") ilog.write (because[dep]+": Requires [dep] in "+src_file+"\n") ilog.write ("============================================\n"); ilog.close() ilog = open(src_file+".log","a") ilog.write (text +"\n") ilog.write ("============================================\n"); ilog.close() ilog = open("reduce-headers-kept.log","a") ilog.write (src_file + " " + text +"\n") ilog.close() if verbose: verbose.write (text + "\n") if keep: # add all headers 'consumes' to src_requires list, and mark as seen for h in newlist: if not h in header_seen: header_seen.append (h) if header_dict.get(h): consume = ii_macro_consume (header_dict[h]) for dep in consume: if dep not in src_requires: src_requires.add (dep) if because.get(dep) == None: because[dep] = inc_file src_tmp = copy.deepcopy (src_data) else: print (message + " --> removing " + inc_file + "\n") rmcount += 1 if verbose: verbose.write (message + " --> removing " + inc_file + "\n") if remove_count.get(inc_file) == None: remove_count[inc_file] = 1 else: remove_count[inc_file] += 1 src_data = copy.deepcopy (src_tmp) except: print ("Interuption: restoring original file") out = open(src_file, "w") for line in src_orig: out.write (line) out.close() raise # copy current version, since it is the "right" one now. out = open(src_file, "w") for line in src_data: out.write (line) out.close() # Try a final host bootstrap build to make sure everything is kosher. if hostbuild: res = get_make_output (build_dir, "all") rc = res[0] if (rc != 0): # host build failed! return to original version print ("Error: " + src_file + " Failed to bootstrap at end!!! restoring.") print (" Bad version at " + src_file + ".bad") os.rename (src_file, src_file + ".bad") out = open(src_file, "w") for line in src_orig: out.write (line) out.close() return src_file + ": failed to build after reduction. Restored original" if src_data == src_orig: summary = src_file + ": No change." else: summary = src_file + ": Reduction performed, "+str(rmcount)+" includes removed." print (summary) return summary only_h = list () ignore_cond = False usage = False src = list() only_targs = list () for x in sys.argv[1:]: if x[0:2] == "-b": build_dir = x[2:] elif x[0:2] == "-f": fn = normalize_header (x[2:]) if fn not in only_h: only_h.append (fn) elif x[0:2] == "-h": usage = True elif x[0:2] == "-d": ignore_cond = True elif x[0:2] == "-D": ignore_list.append(x[2:]) elif x[0:2] == "-T": only_targs.append(x[2:]) elif x[0:2] == "-t": target_dir = x[2:] elif x[0] == "-": print ("Error: Unrecognized option " + x) usgae = True else: if not os.path.exists (x): print ("Error: specified file " + x + " does not exist.") usage = True else: src.append (x) if target_dir: build_target_dict (target_dir, only_targs) if build_dir == "" and target_dir == "": print ("Error: Must specify a build directory, and/or a target directory.") usage = True if build_dir and not os.path.exists (build_dir): print ("Error: specified build directory does not exist : " + build_dir) usage = True if target_dir and not os.path.exists (target_dir): print ("Error: specified target directory does not exist : " + target_dir) usage = True if usage: print ("Attempts to remove extraneous include files from source files.") print (" ") print ("Should be run from the main gcc source directory, and works on a target") print ("directory, as we attempt to make the 'all' target.") print (" ") print ("By default, gcc-reorder-includes is run on each file before attempting") print ("to remove includes. this removes duplicates and puts some headers in a") print ("canonical ordering") print (" ") print ("The build directory should be ready to compile via make. Time is saved") print ("if the build is already complete, so that only changes need to be built.") print (" ") print ("Usage: [options] file1.c [file2.c] ... [filen.c]") print (" -bdir : the root build directory to attempt buiding .o files.") print (" -tdir : the target build directory") print (" -d : Ignore conditional macro dependencies.") print (" ") print (" -Dmacro : Ignore a specific macro for dependencies") print (" -Ttarget : Only consider target in target directory.") print (" -fheader : Specifies a specific .h file to be considered.") print (" ") print (" -D, -T, and -f can be specified mulitple times and are aggregated.") print (" ") print (" The original file will be in filen.bak") print (" ") sys.exit (0) if only_h: print ("Attempting to remove only these files:") for x in only_h: print (x) print (" ") logfile = open("reduce-headers.log","w") for x in src: msg = try_to_remove (x, only_h, logfile) ilog = open("reduce-headers.sum","a") ilog.write (msg + "\n") ilog.close() ilog = open("reduce-headers.sum","a") ilog.write ("===============================================================\n") for x in remove_count: msg = x + ": Removed " + str(remove_count[x]) + " times." print (msg) logfile.write (msg + "\n") ilog.write (msg + "\n")