1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
|
#! /usr/bin/python3
import os.path
import sys
import shlex
import re
import tempfile
import copy
from headerutils import *
requires = { }
provides = { }
no_remove = [ "system.h", "coretypes.h", "config.h" , "bconfig.h", "backend.h" ]
# These targets are the ones which provide "coverage". Typically, if any
# target is going to fail compilation, it's one of these. This was determined
# during the initial runs of reduce-headers... On a full set of target builds,
# every failure which occured was triggered by one of these.
# This list is used during target-list construction simply to put any of these
# *first* in the candidate list, increasing the probability that a failure is
# found quickly.
target_priority = [
"aarch64-linux-gnu",
"arm-netbsdelf",
"c6x-elf",
"epiphany-elf",
"i686-mingw32crt",
"i686-pc-msdosdjgpp",
"mipsel-elf",
"powerpc-eabisimaltivec",
"rs6000-ibm-aix5.1.0",
"sh-superh-elf",
"sparc64-elf"
]
target_dir = ""
build_dir = ""
ignore_list = list()
target_builds = list()
target_dict = { }
header_dict = { }
search_path = [ ".", "../include", "../libcpp/include" ]
remove_count = { }
# Given a header name, normalize it. ie. cp/cp-tree.h could be in gcc, while
# the same header could be referenced from within the cp subdirectory as
# just cp-tree.h
# for now, just assume basenames are unique
def normalize_header (header):
return os.path.basename (header)
# Adds a header file and its sub-includes to the global dictionary if they
# aren't already there. Specify s_path since different build directories may
# append themselves on demand to the global list.
# return entry for the specified header, knowing all sub entries are completed
def get_header_info (header, s_path):
global header_dict
global empty_iinfo
process_list = list ()
location = ""
bname = ""
bname_iinfo = empty_iinfo
for path in s_path:
if os.path.exists (path + "/" + header):
location = path + "/" + header
break
if location:
bname = normalize_header (location)
if header_dict.get (bname):
bname_iinfo = header_dict[bname]
loc2 = ii_path (bname_iinfo)+ "/" + bname
if loc2[:2] == "./":
loc2 = loc2[2:]
if location[:2] == "./":
location = location[2:]
if loc2 != location:
# Don't use the cache if it isnt the right one.
bname_iinfo = process_ii_macro (location)
return bname_iinfo
bname_iinfo = process_ii_macro (location)
header_dict[bname] = bname_iinfo
# now decend into the include tree
for i in ii_include_list (bname_iinfo):
get_header_info (i, s_path)
else:
# if the file isnt in the source directories, look in the build and target
# directories. If it is here, then aggregate all the versions.
location = build_dir + "/gcc/" + header
build_inc = target_inc = False
if os.path.exists (location):
build_inc = True
for x in target_dict:
location = target_dict[x] + "/gcc/" + header
if os.path.exists (location):
target_inc = True
break
if (build_inc or target_inc):
bname = normalize_header(header)
defines = set()
consumes = set()
incl = set()
if build_inc:
iinfo = process_ii_macro (build_dir + "/gcc/" + header)
defines = set (ii_macro_define (iinfo))
consumes = set (ii_macro_consume (iinfo))
incl = set (ii_include_list (iinfo))
if (target_inc):
for x in target_dict:
location = target_dict[x] + "/gcc/" + header
if os.path.exists (location):
iinfo = process_ii_macro (location)
defines.update (ii_macro_define (iinfo))
consumes.update (ii_macro_consume (iinfo))
incl.update (ii_include_list (iinfo))
bname_iinfo = (header, "build", list(incl), list(), list(consumes), list(defines), list(), list())
header_dict[bname] = bname_iinfo
for i in incl:
get_header_info (i, s_path)
return bname_iinfo
# return a list of all headers brought in by this header
def all_headers (fname):
global header_dict
headers_stack = list()
headers_list = list()
if header_dict.get (fname) == None:
return list ()
for y in ii_include_list (header_dict[fname]):
headers_stack.append (y)
while headers_stack:
h = headers_stack.pop ()
hn = normalize_header (h)
if hn not in headers_list:
headers_list.append (hn)
if header_dict.get(hn):
for y in ii_include_list (header_dict[hn]):
if normalize_header (y) not in headers_list:
headers_stack.append (y)
return headers_list
# Search bld_dir for all target tuples, confirm that they have a build path with
# bld_dir/target-tuple/gcc, and build a dictionary of build paths indexed by
# target tuple..
def build_target_dict (bld_dir, just_these):
global target_dict
target_doct = { }
error = False
if os.path.exists (bld_dir):
if just_these:
ls = just_these
else:
ls = os.listdir(bld_dir)
for t in ls:
if t.find("-") != -1:
target = t.strip()
tpath = bld_dir + "/" + target
if not os.path.exists (tpath + "/gcc"):
print ("Error: gcc build directory for target " + t + " Does not exist: " + tpath + "/gcc")
error = True
else:
target_dict[target] = tpath
if error:
target_dict = { }
def get_obj_name (src_file):
if src_file[-2:] == ".c":
return src_file.replace (".c", ".o")
elif src_file[-3:] == ".cc":
return src_file.replace (".cc", ".o")
return ""
def target_obj_exists (target, obj_name):
global target_dict
# look in a subdir if src has a subdir, then check gcc base directory.
if target_dict.get(target):
obj = target_dict[target] + "/gcc/" + obj_name
if not os.path.exists (obj):
obj = target_dict[target] + "/gcc/" + os.path.basename(obj_name)
if os.path.exists (obj):
return True
return False
# Given a src file, return a list of targets which may build this file.
def find_targets (src_file):
global target_dict
targ_list = list()
obj_name = get_obj_name (src_file)
if not obj_name:
print ("Error: " + src_file + " - Cannot determine object name.")
return list()
# Put the high priority targets which tend to trigger failures first
for target in target_priority:
if target_obj_exists (target, obj_name):
targ_list.append ((target, target_dict[target]))
for target in target_dict:
if target not in target_priority and target_obj_exists (target, obj_name):
targ_list.append ((target, target_dict[target]))
return targ_list
def try_to_remove (src_file, h_list, verbose):
global target_dict
global header_dict
global build_dir
# build from scratch each time
header_dict = { }
summary = ""
rmcount = 0
because = { }
src_info = process_ii_macro_src (src_file)
src_data = ii_src (src_info)
if src_data:
inclist = ii_include_list_non_cond (src_info)
# work is done if there are no includes to check
if not inclist:
return src_file + ": No include files to attempt to remove"
# work on the include list in reverse.
inclist.reverse()
# Get the target list
targ_list = list()
targ_list = find_targets (src_file)
spath = search_path
if os.path.dirname (src_file):
spath.append (os.path.dirname (src_file))
hostbuild = True
if src_file.find("config/") != -1:
# config files dont usually build on the host
hostbuild = False
obn = get_obj_name (os.path.basename (src_file))
if obn and os.path.exists (build_dir + "/gcc/" + obn):
hostbuild = True
if not target_dict:
summary = src_file + ": Target builds are required for config files. None found."
print (summary)
return summary
if not targ_list:
summary =src_file + ": Cannot find any targets which build this file."
print (summary)
return summary
if hostbuild:
# confirm it actually builds before we do anything
print ("Confirming source file builds")
res = get_make_output (build_dir + "/gcc", "all")
if res[0] != 0:
message = "Error: " + src_file + " does not build currently."
summary = src_file + " does not build on host."
print (message)
print (res[1])
if verbose:
verbose.write (message + "\n")
verbose.write (res[1]+ "\n")
return summary
src_requires = set (ii_macro_consume (src_info))
for macro in src_requires:
because[macro] = src_file
header_seen = list ()
os.rename (src_file, src_file + ".bak")
src_orig = copy.deepcopy (src_data)
src_tmp = copy.deepcopy (src_data)
try:
# process the includes from bottom to top. This is because we know that
# later includes have are known to be needed, so any dependency from this
# header is a true dependency
for inc_file in inclist:
inc_file_norm = normalize_header (inc_file)
if inc_file in no_remove:
continue
if len (h_list) != 0 and inc_file_norm not in h_list:
continue
if inc_file_norm[0:3] == "gt-":
continue
if inc_file_norm[0:6] == "gtype-":
continue
if inc_file_norm.replace(".h",".c") == os.path.basename(src_file):
continue
lookfor = ii_src_line(src_info)[inc_file]
src_tmp.remove (lookfor)
message = "Trying " + src_file + " without " + inc_file
print (message)
if verbose:
verbose.write (message + "\n")
out = open(src_file, "w")
for line in src_tmp:
out.write (line)
out.close()
keep = False
if hostbuild:
res = get_make_output (build_dir + "/gcc", "all")
else:
res = (0, "")
rc = res[0]
message = "Passed Host build"
if (rc != 0):
# host build failed
message = "Compilation failed:\n";
keep = True
else:
if targ_list:
objfile = get_obj_name (src_file)
t1 = targ_list[0]
if objfile and os.path.exists(t1[1] +"/gcc/"+objfile):
res = get_make_output_parallel (targ_list, objfile, 0)
else:
res = get_make_output_parallel (targ_list, "all-gcc", 0)
rc = res[0]
if rc != 0:
message = "Compilation failed on TARGET : " + res[2]
keep = True
else:
message = "Passed host and target builds"
if keep:
print (message + "\n")
if (rc != 0):
if verbose:
verbose.write (message + "\n");
verbose.write (res[1])
verbose.write ("\n");
if os.path.exists (inc_file):
ilog = open(inc_file+".log","a")
ilog.write (message + " for " + src_file + ":\n\n");
ilog.write ("============================================\n");
ilog.write (res[1])
ilog.write ("\n");
ilog.close()
if os.path.exists (src_file):
ilog = open(src_file+".log","a")
ilog.write (message + " for " +inc_file + ":\n\n");
ilog.write ("============================================\n");
ilog.write (res[1])
ilog.write ("\n");
ilog.close()
# Given a sequence where :
# #include "tm.h"
# #include "target.h" // includes tm.h
# target.h was required, and when attempting to remove tm.h we'd see that
# all the macro defintions are "required" since they all look like:
# #ifndef HAVE_blah
# #define HAVE_blah
# endif
# when target.h was found to be required, tm.h will be tagged as included.
# so when we get this far, we know we dont have to check the macros for
# tm.h since we know it is already been included.
if inc_file_norm not in header_seen:
iinfo = get_header_info (inc_file, spath)
newlist = all_headers (inc_file_norm)
if ii_path(iinfo) == "build" and not target_dict:
keep = True
text = message + " : Will not remove a build file without some targets."
print (text)
ilog = open(src_file+".log","a")
ilog.write (text +"\n")
ilog.write ("============================================\n");
ilog.close()
ilog = open("reduce-headers-kept.log","a")
ilog.write (src_file + " " + text +"\n")
ilog.close()
else:
newlist = list()
if not keep and inc_file_norm not in header_seen:
# now look for any macro requirements.
for h in newlist:
if not h in header_seen:
if header_dict.get(h):
defined = ii_macro_define (header_dict[h])
for dep in defined:
if dep in src_requires and dep not in ignore_list:
keep = True;
text = message + ", but must keep " + inc_file + " because it provides " + dep
if because.get(dep) != None:
text = text + " Possibly required by " + because[dep]
print (text)
ilog = open(inc_file+".log","a")
ilog.write (because[dep]+": Requires [dep] in "+src_file+"\n")
ilog.write ("============================================\n");
ilog.close()
ilog = open(src_file+".log","a")
ilog.write (text +"\n")
ilog.write ("============================================\n");
ilog.close()
ilog = open("reduce-headers-kept.log","a")
ilog.write (src_file + " " + text +"\n")
ilog.close()
if verbose:
verbose.write (text + "\n")
if keep:
# add all headers 'consumes' to src_requires list, and mark as seen
for h in newlist:
if not h in header_seen:
header_seen.append (h)
if header_dict.get(h):
consume = ii_macro_consume (header_dict[h])
for dep in consume:
if dep not in src_requires:
src_requires.add (dep)
if because.get(dep) == None:
because[dep] = inc_file
src_tmp = copy.deepcopy (src_data)
else:
print (message + " --> removing " + inc_file + "\n")
rmcount += 1
if verbose:
verbose.write (message + " --> removing " + inc_file + "\n")
if remove_count.get(inc_file) == None:
remove_count[inc_file] = 1
else:
remove_count[inc_file] += 1
src_data = copy.deepcopy (src_tmp)
except:
print ("Interuption: restoring original file")
out = open(src_file, "w")
for line in src_orig:
out.write (line)
out.close()
raise
# copy current version, since it is the "right" one now.
out = open(src_file, "w")
for line in src_data:
out.write (line)
out.close()
# Try a final host bootstrap build to make sure everything is kosher.
if hostbuild:
res = get_make_output (build_dir, "all")
rc = res[0]
if (rc != 0):
# host build failed! return to original version
print ("Error: " + src_file + " Failed to bootstrap at end!!! restoring.")
print (" Bad version at " + src_file + ".bad")
os.rename (src_file, src_file + ".bad")
out = open(src_file, "w")
for line in src_orig:
out.write (line)
out.close()
return src_file + ": failed to build after reduction. Restored original"
if src_data == src_orig:
summary = src_file + ": No change."
else:
summary = src_file + ": Reduction performed, "+str(rmcount)+" includes removed."
print (summary)
return summary
only_h = list ()
ignore_cond = False
usage = False
src = list()
only_targs = list ()
for x in sys.argv[1:]:
if x[0:2] == "-b":
build_dir = x[2:]
elif x[0:2] == "-f":
fn = normalize_header (x[2:])
if fn not in only_h:
only_h.append (fn)
elif x[0:2] == "-h":
usage = True
elif x[0:2] == "-d":
ignore_cond = True
elif x[0:2] == "-D":
ignore_list.append(x[2:])
elif x[0:2] == "-T":
only_targs.append(x[2:])
elif x[0:2] == "-t":
target_dir = x[2:]
elif x[0] == "-":
print ("Error: Unrecognized option " + x)
usgae = True
else:
if not os.path.exists (x):
print ("Error: specified file " + x + " does not exist.")
usage = True
else:
src.append (x)
if target_dir:
build_target_dict (target_dir, only_targs)
if build_dir == "" and target_dir == "":
print ("Error: Must specify a build directory, and/or a target directory.")
usage = True
if build_dir and not os.path.exists (build_dir):
print ("Error: specified build directory does not exist : " + build_dir)
usage = True
if target_dir and not os.path.exists (target_dir):
print ("Error: specified target directory does not exist : " + target_dir)
usage = True
if usage:
print ("Attempts to remove extraneous include files from source files.")
print (" ")
print ("Should be run from the main gcc source directory, and works on a target")
print ("directory, as we attempt to make the 'all' target.")
print (" ")
print ("By default, gcc-reorder-includes is run on each file before attempting")
print ("to remove includes. this removes duplicates and puts some headers in a")
print ("canonical ordering")
print (" ")
print ("The build directory should be ready to compile via make. Time is saved")
print ("if the build is already complete, so that only changes need to be built.")
print (" ")
print ("Usage: [options] file1.c [file2.c] ... [filen.c]")
print (" -bdir : the root build directory to attempt buiding .o files.")
print (" -tdir : the target build directory")
print (" -d : Ignore conditional macro dependencies.")
print (" ")
print (" -Dmacro : Ignore a specific macro for dependencies")
print (" -Ttarget : Only consider target in target directory.")
print (" -fheader : Specifies a specific .h file to be considered.")
print (" ")
print (" -D, -T, and -f can be specified mulitple times and are aggregated.")
print (" ")
print (" The original file will be in filen.bak")
print (" ")
sys.exit (0)
if only_h:
print ("Attempting to remove only these files:")
for x in only_h:
print (x)
print (" ")
logfile = open("reduce-headers.log","w")
for x in src:
msg = try_to_remove (x, only_h, logfile)
ilog = open("reduce-headers.sum","a")
ilog.write (msg + "\n")
ilog.close()
ilog = open("reduce-headers.sum","a")
ilog.write ("===============================================================\n")
for x in remove_count:
msg = x + ": Removed " + str(remove_count[x]) + " times."
print (msg)
logfile.write (msg + "\n")
ilog.write (msg + "\n")
|