1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
|
#!/bin/sh
# This script automatically test the given tool with the tool's test cases,
# reporting anything of interest.
# Written by Mike Stump <mrs@cygnus.com>
# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
usage()
{
if [ -n "$1" ] ; then
echo "$0: Error: $1" >&2
echo >&2
fi
cat >&2 <<EOUSAGE
Usage: $0 [-strict] PREVIOUS CURRENT
Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
Unless -strict is given, these discrepancies are not counted as errors:
missing/extra .sum files when comparing directories
tests that failed in PREVIOUS but pass in CURRENT
tests that were not in PREVIOUS but appear in CURRENT
tests in PREVIOUS that are missing in CURRENT
Exit with the following values:
0 if there is nothing of interest
1 if there are errors when comparing single test case files
N for the number of errors found when comparing directories
EOUSAGE
exit 2
}
export LC_ALL=C
tool=gxx
TMPDIR=${TMPDIR:-/tmp}
tmp1=$TMPDIR/$tool-testing.$$a
tmp2=$TMPDIR/$tool-testing.$$b
now_s=$TMPDIR/$tool-testing.$$d
before_s=$TMPDIR/$tool-testing.$$e
now_u=$TMPDIR/$tool-uniq.$$d
before_u=$TMPDIR/$tool-uniq.$$e
lst1=$TMPDIR/$tool-lst1.$$
lst2=$TMPDIR/$tool-lst2.$$
lst3=$TMPDIR/$tool-lst3.$$
lst4=$TMPDIR/$tool-lst4.$$
lst5=$TMPDIR/$tool-lst5.$$
sum1=$TMPDIR/$tool-sum1.$$
sum2=$TMPDIR/$tool-sum2.$$
tmps="$tmp1 $tmp2 $now_s $before_s $now_u $before_u $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
[ "$1" = "-strict" ] && strict=$1 && shift
[ "$1" = "-?" ] && usage
[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
trap "rm -f $tmps" 0 1 2 3 5 9 13 15
exit_status=0
if [ -d "$1" -a -d "$2" ] ; then
find "$1/" -name '*.sum' >$lst1
find "$2/" -name '*.sum' >$lst2
echo "# Comparing directories"
echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
echo
# remove leading directory components to compare
sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
comm -23 $lst3 $lst4 >$lst5
if [ -s $lst5 ] ; then
echo "# Extra sum files in Dir1=$1"
sed -e "s|^|< $1/|" $lst5
echo
[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
fi
comm -13 $lst3 $lst4 >$lst5
if [ -s $lst5 ] ; then
echo "# Extra sum files in Dir2=$2"
sed -e "s|^|> $2/|" $lst5
echo
[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
fi
comm -12 $lst3 $lst4 | sort -u >$lst5
if [ ! -s $lst5 ] ; then
echo "# No common sum files"
exit_status=`expr $exit_status + 1`
exit $exit_status
fi
cmnsums=`cat $lst5 | wc -l`
echo "# Comparing $cmnsums common sum files"
( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
ret=$?
if [ $ret -ne 0 ]; then
exit_status=`expr $exit_status + 1`
echo "## Differences found"
fi
if [ $exit_status -ne 0 ]; then
echo "# $exit_status differences in $cmnsums common sum files found"
else
echo "# No differences found in $cmnsums common sum files"
fi
exit $exit_status
elif [ -d "$1" -o -d "$2" ] ; then
usage "Must specify either two directories or two files"
fi
sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^[ ]*=== [^ ]* tests ===$/ {tool = $2} /^Running target / {target = $3} { if (tool != "") { sub(/:[ ]/, "&"tool": " ); }; if (target != "unix") { sub(/:[ ]/, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1
sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^[ ]*=== [^ ]* tests ===$/ {tool = $2} /^Running target / {target = $3} { if (tool != "") { sub(/:[ ]/, "&"tool": " ); }; if (target != "unix") { sub(/:[ ]/, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2
before=$tmp1
now=$tmp2
if sort -k 2 </dev/null >/dev/null 2>&1; then
skip1='-k 2'
else
skip1='+1'
fi
sort -t ':' $skip1 "$now" > "$now_s"
sort -t ':' $skip1 "$before" > "$before_s"
# Report non-unique test names and print the diff between the two
# lists if they are different.
#
# If the results include several configurations (schedule of
# variations), we report duplicates several times with different
# target/tool prefixes because at this stage we do not know if the
# target and/or tool prefix was inserted.
# If we used the input files (so generally several times the same
# results in one section per target), we would incorreclty detect
# duplicates (as many as targets)
grep -E '^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED|UNSUPPORTED|UNTESTED|ERROR):' "$now_s" | uniq -cd > "$now_u"
grep -E '^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED|UNSUPPORTED|UNTESTED|ERROR):' "$before_s" | uniq -cd > "$before_u"
same_uniq=" now"
cmp -s "$before_u" "$now_u" && same_uniq=""
if [ -s "$now_u" ]; then
echo "Non-unique test names$same_uniq: (Eeek!)"
cat "$now_u"
echo
exit_status=1
fi
if [ -s "$before_u" -a "x$same_uniq" != "x" ]; then
echo "Changes to non-unique test names:"
diff -u "$before_u" "$now_u" | grep -E '^[-\\+] '
echo
exit_status=1
fi
grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2
grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
num=`cat $tmp2 | wc -l`
echo "Tests that now fail, but worked before ($num tests):"
echo
cat $tmp2
echo
exit_status=1
fi
grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2
grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
num=`cat $tmp2 | wc -l`
echo "Tests that now work, but didn't before ($num tests):"
echo
cat $tmp2
[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
echo
fi
grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2
grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
num=`cat $tmp2 | wc -l`
echo "New tests that FAIL ($num tests):"
echo
cat $tmp2
echo
exit_status=1
fi
grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2
grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
num=`cat $tmp2 | wc -l`
echo "New tests that PASS ($num tests):"
echo
cat $tmp2
[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
echo
fi
grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2
grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
num=`cat $tmp2 | wc -l`
echo "Old tests that passed, that have disappeared ($num tests): (Eeek!)"
echo
cat $tmp2
[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
echo
fi
grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2
grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
num=`cat $tmp2 | wc -l`
echo "Old tests that failed, that have disappeared ($num tests): (Eeek!)"
echo
cat $tmp2
[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
echo
fi
exit $exit_status
|
