1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
|
<chapter id=runningtests>
<title>Running Tests</title>
<para>There are two ways to execute a test suite. The most
common way is when there is existing support in the
<filename>Makefile</filename>. This support consists of a
<emphasis>check</emphasis> target. The other way is to execute the
<command>runtest</command> program directly. To run
<command>runtest</command> directcly from the command line requires
either all the correct options, or the <xref linkend=local> must be setup
correctly.</para>
<sect1 id=makecheck xreflabel="Make Check">
<title>Make check</title>
<para>To run tests from an existing collection, first use
<command>configure</command> as usual to set up the
build directory. Then try typing:</para>
<screen>
make check
</screen>
<para>If the <emphasis>check</emphasis> target exists, it
usually saves you some trouble. For instance, it can set up any
auxiliary programs or other files needed by the tests. The most
common file the check builds is the
<emphasis>site.exp</emphasis>. The site.exp file contains
various variables that DejaGnu used to dertermine the
configuration of the program being tested. This is mostly for
supporting remote testing.</para>
<para>The <emphasis>check</emphasis> target is supported by GNU
<productname>Automake</productname>. To have DejaGnu support added to your
generated <filename>Makefile.in</filename>, just add the keyword
dejagnu to the AUTOMAKE_OPTIONS variable in your
<filename>Makefile.am</filename> file.</para>
<para>Once you have run <emphasis>make check</emphasis> to build
any auxiliary files, you can invoke the test driver
<command>runtest</command> directly to repeat the tests.
You will also have to execute <command>runtest</command>
directly for test collections with no
<emphasis>check</emphasis> target in the
<filename>Makefile</filename>.</para>
</sect1>
<sect1 id=runtest xreflabel="Runtest">
<title>Runtest</title>
<para><command>runtest</command> is the executable test driver
for DejaGnu. You can specify two kinds of things on the
<command>runtest</command> command line: command line options,
and Tcl variables for the test scripts. The options are listed
alphabetically below.</para>
<para><command>runtest</command> returns an exit code of
<emphasis>1</emphasis> if any test has an unexpected result; otherwise
(if all tests pass or fail as expected) it returns <emphasis>0</emphasis>
as the exit code.</para>
<sect2 id=outputs xreflabel="Output States">
<title>Output States</title>
<para><filename>runtest</filename> flags the outcome of each
test as one of these cases. <xref linkend=posix> for a
discussion of how POSIX specifies the meanings of these
cases.</para>
<variablelist>
<varlistentry>
<term>PASS</term>
<listitem><para>The most desirable outcome: the test succeeded, and
was expected to succeed.</para></listitem>
</varlistentry>
<varlistentry>
<term>XPASS</term>
<listitem><para>A pleasant kind of failure: a test was expected to
fail, but succeeded. This may indicate progress; inspect the test
case to determine whether you should amend it to stop expecting
failure.</para></listitem>
</varlistentry>
<varlistentry>
<term>FAIL</term>
<listitem><para>A test failed, although it was expected to succeed.
This may indicate regress; inspect the test case and the failing
software to ocate the bug.</para></listitem>
</varlistentry>
<varlistentry>
<term>XFAIL</term>
<listitem><para>A test failed, but it was expected to fail. This
result indicates no change in a known bug. If a test fails because
the operating system where the test runs lacks some facility required
by the test, the outcome is <emphasis>UNSUPPORTED</emphasis>
instead.</para></listitem>
</varlistentry>
<varlistentry>
<term>UNRESOLVED</term>
<listitem><para>Output from a test requires manual inspection; the
test suite could not automatically determine the outcome. For
example, your tests can report this outcome is when a test does not
complete as expected.</para></listitem>
</varlistentry>
<varlistentry>
<term>UNTESTED</term>
<listitem><para>A test case is not yet complete, and in particular
cannot yet produce a <emphasis>PASS</emphasis> or
<emphasis>FAIL</emphasis>. You can also use this outcome in dummy
``tests'' that note explicitly the absence of a real test case for a
particular property.</para></listitem>
</varlistentry>
<varlistentry>
<term>UNSUPPORTED</term>
<listitem><para>A test depends on a conditionally available feature
that does not exist (in the configured testing environment). For
example, you can use this outcome to report on a test case that does
not work on a particular target because its operating system support
does not include a required subroutine.</para></listitem>
</varlistentry>
</variablelist>
<para>runtest may also display the following messages:</para>
<variablelist>
<varlistentry>
<term>ERROR</term>
<listitem><para>Indicates a major problem (detected by the test case
itself) in running the test. This is usually an unrecoverable error,
such as a missing file or loss of communication to the target. (POSIX
test suites should not emit this message; use
<emphasis>UNSUPPORTED</emphasis>, <emphasis>UNTESTED</emphasis>, or
<emphasis>UNRESOLVED</emphasis> instead, as
appropriate.)</para></listitem>
</varlistentry>
<varlistentry>
<term>WARNING</term>
<listitem><para>Indicates a possible problem in running the
test. Usually warnings correspond to recoverable errors, or display
an important message about the following tests.</para></listitem>
</varlistentry>
<varlistentry>
<term>NOTE</term>
<listitem><para>An informational message about the test
case.</para></listitem>
</varlistentry>
</variablelist>
</sect2>
<sect2 id=invoking xreflabel="Invoking Runtest">
<title>Invoking Runtest</title>
<para>This is the full set of command line options that
<filename>runtest</filename> recognizes. Arguments may be
abbreviated to the shortest unique string.</para>
<variablelist>
<varlistentry>
<term><option>--all</option> (-a)</term>
<listitem><para>Display all test output. By default,
<emphasis>runtest</emphasis> shows only the output of tests that
produce unexpected results; that is, tests with status
<emphasis>FAIL</emphasis> (unexpected failure),
<emphasis>XPASS</emphasis> (unexpected success), or
<emphasis>ERROR</emphasis> (a severe error in the test case
itself). Specify <emphasis>--all</emphasis> to see output for tests
with status <emphasis>PASS</emphasis> (success, as expected)
<emphasis>XFAIL</emphasis> (failure, as expected), or
<emphasis>WARNING</emphasis> (minor error in the test case
itself).</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--build [string]</option></term>
<listitem><para><emphasis>string</emphasis> is a full configuration
``triple'' name as used by <command>configure</command>. This
is the type of machine DejaGnu and the tools to be tested are built
on. For a normal cross this is the same as the host, but for a
canadian cross, they are seperate.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--host [string]</option></term>
<listitem><para><symbol>string</symbol> is a full configuration
``triple'' name as used by <emphasis>configure</emphasis>. Use this
option to override the default string recorded by your
configuration's choice of host. This choice does not change how
anything is actually configured unless --build is also specified; it
affects <emphasis>only</emphasis> DejaGnu procedures that compare the
host string with particular values. The procedures
<emphasis>ishost</emphasis>, <emphasis>istarget</emphasis>,
<emphasis>isnative</emphasis>, and <emphasis>setup</emphasis>xfail}
are affected by <emphasis>--host</emphasis>. In this usage,
<emphasis>host</emphasis> refers to the machine that the tests are to
be run on, which may not be the same as the
<emphasis>build</emphasis> machine. If <emphasis>--build</emphasis>
is also specified, then <emphasis>--host</emphasis> refers to the
machine that the tests wil, be run on, not the machine DejaGnu is run
on.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--host_board [name]</option></term>
<listitem><para>The host board to use.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--target [string]</option></term>
<listitem><para>Use this option to override the default setting
(running native tests). <emphasis>string</emphasis> is a full
configuration ``triple'' name of the form
<emphasis>cpu-vendor-os</emphasis> as used by
<command>configure</command>. This option changes the
configuration <emphasis>runtest</emphasis> uses for the default tool
names, and other setup information.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--debug</option> (-de)</term>
<listitem><para>Turns on the <emphasis>expect</emphasis> internal
debugging output. Debugging output is displayed as part of the
<emphasis>runtest</emphasis> output, and logged to a file called
<filename>dbg.log</filename>. The extra debugging output does
<emphasis>not</emphasis> appear on standard output, unless the
verbose level is greater than 2 (for instance, to see debug output
immediately, specify <emphasis>--debug</emphasis>-v -v}). The
debugging output shows all attempts at matching the test output of
the tool with the scripted patterns describing expected output. The
output generated with <emphasis>--strace</emphasis> also goes into
<filename>dbg.log</filename>.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--help</option> (-he)</term>
<listitem><para>Prints out a short summary of the
<emphasis>runtest</emphasis> options, then exits (even if you also
specify other options).</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--ignore [name(s)] </option></term>
<listitem><para>The names of specific tests to
ignore.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--objdir [path]</option></term>
<listitem><para>Use <emphasis>path</emphasis> as the top directory
containing any auxiliary compiled test code. This defaults to
<filename>.</filename>. Use this option to locate pre-compiled test
code. You can normally prepare any auxiliary files needed with
<emphasis>make</emphasis>.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--outdir [path]</option></term>
<listitem><para>Write output logs in directory
<filename>path</filename>. The default is <emphasis>.},
the</emphasis> directory where you start
<emphasis>runtest</emphasis>. This option affects only the summary
and the detailed log files
<filename>tool.sum</filename> and
<filename>tool.log</filename>. The DejaGnu debug
log <filename>dbg.log</filename> always appears (when requested) in
the local directory.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--reboot [name]</option></term>
<listitem><para>Reboot the target board when
<emphasis>runtest</emphasis> initializes. Usually, when running tests
on a separate target board, it is safer to reboot the target to be
certain of its state. However, when developing test scripts,
rebooting takes a lot of time.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--srcdir [path]</option></term>
<listitem><para>Use <filename>path</filename> as the top directory
for test scripts to run. <emphasis>runtest</emphasis> looks in this
directory for any subdirectory whose name begins with the toolname
(specified with <emphasis>--tool</emphasis>). For instance, with
<emphasis>--tool</emphasis>gdb}, <emphasis>runtest</emphasis> uses
tests in subdirectories <filename>gdb.*</filename> (with the usual
shell-like filename expansion). If you do not use
<emphasis>--srcdir</emphasis>, <emphasis>runtest</emphasis> looks for
test directories under the current working
directory.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--strace [number]</option></term>
<listitem><para>Turn on internal tracing for
<emphasis>expect</emphasis>, to n levels deep. By adjusting the
level, you can control the extent to which your output expands
multi-level Tcl statements. This allows you to ignore some levels of
<emphasis>case</emphasis> or <emphasis>if</emphasis> statements.
Each procedure call or control structure counts as one ``level''. The
output is recorded in the same file, <filename>dbg.log</filename>,
used for output from <emphasis>--debug</emphasis>.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--connect [program]</option></term>
<listitem><para>Connect to a target testing environment as specified
by <emphasis>type</emphasis>, if the target is not the computer
running <emphasis>runtest</emphasis>. For example, use
<emphasis>--connect</emphasis> to change the program used to connect
to a ``bare board'' boot monitor. The choices for
<emphasis>type</emphasis> in the DejaGnu 1.4 distribution are
<emphasis>rlogin</emphasis>, <emphasis>telnet</emphasis>,
<emphasis>rsh</emphasis>, <emphasis>tip</emphasis>,
<emphasis>kermit</emphasis>, and <emphasis>mondfe</emphasis>.</para>
<para>The default for this option depends on the configuration most
convenient communication method available, but often other
alternatives work as well; you may find it useful to try alternative
connect methods if you suspect a communication problem with your
testing target.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--baud [number]</option></term>
<listitem><para>Set the default baud rate to something other than
9600. (Some serial interface programs, like <emphasis>tip</emphasis>,
use a separate initialization file instead of this
value.)</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--target_board [name(s)]</option></term>
<listitem><para>The list of target boards to run tests
on.</para></listitem>
</varlistentry>
<varlistentry id=tool-opt>
<term><option>--tool[name(s)]</option></term>
<listitem><para>Specifies which test suite to run, and what
initialization module to use. <option>--tool</option> is used
<emphasis>only</emphasis> for these two purposes. It is
<emphasis>not</emphasis> used to name the executable program to
test. Executable tool names (and paths) are recorded in
<filename>site.exp</filename> and you can override them by specifying
Tcl variables on the command line.</para>
<para>For example, including "<option>--tool</option> gcc" on the
<emphasis>runtest</emphasis> command line runs tests from all test
subdirectories whose names match <filename>gcc.*</filename>, and uses
one of the initialization modules named
<filename>config/*-gcc.exp</filename>. To specify the name of the
compiler (perhaps as an alternative path to what
<emphasis>runtest</emphasis> would use by default), use
<emphasis>GCC=binname</emphasis> on the <emphasis>runtest</emphasis>
command line.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--tool_exec [name]</option></term>
<listitem><para>The path to the tool executable to
test.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--tool_opts [options]</option></term>
<listitem><para>A list of additional options to pass to the
tool.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--verbose</option> (-v)</term>
<listitem><para>Turns on more output. Repeating this option increases
the amount of output displayed. Level one (<emphasis>-v</emphasis>)
is simply test output. Level two (<emphasis>-v</emphasis>-v}) shows
messages on options, configuration, and process control. Verbose
messages appear in the detailed (<filename>*.log</filename>) log
file, but not in the summary (<filename>*.sum</filename>) log
file.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--version</option> (-V)</term>
<listitem><para>Prints out the version numbers of DejaGnu,
<emphasis>expect</emphasis> and Tcl, and exits without running any
tests.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--D[0-1]</option></term>
<listitem><para>Start the internal Tcl debugger. The Tcl debugger
supports breakpoints, single stepping, and other common debugging
activities. See the document "Debugger for Tcl Applications" by Don
Libes. (Distributed in PostScript form with
<emphasis>expect</emphasis> as the file
<filename>expect/tcl-debug.ps.</filename>. If you specify
<emphasis>-D1</emphasis>, the <emphasis>expect</emphasis> shell stops
at a breakpoint as soon as DejaGnu invokes it. If you specify
<emphasis>-D0</emphasis>, DejaGnu starts as usual, but you can enter
the debugger by sending an interrupt (e.g. by typing
<keycombo><keycap>C</keycap><keycap>c</keycap></keycombo>).
</para></listitem>
</varlistentry>
<varlistentry>
<term><filename>testfile</filename>.exp[=arg(s)]</term>
<listitem><para>Specify the names of testsuites to run. By default,
<emphasis>runtest</emphasis> runs all tests for the tool, but you can
restrict it to particular testsuites by giving the names of the
<emphasis>.exp expect</emphasis> scripts that control
them. <emphasis>testsuite</emphasis>.exp may not include path
information; use plain filenames.</para></listitem>
</varlistentry>
<varlistentry>
<term><filename>testfile</filename>.exp="testfile1 ..."</term>
<listitem><para>Specify a subset of tests in a suite to run. For
compiler or assembler tests, which often use a single
<emphasis>.exp</emphasis> script covering many different source
files, this option allows you to further restrict the tests by
listing particular source files to compile. Some tools even support
wildcards here. The wildcards supported depend upon the tool, but
typically they are <emphasis>?</emphasis>, <emphasis>*</emphasis>,
and <emphasis>[chars]</emphasis>.</para></listitem>
</varlistentry>
<varlistentry>
<term><symbol>tclvar</symbol>=value</term>
<listitem><para>You can define Tcl variables for use by your test
scripts in the same style used with <emphasis>make</emphasis> for
environment variables. For example, <emphasis>runtest
GDB=gdb.old</emphasis> defines a variable called
<command>GDB</command>; when your scripts refer to
<symbol>$GDB</symbol> in this run, they use the value
<emphasis>gdb.old</emphasis>.</para>
<para>The default Tcl variables used for most tools are defined in
the main DejaGnu <emphasis>Makefile</emphasis>; their values are
captured in the <filename>site.exp</filename> file.</para></listitem>
</varlistentry>
</variablelist>
</sect2>
<sect2 id=common xreflabel="Common Operations">
<title>Common Options</title>
<para>Typically, you don't need must to use any command-line options.
<option>--tool</option> used is only required when there are more than
one test suite in the same directory. The default options are in the
local site.exp file, created by "make site.exp".</para>
<para>For example, if the directory <filename>gdb/testsuite</filename>
contains a collection of DejaGnu tests for GDB, you can run them like
this:</para>
<screen>
eg$ cd gdb/testsuite
eg$ runtest --tool gdb
</screen>
<para>Test output follows, ending with:</para>
<screen>
=== gdb Summary ===
# of expected passes 508
# of expected failures 103
/usr/latest/bin/gdb version 4.14.4 -nx
</screen>
<para>You can use the option <emphasis>--srcdir</emphasis> to point to
some other directory containing a collection of tests:</para>
<screen>
eg$ runtest--srcdir /devo/gdb/testsuite
</screen>
<para>By default, <command>runtest</command> prints only the
names of the tests it runs, output from any tests that have unexpected
results, and a summary showing how many tests passed and how many
failed. To display output from all tests (whether or not they behave
as expected), use the <emphasis>--all</emphasis> option. For more
verbose output about processes being run, communication, and so on, use
<emphasis>--verbose</emphasis>. To see even more output, use multiple
<emphasis>--verbose</emphasis> options. for a more detailed explanation
of each <command>runtest</command> option.</para>
<para>Test output goes into two files in your current directory:
summary output in <filename>tool.sum</filename>,
and detailed output in <filename>
tool.log</filename>. (<emphasis>tool</emphasis>
refers to the collection of tests; for example, after a run with
<emphasis>--tool</emphasis> gdb, look for output files
<filename>gdb.sum</filename> and
<filename>gdb.log</filename>.)</para>
</sect2>
</sect1>
<sect1 id=outputfiles xreflabel="Output Files">
<title>The files DejaGnu produces.</title>
<para>DejaGnu always writes two kinds of output files: summary
logs and detailed logs. The contents of both of these are
determined by your tests.</para>
<para>For troubleshooting, a third kind of output file is useful:
use <option>--debug</option> to request an output file showing
details of what <productname>Expect</productname> is doing
internally.</para>
<sect2 id=sum xreflabel="Summary File">
<title>Summary File</title>
<para>DejaGnu always produces a summary output file
<filename>tool.sum</filename>. This summary shows the names of
all test files run; for each test file, one line of output from
each <command>pass</command> command (showing status
<emphasis>PASS</emphasis> or <emphasis>XPASS</emphasis>) or
<command>fail</command> command (status
<emphasis>FAIL</emphasis> or <emphasis>XFAIL</emphasis>);
trailing summary statistics that count passing and failing tests
(expected and unexpected); and the full pathname and version
number of the tool tested. (All possible outcomes, and all
errors, are always reflected in the summary output file,
regardless of whether or not you specify
<option>--all</option>.)</para>
<para>If any of your tests use the procedures
<command>unresolved</command>, <command>unsupported</command>,
or <command>runtested</command>, the summary output also
tabulates the corresponding outcomes.</para>
<para>For example, after <command>runtest --tool
binutils</command>, look for a summary log in
<filename>binutils.sum</filename>. Normally, DejaGnu writes this
file in your current working directory; use the
<option>--outdir</option> option to select a different
directory.</para>
<example>
<title>Here is a short sample summary log</title>
<screen>
Test Run By rob on Mon May 25 21:40:57 PDT 1992
=== gdb tests ===
Running ./gdb.t00/echo.exp ...
PASS: Echo test
Running ./gdb.all/help.exp ...
PASS: help add-symbol-file
PASS: help aliases
PASS: help breakpoint "bre" abbreviation
FAIL: help run "r" abbreviation
Running ./gdb.t10/crossload.exp ...
PASS: m68k-elf (elf-big) explicit format; loaded
XFAIL: mips-ecoff (ecoff-bigmips) "ptype v_signed_char" signed C types
=== gdb Summary ===
# of expected passes 5
# of expected failures 1
# of unexpected failures 1
/usr/latest/bin/gdb version 4.6.5 -q
</screen>
</example>
</sect2>
<sect2 id=log xreflabel="Log File">
<title>Log File</title>
<para>DejaGnu also saves a detailed log file
<filename>tool.log</filename>, showing any output generated by
tests as well as the summary output. For example, after
<command>runtest --tool binutils</command>, look for a detailed
log in <filename>binutils.log</filename>. Normally, DejaGnu
writes this file in your current working directory; use the
<option>--outdir</option> option to select a different
directory.</para>
<example>
<title>Here is a brief example showing a detailed log for
<productname>G++</productname> tests</title>
<screen>
Test Run By rob on Mon May 25 21:40:43 PDT 1992
=== g++ tests ===
--- Running ./g++.other/t01-1.exp ---
PASS: operate delete
--- Running ./g++.other/t01-2.exp ---
FAIL: i960 bug EOF
p0000646.C: In function `int warn_return_1 ()':
p0000646.C:109: warning: control reaches end of non-void function
p0000646.C: In function `int warn_return_arg (int)':
p0000646.C:117: warning: control reaches end of non-void function
p0000646.C: In function `int warn_return_sum (int, int)':
p0000646.C:125: warning: control reaches end of non-void function
p0000646.C: In function `struct foo warn_return_foo ()':
p0000646.C:132: warning: control reaches end of non-void function
--- Running ./g++.other/t01-4.exp ---
FAIL: abort
900403_04.C:8: zero width for bit-field `foo'
--- Running ./g++.other/t01-3.exp ---
FAIL: segment violation
900519_12.C:9: parse error before `;'
900519_12.C:12: Segmentation violation
/usr/latest/bin/gcc: Internal compiler error: program cc1plus got fatal signal
=== g++ Summary ===
# of expected passes 1
# of expected failures 3
/usr/latest/bin/g++ version cygnus-2.0.1
</screen>
</example>
</sect2>
<sect2 id=debugfile xreflabel="Debug Log File">
<title>Debug Log File</title>
<para>With the <option>--debug</option> option, you can request
a log file showing the output from
<productname>Expect</productname> itself, running in debugging
mode. This file (<filename>dbg.log</filename>, in the directory
where you start <command>runtest</command>) shows each pattern
<productname>Expect</productname> considers in analyzing test
output.</para>
<para>This file reflects each <command>send</command> command,
showing the string sent as input to the tool under test; and
each <productname>Expect</productname> command, showing each
pattern it compares with the tool output.</para>
<example>
<title>The log messages begin with a message of the form</title>
<screen>
expect: does {<symbol>tool output</symbol>} (spawn_id <symbol>n</symbol>)
match pattern {<emphasis>expected pattern</emphasis>}?
</screen>
</example>
<para>For every unsuccessful match,
<productname>Expect</productname> issues a
<emphasis>no</emphasis> after this message; if other patterns
are specified for the same <productname>Expect</productname>
command, they are reflected also, but without the first part of
the message (<emphasis>expect... match pattern</emphasis>).</para>
<para>When <productname>Expect</productname> finds a match, the
log for the successful match ends with <emphasis>yes</emphasis>,
followed by a record of the <productname>Expect</productname>
variables set to describe a successful match.</para>
<example>
<title>Here is an excerpt from the debugging log for a
<productname>GDB</productname> test:</title>
<screen>
send: sent {break gdbme.c:34\n} to spawn id 6
expect: does {} (spawn_id 6) match pattern {Breakpoint.*at.* file
gdbme.c, line 34.*\(gdb\) $}? no
{.*\(gdb\) $}? no
expect: does {} (spawn_id 0) match pattern {return} ? no
{\(y or n\) }? no
{buffer_full}? no
{virtual}? no
{memory}? no
{exhausted}? no
{Undefined}? no
{command}? no
break gdbme.c:34
Breakpoint 8 at 0x23d8: file gdbme.c, line 34.
(gdb) expect: does {break gdbme.c:34\r\nBreakpoint 8 at 0x23d8:
file gdbme.c, line 34.\r\n(gdb) } (spawn_id 6) match pattern
{Breakpoint.*at.* file gdbme.c, line 34.*\(gdb\) $}? yes
expect: set expect_out(0,start) {18}
expect: set expect_out(0,end) {71}
expect: set expect_out(0,string) {Breakpoint 8 at 0x23d8: file
gdbme.c, line 34.\r\n(gdb) }
epect: set expect_out(spawn_id) {6}
expect: set expect_out(buffer) {break gdbme.c:34\r\nBreakpoint 8
at 0x23d8: file gdbme.c, line 34.\r\n(gdb) }
PASS: 70 0 breakpoint line number in file
</screen>
</example>
<para>This example exhibits three properties of
<productname>Expect</productname> and
<productname>DejaGnu</productname> that might be surprising at
first glance:</para>
<itemizedlist mark="bullet">
<listitem><para>Empty output for the first attempted match. The
first set of attempted matches shown ran against the output
<emphasis>{}</emphasis> --- that is, no
output. <productname>Expect</productname> begins
attempting to match the patterns supplied immediately; often,
the first pass is against incomplete output (or completely
before all output, as in this case).</para></listitem>
<listitem><para>Interspersed tool output. The beginning of
the log entry for the second attempted match may be hard to
spot: this is because the prompt <emphasis>{(gdb) }</emphasis>
appears on the same line, just before the
<emphasis>expect:</emphasis> that marks the beginning of the
log entry.</para></listitem>
<listitem><para>Fail-safe patterns. Many of the patterns
tested are fail-safe patterns provided by
<productname>GDB</productname> testing utilities, to reduce
possible indeterminacy. It is useful to anticipate potential
variations caused by extreme system conditions
(<productname>GDB</productname> might issue the message
<emphasis>virtual memory exhausted</emphasis> in rare
circumstances), or by changes in the tested program
(<emphasis>Undefined command</emphasis> is the likeliest
outcome if the name of a tested command changes).</para>
<para>The pattern <emphasis>{return}</emphasis> is a
particularly interesting fail-safe to notice; it checks for an
unexpected <keycap>RET</keycap> prompt. This may happen,
for example, if the tested tool can filter output through a
pager.</para>
<para>These fail-safe patterns (like the debugging log itself)
are primarily useful while developing test scripts. Use the
<command>error</command> procedure to make the actions for
fail-safe patterns produce messages starting with
<emphasis>ERROR</emphasis> on standard output, and in the
detailed log file.</para></listitem>
</itemizedlist>
</sect2>
</sect1>
</chapter>
<chapter id=Customizing xreflabel="Customizing DejaGnu">
<title>Customizing DejaGnu</title>
<para>The site configuration file, <filename>site.exp</filename>,
captures configuration-dependent values and propagates them to the
DejaGnu test environment using Tcl variables. This ties the
DejaGnu test scripts into the <command>configure</command> and
<command>make</command> programs. If this file is setup correctly,
it is possible to execute a test suite merely by typing
<command>runtest</command>.</para>
<para>DejaGnu supports two <filename>site.exp</filename>
files. The multiple instances of <filename>site.exp</filename> are
loaded in a fixed order built into DejaGnu. The first file loaded
is the local file <filename>site.exp</filename>, and then the
optional global <filename>site.exp</filename> file as
pointed to by the <symbol>DEJAGNU</symbol> environment
variable.</para>
<para>There is an optional <emphasis>master</emphasis>
<filename>site.exp</filename>, capturing configuration values that
apply to DejaGnu across the board, in each configuration-specific
subdirectory of the DejaGnu library directory.
<command>runtest</command> loads these values first. The master
<filename>site.exp</filename> contains the default values for all
targets and hosts supported by DejaGnu. This master file is
identified by setting the environment variable
<symbol>DEJAGNU</symbol> to the name of the file. This is also
refered to as the ``global'' config file.</para>
<para>Any directory containing a configured test suite also has a
local <filename>site.exp</filename>, capturing configuration values
specific to the tool under test. Since <command>runtest</command>
loads these values last, the individual test configuration can
either rely on and use, or override, any of the global values from
the global <filename>site.exp</filename> file.</para>
<para>You can usually generate or update the testsuite's local
<filename>site.exp</filename> by typing <command>make
site.exp</command> in the test suite directory, after the test
suite is configured.</para>
<para>You can also have a file in your home directory called
<filename>.dejagnurc</filename>. This gets loaded first before the
other config files. Usually this is used for personal stuff, like
setting the <symbol>all_flag</symbol> so all the output gets
printed, or your own verbosity levels. This file is usually
restricted to setting command line options.</para>
<para>You can further override the default values in a
user-editable section of any <filename>site.exp</filename>, or by
setting variables on the <command>runtest</command> command
line.</para>
<sect1 id=local xreflabel="Local Config File">
<title>Local Config File</title>
<para>It is usually more convenient to keep these <emphasis>manual
overrides</emphasis> in the <filename>site.exp</filename>
local to each test directory, rather than in the global
<filename>site.exp</filename> in the installed DejaGnu
library. This file is mostly for supplying tool specific info
that is required by the test suite.</para>
<para>All local <filename>site.exp</filename> files have
two sections, separated by comment text. The first section is
the part that is generated by <command>make</command>. It is
essentially a collection of Tcl variable definitions based on
<filename>Makefile</filename> environment variables. Since they
are generated by <command>make</command>, they contain the
values as specified by <command>configure</command>. (You can
also customize these values by using the <option>--site</option>
option to <command>configure</command>.) In particular, this
section contains the <filename>Makefile</filename>
variables for host and target configuration data. Do not edit
this first section; if you do, your changes are replaced next
time you run <command>make</command>.</para>
<example>
<title>The first section starts with</title>
<programlisting>
## these variables are automatically generated by make ##
# Do not edit here. If you wish to override these values
# add them to the last section
</programlisting>
</example>
<para>In the second section, you can override any default values
(locally to DejaGnu) for all the variables. The second section
can also contain your preferred defaults for all the command
line options to <command>runtest</command>. This allows you to
easily customize <command>runtest</command> for your preferences
in each configured test-suite tree, so that you need not type
options repeatedly on the command line. (The second section may
also be empty, if you do not wish to override any defaults.)</para>
<example>
<title>The first section ends with this line</title>
<programlisting>
## All variables above are generated by configure. Do Not Edit ##
</programlisting>
</example>
<para>You can make any changes under this line. If you wish to
redefine a variable in the top section, then just put a
duplicate value in this second section. Usually the values
defined in this config file are related to the configuration of
the test run. This is the ideal place to set the variables
<symbol>host_triplet</symbol>, <symbol>build_triplet</symbol>,
<symbol>target_triplet</symbol>. All other variables are tool
dependant. ie for testing a compiler, the value for
<symbol>CC</symbol> might be set to a freshly built binary, as
opposed to one in the user's path.</para>
<para>Here's an example local site.exp file, as used for
<productname>GCC/G++</productname> testing.</para>
<example>
<title>Local Config File</title>
<programlisting>
## these variables are automatically generated by make ##
# Do not edit here. If you wish to override these values
# add them to the last section
set rootme "/build/devo-builds/i586-pc-linux-gnulibc1/gcc"
set host_triplet i586-pc-linux-gnulibc1
set build_triplet i586-pc-linux-gnulibc1
set target_triplet i586-pc-linux-gnulibc1
set target_alias i586-pc-linux-gnulibc1
set CFLAGS ""
set CXXFLAGS "-I/build/devo-builds/i586-pc-linux-gnulibc1/gcc/../libio -I$srcdir/../libg++/src -I$srcdir/../libio -I$srcdir/../libstdc++ -I$srcdir/../libstdc++/stl -L/build/devo-builds/i586-pc-linux-gnulibc1/gcc/../libg++ -L/build/devo-builds/i586-pc-linux-gnulibc1/gcc/../libstdc++"
append LDFLAGS " -L/build/devo-builds/i586-pc-linux-gnulibc1/gcc/../ld"
set tmpdir /build/devo-builds/i586-pc-linux-gnulibc1/gcc/testsuite
set srcdir "${srcdir}/testsuite"
## All variables above are generated by configure. Do Not Edit ##
</programlisting>
</example>
<para>This file defines the required fields for a local config
file, namely the three config triplets, and the srcdir. It also
defines several other Tcl variables that are used exclusivly by
the GCC test suite. For most test cases, the CXXFLAGS and LDFLAGS
are supplied by DejaGnu itself for cross testing, but to test a
compiler, GCC needs to manipulate these itself.</para>
</sect1>
<sect1 id=global xreflabel="Global Config File">
<title>Global Config File</title>
<para>The master config file is where all the target specific
config variables get set for a whole site get set. The idea is
that for a centralized testing lab where people have to share a
target between multiple developers. There are settings for both
remote targets and remote hosts. Here's an example of a Master
Config File (also called the Global config file) for a
<emphasis>canadian cross</emphasis>. A canadian cross is when
you build and test a cross compiler on a machine other than the
one it's to be hosted on.</para>
<para>Here we have the config settings for our California
office. Note that all config values are site dependant. Here we
have two sets of values that we use for testing m68k-aout cross
compilers. As both of these target boards has a different
debugging protocol, we test on both of them in sequence.</para>
<example>
<title>Global Config file</title>
<programlisting>
# Make sure we look in the right place for the board description files.
if ![info exists boards_dir] {
set boards_dir {}
}
lappend boards_dir "/nfs/cygint/s1/cygnus/dejagnu/boards"
verbose "Global Config File: target_triplet is $target_triplet" 2
global target_list
case "$target_triplet" in {
{ "native" } {
set target_list "unix"
}
{ "sparc64-*elf" } {
set target_list "sparc64-sim"
}
{ "mips-*elf" } {
set target_list "mips-sim wilma barney"
}
{ "mips-lsi-elf" } {
set target_list "mips-lsi-sim{,soft-float,el}"
}
{ "sh-*hms" } {
set target_list { "sh-hms-sim" "bloozy" }
}
}
</programlisting>
</example>
<para>In this case, we have support for several cross compilers,
that all run on this host. For testing on operating systems that
don't support Expect, DejaGnu can be run on the local build
machine, and it can connect to the remote host and run all the
tests for this cross compiler on that host. All the remote OS
requires is a working telnetd.</para>
<para>As you can see, all one does is set the variable
<symbol>target_list</symbol> to the list of targets and options to
test. The simple settings, like for
<emphasis>sparc64-elf</emphasis> only require setting the name of
the single board config file. The <emphasis>mips-elf</emphasis>
target is more complicated. Here it sets the list to three target
boards. One is the default mips target, and both
<emphasis>wilma</emphasis> <emphasis>barney</emphasis> are
symbolic names for other mips boards. Symbolic names are covered
in the <xref linkend=addboard> chapter. The more complicated
example is the one for <emphasis>mips-lsi-elf</emphasis>. This one
runs the tests with multiple iterations using all possible
combinations of the <option>--soft-float</option> and the
<option>--el</option> (little endian) option. Needless to say,
this last feature is mostly compiler specific.</para>
</sect1>
<sect1 id=boardconfig xreflabel="Board Config File">
<title>Board Config File</title>
<para>The board config file is where board specfic config data
is stored. A board config file contains all the higher-level
configuration settings. There is a rough inheritance scheme, where it is
possible to base a new board description file on an existing one. There
are also collections of custom procedures for common environments. For
more information on adding a new board config file, go to the <xref
linkend=addboard> chapter. </para>
<para>An example board config file for a GNU simulator is as
follows. <function>set_board_info</function> is a procedure that sets the
field name to the specified value. The procedures in square brackets
<emphasis>[]</emphasis> are <emphasis>helper procedures</emphasis>. Thes
are used to find parts of a tool chain required to build an executable
image that may reside in various locations. This is mostly of use for
when the startup code, the standard C lobraries, or the tool chain itself
is part of your build tree.</para>
<example>
<title>Board Config File</title>
<programlisting>
# This is a list of toolchains that are supported on this board.
set_board_info target_install {sparc64-elf}
# Load the generic configuration for this board. This will define any
# routines needed by the tool to communicate with the board.
load_generic_config "sim"
# We need this for find_gcc and *_include_flags/*_link_flags.
load_base_board_description "basic-sim"
# Use long64 by default.
process_multilib_options "long64"
setup_sim sparc64
# We only support newlib on this target. We assume that all multilib
# options have been specified before we get here.
set_board_info compiler "[find_gcc]"
set_board_info cflags "[libgloss_include_flags] [newlib_include_flags]"
set_board_info ldflags "[libgloss_link_flags] [newlib_link_flags]"
# No linker script.
set_board_info ldscript "";
# Used by a few gcc.c-torture testcases to delimit how large the
# stack can be.
set_board_info gcc,stack_size 16384
# The simulator doesn't return exit statuses and we need to indicate this
# the standard GCC wrapper will work with this target.
set_board_info needs_status_wrapper 1
# We can't pass arguments to programs.
set_board_info noargs 1
</programlisting>
</example>
<para>There are five helper procedures used in this example. The first
one, <function>find gcc</function> looks for a copy of the GNU compiler in
your build tree, or it uses the one in your path. This will also return
the proper transformed name for a cross compiler if you whole build tree
is configured for one. The next helper procedures are
<function>libgloss_include_flags</function> &
<function>libgloss_link_flags</function>. These return the proper flags to
compiler and link an executable image using <xref
linkend=libgloss>, the GNU BSP (Board Support Package). The final
procedures are <function>newlib_include_flag</function> &
<function>newlib_include_flag</function>. These find the Newlib C
library, which is a reentrant standard C library for embedded systems
comprising of non GPL'd code.</para>
</sect1>
<sect1 id=releng xreflabel="Remote Host Testing">
<title>Remote Host Testing</title>
<note><para>Thanks to Dj Delorie for the original paper that
this section is based on.</para></note>
<para>DejaGnu also supports running the tests on a remote
host. To set this up, the remote host needs an ftp server, and a
telnet server. Currently foreign operating systems used as
remote hosts are VxWorks, VRTX, Dos/Win3.1, MacOS, and
win95/win98/NT.</para>
<para>The recommended source for a win95/win98/NT based ftp
server is to get IIS (either IIS 1 or Personal Web Server) from
<ulink
URL="http://www.microsoft.com">http://www.microsoft.com</ulink>.
When you install it, make sure you install the FTP server - it's
not selected by default. Go into the IIS manager and change the
FTP server so that it does not allow anonymous ftp. Set the home
directory to the root directory (i.e. c:\) of a suitable
drive. Allow writing via ftp.</para>
<para>It will create an account like IUSR_FOOBAR where foobar is
the name of your machine. Go into the user editor and give that
account a password that you don't mind hanging around in the
clear (i.e. not the same as your admin or personal
passwords). Also, add it to all the various permission groups.</para>
<para>You'll also need a telnet server. For win95/win98/NT, go
to the <ulink URL="http://ataman.com">Ataman</ulink> web site,
pick up the Ataman Remote Logon Services for Windows, and
install it. You can get started on the eval period anyway. Add
IUSR_FOOBAR to the list of allowed users, set the HOME directory
to be the same as the FTP default directory. Change the Mode
prompt to simple.</para>
<para>Ok, now you need to pick a directory name to do all the
testing in. For the sake of this example, we'll call it piggy
(i.e. c:\piggy). Create this directory.</para>
<para>You'll need a unix machine. Create a directory for the
scripts you'll need. For this example, we'll use
/usr/local/swamp/testing. You'll need to have a source tree
somewhere, say /usr/src/devo. Now, copy some files from
releng's area in SV to your machine:</para>
<example>
<title>Remote host setup</title>
<screen>
cd /usr/local/swamp/testing
mkdir boards
scp darkstar.welcomehome.org:/dejagnu/cst/bin/MkTestDir .
scp darkstar.welcomehome.org:/dejagnu/site.exp .
scp darkstar.welcomehome.org:/dejagnu/boards/useless98r2.exp boards/foobar.exp
export DEJAGNU=/usr/local/swamp/testing/site.exp
</screen>
</example>
<para>You must edit the boards/foobar.exp file to reflect your
machine; change the hostname (foobar.com), username
(iusr_foobar), password, and ftp_directory (c:/piggy) to match
what you selected.</para>
<para>Edit the global <filename> site.exp</filename> to reflect your
boards directory:</para>
<example>
<title>Add The Board Directory</title>
<programlisting>
lappend boards_dir "/usr/local/swamp/testing/boards"
</programlisting>
</example>
<para>Now run MkTestDir, which is in the contrib
directory. The first parameter is the toolchain prefix, the
second is the location of your devo tree. If you are testing a
cross compiler (ex: you have sh-hms-gcc.exe in your PATH on
the PC), do something like this:</para>
<example>
<title>Setup Cross Remote Testing</title>
<programlisting>
./MkTestDir sh-hms /usr/dejagnu/src/devo
</programlisting>
</example>
<para>If you are testing a native PC compiler (ex: you have
gcc.exe in your PATH on the PC), do this:</para>
<example>
<title>Setup Native Remote Testing</title>
<programlisting>
./MkTestDir '' /usr/dejagnu/src/devo
</programlisting>
</example>
<para>To test the setup, <command>ftp</command> to your PC
using the username (iusr_foobar) and password you selected. CD
to the test directory. Upload a file to the PC. Now telnet to
your PC using the same username and password. CD to the test
directory. Make sure the file is there. Type "set" and/or "gcc
-v" (or sh-hms-gcc -v) and make sure the default PATH contains
the installation you want to test.</para>
<example>
<title>Run Test Remotely</title>
<programlisting>
cd /usr/local/swamp/testing
make -k -w check RUNTESTFLAGS="--host_board foobar --target_board foobar -v -v" > check.out 2>&1
</programlisting>
</example>
<para>To run a specific test, use a command like this (for
this example, you'd run this from the gcc directory that
MkTestDir created):</para>
<example>
<title>Run a Test Remotely</title>
<programlisting>
make check RUNTESTFLAGS="--host_board sloth --target_board sloth -v compile.exp=921202-1.c"
</programlisting>
</example>
<para>Note: if you are testing a cross-compiler, put in the
correct target board. You'll also have to download more .exp
files and modify them for your local configuration. The -v's
are optional.</para>
</sect1>
<sect1 id=configfile xreflabel="Config File Values">
<title>Config File Values</title>
<para>DejaGnu uses a named array in Tcl to hold all the info for
each machine. In the case of a canadian cross, this means host
information as well as target information. The named array is
called <symbol>target_info</symbol>, and it has two indices. The
following fields are part of the array.</para>
<sect2 id=optiondefs xreflabel="Option Variables">
<title>Command Line Option Variables</title>
<para>In the user editable second section of the <xref
linkend=personal> you can not only override the configuration
variables captured in the first section, but also specify
default values for all on the <command>runtest</command>
command line options. Save for <option>--debug</option>,
<option>--help</option>, and <option>--version</option>, each
command line option has an associated Tcl variable. Use the
Tcl <command>set</command> command to specify a new default
value (as for the configuration variables). The following
table describes the correspondence between command line
options and variables you can set in
<filename>site.exp</filename>. <xref linkend=invoking>, for
explanations of the command-line options.</para>
<para><table frame=all rowsep=0 colsep=0>
<title>Tcl Variables For Command Line Options</title>
<tgroup cols=3 align="char" rowsep=1 colsep=0>
<thead><row>
<entry>runtest</entry><entry>Tcl</entry>
<entry>option</entry><entry>variable</entry><entry>description</entry>
</row></thead>
<tbody>
<row>
<entry>--all</entry>
<entry>all_flag</entry>
<entry>display all test results if set</entry>
</row>
<row>
<entry>--baud</entry>
<entry>baud</entry>
<entry>set the default baud rate to something other than
9600.</entry>
</row>
<row>
<entry>--connect</entry>
<entry>connectmode</entry>
<entry><command>rlogin</command>,
<command>telnet</command>, <command>rsh</command>,
<command>kermit</command>, <command>tip</command>, or
<command>mondfe</command></entry>
</row>
<row>
<entry>--outdir</entry>
<entry>outdir</entry>
<entry>directory for <filename>tool.sum</filename> and
<filename>tool.log.</filename></entry>
</row>
<row>
<entry>--objdir</entry>
<entry>objdir</entry>
<entry>directory for pre-compiled binaries</entry>
</row>
<row>
<entry>--reboot</entry>
<entry>reboot</entry>
<entry>reboot the target if set to
<emphasis>"1"</emphasis>; do not reboot if set to
<emphasis>"0"</emphasis> (the default).</entry>
</row>
<row>
<entry>--srcdir</entry>
<entry>srcdir</entry>
<entry>directory of test subdirectories</entry>
</row>
<row>
<entry>--strace</entry>
<entry>tracelevel</entry>
<entry>a number: Tcl trace depth</entry>
</row>
<row>
<entry>--tool</entry>
<entry>tool</entry>
<entry>name of tool to test; identifies init, test subdir</entry>
</row>
<row>
<entry>--verbose</entry>
<entry>verbose</entry>
<entry>verbosity level. As option, use multiple times; as
variable, set a number, 0 or greater.</entry>
</row>
<row>
<entry>--target</entry>
<entry>target_triplet</entry>
<entry>The canonical configuration string for the target.</entry>
</row>
<row>
<entry>--host</entry>
<entry>host_triplet</entry>
<entry>The canonical configuration string for the host.</entry>
</row>
<row>
<entry>--build</entry>
<entry>build_triplet</entry>
<entry>The canonical configuration string for the build
host.</entry>
</row>
<row>
<entry>--mail</entry>
<entry>address</entry>
<entry>Email the output log to the specified address.</entry>
</row>
</tbody>
</tgroup>
</table>
</para>
</sect2>
<sect2 id=personal xreflabel="Personal Config File">
<title>Personal Config File</title>
<para>The personal config file is used to customize
<command>runtest's</command> behaviour for each person. It's
typically used to set the user prefered setting for verbosity,
and any experimental Tcl procedures. My personal
<filename>~/.dejagnurc</filename> file looks like:</para>
<example>
<title>Personal Config File</title>
<programlisting>
set all_flag 1
set RLOGIN /usr/ucb/rlogin
set RSH /usr/local/sbin/ssh
</programlisting>
</example>
<para>Here I set <symbol>all_flag</symbol> so I see all the test
cases that PASS along with the ones that FAIL. I also set
<symbol>RLOGIN</symbol> to the BSD version. I have
<productname>Kerberos</productname> installed, and when I rlogin
to a target board, it usually isn't supported. So I use the non
secure version rather than the default that's in my path. I also
set <symbol>RSH</symbol> to the <productname>SSH</productname>
secure shell, as rsh is mostly used to test unix
machines within a local network here.</para>
</sect2>
</sect1>
</chapter>
<chapter id=Extending xreflabel="Extending DejaGnu">
<title>Extending DejaGnu</title>
<sect1 id=addsuite xreflabel="Adding a new Test Suite">
<title>Adding A New Test Suite</title>
<para>The testsuite for a new tool should always be located in that tools
source directory. DejaGnu require the directory be named
<filename>testsuite</filename>. Under this directory, the test cases go
in a subdirectory whose name begins with the tool name. For example, for
a tool named <emphasis>flubber</emphasis>, each subdirectory containing
testsuites must start with <emphasis>"flubber."</emphasis>.</para>
</sect1>
<sect1 id=addtool xreflabel="Adding A New Tool">
<title>Adding A New Tool</title>
<para>In general, the best way to learn how to write (code or even prose)
is to read something similar. This principle applies to test cases and
to test suites. Unfortunately, well-established test suites have a way
of developing their own conventions: as test writers become more
experienced with DejaGnu and with Tcl, they accumulate more utilities,
and take advantage of more and more features of
<productname>Expect</productname> and <productname>Tcl</productname> in
general.</para>
<para>Inspecting such established test suites may make the prospect of
creating an entirely new test suite appear overwhelming. Nevertheless,
it is quite straightforward to get a new test suite going.</para>
<para>There is one test suite that is guaranteed not to grow more
elaborate over time: both it and the tool it tests were created expressly
to illustrate what it takes to get started with DejaGnu. The
<filename>example/</filename> directory of the DejaGnu distribution
contains both an interactive tool called <command>calc</command>, and a
test suite for it. Reading this test suite, and experimenting with it,
is a good way to supplement the information in this section. (Thanks to
Robert Lupton for creating calc and its test suite---and also the first
version of this section of the manual!)</para>
<para>To help orient you further in this task, here is an outline of the
steps to begin building a test suite for a program example.</para>
<itemizedlist mark=bullet>
<listitem><para>Create or select a directory to contain your new
collection of tests. Change into that directory (shown here as
<filename>testsuite</filename>):</para>
<para>Create a <filename>configure.in</filename> file in this directory,
to control configuration-dependent choices for your tests. So far as
DejaGnu is concerned, the important thing is to set a value for the
variable <symbol>target_abbrev</symbol>; this value is the link to the
init file you will write soon. (For simplicity, we assume the
environment is Unix, and use <emphasis>unix</emphasis> as the
value.)</para>
<para>What else is needed in <filename>configure.in</filename> depends on
the requirements of your tool, your intended test environments, and which
configure system you use. This example is a minimal configure.in for use
with <productname>GNU Autoconf</productname>. </para></listitem>
<listitem><para>Create <filename>Makefile.in</filename> (if you are using
Autoconf), or <filename>Makefile.am</filename>(if you are using
Automake), the source file used by configure to build your
<filename>Makefile</filename>. If you are using GNU Automake.just add the
keyword <emphasis>dejagnu</emphasis> to the
<emphasis>AUTOMAKE_OPTIONS</emphasis> variable in your
<filename>Makefile.am</filename> file. This will add all the Makefile
support needed to run DejaGnu, and support the <xref linkend=makecheck>
target.</para>
<para>You also need to include two targets important to DejaGnu:
<emphasis>check</emphasis>, to run the tests, and
<emphasis>site.exp</emphasis>, to set up the Tcl copies of
configuration-dependent values. This is called the <xref linkend=local>
The check target must run the <command>runtest</command> program to
execute the tests.</para>
<para>The <filename>site.exp</filename> target should usually set up
(among other things) the $tool variable for the name of your program. If
the local site.exp file is setup correctly, it is possible to execute the
tests by merely typing <command>runtest</command> on the command
line.</para>
<example>
<title>Sample Makefile.in Fragment</title>
<programlisting>
# Look for a local version of DejaGnu, otherwise use one in the path
RUNTEST = `if test -f $(top_srcdir)/../dejagnu/runtest; then \
echo $(top_srcdir) ../dejagnu/runtest; \
else \
echo runtest; \
fi`
# The flags to pass to runtest
RUNTESTFLAGS =
# Execute the tests
check: site.exp all
$(RUNTEST) $(RUNTESTFLAGS) \
--tool <symbol>${example}</symbol> --srcdir $(srcdir)
# Make the local config file
site.exp: ./config.status Makefile
@echo "Making a new config file..."
-@rm -f ./tmp?
@touch site.exp
-@mv site.exp site.bak
@echo "## these variables are automatically\
generated by make ##" > ./tmp0
@echo "# Do not edit here. If you wish to\
override these values" >> ./tmp0
@echo "# add them to the last section" >> ./tmp0
@echo "set host_os ${host_os}" >> ./tmp0
@echo "set host_alias ${host_alias}" >> ./tmp0
@echo "set host_cpu ${host_cpu}" >> ./tmp0
@echo "set host_vendor ${host_vendor}" >> ./tmp0
@echo "set target_os ${target_os}" >> ./tmp0
@echo "set target_alias ${target_alias}" >> ./tmp0
@echo "set target_cpu ${target_cpu}" >> ./tmp0
@echo "set target_vendor ${target_vendor}" >> ./tmp0
@echo "set host_triplet ${host_canonical}" >> ./tmp0
@echo "set target_triplet ${target_canonical}">>./tmp0
@echo "set tool binutils" >> ./tmp0
@echo "set srcdir ${srcdir}" >> ./tmp0
@echo "set objdir `pwd`" >> ./tmp0
@echo "set <symbol>${examplename}</symbol> <symbol>${example}</symbol>" >> ./tmp0
@echo "## All variables above are generated by\
configure. Do Not Edit ##" >> ./tmp0
@cat ./tmp0 > site.exp
@sed < site.bak \
-e '1,/^## All variables above are.*##/ d' \
>> site.exp
-@rm -f ./tmp?
</programlisting>
</example>
</listitem>
<listitem><para>Create a directory (in <filename>testsuite</filename>)
called <filename>config</filename>. Make a <emphasis>Tool Init
File</emphasis> in this directory. Its name must start with the
<symbol>target_abbrev</symbol> value, or be named
<filename>default.exp</filename> so call it
<filename>config/unix.exp</filename> for our Unix based example. This
is the file that contains the target-dependent procedures.
Fortunately, on Unix, most of them do not have to do very much in
order for <command>runtest</command> to run.</para>
<para>If the program being tested is not interactive, you can get
away with this minimal <filename>unix.exp</filename> to begin
with:</para>
<example>
<title>Simple Batch Program Tool Init File</title>
<programlisting>
proc foo_exit {} {}
proc foo_version {} {}
</programlisting>
</example>
<para>If the program being tested is interactive, however, you might
as well define a <emphasis>start</emphasis> routine and invoke it by
using an init file like this:</para>
<example>
<title>Simple Interactive Program Tool Init File</title>
<programlisting>
proc foo_exit {} {}
proc foo_version {} {}
proc foo_start {} {
global ${examplename}
spawn ${examplename}
expect {
-re "" {}
}
}
# Start the program running we want to test
foo_start
</programlisting>
</example>
</listitem>
<listitem><para>Create a directory whose name begins with your tool's
name, to contain tests. For example, if your tool's name is
<emphasis>gcc</emphasis>, then the directories all need to start with
<emphasis>"gcc."</emphasis>.</para></listitem>
<listitem><para>Create a sample test file. Its name must end with
<filename>.exp</filename>. You can use
<filename>first-try.exp</filename>. To begin with, just write there a
line of Tcl code to issue a message.</para>
<example>
<title>Testing A New Tool Config</title>
<programlisting>
send_user "Testing: one, two...\n"
</programlisting>
</example>
</listitem>
<listitem><para>Back in the <filename>testsuite</filename> (top
level) directory, run <command>configure</command>. Typically you do
this while in the build directory. You may have to specify more of a
path, if a suitable configure is not available in your execution
path.</para></listitem>
<listitem><para>e now ready to triumphantly type <command>make
check</command> or <command>runtest</command>. You should see
something like this:</para>
<example>
<title>Example Test Case Run</title>
<screen>
Test Run By rhl on Fri Jan 29 16:25:44 EST 1993
=== example tests ===
Running ./example.0/first-try.exp ...
Testing: one, two...
=== example Summary ===
</screen>
</example>
<para>There is no output in the summary, because so far the example
does not call any of the procedures that establish a test
outcome.</para></listitem>
<listitem><para>Write some real tests. For an interactive tool, you
should probably write a real exit routine in fairly short order. In
any case, you should also write a real version routine
soon. </para></listitem>
</itemizedlist>
</sect1>
<sect1 id=addtarget xreflabel="Adding A New Target">
<title>Adding A New Target</title>
<para>DejaGnu has some additional requirements for target support, beyond
the general-purpose provisions of configure. DejaGnu must actively
communicate with the target, rather than simply generating or managing
code for the target architecture. Therefore, each tool requires an
initialization module for each target. For new targets, you must supply
a few Tcl procedures to adapt DejaGnu to the target. This permits
DejaGnu itself to remain target independent.</para>
<para>Usually the best way to write a new initialization module is to
edit an existing initialization module; some trial and error will be
required. If necessary, you can use the @samp{--debug} option to see what
is really going on.</para>
<para>When you code an initialization module, be generous in printing
information controlled by the <function>verbose</function>
procedure.</para>
<para>For cross targets, most of the work is in getting the
communications right. Communications code (for several situations
involving IP networks or serial lines) is available in a DejaGnu library
file.</para>
<para>If you suspect a communication problem, try running the connection
interactively from <productname>Expect</productname>. (There are three
ways of running <productname>Expect</productname> as an interactive
interpreter. You can run <productname>Expect</productname> with no
arguments, and control it completely interactively; or you can use
<command>expect -i</command> together with other command-line options and
arguments; or you can run the command <command>interpreter</command> from
any <productname>Expect</productname> procedure. Use
<command>return</command> to get back to the calling procedure (if any),
or <command>return -tcl</command> to make the calling procedure itself
return to its caller; use <command>exi</command>t or end-of-file to leave
Expect altogether.) Run the program whose name is recorded in
<symbol>$connectmode</symbol>, with the arguments in
<symbol>$targetname</symbol>, to establish a connection. You should at
least be able to get a prompt from any target that is physically
connected.</para>
</sect1>
<sect1 id=addboard xreflabel="Adding A New Board">
<title>Adding A New Board</title>
<para>Adding a new board consists of creating a new board config
file. Examples are in
<filename>dejagnu/baseboards</filename>. Usually to make a new
board file, it's easiest to copy an existing one. It is also
possible to have your file be based on a
<emphasis>baseboard</emphasis> file with only one or two
changes needed. Typically, this can be as simple as just
changing the linker script. Once the new baseboard file is done,
add it to the <symbol>boards_DATA</symbol> list in the
<filename>dejagnu/baseboards/Makefile.am</filename>, and regenerate the
Makefile.in using automake. Then just rebuild and install DejaGnu. You
can test it by:</para>
<para>There is a crude inheritance scheme going on with board files, so
you can include one board file into another, The two main procedures used
to do this are <function>load_generic_config</function> and
<function>load_base_board_description</function>. The generic config file
contains other procedures used for a certain class of target. The
board description file is where the board specfic settings go. Commonly
there are similar target environments with just different
processors.</para>
<example>
<title>Testing a New Board Config File</title>
<screen>
make check RUNTESTFLAGS="--target_board=<emphasis>newboardfile</emphasis>".
</screen>
</example>
<para>Here's an example of a board config file. There are
several <emphasis>helper procedures</emphasis> used in this
example. A helper procedure is one that look for a tool of files
in commonly installed locations. These are mostly used when
testing in the build tree, because the executables to be tested
are in the same tree as the new dejagnu files. The helper
procedures are the ones in square braces
<emphasis>[]</emphasis>, which is the Tcl execution characters.</para>
<example>
<title>Example Board Config File</title>
<programlisting>
# Load the generic configuration for this board. This will define a basic
# set of routines needed by the tool to communicate with the board.
load_generic_config "sim"
# basic-sim.exp is a basic description for the standard Cygnus simulator.
load_base_board_description "basic-sim"
# The compiler used to build for this board. This has *nothing* to do
# with what compiler is tested if we're testing gcc.
set_board_info compiler "[find_gcc]"
# We only support newlib on this target.
# However, we include libgloss so we can find the linker scripts.
set_board_info cflags "[newlib_include_flags] [libgloss_include_flags]"
set_board_info ldflags "[newlib_link_flags]"
# No linker script for this board.
set_board_info ldscript "-Tsim.ld";
# The simulator doesn't return exit statuses and we need to indicate this.
set_board_info needs_status_wrapper 1
# Can't pass arguments to this target.
set_board_info noargs 1
# No signals.
set_board_info gdb,nosignals 1
# And it can't call functions.
set_board_info gdb,cannot_call_functions 1
</programlisting>
</example>
</sect1>
<sect1 id=boarddefs xreflabel="Board File Values">
<title>Board Config File Values</title>
<para>These fields are all in the <symbol>board_info</symbol> These are
all set by using the <function>set_board_info</function> procedure. The
parameters are the field name, followed by the value to set the field
to.</para>
<para><table frame=all rowsep=0 colsep=0>
<title>Common Board Info Fields</title>
<tgroup cols=3 align="char" rowsep=1 colsep=0>
<thead><row>
<entry>Field</entry>
<entry>Sample Value</entry>
<entry>Description</entry>
</row></thead>
<tbody>
<row>
<entry>compiler</entry>
<entry>"[find_gcc]"</entry>
<entry>The path to the compiler to use.</entry>
</row>
<row>
<entry>cflags</entry>
<entry>"-mca"</entry>
<entry>Compilation flags for the compiler.</entry>
</row>
<row>
<entry>ldflags</entry>
<entry>"[libgloss_link_flags] [newlib_link_flags]"</entry>
<entry>Linking flags for the compiler.</entry>
</row>
<row>
<entry>ldscript</entry>
<entry>"-Wl,-Tidt.ld"</entry>
<entry>The linker script to use when cross compiling.</entry>
</row>
<row>
<entry>libs</entry>
<entry>"-lgcc"</entry>
<entry>Any additional libraries to link in.</entry>
</row>
<row>
<entry>shell_prompt</entry>
<entry>"cygmon>"</entry>
<entry>The command prompt of the remote shell.</entry>
</row>
<row>
<entry>hex_startaddr</entry>
<entry>"0xa0020000"</entry>
<entry>The Starting address as a string.</entry>
</row>
<row>
<entry>start_addr</entry>
<entry>0xa0008000</entry>
<entry>The starting address as a value.</entry>
</row>
<row>
<entry>startaddr</entry>
<entry>"a0020000"</entry>
<entry></entry>
</row>
<row>
<entry>exit_statuses_bad</entry>
<entry>1</entry>
<entry>Whether there is an accurate exit status.</entry>
</row>
<row>
<entry>reboot_delay</entry>
<entry>10</entry>
<entry>The delay between power off and power on.</entry>
</row>
<row>
<entry>unreliable</entry>
<entry>1</entry>
<entry>Whether communication with the board is unreliable.</entry>
</row>
<row>
<entry>sim</entry>
<entry>[find_sim]</entry>
<entry>The path to the simulator to use.</entry>
</row>
<row>
<entry>objcopy</entry>
<entry>$tempfil</entry>
<entry>The path to the <command>objcopy</command> program.</entry>
</row>
<row>
<entry>support_libs</entry>
<entry>"${prefix_dir}/i386-coff/"</entry>
<entry>Support libraries needed for cross compiling.</entry>
</row>
<row>
<entry>addl_link_flags</entry>
<entry>"-N"</entry>
<entry>Additional link flags, rarely used.</entry>
</row>
</tbody>
</tgroup>
</table>
</para>
<para>These fields are used by the GCC and GDB tests, and are mostly
only useful to somewhat trying to debug a new board file for one of
these tools. Many of these are used only by a few testcases, and their
purpose is esoteric. These are listed with sample values as a guide to
better guessing if you need to change any of these.</para>
<para><table frame=all rowsep=0 colsep=0>
<title>Board Info Fields For GCC & GDB</title>
<tgroup cols=3 align="char" rowsep=1 colsep=0>
<thead><row>
<entry>Field</entry>
<entry>Sample Value</entry>
<entry>Description</entry>
</row></thead>
<tbody>
<row>
<entry>strip</entry>
<entry>$tempfile</entry>
<entry>Strip the executable of symbols.</entry>
</row>
<row>
<entry>gdb_load_offset</entry>
<entry>"0x40050000"</entry>
</row>
<row>
<entry>gdb_protocol</entry>
<entry>"remote"</entry>
<entry>The GDB debugging protocol to use.</entry>
</row>
<row>
<entry>gdb_sect_offset</entry>
<entry>"0x41000000";</entry>
</row>
<row>
<entry>gdb_stub_ldscript</entry>
<entry>"-Wl,-Teva-stub.ld"</entry>
<entry>The linker script to use with a GDB stub.</entry>
</row>
<row>
<entry>gdb_init_command</entry>
<entry>"set mipsfpu none"</entry>
</row>
<row>
<entry>gdb,cannot_call_functions</entry>
<entry>1</entry>
<entry>Whether GDB can call functions on the target,</entry>
</row>
<row>
<entry>gdb,noargs</entry>
<entry>1</entry>
<entry>Whether the target can take command line arguments.</entry>
</row>
<row>
<entry>gdb,nosignals</entry>
<entry>1</entry>
<entry>Whether there are signals on the target.</entry>
</row>
<row>
<entry>gdb,short_int</entry>
<entry>1</entry>
</row>
<row>
<entry>gdb,start_symbol</entry>
<entry>"_start";</entry>
<entry>The starting symbol in the executable.</entry>
</row>
<row>
<entry>gdb,target_sim_options</entry>
<entry>"-sparclite"</entry>
<entry>Special options to pass to the simulator.</entry>
</row>
<row>
<entry>gdb,timeout</entry>
<entry>540</entry>
<entry>Timeout value to use for remote communication.</entry>
</row>
<row>
<entry>gdb_init_command</entry>
<entry>"print/x \$fsr = 0x0"</entry>
</row>
<row>
<entry>gdb_load_offset</entry>
<entry>"0x12020000"</entry>
</row>
<row>
<entry>gdb_opts</entry>
<entry>"--command gdbinit"</entry>
</row>
<row>
<entry>gdb_prompt</entry>
<entry>"\\(gdb960\\)"</entry>
<entry>The prompt GDB is using.</entry>
</row>
<row>
<entry>gdb_run_command</entry>
<entry>"jump start"</entry>
</row>
<row>
<entry>gdb_stub_offset</entry>
<entry>"0x12010000"</entry>
</row>
<row>
<entry>use_gdb_stub</entry>
<entry>1</entry>
<entry>Whether to use a GDB stub.</entry>
</row>
<row>
<entry>use_vma_offset</entry>
<entry>1</entry>
</row>
<row>
<entry>wrap_m68k_aout</entry>
<entry>1</entry>
</row>
<row>
<entry>gcc,no_label_values</entry>
<entry>1</entry>
</row>
<row>
<entry>gcc,no_trampolines</entry>
<entry>1</entry>
</row>
<row>
<entry>gcc,no_varargs</entry>
<entry>1</entry>
</row>
<row>
<entry>gcc,stack_size</entry>
<entry>16384</entry>
<entry>Stack size to use with some GCC testcases.</entry>
</row>
<row>
<entry>ieee_multilib_flags</entry>
<entry>"-mieee";</entry>
</row>
<row>
<entry>is_simulator</entry>
<entry>1</entry>
</row>
<row>
<entry>needs_status_wrapper</entry>
<entry>1</entry>
</row>
<row>
<entry>no_double</entry>
<entry>1</entry>
</row>
<row>
<entry>no_long_long</entry>
<entry>1</entry>
</row>
<row>
<entry>noargs</entry>
<entry>1</entry>
</row>
<row>
<entry>nullstone,lib</entry>
<entry>"mips-clock.c"</entry>
</row>
<row>
<entry>nullstone,ticks_per_sec</entry>
<entry>3782018</entry>
</row>
<row>
<entry>sys_speed_value</entry>
<entry>200</entry>
</row>
<row>
<entry>target_install</entry>
<entry>{sh-hms}</entry>
</row>
</tbody>
</tgroup>
</table>
</para>
</sect1>
<sect1 id=writing xreflabel="Writing A Test Case">
<title>Writing A Test Case</title>
<para>The easiest way to prepare a new test case is to base it
on an existing one for a similar situation. There are two major
categories of tests: batch or interactive. Batch oriented tests
are usually easier to write.</para>
<para>The GCC tests are a good example of batch oriented tests.
All GCC tests consist primarily of a call to a single common
procedure, Since all the tests either have no output, or only
have a few warning messages when successfully compiled. Any
non-warning output is a test failure. All the C code needed is
kept in the test directory. The test driver, written in Tcl,
need only get a listing of all the C files in the directory, and
compile them all using a generic procedure. This procedure and a
few others supporting for these tests are kept in the library
module <filename>lib/c-torture.exp</filename> in the GCC test
suite. Most tests of this kind use very few
<productname>expect</productname> features, and are coded almost
purely in Tcl.</para>
<para>Writing the complete suite of C tests, then, consisted of
these steps:</para>
<itemizedlist mark=bullet>
<listitem><para>Copying all the C code into the test directory.
These tests were based on the C-torture test created by Torbjorn
Granlund (on behalf of the Free Software Foundation) for GCC
development.</para></listitem>
<listitem><para>Writing (and debugging) the generic Tcl procedures for
compilation.</para></listitem>
<listitem><para>Writing the simple test driver: its main task is to
search the directory (using the Tcl procedure
<emphasis>glob</emphasis> for filename expansion with wildcards)
and call a Tcl procedure with each filename. It also checks for
a few errors from the testing procedure.</para></listitem>
</itemizedlist>
<para>Testing interactive programs is intrinsically more
complex. Tests for most interactive programs require some trial
and error before they are complete.</para>
<para>However, some interactive programs can be tested in a
simple fashion reminiscent of batch tests. For example, prior
to the creation of DejaGnu, the GDB distribution already
included a wide-ranging testing procedure. This procedure was
very robust, and had already undergone much more debugging and
error checking than many recent DejaGnu test cases.
Accordingly, the best approach was simply to encapsulate the
existing GDB tests, for reporting purposes. Thereafter, new GDB
tests built up a family of Tcl procedures specialized for GDB
testing.</para>
</sect1>
<sect1 id=debugging xreflabel="Debugging A Test Case">
<title>Debugging A Test Case</title>
<para>These are the kinds of debugging information available
from DejaGnu:</para>
<itemizedlist mark=bullet>
<listitem><para>Output controlled by test scripts themselves,
explicitly allowed for by the test author. This kind of
debugging output appears in the detailed output recorded in the
DejaGnu log file. To do the same for new tests, use the
<command>verbose</command> procedure (which in turn uses the
variable also called <emphasis>verbose</emphasis>) to control
how much output to generate. This will make it easier for other
people running the test to debug it if necessary. Whenever
possible, if <emphasis>$verbose</emphasis> is
<emphasis>0</emphasis>, there should be no output other than the
output from <emphasis>pass</emphasis>,
<emphasis>fail</emphasis>, <emphasis>error</emphasis>, and
<emphasis>warning</emphasis>. Then, to whatever extent is
appropriate for the particular test, allow successively higher
values of <emphasis>$verbose</emphasis> to generate more
information. Be kind to other programmers who use your tests:
provide for a lot of debugging information.</para></listitem>
<listitem><para>Output from the internal debugging functions of
Tcl and <productname>Expect</productname>. There is a command
line options for each; both forms of debugging output are
recorded in the file <filename>dbg.log</filename> in the current
directory.</para>
<para>Use <option>--debug</option> for information from the
expect level; it generates displays of the expect attempts to
match the tool output with the patterns specified. This output
can be very helpful while developing test scripts, since it
shows precisely the characters received. Iterating between the
latest attempt at a new test script and the corresponding
<filename>dbg.log</filename> can allow you to create the final
patterns by ``cut and paste''. This is sometimes the best way
to write a test case.</para></listitem>
<listitem><para>Use <option>--strace</option> to see more
detail at the Tcl level; this shows how Tcl procedure
definitions expand, as they execute. The associated number
controls the depth of definitions expanded.</para></listitem>
<listitem><para>Finally, if the value of
<emphasis>verbose</emphasis> is 3 or greater,DejaGnu turns on
the expect command <command>log_user</command>. This command
prints all expect actions to the expect standard output, to the
detailed log file, and (if <option>--debug</option> is on) to
<filename>dbg.log</filename>.</para></listitem>
</itemizedlist>
</sect1>
<sect1 id=adding xreflabel="Adding A Test Case To A Test Suite">
<title>Adding A Test Case To A Test Suite.</title>
<para>There are two slightly different ways to add a test
case. One is to add the test case to an existing directory. The
other is to create a new directory to hold your test. The
existing test directories represent several styles of testing,
all of which are slightly different; examine the directories for
the tool of interest to see which (if any) is most suitable.</para>
<para>Adding a GCC test can be very simple: just add the C code
to any directory beginning with <filename>gcc</filename>. and it
runs on the next <programlisting>runtest --tool
gcc</programlisting>.</para>
<para>To add a test to GDB, first add any source code you will
need to the test directory. Then you can either create a new
expect file, or add your test to an existing one (any
file with a <emphasis>.exp</emphasis> suffix). Creating a new
.exp file is probably a better idea if the test is significantly
different from existing tests. Adding it as a separate file also
makes upgrading easier. If the C code has to be already compiled
before the test will run, then you'll have to add it to the
<filename>Makefile.in</filename> file for that test directory,
then run <command>configure</command> and
<command>make</command>.</para>
<para>Adding a test by creating a new directory is very
similar:</para>
<itemizedlist mark=bullet>
<listitem><para>Create the new directory. All subdirectory names
begin with the name of the tool to test; e.g. G++ tests might be
in a directory called <filename>g++.other</filename>. There can
be multiple test directories that start with the same tool name
(such as <emphasis>g++</emphasis>).</para></listitem>
<listitem><para>Add the new directory name to the
<symbol>configdirs</symbol> definition in the
<filename>configure.in</filename> file for the test suite
directory. This way when <command>make</command> and
<command>configure</command> next run, they include the new
directory.</para></listitem>
<listitem><para>Add the new test case to the directory, as
above. </para></listitem>
<listitem><para>To add support in the new directory for
configure and make, you must also create a
<filename>Makefile.in</filename> and a
<filename>configure.in</filename>.</para></listitem>
</itemizedlist>
</sect1>
<sect1 id=hints xreflabel="Hints On Writing A Test Case">
<title>Hints On Writing A Test Case</title>
<para>It is safest to write patterns that match all the output
generated by the tested program; this is called closure.
If a pattern does not match the entire output, any output that
remains will be examined by the next <command>expect</command>
command. In this situation, the precise boundary that determines
which <command>expect</command> command sees what is very
sensitive to timing between the Expect task and the task running
the tested tool. As a result, the test may sometimes appear to
work, but is likely to have unpredictable results. (This problem
is particularly likely for interactive tools, but can also
affect batch tools---especially for tests that take a long time
to finish.) The best way to ensure closure is to use the
<option>-re</option> option for the <command>expect</command>
command to write the pattern as a full regular expressions; then
you can match the end of output using a <emphasis>$</emphasis>.
It is also a good idea to write patterns that match all
available output by using <emphasis>.*\</emphasis> after the
text of interest; this will also match any intervening blank
lines. Sometimes an alternative is to match end of line using
<emphasis>\r</emphasis> or <emphasis>\n</emphasis>, but this is
usually too dependent on terminal settings.</para>
<para>Always escape punctuation, such as <emphasis>(</emphasis>
or <emphasis>"</emphasis>, in your patterns; for example, write
<emphasis>\(</emphasis>. If you forget to escape punctuation,
you will usually see an error message like <programlisting>extra
characters after close-quote.</programlisting></para>
<para>If you have trouble understanding why a pattern does not
match the program output, try using the <option>--debug</option>
option to <command>runtest</command>, and examine the debug log
carefully.</para>
<para>Be careful not to neglect output generated by setup rather
than by the interesting parts of a test case. For example,
while testing GDB, I issue a send <emphasis>set height
0\n</emphasis> command. The purpose is simply to make sure GDB
never calls a paging program. The <emphasis>set
height</emphasis> command in GDB does not generate any
output; but running any command makes GDB issue a new
<emphasis>(gdb) </emphasis> prompt. If there were no
<command>expect</command> command to match this prompt, the
output <emphasis>(gdb) </emphasis> begins the text seen by the
next <command>expect</command> command---which might make that
pattern fail to match.</para>
<para>To preserve basic sanity, I also recommended that no test
ever pass if there was any kind of problem in the test case. To
take an extreme case, tests that pass even when the tool will
not spawn are misleading. Ideally, a test in this sort of
situation should not fail either. Instead, print an error
message by calling one of the DejaGnu procedures
<command>error</command> or <command>warning</command>.</para>
</sect1>
<sect1 id=tvariables xreflabel="Test Case Variables">
<title>Special variables used by test cases.</title>
<para>There are special variables used by test cases. These contain
other information from DejaGnu. Your test cases can use these variables,
with conventional meanings (as well as the variables saved in
<filename>site.exp</filename>. You can use the value of these variables,
but they should never be changed.</para>
<variablelist>
<varlistentry>
<term>$prms_id</term>
<listitem><para>The tracking system (e.g. GNATS) number identifying
a corresponding bugreport. (<emphasis>0</emphasis>} if you do not
specify it in the test script.)</para></listitem>
</varlistentry>
<varlistentry>
<term>$item bug_id</term>
<listitem><para>An optional bug id; may reflect a bug
identification from another organization. (<emphasis>0</emphasis>
if you do not specify it.)</para></listitem>
</varlistentry>
<varlistentry>
<term>$subdir</term>
<listitem><para>The subdirectory for the current test
case.</para></listitem>
</varlistentry>
<varlistentry>
<term>$expect_out(buffer)</term>
<listitem><para>The output from the last command. This is an
internal variable set by Expect. More information can be found in
the Expect manual.</para></listitem>
</varlistentry>
<varlistentry>
<term>$exec_output</term>
<listitem><para>This is the output from a
<function>${tool}_load</function> command. This only applies to
tools like GCC and GAS which produce an object file that must in
turn be executed to complete a test.</para></listitem>
</varlistentry>
<varlistentry>
<term>$comp_output</term>
<listitem><para>This is the output from a
<function>${tool}_start</function> command. This is conventionally
used for batch oriented programs, like GCC and GAS, that may
produce interesting output (warnings, errors) without further
interaction.</para></listitem>
</varlistentry>
</variablelist>
</sect1>
</chapter>
<!-- Keep this comment at the end of the file
Local variables:
mode: sgml
sgml-omittag:t
sgml-shorttag:t
sgml-namecase-general:t
sgml-general-insert-case:lower
sgml-minimize-attributes:nil
sgml-always-quote-attributes:t
sgml-indent-step:1
sgml-indent-data:nil
sgml-parent-document:nil
sgml-exposed-tags:nil
sgml-local-catalogs:nil
sgml-local-ecat-files:nil
End:
-->
|