diff options
author | Ian Lance Taylor <iant@golang.org> | 2022-02-11 14:53:56 -0800 |
---|---|---|
committer | Ian Lance Taylor <iant@golang.org> | 2022-02-11 15:01:19 -0800 |
commit | 8dc2499aa62f768c6395c9754b8cabc1ce25c494 (patch) | |
tree | 43d7fd2bbfd7ad8c9625a718a5e8718889351994 /libgo/go/testing/testing.go | |
parent | 9a56779dbc4e2d9c15be8d31e36f2f59be7331a8 (diff) | |
download | gcc-8dc2499aa62f768c6395c9754b8cabc1ce25c494.zip gcc-8dc2499aa62f768c6395c9754b8cabc1ce25c494.tar.gz gcc-8dc2499aa62f768c6395c9754b8cabc1ce25c494.tar.bz2 |
libgo: update to Go1.18beta2
gotools/
* Makefile.am (go_cmd_cgo_files): Add ast_go118.go
(check-go-tool): Copy golang.org/x/tools directories.
* Makefile.in: Regenerate.
Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/384695
Diffstat (limited to 'libgo/go/testing/testing.go')
-rw-r--r-- | libgo/go/testing/testing.go | 350 |
1 files changed, 296 insertions, 54 deletions
diff --git a/libgo/go/testing/testing.go b/libgo/go/testing/testing.go index 4c823dc..827b23b 100644 --- a/libgo/go/testing/testing.go +++ b/libgo/go/testing/testing.go @@ -34,7 +34,7 @@ // its -bench flag is provided. Benchmarks are run sequentially. // // For a description of the testing flags, see -// https://golang.org/cmd/go/#hdr-Testing_flags +// https://golang.org/cmd/go/#hdr-Testing_flags. // // A sample benchmark function looks like this: // func BenchmarkRandInt(b *testing.B) { @@ -75,6 +75,14 @@ // }) // } // +// A detailed specification of the benchmark results format is given +// in https://golang.org/design/14313-benchmark-format. +// +// There are standard tools for working with benchmark results at +// https://golang.org/x/perf/cmd. +// In particular, https://golang.org/x/perf/cmd/benchstat performs +// statistically robust A/B comparisons. +// // Examples // // The package also runs and verifies example code. Example functions may @@ -132,6 +140,71 @@ // example function, at least one other function, type, variable, or constant // declaration, and no test or benchmark functions. // +// Fuzzing +// +// 'go test' and the testing package support fuzzing, a testing technique where +// a function is called with randomly generated inputs to find bugs not +// anticipated by unit tests. +// +// Functions of the form +// func FuzzXxx(*testing.F) +// are considered fuzz tests. +// +// For example: +// +// func FuzzHex(f *testing.F) { +// for _, seed := range [][]byte{{}, {0}, {9}, {0xa}, {0xf}, {1, 2, 3, 4}} { +// f.Add(seed) +// } +// f.Fuzz(func(t *testing.T, in []byte) { +// enc := hex.EncodeToString(in) +// out, err := hex.DecodeString(enc) +// if err != nil { +// t.Fatalf("%v: decode: %v", in, err) +// } +// if !bytes.Equal(in, out) { +// t.Fatalf("%v: not equal after round trip: %v", in, out) +// } +// }) +// } +// +// A fuzz test maintains a seed corpus, or a set of inputs which are run by +// default, and can seed input generation. Seed inputs may be registered by +// calling (*F).Add or by storing files in the directory testdata/fuzz/<Name> +// (where <Name> is the name of the fuzz test) within the package containing +// the fuzz test. Seed inputs are optional, but the fuzzing engine may find +// bugs more efficiently when provided with a set of small seed inputs with good +// code coverage. These seed inputs can also serve as regression tests for bugs +// identified through fuzzing. +// +// The function passed to (*F).Fuzz within the fuzz test is considered the fuzz +// target. A fuzz target must accept a *T parameter, followed by one or more +// parameters for random inputs. The types of arguments passed to (*F).Add must +// be identical to the types of these parameters. The fuzz target may signal +// that it's found a problem the same way tests do: by calling T.Fail (or any +// method that calls it like T.Error or T.Fatal) or by panicking. +// +// When fuzzing is enabled (by setting the -fuzz flag to a regular expression +// that matches a specific fuzz test), the fuzz target is called with arguments +// generated by repeatedly making random changes to the seed inputs. On +// supported platforms, 'go test' compiles the test executable with fuzzing +// coverage instrumentation. The fuzzing engine uses that instrumentation to +// find and cache inputs that expand coverage, increasing the likelihood of +// finding bugs. If the fuzz target fails for a given input, the fuzzing engine +// writes the inputs that caused the failure to a file in the directory +// testdata/fuzz/<Name> within the package directory. This file later serves as +// a seed input. If the file can't be written at that location (for example, +// because the directory is read-only), the fuzzing engine writes the file to +// the fuzz cache directory within the build cache instead. +// +// When fuzzing is disabled, the fuzz target is called with the seed inputs +// registered with F.Add and seed inputs from testdata/fuzz/<Name>. In this +// mode, the fuzz test acts much like a regular test, with subtests started +// with F.Fuzz instead of T.Run. +// +// TODO(#48255): write and link to documentation that will be helpful to users +// who are unfamiliar with fuzzing. +// // Skipping // // Tests or benchmarks may be skipped at run time with a call to @@ -144,6 +217,21 @@ // ... // } // +// The Skip method of *T can be used in a fuzz target if the input is invalid, +// but should not be considered a failing input. For example: +// +// func FuzzJSONMarshalling(f *testing.F) { +// f.Fuzz(func(t *testing.T, b []byte) { +// var v interface{} +// if err := json.Unmarshal(b, &v); err != nil { +// t.Skip() +// } +// if _, err := json.Marshal(v); err != nil { +// t.Error("Marshal: %v", err) +// } +// }) +// } +// // Subtests and Sub-benchmarks // // The Run methods of T and B allow defining subtests and sub-benchmarks, @@ -163,17 +251,25 @@ // of the top-level test and the sequence of names passed to Run, separated by // slashes, with an optional trailing sequence number for disambiguation. // -// The argument to the -run and -bench command-line flags is an unanchored regular +// The argument to the -run, -bench, and -fuzz command-line flags is an unanchored regular // expression that matches the test's name. For tests with multiple slash-separated // elements, such as subtests, the argument is itself slash-separated, with // expressions matching each name element in turn. Because it is unanchored, an // empty expression matches any string. // For example, using "matching" to mean "whose name contains": // -// go test -run '' # Run all tests. -// go test -run Foo # Run top-level tests matching "Foo", such as "TestFooBar". -// go test -run Foo/A= # For top-level tests matching "Foo", run subtests matching "A=". -// go test -run /A=1 # For all top-level tests, run subtests matching "A=1". +// go test -run '' # Run all tests. +// go test -run Foo # Run top-level tests matching "Foo", such as "TestFooBar". +// go test -run Foo/A= # For top-level tests matching "Foo", run subtests matching "A=". +// go test -run /A=1 # For all top-level tests, run subtests matching "A=1". +// go test -fuzz FuzzFoo # Fuzz the target matching "FuzzFoo" +// +// The -run argument can also be used to run a specific value in the seed +// corpus, for debugging. For example: +// go test -run=FuzzFoo/9ddb952d9814 +// +// The -fuzz and -run flags can both be set, in order to fuzz a target but +// skip the execution of all other tests. // // Subtests can also be used to control parallelism. A parent test will only // complete once all of its subtests complete. In this example, all tests are @@ -246,6 +342,7 @@ import ( "io" "math/rand" "os" + "reflect" "runtime" "runtime/debug" "runtime/trace" @@ -307,6 +404,7 @@ func Init() { shuffle = flag.String("test.shuffle", "off", "randomize the execution order of tests and benchmarks") initBenchmarkFlags() + initFuzzFlags() } var ( @@ -355,7 +453,7 @@ func newChattyPrinter(w io.Writer) *chattyPrinter { // Updatef prints a message about the status of the named test to w. // // The formatted message must include the test name itself. -func (p *chattyPrinter) Updatef(testName, format string, args ...interface{}) { +func (p *chattyPrinter) Updatef(testName, format string, args ...any) { p.lastNameMu.Lock() defer p.lastNameMu.Unlock() @@ -369,7 +467,7 @@ func (p *chattyPrinter) Updatef(testName, format string, args ...interface{}) { // Printf prints a message, generated by the named test, that does not // necessarily mention that tests's name itself. -func (p *chattyPrinter) Printf(testName, format string, args ...interface{}) { +func (p *chattyPrinter) Printf(testName, format string, args ...any) { p.lastNameMu.Lock() defer p.lastNameMu.Unlock() @@ -403,6 +501,7 @@ type common struct { cleanupName string // Name of the cleanup function. cleanupPc []uintptr // The stack trace at the point where Cleanup was called. finished bool // Test function has completed. + inFuzzFn bool // Whether the fuzz target, if this is one, is running. chatty *chattyPrinter // A copy of chattyPrinter, if the chatty flag is set. bench bool // Whether the current test is a benchmark. @@ -416,7 +515,7 @@ type common struct { name string // Name of test or benchmark. start time.Time // Time test or benchmark started duration time.Duration - barrier chan bool // To signal parallel subtests they may start. + barrier chan bool // To signal parallel subtests they may start. Nil when T.Parallel is not present (B) or not usable (when fuzzing). signal chan bool // To signal a test is done. sub []*T // Queue of subtests to be run in parallel. @@ -458,6 +557,12 @@ func Verbose() bool { return *chatty } +func (c *common) checkFuzzFn(name string) { + if c.inFuzzFn { + panic(fmt.Sprintf("testing: f.%s was called inside the fuzz target, use t.%s instead", name, name)) + } +} + // frameSkip searches, starting after skip frames, for the first caller frame // in a function not marked as a helper and returns that frame. // The search stops if it finds a tRunner function that @@ -483,6 +588,9 @@ func (c *common) frameSkip(skip int) runtime.Frame { var firstFrame, prevFrame, frame runtime.Frame for more := true; more; prevFrame = frame { frame, more = frames.Next() + if frame.Function == "runtime.gopanic" { + continue + } if frame.Function == c.cleanupName { frames = runtime.CallersFrames(c.cleanupPc) continue @@ -572,7 +680,7 @@ func (c *common) decorate(s string, skip int) string { // flushToParent writes c.output to the parent after first writing the header // with the given format and arguments. -func (c *common) flushToParent(testName, format string, args ...interface{}) { +func (c *common) flushToParent(testName, format string, args ...any) { p := c.parent p.mu.Lock() defer p.mu.Unlock() @@ -632,24 +740,24 @@ func fmtDuration(d time.Duration) string { return fmt.Sprintf("%.2fs", d.Seconds()) } -// TB is the interface common to T and B. +// TB is the interface common to T, B, and F. type TB interface { Cleanup(func()) - Error(args ...interface{}) - Errorf(format string, args ...interface{}) + Error(args ...any) + Errorf(format string, args ...any) Fail() FailNow() Failed() bool - Fatal(args ...interface{}) - Fatalf(format string, args ...interface{}) + Fatal(args ...any) + Fatalf(format string, args ...any) Helper() - Log(args ...interface{}) - Logf(format string, args ...interface{}) + Log(args ...any) + Logf(format string, args ...any) Name() string Setenv(key, value string) - Skip(args ...interface{}) + Skip(args ...any) SkipNow() - Skipf(format string, args ...interface{}) + Skipf(format string, args ...any) Skipped() bool TempDir() string @@ -729,6 +837,7 @@ func (c *common) Failed() bool { // created during the test. Calling FailNow does not stop // those other goroutines. func (c *common) FailNow() { + c.checkFuzzFn("FailNow") c.Fail() // Calling runtime.Goexit will exit the goroutine, which @@ -803,14 +912,20 @@ func (c *common) logDepth(s string, depth int) { // and records the text in the error log. For tests, the text will be printed only if // the test fails or the -test.v flag is set. For benchmarks, the text is always // printed to avoid having performance depend on the value of the -test.v flag. -func (c *common) Log(args ...interface{}) { c.log(fmt.Sprintln(args...)) } +func (c *common) Log(args ...any) { + c.checkFuzzFn("Log") + c.log(fmt.Sprintln(args...)) +} // Logf formats its arguments according to the format, analogous to Printf, and // records the text in the error log. A final newline is added if not provided. For // tests, the text will be printed only if the test fails or the -test.v flag is // set. For benchmarks, the text is always printed to avoid having performance // depend on the value of the -test.v flag. -func (c *common) Logf(format string, args ...interface{}) { c.log(fmt.Sprintf(format, args...)) } +func (c *common) Logf(format string, args ...any) { + c.checkFuzzFn("Logf") + c.log(fmt.Sprintf(format, args...)) +} // This is needed for gccgo to get the tests to pass, because // runtime.Callers doesn't correctly handle skips that land in the @@ -819,7 +934,8 @@ func (c *common) Logf(format string, args ...interface{}) { c.log(fmt.Sprintf(fo //go:noinline // Error is equivalent to Log followed by Fail. -func (c *common) Error(args ...interface{}) { +func (c *common) Error(args ...any) { + c.checkFuzzFn("Error") c.log(fmt.Sprintln(args...)) c.Fail() } @@ -831,7 +947,8 @@ func (c *common) Error(args ...interface{}) { //go:noinline // Errorf is equivalent to Logf followed by Fail. -func (c *common) Errorf(format string, args ...interface{}) { +func (c *common) Errorf(format string, args ...any) { + c.checkFuzzFn("Errorf") c.log(fmt.Sprintf(format, args...)) c.Fail() } @@ -843,7 +960,8 @@ func (c *common) Errorf(format string, args ...interface{}) { //go:noinline // Fatal is equivalent to Log followed by FailNow. -func (c *common) Fatal(args ...interface{}) { +func (c *common) Fatal(args ...any) { + c.checkFuzzFn("Fatal") c.log(fmt.Sprintln(args...)) c.FailNow() } @@ -855,7 +973,8 @@ func (c *common) Fatal(args ...interface{}) { //go:noinline // Fatalf is equivalent to Logf followed by FailNow. -func (c *common) Fatalf(format string, args ...interface{}) { +func (c *common) Fatalf(format string, args ...any) { + c.checkFuzzFn("Fatalf") c.log(fmt.Sprintf(format, args...)) c.FailNow() } @@ -867,13 +986,15 @@ func (c *common) Fatalf(format string, args ...interface{}) { //go:noinline // Skip is equivalent to Log followed by SkipNow. -func (c *common) Skip(args ...interface{}) { +func (c *common) Skip(args ...any) { + c.checkFuzzFn("Skip") c.log(fmt.Sprintln(args...)) c.SkipNow() } // Skipf is equivalent to Logf followed by SkipNow. -func (c *common) Skipf(format string, args ...interface{}) { +func (c *common) Skipf(format string, args ...any) { + c.checkFuzzFn("Skipf") c.log(fmt.Sprintf(format, args...)) c.SkipNow() } @@ -887,6 +1008,7 @@ func (c *common) Skipf(format string, args ...interface{}) { // other goroutines created during the test. Calling SkipNow does not stop // those other goroutines. func (c *common) SkipNow() { + c.checkFuzzFn("SkipNow") c.mu.Lock() c.skipped = true c.finished = true @@ -926,6 +1048,7 @@ func (c *common) Helper() { // subtests complete. Cleanup functions will be called in last added, // first called order. func (c *common) Cleanup(f func()) { + c.checkFuzzFn("Cleanup") var pc [maxStackLen]uintptr // Skip two extra frames to account for this function and runtime.Callers itself. n := runtime.Callers(2, pc[:]) @@ -959,6 +1082,7 @@ func (c *common) Cleanup(f func()) { // Each subsequent call to t.TempDir returns a unique directory; // if the directory creation fails, TempDir terminates the test by calling Fatal. func (c *common) TempDir() string { + c.checkFuzzFn("TempDir") // Use a single parent directory for all the temporary directories // created by a test, each numbered sequentially. c.tempDirMu.Lock() @@ -999,7 +1123,7 @@ func (c *common) TempDir() string { c.tempDir, c.tempDirErr = os.MkdirTemp("", pattern) if c.tempDirErr == nil { c.Cleanup(func() { - if err := os.RemoveAll(c.tempDir); err != nil { + if err := removeAll(c.tempDir); err != nil { c.Errorf("TempDir RemoveAll cleanup: %v", err) } }) @@ -1018,12 +1142,43 @@ func (c *common) TempDir() string { return dir } +// removeAll is like os.RemoveAll, but retries Windows "Access is denied." +// errors up to an arbitrary timeout. +// +// Those errors have been known to occur spuriously on at least the +// windows-amd64-2012 builder (https://go.dev/issue/50051), and can only occur +// legitimately if the test leaves behind a temp file that either is still open +// or the test otherwise lacks permission to delete. In the case of legitimate +// failures, a failing test may take a bit longer to fail, but once the test is +// fixed the extra latency will go away. +func removeAll(path string) error { + const arbitraryTimeout = 2 * time.Second + var ( + start time.Time + nextSleep = 1 * time.Millisecond + ) + for { + err := os.RemoveAll(path) + if !isWindowsAccessDenied(err) { + return err + } + if start.IsZero() { + start = time.Now() + } else if d := time.Since(start) + nextSleep; d >= arbitraryTimeout { + return err + } + time.Sleep(nextSleep) + nextSleep += time.Duration(rand.Int63n(int64(nextSleep))) + } +} + // Setenv calls os.Setenv(key, value) and uses Cleanup to // restore the environment variable to its original value // after the test. // // This cannot be used in parallel tests. func (c *common) Setenv(key, value string) { + c.checkFuzzFn("Setenv") prevValue, ok := os.LookupEnv(key) if err := os.Setenv(key, value); err != nil { @@ -1052,7 +1207,7 @@ const ( // runCleanup is called at the end of the test. // If catchPanic is true, this will catch panics, and return the recovered // value if any. -func (c *common) runCleanup(ph panicHandling) (panicVal interface{}) { +func (c *common) runCleanup(ph panicHandling) (panicVal any) { if ph == recoverAndReturnPanic { defer func() { panicVal = recover() @@ -1116,6 +1271,12 @@ func (t *T) Parallel() { panic("testing: t.Parallel called after t.Setenv; cannot set environment variables in parallel tests") } t.isParallel = true + if t.parent.barrier == nil { + // T.Parallel has no effect when fuzzing. + // Multiple processes may run in parallel, but only one input can run at a + // time per process so we can attribute crashes to specific inputs. + return + } // We don't want to include the time we spend waiting for serial tests // in the test duration. Record the elapsed time thus far and reset the @@ -1188,7 +1349,14 @@ func tRunner(t *T, fn func(t *T)) { t.Errorf("race detected during execution of test") } - // If the test panicked, print any test output before dying. + // Check if the test panicked or Goexited inappropriately. + // + // If this happens in a normal test, print output but continue panicking. + // tRunner is called in its own goroutine, so this terminates the process. + // + // If this happens while fuzzing, recover from the panic and treat it like a + // normal failure. It's important that the process keeps running in order to + // find short inputs that cause panics. err := recover() signal := true @@ -1209,6 +1377,19 @@ func tRunner(t *T, fn func(t *T)) { } } } + + if err != nil && t.context.isFuzzing { + prefix := "panic: " + if err == errNilPanicOrGoexit { + prefix = "" + } + t.Errorf("%s%s\n%s\n", prefix, err, string(debug.Stack())) + t.mu.Lock() + t.finished = true + t.mu.Unlock() + err = nil + } + // Use a deferred call to ensure that we report that the test is // complete even if a cleanup function calls t.FailNow. See issue 41355. didPanic := false @@ -1225,7 +1406,7 @@ func tRunner(t *T, fn func(t *T)) { t.signal <- signal }() - doPanic := func(err interface{}) { + doPanic := func(err any) { t.Fail() if r := t.runCleanup(recoverAndReturnPanic); r != nil { t.Logf("cleanup panicked with %v", r) @@ -1278,7 +1459,7 @@ func tRunner(t *T, fn func(t *T)) { t.report() // Report after all subtests have finished. // Do not lock t.done to allow race detector to detect race in case - // the user does not appropriately synchronizes a goroutine. + // the user does not appropriately synchronize a goroutine. t.done = true if t.parent != nil && atomic.LoadInt32(&t.hasSub) == 0 { t.setRan() @@ -1363,6 +1544,12 @@ type testContext struct { match *matcher deadline time.Time + // isFuzzing is true in the context used when generating random inputs + // for fuzz targets. isFuzzing is false when running normal tests and + // when running fuzz tests as unit tests (without -fuzz or when -fuzz + // does not match). + isFuzzing bool + mu sync.Mutex // Channel used to signal tests that are ready to be run in parallel. @@ -1426,6 +1613,16 @@ func (f matchStringOnly) ImportPath() string { return " func (f matchStringOnly) StartTestLog(io.Writer) {} func (f matchStringOnly) StopTestLog() error { return errMain } func (f matchStringOnly) SetPanicOnExit0(bool) {} +func (f matchStringOnly) CoordinateFuzzing(time.Duration, int64, time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error { + return errMain +} +func (f matchStringOnly) RunFuzzWorker(func(corpusEntry) error) error { return errMain } +func (f matchStringOnly) ReadCorpus(string, []reflect.Type) ([]corpusEntry, error) { + return nil, errMain +} +func (f matchStringOnly) CheckCorpus([]any, []reflect.Type) error { return nil } +func (f matchStringOnly) ResetCoverage() {} +func (f matchStringOnly) SnapshotCoverage() {} // Main is an internal function, part of the implementation of the "go test" command. // It was exported because it is cross-package and predates "internal" packages. @@ -1434,15 +1631,16 @@ func (f matchStringOnly) SetPanicOnExit0(bool) {} // new functionality is added to the testing package. // Systems simulating "go test" should be updated to use MainStart. func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) { - os.Exit(MainStart(matchStringOnly(matchString), tests, benchmarks, examples).Run()) + os.Exit(MainStart(matchStringOnly(matchString), tests, benchmarks, nil, examples).Run()) } // M is a type passed to a TestMain function to run the actual tests. type M struct { - deps testDeps - tests []InternalTest - benchmarks []InternalBenchmark - examples []InternalExample + deps testDeps + tests []InternalTest + benchmarks []InternalBenchmark + fuzzTargets []InternalFuzzTarget + examples []InternalExample timer *time.Timer afterOnce sync.Once @@ -1467,18 +1665,25 @@ type testDeps interface { StartTestLog(io.Writer) StopTestLog() error WriteProfileTo(string, io.Writer, int) error + CoordinateFuzzing(time.Duration, int64, time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error + RunFuzzWorker(func(corpusEntry) error) error + ReadCorpus(string, []reflect.Type) ([]corpusEntry, error) + CheckCorpus([]any, []reflect.Type) error + ResetCoverage() + SnapshotCoverage() } // MainStart is meant for use by tests generated by 'go test'. // It is not meant to be called directly and is not subject to the Go 1 compatibility document. // It may change signature from release to release. -func MainStart(deps testDeps, tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) *M { +func MainStart(deps testDeps, tests []InternalTest, benchmarks []InternalBenchmark, fuzzTargets []InternalFuzzTarget, examples []InternalExample) *M { Init() return &M{ - deps: deps, - tests: tests, - benchmarks: benchmarks, - examples: examples, + deps: deps, + tests: tests, + benchmarks: benchmarks, + fuzzTargets: fuzzTargets, + examples: examples, } } @@ -1505,9 +1710,15 @@ func (m *M) Run() (code int) { m.exitCode = 2 return } + if *matchFuzz != "" && *fuzzCacheDir == "" { + fmt.Fprintln(os.Stderr, "testing: -test.fuzzcachedir must be set if -test.fuzz is set") + flag.Usage() + m.exitCode = 2 + return + } if len(*matchList) != 0 { - listTests(m.deps.MatchString, m.tests, m.benchmarks, m.examples) + listTests(m.deps.MatchString, m.tests, m.benchmarks, m.fuzzTargets, m.examples) m.exitCode = 0 return } @@ -1535,22 +1746,42 @@ func (m *M) Run() (code int) { m.before() defer m.after() - deadline := m.startAlarm() - haveExamples = len(m.examples) > 0 - testRan, testOk := runTests(m.deps.MatchString, m.tests, deadline) - exampleRan, exampleOk := runExamples(m.deps.MatchString, m.examples) - m.stopAlarm() - if !testRan && !exampleRan && *matchBenchmarks == "" { - fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") + + // Run tests, examples, and benchmarks unless this is a fuzz worker process. + // Workers start after this is done by their parent process, and they should + // not repeat this work. + if !*isFuzzWorker { + deadline := m.startAlarm() + haveExamples = len(m.examples) > 0 + testRan, testOk := runTests(m.deps.MatchString, m.tests, deadline) + fuzzTargetsRan, fuzzTargetsOk := runFuzzTests(m.deps, m.fuzzTargets, deadline) + exampleRan, exampleOk := runExamples(m.deps.MatchString, m.examples) + m.stopAlarm() + if !testRan && !exampleRan && !fuzzTargetsRan && *matchBenchmarks == "" && *matchFuzz == "" { + fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") + } + if !testOk || !exampleOk || !fuzzTargetsOk || !runBenchmarks(m.deps.ImportPath(), m.deps.MatchString, m.benchmarks) || race.Errors() > 0 { + fmt.Println("FAIL") + m.exitCode = 1 + return + } } - if !testOk || !exampleOk || !runBenchmarks(m.deps.ImportPath(), m.deps.MatchString, m.benchmarks) || race.Errors() > 0 { + + fuzzingOk := runFuzzing(m.deps, m.fuzzTargets) + if !fuzzingOk { fmt.Println("FAIL") - m.exitCode = 1 + if *isFuzzWorker { + m.exitCode = fuzzWorkerExitCode + } else { + m.exitCode = 1 + } return } - fmt.Println("PASS") m.exitCode = 0 + if !*isFuzzWorker { + fmt.Println("PASS") + } return } @@ -1571,7 +1802,7 @@ func (t *T) report() { } } -func listTests(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) { +func listTests(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, fuzzTargets []InternalFuzzTarget, examples []InternalExample) { if _, err := matchString(*matchList, "non-empty"); err != nil { fmt.Fprintf(os.Stderr, "testing: invalid regexp in -test.list (%q): %s\n", *matchList, err) os.Exit(1) @@ -1587,6 +1818,11 @@ func listTests(matchString func(pat, str string) (bool, error), tests []Internal fmt.Println(bench.Name) } } + for _, fuzzTarget := range fuzzTargets { + if ok, _ := matchString(*matchList, fuzzTarget.Name); ok { + fmt.Println(fuzzTarget.Name) + } + } for _, example := range examples { if ok, _ := matchString(*matchList, example.Name); ok { fmt.Println(example.Name) @@ -1616,6 +1852,12 @@ func runTests(matchString func(pat, str string) (bool, error), tests []InternalT if shouldFailFast() { break } + if i > 0 && !ran { + // There were no tests to run on the first + // iteration. This won't change, so no reason + // to keep trying. + break + } ctx := newTestContext(*parallel, newMatcher(matchString, *match, "-test.run")) ctx.deadline = deadline t := &T{ |