diff options
author | Ian Lance Taylor <iant@golang.org> | 2022-02-11 14:53:56 -0800 |
---|---|---|
committer | Ian Lance Taylor <iant@golang.org> | 2022-02-11 15:01:19 -0800 |
commit | 8dc2499aa62f768c6395c9754b8cabc1ce25c494 (patch) | |
tree | 43d7fd2bbfd7ad8c9625a718a5e8718889351994 /libgo/go/testing | |
parent | 9a56779dbc4e2d9c15be8d31e36f2f59be7331a8 (diff) | |
download | gcc-8dc2499aa62f768c6395c9754b8cabc1ce25c494.zip gcc-8dc2499aa62f768c6395c9754b8cabc1ce25c494.tar.gz gcc-8dc2499aa62f768c6395c9754b8cabc1ce25c494.tar.bz2 |
libgo: update to Go1.18beta2
gotools/
* Makefile.am (go_cmd_cgo_files): Add ast_go118.go
(check-go-tool): Copy golang.org/x/tools directories.
* Makefile.in: Regenerate.
Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/384695
Diffstat (limited to 'libgo/go/testing')
-rw-r--r-- | libgo/go/testing/allocs_test.go | 2 | ||||
-rw-r--r-- | libgo/go/testing/benchmark.go | 32 | ||||
-rw-r--r-- | libgo/go/testing/example.go | 2 | ||||
-rw-r--r-- | libgo/go/testing/fstest/mapfs.go | 8 | ||||
-rw-r--r-- | libgo/go/testing/fstest/mapfs_test.go | 28 | ||||
-rw-r--r-- | libgo/go/testing/fstest/testfs.go | 2 | ||||
-rw-r--r-- | libgo/go/testing/fuzz.go | 704 | ||||
-rw-r--r-- | libgo/go/testing/helper_test.go | 35 | ||||
-rw-r--r-- | libgo/go/testing/helperfuncs_test.go | 32 | ||||
-rw-r--r-- | libgo/go/testing/internal/testdeps/deps.go | 71 | ||||
-rw-r--r-- | libgo/go/testing/match.go | 197 | ||||
-rw-r--r-- | libgo/go/testing/match_test.go | 128 | ||||
-rw-r--r-- | libgo/go/testing/quick/quick.go | 20 | ||||
-rw-r--r-- | libgo/go/testing/run_example.go | 1 | ||||
-rw-r--r-- | libgo/go/testing/run_example_js.go | 1 | ||||
-rw-r--r-- | libgo/go/testing/sub_test.go | 9 | ||||
-rw-r--r-- | libgo/go/testing/testing.go | 350 | ||||
-rw-r--r-- | libgo/go/testing/testing_other.go | 13 | ||||
-rw-r--r-- | libgo/go/testing/testing_windows.go | 18 |
19 files changed, 1458 insertions, 195 deletions
diff --git a/libgo/go/testing/allocs_test.go b/libgo/go/testing/allocs_test.go index 5b346aa..bbd3ae7 100644 --- a/libgo/go/testing/allocs_test.go +++ b/libgo/go/testing/allocs_test.go @@ -6,7 +6,7 @@ package testing_test import "testing" -var global interface{} +var global any var allocsPerRunTests = []struct { name string diff --git a/libgo/go/testing/benchmark.go b/libgo/go/testing/benchmark.go index 15b4426..d8ec217 100644 --- a/libgo/go/testing/benchmark.go +++ b/libgo/go/testing/benchmark.go @@ -32,35 +32,36 @@ var ( matchBenchmarks *string benchmarkMemory *bool - benchTime = benchTimeFlag{d: 1 * time.Second} // changed during test of testing package + benchTime = durationOrCountFlag{d: 1 * time.Second} // changed during test of testing package ) -type benchTimeFlag struct { - d time.Duration - n int +type durationOrCountFlag struct { + d time.Duration + n int + allowZero bool } -func (f *benchTimeFlag) String() string { +func (f *durationOrCountFlag) String() string { if f.n > 0 { return fmt.Sprintf("%dx", f.n) } - return time.Duration(f.d).String() + return f.d.String() } -func (f *benchTimeFlag) Set(s string) error { +func (f *durationOrCountFlag) Set(s string) error { if strings.HasSuffix(s, "x") { n, err := strconv.ParseInt(s[:len(s)-1], 10, 0) - if err != nil || n <= 0 { + if err != nil || n < 0 || (!f.allowZero && n == 0) { return fmt.Errorf("invalid count") } - *f = benchTimeFlag{n: int(n)} + *f = durationOrCountFlag{n: int(n)} return nil } d, err := time.ParseDuration(s) - if err != nil || d <= 0 { + if err != nil || d < 0 || (!f.allowZero && d == 0) { return fmt.Errorf("invalid duration") } - *f = benchTimeFlag{d: d} + *f = durationOrCountFlag{d: d} return nil } @@ -98,7 +99,7 @@ type B struct { previousN int // number of iterations in the previous run previousDuration time.Duration // total duration of the previous run benchFunc func(b *B) - benchTime benchTimeFlag + benchTime durationOrCountFlag bytes int64 missingBytes bool // one of the subbenchmarks does not have bytes set. timerOn bool @@ -298,7 +299,12 @@ func (b *B) launch() { // Run the benchmark for at least the specified amount of time. if b.benchTime.n > 0 { - b.runN(b.benchTime.n) + // We already ran a single iteration in run1. + // If -benchtime=1x was requested, use that result. + // See https://golang.org/issue/32051. + if b.benchTime.n > 1 { + b.runN(b.benchTime.n) + } } else { d := b.benchTime.d for n := int64(1); !b.failed && b.duration < d && n < 1e9; { diff --git a/libgo/go/testing/example.go b/libgo/go/testing/example.go index 0217c5d..f33e8d2 100644 --- a/libgo/go/testing/example.go +++ b/libgo/go/testing/example.go @@ -64,7 +64,7 @@ func sortLines(output string) string { // If recovered is non-nil, it'll panic with that value. // If the test panicked with nil, or invoked runtime.Goexit, it'll be // made to fail and panic with errNilPanicOrGoexit -func (eg *InternalExample) processRunResult(stdout string, timeSpent time.Duration, finished bool, recovered interface{}) (passed bool) { +func (eg *InternalExample) processRunResult(stdout string, timeSpent time.Duration, finished bool, recovered any) (passed bool) { passed = true dstr := fmtDuration(timeSpent) var fail string diff --git a/libgo/go/testing/fstest/mapfs.go b/libgo/go/testing/fstest/mapfs.go index 9fef2f4..4595b73 100644 --- a/libgo/go/testing/fstest/mapfs.go +++ b/libgo/go/testing/fstest/mapfs.go @@ -37,7 +37,7 @@ type MapFile struct { Data []byte // file content Mode fs.FileMode // FileInfo.Mode ModTime time.Time // FileInfo.ModTime - Sys interface{} // FileInfo.Sys + Sys any // FileInfo.Sys } var _ fs.FS = MapFS(nil) @@ -66,7 +66,9 @@ func (fsys MapFS) Open(name string) (fs.File, error) { for fname, f := range fsys { i := strings.Index(fname, "/") if i < 0 { - list = append(list, mapFileInfo{fname, f}) + if fname != "." { + list = append(list, mapFileInfo{fname, f}) + } } else { need[fname[:i]] = true } @@ -154,7 +156,7 @@ func (i *mapFileInfo) Mode() fs.FileMode { return i.f.Mode } func (i *mapFileInfo) Type() fs.FileMode { return i.f.Mode.Type() } func (i *mapFileInfo) ModTime() time.Time { return i.f.ModTime } func (i *mapFileInfo) IsDir() bool { return i.f.Mode&fs.ModeDir != 0 } -func (i *mapFileInfo) Sys() interface{} { return i.f.Sys } +func (i *mapFileInfo) Sys() any { return i.f.Sys } func (i *mapFileInfo) Info() (fs.FileInfo, error) { return i, nil } // An openMapFile is a regular (non-directory) fs.File open for reading. diff --git a/libgo/go/testing/fstest/mapfs_test.go b/libgo/go/testing/fstest/mapfs_test.go index 2abedd6..c8d2928 100644 --- a/libgo/go/testing/fstest/mapfs_test.go +++ b/libgo/go/testing/fstest/mapfs_test.go @@ -5,6 +5,9 @@ package fstest import ( + "fmt" + "io/fs" + "strings" "testing" ) @@ -17,3 +20,28 @@ func TestMapFS(t *testing.T) { t.Fatal(err) } } + +func TestMapFSChmodDot(t *testing.T) { + m := MapFS{ + "a/b.txt": &MapFile{Mode: 0666}, + ".": &MapFile{Mode: 0777 | fs.ModeDir}, + } + buf := new(strings.Builder) + fs.WalkDir(m, ".", func(path string, d fs.DirEntry, err error) error { + fi, err := d.Info() + if err != nil { + return err + } + fmt.Fprintf(buf, "%s: %v\n", path, fi.Mode()) + return nil + }) + want := ` +.: drwxrwxrwx +a: d--------- +a/b.txt: -rw-rw-rw- +`[1:] + got := buf.String() + if want != got { + t.Errorf("MapFS modes want:\n%s\ngot:\n%s\n", want, got) + } +} diff --git a/libgo/go/testing/fstest/testfs.go b/libgo/go/testing/fstest/testfs.go index 5c4f30a..9a65fbb 100644 --- a/libgo/go/testing/fstest/testfs.go +++ b/libgo/go/testing/fstest/testfs.go @@ -105,7 +105,7 @@ type fsTester struct { } // errorf adds an error line to errText. -func (t *fsTester) errorf(format string, args ...interface{}) { +func (t *fsTester) errorf(format string, args ...any) { if len(t.errText) > 0 { t.errText = append(t.errText, '\n') } diff --git a/libgo/go/testing/fuzz.go b/libgo/go/testing/fuzz.go new file mode 100644 index 0000000..e1d7544 --- /dev/null +++ b/libgo/go/testing/fuzz.go @@ -0,0 +1,704 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testing + +import ( + "bytes" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "runtime" + "sync/atomic" + "time" +) + +func initFuzzFlags() { + matchFuzz = flag.String("test.fuzz", "", "run the fuzz test matching `regexp`") + flag.Var(&fuzzDuration, "test.fuzztime", "time to spend fuzzing; default is to run indefinitely") + flag.Var(&minimizeDuration, "test.fuzzminimizetime", "time to spend minimizing a value after finding a failing input") + + fuzzCacheDir = flag.String("test.fuzzcachedir", "", "directory where interesting fuzzing inputs are stored (for use only by cmd/go)") + isFuzzWorker = flag.Bool("test.fuzzworker", false, "coordinate with the parent process to fuzz random values (for use only by cmd/go)") +} + +var ( + matchFuzz *string + fuzzDuration durationOrCountFlag + minimizeDuration = durationOrCountFlag{d: 60 * time.Second, allowZero: true} + fuzzCacheDir *string + isFuzzWorker *bool + + // corpusDir is the parent directory of the fuzz test's seed corpus within + // the package. + corpusDir = "testdata/fuzz" +) + +// fuzzWorkerExitCode is used as an exit code by fuzz worker processes after an +// internal error. This distinguishes internal errors from uncontrolled panics +// and other failiures. Keep in sync with internal/fuzz.workerExitCode. +const fuzzWorkerExitCode = 70 + +// InternalFuzzTarget is an internal type but exported because it is +// cross-package; it is part of the implementation of the "go test" command. +type InternalFuzzTarget struct { + Name string + Fn func(f *F) +} + +// F is a type passed to fuzz tests. +// +// Fuzz tests run generated inputs against a provided fuzz target, which can +// find and report potential bugs in the code being tested. +// +// A fuzz test runs the seed corpus by default, which includes entries provided +// by (*F).Add and entries in the testdata/fuzz/<FuzzTestName> directory. After +// any necessary setup and calls to (*F).Add, the fuzz test must then call +// (*F).Fuzz to provide the fuzz target. See the testing package documentation +// for an example, and see the F.Fuzz and F.Add method documentation for +// details. +// +// *F methods can only be called before (*F).Fuzz. Once the test is +// executing the fuzz target, only (*T) methods can be used. The only *F methods +// that are allowed in the (*F).Fuzz function are (*F).Failed and (*F).Name. +type F struct { + common + fuzzContext *fuzzContext + testContext *testContext + + // inFuzzFn is true when the fuzz function is running. Most F methods cannot + // be called when inFuzzFn is true. + inFuzzFn bool + + // corpus is a set of seed corpus entries, added with F.Add and loaded + // from testdata. + corpus []corpusEntry + + result fuzzResult + fuzzCalled bool +} + +var _ TB = (*F)(nil) + +// corpusEntry is an alias to the same type as internal/fuzz.CorpusEntry. +// We use a type alias because we don't want to export this type, and we can't +// import internal/fuzz from testing. +type corpusEntry = struct { + Parent string + Path string + Data []byte + Values []any + Generation int + IsSeed bool +} + +// Helper marks the calling function as a test helper function. +// When printing file and line information, that function will be skipped. +// Helper may be called simultaneously from multiple goroutines. +func (f *F) Helper() { + if f.inFuzzFn { + panic("testing: f.Helper was called inside the fuzz target, use t.Helper instead") + } + + // common.Helper is inlined here. + // If we called it, it would mark F.Helper as the helper + // instead of the caller. + f.mu.Lock() + defer f.mu.Unlock() + if f.helperPCs == nil { + f.helperPCs = make(map[uintptr]struct{}) + } + // repeating code from callerName here to save walking a stack frame + var pc [1]uintptr + n := runtime.Callers(2, pc[:]) // skip runtime.Callers + Helper + if n == 0 { + panic("testing: zero callers found") + } + if _, found := f.helperPCs[pc[0]]; !found { + f.helperPCs[pc[0]] = struct{}{} + f.helperNames = nil // map will be recreated next time it is needed + } +} + +// Fail marks the function as having failed but continues execution. +func (f *F) Fail() { + // (*F).Fail may be called by (*T).Fail, which we should allow. However, we + // shouldn't allow direct (*F).Fail calls from inside the (*F).Fuzz function. + if f.inFuzzFn { + panic("testing: f.Fail was called inside the fuzz target, use t.Fail instead") + } + f.common.Helper() + f.common.Fail() +} + +// Skipped reports whether the test was skipped. +func (f *F) Skipped() bool { + // (*F).Skipped may be called by tRunner, which we should allow. However, we + // shouldn't allow direct (*F).Skipped calls from inside the (*F).Fuzz function. + if f.inFuzzFn { + panic("testing: f.Skipped was called inside the fuzz target, use t.Skipped instead") + } + f.common.Helper() + return f.common.Skipped() +} + +// Add will add the arguments to the seed corpus for the fuzz test. This will be +// a no-op if called after or within the fuzz target, and args must match the +// arguments for the fuzz target. +func (f *F) Add(args ...any) { + var values []any + for i := range args { + if t := reflect.TypeOf(args[i]); !supportedTypes[t] { + panic(fmt.Sprintf("testing: unsupported type to Add %v", t)) + } + values = append(values, args[i]) + } + f.corpus = append(f.corpus, corpusEntry{Values: values, IsSeed: true, Path: fmt.Sprintf("seed#%d", len(f.corpus))}) +} + +// supportedTypes represents all of the supported types which can be fuzzed. +var supportedTypes = map[reflect.Type]bool{ + reflect.TypeOf(([]byte)("")): true, + reflect.TypeOf((string)("")): true, + reflect.TypeOf((bool)(false)): true, + reflect.TypeOf((byte)(0)): true, + reflect.TypeOf((rune)(0)): true, + reflect.TypeOf((float32)(0)): true, + reflect.TypeOf((float64)(0)): true, + reflect.TypeOf((int)(0)): true, + reflect.TypeOf((int8)(0)): true, + reflect.TypeOf((int16)(0)): true, + reflect.TypeOf((int32)(0)): true, + reflect.TypeOf((int64)(0)): true, + reflect.TypeOf((uint)(0)): true, + reflect.TypeOf((uint8)(0)): true, + reflect.TypeOf((uint16)(0)): true, + reflect.TypeOf((uint32)(0)): true, + reflect.TypeOf((uint64)(0)): true, +} + +// Fuzz runs the fuzz function, ff, for fuzz testing. If ff fails for a set of +// arguments, those arguments will be added to the seed corpus. +// +// ff must be a function with no return value whose first argument is *T and +// whose remaining arguments are the types to be fuzzed. +// For example: +// +// f.Fuzz(func(t *testing.T, b []byte, i int) { ... }) +// +// The following types are allowed: []byte, string, bool, byte, rune, float32, +// float64, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64. +// More types may be supported in the future. +// +// ff must not call any *F methods, e.g. (*F).Log, (*F).Error, (*F).Skip. Use +// the corresponding *T method instead. The only *F methods that are allowed in +// the (*F).Fuzz function are (*F).Failed and (*F).Name. +// +// This function should be fast and deterministic, and its behavior should not +// depend on shared state. No mutatable input arguments, or pointers to them, +// should be retained between executions of the fuzz function, as the memory +// backing them may be mutated during a subsequent invocation. ff must not +// modify the underlying data of the arguments provided by the fuzzing engine. +// +// When fuzzing, F.Fuzz does not return until a problem is found, time runs out +// (set with -fuzztime), or the test process is interrupted by a signal. F.Fuzz +// should be called exactly once, unless F.Skip or F.Fail is called beforehand. +func (f *F) Fuzz(ff any) { + if f.fuzzCalled { + panic("testing: F.Fuzz called more than once") + } + f.fuzzCalled = true + if f.failed { + return + } + f.Helper() + + // ff should be in the form func(*testing.T, ...interface{}) + fn := reflect.ValueOf(ff) + fnType := fn.Type() + if fnType.Kind() != reflect.Func { + panic("testing: F.Fuzz must receive a function") + } + if fnType.NumIn() < 2 || fnType.In(0) != reflect.TypeOf((*T)(nil)) { + panic("testing: fuzz target must receive at least two arguments, where the first argument is a *T") + } + + // Save the types of the function to compare against the corpus. + var types []reflect.Type + for i := 1; i < fnType.NumIn(); i++ { + t := fnType.In(i) + if !supportedTypes[t] { + panic(fmt.Sprintf("testing: unsupported type for fuzzing %v", t)) + } + types = append(types, t) + } + + // Load the testdata seed corpus. Check types of entries in the testdata + // corpus and entries declared with F.Add. + // + // Don't load the seed corpus if this is a worker process; we won't use it. + if f.fuzzContext.mode != fuzzWorker { + for _, c := range f.corpus { + if err := f.fuzzContext.deps.CheckCorpus(c.Values, types); err != nil { + // TODO(#48302): Report the source location of the F.Add call. + f.Fatal(err) + } + } + + // Load seed corpus + c, err := f.fuzzContext.deps.ReadCorpus(filepath.Join(corpusDir, f.name), types) + if err != nil { + f.Fatal(err) + } + for i := range c { + c[i].IsSeed = true // these are all seed corpus values + if f.fuzzContext.mode == fuzzCoordinator { + // If this is the coordinator process, zero the values, since we don't need + // to hold onto them. + c[i].Values = nil + } + } + + f.corpus = append(f.corpus, c...) + } + + // run calls fn on a given input, as a subtest with its own T. + // run is analogous to T.Run. The test filtering and cleanup works similarly. + // fn is called in its own goroutine. + run := func(captureOut io.Writer, e corpusEntry) (ok bool) { + if e.Values == nil { + // The corpusEntry must have non-nil Values in order to run the + // test. If Values is nil, it is a bug in our code. + panic(fmt.Sprintf("corpus file %q was not unmarshaled", e.Path)) + } + if shouldFailFast() { + return true + } + testName := f.name + if e.Path != "" { + testName = fmt.Sprintf("%s/%s", testName, filepath.Base(e.Path)) + } + if f.testContext.isFuzzing { + // Don't preserve subtest names while fuzzing. If fn calls T.Run, + // there will be a very large number of subtests with duplicate names, + // which will use a large amount of memory. The subtest names aren't + // useful since there's no way to re-run them deterministically. + f.testContext.match.clearSubNames() + } + + // Record the stack trace at the point of this call so that if the subtest + // function - which runs in a separate stack - is marked as a helper, we can + // continue walking the stack into the parent test. + var pc [maxStackLen]uintptr + n := runtime.Callers(2, pc[:]) + t := &T{ + common: common{ + barrier: make(chan bool), + signal: make(chan bool), + name: testName, + parent: &f.common, + level: f.level + 1, + creator: pc[:n], + chatty: f.chatty, + }, + context: f.testContext, + } + if captureOut != nil { + // t.parent aliases f.common. + t.parent.w = captureOut + } + t.w = indenter{&t.common} + if t.chatty != nil { + // TODO(#48132): adjust this to work with test2json. + t.chatty.Updatef(t.name, "=== RUN %s\n", t.name) + } + f.common.inFuzzFn, f.inFuzzFn = true, true + go tRunner(t, func(t *T) { + args := []reflect.Value{reflect.ValueOf(t)} + for _, v := range e.Values { + args = append(args, reflect.ValueOf(v)) + } + // Before resetting the current coverage, defer the snapshot so that + // we make sure it is called right before the tRunner function + // exits, regardless of whether it was executed cleanly, panicked, + // or if the fuzzFn called t.Fatal. + if f.testContext.isFuzzing { + defer f.fuzzContext.deps.SnapshotCoverage() + f.fuzzContext.deps.ResetCoverage() + } + fn.Call(args) + }) + <-t.signal + f.common.inFuzzFn, f.inFuzzFn = false, false + return !t.Failed() + } + + switch f.fuzzContext.mode { + case fuzzCoordinator: + // Fuzzing is enabled, and this is the test process started by 'go test'. + // Act as the coordinator process, and coordinate workers to perform the + // actual fuzzing. + corpusTargetDir := filepath.Join(corpusDir, f.name) + cacheTargetDir := filepath.Join(*fuzzCacheDir, f.name) + err := f.fuzzContext.deps.CoordinateFuzzing( + fuzzDuration.d, + int64(fuzzDuration.n), + minimizeDuration.d, + int64(minimizeDuration.n), + *parallel, + f.corpus, + types, + corpusTargetDir, + cacheTargetDir) + if err != nil { + f.result = fuzzResult{Error: err} + f.Fail() + fmt.Fprintf(f.w, "%v\n", err) + if crashErr, ok := err.(fuzzCrashError); ok { + crashPath := crashErr.CrashPath() + fmt.Fprintf(f.w, "Failing input written to %s\n", crashPath) + testName := filepath.Base(crashPath) + fmt.Fprintf(f.w, "To re-run:\ngo test -run=%s/%s\n", f.name, testName) + } + } + // TODO(jayconrod,katiehockman): Aggregate statistics across workers + // and add to FuzzResult (ie. time taken, num iterations) + + case fuzzWorker: + // Fuzzing is enabled, and this is a worker process. Follow instructions + // from the coordinator. + if err := f.fuzzContext.deps.RunFuzzWorker(func(e corpusEntry) error { + // Don't write to f.w (which points to Stdout) if running from a + // fuzz worker. This would become very verbose, particularly during + // minimization. Return the error instead, and let the caller deal + // with the output. + var buf bytes.Buffer + if ok := run(&buf, e); !ok { + return errors.New(buf.String()) + } + return nil + }); err != nil { + // Internal errors are marked with f.Fail; user code may call this too, before F.Fuzz. + // The worker will exit with fuzzWorkerExitCode, indicating this is a failure + // (and 'go test' should exit non-zero) but a failing input should not be recorded. + f.Errorf("communicating with fuzzing coordinator: %v", err) + } + + default: + // Fuzzing is not enabled, or will be done later. Only run the seed + // corpus now. + for _, e := range f.corpus { + name := fmt.Sprintf("%s/%s", f.name, filepath.Base(e.Path)) + if _, ok, _ := f.testContext.match.fullName(nil, name); ok { + run(f.w, e) + } + } + } +} + +func (f *F) report() { + if *isFuzzWorker || f.parent == nil { + return + } + dstr := fmtDuration(f.duration) + format := "--- %s: %s (%s)\n" + if f.Failed() { + f.flushToParent(f.name, format, "FAIL", f.name, dstr) + } else if f.chatty != nil { + if f.Skipped() { + f.flushToParent(f.name, format, "SKIP", f.name, dstr) + } else { + f.flushToParent(f.name, format, "PASS", f.name, dstr) + } + } +} + +// fuzzResult contains the results of a fuzz run. +type fuzzResult struct { + N int // The number of iterations. + T time.Duration // The total time taken. + Error error // Error is the error from the failing input +} + +func (r fuzzResult) String() string { + if r.Error == nil { + return "" + } + return r.Error.Error() +} + +// fuzzCrashError is satisfied by a failing input detected while fuzzing. +// These errors are written to the seed corpus and can be re-run with 'go test'. +// Errors within the fuzzing framework (like I/O errors between coordinator +// and worker processes) don't satisfy this interface. +type fuzzCrashError interface { + error + Unwrap() error + + // CrashPath returns the path of the subtest that corresponds to the saved + // crash input file in the seed corpus. The test can be re-run with go test + // -run=$test/$name $test is the fuzz test name, and $name is the + // filepath.Base of the string returned here. + CrashPath() string +} + +// fuzzContext holds fields common to all fuzz tests. +type fuzzContext struct { + deps testDeps + mode fuzzMode +} + +type fuzzMode uint8 + +const ( + seedCorpusOnly fuzzMode = iota + fuzzCoordinator + fuzzWorker +) + +// runFuzzTests runs the fuzz tests matching the pattern for -run. This will +// only run the (*F).Fuzz function for each seed corpus without using the +// fuzzing engine to generate or mutate inputs. +func runFuzzTests(deps testDeps, fuzzTests []InternalFuzzTarget, deadline time.Time) (ran, ok bool) { + ok = true + if len(fuzzTests) == 0 || *isFuzzWorker { + return ran, ok + } + m := newMatcher(deps.MatchString, *match, "-test.run") + tctx := newTestContext(*parallel, m) + tctx.deadline = deadline + var mFuzz *matcher + if *matchFuzz != "" { + mFuzz = newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz") + } + fctx := &fuzzContext{deps: deps, mode: seedCorpusOnly} + root := common{w: os.Stdout} // gather output in one place + if Verbose() { + root.chatty = newChattyPrinter(root.w) + } + for _, ft := range fuzzTests { + if shouldFailFast() { + break + } + testName, matched, _ := tctx.match.fullName(nil, ft.Name) + if !matched { + continue + } + if mFuzz != nil { + if _, fuzzMatched, _ := mFuzz.fullName(nil, ft.Name); fuzzMatched { + // If this will be fuzzed, then don't run the seed corpus + // right now. That will happen later. + continue + } + } + f := &F{ + common: common{ + signal: make(chan bool), + barrier: make(chan bool), + name: testName, + parent: &root, + level: root.level + 1, + chatty: root.chatty, + }, + testContext: tctx, + fuzzContext: fctx, + } + f.w = indenter{&f.common} + if f.chatty != nil { + // TODO(#48132): adjust this to work with test2json. + f.chatty.Updatef(f.name, "=== RUN %s\n", f.name) + } + + go fRunner(f, ft.Fn) + <-f.signal + } + return root.ran, !root.Failed() +} + +// runFuzzing runs the fuzz test matching the pattern for -fuzz. Only one such +// fuzz test must match. This will run the fuzzing engine to generate and +// mutate new inputs against the fuzz target. +// +// If fuzzing is disabled (-test.fuzz is not set), runFuzzing +// returns immediately. +func runFuzzing(deps testDeps, fuzzTests []InternalFuzzTarget) (ok bool) { + if len(fuzzTests) == 0 || *matchFuzz == "" { + return true + } + m := newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz") + tctx := newTestContext(1, m) + tctx.isFuzzing = true + fctx := &fuzzContext{ + deps: deps, + } + root := common{w: os.Stdout} + if *isFuzzWorker { + root.w = io.Discard + fctx.mode = fuzzWorker + } else { + fctx.mode = fuzzCoordinator + } + if Verbose() && !*isFuzzWorker { + root.chatty = newChattyPrinter(root.w) + } + var fuzzTest *InternalFuzzTarget + var testName string + var matched []string + for i := range fuzzTests { + name, ok, _ := tctx.match.fullName(nil, fuzzTests[i].Name) + if !ok { + continue + } + matched = append(matched, name) + fuzzTest = &fuzzTests[i] + testName = name + } + if len(matched) == 0 { + fmt.Fprintln(os.Stderr, "testing: warning: no fuzz tests to fuzz") + return true + } + if len(matched) > 1 { + fmt.Fprintf(os.Stderr, "testing: will not fuzz, -fuzz matches more than one fuzz test: %v\n", matched) + return false + } + + f := &F{ + common: common{ + signal: make(chan bool), + barrier: nil, // T.Parallel has no effect when fuzzing. + name: testName, + parent: &root, + level: root.level + 1, + chatty: root.chatty, + }, + fuzzContext: fctx, + testContext: tctx, + } + f.w = indenter{&f.common} + if f.chatty != nil { + // TODO(#48132): adjust this to work with test2json. + f.chatty.Updatef(f.name, "=== FUZZ %s\n", f.name) + } + go fRunner(f, fuzzTest.Fn) + <-f.signal + return !f.failed +} + +// fRunner wraps a call to a fuzz test and ensures that cleanup functions are +// called and status flags are set. fRunner should be called in its own +// goroutine. To wait for its completion, receive from f.signal. +// +// fRunner is analogous to tRunner, which wraps subtests started with T.Run. +// Unit tests and fuzz tests work a little differently, so for now, these +// functions aren't consolidated. In particular, because there are no F.Run and +// F.Parallel methods, i.e., no fuzz sub-tests or parallel fuzz tests, a few +// simplifications are made. We also require that F.Fuzz, F.Skip, or F.Fail is +// called. +func fRunner(f *F, fn func(*F)) { + // When this goroutine is done, either because runtime.Goexit was called, a + // panic started, or fn returned normally, record the duration and send + // t.signal, indicating the fuzz test is done. + defer func() { + // Detect whether the fuzz test panicked or called runtime.Goexit + // without calling F.Fuzz, F.Fail, or F.Skip. If it did, panic (possibly + // replacing a nil panic value). Nothing should recover after fRunner + // unwinds, so this should crash the process and print stack. + // Unfortunately, recovering here adds stack frames, but the location of + // the original panic should still be + // clear. + if f.Failed() { + atomic.AddUint32(&numFailed, 1) + } + err := recover() + if err == nil { + f.mu.RLock() + fuzzNotCalled := !f.fuzzCalled && !f.skipped && !f.failed + if !f.finished && !f.skipped && !f.failed { + err = errNilPanicOrGoexit + } + f.mu.RUnlock() + if fuzzNotCalled && err == nil { + f.Error("returned without calling F.Fuzz, F.Fail, or F.Skip") + } + } + + // Use a deferred call to ensure that we report that the test is + // complete even if a cleanup function calls F.FailNow. See issue 41355. + didPanic := false + defer func() { + if !didPanic { + // Only report that the test is complete if it doesn't panic, + // as otherwise the test binary can exit before the panic is + // reported to the user. See issue 41479. + f.signal <- true + } + }() + + // If we recovered a panic or inappropriate runtime.Goexit, fail the test, + // flush the output log up to the root, then panic. + doPanic := func(err any) { + f.Fail() + if r := f.runCleanup(recoverAndReturnPanic); r != nil { + f.Logf("cleanup panicked with %v", r) + } + for root := &f.common; root.parent != nil; root = root.parent { + root.mu.Lock() + root.duration += time.Since(root.start) + d := root.duration + root.mu.Unlock() + root.flushToParent(root.name, "--- FAIL: %s (%s)\n", root.name, fmtDuration(d)) + } + didPanic = true + panic(err) + } + if err != nil { + doPanic(err) + } + + // No panic or inappropriate Goexit. + f.duration += time.Since(f.start) + + if len(f.sub) > 0 { + // Unblock inputs that called T.Parallel while running the seed corpus. + // This only affects fuzz tests run as normal tests. + // While fuzzing, T.Parallel has no effect, so f.sub is empty, and this + // branch is not taken. f.barrier is nil in that case. + f.testContext.release() + close(f.barrier) + // Wait for the subtests to complete. + for _, sub := range f.sub { + <-sub.signal + } + cleanupStart := time.Now() + err := f.runCleanup(recoverAndReturnPanic) + f.duration += time.Since(cleanupStart) + if err != nil { + doPanic(err) + } + } + + // Report after all subtests have finished. + f.report() + f.done = true + f.setRan() + }() + defer func() { + if len(f.sub) == 0 { + f.runCleanup(normalPanic) + } + }() + + f.start = time.Now() + fn(f) + + // Code beyond this point will not be executed when FailNow or SkipNow + // is invoked. + f.mu.Lock() + f.finished = true + f.mu.Unlock() +} diff --git a/libgo/go/testing/helper_test.go b/libgo/go/testing/helper_test.go index b27fd62..6175410 100644 --- a/libgo/go/testing/helper_test.go +++ b/libgo/go/testing/helper_test.go @@ -33,6 +33,9 @@ helperfuncs_test.go:45: 5 helperfuncs_test.go:21: 6 helperfuncs_test.go:44: 7 helperfuncs_test.go:56: 8 +--- FAIL: Test/sub2 (?s) +helperfuncs_test.go:71: 11 +helperfuncs_test.go:75: recover 12 helperfuncs_test.go:64: 9 helperfuncs_test.go:60: 10 ` @@ -71,38 +74,6 @@ func TestTBHelperParallel(t *T) { } } -func TestTBHelperLineNumer(t *T) { - var buf bytes.Buffer - ctx := newTestContext(1, newMatcher(regexp.MatchString, "", "")) - t1 := &T{ - common: common{ - signal: make(chan bool), - w: &buf, - }, - context: ctx, - } - t1.Run("Test", func(t *T) { - helperA := func(t *T) { - t.Helper() - t.Run("subtest", func(t *T) { - t.Helper() - t.Fatal("fatal error message") - }) - } - helperA(t) - }) - - want := "helper_test.go:92: fatal error message" - got := "" - lines := strings.Split(strings.TrimSpace(buf.String()), "\n") - if len(lines) > 0 { - got = strings.TrimSpace(lines[len(lines)-1]) - } - if got != want { - t.Errorf("got output:\n\n%v\nwant:\n\n%v", got, want) - } -} - type noopWriter int func (nw *noopWriter) Write(b []byte) (int, error) { return len(b), nil } diff --git a/libgo/go/testing/helperfuncs_test.go b/libgo/go/testing/helperfuncs_test.go index df0476e..272b33c 100644 --- a/libgo/go/testing/helperfuncs_test.go +++ b/libgo/go/testing/helperfuncs_test.go @@ -65,6 +65,14 @@ func testHelper(t *T) { t.Helper() t.Error("9") }) + + // Check that helper-ness propagates up through subtests + // to helpers above. See https://golang.org/issue/44887. + helperSubCallingHelper(t, "11") + + // Check that helper-ness propagates up through panic/recover. + // See https://golang.org/issue/31154. + recoverHelper(t, "12") } func parallelTestHelper(t *T) { @@ -78,3 +86,27 @@ func parallelTestHelper(t *T) { } wg.Wait() } + +func helperSubCallingHelper(t *T, msg string) { + t.Helper() + t.Run("sub2", func(t *T) { + t.Helper() + t.Fatal(msg) + }) +} + +func recoverHelper(t *T, msg string) { + t.Helper() + defer func() { + t.Helper() + if err := recover(); err != nil { + t.Errorf("recover %s", err) + } + }() + doPanic(t, msg) +} + +func doPanic(t *T, msg string) { + t.Helper() + panic(msg) +} diff --git a/libgo/go/testing/internal/testdeps/deps.go b/libgo/go/testing/internal/testdeps/deps.go index 3608d33..2e85a41 100644 --- a/libgo/go/testing/internal/testdeps/deps.go +++ b/libgo/go/testing/internal/testdeps/deps.go @@ -12,12 +12,18 @@ package testdeps import ( "bufio" + "context" + "internal/fuzz" "internal/testlog" "io" + "os" + "os/signal" + "reflect" "regexp" "runtime/pprof" "strings" "sync" + "time" ) // TestDeps is an implementation of the testing.testDeps interface, @@ -126,3 +132,68 @@ func (TestDeps) StopTestLog() error { func (TestDeps) SetPanicOnExit0(v bool) { testlog.SetPanicOnExit0(v) } + +func (TestDeps) CoordinateFuzzing( + timeout time.Duration, + limit int64, + minimizeTimeout time.Duration, + minimizeLimit int64, + parallel int, + seed []fuzz.CorpusEntry, + types []reflect.Type, + corpusDir, + cacheDir string) (err error) { + // Fuzzing may be interrupted with a timeout or if the user presses ^C. + // In either case, we'll stop worker processes gracefully and save + // crashers and interesting values. + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + err = fuzz.CoordinateFuzzing(ctx, fuzz.CoordinateFuzzingOpts{ + Log: os.Stderr, + Timeout: timeout, + Limit: limit, + MinimizeTimeout: minimizeTimeout, + MinimizeLimit: minimizeLimit, + Parallel: parallel, + Seed: seed, + Types: types, + CorpusDir: corpusDir, + CacheDir: cacheDir, + }) + if err == ctx.Err() { + return nil + } + return err +} + +func (TestDeps) RunFuzzWorker(fn func(fuzz.CorpusEntry) error) error { + // Worker processes may or may not receive a signal when the user presses ^C + // On POSIX operating systems, a signal sent to a process group is delivered + // to all processes in that group. This is not the case on Windows. + // If the worker is interrupted, return quickly and without error. + // If only the coordinator process is interrupted, it tells each worker + // process to stop by closing its "fuzz_in" pipe. + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + err := fuzz.RunFuzzWorker(ctx, fn) + if err == ctx.Err() { + return nil + } + return err +} + +func (TestDeps) ReadCorpus(dir string, types []reflect.Type) ([]fuzz.CorpusEntry, error) { + return fuzz.ReadCorpus(dir, types) +} + +func (TestDeps) CheckCorpus(vals []any, types []reflect.Type) error { + return fuzz.CheckCorpus(vals, types) +} + +func (TestDeps) ResetCoverage() { + fuzz.ResetCoverage() +} + +func (TestDeps) SnapshotCoverage() { + fuzz.SnapshotCoverage() +} diff --git a/libgo/go/testing/match.go b/libgo/go/testing/match.go index b18c6e7..d530f70 100644 --- a/libgo/go/testing/match.go +++ b/libgo/go/testing/match.go @@ -14,36 +14,52 @@ import ( // matcher sanitizes, uniques, and filters names of subtests and subbenchmarks. type matcher struct { - filter []string + filter filterMatch matchFunc func(pat, str string) (bool, error) - mu sync.Mutex - subNames map[string]int64 + mu sync.Mutex + + // subNames is used to deduplicate subtest names. + // Each key is the subtest name joined to the deduplicated name of the parent test. + // Each value is the count of the number of occurrences of the given subtest name + // already seen. + subNames map[string]int32 +} + +type filterMatch interface { + // matches checks the name against the receiver's pattern strings using the + // given match function. + matches(name []string, matchString func(pat, str string) (bool, error)) (ok, partial bool) + + // verify checks that the receiver's pattern strings are valid filters by + // calling the given match function. + verify(name string, matchString func(pat, str string) (bool, error)) error } +// simpleMatch matches a test name if all of the pattern strings match in +// sequence. +type simpleMatch []string + +// alternationMatch matches a test name if one of the alternations match. +type alternationMatch []filterMatch + // TODO: fix test_main to avoid race and improve caching, also allowing to // eliminate this Mutex. var matchMutex sync.Mutex func newMatcher(matchString func(pat, str string) (bool, error), patterns, name string) *matcher { - var filter []string + var impl filterMatch if patterns != "" { - filter = splitRegexp(patterns) - for i, s := range filter { - filter[i] = rewrite(s) - } - // Verify filters before doing any processing. - for i, s := range filter { - if _, err := matchString(s, "non-empty"); err != nil { - fmt.Fprintf(os.Stderr, "testing: invalid regexp for element %d of %s (%q): %s\n", i, name, s, err) - os.Exit(1) - } + impl = splitRegexp(patterns) + if err := impl.verify(name, matchString); err != nil { + fmt.Fprintf(os.Stderr, "testing: invalid regexp for %s\n", err) + os.Exit(1) } } return &matcher{ - filter: filter, + filter: impl, matchFunc: matchString, - subNames: map[string]int64{}, + subNames: map[string]int32{}, } } @@ -60,22 +76,74 @@ func (m *matcher) fullName(c *common, subname string) (name string, ok, partial matchMutex.Lock() defer matchMutex.Unlock() + if m.filter == nil { + return name, true, false + } + // We check the full array of paths each time to allow for the case that // a pattern contains a '/'. elem := strings.Split(name, "/") - for i, s := range elem { - if i >= len(m.filter) { + ok, partial = m.filter.matches(elem, m.matchFunc) + return name, ok, partial +} + +// clearSubNames clears the matcher's internal state, potentially freeing +// memory. After this is called, T.Name may return the same strings as it did +// for earlier subtests. +func (m *matcher) clearSubNames() { + m.mu.Lock() + defer m.mu.Unlock() + for key := range m.subNames { + delete(m.subNames, key) + } +} + +func (m simpleMatch) matches(name []string, matchString func(pat, str string) (bool, error)) (ok, partial bool) { + for i, s := range name { + if i >= len(m) { break } - if ok, _ := m.matchFunc(m.filter[i], s); !ok { - return name, false, false + if ok, _ := matchString(m[i], s); !ok { + return false, false + } + } + return true, len(name) < len(m) +} + +func (m simpleMatch) verify(name string, matchString func(pat, str string) (bool, error)) error { + for i, s := range m { + m[i] = rewrite(s) + } + // Verify filters before doing any processing. + for i, s := range m { + if _, err := matchString(s, "non-empty"); err != nil { + return fmt.Errorf("element %d of %s (%q): %s", i, name, s, err) + } + } + return nil +} + +func (m alternationMatch) matches(name []string, matchString func(pat, str string) (bool, error)) (ok, partial bool) { + for _, m := range m { + if ok, partial = m.matches(name, matchString); ok { + return ok, partial + } + } + return false, false +} + +func (m alternationMatch) verify(name string, matchString func(pat, str string) (bool, error)) error { + for i, m := range m { + if err := m.verify(name, matchString); err != nil { + return fmt.Errorf("alternation %d of %s", i, err) } } - return name, true, len(elem) < len(m.filter) + return nil } -func splitRegexp(s string) []string { - a := make([]string, 0, strings.Count(s, "/")) +func splitRegexp(s string) filterMatch { + a := make(simpleMatch, 0, strings.Count(s, "/")) + b := make(alternationMatch, 0, strings.Count(s, "|")) cs := 0 cp := 0 for i := 0; i < len(s); { @@ -103,31 +171,88 @@ func splitRegexp(s string) []string { i = 0 continue } + case '|': + if cs == 0 && cp == 0 { + a = append(a, s[:i]) + s = s[i+1:] + i = 0 + b = append(b, a) + a = make(simpleMatch, 0, len(a)) + continue + } } i++ } - return append(a, s) + + a = append(a, s) + if len(b) == 0 { + return a + } + return append(b, a) } // unique creates a unique name for the given parent and subname by affixing it // with one or more counts, if necessary. func (m *matcher) unique(parent, subname string) string { - name := fmt.Sprintf("%s/%s", parent, subname) - empty := subname == "" + base := parent + "/" + subname + for { - next, exists := m.subNames[name] - if !empty && !exists { - m.subNames[name] = 1 // next count is 1 - return name + n := m.subNames[base] + if n < 0 { + panic("subtest count overflow") } - // Name was already used. We increment with the count and append a - // string with the count. - m.subNames[name] = next + 1 + m.subNames[base] = n + 1 + + if n == 0 && subname != "" { + prefix, nn := parseSubtestNumber(base) + if len(prefix) < len(base) && nn < m.subNames[prefix] { + // This test is explicitly named like "parent/subname#NN", + // and #NN was already used for the NNth occurrence of "parent/subname". + // Loop to add a disambiguating suffix. + continue + } + return base + } + + name := fmt.Sprintf("%s#%02d", base, n) + if m.subNames[name] != 0 { + // This is the nth occurrence of base, but the name "parent/subname#NN" + // collides with the first occurrence of a subtest *explicitly* named + // "parent/subname#NN". Try the next number. + continue + } + + return name + } +} + +// parseSubtestNumber splits a subtest name into a "#%02d"-formatted int32 +// suffix (if present), and a prefix preceding that suffix (always). +func parseSubtestNumber(s string) (prefix string, nn int32) { + i := strings.LastIndex(s, "#") + if i < 0 { + return s, 0 + } + + prefix, suffix := s[:i], s[i+1:] + if len(suffix) < 2 || (len(suffix) > 2 && suffix[0] == '0') { + // Even if suffix is numeric, it is not a possible output of a "%02" format + // string: it has either too few digits or too many leading zeroes. + return s, 0 + } + if suffix == "00" { + if !strings.HasSuffix(prefix, "/") { + // We only use "#00" as a suffix for subtests named with the empty + // string — it isn't a valid suffix if the subtest name is non-empty. + return s, 0 + } + } - // Add a count to guarantee uniqueness. - name = fmt.Sprintf("%s#%02d", name, next) - empty = false + n, err := strconv.ParseInt(suffix, 10, 32) + if err != nil || n < 0 { + return s, 0 } + return prefix, int32(n) } // rewrite rewrites a subname to having only printable characters and no white diff --git a/libgo/go/testing/match_test.go b/libgo/go/testing/match_test.go index 8c09dc6..206ac0b 100644 --- a/libgo/go/testing/match_test.go +++ b/libgo/go/testing/match_test.go @@ -5,8 +5,10 @@ package testing import ( + "fmt" "reflect" "regexp" + "strings" "unicode" ) @@ -25,10 +27,11 @@ func TestIsSpace(t *T) { } func TestSplitRegexp(t *T) { - res := func(s ...string) []string { return s } + res := func(s ...string) filterMatch { return simpleMatch(s) } + alt := func(m ...filterMatch) filterMatch { return alternationMatch(m) } testCases := []struct { pattern string - result []string + result filterMatch }{ // Correct patterns // If a regexp pattern is correct, all split regexps need to be correct @@ -49,6 +52,8 @@ func TestSplitRegexp(t *T) { {`([)/][(])`, res(`([)/][(])`)}, {"[(]/[)]", res("[(]", "[)]")}, + {"A/B|C/D", alt(res("A", "B"), res("C", "D"))}, + // Faulty patterns // Errors in original should produce at least one faulty regexp in results. {")/", res(")/")}, @@ -71,10 +76,8 @@ func TestSplitRegexp(t *T) { // needs to have an error as well. if _, err := regexp.Compile(tc.pattern); err != nil { ok := true - for _, re := range a { - if _, err := regexp.Compile(re); err != nil { - ok = false - } + if err := a.verify("", regexp.MatchString); err != nil { + ok = false } if ok { t.Errorf("%s: expected error in any of %q", tc.pattern, a) @@ -113,6 +116,10 @@ func TestMatcher(t *T) { {"TestFoo/", "TestBar", "x", false, false}, {"TestFoo/bar/baz", "TestBar", "x/bar/baz", false, false}, + {"A/B|C/D", "TestA", "B", true, false}, + {"A/B|C/D", "TestC", "D", true, false}, + {"A/B|C/D", "TestA", "C", false, false}, + // subtests only {"", "TestFoo", "x", true, false}, {"/", "TestFoo", "x", true, false}, @@ -142,45 +149,90 @@ func TestMatcher(t *T) { } } +var namingTestCases = []struct{ name, want string }{ + // Uniqueness + {"", "x/#00"}, + {"", "x/#01"}, + {"#0", "x/#0"}, // Doesn't conflict with #00 because the number of digits differs. + {"#00", "x/#00#01"}, // Conflicts with implicit #00 (used above), so add a suffix. + {"#", "x/#"}, + {"#", "x/##01"}, + + {"t", "x/t"}, + {"t", "x/t#01"}, + {"t", "x/t#02"}, + {"t#00", "x/t#00"}, // Explicit "#00" doesn't conflict with the unsuffixed first subtest. + + {"a#01", "x/a#01"}, // user has subtest with this name. + {"a", "x/a"}, // doesn't conflict with this name. + {"a", "x/a#02"}, // This string is claimed now, so resume + {"a", "x/a#03"}, // with counting. + {"a#02", "x/a#02#01"}, // We already used a#02 once, so add a suffix. + + {"b#00", "x/b#00"}, + {"b", "x/b"}, // Implicit 0 doesn't conflict with explicit "#00". + {"b", "x/b#01"}, + {"b#9223372036854775807", "x/b#9223372036854775807"}, // MaxInt64 + {"b", "x/b#02"}, + {"b", "x/b#03"}, + + // Sanitizing + {"A:1 B:2", "x/A:1_B:2"}, + {"s\t\r\u00a0", "x/s___"}, + {"\x01", `x/\x01`}, + {"\U0010ffff", `x/\U0010ffff`}, +} + func TestNaming(t *T) { m := newMatcher(regexp.MatchString, "", "") - parent := &common{name: "x", level: 1} // top-level test. - // Rig the matcher with some preloaded values. - m.subNames["x/b"] = 1000 + for i, tc := range namingTestCases { + if got, _, _ := m.fullName(parent, tc.name); got != tc.want { + t.Errorf("%d:%s: got %q; want %q", i, tc.name, got, tc.want) + } + } +} - testCases := []struct { - name, want string - }{ - // Uniqueness - {"", "x/#00"}, - {"", "x/#01"}, - - {"t", "x/t"}, - {"t", "x/t#01"}, - {"t", "x/t#02"}, - - {"a#01", "x/a#01"}, // user has subtest with this name. - {"a", "x/a"}, // doesn't conflict with this name. - {"a", "x/a#01#01"}, // conflict, add disambiguating string. - {"a", "x/a#02"}, // This string is claimed now, so resume - {"a", "x/a#03"}, // with counting. - {"a#02", "x/a#02#01"}, - - {"b", "x/b#1000"}, // rigged, see above - {"b", "x/b#1001"}, - - // // Sanitizing - {"A:1 B:2", "x/A:1_B:2"}, - {"s\t\r\u00a0", "x/s___"}, - {"\x01", `x/\x01`}, - {"\U0010ffff", `x/\U0010ffff`}, +func FuzzNaming(f *F) { + for _, tc := range namingTestCases { + f.Add(tc.name) + } + parent := &common{name: "x", level: 1} + var m *matcher + var seen map[string]string + reset := func() { + m = newMatcher(regexp.MatchString, "", "") + seen = make(map[string]string) } + reset() - for i, tc := range testCases { - if got, _, _ := m.fullName(parent, tc.name); got != tc.want { - t.Errorf("%d:%s: got %q; want %q", i, tc.name, got, tc.want) + f.Fuzz(func(t *T, subname string) { + if len(subname) > 10 { + // Long names attract the OOM killer. + t.Skip() } + name := m.unique(parent.name, subname) + if !strings.Contains(name, "/"+subname) { + t.Errorf("name %q does not contain subname %q", name, subname) + } + if prev, ok := seen[name]; ok { + t.Errorf("name %q generated by both %q and %q", name, prev, subname) + } + if len(seen) > 1e6 { + // Free up memory. + reset() + } + seen[name] = subname + }) +} + +// GoString returns a string that is more readable than the default, which makes +// it easier to read test errors. +func (m alternationMatch) GoString() string { + s := make([]string, len(m)) + for i, m := range m { + s[i] = fmt.Sprintf("%#v", m) } + return fmt.Sprintf("(%s)", strings.Join(s, " | ")) } diff --git a/libgo/go/testing/quick/quick.go b/libgo/go/testing/quick/quick.go index c01647e..e73d307 100644 --- a/libgo/go/testing/quick/quick.go +++ b/libgo/go/testing/quick/quick.go @@ -113,7 +113,7 @@ func sizedValue(t reflect.Type, rand *rand.Rand, size int) (value reflect.Value, } v.SetMapIndex(key, value) } - case reflect.Ptr: + case reflect.Pointer: if rand.Intn(size) == 0 { v.Set(reflect.Zero(concrete)) // Generate nil pointer. } else { @@ -227,7 +227,7 @@ func (s SetupError) Error() string { return string(s) } // A CheckError is the result of Check finding an error. type CheckError struct { Count int - In []interface{} + In []any } func (s *CheckError) Error() string { @@ -237,8 +237,8 @@ func (s *CheckError) Error() string { // A CheckEqualError is the result CheckEqual finding an error. type CheckEqualError struct { CheckError - Out1 []interface{} - Out2 []interface{} + Out1 []any + Out2 []any } func (s *CheckEqualError) Error() string { @@ -260,7 +260,7 @@ func (s *CheckEqualError) Error() string { // t.Error(err) // } // } -func Check(f interface{}, config *Config) error { +func Check(f any, config *Config) error { if config == nil { config = &defaultConfig } @@ -299,7 +299,7 @@ func Check(f interface{}, config *Config) error { // It calls f and g repeatedly with arbitrary values for each argument. // If f and g return different answers, CheckEqual returns a *CheckEqualError // describing the input and the outputs. -func CheckEqual(f, g interface{}, config *Config) error { +func CheckEqual(f, g any, config *Config) error { if config == nil { config = &defaultConfig } @@ -358,7 +358,7 @@ func arbitraryValues(args []reflect.Value, f reflect.Type, config *Config, rand return } -func functionAndType(f interface{}) (v reflect.Value, t reflect.Type, ok bool) { +func functionAndType(f any) (v reflect.Value, t reflect.Type, ok bool) { v = reflect.ValueOf(f) ok = v.Kind() == reflect.Func if !ok { @@ -368,15 +368,15 @@ func functionAndType(f interface{}) (v reflect.Value, t reflect.Type, ok bool) { return } -func toInterfaces(values []reflect.Value) []interface{} { - ret := make([]interface{}, len(values)) +func toInterfaces(values []reflect.Value) []any { + ret := make([]any, len(values)) for i, v := range values { ret[i] = v.Interface() } return ret } -func toString(interfaces []interface{}) string { +func toString(interfaces []any) string { s := make([]string, len(interfaces)) for i, v := range interfaces { s[i] = fmt.Sprintf("%#v", v) diff --git a/libgo/go/testing/run_example.go b/libgo/go/testing/run_example.go index d9e342d..e7eab1e 100644 --- a/libgo/go/testing/run_example.go +++ b/libgo/go/testing/run_example.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !js -// +build !js // TODO(@musiol, @odeke-em): re-unify this entire file back into // example.go when js/wasm gets an os.Pipe implementation diff --git a/libgo/go/testing/run_example_js.go b/libgo/go/testing/run_example_js.go index d914633..adef951 100644 --- a/libgo/go/testing/run_example_js.go +++ b/libgo/go/testing/run_example_js.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build js -// +build js package testing diff --git a/libgo/go/testing/sub_test.go b/libgo/go/testing/sub_test.go index 6c7d83a..6a5add6 100644 --- a/libgo/go/testing/sub_test.go +++ b/libgo/go/testing/sub_test.go @@ -480,9 +480,10 @@ func TestTRun(t *T) { buf := &bytes.Buffer{} root := &T{ common: common{ - signal: make(chan bool), - name: "Test", - w: buf, + signal: make(chan bool), + barrier: make(chan bool), + name: "Test", + w: buf, }, context: ctx, } @@ -669,7 +670,7 @@ func TestBRun(t *T) { w: buf, }, benchFunc: func(b *B) { ok = b.Run("test", tc.f) }, // Use Run to catch failure. - benchTime: benchTimeFlag{d: 1 * time.Microsecond}, + benchTime: durationOrCountFlag{d: 1 * time.Microsecond}, } if tc.chatty { root.chatty = newChattyPrinter(root.w) diff --git a/libgo/go/testing/testing.go b/libgo/go/testing/testing.go index 4c823dc..827b23b 100644 --- a/libgo/go/testing/testing.go +++ b/libgo/go/testing/testing.go @@ -34,7 +34,7 @@ // its -bench flag is provided. Benchmarks are run sequentially. // // For a description of the testing flags, see -// https://golang.org/cmd/go/#hdr-Testing_flags +// https://golang.org/cmd/go/#hdr-Testing_flags. // // A sample benchmark function looks like this: // func BenchmarkRandInt(b *testing.B) { @@ -75,6 +75,14 @@ // }) // } // +// A detailed specification of the benchmark results format is given +// in https://golang.org/design/14313-benchmark-format. +// +// There are standard tools for working with benchmark results at +// https://golang.org/x/perf/cmd. +// In particular, https://golang.org/x/perf/cmd/benchstat performs +// statistically robust A/B comparisons. +// // Examples // // The package also runs and verifies example code. Example functions may @@ -132,6 +140,71 @@ // example function, at least one other function, type, variable, or constant // declaration, and no test or benchmark functions. // +// Fuzzing +// +// 'go test' and the testing package support fuzzing, a testing technique where +// a function is called with randomly generated inputs to find bugs not +// anticipated by unit tests. +// +// Functions of the form +// func FuzzXxx(*testing.F) +// are considered fuzz tests. +// +// For example: +// +// func FuzzHex(f *testing.F) { +// for _, seed := range [][]byte{{}, {0}, {9}, {0xa}, {0xf}, {1, 2, 3, 4}} { +// f.Add(seed) +// } +// f.Fuzz(func(t *testing.T, in []byte) { +// enc := hex.EncodeToString(in) +// out, err := hex.DecodeString(enc) +// if err != nil { +// t.Fatalf("%v: decode: %v", in, err) +// } +// if !bytes.Equal(in, out) { +// t.Fatalf("%v: not equal after round trip: %v", in, out) +// } +// }) +// } +// +// A fuzz test maintains a seed corpus, or a set of inputs which are run by +// default, and can seed input generation. Seed inputs may be registered by +// calling (*F).Add or by storing files in the directory testdata/fuzz/<Name> +// (where <Name> is the name of the fuzz test) within the package containing +// the fuzz test. Seed inputs are optional, but the fuzzing engine may find +// bugs more efficiently when provided with a set of small seed inputs with good +// code coverage. These seed inputs can also serve as regression tests for bugs +// identified through fuzzing. +// +// The function passed to (*F).Fuzz within the fuzz test is considered the fuzz +// target. A fuzz target must accept a *T parameter, followed by one or more +// parameters for random inputs. The types of arguments passed to (*F).Add must +// be identical to the types of these parameters. The fuzz target may signal +// that it's found a problem the same way tests do: by calling T.Fail (or any +// method that calls it like T.Error or T.Fatal) or by panicking. +// +// When fuzzing is enabled (by setting the -fuzz flag to a regular expression +// that matches a specific fuzz test), the fuzz target is called with arguments +// generated by repeatedly making random changes to the seed inputs. On +// supported platforms, 'go test' compiles the test executable with fuzzing +// coverage instrumentation. The fuzzing engine uses that instrumentation to +// find and cache inputs that expand coverage, increasing the likelihood of +// finding bugs. If the fuzz target fails for a given input, the fuzzing engine +// writes the inputs that caused the failure to a file in the directory +// testdata/fuzz/<Name> within the package directory. This file later serves as +// a seed input. If the file can't be written at that location (for example, +// because the directory is read-only), the fuzzing engine writes the file to +// the fuzz cache directory within the build cache instead. +// +// When fuzzing is disabled, the fuzz target is called with the seed inputs +// registered with F.Add and seed inputs from testdata/fuzz/<Name>. In this +// mode, the fuzz test acts much like a regular test, with subtests started +// with F.Fuzz instead of T.Run. +// +// TODO(#48255): write and link to documentation that will be helpful to users +// who are unfamiliar with fuzzing. +// // Skipping // // Tests or benchmarks may be skipped at run time with a call to @@ -144,6 +217,21 @@ // ... // } // +// The Skip method of *T can be used in a fuzz target if the input is invalid, +// but should not be considered a failing input. For example: +// +// func FuzzJSONMarshalling(f *testing.F) { +// f.Fuzz(func(t *testing.T, b []byte) { +// var v interface{} +// if err := json.Unmarshal(b, &v); err != nil { +// t.Skip() +// } +// if _, err := json.Marshal(v); err != nil { +// t.Error("Marshal: %v", err) +// } +// }) +// } +// // Subtests and Sub-benchmarks // // The Run methods of T and B allow defining subtests and sub-benchmarks, @@ -163,17 +251,25 @@ // of the top-level test and the sequence of names passed to Run, separated by // slashes, with an optional trailing sequence number for disambiguation. // -// The argument to the -run and -bench command-line flags is an unanchored regular +// The argument to the -run, -bench, and -fuzz command-line flags is an unanchored regular // expression that matches the test's name. For tests with multiple slash-separated // elements, such as subtests, the argument is itself slash-separated, with // expressions matching each name element in turn. Because it is unanchored, an // empty expression matches any string. // For example, using "matching" to mean "whose name contains": // -// go test -run '' # Run all tests. -// go test -run Foo # Run top-level tests matching "Foo", such as "TestFooBar". -// go test -run Foo/A= # For top-level tests matching "Foo", run subtests matching "A=". -// go test -run /A=1 # For all top-level tests, run subtests matching "A=1". +// go test -run '' # Run all tests. +// go test -run Foo # Run top-level tests matching "Foo", such as "TestFooBar". +// go test -run Foo/A= # For top-level tests matching "Foo", run subtests matching "A=". +// go test -run /A=1 # For all top-level tests, run subtests matching "A=1". +// go test -fuzz FuzzFoo # Fuzz the target matching "FuzzFoo" +// +// The -run argument can also be used to run a specific value in the seed +// corpus, for debugging. For example: +// go test -run=FuzzFoo/9ddb952d9814 +// +// The -fuzz and -run flags can both be set, in order to fuzz a target but +// skip the execution of all other tests. // // Subtests can also be used to control parallelism. A parent test will only // complete once all of its subtests complete. In this example, all tests are @@ -246,6 +342,7 @@ import ( "io" "math/rand" "os" + "reflect" "runtime" "runtime/debug" "runtime/trace" @@ -307,6 +404,7 @@ func Init() { shuffle = flag.String("test.shuffle", "off", "randomize the execution order of tests and benchmarks") initBenchmarkFlags() + initFuzzFlags() } var ( @@ -355,7 +453,7 @@ func newChattyPrinter(w io.Writer) *chattyPrinter { // Updatef prints a message about the status of the named test to w. // // The formatted message must include the test name itself. -func (p *chattyPrinter) Updatef(testName, format string, args ...interface{}) { +func (p *chattyPrinter) Updatef(testName, format string, args ...any) { p.lastNameMu.Lock() defer p.lastNameMu.Unlock() @@ -369,7 +467,7 @@ func (p *chattyPrinter) Updatef(testName, format string, args ...interface{}) { // Printf prints a message, generated by the named test, that does not // necessarily mention that tests's name itself. -func (p *chattyPrinter) Printf(testName, format string, args ...interface{}) { +func (p *chattyPrinter) Printf(testName, format string, args ...any) { p.lastNameMu.Lock() defer p.lastNameMu.Unlock() @@ -403,6 +501,7 @@ type common struct { cleanupName string // Name of the cleanup function. cleanupPc []uintptr // The stack trace at the point where Cleanup was called. finished bool // Test function has completed. + inFuzzFn bool // Whether the fuzz target, if this is one, is running. chatty *chattyPrinter // A copy of chattyPrinter, if the chatty flag is set. bench bool // Whether the current test is a benchmark. @@ -416,7 +515,7 @@ type common struct { name string // Name of test or benchmark. start time.Time // Time test or benchmark started duration time.Duration - barrier chan bool // To signal parallel subtests they may start. + barrier chan bool // To signal parallel subtests they may start. Nil when T.Parallel is not present (B) or not usable (when fuzzing). signal chan bool // To signal a test is done. sub []*T // Queue of subtests to be run in parallel. @@ -458,6 +557,12 @@ func Verbose() bool { return *chatty } +func (c *common) checkFuzzFn(name string) { + if c.inFuzzFn { + panic(fmt.Sprintf("testing: f.%s was called inside the fuzz target, use t.%s instead", name, name)) + } +} + // frameSkip searches, starting after skip frames, for the first caller frame // in a function not marked as a helper and returns that frame. // The search stops if it finds a tRunner function that @@ -483,6 +588,9 @@ func (c *common) frameSkip(skip int) runtime.Frame { var firstFrame, prevFrame, frame runtime.Frame for more := true; more; prevFrame = frame { frame, more = frames.Next() + if frame.Function == "runtime.gopanic" { + continue + } if frame.Function == c.cleanupName { frames = runtime.CallersFrames(c.cleanupPc) continue @@ -572,7 +680,7 @@ func (c *common) decorate(s string, skip int) string { // flushToParent writes c.output to the parent after first writing the header // with the given format and arguments. -func (c *common) flushToParent(testName, format string, args ...interface{}) { +func (c *common) flushToParent(testName, format string, args ...any) { p := c.parent p.mu.Lock() defer p.mu.Unlock() @@ -632,24 +740,24 @@ func fmtDuration(d time.Duration) string { return fmt.Sprintf("%.2fs", d.Seconds()) } -// TB is the interface common to T and B. +// TB is the interface common to T, B, and F. type TB interface { Cleanup(func()) - Error(args ...interface{}) - Errorf(format string, args ...interface{}) + Error(args ...any) + Errorf(format string, args ...any) Fail() FailNow() Failed() bool - Fatal(args ...interface{}) - Fatalf(format string, args ...interface{}) + Fatal(args ...any) + Fatalf(format string, args ...any) Helper() - Log(args ...interface{}) - Logf(format string, args ...interface{}) + Log(args ...any) + Logf(format string, args ...any) Name() string Setenv(key, value string) - Skip(args ...interface{}) + Skip(args ...any) SkipNow() - Skipf(format string, args ...interface{}) + Skipf(format string, args ...any) Skipped() bool TempDir() string @@ -729,6 +837,7 @@ func (c *common) Failed() bool { // created during the test. Calling FailNow does not stop // those other goroutines. func (c *common) FailNow() { + c.checkFuzzFn("FailNow") c.Fail() // Calling runtime.Goexit will exit the goroutine, which @@ -803,14 +912,20 @@ func (c *common) logDepth(s string, depth int) { // and records the text in the error log. For tests, the text will be printed only if // the test fails or the -test.v flag is set. For benchmarks, the text is always // printed to avoid having performance depend on the value of the -test.v flag. -func (c *common) Log(args ...interface{}) { c.log(fmt.Sprintln(args...)) } +func (c *common) Log(args ...any) { + c.checkFuzzFn("Log") + c.log(fmt.Sprintln(args...)) +} // Logf formats its arguments according to the format, analogous to Printf, and // records the text in the error log. A final newline is added if not provided. For // tests, the text will be printed only if the test fails or the -test.v flag is // set. For benchmarks, the text is always printed to avoid having performance // depend on the value of the -test.v flag. -func (c *common) Logf(format string, args ...interface{}) { c.log(fmt.Sprintf(format, args...)) } +func (c *common) Logf(format string, args ...any) { + c.checkFuzzFn("Logf") + c.log(fmt.Sprintf(format, args...)) +} // This is needed for gccgo to get the tests to pass, because // runtime.Callers doesn't correctly handle skips that land in the @@ -819,7 +934,8 @@ func (c *common) Logf(format string, args ...interface{}) { c.log(fmt.Sprintf(fo //go:noinline // Error is equivalent to Log followed by Fail. -func (c *common) Error(args ...interface{}) { +func (c *common) Error(args ...any) { + c.checkFuzzFn("Error") c.log(fmt.Sprintln(args...)) c.Fail() } @@ -831,7 +947,8 @@ func (c *common) Error(args ...interface{}) { //go:noinline // Errorf is equivalent to Logf followed by Fail. -func (c *common) Errorf(format string, args ...interface{}) { +func (c *common) Errorf(format string, args ...any) { + c.checkFuzzFn("Errorf") c.log(fmt.Sprintf(format, args...)) c.Fail() } @@ -843,7 +960,8 @@ func (c *common) Errorf(format string, args ...interface{}) { //go:noinline // Fatal is equivalent to Log followed by FailNow. -func (c *common) Fatal(args ...interface{}) { +func (c *common) Fatal(args ...any) { + c.checkFuzzFn("Fatal") c.log(fmt.Sprintln(args...)) c.FailNow() } @@ -855,7 +973,8 @@ func (c *common) Fatal(args ...interface{}) { //go:noinline // Fatalf is equivalent to Logf followed by FailNow. -func (c *common) Fatalf(format string, args ...interface{}) { +func (c *common) Fatalf(format string, args ...any) { + c.checkFuzzFn("Fatalf") c.log(fmt.Sprintf(format, args...)) c.FailNow() } @@ -867,13 +986,15 @@ func (c *common) Fatalf(format string, args ...interface{}) { //go:noinline // Skip is equivalent to Log followed by SkipNow. -func (c *common) Skip(args ...interface{}) { +func (c *common) Skip(args ...any) { + c.checkFuzzFn("Skip") c.log(fmt.Sprintln(args...)) c.SkipNow() } // Skipf is equivalent to Logf followed by SkipNow. -func (c *common) Skipf(format string, args ...interface{}) { +func (c *common) Skipf(format string, args ...any) { + c.checkFuzzFn("Skipf") c.log(fmt.Sprintf(format, args...)) c.SkipNow() } @@ -887,6 +1008,7 @@ func (c *common) Skipf(format string, args ...interface{}) { // other goroutines created during the test. Calling SkipNow does not stop // those other goroutines. func (c *common) SkipNow() { + c.checkFuzzFn("SkipNow") c.mu.Lock() c.skipped = true c.finished = true @@ -926,6 +1048,7 @@ func (c *common) Helper() { // subtests complete. Cleanup functions will be called in last added, // first called order. func (c *common) Cleanup(f func()) { + c.checkFuzzFn("Cleanup") var pc [maxStackLen]uintptr // Skip two extra frames to account for this function and runtime.Callers itself. n := runtime.Callers(2, pc[:]) @@ -959,6 +1082,7 @@ func (c *common) Cleanup(f func()) { // Each subsequent call to t.TempDir returns a unique directory; // if the directory creation fails, TempDir terminates the test by calling Fatal. func (c *common) TempDir() string { + c.checkFuzzFn("TempDir") // Use a single parent directory for all the temporary directories // created by a test, each numbered sequentially. c.tempDirMu.Lock() @@ -999,7 +1123,7 @@ func (c *common) TempDir() string { c.tempDir, c.tempDirErr = os.MkdirTemp("", pattern) if c.tempDirErr == nil { c.Cleanup(func() { - if err := os.RemoveAll(c.tempDir); err != nil { + if err := removeAll(c.tempDir); err != nil { c.Errorf("TempDir RemoveAll cleanup: %v", err) } }) @@ -1018,12 +1142,43 @@ func (c *common) TempDir() string { return dir } +// removeAll is like os.RemoveAll, but retries Windows "Access is denied." +// errors up to an arbitrary timeout. +// +// Those errors have been known to occur spuriously on at least the +// windows-amd64-2012 builder (https://go.dev/issue/50051), and can only occur +// legitimately if the test leaves behind a temp file that either is still open +// or the test otherwise lacks permission to delete. In the case of legitimate +// failures, a failing test may take a bit longer to fail, but once the test is +// fixed the extra latency will go away. +func removeAll(path string) error { + const arbitraryTimeout = 2 * time.Second + var ( + start time.Time + nextSleep = 1 * time.Millisecond + ) + for { + err := os.RemoveAll(path) + if !isWindowsAccessDenied(err) { + return err + } + if start.IsZero() { + start = time.Now() + } else if d := time.Since(start) + nextSleep; d >= arbitraryTimeout { + return err + } + time.Sleep(nextSleep) + nextSleep += time.Duration(rand.Int63n(int64(nextSleep))) + } +} + // Setenv calls os.Setenv(key, value) and uses Cleanup to // restore the environment variable to its original value // after the test. // // This cannot be used in parallel tests. func (c *common) Setenv(key, value string) { + c.checkFuzzFn("Setenv") prevValue, ok := os.LookupEnv(key) if err := os.Setenv(key, value); err != nil { @@ -1052,7 +1207,7 @@ const ( // runCleanup is called at the end of the test. // If catchPanic is true, this will catch panics, and return the recovered // value if any. -func (c *common) runCleanup(ph panicHandling) (panicVal interface{}) { +func (c *common) runCleanup(ph panicHandling) (panicVal any) { if ph == recoverAndReturnPanic { defer func() { panicVal = recover() @@ -1116,6 +1271,12 @@ func (t *T) Parallel() { panic("testing: t.Parallel called after t.Setenv; cannot set environment variables in parallel tests") } t.isParallel = true + if t.parent.barrier == nil { + // T.Parallel has no effect when fuzzing. + // Multiple processes may run in parallel, but only one input can run at a + // time per process so we can attribute crashes to specific inputs. + return + } // We don't want to include the time we spend waiting for serial tests // in the test duration. Record the elapsed time thus far and reset the @@ -1188,7 +1349,14 @@ func tRunner(t *T, fn func(t *T)) { t.Errorf("race detected during execution of test") } - // If the test panicked, print any test output before dying. + // Check if the test panicked or Goexited inappropriately. + // + // If this happens in a normal test, print output but continue panicking. + // tRunner is called in its own goroutine, so this terminates the process. + // + // If this happens while fuzzing, recover from the panic and treat it like a + // normal failure. It's important that the process keeps running in order to + // find short inputs that cause panics. err := recover() signal := true @@ -1209,6 +1377,19 @@ func tRunner(t *T, fn func(t *T)) { } } } + + if err != nil && t.context.isFuzzing { + prefix := "panic: " + if err == errNilPanicOrGoexit { + prefix = "" + } + t.Errorf("%s%s\n%s\n", prefix, err, string(debug.Stack())) + t.mu.Lock() + t.finished = true + t.mu.Unlock() + err = nil + } + // Use a deferred call to ensure that we report that the test is // complete even if a cleanup function calls t.FailNow. See issue 41355. didPanic := false @@ -1225,7 +1406,7 @@ func tRunner(t *T, fn func(t *T)) { t.signal <- signal }() - doPanic := func(err interface{}) { + doPanic := func(err any) { t.Fail() if r := t.runCleanup(recoverAndReturnPanic); r != nil { t.Logf("cleanup panicked with %v", r) @@ -1278,7 +1459,7 @@ func tRunner(t *T, fn func(t *T)) { t.report() // Report after all subtests have finished. // Do not lock t.done to allow race detector to detect race in case - // the user does not appropriately synchronizes a goroutine. + // the user does not appropriately synchronize a goroutine. t.done = true if t.parent != nil && atomic.LoadInt32(&t.hasSub) == 0 { t.setRan() @@ -1363,6 +1544,12 @@ type testContext struct { match *matcher deadline time.Time + // isFuzzing is true in the context used when generating random inputs + // for fuzz targets. isFuzzing is false when running normal tests and + // when running fuzz tests as unit tests (without -fuzz or when -fuzz + // does not match). + isFuzzing bool + mu sync.Mutex // Channel used to signal tests that are ready to be run in parallel. @@ -1426,6 +1613,16 @@ func (f matchStringOnly) ImportPath() string { return " func (f matchStringOnly) StartTestLog(io.Writer) {} func (f matchStringOnly) StopTestLog() error { return errMain } func (f matchStringOnly) SetPanicOnExit0(bool) {} +func (f matchStringOnly) CoordinateFuzzing(time.Duration, int64, time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error { + return errMain +} +func (f matchStringOnly) RunFuzzWorker(func(corpusEntry) error) error { return errMain } +func (f matchStringOnly) ReadCorpus(string, []reflect.Type) ([]corpusEntry, error) { + return nil, errMain +} +func (f matchStringOnly) CheckCorpus([]any, []reflect.Type) error { return nil } +func (f matchStringOnly) ResetCoverage() {} +func (f matchStringOnly) SnapshotCoverage() {} // Main is an internal function, part of the implementation of the "go test" command. // It was exported because it is cross-package and predates "internal" packages. @@ -1434,15 +1631,16 @@ func (f matchStringOnly) SetPanicOnExit0(bool) {} // new functionality is added to the testing package. // Systems simulating "go test" should be updated to use MainStart. func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) { - os.Exit(MainStart(matchStringOnly(matchString), tests, benchmarks, examples).Run()) + os.Exit(MainStart(matchStringOnly(matchString), tests, benchmarks, nil, examples).Run()) } // M is a type passed to a TestMain function to run the actual tests. type M struct { - deps testDeps - tests []InternalTest - benchmarks []InternalBenchmark - examples []InternalExample + deps testDeps + tests []InternalTest + benchmarks []InternalBenchmark + fuzzTargets []InternalFuzzTarget + examples []InternalExample timer *time.Timer afterOnce sync.Once @@ -1467,18 +1665,25 @@ type testDeps interface { StartTestLog(io.Writer) StopTestLog() error WriteProfileTo(string, io.Writer, int) error + CoordinateFuzzing(time.Duration, int64, time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error + RunFuzzWorker(func(corpusEntry) error) error + ReadCorpus(string, []reflect.Type) ([]corpusEntry, error) + CheckCorpus([]any, []reflect.Type) error + ResetCoverage() + SnapshotCoverage() } // MainStart is meant for use by tests generated by 'go test'. // It is not meant to be called directly and is not subject to the Go 1 compatibility document. // It may change signature from release to release. -func MainStart(deps testDeps, tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) *M { +func MainStart(deps testDeps, tests []InternalTest, benchmarks []InternalBenchmark, fuzzTargets []InternalFuzzTarget, examples []InternalExample) *M { Init() return &M{ - deps: deps, - tests: tests, - benchmarks: benchmarks, - examples: examples, + deps: deps, + tests: tests, + benchmarks: benchmarks, + fuzzTargets: fuzzTargets, + examples: examples, } } @@ -1505,9 +1710,15 @@ func (m *M) Run() (code int) { m.exitCode = 2 return } + if *matchFuzz != "" && *fuzzCacheDir == "" { + fmt.Fprintln(os.Stderr, "testing: -test.fuzzcachedir must be set if -test.fuzz is set") + flag.Usage() + m.exitCode = 2 + return + } if len(*matchList) != 0 { - listTests(m.deps.MatchString, m.tests, m.benchmarks, m.examples) + listTests(m.deps.MatchString, m.tests, m.benchmarks, m.fuzzTargets, m.examples) m.exitCode = 0 return } @@ -1535,22 +1746,42 @@ func (m *M) Run() (code int) { m.before() defer m.after() - deadline := m.startAlarm() - haveExamples = len(m.examples) > 0 - testRan, testOk := runTests(m.deps.MatchString, m.tests, deadline) - exampleRan, exampleOk := runExamples(m.deps.MatchString, m.examples) - m.stopAlarm() - if !testRan && !exampleRan && *matchBenchmarks == "" { - fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") + + // Run tests, examples, and benchmarks unless this is a fuzz worker process. + // Workers start after this is done by their parent process, and they should + // not repeat this work. + if !*isFuzzWorker { + deadline := m.startAlarm() + haveExamples = len(m.examples) > 0 + testRan, testOk := runTests(m.deps.MatchString, m.tests, deadline) + fuzzTargetsRan, fuzzTargetsOk := runFuzzTests(m.deps, m.fuzzTargets, deadline) + exampleRan, exampleOk := runExamples(m.deps.MatchString, m.examples) + m.stopAlarm() + if !testRan && !exampleRan && !fuzzTargetsRan && *matchBenchmarks == "" && *matchFuzz == "" { + fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") + } + if !testOk || !exampleOk || !fuzzTargetsOk || !runBenchmarks(m.deps.ImportPath(), m.deps.MatchString, m.benchmarks) || race.Errors() > 0 { + fmt.Println("FAIL") + m.exitCode = 1 + return + } } - if !testOk || !exampleOk || !runBenchmarks(m.deps.ImportPath(), m.deps.MatchString, m.benchmarks) || race.Errors() > 0 { + + fuzzingOk := runFuzzing(m.deps, m.fuzzTargets) + if !fuzzingOk { fmt.Println("FAIL") - m.exitCode = 1 + if *isFuzzWorker { + m.exitCode = fuzzWorkerExitCode + } else { + m.exitCode = 1 + } return } - fmt.Println("PASS") m.exitCode = 0 + if !*isFuzzWorker { + fmt.Println("PASS") + } return } @@ -1571,7 +1802,7 @@ func (t *T) report() { } } -func listTests(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) { +func listTests(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, fuzzTargets []InternalFuzzTarget, examples []InternalExample) { if _, err := matchString(*matchList, "non-empty"); err != nil { fmt.Fprintf(os.Stderr, "testing: invalid regexp in -test.list (%q): %s\n", *matchList, err) os.Exit(1) @@ -1587,6 +1818,11 @@ func listTests(matchString func(pat, str string) (bool, error), tests []Internal fmt.Println(bench.Name) } } + for _, fuzzTarget := range fuzzTargets { + if ok, _ := matchString(*matchList, fuzzTarget.Name); ok { + fmt.Println(fuzzTarget.Name) + } + } for _, example := range examples { if ok, _ := matchString(*matchList, example.Name); ok { fmt.Println(example.Name) @@ -1616,6 +1852,12 @@ func runTests(matchString func(pat, str string) (bool, error), tests []InternalT if shouldFailFast() { break } + if i > 0 && !ran { + // There were no tests to run on the first + // iteration. This won't change, so no reason + // to keep trying. + break + } ctx := newTestContext(*parallel, newMatcher(matchString, *match, "-test.run")) ctx.deadline = deadline t := &T{ diff --git a/libgo/go/testing/testing_other.go b/libgo/go/testing/testing_other.go new file mode 100644 index 0000000..29496d8 --- /dev/null +++ b/libgo/go/testing/testing_other.go @@ -0,0 +1,13 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !windows + +package testing + +// isWindowsAccessDenied reports whether err is ERROR_ACCESS_DENIED, +// which is defined only on Windows. +func isWindowsAccessDenied(err error) bool { + return false +} diff --git a/libgo/go/testing/testing_windows.go b/libgo/go/testing/testing_windows.go new file mode 100644 index 0000000..bc76cb8 --- /dev/null +++ b/libgo/go/testing/testing_windows.go @@ -0,0 +1,18 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build windows + +package testing + +import ( + "errors" + "syscall" +) + +// isWindowsAccessDenied reports whether err is ERROR_ACCESS_DENIED, +// which is defined only on Windows. +func isWindowsAccessDenied(err error) bool { + return errors.Is(err, syscall.ERROR_ACCESS_DENIED) +} |