diff options
| author | Ian Lance Taylor <iant@golang.org> | 2020-02-05 14:33:27 -0800 |
|---|---|---|
| committer | Ian Lance Taylor <iant@golang.org> | 2020-02-15 09:14:10 -0800 |
| commit | 0b3c2eed35d608d6541ecf004a9576b4eae0b4ef (patch) | |
| tree | c92c05d53eb054d8085d069800f4e9b586fef5a3 /libgo/go | |
| parent | 17edb3310d8ce9d5f6c9e53f6c1f7d611c2a5a41 (diff) | |
| download | gcc-0b3c2eed35d608d6541ecf004a9576b4eae0b4ef.zip gcc-0b3c2eed35d608d6541ecf004a9576b4eae0b4ef.tar.gz gcc-0b3c2eed35d608d6541ecf004a9576b4eae0b4ef.tar.bz2 | |
libgo: update to Go1.14rc1 release
Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/218017
Diffstat (limited to 'libgo/go')
187 files changed, 1514 insertions, 1816 deletions
diff --git a/libgo/go/cmd/go/alldocs.go b/libgo/go/cmd/go/alldocs.go index 54e7a8b..971a756 100644 --- a/libgo/go/cmd/go/alldocs.go +++ b/libgo/go/cmd/go/alldocs.go @@ -907,7 +907,7 @@ // Main bool // is this the main module? // Indirect bool // is this module only an indirect dependency of main module? // Dir string // directory holding files for this module, if any -// GoMod string // path to go.mod file for this module, if any +// GoMod string // path to go.mod file used when loading this module, if any // GoVersion string // go version used in module // Error *ModuleError // error loading module // } @@ -916,6 +916,9 @@ // Err string // the error itself // } // +// The file GoMod refers to may be outside the module directory if the +// module is in the module cache or if the -modfile flag is used. +// // The default output is to print the module path and then // information about the version and replacement if any. // For example, 'go list -m all' might print: @@ -1020,7 +1023,9 @@ // execution. The "go mod download" command is useful mainly for pre-filling // the local cache or to compute the answers for a Go module proxy. // -// By default, download reports errors to standard error but is otherwise silent. +// By default, download writes nothing to standard output. It may print progress +// messages and errors to standard error. +// // The -json flag causes download to print a sequence of JSON objects // to standard output, describing each downloaded module (or failure), // corresponding to this Go struct: @@ -2346,14 +2351,15 @@ // // Module support // -// Go 1.13 includes support for Go modules. Module-aware mode is active by default -// whenever a go.mod file is found in, or in a parent of, the current directory. +// The go command includes support for Go modules. Module-aware mode is active +// by default whenever a go.mod file is found in the current directory or in +// any parent directory. // // The quickest way to take advantage of module support is to check out your // repository, create a go.mod file (described in the next section) there, and run // go commands from within that file tree. // -// For more fine-grained control, Go 1.13 continues to respect +// For more fine-grained control, the go command continues to respect // a temporary environment variable, GO111MODULE, which can be set to one // of three string values: off, on, or auto (the default). // If GO111MODULE=on, then the go command requires the use of modules, diff --git a/libgo/go/cmd/go/go_test.go b/libgo/go/cmd/go/go_test.go index ebd0c7a..d535ea0 100644 --- a/libgo/go/cmd/go/go_test.go +++ b/libgo/go/cmd/go/go_test.go @@ -638,7 +638,7 @@ func (tg *testgoData) grepStderrNot(match, msg string) { } // grepBothNot looks for a regular expression in the test run's -// standard output or stand error and fails, logging msg, if it is +// standard output or standard error and fails, logging msg, if it is // found. func (tg *testgoData) grepBothNot(match, msg string) { tg.t.Helper() @@ -913,6 +913,7 @@ func TestNewReleaseRebuildsStalePackagesInGOPATH(t *testing.T) { tg := testgo(t) defer tg.cleanup() + tg.parallel() // Copy the runtime packages into a temporary GOROOT // so that we can change files. @@ -1026,28 +1027,6 @@ func TestInternalPackagesOutsideGOROOTAreRespected(t *testing.T) { tg.grepBoth(`testinternal2(\/|\\)p\.go\:3\:8\: use of internal package .*internal/w not allowed`, "wrote error message for testdata/testinternal2") } -func TestRunInternal(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - dir := filepath.Join(tg.pwd(), "testdata") - tg.setenv("GOPATH", dir) - tg.run("run", filepath.Join(dir, "src/run/good.go")) - tg.runFail("run", filepath.Join(dir, "src/run/bad.go")) - tg.grepStderr(`testdata(\/|\\)src(\/|\\)run(\/|\\)bad\.go\:3\:8\: use of internal package run/subdir/internal/private not allowed`, "unexpected error for run/bad.go") -} - -func TestRunPkg(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - dir := filepath.Join(tg.pwd(), "testdata") - tg.setenv("GOPATH", dir) - tg.run("run", "hello") - tg.grepStderr("hello, world", "did not find hello, world") - tg.cd(filepath.Join(dir, "src/hello")) - tg.run("run", ".") - tg.grepStderr("hello, world", "did not find hello, world") -} - func TestInternalPackageErrorsAreHandled(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -1062,56 +1041,6 @@ func TestInternalCache(t *testing.T) { tg.grepStderr("internal", "did not fail to build p") } -func TestImportCommandMatch(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/importcom")) - tg.run("build", "./testdata/importcom/works.go") -} - -func TestImportCommentMismatch(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/importcom")) - tg.runFail("build", "./testdata/importcom/wrongplace.go") - tg.grepStderr(`wrongplace expects import "my/x"`, "go build did not mention incorrect import") -} - -func TestImportCommentSyntaxError(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/importcom")) - tg.runFail("build", "./testdata/importcom/bad.go") - tg.grepStderr("cannot parse import comment", "go build did not mention syntax error") -} - -func TestImportCommentConflict(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/importcom")) - tg.runFail("build", "./testdata/importcom/conflict.go") - tg.grepStderr("found import comments", "go build did not mention comment conflict") -} - -func TestImportCycle(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - tg.parallel() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/importcycle")) - tg.runFail("build", "selfimport") - - count := tg.grepCountBoth("import cycle not allowed") - if count == 0 { - t.Fatal("go build did not mention cyclical import") - } - if count > 1 { - t.Fatal("go build mentioned import cycle more than once") - } - - // Don't hang forever. - tg.run("list", "-e", "-json", "selfimport") -} - // cmd/go: custom import path checking should not apply to Go packages without import comment. func TestIssue10952(t *testing.T) { testenv.MustHaveExternalNetwork(t) @@ -1217,24 +1146,6 @@ func TestAccidentalGitCheckout(t *testing.T) { } } -func TestErrorMessageForSyntaxErrorInTestGoFileSaysFAIL(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.runFail("test", "syntaxerror") - tg.grepStderr("x_test.go:", "did not diagnose error") - tg.grepStdout("FAIL", "go test did not say FAIL") -} - -func TestWildcardsDoNotLookInUselessDirectories(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.runFail("list", "...") - tg.grepBoth("badpkg", "go list ... failure does not mention badpkg") - tg.run("list", "m...") -} - func TestRelativeImportsGoTest(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -1673,6 +1584,7 @@ func TestDefaultGOPATHGet(t *testing.T) { tg := testgo(t) defer tg.cleanup() + tg.parallel() tg.setenv("GOPATH", "") tg.tempDir("home") tg.setenv(homeEnvName(), tg.path("home")) @@ -1697,6 +1609,7 @@ func TestDefaultGOPATHGet(t *testing.T) { func TestDefaultGOPATHPrintedSearchList(t *testing.T) { tg := testgo(t) defer tg.cleanup() + tg.parallel() tg.setenv("GOPATH", "") tg.tempDir("home") tg.setenv(homeEnvName(), tg.path("home")) @@ -1819,16 +1732,6 @@ func TestGoTestMutexprofileDashOControlsBinaryLocation(t *testing.T) { tg.wantExecutable("myerrors.test"+exeSuffix, "go test -mutexprofile -o myerrors.test did not create myerrors.test") } -func TestGoBuildNonMain(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - // TODO: tg.parallel() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.runFail("build", "-buildmode=exe", "-o", "not_main"+exeSuffix, "not_main") - tg.grepStderr("-buildmode=exe requires exactly one main package", "go build with -o and -buildmode=exe should on a non-main package should throw an error") - tg.mustNotExist("not_main" + exeSuffix) -} - func TestGoTestDashCDashOControlsBinaryLocation(t *testing.T) { skipIfGccgo(t, "gccgo has no standard packages") tooSlow(t) @@ -2192,33 +2095,6 @@ func TestCoverageNoStatements(t *testing.T) { tg.grepStdout("[no statements]", "expected [no statements] for pkg4") } -func TestCoverageImportMainLoop(t *testing.T) { - skipIfGccgo(t, "gccgo has no cover tool") - tg := testgo(t) - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.runFail("test", "importmain/test") - tg.grepStderr("not an importable package", "did not detect import main") - tg.runFail("test", "-cover", "importmain/test") - tg.grepStderr("not an importable package", "did not detect import main") -} - -func TestCoveragePattern(t *testing.T) { - skipIfGccgo(t, "gccgo has no cover tool") - tooSlow(t) - tg := testgo(t) - defer tg.cleanup() - tg.parallel() - tg.makeTempdir() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - - // If coverpkg=sleepy... expands by package loading - // (as opposed to pattern matching on deps) - // then it will try to load sleepybad, which does not compile, - // and the test command will fail. - tg.run("test", "-coverprofile="+tg.path("cover.out"), "-coverpkg=sleepy...", "-run=^$", "sleepy1") -} - func TestCoverageErrorLine(t *testing.T) { skipIfGccgo(t, "gccgo has no cover tool") tooSlow(t) @@ -2291,20 +2167,6 @@ func TestCoverageDashC(t *testing.T) { tg.wantExecutable(tg.path("coverdep"), "go -test -c -coverprofile did not create executable") } -func TestPluginNonMain(t *testing.T) { - wd, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - - pkg := filepath.Join(wd, "testdata", "testdep", "p2") - - tg := testgo(t) - defer tg.cleanup() - - tg.runFail("build", "-buildmode=plugin", pkg) -} - func TestTestEmpty(t *testing.T) { if !canRace { t.Skip("no race detector") @@ -2389,39 +2251,6 @@ func main() { tg.grepStderrNot(`os.Stat .* no such file or directory`, "unexpected stat of archive file") } -func TestCoverageWithCgo(t *testing.T) { - skipIfGccgo(t, "gccgo has no cover tool") - tooSlow(t) - if !canCgo { - t.Skip("skipping because cgo not enabled") - } - - for _, dir := range []string{"cgocover", "cgocover2", "cgocover3", "cgocover4"} { - t.Run(dir, func(t *testing.T) { - tg := testgo(t) - tg.parallel() - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.run("test", "-short", "-cover", dir) - data := tg.getStdout() + tg.getStderr() - checkCoverage(tg, data) - }) - } -} - -func TestCgoAsmError(t *testing.T) { - if !canCgo { - t.Skip("skipping because cgo not enabled") - } - - tg := testgo(t) - tg.parallel() - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.runFail("build", "cgoasm") - tg.grepBoth("package using cgo has Go assembly file", "did not detect Go assembly file") -} - func TestCgoDependsOnSyscall(t *testing.T) { if testing.Short() { t.Skip("skipping test that removes $GOROOT/pkg/*_race in short mode") @@ -2435,6 +2264,8 @@ func TestCgoDependsOnSyscall(t *testing.T) { tg := testgo(t) defer tg.cleanup() + tg.parallel() + files, err := filepath.Glob(filepath.Join(runtime.GOROOT(), "pkg", "*_race")) tg.must(err) for _, file := range files { @@ -2648,14 +2479,6 @@ func TestListTemplateContextFunction(t *testing.T) { } } -// cmd/go: "go test" should fail if package does not build -func TestIssue7108(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.runFail("test", "notest") -} - func TestGoBuildTestOnly(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -2677,17 +2500,6 @@ func TestGoBuildTestOnly(t *testing.T) { tg.run("install", "./testonly...") } -func TestGoTestDetectsTestOnlyImportCycles(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.runFail("test", "-c", "testcycle/p3") - tg.grepStderr("import cycle not allowed in test", "go test testcycle/p3 produced unexpected error") - - tg.runFail("test", "-c", "testcycle/q1") - tg.grepStderr("import cycle not allowed in test", "go test testcycle/q1 produced unexpected error") -} - func TestGoTestFooTestWorks(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -2715,29 +2527,6 @@ func TestGoTestMainAsNormalTest(t *testing.T) { tg.grepBoth(okPattern, "go test did not say ok") } -func TestGoTestMainTwice(t *testing.T) { - if testing.Short() { - t.Skip("Skipping in short mode") - } - tg := testgo(t) - defer tg.cleanup() - tg.makeTempdir() - tg.setenv("GOCACHE", tg.tempdir) - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.run("test", "-v", "multimain") - if strings.Count(tg.getStdout(), "notwithstanding") != 2 { - t.Fatal("tests did not run twice") - } -} - -func TestGoTestFlagsAfterPackage(t *testing.T) { - tooSlow(t) - tg := testgo(t) - defer tg.cleanup() - tg.run("test", "testdata/flag_test.go", "-v", "-args", "-v=7") // Two distinct -v flags. - tg.run("test", "-v", "testdata/flag_test.go", "-args", "-v=7") // Two distinct -v flags. -} - func TestGoTestXtestonlyWorks(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -2829,20 +2618,6 @@ func TestGoGenerateXTestPkgName(t *testing.T) { } } -func TestGoGenerateBadImports(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("skipping because windows has no echo command") - } - - // This package has an invalid import causing an import cycle, - // but go generate is supposed to still run. - tg := testgo(t) - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.run("generate", "gencycle") - tg.grepStdout("hello world", "go generate gencycle did not run generator") -} - func TestGoGetCustomDomainWildcard(t *testing.T) { testenv.MustHaveExternalNetwork(t) testenv.MustHaveExecPath(t, "git") @@ -3268,43 +3043,6 @@ func TestGoTestRaceInstallCgo(t *testing.T) { } } -func TestGoTestRaceFailures(t *testing.T) { - tooSlow(t) - - if !canRace { - t.Skip("skipping because race detector not supported") - } - - tg := testgo(t) - tg.parallel() - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - - tg.run("test", "testrace") - - tg.runFail("test", "-race", "testrace") - tg.grepStdout("FAIL: TestRace", "TestRace did not fail") - tg.grepBothNot("PASS", "something passed") - - tg.runFail("test", "-race", "testrace", "-run", "XXX", "-bench", ".") - tg.grepStdout("FAIL: BenchmarkRace", "BenchmarkRace did not fail") - tg.grepBothNot("PASS", "something passed") -} - -func TestGoTestImportErrorStack(t *testing.T) { - const out = `package testdep/p1 (test) - imports testdep/p2 - imports testdep/p3: build constraints exclude all Go files ` - - tg := testgo(t) - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.runFail("test", "testdep/p1") - if !strings.Contains(tg.stderr.String(), out) { - t.Fatalf("did not give full import stack:\n\n%s", tg.stderr.String()) - } -} - func TestGoGetUpdate(t *testing.T) { // golang.org/issue/9224. // The recursive updating was trying to walk to @@ -3627,27 +3365,6 @@ func TestGoGetUpdateAllDoesNotTryToLoadDuplicates(t *testing.T) { tg.grepStderrNot("duplicate loads of", "did not remove old packages from cache") } -// Issue 17119 more duplicate load errors -func TestIssue17119(t *testing.T) { - testenv.MustHaveExternalNetwork(t) - - tg := testgo(t) - defer tg.cleanup() - tg.parallel() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.runFail("build", "dupload") - tg.grepBothNot("duplicate load|internal error", "internal error") -} - -func TestFatalInBenchmarkCauseNonZeroExitStatus(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - // TODO: tg.parallel() - tg.runFail("test", "-run", "^$", "-bench", ".", "./testdata/src/benchfatal") - tg.grepBothNot("^ok", "test passed unexpectedly") - tg.grepBoth("FAIL.*benchfatal", "test did not run everything") -} - func TestBinaryOnlyPackages(t *testing.T) { tooSlow(t) @@ -3813,16 +3530,6 @@ func TestMatchesNoTests(t *testing.T) { tg.grepBoth(noMatchesPattern, "go test did not say [no tests to run]") } -func TestMatchesNoTestsDoesNotOverrideBuildFailure(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - tg.parallel() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.runFail("test", "-run", "ThisWillNotMatch", "syntaxerror") - tg.grepBothNot(noMatchesPattern, "go test did say [no tests to run]") - tg.grepBoth("FAIL", "go test did not say FAIL") -} - func TestMatchesNoBenchmarksIsOK(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -3850,18 +3557,6 @@ func TestMatchesOnlyBenchmarkIsOK(t *testing.T) { tg.grepBoth(okPattern, "go test did not say ok") } -func TestBenchmarkLabels(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - // TODO: tg.parallel() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.run("test", "-run", "^$", "-bench", ".", "bench") - tg.grepStdout(`(?m)^goos: `+runtime.GOOS, "go test did not print goos") - tg.grepStdout(`(?m)^goarch: `+runtime.GOARCH, "go test did not print goarch") - tg.grepStdout(`(?m)^pkg: bench`, "go test did not say pkg: bench") - tg.grepBothNot(`(?s)pkg:.*pkg:`, "go test said pkg multiple times") -} - func TestBenchmarkLabelsOutsideGOPATH(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -4261,25 +3956,6 @@ func TestCgoFlagContainsSpace(t *testing.T) { tg.grepStderrNot(`"-L[^"]+c flags".*"-L[^"]+c flags"`, "found too many quoted ld flags") } -// Issue #20435. -func TestGoTestRaceCoverModeFailures(t *testing.T) { - tooSlow(t) - if !canRace { - t.Skip("skipping because race detector not supported") - } - - tg := testgo(t) - tg.parallel() - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - - tg.run("test", "testrace") - - tg.runFail("test", "-race", "-covermode=set", "testrace") - tg.grepStderr(`-covermode must be "atomic", not "set", when -race is enabled`, "-race -covermode=set was allowed") - tg.grepBothNot("PASS", "something passed") -} - // Issue 9737: verify that GOARM and GO386 affect the computed build ID. func TestBuildIDContainsArchModeEnv(t *testing.T) { if testing.Short() { @@ -4319,60 +3995,6 @@ func main() {}`) })) } -func TestTestRegexps(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.run("test", "-cpu=1", "-run=X/Y", "-bench=X/Y", "-count=2", "-v", "testregexp") - var lines []string - for _, line := range strings.SplitAfter(tg.getStdout(), "\n") { - if strings.Contains(line, "=== RUN") || strings.Contains(line, "--- BENCH") || strings.Contains(line, "LOG") { - lines = append(lines, line) - } - } - - // Important parts: - // TestX is run, twice - // TestX/Y is run, twice - // TestXX is run, twice - // TestZ is not run - // BenchmarkX is run but only with N=1, once - // BenchmarkXX is run but only with N=1, once - // BenchmarkX/Y is run in full, twice - want := `=== RUN TestX - TestX: x_test.go:6: LOG: X running -=== RUN TestX/Y - TestX/Y: x_test.go:8: LOG: Y running -=== RUN TestXX - TestXX: z_test.go:10: LOG: XX running -=== RUN TestX - TestX: x_test.go:6: LOG: X running -=== RUN TestX/Y - TestX/Y: x_test.go:8: LOG: Y running -=== RUN TestXX - TestXX: z_test.go:10: LOG: XX running - BenchmarkX: x_test.go:13: LOG: X running N=1 - BenchmarkX/Y: x_test.go:15: LOG: Y running N=1 - BenchmarkX/Y: x_test.go:15: LOG: Y running N=100 - BenchmarkX/Y: x_test.go:15: LOG: Y running N=10000 - BenchmarkX/Y: x_test.go:15: LOG: Y running N=1000000 - BenchmarkX/Y: x_test.go:15: LOG: Y running N=100000000 - BenchmarkX/Y: x_test.go:15: LOG: Y running N=1000000000 - BenchmarkX/Y: x_test.go:15: LOG: Y running N=1 - BenchmarkX/Y: x_test.go:15: LOG: Y running N=100 - BenchmarkX/Y: x_test.go:15: LOG: Y running N=10000 - BenchmarkX/Y: x_test.go:15: LOG: Y running N=1000000 - BenchmarkX/Y: x_test.go:15: LOG: Y running N=100000000 - BenchmarkX/Y: x_test.go:15: LOG: Y running N=1000000000 - BenchmarkXX: z_test.go:18: LOG: XX running N=1 -` - - have := strings.Join(lines, "") - if have != want { - t.Errorf("reduced output:<<<\n%s>>> want:<<<\n%s>>>", have, want) - } -} - func TestListTests(t *testing.T) { tooSlow(t) var tg *testgoData @@ -4408,6 +4030,7 @@ func TestBuildmodePIE(t *testing.T) { tg := testgo(t) defer tg.cleanup() + tg.parallel() tg.tempFile("main.go", `package main; func main() { print("hello") }`) src := tg.path("main.go") @@ -4571,6 +4194,7 @@ func TestUpxCompression(t *testing.T) { tg := testgo(t) defer tg.cleanup() + tg.parallel() tg.tempFile("main.go", `package main; import "fmt"; func main() { fmt.Print("hello upx") }`) src := tg.path("main.go") @@ -4964,14 +4588,6 @@ func TestInstallDeps(t *testing.T) { tg.mustExist(p1) } -func TestGoTestMinusN(t *testing.T) { - // Intent here is to verify that 'go test -n' works without crashing. - // This reuses flag_test.go, but really any test would do. - tg := testgo(t) - defer tg.cleanup() - tg.run("test", "testdata/flag_test.go", "-n", "-args", "-v=7") -} - func TestGoTestJSON(t *testing.T) { skipIfGccgo(t, "gccgo does not have standard packages") tooSlow(t) @@ -5109,6 +4725,7 @@ func init() {} func TestBadCommandLines(t *testing.T) { tg := testgo(t) defer tg.cleanup() + tg.parallel() tg.tempFile("src/x/x.go", "package x\n") tg.setenv("GOPATH", tg.path(".")) @@ -5329,6 +4946,7 @@ func TestCgoCache(t *testing.T) { func TestFilepathUnderCwdFormat(t *testing.T) { tg := testgo(t) defer tg.cleanup() + tg.parallel() tg.run("test", "-x", "-cover", "log") tg.grepStderrNot(`\.log\.cover\.go`, "-x output should contain correctly formatted filepath under cwd") } @@ -5433,16 +5051,6 @@ func TestCDAndGOPATHAreDifferent(t *testing.T) { } } -// Issue 26242. -func TestGoTestWithoutTests(t *testing.T) { - tg := testgo(t) - defer tg.cleanup() - tg.parallel() - tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.run("test", "testnorun") - tg.grepStdout(`testnorun\t\[no test files\]`, "do not want test to run") -} - // Issue 25579. func TestGoBuildDashODevNull(t *testing.T) { tooSlow(t) diff --git a/libgo/go/cmd/go/internal/clean/clean.go b/libgo/go/cmd/go/internal/clean/clean.go index 5f4bf4e..69e1748 100644 --- a/libgo/go/cmd/go/internal/clean/clean.go +++ b/libgo/go/cmd/go/internal/clean/clean.go @@ -178,7 +178,9 @@ func runClean(cmd *base.Command, args []string) { } } if err != nil { - base.Errorf("go clean -testcache: %v", err) + if _, statErr := os.Stat(dir); !os.IsNotExist(statErr) { + base.Errorf("go clean -testcache: %v", err) + } } } } diff --git a/libgo/go/cmd/go/internal/list/list.go b/libgo/go/cmd/go/internal/list/list.go index b393c67..8d979e2 100644 --- a/libgo/go/cmd/go/internal/list/list.go +++ b/libgo/go/cmd/go/internal/list/list.go @@ -211,7 +211,7 @@ applied to a Go struct, but now a Module struct: Main bool // is this the main module? Indirect bool // is this module only an indirect dependency of main module? Dir string // directory holding files for this module, if any - GoMod string // path to go.mod file for this module, if any + GoMod string // path to go.mod file used when loading this module, if any GoVersion string // go version used in module Error *ModuleError // error loading module } @@ -220,6 +220,9 @@ applied to a Go struct, but now a Module struct: Err string // the error itself } +The file GoMod refers to may be outside the module directory if the +module is in the module cache or if the -modfile flag is used. + The default output is to print the module path and then information about the version and replacement if any. For example, 'go list -m all' might print: @@ -387,15 +390,24 @@ func runList(cmd *base.Command, args []string) { modload.InitMod() // Parses go.mod and sets cfg.BuildMod. if cfg.BuildMod == "vendor" { + const actionDisabledFormat = "go list -m: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)" + + if *listVersions { + base.Fatalf(actionDisabledFormat, "determine available versions") + } + if *listU { + base.Fatalf(actionDisabledFormat, "determine available upgrades") + } + for _, arg := range args { // In vendor mode, the module graph is incomplete: it contains only the // explicit module dependencies and the modules that supply packages in // the import graph. Reject queries that imply more information than that. if arg == "all" { - base.Fatalf("go list -m: can't compute 'all' using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)") + base.Fatalf(actionDisabledFormat, "compute 'all'") } if strings.Contains(arg, "...") { - base.Fatalf("go list -m: can't match module patterns using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)") + base.Fatalf(actionDisabledFormat, "match module patterns") } } } diff --git a/libgo/go/cmd/go/internal/modcmd/download.go b/libgo/go/cmd/go/internal/modcmd/download.go index 5db0e46..7d5294d 100644 --- a/libgo/go/cmd/go/internal/modcmd/download.go +++ b/libgo/go/cmd/go/internal/modcmd/download.go @@ -30,7 +30,9 @@ The go command will automatically download modules as needed during ordinary execution. The "go mod download" command is useful mainly for pre-filling the local cache or to compute the answers for a Go module proxy. -By default, download reports errors to standard error but is otherwise silent. +By default, download writes nothing to standard output. It may print progress +messages and errors to standard error. + The -json flag causes download to print a sequence of JSON objects to standard output, describing each downloaded module (or failure), corresponding to this Go struct: diff --git a/libgo/go/cmd/go/internal/modfetch/cache.go b/libgo/go/cmd/go/internal/modfetch/cache.go index 104fce8..947192b 100644 --- a/libgo/go/cmd/go/internal/modfetch/cache.go +++ b/libgo/go/cmd/go/internal/modfetch/cache.go @@ -13,7 +13,6 @@ import ( "os" "path/filepath" "strings" - "time" "cmd/go/internal/base" "cmd/go/internal/cfg" @@ -28,8 +27,6 @@ import ( var PkgMod string // $GOPATH/pkg/mod; set by package modload -const logFindingDelay = 1 * time.Second - func cacheDir(path string) (string, error) { if PkgMod == "" { return "", fmt.Errorf("internal error: modfetch.PkgMod not set") @@ -140,11 +137,6 @@ func (r *cachingRepo) Versions(prefix string) ([]string, error) { err error } c := r.cache.Do("versions:"+prefix, func() interface{} { - logTimer := time.AfterFunc(logFindingDelay, func() { - fmt.Fprintf(os.Stderr, "go: finding versions for %s\n", r.path) - }) - defer logTimer.Stop() - list, err := r.r.Versions(prefix) return cached{list, err} }).(cached) @@ -167,11 +159,6 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { return cachedInfo{info, nil} } - logTimer := time.AfterFunc(logFindingDelay, func() { - fmt.Fprintf(os.Stderr, "go: finding %s %s\n", r.path, rev) - }) - defer logTimer.Stop() - info, err = r.r.Stat(rev) if err == nil { // If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78, @@ -199,11 +186,6 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { func (r *cachingRepo) Latest() (*RevInfo, error) { c := r.cache.Do("latest:", func() interface{} { - logTimer := time.AfterFunc(logFindingDelay, func() { - fmt.Fprintf(os.Stderr, "go: finding %s latest\n", r.path) - }) - defer logTimer.Stop() - info, err := r.r.Latest() // Save info for likely future Stat call. diff --git a/libgo/go/cmd/go/internal/modfetch/codehost/git.go b/libgo/go/cmd/go/internal/modfetch/codehost/git.go index e329cbc..f08df51 100644 --- a/libgo/go/cmd/go/internal/modfetch/codehost/git.go +++ b/libgo/go/cmd/go/internal/modfetch/codehost/git.go @@ -682,8 +682,11 @@ func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) { semtag := line[len(prefix):] // Consider only tags that are valid and complete (not just major.minor prefixes). - if c := semver.Canonical(semtag); c != "" && strings.HasPrefix(semtag, c) && (major == "" || semver.Major(c) == major) { - highest = semver.Max(highest, semtag) + // NOTE: Do not replace the call to semver.Compare with semver.Max. + // We want to return the actual tag, not a canonicalized version of it, + // and semver.Max currently canonicalizes (see golang.org/issue/32700). + if c := semver.Canonical(semtag); c != "" && strings.HasPrefix(semtag, c) && (major == "" || semver.Major(c) == major) && semver.Compare(semtag, highest) > 0 { + highest = semtag } } diff --git a/libgo/go/cmd/go/internal/modfetch/coderepo.go b/libgo/go/cmd/go/internal/modfetch/coderepo.go index de757ec..d1d24a40 100644 --- a/libgo/go/cmd/go/internal/modfetch/coderepo.go +++ b/libgo/go/cmd/go/internal/modfetch/coderepo.go @@ -191,22 +191,6 @@ func (r *codeRepo) appendIncompatibleVersions(list, incompatible []string) ([]st return list, nil } - // We assume that if the latest release of any major version has a go.mod - // file, all subsequent major versions will also have go.mod files (and thus - // be ineligible for use as +incompatible versions). - // If we're wrong about a major version, users will still be able to 'go get' - // specific higher versions explicitly — they just won't affect 'latest' or - // appear in 'go list'. - // - // Conversely, we assume that if the latest release of any major version lacks - // a go.mod file, all versions also lack go.mod files. If we're wrong, we may - // include a +incompatible version that isn't really valid, but most - // operations won't try to use that version anyway. - // - // These optimizations bring - // 'go list -versions -m github.com/openshift/origin' down from 1m58s to 0m37s. - // That's still not great, but a substantial improvement. - versionHasGoMod := func(v string) (bool, error) { _, err := r.code.ReadFile(v, "go.mod", codehost.MaxGoMod) if err == nil { @@ -241,32 +225,41 @@ func (r *codeRepo) appendIncompatibleVersions(list, incompatible []string) ([]st } } - var lastMajor string + var ( + lastMajor string + lastMajorHasGoMod bool + ) for i, v := range incompatible { major := semver.Major(v) - if major == lastMajor { - list = append(list, v+"+incompatible") - continue - } - rem := incompatible[i:] - j := sort.Search(len(rem), func(j int) bool { - return semver.Major(rem[j]) != major - }) - latestAtMajor := rem[j-1] - - ok, err := versionHasGoMod(latestAtMajor) - if err != nil { - return nil, err - } - if ok { - // This major version has a go.mod file, so it is not allowed as - // +incompatible. Subsequent major versions are likely to also have - // go.mod files, so stop here. - break + if major != lastMajor { + rem := incompatible[i:] + j := sort.Search(len(rem), func(j int) bool { + return semver.Major(rem[j]) != major + }) + latestAtMajor := rem[j-1] + + var err error + lastMajor = major + lastMajorHasGoMod, err = versionHasGoMod(latestAtMajor) + if err != nil { + return nil, err + } } - lastMajor = major + if lastMajorHasGoMod { + // The latest release of this major version has a go.mod file, so it is + // not allowed as +incompatible. It would be confusing to include some + // minor versions of this major version as +incompatible but require + // semantic import versioning for others, so drop all +incompatible + // versions for this major version. + // + // If we're wrong about a minor version in the middle, users will still be + // able to 'go get' specific tags for that version explicitly — they just + // won't appear in 'go list' or as the results for queries with inequality + // bounds. + continue + } list = append(list, v+"+incompatible") } @@ -708,7 +701,7 @@ func (r *codeRepo) findDir(version string) (rev, dir string, gomod []byte, err e return "", "", nil, fmt.Errorf("reading %s/%s at revision %s: %v", r.pathPrefix, file1, rev, err1) } mpath1 := modfile.ModulePath(gomod1) - found1 := err1 == nil && isMajor(mpath1, r.pathMajor) + found1 := err1 == nil && (isMajor(mpath1, r.pathMajor) || r.canReplaceMismatchedVersionDueToBug(mpath1)) var file2 string if r.pathMajor != "" && r.codeRoot != r.modPath && !strings.HasPrefix(r.pathMajor, ".") { @@ -817,6 +810,17 @@ func isMajor(mpath, pathMajor string) bool { return pathMajor[1:] == mpathMajor[1:] } +// canReplaceMismatchedVersionDueToBug reports whether versions of r +// could replace versions of mpath with otherwise-mismatched major versions +// due to a historical bug in the Go command (golang.org/issue/34254). +func (r *codeRepo) canReplaceMismatchedVersionDueToBug(mpath string) bool { + // The bug caused us to erroneously accept unversioned paths as replacements + // for versioned gopkg.in paths. + unversioned := r.pathMajor == "" + replacingGopkgIn := strings.HasPrefix(mpath, "gopkg.in/") + return unversioned && replacingGopkgIn +} + func (r *codeRepo) GoMod(version string) (data []byte, err error) { if version != module.CanonicalVersion(version) { return nil, fmt.Errorf("version %s is not canonical", version) diff --git a/libgo/go/cmd/go/internal/modload/build.go b/libgo/go/cmd/go/internal/modload/build.go index 292fd45..d0642bc 100644 --- a/libgo/go/cmd/go/internal/modload/build.go +++ b/libgo/go/cmd/go/internal/modload/build.go @@ -112,7 +112,7 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { } if HasModRoot() { info.Dir = ModRoot() - info.GoMod = filepath.Join(info.Dir, "go.mod") + info.GoMod = ModFilePath() if modFile.Go != nil { info.GoVersion = modFile.Go.Version } diff --git a/libgo/go/cmd/go/internal/modload/help.go b/libgo/go/cmd/go/internal/modload/help.go index b47f3de..66c1f70 100644 --- a/libgo/go/cmd/go/internal/modload/help.go +++ b/libgo/go/cmd/go/internal/modload/help.go @@ -21,14 +21,15 @@ which source files are used in a given build. Module support -Go 1.13 includes support for Go modules. Module-aware mode is active by default -whenever a go.mod file is found in, or in a parent of, the current directory. +The go command includes support for Go modules. Module-aware mode is active +by default whenever a go.mod file is found in the current directory or in +any parent directory. The quickest way to take advantage of module support is to check out your repository, create a go.mod file (described in the next section) there, and run go commands from within that file tree. -For more fine-grained control, Go 1.13 continues to respect +For more fine-grained control, the go command continues to respect a temporary environment variable, GO111MODULE, which can be set to one of three string values: off, on, or auto (the default). If GO111MODULE=on, then the go command requires the use of modules, diff --git a/libgo/go/cmd/go/internal/modload/import.go b/libgo/go/cmd/go/internal/modload/import.go index 9ae2900..5906d64 100644 --- a/libgo/go/cmd/go/internal/modload/import.go +++ b/libgo/go/cmd/go/internal/modload/import.go @@ -203,7 +203,12 @@ func Import(path string) (m module.Version, dir string, err error) { latest := map[string]string{} // path -> version for _, r := range modFile.Replace { if maybeInModule(path, r.Old.Path) { - latest[r.Old.Path] = semver.Max(r.Old.Version, latest[r.Old.Path]) + // Don't use semver.Max here; need to preserve +incompatible suffix. + v := latest[r.Old.Path] + if semver.Compare(r.Old.Version, v) > 0 { + v = r.Old.Version + } + latest[r.Old.Path] = v } } @@ -264,6 +269,8 @@ func Import(path string) (m module.Version, dir string, err error) { return module.Version{}, "", &ImportMissingError{Path: path} } + fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) + candidates, err := QueryPackage(path, "latest", Allowed) if err != nil { if errors.Is(err, os.ErrNotExist) { diff --git a/libgo/go/cmd/go/internal/modload/load.go b/libgo/go/cmd/go/internal/modload/load.go index 408c790..7a8391d 100644 --- a/libgo/go/cmd/go/internal/modload/load.go +++ b/libgo/go/cmd/go/internal/modload/load.go @@ -1324,6 +1324,21 @@ func fetch(mod module.Version) (dir string, isLocal bool, err error) { if !filepath.IsAbs(dir) { dir = filepath.Join(ModRoot(), dir) } + // Ensure that the replacement directory actually exists: + // dirInModule does not report errors for missing modules, + // so if we don't report the error now, later failures will be + // very mysterious. + if _, err := os.Stat(dir); err != nil { + if os.IsNotExist(err) { + // Semantically the module version itself “exists” — we just don't + // have its source code. Remove the equivalence to os.ErrNotExist, + // and make the message more concise while we're at it. + err = fmt.Errorf("replacement directory %s does not exist", r.Path) + } else { + err = fmt.Errorf("replacement directory %s: %w", r.Path, err) + } + return dir, true, module.VersionError(mod, err) + } return dir, true, nil } mod = r diff --git a/libgo/go/cmd/go/internal/modload/query.go b/libgo/go/cmd/go/internal/modload/query.go index 53278b9..031e459 100644 --- a/libgo/go/cmd/go/internal/modload/query.go +++ b/libgo/go/cmd/go/internal/modload/query.go @@ -79,7 +79,7 @@ func queryProxy(proxy, path, query, current string, allowed func(module.Version) if current != "" && !semver.IsValid(current) { return nil, fmt.Errorf("invalid previous version %q", current) } - if cfg.BuildMod != "" && cfg.BuildMod != "mod" { + if cfg.BuildMod == "vendor" { return nil, errQueryDisabled } if allowed == nil { diff --git a/libgo/go/cmd/go/internal/modload/query_test.go b/libgo/go/cmd/go/internal/modload/query_test.go index 9c91c05..15470e2 100644 --- a/libgo/go/cmd/go/internal/modload/query_test.go +++ b/libgo/go/cmd/go/internal/modload/query_test.go @@ -64,7 +64,7 @@ var queryTests = []struct { git add go.mod git commit -m v1 go.mod git tag start - for i in v0.0.0-pre1 v0.0.0 v0.0.1 v0.0.2 v0.0.3 v0.1.0 v0.1.1 v0.1.2 v0.3.0 v1.0.0 v1.1.0 v1.9.0 v1.9.9 v1.9.10-pre1 v1.9.10-pre2+metadata; do + for i in v0.0.0-pre1 v0.0.0 v0.0.1 v0.0.2 v0.0.3 v0.1.0 v0.1.1 v0.1.2 v0.3.0 v1.0.0 v1.1.0 v1.9.0 v1.9.9 v1.9.10-pre1 v1.9.10-pre2+metadata unversioned; do echo before $i >status git add status git commit -m "before $i" status @@ -107,6 +107,7 @@ var queryTests = []struct { {path: queryRepo, query: "v0.2", err: `no matching versions for query "v0.2"`}, {path: queryRepo, query: "v0.0", vers: "v0.0.3"}, {path: queryRepo, query: "v1.9.10-pre2+metadata", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, + {path: queryRepo, query: "ed5ffdaa", vers: "v1.9.10-pre2.0.20191220134614-ed5ffdaa1f5e"}, // golang.org/issue/29262: The major version for for a module without a suffix // should be based on the most recent tag (v1 as appropriate, not v0 @@ -162,10 +163,14 @@ var queryTests = []struct { {path: queryRepoV2, query: "v2.6.0-pre1", vers: "v2.6.0-pre1"}, {path: queryRepoV2, query: "latest", vers: "v2.5.5"}, - // e0cf3de987e6 is the latest commit on the master branch, and it's actually - // v1.19.10-pre1, not anything resembling v3: attempting to query it as such - // should fail. + // Commit e0cf3de987e6 is actually v1.19.10-pre1, not anything resembling v3, + // and it has a go.mod file with a non-v3 module path. Attempting to query it + // as the v3 module should fail. {path: queryRepoV3, query: "e0cf3de987e6", err: `vcs-test.golang.org/git/querytest.git/v3@v3.0.0-20180704024501-e0cf3de987e6: invalid version: go.mod has non-.../v3 module path "vcs-test.golang.org/git/querytest.git" (and .../v3/go.mod does not exist) at revision e0cf3de987e6`}, + + // The querytest repo does not have any commits tagged with major version 3, + // and the latest commit in the repo has a go.mod file specifying a non-v3 path. + // That should prevent us from resolving any version for the /v3 path. {path: queryRepoV3, query: "latest", err: `no matching versions for query "latest"`}, {path: emptyRepo, query: "latest", vers: "v0.0.0-20180704023549-7bb914627242"}, diff --git a/libgo/go/cmd/go/internal/work/gc.go b/libgo/go/cmd/go/internal/work/gc.go index 5702469..7d17c0c 100644 --- a/libgo/go/cmd/go/internal/work/gc.go +++ b/libgo/go/cmd/go/internal/work/gc.go @@ -227,8 +227,8 @@ func (a *Action) trimpath() string { // For "go build -trimpath", rewrite package source directory // to a file system-independent path (just the import path). if cfg.BuildTrimpath { - if m := a.Package.Module; m != nil { - rewrite += ";" + m.Dir + "=>" + m.Path + "@" + m.Version + if m := a.Package.Module; m != nil && m.Version != "" { + rewrite += ";" + a.Package.Dir + "=>" + m.Path + "@" + m.Version + strings.TrimPrefix(a.Package.ImportPath, m.Path) } else { rewrite += ";" + a.Package.Dir + "=>" + a.Package.ImportPath } diff --git a/libgo/go/cmd/go/testdata/badmod/go.mod b/libgo/go/cmd/go/testdata/badmod/go.mod deleted file mode 100644 index f7f6423..0000000 --- a/libgo/go/cmd/go/testdata/badmod/go.mod +++ /dev/null @@ -1 +0,0 @@ -module m diff --git a/libgo/go/cmd/go/testdata/badmod/x.go b/libgo/go/cmd/go/testdata/badmod/x.go deleted file mode 100644 index 579fb08..0000000 --- a/libgo/go/cmd/go/testdata/badmod/x.go +++ /dev/null @@ -1,4 +0,0 @@ -package x - -import _ "appengine" -import _ "nonexistent.rsc.io" // domain does not exist diff --git a/libgo/go/cmd/go/testdata/flag_test.go b/libgo/go/cmd/go/testdata/flag_test.go deleted file mode 100644 index ddf613d..0000000 --- a/libgo/go/cmd/go/testdata/flag_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package flag_test - -import ( - "flag" - "log" - "testing" -) - -var v = flag.Int("v", 0, "v flag") - -// Run this as go test pkg -v=7 -func TestVFlagIsSet(t *testing.T) { - if *v != 7 { - log.Fatal("v flag not set") - } -} diff --git a/libgo/go/cmd/go/testdata/importcom/bad.go b/libgo/go/cmd/go/testdata/importcom/bad.go deleted file mode 100644 index e104c2e..0000000 --- a/libgo/go/cmd/go/testdata/importcom/bad.go +++ /dev/null @@ -1,3 +0,0 @@ -package p - -import "bad" diff --git a/libgo/go/cmd/go/testdata/importcom/conflict.go b/libgo/go/cmd/go/testdata/importcom/conflict.go deleted file mode 100644 index 995556c..0000000 --- a/libgo/go/cmd/go/testdata/importcom/conflict.go +++ /dev/null @@ -1,3 +0,0 @@ -package p - -import "conflict" diff --git a/libgo/go/cmd/go/testdata/importcom/src/bad/bad.go b/libgo/go/cmd/go/testdata/importcom/src/bad/bad.go deleted file mode 100644 index bc51fd3..0000000 --- a/libgo/go/cmd/go/testdata/importcom/src/bad/bad.go +++ /dev/null @@ -1 +0,0 @@ -package bad // import diff --git a/libgo/go/cmd/go/testdata/importcom/src/conflict/a.go b/libgo/go/cmd/go/testdata/importcom/src/conflict/a.go deleted file mode 100644 index 2d67703..0000000 --- a/libgo/go/cmd/go/testdata/importcom/src/conflict/a.go +++ /dev/null @@ -1 +0,0 @@ -package conflict // import "a" diff --git a/libgo/go/cmd/go/testdata/importcom/src/conflict/b.go b/libgo/go/cmd/go/testdata/importcom/src/conflict/b.go deleted file mode 100644 index 8fcfb3c..0000000 --- a/libgo/go/cmd/go/testdata/importcom/src/conflict/b.go +++ /dev/null @@ -1 +0,0 @@ -package conflict /* import "b" */ diff --git a/libgo/go/cmd/go/testdata/importcom/src/works/x/x.go b/libgo/go/cmd/go/testdata/importcom/src/works/x/x.go deleted file mode 100644 index 044c6ec..0000000 --- a/libgo/go/cmd/go/testdata/importcom/src/works/x/x.go +++ /dev/null @@ -1 +0,0 @@ -package x // import "works/x" diff --git a/libgo/go/cmd/go/testdata/importcom/src/works/x/x1.go b/libgo/go/cmd/go/testdata/importcom/src/works/x/x1.go deleted file mode 100644 index 2449b29..0000000 --- a/libgo/go/cmd/go/testdata/importcom/src/works/x/x1.go +++ /dev/null @@ -1 +0,0 @@ -package x // important! not an import comment diff --git a/libgo/go/cmd/go/testdata/importcom/src/wrongplace/x.go b/libgo/go/cmd/go/testdata/importcom/src/wrongplace/x.go deleted file mode 100644 index b89849d..0000000 --- a/libgo/go/cmd/go/testdata/importcom/src/wrongplace/x.go +++ /dev/null @@ -1 +0,0 @@ -package x // import "my/x" diff --git a/libgo/go/cmd/go/testdata/importcom/works.go b/libgo/go/cmd/go/testdata/importcom/works.go deleted file mode 100644 index 31b55d0..0000000 --- a/libgo/go/cmd/go/testdata/importcom/works.go +++ /dev/null @@ -1,3 +0,0 @@ -package p - -import _ "works/x" diff --git a/libgo/go/cmd/go/testdata/importcom/wrongplace.go b/libgo/go/cmd/go/testdata/importcom/wrongplace.go deleted file mode 100644 index e2535e0..0000000 --- a/libgo/go/cmd/go/testdata/importcom/wrongplace.go +++ /dev/null @@ -1,3 +0,0 @@ -package p - -import "wrongplace" diff --git a/libgo/go/cmd/go/testdata/importcycle/src/selfimport/selfimport.go b/libgo/go/cmd/go/testdata/importcycle/src/selfimport/selfimport.go deleted file mode 100644 index dc63c4b..0000000 --- a/libgo/go/cmd/go/testdata/importcycle/src/selfimport/selfimport.go +++ /dev/null @@ -1,3 +0,0 @@ -package selfimport - -import "selfimport" diff --git a/libgo/go/cmd/go/testdata/script/README b/libgo/go/cmd/go/testdata/script/README index 2782a09..7dba6b3 100644 --- a/libgo/go/cmd/go/testdata/script/README +++ b/libgo/go/cmd/go/testdata/script/README @@ -40,7 +40,7 @@ Scripts also have access to these other environment variables: goversion=<current Go version; for example, 1.12> :=<OS-specific path list separator> -The scripts supporting files are unpacked relative to $GOPATH/src (aka $WORK/gopath/src) +The scripts' supporting files are unpacked relative to $GOPATH/src (aka $WORK/gopath/src) and then the script begins execution in that directory as well. Thus the example above runs in $WORK/gopath/src with GOPATH=$WORK/gopath and $WORK/gopath/src/hello.go containing the listed contents. diff --git a/libgo/go/cmd/go/testdata/script/clean_testcache.txt b/libgo/go/cmd/go/testdata/script/clean_testcache.txt index dd78464..b3f32fe 100644 --- a/libgo/go/cmd/go/testdata/script/clean_testcache.txt +++ b/libgo/go/cmd/go/testdata/script/clean_testcache.txt @@ -9,6 +9,13 @@ go clean -testcache go test x_test.go ! stdout 'cached' +# golang.org/issue/29100: 'go clean -testcache' should succeed +# if the cache directory doesn't exist at all. +# It should not write a testexpire.txt file, since there are no +# test results that need to be invalidated in the first place. +env GOCACHE=$WORK/nonexistent +go clean -testcache +! exists $WORK/nonexistent -- x/x_test.go -- package x_test @@ -16,4 +23,4 @@ import ( "testing" ) func TestMain(t *testing.T) { -}
\ No newline at end of file +} diff --git a/libgo/go/cmd/go/testdata/script/link_syso_issue33139.txt b/libgo/go/cmd/go/testdata/script/link_syso_issue33139.txt index c2ca27a..03169bf 100644 --- a/libgo/go/cmd/go/testdata/script/link_syso_issue33139.txt +++ b/libgo/go/cmd/go/testdata/script/link_syso_issue33139.txt @@ -8,6 +8,10 @@ # See: https://github.com/golang/go/issues/8912 [linux] [ppc64] skip +# External linking is not supported on linux/riscv64. +# See: https://github.com/golang/go/issues/36739 +[linux] [riscv64] skip + # External linking is not supported on darwin/386 (10.14+). # See: https://github.com/golang/go/issues/31751 [darwin] [386] skip diff --git a/libgo/go/cmd/go/testdata/script/mod_get_test.txt b/libgo/go/cmd/go/testdata/script/mod_get_test.txt index f921168..3680ca2 100644 --- a/libgo/go/cmd/go/testdata/script/mod_get_test.txt +++ b/libgo/go/cmd/go/testdata/script/mod_get_test.txt @@ -33,7 +33,7 @@ grep 'rsc.io/quote v1.5.1$' go.mod # 'go get all' should consider test dependencies with or without -t. cp go.mod.empty go.mod -go get all +go get -d all grep 'rsc.io/quote v1.5.2$' go.mod -- go.mod.empty -- diff --git a/libgo/go/cmd/go/testdata/script/mod_load_badchain.txt b/libgo/go/cmd/go/testdata/script/mod_load_badchain.txt index 2c532f1..67d9a15 100644 --- a/libgo/go/cmd/go/testdata/script/mod_load_badchain.txt +++ b/libgo/go/cmd/go/testdata/script/mod_load_badchain.txt @@ -75,12 +75,14 @@ go: example.com/badchain/a@v1.1.0 requires module declares its path as: badchain.example.com/c but was required as: example.com/badchain/c -- list-missing-expected -- +go: finding module for package example.com/badchain/c go: found example.com/badchain/c in example.com/badchain/c v1.1.0 go: m/use imports example.com/badchain/c: example.com/badchain/c@v1.1.0: parsing go.mod: module declares its path as: badchain.example.com/c but was required as: example.com/badchain/c -- list-missing-test-expected -- +go: finding module for package example.com/badchain/c go: found example.com/badchain/c in example.com/badchain/c v1.1.0 go: m/testuse tested by m/testuse.test imports diff --git a/libgo/go/cmd/go/testdata/script/mod_readonly.txt b/libgo/go/cmd/go/testdata/script/mod_readonly.txt index 1d1771e..77fc735 100644 --- a/libgo/go/cmd/go/testdata/script/mod_readonly.txt +++ b/libgo/go/cmd/go/testdata/script/mod_readonly.txt @@ -34,6 +34,11 @@ go list all go clean -modcache go list all +# -mod=readonly must not cause 'go list -m' to fail. +# (golang.org/issue/36478) +go list -m all +! stderr 'cannot query module' + # -mod=readonly should reject inconsistent go.mod files # (ones that would be rewritten). go mod edit -require rsc.io/sampler@v1.2.0 diff --git a/libgo/go/cmd/go/testdata/script/mod_replace_gopkgin.txt b/libgo/go/cmd/go/testdata/script/mod_replace_gopkgin.txt index 6608fb1..28c1196 100644 --- a/libgo/go/cmd/go/testdata/script/mod_replace_gopkgin.txt +++ b/libgo/go/cmd/go/testdata/script/mod_replace_gopkgin.txt @@ -15,10 +15,28 @@ env GOSUMDB=off # Replacing gopkg.in/[…].vN with a repository with a root go.mod file # specifying […].vN and a compatible version should succeed, even if # the replacement path is not a gopkg.in path. -cd dot-to-dot -go list gopkg.in/src-d/go-git.v4 +cd 4-to-4 +go list -m gopkg.in/src-d/go-git.v4 --- dot-to-dot/go.mod -- +# Previous versions of the "go" command accepted v0 and v1 pseudo-versions +# as replacements for gopkg.in/[…].v4. +# As a special case, we continue to accept those. + +cd ../4-to-0 +go list -m gopkg.in/src-d/go-git.v4 + +cd ../4-to-1 +go list -m gopkg.in/src-d/go-git.v4 + +cd ../4-to-incompatible +go list -m gopkg.in/src-d/go-git.v4 + +# A mismatched gopkg.in path should not be able to replace a different major version. +cd ../3-to-gomod-4 +! go list -m gopkg.in/src-d/go-git.v3 +stderr '^go: gopkg\.in/src-d/go-git\.v3@v3.0.0-20190801152248-0d1a009cbb60: invalid version: go\.mod has non-\.\.\.\.v3 module path "gopkg\.in/src-d/go-git\.v4" at revision 0d1a009cbb60$' + +-- 4-to-4/go.mod -- module golang.org/issue/34254 go 1.13 @@ -26,3 +44,36 @@ go 1.13 require gopkg.in/src-d/go-git.v4 v4.13.1 replace gopkg.in/src-d/go-git.v4 v4.13.1 => github.com/src-d/go-git/v4 v4.13.1 +-- 4-to-1/go.mod -- +module golang.org/issue/34254 + +go 1.13 + +require gopkg.in/src-d/go-git.v4 v4.13.1 + +replace gopkg.in/src-d/go-git.v4 v4.13.1 => github.com/src-d/go-git v1.0.1-0.20190801152248-0d1a009cbb60 +-- 4-to-0/go.mod -- +module golang.org/issue/34254 + +go 1.13 + +require gopkg.in/src-d/go-git.v4 v4.13.1 + +replace gopkg.in/src-d/go-git.v4 v4.13.1 => github.com/src-d/go-git v0.0.0-20190801152248-0d1a009cbb60 +-- 4-to-incompatible/go.mod -- +module golang.org/issue/34254 + +go 1.13 + +require gopkg.in/src-d/go-git.v4 v4.13.1 + +replace gopkg.in/src-d/go-git.v4 v4.13.1 => github.com/src-d/go-git v4.6.0+incompatible +-- 3-to-gomod-4/go.mod -- +module golang.org/issue/34254 +go 1.13 + +require gopkg.in/src-d/go-git.v3 v3.2.0 + +// This replacement has a go.mod file declaring its path to be +// gopkg.in/src-d/go-git.v4, so it cannot be used as a replacement for v3. +replace gopkg.in/src-d/go-git.v3 v3.2.0 => gopkg.in/src-d/go-git.v3 v3.0.0-20190801152248-0d1a009cbb60 diff --git a/libgo/go/cmd/go/testdata/script/mod_replace_import.txt b/libgo/go/cmd/go/testdata/script/mod_replace_import.txt index 941ef61..fd5b04a 100644 --- a/libgo/go/cmd/go/testdata/script/mod_replace_import.txt +++ b/libgo/go/cmd/go/testdata/script/mod_replace_import.txt @@ -28,7 +28,8 @@ stdout 'example.com/v v1.12.0 => ./v12' cd fail ! go list all stdout 'localhost.fail' -stderr '^can.t load package: m.go:3:8: module w@latest found \(v0.0.0-00010101000000-000000000000, replaced by ../w\), but does not contain package w$' +stderr '^can''t load package: m.go:4:2: module w@latest found \(v0.0.0-00010101000000-000000000000, replaced by ../w\), but does not contain package w$' +stderr '^can''t load package: m.go:5:2: nonexist@v0.1.0: replacement directory ../nonexist does not exist$' -- go.mod -- module example.com/m @@ -54,6 +55,10 @@ replace ( example.com/v => ./v ) +replace ( + example.com/i v2.0.0+incompatible => ./i2 +) + -- m.go -- package main import ( @@ -61,6 +66,7 @@ import ( _ "example.com/x/v3" _ "example.com/y/z/w" _ "example.com/v" + _ "example.com/i" ) func main() {} @@ -115,10 +121,18 @@ module v.localhost -- v/v.go -- package v +-- i2/go.mod -- +module example.com/i +-- i2/i.go -- +package i + -- fail/m.go -- package main -import _ "w" +import ( + _ "w" + _ "nonexist" +) func main() {} @@ -127,3 +141,4 @@ module localhost.fail replace w => ../w +replace nonexist v0.1.0 => ../nonexist diff --git a/libgo/go/cmd/go/testdata/script/mod_run_internal.txt b/libgo/go/cmd/go/testdata/script/mod_run_internal.txt deleted file mode 100644 index 653ad28..0000000 --- a/libgo/go/cmd/go/testdata/script/mod_run_internal.txt +++ /dev/null @@ -1,46 +0,0 @@ -env GO111MODULE=on - -go list -e -f '{{.Incomplete}}' runbad1.go -stdout true -! go run runbad1.go -stderr 'use of internal package m/x/internal not allowed' - -go list -e -f '{{.Incomplete}}' runbad2.go -stdout true -! go run runbad2.go -stderr 'use of internal package m/x/internal/y not allowed' - -go list -e -f '{{.Incomplete}}' runok.go -stdout false -go run runok.go - --- go.mod -- -module m - --- x/internal/internal.go -- -package internal - --- x/internal/y/y.go -- -package y - --- internal/internal.go -- -package internal - --- internal/z/z.go -- -package z - --- runbad1.go -- -package main -import _ "m/x/internal" -func main() {} - --- runbad2.go -- -package main -import _ "m/x/internal/y" -func main() {} - --- runok.go -- -package main -import _ "m/internal" -import _ "m/internal/z" -func main() {} diff --git a/libgo/go/cmd/go/testdata/script/mod_vendor.txt b/libgo/go/cmd/go/testdata/script/mod_vendor.txt index bb3e634..2622916 100644 --- a/libgo/go/cmd/go/testdata/script/mod_vendor.txt +++ b/libgo/go/cmd/go/testdata/script/mod_vendor.txt @@ -38,6 +38,12 @@ stdout 'src[\\/]vendor[\\/]x' go list -mod=vendor -f '{{.Version}} {{.Dir}}' -m x stdout '^v1.0.0 $' +# -mod=vendor should cause 'go list' flags that look up versions to fail. +! go list -mod=vendor -versions -m x +stderr '^go list -m: can''t determine available versions using the vendor directory\n\t\(Use -mod=mod or -mod=readonly to bypass.\)$' +! go list -mod=vendor -u -m x +stderr '^go list -m: can''t determine available upgrades using the vendor directory\n\t\(Use -mod=mod or -mod=readonly to bypass.\)$' + # 'go list -mod=vendor -m' on a transitive dependency that does not # provide vendored packages should give a helpful error rather than # 'not a known dependency'. diff --git a/libgo/go/cmd/go/testdata/script/modfile_flag.txt b/libgo/go/cmd/go/testdata/script/modfile_flag.txt index 1409be9..f05bf03 100644 --- a/libgo/go/cmd/go/testdata/script/modfile_flag.txt +++ b/libgo/go/cmd/go/testdata/script/modfile_flag.txt @@ -11,6 +11,15 @@ cp go.sum go.sum.orig go mod init example.com/m grep example.com/m go.alt.mod +# 'go env GOMOD' should print the path to the real file. +# 'go env' does not recognize the '-modfile' flag. +go env GOMOD +stdout '^\$WORK[/\\]gopath[/\\]src[/\\]go.mod$' + +# 'go list -m' should print the effective go.mod file as GoMod though. +go list -m -f '{{.GoMod}}' +stdout '^go.alt.mod$' + # go mod edit should operate on the alternate file go mod edit -require rsc.io/quote@v1.5.2 grep rsc.io/quote go.alt.mod diff --git a/libgo/go/cmd/go/testdata/script/test_badtest.txt b/libgo/go/cmd/go/testdata/script/test_badtest.txt index f5db694..75b3b68 100644 --- a/libgo/go/cmd/go/testdata/script/test_badtest.txt +++ b/libgo/go/cmd/go/testdata/script/test_badtest.txt @@ -1,11 +1,21 @@ env GO111MODULE=off -! go test badtest/... +! go test badtest/badexec ! stdout ^ok stdout ^FAIL\tbadtest/badexec + +! go test badtest/badsyntax +! stdout ^ok stdout ^FAIL\tbadtest/badsyntax + +! go test badtest/badvar +! stdout ^ok stdout ^FAIL\tbadtest/badvar +! go test notest +! stdout ^ok +stderr '^notest.hello.go:6:1: .*declaration' # Exercise issue #7108 + -- badtest/badexec/x_test.go -- package badexec @@ -30,3 +40,10 @@ package badvar_test func f() { _ = notdefined } +-- notest/hello.go -- +package notest + +func hello() { + println("hello world") +} +Hello world diff --git a/libgo/go/cmd/go/testdata/src/badc/x.c b/libgo/go/cmd/go/testdata/src/badc/x.c deleted file mode 100644 index f6cbf69..0000000 --- a/libgo/go/cmd/go/testdata/src/badc/x.c +++ /dev/null @@ -1 +0,0 @@ -// C code! diff --git a/libgo/go/cmd/go/testdata/src/badc/x.go b/libgo/go/cmd/go/testdata/src/badc/x.go deleted file mode 100644 index bfa1de2..0000000 --- a/libgo/go/cmd/go/testdata/src/badc/x.go +++ /dev/null @@ -1 +0,0 @@ -package badc diff --git a/libgo/go/cmd/go/testdata/src/badpkg/x.go b/libgo/go/cmd/go/testdata/src/badpkg/x.go deleted file mode 100644 index dda35e8..0000000 --- a/libgo/go/cmd/go/testdata/src/badpkg/x.go +++ /dev/null @@ -1 +0,0 @@ -pkg badpkg diff --git a/libgo/go/cmd/go/testdata/src/bench/x_test.go b/libgo/go/cmd/go/testdata/src/bench/x_test.go deleted file mode 100644 index 32cabf8..0000000 --- a/libgo/go/cmd/go/testdata/src/bench/x_test.go +++ /dev/null @@ -1,6 +0,0 @@ -package bench - -import "testing" - -func Benchmark(b *testing.B) { -} diff --git a/libgo/go/cmd/go/testdata/src/benchfatal/x_test.go b/libgo/go/cmd/go/testdata/src/benchfatal/x_test.go deleted file mode 100644 index 8d3a5de..0000000 --- a/libgo/go/cmd/go/testdata/src/benchfatal/x_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package benchfatal - -import "testing" - -func BenchmarkThatCallsFatal(b *testing.B) { - b.Fatal("called by benchmark") -} diff --git a/libgo/go/cmd/go/testdata/src/cgoasm/p.go b/libgo/go/cmd/go/testdata/src/cgoasm/p.go deleted file mode 100644 index 148b47f..0000000 --- a/libgo/go/cmd/go/testdata/src/cgoasm/p.go +++ /dev/null @@ -1,8 +0,0 @@ -package p - -/* -// hi -*/ -import "C" - -func F() {} diff --git a/libgo/go/cmd/go/testdata/src/cgoasm/p.s b/libgo/go/cmd/go/testdata/src/cgoasm/p.s deleted file mode 100644 index aaade03..0000000 --- a/libgo/go/cmd/go/testdata/src/cgoasm/p.s +++ /dev/null @@ -1,2 +0,0 @@ -TEXT asm(SB),$0 - RET diff --git a/libgo/go/cmd/go/testdata/src/cgocover/p.go b/libgo/go/cmd/go/testdata/src/cgocover/p.go deleted file mode 100644 index a6a3891..0000000 --- a/libgo/go/cmd/go/testdata/src/cgocover/p.go +++ /dev/null @@ -1,19 +0,0 @@ -package p - -/* -void -f(void) -{ -} -*/ -import "C" - -var b bool - -func F() { - if b { - for { - } - } - C.f() -} diff --git a/libgo/go/cmd/go/testdata/src/cgocover/p_test.go b/libgo/go/cmd/go/testdata/src/cgocover/p_test.go deleted file mode 100644 index a8f057e..0000000 --- a/libgo/go/cmd/go/testdata/src/cgocover/p_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package p - -import "testing" - -func TestF(t *testing.T) { - F() -} diff --git a/libgo/go/cmd/go/testdata/src/cgocover2/p.go b/libgo/go/cmd/go/testdata/src/cgocover2/p.go deleted file mode 100644 index a6a3891..0000000 --- a/libgo/go/cmd/go/testdata/src/cgocover2/p.go +++ /dev/null @@ -1,19 +0,0 @@ -package p - -/* -void -f(void) -{ -} -*/ -import "C" - -var b bool - -func F() { - if b { - for { - } - } - C.f() -} diff --git a/libgo/go/cmd/go/testdata/src/cgocover2/x_test.go b/libgo/go/cmd/go/testdata/src/cgocover2/x_test.go deleted file mode 100644 index f4790d2..0000000 --- a/libgo/go/cmd/go/testdata/src/cgocover2/x_test.go +++ /dev/null @@ -1,10 +0,0 @@ -package p_test - -import ( - . "cgocover2" - "testing" -) - -func TestF(t *testing.T) { - F() -} diff --git a/libgo/go/cmd/go/testdata/src/cgocover3/p.go b/libgo/go/cmd/go/testdata/src/cgocover3/p.go deleted file mode 100644 index a6a3891..0000000 --- a/libgo/go/cmd/go/testdata/src/cgocover3/p.go +++ /dev/null @@ -1,19 +0,0 @@ -package p - -/* -void -f(void) -{ -} -*/ -import "C" - -var b bool - -func F() { - if b { - for { - } - } - C.f() -} diff --git a/libgo/go/cmd/go/testdata/src/cgocover3/p_test.go b/libgo/go/cmd/go/testdata/src/cgocover3/p_test.go deleted file mode 100644 index c89cd18..0000000 --- a/libgo/go/cmd/go/testdata/src/cgocover3/p_test.go +++ /dev/null @@ -1 +0,0 @@ -package p diff --git a/libgo/go/cmd/go/testdata/src/cgocover3/x_test.go b/libgo/go/cmd/go/testdata/src/cgocover3/x_test.go deleted file mode 100644 index 97d0e0f..0000000 --- a/libgo/go/cmd/go/testdata/src/cgocover3/x_test.go +++ /dev/null @@ -1,10 +0,0 @@ -package p_test - -import ( - . "cgocover3" - "testing" -) - -func TestF(t *testing.T) { - F() -} diff --git a/libgo/go/cmd/go/testdata/src/cgocover4/notcgo.go b/libgo/go/cmd/go/testdata/src/cgocover4/notcgo.go deleted file mode 100644 index c89cd18..0000000 --- a/libgo/go/cmd/go/testdata/src/cgocover4/notcgo.go +++ /dev/null @@ -1 +0,0 @@ -package p diff --git a/libgo/go/cmd/go/testdata/src/cgocover4/p.go b/libgo/go/cmd/go/testdata/src/cgocover4/p.go deleted file mode 100644 index a6a3891..0000000 --- a/libgo/go/cmd/go/testdata/src/cgocover4/p.go +++ /dev/null @@ -1,19 +0,0 @@ -package p - -/* -void -f(void) -{ -} -*/ -import "C" - -var b bool - -func F() { - if b { - for { - } - } - C.f() -} diff --git a/libgo/go/cmd/go/testdata/src/cgocover4/x_test.go b/libgo/go/cmd/go/testdata/src/cgocover4/x_test.go deleted file mode 100644 index fd9bae7..0000000 --- a/libgo/go/cmd/go/testdata/src/cgocover4/x_test.go +++ /dev/null @@ -1,10 +0,0 @@ -package p_test - -import ( - . "cgocover4" - "testing" -) - -func TestF(t *testing.T) { - F() -} diff --git a/libgo/go/cmd/go/testdata/src/dupload/dupload.go b/libgo/go/cmd/go/testdata/src/dupload/dupload.go deleted file mode 100644 index 2f07852..0000000 --- a/libgo/go/cmd/go/testdata/src/dupload/dupload.go +++ /dev/null @@ -1,8 +0,0 @@ -package main - -import ( - _ "dupload/p2" - _ "p" -) - -func main() {} diff --git a/libgo/go/cmd/go/testdata/src/dupload/p/p.go b/libgo/go/cmd/go/testdata/src/dupload/p/p.go deleted file mode 100644 index c89cd18..0000000 --- a/libgo/go/cmd/go/testdata/src/dupload/p/p.go +++ /dev/null @@ -1 +0,0 @@ -package p diff --git a/libgo/go/cmd/go/testdata/src/dupload/p2/p2.go b/libgo/go/cmd/go/testdata/src/dupload/p2/p2.go deleted file mode 100644 index 8a80979..0000000 --- a/libgo/go/cmd/go/testdata/src/dupload/p2/p2.go +++ /dev/null @@ -1,3 +0,0 @@ -package p2 - -import _ "dupload/vendor/p" diff --git a/libgo/go/cmd/go/testdata/src/dupload/vendor/p/p.go b/libgo/go/cmd/go/testdata/src/dupload/vendor/p/p.go deleted file mode 100644 index c89cd18..0000000 --- a/libgo/go/cmd/go/testdata/src/dupload/vendor/p/p.go +++ /dev/null @@ -1 +0,0 @@ -package p diff --git a/libgo/go/cmd/go/testdata/src/gencycle/gencycle.go b/libgo/go/cmd/go/testdata/src/gencycle/gencycle.go deleted file mode 100644 index 600afd9..0000000 --- a/libgo/go/cmd/go/testdata/src/gencycle/gencycle.go +++ /dev/null @@ -1,5 +0,0 @@ -//go:generate echo hello world - -package gencycle - -import _ "gencycle" diff --git a/libgo/go/cmd/go/testdata/src/importmain/ismain/main.go b/libgo/go/cmd/go/testdata/src/importmain/ismain/main.go deleted file mode 100644 index bf01907..0000000 --- a/libgo/go/cmd/go/testdata/src/importmain/ismain/main.go +++ /dev/null @@ -1,5 +0,0 @@ -package main - -import _ "importmain/test" - -func main() {} diff --git a/libgo/go/cmd/go/testdata/src/importmain/test/test.go b/libgo/go/cmd/go/testdata/src/importmain/test/test.go deleted file mode 100644 index 56e5404..0000000 --- a/libgo/go/cmd/go/testdata/src/importmain/test/test.go +++ /dev/null @@ -1 +0,0 @@ -package test diff --git a/libgo/go/cmd/go/testdata/src/importmain/test/test_test.go b/libgo/go/cmd/go/testdata/src/importmain/test/test_test.go deleted file mode 100644 index 2268a82..0000000 --- a/libgo/go/cmd/go/testdata/src/importmain/test/test_test.go +++ /dev/null @@ -1,6 +0,0 @@ -package test_test - -import "testing" -import _ "importmain/ismain" - -func TestCase(t *testing.T) {} diff --git a/libgo/go/cmd/go/testdata/src/multimain/multimain_test.go b/libgo/go/cmd/go/testdata/src/multimain/multimain_test.go deleted file mode 100644 index 007a86a..0000000 --- a/libgo/go/cmd/go/testdata/src/multimain/multimain_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package multimain_test - -import "testing" - -func TestMain(m *testing.M) { - // Some users run m.Run multiple times, changing - // some kind of global state between runs. - // This used to work so I guess now it has to keep working. - // See golang.org/issue/23129. - m.Run() - m.Run() -} - -func Test(t *testing.T) { - t.Log("notwithstanding") -} diff --git a/libgo/go/cmd/go/testdata/src/not_main/not_main.go b/libgo/go/cmd/go/testdata/src/not_main/not_main.go deleted file mode 100644 index 75a397c..0000000 --- a/libgo/go/cmd/go/testdata/src/not_main/not_main.go +++ /dev/null @@ -1,3 +0,0 @@ -package not_main - -func F() {} diff --git a/libgo/go/cmd/go/testdata/src/notest/hello.go b/libgo/go/cmd/go/testdata/src/notest/hello.go deleted file mode 100644 index 7c42c32..0000000 --- a/libgo/go/cmd/go/testdata/src/notest/hello.go +++ /dev/null @@ -1,6 +0,0 @@ -package notest - -func hello() { - println("hello world") -} -Hello world diff --git a/libgo/go/cmd/go/testdata/src/run/bad.go b/libgo/go/cmd/go/testdata/src/run/bad.go deleted file mode 100644 index c1cc3ac..0000000 --- a/libgo/go/cmd/go/testdata/src/run/bad.go +++ /dev/null @@ -1,5 +0,0 @@ -package main - -import _ "run/subdir/internal/private" - -func main() {} diff --git a/libgo/go/cmd/go/testdata/src/run/good.go b/libgo/go/cmd/go/testdata/src/run/good.go deleted file mode 100644 index 0b67dce..0000000 --- a/libgo/go/cmd/go/testdata/src/run/good.go +++ /dev/null @@ -1,5 +0,0 @@ -package main - -import _ "run/internal" - -func main() {} diff --git a/libgo/go/cmd/go/testdata/src/run/internal/internal.go b/libgo/go/cmd/go/testdata/src/run/internal/internal.go deleted file mode 100644 index 5bf0569..0000000 --- a/libgo/go/cmd/go/testdata/src/run/internal/internal.go +++ /dev/null @@ -1 +0,0 @@ -package internal diff --git a/libgo/go/cmd/go/testdata/src/run/subdir/internal/private/private.go b/libgo/go/cmd/go/testdata/src/run/subdir/internal/private/private.go deleted file mode 100644 index 735e4dc..0000000 --- a/libgo/go/cmd/go/testdata/src/run/subdir/internal/private/private.go +++ /dev/null @@ -1 +0,0 @@ -package private diff --git a/libgo/go/cmd/go/testdata/src/sleepy1/p_test.go b/libgo/go/cmd/go/testdata/src/sleepy1/p_test.go deleted file mode 100644 index 333be7d..0000000 --- a/libgo/go/cmd/go/testdata/src/sleepy1/p_test.go +++ /dev/null @@ -1,10 +0,0 @@ -package p - -import ( - "testing" - "time" -) - -func Test1(t *testing.T) { - time.Sleep(200 * time.Millisecond) -} diff --git a/libgo/go/cmd/go/testdata/src/sleepy2/p_test.go b/libgo/go/cmd/go/testdata/src/sleepy2/p_test.go deleted file mode 100644 index 333be7d..0000000 --- a/libgo/go/cmd/go/testdata/src/sleepy2/p_test.go +++ /dev/null @@ -1,10 +0,0 @@ -package p - -import ( - "testing" - "time" -) - -func Test1(t *testing.T) { - time.Sleep(200 * time.Millisecond) -} diff --git a/libgo/go/cmd/go/testdata/src/sleepybad/p.go b/libgo/go/cmd/go/testdata/src/sleepybad/p.go deleted file mode 100644 index e05b403..0000000 --- a/libgo/go/cmd/go/testdata/src/sleepybad/p.go +++ /dev/null @@ -1,5 +0,0 @@ -package p - -// missing import - -var _ = io.DoesNotExist diff --git a/libgo/go/cmd/go/testdata/src/syntaxerror/x.go b/libgo/go/cmd/go/testdata/src/syntaxerror/x.go deleted file mode 100644 index c89cd18..0000000 --- a/libgo/go/cmd/go/testdata/src/syntaxerror/x.go +++ /dev/null @@ -1 +0,0 @@ -package p diff --git a/libgo/go/cmd/go/testdata/src/syntaxerror/x_test.go b/libgo/go/cmd/go/testdata/src/syntaxerror/x_test.go deleted file mode 100644 index 2460743..0000000 --- a/libgo/go/cmd/go/testdata/src/syntaxerror/x_test.go +++ /dev/null @@ -1,4 +0,0 @@ -package p - -func f() (x.y, z int) { -} diff --git a/libgo/go/cmd/go/testdata/src/testcycle/p1/p1.go b/libgo/go/cmd/go/testdata/src/testcycle/p1/p1.go deleted file mode 100644 index 65ab76d..0000000 --- a/libgo/go/cmd/go/testdata/src/testcycle/p1/p1.go +++ /dev/null @@ -1,7 +0,0 @@ -package p1 - -import _ "testcycle/p2" - -func init() { - println("p1 init") -} diff --git a/libgo/go/cmd/go/testdata/src/testcycle/p1/p1_test.go b/libgo/go/cmd/go/testdata/src/testcycle/p1/p1_test.go deleted file mode 100644 index 75abb13..0000000 --- a/libgo/go/cmd/go/testdata/src/testcycle/p1/p1_test.go +++ /dev/null @@ -1,6 +0,0 @@ -package p1 - -import "testing" - -func Test(t *testing.T) { -} diff --git a/libgo/go/cmd/go/testdata/src/testcycle/p2/p2.go b/libgo/go/cmd/go/testdata/src/testcycle/p2/p2.go deleted file mode 100644 index 7e26cdf..0000000 --- a/libgo/go/cmd/go/testdata/src/testcycle/p2/p2.go +++ /dev/null @@ -1,7 +0,0 @@ -package p2 - -import _ "testcycle/p3" - -func init() { - println("p2 init") -} diff --git a/libgo/go/cmd/go/testdata/src/testcycle/p3/p3.go b/libgo/go/cmd/go/testdata/src/testcycle/p3/p3.go deleted file mode 100644 index bb0a2f4..0000000 --- a/libgo/go/cmd/go/testdata/src/testcycle/p3/p3.go +++ /dev/null @@ -1,5 +0,0 @@ -package p3 - -func init() { - println("p3 init") -} diff --git a/libgo/go/cmd/go/testdata/src/testcycle/p3/p3_test.go b/libgo/go/cmd/go/testdata/src/testcycle/p3/p3_test.go deleted file mode 100644 index 9b4b075..0000000 --- a/libgo/go/cmd/go/testdata/src/testcycle/p3/p3_test.go +++ /dev/null @@ -1,10 +0,0 @@ -package p3 - -import ( - "testing" - - _ "testcycle/p1" -) - -func Test(t *testing.T) { -} diff --git a/libgo/go/cmd/go/testdata/src/testcycle/q1/q1.go b/libgo/go/cmd/go/testdata/src/testcycle/q1/q1.go deleted file mode 100644 index 7a471f0..0000000 --- a/libgo/go/cmd/go/testdata/src/testcycle/q1/q1.go +++ /dev/null @@ -1 +0,0 @@ -package q1 diff --git a/libgo/go/cmd/go/testdata/src/testcycle/q1/q1_test.go b/libgo/go/cmd/go/testdata/src/testcycle/q1/q1_test.go deleted file mode 100644 index ca81bd2..0000000 --- a/libgo/go/cmd/go/testdata/src/testcycle/q1/q1_test.go +++ /dev/null @@ -1,6 +0,0 @@ -package q1 - -import "testing" -import _ "testcycle/q1" - -func Test(t *testing.T) {} diff --git a/libgo/go/cmd/go/testdata/src/testdep/p1/p1.go b/libgo/go/cmd/go/testdata/src/testdep/p1/p1.go deleted file mode 100644 index a457035..0000000 --- a/libgo/go/cmd/go/testdata/src/testdep/p1/p1.go +++ /dev/null @@ -1 +0,0 @@ -package p1 diff --git a/libgo/go/cmd/go/testdata/src/testdep/p1/p1_test.go b/libgo/go/cmd/go/testdata/src/testdep/p1/p1_test.go deleted file mode 100644 index 8be7533..0000000 --- a/libgo/go/cmd/go/testdata/src/testdep/p1/p1_test.go +++ /dev/null @@ -1,3 +0,0 @@ -package p1 - -import _ "testdep/p2" diff --git a/libgo/go/cmd/go/testdata/src/testdep/p2/p2.go b/libgo/go/cmd/go/testdata/src/testdep/p2/p2.go deleted file mode 100644 index 15ba2ea..0000000 --- a/libgo/go/cmd/go/testdata/src/testdep/p2/p2.go +++ /dev/null @@ -1,3 +0,0 @@ -package p2 - -import _ "testdep/p3" diff --git a/libgo/go/cmd/go/testdata/src/testdep/p3/p3.go b/libgo/go/cmd/go/testdata/src/testdep/p3/p3.go deleted file mode 100644 index 0219e7f..0000000 --- a/libgo/go/cmd/go/testdata/src/testdep/p3/p3.go +++ /dev/null @@ -1,3 +0,0 @@ -// +build ignore - -package ignored diff --git a/libgo/go/cmd/go/testdata/src/testnorun/p.go b/libgo/go/cmd/go/testdata/src/testnorun/p.go deleted file mode 100644 index 71a9a56..0000000 --- a/libgo/go/cmd/go/testdata/src/testnorun/p.go +++ /dev/null @@ -1,5 +0,0 @@ -package p - -func init() { - panic("go test must not link and run test binaries without tests") -} diff --git a/libgo/go/cmd/go/testdata/src/testrace/race_test.go b/libgo/go/cmd/go/testdata/src/testrace/race_test.go deleted file mode 100644 index 7ec0c6d..0000000 --- a/libgo/go/cmd/go/testdata/src/testrace/race_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package testrace - -import "testing" - -func TestRace(t *testing.T) { - for i := 0; i < 10; i++ { - c := make(chan int) - x := 1 - go func() { - x = 2 - c <- 1 - }() - x = 3 - <-c - _ = x - } -} - -func BenchmarkRace(b *testing.B) { - for i := 0; i < b.N; i++ { - c := make(chan int) - x := 1 - go func() { - x = 2 - c <- 1 - }() - x = 3 - <-c - _ = x - } -} diff --git a/libgo/go/cmd/go/testdata/src/testregexp/x_test.go b/libgo/go/cmd/go/testdata/src/testregexp/x_test.go deleted file mode 100644 index 7573e79..0000000 --- a/libgo/go/cmd/go/testdata/src/testregexp/x_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package x - -import "testing" - -func TestX(t *testing.T) { - t.Logf("LOG: X running") - t.Run("Y", func(t *testing.T) { - t.Logf("LOG: Y running") - }) -} - -func BenchmarkX(b *testing.B) { - b.Logf("LOG: X running N=%d", b.N) - b.Run("Y", func(b *testing.B) { - b.Logf("LOG: Y running N=%d", b.N) - }) -} diff --git a/libgo/go/cmd/go/testdata/src/testregexp/z_test.go b/libgo/go/cmd/go/testdata/src/testregexp/z_test.go deleted file mode 100644 index 4fd1979..0000000 --- a/libgo/go/cmd/go/testdata/src/testregexp/z_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package x - -import "testing" - -func TestZ(t *testing.T) { - t.Logf("LOG: Z running") -} - -func TestXX(t *testing.T) { - t.Logf("LOG: XX running") -} - -func BenchmarkZ(b *testing.B) { - b.Logf("LOG: Z running N=%d", b.N) -} - -func BenchmarkXX(b *testing.B) { - b.Logf("LOG: XX running N=%d", b.N) -} diff --git a/libgo/go/crypto/tls/tls.go b/libgo/go/crypto/tls/tls.go index 228f4a7..af44485 100644 --- a/libgo/go/crypto/tls/tls.go +++ b/libgo/go/crypto/tls/tls.go @@ -116,9 +116,10 @@ func DialWithDialer(dialer *net.Dialer, network, addr string, config *Config) (* if timeout != 0 { errChannel = make(chan error, 2) - time.AfterFunc(timeout, func() { + timer := time.AfterFunc(timeout, func() { errChannel <- timeoutError{} }) + defer timer.Stop() } rawConn, err := dialer.Dial(network, addr) diff --git a/libgo/go/crypto/x509/root_cgo_darwin.go b/libgo/go/crypto/x509/root_cgo_darwin.go index 255a8d3..8a54282 100644 --- a/libgo/go/crypto/x509/root_cgo_darwin.go +++ b/libgo/go/crypto/x509/root_cgo_darwin.go @@ -159,7 +159,7 @@ static Boolean isRootCertificate(SecCertificateRef cert, CFErrorRef *errRef) { // // Note: The CFDataRef returned in pemRoots and untrustedPemRoots must // be released (using CFRelease) after we've consumed its content. -int CopyPEMRoots(CFDataRef *pemRoots, CFDataRef *untrustedPemRoots, bool debugDarwinRoots) { +static int CopyPEMRoots(CFDataRef *pemRoots, CFDataRef *untrustedPemRoots, bool debugDarwinRoots) { int i; if (debugDarwinRoots) { diff --git a/libgo/go/crypto/x509/root_windows.go b/libgo/go/crypto/x509/root_windows.go index 54ab1dc..34d5853 100644 --- a/libgo/go/crypto/x509/root_windows.go +++ b/libgo/go/crypto/x509/root_windows.go @@ -219,10 +219,26 @@ func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate if err != nil { return nil, err } + if len(chain) < 1 { + return nil, errors.New("x509: internal error: system verifier returned an empty chain") + } - chains = append(chains, chain) + // Mitigate CVE-2020-0601, where the Windows system verifier might be + // tricked into using custom curve parameters for a trusted root, by + // double-checking all ECDSA signatures. If the system was tricked into + // using spoofed parameters, the signature will be invalid for the correct + // ones we parsed. (We don't support custom curves ourselves.) + for i, parent := range chain[1:] { + if parent.PublicKeyAlgorithm != ECDSA { + continue + } + if err := parent.CheckSignature(chain[i].SignatureAlgorithm, + chain[i].RawTBSCertificate, chain[i].Signature); err != nil { + return nil, err + } + } - return chains, nil + return [][]*Certificate{chain}, nil } func loadSystemRoots() (*CertPool, error) { diff --git a/libgo/go/database/sql/sql_test.go b/libgo/go/database/sql/sql_test.go index ed0099e..6f59260 100644 --- a/libgo/go/database/sql/sql_test.go +++ b/libgo/go/database/sql/sql_test.go @@ -629,7 +629,8 @@ func TestPoolExhaustOnCancel(t *testing.T) { go func() { rows, err := db.Query("SELECT|people|name,photo|") if err != nil { - t.Fatalf("Query: %v", err) + t.Errorf("Query: %v", err) + return } rows.Close() saturateDone.Done() @@ -637,6 +638,9 @@ func TestPoolExhaustOnCancel(t *testing.T) { } saturate.Wait() + if t.Failed() { + t.FailNow() + } state = 2 // Now cancel the request while it is waiting. diff --git a/libgo/go/go/build/build_test.go b/libgo/go/go/build/build_test.go index 086a42e..4d11223 100644 --- a/libgo/go/go/build/build_test.go +++ b/libgo/go/go/build/build_test.go @@ -342,20 +342,38 @@ func TestImportDirNotExist(t *testing.T) { {"Import(full, FindOnly)", "go/build/doesnotexist", "", FindOnly}, {"Import(local, FindOnly)", "./doesnotexist", filepath.Join(ctxt.GOROOT, "src/go/build"), FindOnly}, } - for _, test := range tests { - p, err := ctxt.Import(test.path, test.srcDir, test.mode) - if err == nil || !strings.HasPrefix(err.Error(), "cannot find package") { - t.Errorf(`%s got error: %q, want "cannot find package" error`, test.label, err) - } - // If an error occurs, build.Import is documented to return - // a non-nil *Package containing partial information. - if p == nil { - t.Fatalf(`%s got nil p, want non-nil *Package`, test.label) - } - // Verify partial information in p. - if p.ImportPath != "go/build/doesnotexist" { - t.Errorf(`%s got p.ImportPath: %q, want "go/build/doesnotexist"`, test.label, p.ImportPath) - } + + defer os.Setenv("GO111MODULE", os.Getenv("GO111MODULE")) + + for _, GO111MODULE := range []string{"off", "on"} { + t.Run("GO111MODULE="+GO111MODULE, func(t *testing.T) { + os.Setenv("GO111MODULE", GO111MODULE) + + for _, test := range tests { + p, err := ctxt.Import(test.path, test.srcDir, test.mode) + + errOk := (err != nil && strings.HasPrefix(err.Error(), "cannot find package")) + wantErr := `"cannot find package" error` + if test.srcDir == "" { + if err != nil && strings.Contains(err.Error(), "is not in GOROOT") { + errOk = true + } + wantErr = `"cannot find package" or "is not in GOROOT" error` + } + if !errOk { + t.Errorf("%s got error: %q, want %s", test.label, err, wantErr) + } + // If an error occurs, build.Import is documented to return + // a non-nil *Package containing partial information. + if p == nil { + t.Fatalf(`%s got nil p, want non-nil *Package`, test.label) + } + // Verify partial information in p. + if p.ImportPath != "go/build/doesnotexist" { + t.Errorf(`%s got p.ImportPath: %q, want "go/build/doesnotexist"`, test.label, p.ImportPath) + } + } + }) } } diff --git a/libgo/go/go/build/deps_test.go b/libgo/go/go/build/deps_test.go index fd256ee..a64c2b3 100644 --- a/libgo/go/go/build/deps_test.go +++ b/libgo/go/go/build/deps_test.go @@ -168,7 +168,7 @@ var pkgDeps = map[string][]string{ }, "internal/cfg": {"L0"}, - "internal/poll": {"L0", "internal/oserror", "internal/race", "syscall", "time", "unicode/utf16", "unicode/utf8", "internal/syscall/windows"}, + "internal/poll": {"L0", "internal/oserror", "internal/race", "syscall", "time", "unicode/utf16", "unicode/utf8", "internal/syscall/windows", "internal/syscall/unix"}, "internal/testlog": {"L0"}, "os": {"L1", "os", "syscall", "time", "internal/oserror", "internal/poll", "internal/syscall/windows", "internal/syscall/unix", "internal/testlog"}, "path/filepath": {"L2", "os", "syscall", "internal/syscall/windows"}, diff --git a/libgo/go/go/doc/example.go b/libgo/go/go/doc/example.go index 868db8a..a010d3a 100644 --- a/libgo/go/go/doc/example.go +++ b/libgo/go/go/doc/example.go @@ -62,7 +62,7 @@ func Examples(testFiles ...*ast.File) []*Example { if !ok || f.Recv != nil { continue } - if params := f.Type.Params; params.List != nil { + if params := f.Type.Params; len(params.List) != 0 { continue // function has params; not a valid example } numDecl++ diff --git a/libgo/go/go/types/builtins.go b/libgo/go/go/types/builtins.go index af374b7..3756303 100644 --- a/libgo/go/go/types/builtins.go +++ b/libgo/go/go/types/builtins.go @@ -559,7 +559,7 @@ func (check *Checker) builtin(x *operand, call *ast.CallExpr, id builtinId) (_ b base := derefStructPtr(x.typ) sel := selx.Sel.Name - obj, index, indirect := check.LookupFieldOrMethod(base, false, check.pkg, sel) + obj, index, indirect := check.lookupFieldOrMethod(base, false, check.pkg, sel) switch obj.(type) { case nil: check.invalidArg(x.pos(), "%s has no single field %s", base, sel) diff --git a/libgo/go/go/types/call.go b/libgo/go/go/types/call.go index 31f9372..689ef87 100644 --- a/libgo/go/go/types/call.go +++ b/libgo/go/go/types/call.go @@ -370,7 +370,7 @@ func (check *Checker) selector(x *operand, e *ast.SelectorExpr) { goto Error } - obj, index, indirect = check.LookupFieldOrMethod(x.typ, x.mode == variable, check.pkg, sel) + obj, index, indirect = check.lookupFieldOrMethod(x.typ, x.mode == variable, check.pkg, sel) if obj == nil { switch { case index != nil: diff --git a/libgo/go/go/types/lookup.go b/libgo/go/go/types/lookup.go index 648e100..342c8ba 100644 --- a/libgo/go/go/types/lookup.go +++ b/libgo/go/go/types/lookup.go @@ -33,19 +33,19 @@ package types // the method's formal receiver base type, nor was the receiver addressable. // func LookupFieldOrMethod(T Type, addressable bool, pkg *Package, name string) (obj Object, index []int, indirect bool) { - return (*Checker)(nil).LookupFieldOrMethod(T, addressable, pkg, name) + return (*Checker)(nil).lookupFieldOrMethod(T, addressable, pkg, name) } -// Internal use of Checker.LookupFieldOrMethod: If the obj result is a method +// Internal use of Checker.lookupFieldOrMethod: If the obj result is a method // associated with a concrete (non-interface) type, the method's signature // may not be fully set up. Call Checker.objDecl(obj, nil) before accessing // the method's type. // TODO(gri) Now that we provide the *Checker, we can probably remove this -// caveat by calling Checker.objDecl from LookupFieldOrMethod. Investigate. +// caveat by calling Checker.objDecl from lookupFieldOrMethod. Investigate. -// LookupFieldOrMethod is like the external version but completes interfaces +// lookupFieldOrMethod is like the external version but completes interfaces // as necessary. -func (check *Checker) LookupFieldOrMethod(T Type, addressable bool, pkg *Package, name string) (obj Object, index []int, indirect bool) { +func (check *Checker) lookupFieldOrMethod(T Type, addressable bool, pkg *Package, name string) (obj Object, index []int, indirect bool) { // Methods cannot be associated to a named pointer type // (spec: "The type denoted by T is called the receiver base type; // it must not be a pointer or interface type and it must be declared @@ -55,7 +55,7 @@ func (check *Checker) LookupFieldOrMethod(T Type, addressable bool, pkg *Package // not have found it for T (see also issue 8590). if t, _ := T.(*Named); t != nil { if p, _ := t.underlying.(*Pointer); p != nil { - obj, index, indirect = check.lookupFieldOrMethod(p, false, pkg, name) + obj, index, indirect = check.rawLookupFieldOrMethod(p, false, pkg, name) if _, ok := obj.(*Func); ok { return nil, nil, false } @@ -63,7 +63,7 @@ func (check *Checker) LookupFieldOrMethod(T Type, addressable bool, pkg *Package } } - return check.lookupFieldOrMethod(T, addressable, pkg, name) + return check.rawLookupFieldOrMethod(T, addressable, pkg, name) } // TODO(gri) The named type consolidation and seen maps below must be @@ -71,8 +71,8 @@ func (check *Checker) LookupFieldOrMethod(T Type, addressable bool, pkg *Package // types always have only one representation (even when imported // indirectly via different packages.) -// lookupFieldOrMethod should only be called by LookupFieldOrMethod and missingMethod. -func (check *Checker) lookupFieldOrMethod(T Type, addressable bool, pkg *Package, name string) (obj Object, index []int, indirect bool) { +// rawLookupFieldOrMethod should only be called by lookupFieldOrMethod and missingMethod. +func (check *Checker) rawLookupFieldOrMethod(T Type, addressable bool, pkg *Package, name string) (obj Object, index []int, indirect bool) { // WARNING: The code in this function is extremely subtle - do not modify casually! // This function and NewMethodSet should be kept in sync. @@ -297,7 +297,7 @@ func (check *Checker) missingMethod(V Type, T *Interface, static bool) (method * // A concrete type implements T if it implements all methods of T. for _, m := range T.allMethods { - obj, _, _ := check.lookupFieldOrMethod(V, false, m.pkg, m.name) + obj, _, _ := check.rawLookupFieldOrMethod(V, false, m.pkg, m.name) // we must have a method (not a field of matching function type) f, _ := obj.(*Func) diff --git a/libgo/go/golang.org/x/crypto/cryptobyte/asn1.go b/libgo/go/golang.org/x/crypto/cryptobyte/asn1.go index 528b9bf..f930f7e 100644 --- a/libgo/go/golang.org/x/crypto/cryptobyte/asn1.go +++ b/libgo/go/golang.org/x/crypto/cryptobyte/asn1.go @@ -470,7 +470,8 @@ func (s *String) ReadASN1GeneralizedTime(out *time.Time) bool { // It reports whether the read was successful. func (s *String) ReadASN1BitString(out *encoding_asn1.BitString) bool { var bytes String - if !s.ReadASN1(&bytes, asn1.BIT_STRING) || len(bytes) == 0 { + if !s.ReadASN1(&bytes, asn1.BIT_STRING) || len(bytes) == 0 || + len(bytes)*8/8 != len(bytes) { return false } @@ -740,7 +741,7 @@ func (s *String) readASN1(out *String, outTag *asn1.Tag, skipHeader bool) bool { length = headerLen + len32 } - if uint32(int(length)) != length || !s.ReadBytes((*[]byte)(out), int(length)) { + if int(length) < 0 || !s.ReadBytes((*[]byte)(out), int(length)) { return false } if skipHeader && !out.Skip(int(headerLen)) { diff --git a/libgo/go/golang.org/x/crypto/cryptobyte/string.go b/libgo/go/golang.org/x/crypto/cryptobyte/string.go index 39bf98a..589d297 100644 --- a/libgo/go/golang.org/x/crypto/cryptobyte/string.go +++ b/libgo/go/golang.org/x/crypto/cryptobyte/string.go @@ -24,7 +24,7 @@ type String []byte // read advances a String by n bytes and returns them. If less than n bytes // remain, it returns nil. func (s *String) read(n int) []byte { - if len(*s) < n { + if len(*s) < n || n < 0 { return nil } v := (*s)[:n] @@ -105,11 +105,6 @@ func (s *String) readLengthPrefixed(lenLen int, outChild *String) bool { length = length << 8 length = length | uint32(b) } - if int(length) < 0 { - // This currently cannot overflow because we read uint24 at most, but check - // anyway in case that changes in the future. - return false - } v := s.read(int(length)) if v == nil { return false diff --git a/libgo/go/golang.org/x/crypto/poly1305/sum_noasm.go b/libgo/go/golang.org/x/crypto/poly1305/sum_noasm.go index 1682eda45..32a9cef 100644 --- a/libgo/go/golang.org/x/crypto/poly1305/sum_noasm.go +++ b/libgo/go/golang.org/x/crypto/poly1305/sum_noasm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build s390x,!go1.11 !arm,!amd64,!s390x,!ppc64le gccgo appengine nacl +// +build s390x,!go1.11 !amd64,!s390x,!ppc64le gccgo appengine nacl package poly1305 diff --git a/libgo/go/golang.org/x/mod/sumdb/note/note.go b/libgo/go/golang.org/x/mod/sumdb/note/note.go index 3c8e67b..467d25e 100644 --- a/libgo/go/golang.org/x/mod/sumdb/note/note.go +++ b/libgo/go/golang.org/x/mod/sumdb/note/note.go @@ -4,9 +4,6 @@ // Package note defines the notes signed by the Go module database server. // -// This package is part of a DRAFT of what the Go module database server will look like. -// Do not assume the details here are final! -// // A note is text signed by one or more server keys. // The text should be ignored unless the note is signed by // a trusted server key and the signature has been verified diff --git a/libgo/go/golang.org/x/sys/cpu/cpu_riscv64.go b/libgo/go/golang.org/x/sys/cpu/cpu_riscv64.go new file mode 100644 index 0000000..18c8a48 --- /dev/null +++ b/libgo/go/golang.org/x/sys/cpu/cpu_riscv64.go @@ -0,0 +1,7 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build riscv64 + +package cpu diff --git a/libgo/go/golang.org/x/tools/go/analysis/passes/asmdecl/asmdecl.go b/libgo/go/golang.org/x/tools/go/analysis/passes/asmdecl/asmdecl.go index 5c93a4f..e6bfe71 100644 --- a/libgo/go/golang.org/x/tools/go/analysis/passes/asmdecl/asmdecl.go +++ b/libgo/go/golang.org/x/tools/go/analysis/passes/asmdecl/asmdecl.go @@ -87,6 +87,7 @@ var ( asmArchMips64LE = asmArch{name: "mips64le", bigEndian: false, stack: "R29", lr: true} asmArchPpc64 = asmArch{name: "ppc64", bigEndian: true, stack: "R1", lr: true} asmArchPpc64LE = asmArch{name: "ppc64le", bigEndian: false, stack: "R1", lr: true} + asmArchRISCV64 = asmArch{name: "riscv64", bigEndian: false, stack: "SP", lr: true} asmArchS390X = asmArch{name: "s390x", bigEndian: true, stack: "R15", lr: true} asmArchWasm = asmArch{name: "wasm", bigEndian: false, stack: "SP", lr: false} @@ -101,6 +102,7 @@ var ( &asmArchMips64LE, &asmArchPpc64, &asmArchPpc64LE, + &asmArchRISCV64, &asmArchS390X, &asmArchWasm, } diff --git a/libgo/go/html/escape.go b/libgo/go/html/escape.go index dae404f..1dc1287 100644 --- a/libgo/go/html/escape.go +++ b/libgo/go/html/escape.go @@ -12,7 +12,7 @@ import ( // These replacements permit compatibility with old numeric entities that // assumed Windows-1252 encoding. -// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference +// https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state var replacementTable = [...]rune{ '\u20AC', // First entry is what 0x80 should be replaced with. '\u0081', diff --git a/libgo/go/internal/poll/fcntl_js.go b/libgo/go/internal/poll/fcntl_js.go new file mode 100644 index 0000000..120fc11 --- /dev/null +++ b/libgo/go/internal/poll/fcntl_js.go @@ -0,0 +1,14 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build js,wasm + +package poll + +import "syscall" + +// fcntl not supported on js/wasm +func fcntl(fd int, cmd int, arg int) (int, error) { + return 0, syscall.ENOSYS +} diff --git a/libgo/go/internal/poll/fcntl_libc.go b/libgo/go/internal/poll/fcntl_libc.go new file mode 100644 index 0000000..ed00f76 --- /dev/null +++ b/libgo/go/internal/poll/fcntl_libc.go @@ -0,0 +1,26 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build aix darwin solaris + +package poll + +import ( + "syscall" +) + +// Use a helper function to call fcntl. This is defined in C in +// libgo/runtime. +//extern __go_fcntl_uintptr +func libc_fcntl(uintptr, uintptr, uintptr) (uintptr, uintptr) + +func fcntl(fd int, cmd int, arg int) (int, error) { + syscall.Entersyscall() + r, e := libc_fcntl(uintptr(fd), uintptr(cmd), uintptr(arg)) + syscall.Exitsyscall() + if e != 0 { + return int(r), syscall.Errno(e) + } + return int(r), nil +} diff --git a/libgo/go/internal/poll/fcntl_syscall.go b/libgo/go/internal/poll/fcntl_syscall.go new file mode 100644 index 0000000..d232e51 --- /dev/null +++ b/libgo/go/internal/poll/fcntl_syscall.go @@ -0,0 +1,26 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build dragonfly freebsd linux netbsd openbsd + +package poll + +import ( + "syscall" +) + +// Use a helper function to call fcntl. This is defined in C in +// libgo/runtime. +//extern __go_fcntl_uintptr +func libc_fcntl(uintptr, uintptr, uintptr) (uintptr, uintptr) + +func fcntl(fd int, cmd int, arg int) (int, error) { + syscall.Entersyscall() + r, e := libc_fcntl(uintptr(fd), uintptr(cmd), uintptr(arg)) + syscall.Exitsyscall() + if e != 0 { + return int(r), syscall.Errno(e) + } + return int(r), nil +} diff --git a/libgo/go/internal/poll/fd_fsync_darwin.go b/libgo/go/internal/poll/fd_fsync_darwin.go index 6cd3f91..9175149 100644 --- a/libgo/go/internal/poll/fd_fsync_darwin.go +++ b/libgo/go/internal/poll/fd_fsync_darwin.go @@ -4,10 +4,7 @@ package poll -import ( - "syscall" - _ "unsafe" // for go:linkname -) +import "syscall" // Fsync invokes SYS_FCNTL with SYS_FULLFSYNC because // on OS X, SYS_FSYNC doesn't fully flush contents to disk. @@ -21,18 +18,3 @@ func (fd *FD) Fsync() error { _, e1 := fcntl(fd.Sysfd, syscall.F_FULLFSYNC, 0) return e1 } - -// Use a helper function to call fcntl. This is defined in C in -// libgo/runtime. -//extern __go_fcntl_uintptr -func libc_fcntl(uintptr, uintptr, uintptr) (uintptr, uintptr) - -func fcntl(fd int, cmd int, arg int) (int, error) { - syscall.Entersyscall() - r, e := libc_fcntl(uintptr(fd), uintptr(cmd), uintptr(arg)) - syscall.Exitsyscall() - if e != 0 { - return int(r), syscall.Errno(e) - } - return int(r), nil -} diff --git a/libgo/go/internal/poll/fd_fsync_posix.go b/libgo/go/internal/poll/fd_fsync_posix.go index 67b76f8..dfc8b77 100644 --- a/libgo/go/internal/poll/fd_fsync_posix.go +++ b/libgo/go/internal/poll/fd_fsync_posix.go @@ -8,11 +8,6 @@ package poll import "syscall" -// Use a helper function to call fcntl. This is defined in C in -// libgo/runtime. -//extern __go_fcntl_uintptr -func libc_fcntl(uintptr, uintptr, uintptr) (uintptr, uintptr) - // Fsync wraps syscall.Fsync. func (fd *FD) Fsync() error { if err := fd.incref(); err != nil { @@ -21,13 +16,3 @@ func (fd *FD) Fsync() error { defer fd.decref() return syscall.Fsync(fd.Sysfd) } - -func fcntl(fd int, cmd int, arg int) (int, error) { - syscall.Entersyscall() - r, e := libc_fcntl(uintptr(fd), uintptr(cmd), uintptr(arg)) - syscall.Exitsyscall() - if e != 0 { - return int(r), syscall.Errno(e) - } - return int(r), nil -} diff --git a/libgo/go/internal/poll/fd_unix.go b/libgo/go/internal/poll/fd_unix.go index 6b8e476..213e815 100644 --- a/libgo/go/internal/poll/fd_unix.go +++ b/libgo/go/internal/poll/fd_unix.go @@ -451,7 +451,7 @@ var tryDupCloexec = int32(1) // DupCloseOnExec dups fd and marks it close-on-exec. func DupCloseOnExec(fd int) (int, string, error) { - if atomic.LoadInt32(&tryDupCloexec) == 1 { + if syscall.F_DUPFD_CLOEXEC != 0 && atomic.LoadInt32(&tryDupCloexec) == 1 { r0, e1 := fcntl(fd, syscall.F_DUPFD_CLOEXEC, 0) if e1 == nil { return r0, "", nil diff --git a/libgo/go/internal/syscall/unix/nonblocking.go b/libgo/go/internal/syscall/unix/nonblocking.go index cff5a53..9b39bb2 100644 --- a/libgo/go/internal/syscall/unix/nonblocking.go +++ b/libgo/go/internal/syscall/unix/nonblocking.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix dragonfly freebsd hurd linux netbsd openbsd solaris +// +build dragonfly freebsd linux netbsd openbsd package unix diff --git a/libgo/go/internal/syscall/unix/nonblocking_darwin.go b/libgo/go/internal/syscall/unix/nonblocking_libc.go index e3dd3a0..464314d 100644 --- a/libgo/go/internal/syscall/unix/nonblocking_darwin.go +++ b/libgo/go/internal/syscall/unix/nonblocking_libc.go @@ -2,23 +2,19 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin +// +build aix darwin solaris package unix -import ( - "syscall" - _ "unsafe" // for go:linkname -) +import "syscall" + +//extern __go_fcntl_uintptr +func fcntl(uintptr, uintptr, uintptr) (uintptr, uintptr) func IsNonblock(fd int) (nonblocking bool, err error) { - flag, e1 := fcntl(fd, syscall.F_GETFL, 0) - if e1 != nil { - return false, e1 + flag, e1 := fcntl(uintptr(fd), syscall.F_GETFL, 0) + if e1 != 0 { + return false, syscall.Errno(e1) } return flag&syscall.O_NONBLOCK != 0, nil } - -// Implemented in syscall/syscall_darwin.go. -//go:linkname fcntl syscall.fcntl -func fcntl(fd int, cmd int, arg int) (int, error) diff --git a/libgo/go/io/example_test.go b/libgo/go/io/example_test.go index edcd008..2eaab67 100644 --- a/libgo/go/io/example_test.go +++ b/libgo/go/io/example_test.go @@ -59,7 +59,7 @@ func ExampleCopyN() { func ExampleReadAtLeast() { r := strings.NewReader("some io.Reader stream to be read\n") - buf := make([]byte, 33) + buf := make([]byte, 14) if _, err := io.ReadAtLeast(r, buf, 4); err != nil { log.Fatal(err) } @@ -78,10 +78,9 @@ func ExampleReadAtLeast() { } // Output: - // some io.Reader stream to be read - // + // some io.Reader // error: short buffer - // error: EOF + // error: unexpected EOF } func ExampleReadFull() { diff --git a/libgo/go/math/big/arith_decl.go b/libgo/go/math/big/arith_decl.go index 61df0df..0a139f1 100644 --- a/libgo/go/math/big/arith_decl.go +++ b/libgo/go/math/big/arith_decl.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // +build ignore -// +build !math_big_pure_go +// +build !math_big_pure_go,!riscv64 package big diff --git a/libgo/go/math/big/arith_decl_pure.go b/libgo/go/math/big/arith_decl_pure.go index ee8f922..8853eb6 100644 --- a/libgo/go/math/big/arith_decl_pure.go +++ b/libgo/go/math/big/arith_decl_pure.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// -build math_big_pure_go +// -build math_big_pure_go riscv64 package big diff --git a/libgo/go/math/big/int.go b/libgo/go/math/big/int.go index bf1fa73..bec0a81 100644 --- a/libgo/go/math/big/int.go +++ b/libgo/go/math/big/int.go @@ -504,9 +504,14 @@ func (z *Int) Exp(x, y, m *Int) *Int { // GCD sets z to the greatest common divisor of a and b and returns z. // If x or y are not nil, GCD sets their value such that z = a*x + b*y. +// +// a and b may be positive, zero or negative. // Regardless of the signs of a and b, z is always >= 0. +// // If a == b == 0, GCD sets z = x = y = 0. +// // If a == 0 and b != 0, GCD sets z = |b|, x = 0, y = sign(b) * 1. +// // If a != 0 and b == 0, GCD sets z = |a|, x = sign(a) * 1, y = 0. func (z *Int) GCD(x, y, a, b *Int) *Int { if len(a.abs) == 0 || len(b.abs) == 0 { diff --git a/libgo/go/net/dial_test.go b/libgo/go/net/dial_test.go index ae40079..493cdfc 100644 --- a/libgo/go/net/dial_test.go +++ b/libgo/go/net/dial_test.go @@ -174,7 +174,7 @@ func dialClosedPort(t *testing.T) (actual, expected time.Duration) { } addr := l.Addr().String() l.Close() - // On OpenBSD, interference from TestSelfConnect is mysteriously + // On OpenBSD, interference from TestTCPSelfConnect is mysteriously // causing the first attempt to hang for a few seconds, so we throw // away the first result and keep the second. for i := 1; ; i++ { diff --git a/libgo/go/net/dnsclient_unix_test.go b/libgo/go/net/dnsclient_unix_test.go index 6d72817..e8f81e8 100644 --- a/libgo/go/net/dnsclient_unix_test.go +++ b/libgo/go/net/dnsclient_unix_test.go @@ -173,7 +173,7 @@ func TestAvoidDNSName(t *testing.T) { // Without stuff before onion/local, they're fine to // use DNS. With a search path, - // "onion.vegegtables.com" can use DNS. Without a + // "onion.vegetables.com" can use DNS. Without a // search path (or with a trailing dot), the queries // are just kinda useless, but don't reveal anything // private. diff --git a/libgo/go/net/http/cgi/matryoshka_test.go b/libgo/go/net/http/cgi/integration_test.go index 32d59c0..32d59c0 100644 --- a/libgo/go/net/http/cgi/matryoshka_test.go +++ b/libgo/go/net/http/cgi/integration_test.go diff --git a/libgo/go/net/http/client.go b/libgo/go/net/http/client.go index 6a8c59a..a496f1c 100644 --- a/libgo/go/net/http/client.go +++ b/libgo/go/net/http/client.go @@ -288,10 +288,17 @@ func timeBeforeContextDeadline(t time.Time, ctx context.Context) bool { // knownRoundTripperImpl reports whether rt is a RoundTripper that's // maintained by the Go team and known to implement the latest -// optional semantics (notably contexts). -func knownRoundTripperImpl(rt RoundTripper) bool { - switch rt.(type) { - case *Transport, *http2Transport: +// optional semantics (notably contexts). The Request is used +// to check whether this particular request is using an alternate protocol, +// in which case we need to check the RoundTripper for that protocol. +func knownRoundTripperImpl(rt RoundTripper, req *Request) bool { + switch t := rt.(type) { + case *Transport: + if altRT := t.alternateRoundTripper(req); altRT != nil { + return knownRoundTripperImpl(altRT, req) + } + return true + case *http2Transport, http2noDialH2RoundTripper: return true } // There's a very minor chance of a false positive with this. @@ -319,7 +326,7 @@ func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTi if deadline.IsZero() { return nop, alwaysFalse } - knownTransport := knownRoundTripperImpl(rt) + knownTransport := knownRoundTripperImpl(rt, req) oldCtx := req.Context() if req.Cancel == nil && knownTransport { diff --git a/libgo/go/net/http/httputil/reverseproxy.go b/libgo/go/net/http/httputil/reverseproxy.go index e8f7df2..4d6a085 100644 --- a/libgo/go/net/http/httputil/reverseproxy.go +++ b/libgo/go/net/http/httputil/reverseproxy.go @@ -24,6 +24,14 @@ import ( // ReverseProxy is an HTTP Handler that takes an incoming request and // sends it to another server, proxying the response back to the // client. +// +// ReverseProxy automatically sets the client IP as the value of the +// X-Forwarded-For header. +// If an X-Forwarded-For header already exists, the client IP is +// appended to the existing values. +// To prevent IP spoofing, be sure to delete any pre-existing +// X-Forwarded-For header coming from the client or +// an untrusted proxy. type ReverseProxy struct { // Director must be a function which modifies // the request into a new request to be sent diff --git a/libgo/go/net/http/omithttp2.go b/libgo/go/net/http/omithttp2.go index a0b33e9..307d93a 100644 --- a/libgo/go/net/http/omithttp2.go +++ b/libgo/go/net/http/omithttp2.go @@ -36,6 +36,10 @@ type http2erringRoundTripper struct{} func (http2erringRoundTripper) RoundTrip(*Request) (*Response, error) { panic(noHTTP2) } +type http2noDialH2RoundTripper struct{} + +func (http2noDialH2RoundTripper) RoundTrip(*Request) (*Response, error) { panic(noHTTP2) } + type http2noDialClientConnPool struct { http2clientConnPool http2clientConnPool } diff --git a/libgo/go/net/http/request.go b/libgo/go/net/http/request.go index 8dd9fe1..88fa093 100644 --- a/libgo/go/net/http/request.go +++ b/libgo/go/net/http/request.go @@ -1223,17 +1223,17 @@ func parsePostForm(r *Request) (vs url.Values, err error) { // For all requests, ParseForm parses the raw query from the URL and updates // r.Form. // -// For POST, PUT, and PATCH requests, it also parses the request body as a form -// and puts the results into both r.PostForm and r.Form. Request body parameters -// take precedence over URL query string values in r.Form. +// For POST, PUT, and PATCH requests, it also reads the request body, parses it +// as a form and puts the results into both r.PostForm and r.Form. Request body +// parameters take precedence over URL query string values in r.Form. +// +// If the request Body's size has not already been limited by MaxBytesReader, +// the size is capped at 10MB. // // For other HTTP methods, or when the Content-Type is not // application/x-www-form-urlencoded, the request Body is not read, and // r.PostForm is initialized to a non-nil, empty value. // -// If the request Body's size has not already been limited by MaxBytesReader, -// the size is capped at 10MB. -// // ParseMultipartForm calls ParseForm automatically. // ParseForm is idempotent. func (r *Request) ParseForm() error { diff --git a/libgo/go/net/http/transfer.go b/libgo/go/net/http/transfer.go index 1d6a987..2e01a07 100644 --- a/libgo/go/net/http/transfer.go +++ b/libgo/go/net/http/transfer.go @@ -7,7 +7,6 @@ package http import ( "bufio" "bytes" - "compress/gzip" "errors" "fmt" "io" @@ -467,34 +466,6 @@ func suppressedHeaders(status int) []string { return nil } -// proxyingReadCloser is a composite type that accepts and proxies -// io.Read and io.Close calls to its respective Reader and Closer. -// -// It is composed of: -// a) a top-level reader e.g. the result of decompression -// b) a symbolic Closer e.g. the result of decompression, the -// original body and the connection itself. -type proxyingReadCloser struct { - io.Reader - io.Closer -} - -// multiCloser implements io.Closer and allows a bunch of io.Closer values -// to all be closed once. -// Example usage is with proxyingReadCloser if we are decompressing a response -// body on the fly and would like to close both *gzip.Reader and underlying body. -type multiCloser []io.Closer - -func (mc multiCloser) Close() error { - var err error - for _, c := range mc { - if err1 := c.Close(); err1 != nil && err == nil { - err = err1 - } - } - return err -} - // msg is *Request or *Response. func readTransfer(msg interface{}, r *bufio.Reader) (err error) { t := &transferReader{RequestMethod: "GET"} @@ -572,7 +543,7 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { // Prepare body reader. ContentLength < 0 means chunked encoding // or close connection when finished, since multipart is not supported yet switch { - case chunked(t.TransferEncoding) || implicitlyChunked(t.TransferEncoding): + case chunked(t.TransferEncoding): if noResponseBodyExpected(t.RequestMethod) || !bodyAllowedForStatus(t.StatusCode) { t.Body = NoBody } else { @@ -593,21 +564,6 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { } } - // Finally if "gzip" was one of the requested transfer-encodings, - // we'll unzip the concatenated body/payload of the request. - // TODO: As we support more transfer-encodings, extract - // this code and apply the un-codings in reverse. - if t.Body != NoBody && gzipped(t.TransferEncoding) { - zr, err := gzip.NewReader(t.Body) - if err != nil { - return fmt.Errorf("http: failed to gunzip body: %v", err) - } - t.Body = &proxyingReadCloser{ - Reader: zr, - Closer: multiCloser{zr, t.Body}, - } - } - // Unify output switch rr := msg.(type) { case *Request: @@ -627,41 +583,8 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { return nil } -// Checks whether chunked is the last part of the encodings stack -func chunked(te []string) bool { return len(te) > 0 && te[len(te)-1] == "chunked" } - -// implicitlyChunked is a helper to check for implicity of chunked, because -// RFC 7230 Section 3.3.1 says that the sender MUST apply chunked as the final -// payload body to ensure that the message is framed for both the request -// and the body. Since "identity" is incompatible with any other transformational -// encoding cannot co-exist, the presence of "identity" will cause implicitlyChunked -// to return false. -func implicitlyChunked(te []string) bool { - if len(te) == 0 { // No transfer-encodings passed in, so not implicitly chunked. - return false - } - for _, tei := range te { - if tei == "identity" { - return false - } - } - return true -} - -func isGzipTransferEncoding(tei string) bool { - // RFC 7230 4.2.3 requests that "x-gzip" SHOULD be considered the same as "gzip". - return tei == "gzip" || tei == "x-gzip" -} - -// Checks where either of "gzip" or "x-gzip" are contained in transfer encodings. -func gzipped(te []string) bool { - for _, tei := range te { - if isGzipTransferEncoding(tei) { - return true - } - } - return false -} +// Checks whether chunked is part of the encodings stack +func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } // Checks whether the encoding is explicitly "identity". func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" } @@ -697,47 +620,25 @@ func (t *transferReader) fixTransferEncoding() error { encodings := strings.Split(raw[0], ",") te := make([]string, 0, len(encodings)) - - // When adding new encodings, please maintain the invariant: - // if chunked encoding is present, it must always - // come last and it must be applied only once. - // See RFC 7230 Section 3.3.1 Transfer-Encoding. - for i, encoding := range encodings { + // TODO: Even though we only support "identity" and "chunked" + // encodings, the loop below is designed with foresight. One + // invariant that must be maintained is that, if present, + // chunked encoding must always come first. + for _, encoding := range encodings { encoding = strings.ToLower(strings.TrimSpace(encoding)) - + // "identity" encoding is not recorded if encoding == "identity" { - // "identity" should not be mixed with other transfer-encodings/compressions - // because it means "no compression, no transformation". - if len(encodings) != 1 { - return &badStringError{`"identity" when present must be the only transfer encoding`, strings.Join(encodings, ",")} - } - // "identity" is not recorded. break } - - switch { - case encoding == "chunked": - // "chunked" MUST ALWAYS be the last - // encoding as per the loop invariant. - // That is: - // Invalid: [chunked, gzip] - // Valid: [gzip, chunked] - if i+1 != len(encodings) { - return &badStringError{"chunked must be applied only once, as the last encoding", strings.Join(encodings, ",")} - } - // Supported otherwise. - - case isGzipTransferEncoding(encoding): - // Supported - - default: + if encoding != "chunked" { return &unsupportedTEError{fmt.Sprintf("unsupported transfer encoding: %q", encoding)} } - te = te[0 : len(te)+1] te[len(te)-1] = encoding } - + if len(te) > 1 { + return &badStringError{"too many transfer encodings", strings.Join(te, ",")} + } if len(te) > 0 { // RFC 7230 3.3.2 says "A sender MUST NOT send a // Content-Length header field in any message that diff --git a/libgo/go/net/http/transfer_test.go b/libgo/go/net/http/transfer_test.go index a8ce2d3..65009ee 100644 --- a/libgo/go/net/http/transfer_test.go +++ b/libgo/go/net/http/transfer_test.go @@ -7,7 +7,6 @@ package http import ( "bufio" "bytes" - "compress/gzip" "crypto/rand" "fmt" "io" @@ -62,6 +61,7 @@ func TestFinalChunkedBodyReadEOF(t *testing.T) { buf := make([]byte, len(want)) n, err := res.Body.Read(buf) if n != len(want) || err != io.EOF { + t.Logf("body = %#v", res.Body) t.Errorf("Read = %v, %v; want %d, EOF", n, err, len(want)) } if string(buf) != want { @@ -290,7 +290,7 @@ func TestFixTransferEncoding(t *testing.T) { }, { hdr: Header{"Transfer-Encoding": {"chunked, chunked", "identity", "chunked"}}, - wantErr: &badStringError{"chunked must be applied only once, as the last encoding", "chunked, chunked"}, + wantErr: &badStringError{"too many transfer encodings", "chunked,chunked"}, }, { hdr: Header{"Transfer-Encoding": {"chunked"}}, @@ -310,283 +310,3 @@ func TestFixTransferEncoding(t *testing.T) { } } } - -func gzipIt(s string) string { - buf := new(bytes.Buffer) - gw := gzip.NewWriter(buf) - gw.Write([]byte(s)) - gw.Close() - return buf.String() -} - -func TestUnitTestProxyingReadCloserClosesBody(t *testing.T) { - var checker closeChecker - buf := new(bytes.Buffer) - buf.WriteString("Hello, Gophers!") - prc := &proxyingReadCloser{ - Reader: buf, - Closer: &checker, - } - prc.Close() - - read, err := ioutil.ReadAll(prc) - if err != nil { - t.Fatalf("Read error: %v", err) - } - if g, w := string(read), "Hello, Gophers!"; g != w { - t.Errorf("Read mismatch: got %q want %q", g, w) - } - - if checker.closed != true { - t.Fatal("closeChecker.Close was never invoked") - } -} - -func TestGzipTransferEncoding_request(t *testing.T) { - helloWorldGzipped := gzipIt("Hello, World!") - - tests := []struct { - payload string - wantErr string - wantBody string - }{ - - { - // The case of "chunked" properly applied as the last encoding - // and a gzipped request payload that is streamed in 3 parts. - payload: `POST / HTTP/1.1 -Host: golang.org -Transfer-Encoding: gzip, chunked -Content-Type: text/html; charset=UTF-8 - -` + fmt.Sprintf("%02x\r\n%s\r\n%02x\r\n%s\r\n%02x\r\n%s\r\n0\r\n\r\n", - 3, helloWorldGzipped[:3], - 5, helloWorldGzipped[3:8], - len(helloWorldGzipped)-8, helloWorldGzipped[8:]), - wantBody: `Hello, World!`, - }, - - { - // The request specifies "Transfer-Encoding: chunked" so its body must be left untouched. - payload: `PUT / HTTP/1.1 -Host: golang.org -Transfer-Encoding: chunked -Connection: close -Content-Type: text/html; charset=UTF-8 - -` + fmt.Sprintf("%0x\r\n%s\r\n0\r\n\r\n", len(helloWorldGzipped), helloWorldGzipped), - // We want that payload as it was sent. - wantBody: helloWorldGzipped, - }, - - { - // Valid request, the body doesn't have "Transfer-Encoding: chunked" but implicitly encoded - // for chunking as per the advisory from RFC 7230 3.3.1 which advises for cases where. - payload: `POST / HTTP/1.1 -Host: localhost -Transfer-Encoding: gzip -Content-Type: text/html; charset=UTF-8 - -` + fmt.Sprintf("%0x\r\n%s\r\n0\r\n\r\n", len(helloWorldGzipped), helloWorldGzipped), - wantBody: `Hello, World!`, - }, - - { - // Invalid request, the body isn't chunked nor is the connection terminated immediately - // hence invalid as per the advisory from RFC 7230 3.3.1 which advises for cases where - // a Transfer-Encoding that isn't finally chunked is provided. - payload: `PUT / HTTP/1.1 -Host: golang.org -Transfer-Encoding: gzip -Content-Length: 0 -Connection: close -Content-Type: text/html; charset=UTF-8 - -`, - wantErr: `EOF`, - }, - - { - // The case of chunked applied before another encoding. - payload: `PUT / HTTP/1.1 -Location: golang.org -Transfer-Encoding: chunked, gzip -Content-Length: 0 -Connection: close -Content-Type: text/html; charset=UTF-8 - -`, - wantErr: `chunked must be applied only once, as the last encoding "chunked, gzip"`, - }, - - { - // The case of chunked properly applied as the - // last encoding BUT with a bad "Content-Length". - payload: `POST / HTTP/1.1 -Host: golang.org -Transfer-Encoding: gzip, chunked -Content-Length: 10 -Connection: close -Content-Type: text/html; charset=UTF-8 - -` + "0\r\n\r\n", - wantErr: "EOF", - }, - } - - for i, tt := range tests { - req, err := ReadRequest(bufio.NewReader(strings.NewReader(tt.payload))) - if tt.wantErr != "" { - if err == nil || !strings.Contains(err.Error(), tt.wantErr) { - t.Errorf("test %d. Error mismatch\nGot: %v\nWant: %s", i, err, tt.wantErr) - } - continue - } - - if err != nil { - t.Errorf("test %d. Unexpected ReadRequest error: %v\nPayload:\n%s", i, err, tt.payload) - continue - } - - got, err := ioutil.ReadAll(req.Body) - req.Body.Close() - if err != nil { - t.Errorf("test %d. Failed to read response body: %v", i, err) - } - if g, w := string(got), tt.wantBody; g != w { - t.Errorf("test %d. Request body mimsatch\nGot:\n%s\n\nWant:\n%s", i, g, w) - } - } -} - -func TestGzipTransferEncoding_response(t *testing.T) { - helloWorldGzipped := gzipIt("Hello, World!") - - tests := []struct { - payload string - wantErr string - wantBody string - }{ - - { - // The case of "chunked" properly applied as the last encoding - // and a gzipped payload that is streamed in 3 parts. - payload: `HTTP/1.1 302 Found -Location: https://golang.org/ -Transfer-Encoding: gzip, chunked -Connection: close -Content-Type: text/html; charset=UTF-8 - -` + fmt.Sprintf("%02x\r\n%s\r\n%02x\r\n%s\r\n%02x\r\n%s\r\n0\r\n\r\n", - 3, helloWorldGzipped[:3], - 5, helloWorldGzipped[3:8], - len(helloWorldGzipped)-8, helloWorldGzipped[8:]), - wantBody: `Hello, World!`, - }, - - { - // The response specifies "Transfer-Encoding: chunked" so response body must be left untouched. - payload: `HTTP/1.1 302 Found -Location: https://golang.org/ -Transfer-Encoding: chunked -Connection: close -Content-Type: text/html; charset=UTF-8 - -` + fmt.Sprintf("%0x\r\n%s\r\n0\r\n\r\n", len(helloWorldGzipped), helloWorldGzipped), - // We want that payload as it was sent. - wantBody: helloWorldGzipped, - }, - - { - // Valid response, the body doesn't have "Transfer-Encoding: chunked" but implicitly encoded - // for chunking as per the advisory from RFC 7230 3.3.1 which advises for cases where. - payload: `HTTP/1.1 302 Found -Location: https://golang.org/ -Transfer-Encoding: gzip -Connection: close -Content-Type: text/html; charset=UTF-8 - -` + fmt.Sprintf("%0x\r\n%s\r\n0\r\n\r\n", len(helloWorldGzipped), helloWorldGzipped), - wantBody: `Hello, World!`, - }, - - { - // Invalid response, the body isn't chunked nor is the connection terminated immediately - // hence invalid as per the advisory from RFC 7230 3.3.1 which advises for cases where - // a Transfer-Encoding that isn't finally chunked is provided. - payload: `HTTP/1.1 302 Found -Location: https://golang.org/ -Transfer-Encoding: gzip -Content-Length: 0 -Connection: close -Content-Type: text/html; charset=UTF-8 - -`, - wantErr: `EOF`, - }, - - { - // The case of chunked applied before another encoding. - payload: `HTTP/1.1 302 Found -Location: https://golang.org/ -Transfer-Encoding: chunked, gzip -Content-Length: 0 -Connection: close -Content-Type: text/html; charset=UTF-8 - -`, - wantErr: `chunked must be applied only once, as the last encoding "chunked, gzip"`, - }, - - { - // The case of chunked properly applied as the - // last encoding BUT with a bad "Content-Length". - payload: `HTTP/1.1 302 Found -Location: https://golang.org/ -Transfer-Encoding: gzip, chunked -Content-Length: 10 -Connection: close -Content-Type: text/html; charset=UTF-8 - -` + "0\r\n\r\n", - wantErr: "EOF", - }, - - { - // Including "identity" more than once. - payload: `HTTP/1.1 200 OK -Location: https://golang.org/ -Transfer-Encoding: identity, identity -Content-Length: 0 -Connection: close -Content-Type: text/html; charset=UTF-8 - -` + "0\r\n\r\n", - wantErr: `"identity" when present must be the only transfer encoding "identity, identity"`, - }, - } - - for i, tt := range tests { - res, err := ReadResponse(bufio.NewReader(strings.NewReader(tt.payload)), nil) - if tt.wantErr != "" { - if err == nil || !strings.Contains(err.Error(), tt.wantErr) { - t.Errorf("test %d. Error mismatch\nGot: %v\nWant: %s", i, err, tt.wantErr) - } - continue - } - - if err != nil { - t.Errorf("test %d. Unexpected ReadResponse error: %v\nPayload:\n%s", i, err, tt.payload) - continue - } - - got, err := ioutil.ReadAll(res.Body) - res.Body.Close() - if err != nil { - t.Errorf("test %d. Failed to read response body: %v", i, err) - } - if g, w := string(got), tt.wantBody; g != w { - t.Errorf("test %d. Response body mimsatch\nGot:\n%s\n\nWant:\n%s", i, g, w) - } - } -} diff --git a/libgo/go/net/http/transport.go b/libgo/go/net/http/transport.go index 64d8510..d0bfdb4 100644 --- a/libgo/go/net/http/transport.go +++ b/libgo/go/net/http/transport.go @@ -469,6 +469,17 @@ func (t *Transport) useRegisteredProtocol(req *Request) bool { return true } +// alternateRoundTripper returns the alternate RoundTripper to use +// for this request if the Request's URL scheme requires one, +// or nil for the normal case of using the Transport. +func (t *Transport) alternateRoundTripper(req *Request) RoundTripper { + if !t.useRegisteredProtocol(req) { + return nil + } + altProto, _ := t.altProto.Load().(map[string]RoundTripper) + return altProto[req.URL.Scheme] +} + // roundTrip implements a RoundTripper over HTTP. func (t *Transport) roundTrip(req *Request) (*Response, error) { t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) @@ -500,12 +511,9 @@ func (t *Transport) roundTrip(req *Request) (*Response, error) { } } - if t.useRegisteredProtocol(req) { - altProto, _ := t.altProto.Load().(map[string]RoundTripper) - if altRT := altProto[scheme]; altRT != nil { - if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol { - return resp, err - } + if altRT := t.alternateRoundTripper(req); altRT != nil { + if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol { + return resp, err } } if !isHTTP { @@ -1559,15 +1567,16 @@ func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (pconn *pers if hdr == nil { hdr = make(Header) } + if pa := cm.proxyAuth(); pa != "" { + hdr = hdr.Clone() + hdr.Set("Proxy-Authorization", pa) + } connectReq := &Request{ Method: "CONNECT", URL: &url.URL{Opaque: cm.targetAddr}, Host: cm.targetAddr, Header: hdr, } - if pa := cm.proxyAuth(); pa != "" { - connectReq.Header.Set("Proxy-Authorization", pa) - } // If there's no done channel (no deadline or cancellation // from the caller possible), at least set some (long) diff --git a/libgo/go/net/http/transport_test.go b/libgo/go/net/http/transport_test.go index 2256813..1e0334d 100644 --- a/libgo/go/net/http/transport_test.go +++ b/libgo/go/net/http/transport_test.go @@ -1550,6 +1550,44 @@ func TestTransportDialPreservesNetOpProxyError(t *testing.T) { } } +// Issue 36431: calls to RoundTrip should not mutate t.ProxyConnectHeader. +// +// (A bug caused dialConn to instead write the per-request Proxy-Authorization +// header through to the shared Header instance, introducing a data race.) +func TestTransportProxyDialDoesNotMutateProxyConnectHeader(t *testing.T) { + setParallel(t) + defer afterTest(t) + + proxy := httptest.NewTLSServer(NotFoundHandler()) + defer proxy.Close() + c := proxy.Client() + + tr := c.Transport.(*Transport) + tr.Proxy = func(*Request) (*url.URL, error) { + u, _ := url.Parse(proxy.URL) + u.User = url.UserPassword("aladdin", "opensesame") + return u, nil + } + h := tr.ProxyConnectHeader + if h == nil { + h = make(Header) + } + tr.ProxyConnectHeader = h.Clone() + + req, err := NewRequest("GET", "https://golang.fake.tld/", nil) + if err != nil { + t.Fatal(err) + } + _, err = c.Do(req) + if err == nil { + t.Errorf("unexpected Get success") + } + + if !reflect.DeepEqual(tr.ProxyConnectHeader, h) { + t.Errorf("tr.ProxyConnectHeader = %v; want %v", tr.ProxyConnectHeader, h) + } +} + // TestTransportGzipRecursive sends a gzip quine and checks that the // client gets the same value back. This is more cute than anything, // but checks that we don't recurse forever, and checks that @@ -6109,3 +6147,35 @@ func TestTransportDecrementConnWhenIdleConnRemoved(t *testing.T) { t.Errorf("error occurred: %v", err) } } + +// Issue 36820 +// Test that we use the older backward compatible cancellation protocol +// when a RoundTripper is registered via RegisterProtocol. +func TestAltProtoCancellation(t *testing.T) { + defer afterTest(t) + tr := &Transport{} + c := &Client{ + Transport: tr, + Timeout: time.Millisecond, + } + tr.RegisterProtocol("timeout", timeoutProto{}) + _, err := c.Get("timeout://bar.com/path") + if err == nil { + t.Error("request unexpectedly succeeded") + } else if !strings.Contains(err.Error(), timeoutProtoErr.Error()) { + t.Errorf("got error %q, does not contain expected string %q", err, timeoutProtoErr) + } +} + +var timeoutProtoErr = errors.New("canceled as expected") + +type timeoutProto struct{} + +func (timeoutProto) RoundTrip(req *Request) (*Response, error) { + select { + case <-req.Cancel: + return nil, timeoutProtoErr + case <-time.After(5 * time.Second): + return nil, errors.New("request was not canceled") + } +} diff --git a/libgo/go/net/lookup_test.go b/libgo/go/net/lookup_test.go index 8a41510..2bc5592 100644 --- a/libgo/go/net/lookup_test.go +++ b/libgo/go/net/lookup_test.go @@ -998,12 +998,16 @@ func TestConcurrentPreferGoResolversDial(t *testing.T) { defer wg.Done() _, err := r.LookupIPAddr(context.Background(), "google.com") if err != nil { - t.Fatalf("lookup failed for resolver %d: %q", index, err) + t.Errorf("lookup failed for resolver %d: %q", index, err) } }(resolver.Resolver, i) } wg.Wait() + if t.Failed() { + t.FailNow() + } + for i, resolver := range resolvers { if !resolver.dialed { t.Errorf("custom resolver %d not dialed during lookup", i) @@ -1175,12 +1179,9 @@ func TestWithUnexpiredValuesPreserved(t *testing.T) { } } -// Issue 31586: don't crash on null byte in name +// Issue 31597: don't panic on null byte in name func TestLookupNullByte(t *testing.T) { testenv.MustHaveExternalNetwork(t) testenv.SkipFlakyNet(t) - _, err := LookupHost("foo\x00bar") // used to crash on Windows - if err == nil { - t.Errorf("unexpected success") - } + LookupHost("foo\x00bar") // check that it doesn't panic; it used to on Windows } diff --git a/libgo/go/net/net.go b/libgo/go/net/net.go index 38c6b99..1d7e5e7 100644 --- a/libgo/go/net/net.go +++ b/libgo/go/net/net.go @@ -452,6 +452,7 @@ type OpError struct { Addr Addr // Err is the error that occurred during the operation. + // The Error method panics if the error is nil. Err error } diff --git a/libgo/go/os/file.go b/libgo/go/os/file.go index 7995de7..9f8c827 100644 --- a/libgo/go/os/file.go +++ b/libgo/go/os/file.go @@ -204,6 +204,10 @@ func (f *File) WriteAt(b []byte, off int64) (n int, err error) { // relative to the current offset, and 2 means relative to the end. // It returns the new offset and an error, if any. // The behavior of Seek on a file opened with O_APPEND is not specified. +// +// If f is a directory, the behavior of Seek varies by operating +// system; you can seek to the beginning of the directory on Unix-like +// operating systems, but not on Windows. func (f *File) Seek(offset int64, whence int) (ret int64, err error) { if err := f.checkValid("seek"); err != nil { return 0, err diff --git a/libgo/go/reflect/all_test.go b/libgo/go/reflect/all_test.go index c9ed0a9..33d02ae 100644 --- a/libgo/go/reflect/all_test.go +++ b/libgo/go/reflect/all_test.go @@ -4864,6 +4864,9 @@ func TestStructOfExportRules(t *testing.T) { if exported != test.exported { t.Errorf("test-%d: got exported=%v want exported=%v", i, exported, test.exported) } + if field.PkgPath != test.field.PkgPath { + t.Errorf("test-%d: got PkgPath=%q want pkgPath=%q", i, field.PkgPath, test.field.PkgPath) + } }) } } @@ -5327,6 +5330,24 @@ func TestStructOfTooManyFields(t *testing.T) { } } +func TestStructOfDifferentPkgPath(t *testing.T) { + fields := []StructField{ + { + Name: "f1", + PkgPath: "p1", + Type: TypeOf(int(0)), + }, + { + Name: "f2", + PkgPath: "p2", + Type: TypeOf(int(0)), + }, + } + shouldPanic(func() { + StructOf(fields) + }) +} + func TestChanOf(t *testing.T) { // check construction and use of type not in binary type T string diff --git a/libgo/go/reflect/type.go b/libgo/go/reflect/type.go index a350674..9c003a4 100644 --- a/libgo/go/reflect/type.go +++ b/libgo/go/reflect/type.go @@ -1993,6 +1993,7 @@ func StructOf(fields []StructField) Type { lastzero := uintptr(0) repr = append(repr, "struct {"...) + pkgpath := "" for i, field := range fields { if field.Name == "" { panic("reflect.StructOf: field " + strconv.Itoa(i) + " has no name") @@ -2003,11 +2004,18 @@ func StructOf(fields []StructField) Type { if field.Type == nil { panic("reflect.StructOf: field " + strconv.Itoa(i) + " has no type") } - f := runtimeStructField(field) + f, fpkgpath := runtimeStructField(field) ft := f.typ if ft.kind&kindGCProg != 0 { hasGCProg = true } + if fpkgpath != "" { + if pkgpath == "" { + pkgpath = fpkgpath + } else if pkgpath != fpkgpath { + panic("reflect.Struct: fields with different PkgPath " + pkgpath + " and " + fpkgpath) + } + } // Update string and hash name := *f.name @@ -2229,7 +2237,10 @@ func StructOf(fields []StructField) Type { return addToCache(&typ.rtype) } -func runtimeStructField(field StructField) structField { +// runtimeStructField takes a StructField value passed to StructOf and +// returns both the corresponding internal representation, of type +// structField, and the pkgpath value to use for this field. +func runtimeStructField(field StructField) (structField, string) { if field.Anonymous && field.PkgPath != "" { panic("reflect.StructOf: field \"" + field.Name + "\" is anonymous but has PkgPath set") } @@ -2263,13 +2274,14 @@ func runtimeStructField(field StructField) structField { s := field.PkgPath pkgPath = &s } - return structField{ + f := structField{ name: name, pkgPath: pkgPath, typ: field.Type.common(), tag: tag, offsetEmbed: offsetEmbed, } + return f, field.PkgPath } // typeptrdata returns the length in bytes of the prefix of t diff --git a/libgo/go/runtime/chan.go b/libgo/go/runtime/chan.go index 549e566..ec8252b 100644 --- a/libgo/go/runtime/chan.go +++ b/libgo/go/runtime/chan.go @@ -133,21 +133,6 @@ func chanbuf(c *hchan, i uint) unsafe.Pointer { return add(c.buf, uintptr(i)*uintptr(c.elemsize)) } -// full reports whether a send on c would block (that is, the channel is full). -// It uses a single word-sized read of mutable state, so although -// the answer is instantaneously true, the correct answer may have changed -// by the time the calling function receives the return value. -func full(c *hchan) bool { - // c.dataqsiz is immutable (never written after the channel is created) - // so it is safe to read at any time during channel operation. - if c.dataqsiz == 0 { - // Assumes that a pointer read is relaxed-atomic. - return c.recvq.first == nil - } - // Assumes that a uint read is relaxed-atomic. - return c.qcount == c.dataqsiz -} - // entry point for c <- x from compiled code //go:nosplit func chansend1(c *hchan, elem unsafe.Pointer) { @@ -192,7 +177,7 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { // // After observing that the channel is not closed, we observe that the channel is // not ready for sending. Each of these observations is a single word-sized read - // (first c.closed and second full()). + // (first c.closed and second c.recvq.first or c.qcount depending on kind of channel). // Because a closed channel cannot transition from 'ready for sending' to // 'not ready for sending', even if the channel is closed between the two observations, // they imply a moment between the two when the channel was both not yet closed @@ -201,10 +186,9 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { // // It is okay if the reads are reordered here: if we observe that the channel is not // ready for sending and then observe that it is not closed, that implies that the - // channel wasn't closed during the first observation. However, nothing here - // guarantees forward progress. We rely on the side effects of lock release in - // chanrecv() and closechan() to update this thread's view of c.closed and full(). - if !block && c.closed == 0 && full(c) { + // channel wasn't closed during the first observation. + if !block && c.closed == 0 && ((c.dataqsiz == 0 && c.recvq.first == nil) || + (c.dataqsiz > 0 && c.qcount == c.dataqsiz)) { return false } @@ -434,16 +418,6 @@ func closechan(c *hchan) { } } -// empty reports whether a read from c would block (that is, the channel is -// empty). It uses a single atomic read of mutable state. -func empty(c *hchan) bool { - // c.dataqsiz is immutable. - if c.dataqsiz == 0 { - return atomic.Loadp(unsafe.Pointer(&c.sendq.first)) == nil - } - return atomic.Loaduint(&c.qcount) == 0 -} - // entry points for <- c from compiled code //go:nosplit func chanrecv1(c *hchan, elem unsafe.Pointer) { @@ -484,33 +458,21 @@ func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) } // Fast path: check for failed non-blocking operation without acquiring the lock. - if !block && empty(c) { - // After observing that the channel is not ready for receiving, we observe whether the - // channel is closed. - // - // Reordering of these checks could lead to incorrect behavior when racing with a close. - // For example, if the channel was open and not empty, was closed, and then drained, - // reordered reads could incorrectly indicate "open and empty". To prevent reordering, - // we use atomic loads for both checks, and rely on emptying and closing to happen in - // separate critical sections under the same lock. This assumption fails when closing - // an unbuffered channel with a blocked send, but that is an error condition anyway. - if atomic.Load(&c.closed) == 0 { - // Because a channel cannot be reopened, the later observation of the channel - // being not closed implies that it was also not closed at the moment of the - // first observation. We behave as if we observed the channel at that moment - // and report that the receive cannot proceed. - return - } - // The channel is irreversibly closed. Re-check whether the channel has any pending data - // to receive, which could have arrived between the empty and closed checks above. - // Sequential consistency is also required here, when racing with such a send. - if empty(c) { - // The channel is irreversibly closed and empty. - if ep != nil { - typedmemclr(c.elemtype, ep) - } - return true, false - } + // + // After observing that the channel is not ready for receiving, we observe that the + // channel is not closed. Each of these observations is a single word-sized read + // (first c.sendq.first or c.qcount, and second c.closed). + // Because a channel cannot be reopened, the later observation of the channel + // being not closed implies that it was also not closed at the moment of the + // first observation. We behave as if we observed the channel at that moment + // and report that the receive cannot proceed. + // + // The order of operations is important here: reversing the operations can lead to + // incorrect behavior when racing with a close. + if !block && (c.dataqsiz == 0 && c.sendq.first == nil || + c.dataqsiz > 0 && atomic.Loaduint(&c.qcount) == 0) && + atomic.Load(&c.closed) == 0 { + return } var t0 int64 diff --git a/libgo/go/runtime/chan_test.go b/libgo/go/runtime/chan_test.go index ac81d40..c194781 100644 --- a/libgo/go/runtime/chan_test.go +++ b/libgo/go/runtime/chan_test.go @@ -1132,20 +1132,6 @@ func BenchmarkChanPopular(b *testing.B) { wg.Wait() } -func BenchmarkChanClosed(b *testing.B) { - c := make(chan struct{}) - close(c) - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - select { - case <-c: - default: - b.Error("Unreachable") - } - } - }) -} - var ( alwaysFalse = false workSink = 0 diff --git a/libgo/go/runtime/checkptr.go b/libgo/go/runtime/checkptr.go index f478ddd..974f0a0 100644 --- a/libgo/go/runtime/checkptr.go +++ b/libgo/go/runtime/checkptr.go @@ -8,45 +8,22 @@ package runtime import "unsafe" -type ptrAlignError struct { - ptr unsafe.Pointer - elem *_type - n uintptr -} - -func (e ptrAlignError) RuntimeError() {} - -func (e ptrAlignError) Error() string { - return "runtime error: unsafe pointer conversion" -} - func checkptrAlignment(p unsafe.Pointer, elem *_type, n uintptr) { // Check that (*[n]elem)(p) is appropriately aligned. // TODO(mdempsky): What about fieldAlign? if uintptr(p)&(uintptr(elem.align)-1) != 0 { - panic(ptrAlignError{p, elem, n}) + throw("checkptr: unsafe pointer conversion") } // Check that (*[n]elem)(p) doesn't straddle multiple heap objects. if size := n * elem.size; size > 1 && checkptrBase(p) != checkptrBase(add(p, size-1)) { - panic(ptrAlignError{p, elem, n}) + throw("checkptr: unsafe pointer conversion") } } -type ptrArithError struct { - ptr unsafe.Pointer - originals []unsafe.Pointer -} - -func (e ptrArithError) RuntimeError() {} - -func (e ptrArithError) Error() string { - return "runtime error: unsafe pointer arithmetic" -} - func checkptrArithmetic(p unsafe.Pointer, originals []unsafe.Pointer) { if 0 < uintptr(p) && uintptr(p) < minLegalPointer { - panic(ptrArithError{p, originals}) + throw("checkptr: unsafe pointer arithmetic") } // Check that if the computed pointer p points into a heap @@ -63,7 +40,7 @@ func checkptrArithmetic(p unsafe.Pointer, originals []unsafe.Pointer) { } } - panic(ptrArithError{p, originals}) + throw("checkptr: unsafe pointer arithmetic") } // checkptrBase returns the base address for the allocation containing diff --git a/libgo/go/runtime/checkptr_test.go b/libgo/go/runtime/checkptr_test.go new file mode 100644 index 0000000..ab3058f --- /dev/null +++ b/libgo/go/runtime/checkptr_test.go @@ -0,0 +1,50 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "internal/testenv" + "os/exec" + "runtime" + "strings" + "testing" +) + +func TestCheckPtr(t *testing.T) { + if runtime.Compiler == "gccgo" { + t.Skip("gccgo does not have -d=checkptr") + } + t.Parallel() + testenv.MustHaveGoRun(t) + + exe, err := buildTestProg(t, "testprog", "-gcflags=all=-d=checkptr=1") + if err != nil { + t.Fatal(err) + } + + testCases := []struct { + cmd string + want string + }{ + {"CheckPtrAlignment", "fatal error: checkptr: unsafe pointer conversion\n"}, + {"CheckPtrArithmetic", "fatal error: checkptr: unsafe pointer arithmetic\n"}, + {"CheckPtrSize", "fatal error: checkptr: unsafe pointer conversion\n"}, + {"CheckPtrSmall", "fatal error: checkptr: unsafe pointer arithmetic\n"}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.cmd, func(t *testing.T) { + t.Parallel() + got, err := testenv.CleanCmdEnv(exec.Command(exe, tc.cmd)).CombinedOutput() + if err != nil { + t.Log(err) + } + if !strings.HasPrefix(string(got), tc.want) { + t.Errorf("output:\n%s\n\nwant output starting with: %s", got, tc.want) + } + }) + } +} diff --git a/libgo/go/runtime/debug.go b/libgo/go/runtime/debug.go index 1202e36..e480466 100644 --- a/libgo/go/runtime/debug.go +++ b/libgo/go/runtime/debug.go @@ -26,12 +26,12 @@ func GOMAXPROCS(n int) int { return ret } - stopTheWorldGC("GOMAXPROCS") + stopTheWorld("GOMAXPROCS") // newprocs will be processed by startTheWorld newprocs = int32(n) - startTheWorldGC() + startTheWorld() return ret } diff --git a/libgo/go/runtime/export_test.go b/libgo/go/runtime/export_test.go index 9a977d8..b60c19b 100644 --- a/libgo/go/runtime/export_test.go +++ b/libgo/go/runtime/export_test.go @@ -45,6 +45,9 @@ var NetpollGenericInit = netpollGenericInit var ParseRelease = parseRelease +var Memmove = memmove +var MemclrNoHeapPointers = memclrNoHeapPointers + const PreemptMSupported = preemptMSupported type LFNode struct { @@ -573,6 +576,7 @@ const ( PageSize = pageSize PallocChunkPages = pallocChunkPages PageAlloc64Bit = pageAlloc64Bit + PallocSumBytes = pallocSumBytes ) // Expose pallocSum for testing. diff --git a/libgo/go/runtime/extern.go b/libgo/go/runtime/extern.go index e2e601f..96af606 100644 --- a/libgo/go/runtime/extern.go +++ b/libgo/go/runtime/extern.go @@ -78,21 +78,6 @@ It is a comma-separated list of name=val pairs setting these named variables: If the line ends with "(forced)", this GC was forced by a runtime.GC() call. - Setting gctrace to any value > 0 also causes the garbage collector - to emit a summary when memory is released back to the system. - This process of returning memory to the system is called scavenging. - The format of this summary is subject to change. - Currently it is: - scvg#: # MB released printed only if non-zero - scvg#: inuse: # idle: # sys: # released: # consumed: # (MB) - where the fields are as follows: - scvg# the scavenge cycle number, incremented at each scavenge - inuse: # MB used or partially used spans - idle: # MB spans pending scavenging - sys: # MB mapped from the system - released: # MB released to the system - consumed: # MB allocated from the system - madvdontneed: setting madvdontneed=1 will use MADV_DONTNEED instead of MADV_FREE on Linux when returning memory to the kernel. This is less efficient, but causes RSS numbers to drop @@ -112,6 +97,19 @@ It is a comma-separated list of name=val pairs setting these named variables: scavenge: scavenge=1 enables debugging mode of heap scavenger. + scavtrace: setting scavtrace=1 causes the runtime to emit a single line to standard + error, roughly once per GC cycle, summarizing the amount of work done by the + scavenger as well as the total amount of memory returned to the operating system + and an estimate of physical memory utilization. The format of this line is subject + to change, but currently it is: + scav # KiB work, # KiB total, #% util + where the fields are as follows: + # KiB work the amount of memory returned to the OS since the last scav line + # KiB total how much of the heap at this point in time has been released to the OS + #% util the fraction of all unscavenged memory which is in-use + If the line ends with "(forced)", then scavenging was forced by a + debug.FreeOSMemory() call. + scheddetail: setting schedtrace=X and scheddetail=1 causes the scheduler to emit detailed multiline info every X milliseconds, describing state of the scheduler, processors, threads and goroutines. diff --git a/libgo/go/runtime/gcinfo_test.go b/libgo/go/runtime/gcinfo_test.go index fc24f04..ddbe5dd 100644 --- a/libgo/go/runtime/gcinfo_test.go +++ b/libgo/go/runtime/gcinfo_test.go @@ -165,7 +165,7 @@ func infoBigStruct() []byte { typeScalar, typeScalar, typeScalar, typeScalar, // t int; y uint16; u uint64 typePointer, typeScalar, // i string } - case "arm64", "amd64", "mips64", "mips64le", "ppc64", "ppc64le", "s390x", "wasm": + case "arm64", "amd64", "mips64", "mips64le", "ppc64", "ppc64le", "riscv64", "s390x", "wasm": return []byte{ typePointer, // q *int typeScalar, typeScalar, typeScalar, // w byte; e [17]byte diff --git a/libgo/go/runtime/hash64.go b/libgo/go/runtime/hash64.go index cff663a..704bbe6 100644 --- a/libgo/go/runtime/hash64.go +++ b/libgo/go/runtime/hash64.go @@ -6,7 +6,7 @@ // xxhash: https://code.google.com/p/xxhash/ // cityhash: https://code.google.com/p/cityhash/ -// +build amd64 amd64p32 arm64 mips64 mips64le ppc64 ppc64le s390x wasm alpha arm64be ia64 mips64p32 mips64p32le sparc64 riscv64 +// +build amd64 arm64 mips64 mips64le ppc64 ppc64le riscv64 s390x wasm alpha amd64p32 arm64be ia64 mips64p32 mips64p32le sparc64 package runtime diff --git a/libgo/go/runtime/lfstack_64bit.go b/libgo/go/runtime/lfstack_64bit.go index de40a00..af9e7d1 100644 --- a/libgo/go/runtime/lfstack_64bit.go +++ b/libgo/go/runtime/lfstack_64bit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build amd64 arm64 mips64 mips64le ppc64 ppc64le s390x wasm arm64be alpha sparc64 ia64 riscv64 +// +build amd64 arm64 mips64 mips64le ppc64 ppc64le riscv64 s390x wasm arm64be alpha sparc64 ia64 package runtime diff --git a/libgo/go/runtime/malloc.go b/libgo/go/runtime/malloc.go index fda2273..35ace7f 100644 --- a/libgo/go/runtime/malloc.go +++ b/libgo/go/runtime/malloc.go @@ -513,6 +513,7 @@ func mallocinit() { // allocation at 0x40 << 32 because when using 4k pages with 3-level // translation buffers, the user address space is limited to 39 bits // On darwin/arm64, the address space is even smaller. + // // On AIX, mmaps starts at 0x0A00000000000000 for 64-bit. // processes. for i := 0x7f; i >= 0; i-- { diff --git a/libgo/go/runtime/malloc_test.go b/libgo/go/runtime/malloc_test.go index bd30bc1..45555ee 100644 --- a/libgo/go/runtime/malloc_test.go +++ b/libgo/go/runtime/malloc_test.go @@ -206,14 +206,6 @@ type acLink struct { var arenaCollisionSink []*acLink func TestArenaCollision(t *testing.T) { - if GOOS == "darwin" && race.Enabled { - // Skip this test on Darwin in race mode because Darwin 10.10 has - // issues following arena hints and runs out of them in race mode, so - // MAP_FIXED is used to ensure we keep the heap in the memory region the - // race detector expects. - // TODO(mknyszek): Delete this when Darwin 10.10 is no longer supported. - t.Skip("disabled on Darwin with race mode since MAP_FIXED is used") - } testenv.MustHaveExec(t) // Test that mheap.sysAlloc handles collisions with other diff --git a/libgo/go/runtime/memmove_test.go b/libgo/go/runtime/memmove_test.go index 0b2e191..396c130 100644 --- a/libgo/go/runtime/memmove_test.go +++ b/libgo/go/runtime/memmove_test.go @@ -11,7 +11,9 @@ import ( "internal/race" "internal/testenv" . "runtime" + "sync/atomic" "testing" + "unsafe" ) func TestMemmove(t *testing.T) { @@ -206,6 +208,71 @@ func cmpb(a, b []byte) int { return l } +// Ensure that memmove writes pointers atomically, so the GC won't +// observe a partially updated pointer. +func TestMemmoveAtomicity(t *testing.T) { + if race.Enabled { + t.Skip("skip under the race detector -- this test is intentionally racy") + } + + var x int + + for _, backward := range []bool{true, false} { + for _, n := range []int{3, 4, 5, 6, 7, 8, 9, 10, 15, 25, 49} { + n := n + + // test copying [N]*int. + sz := uintptr(n * PtrSize) + name := fmt.Sprint(sz) + if backward { + name += "-backward" + } else { + name += "-forward" + } + t.Run(name, func(t *testing.T) { + // Use overlapping src and dst to force forward/backward copy. + var s [100]*int + src := s[n-1 : 2*n-1] + dst := s[:n] + if backward { + src, dst = dst, src + } + for i := range src { + src[i] = &x + } + for i := range dst { + dst[i] = nil + } + + var ready uint32 + go func() { + sp := unsafe.Pointer(&src[0]) + dp := unsafe.Pointer(&dst[0]) + atomic.StoreUint32(&ready, 1) + for i := 0; i < 10000; i++ { + Memmove(dp, sp, sz) + MemclrNoHeapPointers(dp, sz) + } + atomic.StoreUint32(&ready, 2) + }() + + for atomic.LoadUint32(&ready) == 0 { + Gosched() + } + + for atomic.LoadUint32(&ready) != 2 { + for i := range dst { + p := dst[i] + if p != nil && p != &x { + t.Fatalf("got partially updated pointer %p at dst[%d], want either nil or %p", p, i, &x) + } + } + } + }) + } + } +} + func benchmarkSizes(b *testing.B, sizes []int, fn func(b *testing.B, n int)) { for _, n := range sizes { b.Run(fmt.Sprint(n), func(b *testing.B) { diff --git a/libgo/go/runtime/mgc.go b/libgo/go/runtime/mgc.go index b0040f9..8ded306 100644 --- a/libgo/go/runtime/mgc.go +++ b/libgo/go/runtime/mgc.go @@ -1271,7 +1271,6 @@ func gcStart(trigger gcTrigger) { } // Ok, we're doing it! Stop everybody else - semacquire(&gcsema) semacquire(&worldsema) if trace.enabled { @@ -1370,13 +1369,6 @@ func gcStart(trigger gcTrigger) { work.pauseNS += now - work.pauseStart work.tMark = now }) - - // Release the world sema before Gosched() in STW mode - // because we will need to reacquire it later but before - // this goroutine becomes runnable again, and we could - // self-deadlock otherwise. - semrelease(&worldsema) - // In STW mode, we could block the instant systemstack // returns, so don't do anything important here. Make sure we // block rather than returning to user code. @@ -1446,10 +1438,6 @@ top: return } - // forEachP needs worldsema to execute, and we'll need it to - // stop the world later, so acquire worldsema now. - semacquire(&worldsema) - // Flush all local buffers and collect flushedWork flags. gcMarkDoneFlushed = 0 systemstack(func() { @@ -1510,7 +1498,6 @@ top: // work to do. Keep going. It's possible the // transition condition became true again during the // ragged barrier, so re-check it. - semrelease(&worldsema) goto top } @@ -1587,7 +1574,6 @@ top: now := startTheWorldWithSema(true) work.pauseNS += now - work.pauseStart }) - semrelease(&worldsema) goto top } } @@ -1802,7 +1788,6 @@ func gcMarkTermination(nextTriggerRatio float64) { } semrelease(&worldsema) - semrelease(&gcsema) // Careful: another GC cycle may start now. releasem(mp) diff --git a/libgo/go/runtime/mgcscavenge.go b/libgo/go/runtime/mgcscavenge.go index f3856db..3b60b3d 100644 --- a/libgo/go/runtime/mgcscavenge.go +++ b/libgo/go/runtime/mgcscavenge.go @@ -80,6 +80,17 @@ const ( // maxPagesPerPhysPage is the maximum number of supported runtime pages per // physical page, based on maxPhysPageSize. maxPagesPerPhysPage = maxPhysPageSize / pageSize + + // scavengeCostRatio is the approximate ratio between the costs of using previously + // scavenged memory and scavenging memory. + // + // For most systems the cost of scavenging greatly outweighs the costs + // associated with using scavenged memory, making this constant 0. On other systems + // (especially ones where "sysUsed" is not just a no-op) this cost is non-trivial. + // + // This ratio is used as part of multiplicative factor to help the scavenger account + // for the additional costs of using scavenged memory in its pacing. + scavengeCostRatio = 0.7 * sys.GoosDarwin ) // heapRetained returns an estimate of the current heap RSS. @@ -248,7 +259,7 @@ func bgscavenge(c chan int) { released := uintptr(0) // Time in scavenging critical section. - crit := int64(0) + crit := float64(0) // Run on the system stack since we grab the heap lock, // and a stack growth with the heap lock means a deadlock. @@ -266,16 +277,10 @@ func bgscavenge(c chan int) { // Scavenge one page, and measure the amount of time spent scavenging. start := nanotime() released = mheap_.pages.scavengeOne(physPageSize, false) - crit = nanotime() - start + atomic.Xadduintptr(&mheap_.pages.scavReleased, released) + crit = float64(nanotime() - start) }) - if debug.gctrace > 0 { - if released > 0 { - print("scvg: ", released>>10, " KB released\n") - } - print("scvg: inuse: ", memstats.heap_inuse>>20, ", idle: ", memstats.heap_idle>>20, ", sys: ", memstats.heap_sys>>20, ", released: ", memstats.heap_released>>20, ", consumed: ", (memstats.heap_sys-memstats.heap_released)>>20, " (MB)\n") - } - if released == 0 { lock(&scavenge.lock) scavenge.parked = true @@ -283,6 +288,14 @@ func bgscavenge(c chan int) { continue } + // Multiply the critical time by 1 + the ratio of the costs of using + // scavenged memory vs. scavenging memory. This forces us to pay down + // the cost of reusing this memory eagerly by sleeping for a longer period + // of time and scavenging less frequently. More concretely, we avoid situations + // where we end up scavenging so often that we hurt allocation performance + // because of the additional overheads of using scavenged memory. + crit *= 1 + scavengeCostRatio + // If we spent more than 10 ms (for example, if the OS scheduled us away, or someone // put their machine to sleep) in the critical section, bound the time we use to // calculate at 10 ms to avoid letting the sleep time get arbitrarily high. @@ -298,13 +311,13 @@ func bgscavenge(c chan int) { // much, then scavengeEMWA < idealFraction, so we'll adjust the sleep time // down. adjust := scavengeEWMA / idealFraction - sleepTime := int64(adjust * float64(crit) / (scavengePercent / 100.0)) + sleepTime := int64(adjust * crit / (scavengePercent / 100.0)) // Go to sleep. slept := scavengeSleep(sleepTime) // Compute the new ratio. - fraction := float64(crit) / float64(crit+slept) + fraction := crit / (crit + float64(slept)) // Set a lower bound on the fraction. // Due to OS-related anomalies we may "sleep" for an inordinate amount @@ -348,12 +361,39 @@ func (s *pageAlloc) scavenge(nbytes uintptr, locked bool) uintptr { return released } +// printScavTrace prints a scavenge trace line to standard error. +// +// released should be the amount of memory released since the last time this +// was called, and forced indicates whether the scavenge was forced by the +// application. +func printScavTrace(released uintptr, forced bool) { + printlock() + print("scav ", + released>>10, " KiB work, ", + atomic.Load64(&memstats.heap_released)>>10, " KiB total, ", + (atomic.Load64(&memstats.heap_inuse)*100)/heapRetained(), "% util", + ) + if forced { + print(" (forced)") + } + println() + printunlock() +} + // resetScavengeAddr sets the scavenge start address to the top of the heap's // address space. This should be called each time the scavenger's pacing // changes. // // s.mheapLock must be held. func (s *pageAlloc) resetScavengeAddr() { + released := atomic.Loaduintptr(&s.scavReleased) + if debug.scavtrace > 0 { + printScavTrace(released, false) + } + // Subtract from scavReleased instead of just setting it to zero because + // the scavenger could have increased scavReleased concurrently with the + // load above, and we may miss an update by just blindly zeroing the field. + atomic.Xadduintptr(&s.scavReleased, -released) s.scavAddr = chunkBase(s.end) - 1 } @@ -415,7 +455,10 @@ func (s *pageAlloc) scavengeOne(max uintptr, locked bool) uintptr { // Check the chunk containing the scav addr, starting at the addr // and see if there are any free and unscavenged pages. - if s.summary[len(s.summary)-1][ci].max() >= uint(minPages) { + // + // Only check this if s.scavAddr is covered by any address range + // in s.inUse, so that we know our check of the summary is safe. + if s.inUse.contains(s.scavAddr) && s.summary[len(s.summary)-1][ci].max() >= uint(minPages) { // We only bother looking for a candidate if there at least // minPages free pages at all. It's important that we only // continue if the summary says we can because that's how diff --git a/libgo/go/runtime/mgcscavenge_test.go b/libgo/go/runtime/mgcscavenge_test.go index 518d5ab..58f9e3a 100644 --- a/libgo/go/runtime/mgcscavenge_test.go +++ b/libgo/go/runtime/mgcscavenge_test.go @@ -272,6 +272,9 @@ func TestPallocDataFindScavengeCandidate(t *testing.T) { // Tests end-to-end scavenging on a pageAlloc. func TestPageAllocScavenge(t *testing.T) { + if GOOS == "openbsd" && testing.Short() { + t.Skip("skipping because virtual memory is limited; see #36210") + } type test struct { request, expect uintptr } @@ -279,12 +282,13 @@ func TestPageAllocScavenge(t *testing.T) { if minPages < 1 { minPages = 1 } - tests := map[string]struct { + type setup struct { beforeAlloc map[ChunkIdx][]BitRange beforeScav map[ChunkIdx][]BitRange expect []test afterScav map[ChunkIdx][]BitRange - }{ + } + tests := map[string]setup{ "AllFreeUnscavExhaust": { beforeAlloc: map[ChunkIdx][]BitRange{ BaseChunkIdx: {}, @@ -393,6 +397,26 @@ func TestPageAllocScavenge(t *testing.T) { }, }, } + if PageAlloc64Bit != 0 { + tests["ScavAllVeryDiscontiguous"] = setup{ + beforeAlloc: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {}, + BaseChunkIdx + 0x1000: {}, + }, + beforeScav: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {}, + BaseChunkIdx + 0x1000: {}, + }, + expect: []test{ + {^uintptr(0), 2 * PallocChunkPages * PageSize}, + {^uintptr(0), 0}, + }, + afterScav: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{0, PallocChunkPages}}, + BaseChunkIdx + 0x1000: {{0, PallocChunkPages}}, + }, + } + } for name, v := range tests { v := v runTest := func(t *testing.T, locked bool) { diff --git a/libgo/go/runtime/mheap.go b/libgo/go/runtime/mheap.go index f40589a..c40c9e2 100644 --- a/libgo/go/runtime/mheap.go +++ b/libgo/go/runtime/mheap.go @@ -70,7 +70,7 @@ type mheap struct { // on the swept stack. sweepSpans [2]gcSweepBuf - _ uint32 // align uint64 fields on 32-bit for atomics + // _ uint32 // align uint64 fields on 32-bit for atomics // Proportional sweep // @@ -786,7 +786,9 @@ func (h *mheap) reclaim(npage uintptr) { // reclaimChunk sweeps unmarked spans that start at page indexes [pageIdx, pageIdx+n). // It returns the number of pages returned to the heap. // -// h.lock must be held and the caller must be non-preemptible. +// h.lock must be held and the caller must be non-preemptible. Note: h.lock may be +// temporarily unlocked and re-locked in order to do sweeping or if tracing is +// enabled. func (h *mheap) reclaimChunk(arenas []arenaIdx, pageIdx, n uintptr) uintptr { // The heap lock must be held because this accesses the // heapArena.spans arrays using potentially non-live pointers. @@ -842,8 +844,10 @@ func (h *mheap) reclaimChunk(arenas []arenaIdx, pageIdx, n uintptr) uintptr { n -= uintptr(len(inUse) * 8) } if trace.enabled { + unlock(&h.lock) // Account for pages scanned but not reclaimed. traceGCSweepSpan((n0 - nFreed) * pageSize) + lock(&h.lock) } return nFreed } @@ -1430,11 +1434,8 @@ func (h *mheap) scavengeAll() { unlock(&h.lock) gp.m.mallocing-- - if debug.gctrace > 0 { - if released > 0 { - print("forced scvg: ", released>>20, " MB released\n") - } - print("forced scvg: inuse: ", memstats.heap_inuse>>20, ", idle: ", memstats.heap_idle>>20, ", sys: ", memstats.heap_sys>>20, ", released: ", memstats.heap_released>>20, ", consumed: ", (memstats.heap_sys-memstats.heap_released)>>20, " (MB)\n") + if debug.scavtrace > 0 { + printScavTrace(released, true) } } diff --git a/libgo/go/runtime/mkpreempt.go b/libgo/go/runtime/mkpreempt.go index 615ec18..64e2207 100644 --- a/libgo/go/runtime/mkpreempt.go +++ b/libgo/go/runtime/mkpreempt.go @@ -83,6 +83,7 @@ var arches = map[string]func(){ "mips64x": func() { genMIPS(true) }, "mipsx": func() { genMIPS(false) }, "ppc64x": genPPC64, + "riscv64": genRISCV64, "s390x": genS390X, "wasm": genWasm, } @@ -478,6 +479,11 @@ func genPPC64() { p("JMP (CTR)") } +func genRISCV64() { + p("// No async preemption on riscv64 - see issue 36711") + p("UNDEF") +} + func genS390X() { // Add integer registers R0-R12 // R13 (g), R14 (LR), R15 (SP) are special, and not saved here. diff --git a/libgo/go/runtime/mpagealloc.go b/libgo/go/runtime/mpagealloc.go index 572e6a9..bb751f1 100644 --- a/libgo/go/runtime/mpagealloc.go +++ b/libgo/go/runtime/mpagealloc.go @@ -225,7 +225,9 @@ type pageAlloc struct { // the bitmaps align better on zero-values. chunks [1 << pallocChunksL1Bits]*[1 << pallocChunksL2Bits]pallocData - // The address to start an allocation search with. + // The address to start an allocation search with. It must never + // point to any memory that is not contained in inUse, i.e. + // inUse.contains(searchAddr) must always be true. // // When added with arenaBaseOffset, we guarantee that // all valid heap addresses (when also added with @@ -237,9 +239,15 @@ type pageAlloc struct { // space on architectures with segmented address spaces. searchAddr uintptr - // The address to start a scavenge candidate search with. + // The address to start a scavenge candidate search with. It + // need not point to memory contained in inUse. scavAddr uintptr + // The amount of memory scavenged since the last scavtrace print. + // + // Read and updated atomically. + scavReleased uintptr + // start and end represent the chunk indices // which pageAlloc knows about. It assumes // chunks in the range [start, end) are diff --git a/libgo/go/runtime/mpagealloc_64bit.go b/libgo/go/runtime/mpagealloc_64bit.go index dd44da1..385b7b3 100644 --- a/libgo/go/runtime/mpagealloc_64bit.go +++ b/libgo/go/runtime/mpagealloc_64bit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build amd64 !darwin,arm64 mips64 mips64le ppc64 ppc64le s390x arm64be alpha sparc64 ia64 riscv64 +// +build amd64 !darwin,arm64 mips64 mips64le ppc64 ppc64le riscv64 s390x arm64be alpha sparc64 ia64 // See mpagealloc_32bit.go for why darwin/arm64 is excluded here. diff --git a/libgo/go/runtime/mpagealloc_test.go b/libgo/go/runtime/mpagealloc_test.go index 6c48296..89a4a25 100644 --- a/libgo/go/runtime/mpagealloc_test.go +++ b/libgo/go/runtime/mpagealloc_test.go @@ -41,6 +41,9 @@ func checkPageAlloc(t *testing.T, want, got *PageAlloc) { } func TestPageAllocGrow(t *testing.T) { + if GOOS == "openbsd" && testing.Short() { + t.Skip("skipping because virtual memory is limited; see #36210") + } type test struct { chunks []ChunkIdx inUse []AddrRange @@ -216,15 +219,19 @@ func TestPageAllocGrow(t *testing.T) { } func TestPageAllocAlloc(t *testing.T) { + if GOOS == "openbsd" && testing.Short() { + t.Skip("skipping because virtual memory is limited; see #36210") + } type hit struct { npages, base, scav uintptr } - tests := map[string]struct { + type test struct { scav map[ChunkIdx][]BitRange before map[ChunkIdx][]BitRange after map[ChunkIdx][]BitRange hits []hit - }{ + } + tests := map[string]test{ "AllFree1": { before: map[ChunkIdx][]BitRange{ BaseChunkIdx: {}, @@ -365,7 +372,6 @@ func TestPageAllocAlloc(t *testing.T) { BaseChunkIdx: {{0, 195}}, }, }, - // TODO(mknyszek): Add tests close to the chunk size. "ExhaustPallocChunkPages-3": { before: map[ChunkIdx][]BitRange{ BaseChunkIdx: {}, @@ -565,6 +571,48 @@ func TestPageAllocAlloc(t *testing.T) { }, }, } + if PageAlloc64Bit != 0 { + const chunkIdxBigJump = 0x100000 // chunk index offset which translates to O(TiB) + + // This test attempts to trigger a bug wherein we look at unmapped summary + // memory that isn't just in the case where we exhaust the heap. + // + // It achieves this by placing a chunk such that its summary will be + // at the very end of a physical page. It then also places another chunk + // much further up in the address space, such that any allocations into the + // first chunk do not exhaust the heap and the second chunk's summary is not in the + // page immediately adjacent to the first chunk's summary's page. + // Allocating into this first chunk to exhaustion and then into the second + // chunk may then trigger a check in the allocator which erroneously looks at + // unmapped summary memory and crashes. + + // Figure out how many chunks are in a physical page, then align BaseChunkIdx + // to a physical page in the chunk summary array. Here we only assume that + // each summary array is aligned to some physical page. + sumsPerPhysPage := ChunkIdx(PhysPageSize / PallocSumBytes) + baseChunkIdx := BaseChunkIdx &^ (sumsPerPhysPage - 1) + tests["DiscontiguousMappedSumBoundary"] = test{ + before: map[ChunkIdx][]BitRange{ + baseChunkIdx + sumsPerPhysPage - 1: {}, + baseChunkIdx + chunkIdxBigJump: {}, + }, + scav: map[ChunkIdx][]BitRange{ + baseChunkIdx + sumsPerPhysPage - 1: {}, + baseChunkIdx + chunkIdxBigJump: {}, + }, + hits: []hit{ + {PallocChunkPages - 1, PageBase(baseChunkIdx+sumsPerPhysPage-1, 0), 0}, + {1, PageBase(baseChunkIdx+sumsPerPhysPage-1, PallocChunkPages-1), 0}, + {1, PageBase(baseChunkIdx+chunkIdxBigJump, 0), 0}, + {PallocChunkPages - 1, PageBase(baseChunkIdx+chunkIdxBigJump, 1), 0}, + {1, 0, 0}, + }, + after: map[ChunkIdx][]BitRange{ + baseChunkIdx + sumsPerPhysPage - 1: {{0, PallocChunkPages}}, + baseChunkIdx + chunkIdxBigJump: {{0, PallocChunkPages}}, + }, + } + } for name, v := range tests { v := v t.Run(name, func(t *testing.T) { @@ -589,6 +637,9 @@ func TestPageAllocAlloc(t *testing.T) { } func TestPageAllocExhaust(t *testing.T) { + if GOOS == "openbsd" && testing.Short() { + t.Skip("skipping because virtual memory is limited; see #36210") + } for _, npages := range []uintptr{1, 2, 3, 4, 5, 8, 16, 64, 1024, 1025, 2048, 2049} { npages := npages t.Run(fmt.Sprintf("%d", npages), func(t *testing.T) { @@ -638,6 +689,9 @@ func TestPageAllocExhaust(t *testing.T) { } func TestPageAllocFree(t *testing.T) { + if GOOS == "openbsd" && testing.Short() { + t.Skip("skipping because virtual memory is limited; see #36210") + } tests := map[string]struct { before map[ChunkIdx][]BitRange after map[ChunkIdx][]BitRange @@ -867,6 +921,9 @@ func TestPageAllocFree(t *testing.T) { } func TestPageAllocAllocAndFree(t *testing.T) { + if GOOS == "openbsd" && testing.Short() { + t.Skip("skipping because virtual memory is limited; see #36210") + } type hit struct { alloc bool npages uintptr diff --git a/libgo/go/runtime/mpagecache_test.go b/libgo/go/runtime/mpagecache_test.go index 6fdaa04..b8cc0bd 100644 --- a/libgo/go/runtime/mpagecache_test.go +++ b/libgo/go/runtime/mpagecache_test.go @@ -180,6 +180,9 @@ func TestPageCacheAlloc(t *testing.T) { } func TestPageCacheFlush(t *testing.T) { + if GOOS == "openbsd" && testing.Short() { + t.Skip("skipping because virtual memory is limited; see #36210") + } bits64ToBitRanges := func(bits uint64, base uint) []BitRange { var ranges []BitRange start, size := uint(0), uint(0) @@ -254,6 +257,9 @@ func TestPageCacheFlush(t *testing.T) { } func TestPageAllocAllocToCache(t *testing.T) { + if GOOS == "openbsd" && testing.Short() { + t.Skip("skipping because virtual memory is limited; see #36210") + } tests := map[string]struct { before map[ChunkIdx][]BitRange scav map[ChunkIdx][]BitRange diff --git a/libgo/go/runtime/mpallocbits.go b/libgo/go/runtime/mpallocbits.go index 9d01ff8..a801134 100644 --- a/libgo/go/runtime/mpallocbits.go +++ b/libgo/go/runtime/mpallocbits.go @@ -202,17 +202,11 @@ func (b *pallocBits) summarize() pallocSum { // If find fails to find any free space, it returns an index of ^uint(0) and // the new searchIdx should be ignored. // -// The returned searchIdx is always the index of the first free page found -// in this bitmap during the search, except if npages == 1, in which -// case it will be the index just after the first free page, because the -// index returned as the first result is assumed to be allocated and so -// represents a minor optimization for that case. +// Note that if npages == 1, the two returned values will always be identical. func (b *pallocBits) find(npages uintptr, searchIdx uint) (uint, uint) { if npages == 1 { addr := b.find1(searchIdx) - // Return a searchIdx of addr + 1 since we assume addr will be - // allocated. - return addr, addr + 1 + return addr, addr } else if npages <= 64 { return b.findSmallN(npages, searchIdx) } diff --git a/libgo/go/runtime/mranges.go b/libgo/go/runtime/mranges.go index c14e5c7..b133851 100644 --- a/libgo/go/runtime/mranges.go +++ b/libgo/go/runtime/mranges.go @@ -29,6 +29,11 @@ func (a addrRange) size() uintptr { return a.limit - a.base } +// contains returns whether or not the range contains a given address. +func (a addrRange) contains(addr uintptr) bool { + return addr >= a.base && addr < a.limit +} + // subtract takes the addrRange toPrune and cuts out any overlap with // from, then returns the new range. subtract assumes that a and b // either don't overlap at all, only overlap on one side, or are equal. @@ -87,6 +92,15 @@ func (a *addrRanges) findSucc(base uintptr) int { return len(a.ranges) } +// contains returns true if a covers the address addr. +func (a *addrRanges) contains(addr uintptr) bool { + i := a.findSucc(addr) + if i == 0 { + return false + } + return a.ranges[i-1].contains(addr) +} + // add inserts a new address range to a. // // r must not overlap with any address range in a. diff --git a/libgo/go/runtime/preempt_nonwindows.go b/libgo/go/runtime/preempt_nonwindows.go new file mode 100644 index 0000000..3066a152 --- /dev/null +++ b/libgo/go/runtime/preempt_nonwindows.go @@ -0,0 +1,13 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !windows + +package runtime + +//go:nosplit +func osPreemptExtEnter(mp *m) {} + +//go:nosplit +func osPreemptExtExit(mp *m) {} diff --git a/libgo/go/runtime/proc.go b/libgo/go/runtime/proc.go index e3f934a..f75cacf 100644 --- a/libgo/go/runtime/proc.go +++ b/libgo/go/runtime/proc.go @@ -841,23 +841,8 @@ func casGFromPreempted(gp *g, old, new uint32) bool { // goroutines. func stopTheWorld(reason string) { semacquire(&worldsema) - gp := getg() - gp.m.preemptoff = reason - systemstack(func() { - // Mark the goroutine which called stopTheWorld preemptible so its - // stack may be scanned. - // This lets a mark worker scan us while we try to stop the world - // since otherwise we could get in a mutual preemption deadlock. - // We must not modify anything on the G stack because a stack shrink - // may occur. A stack shrink is otherwise OK though because in order - // to return from this function (and to leave the system stack) we - // must have preempted all goroutines, including any attempting - // to scan our stack, in which case, any stack shrinking will - // have already completed by the time we exit. - casgstatus(gp, _Grunning, _Gwaiting) - stopTheWorldWithSema() - casgstatus(gp, _Gwaiting, _Grunning) - }) + getg().m.preemptoff = reason + systemstack(stopTheWorldWithSema) } // startTheWorld undoes the effects of stopTheWorld. @@ -869,31 +854,10 @@ func startTheWorld() { getg().m.preemptoff = "" } -// stopTheWorldGC has the same effect as stopTheWorld, but blocks -// until the GC is not running. It also blocks a GC from starting -// until startTheWorldGC is called. -func stopTheWorldGC(reason string) { - semacquire(&gcsema) - stopTheWorld(reason) -} - -// startTheWorldGC undoes the effects of stopTheWorldGC. -func startTheWorldGC() { - startTheWorld() - semrelease(&gcsema) -} - -// Holding worldsema grants an M the right to try to stop the world. +// Holding worldsema grants an M the right to try to stop the world +// and prevents gomaxprocs from changing concurrently. var worldsema uint32 = 1 -// Holding gcsema grants the M the right to block a GC, and blocks -// until the current GC is done. In particular, it prevents gomaxprocs -// from changing concurrently. -// -// TODO(mknyszek): Once gomaxprocs and the execution tracer can handle -// being changed/enabled during a GC, remove this. -var gcsema uint32 = 1 - // stopTheWorldWithSema is the core implementation of stopTheWorld. // The caller is responsible for acquiring worldsema and disabling // preemption first and then should stopTheWorldWithSema on the system @@ -2577,6 +2541,27 @@ func dropg() { // We pass now in and out to avoid extra calls of nanotime. //go:yeswritebarrierrec func checkTimers(pp *p, now int64) (rnow, pollUntil int64, ran bool) { + // If there are no timers to adjust, and the first timer on + // the heap is not yet ready to run, then there is nothing to do. + if atomic.Load(&pp.adjustTimers) == 0 { + next := int64(atomic.Load64(&pp.timer0When)) + if next == 0 { + return now, 0, false + } + if now == 0 { + now = nanotime() + } + if now < next { + // Next timer is not ready to run. + // But keep going if we would clear deleted timers. + // This corresponds to the condition below where + // we decide whether to call clearDeletedTimers. + if pp != getg().m.p.ptr() || int(atomic.Load(&pp.deletedTimers)) <= int(atomic.Load(&pp.numTimers)/4) { + return now, next, false + } + } + } + lock(&pp.timersLock) adjusttimers(pp) @@ -2599,6 +2584,13 @@ func checkTimers(pp *p, now int64) (rnow, pollUntil int64, ran bool) { } } + // If this is the local P, and there are a lot of deleted timers, + // clear them out. We only do this for the local P to reduce + // lock contention on timersLock. + if pp == getg().m.p.ptr() && int(atomic.Load(&pp.deletedTimers)) > len(pp.timers)/4 { + clearDeletedTimers(pp) + } + unlock(&pp.timersLock) return rnow, pollUntil, ran @@ -2723,7 +2715,7 @@ func preemptPark(gp *g) { } // goyield is like Gosched, but it: -// - does not emit a GoSched trace event +// - emits a GoPreempt trace event instead of a GoSched trace event // - puts the current G on the runq of the current P instead of the globrunq func goyield() { checkTimeouts() @@ -2731,6 +2723,9 @@ func goyield() { } func goyield_m(gp *g) { + if trace.enabled { + traceGoPreempt() + } pp := gp.m.p.ptr() casgstatus(gp, _Grunning, _Grunnable) dropg() @@ -3816,7 +3811,10 @@ func (pp *p) destroy() { lock(&pp.timersLock) moveTimers(plocal, pp.timers) pp.timers = nil + pp.numTimers = 0 pp.adjustTimers = 0 + pp.deletedTimers = 0 + atomic.Store64(&pp.timer0When, 0) unlock(&pp.timersLock) unlock(&plocal.timersLock) } @@ -4122,23 +4120,26 @@ func checkdead() { } // Maybe jump time forward for playground. - _p_ := timejump() - if _p_ != nil { - for pp := &sched.pidle; *pp != 0; pp = &(*pp).ptr().link { - if (*pp).ptr() == _p_ { - *pp = _p_.link - break + if faketime != 0 { + when, _p_ := timeSleepUntil() + if _p_ != nil { + faketime = when + for pp := &sched.pidle; *pp != 0; pp = &(*pp).ptr().link { + if (*pp).ptr() == _p_ { + *pp = _p_.link + break + } } + mp := mget() + if mp == nil { + // There should always be a free M since + // nothing is running. + throw("checkdead: no m for timer") + } + mp.nextp.set(_p_) + notewakeup(&mp.park) + return } - mp := mget() - if mp == nil { - // There should always be a free M since - // nothing is running. - throw("checkdead: no m for timer") - } - mp.nextp.set(_p_) - notewakeup(&mp.park) - return } // There are no goroutines running, so we can look at the P's. @@ -4183,7 +4184,7 @@ func sysmon() { } usleep(delay) now := nanotime() - next := timeSleepUntil() + next, _ := timeSleepUntil() if debug.schedtrace <= 0 && (sched.gcwaiting != 0 || atomic.Load(&sched.npidle) == uint32(gomaxprocs)) { lock(&sched.lock) if atomic.Load(&sched.gcwaiting) != 0 || atomic.Load(&sched.npidle) == uint32(gomaxprocs) { @@ -4205,7 +4206,7 @@ func sysmon() { osRelax(false) } now = nanotime() - next = timeSleepUntil() + next, _ = timeSleepUntil() lock(&sched.lock) atomic.Store(&sched.sysmonwait, 0) noteclear(&sched.sysmonnote) diff --git a/libgo/go/runtime/runtime1.go b/libgo/go/runtime/runtime1.go index 60aa90f..6edf7a5 100644 --- a/libgo/go/runtime/runtime1.go +++ b/libgo/go/runtime/runtime1.go @@ -323,6 +323,7 @@ var debug struct { madvdontneed int32 // for Linux; issue 28466 sbrk int32 scavenge int32 + scavtrace int32 scheddetail int32 schedtrace int32 tracebackancestors int32 @@ -343,6 +344,7 @@ var dbgvars = []dbgVar{ {"madvdontneed", &debug.madvdontneed}, {"sbrk", &debug.sbrk}, {"scavenge", &debug.scavenge}, + {"scavtrace", &debug.scavtrace}, {"scheddetail", &debug.scheddetail}, {"schedtrace", &debug.schedtrace}, {"tracebackancestors", &debug.tracebackancestors}, diff --git a/libgo/go/runtime/runtime2.go b/libgo/go/runtime/runtime2.go index d50f82a..f5bfc08 100644 --- a/libgo/go/runtime/runtime2.go +++ b/libgo/go/runtime/runtime2.go @@ -677,6 +677,11 @@ type p struct { _ uint32 // Alignment for atomic fields below + // The when field of the first entry on the timer heap. + // This is updated using atomic functions. + // This is 0 if the timer heap is empty. + timer0When uint64 + // Per-P GC state gcAssistTime int64 // Nanoseconds in assistAlloc gcFractionalMarkTime int64 // Nanoseconds in fractional mark worker (atomic) @@ -708,12 +713,20 @@ type p struct { // Must hold timersLock to access. timers []*timer + // Number of timers in P's heap. + // Modified using atomic instructions. + numTimers uint32 + // Number of timerModifiedEarlier timers on P's heap. // This should only be modified while holding timersLock, // or while the timer status is in a transient state // such as timerModifying. adjustTimers uint32 + // Number of timerDeleted timers in P's heap. + // Modified using atomic instructions. + deletedTimers uint32 + // Race context used while executing timer functions. // Not for gccgo: timerRaceCtx uintptr diff --git a/libgo/go/runtime/sema.go b/libgo/go/runtime/sema.go index fb16796..b6fab6d 100644 --- a/libgo/go/runtime/sema.go +++ b/libgo/go/runtime/sema.go @@ -199,9 +199,9 @@ func semrelease1(addr *uint32, handoff bool, skipframes int) { // the waiter G immediately. // Note that waiter inherits our time slice: this is desirable // to avoid having a highly contended semaphore hog the P - // indefinitely. goyield is like Gosched, but it does not emit a - // GoSched trace event and, more importantly, puts the current G - // on the local runq instead of the global one. + // indefinitely. goyield is like Gosched, but it emits a + // "preempted" trace event instead and, more importantly, puts + // the current G on the local runq instead of the global one. // We only do this in the starving regime (handoff=true), as in // the non-starving case it is possible for a different waiter // to acquire the semaphore while we are yielding/scheduling, diff --git a/libgo/go/runtime/signal_unix.go b/libgo/go/runtime/signal_unix.go index 29f9443..150345f 100644 --- a/libgo/go/runtime/signal_unix.go +++ b/libgo/go/runtime/signal_unix.go @@ -399,6 +399,16 @@ func sigtrampgo(sig uint32, info *_siginfo_t, ctx unsafe.Pointer) { sigprofNonGo(pc) return } + if sig == sigPreempt && preemptMSupported && debug.asyncpreemptoff == 0 { + // This is probably a signal from preemptM sent + // while executing Go code but received while + // executing non-Go code. + // We got past sigfwdgo, so we know that there is + // no non-Go signal handler for sigPreempt. + // The default behavior for sigPreempt is to ignore + // the signal, so badsignal will be a no-op anyway. + return + } badsignal(uintptr(sig), &c) return } diff --git a/libgo/go/runtime/testdata/testprog/checkptr.go b/libgo/go/runtime/testdata/testprog/checkptr.go new file mode 100644 index 0000000..177db38 --- /dev/null +++ b/libgo/go/runtime/testdata/testprog/checkptr.go @@ -0,0 +1,36 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "unsafe" + +func init() { + register("CheckPtrAlignment", CheckPtrAlignment) + register("CheckPtrArithmetic", CheckPtrArithmetic) + register("CheckPtrSize", CheckPtrSize) + register("CheckPtrSmall", CheckPtrSmall) +} + +func CheckPtrAlignment() { + var x [2]int64 + p := unsafe.Pointer(&x[0]) + sink2 = (*int64)(unsafe.Pointer(uintptr(p) + 1)) +} + +func CheckPtrArithmetic() { + var x int + i := uintptr(unsafe.Pointer(&x)) + sink2 = (*int)(unsafe.Pointer(i)) +} + +func CheckPtrSize() { + p := new(int64) + sink2 = p + sink2 = (*[100]int64)(unsafe.Pointer(p)) +} + +func CheckPtrSmall() { + sink2 = unsafe.Pointer(uintptr(1)) +} diff --git a/libgo/go/runtime/time.go b/libgo/go/runtime/time.go index ded68ed..d0dd3a4 100644 --- a/libgo/go/runtime/time.go +++ b/libgo/go/runtime/time.go @@ -73,14 +73,15 @@ type timer struct { // timerNoStatus -> timerWaiting // anything else -> panic: invalid value // deltimer: -// timerWaiting -> timerDeleted -// timerModifiedXX -> timerDeleted -// timerNoStatus -> do nothing -// timerDeleted -> do nothing -// timerRemoving -> do nothing -// timerRemoved -> do nothing -// timerRunning -> wait until status changes -// timerMoving -> wait until status changes +// timerWaiting -> timerDeleted +// timerModifiedEarlier -> timerModifying -> timerDeleted +// timerModifiedLater -> timerDeleted +// timerNoStatus -> do nothing +// timerDeleted -> do nothing +// timerRemoving -> do nothing +// timerRemoved -> do nothing +// timerRunning -> wait until status changes +// timerMoving -> wait until status changes // timerModifying -> panic: concurrent deltimer/modtimer calls // modtimer: // timerWaiting -> timerModifying -> timerModifiedXX @@ -168,6 +169,10 @@ const ( // maxWhen is the maximum value for timer's when field. const maxWhen = 1<<63 - 1 +// verifyTimers can be set to true to add debugging checks that the +// timer heaps are valid. +const verifyTimers = false + // Package time APIs. // Godoc uses the comments in package time, not these. @@ -283,7 +288,12 @@ func doaddtimer(pp *p, t *timer) bool { t.pp.set(pp) i := len(pp.timers) pp.timers = append(pp.timers, t) - return siftupTimer(pp.timers, i) + ok := siftupTimer(pp.timers, i) + if t == pp.timers[0] { + atomic.Store64(&pp.timer0When, uint64(t.when)) + } + atomic.Xadd(&pp.numTimers, 1) + return ok } // deltimer deletes the timer t. It may be on some other P, so we can't @@ -294,7 +304,9 @@ func deltimer(t *timer) bool { for { switch s := atomic.Load(&t.status); s { case timerWaiting, timerModifiedLater: + tpp := t.pp.ptr() if atomic.Cas(&t.status, s, timerDeleted) { + atomic.Xadd(&tpp.deletedTimers, 1) // Timer was not yet run. return true } @@ -305,6 +317,7 @@ func deltimer(t *timer) bool { if !atomic.Cas(&t.status, timerModifying, timerDeleted) { badTimer() } + atomic.Xadd(&tpp.deletedTimers, 1) // Timer was not yet run. return true } @@ -355,6 +368,10 @@ func dodeltimer(pp *p, i int) bool { ok = false } } + if i == 0 { + updateTimer0When(pp) + } + atomic.Xadd(&pp.numTimers, -1) return ok } @@ -378,6 +395,8 @@ func dodeltimer0(pp *p) bool { if last > 0 { ok = siftdownTimer(pp.timers, 0) } + updateTimer0When(pp) + atomic.Xadd(&pp.numTimers, -1) return ok } @@ -485,6 +504,7 @@ func resettimer(t *timer, when int64) { return } case timerDeleted: + tpp := t.pp.ptr() if atomic.Cas(&t.status, s, timerModifying) { t.nextwhen = when newStatus := uint32(timerModifiedLater) @@ -495,6 +515,7 @@ func resettimer(t *timer, when int64) { if !atomic.Cas(&t.status, timerModifying, newStatus) { badTimer() } + atomic.Xadd(&tpp.deletedTimers, -1) if newStatus == timerModifiedEarlier { wakeNetPoller(when) } @@ -542,6 +563,7 @@ func cleantimers(pp *p) bool { if !atomic.Cas(&t.status, timerRemoving, timerRemoved) { return false } + atomic.Xadd(&pp.deletedTimers, -1) case timerModifiedEarlier, timerModifiedLater: if !atomic.Cas(&t.status, s, timerMoving) { continue @@ -630,9 +652,13 @@ func adjusttimers(pp *p) { return } if atomic.Load(&pp.adjustTimers) == 0 { + if verifyTimers { + verifyTimerHeap(pp) + } return } var moved []*timer +loop: for i := 0; i < len(pp.timers); i++ { t := pp.timers[i] if t.pp.ptr() != pp { @@ -647,6 +673,7 @@ func adjusttimers(pp *p) { if !atomic.Cas(&t.status, timerRemoving, timerRemoved) { badTimer() } + atomic.Xadd(&pp.deletedTimers, -1) // Look at this heap position again. i-- } @@ -664,10 +691,11 @@ func adjusttimers(pp *p) { moved = append(moved, t) if s == timerModifiedEarlier { if n := atomic.Xadd(&pp.adjustTimers, -1); int32(n) <= 0 { - addAdjustedTimers(pp, moved) - return + break loop } } + // Look at this heap position again. + i-- } case timerNoStatus, timerRunning, timerRemoving, timerRemoved, timerMoving: badTimer() @@ -685,6 +713,10 @@ func adjusttimers(pp *p) { if len(moved) > 0 { addAdjustedTimers(pp, moved) } + + if verifyTimers { + verifyTimerHeap(pp) + } } // addAdjustedTimers adds any timers we adjusted in adjusttimers @@ -708,17 +740,11 @@ func addAdjustedTimers(pp *p, moved []*timer) { // The netpoller M will wake up and adjust timers before sleeping again. //go:nowritebarrierrec func nobarrierWakeTime(pp *p) int64 { - lock(&pp.timersLock) - ret := int64(0) - if len(pp.timers) > 0 { - if atomic.Load(&pp.adjustTimers) > 0 { - ret = nanotime() - } else { - ret = pp.timers[0].when - } + if atomic.Load(&pp.adjustTimers) > 0 { + return nanotime() + } else { + return int64(atomic.Load64(&pp.timer0When)) } - unlock(&pp.timersLock) - return ret } // runtimer examines the first timer in timers. If it is ready based on now, @@ -759,6 +785,7 @@ func runtimer(pp *p, now int64) int64 { if !atomic.Cas(&t.status, timerRemoving, timerRemoved) { badTimer() } + atomic.Xadd(&pp.deletedTimers, -1) if len(pp.timers) == 0 { return -1 } @@ -817,6 +844,7 @@ func runOneTimer(pp *p, t *timer, now int64) { if !atomic.Cas(&t.status, timerRunning, timerWaiting) { badTimer() } + updateTimer0When(pp) } else { // Remove from heap. if !dodeltimer0(pp) { @@ -834,69 +862,131 @@ func runOneTimer(pp *p, t *timer, now int64) { lock(&pp.timersLock) } -func timejump() *p { - if faketime == 0 { - return nil - } - - // Nothing is running, so we can look at all the P's. - // Determine a timer bucket with minimum when. - var ( - minT *timer - minWhen int64 - minP *p - ) - for _, pp := range allp { - if pp.status != _Pidle && pp.status != _Pdead { - throw("non-idle P in timejump") - } - if len(pp.timers) == 0 { - continue - } - c := pp.adjustTimers - for _, t := range pp.timers { +// clearDeletedTimers removes all deleted timers from the P's timer heap. +// This is used to avoid clogging up the heap if the program +// starts a lot of long-running timers and then stops them. +// For example, this can happen via context.WithTimeout. +// +// This is the only function that walks through the entire timer heap, +// other than moveTimers which only runs when the world is stopped. +// +// The caller must have locked the timers for pp. +func clearDeletedTimers(pp *p) { + cdel := int32(0) + cearlier := int32(0) + to := 0 + changedHeap := false + timers := pp.timers +nextTimer: + for _, t := range timers { + for { switch s := atomic.Load(&t.status); s { case timerWaiting: - if minT == nil || t.when < minWhen { - minT = t - minWhen = t.when - minP = pp + if changedHeap { + timers[to] = t + siftupTimer(timers, to) } + to++ + continue nextTimer case timerModifiedEarlier, timerModifiedLater: - if minT == nil || t.nextwhen < minWhen { - minT = t - minWhen = t.nextwhen - minP = pp + if atomic.Cas(&t.status, s, timerMoving) { + t.when = t.nextwhen + timers[to] = t + siftupTimer(timers, to) + to++ + changedHeap = true + if !atomic.Cas(&t.status, timerMoving, timerWaiting) { + badTimer() + } + if s == timerModifiedEarlier { + cearlier++ + } + continue nextTimer } - if s == timerModifiedEarlier { - c-- + case timerDeleted: + if atomic.Cas(&t.status, s, timerRemoving) { + t.pp = 0 + cdel++ + if !atomic.Cas(&t.status, timerRemoving, timerRemoved) { + badTimer() + } + changedHeap = true + continue nextTimer } - case timerRunning, timerModifying, timerMoving: + case timerModifying: + // Loop until modification complete. + osyield() + case timerNoStatus, timerRemoved: + // We should not see these status values in a timer heap. + badTimer() + case timerRunning, timerRemoving, timerMoving: + // Some other P thinks it owns this timer, + // which should not happen. + badTimer() + default: badTimer() - } - // The timers are sorted, so we only have to check - // the first timer for each P, unless there are - // some timerModifiedEarlier timers. The number - // of timerModifiedEarlier timers is in the adjustTimers - // field, used to initialize c, above. - if c == 0 { - break } } } - if minT == nil || minWhen <= faketime { - return nil + // Set remaining slots in timers slice to nil, + // so that the timer values can be garbage collected. + for i := to; i < len(timers); i++ { + timers[i] = nil + } + + atomic.Xadd(&pp.deletedTimers, -cdel) + atomic.Xadd(&pp.numTimers, -cdel) + atomic.Xadd(&pp.adjustTimers, -cearlier) + + timers = timers[:to] + pp.timers = timers + updateTimer0When(pp) + + if verifyTimers { + verifyTimerHeap(pp) + } +} + +// verifyTimerHeap verifies that the timer heap is in a valid state. +// This is only for debugging, and is only called if verifyTimers is true. +// The caller must have locked the timers. +func verifyTimerHeap(pp *p) { + for i, t := range pp.timers { + if i == 0 { + // First timer has no parent. + continue + } + + // The heap is 4-ary. See siftupTimer and siftdownTimer. + p := (i - 1) / 4 + if t.when < pp.timers[p].when { + print("bad timer heap at ", i, ": ", p, ": ", pp.timers[p].when, ", ", i, ": ", t.when, "\n") + throw("bad timer heap") + } + } + if numTimers := int(atomic.Load(&pp.numTimers)); len(pp.timers) != numTimers { + println("timer heap len", len(pp.timers), "!= numTimers", numTimers) + throw("bad timer heap len") } +} - faketime = minWhen - return minP +// updateTimer0When sets the P's timer0When field. +// The caller must have locked the timers for pp. +func updateTimer0When(pp *p) { + if len(pp.timers) == 0 { + atomic.Store64(&pp.timer0When, 0) + } else { + atomic.Store64(&pp.timer0When, uint64(pp.timers[0].when)) + } } -// timeSleepUntil returns the time when the next timer should fire. -// This is only called by sysmon. -func timeSleepUntil() int64 { +// timeSleepUntil returns the time when the next timer should fire, +// and the P that holds the timer heap that that timer is on. +// This is only called by sysmon and checkdead. +func timeSleepUntil() (int64, *p) { next := int64(maxWhen) + var pret *p // Prevent allp slice changes. This is like retake. lock(&allpLock) @@ -907,8 +997,17 @@ func timeSleepUntil() int64 { continue } - lock(&pp.timersLock) c := atomic.Load(&pp.adjustTimers) + if c == 0 { + w := int64(atomic.Load64(&pp.timer0When)) + if w != 0 && w < next { + next = w + pret = pp + } + continue + } + + lock(&pp.timersLock) for _, t := range pp.timers { switch s := atomic.Load(&t.status); s { case timerWaiting: @@ -943,7 +1042,7 @@ func timeSleepUntil() int64 { } unlock(&allpLock) - return next + return next, pret } // Heap maintenance algorithms. diff --git a/libgo/go/runtime/trace.go b/libgo/go/runtime/trace.go index 81ff0ca..358674b 100644 --- a/libgo/go/runtime/trace.go +++ b/libgo/go/runtime/trace.go @@ -181,12 +181,9 @@ func traceBufPtrOf(b *traceBuf) traceBufPtr { // Most clients should use the runtime/trace package or the testing package's // -test.trace flag instead of calling StartTrace directly. func StartTrace() error { - // Stop the world so that we can take a consistent snapshot + // Stop the world, so that we can take a consistent snapshot // of all goroutines at the beginning of the trace. - // Do not stop the world during GC so we ensure we always see - // a consistent view of GC-related events (e.g. a start is always - // paired with an end). - stopTheWorldGC("start tracing") + stopTheWorld("start tracing") // We are in stop-the-world, but syscalls can finish and write to trace concurrently. // Exitsyscall could check trace.enabled long before and then suddenly wake up @@ -197,7 +194,7 @@ func StartTrace() error { if trace.enabled || trace.shutdown { unlock(&trace.bufLock) - startTheWorldGC() + startTheWorld() return errorString("tracing is already enabled") } @@ -268,7 +265,7 @@ func StartTrace() error { unlock(&trace.bufLock) - startTheWorldGC() + startTheWorld() return nil } @@ -277,14 +274,14 @@ func StartTrace() error { func StopTrace() { // Stop the world so that we can collect the trace buffers from all p's below, // and also to avoid races with traceEvent. - stopTheWorldGC("stop tracing") + stopTheWorld("stop tracing") // See the comment in StartTrace. lock(&trace.bufLock) if !trace.enabled { unlock(&trace.bufLock) - startTheWorldGC() + startTheWorld() return } @@ -321,7 +318,7 @@ func StopTrace() { trace.shutdown = true unlock(&trace.bufLock) - startTheWorldGC() + startTheWorld() // The world is started but we've set trace.shutdown, so new tracing can't start. // Wait for the trace reader to flush pending buffers and stop. diff --git a/libgo/go/runtime/trace/trace_stack_test.go b/libgo/go/runtime/trace/trace_stack_test.go index e3608c6..62c06e6 100644 --- a/libgo/go/runtime/trace/trace_stack_test.go +++ b/libgo/go/runtime/trace/trace_stack_test.go @@ -233,7 +233,6 @@ func TestTraceSymbolize(t *testing.T) { }}, {trace.EvGomaxprocs, []frame{ {"runtime.startTheWorld", 0}, // this is when the current gomaxprocs is logged. - {"runtime.startTheWorldGC", 0}, {"runtime.GOMAXPROCS", 0}, {"runtime/trace_test.TestTraceSymbolize", 0}, {"testing.tRunner", 0}, diff --git a/libgo/go/runtime/utf8.go b/libgo/go/runtime/utf8.go index 6590472..a404a33 100644 --- a/libgo/go/runtime/utf8.go +++ b/libgo/go/runtime/utf8.go @@ -13,7 +13,7 @@ import _ "unsafe" // For go:linkname. // Numbers fundamental to the encoding. const ( runeError = '\uFFFD' // the "error" Rune or "Unicode replacement character" - runeSelf = 0x80 // characters below Runeself are represented as themselves in a single byte. + runeSelf = 0x80 // characters below runeSelf are represented as themselves in a single byte. maxRune = '\U0010FFFF' // Maximum valid Unicode code point. ) diff --git a/libgo/go/strconv/quote.go b/libgo/go/strconv/quote.go index b50496a..bcbdbc5 100644 --- a/libgo/go/strconv/quote.go +++ b/libgo/go/strconv/quote.go @@ -145,8 +145,9 @@ func AppendQuoteToASCII(dst []byte, s string) []byte { } // QuoteToGraphic returns a double-quoted Go string literal representing s. -// The returned string uses Go escape sequences (\t, \n, \xFF, \u0100) for -// non-ASCII characters and non-printable characters as defined by IsGraphic. +// The returned string leaves Unicode graphic characters, as defined by +// IsGraphic, unchanged and uses Go escape sequences (\t, \n, \xFF, \u0100) +// for non-graphic characters. func QuoteToGraphic(s string) string { return quoteWith(s, '"', false, true) } @@ -185,9 +186,9 @@ func AppendQuoteRuneToASCII(dst []byte, r rune) []byte { } // QuoteRuneToGraphic returns a single-quoted Go character literal representing -// the rune. The returned string uses Go escape sequences (\t, \n, \xFF, -// \u0100) for non-ASCII characters and non-printable characters as defined -// by IsGraphic. +// the rune. If the rune is not a Unicode graphic character, +// as defined by IsGraphic, the returned string will use a Go escape sequence +// (\t, \n, \xFF, \u0100). func QuoteRuneToGraphic(r rune) string { return quoteRuneWith(r, '\'', false, true) } diff --git a/libgo/go/strings/strings.go b/libgo/go/strings/strings.go index 69f51b6..238d657 100644 --- a/libgo/go/strings/strings.go +++ b/libgo/go/strings/strings.go @@ -420,24 +420,24 @@ func FieldsFunc(s string, f func(rune) bool) []string { return a } -// Join concatenates the elements of a to create a single string. The separator string -// sep is placed between elements in the resulting string. -func Join(a []string, sep string) string { - switch len(a) { +// Join concatenates the elements of its first argument to create a single string. The separator +// string sep is placed between elements in the resulting string. +func Join(elems []string, sep string) string { + switch len(elems) { case 0: return "" case 1: - return a[0] + return elems[0] } - n := len(sep) * (len(a) - 1) - for i := 0; i < len(a); i++ { - n += len(a[i]) + n := len(sep) * (len(elems) - 1) + for i := 0; i < len(elems); i++ { + n += len(elems[i]) } var b Builder b.Grow(n) - b.WriteString(a[0]) - for _, s := range a[1:] { + b.WriteString(elems[0]) + for _, s := range elems[1:] { b.WriteString(sep) b.WriteString(s) } diff --git a/libgo/go/syscall/syscall_aix.go b/libgo/go/syscall/syscall_aix.go index f4cac01..af0b883 100644 --- a/libgo/go/syscall/syscall_aix.go +++ b/libgo/go/syscall/syscall_aix.go @@ -4,7 +4,9 @@ package syscall -import "unsafe" +import ( + "unsafe" +) func (ts *StTimespec) Unix() (sec int64, nsec int64) { return int64(ts.Sec), int64(ts.Nsec) diff --git a/libgo/go/testing/benchmark.go b/libgo/go/testing/benchmark.go index 0412772..93f461b 100644 --- a/libgo/go/testing/benchmark.go +++ b/libgo/go/testing/benchmark.go @@ -86,7 +86,7 @@ type InternalBenchmark struct { // may be called simultaneously from multiple goroutines. // // Like in tests, benchmark logs are accumulated during execution -// and dumped to standard error when done. Unlike in tests, benchmark logs +// and dumped to standard output when done. Unlike in tests, benchmark logs // are always printed, so as not to hide output whose existence may be // affecting benchmark results. type B struct { diff --git a/libgo/go/testing/panic_test.go b/libgo/go/testing/panic_test.go index 3491510..6b8b953 100644 --- a/libgo/go/testing/panic_test.go +++ b/libgo/go/testing/panic_test.go @@ -16,6 +16,9 @@ import ( ) var testPanicTest = flag.String("test_panic_test", "", "TestPanic: indicates which test should panic") +var testPanicParallel = flag.Bool("test_panic_parallel", false, "TestPanic: run subtests in parallel") +var testPanicCleanup = flag.Bool("test_panic_cleanup", false, "TestPanic: indicates whether test should call Cleanup") +var testPanicCleanupPanic = flag.String("test_panic_cleanup_panic", "", "TestPanic: indicate whether test should call Cleanup function that panics") func TestPanic(t *testing.T) { testenv.MustHaveExec(t) @@ -40,6 +43,98 @@ func TestPanic(t *testing.T) { --- FAIL: TestPanicHelper/1 (N.NNs) panic_test.go:NNN: TestPanicHelper/1 `, + }, { + desc: "subtest panics with cleanup", + flags: []string{"-test_panic_test=TestPanicHelper/1", "-test_panic_cleanup"}, + want: ` +ran inner cleanup 1 +ran middle cleanup 1 +ran outer cleanup +--- FAIL: TestPanicHelper (N.NNs) + panic_test.go:NNN: TestPanicHelper + --- FAIL: TestPanicHelper/1 (N.NNs) + panic_test.go:NNN: TestPanicHelper/1 +`, + }, { + desc: "subtest panics with outer cleanup panic", + flags: []string{"-test_panic_test=TestPanicHelper/1", "-test_panic_cleanup", "-test_panic_cleanup_panic=outer"}, + want: ` +ran inner cleanup 1 +ran middle cleanup 1 +ran outer cleanup +--- FAIL: TestPanicHelper (N.NNs) + panic_test.go:NNN: TestPanicHelper +`, + }, { + desc: "subtest panics with middle cleanup panic", + flags: []string{"-test_panic_test=TestPanicHelper/1", "-test_panic_cleanup", "-test_panic_cleanup_panic=middle"}, + want: ` +ran inner cleanup 1 +ran middle cleanup 1 +ran outer cleanup +--- FAIL: TestPanicHelper (N.NNs) + panic_test.go:NNN: TestPanicHelper + --- FAIL: TestPanicHelper/1 (N.NNs) + panic_test.go:NNN: TestPanicHelper/1 +`, + }, { + desc: "subtest panics with inner cleanup panic", + flags: []string{"-test_panic_test=TestPanicHelper/1", "-test_panic_cleanup", "-test_panic_cleanup_panic=inner"}, + want: ` +ran inner cleanup 1 +ran middle cleanup 1 +ran outer cleanup +--- FAIL: TestPanicHelper (N.NNs) + panic_test.go:NNN: TestPanicHelper + --- FAIL: TestPanicHelper/1 (N.NNs) + panic_test.go:NNN: TestPanicHelper/1 +`, + }, { + desc: "parallel subtest panics with cleanup", + flags: []string{"-test_panic_test=TestPanicHelper/1", "-test_panic_cleanup", "-test_panic_parallel"}, + want: ` +ran inner cleanup 1 +ran middle cleanup 1 +ran outer cleanup +--- FAIL: TestPanicHelper (N.NNs) + panic_test.go:NNN: TestPanicHelper + --- FAIL: TestPanicHelper/1 (N.NNs) + panic_test.go:NNN: TestPanicHelper/1 +`, + }, { + desc: "parallel subtest panics with outer cleanup panic", + flags: []string{"-test_panic_test=TestPanicHelper/1", "-test_panic_cleanup", "-test_panic_cleanup_panic=outer", "-test_panic_parallel"}, + want: ` +ran inner cleanup 1 +ran middle cleanup 1 +ran outer cleanup +--- FAIL: TestPanicHelper (N.NNs) + panic_test.go:NNN: TestPanicHelper +`, + }, { + desc: "parallel subtest panics with middle cleanup panic", + flags: []string{"-test_panic_test=TestPanicHelper/1", "-test_panic_cleanup", "-test_panic_cleanup_panic=middle", "-test_panic_parallel"}, + want: ` +ran inner cleanup 1 +ran middle cleanup 1 +ran outer cleanup +--- FAIL: TestPanicHelper (N.NNs) + panic_test.go:NNN: TestPanicHelper + --- FAIL: TestPanicHelper/1 (N.NNs) + panic_test.go:NNN: TestPanicHelper/1 +`, + }, { + desc: "parallel subtest panics with inner cleanup panic", + flags: []string{"-test_panic_test=TestPanicHelper/1", "-test_panic_cleanup", "-test_panic_cleanup_panic=inner", "-test_panic_parallel"}, + want: ` +ran inner cleanup 1 +ran middle cleanup 1 +ran outer cleanup +--- FAIL: TestPanicHelper (N.NNs) + panic_test.go:NNN: TestPanicHelper + --- FAIL: TestPanicHelper/1 (N.NNs) + panic_test.go:NNN: TestPanicHelper/1 +`, }} for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { @@ -72,10 +167,42 @@ func TestPanicHelper(t *testing.T) { if t.Name() == *testPanicTest { panic("panic") } + switch *testPanicCleanupPanic { + case "", "outer", "middle", "inner": + default: + t.Fatalf("bad -test_panic_cleanup_panic: %s", *testPanicCleanupPanic) + } + t.Cleanup(func() { + fmt.Println("ran outer cleanup") + if *testPanicCleanupPanic == "outer" { + panic("outer cleanup") + } + }) for i := 0; i < 3; i++ { + i := i t.Run(fmt.Sprintf("%v", i), func(t *testing.T) { + chosen := t.Name() == *testPanicTest + if chosen && *testPanicCleanup { + t.Cleanup(func() { + fmt.Printf("ran middle cleanup %d\n", i) + if *testPanicCleanupPanic == "middle" { + panic("middle cleanup") + } + }) + } + if chosen && *testPanicParallel { + t.Parallel() + } t.Log(t.Name()) - if t.Name() == *testPanicTest { + if chosen { + if *testPanicCleanup { + t.Cleanup(func() { + fmt.Printf("ran inner cleanup %d\n", i) + if *testPanicCleanupPanic == "inner" { + panic("inner cleanup") + } + }) + } panic("panic") } }) diff --git a/libgo/go/testing/sub_test.go b/libgo/go/testing/sub_test.go index 3f0f71f..3dc30ee 100644 --- a/libgo/go/testing/sub_test.go +++ b/libgo/go/testing/sub_test.go @@ -460,6 +460,21 @@ func TestTRun(t *T) { <-ch t.Errorf("error") }, + }, { + // If a subtest panics we should run cleanups. + desc: "cleanup when subtest panics", + ok: false, + chatty: false, + output: ` +--- FAIL: cleanup when subtest panics (N.NNs) + --- FAIL: cleanup when subtest panics/sub (N.NNs) + sub_test.go:NNN: running cleanup`, + f: func(t *T) { + t.Cleanup(func() { t.Log("running cleanup") }) + t.Run("sub", func(t2 *T) { + t2.FailNow() + }) + }, }} for _, tc := range testCases { ctx := newTestContext(tc.maxPar, newMatcher(regexp.MatchString, "", "")) @@ -855,3 +870,19 @@ func TestRunCleanup(t *T) { t.Errorf("unexpected outer cleanup count; got %d want 0", outerCleanup) } } + +func TestCleanupParallelSubtests(t *T) { + ranCleanup := 0 + t.Run("test", func(t *T) { + t.Cleanup(func() { ranCleanup++ }) + t.Run("x", func(t *T) { + t.Parallel() + if ranCleanup > 0 { + t.Error("outer cleanup ran before parallel subtest") + } + }) + }) + if ranCleanup != 1 { + t.Errorf("unexpected cleanup count; got %d want 1", ranCleanup) + } +} diff --git a/libgo/go/testing/testing.go b/libgo/go/testing/testing.go index e05314e..67892d7 100644 --- a/libgo/go/testing/testing.go +++ b/libgo/go/testing/testing.go @@ -812,9 +812,9 @@ func (c *common) Helper() { c.helpers[callerName(1)] = struct{}{} } -// Cleanup registers a function to be called when the test finishes. -// Cleanup functions will be called in last added, first called -// order. +// Cleanup registers a function to be called when the test and all its +// subtests complete. Cleanup functions will be called in last added, +// first called order. func (c *common) Cleanup(f func()) { c.mu.Lock() defer c.mu.Unlock() @@ -827,15 +827,34 @@ func (c *common) Cleanup(f func()) { } } +// panicHanding is an argument to runCleanup. +type panicHandling int + +const ( + normalPanic panicHandling = iota + recoverAndReturnPanic +) + // runCleanup is called at the end of the test. -func (c *common) runCleanup() { +// If catchPanic is true, this will catch panics, and return the recovered +// value if any. +func (c *common) runCleanup(ph panicHandling) (panicVal interface{}) { c.mu.Lock() cleanup := c.cleanup c.cleanup = nil c.mu.Unlock() - if cleanup != nil { - cleanup() + if cleanup == nil { + return nil + } + + if ph == recoverAndReturnPanic { + defer func() { + panicVal = recover() + }() } + + cleanup() + return nil } // callerName gives the function name (qualified with a package path) @@ -938,19 +957,29 @@ func tRunner(t *T, fn func(t *T)) { } } } - if err != nil { + + doPanic := func(err interface{}) { t.Fail() + if r := t.runCleanup(recoverAndReturnPanic); r != nil { + t.Logf("cleanup panicked with %v", r) + } // Flush the output log up to the root before dying. t.mu.Lock() root := &t.common for ; root.parent != nil; root = root.parent { root.duration += time.Since(root.start) fmt.Fprintf(root.parent.w, "--- FAIL: %s (%s)\n", root.name, fmtDuration(root.duration)) + if r := root.parent.runCleanup(recoverAndReturnPanic); r != nil { + fmt.Fprintf(root.parent.w, "cleanup panicked with %v", r) + } root.parent.mu.Lock() io.Copy(root.parent.w, bytes.NewReader(root.output)) } panic(err) } + if err != nil { + doPanic(err) + } t.duration += time.Since(t.start) @@ -964,6 +993,12 @@ func tRunner(t *T, fn func(t *T)) { for _, sub := range t.sub { <-sub.signal } + cleanupStart := time.Now() + err := t.runCleanup(recoverAndReturnPanic) + t.duration += time.Since(cleanupStart) + if err != nil { + doPanic(err) + } if !t.isParallel { // Reacquire the count for sequential tests. See comment in Run. t.context.waitParallel() @@ -983,7 +1018,11 @@ func tRunner(t *T, fn func(t *T)) { } t.signal <- signal }() - defer t.runCleanup() + defer func() { + if len(t.sub) == 0 { + t.runCleanup(normalPanic) + } + }() t.start = time.Now() t.raceErrors = -race.Errors() diff --git a/libgo/go/text/template/exec_test.go b/libgo/go/text/template/exec_test.go index c64b835..f92ac6f 100644 --- a/libgo/go/text/template/exec_test.go +++ b/libgo/go/text/template/exec_test.go @@ -502,6 +502,7 @@ var execTests = []execTest{ {"map MUI64S", "{{index .MUI64S 3}}", "ui643", tVal, true}, {"map MI8S", "{{index .MI8S 3}}", "i83", tVal, true}, {"map MUI8S", "{{index .MUI8S 2}}", "u82", tVal, true}, + {"index of an interface field", "{{index .Empty3 0}}", "7", tVal, true}, // Slicing. {"slice[:]", "{{slice .SI}}", "[3 4 5]", tVal, true}, @@ -527,12 +528,14 @@ var execTests = []execTest{ {"string[1:2]", "{{slice .S 1 2}}", "y", tVal, true}, {"out of range", "{{slice .S 1 5}}", "", tVal, false}, {"3-index slice of string", "{{slice .S 1 2 2}}", "", tVal, false}, + {"slice of an interface field", "{{slice .Empty3 0 1}}", "[7]", tVal, true}, // Len. {"slice", "{{len .SI}}", "3", tVal, true}, {"map", "{{len .MSI }}", "3", tVal, true}, {"len of int", "{{len 3}}", "", tVal, false}, {"len of nothing", "{{len .Empty0}}", "", tVal, false}, + {"len of an interface field", "{{len .Empty3}}", "2", tVal, true}, // With. {"with true", "{{with true}}{{.}}{{end}}", "true", tVal, true}, diff --git a/libgo/go/text/template/funcs.go b/libgo/go/text/template/funcs.go index 0568c79..46125bc 100644 --- a/libgo/go/text/template/funcs.go +++ b/libgo/go/text/template/funcs.go @@ -264,13 +264,13 @@ func slice(item reflect.Value, indexes ...reflect.Value) (reflect.Value, error) return reflect.Value{}, fmt.Errorf("invalid slice index: %d > %d", idx[0], idx[1]) } if len(indexes) < 3 { - return item.Slice(idx[0], idx[1]), nil + return v.Slice(idx[0], idx[1]), nil } // given item[i:j:k], make sure i <= j <= k. if idx[1] > idx[2] { return reflect.Value{}, fmt.Errorf("invalid slice index: %d > %d", idx[1], idx[2]) } - return item.Slice3(idx[0], idx[1], idx[2]), nil + return v.Slice3(idx[0], idx[1], idx[2]), nil } // Length diff --git a/libgo/go/text/template/parse/lex.go b/libgo/go/text/template/parse/lex.go index 3d57708..30371f2 100644 --- a/libgo/go/text/template/parse/lex.go +++ b/libgo/go/text/template/parse/lex.go @@ -411,7 +411,6 @@ func lexInsideAction(l *lexer) stateFn { } case r <= unicode.MaxASCII && unicode.IsPrint(r): l.emit(itemChar) - return lexInsideAction default: return l.errorf("unrecognized character in action: %#U", r) } diff --git a/libgo/go/time/format.go b/libgo/go/time/format.go index b531cb4..9beb5d9 100644 --- a/libgo/go/time/format.go +++ b/libgo/go/time/format.go @@ -792,6 +792,9 @@ func skip(value, prefix string) (string, error) { // Years must be in the range 0000..9999. The day of the week is checked // for syntax but it is otherwise ignored. // +// For layouts specifying the two-digit year 06, a value NN >= 69 will be treated +// as 19NN and a value NN < 69 will be treated as 20NN. +// // In the absence of a time zone indicator, Parse returns a time in UTC. // // When parsing a time with a zone offset like -0700, if the offset corresponds diff --git a/libgo/go/time/sleep_test.go b/libgo/go/time/sleep_test.go index 26cc488..9c4de6d 100644 --- a/libgo/go/time/sleep_test.go +++ b/libgo/go/time/sleep_test.go @@ -357,7 +357,7 @@ func TestTimerStopStress(t *testing.T) { for i := 0; i < 100; i++ { go func(i int) { timer := AfterFunc(2*Second, func() { - t.Fatalf("timer %d was not stopped", i) + t.Errorf("timer %d was not stopped", i) }) Sleep(1 * Second) timer.Stop() diff --git a/libgo/go/time/time.go b/libgo/go/time/time.go index 10a132f..5dc9fa6 100644 --- a/libgo/go/time/time.go +++ b/libgo/go/time/time.go @@ -1148,6 +1148,9 @@ func (t Time) Zone() (name string, offset int) { // Unix returns t as a Unix time, the number of seconds elapsed // since January 1, 1970 UTC. The result does not depend on the // location associated with t. +// Unix-like operating systems often record time as a 32-bit +// count of seconds, but since the method here returns a 64-bit +// value it is valid for billions of years into the past or future. func (t Time) Unix() int64 { return t.unixSec() } diff --git a/libgo/go/unicode/utf8/utf8.go b/libgo/go/unicode/utf8/utf8.go index b722a03..b8368fc 100644 --- a/libgo/go/unicode/utf8/utf8.go +++ b/libgo/go/unicode/utf8/utf8.go @@ -14,7 +14,7 @@ package utf8 // Numbers fundamental to the encoding. const ( RuneError = '\uFFFD' // the "error" Rune or "Unicode replacement character" - RuneSelf = 0x80 // characters below Runeself are represented as themselves in a single byte. + RuneSelf = 0x80 // characters below RuneSelf are represented as themselves in a single byte. MaxRune = '\U0010FFFF' // Maximum valid Unicode code point. UTFMax = 4 // maximum number of bytes of a UTF-8 encoded Unicode character. ) |