diff options
author | Ian Lance Taylor <iant@golang.org> | 2020-12-23 09:57:37 -0800 |
---|---|---|
committer | Ian Lance Taylor <iant@golang.org> | 2020-12-30 15:13:24 -0800 |
commit | cfcbb4227fb20191e04eb8d7766ae6202f526afd (patch) | |
tree | e2effea96f6f204451779f044415c2385e45042b /libgo/go/cmd | |
parent | 0696141107d61483f38482b941549959a0d7f613 (diff) | |
download | gcc-cfcbb4227fb20191e04eb8d7766ae6202f526afd.zip gcc-cfcbb4227fb20191e04eb8d7766ae6202f526afd.tar.gz gcc-cfcbb4227fb20191e04eb8d7766ae6202f526afd.tar.bz2 |
libgo: update to Go1.16beta1 release
This does not yet include support for the //go:embed directive added
in this release.
* Makefile.am (check-runtime): Don't create check-runtime-dir.
(mostlyclean-local): Don't remove check-runtime-dir.
(check-go-tool, check-vet): Copy in go.mod and modules.txt.
(check-cgo-test, check-carchive-test): Add go.mod file.
* Makefile.in: Regenerate.
Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/280172
Diffstat (limited to 'libgo/go/cmd')
281 files changed, 12685 insertions, 4833 deletions
diff --git a/libgo/go/cmd/buildid/buildid.go b/libgo/go/cmd/buildid/buildid.go index 1c7b228..8e02a7a 100644 --- a/libgo/go/cmd/buildid/buildid.go +++ b/libgo/go/cmd/buildid/buildid.go @@ -22,21 +22,6 @@ func usage() { var wflag = flag.Bool("w", false, "write build ID") -// taken from cmd/go/internal/work/buildid.go -func hashToString(h [32]byte) string { - const b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" - const chunks = 5 - var dst [chunks * 4]byte - for i := 0; i < chunks; i++ { - v := uint32(h[3*i])<<16 | uint32(h[3*i+1])<<8 | uint32(h[3*i+2]) - dst[4*i+0] = b64[(v>>18)&0x3F] - dst[4*i+1] = b64[(v>>12)&0x3F] - dst[4*i+2] = b64[(v>>6)&0x3F] - dst[4*i+3] = b64[v&0x3F] - } - return string(dst[:]) -} - func main() { log.SetPrefix("buildid: ") log.SetFlags(0) @@ -63,12 +48,12 @@ func main() { log.Fatal(err) } matches, hash, err := buildid.FindAndHash(f, id, 0) + f.Close() if err != nil { log.Fatal(err) } - f.Close() - newID := id[:strings.LastIndex(id, "/")] + "/" + hashToString(hash) + newID := id[:strings.LastIndex(id, "/")] + "/" + buildid.HashToString(hash) if len(newID) != len(id) { log.Fatalf("%s: build ID length mismatch %q vs %q", file, id, newID) } @@ -77,7 +62,7 @@ func main() { return } - f, err = os.OpenFile(file, os.O_WRONLY, 0) + f, err = os.OpenFile(file, os.O_RDWR, 0) if err != nil { log.Fatal(err) } diff --git a/libgo/go/cmd/cgo/ast.go b/libgo/go/cmd/cgo/ast.go index 54d6bc2..a073407 100644 --- a/libgo/go/cmd/cgo/ast.go +++ b/libgo/go/cmd/cgo/ast.go @@ -13,7 +13,6 @@ import ( "go/scanner" "go/token" "os" - "path/filepath" "strings" ) @@ -44,14 +43,7 @@ func sourceLine(n ast.Node) int { // attached to the import "C" comment, a list of references to C.xxx, // a list of exported functions, and the actual AST, to be rewritten and // printed. -func (f *File) ParseGo(name string, src []byte) { - // Create absolute path for file, so that it will be used in error - // messages and recorded in debug line number information. - // This matches the rest of the toolchain. See golang.org/issue/5122. - if aname, err := filepath.Abs(name); err == nil { - name = aname - } - +func (f *File) ParseGo(abspath string, src []byte) { // Two different parses: once with comments, once without. // The printer is not good enough at printing comments in the // right place when we start editing the AST behind its back, @@ -60,8 +52,8 @@ func (f *File) ParseGo(name string, src []byte) { // and reprinting. // In cgo mode, we ignore ast2 and just apply edits directly // the text behind ast1. In godefs mode we modify and print ast2. - ast1 := parse(name, src, parser.ParseComments) - ast2 := parse(name, src, 0) + ast1 := parse(abspath, src, parser.ParseComments) + ast2 := parse(abspath, src, 0) f.Package = ast1.Name.Name f.Name = make(map[string]*Name) @@ -88,7 +80,7 @@ func (f *File) ParseGo(name string, src []byte) { cg = d.Doc } if cg != nil { - f.Preamble += fmt.Sprintf("#line %d %q\n", sourceLine(cg), name) + f.Preamble += fmt.Sprintf("#line %d %q\n", sourceLine(cg), abspath) f.Preamble += commentText(cg) + "\n" f.Preamble += "#line 1 \"cgo-generated-wrapper\"\n" } diff --git a/libgo/go/cmd/cgo/doc.go b/libgo/go/cmd/cgo/doc.go index ca18c45..e782c86 100644 --- a/libgo/go/cmd/cgo/doc.go +++ b/libgo/go/cmd/cgo/doc.go @@ -112,6 +112,13 @@ The default C and C++ compilers may be changed by the CC and CXX environment variables, respectively; those environment variables may include command line options. +The cgo tool will always invoke the C compiler with the source file's +directory in the include path; i.e. -I${SRCDIR} is always implied. This +means that if a header file foo/bar.h exists both in the source +directory and also in the system include directory (or some other place +specified by a -I flag), then "#include <foo/bar.h>" will always find the +local version in preference to any other version. + The cgo tool is enabled by default for native builds on systems where it is expected to work. It is disabled by default when cross-compiling. You can control this by setting the CGO_ENABLED @@ -714,7 +721,7 @@ linkage to the desired libraries. The main function is provided by _cgo_main.c: int main() { return 0; } - void crosscall2(void(*fn)(void*, int, uintptr_t), void *a, int c, uintptr_t ctxt) { } + void crosscall2(void(*fn)(void*), void *a, int c, uintptr_t ctxt) { } uintptr_t _cgo_wait_runtime_init_done(void) { return 0; } void _cgo_release_context(uintptr_t ctxt) { } char* _cgo_topofstack(void) { return (char*)0; } diff --git a/libgo/go/cmd/cgo/gcc.go b/libgo/go/cmd/cgo/gcc.go index 37bafcf..796ff8a 100644 --- a/libgo/go/cmd/cgo/gcc.go +++ b/libgo/go/cmd/cgo/gcc.go @@ -316,7 +316,7 @@ func (p *Package) guessKinds(f *File) []*Name { continue } - if goos == "darwin" && strings.HasSuffix(n.C, "Ref") { + if (goos == "darwin" || goos == "ios") && strings.HasSuffix(n.C, "Ref") { // For FooRef, find out if FooGetTypeID exists. s := n.C[:len(n.C)-3] + "GetTypeID" n := &Name{Go: s, C: s} @@ -387,7 +387,18 @@ func (p *Package) guessKinds(f *File) []*Name { fmt.Fprintf(&b, "#line 1 \"completed\"\n"+ "int __cgo__1 = __cgo__2;\n") - stderr := p.gccErrors(b.Bytes()) + // We need to parse the output from this gcc command, so ensure that it + // doesn't have any ANSI escape sequences in it. (TERM=dumb is + // insufficient; if the user specifies CGO_CFLAGS=-fdiagnostics-color, + // GCC will ignore TERM, and GCC can also be configured at compile-time + // to ignore TERM.) + stderr := p.gccErrors(b.Bytes(), "-fdiagnostics-color=never") + if strings.Contains(stderr, "unrecognized command line option") { + // We're using an old version of GCC that doesn't understand + // -fdiagnostics-color. Those versions can't print color anyway, + // so just rerun without that option. + stderr = p.gccErrors(b.Bytes()) + } if stderr == "" { fatalf("%s produced no output\non input:\n%s", p.gccBaseCmd()[0], b.Bytes()) } @@ -1995,22 +2006,25 @@ func (p *Package) gccDefines(stdin []byte) string { // gccErrors runs gcc over the C program stdin and returns // the errors that gcc prints. That is, this function expects // gcc to fail. -func (p *Package) gccErrors(stdin []byte) string { +func (p *Package) gccErrors(stdin []byte, extraArgs ...string) string { // TODO(rsc): require failure args := p.gccCmd() // Optimization options can confuse the error messages; remove them. - nargs := make([]string, 0, len(args)) + nargs := make([]string, 0, len(args)+len(extraArgs)) for _, arg := range args { if !strings.HasPrefix(arg, "-O") { nargs = append(nargs, arg) } } - // Force -O0 optimization but keep the trailing "-" at the end. - nargs = append(nargs, "-O0") - nl := len(nargs) - nargs[nl-2], nargs[nl-1] = nargs[nl-1], nargs[nl-2] + // Force -O0 optimization and append extra arguments, but keep the + // trailing "-" at the end. + li := len(nargs) - 1 + last := nargs[li] + nargs[li] = "-O0" + nargs = append(nargs, extraArgs...) + nargs = append(nargs, last) if *debugGcc { fmt.Fprintf(os.Stderr, "$ %s <<EOF\n", strings.Join(nargs, " ")) @@ -2856,21 +2870,11 @@ func (c *typeConv) Struct(dt *dwarf.StructType, pos token.Pos) (expr *ast.Struct tgo := t.Go size := t.Size talign := t.Align - if f.BitSize > 0 { - switch f.BitSize { - case 8, 16, 32, 64: - default: - continue - } - size = f.BitSize / 8 - name := tgo.(*ast.Ident).String() - if strings.HasPrefix(name, "int") { - name = "int" - } else { - name = "uint" - } - tgo = ast.NewIdent(name + fmt.Sprint(f.BitSize)) - talign = size + if f.BitOffset > 0 || f.BitSize > 0 { + // The layout of bitfields is implementation defined, + // so we don't know how they correspond to Go fields + // even if they are aligned at byte boundaries. + continue } if talign > 0 && f.ByteOffset%talign != 0 { @@ -3096,7 +3100,7 @@ func (c *typeConv) badCFType(dt *dwarf.TypedefType) bool { // We identify the correct set of types as those ending in Ref and for which // there exists a corresponding GetTypeID function. // See comment below for details about the bad pointers. - if goos != "darwin" { + if goos != "darwin" && goos != "ios" { return false } s := dt.Name diff --git a/libgo/go/cmd/cgo/godefs.go b/libgo/go/cmd/cgo/godefs.go index b4fd9c5..c0d59ae 100644 --- a/libgo/go/cmd/cgo/godefs.go +++ b/libgo/go/cmd/cgo/godefs.go @@ -16,7 +16,7 @@ import ( ) // godefs returns the output for -godefs mode. -func (p *Package) godefs(f *File, srcfile string) string { +func (p *Package) godefs(f *File) string { var buf bytes.Buffer fmt.Fprintf(&buf, "// Code generated by cmd/cgo -godefs; DO NOT EDIT.\n") diff --git a/libgo/go/cmd/cgo/main.go b/libgo/go/cmd/cgo/main.go index 4952118..c47e991 100644 --- a/libgo/go/cmd/cgo/main.go +++ b/libgo/go/cmd/cgo/main.go @@ -247,6 +247,8 @@ var gccgopkgpath = flag.String("gccgopkgpath", "", "-fgo-pkgpath option used wit var gccgoMangler func(string) string var importRuntimeCgo = flag.Bool("import_runtime_cgo", true, "import runtime/cgo in generated code") var importSyscall = flag.Bool("import_syscall", true, "import syscall in generated code") +var trimpath = flag.String("trimpath", "", "applies supplied rewrites or trims prefixes to recorded source file paths") + var goarch, goos string func main() { @@ -326,6 +328,13 @@ func main() { input = filepath.Join(*srcDir, input) } + // Create absolute path for file, so that it will be used in error + // messages and recorded in debug line number information. + // This matches the rest of the toolchain. See golang.org/issue/5122. + if aname, err := filepath.Abs(input); err == nil { + input = aname + } + b, err := ioutil.ReadFile(input) if err != nil { fatalf("%s", err) @@ -334,6 +343,10 @@ func main() { fatalf("%s", err) } + // Apply trimpath to the file path. The path won't be read from after this point. + input, _ = objabi.ApplyRewrites(input, *trimpath) + goFiles[i] = input + f := new(File) f.Edit = edit.NewBuffer(b) f.ParseGo(input, b) @@ -371,7 +384,7 @@ func main() { p.PackagePath = f.Package p.Record(f) if *godefs { - os.Stdout.WriteString(p.godefs(f, input)) + os.Stdout.WriteString(p.godefs(f)) } else { p.writeOutput(f, input) } diff --git a/libgo/go/cmd/cgo/out.go b/libgo/go/cmd/cgo/out.go index 1c143d7..6cdebac 100644 --- a/libgo/go/cmd/cgo/out.go +++ b/libgo/go/cmd/cgo/out.go @@ -64,14 +64,14 @@ func (p *Package) writeDefs() { // Write C main file for using gcc to resolve imports. fmt.Fprintf(fm, "int main() { return 0; }\n") if *importRuntimeCgo { - fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*, int, __SIZE_TYPE__), void *a, int c, __SIZE_TYPE__ ctxt) { }\n") + fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*), void *a, int c, __SIZE_TYPE__ ctxt) { }\n") fmt.Fprintf(fm, "__SIZE_TYPE__ _cgo_wait_runtime_init_done(void) { return 0; }\n") fmt.Fprintf(fm, "void _cgo_release_context(__SIZE_TYPE__ ctxt) { }\n") fmt.Fprintf(fm, "char* _cgo_topofstack(void) { return (char*)0; }\n") } else { // If we're not importing runtime/cgo, we *are* runtime/cgo, // which provides these functions. We just need a prototype. - fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*, int, __SIZE_TYPE__), void *a, int c, __SIZE_TYPE__ ctxt);\n") + fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*), void *a, int c, __SIZE_TYPE__ ctxt);\n") fmt.Fprintf(fm, "__SIZE_TYPE__ _cgo_wait_runtime_init_done(void);\n") fmt.Fprintf(fm, "void _cgo_release_context(__SIZE_TYPE__);\n") } @@ -251,6 +251,7 @@ func (p *Package) writeDefs() { if err != nil { fatalf("%s", err) } + defer fgcch.Close() _, err = io.Copy(fexp, fgcch) if err != nil { fatalf("%s", err) @@ -879,7 +880,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { fmt.Fprintf(fgcc, "#pragma GCC diagnostic ignored \"-Wpragmas\"\n") fmt.Fprintf(fgcc, "#pragma GCC diagnostic ignored \"-Waddress-of-packed-member\"\n") - fmt.Fprintf(fgcc, "extern void crosscall2(void (*fn)(void *, int, __SIZE_TYPE__), void *, int, __SIZE_TYPE__);\n") + fmt.Fprintf(fgcc, "extern void crosscall2(void (*fn)(void *), void *, int, __SIZE_TYPE__);\n") fmt.Fprintf(fgcc, "extern __SIZE_TYPE__ _cgo_wait_runtime_init_done(void);\n") fmt.Fprintf(fgcc, "extern void _cgo_release_context(__SIZE_TYPE__);\n\n") fmt.Fprintf(fgcc, "extern char* _cgo_topofstack(void);") @@ -889,59 +890,48 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { for _, exp := range p.ExpFunc { fn := exp.Func - // Construct a gcc struct matching the gc argument and - // result frame. The gcc struct will be compiled with - // __attribute__((packed)) so all padding must be accounted - // for explicitly. + // Construct a struct that will be used to communicate + // arguments from C to Go. The C and Go definitions + // just have to agree. The gcc struct will be compiled + // with __attribute__((packed)) so all padding must be + // accounted for explicitly. ctype := "struct {\n" + gotype := new(bytes.Buffer) + fmt.Fprintf(gotype, "struct {\n") off := int64(0) npad := 0 - if fn.Recv != nil { - t := p.cgoType(fn.Recv.List[0].Type) - ctype += fmt.Sprintf("\t\t%s recv;\n", t.C) + argField := func(typ ast.Expr, namePat string, args ...interface{}) { + name := fmt.Sprintf(namePat, args...) + t := p.cgoType(typ) + if off%t.Align != 0 { + pad := t.Align - off%t.Align + ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) + off += pad + npad++ + } + ctype += fmt.Sprintf("\t\t%s %s;\n", t.C, name) + fmt.Fprintf(gotype, "\t\t%s ", name) + noSourceConf.Fprint(gotype, fset, typ) + fmt.Fprintf(gotype, "\n") off += t.Size } + if fn.Recv != nil { + argField(fn.Recv.List[0].Type, "recv") + } fntype := fn.Type forFieldList(fntype.Params, func(i int, aname string, atype ast.Expr) { - t := p.cgoType(atype) - if off%t.Align != 0 { - pad := t.Align - off%t.Align - ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) - off += pad - npad++ - } - ctype += fmt.Sprintf("\t\t%s p%d;\n", t.C, i) - off += t.Size + argField(atype, "p%d", i) }) - if off%p.PtrSize != 0 { - pad := p.PtrSize - off%p.PtrSize - ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) - off += pad - npad++ - } forFieldList(fntype.Results, func(i int, aname string, atype ast.Expr) { - t := p.cgoType(atype) - if off%t.Align != 0 { - pad := t.Align - off%t.Align - ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) - off += pad - npad++ - } - ctype += fmt.Sprintf("\t\t%s r%d;\n", t.C, i) - off += t.Size + argField(atype, "r%d", i) }) - if off%p.PtrSize != 0 { - pad := p.PtrSize - off%p.PtrSize - ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) - off += pad - npad++ - } if ctype == "struct {\n" { ctype += "\t\tchar unused;\n" // avoid empty struct } ctype += "\t}" + fmt.Fprintf(gotype, "\t}") // Get the return type of the wrapper function // compiled by gcc. @@ -966,7 +956,11 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { } // Build the wrapper function compiled by gcc. - s := fmt.Sprintf("%s %s(", gccResult, exp.ExpName) + gccExport := "" + if goos == "windows" { + gccExport = "__declspec(dllexport)" + } + s := fmt.Sprintf("%s %s %s(", gccExport, gccResult, exp.ExpName) if fn.Recv != nil { s += p.cgoType(fn.Recv.List[0].Type).C.String() s += " recv" @@ -988,12 +982,23 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { } fmt.Fprintf(fgcch, "extern %s;\n", s) - fmt.Fprintf(fgcc, "extern void _cgoexp%s_%s(void *, int, __SIZE_TYPE__);\n", cPrefix, exp.ExpName) + fmt.Fprintf(fgcc, "extern void _cgoexp%s_%s(void *);\n", cPrefix, exp.ExpName) fmt.Fprintf(fgcc, "\nCGO_NO_SANITIZE_THREAD") fmt.Fprintf(fgcc, "\n%s\n", s) fmt.Fprintf(fgcc, "{\n") fmt.Fprintf(fgcc, "\t__SIZE_TYPE__ _cgo_ctxt = _cgo_wait_runtime_init_done();\n") - fmt.Fprintf(fgcc, "\t%s %v _cgo_a;\n", ctype, p.packedAttribute()) + // The results part of the argument structure must be + // initialized to 0 so the write barriers generated by + // the assignments to these fields in Go are safe. + // + // We use a local static variable to get the zeroed + // value of the argument type. This avoids including + // string.h for memset, and is also robust to C++ + // types with constructors. Both GCC and LLVM optimize + // this into just zeroing _cgo_a. + fmt.Fprintf(fgcc, "\ttypedef %s %v _cgo_argtype;\n", ctype, p.packedAttribute()) + fmt.Fprintf(fgcc, "\tstatic _cgo_argtype _cgo_zero;\n") + fmt.Fprintf(fgcc, "\t_cgo_argtype _cgo_a = _cgo_zero;\n") if gccResult != "void" && (len(fntype.Results.List) > 1 || len(fntype.Results.List[0].Names) > 1) { fmt.Fprintf(fgcc, "\t%s r;\n", gccResult) } @@ -1022,82 +1027,28 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { fmt.Fprintf(fgcc, "}\n") // Build the wrapper function compiled by cmd/compile. - goname := "_cgoexpwrap" + cPrefix + "_" - if fn.Recv != nil { - goname += fn.Recv.List[0].Names[0].Name + "_" - } - goname += exp.Func.Name.Name + // This unpacks the argument struct above and calls the Go function. fmt.Fprintf(fgo2, "//go:cgo_export_dynamic %s\n", exp.ExpName) fmt.Fprintf(fgo2, "//go:linkname _cgoexp%s_%s _cgoexp%s_%s\n", cPrefix, exp.ExpName, cPrefix, exp.ExpName) fmt.Fprintf(fgo2, "//go:cgo_export_static _cgoexp%s_%s\n", cPrefix, exp.ExpName) - fmt.Fprintf(fgo2, "//go:nosplit\n") // no split stack, so no use of m or g - fmt.Fprintf(fgo2, "//go:norace\n") // must not have race detector calls inserted - fmt.Fprintf(fgo2, "func _cgoexp%s_%s(a unsafe.Pointer, n int32, ctxt uintptr) {\n", cPrefix, exp.ExpName) - fmt.Fprintf(fgo2, "\tfn := %s\n", goname) - // The indirect here is converting from a Go function pointer to a C function pointer. - fmt.Fprintf(fgo2, "\t_cgo_runtime_cgocallback(**(**unsafe.Pointer)(unsafe.Pointer(&fn)), a, uintptr(n), ctxt);\n") - fmt.Fprintf(fgo2, "}\n") + fmt.Fprintf(fgo2, "func _cgoexp%s_%s(a *%s) {\n", cPrefix, exp.ExpName, gotype) fmt.Fprintf(fm, "int _cgoexp%s_%s;\n", cPrefix, exp.ExpName) - // This code uses printer.Fprint, not conf.Fprint, - // because we don't want //line comments in the middle - // of the function types. - fmt.Fprintf(fgo2, "\n") - fmt.Fprintf(fgo2, "func %s(", goname) - comma := false - if fn.Recv != nil { - fmt.Fprintf(fgo2, "recv ") - printer.Fprint(fgo2, fset, fn.Recv.List[0].Type) - comma = true - } - forFieldList(fntype.Params, - func(i int, aname string, atype ast.Expr) { - if comma { - fmt.Fprintf(fgo2, ", ") - } - fmt.Fprintf(fgo2, "p%d ", i) - printer.Fprint(fgo2, fset, atype) - comma = true - }) - fmt.Fprintf(fgo2, ")") if gccResult != "void" { - fmt.Fprint(fgo2, " (") + // Write results back to frame. + fmt.Fprintf(fgo2, "\t") forFieldList(fntype.Results, func(i int, aname string, atype ast.Expr) { if i > 0 { - fmt.Fprint(fgo2, ", ") - } - fmt.Fprintf(fgo2, "r%d ", i) - printer.Fprint(fgo2, fset, atype) - }) - fmt.Fprint(fgo2, ")") - } - fmt.Fprint(fgo2, " {\n") - if gccResult == "void" { - fmt.Fprint(fgo2, "\t") - } else { - // Verify that any results don't contain any - // Go pointers. - addedDefer := false - forFieldList(fntype.Results, - func(i int, aname string, atype ast.Expr) { - if !p.hasPointer(nil, atype, false) { - return + fmt.Fprintf(fgo2, ", ") } - if !addedDefer { - fmt.Fprint(fgo2, "\tdefer func() {\n") - addedDefer = true - } - fmt.Fprintf(fgo2, "\t\t_cgoCheckResult(r%d)\n", i) + fmt.Fprintf(fgo2, "a.r%d", i) }) - if addedDefer { - fmt.Fprint(fgo2, "\t}()\n") - } - fmt.Fprint(fgo2, "\treturn ") + fmt.Fprintf(fgo2, " = ") } if fn.Recv != nil { - fmt.Fprintf(fgo2, "recv.") + fmt.Fprintf(fgo2, "a.recv.") } fmt.Fprintf(fgo2, "%s(", exp.Func.Name) forFieldList(fntype.Params, @@ -1105,9 +1056,20 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { if i > 0 { fmt.Fprint(fgo2, ", ") } - fmt.Fprintf(fgo2, "p%d", i) + fmt.Fprintf(fgo2, "a.p%d", i) }) fmt.Fprint(fgo2, ")\n") + if gccResult != "void" { + // Verify that any results don't contain any + // Go pointers. + forFieldList(fntype.Results, + func(i int, aname string, atype ast.Expr) { + if !p.hasPointer(nil, atype, false) { + return + } + fmt.Fprintf(fgo2, "\t_cgoCheckResult(a.r%d)\n", i) + }) + } fmt.Fprint(fgo2, "}\n") } @@ -1604,9 +1566,6 @@ const goProlog = ` //go:linkname _cgo_runtime_cgocall runtime.cgocall func _cgo_runtime_cgocall(unsafe.Pointer, uintptr) int32 -//go:linkname _cgo_runtime_cgocallback runtime.cgocallback -func _cgo_runtime_cgocallback(unsafe.Pointer, unsafe.Pointer, uintptr, uintptr) - //go:linkname _cgoCheckPointer runtime.cgoCheckPointer func _cgoCheckPointer(interface{}, interface{}) diff --git a/libgo/go/cmd/go.mod b/libgo/go/cmd/go.mod new file mode 100644 index 0000000..031b8d4 --- /dev/null +++ b/libgo/go/cmd/go.mod @@ -0,0 +1,12 @@ +module cmd + +go 1.16 + +require ( + github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2 + golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff + golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 + golang.org/x/mod v0.4.0 + golang.org/x/sys v0.0.0-20201204225414-ed752295db88 // indirect + golang.org/x/tools v0.0.0-20201211025543-abf6a1d87e11 +) diff --git a/libgo/go/cmd/go/alldocs.go b/libgo/go/cmd/go/alldocs.go index 68bad3c..c4913ce 100644 --- a/libgo/go/cmd/go/alldocs.go +++ b/libgo/go/cmd/go/alldocs.go @@ -49,10 +49,11 @@ // modules modules, module versions, and more // module-get module-aware go get // module-auth module authentication using go.sum -// module-private module configuration for non-public modules // packages package lists and patterns +// private configuration for downloading non-public code // testflag testing flags // testfunc testing functions +// vcs controlling version control with GOVCS // // Use "go help <topic>" for more information about that topic. // @@ -71,7 +72,7 @@ // // Usage: // -// go build [-o output] [-i] [build flags] [packages] +// go build [-o output] [build flags] [packages] // // Build compiles the packages named by the import paths, // along with their dependencies, but it does not install the results. @@ -93,10 +94,12 @@ // // The -o flag forces build to write the resulting executable or object // to the named output file or directory, instead of the default behavior described -// in the last two paragraphs. If the named output is a directory that exists, -// then any resulting executables will be written to that directory. +// in the last two paragraphs. If the named output is an existing directory or +// ends with a slash or backslash, then any resulting executables +// will be written to that directory. // // The -i flag installs the packages that are dependencies of the target. +// The -i flag is deprecated. Compiled packages are cached automatically. // // The build flags are shared by the build, clean, get, install, list, run, // and test commands: @@ -161,6 +164,17 @@ // directory, but it is not accessed. When -modfile is specified, an // alternate go.sum file is also used: its path is derived from the // -modfile flag by trimming the ".mod" extension and appending ".sum". +// -overlay file +// read a JSON config file that provides an overlay for build operations. +// The file is a JSON struct with a single field, named 'Replace', that +// maps each disk file path (a string) to its backing file path, so that +// a build will run as if the disk file path exists with the contents +// given by the backing file paths, or as if the disk file path does not +// exist if its backing file path is empty. Support for the -overlay flag +// has some limitations:importantly, cgo files included from outside the +// include path must be in the same directory as the Go package they are +// included from, and overlays will not appear when binaries and tests are +// run through go run and go test respectively. // -pkgdir dir // install and load all packages from dir instead of the usual locations. // For example, when building with a non-standard configuration, @@ -616,15 +630,15 @@ // dependency should be removed entirely, downgrading or removing modules // depending on it as needed. // -// The version suffix @latest explicitly requests the latest minor release of the -// module named by the given path. The suffix @upgrade is like @latest but +// The version suffix @latest explicitly requests the latest minor release of +// the module named by the given path. The suffix @upgrade is like @latest but // will not downgrade a module if it is already required at a revision or // pre-release version newer than the latest released version. The suffix // @patch requests the latest patch release: the latest released version // with the same major and minor version numbers as the currently required // version. Like @upgrade, @patch will not downgrade a module already required -// at a newer version. If the path is not already required, @upgrade and @patch -// are equivalent to @latest. +// at a newer version. If the path is not already required, @upgrade is +// equivalent to @latest, and @patch is disallowed. // // Although get defaults to using the latest version of the module containing // a named package, it does not use the latest version of that module's @@ -661,14 +675,27 @@ // this automatically as well. // // The -insecure flag permits fetching from repositories and resolving -// custom domains using insecure schemes such as HTTP. Use with caution. The -// GOINSECURE environment variable is usually a better alternative, since it -// provides control over which modules may be retrieved using an insecure scheme. -// See 'go help environment' for details. +// custom domains using insecure schemes such as HTTP, and also bypassess +// module sum validation using the checksum database. Use with caution. +// This flag is deprecated and will be removed in a future version of go. +// To permit the use of insecure schemes, use the GOINSECURE environment +// variable instead. To bypass module sum validation, use GOPRIVATE or +// GONOSUMDB. See 'go help environment' for details. // // The second step is to download (if needed), build, and install // the named packages. // +// The -d flag instructs get to skip this step, downloading source code +// needed to build the named packages and their dependencies, but not +// building or installing. +// +// Building and installing packages with get is deprecated. In a future release, +// the -d flag will be enabled by default, and 'go get' will be only be used to +// adjust dependencies of the current module. To install a package using +// dependencies from the current module, use 'go install'. To install a package +// ignoring the current module, use 'go install' with an @version suffix like +// "@latest" after each argument. +// // If an argument names a module but not a package (because there is no // Go source code in the module's root directory), then the install step // is skipped for that argument, instead of causing a build failure. @@ -680,10 +707,6 @@ // adds the latest golang.org/x/perf and then installs the commands in that // latest version. // -// The -d flag instructs get to download the source code needed to build -// the named packages, including downloading necessary dependencies, -// but not to build and install them. -// // With no package arguments, 'go get' applies to Go package in the // current directory, if any. In particular, 'go get -u' and // 'go get -u=patch' update all the dependencies of that package. @@ -706,7 +729,7 @@ // // Usage: // -// go install [-i] [build flags] [packages] +// go install [build flags] [packages] // // Install compiles and installs the packages named by the import paths. // @@ -715,11 +738,39 @@ // environment variable is not set. Executables in $GOROOT // are installed in $GOROOT/bin or $GOTOOLDIR instead of $GOBIN. // +// If the arguments have version suffixes (like @latest or @v1.0.0), "go install" +// builds packages in module-aware mode, ignoring the go.mod file in the current +// directory or any parent directory, if there is one. This is useful for +// installing executables without affecting the dependencies of the main module. +// To eliminate ambiguity about which module versions are used in the build, the +// arguments must satisfy the following constraints: +// +// - Arguments must be package paths or package patterns (with "..." wildcards). +// They must not be standard packages (like fmt), meta-patterns (std, cmd, +// all), or relative or absolute file paths. +// - All arguments must have the same version suffix. Different queries are not +// allowed, even if they refer to the same version. +// - All arguments must refer to packages in the same module at the same version. +// - No module is considered the "main" module. If the module containing +// packages named on the command line has a go.mod file, it must not contain +// directives (replace and exclude) that would cause it to be interpreted +// differently than if it were the main module. The module must not require +// a higher version of itself. +// - Package path arguments must refer to main packages. Pattern arguments +// will only match main packages. +// +// If the arguments don't have version suffixes, "go install" may run in +// module-aware mode or GOPATH mode, depending on the GO111MODULE environment +// variable and the presence of a go.mod file. See 'go help modules' for details. +// If module-aware mode is enabled, "go install" runs in the context of the main +// module. +// // When module-aware mode is disabled, other packages are installed in the // directory $GOPATH/pkg/$GOOS_$GOARCH. When module-aware mode is enabled, // other packages are built and cached but not installed. // // The -i flag installs the dependencies of the named packages as well. +// The -i flag is deprecated. Compiled packages are cached automatically. // // For more about the build flags, see 'go help build'. // For more about specifying packages, see 'go help packages'. @@ -766,26 +817,28 @@ // BinaryOnly bool // binary-only package (no longer supported) // ForTest string // package is only for use in named test // Export string // file containing export data (when using -export) +// BuildID string // build ID of the compiled package (when using -export) // Module *Module // info about package's containing module, if any (can be nil) // Match []string // command-line patterns matching this package // DepOnly bool // package is only a dependency, not explicitly listed // // // Source files -// GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) -// CgoFiles []string // .go source files that import "C" -// CompiledGoFiles []string // .go files presented to compiler (when using -compiled) -// IgnoredGoFiles []string // .go source files ignored due to build constraints -// CFiles []string // .c source files -// CXXFiles []string // .cc, .cxx and .cpp source files -// MFiles []string // .m source files -// HFiles []string // .h, .hh, .hpp and .hxx source files -// FFiles []string // .f, .F, .for and .f90 Fortran source files -// SFiles []string // .s source files -// SwigFiles []string // .swig files -// SwigCXXFiles []string // .swigcxx files -// SysoFiles []string // .syso object files to add to archive -// TestGoFiles []string // _test.go files in package -// XTestGoFiles []string // _test.go files outside package +// GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) +// CgoFiles []string // .go source files that import "C" +// CompiledGoFiles []string // .go files presented to compiler (when using -compiled) +// IgnoredGoFiles []string // .go source files ignored due to build constraints +// IgnoredOtherFiles []string // non-.go source files ignored due to build constraints +// CFiles []string // .c source files +// CXXFiles []string // .cc, .cxx and .cpp source files +// MFiles []string // .m source files +// HFiles []string // .h, .hh, .hpp and .hxx source files +// FFiles []string // .f, .F, .for and .f90 Fortran source files +// SFiles []string // .s source files +// SwigFiles []string // .swig files +// SwigCXXFiles []string // .swigcxx files +// SysoFiles []string // .syso object files to add to archive +// TestGoFiles []string // _test.go files in package +// XTestGoFiles []string // _test.go files outside package // // // Cgo directives // CgoCFLAGS []string // cgo: flags for C compiler @@ -916,6 +969,7 @@ // Dir string // directory holding files for this module, if any // GoMod string // path to go.mod file used when loading this module, if any // GoVersion string // go version used in module +// Retracted string // retraction information, if any (with -retracted or -u) // Error *ModuleError // error loading module // } // @@ -947,14 +1001,16 @@ // The -u flag adds information about available upgrades. // When the latest version of a given module is newer than // the current one, list -u sets the Module's Update field -// to information about the newer module. +// to information about the newer module. list -u will also set +// the module's Retracted field if the current version is retracted. // The Module's String method indicates an available upgrade by // formatting the newer version in brackets after the current version. +// If a version is retracted, the string "(retracted)" will follow it. // For example, 'go list -m -u all' might print: // // my/main/module // golang.org/x/text v0.3.0 [v0.4.0] => /tmp/text -// rsc.io/pdf v0.1.1 [v0.1.2] +// rsc.io/pdf v0.1.1 (retracted) [v0.1.2] // // (For tools, 'go list -m -u -json all' may be more convenient to parse.) // @@ -964,6 +1020,14 @@ // the default output format to display the module path followed by the // space-separated version list. // +// The -retracted flag causes list to report information about retracted +// module versions. When -retracted is used with -f or -json, the Retracted +// field will be set to a string explaining why the version was retracted. +// The string is taken from comments on the retract directive in the +// module's go.mod file. When -retracted is used with -versions, retracted +// versions are listed together with unretracted versions. The -retracted +// flag may be used with or without -m. +// // The arguments to list -m are interpreted as a list of modules, not packages. // The main module is the module containing the current directory. // The active modules are the main module and its dependencies. @@ -1100,9 +1164,14 @@ // module path and version pair. If the @v is omitted, a replacement without // a version on the left side is dropped. // +// The -retract=version and -dropretract=version flags add and drop a +// retraction on the given version. The version may be a single version +// like "v1.2.3" or a closed interval like "[v1.1.0,v1.1.9]". Note that +// -retract=version is a no-op if that retraction already exists. +// // The -require, -droprequire, -exclude, -dropexclude, -replace, -// and -dropreplace editing flags may be repeated, and the changes -// are applied in the order given. +// -dropreplace, -retract, and -dropretract editing flags may be repeated, +// and the changes are applied in the order given. // // The -go=version flag sets the expected Go language version. // @@ -1136,6 +1205,15 @@ // New Module // } // +// type Retract struct { +// Low string +// High string +// Rationale string +// } +// +// Retract entries representing a single version (not an interval) will have +// the "Low" and "High" fields set to the same value. +// // Note that this only describes the go.mod file itself, not other modules // referred to indirectly. For the full set of modules available to a build, // use 'go list -m -json all'. @@ -1163,19 +1241,24 @@ // // go mod init [module] // -// Init initializes and writes a new go.mod to the current directory, -// in effect creating a new module rooted at the current directory. -// The file go.mod must not already exist. -// If possible, init will guess the module path from import comments -// (see 'go help importpath') or from version control configuration. -// To override this guess, supply the module path as an argument. +// Init initializes and writes a new go.mod file in the current directory, in +// effect creating a new module rooted at the current directory. The go.mod file +// must not already exist. +// +// Init accepts one optional argument, the module path for the new module. If the +// module path argument is omitted, init will attempt to infer the module path +// using import comments in .go files, vendoring tool configuration files (like +// Gopkg.lock), and the current directory (if in GOPATH). +// +// If a configuration file for a vendoring tool is present, init will attempt to +// import module requirements from it. // // // Add missing and remove unused modules // // Usage: // -// go mod tidy [-v] +// go mod tidy [-e] [-v] // // Tidy makes sure go.mod matches the source code in the module. // It adds any missing modules necessary to build the current module's @@ -1186,12 +1269,15 @@ // The -v flag causes tidy to print information about removed modules // to standard error. // +// The -e flag causes tidy to attempt to proceed despite errors +// encountered while loading packages. +// // // Make vendored copy of dependencies // // Usage: // -// go mod vendor [-v] +// go mod vendor [-e] [-v] // // Vendor resets the main module's vendor directory to include all packages // needed to build and test all the main module's packages. @@ -1200,6 +1286,9 @@ // The -v flag causes vendor to print the names of vendored // modules and packages to standard error. // +// The -e flag causes vendor to attempt to proceed despite errors +// encountered while loading packages. +// // // Verify dependencies have expected content // @@ -1388,6 +1477,7 @@ // -i // Install packages that are dependencies of the test. // Do not run the test. +// The -i flag is deprecated. Compiled packages are cached automatically. // // -json // Convert test output to JSON suitable for automated processing. @@ -1546,6 +1636,9 @@ // Using GOOS=illumos matches build tags and files as for GOOS=solaris // in addition to illumos tags and files. // +// Using GOOS=ios matches build tags and files as for GOOS=darwin +// in addition to ios tags and files. +// // To keep a file from being considered for the build: // // // +build ignore @@ -1733,7 +1826,7 @@ // Comma-separated list of glob patterns (in the syntax of Go's path.Match) // of module path prefixes that should always be fetched directly // or that should not be compared against the checksum database. -// See 'go help module-private'. +// See 'go help private'. // GOROOT // The root of the go tree. // GOSUMDB @@ -1789,8 +1882,8 @@ // For GOARCH=arm, the ARM architecture for which to compile. // Valid values are 5, 6, 7. // GO386 -// For GOARCH=386, the floating point instruction set. -// Valid values are 387, sse2. +// For GOARCH=386, how to implement floating point instructions. +// Valid values are sse2 (default), softfloat. // GOMIPS // For GOARCH=mips{,le}, whether to use floating point instructions. // Valid values are hardfloat (default), softfloat. @@ -1839,6 +1932,8 @@ // If module-aware mode is disabled, GOMOD will be the empty string. // GOTOOLDIR // The directory where the go tools (compile, cover, doc, etc...) are installed. +// GOVERSION +// The version of the installed Go tree, as reported by runtime.Version. // // // File types @@ -1894,21 +1989,23 @@ // require new/thing/v2 v2.3.4 // exclude old/thing v1.2.3 // replace bad/thing v1.4.5 => good/thing v1.4.5 +// retract v1.5.6 // // The verbs are // module, to define the module path; // go, to set the expected language version; // require, to require a particular module at a given version or later; -// exclude, to exclude a particular module version from use; and -// replace, to replace a module version with a different module version. +// exclude, to exclude a particular module version from use; +// replace, to replace a module version with a different module version; and +// retract, to indicate a previously released version should not be used. // Exclude and replace apply only in the main module's go.mod and are ignored -// in dependencies. See https://research.swtch.com/vgo-mvs for details. +// in dependencies. See https://golang.org/ref/mod for details. // // The leading verb can be factored out of adjacent lines to create a block, // like in Go imports: // // require ( -// new/thing v2.3.4 +// new/thing/v2 v2.3.4 // old/thing v1.2.3 // ) // @@ -2146,6 +2243,10 @@ // // The -insecure flag permits fetching from repositories and resolving // custom domains using insecure schemes such as HTTP. Use with caution. +// This flag is deprecated and will be removed in a future version of go. +// The GOINSECURE environment variable should be used instead, since it +// provides control over which packages may be retrieved using an insecure +// scheme. See 'go help environment' for details. // // The -t flag instructs get to also download the packages required to build // the tests for the specified packages. @@ -2556,72 +2657,63 @@ // // Maintaining module requirements // -// The go.mod file is meant to be readable and editable by both -// programmers and tools. The go command itself automatically updates the go.mod file -// to maintain a standard formatting and the accuracy of require statements. -// -// Any go command that finds an unfamiliar import will look up the module -// containing that import and add the latest version of that module -// to go.mod automatically. In most cases, therefore, it suffices to -// add an import to source code and run 'go build', 'go test', or even 'go list': -// as part of analyzing the package, the go command will discover -// and resolve the import and update the go.mod file. -// -// Any go command can determine that a module requirement is -// missing and must be added, even when considering only a single -// package from the module. On the other hand, determining that a module requirement -// is no longer necessary and can be deleted requires a full view of -// all packages in the module, across all possible build configurations -// (architectures, operating systems, build tags, and so on). -// The 'go mod tidy' command builds that view and then -// adds any missing module requirements and removes unnecessary ones. +// The go.mod file is meant to be readable and editable by both programmers and +// tools. Most updates to dependencies can be performed using "go get" and +// "go mod tidy". Other module-aware build commands may be invoked using the +// -mod=mod flag to automatically add missing requirements and fix inconsistencies. +// +// The "go get" command updates go.mod to change the module versions used in a +// build. An upgrade of one module may imply upgrading others, and similarly a +// downgrade of one module may imply downgrading others. The "go get" command +// makes these implied changes as well. See "go help module-get". +// +// The "go mod" command provides other functionality for use in maintaining +// and understanding modules and go.mod files. See "go help mod", particularly +// "go help mod tidy" and "go help mod edit". // // As part of maintaining the require statements in go.mod, the go command // tracks which ones provide packages imported directly by the current module // and which ones provide packages only used indirectly by other module // dependencies. Requirements needed only for indirect uses are marked with a -// "// indirect" comment in the go.mod file. Indirect requirements are +// "// indirect" comment in the go.mod file. Indirect requirements may be // automatically removed from the go.mod file once they are implied by other // direct requirements. Indirect requirements only arise when using modules // that fail to state some of their own dependencies or when explicitly // upgrading a module's dependencies ahead of its own stated requirements. // -// Because of this automatic maintenance, the information in go.mod is an -// up-to-date, readable description of the build. -// -// The 'go get' command updates go.mod to change the module versions used in a -// build. An upgrade of one module may imply upgrading others, and similarly a -// downgrade of one module may imply downgrading others. The 'go get' command -// makes these implied changes as well. If go.mod is edited directly, commands -// like 'go build' or 'go list' will assume that an upgrade is intended and -// automatically make any implied upgrades and update go.mod to reflect them. -// -// The 'go mod' command provides other functionality for use in maintaining -// and understanding modules and go.mod files. See 'go help mod'. -// -// The -mod build flag provides additional control over updating and use of go.mod. -// -// If invoked with -mod=readonly, the go command is disallowed from the implicit -// automatic updating of go.mod described above. Instead, it fails when any changes -// to go.mod are needed. This setting is most useful to check that go.mod does -// not need updates, such as in a continuous integration and testing system. -// The "go get" command remains permitted to update go.mod even with -mod=readonly, -// and the "go mod" commands do not take the -mod flag (or any other build flags). +// The -mod build flag provides additional control over the updating and use of +// go.mod for commands that build packages like "go build" and "go test". +// +// If invoked with -mod=readonly (the default in most situations), the go command +// reports an error if a package named on the command line or an imported package +// is not provided by any module in the build list computed from the main module's +// requirements. The go command also reports an error if a module's checksum is +// missing from go.sum (see Module downloading and verification). Either go.mod or +// go.sum must be updated in these situations. +// +// If invoked with -mod=mod, the go command automatically updates go.mod and +// go.sum, fixing inconsistencies and adding missing requirements and checksums +// as needed. If the go command finds an unfamiliar import, it looks up the +// module containing that import and adds a requirement for the latest version +// of that module to go.mod. In most cases, therefore, one may add an import to +// source code and run "go build", "go test", or even "go list" with -mod=mod: +// as part of analyzing the package, the go command will resolve the import and +// update the go.mod file. // // If invoked with -mod=vendor, the go command loads packages from the main // module's vendor directory instead of downloading modules to and loading packages // from the module cache. The go command assumes the vendor directory holds // correct copies of dependencies, and it does not compute the set of required // module versions from go.mod files. However, the go command does check that -// vendor/modules.txt (generated by 'go mod vendor') contains metadata consistent +// vendor/modules.txt (generated by "go mod vendor") contains metadata consistent // with go.mod. // -// If invoked with -mod=mod, the go command loads modules from the module cache -// even if there is a vendor directory present. +// If the go command is not invoked with a -mod flag, and the vendor directory +// is present, and the "go" version in go.mod is 1.14 or higher, the go command +// will act as if it were invoked with -mod=vendor. Otherwise, the -mod flag +// defaults to -mod=readonly. // -// If the go command is not invoked with a -mod flag and the vendor directory -// is present and the "go" version in go.mod is 1.14 or higher, the go command -// will act as if it were invoked with -mod=vendor. +// Note that neither "go get" nor the "go mod" subcommands accept the -mod flag. // // Pseudo-versions // @@ -2806,7 +2898,7 @@ // followed by a pipe character, indicating it is safe to fall back on any error. // // The GOPRIVATE and GONOPROXY environment variables allow bypassing -// the proxy for selected modules. See 'go help module-private' for details. +// the proxy for selected modules. See 'go help private' for details. // // No matter the source of the modules, the go command checks downloads against // known checksums, to detect unexpected changes in the content of any specific @@ -2926,52 +3018,7 @@ // accepted, at the cost of giving up the security guarantee of verified repeatable // downloads for all modules. A better way to bypass the checksum database // for specific modules is to use the GOPRIVATE or GONOSUMDB environment -// variables. See 'go help module-private' for details. -// -// The 'go env -w' command (see 'go help env') can be used to set these variables -// for future go command invocations. -// -// -// Module configuration for non-public modules -// -// The go command defaults to downloading modules from the public Go module -// mirror at proxy.golang.org. It also defaults to validating downloaded modules, -// regardless of source, against the public Go checksum database at sum.golang.org. -// These defaults work well for publicly available source code. -// -// The GOPRIVATE environment variable controls which modules the go command -// considers to be private (not available publicly) and should therefore not use the -// proxy or checksum database. The variable is a comma-separated list of -// glob patterns (in the syntax of Go's path.Match) of module path prefixes. -// For example, -// -// GOPRIVATE=*.corp.example.com,rsc.io/private -// -// causes the go command to treat as private any module with a path prefix -// matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private, -// and rsc.io/private/quux. -// -// The GOPRIVATE environment variable may be used by other tools as well to -// identify non-public modules. For example, an editor could use GOPRIVATE -// to decide whether to hyperlink a package import to a godoc.org page. -// -// For fine-grained control over module download and validation, the GONOPROXY -// and GONOSUMDB environment variables accept the same kind of glob list -// and override GOPRIVATE for the specific decision of whether to use the proxy -// and checksum database, respectively. -// -// For example, if a company ran a module proxy serving private modules, -// users would configure go using: -// -// GOPRIVATE=*.corp.example.com -// GOPROXY=proxy.example.com -// GONOPROXY=none -// -// This would tell the go command and other tools that modules beginning with -// a corp.example.com subdomain are private but that the company proxy should -// be used for downloading both public and private modules, because -// GONOPROXY has been set to a pattern that won't match any modules, -// overriding GOPRIVATE. +// variables. See 'go help private' for details. // // The 'go env -w' command (see 'go help env') can be used to set these variables // for future go command invocations. @@ -3061,6 +3108,56 @@ // by the go tool, as are directories named "testdata". // // +// Configuration for downloading non-public code +// +// The go command defaults to downloading modules from the public Go module +// mirror at proxy.golang.org. It also defaults to validating downloaded modules, +// regardless of source, against the public Go checksum database at sum.golang.org. +// These defaults work well for publicly available source code. +// +// The GOPRIVATE environment variable controls which modules the go command +// considers to be private (not available publicly) and should therefore not use the +// proxy or checksum database. The variable is a comma-separated list of +// glob patterns (in the syntax of Go's path.Match) of module path prefixes. +// For example, +// +// GOPRIVATE=*.corp.example.com,rsc.io/private +// +// causes the go command to treat as private any module with a path prefix +// matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private, +// and rsc.io/private/quux. +// +// The GOPRIVATE environment variable may be used by other tools as well to +// identify non-public modules. For example, an editor could use GOPRIVATE +// to decide whether to hyperlink a package import to a godoc.org page. +// +// For fine-grained control over module download and validation, the GONOPROXY +// and GONOSUMDB environment variables accept the same kind of glob list +// and override GOPRIVATE for the specific decision of whether to use the proxy +// and checksum database, respectively. +// +// For example, if a company ran a module proxy serving private modules, +// users would configure go using: +// +// GOPRIVATE=*.corp.example.com +// GOPROXY=proxy.example.com +// GONOPROXY=none +// +// This would tell the go command and other tools that modules beginning with +// a corp.example.com subdomain are private but that the company proxy should +// be used for downloading both public and private modules, because +// GONOPROXY has been set to a pattern that won't match any modules, +// overriding GOPRIVATE. +// +// The GOPRIVATE variable is also used to define the "public" and "private" +// patterns for the GOVCS variable; see 'go help vcs'. For that usage, +// GOPRIVATE applies even in GOPATH mode. In that case, it matches import paths +// instead of module paths. +// +// The 'go env -w' command (see 'go help env') can be used to set these variables +// for future go command invocations. +// +// // Testing flags // // The 'go test' command takes both flags that apply to 'go test' itself @@ -3353,4 +3450,77 @@ // See the documentation of the testing package for more information. // // +// Controlling version control with GOVCS +// +// The 'go get' command can run version control commands like git +// to download imported code. This functionality is critical to the decentralized +// Go package ecosystem, in which code can be imported from any server, +// but it is also a potential security problem, if a malicious server finds a +// way to cause the invoked version control command to run unintended code. +// +// To balance the functionality and security concerns, the 'go get' command +// by default will only use git and hg to download code from public servers. +// But it will use any known version control system (bzr, fossil, git, hg, svn) +// to download code from private servers, defined as those hosting packages +// matching the GOPRIVATE variable (see 'go help private'). The rationale behind +// allowing only Git and Mercurial is that these two systems have had the most +// attention to issues of being run as clients of untrusted servers. In contrast, +// Bazaar, Fossil, and Subversion have primarily been used in trusted, +// authenticated environments and are not as well scrutinized as attack surfaces. +// +// The version control command restrictions only apply when using direct version +// control access to download code. When downloading modules from a proxy, +// 'go get' uses the proxy protocol instead, which is always permitted. +// By default, the 'go get' command uses the Go module mirror (proxy.golang.org) +// for public packages and only falls back to version control for private +// packages or when the mirror refuses to serve a public package (typically for +// legal reasons). Therefore, clients can still access public code served from +// Bazaar, Fossil, or Subversion repositories by default, because those downloads +// use the Go module mirror, which takes on the security risk of running the +// version control commands, using a custom sandbox. +// +// The GOVCS variable can be used to change the allowed version control systems +// for specific packages (identified by a module or import path). +// The GOVCS variable applies both when using modules and when using GOPATH. +// When using modules, the patterns match against the module path. +// When using GOPATH, the patterns match against the import path +// corresponding to the root of the version control repository. +// +// The general form of the GOVCS setting is a comma-separated list of +// pattern:vcslist rules. The pattern is a glob pattern that must match +// one or more leading elements of the module or import path. The vcslist +// is a pipe-separated list of allowed version control commands, or "all" +// to allow use of any known command, or "off" to allow nothing. +// The earliest matching pattern in the list applies, even if later patterns +// might also match. +// +// For example, consider: +// +// GOVCS=github.com:git,evil.com:off,*:git|hg +// +// With this setting, code with an module or import path beginning with +// github.com/ can only use git; paths on evil.com cannot use any version +// control command, and all other paths (* matches everything) can use +// only git or hg. +// +// The special patterns "public" and "private" match public and private +// module or import paths. A path is private if it matches the GOPRIVATE +// variable; otherwise it is public. +// +// If no rules in the GOVCS variable match a particular module or import path, +// the 'go get' command applies its default rule, which can now be summarized +// in GOVCS notation as 'public:git|hg,private:all'. +// +// To allow unfettered use of any version control system for any package, use: +// +// GOVCS=*:all +// +// To disable all use of version control, use: +// +// GOVCS=*:off +// +// The 'go env -w' command (see 'go help env') can be used to set the GOVCS +// variable for future go command invocations. +// +// package main diff --git a/libgo/go/cmd/go/go_test.go b/libgo/go/cmd/go/go_test.go index 021930a..c472620 100644 --- a/libgo/go/cmd/go/go_test.go +++ b/libgo/go/cmd/go/go_test.go @@ -9,13 +9,14 @@ import ( "debug/elf" "debug/macho" "debug/pe" + "encoding/binary" "flag" "fmt" "go/format" "internal/race" "internal/testenv" "io" - "io/ioutil" + "io/fs" "log" "os" "os/exec" @@ -30,77 +31,42 @@ import ( "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/robustio" + "cmd/go/internal/work" "cmd/internal/sys" ) +func init() { + // GOVCS defaults to public:git|hg,private:all, + // which breaks many tests here - they can't use non-git, non-hg VCS at all! + // Change to fully permissive. + // The tests of the GOVCS setting itself are in ../../testdata/script/govcs.txt. + os.Setenv("GOVCS", "*:all") +} + var ( - canRun = true // whether we can run go or ./testgo canRace = false // whether we can run the race detector canCgo = false // whether we can use cgo canMSan = false // whether we can run the memory sanitizer - - exeSuffix string // ".exe" on Windows - - skipExternal = false // skip external tests ) +var exeSuffix string = func() string { + if runtime.GOOS == "windows" { + return ".exe" + } + return "" +}() + func tooSlow(t *testing.T) { if testing.Short() { // In -short mode; skip test, except run it on the {darwin,linux,windows}/amd64 builders. if testenv.Builder() != "" && runtime.GOARCH == "amd64" && (runtime.GOOS == "linux" || runtime.GOOS == "darwin" || runtime.GOOS == "windows") { return } + t.Helper() t.Skip("skipping test in -short mode") } } -func init() { - switch runtime.GOOS { - case "android", "js": - canRun = false - case "darwin": - switch runtime.GOARCH { - case "arm64": - canRun = false - } - case "linux": - switch runtime.GOARCH { - case "arm": - // many linux/arm machines are too slow to run - // the full set of external tests. - skipExternal = true - case "mips", "mipsle", "mips64", "mips64le": - // Also slow. - skipExternal = true - if testenv.Builder() != "" { - // On the builders, skip the cmd/go - // tests. They're too slow and already - // covered by other ports. There's - // nothing os/arch specific in the - // tests. - canRun = false - } - } - case "freebsd": - switch runtime.GOARCH { - case "arm": - // many freebsd/arm machines are too slow to run - // the full set of external tests. - skipExternal = true - canRun = false - } - case "plan9": - switch runtime.GOARCH { - case "arm": - // many plan9/arm machines are too slow to run - // the full set of external tests. - skipExternal = true - } - case "windows": - exeSuffix = ".exe" - } -} - // testGOROOT is the GOROOT to use when running testgo, a cmd/go binary // build from this process's current GOROOT, but run from a different // (temp) directory. @@ -134,7 +100,7 @@ func TestMain(m *testing.M) { // Run with a temporary TMPDIR to check that the tests don't // leave anything behind. - topTmpdir, err := ioutil.TempDir("", "cmd-go-test-") + topTmpdir, err := os.MkdirTemp("", "cmd-go-test-") if err != nil { log.Fatal(err) } @@ -143,7 +109,7 @@ func TestMain(m *testing.M) { } os.Setenv(tempEnvName(), topTmpdir) - dir, err := ioutil.TempDir(topTmpdir, "tmpdir") + dir, err := os.MkdirTemp(topTmpdir, "tmpdir") if err != nil { log.Fatal(err) } @@ -153,7 +119,7 @@ func TestMain(m *testing.M) { } testGOCACHE = cache.DefaultDir() - if canRun { + if testenv.HasGoBuild() { testBin = filepath.Join(testTmpDir, "testbin") if err := os.Mkdir(testBin, 0777); err != nil { log.Fatal(err) @@ -224,7 +190,7 @@ func TestMain(m *testing.M) { cmd.Stderr = new(strings.Builder) if out, err := cmd.Output(); err != nil { fmt.Fprintf(os.Stderr, "running testgo failed: %v\n%s", err, cmd.Stderr) - canRun = false + os.Exit(2) } else { canCgo, err = strconv.ParseBool(strings.TrimSpace(string(out))) if err != nil { @@ -324,10 +290,7 @@ func skipIfGccgo(t *testing.T, msg string) { func testgo(t *testing.T) *testgoData { t.Helper() testenv.MustHaveGoBuild(t) - - if skipExternal { - t.Skipf("skipping external tests on %s/%s", runtime.GOOS, runtime.GOARCH) - } + testenv.SkipIfShortAndSlow(t) return &testgoData{t: t} } @@ -416,9 +379,6 @@ func (tg *testgoData) goTool() string { // returning exit status. func (tg *testgoData) doRun(args []string) error { tg.t.Helper() - if !canRun { - panic("testgoData.doRun called but canRun false") - } if tg.inParallel { for _, arg := range args { if strings.HasPrefix(arg, "testdata") || strings.HasPrefix(arg, "./testdata") { @@ -656,7 +616,7 @@ func (tg *testgoData) makeTempdir() { tg.t.Helper() if tg.tempdir == "" { var err error - tg.tempdir, err = ioutil.TempDir("", "gotest") + tg.tempdir, err = os.MkdirTemp("", "gotest") tg.must(err) } } @@ -673,7 +633,7 @@ func (tg *testgoData) tempFile(path, contents string) { bytes = formatted } } - tg.must(ioutil.WriteFile(filepath.Join(tg.tempdir, path), bytes, 0644)) + tg.must(os.WriteFile(filepath.Join(tg.tempdir, path), bytes, 0644)) } // tempDir adds a temporary directory for a run of testgo. @@ -814,7 +774,7 @@ func (tg *testgoData) cleanup() { func removeAll(dir string) error { // module cache has 0444 directories; // make them writable in order to remove content. - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error { // chmod not only directories, but also things that we couldn't even stat // due to permission errors: they may also be unreadable directories. if err != nil || info.IsDir() { @@ -860,8 +820,8 @@ func TestNewReleaseRebuildsStalePackagesInGOPATH(t *testing.T) { } { srcdir := filepath.Join(testGOROOT, copydir) tg.tempDir(filepath.Join("goroot", copydir)) - err := filepath.Walk(srcdir, - func(path string, info os.FileInfo, err error) error { + err := filepath.WalkDir(srcdir, + func(path string, info fs.DirEntry, err error) error { if err != nil { return err } @@ -873,13 +833,13 @@ func TestNewReleaseRebuildsStalePackagesInGOPATH(t *testing.T) { return err } dest := filepath.Join("goroot", copydir, srcrel) - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { return err } tg.tempFile(dest, string(data)) - if err := os.Chmod(tg.path(dest), info.Mode()|0200); err != nil { - return err + if strings.Contains(copydir, filepath.Join("pkg", "tool")) { + os.Chmod(tg.path(dest), 0777) } return nil }) @@ -890,18 +850,18 @@ func TestNewReleaseRebuildsStalePackagesInGOPATH(t *testing.T) { tg.setenv("GOROOT", tg.path("goroot")) addVar := func(name string, idx int) (restore func()) { - data, err := ioutil.ReadFile(name) + data, err := os.ReadFile(name) if err != nil { t.Fatal(err) } old := data data = append(data, fmt.Sprintf("var DummyUnusedVar%d bool\n", idx)...) - if err := ioutil.WriteFile(name, append(data, '\n'), 0666); err != nil { + if err := os.WriteFile(name, append(data, '\n'), 0666); err != nil { t.Fatal(err) } tg.sleep() return func() { - if err := ioutil.WriteFile(name, old, 0666); err != nil { + if err := os.WriteFile(name, old, 0666); err != nil { t.Fatal(err) } } @@ -1237,6 +1197,18 @@ func TestGoListExport(t *testing.T) { if _, err := os.Stat(file); err != nil { t.Fatalf("cannot find .Export result %s: %v", file, err) } + + tg.run("list", "-export", "-f", "{{.BuildID}}", "strings") + buildID := strings.TrimSpace(tg.stdout.String()) + if buildID == "" { + t.Fatalf(".BuildID with -export was empty") + } + + tg.run("tool", "buildid", file) + toolBuildID := strings.TrimSpace(tg.stdout.String()) + if buildID != toolBuildID { + t.Fatalf(".BuildID with -export %q disagrees with 'go tool buildid' %q", buildID, toolBuildID) + } } // Issue 4096. Validate the output of unsuccessful go install foo/quxx. @@ -1394,6 +1366,30 @@ func TestLdflagsArgumentsWithSpacesIssue3941(t *testing.T) { tg.grepStderr("^hello world", `ldflags -X "main.extern=hello world"' failed`) } +func TestLdFlagsLongArgumentsIssue42295(t *testing.T) { + // Test the extremely long command line arguments that contain '\n' characters + // get encoded and passed correctly. + skipIfGccgo(t, "gccgo does not support -ldflags -X") + tooSlow(t) + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("main.go", `package main + var extern string + func main() { + print(extern) + }`) + testStr := "test test test test test \n\\ " + var buf bytes.Buffer + for buf.Len() < work.ArgLengthForResponseFile+1 { + buf.WriteString(testStr) + } + tg.run("run", "-ldflags", fmt.Sprintf(`-X "main.extern=%s"`, buf.String()), tg.path("main.go")) + if tg.stderr.String() != buf.String() { + t.Errorf("strings differ") + } +} + func TestGoTestDashCDashOControlsBinaryLocation(t *testing.T) { skipIfGccgo(t, "gccgo has no standard packages") tooSlow(t) @@ -1914,6 +1910,18 @@ func TestGoEnv(t *testing.T) { tg.grepStdout("gcc", "CC not found") tg.run("env", "GOGCCFLAGS") tg.grepStdout("-ffaster", "CC arguments not found") + + tg.run("env", "GOVERSION") + envVersion := strings.TrimSpace(tg.stdout.String()) + + tg.run("version") + cmdVersion := strings.TrimSpace(tg.stdout.String()) + + // If 'go version' is "go version <version> <goos>/<goarch>", then + // 'go env GOVERSION' is just "<version>". + if cmdVersion == envVersion || !strings.Contains(cmdVersion, envVersion) { + t.Fatalf("'go env GOVERSION' %q should be a shorter substring of 'go version' %q", envVersion, cmdVersion) + } } const ( @@ -2019,7 +2027,7 @@ func main() { tg.run("build", "-o", exe, "p") } -func copyFile(src, dst string, perm os.FileMode) error { +func copyFile(src, dst string, perm fs.FileMode) error { sf, err := os.Open(src) if err != nil { return err @@ -2058,7 +2066,7 @@ func TestBuildmodePIE(t *testing.T) { platform := fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH) switch platform { - case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/s390x", + case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/riscv64", "linux/s390x", "android/amd64", "android/arm", "android/arm64", "android/386", "freebsd/amd64", "windows/386", "windows/amd64", "windows/arm": @@ -2166,6 +2174,38 @@ func testBuildmodePIE(t *testing.T, useCgo, setBuildmodeToPIE bool) { if (dc & pe.IMAGE_DLLCHARACTERISTICS_DYNAMIC_BASE) == 0 { t.Error("IMAGE_DLLCHARACTERISTICS_DYNAMIC_BASE flag is not set") } + if useCgo { + // Test that only one symbol is exported (#40795). + // PIE binaries don´t require .edata section but unfortunately + // binutils doesn´t generate a .reloc section unless there is + // at least one symbol exported. + // See https://sourceware.org/bugzilla/show_bug.cgi?id=19011 + section := f.Section(".edata") + if section == nil { + t.Fatalf(".edata section is not present") + } + // TODO: deduplicate this struct from cmd/link/internal/ld/pe.go + type IMAGE_EXPORT_DIRECTORY struct { + _ [2]uint32 + _ [2]uint16 + _ [2]uint32 + NumberOfFunctions uint32 + NumberOfNames uint32 + _ [3]uint32 + } + var e IMAGE_EXPORT_DIRECTORY + if err := binary.Read(section.Open(), binary.LittleEndian, &e); err != nil { + t.Fatalf("binary.Read failed: %v", err) + } + + // Only _cgo_dummy_export should be exported + if e.NumberOfFunctions != 1 { + t.Fatalf("got %d exported functions; want 1", e.NumberOfFunctions) + } + if e.NumberOfNames != 1 { + t.Fatalf("got %d exported names; want 1", e.NumberOfNames) + } + } default: panic("unreachable") } @@ -2658,7 +2698,7 @@ echo $* >>`+tg.path("pkg-config.out")) tg.setenv("GOPATH", tg.path(".")) tg.setenv("PKG_CONFIG", tg.path("pkg-config.sh")) tg.run("build", "x") - out, err := ioutil.ReadFile(tg.path("pkg-config.out")) + out, err := os.ReadFile(tg.path("pkg-config.out")) tg.must(err) out = bytes.TrimSpace(out) want := "--cflags --static --static -- a a\n--libs --static --static -- a a" diff --git a/libgo/go/cmd/go/go_windows_test.go b/libgo/go/cmd/go/go_windows_test.go index 3999166..3094212 100644 --- a/libgo/go/cmd/go/go_windows_test.go +++ b/libgo/go/cmd/go/go_windows_test.go @@ -2,11 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package main +package main_test import ( - "internal/testenv" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -17,16 +15,18 @@ import ( ) func TestAbsolutePath(t *testing.T) { - t.Parallel() + tg := testgo(t) + defer tg.cleanup() + tg.parallel() - tmp, err := ioutil.TempDir("", "TestAbsolutePath") + tmp, err := os.MkdirTemp("", "TestAbsolutePath") if err != nil { t.Fatal(err) } defer robustio.RemoveAll(tmp) file := filepath.Join(tmp, "a.go") - err = ioutil.WriteFile(file, []byte{}, 0644) + err = os.WriteFile(file, []byte{}, 0644) if err != nil { t.Fatal(err) } @@ -38,7 +38,7 @@ func TestAbsolutePath(t *testing.T) { noVolume := file[len(filepath.VolumeName(file)):] wrongPath := filepath.Join(dir, noVolume) - cmd := exec.Command(testenv.GoToolPath(t), "build", noVolume) + cmd := exec.Command(tg.goTool(), "build", noVolume) cmd.Dir = dir output, err := cmd.CombinedOutput() if err == nil { diff --git a/libgo/go/cmd/go/help_test.go b/libgo/go/cmd/go/help_test.go index 78d63ff..abfc3db 100644 --- a/libgo/go/cmd/go/help_test.go +++ b/libgo/go/cmd/go/help_test.go @@ -6,7 +6,7 @@ package main_test import ( "bytes" - "io/ioutil" + "os" "testing" "cmd/go/internal/help" @@ -23,7 +23,7 @@ func TestDocsUpToDate(t *testing.T) { buf := new(bytes.Buffer) // Match the command in mkalldocs.sh that generates alldocs.go. help.Help(buf, []string{"documentation"}) - data, err := ioutil.ReadFile("alldocs.go") + data, err := os.ReadFile("alldocs.go") if err != nil { t.Fatalf("error reading alldocs.go: %v", err) } diff --git a/libgo/go/cmd/go/init_test.go b/libgo/go/cmd/go/init_test.go index ed90a77..5a5cbe5 100644 --- a/libgo/go/cmd/go/init_test.go +++ b/libgo/go/cmd/go/init_test.go @@ -7,6 +7,7 @@ package main_test import ( "internal/testenv" "os/exec" + "sync/atomic" "testing" ) @@ -15,20 +16,27 @@ import ( // the benchmark if any changes were done. func BenchmarkExecGoEnv(b *testing.B) { testenv.MustHaveExec(b) - b.StopTimer() gotool, err := testenv.GoTool() if err != nil { b.Fatal(err) } - for i := 0; i < b.N; i++ { - cmd := exec.Command(gotool, "env", "GOARCH") - b.StartTimer() - err := cmd.Run() - b.StopTimer() + // We collect extra metrics. + var n, userTime, systemTime int64 - if err != nil { - b.Fatal(err) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + cmd := exec.Command(gotool, "env", "GOARCH") + + if err := cmd.Run(); err != nil { + b.Fatal(err) + } + atomic.AddInt64(&n, 1) + atomic.AddInt64(&userTime, int64(cmd.ProcessState.UserTime())) + atomic.AddInt64(&systemTime, int64(cmd.ProcessState.SystemTime())) } - } + }) + b.ReportMetric(float64(userTime)/float64(n), "user-ns/op") + b.ReportMetric(float64(systemTime)/float64(n), "sys-ns/op") } diff --git a/libgo/go/cmd/go/internal/auth/netrc.go b/libgo/go/cmd/go/internal/auth/netrc.go index 7a9bdbb..0107f20 100644 --- a/libgo/go/cmd/go/internal/auth/netrc.go +++ b/libgo/go/cmd/go/internal/auth/netrc.go @@ -5,7 +5,6 @@ package auth import ( - "io/ioutil" "os" "path/filepath" "runtime" @@ -99,7 +98,7 @@ func readNetrc() { return } - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { if !os.IsNotExist(err) { netrcErr = err diff --git a/libgo/go/cmd/go/internal/base/base.go b/libgo/go/cmd/go/internal/base/base.go index ab2f1bb..004588c 100644 --- a/libgo/go/cmd/go/internal/base/base.go +++ b/libgo/go/cmd/go/internal/base/base.go @@ -7,6 +7,7 @@ package base import ( + "context" "flag" "fmt" "log" @@ -24,7 +25,7 @@ import ( type Command struct { // Run runs the command. // The args are the arguments after the command name. - Run func(cmd *Command, args []string) + Run func(ctx context.Context, cmd *Command, args []string) // UsageLine is the one-line usage message. // The words between "go" and the first flag or argument in the line are taken to be the command name. @@ -55,6 +56,20 @@ var Go = &Command{ // Commands initialized in package main } +// hasFlag reports whether a command or any of its subcommands contain the given +// flag. +func hasFlag(c *Command, name string) bool { + if f := c.Flag.Lookup(name); f != nil { + return true + } + for _, sub := range c.Commands { + if hasFlag(sub, name) { + return true + } + } + return false +} + // LongName returns the command's long name: all the words in the usage line between "go" and a flag or argument, func (c *Command) LongName() string { name := c.UsageLine diff --git a/libgo/go/cmd/go/internal/base/flag.go b/libgo/go/cmd/go/internal/base/flag.go index 6727196..677f819 100644 --- a/libgo/go/cmd/go/internal/base/flag.go +++ b/libgo/go/cmd/go/internal/base/flag.go @@ -8,6 +8,7 @@ import ( "flag" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/str" ) @@ -28,13 +29,43 @@ func (v *StringsFlag) String() string { return "<StringsFlag>" } +// explicitStringFlag is like a regular string flag, but it also tracks whether +// the string was set explicitly to a non-empty value. +type explicitStringFlag struct { + value *string + explicit *bool +} + +func (f explicitStringFlag) String() string { + if f.value == nil { + return "" + } + return *f.value +} + +func (f explicitStringFlag) Set(v string) error { + *f.value = v + if v != "" { + *f.explicit = true + } + return nil +} + // AddBuildFlagsNX adds the -n and -x build flags to the flag set. func AddBuildFlagsNX(flags *flag.FlagSet) { flags.BoolVar(&cfg.BuildN, "n", false, "") flags.BoolVar(&cfg.BuildX, "x", false, "") } -// AddLoadFlags adds the -mod build flag to the flag set. -func AddLoadFlags(flags *flag.FlagSet) { - flags.StringVar(&cfg.BuildMod, "mod", "", "") +// AddModFlag adds the -mod build flag to the flag set. +func AddModFlag(flags *flag.FlagSet) { + flags.Var(explicitStringFlag{value: &cfg.BuildMod, explicit: &cfg.BuildModExplicit}, "mod", "") +} + +// AddModCommonFlags adds the module-related flags common to build commands +// and 'go mod' subcommands. +func AddModCommonFlags(flags *flag.FlagSet) { + flags.BoolVar(&cfg.ModCacheRW, "modcacherw", false, "") + flags.StringVar(&cfg.ModFile, "modfile", "", "") + flags.StringVar(&fsys.OverlayFile, "overlay", "", "") } diff --git a/libgo/go/cmd/go/internal/base/goflags.go b/libgo/go/cmd/go/internal/base/goflags.go index f29cc3c..267006b 100644 --- a/libgo/go/cmd/go/internal/base/goflags.go +++ b/libgo/go/cmd/go/internal/base/goflags.go @@ -13,15 +13,7 @@ import ( "cmd/go/internal/cfg" ) -var ( - goflags []string // cached $GOFLAGS list; can be -x or --x form - knownFlag = make(map[string]bool) // flags allowed to appear in $GOFLAGS; no leading dashes -) - -// AddKnownFlag adds name to the list of known flags for use in $GOFLAGS. -func AddKnownFlag(name string) { - knownFlag[name] = true -} +var goflags []string // cached $GOFLAGS list; can be -x or --x form // GOFLAGS returns the flags from $GOFLAGS. // The list can be assumed to contain one string per flag, @@ -38,22 +30,12 @@ func InitGOFLAGS() { return } - // Build list of all flags for all commands. - // If no command has that flag, then we report the problem. - // This catches typos while still letting users record flags in GOFLAGS - // that only apply to a subset of go commands. - // Commands using CustomFlags can report their flag names - // by calling AddKnownFlag instead. - var walkFlags func(*Command) - walkFlags = func(cmd *Command) { - for _, sub := range cmd.Commands { - walkFlags(sub) - } - cmd.Flag.VisitAll(func(f *flag.Flag) { - knownFlag[f.Name] = true - }) + goflags = strings.Fields(cfg.Getenv("GOFLAGS")) + if len(goflags) == 0 { + // nothing to do; avoid work on later InitGOFLAGS call + goflags = []string{} + return } - walkFlags(Go) // Ignore bad flag in go env and go bug, because // they are what people reach for when debugging @@ -61,11 +43,6 @@ func InitGOFLAGS() { // (Both will show the GOFLAGS setting if let succeed.) hideErrors := cfg.CmdName == "env" || cfg.CmdName == "bug" - goflags = strings.Fields(cfg.Getenv("GOFLAGS")) - if goflags == nil { - goflags = []string{} // avoid work on later InitGOFLAGS call - } - // Each of the words returned by strings.Fields must be its own flag. // To set flag arguments use -x=value instead of -x value. // For boolean flags, -x is fine instead of -x=true. @@ -85,7 +62,7 @@ func InitGOFLAGS() { if i := strings.Index(name, "="); i >= 0 { name = name[:i] } - if !knownFlag[name] { + if !hasFlag(Go, name) { if hideErrors { continue } @@ -115,7 +92,11 @@ func SetFromGOFLAGS(flags *flag.FlagSet) { } for _, goflag := range goflags { name, value, hasValue := goflag, "", false - if i := strings.Index(goflag, "="); i >= 0 { + // Ignore invalid flags like '=' or '=value'. + // If it is not reported in InitGOFlags it means we don't want to report it. + if i := strings.Index(goflag, "="); i == 0 { + continue + } else if i > 0 { name, value, hasValue = goflag[:i], goflag[i+1:], true } if strings.HasPrefix(name, "--") { diff --git a/libgo/go/cmd/go/internal/base/signal.go b/libgo/go/cmd/go/internal/base/signal.go index 54d1187..05befcf 100644 --- a/libgo/go/cmd/go/internal/base/signal.go +++ b/libgo/go/cmd/go/internal/base/signal.go @@ -15,7 +15,7 @@ var Interrupted = make(chan struct{}) // processSignals setups signal handler. func processSignals() { - sig := make(chan os.Signal) + sig := make(chan os.Signal, 1) signal.Notify(sig, signalsToIgnore...) go func() { <-sig diff --git a/libgo/go/cmd/go/internal/bug/bug.go b/libgo/go/cmd/go/internal/bug/bug.go index fe71281..1085fea 100644 --- a/libgo/go/cmd/go/internal/bug/bug.go +++ b/libgo/go/cmd/go/internal/bug/bug.go @@ -7,9 +7,9 @@ package bug import ( "bytes" + "context" "fmt" "io" - "io/ioutil" urlpkg "net/url" "os" "os/exec" @@ -37,7 +37,7 @@ func init() { CmdBug.Flag.BoolVar(&cfg.BuildV, "v", false, "") } -func runBug(cmd *base.Command, args []string) { +func runBug(ctx context.Context, cmd *base.Command, args []string) { if len(args) > 0 { base.Fatalf("go bug: bug takes no arguments") } @@ -104,7 +104,7 @@ func printGoDetails(w io.Writer) { func printOSDetails(w io.Writer) { switch runtime.GOOS { - case "darwin": + case "darwin", "ios": printCmdOut(w, "uname -v: ", "uname", "-v") printCmdOut(w, "", "sw_vers") case "linux": @@ -116,7 +116,7 @@ func printOSDetails(w io.Writer) { case "illumos", "solaris": // Be sure to use the OS-supplied uname, in "/usr/bin": printCmdOut(w, "uname -srv: ", "/usr/bin/uname", "-srv") - out, err := ioutil.ReadFile("/etc/release") + out, err := os.ReadFile("/etc/release") if err == nil { fmt.Fprintf(w, "/etc/release: %s\n", out) } else { @@ -176,7 +176,7 @@ func printGlibcVersion(w io.Writer) { src := []byte(`int main() {}`) srcfile := filepath.Join(tempdir, "go-bug.c") outfile := filepath.Join(tempdir, "go-bug") - err := ioutil.WriteFile(srcfile, src, 0644) + err := os.WriteFile(srcfile, src, 0644) if err != nil { return } diff --git a/libgo/go/cmd/go/internal/cache/cache.go b/libgo/go/cmd/go/internal/cache/cache.go index 15545ac..41f9216 100644 --- a/libgo/go/cmd/go/internal/cache/cache.go +++ b/libgo/go/cmd/go/internal/cache/cache.go @@ -12,7 +12,7 @@ import ( "errors" "fmt" "io" - "io/ioutil" + "io/fs" "os" "path/filepath" "strconv" @@ -54,7 +54,7 @@ func Open(dir string) (*Cache, error) { return nil, err } if !info.IsDir() { - return nil, &os.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")} + return nil, &fs.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")} } for i := 0; i < 256; i++ { name := filepath.Join(dir, fmt.Sprintf("%02x", i)) @@ -238,7 +238,7 @@ func (c *Cache) GetBytes(id ActionID) ([]byte, Entry, error) { if err != nil { return nil, entry, err } - data, _ := ioutil.ReadFile(c.OutputFile(entry.OutputID)) + data, _ := os.ReadFile(c.OutputFile(entry.OutputID)) if sha256.Sum256(data) != entry.OutputID { return nil, entry, &entryNotFoundError{Err: errors.New("bad checksum")} } @@ -377,7 +377,7 @@ func (c *Cache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify // Truncate the file only *after* writing it. // (This should be a no-op, but truncate just in case of previous corruption.) // - // This differs from ioutil.WriteFile, which truncates to 0 *before* writing + // This differs from os.WriteFile, which truncates to 0 *before* writing // via os.O_TRUNC. Truncating only after writing ensures that a second write // of the same content to the same file is idempotent, and does not — even // temporarily! — undo the effect of the first write. diff --git a/libgo/go/cmd/go/internal/cache/cache_test.go b/libgo/go/cmd/go/internal/cache/cache_test.go index 1988c34..a865b97 100644 --- a/libgo/go/cmd/go/internal/cache/cache_test.go +++ b/libgo/go/cmd/go/internal/cache/cache_test.go @@ -8,7 +8,6 @@ import ( "bytes" "encoding/binary" "fmt" - "io/ioutil" "os" "path/filepath" "testing" @@ -20,7 +19,7 @@ func init() { } func TestBasic(t *testing.T) { - dir, err := ioutil.TempDir("", "cachetest-") + dir, err := os.MkdirTemp("", "cachetest-") if err != nil { t.Fatal(err) } @@ -65,7 +64,7 @@ func TestBasic(t *testing.T) { } func TestGrowth(t *testing.T) { - dir, err := ioutil.TempDir("", "cachetest-") + dir, err := os.MkdirTemp("", "cachetest-") if err != nil { t.Fatal(err) } @@ -118,7 +117,7 @@ func TestVerifyPanic(t *testing.T) { t.Fatal("initEnv did not set verify") } - dir, err := ioutil.TempDir("", "cachetest-") + dir, err := os.MkdirTemp("", "cachetest-") if err != nil { t.Fatal(err) } @@ -151,7 +150,7 @@ func dummyID(x int) [HashSize]byte { } func TestCacheTrim(t *testing.T) { - dir, err := ioutil.TempDir("", "cachetest-") + dir, err := os.MkdirTemp("", "cachetest-") if err != nil { t.Fatal(err) } @@ -207,7 +206,7 @@ func TestCacheTrim(t *testing.T) { t.Fatal(err) } c.OutputFile(entry.OutputID) - data, err := ioutil.ReadFile(filepath.Join(dir, "trim.txt")) + data, err := os.ReadFile(filepath.Join(dir, "trim.txt")) if err != nil { t.Fatal(err) } @@ -220,7 +219,7 @@ func TestCacheTrim(t *testing.T) { t.Fatal(err) } c.OutputFile(entry.OutputID) - data2, err := ioutil.ReadFile(filepath.Join(dir, "trim.txt")) + data2, err := os.ReadFile(filepath.Join(dir, "trim.txt")) if err != nil { t.Fatal(err) } diff --git a/libgo/go/cmd/go/internal/cache/default.go b/libgo/go/cmd/go/internal/cache/default.go index 9f8dd8a..0b1c1e0 100644 --- a/libgo/go/cmd/go/internal/cache/default.go +++ b/libgo/go/cmd/go/internal/cache/default.go @@ -6,7 +6,6 @@ package cache import ( "fmt" - "io/ioutil" "os" "path/filepath" "sync" @@ -49,7 +48,7 @@ func initDefaultCache() { } if _, err := os.Stat(filepath.Join(dir, "README")); err != nil { // Best effort. - ioutil.WriteFile(filepath.Join(dir, "README"), []byte(cacheREADME), 0666) + os.WriteFile(filepath.Join(dir, "README"), []byte(cacheREADME), 0666) } c, err := Open(dir) diff --git a/libgo/go/cmd/go/internal/cache/hash_test.go b/libgo/go/cmd/go/internal/cache/hash_test.go index 3bf7143..a035677 100644 --- a/libgo/go/cmd/go/internal/cache/hash_test.go +++ b/libgo/go/cmd/go/internal/cache/hash_test.go @@ -6,7 +6,6 @@ package cache import ( "fmt" - "io/ioutil" "os" "testing" ) @@ -28,7 +27,7 @@ func TestHash(t *testing.T) { } func TestHashFile(t *testing.T) { - f, err := ioutil.TempFile("", "cmd-go-test-") + f, err := os.CreateTemp("", "cmd-go-test-") if err != nil { t.Fatal(err) } diff --git a/libgo/go/cmd/go/internal/cfg/cfg.go b/libgo/go/cmd/go/internal/cfg/cfg.go index 7f8f8e9..c48904e 100644 --- a/libgo/go/cmd/go/internal/cfg/cfg.go +++ b/libgo/go/cmd/go/internal/cfg/cfg.go @@ -11,13 +11,15 @@ import ( "fmt" "go/build" "internal/cfg" - "io/ioutil" + "io" "os" "path/filepath" "runtime" "strings" "sync" + "cmd/go/internal/fsys" + "cmd/internal/objabi" ) @@ -27,7 +29,8 @@ var ( BuildBuildmode string // -buildmode flag BuildContext = defaultContext() BuildMod string // -mod flag - BuildModReason string // reason -mod flag is set, if set by default + BuildModExplicit bool // whether -mod was set explicitly + BuildModReason string // reason -mod was set, if set by default BuildI bool // -i flag BuildLinkshared bool // -linkshared flag BuildMSan bool // -msan flag @@ -48,9 +51,12 @@ var ( ModCacheRW bool // -modcacherw flag ModFile string // -modfile flag + Insecure bool // -insecure flag + CmdName string // "build", "install", "list", "mod tidy", etc. DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable) + DebugTrace string // -debug-trace flag ) func defaultContext() build.Context { @@ -100,6 +106,15 @@ func defaultContext() build.Context { // Nothing to do here. } + ctxt.OpenFile = func(path string) (io.ReadCloser, error) { + return fsys.Open(path) + } + ctxt.ReadDir = fsys.ReadDir + ctxt.IsDir = func(path string) bool { + isDir, err := fsys.IsDir(path) + return err == nil && isDir + } + return ctxt } @@ -171,7 +186,7 @@ func initEnvCache() { if file == "" { return } - data, err := ioutil.ReadFile(file) + data, err := os.ReadFile(file) if err != nil { return } @@ -252,6 +267,7 @@ var ( GONOPROXY = envOr("GONOPROXY", GOPRIVATE) GONOSUMDB = envOr("GONOSUMDB", GOPRIVATE) GOINSECURE = Getenv("GOINSECURE") + GOVCS = Getenv("GOVCS") ) var SumdbDir = gopathDir("pkg/sumdb") diff --git a/libgo/go/cmd/go/internal/clean/clean.go b/libgo/go/cmd/go/internal/clean/clean.go index 99704cb..b1d40fe 100644 --- a/libgo/go/cmd/go/internal/clean/clean.go +++ b/libgo/go/cmd/go/internal/clean/clean.go @@ -6,8 +6,9 @@ package clean import ( + "context" "fmt" - "io/ioutil" + "io" "os" "path/filepath" "strconv" @@ -105,7 +106,7 @@ func init() { work.AddBuildFlags(CmdClean, work.DefaultBuildFlags) } -func runClean(cmd *base.Command, args []string) { +func runClean(ctx context.Context, cmd *base.Command, args []string) { // golang.org/issue/29925: only load packages before cleaning if // either the flags and arguments explicitly imply a package, // or no other target (such as a cache) was requested to be cleaned. @@ -116,7 +117,7 @@ func runClean(cmd *base.Command, args []string) { } if cleanPkg { - for _, pkg := range load.PackagesAndErrors(args) { + for _, pkg := range load.PackagesAndErrors(ctx, args) { clean(pkg) } } @@ -171,7 +172,7 @@ func runClean(cmd *base.Command, args []string) { f, err := lockedfile.Edit(filepath.Join(dir, "testexpire.txt")) if err == nil { now := time.Now().UnixNano() - buf, _ := ioutil.ReadAll(f) + buf, _ := io.ReadAll(f) prev, _ := strconv.ParseInt(strings.TrimSpace(string(buf)), 10, 64) if now > prev { if err = f.Truncate(0); err == nil { @@ -242,7 +243,7 @@ func clean(p *load.Package) { base.Errorf("%v", p.Error) return } - dirs, err := ioutil.ReadDir(p.Dir) + dirs, err := os.ReadDir(p.Dir) if err != nil { base.Errorf("go clean %s: %v", p.Dir, err) return @@ -274,6 +275,8 @@ func clean(p *load.Package) { allRemove = append(allRemove, elem, elem+".exe", + p.DefaultExecName(), + p.DefaultExecName()+".exe", ) } @@ -281,16 +284,28 @@ func clean(p *load.Package) { allRemove = append(allRemove, elem+".test", elem+".test.exe", + p.DefaultExecName()+".test", + p.DefaultExecName()+".test.exe", ) - // Remove a potential executable for each .go file in the directory that + // Remove a potential executable, test executable for each .go file in the directory that // is not part of the directory's package. for _, dir := range dirs { name := dir.Name() if packageFile[name] { continue } - if !dir.IsDir() && strings.HasSuffix(name, ".go") { + + if dir.IsDir() { + continue + } + + if strings.HasSuffix(name, "_test.go") { + base := name[:len(name)-len("_test.go")] + allRemove = append(allRemove, base+".test", base+".test.exe") + } + + if strings.HasSuffix(name, ".go") { // TODO(adg,rsc): check that this .go file is actually // in "package main", and therefore capable of building // to an executable file. diff --git a/libgo/go/cmd/go/internal/doc/doc.go b/libgo/go/cmd/go/internal/doc/doc.go index 4ff08bb..67f76e2 100644 --- a/libgo/go/cmd/go/internal/doc/doc.go +++ b/libgo/go/cmd/go/internal/doc/doc.go @@ -8,6 +8,7 @@ package doc import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "context" ) var CmdDoc = &base.Command{ @@ -129,6 +130,6 @@ Flags: `, } -func runDoc(cmd *base.Command, args []string) { +func runDoc(ctx context.Context, cmd *base.Command, args []string) { base.Run(cfg.BuildToolexec, base.Tool("doc"), args) } diff --git a/libgo/go/cmd/go/internal/envcmd/env.go b/libgo/go/cmd/go/internal/envcmd/env.go index 252025d..6937187 100644 --- a/libgo/go/cmd/go/internal/envcmd/env.go +++ b/libgo/go/cmd/go/internal/envcmd/env.go @@ -6,10 +6,10 @@ package envcmd import ( + "context" "encoding/json" "fmt" "go/build" - "io/ioutil" "os" "path/filepath" "runtime" @@ -20,6 +20,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cache" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/work" @@ -62,9 +63,6 @@ var ( ) func MkEnv() []cfg.EnvVar { - var b work.Builder - b.Init() - envFile, _ := cfg.EnvFile() env := []cfg.EnvVar{ {Name: "GO111MODULE", Value: cfg.Getenv("GO111MODULE")}, @@ -88,6 +86,8 @@ func MkEnv() []cfg.EnvVar { {Name: "GOSUMDB", Value: cfg.GOSUMDB}, {Name: "GOTMPDIR", Value: cfg.Getenv("GOTMPDIR")}, {Name: "GOTOOLDIR", Value: base.ToolDir}, + {Name: "GOVCS", Value: cfg.GOVCS}, + {Name: "GOVERSION", Value: runtime.Version()}, } if work.GccgoBin != "" { @@ -186,7 +186,7 @@ func argKey(arg string) string { return arg[:i] } -func runEnv(cmd *base.Command, args []string) { +func runEnv(ctx context.Context, cmd *base.Command, args []string) { if *envJson && *envU { base.Fatalf("go env: cannot use -json with -u") } @@ -199,12 +199,26 @@ func runEnv(cmd *base.Command, args []string) { env := cfg.CmdEnv env = append(env, ExtraEnvVars()...) + if err := fsys.Init(base.Cwd); err != nil { + base.Fatalf("go: %v", err) + } + // Do we need to call ExtraEnvVarsCostly, which is a bit expensive? - // Only if we're listing all environment variables ("go env") - // or the variables being requested are in the extra list. - needCostly := true - if len(args) > 0 { + needCostly := false + if *envU || *envW { + // We're overwriting or removing default settings, + // so it doesn't really matter what the existing settings are. + // + // Moreover, we haven't validated the new settings yet, so it is + // important that we NOT perform any actions based on them, + // such as initializing the builder to compute other variables. + } else if len(args) == 0 { + // We're listing all environment variables ("go env"), + // including the expensive ones. + needCostly = true + } else { needCostly = false + checkCostly: for _, arg := range args { switch argKey(arg) { case "CGO_CFLAGS", @@ -215,6 +229,7 @@ func runEnv(cmd *base.Command, args []string) { "PKG_CONFIG", "GOGCCFLAGS": needCostly = true + break checkCostly } } } @@ -266,6 +281,13 @@ func runEnv(cmd *base.Command, args []string) { } } + gotmp, okGOTMP := add["GOTMPDIR"] + if okGOTMP { + if !filepath.IsAbs(gotmp) && gotmp != "" { + base.Fatalf("go env -w: GOTMPDIR must be an absolute path") + } + } + updateEnvFile(add, nil) return } @@ -377,7 +399,7 @@ func getOrigEnv(key string) string { func checkEnvWrite(key, val string) error { switch key { - case "GOEXE", "GOGCCFLAGS", "GOHOSTARCH", "GOHOSTOS", "GOMOD", "GOTOOLDIR": + case "GOEXE", "GOGCCFLAGS", "GOHOSTARCH", "GOHOSTOS", "GOMOD", "GOTOOLDIR", "GOVERSION": return fmt.Errorf("%s cannot be modified", key) case "GOENV": return fmt.Errorf("%s can only be set using the OS environment", key) @@ -405,6 +427,11 @@ func checkEnvWrite(key, val string) error { if !filepath.IsAbs(val) && val != "" { return fmt.Errorf("GOPATH entry is relative; must be absolute path: %q", val) } + // Make sure CC and CXX are absolute paths + case "CC", "CXX": + if !filepath.IsAbs(val) && val != "" && val != filepath.Base(val) { + return fmt.Errorf("%s entry is relative; must be absolute path: %q", key, val) + } } if !utf8.ValidString(val) { @@ -424,7 +451,7 @@ func updateEnvFile(add map[string]string, del map[string]bool) { if file == "" { base.Fatalf("go env: cannot find go env config: %v", err) } - data, err := ioutil.ReadFile(file) + data, err := os.ReadFile(file) if err != nil && (!os.IsNotExist(err) || len(add) == 0) { base.Fatalf("go env: reading go env config: %v", err) } @@ -478,11 +505,11 @@ func updateEnvFile(add map[string]string, del map[string]bool) { } data = []byte(strings.Join(lines, "")) - err = ioutil.WriteFile(file, data, 0666) + err = os.WriteFile(file, data, 0666) if err != nil { // Try creating directory. os.MkdirAll(filepath.Dir(file), 0777) - err = ioutil.WriteFile(file, data, 0666) + err = os.WriteFile(file, data, 0666) if err != nil { base.Fatalf("go env: writing go env config: %v", err) } @@ -499,7 +526,10 @@ func lineToKey(line string) string { } // sortKeyValues sorts a sequence of lines by key. -// It differs from sort.Strings in that GO386= sorts after GO=. +// It differs from sort.Strings in that keys which are GOx where x is an ASCII +// character smaller than = sort after GO=. +// (There are no such keys currently. It used to matter for GO386 which was +// removed in Go 1.16.) func sortKeyValues(lines []string) { sort.Slice(lines, func(i, j int) bool { return lineToKey(lines[i]) < lineToKey(lines[j]) diff --git a/libgo/go/cmd/go/internal/fix/fix.go b/libgo/go/cmd/go/internal/fix/fix.go index 4d741df..c7588c6 100644 --- a/libgo/go/cmd/go/internal/fix/fix.go +++ b/libgo/go/cmd/go/internal/fix/fix.go @@ -11,6 +11,7 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/str" + "context" "fmt" "os" ) @@ -31,9 +32,21 @@ See also: go fmt, go vet. `, } -func runFix(cmd *base.Command, args []string) { +func runFix(ctx context.Context, cmd *base.Command, args []string) { + pkgs := load.PackagesAndErrors(ctx, args) + w := 0 + for _, pkg := range pkgs { + if pkg.Error != nil { + base.Errorf("%v", pkg.Error) + continue + } + pkgs[w] = pkg + w++ + } + pkgs = pkgs[:w] + printed := false - for _, pkg := range load.Packages(args) { + for _, pkg := range pkgs { if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main { if !printed { fmt.Fprintf(os.Stderr, "go: not fixing packages in dependency modules\n") diff --git a/libgo/go/cmd/go/internal/fmtcmd/fmt.go b/libgo/go/cmd/go/internal/fmtcmd/fmt.go index d6894ed..b0c1c59 100644 --- a/libgo/go/cmd/go/internal/fmtcmd/fmt.go +++ b/libgo/go/cmd/go/internal/fmtcmd/fmt.go @@ -6,6 +6,7 @@ package fmtcmd import ( + "context" "errors" "fmt" "os" @@ -22,7 +23,8 @@ import ( func init() { base.AddBuildFlagsNX(&CmdFmt.Flag) - base.AddLoadFlags(&CmdFmt.Flag) + base.AddModFlag(&CmdFmt.Flag) + base.AddModCommonFlags(&CmdFmt.Flag) } var CmdFmt = &base.Command{ @@ -48,7 +50,7 @@ See also: go fix, go vet. `, } -func runFmt(cmd *base.Command, args []string) { +func runFmt(ctx context.Context, cmd *base.Command, args []string) { printed := false gofmt := gofmtPath() procs := runtime.GOMAXPROCS(0) @@ -63,7 +65,7 @@ func runFmt(cmd *base.Command, args []string) { } }() } - for _, pkg := range load.PackagesAndErrors(args) { + for _, pkg := range load.PackagesAndErrors(ctx, args) { if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main { if !printed { fmt.Fprintf(os.Stderr, "go: not formatting packages in dependency modules\n") diff --git a/libgo/go/cmd/go/internal/fsys/fsys.go b/libgo/go/cmd/go/internal/fsys/fsys.go new file mode 100644 index 0000000..7b06c3c --- /dev/null +++ b/libgo/go/cmd/go/internal/fsys/fsys.go @@ -0,0 +1,689 @@ +// Package fsys is an abstraction for reading files that +// allows for virtual overlays on top of the files on disk. +package fsys + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "time" +) + +// OverlayFile is the path to a text file in the OverlayJSON format. +// It is the value of the -overlay flag. +var OverlayFile string + +// OverlayJSON is the format overlay files are expected to be in. +// The Replace map maps from overlaid paths to replacement paths: +// the Go command will forward all reads trying to open +// each overlaid path to its replacement path, or consider the overlaid +// path not to exist if the replacement path is empty. +type OverlayJSON struct { + Replace map[string]string +} + +type node struct { + actualFilePath string // empty if a directory + children map[string]*node // path element → file or directory +} + +func (n *node) isDir() bool { + return n.actualFilePath == "" && n.children != nil +} + +func (n *node) isDeleted() bool { + return n.actualFilePath == "" && n.children == nil +} + +// TODO(matloob): encapsulate these in an io/fs-like interface +var overlay map[string]*node // path -> file or directory node +var cwd string // copy of base.Cwd to avoid dependency + +// Canonicalize a path for looking it up in the overlay. +// Important: filepath.Join(cwd, path) doesn't always produce +// the correct absolute path if path is relative, because on +// Windows producing the correct absolute path requires making +// a syscall. So this should only be used when looking up paths +// in the overlay, or canonicalizing the paths in the overlay. +func canonicalize(path string) string { + if path == "" { + return "" + } + if filepath.IsAbs(path) { + return filepath.Clean(path) + } + + if v := filepath.VolumeName(cwd); v != "" && path[0] == filepath.Separator { + // On Windows filepath.Join(cwd, path) doesn't always work. In general + // filepath.Abs needs to make a syscall on Windows. Elsewhere in cmd/go + // use filepath.Join(cwd, path), but cmd/go specifically supports Windows + // paths that start with "\" which implies the path is relative to the + // volume of the working directory. See golang.org/issue/8130. + return filepath.Join(v, path) + } + + // Make the path absolute. + return filepath.Join(cwd, path) +} + +// Init initializes the overlay, if one is being used. +func Init(wd string) error { + if overlay != nil { + // already initialized + return nil + } + + cwd = wd + + if OverlayFile == "" { + return nil + } + + b, err := os.ReadFile(OverlayFile) + if err != nil { + return fmt.Errorf("reading overlay file: %v", err) + } + + var overlayJSON OverlayJSON + if err := json.Unmarshal(b, &overlayJSON); err != nil { + return fmt.Errorf("parsing overlay JSON: %v", err) + } + + return initFromJSON(overlayJSON) +} + +func initFromJSON(overlayJSON OverlayJSON) error { + // Canonicalize the paths in in the overlay map. + // Use reverseCanonicalized to check for collisions: + // no two 'from' paths should canonicalize to the same path. + overlay = make(map[string]*node) + reverseCanonicalized := make(map[string]string) // inverse of canonicalize operation, to check for duplicates + // Build a table of file and directory nodes from the replacement map. + + // Remove any potential non-determinism from iterating over map by sorting it. + replaceFrom := make([]string, 0, len(overlayJSON.Replace)) + for k := range overlayJSON.Replace { + replaceFrom = append(replaceFrom, k) + } + sort.Strings(replaceFrom) + + for _, from := range replaceFrom { + to := overlayJSON.Replace[from] + // Canonicalize paths and check for a collision. + if from == "" { + return fmt.Errorf("empty string key in overlay file Replace map") + } + cfrom := canonicalize(from) + if to != "" { + // Don't canonicalize "", meaning to delete a file, because then it will turn into ".". + to = canonicalize(to) + } + if otherFrom, seen := reverseCanonicalized[cfrom]; seen { + return fmt.Errorf( + "paths %q and %q both canonicalize to %q in overlay file Replace map", otherFrom, from, cfrom) + } + reverseCanonicalized[cfrom] = from + from = cfrom + + // Create node for overlaid file. + dir, base := filepath.Dir(from), filepath.Base(from) + if n, ok := overlay[from]; ok { + // All 'from' paths in the overlay are file paths. Since the from paths + // are in a map, they are unique, so if the node already exists we added + // it below when we create parent directory nodes. That is, that + // both a file and a path to one of its parent directories exist as keys + // in the Replace map. + // + // This only applies if the overlay directory has any files or directories + // in it: placeholder directories that only contain deleted files don't + // count. They are safe to be overwritten with actual files. + for _, f := range n.children { + if !f.isDeleted() { + return fmt.Errorf("invalid overlay: path %v is used as both file and directory", from) + } + } + } + overlay[from] = &node{actualFilePath: to} + + // Add parent directory nodes to overlay structure. + childNode := overlay[from] + for { + dirNode := overlay[dir] + if dirNode == nil || dirNode.isDeleted() { + dirNode = &node{children: make(map[string]*node)} + overlay[dir] = dirNode + } + if childNode.isDeleted() { + // Only create one parent for a deleted file: + // the directory only conditionally exists if + // there are any non-deleted children, so + // we don't create their parents. + if dirNode.isDir() { + dirNode.children[base] = childNode + } + break + } + if !dirNode.isDir() { + // This path already exists as a file, so it can't be a parent + // directory. See comment at error above. + return fmt.Errorf("invalid overlay: path %v is used as both file and directory", dir) + } + dirNode.children[base] = childNode + parent := filepath.Dir(dir) + if parent == dir { + break // reached the top; there is no parent + } + dir, base = parent, filepath.Base(dir) + childNode = dirNode + } + } + + return nil +} + +// IsDir returns true if path is a directory on disk or in the +// overlay. +func IsDir(path string) (bool, error) { + path = canonicalize(path) + + if _, ok := parentIsOverlayFile(path); ok { + return false, nil + } + + if n, ok := overlay[path]; ok { + return n.isDir(), nil + } + + fi, err := os.Stat(path) + if err != nil { + return false, err + } + + return fi.IsDir(), nil +} + +// parentIsOverlayFile returns whether name or any of +// its parents are files in the overlay, and the first parent found, +// including name itself, that's a file in the overlay. +func parentIsOverlayFile(name string) (string, bool) { + if overlay != nil { + // Check if name can't possibly be a directory because + // it or one of its parents is overlaid with a file. + // TODO(matloob): Maybe save this to avoid doing it every time? + prefix := name + for { + node := overlay[prefix] + if node != nil && !node.isDir() { + return prefix, true + } + parent := filepath.Dir(prefix) + if parent == prefix { + break + } + prefix = parent + } + } + + return "", false +} + +// errNotDir is used to communicate from ReadDir to IsDirWithGoFiles +// that the argument is not a directory, so that IsDirWithGoFiles doesn't +// return an error. +var errNotDir = errors.New("not a directory") + +// readDir reads a dir on disk, returning an error that is errNotDir if the dir is not a directory. +// Unfortunately, the error returned by ioutil.ReadDir if dir is not a directory +// can vary depending on the OS (Linux, Mac, Windows return ENOTDIR; BSD returns EINVAL). +func readDir(dir string) ([]fs.FileInfo, error) { + fis, err := ioutil.ReadDir(dir) + if err == nil { + return fis, nil + } + + if os.IsNotExist(err) { + return nil, err + } + if dirfi, staterr := os.Stat(dir); staterr == nil && !dirfi.IsDir() { + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} + } + return nil, err +} + +// ReadDir provides a slice of fs.FileInfo entries corresponding +// to the overlaid files in the directory. +func ReadDir(dir string) ([]fs.FileInfo, error) { + dir = canonicalize(dir) + if _, ok := parentIsOverlayFile(dir); ok { + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} + } + + dirNode := overlay[dir] + if dirNode == nil { + return readDir(dir) + } + if dirNode.isDeleted() { + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: fs.ErrNotExist} + } + diskfis, err := readDir(dir) + if err != nil && !os.IsNotExist(err) && !errors.Is(err, errNotDir) { + return nil, err + } + + // Stat files in overlay to make composite list of fileinfos + files := make(map[string]fs.FileInfo) + for _, f := range diskfis { + files[f.Name()] = f + } + for name, to := range dirNode.children { + switch { + case to.isDir(): + files[name] = fakeDir(name) + case to.isDeleted(): + delete(files, name) + default: + // This is a regular file. + f, err := os.Lstat(to.actualFilePath) + if err != nil { + files[name] = missingFile(name) + continue + } else if f.IsDir() { + return nil, fmt.Errorf("for overlay of %q to %q: overlay Replace entries can't point to dirctories", + filepath.Join(dir, name), to.actualFilePath) + } + // Add a fileinfo for the overlaid file, so that it has + // the original file's name, but the overlaid file's metadata. + files[name] = fakeFile{name, f} + } + } + sortedFiles := diskfis[:0] + for _, f := range files { + sortedFiles = append(sortedFiles, f) + } + sort.Slice(sortedFiles, func(i, j int) bool { return sortedFiles[i].Name() < sortedFiles[j].Name() }) + return sortedFiles, nil +} + +// OverlayPath returns the path to the overlaid contents of the +// file, the empty string if the overlay deletes the file, or path +// itself if the file is not in the overlay, the file is a directory +// in the overlay, or there is no overlay. +// It returns true if the path is overlaid with a regular file +// or deleted, and false otherwise. +func OverlayPath(path string) (string, bool) { + if p, ok := overlay[canonicalize(path)]; ok && !p.isDir() { + return p.actualFilePath, ok + } + + return path, false +} + +// Open opens the file at or overlaid on the given path. +func Open(path string) (*os.File, error) { + return OpenFile(path, os.O_RDONLY, 0) +} + +// OpenFile opens the file at or overlaid on the given path with the flag and perm. +func OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) { + cpath := canonicalize(path) + if node, ok := overlay[cpath]; ok { + // Opening a file in the overlay. + if node.isDir() { + return nil, &fs.PathError{Op: "OpenFile", Path: path, Err: errors.New("fsys.OpenFile doesn't support opening directories yet")} + } + // We can't open overlaid paths for write. + if perm != os.FileMode(os.O_RDONLY) { + return nil, &fs.PathError{Op: "OpenFile", Path: path, Err: errors.New("overlaid files can't be opened for write")} + } + return os.OpenFile(node.actualFilePath, flag, perm) + } + if parent, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { + // The file is deleted explicitly in the Replace map, + // or implicitly because one of its parent directories was + // replaced by a file. + return nil, &fs.PathError{ + Op: "Open", + Path: path, + Err: fmt.Errorf("file %s does not exist: parent directory %s is replaced by a file in overlay", path, parent), + } + } + return os.OpenFile(cpath, flag, perm) +} + +// IsDirWithGoFiles reports whether dir is a directory containing Go files +// either on disk or in the overlay. +func IsDirWithGoFiles(dir string) (bool, error) { + fis, err := ReadDir(dir) + if os.IsNotExist(err) || errors.Is(err, errNotDir) { + return false, nil + } + if err != nil { + return false, err + } + + var firstErr error + for _, fi := range fis { + if fi.IsDir() { + continue + } + + // TODO(matloob): this enforces that the "from" in the map + // has a .go suffix, but the actual destination file + // doesn't need to have a .go suffix. Is this okay with the + // compiler? + if !strings.HasSuffix(fi.Name(), ".go") { + continue + } + if fi.Mode().IsRegular() { + return true, nil + } + + // fi is the result of an Lstat, so it doesn't follow symlinks. + // But it's okay if the file is a symlink pointing to a regular + // file, so use os.Stat to follow symlinks and check that. + actualFilePath, _ := OverlayPath(filepath.Join(dir, fi.Name())) + fi, err := os.Stat(actualFilePath) + if err == nil && fi.Mode().IsRegular() { + return true, nil + } + if err != nil && firstErr == nil { + firstErr = err + } + } + + // No go files found in directory. + return false, firstErr +} + +// walk recursively descends path, calling walkFn. Copied, with some +// modifications from path/filepath.walk. +func walk(path string, info fs.FileInfo, walkFn filepath.WalkFunc) error { + if !info.IsDir() { + return walkFn(path, info, nil) + } + + fis, readErr := ReadDir(path) + walkErr := walkFn(path, info, readErr) + // If readErr != nil, walk can't walk into this directory. + // walkErr != nil means walkFn want walk to skip this directory or stop walking. + // Therefore, if one of readErr and walkErr isn't nil, walk will return. + if readErr != nil || walkErr != nil { + // The caller's behavior is controlled by the return value, which is decided + // by walkFn. walkFn may ignore readErr and return nil. + // If walkFn returns SkipDir, it will be handled by the caller. + // So walk should return whatever walkFn returns. + return walkErr + } + + for _, fi := range fis { + filename := filepath.Join(path, fi.Name()) + if walkErr = walk(filename, fi, walkFn); walkErr != nil { + if !fi.IsDir() || walkErr != filepath.SkipDir { + return walkErr + } + } + } + return nil +} + +// Walk walks the file tree rooted at root, calling walkFn for each file or +// directory in the tree, including root. +func Walk(root string, walkFn filepath.WalkFunc) error { + info, err := Lstat(root) + if err != nil { + err = walkFn(root, nil, err) + } else { + err = walk(root, info, walkFn) + } + if err == filepath.SkipDir { + return nil + } + return err +} + +// lstat implements a version of os.Lstat that operates on the overlay filesystem. +func Lstat(path string) (fs.FileInfo, error) { + return overlayStat(path, os.Lstat, "lstat") +} + +// Stat implements a version of os.Stat that operates on the overlay filesystem. +func Stat(path string) (fs.FileInfo, error) { + return overlayStat(path, os.Stat, "stat") +} + +// overlayStat implements lstat or Stat (depending on whether os.Lstat or os.Stat is passed in). +func overlayStat(path string, osStat func(string) (fs.FileInfo, error), opName string) (fs.FileInfo, error) { + cpath := canonicalize(path) + + if _, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { + return nil, &fs.PathError{Op: opName, Path: cpath, Err: fs.ErrNotExist} + } + + node, ok := overlay[cpath] + if !ok { + // The file or directory is not overlaid. + return osStat(path) + } + + switch { + case node.isDeleted(): + return nil, &fs.PathError{Op: "lstat", Path: cpath, Err: fs.ErrNotExist} + case node.isDir(): + return fakeDir(filepath.Base(path)), nil + default: + fi, err := osStat(node.actualFilePath) + if err != nil { + return nil, err + } + return fakeFile{name: filepath.Base(path), real: fi}, nil + } +} + +// fakeFile provides an fs.FileInfo implementation for an overlaid file, +// so that the file has the name of the overlaid file, but takes all +// other characteristics of the replacement file. +type fakeFile struct { + name string + real fs.FileInfo +} + +func (f fakeFile) Name() string { return f.name } +func (f fakeFile) Size() int64 { return f.real.Size() } +func (f fakeFile) Mode() fs.FileMode { return f.real.Mode() } +func (f fakeFile) ModTime() time.Time { return f.real.ModTime() } +func (f fakeFile) IsDir() bool { return f.real.IsDir() } +func (f fakeFile) Sys() interface{} { return f.real.Sys() } + +// missingFile provides an fs.FileInfo for an overlaid file where the +// destination file in the overlay doesn't exist. It returns zero values +// for the fileInfo methods other than Name, set to the file's name, and Mode +// set to ModeIrregular. +type missingFile string + +func (f missingFile) Name() string { return string(f) } +func (f missingFile) Size() int64 { return 0 } +func (f missingFile) Mode() fs.FileMode { return fs.ModeIrregular } +func (f missingFile) ModTime() time.Time { return time.Unix(0, 0) } +func (f missingFile) IsDir() bool { return false } +func (f missingFile) Sys() interface{} { return nil } + +// fakeDir provides an fs.FileInfo implementation for directories that are +// implicitly created by overlaid files. Each directory in the +// path of an overlaid file is considered to exist in the overlay filesystem. +type fakeDir string + +func (f fakeDir) Name() string { return string(f) } +func (f fakeDir) Size() int64 { return 0 } +func (f fakeDir) Mode() fs.FileMode { return fs.ModeDir | 0500 } +func (f fakeDir) ModTime() time.Time { return time.Unix(0, 0) } +func (f fakeDir) IsDir() bool { return true } +func (f fakeDir) Sys() interface{} { return nil } + +// Glob is like filepath.Glob but uses the overlay file system. +func Glob(pattern string) (matches []string, err error) { + // Check pattern is well-formed. + if _, err := filepath.Match(pattern, ""); err != nil { + return nil, err + } + if !hasMeta(pattern) { + if _, err = Lstat(pattern); err != nil { + return nil, nil + } + return []string{pattern}, nil + } + + dir, file := filepath.Split(pattern) + volumeLen := 0 + if runtime.GOOS == "windows" { + volumeLen, dir = cleanGlobPathWindows(dir) + } else { + dir = cleanGlobPath(dir) + } + + if !hasMeta(dir[volumeLen:]) { + return glob(dir, file, nil) + } + + // Prevent infinite recursion. See issue 15879. + if dir == pattern { + return nil, filepath.ErrBadPattern + } + + var m []string + m, err = Glob(dir) + if err != nil { + return + } + for _, d := range m { + matches, err = glob(d, file, matches) + if err != nil { + return + } + } + return +} + +// cleanGlobPath prepares path for glob matching. +func cleanGlobPath(path string) string { + switch path { + case "": + return "." + case string(filepath.Separator): + // do nothing to the path + return path + default: + return path[0 : len(path)-1] // chop off trailing separator + } +} + +func volumeNameLen(path string) int { + isSlash := func(c uint8) bool { + return c == '\\' || c == '/' + } + if len(path) < 2 { + return 0 + } + // with drive letter + c := path[0] + if path[1] == ':' && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') { + return 2 + } + // is it UNC? https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx + if l := len(path); l >= 5 && isSlash(path[0]) && isSlash(path[1]) && + !isSlash(path[2]) && path[2] != '.' { + // first, leading `\\` and next shouldn't be `\`. its server name. + for n := 3; n < l-1; n++ { + // second, next '\' shouldn't be repeated. + if isSlash(path[n]) { + n++ + // third, following something characters. its share name. + if !isSlash(path[n]) { + if path[n] == '.' { + break + } + for ; n < l; n++ { + if isSlash(path[n]) { + break + } + } + return n + } + break + } + } + } + return 0 +} + +// cleanGlobPathWindows is windows version of cleanGlobPath. +func cleanGlobPathWindows(path string) (prefixLen int, cleaned string) { + vollen := volumeNameLen(path) + switch { + case path == "": + return 0, "." + case vollen+1 == len(path) && os.IsPathSeparator(path[len(path)-1]): // /, \, C:\ and C:/ + // do nothing to the path + return vollen + 1, path + case vollen == len(path) && len(path) == 2: // C: + return vollen, path + "." // convert C: into C:. + default: + if vollen >= len(path) { + vollen = len(path) - 1 + } + return vollen, path[0 : len(path)-1] // chop off trailing separator + } +} + +// glob searches for files matching pattern in the directory dir +// and appends them to matches. If the directory cannot be +// opened, it returns the existing matches. New matches are +// added in lexicographical order. +func glob(dir, pattern string, matches []string) (m []string, e error) { + m = matches + fi, err := Stat(dir) + if err != nil { + return // ignore I/O error + } + if !fi.IsDir() { + return // ignore I/O error + } + + list, err := ReadDir(dir) + if err != nil { + return // ignore I/O error + } + + var names []string + for _, info := range list { + names = append(names, info.Name()) + } + sort.Strings(names) + + for _, n := range names { + matched, err := filepath.Match(pattern, n) + if err != nil { + return m, err + } + if matched { + m = append(m, filepath.Join(dir, n)) + } + } + return +} + +// hasMeta reports whether path contains any of the magic characters +// recognized by filepath.Match. +func hasMeta(path string) bool { + magicChars := `*?[` + if runtime.GOOS != "windows" { + magicChars = `*?[\` + } + return strings.ContainsAny(path, magicChars) +} diff --git a/libgo/go/cmd/go/internal/fsys/fsys_test.go b/libgo/go/cmd/go/internal/fsys/fsys_test.go new file mode 100644 index 0000000..7f175c7 --- /dev/null +++ b/libgo/go/cmd/go/internal/fsys/fsys_test.go @@ -0,0 +1,1094 @@ +package fsys + +import ( + "cmd/go/internal/txtar" + "encoding/json" + "errors" + "fmt" + "internal/testenv" + "io" + "io/fs" + "os" + "path/filepath" + "reflect" + "testing" +) + +// initOverlay resets the overlay state to reflect the config. +// config should be a text archive string. The comment is the overlay config +// json, and the files, in the archive are laid out in a temp directory +// that cwd is set to. +func initOverlay(t *testing.T, config string) { + t.Helper() + + // Create a temporary directory and chdir to it. + prevwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + cwd = filepath.Join(t.TempDir(), "root") + if err := os.Mkdir(cwd, 0777); err != nil { + t.Fatal(err) + } + if err := os.Chdir(cwd); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + overlay = nil + if err := os.Chdir(prevwd); err != nil { + t.Fatal(err) + } + }) + + a := txtar.Parse([]byte(config)) + for _, f := range a.Files { + name := filepath.Join(cwd, f.Name) + if err := os.MkdirAll(filepath.Dir(name), 0777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(name, f.Data, 0666); err != nil { + t.Fatal(err) + } + } + + var overlayJSON OverlayJSON + if err := json.Unmarshal(a.Comment, &overlayJSON); err != nil { + t.Fatal(fmt.Errorf("parsing overlay JSON: %v", err)) + } + + initFromJSON(overlayJSON) +} + +func TestIsDir(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "subdir2/file2.txt": "overlayfiles/subdir2_file2.txt", + "subdir4": "overlayfiles/subdir4", + "subdir3/file3b.txt": "overlayfiles/subdir3_file3b.txt", + "subdir5": "", + "subdir6": "" + } +} +-- subdir1/file1.txt -- + +-- subdir3/file3a.txt -- +33 +-- subdir4/file4.txt -- +444 +-- overlayfiles/subdir2_file2.txt -- +2 +-- overlayfiles/subdir3_file3b.txt -- +66666 +-- overlayfiles/subdir4 -- +x +-- subdir6/file6.txt -- +six +`) + + testCases := []struct { + path string + want, wantErr bool + }{ + {"", true, true}, + {".", true, false}, + {cwd, true, false}, + {cwd + string(filepath.Separator), true, false}, + // subdir1 is only on disk + {filepath.Join(cwd, "subdir1"), true, false}, + {"subdir1", true, false}, + {"subdir1" + string(filepath.Separator), true, false}, + {"subdir1/file1.txt", false, false}, + {"subdir1/doesntexist.txt", false, true}, + {"doesntexist", false, true}, + // subdir2 is only in overlay + {filepath.Join(cwd, "subdir2"), true, false}, + {"subdir2", true, false}, + {"subdir2" + string(filepath.Separator), true, false}, + {"subdir2/file2.txt", false, false}, + {"subdir2/doesntexist.txt", false, true}, + // subdir3 has files on disk and in overlay + {filepath.Join(cwd, "subdir3"), true, false}, + {"subdir3", true, false}, + {"subdir3" + string(filepath.Separator), true, false}, + {"subdir3/file3a.txt", false, false}, + {"subdir3/file3b.txt", false, false}, + {"subdir3/doesntexist.txt", false, true}, + // subdir4 is overlaid with a file + {filepath.Join(cwd, "subdir4"), false, false}, + {"subdir4", false, false}, + {"subdir4" + string(filepath.Separator), false, false}, + {"subdir4/file4.txt", false, false}, + {"subdir4/doesntexist.txt", false, false}, + // subdir5 doesn't exist, and is overlaid with a "delete" entry + {filepath.Join(cwd, "subdir5"), false, false}, + {"subdir5", false, false}, + {"subdir5" + string(filepath.Separator), false, false}, + {"subdir5/file5.txt", false, false}, + {"subdir5/doesntexist.txt", false, false}, + // subdir6 does exist, and is overlaid with a "delete" entry + {filepath.Join(cwd, "subdir6"), false, false}, + {"subdir6", false, false}, + {"subdir6" + string(filepath.Separator), false, false}, + {"subdir6/file6.txt", false, false}, + {"subdir6/doesntexist.txt", false, false}, + } + + for _, tc := range testCases { + got, err := IsDir(tc.path) + if err != nil { + if !tc.wantErr { + t.Errorf("IsDir(%q): got error with string %q, want no error", tc.path, err.Error()) + } + continue + } + if tc.wantErr { + t.Errorf("IsDir(%q): got no error, want error", tc.path) + } + if tc.want != got { + t.Errorf("IsDir(%q) = %v, want %v", tc.path, got, tc.want) + } + } +} + +const readDirOverlay = ` +{ + "Replace": { + "subdir2/file2.txt": "overlayfiles/subdir2_file2.txt", + "subdir4": "overlayfiles/subdir4", + "subdir3/file3b.txt": "overlayfiles/subdir3_file3b.txt", + "subdir5": "", + "subdir6/asubsubdir/afile.txt": "overlayfiles/subdir6_asubsubdir_afile.txt", + "subdir6/asubsubdir/zfile.txt": "overlayfiles/subdir6_asubsubdir_zfile.txt", + "subdir6/zsubsubdir/file.txt": "overlayfiles/subdir6_zsubsubdir_file.txt", + "subdir7/asubsubdir/file.txt": "overlayfiles/subdir7_asubsubdir_file.txt", + "subdir7/zsubsubdir/file.txt": "overlayfiles/subdir7_zsubsubdir_file.txt", + "subdir8/doesntexist": "this_file_doesnt_exist_anywhere", + "other/pointstodir": "overlayfiles/this_is_a_directory", + "parentoverwritten/subdir1": "overlayfiles/parentoverwritten_subdir1", + "subdir9/this_file_is_overlaid.txt": "overlayfiles/subdir9_this_file_is_overlaid.txt", + "subdir10/only_deleted_file.txt": "", + "subdir11/deleted.txt": "", + "subdir11": "overlayfiles/subdir11", + "textfile.txt/file.go": "overlayfiles/textfile_txt_file.go" + } +} +-- subdir1/file1.txt -- + +-- subdir3/file3a.txt -- +33 +-- subdir4/file4.txt -- +444 +-- subdir6/file.txt -- +-- subdir6/asubsubdir/file.txt -- +-- subdir6/anothersubsubdir/file.txt -- +-- subdir9/this_file_is_overlaid.txt -- +-- subdir10/only_deleted_file.txt -- +this will be deleted in overlay +-- subdir11/deleted.txt -- +-- parentoverwritten/subdir1/subdir2/subdir3/file.txt -- +-- textfile.txt -- +this will be overridden by textfile.txt/file.go +-- overlayfiles/subdir2_file2.txt -- +2 +-- overlayfiles/subdir3_file3b.txt -- +66666 +-- overlayfiles/subdir4 -- +x +-- overlayfiles/subdir6_asubsubdir_afile.txt -- +-- overlayfiles/subdir6_asubsubdir_zfile.txt -- +-- overlayfiles/subdir6_zsubsubdir_file.txt -- +-- overlayfiles/subdir7_asubsubdir_file.txt -- +-- overlayfiles/subdir7_zsubsubdir_file.txt -- +-- overlayfiles/parentoverwritten_subdir1 -- +x +-- overlayfiles/subdir9_this_file_is_overlaid.txt -- +99999999 +-- overlayfiles/subdir11 -- +-- overlayfiles/this_is_a_directory/file.txt -- +-- overlayfiles/textfile_txt_file.go -- +x +` + +func TestReadDir(t *testing.T) { + initOverlay(t, readDirOverlay) + + type entry struct { + name string + size int64 + isDir bool + } + + testCases := []struct { + dir string + want []entry + }{ + { + ".", []entry{ + {"other", 0, true}, + {"overlayfiles", 0, true}, + {"parentoverwritten", 0, true}, + {"subdir1", 0, true}, + {"subdir10", 0, true}, + {"subdir11", 0, false}, + {"subdir2", 0, true}, + {"subdir3", 0, true}, + {"subdir4", 2, false}, + // no subdir5. + {"subdir6", 0, true}, + {"subdir7", 0, true}, + {"subdir8", 0, true}, + {"subdir9", 0, true}, + {"textfile.txt", 0, true}, + }, + }, + { + "subdir1", []entry{ + {"file1.txt", 1, false}, + }, + }, + { + "subdir2", []entry{ + {"file2.txt", 2, false}, + }, + }, + { + "subdir3", []entry{ + {"file3a.txt", 3, false}, + {"file3b.txt", 6, false}, + }, + }, + { + "subdir6", []entry{ + {"anothersubsubdir", 0, true}, + {"asubsubdir", 0, true}, + {"file.txt", 0, false}, + {"zsubsubdir", 0, true}, + }, + }, + { + "subdir6/asubsubdir", []entry{ + {"afile.txt", 0, false}, + {"file.txt", 0, false}, + {"zfile.txt", 0, false}, + }, + }, + { + "subdir8", []entry{ + {"doesntexist", 0, false}, // entry is returned even if destination file doesn't exist + }, + }, + { + // check that read dir actually redirects files that already exist + // the original this_file_is_overlaid.txt is empty + "subdir9", []entry{ + {"this_file_is_overlaid.txt", 9, false}, + }, + }, + { + "subdir10", []entry{}, + }, + { + "parentoverwritten", []entry{ + {"subdir1", 2, false}, + }, + }, + { + "textfile.txt", []entry{ + {"file.go", 2, false}, + }, + }, + } + + for _, tc := range testCases { + dir, want := tc.dir, tc.want + infos, err := ReadDir(dir) + if err != nil { + t.Errorf("ReadDir(%q): %v", dir, err) + continue + } + // Sorted diff of want and infos. + for len(infos) > 0 || len(want) > 0 { + switch { + case len(want) == 0 || len(infos) > 0 && infos[0].Name() < want[0].name: + t.Errorf("ReadDir(%q): unexpected entry: %s IsDir=%v Size=%v", dir, infos[0].Name(), infos[0].IsDir(), infos[0].Size()) + infos = infos[1:] + case len(infos) == 0 || len(want) > 0 && want[0].name < infos[0].Name(): + t.Errorf("ReadDir(%q): missing entry: %s IsDir=%v Size=%v", dir, want[0].name, want[0].isDir, want[0].size) + want = want[1:] + default: + infoSize := infos[0].Size() + if want[0].isDir { + infoSize = 0 + } + if infos[0].IsDir() != want[0].isDir || want[0].isDir && infoSize != want[0].size { + t.Errorf("ReadDir(%q): %s: IsDir=%v Size=%v, want IsDir=%v Size=%v", dir, want[0].name, infos[0].IsDir(), infoSize, want[0].isDir, want[0].size) + } + infos = infos[1:] + want = want[1:] + } + } + } + + errCases := []string{ + "subdir1/file1.txt", // regular file on disk + "subdir2/file2.txt", // regular file in overlay + "subdir4", // directory overlaid with regular file + "subdir5", // directory deleted in overlay + "parentoverwritten/subdir1/subdir2/subdir3", // parentoverwritten/subdir1 overlaid with regular file + "parentoverwritten/subdir1/subdir2", // parentoverwritten/subdir1 overlaid with regular file + "subdir11", // directory with deleted child, overlaid with regular file + "other/pointstodir", + } + + for _, dir := range errCases { + _, err := ReadDir(dir) + if _, ok := err.(*fs.PathError); !ok { + t.Errorf("ReadDir(%q): err = %T (%v), want fs.PathError", dir, err, err) + } + } +} + +func TestGlob(t *testing.T) { + initOverlay(t, readDirOverlay) + + testCases := []struct { + pattern string + match []string + }{ + { + "*o*", + []string{ + "other", + "overlayfiles", + "parentoverwritten", + }, + }, + { + "subdir2/file2.txt", + []string{ + "subdir2/file2.txt", + }, + }, + { + "*/*.txt", + []string{ + "overlayfiles/subdir2_file2.txt", + "overlayfiles/subdir3_file3b.txt", + "overlayfiles/subdir6_asubsubdir_afile.txt", + "overlayfiles/subdir6_asubsubdir_zfile.txt", + "overlayfiles/subdir6_zsubsubdir_file.txt", + "overlayfiles/subdir7_asubsubdir_file.txt", + "overlayfiles/subdir7_zsubsubdir_file.txt", + "overlayfiles/subdir9_this_file_is_overlaid.txt", + "subdir1/file1.txt", + "subdir2/file2.txt", + "subdir3/file3a.txt", + "subdir3/file3b.txt", + "subdir6/file.txt", + "subdir9/this_file_is_overlaid.txt", + }, + }, + } + + for _, tc := range testCases { + pattern := tc.pattern + match, err := Glob(pattern) + if err != nil { + t.Errorf("Glob(%q): %v", pattern, err) + continue + } + want := tc.match + for i, name := range want { + if name != tc.pattern { + want[i] = filepath.FromSlash(name) + } + } + for len(match) > 0 || len(want) > 0 { + switch { + case len(match) == 0 || len(want) > 0 && want[0] < match[0]: + t.Errorf("Glob(%q): missing match: %s", pattern, want[0]) + want = want[1:] + case len(want) == 0 || len(match) > 0 && match[0] < want[0]: + t.Errorf("Glob(%q): extra match: %s", pattern, match[0]) + match = match[1:] + default: + want = want[1:] + match = match[1:] + } + } + } +} + +func TestOverlayPath(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "subdir2/file2.txt": "overlayfiles/subdir2_file2.txt", + "subdir3/doesntexist": "this_file_doesnt_exist_anywhere", + "subdir4/this_file_is_overlaid.txt": "overlayfiles/subdir4_this_file_is_overlaid.txt", + "subdir5/deleted.txt": "", + "parentoverwritten/subdir1": "" + } +} +-- subdir1/file1.txt -- +file 1 +-- subdir4/this_file_is_overlaid.txt -- +these contents are replaced by the overlay +-- parentoverwritten/subdir1/subdir2/subdir3/file.txt -- +-- subdir5/deleted.txt -- +deleted +-- overlayfiles/subdir2_file2.txt -- +file 2 +-- overlayfiles/subdir4_this_file_is_overlaid.txt -- +99999999 +`) + + testCases := []struct { + path string + wantPath string + wantOK bool + }{ + {"subdir1/file1.txt", "subdir1/file1.txt", false}, + // OverlayPath returns false for directories + {"subdir2", "subdir2", false}, + {"subdir2/file2.txt", filepath.Join(cwd, "overlayfiles/subdir2_file2.txt"), true}, + // OverlayPath doesn't stat a file to see if it exists, so it happily returns + // the 'to' path and true even if the 'to' path doesn't exist on disk. + {"subdir3/doesntexist", filepath.Join(cwd, "this_file_doesnt_exist_anywhere"), true}, + // Like the subdir2/file2.txt case above, but subdir4 exists on disk, but subdir2 does not. + {"subdir4/this_file_is_overlaid.txt", filepath.Join(cwd, "overlayfiles/subdir4_this_file_is_overlaid.txt"), true}, + {"subdir5", "subdir5", false}, + {"subdir5/deleted.txt", "", true}, + } + + for _, tc := range testCases { + gotPath, gotOK := OverlayPath(tc.path) + if gotPath != tc.wantPath || gotOK != tc.wantOK { + t.Errorf("OverlayPath(%q): got %v, %v; want %v, %v", + tc.path, gotPath, gotOK, tc.wantPath, tc.wantOK) + } + } +} + +func TestOpen(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "subdir2/file2.txt": "overlayfiles/subdir2_file2.txt", + "subdir3/doesntexist": "this_file_doesnt_exist_anywhere", + "subdir4/this_file_is_overlaid.txt": "overlayfiles/subdir4_this_file_is_overlaid.txt", + "subdir5/deleted.txt": "", + "parentoverwritten/subdir1": "", + "childoverlay/subdir1.txt/child.txt": "overlayfiles/child.txt", + "subdir11/deleted.txt": "", + "subdir11": "overlayfiles/subdir11", + "parentdeleted": "", + "parentdeleted/file.txt": "overlayfiles/parentdeleted_file.txt" + } +} +-- subdir11/deleted.txt -- +-- subdir1/file1.txt -- +file 1 +-- subdir4/this_file_is_overlaid.txt -- +these contents are replaced by the overlay +-- parentoverwritten/subdir1/subdir2/subdir3/file.txt -- +-- childoverlay/subdir1.txt -- +this file doesn't exist because the path +childoverlay/subdir1.txt/child.txt is in the overlay +-- subdir5/deleted.txt -- +deleted +-- parentdeleted -- +this will be deleted so that parentdeleted/file.txt can exist +-- overlayfiles/subdir2_file2.txt -- +file 2 +-- overlayfiles/subdir4_this_file_is_overlaid.txt -- +99999999 +-- overlayfiles/child.txt -- +-- overlayfiles/subdir11 -- +11 +-- overlayfiles/parentdeleted_file.txt -- +this can exist because the parent directory is deleted +`) + + testCases := []struct { + path string + wantContents string + isErr bool + }{ + {"subdir1/file1.txt", "file 1\n", false}, + {"subdir2/file2.txt", "file 2\n", false}, + {"subdir3/doesntexist", "", true}, + {"subdir4/this_file_is_overlaid.txt", "99999999\n", false}, + {"subdir5/deleted.txt", "", true}, + {"parentoverwritten/subdir1/subdir2/subdir3/file.txt", "", true}, + {"childoverlay/subdir1.txt", "", true}, + {"subdir11", "11\n", false}, + {"parentdeleted/file.txt", "this can exist because the parent directory is deleted\n", false}, + } + + for _, tc := range testCases { + f, err := Open(tc.path) + if tc.isErr { + if err == nil { + f.Close() + t.Errorf("Open(%q): got no error, but want error", tc.path) + } + continue + } + if err != nil { + t.Errorf("Open(%q): got error %v, want nil", tc.path, err) + continue + } + contents, err := io.ReadAll(f) + if err != nil { + t.Errorf("unexpected error reading contents of file: %v", err) + } + if string(contents) != tc.wantContents { + t.Errorf("contents of file opened with Open(%q): got %q, want %q", + tc.path, contents, tc.wantContents) + } + f.Close() + } +} + +func TestIsDirWithGoFiles(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "goinoverlay/file.go": "dummy", + "directory/removed/by/file": "dummy", + "directory_with_go_dir/dir.go/file.txt": "dummy", + "otherdirectory/deleted.go": "", + "nonexistentdirectory/deleted.go": "", + "textfile.txt/file.go": "dummy" + } +} +-- dummy -- +a destination file for the overlay entries to point to +contents don't matter for this test +-- nogo/file.txt -- +-- goondisk/file.go -- +-- goinoverlay/file.txt -- +-- directory/removed/by/file/in/overlay/file.go -- +-- otherdirectory/deleted.go -- +-- textfile.txt -- +`) + + testCases := []struct { + dir string + want bool + wantErr bool + }{ + {"nogo", false, false}, + {"goondisk", true, false}, + {"goinoverlay", true, false}, + {"directory/removed/by/file/in/overlay", false, false}, + {"directory_with_go_dir", false, false}, + {"otherdirectory", false, false}, + {"nonexistentdirectory", false, false}, + {"textfile.txt", true, false}, + } + + for _, tc := range testCases { + got, gotErr := IsDirWithGoFiles(tc.dir) + if tc.wantErr { + if gotErr == nil { + t.Errorf("IsDirWithGoFiles(%q): got %v, %v; want non-nil error", tc.dir, got, gotErr) + } + continue + } + if gotErr != nil { + t.Errorf("IsDirWithGoFiles(%q): got %v, %v; want nil error", tc.dir, got, gotErr) + } + if got != tc.want { + t.Errorf("IsDirWithGoFiles(%q) = %v; want %v", tc.dir, got, tc.want) + } + } +} + +func TestWalk(t *testing.T) { + // The root of the walk must be a name with an actual basename, not just ".". + // Walk uses Lstat to obtain the name of the root, and Lstat on platforms + // other than Plan 9 reports the name "." instead of the actual base name of + // the directory. (See https://golang.org/issue/42115.) + + type file struct { + path string + name string + size int64 + mode fs.FileMode + isDir bool + } + testCases := []struct { + name string + overlay string + root string + wantFiles []file + }{ + {"no overlay", ` +{} +-- dir/file.txt -- +`, + "dir", + []file{ + {"dir", "dir", 0, fs.ModeDir | 0700, true}, + {"dir/file.txt", "file.txt", 0, 0600, false}, + }, + }, + {"overlay with different file", ` +{ + "Replace": { + "dir/file.txt": "dir/other.txt" + } +} +-- dir/file.txt -- +-- dir/other.txt -- +contents of other file +`, + "dir", + []file{ + {"dir", "dir", 0, fs.ModeDir | 0500, true}, + {"dir/file.txt", "file.txt", 23, 0600, false}, + {"dir/other.txt", "other.txt", 23, 0600, false}, + }, + }, + {"overlay with new file", ` +{ + "Replace": { + "dir/file.txt": "dir/other.txt" + } +} +-- dir/other.txt -- +contents of other file +`, + "dir", + []file{ + {"dir", "dir", 0, fs.ModeDir | 0500, true}, + {"dir/file.txt", "file.txt", 23, 0600, false}, + {"dir/other.txt", "other.txt", 23, 0600, false}, + }, + }, + {"overlay with new directory", ` +{ + "Replace": { + "dir/subdir/file.txt": "dir/other.txt" + } +} +-- dir/other.txt -- +contents of other file +`, + "dir", + []file{ + {"dir", "dir", 0, fs.ModeDir | 0500, true}, + {"dir/other.txt", "other.txt", 23, 0600, false}, + {"dir/subdir", "subdir", 0, fs.ModeDir | 0500, true}, + {"dir/subdir/file.txt", "file.txt", 23, 0600, false}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + initOverlay(t, tc.overlay) + + var got []file + Walk(tc.root, func(path string, info fs.FileInfo, err error) error { + got = append(got, file{path, info.Name(), info.Size(), info.Mode(), info.IsDir()}) + return nil + }) + + if len(got) != len(tc.wantFiles) { + t.Errorf("Walk: saw %#v in walk; want %#v", got, tc.wantFiles) + } + for i := 0; i < len(got) && i < len(tc.wantFiles); i++ { + wantPath := filepath.FromSlash(tc.wantFiles[i].path) + if got[i].path != wantPath { + t.Errorf("path of file #%v in walk, got %q, want %q", i, got[i].path, wantPath) + } + if got[i].name != tc.wantFiles[i].name { + t.Errorf("name of file #%v in walk, got %q, want %q", i, got[i].name, tc.wantFiles[i].name) + } + if got[i].mode&(fs.ModeDir|0700) != tc.wantFiles[i].mode { + t.Errorf("mode&(fs.ModeDir|0700) for mode of file #%v in walk, got %v, want %v", i, got[i].mode&(fs.ModeDir|0700), tc.wantFiles[i].mode) + } + if got[i].isDir != tc.wantFiles[i].isDir { + t.Errorf("isDir for file #%v in walk, got %v, want %v", i, got[i].isDir, tc.wantFiles[i].isDir) + } + if tc.wantFiles[i].isDir { + continue // don't check size for directories + } + if got[i].size != tc.wantFiles[i].size { + t.Errorf("size of file #%v in walk, got %v, want %v", i, got[i].size, tc.wantFiles[i].size) + } + } + }) + } +} + +func TestWalkSkipDir(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "dir/skip/file.go": "dummy.txt", + "dir/dontskip/file.go": "dummy.txt", + "dir/dontskip/skip/file.go": "dummy.txt" + } +} +-- dummy.txt -- +`) + + var seen []string + Walk("dir", func(path string, info fs.FileInfo, err error) error { + seen = append(seen, filepath.ToSlash(path)) + if info.Name() == "skip" { + return filepath.SkipDir + } + return nil + }) + + wantSeen := []string{"dir", "dir/dontskip", "dir/dontskip/file.go", "dir/dontskip/skip", "dir/skip"} + + if len(seen) != len(wantSeen) { + t.Errorf("paths seen in walk: got %v entries; want %v entries", len(seen), len(wantSeen)) + } + + for i := 0; i < len(seen) && i < len(wantSeen); i++ { + if seen[i] != wantSeen[i] { + t.Errorf("path #%v seen walking tree: want %q, got %q", i, seen[i], wantSeen[i]) + } + } +} + +func TestWalkError(t *testing.T) { + initOverlay(t, "{}") + + alreadyCalled := false + err := Walk("foo", func(path string, info fs.FileInfo, err error) error { + if alreadyCalled { + t.Fatal("expected walk function to be called exactly once, but it was called more than once") + } + alreadyCalled = true + return errors.New("returned from function") + }) + if !alreadyCalled { + t.Fatal("expected walk function to be called exactly once, but it was never called") + + } + if err == nil { + t.Fatalf("Walk: got no error, want error") + } + if err.Error() != "returned from function" { + t.Fatalf("Walk: got error %v, want \"returned from function\" error", err) + } +} + +func TestWalkSymlink(t *testing.T) { + testenv.MustHaveSymlink(t) + + initOverlay(t, `{ + "Replace": {"overlay_symlink": "symlink"} +} +-- dir/file --`) + + // Create symlink + if err := os.Symlink("dir", "symlink"); err != nil { + t.Error(err) + } + + testCases := []struct { + name string + dir string + wantFiles []string + }{ + {"control", "dir", []string{"dir", "dir" + string(filepath.Separator) + "file"}}, + // ensure Walk doesn't walk into the directory pointed to by the symlink + // (because it's supposed to use Lstat instead of Stat). + {"symlink_to_dir", "symlink", []string{"symlink"}}, + {"overlay_to_symlink_to_dir", "overlay_symlink", []string{"overlay_symlink"}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var got []string + + err := Walk(tc.dir, func(path string, info fs.FileInfo, err error) error { + got = append(got, path) + if err != nil { + t.Errorf("walkfn: got non nil err argument: %v, want nil err argument", err) + } + return nil + }) + if err != nil { + t.Errorf("Walk: got error %q, want nil", err) + } + + if !reflect.DeepEqual(got, tc.wantFiles) { + t.Errorf("files examined by walk: got %v, want %v", got, tc.wantFiles) + } + }) + } + +} + +func TestLstat(t *testing.T) { + type file struct { + name string + size int64 + mode fs.FileMode // mode & (fs.ModeDir|0x700): only check 'user' permissions + isDir bool + } + + testCases := []struct { + name string + overlay string + path string + + want file + wantErr bool + }{ + { + "regular_file", + `{} +-- file.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "new_file_in_overlay", + `{"Replace": {"file.txt": "dummy.txt"}} +-- dummy.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "file_replaced_in_overlay", + `{"Replace": {"file.txt": "dummy.txt"}} +-- file.txt -- +-- dummy.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "file_cant_exist", + `{"Replace": {"deleted": "dummy.txt"}} +-- deleted/file.txt -- +-- dummy.txt -- +`, + "deleted/file.txt", + file{}, + true, + }, + { + "deleted", + `{"Replace": {"deleted": ""}} +-- deleted -- +`, + "deleted", + file{}, + true, + }, + { + "dir_on_disk", + `{} +-- dir/foo.txt -- +`, + "dir", + file{"dir", 0, 0700 | fs.ModeDir, true}, + false, + }, + { + "dir_in_overlay", + `{"Replace": {"dir/file.txt": "dummy.txt"}} +-- dummy.txt -- +`, + "dir", + file{"dir", 0, 0500 | fs.ModeDir, true}, + false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + initOverlay(t, tc.overlay) + got, err := Lstat(tc.path) + if tc.wantErr { + if err == nil { + t.Errorf("lstat(%q): got no error, want error", tc.path) + } + return + } + if err != nil { + t.Fatalf("lstat(%q): got error %v, want no error", tc.path, err) + } + if got.Name() != tc.want.name { + t.Errorf("lstat(%q).Name(): got %q, want %q", tc.path, got.Name(), tc.want.name) + } + if got.Mode()&(fs.ModeDir|0700) != tc.want.mode { + t.Errorf("lstat(%q).Mode()&(fs.ModeDir|0700): got %v, want %v", tc.path, got.Mode()&(fs.ModeDir|0700), tc.want.mode) + } + if got.IsDir() != tc.want.isDir { + t.Errorf("lstat(%q).IsDir(): got %v, want %v", tc.path, got.IsDir(), tc.want.isDir) + } + if tc.want.isDir { + return // don't check size for directories + } + if got.Size() != tc.want.size { + t.Errorf("lstat(%q).Size(): got %v, want %v", tc.path, got.Size(), tc.want.size) + } + }) + } +} + +func TestStat(t *testing.T) { + testenv.MustHaveSymlink(t) + + type file struct { + name string + size int64 + mode os.FileMode // mode & (os.ModeDir|0x700): only check 'user' permissions + isDir bool + } + + testCases := []struct { + name string + overlay string + path string + + want file + wantErr bool + }{ + { + "regular_file", + `{} +-- file.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "new_file_in_overlay", + `{"Replace": {"file.txt": "dummy.txt"}} +-- dummy.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "file_replaced_in_overlay", + `{"Replace": {"file.txt": "dummy.txt"}} +-- file.txt -- +-- dummy.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "file_cant_exist", + `{"Replace": {"deleted": "dummy.txt"}} +-- deleted/file.txt -- +-- dummy.txt -- +`, + "deleted/file.txt", + file{}, + true, + }, + { + "deleted", + `{"Replace": {"deleted": ""}} +-- deleted -- +`, + "deleted", + file{}, + true, + }, + { + "dir_on_disk", + `{} +-- dir/foo.txt -- +`, + "dir", + file{"dir", 0, 0700 | os.ModeDir, true}, + false, + }, + { + "dir_in_overlay", + `{"Replace": {"dir/file.txt": "dummy.txt"}} +-- dummy.txt -- +`, + "dir", + file{"dir", 0, 0500 | os.ModeDir, true}, + false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + initOverlay(t, tc.overlay) + got, err := Stat(tc.path) + if tc.wantErr { + if err == nil { + t.Errorf("Stat(%q): got no error, want error", tc.path) + } + return + } + if err != nil { + t.Fatalf("Stat(%q): got error %v, want no error", tc.path, err) + } + if got.Name() != tc.want.name { + t.Errorf("Stat(%q).Name(): got %q, want %q", tc.path, got.Name(), tc.want.name) + } + if got.Mode()&(os.ModeDir|0700) != tc.want.mode { + t.Errorf("Stat(%q).Mode()&(os.ModeDir|0700): got %v, want %v", tc.path, got.Mode()&(os.ModeDir|0700), tc.want.mode) + } + if got.IsDir() != tc.want.isDir { + t.Errorf("Stat(%q).IsDir(): got %v, want %v", tc.path, got.IsDir(), tc.want.isDir) + } + if tc.want.isDir { + return // don't check size for directories + } + if got.Size() != tc.want.size { + t.Errorf("Stat(%q).Size(): got %v, want %v", tc.path, got.Size(), tc.want.size) + } + }) + } +} + +func TestStatSymlink(t *testing.T) { + testenv.MustHaveSymlink(t) + + initOverlay(t, `{ + "Replace": {"file.go": "symlink"} +} +-- to.go -- +0123456789 +`) + + // Create symlink + if err := os.Symlink("to.go", "symlink"); err != nil { + t.Error(err) + } + + f := "file.go" + fi, err := Stat(f) + if err != nil { + t.Errorf("Stat(%q): got error %q, want nil error", f, err) + } + + if !fi.Mode().IsRegular() { + t.Errorf("Stat(%q).Mode(): got %v, want regular mode", f, fi.Mode()) + } + + if fi.Size() != 11 { + t.Errorf("Stat(%q).Size(): got %v, want 11", f, fi.Size()) + } +} diff --git a/libgo/go/cmd/go/internal/generate/generate.go b/libgo/go/cmd/go/internal/generate/generate.go index 093b198..c740194 100644 --- a/libgo/go/cmd/go/internal/generate/generate.go +++ b/libgo/go/cmd/go/internal/generate/generate.go @@ -8,11 +8,11 @@ package generate import ( "bufio" "bytes" + "context" "fmt" "go/parser" "go/token" "io" - "io/ioutil" "log" "os" "os/exec" @@ -160,7 +160,7 @@ func init() { CmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "") } -func runGenerate(cmd *base.Command, args []string) { +func runGenerate(ctx context.Context, cmd *base.Command, args []string) { load.IgnoreImports = true if generateRunFlag != "" { @@ -175,7 +175,7 @@ func runGenerate(cmd *base.Command, args []string) { // Even if the arguments are .go files, this loop suffices. printed := false - for _, pkg := range load.PackagesAndErrors(args) { + for _, pkg := range load.PackagesAndErrors(ctx, args) { if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main { if !printed { fmt.Fprintf(os.Stderr, "go: not generating in packages in dependency modules\n") @@ -200,7 +200,7 @@ func runGenerate(cmd *base.Command, args []string) { // generate runs the generation directives for a single file. func generate(absFile string) bool { - src, err := ioutil.ReadFile(absFile) + src, err := os.ReadFile(absFile) if err != nil { log.Fatalf("generate: %s", err) } diff --git a/libgo/go/cmd/go/internal/get/get.go b/libgo/go/cmd/go/internal/get/get.go index d38350c..94a42c4 100644 --- a/libgo/go/cmd/go/internal/get/get.go +++ b/libgo/go/cmd/go/internal/get/get.go @@ -6,6 +6,7 @@ package get import ( + "context" "fmt" "os" "path/filepath" @@ -17,8 +18,11 @@ import ( "cmd/go/internal/load" "cmd/go/internal/search" "cmd/go/internal/str" + "cmd/go/internal/vcs" "cmd/go/internal/web" "cmd/go/internal/work" + + "golang.org/x/mod/module" ) var CmdGet = &base.Command{ @@ -41,6 +45,10 @@ before resolving dependencies or building the code. The -insecure flag permits fetching from repositories and resolving custom domains using insecure schemes such as HTTP. Use with caution. +This flag is deprecated and will be removed in a future version of go. +The GOINSECURE environment variable should be used instead, since it +provides control over which packages may be retrieved using an insecure +scheme. See 'go help environment' for details. The -t flag instructs get to also download the packages required to build the tests for the specified packages. @@ -102,17 +110,15 @@ var ( getT = CmdGet.Flag.Bool("t", false, "") getU = CmdGet.Flag.Bool("u", false, "") getFix = CmdGet.Flag.Bool("fix", false, "") - - Insecure bool ) func init() { work.AddBuildFlags(CmdGet, work.OmitModFlag|work.OmitModCommonFlags) CmdGet.Run = runGet // break init loop - CmdGet.Flag.BoolVar(&Insecure, "insecure", Insecure, "") + CmdGet.Flag.BoolVar(&cfg.Insecure, "insecure", cfg.Insecure, "") } -func runGet(cmd *base.Command, args []string) { +func runGet(ctx context.Context, cmd *base.Command, args []string) { if cfg.ModulesEnabled { // Should not happen: main.go should install the separate module-enabled get code. base.Fatalf("go get: modules not implemented") @@ -123,6 +129,9 @@ func runGet(cmd *base.Command, args []string) { if *getF && !*getU { base.Fatalf("go get: cannot use -f flag without -u") } + if cfg.Insecure { + fmt.Fprintf(os.Stderr, "go get: -insecure flag is deprecated; see 'go help get' for details\n") + } // Disable any prompting for passwords by Git. // Only has an effect for 2.3.0 or later, but avoiding @@ -171,17 +180,18 @@ func runGet(cmd *base.Command, args []string) { // everything. load.ClearPackageCache() - pkgs := load.PackagesForBuild(args) + pkgs := load.PackagesAndErrors(ctx, args) + load.CheckPackageErrors(pkgs) // Phase 3. Install. if *getD { // Download only. - // Check delayed until now so that importPaths - // and packagesForBuild have a chance to print errors. + // Check delayed until now so that downloadPaths + // and CheckPackageErrors have a chance to print errors. return } - work.InstallPackages(args, pkgs) + work.InstallPackages(ctx, args, pkgs) } // downloadPaths prepares the list of paths to pass to download. @@ -245,9 +255,9 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) load1 := func(path string, mode int) *load.Package { if parent == nil { mode := 0 // don't do module or vendor resolution - return load.LoadImport(path, base.Cwd, nil, stk, nil, mode) + return load.LoadImport(context.TODO(), path, base.Cwd, nil, stk, nil, mode) } - return load.LoadImport(path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) + return load.LoadImport(context.TODO(), path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) } p := load1(arg, mode) @@ -402,17 +412,12 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) // to make the first copy of or update a copy of the given package. func downloadPackage(p *load.Package) error { var ( - vcs *vcsCmd + vcsCmd *vcs.Cmd repo, rootPath string err error blindRepo bool // set if the repo has unusual configuration ) - security := web.SecureOnly - if Insecure { - security = web.Insecure - } - // p can be either a real package, or a pseudo-package whose “import path” is // actually a wildcard pattern. // Trim the path at the element containing the first wildcard, @@ -426,22 +431,26 @@ func downloadPackage(p *load.Package) error { } importPrefix = importPrefix[:slash] } - if err := CheckImportPath(importPrefix); err != nil { + if err := module.CheckImportPath(importPrefix); err != nil { return fmt.Errorf("%s: invalid import path: %v", p.ImportPath, err) } + security := web.SecureOnly + if cfg.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, importPrefix) { + security = web.Insecure + } if p.Internal.Build.SrcRoot != "" { // Directory exists. Look for checkout along path to src. - vcs, rootPath, err = vcsFromDir(p.Dir, p.Internal.Build.SrcRoot) + vcsCmd, rootPath, err = vcs.FromDir(p.Dir, p.Internal.Build.SrcRoot) if err != nil { return err } repo = "<local>" // should be unused; make distinctive // Double-check where it came from. - if *getU && vcs.remoteRepo != nil { + if *getU && vcsCmd.RemoteRepo != nil { dir := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath)) - remote, err := vcs.remoteRepo(vcs, dir) + remote, err := vcsCmd.RemoteRepo(vcsCmd, dir) if err != nil { // Proceed anyway. The package is present; we likely just don't understand // the repo configuration (e.g. unusual remote protocol). @@ -449,10 +458,10 @@ func downloadPackage(p *load.Package) error { } repo = remote if !*getF && err == nil { - if rr, err := RepoRootForImportPath(importPrefix, IgnoreMod, security); err == nil { + if rr, err := vcs.RepoRootForImportPath(importPrefix, vcs.IgnoreMod, security); err == nil { repo := rr.Repo - if rr.vcs.resolveRepo != nil { - resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo) + if rr.VCS.ResolveRepo != nil { + resolved, err := rr.VCS.ResolveRepo(rr.VCS, dir, repo) if err == nil { repo = resolved } @@ -466,13 +475,13 @@ func downloadPackage(p *load.Package) error { } else { // Analyze the import path to determine the version control system, // repository, and the import path for the root of the repository. - rr, err := RepoRootForImportPath(importPrefix, IgnoreMod, security) + rr, err := vcs.RepoRootForImportPath(importPrefix, vcs.IgnoreMod, security) if err != nil { return err } - vcs, repo, rootPath = rr.vcs, rr.Repo, rr.Root + vcsCmd, repo, rootPath = rr.VCS, rr.Repo, rr.Root } - if !blindRepo && !vcs.isSecure(repo) && !Insecure { + if !blindRepo && !vcsCmd.IsSecure(repo) && security != web.Insecure { return fmt.Errorf("cannot download, %v uses insecure protocol", repo) } @@ -495,7 +504,7 @@ func downloadPackage(p *load.Package) error { } root := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath)) - if err := checkNestedVCS(vcs, root, p.Internal.Build.SrcRoot); err != nil { + if err := vcs.CheckNested(vcsCmd, root, p.Internal.Build.SrcRoot); err != nil { return err } @@ -511,7 +520,7 @@ func downloadPackage(p *load.Package) error { // Check that this is an appropriate place for the repo to be checked out. // The target directory must either not exist or have a repo checked out already. - meta := filepath.Join(root, "."+vcs.cmd) + meta := filepath.Join(root, "."+vcsCmd.Cmd) if _, err := os.Stat(meta); err != nil { // Metadata file or directory does not exist. Prepare to checkout new copy. // Some version control tools require the target directory not to exist. @@ -532,12 +541,12 @@ func downloadPackage(p *load.Package) error { fmt.Fprintf(os.Stderr, "created GOPATH=%s; see 'go help gopath'\n", p.Internal.Build.Root) } - if err = vcs.create(root, repo); err != nil { + if err = vcsCmd.Create(root, repo); err != nil { return err } } else { // Metadata directory does exist; download incremental updates. - if err = vcs.download(root); err != nil { + if err = vcsCmd.Download(root); err != nil { return err } } @@ -546,12 +555,12 @@ func downloadPackage(p *load.Package) error { // Do not show tag sync in -n; it's noise more than anything, // and since we're not running commands, no tag will be found. // But avoid printing nothing. - fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcs.cmd) + fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcsCmd.Cmd) return nil } // Select and sync to appropriate version of the repository. - tags, err := vcs.tags(root) + tags, err := vcsCmd.Tags(root) if err != nil { return err } @@ -559,7 +568,7 @@ func downloadPackage(p *load.Package) error { if i := strings.Index(vers, " "); i >= 0 { vers = vers[:i] } - if err := vcs.tagSync(root, selectTag(vers, tags)); err != nil { + if err := vcsCmd.TagSync(root, selectTag(vers, tags)); err != nil { return err } diff --git a/libgo/go/cmd/go/internal/get/path.go b/libgo/go/cmd/go/internal/get/path.go deleted file mode 100644 index ce2e0cd..0000000 --- a/libgo/go/cmd/go/internal/get/path.go +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package get - -import ( - "fmt" - "strings" - "unicode" - "unicode/utf8" -) - -// The following functions are copied verbatim from golang.org/x/mod/module/module.go, -// with a change to additionally reject Windows short-names, -// and one to accept arbitrary letters (golang.org/issue/29101). -// -// TODO(bcmills): After the call site for this function is backported, -// consolidate this back down to a single copy. -// -// NOTE: DO NOT MERGE THESE UNTIL WE DECIDE ABOUT ARBITRARY LETTERS IN MODULE MODE. - -// CheckImportPath checks that an import path is valid. -func CheckImportPath(path string) error { - if err := checkPath(path, false); err != nil { - return fmt.Errorf("malformed import path %q: %v", path, err) - } - return nil -} - -// checkPath checks that a general path is valid. -// It returns an error describing why but not mentioning path. -// Because these checks apply to both module paths and import paths, -// the caller is expected to add the "malformed ___ path %q: " prefix. -// fileName indicates whether the final element of the path is a file name -// (as opposed to a directory name). -func checkPath(path string, fileName bool) error { - if !utf8.ValidString(path) { - return fmt.Errorf("invalid UTF-8") - } - if path == "" { - return fmt.Errorf("empty string") - } - if path[0] == '-' { - return fmt.Errorf("leading dash") - } - if strings.Contains(path, "//") { - return fmt.Errorf("double slash") - } - if path[len(path)-1] == '/' { - return fmt.Errorf("trailing slash") - } - elemStart := 0 - for i, r := range path { - if r == '/' { - if err := checkElem(path[elemStart:i], fileName); err != nil { - return err - } - elemStart = i + 1 - } - } - if err := checkElem(path[elemStart:], fileName); err != nil { - return err - } - return nil -} - -// checkElem checks whether an individual path element is valid. -// fileName indicates whether the element is a file name (not a directory name). -func checkElem(elem string, fileName bool) error { - if elem == "" { - return fmt.Errorf("empty path element") - } - if strings.Count(elem, ".") == len(elem) { - return fmt.Errorf("invalid path element %q", elem) - } - if elem[0] == '.' && !fileName { - return fmt.Errorf("leading dot in path element") - } - if elem[len(elem)-1] == '.' { - return fmt.Errorf("trailing dot in path element") - } - - charOK := pathOK - if fileName { - charOK = fileNameOK - } - for _, r := range elem { - if !charOK(r) { - return fmt.Errorf("invalid char %q", r) - } - } - - // Windows disallows a bunch of path elements, sadly. - // See https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file - short := elem - if i := strings.Index(short, "."); i >= 0 { - short = short[:i] - } - for _, bad := range badWindowsNames { - if strings.EqualFold(bad, short) { - return fmt.Errorf("disallowed path element %q", elem) - } - } - - // Reject path components that look like Windows short-names. - // Those usually end in a tilde followed by one or more ASCII digits. - if tilde := strings.LastIndexByte(short, '~'); tilde >= 0 && tilde < len(short)-1 { - suffix := short[tilde+1:] - suffixIsDigits := true - for _, r := range suffix { - if r < '0' || r > '9' { - suffixIsDigits = false - break - } - } - if suffixIsDigits { - return fmt.Errorf("trailing tilde and digits in path element") - } - } - - return nil -} - -// pathOK reports whether r can appear in an import path element. -// -// NOTE: This function DIVERGES from module mode pathOK by accepting Unicode letters. -func pathOK(r rune) bool { - if r < utf8.RuneSelf { - return r == '+' || r == '-' || r == '.' || r == '_' || r == '~' || - '0' <= r && r <= '9' || - 'A' <= r && r <= 'Z' || - 'a' <= r && r <= 'z' - } - return unicode.IsLetter(r) -} - -// fileNameOK reports whether r can appear in a file name. -// For now we allow all Unicode letters but otherwise limit to pathOK plus a few more punctuation characters. -// If we expand the set of allowed characters here, we have to -// work harder at detecting potential case-folding and normalization collisions. -// See note about "safe encoding" below. -func fileNameOK(r rune) bool { - if r < utf8.RuneSelf { - // Entire set of ASCII punctuation, from which we remove characters: - // ! " # $ % & ' ( ) * + , - . / : ; < = > ? @ [ \ ] ^ _ ` { | } ~ - // We disallow some shell special characters: " ' * < > ? ` | - // (Note that some of those are disallowed by the Windows file system as well.) - // We also disallow path separators / : and \ (fileNameOK is only called on path element characters). - // We allow spaces (U+0020) in file names. - const allowed = "!#$%&()+,-.=@[]^_{}~ " - if '0' <= r && r <= '9' || 'A' <= r && r <= 'Z' || 'a' <= r && r <= 'z' { - return true - } - for i := 0; i < len(allowed); i++ { - if rune(allowed[i]) == r { - return true - } - } - return false - } - // It may be OK to add more ASCII punctuation here, but only carefully. - // For example Windows disallows < > \, and macOS disallows :, so we must not allow those. - return unicode.IsLetter(r) -} - -// badWindowsNames are the reserved file path elements on Windows. -// See https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file -var badWindowsNames = []string{ - "CON", - "PRN", - "AUX", - "NUL", - "COM1", - "COM2", - "COM3", - "COM4", - "COM5", - "COM6", - "COM7", - "COM8", - "COM9", - "LPT1", - "LPT2", - "LPT3", - "LPT4", - "LPT5", - "LPT6", - "LPT7", - "LPT8", - "LPT9", -} diff --git a/libgo/go/cmd/go/internal/help/helpdoc.go b/libgo/go/cmd/go/internal/help/helpdoc.go index e1f0521..98f5844 100644 --- a/libgo/go/cmd/go/internal/help/helpdoc.go +++ b/libgo/go/cmd/go/internal/help/helpdoc.go @@ -526,7 +526,7 @@ General-purpose environment variables: Comma-separated list of glob patterns (in the syntax of Go's path.Match) of module path prefixes that should always be fetched directly or that should not be compared against the checksum database. - See 'go help module-private'. + See 'go help private'. GOROOT The root of the go tree. GOSUMDB @@ -582,8 +582,8 @@ Architecture-specific environment variables: For GOARCH=arm, the ARM architecture for which to compile. Valid values are 5, 6, 7. GO386 - For GOARCH=386, the floating point instruction set. - Valid values are 387, sse2. + For GOARCH=386, how to implement floating point instructions. + Valid values are sse2 (default), softfloat. GOMIPS For GOARCH=mips{,le}, whether to use floating point instructions. Valid values are hardfloat (default), softfloat. @@ -632,6 +632,8 @@ Additional information available from 'go env' but not read from the environment If module-aware mode is disabled, GOMOD will be the empty string. GOTOOLDIR The directory where the go tools (compile, cover, doc, etc...) are installed. + GOVERSION + The version of the installed Go tree, as reported by runtime.Version. `, } @@ -838,6 +840,9 @@ in addition to android tags and files. Using GOOS=illumos matches build tags and files as for GOOS=solaris in addition to illumos tags and files. +Using GOOS=ios matches build tags and files as for GOOS=darwin +in addition to ios tags and files. + To keep a file from being considered for the build: // +build ignore diff --git a/libgo/go/cmd/go/internal/imports/build.go b/libgo/go/cmd/go/internal/imports/build.go index eb070ee..50aeabc 100644 --- a/libgo/go/cmd/go/internal/imports/build.go +++ b/libgo/go/cmd/go/internal/imports/build.go @@ -141,6 +141,9 @@ func matchTag(name string, tags map[string]bool, want bool) bool { if name == "solaris" { have = have || tags["illumos"] } + if name == "darwin" { + have = have || tags["ios"] + } return have == want } @@ -158,6 +161,7 @@ func matchTag(name string, tags map[string]bool, want bool) bool { // Exceptions: // if GOOS=android, then files with GOOS=linux are also matched. // if GOOS=illumos, then files with GOOS=solaris are also matched. +// if GOOS=ios, then files with GOOS=darwin are also matched. // // If tags["*"] is true, then MatchFile will consider all possible // GOOS and GOARCH to be available and will consequently @@ -208,6 +212,7 @@ var KnownOS = map[string]bool{ "freebsd": true, "hurd": true, "illumos": true, + "ios": true, "js": true, "linux": true, "nacl": true, // legacy; don't remove diff --git a/libgo/go/cmd/go/internal/imports/read.go b/libgo/go/cmd/go/internal/imports/read.go index 58c2abd..5e27078 100644 --- a/libgo/go/cmd/go/internal/imports/read.go +++ b/libgo/go/cmd/go/internal/imports/read.go @@ -198,7 +198,7 @@ func (r *importReader) readImport(imports *[]string) { r.readString(imports) } -// ReadComments is like ioutil.ReadAll, except that it only reads the leading +// ReadComments is like io.ReadAll, except that it only reads the leading // block of comments in the file. func ReadComments(f io.Reader) ([]byte, error) { r := &importReader{b: bufio.NewReader(f)} @@ -210,7 +210,7 @@ func ReadComments(f io.Reader) ([]byte, error) { return r.buf, r.err } -// ReadImports is like ioutil.ReadAll, except that it expects a Go file as input +// ReadImports is like io.ReadAll, except that it expects a Go file as input // and stops reading the input once the imports have completed. func ReadImports(f io.Reader, reportSyntaxError bool, imports *[]string) ([]byte, error) { r := &importReader{b: bufio.NewReader(f)} diff --git a/libgo/go/cmd/go/internal/imports/scan.go b/libgo/go/cmd/go/internal/imports/scan.go index 3d9b613..ee11a87 100644 --- a/libgo/go/cmd/go/internal/imports/scan.go +++ b/libgo/go/cmd/go/internal/imports/scan.go @@ -6,16 +6,17 @@ package imports import ( "fmt" - "io/ioutil" - "os" + "io/fs" "path/filepath" "sort" "strconv" "strings" + + "cmd/go/internal/fsys" ) func ScanDir(dir string, tags map[string]bool) ([]string, []string, error) { - infos, err := ioutil.ReadDir(dir) + infos, err := fsys.ReadDir(dir) if err != nil { return nil, nil, err } @@ -25,14 +26,14 @@ func ScanDir(dir string, tags map[string]bool) ([]string, []string, error) { // If the directory entry is a symlink, stat it to obtain the info for the // link target instead of the link itself. - if info.Mode()&os.ModeSymlink != 0 { - info, err = os.Stat(filepath.Join(dir, name)) + if info.Mode()&fs.ModeSymlink != 0 { + info, err = fsys.Stat(filepath.Join(dir, name)) if err != nil { continue // Ignore broken symlinks. } } - if info.Mode().IsRegular() && !strings.HasPrefix(name, "_") && strings.HasSuffix(name, ".go") && MatchFile(name, tags) { + if info.Mode().IsRegular() && !strings.HasPrefix(name, "_") && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go") && MatchFile(name, tags) { files = append(files, filepath.Join(dir, name)) } } @@ -49,7 +50,7 @@ func scanFiles(files []string, tags map[string]bool, explicitFiles bool) ([]stri numFiles := 0 Files: for _, name := range files { - r, err := os.Open(name) + r, err := fsys.Open(name) if err != nil { return nil, nil, err } diff --git a/libgo/go/cmd/go/internal/imports/scan_test.go b/libgo/go/cmd/go/internal/imports/scan_test.go index e424656..2d245ee 100644 --- a/libgo/go/cmd/go/internal/imports/scan_test.go +++ b/libgo/go/cmd/go/internal/imports/scan_test.go @@ -7,7 +7,7 @@ package imports import ( "bytes" "internal/testenv" - "io/ioutil" + "os" "path" "path/filepath" "runtime" @@ -57,7 +57,7 @@ func TestScan(t *testing.T) { func TestScanDir(t *testing.T) { testenv.MustHaveGoBuild(t) - dirs, err := ioutil.ReadDir("testdata") + dirs, err := os.ReadDir("testdata") if err != nil { t.Fatal(err) } @@ -66,7 +66,7 @@ func TestScanDir(t *testing.T) { continue } t.Run(dir.Name(), func(t *testing.T) { - tagsData, err := ioutil.ReadFile(filepath.Join("testdata", dir.Name(), "tags.txt")) + tagsData, err := os.ReadFile(filepath.Join("testdata", dir.Name(), "tags.txt")) if err != nil { t.Fatalf("error reading tags: %v", err) } @@ -75,7 +75,7 @@ func TestScanDir(t *testing.T) { tags[t] = true } - wantData, err := ioutil.ReadFile(filepath.Join("testdata", dir.Name(), "want.txt")) + wantData, err := os.ReadFile(filepath.Join("testdata", dir.Name(), "want.txt")) if err != nil { t.Fatalf("error reading want: %v", err) } diff --git a/libgo/go/cmd/go/internal/imports/tags.go b/libgo/go/cmd/go/internal/imports/tags.go index 14b4e21..01b448b 100644 --- a/libgo/go/cmd/go/internal/imports/tags.go +++ b/libgo/go/cmd/go/internal/imports/tags.go @@ -4,17 +4,23 @@ package imports -import "cmd/go/internal/cfg" +import ( + "cmd/go/internal/cfg" + "sync" +) -var tags map[string]bool +var ( + tags map[string]bool + tagsOnce sync.Once +) // Tags returns a set of build tags that are true for the target platform. // It includes GOOS, GOARCH, the compiler, possibly "cgo", // release tags like "go1.13", and user-specified build tags. func Tags() map[string]bool { - if tags == nil { + tagsOnce.Do(func() { tags = loadTags() - } + }) return tags } @@ -36,14 +42,17 @@ func loadTags() map[string]bool { return tags } -var anyTags map[string]bool +var ( + anyTags map[string]bool + anyTagsOnce sync.Once +) // AnyTags returns a special set of build tags that satisfy nearly all // build tag expressions. Only "ignore" and malformed build tag requirements // are considered false. func AnyTags() map[string]bool { - if anyTags == nil { + anyTagsOnce.Do(func() { anyTags = map[string]bool{"*": true} - } + }) return anyTags } diff --git a/libgo/go/cmd/go/internal/imports/testdata/android/.h.go b/libgo/go/cmd/go/internal/imports/testdata/android/.h.go new file mode 100644 index 0000000..53c529e --- /dev/null +++ b/libgo/go/cmd/go/internal/imports/testdata/android/.h.go @@ -0,0 +1,3 @@ +package android + +import _ "h" diff --git a/libgo/go/cmd/go/internal/imports/testdata/illumos/.h.go b/libgo/go/cmd/go/internal/imports/testdata/illumos/.h.go new file mode 100644 index 0000000..53c529e --- /dev/null +++ b/libgo/go/cmd/go/internal/imports/testdata/illumos/.h.go @@ -0,0 +1,3 @@ +package android + +import _ "h" diff --git a/libgo/go/cmd/go/internal/list/list.go b/libgo/go/cmd/go/internal/list/list.go index 6ca1561..ce6f579 100644 --- a/libgo/go/cmd/go/internal/list/list.go +++ b/libgo/go/cmd/go/internal/list/list.go @@ -8,7 +8,9 @@ package list import ( "bufio" "bytes" + "context" "encoding/json" + "fmt" "io" "os" "sort" @@ -19,6 +21,7 @@ import ( "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/load" + "cmd/go/internal/modinfo" "cmd/go/internal/modload" "cmd/go/internal/str" "cmd/go/internal/work" @@ -63,26 +66,28 @@ to -f '{{.ImportPath}}'. The struct being passed to the template is: BinaryOnly bool // binary-only package (no longer supported) ForTest string // package is only for use in named test Export string // file containing export data (when using -export) + BuildID string // build ID of the compiled package (when using -export) Module *Module // info about package's containing module, if any (can be nil) Match []string // command-line patterns matching this package DepOnly bool // package is only a dependency, not explicitly listed // Source files - GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) - CgoFiles []string // .go source files that import "C" - CompiledGoFiles []string // .go files presented to compiler (when using -compiled) - IgnoredGoFiles []string // .go source files ignored due to build constraints - CFiles []string // .c source files - CXXFiles []string // .cc, .cxx and .cpp source files - MFiles []string // .m source files - HFiles []string // .h, .hh, .hpp and .hxx source files - FFiles []string // .f, .F, .for and .f90 Fortran source files - SFiles []string // .s source files - SwigFiles []string // .swig files - SwigCXXFiles []string // .swigcxx files - SysoFiles []string // .syso object files to add to archive - TestGoFiles []string // _test.go files in package - XTestGoFiles []string // _test.go files outside package + GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string // .go source files that import "C" + CompiledGoFiles []string // .go files presented to compiler (when using -compiled) + IgnoredGoFiles []string // .go source files ignored due to build constraints + IgnoredOtherFiles []string // non-.go source files ignored due to build constraints + CFiles []string // .c source files + CXXFiles []string // .cc, .cxx and .cpp source files + MFiles []string // .m source files + HFiles []string // .h, .hh, .hpp and .hxx source files + FFiles []string // .f, .F, .for and .f90 Fortran source files + SFiles []string // .s source files + SwigFiles []string // .swig files + SwigCXXFiles []string // .swigcxx files + SysoFiles []string // .syso object files to add to archive + TestGoFiles []string // _test.go files in package + XTestGoFiles []string // _test.go files outside package // Cgo directives CgoCFLAGS []string // cgo: flags for C compiler @@ -213,6 +218,7 @@ applied to a Go struct, but now a Module struct: Dir string // directory holding files for this module, if any GoMod string // path to go.mod file used when loading this module, if any GoVersion string // go version used in module + Retracted string // retraction information, if any (with -retracted or -u) Error *ModuleError // error loading module } @@ -244,14 +250,16 @@ the replaced source code.) The -u flag adds information about available upgrades. When the latest version of a given module is newer than the current one, list -u sets the Module's Update field -to information about the newer module. +to information about the newer module. list -u will also set +the module's Retracted field if the current version is retracted. The Module's String method indicates an available upgrade by formatting the newer version in brackets after the current version. +If a version is retracted, the string "(retracted)" will follow it. For example, 'go list -m -u all' might print: my/main/module golang.org/x/text v0.3.0 [v0.4.0] => /tmp/text - rsc.io/pdf v0.1.1 [v0.1.2] + rsc.io/pdf v0.1.1 (retracted) [v0.1.2] (For tools, 'go list -m -u -json all' may be more convenient to parse.) @@ -261,6 +269,14 @@ to semantic versioning, earliest to latest. The flag also changes the default output format to display the module path followed by the space-separated version list. +The -retracted flag causes list to report information about retracted +module versions. When -retracted is used with -f or -json, the Retracted +field will be set to a string explaining why the version was retracted. +The string is taken from comments on the retract directive in the +module's go.mod file. When -retracted is used with -versions, retracted +versions are listed together with unretracted versions. The -retracted +flag may be used with or without -m. + The arguments to list -m are interpreted as a list of modules, not packages. The main module is the module containing the current directory. The active modules are the main module and its dependencies. @@ -294,23 +310,24 @@ func init() { } var ( - listCompiled = CmdList.Flag.Bool("compiled", false, "") - listDeps = CmdList.Flag.Bool("deps", false, "") - listE = CmdList.Flag.Bool("e", false, "") - listExport = CmdList.Flag.Bool("export", false, "") - listFmt = CmdList.Flag.String("f", "", "") - listFind = CmdList.Flag.Bool("find", false, "") - listJson = CmdList.Flag.Bool("json", false, "") - listM = CmdList.Flag.Bool("m", false, "") - listU = CmdList.Flag.Bool("u", false, "") - listTest = CmdList.Flag.Bool("test", false, "") - listVersions = CmdList.Flag.Bool("versions", false, "") + listCompiled = CmdList.Flag.Bool("compiled", false, "") + listDeps = CmdList.Flag.Bool("deps", false, "") + listE = CmdList.Flag.Bool("e", false, "") + listExport = CmdList.Flag.Bool("export", false, "") + listFmt = CmdList.Flag.String("f", "", "") + listFind = CmdList.Flag.Bool("find", false, "") + listJson = CmdList.Flag.Bool("json", false, "") + listM = CmdList.Flag.Bool("m", false, "") + listRetracted = CmdList.Flag.Bool("retracted", false, "") + listTest = CmdList.Flag.Bool("test", false, "") + listU = CmdList.Flag.Bool("u", false, "") + listVersions = CmdList.Flag.Bool("versions", false, "") ) var nl = []byte{'\n'} -func runList(cmd *base.Command, args []string) { - modload.LoadTests = *listTest +func runList(ctx context.Context, cmd *base.Command, args []string) { + load.ModResolveTests = *listTest work.BuildInit() out := newTrackingWriter(os.Stdout) defer out.w.Flush() @@ -348,7 +365,7 @@ func runList(cmd *base.Command, args []string) { fm := template.FuncMap{ "join": strings.Join, "context": context, - "module": modload.ModuleInfo, + "module": func(path string) *modinfo.ModulePublic { return modload.ModuleInfo(ctx, path) }, } tmpl, err := template.New("main").Funcs(fm).Parse(*listFmt) if err != nil { @@ -365,6 +382,16 @@ func runList(cmd *base.Command, args []string) { } } + modload.Init() + if *listRetracted { + if cfg.BuildMod == "vendor" { + base.Fatalf("go list -retracted cannot be used when vendoring is enabled") + } + if !modload.Enabled() { + base.Fatalf("go list -retracted can only be used in module-aware mode") + } + } + if *listM { // Module mode. if *listCompiled { @@ -388,7 +415,7 @@ func runList(cmd *base.Command, args []string) { base.Fatalf("go list -m: not using modules") } - modload.InitMod() // Parses go.mod and sets cfg.BuildMod. + modload.LoadModFile(ctx) // Parses go.mod and sets cfg.BuildMod. if cfg.BuildMod == "vendor" { const actionDisabledFormat = "go list -m: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)" @@ -412,9 +439,7 @@ func runList(cmd *base.Command, args []string) { } } - modload.LoadBuildList() - - mods := modload.ListModules(args, *listU, *listVersions) + mods := modload.ListModules(ctx, args, *listU, *listVersions, *listRetracted) if !*listE { for _, m := range mods { if m.Error != nil { @@ -446,11 +471,18 @@ func runList(cmd *base.Command, args []string) { } load.IgnoreImports = *listFind - var pkgs []*load.Package - if *listE { - pkgs = load.PackagesAndErrors(args) - } else { - pkgs = load.Packages(args) + pkgs := load.PackagesAndErrors(ctx, args) + if !*listE { + w := 0 + for _, pkg := range pkgs { + if pkg.Error != nil { + base.Errorf("%v", pkg.Error) + continue + } + pkgs[w] = pkg + w++ + } + pkgs = pkgs[:w] base.ExitIfErrors() } @@ -476,9 +508,9 @@ func runList(cmd *base.Command, args []string) { var pmain, ptest, pxtest *load.Package var err error if *listE { - pmain, ptest, pxtest = load.TestPackagesAndErrors(p, nil) + pmain, ptest, pxtest = load.TestPackagesAndErrors(ctx, p, nil) } else { - pmain, ptest, pxtest, err = load.TestPackagesFor(p, nil) + pmain, ptest, pxtest, err = load.TestPackagesFor(ctx, p, nil) if err != nil { base.Errorf("can't load test package: %s", err) } @@ -520,7 +552,7 @@ func runList(cmd *base.Command, args []string) { // Note that -deps is applied after -test, // so that you only get descriptions of tests for the things named // explicitly on the command line, not for all dependencies. - pkgs = load.PackageList(pkgs) + pkgs = loadPackageList(pkgs) } // Do we need to run a build to gather information? @@ -538,13 +570,15 @@ func runList(cmd *base.Command, args []string) { a.Deps = append(a.Deps, b.AutoAction(work.ModeInstall, work.ModeInstall, p)) } } - b.Do(a) + b.Do(ctx, a) } for _, p := range pkgs { // Show vendor-expanded paths in listing p.TestImports = p.Resolve(p.TestImports) p.XTestImports = p.Resolve(p.XTestImports) + p.TestEmbedFiles = p.ResolveEmbed(p.TestEmbedPatterns) + p.XTestEmbedFiles = p.ResolveEmbed(p.XTestEmbedPatterns) p.DepOnly = !cmdline[p] if *listCompiled { @@ -555,7 +589,7 @@ func runList(cmd *base.Command, args []string) { if *listTest { all := pkgs if !*listDeps { - all = load.PackageList(pkgs) + all = loadPackageList(pkgs) } // Update import paths to distinguish the real package p // from p recompiled for q.test. @@ -605,6 +639,55 @@ func runList(cmd *base.Command, args []string) { } } + // TODO(golang.org/issue/40676): This mechanism could be extended to support + // -u without -m. + if *listRetracted { + // Load retractions for modules that provide packages that will be printed. + // TODO(golang.org/issue/40775): Packages from the same module refer to + // distinct ModulePublic instance. It would be nice if they could all point + // to the same instance. This would require additional global state in + // modload.loaded, so that should be refactored first. For now, we update + // all instances. + modToArg := make(map[*modinfo.ModulePublic]string) + argToMods := make(map[string][]*modinfo.ModulePublic) + var args []string + addModule := func(mod *modinfo.ModulePublic) { + if mod.Version == "" { + return + } + arg := fmt.Sprintf("%s@%s", mod.Path, mod.Version) + if argToMods[arg] == nil { + args = append(args, arg) + } + argToMods[arg] = append(argToMods[arg], mod) + modToArg[mod] = arg + } + for _, p := range pkgs { + if p.Module == nil { + continue + } + addModule(p.Module) + if p.Module.Replace != nil { + addModule(p.Module.Replace) + } + } + + if len(args) > 0 { + listU := false + listVersions := false + rmods := modload.ListModules(ctx, args, listU, listVersions, *listRetracted) + for i, arg := range args { + rmod := rmods[i] + for _, mod := range argToMods[arg] { + mod.Retracted = rmod.Retracted + if rmod.Error != nil && mod.Error == nil { + mod.Error = rmod.Error + } + } + } + } + } + // Record non-identity import mappings in p.ImportMap. for _, p := range pkgs { for i, srcPath := range p.Internal.RawImports { @@ -623,6 +706,23 @@ func runList(cmd *base.Command, args []string) { } } +// loadPackageList is like load.PackageList, but prints error messages and exits +// with nonzero status if listE is not set and any package in the expanded list +// has errors. +func loadPackageList(roots []*load.Package) []*load.Package { + pkgs := load.PackageList(roots) + + if !*listE { + for _, pkg := range pkgs { + if pkg.Error != nil { + base.Errorf("%v", pkg.Error) + } + } + } + + return pkgs +} + // TrackingWriter tracks the last byte written on every write so // we can avoid printing a newline if one was already written or // if there is no output at all. diff --git a/libgo/go/cmd/go/internal/load/pkg.go b/libgo/go/cmd/go/internal/load/pkg.go index 28220ce..b367972 100644 --- a/libgo/go/cmd/go/internal/load/pkg.go +++ b/libgo/go/cmd/go/internal/load/pkg.go @@ -7,14 +7,16 @@ package load import ( "bytes" + "context" "encoding/json" "errors" "fmt" "go/build" "go/scanner" "go/token" - "io/ioutil" + "io/fs" "os" + "path" pathpkg "path" "path/filepath" "runtime" @@ -26,25 +28,14 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/modinfo" + "cmd/go/internal/modload" "cmd/go/internal/par" "cmd/go/internal/search" "cmd/go/internal/str" -) - -var ( - // module initialization hook; never nil, no-op if module use is disabled - ModInit func() - - // module hooks; nil if module use is disabled - ModBinDir func() string // return effective bin directory - ModLookup func(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) // lookup effective meaning of import - ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct - ModImportPaths func(args []string) []*search.Match // expand import paths - ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary - ModInfoProg func(info string, isgccgo bool) []byte // wrap module info in .go code for binary - ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files - ModDirImportPath func(string) string // return effective import path for directory + "cmd/go/internal/trace" + "cmd/internal/sys" ) var IgnoreImports bool // control whether we ignore imports in packages @@ -70,6 +61,7 @@ type PackagePublic struct { ConflictDir string `json:",omitempty"` // Dir is hidden by this other directory ForTest string `json:",omitempty"` // package is only for use in named test Export string `json:",omitempty"` // file containing export data (set by go list -export) + BuildID string `json:",omitempty"` // build ID of the compiled package (set by go list -export) Module *modinfo.ModulePublic `json:",omitempty"` // info about package's module, if any Match []string `json:",omitempty"` // command-line patterns matching this package Goroot bool `json:",omitempty"` // is this package found in the Go root? @@ -87,19 +79,24 @@ type PackagePublic struct { // Source files // If you add to this list you MUST add to p.AllFiles (below) too. // Otherwise file name security lists will not apply to any new additions. - GoFiles []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) - CgoFiles []string `json:",omitempty"` // .go source files that import "C" - CompiledGoFiles []string `json:",omitempty"` // .go output from running cgo on CgoFiles - IgnoredGoFiles []string `json:",omitempty"` // .go source files ignored due to build constraints - CFiles []string `json:",omitempty"` // .c source files - CXXFiles []string `json:",omitempty"` // .cc, .cpp and .cxx source files - MFiles []string `json:",omitempty"` // .m source files - HFiles []string `json:",omitempty"` // .h, .hh, .hpp and .hxx source files - FFiles []string `json:",omitempty"` // .f, .F, .for and .f90 Fortran source files - SFiles []string `json:",omitempty"` // .s source files - SwigFiles []string `json:",omitempty"` // .swig files - SwigCXXFiles []string `json:",omitempty"` // .swigcxx files - SysoFiles []string `json:",omitempty"` // .syso system object files added to package + GoFiles []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string `json:",omitempty"` // .go source files that import "C" + CompiledGoFiles []string `json:",omitempty"` // .go output from running cgo on CgoFiles + IgnoredGoFiles []string `json:",omitempty"` // .go source files ignored due to build constraints + IgnoredOtherFiles []string `json:",omitempty"` // non-.go source files ignored due to build constraints + CFiles []string `json:",omitempty"` // .c source files + CXXFiles []string `json:",omitempty"` // .cc, .cpp and .cxx source files + MFiles []string `json:",omitempty"` // .m source files + HFiles []string `json:",omitempty"` // .h, .hh, .hpp and .hxx source files + FFiles []string `json:",omitempty"` // .f, .F, .for and .f90 Fortran source files + SFiles []string `json:",omitempty"` // .s source files + SwigFiles []string `json:",omitempty"` // .swig files + SwigCXXFiles []string `json:",omitempty"` // .swigcxx files + SysoFiles []string `json:",omitempty"` // .syso system object files added to package + + // Embedded files + EmbedPatterns []string `json:",omitempty"` // //go:embed patterns + EmbedFiles []string `json:",omitempty"` // files and directories matched by EmbedPatterns // Cgo directives CgoCFLAGS []string `json:",omitempty"` // cgo: flags for C compiler @@ -122,10 +119,14 @@ type PackagePublic struct { // Test information // If you add to this list you MUST add to p.AllFiles (below) too. // Otherwise file name security lists will not apply to any new additions. - TestGoFiles []string `json:",omitempty"` // _test.go files in package - TestImports []string `json:",omitempty"` // imports from TestGoFiles - XTestGoFiles []string `json:",omitempty"` // _test.go files outside package - XTestImports []string `json:",omitempty"` // imports from XTestGoFiles + TestGoFiles []string `json:",omitempty"` // _test.go files in package + TestImports []string `json:",omitempty"` // imports from TestGoFiles + TestEmbedPatterns []string `json:",omitempty"` // //go:embed patterns + TestEmbedFiles []string `json:",omitempty"` // //files matched by EmbedPatterns + XTestGoFiles []string `json:",omitempty"` // _test.go files outside package + XTestImports []string `json:",omitempty"` // imports from XTestGoFiles + XTestEmbedPatterns []string `json:",omitempty"` // //go:embed patterns + XTestEmbedFiles []string `json:",omitempty"` // //files matched by EmbedPatterns } // AllFiles returns the names of all the files considered for the package. @@ -134,11 +135,12 @@ type PackagePublic struct { // The go/build package filtered others out (like foo_wrongGOARCH.s) // and that's OK. func (p *Package) AllFiles() []string { - return str.StringList( + files := str.StringList( p.GoFiles, p.CgoFiles, // no p.CompiledGoFiles, because they are from GoFiles or generated by us p.IgnoredGoFiles, + p.IgnoredOtherFiles, p.CFiles, p.CXXFiles, p.MFiles, @@ -151,6 +153,27 @@ func (p *Package) AllFiles() []string { p.TestGoFiles, p.XTestGoFiles, ) + + // EmbedFiles may overlap with the other files. + // Dedup, but delay building the map as long as possible. + // Only files in the current directory (no slash in name) + // need to be checked against the files variable above. + var have map[string]bool + for _, file := range p.EmbedFiles { + if !strings.Contains(file, "/") { + if have == nil { + have = make(map[string]bool) + for _, file := range files { + have[file] = true + } + } + if have[file] { + continue + } + } + files = append(files, file) + } + return files } // Desc returns the package "description", for use in b.showOutput. @@ -180,6 +203,7 @@ type PackageInternal struct { GobinSubdir bool // install target would be subdir of GOBIN BuildInfo string // add this info to package main TestmainGo *[]byte // content for _testmain.go + Embed map[string][]string // //go:embed comment mapping Asmflags []string // -asmflags for this package Gcflags []string // -gcflags for this package @@ -197,7 +221,7 @@ type NoGoError struct { } func (e *NoGoError) Error() string { - if len(e.Package.constraintIgnoredGoFiles()) > 0 { + if len(e.Package.IgnoredGoFiles) > 0 { // Go files exist, but they were ignored due to build constraints. return "build constraints exclude all Go files in " + e.Package.Dir } @@ -260,8 +284,8 @@ func (p *Package) setLoadPackageDataError(err error, path string, stk *ImportSta // package's source files themselves (scanner errors). // // TODO(matloob): Perhaps make each of those the errors in the first group - // (including modload.ImportMissingError, and the corresponding - // "cannot find package %q in any of" GOPATH-mode error + // (including modload.ImportMissingError, ImportMissingSumError, and the + // corresponding "cannot find package %q in any of" GOPATH-mode error // produced in build.(*Context).Import; modload.AmbiguousImportError, // and modload.PackageNotInModuleError; and the malformed module path errors // produced in golang.org/x/mod/module.CheckMod) implement an interface @@ -342,6 +366,7 @@ func (p *Package) copyBuild(pp *build.Package) { p.GoFiles = pp.GoFiles p.CgoFiles = pp.CgoFiles p.IgnoredGoFiles = pp.IgnoredGoFiles + p.IgnoredOtherFiles = pp.IgnoredOtherFiles p.CFiles = pp.CFiles p.CXXFiles = pp.CXXFiles p.MFiles = pp.MFiles @@ -371,6 +396,9 @@ func (p *Package) copyBuild(pp *build.Package) { p.TestImports = nil p.XTestImports = nil } + p.EmbedPatterns = pp.EmbedPatterns + p.TestEmbedPatterns = pp.TestEmbedPatterns + p.XTestEmbedPatterns = pp.XTestEmbedPatterns } // A PackageError describes an error loading information about a package. @@ -432,13 +460,17 @@ type ImportPathError interface { ImportPath() string } +var ( + _ ImportPathError = (*importError)(nil) + _ ImportPathError = (*modload.ImportMissingError)(nil) + _ ImportPathError = (*modload.ImportMissingSumError)(nil) +) + type importError struct { importPath string err error // created with fmt.Errorf } -var _ ImportPathError = (*importError)(nil) - func ImportErrorf(path, format string, args ...interface{}) ImportPathError { err := &importError{importPath: path, err: fmt.Errorf(format, args...)} if errStr := err.Error(); !strings.Contains(errStr, path) { @@ -551,7 +583,7 @@ func ReloadPackageNoFlags(arg string, stk *ImportStack) *Package { }) packageDataCache.Delete(p.ImportPath) } - return LoadImport(arg, base.Cwd, nil, stk, nil, 0) + return LoadImport(context.TODO(), arg, base.Cwd, nil, stk, nil, 0) } // dirToImportPath returns the pseudo-import path we use for a package @@ -603,11 +635,11 @@ const ( // LoadImport does not set tool flags and should only be used by // this package, as part of a bigger load operation, and by GOPATH-based "go get". // TODO(rsc): When GOPATH-based "go get" is removed, unexport this function. -func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { - return loadImport(nil, path, srcDir, parent, stk, importPos, mode) +func LoadImport(ctx context.Context, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { + return loadImport(ctx, nil, path, srcDir, parent, stk, importPos, mode) } -func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { +func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { if path == "" { panic("LoadImport called with empty package path") } @@ -655,7 +687,7 @@ func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportS // Load package. // loadPackageData may return bp != nil even if an error occurs, // in order to return partial information. - p.load(path, stk, importPos, bp, err) + p.load(ctx, path, stk, importPos, bp, err) if !cfg.ModulesEnabled && path != cleanImport(path) { p.Error = &PackageError{ @@ -751,7 +783,7 @@ func loadPackageData(path, parentPath, parentDir, parentRoot string, parentIsStd // For vendored imports, it is the expanded form. // // Note that when modules are enabled, local import paths are normally - // canonicalized by modload.ImportPaths before now. However, if there's an + // canonicalized by modload.LoadPackages before now. However, if there's an // error resolving a local path, it will be returned untransformed // so that 'go list -e' reports something useful. importKey := importSpec{ @@ -768,7 +800,7 @@ func loadPackageData(path, parentPath, parentDir, parentRoot string, parentIsStd r.dir = filepath.Join(parentDir, path) r.path = dirToImportPath(r.dir) } else if cfg.ModulesEnabled { - r.dir, r.path, r.err = ModLookup(parentPath, parentIsStd, path) + r.dir, r.path, r.err = modload.Lookup(parentPath, parentIsStd, path) } else if mode&ResolveImport != 0 { // We do our own path resolution, because we want to // find out the key to use in packageCache without the @@ -799,7 +831,7 @@ func loadPackageData(path, parentPath, parentDir, parentRoot string, parentIsStd } data.p, data.err = cfg.BuildContext.ImportDir(r.dir, buildMode) if data.p.Root == "" && cfg.ModulesEnabled { - if info := ModPackageModuleInfo(path); info != nil { + if info := modload.PackageModuleInfo(path); info != nil { data.p.Root = info.Dir } } @@ -825,7 +857,7 @@ func loadPackageData(path, parentPath, parentDir, parentRoot string, parentIsStd if cfg.GOBIN != "" { data.p.BinDir = cfg.GOBIN } else if cfg.ModulesEnabled { - data.p.BinDir = ModBinDir() + data.p.BinDir = modload.BinDir() } } @@ -893,8 +925,8 @@ var preloadWorkerCount = runtime.GOMAXPROCS(0) // to ensure preload goroutines are no longer active. This is necessary // because of global mutable state that cannot safely be read and written // concurrently. In particular, packageDataCache may be cleared by "go get" -// in GOPATH mode, and modload.loaded (accessed via ModLookup) may be -// modified by modload.ImportPaths (ModImportPaths). +// in GOPATH mode, and modload.loaded (accessed via modload.Lookup) may be +// modified by modload.LoadPackages. type preload struct { cancel chan struct{} sema chan struct{} @@ -961,6 +993,12 @@ func (pre *preload) preloadImports(imports []string, parent *build.Package) { // loadPackageData have completed. The preloader will not make any new calls // to loadPackageData. func (pre *preload) flush() { + // flush is usually deferred. + // Don't hang program waiting for workers on panic. + if v := recover(); v != nil { + panic(v) + } + close(pre.cancel) for i := 0; i < preloadWorkerCount; i++ { pre.sema <- struct{}{} @@ -980,7 +1018,7 @@ var isDirCache par.Cache func isDir(path string) bool { return isDirCache.Do(path, func() interface{} { - fi, err := os.Stat(path) + fi, err := fsys.Stat(path) return err == nil && fi.IsDir() }).(bool) } @@ -1004,7 +1042,7 @@ func ResolveImportPath(parent *Package, path string) (found string) { func resolveImportPath(path, parentPath, parentDir, parentRoot string, parentIsStd bool) (found string) { if cfg.ModulesEnabled { - if _, p, e := ModLookup(parentPath, parentIsStd, path); e == nil { + if _, p, e := modload.Lookup(parentPath, parentIsStd, path); e == nil { return p } return path @@ -1108,7 +1146,7 @@ var ( // goModPath returns the module path in the go.mod in dir, if any. func goModPath(dir string) (path string) { return goModPathCache.Do(dir, func() interface{} { - data, err := ioutil.ReadFile(filepath.Join(dir, "go.mod")) + data, err := os.ReadFile(filepath.Join(dir, "go.mod")) if err != nil { return "" } @@ -1257,9 +1295,9 @@ HaveGoMod: // Otherwise it is not possible to vendor just a/b/c and still import the // non-vendored a/b. See golang.org/issue/13832. func hasGoFiles(dir string) bool { - fis, _ := ioutil.ReadDir(dir) - for _, fi := range fis { - if !fi.IsDir() && strings.HasSuffix(fi.Name(), ".go") { + files, _ := os.ReadDir(dir) + for _, f := range files { + if !f.IsDir() && strings.HasSuffix(f.Name(), ".go") { return true } } @@ -1373,7 +1411,7 @@ func disallowInternal(srcDir string, importer *Package, importerPath string, p * // directory containing them. // If the directory is outside the main module, this will resolve to ".", // which is not a prefix of any valid module. - importerPath = ModDirImportPath(importer.Dir) + importerPath = modload.DirImportPath(importer.Dir) } parentOfInternal := p.ImportPath[:i] if str.HasPathPrefix(importerPath, parentOfInternal) { @@ -1595,7 +1633,7 @@ func (p *Package) DefaultExecName() string { // load populates p using information from bp, err, which should // be the result of calling build.Context.Import. // stk contains the import stack, not including path itself. -func (p *Package) load(path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { +func (p *Package) load(ctx context.Context, path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { p.copyBuild(bp) // The localPrefix is the path we interpret ./ imports relative to. @@ -1631,6 +1669,11 @@ func (p *Package) load(path string, stk *ImportStack, importPos []token.Position p.setLoadPackageDataError(err, path, stk, importPos) } + p.EmbedFiles, p.Internal.Embed, err = p.resolveEmbed(p.EmbedPatterns) + if err != nil { + setError(err) + } + useBindir := p.Name == "main" if !p.Standard { switch cfg.BuildBuildmode { @@ -1656,7 +1699,7 @@ func (p *Package) load(path string, stk *ImportStack, importPos []token.Position elem = full } if p.Internal.Build.BinDir == "" && cfg.ModulesEnabled { - p.Internal.Build.BinDir = ModBinDir() + p.Internal.Build.BinDir = modload.BinDir() } if p.Internal.Build.BinDir != "" { // Install to GOBIN or bin of GOPATH entry. @@ -1690,7 +1733,7 @@ func (p *Package) load(path string, stk *ImportStack, importPos []token.Position // not work for any package that lacks a Target — such as a non-main // package in module mode. We should probably fix that. shlibnamefile := p.Target[:len(p.Target)-2] + ".shlibname" - shlib, err := ioutil.ReadFile(shlibnamefile) + shlib, err := os.ReadFile(shlibnamefile) if err != nil && !os.IsNotExist(err) { base.Fatalf("reading shlibname: %v", err) } @@ -1804,7 +1847,7 @@ func (p *Package) load(path string, stk *ImportStack, importPos []token.Position if path == "C" { continue } - p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) + p1 := LoadImport(ctx, path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) path = p1.ImportPath importPaths[i] = path @@ -1865,13 +1908,169 @@ func (p *Package) load(path string, stk *ImportStack, importPos []token.Position if p.Internal.CmdlineFiles { mainPath = "command-line-arguments" } - p.Module = ModPackageModuleInfo(mainPath) + p.Module = modload.PackageModuleInfo(mainPath) if p.Name == "main" && len(p.DepsErrors) == 0 { - p.Internal.BuildInfo = ModPackageBuildInfo(mainPath, p.Deps) + p.Internal.BuildInfo = modload.PackageBuildInfo(mainPath, p.Deps) } } } +// ResolveEmbed resolves //go:embed patterns and returns only the file list. +// For use by go list to compute p.TestEmbedFiles and p.XTestEmbedFiles. +func (p *Package) ResolveEmbed(patterns []string) []string { + files, _, _ := p.resolveEmbed(patterns) + return files +} + +// resolveEmbed resolves //go:embed patterns to precise file lists. +// It sets files to the list of unique files matched (for go list), +// and it sets pmap to the more precise mapping from +// patterns to files. +// TODO(rsc): All these messages need position information for better error reports. +func (p *Package) resolveEmbed(patterns []string) (files []string, pmap map[string][]string, err error) { + pmap = make(map[string][]string) + have := make(map[string]int) + dirOK := make(map[string]bool) + pid := 0 // pattern ID, to allow reuse of have map + for _, pattern := range patterns { + pid++ + + // Check pattern is valid for //go:embed. + if _, err := path.Match(pattern, ""); err != nil || !validEmbedPattern(pattern) { + return nil, nil, fmt.Errorf("pattern %s: invalid pattern syntax", pattern) + } + + // Glob to find matches. + match, err := fsys.Glob(p.Dir + string(filepath.Separator) + filepath.FromSlash(pattern)) + if err != nil { + return nil, nil, fmt.Errorf("pattern %s: %v", pattern, err) + } + + // Filter list of matches down to the ones that will still exist when + // the directory is packaged up as a module. (If p.Dir is in the module cache, + // only those files exist already, but if p.Dir is in the current module, + // then there may be other things lying around, like symbolic links or .git directories.) + var list []string + for _, file := range match { + rel := filepath.ToSlash(file[len(p.Dir)+1:]) // file, relative to p.Dir + + what := "file" + info, err := fsys.Lstat(file) + if err != nil { + return nil, nil, err + } + if info.IsDir() { + what = "directory" + } + + // Check that directories along path do not begin a new module + // (do not contain a go.mod). + for dir := file; len(dir) > len(p.Dir)+1 && !dirOK[dir]; dir = filepath.Dir(dir) { + if _, err := fsys.Stat(filepath.Join(dir, "go.mod")); err == nil { + return nil, nil, fmt.Errorf("pattern %s: cannot embed %s %s: in different module", pattern, what, rel) + } + if dir != file { + if info, err := fsys.Lstat(dir); err == nil && !info.IsDir() { + return nil, nil, fmt.Errorf("pattern %s: cannot embed %s %s: in non-directory %s", pattern, what, rel, dir[len(p.Dir)+1:]) + } + } + dirOK[dir] = true + if elem := filepath.Base(dir); isBadEmbedName(elem) { + if dir == file { + return nil, nil, fmt.Errorf("pattern %s: cannot embed %s %s: invalid name %s", pattern, what, rel, elem) + } else { + return nil, nil, fmt.Errorf("pattern %s: cannot embed %s %s: in invalid directory %s", pattern, what, rel, elem) + } + } + } + + switch { + default: + return nil, nil, fmt.Errorf("pattern %s: cannot embed irregular file %s", pattern, rel) + + case info.Mode().IsRegular(): + if have[rel] != pid { + have[rel] = pid + list = append(list, rel) + } + + case info.IsDir(): + // Gather all files in the named directory, stopping at module boundaries + // and ignoring files that wouldn't be packaged into a module. + count := 0 + err := fsys.Walk(file, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel := filepath.ToSlash(path[len(p.Dir)+1:]) + name := info.Name() + if path != file && (isBadEmbedName(name) || name[0] == '.' || name[0] == '_') { + // Ignore bad names, assuming they won't go into modules. + // Also avoid hidden files that user may not know about. + // See golang.org/issue/42328. + if info.IsDir() { + return fs.SkipDir + } + return nil + } + if info.IsDir() { + if _, err := fsys.Stat(filepath.Join(path, "go.mod")); err == nil { + return filepath.SkipDir + } + return nil + } + if !info.Mode().IsRegular() { + return nil + } + count++ + if have[rel] != pid { + have[rel] = pid + list = append(list, rel) + } + return nil + }) + if err != nil { + return nil, nil, err + } + if count == 0 { + return nil, nil, fmt.Errorf("pattern %s: cannot embed directory %s: contains no embeddable files", pattern, rel) + } + } + } + + if len(list) == 0 { + return nil, nil, fmt.Errorf("pattern %s: no matching files found", pattern) + } + sort.Strings(list) + pmap[pattern] = list + } + + for file := range have { + files = append(files, file) + } + sort.Strings(files) + return files, pmap, nil +} + +func validEmbedPattern(pattern string) bool { + return pattern != "." && fs.ValidPath(pattern) +} + +// isBadEmbedName reports whether name is the base name of a file that +// can't or won't be included in modules and therefore shouldn't be treated +// as existing for embedding. +func isBadEmbedName(name string) bool { + switch name { + // Empty string should be impossible but make it bad. + case "": + return true + // Version control directories won't be present in module. + case ".bzr", ".hg", ".git", ".svn": + return true + } + return false +} + // collectDeps populates p.Deps and p.DepsErrors by iterating over // p.Internal.Imports. // @@ -1959,37 +2158,40 @@ func LinkerDeps(p *Package) []string { // externalLinkingForced reports whether external linking is being // forced even for programs that do not use cgo. func externalLinkingForced(p *Package) bool { + if !cfg.BuildContext.CgoEnabled { + return false + } + // Some targets must use external linking even inside GOROOT. switch cfg.BuildContext.GOOS { case "android": if cfg.BuildContext.GOARCH != "arm64" { return true } - case "darwin": - if cfg.BuildContext.GOARCH == "arm64" { - return true - } + case "ios": + return true } - if !cfg.BuildContext.CgoEnabled { - return false - } // Currently build modes c-shared, pie (on systems that do not // support PIE with internal linking mode (currently all // systems: issue #18968)), plugin, and -linkshared force // external linking mode, as of course does // -ldflags=-linkmode=external. External linking mode forces // an import of runtime/cgo. - pieCgo := cfg.BuildBuildmode == "pie" + // If there are multiple -linkmode options, the last one wins. + pieCgo := cfg.BuildBuildmode == "pie" && !sys.InternalLinkPIESupported(cfg.BuildContext.GOOS, cfg.BuildContext.GOARCH) linkmodeExternal := false if p != nil { ldflags := BuildLdflags.For(p) - for i, a := range ldflags { - if a == "-linkmode=external" { - linkmodeExternal = true - } - if a == "-linkmode" && i+1 < len(ldflags) && ldflags[i+1] == "external" { + for i := len(ldflags) - 1; i >= 0; i-- { + a := ldflags[i] + if a == "-linkmode=external" || + a == "-linkmode" && i+1 < len(ldflags) && ldflags[i+1] == "external" { linkmodeExternal = true + break + } else if a == "-linkmode=internal" || + a == "-linkmode" && i+1 < len(ldflags) && ldflags[i+1] == "internal" { + break } } } @@ -2024,22 +2226,7 @@ func (p *Package) InternalXGoFiles() []string { // using absolute paths. "Possibly relevant" means that files are not excluded // due to build tags, but files with names beginning with . or _ are still excluded. func (p *Package) InternalAllGoFiles() []string { - return p.mkAbs(str.StringList(p.constraintIgnoredGoFiles(), p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles)) -} - -// constraintIgnoredGoFiles returns the list of Go files ignored for reasons -// other than having a name beginning with '.' or '_'. -func (p *Package) constraintIgnoredGoFiles() []string { - if len(p.IgnoredGoFiles) == 0 { - return nil - } - files := make([]string, 0, len(p.IgnoredGoFiles)) - for _, f := range p.IgnoredGoFiles { - if f != "" && f[0] != '.' && f[0] != '_' { - files = append(files, f) - } - } - return files + return p.mkAbs(str.StringList(p.IgnoredGoFiles, p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles)) } // usesSwig reports whether the package needs to run SWIG. @@ -2077,7 +2264,7 @@ func PackageList(roots []*Package) []*Package { // TestPackageList returns the list of packages in the dag rooted at roots // as visited in a depth-first post-order traversal, including the test // imports of the roots. This ignores errors in test packages. -func TestPackageList(roots []*Package) []*Package { +func TestPackageList(ctx context.Context, roots []*Package) []*Package { seen := map[*Package]bool{} all := []*Package{} var walk func(*Package) @@ -2093,7 +2280,7 @@ func TestPackageList(roots []*Package) []*Package { } walkTest := func(root *Package, path string) { var stk ImportStack - p1 := LoadImport(path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport) + p1 := LoadImport(ctx, path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport) if p1.Error == nil { walk(p1) } @@ -2116,36 +2303,35 @@ func TestPackageList(roots []*Package) []*Package { // TODO(jayconrod): delete this function and set flags automatically // in LoadImport instead. func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { - p := LoadImport(path, srcDir, parent, stk, importPos, mode) + p := LoadImport(context.TODO(), path, srcDir, parent, stk, importPos, mode) setToolFlags(p) return p } -// Packages returns the packages named by the -// command line arguments 'args'. If a named package -// cannot be loaded at all (for example, if the directory does not exist), -// then packages prints an error and does not include that -// package in the results. However, if errors occur trying -// to load dependencies of a named package, the named -// package is still returned, with p.Incomplete = true -// and details in p.DepsErrors. -func Packages(args []string) []*Package { - var pkgs []*Package - for _, pkg := range PackagesAndErrors(args) { - if pkg.Error != nil { - base.Errorf("%v", pkg.Error) - continue - } - pkgs = append(pkgs, pkg) - } - return pkgs -} +// ModResolveTests indicates whether calls to the module loader should also +// resolve test dependencies of the requested packages. +// +// If ModResolveTests is true, then the module loader needs to resolve test +// dependencies at the same time as packages; otherwise, the test dependencies +// of those packages could be missing, and resolving those missing dependencies +// could change the selected versions of modules that provide other packages. +// +// TODO(#40775): Change this from a global variable to an explicit function +// argument where needed. +var ModResolveTests bool + +// PackagesAndErrors returns the packages named by the command line arguments +// 'patterns'. If a named package cannot be loaded, PackagesAndErrors returns +// a *Package with the Error field describing the failure. If errors are found +// loading imported packages, the DepsErrors field is set. The Incomplete field +// may be set as well. +// +// To obtain a flat list of packages, use PackageList. +// To report errors loading packages, use ReportPackageErrors. +func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { + ctx, span := trace.StartSpan(ctx, "load.PackagesAndErrors") + defer span.Done() -// PackagesAndErrors is like 'packages' but returns a -// *Package for every argument, even the ones that -// cannot be loaded at all. -// The packages that fail to load will have p.Error != nil. -func PackagesAndErrors(patterns []string) []*Package { for _, p := range patterns { // Listing is only supported with all patterns referring to either: // - Files that are part of the same directory. @@ -2153,13 +2339,24 @@ func PackagesAndErrors(patterns []string) []*Package { if strings.HasSuffix(p, ".go") { // We need to test whether the path is an actual Go file and not a // package path or pattern ending in '.go' (see golang.org/issue/34653). - if fi, err := os.Stat(p); err == nil && !fi.IsDir() { - return []*Package{GoFilesPackage(patterns)} + if fi, err := fsys.Stat(p); err == nil && !fi.IsDir() { + return []*Package{GoFilesPackage(ctx, patterns)} } } } - matches := ImportPaths(patterns) + var matches []*search.Match + if modload.Init(); cfg.ModulesEnabled { + loadOpts := modload.PackageOpts{ + ResolveMissingImports: true, + LoadTests: ModResolveTests, + SilenceErrors: true, + } + matches, _ = modload.LoadPackages(ctx, loadOpts, patterns...) + } else { + matches = search.ImportPaths(patterns) + } + var ( pkgs []*Package stk ImportStack @@ -2175,7 +2372,7 @@ func PackagesAndErrors(patterns []string) []*Package { if pkg == "" { panic(fmt.Sprintf("ImportPaths returned empty package for pattern %s", m.Pattern())) } - p := loadImport(pre, pkg, base.Cwd, nil, &stk, nil, 0) + p := loadImport(ctx, pre, pkg, base.Cwd, nil, &stk, nil, 0) p.Match = append(p.Match, m.Pattern()) p.Internal.CmdlinePkg = true if m.IsLiteral() { @@ -2220,27 +2417,9 @@ func PackagesAndErrors(patterns []string) []*Package { return pkgs } -func setToolFlags(pkgs ...*Package) { - for _, p := range PackageList(pkgs) { - p.Internal.Asmflags = BuildAsmflags.For(p) - p.Internal.Gcflags = BuildGcflags.For(p) - p.Internal.Ldflags = BuildLdflags.For(p) - p.Internal.Gccgoflags = BuildGccgoflags.For(p) - } -} - -func ImportPaths(args []string) []*search.Match { - if ModInit(); cfg.ModulesEnabled { - return ModImportPaths(args) - } - return search.ImportPaths(args) -} - -// PackagesForBuild is like Packages but exits -// if any of the packages or their dependencies have errors -// (cannot be built). -func PackagesForBuild(args []string) []*Package { - pkgs := PackagesAndErrors(args) +// CheckPackageErrors prints errors encountered loading pkgs and their +// dependencies, then exits with a non-zero status if any errors were found. +func CheckPackageErrors(pkgs []*Package) { printed := map[*PackageError]bool{} for _, pkg := range pkgs { if pkg.Error != nil { @@ -2275,15 +2454,22 @@ func PackagesForBuild(args []string) []*Package { seen[pkg.ImportPath] = true } base.ExitIfErrors() +} - return pkgs +func setToolFlags(pkgs ...*Package) { + for _, p := range PackageList(pkgs) { + p.Internal.Asmflags = BuildAsmflags.For(p) + p.Internal.Gcflags = BuildGcflags.For(p) + p.Internal.Ldflags = BuildLdflags.For(p) + p.Internal.Gccgoflags = BuildGccgoflags.For(p) + } } // GoFilesPackage creates a package for building a collection of Go files // (typically named on the command line). The target is named p.a for // package p or named after the first Go file for package main. -func GoFilesPackage(gofiles []string) *Package { - ModInit() +func GoFilesPackage(ctx context.Context, gofiles []string) *Package { + modload.Init() for _, f := range gofiles { if !strings.HasSuffix(f, ".go") { @@ -2306,10 +2492,10 @@ func GoFilesPackage(gofiles []string) *Package { // to make it look like this is a standard package or // command directory. So that local imports resolve // consistently, the files must all be in the same directory. - var dirent []os.FileInfo + var dirent []fs.FileInfo var dir string for _, file := range gofiles { - fi, err := os.Stat(file) + fi, err := fsys.Stat(file) if err != nil { base.Fatalf("%s", err) } @@ -2327,10 +2513,10 @@ func GoFilesPackage(gofiles []string) *Package { } dirent = append(dirent, fi) } - ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil } + ctxt.ReadDir = func(string) ([]fs.FileInfo, error) { return dirent, nil } if cfg.ModulesEnabled { - ModImportFromFiles(gofiles) + modload.ImportFromFiles(ctx, gofiles) } var err error @@ -2346,7 +2532,7 @@ func GoFilesPackage(gofiles []string) *Package { pkg := new(Package) pkg.Internal.Local = true pkg.Internal.CmdlineFiles = true - pkg.load("command-line-arguments", &stk, nil, bp, err) + pkg.load(ctx, "command-line-arguments", &stk, nil, bp, err) pkg.Internal.LocalPrefix = dirToImportPath(dir) pkg.ImportPath = "command-line-arguments" pkg.Target = "" @@ -2358,7 +2544,7 @@ func GoFilesPackage(gofiles []string) *Package { if cfg.GOBIN != "" { pkg.Target = filepath.Join(cfg.GOBIN, exe) } else if cfg.ModulesEnabled { - pkg.Target = filepath.Join(ModBinDir(), exe) + pkg.Target = filepath.Join(modload.BinDir(), exe) } } diff --git a/libgo/go/cmd/go/internal/load/test.go b/libgo/go/cmd/go/internal/load/test.go index 6d251e8..d884361 100644 --- a/libgo/go/cmd/go/internal/load/test.go +++ b/libgo/go/cmd/go/internal/load/test.go @@ -6,7 +6,7 @@ package load import ( "bytes" - "cmd/go/internal/str" + "context" "errors" "fmt" "go/ast" @@ -20,6 +20,9 @@ import ( "strings" "unicode" "unicode/utf8" + + "cmd/go/internal/str" + "cmd/go/internal/trace" ) var TestMainDeps = []string{ @@ -42,8 +45,8 @@ type TestCover struct { // TestPackagesFor is like TestPackagesAndErrors but it returns // an error if the test packages or their dependencies have errors. // Only test packages without errors are returned. -func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Package, err error) { - pmain, ptest, pxtest = TestPackagesAndErrors(p, cover) +func TestPackagesFor(ctx context.Context, p *Package, cover *TestCover) (pmain, ptest, pxtest *Package, err error) { + pmain, ptest, pxtest = TestPackagesAndErrors(ctx, p, cover) for _, p1 := range []*Package{ptest, pxtest, pmain} { if p1 == nil { // pxtest may be nil @@ -89,7 +92,10 @@ func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Packag // // The caller is expected to have checked that len(p.TestGoFiles)+len(p.XTestGoFiles) > 0, // or else there's no point in any of this. -func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest *Package) { +func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (pmain, ptest, pxtest *Package) { + ctx, span := trace.StartSpan(ctx, "load.TestPackagesAndErrors") + defer span.Done() + pre := newPreload() defer pre.flush() allImports := append([]string{}, p.TestImports...) @@ -99,10 +105,11 @@ func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest * var ptestErr, pxtestErr *PackageError var imports, ximports []*Package var stk ImportStack + var testEmbed, xtestEmbed map[string][]string stk.Push(p.ImportPath + " (test)") rawTestImports := str.StringList(p.TestImports) for i, path := range p.TestImports { - p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) + p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) if str.Contains(p1.Deps, p.ImportPath) || p1.ImportPath == p.ImportPath { // Same error that loadPackage returns (via reusePackage) in pkg.go. // Can't change that code, because that code is only for loading the @@ -116,12 +123,21 @@ func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest * p.TestImports[i] = p1.ImportPath imports = append(imports, p1) } + var err error + p.TestEmbedFiles, testEmbed, err = p.resolveEmbed(p.TestEmbedPatterns) + if err != nil && ptestErr == nil { + ptestErr = &PackageError{ + ImportStack: stk.Copy(), + Err: err, + } + } stk.Pop() + stk.Push(p.ImportPath + "_test") pxtestNeedsPtest := false rawXTestImports := str.StringList(p.XTestImports) for i, path := range p.XTestImports { - p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) + p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) if p1.ImportPath == p.ImportPath { pxtestNeedsPtest = true } else { @@ -129,6 +145,13 @@ func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest * } p.XTestImports[i] = p1.ImportPath } + p.XTestEmbedFiles, xtestEmbed, err = p.resolveEmbed(p.XTestEmbedPatterns) + if err != nil && pxtestErr == nil { + pxtestErr = &PackageError{ + ImportStack: stk.Copy(), + Err: err, + } + } stk.Pop() // Test package. @@ -168,6 +191,14 @@ func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest * m[k] = append(m[k], v...) } ptest.Internal.Build.ImportPos = m + if testEmbed == nil && len(p.Internal.Embed) > 0 { + testEmbed = map[string][]string{} + } + for k, v := range p.Internal.Embed { + testEmbed[k] = v + } + ptest.Internal.Embed = testEmbed + ptest.EmbedFiles = str.StringList(p.EmbedFiles, p.TestEmbedFiles) ptest.collectDeps() } else { ptest = p @@ -185,7 +216,9 @@ func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest * GoFiles: p.XTestGoFiles, Imports: p.XTestImports, ForTest: p.ImportPath, + Module: p.Module, Error: pxtestErr, + EmbedFiles: p.XTestEmbedFiles, }, Internal: PackageInternal{ LocalPrefix: p.Internal.LocalPrefix, @@ -199,6 +232,7 @@ func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest * Gcflags: p.Internal.Gcflags, Ldflags: p.Internal.Ldflags, Gccgoflags: p.Internal.Gccgoflags, + Embed: xtestEmbed, }, } if pxtestNeedsPtest { @@ -216,6 +250,7 @@ func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest * ImportPath: p.ImportPath + ".test", Root: p.Root, Imports: str.StringList(TestMainDeps), + Module: p.Module, }, Internal: PackageInternal{ Build: &build.Package{Name: "main"}, @@ -238,7 +273,7 @@ func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest * if dep == ptest.ImportPath { pmain.Internal.Imports = append(pmain.Internal.Imports, ptest) } else { - p1 := loadImport(pre, dep, "", nil, &stk, nil, 0) + p1 := loadImport(ctx, pre, dep, "", nil, &stk, nil, 0) pmain.Internal.Imports = append(pmain.Internal.Imports, p1) } } diff --git a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock.go b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock.go index aba3eed..05f27c3 100644 --- a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock.go +++ b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock.go @@ -9,6 +9,7 @@ package filelock import ( "errors" + "io/fs" "os" ) @@ -24,7 +25,7 @@ type File interface { Fd() uintptr // Stat returns the FileInfo structure describing file. - Stat() (os.FileInfo, error) + Stat() (fs.FileInfo, error) } // Lock places an advisory write lock on the file, blocking until it can be @@ -87,7 +88,7 @@ var ErrNotSupported = errors.New("operation not supported") // underlyingError returns the underlying error for known os error types. func underlyingError(err error) error { switch err := err.(type) { - case *os.PathError: + case *fs.PathError: return err.Err case *os.LinkError: return err.Err diff --git a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go index c3e09bb..ea480f8 100644 --- a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go +++ b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix solaris +// +build aix solaris,!illumos // This code implements the filelock API using POSIX 'fcntl' locks, which attach // to an (inode, process) pair rather than a file descriptor. To avoid unlocking @@ -12,17 +12,14 @@ // Most platforms provide some alternative API, such as an 'flock' system call // or an F_OFD_SETLK command for 'fcntl', that allows for better concurrency and // does not require per-inode bookkeeping in the application. -// -// TODO(golang.org/issue/35618): add a syscall.Flock binding for Illumos and -// switch it over to use filelock_unix.go. package filelock import ( "errors" "io" + "io/fs" "math/rand" - "os" "sync" "syscall" "time" @@ -66,7 +63,7 @@ func lock(f File, lt lockType) (err error) { mu.Lock() if i, dup := inodes[f]; dup && i != ino { mu.Unlock() - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: errors.New("inode for file changed since last Lock or RLock"), @@ -157,7 +154,7 @@ func lock(f File, lt lockType) (err error) { if err != nil { unlock(f) - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: err, diff --git a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go index 2c88c46..c09530d 100644 --- a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go +++ b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go @@ -6,7 +6,7 @@ package filelock -import "os" +import "io/fs" type lockType int8 @@ -16,7 +16,7 @@ const ( ) func lock(f File, lt lockType) error { - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: ErrNotSupported, @@ -24,7 +24,7 @@ func lock(f File, lt lockType) error { } func unlock(f File) error { - return &os.PathError{ + return &fs.PathError{ Op: "Unlock", Path: f.Name(), Err: ErrNotSupported, diff --git a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go index afdffe3..0798ee4 100644 --- a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go +++ b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go @@ -6,9 +6,7 @@ package filelock -import ( - "os" -) +import "io/fs" type lockType int8 @@ -18,7 +16,7 @@ const ( ) func lock(f File, lt lockType) error { - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: ErrNotSupported, @@ -26,7 +24,7 @@ func lock(f File, lt lockType) error { } func unlock(f File) error { - return &os.PathError{ + return &fs.PathError{ Op: "Unlock", Path: f.Name(), Err: ErrNotSupported, diff --git a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_test.go b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_test.go index faf7344..2ac2052 100644 --- a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_test.go +++ b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_test.go @@ -9,7 +9,6 @@ package filelock_test import ( "fmt" "internal/testenv" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -51,9 +50,9 @@ func mustTempFile(t *testing.T) (f *os.File, remove func()) { t.Helper() base := filepath.Base(t.Name()) - f, err := ioutil.TempFile("", base) + f, err := os.CreateTemp("", base) if err != nil { - t.Fatalf(`ioutil.TempFile("", %q) = %v`, base, err) + t.Fatalf(`os.CreateTemp("", %q) = %v`, base, err) } t.Logf("fd %d = %s", f.Fd(), f.Name()) @@ -161,7 +160,7 @@ func TestRLockExcludesOnlyLock(t *testing.T) { doUnlockTF := false switch runtime.GOOS { - case "aix", "illumos", "solaris": + case "aix", "solaris": // When using POSIX locks (as on Solaris), we can't safely read-lock the // same inode through two different descriptors at the same time: when the // first descriptor is closed, the second descriptor would still be open but diff --git a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go index a1282fc..c06b0f7 100644 --- a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go +++ b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go @@ -2,12 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd hurd linux netbsd openbsd +// +build darwin dragonfly freebsd hurd illumos linux netbsd openbsd package filelock import ( - "os" + "io/fs" "syscall" ) @@ -26,7 +26,7 @@ func lock(f File, lt lockType) (err error) { } } if err != nil { - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: err, diff --git a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go index 43e85e4..19de27e 100644 --- a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go +++ b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go @@ -8,7 +8,7 @@ package filelock import ( "internal/syscall/windows" - "os" + "io/fs" "syscall" ) @@ -34,7 +34,7 @@ func lock(f File, lt lockType) error { err := windows.LockFileEx(syscall.Handle(f.Fd()), uint32(lt), reserved, allBytes, allBytes, ol) if err != nil { - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: err, @@ -47,7 +47,7 @@ func unlock(f File) error { ol := new(syscall.Overlapped) err := windows.UnlockFileEx(syscall.Handle(f.Fd()), reserved, allBytes, allBytes, ol) if err != nil { - return &os.PathError{ + return &fs.PathError{ Op: "Unlock", Path: f.Name(), Err: err, diff --git a/libgo/go/cmd/go/internal/lockedfile/lockedfile.go b/libgo/go/cmd/go/internal/lockedfile/lockedfile.go index 59b2dba..82e1a89 100644 --- a/libgo/go/cmd/go/internal/lockedfile/lockedfile.go +++ b/libgo/go/cmd/go/internal/lockedfile/lockedfile.go @@ -9,7 +9,7 @@ package lockedfile import ( "fmt" "io" - "io/ioutil" + "io/fs" "os" "runtime" ) @@ -35,7 +35,7 @@ type osFile struct { // OpenFile is like os.OpenFile, but returns a locked file. // If flag includes os.O_WRONLY or os.O_RDWR, the file is write-locked; // otherwise, it is read-locked. -func OpenFile(name string, flag int, perm os.FileMode) (*File, error) { +func OpenFile(name string, flag int, perm fs.FileMode) (*File, error) { var ( f = new(File) err error @@ -82,10 +82,10 @@ func Edit(name string) (*File, error) { // non-nil error. func (f *File) Close() error { if f.closed { - return &os.PathError{ + return &fs.PathError{ Op: "close", Path: f.Name(), - Err: os.ErrClosed, + Err: fs.ErrClosed, } } f.closed = true @@ -103,12 +103,12 @@ func Read(name string) ([]byte, error) { } defer f.Close() - return ioutil.ReadAll(f) + return io.ReadAll(f) } // Write opens the named file (creating it with the given permissions if needed), // then write-locks it and overwrites it with the given content. -func Write(name string, content io.Reader, perm os.FileMode) (err error) { +func Write(name string, content io.Reader, perm fs.FileMode) (err error) { f, err := OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) if err != nil { return err @@ -135,7 +135,7 @@ func Transform(name string, t func([]byte) ([]byte, error)) (err error) { } defer f.Close() - old, err := ioutil.ReadAll(f) + old, err := io.ReadAll(f) if err != nil { return err } diff --git a/libgo/go/cmd/go/internal/lockedfile/lockedfile_filelock.go b/libgo/go/cmd/go/internal/lockedfile/lockedfile_filelock.go index f63dd86..efc6646 100644 --- a/libgo/go/cmd/go/internal/lockedfile/lockedfile_filelock.go +++ b/libgo/go/cmd/go/internal/lockedfile/lockedfile_filelock.go @@ -7,18 +7,20 @@ package lockedfile import ( + "io/fs" "os" + "cmd/go/internal/fsys" "cmd/go/internal/lockedfile/internal/filelock" ) -func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { +func openFile(name string, flag int, perm fs.FileMode) (*os.File, error) { // On BSD systems, we could add the O_SHLOCK or O_EXLOCK flag to the OpenFile // call instead of locking separately, but we have to support separate locking // calls for Linux and Windows anyway, so it's simpler to use that approach // consistently. - f, err := os.OpenFile(name, flag&^os.O_TRUNC, perm) + f, err := fsys.OpenFile(name, flag&^os.O_TRUNC, perm) if err != nil { return nil, err } diff --git a/libgo/go/cmd/go/internal/lockedfile/lockedfile_plan9.go b/libgo/go/cmd/go/internal/lockedfile/lockedfile_plan9.go index 4a52c94..70d6edd 100644 --- a/libgo/go/cmd/go/internal/lockedfile/lockedfile_plan9.go +++ b/libgo/go/cmd/go/internal/lockedfile/lockedfile_plan9.go @@ -7,10 +7,13 @@ package lockedfile import ( + "io/fs" "math/rand" "os" "strings" "time" + + "cmd/go/internal/fsys" ) // Opening an exclusive-use file returns an error. @@ -41,7 +44,7 @@ func isLocked(err error) bool { return false } -func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { +func openFile(name string, flag int, perm fs.FileMode) (*os.File, error) { // Plan 9 uses a mode bit instead of explicit lock/unlock syscalls. // // Per http://man.cat-v.org/plan_9/5/stat: “Exclusive use files may be open @@ -55,9 +58,9 @@ func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { // If the file was unpacked or created by some other program, it might not // have the ModeExclusive bit set. Set it before we call OpenFile, so that we // can be confident that a successful OpenFile implies exclusive use. - if fi, err := os.Stat(name); err == nil { - if fi.Mode()&os.ModeExclusive == 0 { - if err := os.Chmod(name, fi.Mode()|os.ModeExclusive); err != nil { + if fi, err := fsys.Stat(name); err == nil { + if fi.Mode()&fs.ModeExclusive == 0 { + if err := os.Chmod(name, fi.Mode()|fs.ModeExclusive); err != nil { return nil, err } } @@ -68,7 +71,7 @@ func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { nextSleep := 1 * time.Millisecond const maxSleep = 500 * time.Millisecond for { - f, err := os.OpenFile(name, flag, perm|os.ModeExclusive) + f, err := fsys.OpenFile(name, flag, perm|fs.ModeExclusive) if err == nil { return f, nil } diff --git a/libgo/go/cmd/go/internal/lockedfile/lockedfile_test.go b/libgo/go/cmd/go/internal/lockedfile/lockedfile_test.go index 416c69d..34327dd 100644 --- a/libgo/go/cmd/go/internal/lockedfile/lockedfile_test.go +++ b/libgo/go/cmd/go/internal/lockedfile/lockedfile_test.go @@ -10,7 +10,6 @@ package lockedfile_test import ( "fmt" "internal/testenv" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -23,7 +22,7 @@ import ( func mustTempDir(t *testing.T) (dir string, remove func()) { t.Helper() - dir, err := ioutil.TempDir("", filepath.Base(t.Name())) + dir, err := os.MkdirTemp("", filepath.Base(t.Name())) if err != nil { t.Fatal(err) } @@ -155,8 +154,8 @@ func TestCanLockExistingFile(t *testing.T) { defer remove() path := filepath.Join(dir, "existing.txt") - if err := ioutil.WriteFile(path, []byte("ok"), 0777); err != nil { - t.Fatalf("ioutil.WriteFile: %v", err) + if err := os.WriteFile(path, []byte("ok"), 0777); err != nil { + t.Fatalf("os.WriteFile: %v", err) } f, err := lockedfile.Edit(path) @@ -201,7 +200,7 @@ func TestSpuriousEDEADLK(t *testing.T) { } defer b.Close() - if err := ioutil.WriteFile(filepath.Join(dir, "locked"), []byte("ok"), 0666); err != nil { + if err := os.WriteFile(filepath.Join(dir, "locked"), []byte("ok"), 0666); err != nil { t.Fatal(err) } diff --git a/libgo/go/cmd/go/internal/modcmd/download.go b/libgo/go/cmd/go/internal/modcmd/download.go index 5844349..ef1ad78 100644 --- a/libgo/go/cmd/go/internal/modcmd/download.go +++ b/libgo/go/cmd/go/internal/modcmd/download.go @@ -5,15 +5,15 @@ package modcmd import ( + "context" "encoding/json" "os" + "runtime" "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modfetch" "cmd/go/internal/modload" - "cmd/go/internal/par" - "cmd/go/internal/work" "golang.org/x/mod/module" ) @@ -63,7 +63,7 @@ func init() { // TODO(jayconrod): https://golang.org/issue/35849 Apply -x to other 'go mod' commands. cmdDownload.Flag.BoolVar(&cfg.BuildX, "x", false, "") - work.AddModCommonFlags(cmdDownload) + base.AddModCommonFlags(&cmdDownload.Flag) } type moduleJSON struct { @@ -78,56 +78,27 @@ type moduleJSON struct { GoModSum string `json:",omitempty"` } -func runDownload(cmd *base.Command, args []string) { +func runDownload(ctx context.Context, cmd *base.Command, args []string) { // Check whether modules are enabled and whether we're in a module. - if cfg.Getenv("GO111MODULE") == "off" { - base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") - } + modload.ForceUseModules = true if !modload.HasModRoot() && len(args) == 0 { base.Fatalf("go mod download: no modules specified (see 'go help mod download')") } if len(args) == 0 { args = []string{"all"} } else if modload.HasModRoot() { - modload.InitMod() // to fill Target - targetAtLatest := modload.Target.Path + "@latest" + modload.LoadModFile(ctx) // to fill Target targetAtUpgrade := modload.Target.Path + "@upgrade" targetAtPatch := modload.Target.Path + "@patch" for _, arg := range args { switch arg { - case modload.Target.Path, targetAtLatest, targetAtUpgrade, targetAtPatch: + case modload.Target.Path, targetAtUpgrade, targetAtPatch: os.Stderr.WriteString("go mod download: skipping argument " + arg + " that resolves to the main module\n") } } } - var mods []*moduleJSON - var work par.Work - listU := false - listVersions := false - for _, info := range modload.ListModules(args, listU, listVersions) { - if info.Replace != nil { - info = info.Replace - } - if info.Version == "" && info.Error == nil { - // main module or module replaced with file path. - // Nothing to download. - continue - } - m := &moduleJSON{ - Path: info.Path, - Version: info.Version, - } - mods = append(mods, m) - if info.Error != nil { - m.Error = info.Error.Err - continue - } - work.Add(m) - } - - work.Do(10, func(item interface{}) { - m := item.(*moduleJSON) + downloadModule := func(m *moduleJSON) { var err error m.Info, err = modfetch.InfoFile(m.Path, m.Version) if err != nil { @@ -145,24 +116,60 @@ func runDownload(cmd *base.Command, args []string) { return } mod := module.Version{Path: m.Path, Version: m.Version} - m.Zip, err = modfetch.DownloadZip(mod) + m.Zip, err = modfetch.DownloadZip(ctx, mod) if err != nil { m.Error = err.Error() return } m.Sum = modfetch.Sum(mod) - m.Dir, err = modfetch.Download(mod) + m.Dir, err = modfetch.Download(ctx, mod) if err != nil { m.Error = err.Error() return } - }) + } + + var mods []*moduleJSON + listU := false + listVersions := false + listRetractions := false + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for _, info := range modload.ListModules(ctx, args, listU, listVersions, listRetractions) { + if info.Replace != nil { + info = info.Replace + } + if info.Version == "" && info.Error == nil { + // main module or module replaced with file path. + // Nothing to download. + continue + } + m := &moduleJSON{ + Path: info.Path, + Version: info.Version, + } + mods = append(mods, m) + if info.Error != nil { + m.Error = info.Error.Err + continue + } + sem <- token{} + go func() { + downloadModule(m) + <-sem + }() + } + + // Fill semaphore channel to wait for goroutines to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } if *downloadJSON { for _, m := range mods { b, err := json.MarshalIndent(m, "", "\t") if err != nil { - base.Fatalf("%v", err) + base.Fatalf("go mod download: %v", err) } os.Stdout.Write(append(b, '\n')) if m.Error != "" { @@ -172,9 +179,12 @@ func runDownload(cmd *base.Command, args []string) { } else { for _, m := range mods { if m.Error != "" { - base.Errorf("%s", m.Error) + base.Errorf("go mod download: %v", m.Error) } } base.ExitIfErrors() } + + // Update go.mod and especially go.sum if needed. + modload.WriteGoMod() } diff --git a/libgo/go/cmd/go/internal/modcmd/edit.go b/libgo/go/cmd/go/internal/modcmd/edit.go index dbbfb96..b203a8a 100644 --- a/libgo/go/cmd/go/internal/modcmd/edit.go +++ b/libgo/go/cmd/go/internal/modcmd/edit.go @@ -8,6 +8,7 @@ package modcmd import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -18,7 +19,6 @@ import ( "cmd/go/internal/lockedfile" "cmd/go/internal/modfetch" "cmd/go/internal/modload" - "cmd/go/internal/work" "golang.org/x/mod/modfile" "golang.org/x/mod/module" @@ -67,9 +67,14 @@ The -dropreplace=old[@v] flag drops a replacement of the given module path and version pair. If the @v is omitted, a replacement without a version on the left side is dropped. +The -retract=version and -dropretract=version flags add and drop a +retraction on the given version. The version may be a single version +like "v1.2.3" or a closed interval like "[v1.1.0,v1.1.9]". Note that +-retract=version is a no-op if that retraction already exists. + The -require, -droprequire, -exclude, -dropexclude, -replace, -and -dropreplace editing flags may be repeated, and the changes -are applied in the order given. +-dropreplace, -retract, and -dropretract editing flags may be repeated, +and the changes are applied in the order given. The -go=version flag sets the expected Go language version. @@ -103,6 +108,15 @@ writing it back to go.mod. The JSON output corresponds to these Go types: New Module } + type Retract struct { + Low string + High string + Rationale string + } + +Retract entries representing a single version (not an interval) will have +the "Low" and "High" fields set to the same value. + Note that this only describes the go.mod file itself, not other modules referred to indirectly. For the full set of modules available to a build, use 'go list -m -json all'. @@ -136,12 +150,14 @@ func init() { cmdEdit.Flag.Var(flagFunc(flagDropReplace), "dropreplace", "") cmdEdit.Flag.Var(flagFunc(flagReplace), "replace", "") cmdEdit.Flag.Var(flagFunc(flagDropExclude), "dropexclude", "") + cmdEdit.Flag.Var(flagFunc(flagRetract), "retract", "") + cmdEdit.Flag.Var(flagFunc(flagDropRetract), "dropretract", "") - work.AddModCommonFlags(cmdEdit) + base.AddModCommonFlags(&cmdEdit.Flag) base.AddBuildFlagsNX(&cmdEdit.Flag) } -func runEdit(cmd *base.Command, args []string) { +func runEdit(ctx context.Context, cmd *base.Command, args []string) { anyFlags := *editModule != "" || *editGo != "" || @@ -251,12 +267,7 @@ func parsePathVersion(flag, arg string) (path, version string) { base.Fatalf("go mod: -%s=%s: invalid path: %v", flag, arg, err) } - // We don't call modfile.CheckPathVersion, because that insists - // on versions being in semver form, but here we want to allow - // versions like "master" or "1234abcdef", which the go command will resolve - // the next time it runs (or during -fix). - // Even so, we need to make sure the version is a valid token. - if modfile.MustQuote(version) { + if !allowedVersionArg(version) { base.Fatalf("go mod: -%s=%s: invalid version %q", flag, arg, version) } @@ -288,12 +299,48 @@ func parsePathVersionOptional(adj, arg string, allowDirPath bool) (path, version return path, version, fmt.Errorf("invalid %s path: %v", adj, err) } } - if path != arg && modfile.MustQuote(version) { + if path != arg && !allowedVersionArg(version) { return path, version, fmt.Errorf("invalid %s version: %q", adj, version) } return path, version, nil } +// parseVersionInterval parses a single version like "v1.2.3" or a closed +// interval like "[v1.2.3,v1.4.5]". Note that a single version has the same +// representation as an interval with equal upper and lower bounds: both +// Low and High are set. +func parseVersionInterval(arg string) (modfile.VersionInterval, error) { + if !strings.HasPrefix(arg, "[") { + if !allowedVersionArg(arg) { + return modfile.VersionInterval{}, fmt.Errorf("invalid version: %q", arg) + } + return modfile.VersionInterval{Low: arg, High: arg}, nil + } + if !strings.HasSuffix(arg, "]") { + return modfile.VersionInterval{}, fmt.Errorf("invalid version interval: %q", arg) + } + s := arg[1 : len(arg)-1] + i := strings.Index(s, ",") + if i < 0 { + return modfile.VersionInterval{}, fmt.Errorf("invalid version interval: %q", arg) + } + low := strings.TrimSpace(s[:i]) + high := strings.TrimSpace(s[i+1:]) + if !allowedVersionArg(low) || !allowedVersionArg(high) { + return modfile.VersionInterval{}, fmt.Errorf("invalid version interval: %q", arg) + } + return modfile.VersionInterval{Low: low, High: high}, nil +} + +// allowedVersionArg returns whether a token may be used as a version in go.mod. +// We don't call modfile.CheckPathVersion, because that insists on versions +// being in semver form, but here we want to allow versions like "master" or +// "1234abcdef", which the go command will resolve the next time it runs (or +// during -fix). Even so, we need to make sure the version is a valid token. +func allowedVersionArg(arg string) bool { + return !modfile.MustQuote(arg) +} + // flagRequire implements the -require flag. func flagRequire(arg string) { path, version := parsePathVersion("require", arg) @@ -376,6 +423,32 @@ func flagDropReplace(arg string) { }) } +// flagRetract implements the -retract flag. +func flagRetract(arg string) { + vi, err := parseVersionInterval(arg) + if err != nil { + base.Fatalf("go mod: -retract=%s: %v", arg, err) + } + edits = append(edits, func(f *modfile.File) { + if err := f.AddRetract(vi, ""); err != nil { + base.Fatalf("go mod: -retract=%s: %v", arg, err) + } + }) +} + +// flagDropRetract implements the -dropretract flag. +func flagDropRetract(arg string) { + vi, err := parseVersionInterval(arg) + if err != nil { + base.Fatalf("go mod: -dropretract=%s: %v", arg, err) + } + edits = append(edits, func(f *modfile.File) { + if err := f.DropRetract(vi); err != nil { + base.Fatalf("go mod: -dropretract=%s: %v", arg, err) + } + }) +} + // fileJSON is the -json output data structure. type fileJSON struct { Module module.Version @@ -383,6 +456,7 @@ type fileJSON struct { Require []requireJSON Exclude []module.Version Replace []replaceJSON + Retract []retractJSON } type requireJSON struct { @@ -396,6 +470,12 @@ type replaceJSON struct { New module.Version } +type retractJSON struct { + Low string `json:",omitempty"` + High string `json:",omitempty"` + Rationale string `json:",omitempty"` +} + // editPrintJSON prints the -json output. func editPrintJSON(modFile *modfile.File) { var f fileJSON @@ -414,6 +494,9 @@ func editPrintJSON(modFile *modfile.File) { for _, r := range modFile.Replace { f.Replace = append(f.Replace, replaceJSON{r.Old, r.New}) } + for _, r := range modFile.Retract { + f.Retract = append(f.Retract, retractJSON{r.Low, r.High, r.Rationale}) + } data, err := json.MarshalIndent(&f, "", "\t") if err != nil { base.Fatalf("go: internal error: %v", err) diff --git a/libgo/go/cmd/go/internal/modcmd/graph.go b/libgo/go/cmd/go/internal/modcmd/graph.go index 27ae935..3277548 100644 --- a/libgo/go/cmd/go/internal/modcmd/graph.go +++ b/libgo/go/cmd/go/internal/modcmd/graph.go @@ -8,14 +8,12 @@ package modcmd import ( "bufio" + "context" "os" "sort" "cmd/go/internal/base" - "cmd/go/internal/cfg" "cmd/go/internal/modload" - "cmd/go/internal/par" - "cmd/go/internal/work" "golang.org/x/mod/module" ) @@ -33,22 +31,16 @@ path@version, except for the main module, which has no @version suffix. } func init() { - work.AddModCommonFlags(cmdGraph) + base.AddModCommonFlags(&cmdGraph.Flag) } -func runGraph(cmd *base.Command, args []string) { +func runGraph(ctx context.Context, cmd *base.Command, args []string) { if len(args) > 0 { base.Fatalf("go mod graph: graph takes no arguments") } - // Checks go mod expected behavior - if !modload.Enabled() { - if cfg.Getenv("GO111MODULE") == "off" { - base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") - } else { - base.Fatalf("go: cannot find main module; see 'go help modules'") - } - } - modload.LoadBuildList() + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot + modload.LoadAllModules(ctx) reqs := modload.MinReqs() format := func(m module.Version) string { @@ -58,23 +50,25 @@ func runGraph(cmd *base.Command, args []string) { return m.Path + "@" + m.Version } - // Note: using par.Work only to manage work queue. - // No parallelism here, so no locking. var out []string var deps int // index in out where deps start - var work par.Work - work.Add(modload.Target) - work.Do(1, func(item interface{}) { - m := item.(module.Version) + seen := map[module.Version]bool{modload.Target: true} + queue := []module.Version{modload.Target} + for len(queue) > 0 { + var m module.Version + m, queue = queue[0], queue[1:] list, _ := reqs.Required(m) for _, r := range list { - work.Add(r) + if !seen[r] { + queue = append(queue, r) + seen[r] = true + } out = append(out, format(m)+" "+format(r)+"\n") } if m == modload.Target { deps = len(out) } - }) + } sort.Slice(out[deps:], func(i, j int) bool { return out[deps+i][0] < out[deps+j][0] diff --git a/libgo/go/cmd/go/internal/modcmd/init.go b/libgo/go/cmd/go/internal/modcmd/init.go index 714ff2e..c081bb5 100644 --- a/libgo/go/cmd/go/internal/modcmd/init.go +++ b/libgo/go/cmd/go/internal/modcmd/init.go @@ -9,46 +9,41 @@ package modcmd import ( "cmd/go/internal/base" "cmd/go/internal/modload" - "cmd/go/internal/work" - "os" - "strings" + "context" ) var cmdInit = &base.Command{ UsageLine: "go mod init [module]", Short: "initialize new module in current directory", Long: ` -Init initializes and writes a new go.mod to the current directory, -in effect creating a new module rooted at the current directory. -The file go.mod must not already exist. -If possible, init will guess the module path from import comments -(see 'go help importpath') or from version control configuration. -To override this guess, supply the module path as an argument. - `, +Init initializes and writes a new go.mod file in the current directory, in +effect creating a new module rooted at the current directory. The go.mod file +must not already exist. + +Init accepts one optional argument, the module path for the new module. If the +module path argument is omitted, init will attempt to infer the module path +using import comments in .go files, vendoring tool configuration files (like +Gopkg.lock), and the current directory (if in GOPATH). + +If a configuration file for a vendoring tool is present, init will attempt to +import module requirements from it. +`, Run: runInit, } func init() { - work.AddModCommonFlags(cmdInit) + base.AddModCommonFlags(&cmdInit.Flag) } -func runInit(cmd *base.Command, args []string) { - modload.CmdModInit = true +func runInit(ctx context.Context, cmd *base.Command, args []string) { if len(args) > 1 { base.Fatalf("go mod init: too many arguments") } + var modPath string if len(args) == 1 { - modload.CmdModModule = args[0] - } - if os.Getenv("GO111MODULE") == "off" { - base.Fatalf("go mod init: modules disabled by GO111MODULE=off; see 'go help modules'") + modPath = args[0] } - modFilePath := modload.ModFilePath() - if _, err := os.Stat(modFilePath); err == nil { - base.Fatalf("go mod init: go.mod already exists") - } - if strings.Contains(modload.CmdModModule, "@") { - base.Fatalf("go mod init: module path must not contain '@'") - } - modload.InitMod() // does all the hard work + + modload.ForceUseModules = true + modload.CreateModFile(ctx, modPath) // does all the hard work } diff --git a/libgo/go/cmd/go/internal/modcmd/tidy.go b/libgo/go/cmd/go/internal/modcmd/tidy.go index af2b04c..fb43e33 100644 --- a/libgo/go/cmd/go/internal/modcmd/tidy.go +++ b/libgo/go/cmd/go/internal/modcmd/tidy.go @@ -9,15 +9,13 @@ package modcmd import ( "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/modfetch" + "cmd/go/internal/imports" "cmd/go/internal/modload" - "cmd/go/internal/work" - - "golang.org/x/mod/module" + "context" ) var cmdTidy = &base.Command{ - UsageLine: "go mod tidy [-v]", + UsageLine: "go mod tidy [-e] [-v]", Short: "add missing and remove unused modules", Long: ` Tidy makes sure go.mod matches the source code in the module. @@ -28,55 +26,47 @@ to go.sum and removes any unnecessary ones. The -v flag causes tidy to print information about removed modules to standard error. + +The -e flag causes tidy to attempt to proceed despite errors +encountered while loading packages. `, + Run: runTidy, } +var tidyE bool // if true, report errors but proceed anyway. + func init() { - cmdTidy.Run = runTidy // break init cycle cmdTidy.Flag.BoolVar(&cfg.BuildV, "v", false, "") - work.AddModCommonFlags(cmdTidy) + cmdTidy.Flag.BoolVar(&tidyE, "e", false, "") + base.AddModCommonFlags(&cmdTidy.Flag) } -func runTidy(cmd *base.Command, args []string) { +func runTidy(ctx context.Context, cmd *base.Command, args []string) { if len(args) > 0 { base.Fatalf("go mod tidy: no arguments allowed") } - modload.LoadALL() + // Tidy aims to make 'go test' reproducible for any package in 'all', so we + // need to include test dependencies. For modules that specify go 1.15 or + // earlier this is a no-op (because 'all' saturates transitive test + // dependencies). + // + // However, with lazy loading (go 1.16+) 'all' includes only the packages that + // are transitively imported by the main module, not the test dependencies of + // those packages. In order to make 'go test' reproducible for the packages + // that are in 'all' but outside of the main module, we must explicitly + // request that their test dependencies be included. + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot + + modload.LoadPackages(ctx, modload.PackageOpts{ + Tags: imports.AnyTags(), + ResolveMissingImports: true, + LoadTests: true, + AllowErrors: tidyE, + }, "all") + modload.TidyBuildList() - modTidyGoSum() // updates memory copy; WriteGoMod on next line flushes it out + modload.TrimGoSum() modload.WriteGoMod() } - -// modTidyGoSum resets the go.sum file content -// to be exactly what's needed for the current go.mod. -func modTidyGoSum() { - // Assuming go.sum already has at least enough from the successful load, - // we only have to tell modfetch what needs keeping. - reqs := modload.Reqs() - keep := make(map[module.Version]bool) - replaced := make(map[module.Version]bool) - var walk func(module.Version) - walk = func(m module.Version) { - // If we build using a replacement module, keep the sum for the replacement, - // since that's the code we'll actually use during a build. - // - // TODO(golang.org/issue/29182): Perhaps we should keep both sums, and the - // sums for both sets of transitive requirements. - r := modload.Replacement(m) - if r.Path == "" { - keep[m] = true - } else { - keep[r] = true - replaced[m] = true - } - list, _ := reqs.Required(m) - for _, r := range list { - if !keep[r] && !replaced[r] { - walk(r) - } - } - } - walk(modload.Target) - modfetch.TrimGoSum(keep) -} diff --git a/libgo/go/cmd/go/internal/modcmd/vendor.go b/libgo/go/cmd/go/internal/modcmd/vendor.go index 8509ceb..1bbb57d 100644 --- a/libgo/go/cmd/go/internal/modcmd/vendor.go +++ b/libgo/go/cmd/go/internal/modcmd/vendor.go @@ -6,9 +6,10 @@ package modcmd import ( "bytes" + "context" "fmt" "io" - "io/ioutil" + "io/fs" "os" "path/filepath" "sort" @@ -16,16 +17,16 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/imports" "cmd/go/internal/modload" - "cmd/go/internal/work" "golang.org/x/mod/module" "golang.org/x/mod/semver" ) var cmdVendor = &base.Command{ - UsageLine: "go mod vendor [-v]", + UsageLine: "go mod vendor [-e] [-v]", Short: "make vendored copy of dependencies", Long: ` Vendor resets the main module's vendor directory to include all packages @@ -34,20 +35,35 @@ It does not include test code for vendored packages. The -v flag causes vendor to print the names of vendored modules and packages to standard error. + +The -e flag causes vendor to attempt to proceed despite errors +encountered while loading packages. `, Run: runVendor, } +var vendorE bool // if true, report errors but proceed anyway + func init() { cmdVendor.Flag.BoolVar(&cfg.BuildV, "v", false, "") - work.AddModCommonFlags(cmdVendor) + cmdVendor.Flag.BoolVar(&vendorE, "e", false, "") + base.AddModCommonFlags(&cmdVendor.Flag) } -func runVendor(cmd *base.Command, args []string) { +func runVendor(ctx context.Context, cmd *base.Command, args []string) { if len(args) != 0 { base.Fatalf("go mod vendor: vendor takes no arguments") } - pkgs := modload.LoadVendor() + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot + + loadOpts := modload.PackageOpts{ + Tags: imports.AnyTags(), + ResolveMissingImports: true, + UseVendorAll: true, + AllowErrors: vendorE, + } + _, pkgs := modload.LoadPackages(ctx, loadOpts, "all") vdir := filepath.Join(modload.ModRoot(), "vendor") if err := os.RemoveAll(vdir); err != nil { @@ -57,7 +73,7 @@ func runVendor(cmd *base.Command, args []string) { modpkgs := make(map[module.Version][]string) for _, pkg := range pkgs { m := modload.PackageModule(pkg) - if m == modload.Target { + if m.Path == "" || m == modload.Target { continue } modpkgs[m] = append(modpkgs[m], pkg) @@ -75,28 +91,38 @@ func runVendor(cmd *base.Command, args []string) { includeAllReplacements = true } + var vendorMods []module.Version + for m := range isExplicit { + vendorMods = append(vendorMods, m) + } + for m := range modpkgs { + if !isExplicit[m] { + vendorMods = append(vendorMods, m) + } + } + module.Sort(vendorMods) + var buf bytes.Buffer - for _, m := range modload.BuildList()[1:] { - if pkgs := modpkgs[m]; len(pkgs) > 0 || isExplicit[m] { - line := moduleLine(m, modload.Replacement(m)) - buf.WriteString(line) + for _, m := range vendorMods { + line := moduleLine(m, modload.Replacement(m)) + buf.WriteString(line) + if cfg.BuildV { + os.Stderr.WriteString(line) + } + if isExplicit[m] { + buf.WriteString("## explicit\n") if cfg.BuildV { - os.Stderr.WriteString(line) + os.Stderr.WriteString("## explicit\n") } - if isExplicit[m] { - buf.WriteString("## explicit\n") - if cfg.BuildV { - os.Stderr.WriteString("## explicit\n") - } - } - sort.Strings(pkgs) - for _, pkg := range pkgs { - fmt.Fprintf(&buf, "%s\n", pkg) - if cfg.BuildV { - fmt.Fprintf(os.Stderr, "%s\n", pkg) - } - vendorPkg(vdir, pkg) + } + pkgs := modpkgs[m] + sort.Strings(pkgs) + for _, pkg := range pkgs { + fmt.Fprintf(&buf, "%s\n", pkg) + if cfg.BuildV { + fmt.Fprintf(os.Stderr, "%s\n", pkg) } + vendorPkg(vdir, pkg) } } @@ -128,7 +154,7 @@ func runVendor(cmd *base.Command, args []string) { base.Fatalf("go mod vendor: %v", err) } - if err := ioutil.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil { + if err := os.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil { base.Fatalf("go mod vendor: %v", err) } } @@ -217,7 +243,7 @@ var metaPrefixes = []string{ } // matchMetadata reports whether info is a metadata file. -func matchMetadata(dir string, info os.FileInfo) bool { +func matchMetadata(dir string, info fs.DirEntry) bool { name := info.Name() for _, p := range metaPrefixes { if strings.HasPrefix(name, p) { @@ -228,12 +254,12 @@ func matchMetadata(dir string, info os.FileInfo) bool { } // matchPotentialSourceFile reports whether info may be relevant to a build operation. -func matchPotentialSourceFile(dir string, info os.FileInfo) bool { +func matchPotentialSourceFile(dir string, info fs.DirEntry) bool { if strings.HasSuffix(info.Name(), "_test.go") { return false } if strings.HasSuffix(info.Name(), ".go") { - f, err := os.Open(filepath.Join(dir, info.Name())) + f, err := fsys.Open(filepath.Join(dir, info.Name())) if err != nil { base.Fatalf("go mod vendor: %v", err) } @@ -254,8 +280,8 @@ func matchPotentialSourceFile(dir string, info os.FileInfo) bool { } // copyDir copies all regular files satisfying match(info) from src to dst. -func copyDir(dst, src string, match func(dir string, info os.FileInfo) bool) { - files, err := ioutil.ReadDir(src) +func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool) { + files, err := os.ReadDir(src) if err != nil { base.Fatalf("go mod vendor: %v", err) } @@ -263,7 +289,7 @@ func copyDir(dst, src string, match func(dir string, info os.FileInfo) bool) { base.Fatalf("go mod vendor: %v", err) } for _, file := range files { - if file.IsDir() || !file.Mode().IsRegular() || !match(src, file) { + if file.IsDir() || !file.Type().IsRegular() || !match(src, file) { continue } r, err := os.Open(filepath.Join(src, file.Name())) diff --git a/libgo/go/cmd/go/internal/modcmd/verify.go b/libgo/go/cmd/go/internal/modcmd/verify.go index b7fd7fa..c83e700 100644 --- a/libgo/go/cmd/go/internal/modcmd/verify.go +++ b/libgo/go/cmd/go/internal/modcmd/verify.go @@ -6,17 +6,16 @@ package modcmd import ( "bytes" + "context" "errors" "fmt" - "io/ioutil" + "io/fs" "os" "runtime" "cmd/go/internal/base" - "cmd/go/internal/cfg" "cmd/go/internal/modfetch" "cmd/go/internal/modload" - "cmd/go/internal/work" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/dirhash" @@ -37,29 +36,23 @@ non-zero status. } func init() { - work.AddModCommonFlags(cmdVerify) + base.AddModCommonFlags(&cmdVerify.Flag) } -func runVerify(cmd *base.Command, args []string) { +func runVerify(ctx context.Context, cmd *base.Command, args []string) { if len(args) != 0 { // NOTE(rsc): Could take a module pattern. base.Fatalf("go mod verify: verify takes no arguments") } - // Checks go mod expected behavior - if !modload.Enabled() || !modload.HasModRoot() { - if cfg.Getenv("GO111MODULE") == "off" { - base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") - } else { - base.Fatalf("go: cannot find main module; see 'go help modules'") - } - } + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot // Only verify up to GOMAXPROCS zips at once. type token struct{} sem := make(chan token, runtime.GOMAXPROCS(0)) // Use a slice of result channels, so that the output is deterministic. - mods := modload.LoadBuildList()[1:] + mods := modload.LoadAllModules(ctx)[1:] errsChans := make([]<-chan []error, len(mods)) for i, mod := range mods { @@ -93,10 +86,10 @@ func verifyMod(mod module.Version) []error { _, zipErr = os.Stat(zip) } dir, dirErr := modfetch.DownloadDir(mod) - data, err := ioutil.ReadFile(zip + "hash") + data, err := os.ReadFile(zip + "hash") if err != nil { - if zipErr != nil && errors.Is(zipErr, os.ErrNotExist) && - dirErr != nil && errors.Is(dirErr, os.ErrNotExist) { + if zipErr != nil && errors.Is(zipErr, fs.ErrNotExist) && + dirErr != nil && errors.Is(dirErr, fs.ErrNotExist) { // Nothing downloaded yet. Nothing to verify. return nil } @@ -105,7 +98,7 @@ func verifyMod(mod module.Version) []error { } h := string(bytes.TrimSpace(data)) - if zipErr != nil && errors.Is(zipErr, os.ErrNotExist) { + if zipErr != nil && errors.Is(zipErr, fs.ErrNotExist) { // ok } else { hZ, err := dirhash.HashZip(zip, dirhash.DefaultHash) @@ -116,7 +109,7 @@ func verifyMod(mod module.Version) []error { errs = append(errs, fmt.Errorf("%s %s: zip has been modified (%v)", mod.Path, mod.Version, zip)) } } - if dirErr != nil && errors.Is(dirErr, os.ErrNotExist) { + if dirErr != nil && errors.Is(dirErr, fs.ErrNotExist) { // ok } else { hD, err := dirhash.HashDir(dir, mod.Path+"@"+mod.Version, dirhash.DefaultHash) diff --git a/libgo/go/cmd/go/internal/modcmd/why.go b/libgo/go/cmd/go/internal/modcmd/why.go index 40d2385..e287c88 100644 --- a/libgo/go/cmd/go/internal/modcmd/why.go +++ b/libgo/go/cmd/go/internal/modcmd/why.go @@ -5,12 +5,13 @@ package modcmd import ( + "context" "fmt" "strings" "cmd/go/internal/base" + "cmd/go/internal/imports" "cmd/go/internal/modload" - "cmd/go/internal/work" "golang.org/x/mod/module" ) @@ -57,25 +58,33 @@ var ( func init() { cmdWhy.Run = runWhy // break init cycle - work.AddModCommonFlags(cmdWhy) + base.AddModCommonFlags(&cmdWhy.Flag) } -func runWhy(cmd *base.Command, args []string) { - loadALL := modload.LoadALL - if *whyVendor { - loadALL = modload.LoadVendor +func runWhy(ctx context.Context, cmd *base.Command, args []string) { + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot + + loadOpts := modload.PackageOpts{ + Tags: imports.AnyTags(), + LoadTests: !*whyVendor, + SilenceErrors: true, + UseVendorAll: *whyVendor, } + if *whyM { listU := false listVersions := false + listRetractions := false for _, arg := range args { if strings.Contains(arg, "@") { base.Fatalf("go mod why: module query not allowed") } } - mods := modload.ListModules(args, listU, listVersions) + mods := modload.ListModules(ctx, args, listU, listVersions, listRetractions) byModule := make(map[module.Version][]string) - for _, path := range loadALL() { + _, pkgs := modload.LoadPackages(ctx, loadOpts, "all") + for _, path := range pkgs { m := modload.PackageModule(path) if m.Path != "" { byModule[m] = append(byModule[m], path) @@ -104,8 +113,11 @@ func runWhy(cmd *base.Command, args []string) { sep = "\n" } } else { - matches := modload.ImportPaths(args) // resolve to packages - loadALL() // rebuild graph, from main module (not from named packages) + // Resolve to packages. + matches, _ := modload.LoadPackages(ctx, loadOpts, args...) + + modload.LoadPackages(ctx, loadOpts, "all") // rebuild graph, from main module (not from named packages) + sep := "" for _, m := range matches { for _, path := range m.Pkgs { diff --git a/libgo/go/cmd/go/internal/modconv/convert.go b/libgo/go/cmd/go/internal/modconv/convert.go index f465a9f..5d4165c 100644 --- a/libgo/go/cmd/go/internal/modconv/convert.go +++ b/libgo/go/cmd/go/internal/modconv/convert.go @@ -7,13 +7,12 @@ package modconv import ( "fmt" "os" + "runtime" "sort" "strings" - "sync" "cmd/go/internal/base" "cmd/go/internal/modfetch" - "cmd/go/internal/par" "golang.org/x/mod/modfile" "golang.org/x/mod/module" @@ -42,46 +41,54 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { // Convert requirements block, which may use raw SHA1 hashes as versions, // to valid semver requirement list, respecting major versions. - var ( - work par.Work - mu sync.Mutex - need = make(map[string]string) - replace = make(map[string]*modfile.Replace) - ) + versions := make([]module.Version, len(mf.Require)) + replace := make(map[string]*modfile.Replace) for _, r := range mf.Replace { replace[r.New.Path] = r replace[r.Old.Path] = r } - for _, r := range mf.Require { + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for i, r := range mf.Require { m := r.Mod if m.Path == "" { continue } if re, ok := replace[m.Path]; ok { - work.Add(re.New) - continue + m = re.New } - work.Add(r.Mod) + sem <- token{} + go func(i int, m module.Version) { + defer func() { <-sem }() + repo, info, err := modfetch.ImportRepoRev(m.Path, m.Version) + if err != nil { + fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), m.Path, m.Version, err) + return + } + + path := repo.ModulePath() + versions[i].Path = path + versions[i].Version = info.Version + }(i, m) + } + // Fill semaphore channel to wait for all tasks to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} } - work.Do(10, func(item interface{}) { - r := item.(module.Version) - repo, info, err := modfetch.ImportRepoRev(r.Path, r.Version) - if err != nil { - fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), r.Path, r.Version, err) - return + need := map[string]string{} + for _, v := range versions { + if v.Path == "" { + continue } - mu.Lock() - path := repo.ModulePath() // Don't use semver.Max here; need to preserve +incompatible suffix. - if v, ok := need[path]; !ok || semver.Compare(v, info.Version) < 0 { - need[path] = info.Version + if needv, ok := need[v.Path]; !ok || semver.Compare(needv, v.Version) < 0 { + need[v.Path] = v.Version } - mu.Unlock() - }) - - var paths []string + } + paths := make([]string, 0, len(need)) for path := range need { paths = append(paths, path) } diff --git a/libgo/go/cmd/go/internal/modconv/convert_test.go b/libgo/go/cmd/go/internal/modconv/convert_test.go index a04a13b..66b9ff4 100644 --- a/libgo/go/cmd/go/internal/modconv/convert_test.go +++ b/libgo/go/cmd/go/internal/modconv/convert_test.go @@ -6,9 +6,9 @@ package modconv import ( "bytes" + "context" "fmt" "internal/testenv" - "io/ioutil" "log" "os" "os/exec" @@ -36,7 +36,7 @@ func testMain(m *testing.M) int { return 0 } - dir, err := ioutil.TempDir("", "modconv-test-") + dir, err := os.MkdirTemp("", "modconv-test-") if err != nil { log.Fatal(err) } @@ -146,6 +146,8 @@ func TestConvertLegacyConfig(t *testing.T) { }, } + ctx := context.Background() + for _, tt := range tests { t.Run(strings.ReplaceAll(tt.path, "/", "_")+"_"+tt.vers, func(t *testing.T) { f, err := modfile.Parse("golden", []byte(tt.gomod), nil) @@ -157,14 +159,14 @@ func TestConvertLegacyConfig(t *testing.T) { t.Fatal(err) } - dir, err := modfetch.Download(module.Version{Path: tt.path, Version: tt.vers}) + dir, err := modfetch.Download(ctx, module.Version{Path: tt.path, Version: tt.vers}) if err != nil { t.Fatal(err) } for name := range Converters { file := filepath.Join(dir, name) - data, err := ioutil.ReadFile(file) + data, err := os.ReadFile(file) if err == nil { f := new(modfile.File) f.AddModuleStmt(tt.path) diff --git a/libgo/go/cmd/go/internal/modconv/modconv_test.go b/libgo/go/cmd/go/internal/modconv/modconv_test.go index ccc4f3d..750525d 100644 --- a/libgo/go/cmd/go/internal/modconv/modconv_test.go +++ b/libgo/go/cmd/go/internal/modconv/modconv_test.go @@ -7,7 +7,7 @@ package modconv import ( "bytes" "fmt" - "io/ioutil" + "os" "path/filepath" "testing" ) @@ -42,7 +42,7 @@ func Test(t *testing.T) { if Converters[extMap[ext]] == nil { t.Fatalf("Converters[%q] == nil", extMap[ext]) } - data, err := ioutil.ReadFile(test) + data, err := os.ReadFile(test) if err != nil { t.Fatal(err) } @@ -50,7 +50,7 @@ func Test(t *testing.T) { if err != nil { t.Fatal(err) } - want, err := ioutil.ReadFile(test[:len(test)-len(ext)] + ".out") + want, err := os.ReadFile(test[:len(test)-len(ext)] + ".out") if err != nil { t.Error(err) } diff --git a/libgo/go/cmd/go/internal/modfetch/cache.go b/libgo/go/cmd/go/internal/modfetch/cache.go index e3074b7..3a2ff63 100644 --- a/libgo/go/cmd/go/internal/modfetch/cache.go +++ b/libgo/go/cmd/go/internal/modfetch/cache.go @@ -10,10 +10,11 @@ import ( "errors" "fmt" "io" - "io/ioutil" + "io/fs" "os" "path/filepath" "strings" + "sync" "cmd/go/internal/base" "cmd/go/internal/cfg" @@ -59,7 +60,7 @@ func CachePath(m module.Version, suffix string) (string, error) { // DownloadDir returns the directory to which m should have been downloaded. // An error will be returned if the module path or version cannot be escaped. -// An error satisfying errors.Is(err, os.ErrNotExist) will be returned +// An error satisfying errors.Is(err, fs.ErrNotExist) will be returned // along with the directory if the directory does not exist or if the directory // is not completely populated. func DownloadDir(m module.Version) (string, error) { @@ -106,14 +107,14 @@ func DownloadDir(m module.Version) (string, error) { // DownloadDirPartialError is returned by DownloadDir if a module directory // exists but was not completely populated. // -// DownloadDirPartialError is equivalent to os.ErrNotExist. +// DownloadDirPartialError is equivalent to fs.ErrNotExist. type DownloadDirPartialError struct { Dir string Err error } func (e *DownloadDirPartialError) Error() string { return fmt.Sprintf("%s: %v", e.Dir, e.Err) } -func (e *DownloadDirPartialError) Is(err error) bool { return err == os.ErrNotExist } +func (e *DownloadDirPartialError) Is(err error) bool { return err == fs.ErrNotExist } // lockVersion locks a file within the module cache that guards the downloading // and extraction of the zipfile for the given module version. @@ -155,16 +156,30 @@ func SideLock() (unlock func(), err error) { type cachingRepo struct { path string cache par.Cache // cache for all operations - r Repo + + once sync.Once + initRepo func() (Repo, error) + r Repo } -func newCachingRepo(r Repo) *cachingRepo { +func newCachingRepo(path string, initRepo func() (Repo, error)) *cachingRepo { return &cachingRepo{ - r: r, - path: r.ModulePath(), + path: path, + initRepo: initRepo, } } +func (r *cachingRepo) repo() Repo { + r.once.Do(func() { + var err error + r.r, err = r.initRepo() + if err != nil { + r.r = errRepo{r.path, err} + } + }) + return r.r +} + func (r *cachingRepo) ModulePath() string { return r.path } @@ -175,7 +190,7 @@ func (r *cachingRepo) Versions(prefix string) ([]string, error) { err error } c := r.cache.Do("versions:"+prefix, func() interface{} { - list, err := r.r.Versions(prefix) + list, err := r.repo().Versions(prefix) return cached{list, err} }).(cached) @@ -197,7 +212,7 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { return cachedInfo{info, nil} } - info, err = r.r.Stat(rev) + info, err = r.repo().Stat(rev) if err == nil { // If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78, // then save the information under the proper version, for future use. @@ -224,7 +239,7 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { func (r *cachingRepo) Latest() (*RevInfo, error) { c := r.cache.Do("latest:", func() interface{} { - info, err := r.r.Latest() + info, err := r.repo().Latest() // Save info for likely future Stat call. if err == nil { @@ -258,7 +273,7 @@ func (r *cachingRepo) GoMod(version string) ([]byte, error) { return cached{text, nil} } - text, err = r.r.GoMod(version) + text, err = r.repo().GoMod(version) if err == nil { if err := checkGoMod(r.path, version, text); err != nil { return cached{text, err} @@ -277,26 +292,11 @@ func (r *cachingRepo) GoMod(version string) ([]byte, error) { } func (r *cachingRepo) Zip(dst io.Writer, version string) error { - return r.r.Zip(dst, version) -} - -// Stat is like Lookup(path).Stat(rev) but avoids the -// repository path resolution in Lookup if the result is -// already cached on local disk. -func Stat(proxy, path, rev string) (*RevInfo, error) { - _, info, err := readDiskStat(path, rev) - if err == nil { - return info, nil - } - repo, err := Lookup(proxy, path) - if err != nil { - return nil, err - } - return repo.Stat(rev) + return r.repo().Zip(dst, version) } -// InfoFile is like Stat but returns the name of the file containing -// the cached information. +// InfoFile is like Lookup(path).Stat(version) but returns the name of the file +// containing the cached information. func InfoFile(path, version string) (string, error) { if !semver.IsValid(version) { return "", fmt.Errorf("invalid version %q", version) @@ -307,10 +307,7 @@ func InfoFile(path, version string) (string, error) { } err := TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, path) - if err == nil { - _, err = repo.Stat(version) - } + _, err := Lookup(proxy, path).Stat(version) return err }) if err != nil { @@ -336,11 +333,7 @@ func GoMod(path, rev string) ([]byte, error) { rev = info.Version } else { err := TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, path) - if err != nil { - return err - } - info, err := repo.Stat(rev) + info, err := Lookup(proxy, path).Stat(rev) if err == nil { rev = info.Version } @@ -357,11 +350,8 @@ func GoMod(path, rev string) ([]byte, error) { return data, nil } - err = TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, path) - if err == nil { - data, err = repo.GoMod(rev) - } + err = TryProxies(func(proxy string) (err error) { + data, err = Lookup(proxy, path).GoMod(rev) return err }) return data, err @@ -492,7 +482,7 @@ func readDiskStatByHash(path, rev string) (file string, info *RevInfo, err error for _, name := range names { if strings.HasSuffix(name, suffix) { v := strings.TrimSuffix(name, ".info") - if IsPseudoVersion(v) && semver.Max(maxVersion, v) == v { + if IsPseudoVersion(v) && semver.Compare(v, maxVersion) > 0 { maxVersion = v file, info, err = readDiskStat(path, strings.TrimSuffix(name, ".info")) } @@ -607,7 +597,7 @@ func rewriteVersionList(dir string) { } defer unlock() - infos, err := ioutil.ReadDir(dir) + infos, err := os.ReadDir(dir) if err != nil { return } diff --git a/libgo/go/cmd/go/internal/modfetch/cache_test.go b/libgo/go/cmd/go/internal/modfetch/cache_test.go index 241c800..722c984 100644 --- a/libgo/go/cmd/go/internal/modfetch/cache_test.go +++ b/libgo/go/cmd/go/internal/modfetch/cache_test.go @@ -5,14 +5,13 @@ package modfetch import ( - "io/ioutil" "os" "path/filepath" "testing" ) func TestWriteDiskCache(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "go-writeCache-test-") + tmpdir, err := os.MkdirTemp("", "go-writeCache-test-") if err != nil { t.Fatal(err) } diff --git a/libgo/go/cmd/go/internal/modfetch/codehost/codehost.go b/libgo/go/cmd/go/internal/modfetch/codehost/codehost.go index d85eddf..86c1c14 100644 --- a/libgo/go/cmd/go/internal/modfetch/codehost/codehost.go +++ b/libgo/go/cmd/go/internal/modfetch/codehost/codehost.go @@ -11,7 +11,7 @@ import ( "crypto/sha256" "fmt" "io" - "io/ioutil" + "io/fs" "os" "os/exec" "path/filepath" @@ -79,9 +79,8 @@ type Repo interface { ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) // RecentTag returns the most recent tag on rev or one of its predecessors - // with the given prefix and major version. - // An empty major string matches any major version. - RecentTag(rev, prefix, major string) (tag string, err error) + // with the given prefix. allowed may be used to filter out unwanted versions. + RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) // DescendsFrom reports whether rev or any of its ancestors has the given tag. // @@ -106,7 +105,7 @@ type FileRev struct { Err error // error if any; os.IsNotExist(Err)==true if rev exists but file does not exist in that rev } -// UnknownRevisionError is an error equivalent to os.ErrNotExist, but for a +// UnknownRevisionError is an error equivalent to fs.ErrNotExist, but for a // revision rather than a file. type UnknownRevisionError struct { Rev string @@ -116,10 +115,10 @@ func (e *UnknownRevisionError) Error() string { return "unknown revision " + e.Rev } func (UnknownRevisionError) Is(err error) bool { - return err == os.ErrNotExist + return err == fs.ErrNotExist } -// ErrNoCommits is an error equivalent to os.ErrNotExist indicating that a given +// ErrNoCommits is an error equivalent to fs.ErrNotExist indicating that a given // repository or module contains no commits. var ErrNoCommits error = noCommitsError{} @@ -129,7 +128,7 @@ func (noCommitsError) Error() string { return "no commits" } func (noCommitsError) Is(err error) bool { - return err == os.ErrNotExist + return err == fs.ErrNotExist } // AllHex reports whether the revision rev is entirely lower-case hexadecimal digits. @@ -189,7 +188,7 @@ func WorkDir(typ, name string) (dir, lockfile string, err error) { } defer unlock() - data, err := ioutil.ReadFile(dir + ".info") + data, err := os.ReadFile(dir + ".info") info, err2 := os.Stat(dir) if err == nil && err2 == nil && info.IsDir() { // Info file and directory both already exist: reuse. @@ -211,7 +210,7 @@ func WorkDir(typ, name string) (dir, lockfile string, err error) { if err := os.MkdirAll(dir, 0777); err != nil { return "", "", err } - if err := ioutil.WriteFile(dir+".info", []byte(key), 0666); err != nil { + if err := os.WriteFile(dir+".info", []byte(key), 0666); err != nil { os.RemoveAll(dir) return "", "", err } @@ -264,6 +263,9 @@ func RunWithStdin(dir string, stdin io.Reader, cmdline ...interface{}) ([]byte, } cmd := str.StringList(cmdline...) + if os.Getenv("TESTGOVCS") == "panic" { + panic(fmt.Sprintf("use of vcs: %v", cmd)) + } if cfg.BuildX { text := new(strings.Builder) if dir != "" { diff --git a/libgo/go/cmd/go/internal/modfetch/codehost/git.go b/libgo/go/cmd/go/internal/modfetch/codehost/git.go index 3192132..8abc039 100644 --- a/libgo/go/cmd/go/internal/modfetch/codehost/git.go +++ b/libgo/go/cmd/go/internal/modfetch/codehost/git.go @@ -9,7 +9,7 @@ import ( "errors" "fmt" "io" - "io/ioutil" + "io/fs" "net/url" "os" "os/exec" @@ -34,13 +34,13 @@ func LocalGitRepo(remote string) (Repo, error) { } // A notExistError wraps another error to retain its original text -// but makes it opaquely equivalent to os.ErrNotExist. +// but makes it opaquely equivalent to fs.ErrNotExist. type notExistError struct { err error } func (e notExistError) Error() string { return e.err.Error() } -func (notExistError) Is(err error) bool { return err == os.ErrNotExist } +func (notExistError) Is(err error) bool { return err == fs.ErrNotExist } const gitWorkDirType = "git3" @@ -188,7 +188,7 @@ func (r *gitRepo) loadRefs() { // For HTTP and HTTPS, that's easy to detect: we'll try to fetch the URL // ourselves and see what code it serves. if u, err := url.Parse(r.remoteURL); err == nil && (u.Scheme == "http" || u.Scheme == "https") { - if _, err := web.GetBytes(u); errors.Is(err, os.ErrNotExist) { + if _, err := web.GetBytes(u); errors.Is(err, fs.ErrNotExist) { gitErr = notExistError{gitErr} } } @@ -505,7 +505,7 @@ func (r *gitRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) { } out, err := Run(r.dir, "git", "cat-file", "blob", info.Name+":"+file) if err != nil { - return nil, os.ErrNotExist + return nil, fs.ErrNotExist } return out, nil } @@ -629,9 +629,9 @@ func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*F case "tag", "commit": switch fileType { default: - f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)} + f.Err = &fs.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)} case "missing": - f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: os.ErrNotExist} + f.Err = &fs.PathError{Path: tag + ":" + file, Op: "read", Err: fs.ErrNotExist} case "blob": f.Data = fileData } @@ -644,7 +644,7 @@ func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*F return missing, nil } -func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) { +func (r *gitRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) { info, err := r.Stat(rev) if err != nil { return "", err @@ -680,7 +680,10 @@ func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) { // NOTE: Do not replace the call to semver.Compare with semver.Max. // We want to return the actual tag, not a canonicalized version of it, // and semver.Max currently canonicalizes (see golang.org/issue/32700). - if c := semver.Canonical(semtag); c != "" && strings.HasPrefix(semtag, c) && (major == "" || semver.Major(c) == major) && semver.Compare(semtag, highest) > 0 { + if c := semver.Canonical(semtag); c == "" || !strings.HasPrefix(semtag, c) || !allowed(semtag) { + continue + } + if semver.Compare(semtag, highest) > 0 { highest = semtag } } @@ -823,12 +826,12 @@ func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, archive, err := Run(r.dir, "git", "-c", "core.autocrlf=input", "-c", "core.eol=lf", "archive", "--format=zip", "--prefix=prefix/", info.Name, args) if err != nil { if bytes.Contains(err.(*RunError).Stderr, []byte("did not match any files")) { - return nil, os.ErrNotExist + return nil, fs.ErrNotExist } return nil, err } - return ioutil.NopCloser(bytes.NewReader(archive)), nil + return io.NopCloser(bytes.NewReader(archive)), nil } // ensureGitAttributes makes sure export-subst and export-ignore features are @@ -859,7 +862,7 @@ func ensureGitAttributes(repoDir string) (err error) { } }() - b, err := ioutil.ReadAll(f) + b, err := io.ReadAll(f) if err != nil { return err } diff --git a/libgo/go/cmd/go/internal/modfetch/codehost/git_test.go b/libgo/go/cmd/go/internal/modfetch/codehost/git_test.go index ba27c70..89a73ba 100644 --- a/libgo/go/cmd/go/internal/modfetch/codehost/git_test.go +++ b/libgo/go/cmd/go/internal/modfetch/codehost/git_test.go @@ -10,7 +10,8 @@ import ( "flag" "fmt" "internal/testenv" - "io/ioutil" + "io" + "io/fs" "log" "os" "os/exec" @@ -52,7 +53,7 @@ func testMain(m *testing.M) int { return 0 } - dir, err := ioutil.TempDir("", "gitrepo-test-") + dir, err := os.MkdirTemp("", "gitrepo-test-") if err != nil { log.Fatal(err) } @@ -210,7 +211,7 @@ var readFileTests = []struct { repo: gitrepo1, rev: "v2.3.4", file: "another.txt", - err: os.ErrNotExist.Error(), + err: fs.ErrNotExist.Error(), }, } @@ -432,7 +433,7 @@ func TestReadZip(t *testing.T) { if tt.err != "" { t.Fatalf("ReadZip: no error, wanted %v", tt.err) } - zipdata, err := ioutil.ReadAll(rc) + zipdata, err := io.ReadAll(rc) if err != nil { t.Fatal(err) } diff --git a/libgo/go/cmd/go/internal/modfetch/codehost/shell.go b/libgo/go/cmd/go/internal/modfetch/codehost/shell.go index 2762c55..ce8b501 100644 --- a/libgo/go/cmd/go/internal/modfetch/codehost/shell.go +++ b/libgo/go/cmd/go/internal/modfetch/codehost/shell.go @@ -14,7 +14,7 @@ import ( "bytes" "flag" "fmt" - "io/ioutil" + "io" "log" "os" "strings" @@ -115,7 +115,7 @@ func main() { fmt.Fprintf(os.Stderr, "?%s\n", err) continue } - data, err := ioutil.ReadAll(rc) + data, err := io.ReadAll(rc) rc.Close() if err != nil { fmt.Fprintf(os.Stderr, "?%s\n", err) @@ -123,7 +123,7 @@ func main() { } if f[3] != "-" { - if err := ioutil.WriteFile(f[3], data, 0666); err != nil { + if err := os.WriteFile(f[3], data, 0666); err != nil { fmt.Fprintf(os.Stderr, "?%s\n", err) continue } diff --git a/libgo/go/cmd/go/internal/modfetch/codehost/vcs.go b/libgo/go/cmd/go/internal/modfetch/codehost/vcs.go index 7284557..c2cca08 100644 --- a/libgo/go/cmd/go/internal/modfetch/codehost/vcs.go +++ b/libgo/go/cmd/go/internal/modfetch/codehost/vcs.go @@ -9,7 +9,7 @@ import ( "fmt" "internal/lazyregexp" "io" - "io/ioutil" + "io/fs" "os" "path/filepath" "sort" @@ -377,7 +377,7 @@ func (r *vcsRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) { out, err := Run(r.dir, r.cmd.readFile(rev, file, r.remote)) if err != nil { - return nil, os.ErrNotExist + return nil, fs.ErrNotExist } return out, nil } @@ -395,7 +395,7 @@ func (r *vcsRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[s return nil, vcsErrorf("ReadFileRevs not implemented") } -func (r *vcsRepo) RecentTag(rev, prefix, major string) (tag string, err error) { +func (r *vcsRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) { // We don't technically need to lock here since we're returning an error // uncondititonally, but doing so anyway will help to avoid baking in // lock-inversion bugs. @@ -432,7 +432,7 @@ func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, if rev == "latest" { rev = r.cmd.latest } - f, err := ioutil.TempFile("", "go-readzip-*.zip") + f, err := os.CreateTemp("", "go-readzip-*.zip") if err != nil { return nil, err } @@ -567,7 +567,7 @@ func bzrParseStat(rev, out string) (*RevInfo, error) { func fossilParseStat(rev, out string) (*RevInfo, error) { for _, line := range strings.Split(out, "\n") { - if strings.HasPrefix(line, "uuid:") { + if strings.HasPrefix(line, "uuid:") || strings.HasPrefix(line, "hash:") { f := strings.Fields(line) if len(f) != 5 || len(f[1]) != 40 || f[4] != "UTC" { return nil, vcsErrorf("unexpected response from fossil info: %q", line) diff --git a/libgo/go/cmd/go/internal/modfetch/coderepo.go b/libgo/go/cmd/go/internal/modfetch/coderepo.go index d043903..2dcbb99 100644 --- a/libgo/go/cmd/go/internal/modfetch/coderepo.go +++ b/libgo/go/cmd/go/internal/modfetch/coderepo.go @@ -10,7 +10,7 @@ import ( "errors" "fmt" "io" - "io/ioutil" + "io/fs" "os" "path" "sort" @@ -419,9 +419,14 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e tagPrefix = r.codeDir + "/" } + isRetracted, err := r.retractedVersions() + if err != nil { + isRetracted = func(string) bool { return false } + } + // tagToVersion returns the version obtained by trimming tagPrefix from tag. - // If the tag is invalid or a pseudo-version, tagToVersion returns an empty - // version. + // If the tag is invalid, retracted, or a pseudo-version, tagToVersion returns + // an empty version. tagToVersion := func(tag string) (v string, tagIsCanonical bool) { if !strings.HasPrefix(tag, tagPrefix) { return "", false @@ -436,6 +441,9 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e if v == "" || !strings.HasPrefix(trimmed, v) { return "", false // Invalid or incomplete version (just vX or vX.Y). } + if isRetracted(v) { + return "", false + } if v == trimmed { tagIsCanonical = true } @@ -500,15 +508,24 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e return checkGoMod() } + // Find the highest tagged version in the revision's history, subject to + // major version and +incompatible constraints. Use that version as the + // pseudo-version base so that the pseudo-version sorts higher. Ignore + // retracted versions. + allowedMajor := func(major string) func(v string) bool { + return func(v string) bool { + return (major == "" || semver.Major(v) == major) && !isRetracted(v) + } + } if pseudoBase == "" { var tag string if r.pseudoMajor != "" || canUseIncompatible() { - tag, _ = r.code.RecentTag(info.Name, tagPrefix, r.pseudoMajor) + tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor(r.pseudoMajor)) } else { // Allow either v1 or v0, but not incompatible higher versions. - tag, _ = r.code.RecentTag(info.Name, tagPrefix, "v1") + tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor("v1")) if tag == "" { - tag, _ = r.code.RecentTag(info.Name, tagPrefix, "v0") + tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor("v0")) } } pseudoBase, _ = tagToVersion(tag) // empty if the tag is invalid @@ -869,6 +886,57 @@ func (r *codeRepo) modPrefix(rev string) string { return r.modPath + "@" + rev } +func (r *codeRepo) retractedVersions() (func(string) bool, error) { + versions, err := r.Versions("") + if err != nil { + return nil, err + } + + for i, v := range versions { + if strings.HasSuffix(v, "+incompatible") { + versions = versions[:i] + break + } + } + if len(versions) == 0 { + return func(string) bool { return false }, nil + } + + var highest string + for i := len(versions) - 1; i >= 0; i-- { + v := versions[i] + if semver.Prerelease(v) == "" { + highest = v + break + } + } + if highest == "" { + highest = versions[len(versions)-1] + } + + data, err := r.GoMod(highest) + if err != nil { + return nil, err + } + f, err := modfile.ParseLax("go.mod", data, nil) + if err != nil { + return nil, err + } + retractions := make([]modfile.VersionInterval, len(f.Retract)) + for _, r := range f.Retract { + retractions = append(retractions, r.VersionInterval) + } + + return func(v string) bool { + for _, r := range retractions { + if semver.Compare(r.Low, v) <= 0 && semver.Compare(v, r.High) <= 0 { + return true + } + } + return false + }, nil +} + func (r *codeRepo) Zip(dst io.Writer, version string) error { if version != module.CanonicalVersion(version) { return fmt.Errorf("version %s is not canonical", version) @@ -897,7 +965,7 @@ func (r *codeRepo) Zip(dst io.Writer, version string) error { subdir = strings.Trim(subdir, "/") // Spool to local file. - f, err := ioutil.TempFile("", "go-codehost-") + f, err := os.CreateTemp("", "go-codehost-") if err != nil { dl.Close() return err @@ -972,7 +1040,7 @@ type zipFile struct { } func (f zipFile) Path() string { return f.name } -func (f zipFile) Lstat() (os.FileInfo, error) { return f.f.FileInfo(), nil } +func (f zipFile) Lstat() (fs.FileInfo, error) { return f.f.FileInfo(), nil } func (f zipFile) Open() (io.ReadCloser, error) { return f.f.Open() } type dataFile struct { @@ -981,9 +1049,9 @@ type dataFile struct { } func (f dataFile) Path() string { return f.name } -func (f dataFile) Lstat() (os.FileInfo, error) { return dataFileInfo{f}, nil } +func (f dataFile) Lstat() (fs.FileInfo, error) { return dataFileInfo{f}, nil } func (f dataFile) Open() (io.ReadCloser, error) { - return ioutil.NopCloser(bytes.NewReader(f.data)), nil + return io.NopCloser(bytes.NewReader(f.data)), nil } type dataFileInfo struct { @@ -992,7 +1060,7 @@ type dataFileInfo struct { func (fi dataFileInfo) Name() string { return path.Base(fi.f.name) } func (fi dataFileInfo) Size() int64 { return int64(len(fi.f.data)) } -func (fi dataFileInfo) Mode() os.FileMode { return 0644 } +func (fi dataFileInfo) Mode() fs.FileMode { return 0644 } func (fi dataFileInfo) ModTime() time.Time { return time.Time{} } func (fi dataFileInfo) IsDir() bool { return false } func (fi dataFileInfo) Sys() interface{} { return nil } diff --git a/libgo/go/cmd/go/internal/modfetch/coderepo_test.go b/libgo/go/cmd/go/internal/modfetch/coderepo_test.go index 9a0cd7d..02e399f 100644 --- a/libgo/go/cmd/go/internal/modfetch/coderepo_test.go +++ b/libgo/go/cmd/go/internal/modfetch/coderepo_test.go @@ -11,7 +11,6 @@ import ( "hash" "internal/testenv" "io" - "io/ioutil" "log" "os" "reflect" @@ -38,7 +37,7 @@ func testMain(m *testing.M) int { // code, bypass the sum database. cfg.GOSUMDB = "off" - dir, err := ioutil.TempDir("", "gitrepo-test-") + dir, err := os.MkdirTemp("", "gitrepo-test-") if err != nil { log.Fatal(err) } @@ -60,7 +59,6 @@ var altVgotests = map[string]string{ type codeRepoTest struct { vcs string path string - lookErr string mpath string rev string err string @@ -332,9 +330,9 @@ var codeRepoTests = []codeRepoTest{ // package in subdirectory - custom domain // In general we can't reject these definitively in Lookup, // but gopkg.in is special. - vcs: "git", - path: "gopkg.in/yaml.v2/abc", - lookErr: "invalid module path \"gopkg.in/yaml.v2/abc\"", + vcs: "git", + path: "gopkg.in/yaml.v2/abc", + err: "invalid module path \"gopkg.in/yaml.v2/abc\"", }, { // package in subdirectory - github @@ -425,7 +423,7 @@ var codeRepoTests = []codeRepoTest{ func TestCodeRepo(t *testing.T) { testenv.MustHaveExternalNetwork(t) - tmpdir, err := ioutil.TempDir("", "modfetch-test-") + tmpdir, err := os.MkdirTemp("", "modfetch-test-") if err != nil { t.Fatal(err) } @@ -440,16 +438,7 @@ func TestCodeRepo(t *testing.T) { testenv.MustHaveExecPath(t, tt.vcs) } - repo, err := Lookup("direct", tt.path) - if tt.lookErr != "" { - if err != nil && err.Error() == tt.lookErr { - return - } - t.Errorf("Lookup(%q): %v, want error %q", tt.path, err, tt.lookErr) - } - if err != nil { - t.Fatalf("Lookup(%q): %v", tt.path, err) - } + repo := Lookup("direct", tt.path) if tt.mpath == "" { tt.mpath = tt.path @@ -501,9 +490,9 @@ func TestCodeRepo(t *testing.T) { needHash := !testing.Short() && (tt.zipFileHash != "" || tt.zipSum != "") if tt.zip != nil || tt.zipErr != "" || needHash { - f, err := ioutil.TempFile(tmpdir, tt.version+".zip.") + f, err := os.CreateTemp(tmpdir, tt.version+".zip.") if err != nil { - t.Fatalf("ioutil.TempFile: %v", err) + t.Fatalf("os.CreateTemp: %v", err) } zipfile := f.Name() defer func() { @@ -665,7 +654,7 @@ var codeRepoVersionsTests = []struct { func TestCodeRepoVersions(t *testing.T) { testenv.MustHaveExternalNetwork(t) - tmpdir, err := ioutil.TempDir("", "vgo-modfetch-test-") + tmpdir, err := os.MkdirTemp("", "vgo-modfetch-test-") if err != nil { t.Fatal(err) } @@ -680,10 +669,7 @@ func TestCodeRepoVersions(t *testing.T) { testenv.MustHaveExecPath(t, tt.vcs) } - repo, err := Lookup("direct", tt.path) - if err != nil { - t.Fatalf("Lookup(%q): %v", tt.path, err) - } + repo := Lookup("direct", tt.path) list, err := repo.Versions(tt.prefix) if err != nil { t.Fatalf("Versions(%q): %v", tt.prefix, err) @@ -742,7 +728,7 @@ var latestTests = []struct { func TestLatest(t *testing.T) { testenv.MustHaveExternalNetwork(t) - tmpdir, err := ioutil.TempDir("", "vgo-modfetch-test-") + tmpdir, err := os.MkdirTemp("", "vgo-modfetch-test-") if err != nil { t.Fatal(err) } @@ -758,10 +744,7 @@ func TestLatest(t *testing.T) { testenv.MustHaveExecPath(t, tt.vcs) } - repo, err := Lookup("direct", tt.path) - if err != nil { - t.Fatalf("Lookup(%q): %v", tt.path, err) - } + repo := Lookup("direct", tt.path) info, err := repo.Latest() if err != nil { if tt.err != "" { diff --git a/libgo/go/cmd/go/internal/modfetch/fetch.go b/libgo/go/cmd/go/internal/modfetch/fetch.go index fd7a5ce..debeb3f 100644 --- a/libgo/go/cmd/go/internal/modfetch/fetch.go +++ b/libgo/go/cmd/go/internal/modfetch/fetch.go @@ -7,10 +7,11 @@ package modfetch import ( "archive/zip" "bytes" + "context" "errors" "fmt" "io" - "io/ioutil" + "io/fs" "os" "path/filepath" "sort" @@ -23,6 +24,7 @@ import ( "cmd/go/internal/par" "cmd/go/internal/renameio" "cmd/go/internal/robustio" + "cmd/go/internal/trace" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/dirhash" @@ -34,7 +36,7 @@ var downloadCache par.Cache // Download downloads the specific module version to the // local download cache and returns the name of the directory // corresponding to the root of the module's file tree. -func Download(mod module.Version) (dir string, err error) { +func Download(ctx context.Context, mod module.Version) (dir string, err error) { if cfg.GOMODCACHE == "" { // modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE // is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen. @@ -47,7 +49,7 @@ func Download(mod module.Version) (dir string, err error) { err error } c := downloadCache.Do(mod, func() interface{} { - dir, err := download(mod) + dir, err := download(ctx, mod) if err != nil { return cached{"", err} } @@ -57,22 +59,22 @@ func Download(mod module.Version) (dir string, err error) { return c.dir, c.err } -func download(mod module.Version) (dir string, err error) { - // If the directory exists, and no .partial file exists, the module has - // already been completely extracted. .partial files may be created when a - // module zip directory is extracted in place instead of being extracted to a - // temporary directory and renamed. +func download(ctx context.Context, mod module.Version) (dir string, err error) { + ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String()) + defer span.Done() + dir, err = DownloadDir(mod) if err == nil { + // The directory has already been completely extracted (no .partial file exists). return dir, nil - } else if dir == "" || !errors.Is(err, os.ErrNotExist) { + } else if dir == "" || !errors.Is(err, fs.ErrNotExist) { return "", err } // To avoid cluttering the cache with extraneous files, // DownloadZip uses the same lockfile as Download. // Invoke DownloadZip before locking the file. - zipfile, err := DownloadZip(mod) + zipfile, err := DownloadZip(ctx, mod) if err != nil { return "", err } @@ -83,6 +85,9 @@ func download(mod module.Version) (dir string, err error) { } defer unlock() + ctx, span = trace.StartSpan(ctx, "unzip "+zipfile) + defer span.Done() + // Check whether the directory was populated while we were waiting on the lock. _, dirErr := DownloadDir(mod) if dirErr == nil { @@ -90,10 +95,11 @@ func download(mod module.Version) (dir string, err error) { } _, dirExists := dirErr.(*DownloadDirPartialError) - // Clean up any remaining temporary directories from previous runs, as well - // as partially extracted diectories created by future versions of cmd/go. - // This is only safe to do because the lock file ensures that their writers - // are no longer active. + // Clean up any remaining temporary directories created by old versions + // (before 1.16), as well as partially extracted directories (indicated by + // DownloadDirPartialError, usually because of a .partial file). This is only + // safe to do because the lock file ensures that their writers are no longer + // active. parentDir := filepath.Dir(dir) tmpPrefix := filepath.Base(dir) + ".tmp-" if old, err := filepath.Glob(filepath.Join(parentDir, tmpPrefix+"*")); err == nil { @@ -111,91 +117,49 @@ func download(mod module.Version) (dir string, err error) { if err != nil { return "", err } - if err := os.Remove(partialPath); err != nil && !os.IsNotExist(err) { - return "", err - } - // Extract the module zip directory. + // Extract the module zip directory at its final location. // - // By default, we extract to a temporary directory, then atomically rename to - // its final location. We use the existence of the source directory to signal - // that it has been extracted successfully (see DownloadDir). If someone - // deletes the entire directory (e.g., as an attempt to prune out file - // corruption), the module cache will still be left in a recoverable - // state. + // To prevent other processes from reading the directory if we crash, + // create a .partial file before extracting the directory, and delete + // the .partial file afterward (all while holding the lock). // - // Unfortunately, os.Rename may fail with ERROR_ACCESS_DENIED on Windows if - // another process opens files in the temporary directory. This is partially - // mitigated by using robustio.Rename, which retries os.Rename for a short - // time. + // Before Go 1.16, we extracted to a temporary directory with a random name + // then renamed it into place with os.Rename. On Windows, this failed with + // ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner) + // opened files in the temporary directory. // - // To avoid this error completely, if unzipInPlace is set, we instead create a - // .partial file (indicating the directory isn't fully extracted), then we - // extract the directory at its final location, then we delete the .partial - // file. This is not the default behavior because older versions of Go may - // simply stat the directory to check whether it exists without looking for a - // .partial file. If multiple versions run concurrently, the older version may - // assume a partially extracted directory is complete. - // TODO(golang.org/issue/36568): when these older versions are no longer - // supported, remove the old default behavior and the unzipInPlace flag. + // Go 1.14.2 and higher respect .partial files. Older versions may use + // partially extracted directories. 'go mod verify' can detect this, + // and 'go clean -modcache' can fix it. if err := os.MkdirAll(parentDir, 0777); err != nil { return "", err } - - if unzipInPlace { - if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil { - return "", err - } - if err := modzip.Unzip(dir, mod, zipfile); err != nil { - fmt.Fprintf(os.Stderr, "-> %s\n", err) - if rmErr := RemoveAll(dir); rmErr == nil { - os.Remove(partialPath) - } - return "", err - } - if err := os.Remove(partialPath); err != nil { - return "", err - } - } else { - tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix) - if err != nil { - return "", err - } - if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil { - fmt.Fprintf(os.Stderr, "-> %s\n", err) - RemoveAll(tmpDir) - return "", err - } - if err := robustio.Rename(tmpDir, dir); err != nil { - RemoveAll(tmpDir) - return "", err + if err := os.WriteFile(partialPath, nil, 0666); err != nil { + return "", err + } + if err := modzip.Unzip(dir, mod, zipfile); err != nil { + fmt.Fprintf(os.Stderr, "-> %s\n", err) + if rmErr := RemoveAll(dir); rmErr == nil { + os.Remove(partialPath) } + return "", err + } + if err := os.Remove(partialPath); err != nil { + return "", err } if !cfg.ModCacheRW { - // Make dir read-only only *after* renaming it. - // os.Rename was observed to fail for read-only directories on macOS. makeDirsReadOnly(dir) } return dir, nil } -var unzipInPlace bool - -func init() { - for _, f := range strings.Split(os.Getenv("GODEBUG"), ",") { - if f == "modcacheunzipinplace=1" { - unzipInPlace = true - break - } - } -} - var downloadZipCache par.Cache // DownloadZip downloads the specific module version to the // local zip cache and returns the name of the zip file. -func DownloadZip(mod module.Version) (zipfile string, err error) { +func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) { // The par.Cache here avoids duplicate work. type cached struct { zipfile string @@ -230,7 +194,7 @@ func DownloadZip(mod module.Version) (zipfile string, err error) { if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil { return cached{"", err} } - if err := downloadZip(mod, zipfile); err != nil { + if err := downloadZip(ctx, mod, zipfile); err != nil { return cached{"", err} } return cached{zipfile, nil} @@ -238,7 +202,10 @@ func DownloadZip(mod module.Version) (zipfile string, err error) { return c.zipfile, c.err } -func downloadZip(mod module.Version, zipfile string) (err error) { +func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) { + ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile) + defer span.Done() + // Clean up any remaining tempfiles from previous runs. // This is only safe to do because the lock file ensures that their // writers are no longer active. @@ -255,7 +222,7 @@ func downloadZip(mod module.Version, zipfile string) (err error) { // contents of the file (by hashing it) before we commit it. Because the file // is zip-compressed, we need an actual file — or at least an io.ReaderAt — to // validate it: we can't just tee the stream as we write it. - f, err := ioutil.TempFile(filepath.Dir(zipfile), filepath.Base(renameio.Pattern(zipfile))) + f, err := os.CreateTemp(filepath.Dir(zipfile), filepath.Base(renameio.Pattern(zipfile))) if err != nil { return err } @@ -266,12 +233,28 @@ func downloadZip(mod module.Version, zipfile string) (err error) { } }() + var unrecoverableErr error err = TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, mod.Path) + if unrecoverableErr != nil { + return unrecoverableErr + } + repo := Lookup(proxy, mod.Path) + err := repo.Zip(f, mod.Version) if err != nil { - return err + // Zip may have partially written to f before failing. + // (Perhaps the server crashed while sending the file?) + // Since we allow fallback on error in some cases, we need to fix up the + // file to be empty again for the next attempt. + if _, err := f.Seek(0, io.SeekStart); err != nil { + unrecoverableErr = err + return err + } + if err := f.Truncate(0); err != nil { + unrecoverableErr = err + return err + } } - return repo.Zip(f, mod.Version) + return err }) if err != nil { return err @@ -331,12 +314,13 @@ func downloadZip(mod module.Version, zipfile string) (err error) { func makeDirsReadOnly(dir string) { type pathMode struct { path string - mode os.FileMode + mode fs.FileMode } var dirs []pathMode // in lexical order - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err == nil && info.Mode()&0222 != 0 { - if info.IsDir() { + filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err == nil && d.IsDir() { + info, err := d.Info() + if err == nil && info.Mode()&0222 != 0 { dirs = append(dirs, pathMode{path, info.Mode()}) } } @@ -353,7 +337,7 @@ func makeDirsReadOnly(dir string) { // any permission changes needed to do so. func RemoveAll(dir string) error { // Module cache has 0555 directories; make them writable in order to remove content. - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error { if err != nil { return nil // ignore errors walking in file system } @@ -374,12 +358,14 @@ type modSum struct { var goSum struct { mu sync.Mutex - m map[module.Version][]string // content of go.sum file (+ go.modverify if present) - checked map[modSum]bool // sums actually checked during execution - dirty bool // whether we added any new sums to m + m map[module.Version][]string // content of go.sum file + status map[modSum]modSumStatus // state of sums in m overwrite bool // if true, overwrite go.sum without incorporating its contents enabled bool // whether to use go.sum at all - modverify string // path to go.modverify, to be deleted +} + +type modSumStatus struct { + used, dirty bool } // initGoSum initializes the go.sum data. @@ -395,7 +381,7 @@ func initGoSum() (bool, error) { } goSum.m = make(map[module.Version][]string) - goSum.checked = make(map[modSum]bool) + goSum.status = make(map[modSum]modSumStatus) data, err := lockedfile.Read(GoSumFile) if err != nil && !os.IsNotExist(err) { return false, err @@ -403,19 +389,6 @@ func initGoSum() (bool, error) { goSum.enabled = true readGoSum(goSum.m, GoSumFile, data) - // Add old go.modverify file. - // We'll delete go.modverify in WriteGoSum. - alt := strings.TrimSuffix(GoSumFile, ".sum") + ".modverify" - if data, err := renameio.ReadFile(alt); err == nil { - migrate := make(map[module.Version][]string) - readGoSum(migrate, alt, data) - for mod, sums := range migrate { - for _, sum := range sums { - addModSumLocked(mod, sum) - } - } - goSum.modverify = alt - } return true, nil } @@ -455,6 +428,28 @@ func readGoSum(dst map[module.Version][]string, file string, data []byte) error return nil } +// HaveSum returns true if the go.sum file contains an entry for mod. +// The entry's hash must be generated with a known hash algorithm. +// mod.Version may have a "/go.mod" suffix to distinguish sums for +// .mod and .zip files. +func HaveSum(mod module.Version) bool { + goSum.mu.Lock() + defer goSum.mu.Unlock() + inited, err := initGoSum() + if err != nil || !inited { + return false + } + for _, h := range goSum.m[mod] { + if !strings.HasPrefix(h, "h1:") { + continue + } + if !goSum.status[modSum{mod, h}].dirty { + return true + } + } + return false +} + // checkMod checks the given module's checksum. func checkMod(mod module.Version) { if cfg.GOMODCACHE == "" { @@ -469,7 +464,7 @@ func checkMod(mod module.Version) { } data, err := renameio.ReadFile(ziphash) if err != nil { - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes. return } @@ -488,7 +483,7 @@ func checkMod(mod module.Version) { // goModSum returns the checksum for the go.mod contents. func goModSum(data []byte) (string, error) { return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) { - return ioutil.NopCloser(bytes.NewReader(data)), nil + return io.NopCloser(bytes.NewReader(data)), nil }) } @@ -504,6 +499,9 @@ func checkGoMod(path, version string, data []byte) error { } // checkModSum checks that the recorded checksum for mod is h. +// +// mod.Version may have the additional suffix "/go.mod" to request the checksum +// for the module's go.mod file only. func checkModSum(mod module.Version, h string) error { // We lock goSum when manipulating it, // but we arrange to release the lock when calling checkSumDB, @@ -518,6 +516,11 @@ func checkModSum(mod module.Version, h string) error { return err } done := inited && haveModSumLocked(mod, h) + if inited { + st := goSum.status[modSum{mod, h}] + st.used = true + goSum.status[modSum{mod, h}] = st + } goSum.mu.Unlock() if done { @@ -537,6 +540,9 @@ func checkModSum(mod module.Version, h string) error { if inited { goSum.mu.Lock() addModSumLocked(mod, h) + st := goSum.status[modSum{mod, h}] + st.dirty = true + goSum.status[modSum{mod, h}] = st goSum.mu.Unlock() } return nil @@ -546,7 +552,6 @@ func checkModSum(mod module.Version, h string) error { // If it finds a conflicting pair instead, it calls base.Fatalf. // goSum.mu must be locked. func haveModSumLocked(mod module.Version, h string) bool { - goSum.checked[modSum{mod, h}] = true for _, vh := range goSum.m[mod] { if h == vh { return true @@ -568,15 +573,21 @@ func addModSumLocked(mod module.Version, h string) { fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h) } goSum.m[mod] = append(goSum.m[mod], h) - goSum.dirty = true } // checkSumDB checks the mod, h pair against the Go checksum database. // It calls base.Fatalf if the hash is to be rejected. func checkSumDB(mod module.Version, h string) error { + modWithoutSuffix := mod + noun := "module" + if strings.HasSuffix(mod.Version, "/go.mod") { + noun = "go.mod" + modWithoutSuffix.Version = strings.TrimSuffix(mod.Version, "/go.mod") + } + db, lines, err := lookupSumDB(mod) if err != nil { - return module.VersionError(mod, fmt.Errorf("verifying module: %v", err)) + return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: %v", noun, err)) } have := mod.Path + " " + mod.Version + " " + h @@ -586,7 +597,7 @@ func checkSumDB(mod module.Version, h string) error { return nil } if strings.HasPrefix(line, prefix) { - return module.VersionError(mod, fmt.Errorf("verifying module: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, h, db, line[len(prefix)-len("h1:"):])) + return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, noun, h, db, line[len(prefix)-len("h1:"):])) } } return nil @@ -612,18 +623,35 @@ func Sum(mod module.Version) string { } // WriteGoSum writes the go.sum file if it needs to be updated. -func WriteGoSum() { +// +// keep is used to check whether a newly added sum should be saved in go.sum. +// It should have entries for both module content sums and go.mod sums +// (version ends with "/go.mod"). Existing sums will be preserved unless they +// have been marked for deletion with TrimGoSum. +func WriteGoSum(keep map[module.Version]bool) { goSum.mu.Lock() defer goSum.mu.Unlock() + // If we haven't read the go.sum file yet, don't bother writing it. if !goSum.enabled { - // If we haven't read the go.sum file yet, don't bother writing it: at best, - // we could rename the go.modverify file if it isn't empty, but we haven't - // needed to touch it so far — how important could it be? return } - if !goSum.dirty { - // Don't bother opening the go.sum file if we don't have anything to add. + + // Check whether we need to add sums for which keep[m] is true or remove + // unused sums marked with TrimGoSum. If there are no changes to make, + // just return without opening go.sum. + dirty := false +Outer: + for m, hs := range goSum.m { + for _, h := range hs { + st := goSum.status[modSum{m, h}] + if st.dirty && (!st.used || keep[m]) { + dirty = true + break Outer + } + } + } + if !dirty { return } if cfg.BuildMod == "readonly" { @@ -644,9 +672,10 @@ func WriteGoSum() { // them without good reason. goSum.m = make(map[module.Version][]string, len(goSum.m)) readGoSum(goSum.m, GoSumFile, data) - for ms := range goSum.checked { - addModSumLocked(ms.mod, ms.sum) - goSum.dirty = true + for ms, st := range goSum.status { + if st.used { + addModSumLocked(ms.mod, ms.sum) + } } } @@ -661,7 +690,10 @@ func WriteGoSum() { list := goSum.m[m] sort.Strings(list) for _, h := range list { - fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h) + st := goSum.status[modSum{m, h}] + if !st.dirty || (st.used && keep[m]) { + fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h) + } } } return buf.Bytes(), nil @@ -671,16 +703,16 @@ func WriteGoSum() { base.Fatalf("go: updating go.sum: %v", err) } - goSum.checked = make(map[modSum]bool) - goSum.dirty = false + goSum.status = make(map[modSum]modSumStatus) goSum.overwrite = false - - if goSum.modverify != "" { - os.Remove(goSum.modverify) // best effort - } } -// TrimGoSum trims go.sum to contain only the modules for which keep[m] is true. +// TrimGoSum trims go.sum to contain only the modules needed for reproducible +// builds. +// +// keep is used to check whether a sum should be retained in go.mod. It should +// have entries for both module content sums and go.mod sums (version ends +// with "/go.mod"). func TrimGoSum(keep map[module.Version]bool) { goSum.mu.Lock() defer goSum.mu.Unlock() @@ -692,13 +724,11 @@ func TrimGoSum(keep map[module.Version]bool) { return } - for m := range goSum.m { - // If we're keeping x@v we also keep x@v/go.mod. - // Map x@v/go.mod back to x@v for the keep lookup. - noGoMod := module.Version{Path: m.Path, Version: strings.TrimSuffix(m.Version, "/go.mod")} - if !keep[m] && !keep[noGoMod] { - delete(goSum.m, m) - goSum.dirty = true + for m, hs := range goSum.m { + if !keep[m] { + for _, h := range hs { + goSum.status[modSum{m, h}] = modSumStatus{used: false, dirty: true} + } goSum.overwrite = true } } @@ -818,16 +848,16 @@ the checksum database is not consulted, and all unrecognized modules are accepted, at the cost of giving up the security guarantee of verified repeatable downloads for all modules. A better way to bypass the checksum database for specific modules is to use the GOPRIVATE or GONOSUMDB environment -variables. See 'go help module-private' for details. +variables. See 'go help private' for details. The 'go env -w' command (see 'go help env') can be used to set these variables for future go command invocations. `, } -var HelpModulePrivate = &base.Command{ - UsageLine: "module-private", - Short: "module configuration for non-public modules", +var HelpPrivate = &base.Command{ + UsageLine: "private", + Short: "configuration for downloading non-public code", Long: ` The go command defaults to downloading modules from the public Go module mirror at proxy.golang.org. It also defaults to validating downloaded modules, @@ -868,6 +898,11 @@ be used for downloading both public and private modules, because GONOPROXY has been set to a pattern that won't match any modules, overriding GOPRIVATE. +The GOPRIVATE variable is also used to define the "public" and "private" +patterns for the GOVCS variable; see 'go help vcs'. For that usage, +GOPRIVATE applies even in GOPATH mode. In that case, it matches import paths +instead of module paths. + The 'go env -w' command (see 'go help env') can be used to set these variables for future go command invocations. `, diff --git a/libgo/go/cmd/go/internal/modfetch/insecure.go b/libgo/go/cmd/go/internal/modfetch/insecure.go index 8420432..012d05f 100644 --- a/libgo/go/cmd/go/internal/modfetch/insecure.go +++ b/libgo/go/cmd/go/internal/modfetch/insecure.go @@ -6,11 +6,11 @@ package modfetch import ( "cmd/go/internal/cfg" - "cmd/go/internal/get" - "cmd/go/internal/str" + + "golang.org/x/mod/module" ) // allowInsecure reports whether we are allowed to fetch this path in an insecure manner. func allowInsecure(path string) bool { - return get.Insecure || str.GlobsMatchPath(cfg.GOINSECURE, path) + return cfg.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, path) } diff --git a/libgo/go/cmd/go/internal/modfetch/proxy.go b/libgo/go/cmd/go/internal/modfetch/proxy.go index 1c35d0b..d75b4da 100644 --- a/libgo/go/cmd/go/internal/modfetch/proxy.go +++ b/libgo/go/cmd/go/internal/modfetch/proxy.go @@ -9,9 +9,8 @@ import ( "errors" "fmt" "io" - "io/ioutil" + "io/fs" "net/url" - "os" "path" pathpkg "path" "path/filepath" @@ -186,7 +185,7 @@ func proxyList() ([]proxySpec, error) { // TryProxies iterates f over each configured proxy (including "noproxy" and // "direct" if applicable) until f returns no error or until f returns an -// error that is not equivalent to os.ErrNotExist on a proxy configured +// error that is not equivalent to fs.ErrNotExist on a proxy configured // not to fall back on errors. // // TryProxies then returns that final error. @@ -222,7 +221,7 @@ func TryProxies(f func(proxy string) error) error { if err == nil { return nil } - isNotExistErr := errors.Is(err, os.ErrNotExist) + isNotExistErr := errors.Is(err, fs.ErrNotExist) if proxy.url == "direct" || (proxy.url == "noproxy" && err != errUseProxy) { bestErr = err @@ -242,8 +241,9 @@ func TryProxies(f func(proxy string) error) error { } type proxyRepo struct { - url *url.URL - path string + url *url.URL + path string + redactedURL string } func newProxyRepo(baseURL, path string) (Repo, error) { @@ -268,10 +268,10 @@ func newProxyRepo(baseURL, path string) (Repo, error) { if err != nil { return nil, err } - + redactedURL := base.Redacted() base.Path = strings.TrimSuffix(base.Path, "/") + "/" + enc base.RawPath = strings.TrimSuffix(base.RawPath, "/") + "/" + pathEscape(enc) - return &proxyRepo{base, path}, nil + return &proxyRepo{base, path, redactedURL}, nil } func (p *proxyRepo) ModulePath() string { @@ -304,7 +304,7 @@ func (p *proxyRepo) getBytes(path string) ([]byte, error) { return nil, err } defer body.Close() - return ioutil.ReadAll(body) + return io.ReadAll(body) } func (p *proxyRepo) getBody(path string) (io.ReadCloser, error) { @@ -413,7 +413,7 @@ func (p *proxyRepo) Stat(rev string) (*RevInfo, error) { } info := new(RevInfo) if err := json.Unmarshal(data, info); err != nil { - return nil, p.versionError(rev, err) + return nil, p.versionError(rev, fmt.Errorf("invalid response from proxy %q: %w", p.redactedURL, err)) } if info.Version != rev && rev == module.CanonicalVersion(rev) && module.Check(p.path, rev) == nil { // If we request a correct, appropriate version for the module path, the @@ -427,14 +427,14 @@ func (p *proxyRepo) Stat(rev string) (*RevInfo, error) { func (p *proxyRepo) Latest() (*RevInfo, error) { data, err := p.getBytes("@latest") if err != nil { - if !errors.Is(err, os.ErrNotExist) { + if !errors.Is(err, fs.ErrNotExist) { return nil, p.versionError("", err) } return p.latest() } info := new(RevInfo) if err := json.Unmarshal(data, info); err != nil { - return nil, p.versionError("", err) + return nil, p.versionError("", fmt.Errorf("invalid response from proxy %q: %w", p.redactedURL, err)) } return info, nil } diff --git a/libgo/go/cmd/go/internal/modfetch/pseudo.go b/libgo/go/cmd/go/internal/modfetch/pseudo.go index 20c0b06..93eb0fa 100644 --- a/libgo/go/cmd/go/internal/modfetch/pseudo.go +++ b/libgo/go/cmd/go/internal/modfetch/pseudo.go @@ -76,6 +76,12 @@ func PseudoVersion(major, older string, t time.Time, rev string) string { return v + incDecimal(patch) + "-0." + segment + build } +// ZeroPseudoVersion returns a pseudo-version with a zero timestamp and +// revision, which may be used as a placeholder. +func ZeroPseudoVersion(major string) string { + return PseudoVersion(major, "", time.Time{}, "000000000000") +} + // incDecimal returns the decimal string incremented by 1. func incDecimal(decimal string) string { // Scan right to left turning 9s to 0s until you find a digit to increment. @@ -120,6 +126,12 @@ func IsPseudoVersion(v string) bool { return strings.Count(v, "-") >= 2 && semver.IsValid(v) && pseudoVersionRE.MatchString(v) } +// IsZeroPseudoVersion returns whether v is a pseudo-version with a zero base, +// timestamp, and revision, as returned by ZeroPseudoVersion. +func IsZeroPseudoVersion(v string) bool { + return v == ZeroPseudoVersion(semver.Major(v)) +} + // PseudoVersionTime returns the time stamp of the pseudo-version v. // It returns an error if v is not a pseudo-version or if the time stamp // embedded in the pseudo-version is not a valid time. diff --git a/libgo/go/cmd/go/internal/modfetch/repo.go b/libgo/go/cmd/go/internal/modfetch/repo.go index f03bdd8..af9e24c 100644 --- a/libgo/go/cmd/go/internal/modfetch/repo.go +++ b/libgo/go/cmd/go/internal/modfetch/repo.go @@ -7,18 +7,19 @@ package modfetch import ( "fmt" "io" + "io/fs" "os" "sort" "strconv" "time" "cmd/go/internal/cfg" - "cmd/go/internal/get" "cmd/go/internal/modfetch/codehost" "cmd/go/internal/par" - "cmd/go/internal/str" + "cmd/go/internal/vcs" web "cmd/go/internal/web" + "golang.org/x/mod/module" "golang.org/x/mod/semver" ) @@ -32,8 +33,17 @@ type Repo interface { // Versions lists all known versions with the given prefix. // Pseudo-versions are not included. + // // Versions should be returned sorted in semver order // (implementations can use SortVersions). + // + // Versions returns a non-nil error only if there was a problem + // fetching the list of versions: it may return an empty list + // along with a nil error if the list of matching versions + // is known to be empty. + // + // If the underlying repository does not exist, + // Versions returns an error matching errors.Is(_, os.NotExist). Versions(prefix string) ([]string, error) // Stat returns information about the revision rev. @@ -188,27 +198,26 @@ type lookupCacheKey struct { // // A successful return does not guarantee that the module // has any defined versions. -func Lookup(proxy, path string) (Repo, error) { +func Lookup(proxy, path string) Repo { if traceRepo { defer logCall("Lookup(%q, %q)", proxy, path)() } type cached struct { - r Repo - err error + r Repo } c := lookupCache.Do(lookupCacheKey{proxy, path}, func() interface{} { - r, err := lookup(proxy, path) - if err == nil { - if traceRepo { + r := newCachingRepo(path, func() (Repo, error) { + r, err := lookup(proxy, path) + if err == nil && traceRepo { r = newLoggingRepo(r) } - r = newCachingRepo(r) - } - return cached{r, err} + return r, err + }) + return cached{r} }).(cached) - return c.r, c.err + return c.r } // lookup returns the module with the given module path. @@ -217,7 +226,7 @@ func lookup(proxy, path string) (r Repo, err error) { return nil, errLookupDisabled } - if str.GlobsMatchPath(cfg.GONOPROXY, path) { + if module.MatchPrefixPatterns(cfg.GONOPROXY, path) { switch proxy { case "noproxy", "direct": return lookupDirect(path) @@ -228,7 +237,7 @@ func lookup(proxy, path string) (r Repo, err error) { switch proxy { case "off": - return nil, errProxyOff + return errRepo{path, errProxyOff}, nil case "direct": return lookupDirect(path) case "noproxy": @@ -261,13 +270,13 @@ func lookupDirect(path string) (Repo, error) { if allowInsecure(path) { security = web.Insecure } - rr, err := get.RepoRootForImportPath(path, get.PreferMod, security) + rr, err := vcs.RepoRootForImportPath(path, vcs.PreferMod, security) if err != nil { // We don't know where to find code for a module with this path. return nil, notExistError{err: err} } - if rr.VCS == "mod" { + if rr.VCS.Name == "mod" { // Fetch module from proxy with base URL rr.Repo. return newProxyRepo(rr.Repo, path) } @@ -279,8 +288,8 @@ func lookupDirect(path string) (Repo, error) { return newCodeRepo(code, rr.Root, path) } -func lookupCodeRepo(rr *get.RepoRoot) (codehost.Repo, error) { - code, err := codehost.NewRepo(rr.VCS, rr.Repo) +func lookupCodeRepo(rr *vcs.RepoRoot) (codehost.Repo, error) { + code, err := codehost.NewRepo(rr.VCS.Cmd, rr.Repo) if err != nil { if _, ok := err.(*codehost.VCSError); ok { return nil, err @@ -306,7 +315,7 @@ func ImportRepoRev(path, rev string) (Repo, *RevInfo, error) { if allowInsecure(path) { security = web.Insecure } - rr, err := get.RepoRootForImportPath(path, get.IgnoreMod, security) + rr, err := vcs.RepoRootForImportPath(path, vcs.IgnoreMod, security) if err != nil { return nil, nil, err } @@ -407,7 +416,24 @@ func (l *loggingRepo) Zip(dst io.Writer, version string) error { return l.r.Zip(dst, version) } -// A notExistError is like os.ErrNotExist, but with a custom message +// errRepo is a Repo that returns the same error for all operations. +// +// It is useful in conjunction with caching, since cache hits will not attempt +// the prohibited operations. +type errRepo struct { + modulePath string + err error +} + +func (r errRepo) ModulePath() string { return r.modulePath } + +func (r errRepo) Versions(prefix string) (tags []string, err error) { return nil, r.err } +func (r errRepo) Stat(rev string) (*RevInfo, error) { return nil, r.err } +func (r errRepo) Latest() (*RevInfo, error) { return nil, r.err } +func (r errRepo) GoMod(version string) ([]byte, error) { return nil, r.err } +func (r errRepo) Zip(dst io.Writer, version string) error { return r.err } + +// A notExistError is like fs.ErrNotExist, but with a custom message type notExistError struct { err error } @@ -421,7 +447,7 @@ func (e notExistError) Error() string { } func (notExistError) Is(target error) bool { - return target == os.ErrNotExist + return target == fs.ErrNotExist } func (e notExistError) Unwrap() error { diff --git a/libgo/go/cmd/go/internal/modfetch/sumdb.go b/libgo/go/cmd/go/internal/modfetch/sumdb.go index 7973f47..4fbc54d 100644 --- a/libgo/go/cmd/go/internal/modfetch/sumdb.go +++ b/libgo/go/cmd/go/internal/modfetch/sumdb.go @@ -12,7 +12,8 @@ import ( "bytes" "errors" "fmt" - "io/ioutil" + "io" + "io/fs" "net/url" "os" "path/filepath" @@ -22,9 +23,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/get" "cmd/go/internal/lockedfile" - "cmd/go/internal/str" "cmd/go/internal/web" "golang.org/x/mod/module" @@ -34,7 +33,7 @@ import ( // useSumDB reports whether to use the Go checksum database for the given module. func useSumDB(mod module.Version) bool { - return cfg.GOSUMDB != "off" && !get.Insecure && !str.GlobsMatchPath(cfg.GONOSUMDB, mod.Path) + return cfg.GOSUMDB != "off" && !cfg.Insecure && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path) } // lookupSumDB returns the Go checksum database's go.sum lines for the given module, @@ -184,7 +183,7 @@ func (c *dbClient) initBase() { return nil } }) - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // No proxies, or all proxies failed (with 404, 410, or were were allowed // to fall back), or we reached an explicit "direct" or "off". c.base = c.direct @@ -205,7 +204,7 @@ func (c *dbClient) ReadConfig(file string) (data []byte, err error) { } targ := filepath.Join(cfg.SumdbDir, file) data, err = lockedfile.Read(targ) - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // Treat non-existent as empty, to bootstrap the "latest" file // the first time we connect to a given database. return []byte{}, nil @@ -229,7 +228,7 @@ func (*dbClient) WriteConfig(file string, old, new []byte) error { return err } defer f.Close() - data, err := ioutil.ReadAll(f) + data, err := io.ReadAll(f) if err != nil { return err } @@ -259,7 +258,7 @@ func (*dbClient) ReadCache(file string) ([]byte, error) { // during which the empty file can be locked for reading. // Treat observing an empty file as file not found. if err == nil && len(data) == 0 { - err = &os.PathError{Op: "read", Path: targ, Err: os.ErrNotExist} + err = &fs.PathError{Op: "read", Path: targ, Err: fs.ErrNotExist} } return data, err } diff --git a/libgo/go/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go b/libgo/go/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go index eac9b32..d9ba8ef 100644 --- a/libgo/go/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go +++ b/libgo/go/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go @@ -16,6 +16,7 @@ package zip_sum_test import ( + "context" "crypto/sha256" "encoding/csv" "encoding/hex" @@ -23,7 +24,6 @@ import ( "fmt" "internal/testenv" "io" - "io/ioutil" "os" "path/filepath" "strings" @@ -80,7 +80,7 @@ func TestZipSums(t *testing.T) { if *modCacheDir != "" { cfg.BuildContext.GOPATH = *modCacheDir } else { - tmpDir, err := ioutil.TempDir("", "TestZipSums") + tmpDir, err := os.MkdirTemp("", "TestZipSums") if err != nil { t.Fatal(err) } @@ -119,7 +119,7 @@ func TestZipSums(t *testing.T) { name := fmt.Sprintf("%s@%s", strings.ReplaceAll(test.m.Path, "/", "_"), test.m.Version) t.Run(name, func(t *testing.T) { t.Parallel() - zipPath, err := modfetch.DownloadZip(test.m) + zipPath, err := modfetch.DownloadZip(context.Background(), test.m) if err != nil { if *updateTestData { t.Logf("%s: could not download module: %s (will remove from testdata)", test.m, err) diff --git a/libgo/go/cmd/go/internal/modget/get.go b/libgo/go/cmd/go/internal/modget/get.go index 4c69824..8463ec4 100644 --- a/libgo/go/cmd/go/internal/modget/get.go +++ b/libgo/go/cmd/go/internal/modget/get.go @@ -5,25 +5,47 @@ // Package modget implements the module-aware ``go get'' command. package modget +// The arguments to 'go get' are patterns with optional version queries, with +// the version queries defaulting to "upgrade". +// +// The patterns are normally interpreted as package patterns. However, if a +// pattern cannot match a package, it is instead interpreted as a *module* +// pattern. For version queries such as "upgrade" and "patch" that depend on the +// selected version of a module (or of the module containing a package), +// whether a pattern denotes a package or module may change as updates are +// applied (see the example in mod_get_patchmod.txt). +// +// There are a few other ambiguous cases to resolve, too. A package can exist in +// two different modules at the same version: for example, the package +// example.com/foo might be found in module example.com and also in module +// example.com/foo, and those modules may have independent v0.1.0 tags — so the +// input 'example.com/foo@v0.1.0' could syntactically refer to the variant of +// the package loaded from either module! (See mod_get_ambiguous_pkg.txt.) +// If the argument is ambiguous, the user can often disambiguate by specifying +// explicit versions for *all* of the potential module paths involved. + import ( + "context" "errors" "fmt" "os" "path/filepath" + "reflect" + "runtime" "sort" "strings" "sync" "cmd/go/internal/base" - "cmd/go/internal/get" + "cmd/go/internal/cfg" "cmd/go/internal/imports" "cmd/go/internal/load" "cmd/go/internal/modload" - "cmd/go/internal/mvs" "cmd/go/internal/par" "cmd/go/internal/search" "cmd/go/internal/work" + "golang.org/x/mod/modfile" "golang.org/x/mod/module" "golang.org/x/mod/semver" ) @@ -69,15 +91,15 @@ downgrades the dependency. The version suffix @none indicates that the dependency should be removed entirely, downgrading or removing modules depending on it as needed. -The version suffix @latest explicitly requests the latest minor release of the -module named by the given path. The suffix @upgrade is like @latest but +The version suffix @latest explicitly requests the latest minor release of +the module named by the given path. The suffix @upgrade is like @latest but will not downgrade a module if it is already required at a revision or pre-release version newer than the latest released version. The suffix @patch requests the latest patch release: the latest released version with the same major and minor version numbers as the currently required version. Like @upgrade, @patch will not downgrade a module already required -at a newer version. If the path is not already required, @upgrade and @patch -are equivalent to @latest. +at a newer version. If the path is not already required, @upgrade is +equivalent to @latest, and @patch is disallowed. Although get defaults to using the latest version of the module containing a named package, it does not use the latest version of that module's @@ -114,14 +136,27 @@ require downgrading other dependencies, and 'go get' does this automatically as well. The -insecure flag permits fetching from repositories and resolving -custom domains using insecure schemes such as HTTP. Use with caution. The -GOINSECURE environment variable is usually a better alternative, since it -provides control over which modules may be retrieved using an insecure scheme. -See 'go help environment' for details. +custom domains using insecure schemes such as HTTP, and also bypassess +module sum validation using the checksum database. Use with caution. +This flag is deprecated and will be removed in a future version of go. +To permit the use of insecure schemes, use the GOINSECURE environment +variable instead. To bypass module sum validation, use GOPRIVATE or +GONOSUMDB. See 'go help environment' for details. The second step is to download (if needed), build, and install the named packages. +The -d flag instructs get to skip this step, downloading source code +needed to build the named packages and their dependencies, but not +building or installing. + +Building and installing packages with get is deprecated. In a future release, +the -d flag will be enabled by default, and 'go get' will be only be used to +adjust dependencies of the current module. To install a package using +dependencies from the current module, use 'go install'. To install a package +ignoring the current module, use 'go install' with an @version suffix like +"@latest" after each argument. + If an argument names a module but not a package (because there is no Go source code in the module's root directory), then the install step is skipped for that argument, instead of causing a build failure. @@ -133,10 +168,6 @@ the module versions. For example, 'go get golang.org/x/perf/cmd/...' adds the latest golang.org/x/perf and then installs the commands in that latest version. -The -d flag instructs get to download the source code needed to build -the named packages, including downloading necessary dependencies, -but not to build and install them. - With no package arguments, 'go get' applies to Go package in the current directory, if any. In particular, 'go get -u' and 'go get -u=patch' update all the dependencies of that package. @@ -173,6 +204,82 @@ Usage: ` + CmdGet.UsageLine + ` ` + CmdGet.Long, } +var HelpVCS = &base.Command{ + UsageLine: "vcs", + Short: "controlling version control with GOVCS", + Long: ` +The 'go get' command can run version control commands like git +to download imported code. This functionality is critical to the decentralized +Go package ecosystem, in which code can be imported from any server, +but it is also a potential security problem, if a malicious server finds a +way to cause the invoked version control command to run unintended code. + +To balance the functionality and security concerns, the 'go get' command +by default will only use git and hg to download code from public servers. +But it will use any known version control system (bzr, fossil, git, hg, svn) +to download code from private servers, defined as those hosting packages +matching the GOPRIVATE variable (see 'go help private'). The rationale behind +allowing only Git and Mercurial is that these two systems have had the most +attention to issues of being run as clients of untrusted servers. In contrast, +Bazaar, Fossil, and Subversion have primarily been used in trusted, +authenticated environments and are not as well scrutinized as attack surfaces. + +The version control command restrictions only apply when using direct version +control access to download code. When downloading modules from a proxy, +'go get' uses the proxy protocol instead, which is always permitted. +By default, the 'go get' command uses the Go module mirror (proxy.golang.org) +for public packages and only falls back to version control for private +packages or when the mirror refuses to serve a public package (typically for +legal reasons). Therefore, clients can still access public code served from +Bazaar, Fossil, or Subversion repositories by default, because those downloads +use the Go module mirror, which takes on the security risk of running the +version control commands, using a custom sandbox. + +The GOVCS variable can be used to change the allowed version control systems +for specific packages (identified by a module or import path). +The GOVCS variable applies both when using modules and when using GOPATH. +When using modules, the patterns match against the module path. +When using GOPATH, the patterns match against the import path +corresponding to the root of the version control repository. + +The general form of the GOVCS setting is a comma-separated list of +pattern:vcslist rules. The pattern is a glob pattern that must match +one or more leading elements of the module or import path. The vcslist +is a pipe-separated list of allowed version control commands, or "all" +to allow use of any known command, or "off" to allow nothing. +The earliest matching pattern in the list applies, even if later patterns +might also match. + +For example, consider: + + GOVCS=github.com:git,evil.com:off,*:git|hg + +With this setting, code with an module or import path beginning with +github.com/ can only use git; paths on evil.com cannot use any version +control command, and all other paths (* matches everything) can use +only git or hg. + +The special patterns "public" and "private" match public and private +module or import paths. A path is private if it matches the GOPRIVATE +variable; otherwise it is public. + +If no rules in the GOVCS variable match a particular module or import path, +the 'go get' command applies its default rule, which can now be summarized +in GOVCS notation as 'public:git|hg,private:all'. + +To allow unfettered use of any version control system for any package, use: + + GOVCS=*:all + +To disable all use of version control, use: + + GOVCS=*:off + +The 'go env -w' command (see 'go help env') can be used to set the GOVCS +variable for future go command invocations. +`, +} + var ( getD = CmdGet.Flag.Bool("d", false, "") getF = CmdGet.Flag.Bool("f", false, "") @@ -180,23 +287,29 @@ var ( getM = CmdGet.Flag.Bool("m", false, "") getT = CmdGet.Flag.Bool("t", false, "") getU upgradeFlag - // -insecure is get.Insecure + // -insecure is cfg.Insecure // -v is cfg.BuildV ) // upgradeFlag is a custom flag.Value for -u. -type upgradeFlag string +type upgradeFlag struct { + rawVersion string + version string +} func (*upgradeFlag) IsBoolFlag() bool { return true } // allow -u func (v *upgradeFlag) Set(s string) error { if s == "false" { - s = "" + v.version = "" + v.rawVersion = "" + } else if s == "true" { + v.version = "upgrade" + v.rawVersion = "" + } else { + v.version = s + v.rawVersion = s } - if s == "true" { - s = "upgrade" - } - *v = upgradeFlag(s) return nil } @@ -205,66 +318,16 @@ func (v *upgradeFlag) String() string { return "" } func init() { work.AddBuildFlags(CmdGet, work.OmitModFlag) CmdGet.Run = runGet // break init loop - CmdGet.Flag.BoolVar(&get.Insecure, "insecure", get.Insecure, "") + CmdGet.Flag.BoolVar(&cfg.Insecure, "insecure", cfg.Insecure, "") CmdGet.Flag.Var(&getU, "u", "") } -// A getArg holds a parsed positional argument for go get (path@vers). -type getArg struct { - // raw is the original argument, to be printed in error messages. - raw string - - // path is the part of the argument before "@" (or the whole argument - // if there is no "@"). path specifies the modules or packages to get. - path string - - // vers is the part of the argument after "@" or an implied - // "upgrade" or "patch" if there is no "@". vers specifies the - // module version to get. - vers string -} - -// querySpec describes a query for a specific module. path may be a -// module path, package path, or package pattern. vers is a version -// query string from a command line argument. -type querySpec struct { - // path is a module path, package path, or package pattern that - // specifies which module to query. - path string - - // vers specifies what version of the module to get. - vers string - - // forceModulePath is true if path should be interpreted as a module path. - // If forceModulePath is true, prevM must be set. - forceModulePath bool - - // prevM is the previous version of the module. prevM is needed - // to determine the minor version number if vers is "patch". It's also - // used to avoid downgrades from prerelease versions newer than - // "latest" and "patch". If prevM is set, forceModulePath must be true. - prevM module.Version -} - -// query holds the state for a query made for a specific module. -// After a query is performed, we know the actual module path and -// version and whether any packages were matched by the query path. -type query struct { - querySpec - - // arg is the command line argument that matched the specified module. - arg string - - // m is the module path and version found by the query. - m module.Version -} - -func runGet(cmd *base.Command, args []string) { - switch getU { +func runGet(ctx context.Context, cmd *base.Command, args []string) { + switch getU.version { case "", "upgrade", "patch": // ok default: - base.Fatalf("go get: unknown upgrade flag -u=%s", getU) + base.Fatalf("go get: unknown upgrade flag -u=%s", getU.rawVersion) } if *getF { fmt.Fprintf(os.Stderr, "go get: -f flag is a no-op when using modules\n") @@ -275,770 +338,1394 @@ func runGet(cmd *base.Command, args []string) { if *getM { base.Fatalf("go get: -m flag is no longer supported; consider -d to skip building packages") } - modload.LoadTests = *getT - - buildList := modload.LoadBuildList() - buildList = buildList[:len(buildList):len(buildList)] // copy on append - versionByPath := make(map[string]string) - for _, m := range buildList { - versionByPath[m.Path] = m.Version + if cfg.Insecure { + fmt.Fprintf(os.Stderr, "go get: -insecure flag is deprecated; see 'go help get' for details\n") } + load.ModResolveTests = *getT // Do not allow any updating of go.mod until we've applied // all the requested changes and checked that the result matches // what was requested. modload.DisallowWriteGoMod() - // Allow looking up modules for import paths outside of a module. + // Allow looking up modules for import paths when outside of a module. // 'go get' is expected to do this, unlike other commands. modload.AllowMissingModuleImports() - // Parse command-line arguments and report errors. The command-line - // arguments are of the form path@version or simply path, with implicit - // @upgrade. path@none is "downgrade away". - var gets []getArg - var queries []*query - for _, arg := range search.CleanPatterns(args) { - // Argument is path or path@vers. - path := arg - vers := "" - if i := strings.Index(arg, "@"); i >= 0 { - path, vers = arg[:i], arg[i+1:] + modload.LoadModFile(ctx) // Initializes modload.Target. + + queries := parseArgs(ctx, args) + + r := newResolver(ctx, queries) + r.performLocalQueries(ctx) + r.performPathQueries(ctx) + + for { + r.performWildcardQueries(ctx) + r.performPatternAllQueries(ctx) + + if changed := r.resolveCandidates(ctx, queries, nil); changed { + // 'go get' arguments can be (and often are) package patterns rather than + // (just) modules. A package can be provided by any module with a prefix + // of its import path, and a wildcard can even match packages in modules + // with totally different paths. Because of these effects, and because any + // change to the selected version of a module can bring in entirely new + // module paths as dependencies, we need to reissue queries whenever we + // change the build list. + // + // The result of any version query for a given module — even "upgrade" or + // "patch" — is always relative to the build list at the start of + // the 'go get' command, not an intermediate state, and is therefore + // dederministic and therefore cachable, and the constraints on the + // selected version of each module can only narrow as we iterate. + // + // "all" is functionally very similar to a wildcard pattern. The set of + // packages imported by the main module does not change, and the query + // result for the module containing each such package also does not change + // (it is always relative to the initial build list, before applying + // queries). So the only way that the result of an "all" query can change + // is if some matching package moves from one module in the build list + // to another, which should not happen very often. + continue } - if strings.Contains(vers, "@") || arg != path && vers == "" { - base.Errorf("go get %s: invalid module version syntax", arg) + + // When we load imports, we detect the following conditions: + // + // - missing transitive depencies that need to be resolved from outside the + // current build list (note that these may add new matches for existing + // pattern queries!) + // + // - transitive dependencies that didn't match any other query, + // but need to be upgraded due to the -u flag + // + // - ambiguous import errors. + // TODO(#27899): Try to resolve ambiguous import errors automatically. + upgrades := r.findAndUpgradeImports(ctx, queries) + if changed := r.resolveCandidates(ctx, nil, upgrades); changed { + continue + } + + r.findMissingWildcards(ctx) + if changed := r.resolveCandidates(ctx, r.wildcardQueries, nil); changed { continue } + break + } + + r.checkWildcardVersions(ctx) + + var pkgPatterns []string + for _, q := range queries { + if q.matchesPackages { + pkgPatterns = append(pkgPatterns, q.pattern) + } + } + r.checkPackagesAndRetractions(ctx, pkgPatterns) + + // We've already downloaded modules (and identified direct and indirect + // dependencies) by loading packages in findAndUpgradeImports. + // So if -d is set, we're done after the module work. + // + // Otherwise, we need to build and install the packages matched by + // command line arguments. + // Note that 'go get -u' without arguments is equivalent to + // 'go get -u .', so we'll typically build the package in the current + // directory. + if !*getD && len(pkgPatterns) > 0 { + work.BuildInit() + pkgs := load.PackagesAndErrors(ctx, pkgPatterns) + load.CheckPackageErrors(pkgs) + work.InstallPackages(ctx, pkgPatterns, pkgs) + // TODO(#40276): After Go 1.16, print a deprecation notice when building + // and installing main packages. 'go install pkg' or + // 'go install pkg@version' should be used instead. + // Give the specific argument to use if possible. + } + + if !modload.HasModRoot() { + return + } + + // Everything succeeded. Update go.mod. + oldReqs := reqsFromGoMod(modload.ModFile()) + + modload.AllowWriteGoMod() + modload.WriteGoMod() + modload.DisallowWriteGoMod() + + newReqs := reqsFromGoMod(modload.ModFile()) + r.reportChanges(oldReqs, newReqs) +} + +// parseArgs parses command-line arguments and reports errors. +// +// The command-line arguments are of the form path@version or simply path, with +// implicit @upgrade. path@none is "downgrade away". +func parseArgs(ctx context.Context, rawArgs []string) []*query { + defer base.ExitIfErrors() + + var queries []*query + for _, arg := range search.CleanPatterns(rawArgs) { + q, err := newQuery(arg) + if err != nil { + base.Errorf("go get: %v", err) + continue + } + + // If there were no arguments, CleanPatterns returns ".". Set the raw + // string back to "" for better errors. + if len(rawArgs) == 0 { + q.raw = "" + } + // Guard against 'go get x.go', a common mistake. // Note that package and module paths may end with '.go', so only print an error // if the argument has no version and either has no slash or refers to an existing file. - if strings.HasSuffix(arg, ".go") && vers == "" { - if !strings.Contains(arg, "/") { - base.Errorf("go get %s: arguments must be package or module paths", arg) + if strings.HasSuffix(q.raw, ".go") && q.rawVersion == "" { + if !strings.Contains(q.raw, "/") { + base.Errorf("go get %s: arguments must be package or module paths", q.raw) continue } - if fi, err := os.Stat(arg); err == nil && !fi.IsDir() { - base.Errorf("go get: %s exists as a file, but 'go get' requires package arguments", arg) + if fi, err := os.Stat(q.raw); err == nil && !fi.IsDir() { + base.Errorf("go get: %s exists as a file, but 'go get' requires package arguments", q.raw) continue } } - // If no version suffix is specified, assume @upgrade. - // If -u=patch was specified, assume @patch instead. - if vers == "" { - if getU != "" { - vers = string(getU) + queries = append(queries, q) + } + + return queries +} + +type resolver struct { + localQueries []*query // queries for absolute or relative paths + pathQueries []*query // package path literal queries in original order + wildcardQueries []*query // path wildcard queries in original order + patternAllQueries []*query // queries with the pattern "all" + + // Indexed "none" queries. These are also included in the slices above; + // they are indexed here to speed up noneForPath. + nonesByPath map[string]*query // path-literal "@none" queries indexed by path + wildcardNones []*query // wildcard "@none" queries + + // resolvedVersion maps each module path to the version of that module that + // must be selected in the final build list, along with the first query + // that resolved the module to that version (the “reason”). + resolvedVersion map[string]versionReason + + buildList []module.Version + buildListResolvedVersions int // len(resolvedVersion) when buildList was computed + buildListVersion map[string]string // index of buildList (module path → version) + + initialVersion map[string]string // index of the initial build list at the start of 'go get' + + missing []pathSet // candidates for missing transitive dependencies + + work *par.Queue + + matchInModuleCache par.Cache +} + +type versionReason struct { + version string + reason *query +} + +func newResolver(ctx context.Context, queries []*query) *resolver { + buildList := modload.LoadAllModules(ctx) + initialVersion := make(map[string]string, len(buildList)) + for _, m := range buildList { + initialVersion[m.Path] = m.Version + } + + r := &resolver{ + work: par.NewQueue(runtime.GOMAXPROCS(0)), + resolvedVersion: map[string]versionReason{}, + buildList: buildList, + buildListVersion: initialVersion, + initialVersion: initialVersion, + nonesByPath: map[string]*query{}, + } + + for _, q := range queries { + if q.pattern == "all" { + r.patternAllQueries = append(r.patternAllQueries, q) + } else if q.patternIsLocal { + r.localQueries = append(r.localQueries, q) + } else if q.isWildcard() { + r.wildcardQueries = append(r.wildcardQueries, q) + } else { + r.pathQueries = append(r.pathQueries, q) + } + + if q.version == "none" { + // Index "none" queries to make noneForPath more efficient. + if q.isWildcard() { + r.wildcardNones = append(r.wildcardNones, q) } else { - vers = "upgrade" + // All "<path>@none" queries for the same path are identical; we only + // need to index one copy. + r.nonesByPath[q.pattern] = q } } + } - gets = append(gets, getArg{raw: arg, path: path, vers: vers}) - - // Determine the modules that path refers to, and create queries - // to lookup modules at target versions before loading packages. - // This is an imprecise process, but it helps reduce unnecessary - // queries and package loading. It's also necessary for handling - // patterns like golang.org/x/tools/..., which can't be expanded - // during package loading until they're in the build list. - switch { - case filepath.IsAbs(path) || search.IsRelativePath(path): - // Absolute paths like C:\foo and relative paths like ../foo... - // are restricted to matching packages in the main module. If the path - // is explicit and contains no wildcards (...), check that it is a - // package in the main module. If the path contains wildcards but - // matches no packages, we'll warn after package loading. - if !strings.Contains(path, "...") { - m := search.NewMatch(path) - if pkgPath := modload.DirImportPath(path); pkgPath != "." { - m = modload.TargetPackages(pkgPath) - } - if len(m.Pkgs) == 0 { - for _, err := range m.Errs { - base.Errorf("go get %s: %v", arg, err) - } + return r +} - abs, err := filepath.Abs(path) - if err != nil { - abs = path - } - base.Errorf("go get %s: path %s is not a package in module rooted at %s", arg, abs, modload.ModRoot()) - continue - } - } +// initialSelected returns the version of the module with the given path that +// was selected at the start of this 'go get' invocation. +func (r *resolver) initialSelected(mPath string) (version string) { + v, ok := r.initialVersion[mPath] + if !ok { + return "none" + } + return v +} - if path != arg { - base.Errorf("go get %s: can't request explicit version of path in main module", arg) - continue - } +// selected returns the version of the module with the given path that is +// selected in the resolver's current build list. +func (r *resolver) selected(mPath string) (version string) { + v, ok := r.buildListVersion[mPath] + if !ok { + return "none" + } + return v +} + +// noneForPath returns a "none" query matching the given module path, +// or found == false if no such query exists. +func (r *resolver) noneForPath(mPath string) (nq *query, found bool) { + if nq = r.nonesByPath[mPath]; nq != nil { + return nq, true + } + for _, nq := range r.wildcardNones { + if nq.matchesPath(mPath) { + return nq, true + } + } + return nil, false +} + +// queryModule wraps modload.Query, substituting r.checkAllowedOr to decide +// allowed versions. +func (r *resolver) queryModule(ctx context.Context, mPath, query string, selected func(string) string) (module.Version, error) { + current := r.initialSelected(mPath) + rev, err := modload.Query(ctx, mPath, query, current, r.checkAllowedOr(query, selected)) + if err != nil { + return module.Version{}, err + } + return module.Version{Path: mPath, Version: rev.Version}, nil +} - case strings.Contains(path, "..."): - // Wait until we load packages to look up modules. - // We don't know yet whether any modules in the build list provide - // packages matching the pattern. For example, suppose - // golang.org/x/tools and golang.org/x/tools/playground are separate - // modules, and only golang.org/x/tools is in the build list. If the - // user runs 'go get golang.org/x/tools/playground/...', we should - // add a requirement for golang.org/x/tools/playground. We should not - // upgrade golang.org/x/tools. - - case path == "all": - // If there is no main module, "all" is not meaningful. - if !modload.HasModRoot() { - base.Errorf(`go get %s: cannot match "all": working directory is not part of a module`, arg) +// queryPackage wraps modload.QueryPackage, substituting r.checkAllowedOr to +// decide allowed versions. +func (r *resolver) queryPackages(ctx context.Context, pattern, query string, selected func(string) string) (pkgMods []module.Version, err error) { + results, err := modload.QueryPackages(ctx, pattern, query, selected, r.checkAllowedOr(query, selected)) + if len(results) > 0 { + pkgMods = make([]module.Version, 0, len(results)) + for _, qr := range results { + pkgMods = append(pkgMods, qr.Mod) + } + } + return pkgMods, err +} + +// queryPattern wraps modload.QueryPattern, substituting r.checkAllowedOr to +// decide allowed versions. +func (r *resolver) queryPattern(ctx context.Context, pattern, query string, selected func(string) string) (pkgMods []module.Version, mod module.Version, err error) { + results, modOnly, err := modload.QueryPattern(ctx, pattern, query, selected, r.checkAllowedOr(query, selected)) + if len(results) > 0 { + pkgMods = make([]module.Version, 0, len(results)) + for _, qr := range results { + pkgMods = append(pkgMods, qr.Mod) + } + } + if modOnly != nil { + mod = modOnly.Mod + } + return pkgMods, mod, err +} + +// checkAllowedOr is like modload.CheckAllowed, but it always allows the requested +// and current versions (even if they are retracted or otherwise excluded). +func (r *resolver) checkAllowedOr(requested string, selected func(string) string) modload.AllowedFunc { + return func(ctx context.Context, m module.Version) error { + if m.Version == requested { + return modload.CheckExclusions(ctx, m) + } + if (requested == "upgrade" || requested == "patch") && m.Version == selected(m.Path) { + return nil + } + return modload.CheckAllowed(ctx, m) + } +} + +// matchInModule is a caching wrapper around modload.MatchInModule. +func (r *resolver) matchInModule(ctx context.Context, pattern string, m module.Version) (packages []string, err error) { + type key struct { + pattern string + m module.Version + } + type entry struct { + packages []string + err error + } + + e := r.matchInModuleCache.Do(key{pattern, m}, func() interface{} { + match := modload.MatchInModule(ctx, pattern, m, imports.AnyTags()) + if len(match.Errs) > 0 { + return entry{match.Pkgs, match.Errs[0]} + } + return entry{match.Pkgs, nil} + }).(entry) + + return e.packages, e.err +} + +// queryNone adds a candidate set to q for each module matching q.pattern. +// Each candidate set has only one possible module version: the matched +// module at version "none". +// +// We interpret arguments to 'go get' as packages first, and fall back to +// modules second. However, no module exists at version "none", and therefore no +// package exists at that version either: we know that the argument cannot match +// any packages, and thus it must match modules instead. +func (r *resolver) queryNone(ctx context.Context, q *query) { + if search.IsMetaPackage(q.pattern) { + panic(fmt.Sprintf("internal error: queryNone called with pattern %q", q.pattern)) + } + + if !q.isWildcard() { + q.pathOnce(q.pattern, func() pathSet { + if modload.HasModRoot() && q.pattern == modload.Target.Path { + // The user has explicitly requested to downgrade their own module to + // version "none". This is not an entirely unreasonable request: it + // could plausibly mean “downgrade away everything that depends on any + // explicit version of the main module”, or “downgrade away the + // package with the same path as the main module, found in a module + // with a prefix of the main module's path”. + // + // However, neither of those behaviors would be consistent with the + // plain meaning of the query. To try to reduce confusion, reject the + // query explicitly. + return errSet(&modload.QueryMatchesMainModuleError{Pattern: q.pattern, Query: q.version}) } - // Don't query modules until we load packages. We'll automatically - // look up any missing modules. - case search.IsMetaPackage(path): - base.Errorf("go get %s: explicit requirement on standard-library module %s not allowed", path, path) + return pathSet{mod: module.Version{Path: q.pattern, Version: "none"}} + }) + } + + for _, curM := range r.buildList { + if !q.matchesPath(curM.Path) { continue + } + q.pathOnce(curM.Path, func() pathSet { + if modload.HasModRoot() && curM == modload.Target { + return errSet(&modload.QueryMatchesMainModuleError{Pattern: q.pattern, Query: q.version}) + } + return pathSet{mod: module.Version{Path: curM.Path, Version: "none"}} + }) + } +} - default: - // The argument is a package or module path. - if modload.HasModRoot() { - if m := modload.TargetPackages(path); len(m.Pkgs) != 0 { - // The path is in the main module. Nothing to query. - if vers != "upgrade" && vers != "patch" { - base.Errorf("go get %s: can't request explicit version of path in main module", arg) - } - continue +func (r *resolver) performLocalQueries(ctx context.Context) { + for _, q := range r.localQueries { + q.pathOnce(q.pattern, func() pathSet { + absDetail := "" + if !filepath.IsAbs(q.pattern) { + if absPath, err := filepath.Abs(q.pattern); err == nil { + absDetail = fmt.Sprintf(" (%s)", absPath) } } - first := path - if i := strings.IndexByte(first, '/'); i >= 0 { - first = path + // Absolute paths like C:\foo and relative paths like ../foo... are + // restricted to matching packages in the main module. + pkgPattern := modload.DirImportPath(q.pattern) + if pkgPattern == "." { + return errSet(fmt.Errorf("%s%s is not within module rooted at %s", q.pattern, absDetail, modload.ModRoot())) } - if !strings.Contains(first, ".") { - // The path doesn't have a dot in the first component and cannot be - // queried as a module. It may be a package in the standard library, - // which is fine, so don't report an error unless we encounter - // a problem loading packages below. - continue + + match := modload.MatchInModule(ctx, pkgPattern, modload.Target, imports.AnyTags()) + if len(match.Errs) > 0 { + return pathSet{err: match.Errs[0]} + } + + if len(match.Pkgs) == 0 { + if q.raw == "" || q.raw == "." { + return errSet(fmt.Errorf("no package in current directory")) + } + if !q.isWildcard() { + return errSet(fmt.Errorf("%s%s is not a package in module rooted at %s", q.pattern, absDetail, modload.ModRoot())) + } + search.WarnUnmatched([]*search.Match{match}) + return pathSet{} } - // If we're querying "upgrade" or "patch", we need to know the current - // version of the module. For "upgrade", we want to avoid accidentally - // downgrading from a newer prerelease. For "patch", we need to query - // the correct minor version. - // Here, we check if "path" is the name of a module in the build list - // (other than the main module) and set prevM if so. If "path" isn't - // a module in the build list, the current version doesn't matter - // since it's either an unknown module or a package within a module - // that we'll discover later. - q := &query{querySpec: querySpec{path: path, vers: vers}, arg: arg} - if v, ok := versionByPath[path]; ok && path != modload.Target.Path { - q.prevM = module.Version{Path: path, Version: v} - q.forceModulePath = true + return pathSet{pkgMods: []module.Version{modload.Target}} + }) + } +} + +// performWildcardQueries populates the candidates for each query whose pattern +// is a wildcard. +// +// The candidates for a given module path matching (or containing a package +// matching) a wildcard query depend only on the initial build list, but the set +// of modules may be expanded by other queries, so wildcard queries need to be +// re-evaluated whenever a potentially-matching module path is added to the +// build list. +func (r *resolver) performWildcardQueries(ctx context.Context) { + for _, q := range r.wildcardQueries { + q := q + r.work.Add(func() { + if q.version == "none" { + r.queryNone(ctx, q) + } else { + r.queryWildcard(ctx, q) } - queries = append(queries, q) + }) + } + <-r.work.Idle() +} + +// queryWildcard adds a candidate set to q for each module for which: +// - some version of the module is already in the build list, and +// - that module exists at some version matching q.version, and +// - either the module path itself matches q.pattern, or some package within +// the module at q.version matches q.pattern. +func (r *resolver) queryWildcard(ctx context.Context, q *query) { + // For wildcard patterns, modload.QueryPattern only identifies modules + // matching the prefix of the path before the wildcard. However, the build + // list may already contain other modules with matching packages, and we + // should consider those modules to satisfy the query too. + // We want to match any packages in existing dependencies, but we only want to + // resolve new dependencies if nothing else turns up. + for _, curM := range r.buildList { + if !q.canMatchInModule(curM.Path) { + continue } + q.pathOnce(curM.Path, func() pathSet { + if _, hit := r.noneForPath(curM.Path); hit { + // This module is being removed, so it will no longer be in the build list + // (and thus will no longer match the pattern). + return pathSet{} + } + + if curM.Path == modload.Target.Path && !versionOkForMainModule(q.version) { + if q.matchesPath(curM.Path) { + return errSet(&modload.QueryMatchesMainModuleError{ + Pattern: q.pattern, + Query: q.version, + }) + } + + packages, err := r.matchInModule(ctx, q.pattern, curM) + if err != nil { + return errSet(err) + } + if len(packages) > 0 { + return errSet(&modload.QueryMatchesPackagesInMainModuleError{ + Pattern: q.pattern, + Query: q.version, + Packages: packages, + }) + } + + return r.tryWildcard(ctx, q, curM) + } + + m, err := r.queryModule(ctx, curM.Path, q.version, r.initialSelected) + if err != nil { + if !isNoSuchModuleVersion(err) { + // We can't tell whether a matching version exists. + return errSet(err) + } + // There is no version of curM.Path matching the query. + + // We haven't checked whether curM contains any matching packages at its + // currently-selected version, or whether curM.Path itself matches q. If + // either of those conditions holds, *and* no other query changes the + // selected version of curM, then we will fail in checkWildcardVersions. + // (This could be an error, but it's too soon to tell.) + // + // However, even then the transitive requirements of some other query + // may downgrade this module out of the build list entirely, in which + // case the pattern will no longer include it and it won't be an error. + // + // Either way, punt on the query rather than erroring out just yet. + return pathSet{} + } + + return r.tryWildcard(ctx, q, m) + }) } - base.ExitIfErrors() - // Query modules referenced by command line arguments at requested versions. - // We need to do this before loading packages since patterns that refer to - // packages in unknown modules can't be expanded. This also avoids looking - // up new modules while loading packages, only to downgrade later. - queryCache := make(map[querySpec]*query) - byPath := runQueries(queryCache, queries, nil) + // Even if no modules matched, we shouldn't query for a new module to provide + // the pattern yet: some other query may yet induce a new requirement that + // will match the wildcard. Instead, we'll check in findMissingWildcards. +} - // Add missing modules to the build list. - // We call SetBuildList here and elsewhere, since newUpgrader, - // ImportPathsQuiet, and other functions read the global build list. - for _, q := range queries { - if _, ok := versionByPath[q.m.Path]; !ok && q.m.Version != "none" { - buildList = append(buildList, q.m) - } +// tryWildcard returns a pathSet for module m matching query q. +// If m does not actually match q, tryWildcard returns an empty pathSet. +func (r *resolver) tryWildcard(ctx context.Context, q *query, m module.Version) pathSet { + mMatches := q.matchesPath(m.Path) + packages, err := r.matchInModule(ctx, q.pattern, m) + if err != nil { + return errSet(err) } - versionByPath = nil // out of date now; rebuilt later when needed - modload.SetBuildList(buildList) - - // Upgrade modules specifically named on the command line. This is our only - // chance to upgrade modules without root packages (modOnly below). - // This also skips loading packages at an old version, only to upgrade - // and reload at a new version. - upgrade := make(map[string]*query) - for path, q := range byPath { - if q.path == q.m.Path && q.m.Version != "none" { - upgrade[path] = q - } + if len(packages) > 0 { + return pathSet{pkgMods: []module.Version{m}} } - buildList, err := mvs.UpgradeAll(modload.Target, newUpgrader(upgrade, nil)) - if err != nil { - base.Fatalf("go get: %v", err) + if mMatches { + return pathSet{mod: m} } - modload.SetBuildList(buildList) - base.ExitIfErrors() - prevBuildList := buildList - - // Build a set of module paths that we don't plan to load packages from. - // This includes explicitly requested modules that don't have a root package - // and modules with a target version of "none". - var wg sync.WaitGroup - var modOnlyMu sync.Mutex - modOnly := make(map[string]*query) - for _, q := range queries { - if q.m.Version == "none" { - modOnlyMu.Lock() - modOnly[q.m.Path] = q - modOnlyMu.Unlock() - continue + return pathSet{} +} + +// findMissingWildcards adds a candidate set for each query in r.wildcardQueries +// that has not yet resolved to any version containing packages. +func (r *resolver) findMissingWildcards(ctx context.Context) { + for _, q := range r.wildcardQueries { + if q.version == "none" || q.matchesPackages { + continue // q is not “missing” } - if q.path == q.m.Path { - wg.Add(1) - go func(q *query) { - if hasPkg, err := modload.ModuleHasRootPackage(q.m); err != nil { - base.Errorf("go get: %v", err) - } else if !hasPkg { - modOnlyMu.Lock() - modOnly[q.m.Path] = q - modOnlyMu.Unlock() + r.work.Add(func() { + q.pathOnce(q.pattern, func() pathSet { + pkgMods, mod, err := r.queryPattern(ctx, q.pattern, q.version, r.initialSelected) + if err != nil { + if isNoSuchPackageVersion(err) && len(q.resolved) > 0 { + // q already resolved one or more modules but matches no packages. + // That's ok: this pattern is just a module pattern, and we don't + // need to add any more modules to satisfy it. + return pathSet{} + } + return errSet(err) } - wg.Done() - }(q) - } - } - wg.Wait() - base.ExitIfErrors() - // Build a list of arguments that may refer to packages. - var pkgPatterns []string - var pkgGets []getArg - for _, arg := range gets { - if modOnly[arg.path] == nil && arg.vers != "none" { - pkgPatterns = append(pkgPatterns, arg.path) - pkgGets = append(pkgGets, arg) - } + return pathSet{pkgMods: pkgMods, mod: mod} + }) + }) } + <-r.work.Idle() +} - // Load packages and upgrade the modules that provide them. We do this until - // we reach a fixed point, since modules providing packages may change as we - // change versions. This must terminate because the module graph is finite, - // and the load and upgrade operations may only add and upgrade modules - // in the build list. - var matches []*search.Match - for { - var seenPkgs map[string]bool - seenQuery := make(map[querySpec]bool) - var queries []*query - addQuery := func(q *query) { - if !seenQuery[q.querySpec] { - seenQuery[q.querySpec] = true - queries = append(queries, q) - } - } +// checkWildcardVersions reports an error if any module in the build list has a +// path (or contains a package) matching a query with a wildcard pattern, but +// has a selected version that does *not* match the query. +func (r *resolver) checkWildcardVersions(ctx context.Context) { + defer base.ExitIfErrors() - if len(pkgPatterns) > 0 { - // Don't load packages if pkgPatterns is empty. Both - // modload.ImportPathsQuiet and ModulePackages convert an empty list - // of patterns to []string{"."}, which is not what we want. - matches = modload.ImportPathsQuiet(pkgPatterns, imports.AnyTags()) - seenPkgs = make(map[string]bool) - for i, match := range matches { - arg := pkgGets[i] - - if len(match.Pkgs) == 0 { - // If the pattern did not match any packages, look up a new module. - // If the pattern doesn't match anything on the last iteration, - // we'll print a warning after the outer loop. - if !match.IsLocal() && !match.IsLiteral() && arg.path != "all" { - addQuery(&query{querySpec: querySpec{path: arg.path, vers: arg.vers}, arg: arg.raw}) - } else { - for _, err := range match.Errs { - base.Errorf("go get: %v", err) - } + for _, q := range r.wildcardQueries { + for _, curM := range r.buildList { + if !q.canMatchInModule(curM.Path) { + continue + } + if !q.matchesPath(curM.Path) { + packages, err := r.matchInModule(ctx, q.pattern, curM) + if len(packages) == 0 { + if err != nil { + reportError(q, err) } - continue + continue // curM is not relevant to q. } + } - allStd := true - for _, pkg := range match.Pkgs { - if !seenPkgs[pkg] { - seenPkgs[pkg] = true - if _, _, err := modload.Lookup("", false, pkg); err != nil { - allStd = false - base.Errorf("go get %s: %v", arg.raw, err) - continue - } - } - m := modload.PackageModule(pkg) - if m.Path == "" { - // pkg is in the standard library. - continue - } - allStd = false - if m.Path == modload.Target.Path { - // pkg is in the main module. - continue - } - addQuery(&query{querySpec: querySpec{path: m.Path, vers: arg.vers, forceModulePath: true, prevM: m}, arg: arg.raw}) + rev, err := r.queryModule(ctx, curM.Path, q.version, r.initialSelected) + if err != nil { + reportError(q, err) + continue + } + if rev.Version == curM.Version { + continue // curM already matches q. + } + + if !q.matchesPath(curM.Path) { + m := module.Version{Path: curM.Path, Version: rev.Version} + packages, err := r.matchInModule(ctx, q.pattern, m) + if err != nil { + reportError(q, err) + continue } - if allStd && arg.path != arg.raw { - base.Errorf("go get %s: cannot use pattern %q with explicit version", arg.raw, arg.raw) + if len(packages) == 0 { + // curM at its original version contains a path matching q.pattern, + // but at rev.Version it does not, so (somewhat paradoxically) if + // we changed the version of curM it would no longer match the query. + var version interface{} = m + if rev.Version != q.version { + version = fmt.Sprintf("%s@%s (%s)", m.Path, q.version, m.Version) + } + reportError(q, fmt.Errorf("%v matches packages in %v but not %v: specify a different version for module %s", q, curM, version, m.Path)) + continue } } + + // Since queryModule succeeded and either curM or one of the packages it + // contains matches q.pattern, we should have either selected the version + // of curM matching q, or reported a conflict error (and exited). + // If we're still here and the version doesn't match, + // something has gone very wrong. + reportError(q, fmt.Errorf("internal error: selected %v instead of %v", curM, rev.Version)) } - base.ExitIfErrors() + } +} - // Query target versions for modules providing packages matched by - // command line arguments. - byPath = runQueries(queryCache, queries, modOnly) +// performPathQueries populates the candidates for each query whose pattern is +// a path literal. +// +// The candidate packages and modules for path literals depend only on the +// initial build list, not the current build list, so we only need to query path +// literals once. +func (r *resolver) performPathQueries(ctx context.Context) { + for _, q := range r.pathQueries { + q := q + r.work.Add(func() { + if q.version == "none" { + r.queryNone(ctx, q) + } else { + r.queryPath(ctx, q) + } + }) + } + <-r.work.Idle() +} - // Handle upgrades. This is needed for arguments that didn't match - // modules or matched different modules from a previous iteration. It - // also upgrades modules providing package dependencies if -u is set. - buildList, err := mvs.UpgradeAll(modload.Target, newUpgrader(byPath, seenPkgs)) - if err != nil { - base.Fatalf("go get: %v", err) +// queryPath adds a candidate set to q for the package with path q.pattern. +// The candidate set consists of all modules that could provide q.pattern +// and have a version matching q, plus (if it exists) the module whose path +// is itself q.pattern (at a matching version). +func (r *resolver) queryPath(ctx context.Context, q *query) { + q.pathOnce(q.pattern, func() pathSet { + if search.IsMetaPackage(q.pattern) || q.isWildcard() { + panic(fmt.Sprintf("internal error: queryPath called with pattern %q", q.pattern)) } - modload.SetBuildList(buildList) - base.ExitIfErrors() + if q.version == "none" { + panic(`internal error: queryPath called with version "none"`) + } + + if search.IsStandardImportPath(q.pattern) { + stdOnly := module.Version{} + packages, _ := r.matchInModule(ctx, q.pattern, stdOnly) + if len(packages) > 0 { + if q.rawVersion != "" { + return errSet(fmt.Errorf("can't request explicit version %q of standard library package %s", q.version, q.pattern)) + } - // Stop if no changes have been made to the build list. - buildList = modload.BuildList() - eq := len(buildList) == len(prevBuildList) - for i := 0; eq && i < len(buildList); i++ { - eq = buildList[i] == prevBuildList[i] + q.matchesPackages = true + return pathSet{} // No module needed for standard library. + } } - if eq { - break + + pkgMods, mod, err := r.queryPattern(ctx, q.pattern, q.version, r.initialSelected) + if err != nil { + return errSet(err) } - prevBuildList = buildList + return pathSet{pkgMods: pkgMods, mod: mod} + }) +} + +// performPatternAllQueries populates the candidates for each query whose +// pattern is "all". +// +// The candidate modules for a given package in "all" depend only on the initial +// build list, but we cannot follow the dependencies of a given package until we +// know which candidate is selected — and that selection may depend on the +// results of other queries. We need to re-evaluate the "all" queries whenever +// the module for one or more packages in "all" are resolved. +func (r *resolver) performPatternAllQueries(ctx context.Context) { + if len(r.patternAllQueries) == 0 { + return } - if !*getD { - // Only print warnings after the last iteration, - // and only if we aren't going to build. - search.WarnUnmatched(matches) + + findPackage := func(ctx context.Context, path string, m module.Version) (versionOk bool) { + versionOk = true + for _, q := range r.patternAllQueries { + q.pathOnce(path, func() pathSet { + pkgMods, err := r.queryPackages(ctx, path, q.version, r.initialSelected) + if len(pkgMods) != 1 || pkgMods[0] != m { + // There are candidates other than m for the given path, so we can't + // be certain that m will actually be the module selected to provide + // the package. Don't load its dependencies just yet, because they + // might no longer be dependencies after we resolve the correct + // version. + versionOk = false + } + return pathSet{pkgMods: pkgMods, err: err} + }) + } + return versionOk + } + + r.loadPackages(ctx, []string{"all"}, findPackage) + + // Since we built up the candidate lists concurrently, they may be in a + // nondeterministic order. We want 'go get' to be fully deterministic, + // including in which errors it chooses to report, so sort the candidates + // into a deterministic-but-arbitrary order. + for _, q := range r.patternAllQueries { + sort.Slice(q.candidates, func(i, j int) bool { + return q.candidates[i].path < q.candidates[j].path + }) } +} - // Handle downgrades. - var down []module.Version - for _, m := range modload.BuildList() { - q := byPath[m.Path] - if q != nil && semver.Compare(m.Version, q.m.Version) > 0 { - down = append(down, module.Version{Path: m.Path, Version: q.m.Version}) +// findAndUpgradeImports returns a pathSet for each package that is not yet +// in the build list but is transitively imported by the packages matching the +// given queries (which must already have been resolved). +// +// If the getU flag ("-u") is set, findAndUpgradeImports also returns a +// pathSet for each module that is not constrained by any other +// command-line argument and has an available matching upgrade. +func (r *resolver) findAndUpgradeImports(ctx context.Context, queries []*query) (upgrades []pathSet) { + patterns := make([]string, 0, len(queries)) + for _, q := range queries { + if q.matchesPackages { + patterns = append(patterns, q.pattern) } } - if len(down) > 0 { - buildList, err := mvs.Downgrade(modload.Target, modload.Reqs(), down...) + if len(patterns) == 0 { + return nil + } + + // mu guards concurrent writes to upgrades, which will be sorted + // (to restore determinism) after loading. + var mu sync.Mutex + + findPackage := func(ctx context.Context, path string, m module.Version) (versionOk bool) { + version := "latest" + if m.Path != "" { + if getU.version == "" { + // The user did not request that we upgrade transitive dependencies. + return true + } + if _, ok := r.resolvedVersion[m.Path]; ok { + // We cannot upgrade m implicitly because its version is determined by + // an explicit pattern argument. + return true + } + version = getU.version + } + + // Unlike other queries, the "-u" flag upgrades relative to the build list + // after applying changes so far, not the initial build list. + // This is for two reasons: + // + // - The "-u" flag intentionally applies to transitive dependencies, + // which may not be known or even resolved in advance of applying + // other version changes. + // + // - The "-u" flag, unlike other arguments, does not cause version + // conflicts with other queries. (The other query always wins.) + + pkgMods, err := r.queryPackages(ctx, path, version, r.selected) + for _, u := range pkgMods { + if u == m { + // The selected package version is already upgraded appropriately; there + // is no need to change it. + return true + } + } + if err != nil { - base.Fatalf("go: %v", err) + if isNoSuchPackageVersion(err) || (m.Path == "" && module.CheckPath(path) != nil) { + // We can't find the package because it doesn't — or can't — even exist + // in any module at the latest version. (Note that invalid module paths + // could in general exist due to replacements, so we at least need to + // run the query to check those.) + // + // There is no version change we can make to fix the package, so leave + // it unresolved. Either some other query (perhaps a wildcard matching a + // newly-added dependency for some other missing package) will fill in + // the gaps, or we will report an error (with a better import stack) in + // the final LoadPackages call. + return true + } } - modload.SetBuildList(buildList) - modload.ReloadBuildList() // note: does not update go.mod - base.ExitIfErrors() + + mu.Lock() + upgrades = append(upgrades, pathSet{path: path, pkgMods: pkgMods, err: err}) + mu.Unlock() + return false + } + + r.loadPackages(ctx, patterns, findPackage) + + // Since we built up the candidate lists concurrently, they may be in a + // nondeterministic order. We want 'go get' to be fully deterministic, + // including in which errors it chooses to report, so sort the candidates + // into a deterministic-but-arbitrary order. + sort.Slice(upgrades, func(i, j int) bool { + return upgrades[i].path < upgrades[j].path + }) + return upgrades +} + +// loadPackages loads the packages matching the given patterns, invoking the +// findPackage function for each package that may require a change to the +// build list. +// +// loadPackages invokes the findPackage function for each package loaded from a +// module outside the main module. If the module or version that supplies that +// package needs to be changed due to a query, findPackage may return false +// and the imports of that package will not be loaded. +// +// loadPackages also invokes the findPackage function for each imported package +// that is neither present in the standard library nor in any module in the +// build list. +func (r *resolver) loadPackages(ctx context.Context, patterns []string, findPackage func(ctx context.Context, path string, m module.Version) (versionOk bool)) { + opts := modload.PackageOpts{ + Tags: imports.AnyTags(), + LoadTests: *getT, + SilenceErrors: true, // May be fixed by subsequent upgrades or downgrades. } - // Scan for any upgrades lost by the downgrades. - var lostUpgrades []*query - if len(down) > 0 { - versionByPath = make(map[string]string) - for _, m := range modload.BuildList() { - versionByPath[m.Path] = m.Version + opts.AllowPackage = func(ctx context.Context, path string, m module.Version) error { + if m.Path == "" || m == modload.Target { + // Packages in the standard library and main module are already at their + // latest (and only) available versions. + return nil } - for _, q := range byPath { - if v, ok := versionByPath[q.m.Path]; q.m.Version != "none" && (!ok || semver.Compare(v, q.m.Version) != 0) { - lostUpgrades = append(lostUpgrades, q) - } + if ok := findPackage(ctx, path, m); !ok { + return errVersionChange } - sort.Slice(lostUpgrades, func(i, j int) bool { - return lostUpgrades[i].m.Path < lostUpgrades[j].m.Path - }) + return nil } - if len(lostUpgrades) > 0 { - desc := func(m module.Version) string { - s := m.Path + "@" + m.Version - t := byPath[m.Path] - if t != nil && t.arg != s { - s += " from " + t.arg - } - return s + + _, pkgs := modload.LoadPackages(ctx, opts, patterns...) + for _, path := range pkgs { + const ( + parentPath = "" + parentIsStd = false + ) + _, _, err := modload.Lookup(parentPath, parentIsStd, path) + if err == nil { + continue } - downByPath := make(map[string]module.Version) - for _, d := range down { - downByPath[d.Path] = d + if errors.Is(err, errVersionChange) { + // We already added candidates during loading. + continue } - var buf strings.Builder - fmt.Fprintf(&buf, "go get: inconsistent versions:") - reqs := modload.Reqs() - for _, q := range lostUpgrades { - // We lost q because its build list requires a newer version of something in down. - // Figure out exactly what. - // Repeatedly constructing the build list is inefficient - // if there are MANY command-line arguments, - // but at least all the necessary requirement lists are cached at this point. - list, err := buildListForLostUpgrade(q.m, reqs) - if err != nil { - base.Fatalf("go: %v", err) - } + var ( + importMissing *modload.ImportMissingError + ambiguous *modload.AmbiguousImportError + ) + if !errors.As(err, &importMissing) && !errors.As(err, &ambiguous) { + // The package, which is a dependency of something we care about, has some + // problem that we can't resolve with a version change. + // Leave the error for the final LoadPackages call. + continue + } + + path := path + r.work.Add(func() { + findPackage(ctx, path, module.Version{}) + }) + } + <-r.work.Idle() +} - fmt.Fprintf(&buf, "\n\t%s", desc(q.m)) - sep := " requires" - for _, m := range list { - if down, ok := downByPath[m.Path]; ok && semver.Compare(down.Version, m.Version) < 0 { - fmt.Fprintf(&buf, "%s %s@%s (not %s)", sep, m.Path, m.Version, desc(down)) - sep = "," +// errVersionChange is a sentinel error indicating that a module's version needs +// to be updated before its dependencies can be loaded. +var errVersionChange = errors.New("version change needed") + +// resolveCandidates resolves candidates sets that are attached to the given +// queries and/or needed to provide the given missing-package dependencies. +// +// resolveCandidates starts by resolving one module version from each +// unambiguous pathSet attached to the given queries. +// +// If no unambiguous query results in a change to the build list, +// resolveCandidates modifies the build list by adding one module version from +// each pathSet in missing, but does not mark those versions as resolved +// (so they can still be modified by other queries). +// +// If that still does not result in any changes to the build list, +// resolveCandidates revisits the ambiguous query candidates and resolves them +// arbitrarily in order to guarantee forward progress. +// +// If all pathSets are resolved without any changes to the build list, +// resolveCandidates returns with changed=false. +func (r *resolver) resolveCandidates(ctx context.Context, queries []*query, upgrades []pathSet) (changed bool) { + defer base.ExitIfErrors() + + // Note: this is O(N²) with the number of pathSets in the worst case. + // + // We could perhaps get it down to O(N) if we were to index the pathSets + // by module path, so that we only revisit a given pathSet when the + // version of some module in its containingPackage list has been determined. + // + // However, N tends to be small, and most candidate sets will include only one + // candidate module (so they will be resolved in the first iteration), so for + // now we'll stick to the simple O(N²) approach. + + resolved := 0 + for { + prevResolved := resolved + + for _, q := range queries { + unresolved := q.candidates[:0] + + for _, cs := range q.candidates { + if cs.err != nil { + reportError(q, cs.err) + resolved++ + continue } - } - if sep != "," { - // We have no idea why this happened. - // At least report the problem. - if v := versionByPath[q.m.Path]; v == "" { - fmt.Fprintf(&buf, " removed unexpectedly") - } else { - fmt.Fprintf(&buf, " ended up at %s unexpectedly", v) + + filtered, isPackage, m, unique := r.disambiguate(cs) + if !unique { + unresolved = append(unresolved, filtered) + continue } - fmt.Fprintf(&buf, " (please report at golang.org/issue/new)") + + if m.Path == "" { + // The query is not viable. Choose an arbitrary candidate from + // before filtering and “resolve” it to report a conflict. + isPackage, m = r.chooseArbitrarily(cs) + } + if isPackage { + q.matchesPackages = true + } + r.resolve(q, m) + resolved++ } + + q.candidates = unresolved } - base.Fatalf("%v", buf.String()) - } - // Everything succeeded. Update go.mod. - modload.AllowWriteGoMod() - modload.WriteGoMod() + base.ExitIfErrors() + if resolved == prevResolved { + break // No unambiguous candidate remains. + } + } - // If -d was specified, we're done after the module work. - // We've already downloaded modules by loading packages above. - // Otherwise, we need to build and install the packages matched by - // command line arguments. This may be a different set of packages, - // since we only build packages for the target platform. - // Note that 'go get -u' without arguments is equivalent to - // 'go get -u .', so we'll typically build the package in the current - // directory. - if *getD || len(pkgPatterns) == 0 { - return + if changed := r.updateBuildList(ctx, nil); changed { + // The build list has changed, so disregard any missing packages: they might + // now be determined by requirements in the build list, which we would + // prefer to use instead of arbitrary "latest" versions. + return true } - work.BuildInit() - pkgs := load.PackagesForBuild(pkgPatterns) - work.InstallPackages(pkgPatterns, pkgs) -} -// runQueries looks up modules at target versions in parallel. Results will be -// cached. If the same module is referenced by multiple queries at different -// versions (including earlier queries in the modOnly map), an error will be -// reported. A map from module paths to queries is returned, which includes -// queries and modOnly. -func runQueries(cache map[querySpec]*query, queries []*query, modOnly map[string]*query) map[string]*query { - var lookup par.Work - for _, q := range queries { - if cached := cache[q.querySpec]; cached != nil { - *q = *cached - } else { - cache[q.querySpec] = q - lookup.Add(q) + // Arbitrarily add a "latest" version that provides each missing package, but + // do not mark the version as resolved: we still want to allow the explicit + // queries to modify the resulting versions. + var tentative []module.Version + for _, cs := range upgrades { + if cs.err != nil { + base.Errorf("go get: %v", cs.err) + continue } - } - lookup.Do(10, func(item interface{}) { - q := item.(*query) - if q.vers == "none" { - // Wait for downgrade step. - q.m = module.Version{Path: q.path, Version: "none"} - return + filtered, _, m, unique := r.disambiguate(cs) + if !unique { + _, m = r.chooseArbitrarily(filtered) } - m, err := getQuery(q.path, q.vers, q.prevM, q.forceModulePath) - if err != nil { - base.Errorf("go get %s: %v", q.arg, err) + if m.Path == "" { + // There is no viable candidate for the missing package. + // Leave it unresolved. + continue } - q.m = m - }) + tentative = append(tentative, m) + } base.ExitIfErrors() - - byPath := make(map[string]*query) - check := func(q *query) { - if prev, ok := byPath[q.m.Path]; prev != nil && prev.m != q.m { - base.Errorf("go get: conflicting versions for module %s: %s and %s", q.m.Path, prev.m.Version, q.m.Version) - byPath[q.m.Path] = nil // sentinel to stop errors - return - } else if !ok { - byPath[q.m.Path] = q - } + if changed := r.updateBuildList(ctx, tentative); changed { + return true } + + // The build list will be the same on the next iteration as it was on this + // iteration, so any ambiguous queries will remain so. In order to make + // progress, resolve them arbitrarily but deterministically. + // + // If that results in conflicting versions, the user can re-run 'go get' + // with additional explicit versions for the conflicting packages or + // modules. for _, q := range queries { - check(q) + for _, cs := range q.candidates { + isPackage, m := r.chooseArbitrarily(cs) + if isPackage { + q.matchesPackages = true + } + r.resolve(q, m) + } } - for _, q := range modOnly { - check(q) + return r.updateBuildList(ctx, nil) +} + +// disambiguate eliminates candidates from cs that conflict with other module +// versions that have already been resolved. If there is only one (unique) +// remaining candidate, disambiguate returns that candidate, along with +// an indication of whether that result interprets cs.path as a package +// +// Note: we're only doing very simple disambiguation here. The goal is to +// reproduce the user's intent, not to find a solution that a human couldn't. +// In the vast majority of cases, we expect only one module per pathSet, +// but we want to give some minimal additional tools so that users can add an +// extra argument or two on the command line to resolve simple ambiguities. +func (r *resolver) disambiguate(cs pathSet) (filtered pathSet, isPackage bool, m module.Version, unique bool) { + if len(cs.pkgMods) == 0 && cs.mod.Path == "" { + panic("internal error: resolveIfUnambiguous called with empty pathSet") } - base.ExitIfErrors() - return byPath -} + for _, m := range cs.pkgMods { + if _, ok := r.noneForPath(m.Path); ok { + // A query with version "none" forces the candidate module to version + // "none", so we cannot use any other version for that module. + continue + } -// getQuery evaluates the given (package or module) path and version -// to determine the underlying module version being requested. -// If forceModulePath is set, getQuery must interpret path -// as a module path. -func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) { - if (prevM.Version != "") != forceModulePath { - // We resolve package patterns by calling QueryPattern, which does not - // accept a previous version and therefore cannot take it into account for - // the "latest" or "patch" queries. - // If we are resolving a package path or pattern, the caller has already - // resolved any existing packages to their containing module(s), and - // will set both prevM.Version and forceModulePath for those modules. - // The only remaining package patterns are those that are not already - // provided by the build list, which are indicated by - // an empty prevM.Version. - base.Fatalf("go get: internal error: prevM may be set if and only if forceModulePath is set") - } - - // If the query must be a module path, try only that module path. - if forceModulePath { - if path == modload.Target.Path { - if vers != "latest" { - return module.Version{}, fmt.Errorf("can't get a specific version of the main module") + if m.Path == modload.Target.Path { + if m.Version == modload.Target.Version { + return pathSet{}, true, m, true } + // The main module can only be set to its own version. + continue } - info, err := modload.Query(path, vers, prevM.Version, modload.Allowed) - if err == nil { - if info.Version != vers && info.Version != prevM.Version { - logOncef("go: %s %s => %s", path, vers, info.Version) - } - return module.Version{Path: path, Version: info.Version}, nil + vr, ok := r.resolvedVersion[m.Path] + if !ok { + // m is a viable answer to the query, but other answers may also + // still be viable. + filtered.pkgMods = append(filtered.pkgMods, m) + continue } - // If the query was "upgrade" or "patch" and the current version has been - // replaced, check to see whether the error was for that same version: - // if so, the version was probably replaced because it is invalid, - // and we should keep that replacement without complaining. - if vers == "upgrade" || vers == "patch" { - var vErr *module.InvalidVersionError - if errors.As(err, &vErr) && vErr.Version == prevM.Version && modload.Replacement(prevM).Path != "" { - return prevM, nil - } + if vr.version != m.Version { + // Some query forces the candidate module to a version other than this + // one. + // + // The command could be something like + // + // go get example.com/foo/bar@none example.com/foo/bar/baz@latest + // + // in which case we *cannot* resolve the package from + // example.com/foo/bar (because it is constrained to version + // "none") and must fall through to module example.com/foo@latest. + continue } - return module.Version{}, err + // Some query forces the candidate module *to* the candidate version. + // As a result, this candidate is the only viable choice to provide + // its package(s): any other choice would result in an ambiguous import + // for this path. + // + // For example, consider the command + // + // go get example.com/foo@latest example.com/foo/bar/baz@latest + // + // If modules example.com/foo and example.com/foo/bar both provide + // package example.com/foo/bar/baz, then we *must* resolve the package + // from example.com/foo: if we instead resolved it from + // example.com/foo/bar, we would have two copies of the package. + return pathSet{}, true, m, true } - // If the query may be either a package or a module, try it as a package path. - // If it turns out to only exist as a module, we can detect the resulting - // PackageNotInModuleError and avoid a second round-trip through (potentially) - // all of the configured proxies. - results, err := modload.QueryPattern(path, vers, modload.Allowed) - if err != nil { - // If the path doesn't contain a wildcard, check whether it was actually a - // module path instead. If so, return that. - if !strings.Contains(path, "...") { - var modErr *modload.PackageNotInModuleError - if errors.As(err, &modErr) && modErr.Mod.Path == path { - if modErr.Mod.Version != vers { - logOncef("go: %s %s => %s", path, vers, modErr.Mod.Version) - } - return modErr.Mod, nil - } + if cs.mod.Path != "" { + vr, ok := r.resolvedVersion[cs.mod.Path] + if !ok || vr.version == cs.mod.Version { + filtered.mod = cs.mod } + } - return module.Version{}, err + if len(filtered.pkgMods) == 1 && + (filtered.mod.Path == "" || filtered.mod == filtered.pkgMods[0]) { + // Exactly one viable module contains the package with the given path + // (by far the common case), so we can resolve it unambiguously. + return pathSet{}, true, filtered.pkgMods[0], true } - m := results[0].Mod - if m.Path != path { - logOncef("go: found %s in %s %s", path, m.Path, m.Version) - } else if m.Version != vers { - logOncef("go: %s %s => %s", path, vers, m.Version) + if len(filtered.pkgMods) == 0 { + // All modules that could provide the path as a package conflict with other + // resolved arguments. If it can refer to a module instead, return that; + // otherwise, this pathSet cannot be resolved (and we will return the + // zero module.Version). + return pathSet{}, false, filtered.mod, true } - return m, nil + + // The query remains ambiguous: there are at least two different modules + // to which cs.path could refer. + return filtered, false, module.Version{}, false } -// An upgrader adapts an underlying mvs.Reqs to apply an -// upgrade policy to a list of targets and their dependencies. -type upgrader struct { - mvs.Reqs +// chooseArbitrarily returns an arbitrary (but deterministic) module version +// from among those in the given set. +// +// chooseArbitrarily prefers module paths that were already in the build list at +// the start of 'go get', prefers modules that provide packages over those that +// do not, and chooses the first module meeting those criteria (so biases toward +// longer paths). +func (r *resolver) chooseArbitrarily(cs pathSet) (isPackage bool, m module.Version) { + // Prefer to upgrade some module that was already in the build list. + for _, m := range cs.pkgMods { + if r.initialSelected(m.Path) != "none" { + return true, m + } + } - // cmdline maps a module path to a query made for that module at a - // specific target version. Each query corresponds to a module - // matched by a command line argument. - cmdline map[string]*query + // Otherwise, arbitrarily choose the first module that provides the package. + if len(cs.pkgMods) > 0 { + return true, cs.pkgMods[0] + } - // upgrade is a set of modules providing dependencies of packages - // matched by command line arguments. If -u or -u=patch is set, - // these modules are upgraded accordingly. - upgrade map[string]bool + return false, cs.mod } -// newUpgrader creates an upgrader. cmdline contains queries made at -// specific versions for modules matched by command line arguments. pkgs -// is the set of packages matched by command line arguments. If -u or -u=patch -// is set, modules providing dependencies of pkgs are upgraded accordingly. -func newUpgrader(cmdline map[string]*query, pkgs map[string]bool) *upgrader { - u := &upgrader{ - Reqs: modload.Reqs(), - cmdline: cmdline, - } - if getU != "" { - u.upgrade = make(map[string]bool) - - // Traverse package import graph. - // Initialize work queue with root packages. - seen := make(map[string]bool) - var work []string - add := func(path string) { - if !seen[path] { - seen[path] = true - work = append(work, path) - } +// checkPackagesAndRetractions reloads packages for the given patterns and +// reports missing and ambiguous package errors. It also reports loads and +// reports retractions for resolved modules and modules needed to build +// named packages. +// +// We skip missing-package errors earlier in the process, since we want to +// resolve pathSets ourselves, but at that point, we don't have enough context +// to log the package-import chains leading to each error. +func (r *resolver) checkPackagesAndRetractions(ctx context.Context, pkgPatterns []string) { + defer base.ExitIfErrors() + + // Build a list of modules to load retractions for. Start with versions + // selected based on command line queries. + // + // This is a subset of the build list. If the main module has a lot of + // dependencies, loading retractions for the entire build list would be slow. + relevantMods := make(map[module.Version]struct{}) + for path, reason := range r.resolvedVersion { + relevantMods[module.Version{Path: path, Version: reason.version}] = struct{}{} + } + + // Reload packages, reporting errors for missing and ambiguous imports. + if len(pkgPatterns) > 0 { + // LoadPackages will print errors (since it has more context) but will not + // exit, since we need to load retractions later. + pkgOpts := modload.PackageOpts{ + LoadTests: *getT, + ResolveMissingImports: false, + AllowErrors: true, } - for pkg := range pkgs { - add(pkg) + matches, pkgs := modload.LoadPackages(ctx, pkgOpts, pkgPatterns...) + for _, m := range matches { + if len(m.Errs) > 0 { + base.SetExitStatus(1) + break + } } - for len(work) > 0 { - pkg := work[0] - work = work[1:] - m := modload.PackageModule(pkg) - u.upgrade[m.Path] = true - - // testImports is empty unless test imports were actually loaded, - // i.e., -t was set or "all" was one of the arguments. - imports, testImports := modload.PackageImports(pkg) - for _, imp := range imports { - add(imp) + for _, pkg := range pkgs { + if _, _, err := modload.Lookup("", false, pkg); err != nil { + base.SetExitStatus(1) + if ambiguousErr := (*modload.AmbiguousImportError)(nil); errors.As(err, &ambiguousErr) { + for _, m := range ambiguousErr.Modules { + relevantMods[m] = struct{}{} + } + } } - for _, imp := range testImports { - add(imp) + if m := modload.PackageModule(pkg); m.Path != "" { + relevantMods[m] = struct{}{} } } } - return u -} -// Required returns the requirement list for m. -// For the main module, we override requirements with the modules named -// one the command line, and we include new requirements. Otherwise, -// we defer to u.Reqs. -func (u *upgrader) Required(m module.Version) ([]module.Version, error) { - rs, err := u.Reqs.Required(m) - if err != nil { - return nil, err + // Load and report retractions. + type retraction struct { + m module.Version + err error } - if m != modload.Target { - return rs, nil + retractions := make([]retraction, 0, len(relevantMods)) + for m := range relevantMods { + retractions = append(retractions, retraction{m: m}) } - - overridden := make(map[string]bool) - for i, m := range rs { - if q := u.cmdline[m.Path]; q != nil && q.m.Version != "none" { - rs[i] = q.m - overridden[q.m.Path] = true - } + sort.Slice(retractions, func(i, j int) bool { + return retractions[i].m.Path < retractions[j].m.Path + }) + for i := 0; i < len(retractions); i++ { + i := i + r.work.Add(func() { + err := modload.CheckRetractions(ctx, retractions[i].m) + if retractErr := (*modload.ModuleRetractedError)(nil); errors.As(err, &retractErr) { + retractions[i].err = err + } + }) } - for _, q := range u.cmdline { - if !overridden[q.m.Path] && q.m.Path != modload.Target.Path && q.m.Version != "none" { - rs = append(rs, q.m) + <-r.work.Idle() + var retractPath string + for _, r := range retractions { + if r.err != nil { + fmt.Fprintf(os.Stderr, "go: warning: %v\n", r.err) + if retractPath == "" { + retractPath = r.m.Path + } else { + retractPath = "<module>" + } } } - return rs, nil + if retractPath != "" { + fmt.Fprintf(os.Stderr, "go: run 'go get %s@latest' to switch to the latest unretracted version\n", retractPath) + } } -// Upgrade returns the desired upgrade for m. -// -// If m was requested at a specific version on the command line, then -// Upgrade returns that version. +// reportChanges logs version changes to os.Stderr. // -// If -u is set and m provides a dependency of a package matched by -// command line arguments, then Upgrade may provider a newer tagged version. -// If m is a tagged version, then Upgrade will return the latest tagged -// version (with the same minor version number if -u=patch). -// If m is a pseudo-version, then Upgrade returns the latest tagged version -// only if that version has a time-stamp newer than m. This special case -// prevents accidental downgrades when already using a pseudo-version -// newer than the latest tagged version. +// reportChanges only logs changes to modules named on the command line and to +// explicitly required modules in go.mod. Most changes to indirect requirements +// are not relevant to the user and are not logged. // -// If none of the above cases apply, then Upgrade returns m. -func (u *upgrader) Upgrade(m module.Version) (module.Version, error) { - // Allow pkg@vers on the command line to override the upgrade choice v. - // If q's version is < m.Version, then we're going to downgrade anyway, - // and it's cleaner to avoid moving back and forth and picking up - // extraneous other newer dependencies. - // If q's version is > m.Version, then we're going to upgrade past - // m.Version anyway, and again it's cleaner to avoid moving back and forth - // picking up extraneous other newer dependencies. - if q := u.cmdline[m.Path]; q != nil { - return q.m, nil - } - - if !u.upgrade[m.Path] { - // Not involved in upgrade. Leave alone. - return m, nil - } - - // Run query required by upgrade semantics. - // Note that Query "latest" is not the same as using repo.Latest, - // which may return a pseudoversion for the latest commit. - // Query "latest" returns the newest tagged version or the newest - // prerelease version if there are no non-prereleases, or repo.Latest - // if there aren't any tagged versions. - // If we're querying "upgrade" or "patch", Query will compare the current - // version against the chosen version and will return the current version - // if it is newer. - info, err := modload.Query(m.Path, string(getU), m.Version, modload.Allowed) - if err != nil { - // Report error but return m, to let version selection continue. - // (Reporting the error will fail the command at the next base.ExitIfErrors.) - - // Special case: if the error is for m.Version itself and m.Version has a - // replacement, then keep it and don't report the error: the fact that the - // version is invalid is likely the reason it was replaced to begin with. - var vErr *module.InvalidVersionError - if errors.As(err, &vErr) && vErr.Version == m.Version && modload.Replacement(m).Path != "" { - return m, nil +// reportChanges should be called after WriteGoMod. +func (r *resolver) reportChanges(oldReqs, newReqs []module.Version) { + type change struct { + path, old, new string + } + changes := make(map[string]change) + + // Collect changes in modules matched by command line arguments. + for path, reason := range r.resolvedVersion { + old := r.initialVersion[path] + new := reason.version + if old != new && (old != "" || new != "none") { + changes[path] = change{path, old, new} } + } - // Special case: if the error is "no matching versions" then don't - // even report the error. Because Query does not consider pseudo-versions, - // it may happen that we have a pseudo-version but during -u=patch - // the query v0.0 matches no versions (not even the one we're using). - var noMatch *modload.NoMatchingVersionError - if !errors.As(err, &noMatch) { - base.Errorf("go get: upgrading %s@%s: %v", m.Path, m.Version, err) + // Collect changes to explicit requirements in go.mod. + for _, req := range oldReqs { + path := req.Path + old := req.Version + new := r.buildListVersion[path] + if old != new { + changes[path] = change{path, old, new} + } + } + for _, req := range newReqs { + path := req.Path + old := r.initialVersion[path] + new := req.Version + if old != new { + changes[path] = change{path, old, new} } - return m, nil } - if info.Version != m.Version { - logOncef("go: %s %s => %s", m.Path, getU, info.Version) + sortedChanges := make([]change, 0, len(changes)) + for _, c := range changes { + sortedChanges = append(sortedChanges, c) } - return module.Version{Path: m.Path, Version: info.Version}, nil + sort.Slice(sortedChanges, func(i, j int) bool { + return sortedChanges[i].path < sortedChanges[j].path + }) + for _, c := range sortedChanges { + if c.old == "" { + fmt.Fprintf(os.Stderr, "go get: added %s %s\n", c.path, c.new) + } else if c.new == "none" || c.new == "" { + fmt.Fprintf(os.Stderr, "go get: removed %s %s\n", c.path, c.old) + } else if semver.Compare(c.new, c.old) > 0 { + fmt.Fprintf(os.Stderr, "go get: upgraded %s %s => %s\n", c.path, c.old, c.new) + } else { + fmt.Fprintf(os.Stderr, "go get: downgraded %s %s => %s\n", c.path, c.old, c.new) + } + } + + // TODO(golang.org/issue/33284): attribute changes to command line arguments. + // For modules matched by command line arguments, this probably isn't + // necessary, but it would be useful for unmatched direct dependencies of + // the main module. } -// buildListForLostUpgrade returns the build list for the module graph -// rooted at lost. Unlike mvs.BuildList, the target module (lost) is not -// treated specially. The returned build list may contain a newer version -// of lost. -// -// buildListForLostUpgrade is used after a downgrade has removed a module -// requested at a specific version. This helps us understand the requirements -// implied by each downgrade. -func buildListForLostUpgrade(lost module.Version, reqs mvs.Reqs) ([]module.Version, error) { - return mvs.BuildList(lostUpgradeRoot, &lostUpgradeReqs{Reqs: reqs, lost: lost}) +// resolve records that module m must be at its indicated version (which may be +// "none") due to query q. If some other query forces module m to be at a +// different version, resolve reports a conflict error. +func (r *resolver) resolve(q *query, m module.Version) { + if m.Path == "" { + panic("internal error: resolving a module.Version with an empty path") + } + + if m.Path == modload.Target.Path && m.Version != modload.Target.Version { + reportError(q, &modload.QueryMatchesMainModuleError{ + Pattern: q.pattern, + Query: q.version, + }) + return + } + + vr, ok := r.resolvedVersion[m.Path] + if ok && vr.version != m.Version { + reportConflict(q, m, vr) + return + } + r.resolvedVersion[m.Path] = versionReason{m.Version, q} + q.resolved = append(q.resolved, m) } -var lostUpgradeRoot = module.Version{Path: "lost-upgrade-root", Version: ""} +// updateBuildList updates the module loader's global build list to be +// consistent with r.resolvedVersion, and to include additional modules +// provided that they do not conflict with the resolved versions. +// +// If the additional modules conflict with the resolved versions, they will be +// downgraded to a non-conflicting version (possibly "none"). +func (r *resolver) updateBuildList(ctx context.Context, additions []module.Version) (changed bool) { + if len(additions) == 0 && len(r.resolvedVersion) == r.buildListResolvedVersions { + return false + } + + defer base.ExitIfErrors() -type lostUpgradeReqs struct { - mvs.Reqs - lost module.Version + resolved := make([]module.Version, 0, len(r.resolvedVersion)) + for mPath, rv := range r.resolvedVersion { + if mPath != modload.Target.Path { + resolved = append(resolved, module.Version{Path: mPath, Version: rv.version}) + } + } + + if err := modload.EditBuildList(ctx, additions, resolved); err != nil { + var constraint *modload.ConstraintError + if !errors.As(err, &constraint) { + base.Errorf("go get: %v", err) + return false + } + + reason := func(m module.Version) string { + rv, ok := r.resolvedVersion[m.Path] + if !ok { + panic(fmt.Sprintf("internal error: can't find reason for requirement on %v", m)) + } + return rv.reason.ResolvedString(module.Version{Path: m.Path, Version: rv.version}) + } + for _, c := range constraint.Conflicts { + base.Errorf("go get: %v requires %v, not %v", reason(c.Source), c.Dep, reason(c.Constraint)) + } + return false + } + + buildList := modload.LoadAllModules(ctx) + r.buildListResolvedVersions = len(r.resolvedVersion) + if reflect.DeepEqual(r.buildList, buildList) { + return false + } + r.buildList = buildList + r.buildListVersion = make(map[string]string, len(r.buildList)) + for _, m := range r.buildList { + r.buildListVersion[m.Path] = m.Version + } + return true } -func (r *lostUpgradeReqs) Required(mod module.Version) ([]module.Version, error) { - if mod == lostUpgradeRoot { - return []module.Version{r.lost}, nil +func reqsFromGoMod(f *modfile.File) []module.Version { + reqs := make([]module.Version, len(f.Require)) + for i, r := range f.Require { + reqs[i] = r.Mod } - return r.Reqs.Required(mod) + return reqs } -var loggedLines sync.Map +// isNoSuchModuleVersion reports whether err indicates that the requested module +// does not exist at the requested version, either because the module does not +// exist at all or because it does not include that specific version. +func isNoSuchModuleVersion(err error) bool { + var noMatch *modload.NoMatchingVersionError + return errors.Is(err, os.ErrNotExist) || errors.As(err, &noMatch) +} -func logOncef(format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - if _, dup := loggedLines.LoadOrStore(msg, true); !dup { - fmt.Fprintln(os.Stderr, msg) - } +// isNoSuchPackageVersion reports whether err indicates that the requested +// package does not exist at the requested version, either because no module +// that could contain it exists at that version, or because every such module +// that does exist does not actually contain the package. +func isNoSuchPackageVersion(err error) bool { + var noPackage *modload.PackageNotInModuleError + return isNoSuchModuleVersion(err) || errors.As(err, &noPackage) } diff --git a/libgo/go/cmd/go/internal/modget/query.go b/libgo/go/cmd/go/internal/modget/query.go new file mode 100644 index 0000000..20eb0b6 --- /dev/null +++ b/libgo/go/cmd/go/internal/modget/query.go @@ -0,0 +1,357 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modget + +import ( + "fmt" + "path/filepath" + "regexp" + "strings" + "sync" + + "cmd/go/internal/base" + "cmd/go/internal/modload" + "cmd/go/internal/search" + "cmd/go/internal/str" + + "golang.org/x/mod/module" +) + +// A query describes a command-line argument and the modules and/or packages +// to which that argument may resolve.. +type query struct { + // raw is the original argument, to be printed in error messages. + raw string + + // rawVersion is the portion of raw corresponding to version, if any + rawVersion string + + // pattern is the part of the argument before "@" (or the whole argument + // if there is no "@"), which may match either packages (preferred) or + // modules (if no matching packages). + // + // The pattern may also be "-u", for the synthetic query representing the -u + // (“upgrade”)flag. + pattern string + + // patternIsLocal indicates whether pattern is restricted to match only paths + // local to the main module, such as absolute filesystem paths or paths + // beginning with './'. + // + // A local pattern must resolve to one or more packages in the main module. + patternIsLocal bool + + // version is the part of the argument after "@", or an implied + // "upgrade" or "patch" if there is no "@". version specifies the + // module version to get. + version string + + // matchWildcard, if non-nil, reports whether pattern, which must be a + // wildcard (with the substring "..."), matches the given package or module + // path. + matchWildcard func(path string) bool + + // canMatchWildcard, if non-nil, reports whether the module with the given + // path could lexically contain a package matching pattern, which must be a + // wildcard. + canMatchWildcardInModule func(mPath string) bool + + // conflict is the first query identified as incompatible with this one. + // conflict forces one or more of the modules matching this query to a + // version that does not match version. + conflict *query + + // candidates is a list of sets of alternatives for a path that matches (or + // contains packages that match) the pattern. The query can be resolved by + // choosing exactly one alternative from each set in the list. + // + // A path-literal query results in only one set: the path itself, which + // may resolve to either a package path or a module path. + // + // A wildcard query results in one set for each matching module path, each + // module for which the matching version contains at least one matching + // package, and (if no other modules match) one candidate set for the pattern + // overall if no existing match is identified in the build list. + // + // A query for pattern "all" results in one set for each package transitively + // imported by the main module. + // + // The special query for the "-u" flag results in one set for each + // otherwise-unconstrained package that has available upgrades. + candidates []pathSet + candidatesMu sync.Mutex + + // pathSeen ensures that only one pathSet is added to the query per + // unique path. + pathSeen sync.Map + + // resolved contains the set of modules whose versions have been determined by + // this query, in the order in which they were determined. + // + // The resolver examines the candidate sets for each query, resolving one + // module per candidate set in a way that attempts to avoid obvious conflicts + // between the versions resolved by different queries. + resolved []module.Version + + // matchesPackages is true if the resolved modules provide at least one + // package mathcing q.pattern. + matchesPackages bool +} + +// A pathSet describes the possible options for resolving a specific path +// to a package and/or module. +type pathSet struct { + // path is a package (if "all" or "-u" or a non-wildcard) or module (if + // wildcard) path that could be resolved by adding any of the modules in this + // set. For a wildcard pattern that so far matches no packages, the path is + // the wildcard pattern itself. + // + // Each path must occur only once in a query's candidate sets, and the path is + // added implicitly to each pathSet returned to pathOnce. + path string + + // pkgMods is a set of zero or more modules, each of which contains the + // package with the indicated path. Due to the requirement that imports be + // unambiguous, only one such module can be in the build list, and all others + // must be excluded. + pkgMods []module.Version + + // mod is either the zero Version, or a module that does not contain any + // packages matching the query but for which the module path itself + // matches the query pattern. + // + // We track this module separately from pkgMods because, all else equal, we + // prefer to match a query to a package rather than just a module. Also, + // unlike the modules in pkgMods, this module does not inherently exclude + // any other module in pkgMods. + mod module.Version + + err error +} + +// errSet returns a pathSet containing the given error. +func errSet(err error) pathSet { return pathSet{err: err} } + +// newQuery returns a new query parsed from the raw argument, +// which must be either path or path@version. +func newQuery(raw string) (*query, error) { + pattern := raw + rawVers := "" + if i := strings.Index(raw, "@"); i >= 0 { + pattern, rawVers = raw[:i], raw[i+1:] + if strings.Contains(rawVers, "@") || rawVers == "" { + return nil, fmt.Errorf("invalid module version syntax %q", raw) + } + } + + // If no version suffix is specified, assume @upgrade. + // If -u=patch was specified, assume @patch instead. + version := rawVers + if version == "" { + if getU.version == "" { + version = "upgrade" + } else { + version = getU.version + } + } + + q := &query{ + raw: raw, + rawVersion: rawVers, + pattern: pattern, + patternIsLocal: filepath.IsAbs(pattern) || search.IsRelativePath(pattern), + version: version, + } + if strings.Contains(q.pattern, "...") { + q.matchWildcard = search.MatchPattern(q.pattern) + q.canMatchWildcardInModule = search.TreeCanMatchPattern(q.pattern) + } + if err := q.validate(); err != nil { + return q, err + } + return q, nil +} + +// validate reports a non-nil error if q is not sensible and well-formed. +func (q *query) validate() error { + if q.patternIsLocal { + if q.rawVersion != "" { + return fmt.Errorf("can't request explicit version %q of path %q in main module", q.rawVersion, q.pattern) + } + return nil + } + + if q.pattern == "all" { + // If there is no main module, "all" is not meaningful. + if !modload.HasModRoot() { + return fmt.Errorf(`cannot match "all": working directory is not part of a module`) + } + if !versionOkForMainModule(q.version) { + // TODO(bcmills): "all@none" seems like a totally reasonable way to + // request that we remove all module requirements, leaving only the main + // module and standard library. Perhaps we should implement that someday. + return &modload.QueryMatchesMainModuleError{ + Pattern: q.pattern, + Query: q.version, + } + } + } + + if search.IsMetaPackage(q.pattern) && q.pattern != "all" { + if q.pattern != q.raw { + return fmt.Errorf("can't request explicit version of standard-library pattern %q", q.pattern) + } + } + + return nil +} + +// String returns the original argument from which q was parsed. +func (q *query) String() string { return q.raw } + +// ResolvedString returns a string describing m as a resolved match for q. +func (q *query) ResolvedString(m module.Version) string { + if m.Path != q.pattern { + if m.Version != q.version { + return fmt.Sprintf("%v (matching %s@%s)", m, q.pattern, q.version) + } + return fmt.Sprintf("%v (matching %v)", m, q) + } + if m.Version != q.version { + return fmt.Sprintf("%s@%s (%s)", q.pattern, q.version, m.Version) + } + return q.String() +} + +// isWildcard reports whether q is a pattern that can match multiple paths. +func (q *query) isWildcard() bool { + return q.matchWildcard != nil || (q.patternIsLocal && strings.Contains(q.pattern, "...")) +} + +// matchesPath reports whether the given path matches q.pattern. +func (q *query) matchesPath(path string) bool { + if q.matchWildcard != nil { + return q.matchWildcard(path) + } + return path == q.pattern +} + +// canMatchInModule reports whether the given module path can potentially +// contain q.pattern. +func (q *query) canMatchInModule(mPath string) bool { + if q.canMatchWildcardInModule != nil { + return q.canMatchWildcardInModule(mPath) + } + return str.HasPathPrefix(q.pattern, mPath) +} + +// pathOnce invokes f to generate the pathSet for the given path, +// if one is still needed. +// +// Note that, unlike sync.Once, pathOnce does not guarantee that a concurrent +// call to f for the given path has completed on return. +// +// pathOnce is safe for concurrent use by multiple goroutines, but note that +// multiple concurrent calls will result in the sets being added in +// nondeterministic order. +func (q *query) pathOnce(path string, f func() pathSet) { + if _, dup := q.pathSeen.LoadOrStore(path, nil); dup { + return + } + + cs := f() + + if len(cs.pkgMods) > 0 || cs.mod != (module.Version{}) || cs.err != nil { + cs.path = path + q.candidatesMu.Lock() + q.candidates = append(q.candidates, cs) + q.candidatesMu.Unlock() + } +} + +// reportError logs err concisely using base.Errorf. +func reportError(q *query, err error) { + errStr := err.Error() + + // If err already mentions all of the relevant parts of q, just log err to + // reduce stutter. Otherwise, log both q and err. + // + // TODO(bcmills): Use errors.As to unpack these errors instead of parsing + // strings with regular expressions. + + patternRE := regexp.MustCompile("(?m)(?:[ \t(\"`]|^)" + regexp.QuoteMeta(q.pattern) + "(?:[ @:)\"`]|$)") + if patternRE.MatchString(errStr) { + if q.rawVersion == "" { + base.Errorf("go get: %s", errStr) + return + } + + versionRE := regexp.MustCompile("(?m)(?:[ @(\"`]|^)" + regexp.QuoteMeta(q.version) + "(?:[ :)\"`]|$)") + if versionRE.MatchString(errStr) { + base.Errorf("go get: %s", errStr) + return + } + } + + if qs := q.String(); qs != "" { + base.Errorf("go get %s: %s", qs, errStr) + } else { + base.Errorf("go get: %s", errStr) + } +} + +func reportConflict(pq *query, m module.Version, conflict versionReason) { + if pq.conflict != nil { + // We've already reported a conflict for the proposed query. + // Don't report it again, even if it has other conflicts. + return + } + pq.conflict = conflict.reason + + proposed := versionReason{ + version: m.Version, + reason: pq, + } + if pq.isWildcard() && !conflict.reason.isWildcard() { + // Prefer to report the specific path first and the wildcard second. + proposed, conflict = conflict, proposed + } + reportError(pq, &conflictError{ + mPath: m.Path, + proposed: proposed, + conflict: conflict, + }) +} + +type conflictError struct { + mPath string + proposed versionReason + conflict versionReason +} + +func (e *conflictError) Error() string { + argStr := func(q *query, v string) string { + if v != q.version { + return fmt.Sprintf("%s@%s (%s)", q.pattern, q.version, v) + } + return q.String() + } + + pq := e.proposed.reason + rq := e.conflict.reason + modDetail := "" + if e.mPath != pq.pattern { + modDetail = fmt.Sprintf("for module %s, ", e.mPath) + } + + return fmt.Sprintf("%s%s conflicts with %s", + modDetail, + argStr(pq, e.proposed.version), + argStr(rq, e.conflict.version)) +} + +func versionOkForMainModule(version string) bool { + return version == "upgrade" || version == "patch" +} diff --git a/libgo/go/cmd/go/internal/modinfo/info.go b/libgo/go/cmd/go/internal/modinfo/info.go index 07248d1..897be56 100644 --- a/libgo/go/cmd/go/internal/modinfo/info.go +++ b/libgo/go/cmd/go/internal/modinfo/info.go @@ -21,6 +21,7 @@ type ModulePublic struct { Dir string `json:",omitempty"` // directory holding local copy of files, if any GoMod string `json:",omitempty"` // path to go.mod file describing module, if any GoVersion string `json:",omitempty"` // go version used in module + Retracted []string `json:",omitempty"` // retraction information, if any (with -retracted or -u) Error *ModuleError `json:",omitempty"` // error loading module } @@ -30,18 +31,26 @@ type ModuleError struct { func (m *ModulePublic) String() string { s := m.Path + versionString := func(mm *ModulePublic) string { + v := mm.Version + if len(mm.Retracted) == 0 { + return v + } + return v + " (retracted)" + } + if m.Version != "" { - s += " " + m.Version + s += " " + versionString(m) if m.Update != nil { - s += " [" + m.Update.Version + "]" + s += " [" + versionString(m.Update) + "]" } } if m.Replace != nil { s += " => " + m.Replace.Path if m.Replace.Version != "" { - s += " " + m.Replace.Version + s += " " + versionString(m.Replace) if m.Replace.Update != nil { - s += " [" + m.Replace.Update.Version + "]" + s += " [" + versionString(m.Replace.Update) + "]" } } } diff --git a/libgo/go/cmd/go/internal/modload/build.go b/libgo/go/cmd/go/internal/modload/build.go index e7e7d56..d2f8f86 100644 --- a/libgo/go/cmd/go/internal/modload/build.go +++ b/libgo/go/cmd/go/internal/modload/build.go @@ -6,12 +6,13 @@ package modload import ( "bytes" + "context" "encoding/hex" + "errors" "fmt" "internal/goroot" "os" "path/filepath" - "runtime/debug" "strings" "cmd/go/internal/base" @@ -48,7 +49,7 @@ func findStandardImportPath(path string) string { // PackageModuleInfo returns information about the module that provides // a given package. If modules are not enabled or if the package is in the // standard library or if the package was not successfully loaded with -// ImportPaths or a similar loading function, nil is returned. +// LoadPackages or ImportFromFiles, nil is returned. func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { if isStandardImportPath(pkgpath) || !Enabled() { return nil @@ -57,21 +58,27 @@ func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { if !ok { return nil } - return moduleInfo(m, true) + fromBuildList := true + listRetracted := false + return moduleInfo(context.TODO(), m, fromBuildList, listRetracted) } -func ModuleInfo(path string) *modinfo.ModulePublic { +func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { if !Enabled() { return nil } + listRetracted := false if i := strings.Index(path, "@"); i >= 0 { - return moduleInfo(module.Version{Path: path[:i], Version: path[i+1:]}, false) + m := module.Version{Path: path[:i], Version: path[i+1:]} + fromBuildList := false + return moduleInfo(ctx, m, fromBuildList, listRetracted) } - for _, m := range BuildList() { + for _, m := range buildList { if m.Path == path { - return moduleInfo(m, true) + fromBuildList := true + return moduleInfo(ctx, m, fromBuildList, listRetracted) } } @@ -84,12 +91,12 @@ func ModuleInfo(path string) *modinfo.ModulePublic { } // addUpdate fills in m.Update if an updated version is available. -func addUpdate(m *modinfo.ModulePublic) { +func addUpdate(ctx context.Context, m *modinfo.ModulePublic) { if m.Version == "" { return } - if info, err := Query(m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { + if info, err := Query(ctx, m.Path, "upgrade", m.Version, CheckAllowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { m.Update = &modinfo.ModulePublic{ Path: m.Path, Version: info.Version, @@ -99,11 +106,37 @@ func addUpdate(m *modinfo.ModulePublic) { } // addVersions fills in m.Versions with the list of known versions. -func addVersions(m *modinfo.ModulePublic) { - m.Versions, _ = versions(m.Path) +// Excluded versions will be omitted. If listRetracted is false, retracted +// versions will also be omitted. +func addVersions(ctx context.Context, m *modinfo.ModulePublic, listRetracted bool) { + allowed := CheckAllowed + if listRetracted { + allowed = CheckExclusions + } + m.Versions, _ = versions(ctx, m.Path, allowed) +} + +// addRetraction fills in m.Retracted if the module was retracted by its author. +// m.Error is set if there's an error loading retraction information. +func addRetraction(ctx context.Context, m *modinfo.ModulePublic) { + if m.Version == "" { + return + } + + err := CheckRetractions(ctx, module.Version{Path: m.Path, Version: m.Version}) + var rerr *ModuleRetractedError + if errors.As(err, &rerr) { + if len(rerr.Rationale) == 0 { + m.Retracted = []string{"retracted by module author"} + } else { + m.Retracted = rerr.Rationale + } + } else if err != nil && m.Error == nil { + m.Error = &modinfo.ModuleError{Err: err.Error()} + } } -func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { +func moduleInfo(ctx context.Context, m module.Version, fromBuildList, listRetracted bool) *modinfo.ModulePublic { if m == Target { info := &modinfo.ModulePublic{ Path: m.Path, @@ -125,21 +158,22 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { Version: m.Version, Indirect: fromBuildList && loaded != nil && !loaded.direct[m.Path], } - if loaded != nil { - info.GoVersion = loaded.goVersion[m.Path] + if v, ok := rawGoVersion.Load(m); ok { + info.GoVersion = v.(string) } // completeFromModCache fills in the extra fields in m using the module cache. completeFromModCache := func(m *modinfo.ModulePublic) { + mod := module.Version{Path: m.Path, Version: m.Version} + if m.Version != "" { - if q, err := Query(m.Path, m.Version, "", nil); err != nil { + if q, err := Query(ctx, m.Path, m.Version, "", nil); err != nil { m.Error = &modinfo.ModuleError{Err: err.Error()} } else { m.Version = q.Version m.Time = &q.Time } - mod := module.Version{Path: m.Path, Version: m.Version} gomod, err := modfetch.CachePath(mod, "mod") if err == nil { if info, err := os.Stat(gomod); err == nil && info.Mode().IsRegular() { @@ -150,10 +184,22 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { if err == nil { m.Dir = dir } + + if listRetracted { + addRetraction(ctx, m) + } + } + + if m.GoVersion == "" { + if summary, err := rawGoModSummary(mod); err == nil && summary.goVersionV != "" { + m.GoVersion = summary.goVersionV[1:] + } } } if !fromBuildList { + // If this was an explicitly-versioned argument to 'go mod download' or + // 'go list -m', report the actual requested version, not its replacement. completeFromModCache(info) // Will set m.Error in vendor mode. return info } @@ -177,9 +223,11 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { // worth the cost, and we're going to overwrite the GoMod and Dir from the // replacement anyway. See https://golang.org/issue/27859. info.Replace = &modinfo.ModulePublic{ - Path: r.Path, - Version: r.Version, - GoVersion: info.GoVersion, + Path: r.Path, + Version: r.Version, + } + if v, ok := rawGoVersion.Load(m); ok { + info.Replace.GoVersion = v.(string) } if r.Version == "" { if filepath.IsAbs(r.Path) { @@ -193,14 +241,15 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { completeFromModCache(info.Replace) info.Dir = info.Replace.Dir info.GoMod = info.Replace.GoMod + info.Retracted = info.Replace.Retracted } + info.GoVersion = info.Replace.GoVersion return info } // PackageBuildInfo returns a string containing module version information // for modules providing packages named by path and deps. path and deps must -// name packages that were resolved successfully with ImportPaths or one of -// the Load functions. +// name packages that were resolved successfully with LoadPackages. func PackageBuildInfo(path string, deps []string) string { if isStandardImportPath(path) || !Enabled() { return "" @@ -262,17 +311,13 @@ func mustFindModule(target, path string) module.Version { return Target } - if printStackInDie { - debug.PrintStack() - } base.Fatalf("build %v: cannot find module for path %v", target, path) panic("unreachable") } // findModule searches for the module that contains the package at path. -// If the package was loaded with ImportPaths or one of the other loading -// functions, its containing module and true are returned. Otherwise, -// module.Version{} and false are returend. +// If the package was loaded, its containing module and true are returned. +// Otherwise, module.Version{} and false are returend. func findModule(path string) (module.Version, bool) { if pkg, ok := loaded.pkgCache.Get(path).(*loadPkg); ok { return pkg.mod, pkg.mod != module.Version{} @@ -312,6 +357,6 @@ import _ "unsafe" //go:linkname __set_debug_modinfo__ runtime.setmodinfo func __set_debug_modinfo__(string) func init() { __set_debug_modinfo__(%q) } - `, string(infoStart)+info+string(infoEnd))) +`, string(infoStart)+info+string(infoEnd))) } } diff --git a/libgo/go/cmd/go/internal/modload/buildlist.go b/libgo/go/cmd/go/internal/modload/buildlist.go new file mode 100644 index 0000000..896adeb --- /dev/null +++ b/libgo/go/cmd/go/internal/modload/buildlist.go @@ -0,0 +1,267 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/imports" + "cmd/go/internal/mvs" + "context" + "fmt" + "os" + "strings" + + "golang.org/x/mod/module" +) + +// buildList is the list of modules to use for building packages. +// It is initialized by calling LoadPackages or ImportFromFiles, +// each of which uses loaded.load. +// +// Ideally, exactly ONE of those functions would be called, +// and exactly once. Most of the time, that's true. +// During "go get" it may not be. TODO(rsc): Figure out if +// that restriction can be established, or else document why not. +// +var buildList []module.Version + +// capVersionSlice returns s with its cap reduced to its length. +func capVersionSlice(s []module.Version) []module.Version { + return s[:len(s):len(s)] +} + +// LoadAllModules loads and returns the list of modules matching the "all" +// module pattern, starting with the Target module and in a deterministic +// (stable) order, without loading any packages. +// +// Modules are loaded automatically (and lazily) in LoadPackages: +// LoadAllModules need only be called if LoadPackages is not, +// typically in commands that care about modules but no particular package. +// +// The caller must not modify the returned list, but may append to it. +func LoadAllModules(ctx context.Context) []module.Version { + LoadModFile(ctx) + ReloadBuildList() + WriteGoMod() + return capVersionSlice(buildList) +} + +// Selected returns the selected version of the module with the given path, or +// the empty string if the given module has no selected version +// (either because it is not required or because it is the Target module). +func Selected(path string) (version string) { + if path == Target.Path { + return "" + } + for _, m := range buildList { + if m.Path == path { + return m.Version + } + } + return "" +} + +// EditBuildList edits the global build list by first adding every module in add +// to the existing build list, then adjusting versions (and adding or removing +// requirements as needed) until every module in mustSelect is selected at the +// given version. +// +// (Note that the newly-added modules might not be selected in the resulting +// build list: they could be lower than existing requirements or conflict with +// versions in mustSelect.) +// +// If the versions listed in mustSelect are mutually incompatible (due to one of +// the listed modules requiring a higher version of another), EditBuildList +// returns a *ConstraintError and leaves the build list in its previous state. +func EditBuildList(ctx context.Context, add, mustSelect []module.Version) error { + var upgraded = capVersionSlice(buildList) + if len(add) > 0 { + // First, upgrade the build list with any additions. + // In theory we could just append the additions to the build list and let + // mvs.Downgrade take care of resolving the upgrades too, but the + // diagnostics from Upgrade are currently much better in case of errors. + var err error + upgraded, err = mvs.Upgrade(Target, &mvsReqs{buildList: upgraded}, add...) + if err != nil { + return err + } + } + + downgraded, err := mvs.Downgrade(Target, &mvsReqs{buildList: append(upgraded, mustSelect...)}, mustSelect...) + if err != nil { + return err + } + + final, err := mvs.Upgrade(Target, &mvsReqs{buildList: downgraded}, mustSelect...) + if err != nil { + return err + } + + selected := make(map[string]module.Version, len(final)) + for _, m := range final { + selected[m.Path] = m + } + inconsistent := false + for _, m := range mustSelect { + s, ok := selected[m.Path] + if !ok { + if m.Version != "none" { + panic(fmt.Sprintf("internal error: mvs.BuildList lost %v", m)) + } + continue + } + if s.Version != m.Version { + inconsistent = true + break + } + } + + if !inconsistent { + buildList = final + return nil + } + + // We overshot one or more of the modules in mustSelected, which means that + // Downgrade removed something in mustSelect because it conflicted with + // something else in mustSelect. + // + // Walk the requirement graph to find the conflict. + // + // TODO(bcmills): Ideally, mvs.Downgrade (or a replacement for it) would do + // this directly. + + reqs := &mvsReqs{buildList: final} + reason := map[module.Version]module.Version{} + for _, m := range mustSelect { + reason[m] = m + } + queue := mustSelect[:len(mustSelect):len(mustSelect)] + for len(queue) > 0 { + var m module.Version + m, queue = queue[0], queue[1:] + required, err := reqs.Required(m) + if err != nil { + return err + } + for _, r := range required { + if _, ok := reason[r]; !ok { + reason[r] = reason[m] + queue = append(queue, r) + } + } + } + + var conflicts []Conflict + for _, m := range mustSelect { + s, ok := selected[m.Path] + if !ok { + if m.Version != "none" { + panic(fmt.Sprintf("internal error: mvs.BuildList lost %v", m)) + } + continue + } + if s.Version != m.Version { + conflicts = append(conflicts, Conflict{ + Source: reason[s], + Dep: s, + Constraint: m, + }) + } + } + + return &ConstraintError{ + Conflicts: conflicts, + } +} + +// A ConstraintError describes inconsistent constraints in EditBuildList +type ConstraintError struct { + // Conflict lists the source of the conflict for each version in mustSelect + // that could not be selected due to the requirements of some other version in + // mustSelect. + Conflicts []Conflict +} + +func (e *ConstraintError) Error() string { + b := new(strings.Builder) + b.WriteString("version constraints conflict:") + for _, c := range e.Conflicts { + fmt.Fprintf(b, "\n\t%v requires %v, but %v is requested", c.Source, c.Dep, c.Constraint) + } + return b.String() +} + +// A Conflict documents that Source requires Dep, which conflicts with Constraint. +// (That is, Dep has the same module path as Constraint but a higher version.) +type Conflict struct { + Source module.Version + Dep module.Version + Constraint module.Version +} + +// ReloadBuildList resets the state of loaded packages, then loads and returns +// the build list set by EditBuildList. +func ReloadBuildList() []module.Version { + loaded = loadFromRoots(loaderParams{ + PackageOpts: PackageOpts{ + Tags: imports.Tags(), + }, + listRoots: func() []string { return nil }, + allClosesOverTests: index.allPatternClosesOverTests(), // but doesn't matter because the root list is empty. + }) + return capVersionSlice(buildList) +} + +// TidyBuildList trims the build list to the minimal requirements needed to +// retain the same versions of all packages from the preceding call to +// LoadPackages. +func TidyBuildList() { + used := map[module.Version]bool{Target: true} + for _, pkg := range loaded.pkgs { + used[pkg.mod] = true + } + + keep := []module.Version{Target} + var direct []string + for _, m := range buildList[1:] { + if used[m] { + keep = append(keep, m) + if loaded.direct[m.Path] { + direct = append(direct, m.Path) + } + } else if cfg.BuildV { + if _, ok := index.require[m]; ok { + fmt.Fprintf(os.Stderr, "unused %s\n", m.Path) + } + } + } + + min, err := mvs.Req(Target, direct, &mvsReqs{buildList: keep}) + if err != nil { + base.Fatalf("go: %v", err) + } + buildList = append([]module.Version{Target}, min...) +} + +// checkMultiplePaths verifies that a given module path is used as itself +// or as a replacement for another module, but not both at the same time. +// +// (See https://golang.org/issue/26607 and https://golang.org/issue/34650.) +func checkMultiplePaths() { + firstPath := make(map[module.Version]string, len(buildList)) + for _, mod := range buildList { + src := mod + if rep := Replacement(mod); rep.Path != "" { + src = rep + } + if prev, ok := firstPath[src]; !ok { + firstPath[src] = mod.Path + } else if prev != mod.Path { + base.Errorf("go: %s@%s used for two different module paths (%s and %s)", src.Path, src.Version, prev, mod.Path) + } + } + base.ExitIfErrors() +} diff --git a/libgo/go/cmd/go/internal/modload/help.go b/libgo/go/cmd/go/internal/modload/help.go index d80206b..d81dfd5 100644 --- a/libgo/go/cmd/go/internal/modload/help.go +++ b/libgo/go/cmd/go/internal/modload/help.go @@ -124,72 +124,63 @@ and the build list. For example: Maintaining module requirements -The go.mod file is meant to be readable and editable by both -programmers and tools. The go command itself automatically updates the go.mod file -to maintain a standard formatting and the accuracy of require statements. - -Any go command that finds an unfamiliar import will look up the module -containing that import and add the latest version of that module -to go.mod automatically. In most cases, therefore, it suffices to -add an import to source code and run 'go build', 'go test', or even 'go list': -as part of analyzing the package, the go command will discover -and resolve the import and update the go.mod file. - -Any go command can determine that a module requirement is -missing and must be added, even when considering only a single -package from the module. On the other hand, determining that a module requirement -is no longer necessary and can be deleted requires a full view of -all packages in the module, across all possible build configurations -(architectures, operating systems, build tags, and so on). -The 'go mod tidy' command builds that view and then -adds any missing module requirements and removes unnecessary ones. +The go.mod file is meant to be readable and editable by both programmers and +tools. Most updates to dependencies can be performed using "go get" and +"go mod tidy". Other module-aware build commands may be invoked using the +-mod=mod flag to automatically add missing requirements and fix inconsistencies. + +The "go get" command updates go.mod to change the module versions used in a +build. An upgrade of one module may imply upgrading others, and similarly a +downgrade of one module may imply downgrading others. The "go get" command +makes these implied changes as well. See "go help module-get". + +The "go mod" command provides other functionality for use in maintaining +and understanding modules and go.mod files. See "go help mod", particularly +"go help mod tidy" and "go help mod edit". As part of maintaining the require statements in go.mod, the go command tracks which ones provide packages imported directly by the current module and which ones provide packages only used indirectly by other module dependencies. Requirements needed only for indirect uses are marked with a -"// indirect" comment in the go.mod file. Indirect requirements are +"// indirect" comment in the go.mod file. Indirect requirements may be automatically removed from the go.mod file once they are implied by other direct requirements. Indirect requirements only arise when using modules that fail to state some of their own dependencies or when explicitly upgrading a module's dependencies ahead of its own stated requirements. -Because of this automatic maintenance, the information in go.mod is an -up-to-date, readable description of the build. - -The 'go get' command updates go.mod to change the module versions used in a -build. An upgrade of one module may imply upgrading others, and similarly a -downgrade of one module may imply downgrading others. The 'go get' command -makes these implied changes as well. If go.mod is edited directly, commands -like 'go build' or 'go list' will assume that an upgrade is intended and -automatically make any implied upgrades and update go.mod to reflect them. - -The 'go mod' command provides other functionality for use in maintaining -and understanding modules and go.mod files. See 'go help mod'. - -The -mod build flag provides additional control over updating and use of go.mod. - -If invoked with -mod=readonly, the go command is disallowed from the implicit -automatic updating of go.mod described above. Instead, it fails when any changes -to go.mod are needed. This setting is most useful to check that go.mod does -not need updates, such as in a continuous integration and testing system. -The "go get" command remains permitted to update go.mod even with -mod=readonly, -and the "go mod" commands do not take the -mod flag (or any other build flags). +The -mod build flag provides additional control over the updating and use of +go.mod for commands that build packages like "go build" and "go test". + +If invoked with -mod=readonly (the default in most situations), the go command +reports an error if a package named on the command line or an imported package +is not provided by any module in the build list computed from the main module's +requirements. The go command also reports an error if a module's checksum is +missing from go.sum (see Module downloading and verification). Either go.mod or +go.sum must be updated in these situations. + +If invoked with -mod=mod, the go command automatically updates go.mod and +go.sum, fixing inconsistencies and adding missing requirements and checksums +as needed. If the go command finds an unfamiliar import, it looks up the +module containing that import and adds a requirement for the latest version +of that module to go.mod. In most cases, therefore, one may add an import to +source code and run "go build", "go test", or even "go list" with -mod=mod: +as part of analyzing the package, the go command will resolve the import and +update the go.mod file. If invoked with -mod=vendor, the go command loads packages from the main module's vendor directory instead of downloading modules to and loading packages from the module cache. The go command assumes the vendor directory holds correct copies of dependencies, and it does not compute the set of required module versions from go.mod files. However, the go command does check that -vendor/modules.txt (generated by 'go mod vendor') contains metadata consistent +vendor/modules.txt (generated by "go mod vendor") contains metadata consistent with go.mod. -If invoked with -mod=mod, the go command loads modules from the module cache -even if there is a vendor directory present. +If the go command is not invoked with a -mod flag, and the vendor directory +is present, and the "go" version in go.mod is 1.14 or higher, the go command +will act as if it were invoked with -mod=vendor. Otherwise, the -mod flag +defaults to -mod=readonly. -If the go command is not invoked with a -mod flag and the vendor directory -is present and the "go" version in go.mod is 1.14 or higher, the go command -will act as if it were invoked with -mod=vendor. +Note that neither "go get" nor the "go mod" subcommands accept the -mod flag. Pseudo-versions @@ -374,7 +365,7 @@ list if the error is a 404 or 410 HTTP response or if the current proxy is followed by a pipe character, indicating it is safe to fall back on any error. The GOPRIVATE and GONOPROXY environment variables allow bypassing -the proxy for selected modules. See 'go help module-private' for details. +the proxy for selected modules. See 'go help private' for details. No matter the source of the modules, the go command checks downloads against known checksums, to detect unexpected changes in the content of any specific @@ -432,21 +423,23 @@ verb followed by arguments. For example: require new/thing/v2 v2.3.4 exclude old/thing v1.2.3 replace bad/thing v1.4.5 => good/thing v1.4.5 + retract v1.5.6 The verbs are module, to define the module path; go, to set the expected language version; require, to require a particular module at a given version or later; - exclude, to exclude a particular module version from use; and - replace, to replace a module version with a different module version. + exclude, to exclude a particular module version from use; + replace, to replace a module version with a different module version; and + retract, to indicate a previously released version should not be used. Exclude and replace apply only in the main module's go.mod and are ignored -in dependencies. See https://research.swtch.com/vgo-mvs for details. +in dependencies. See https://golang.org/ref/mod for details. The leading verb can be factored out of adjacent lines to create a block, like in Go imports: require ( - new/thing v2.3.4 + new/thing/v2 v2.3.4 old/thing v1.2.3 ) diff --git a/libgo/go/cmd/go/internal/modload/import.go b/libgo/go/cmd/go/internal/modload/import.go index 4d2bc80..ce56717 100644 --- a/libgo/go/cmd/go/internal/modload/import.go +++ b/libgo/go/cmd/go/internal/modload/import.go @@ -5,18 +5,19 @@ package modload import ( + "context" "errors" "fmt" "go/build" "internal/goroot" + "io/fs" "os" "path/filepath" "sort" "strings" - "time" "cmd/go/internal/cfg" - "cmd/go/internal/load" + "cmd/go/internal/fsys" "cmd/go/internal/modfetch" "cmd/go/internal/par" "cmd/go/internal/search" @@ -30,23 +31,60 @@ type ImportMissingError struct { Module module.Version QueryErr error + // inAll indicates whether Path is in the "all" package pattern, + // and thus would be added by 'go mod tidy'. + inAll bool + + // isStd indicates whether we would expect to find the package in the standard + // library. This is normally true for all dotless import paths, but replace + // directives can cause us to treat the replaced paths as also being in + // modules. + isStd bool + + // replaced the highest replaced version of the module where the replacement + // contains the package. replaced is only set if the replacement is unused. + replaced module.Version + // newMissingVersion is set to a newer version of Module if one is present // in the build list. When set, we can't automatically upgrade. newMissingVersion string } -var _ load.ImportPathError = (*ImportMissingError)(nil) - func (e *ImportMissingError) Error() string { if e.Module.Path == "" { - if search.IsStandardImportPath(e.Path) { + if e.isStd { return fmt.Sprintf("package %s is not in GOROOT (%s)", e.Path, filepath.Join(cfg.GOROOT, "src", e.Path)) } if e.QueryErr != nil { return fmt.Sprintf("cannot find module providing package %s: %v", e.Path, e.QueryErr) } - return "cannot find module providing package " + e.Path + if cfg.BuildMod == "mod" { + return "cannot find module providing package " + e.Path + } + + if e.replaced.Path != "" { + suggestArg := e.replaced.Path + if !modfetch.IsZeroPseudoVersion(e.replaced.Version) { + suggestArg = e.replaced.String() + } + return fmt.Sprintf("module %s provides package %s and is replaced but not required; try 'go get -d %s' to add it", e.replaced.Path, e.Path, suggestArg) + } + + suggestion := "" + if !HasModRoot() { + suggestion = ": working directory is not part of a module" + } else if e.inAll { + suggestion = "; try 'go mod tidy' to add it" + } else { + suggestion = fmt.Sprintf("; try 'go get -d %s' to add it", e.Path) + } + return fmt.Sprintf("no required module provides package %s%s", e.Path, suggestion) } + + if e.newMissingVersion != "" { + return fmt.Sprintf("package %s provided by %s at latest version %s but not at required version %s", e.Path, e.Module.Path, e.Module.Version, e.newMissingVersion) + } + return fmt.Sprintf("missing module for import: %s@%s provides %s", e.Module.Path, e.Module.Version, e.Path) } @@ -97,20 +135,62 @@ func (e *AmbiguousImportError) Error() string { return buf.String() } -var _ load.ImportPathError = &AmbiguousImportError{} +// ImportMissingSumError is reported in readonly mode when we need to check +// if a module in the build list contains a package, but we don't have a sum +// for its .zip file. +type ImportMissingSumError struct { + importPath string + found, inAll bool +} -// Import finds the module and directory in the build list -// containing the package with the given import path. -// The answer must be unique: Import returns an error -// if multiple modules attempt to provide the same package. -// Import can return a module with an empty m.Path, for packages in the standard library. -// Import can return an empty directory string, for fake packages like "C" and "unsafe". +func (e *ImportMissingSumError) Error() string { + var message string + if e.found { + message = fmt.Sprintf("missing go.sum entry needed to verify package %s is provided by exactly one module", e.importPath) + } else { + message = fmt.Sprintf("missing go.sum entry for module providing package %s", e.importPath) + } + if e.inAll { + return message + "; try 'go mod tidy' to add it" + } + return message +} + +func (e *ImportMissingSumError) ImportPath() string { + return e.importPath +} + +type invalidImportError struct { + importPath string + err error +} + +func (e *invalidImportError) ImportPath() string { + return e.importPath +} + +func (e *invalidImportError) Error() string { + return e.err.Error() +} + +func (e *invalidImportError) Unwrap() error { + return e.err +} + +// importFromBuildList finds the module and directory in the build list +// containing the package with the given import path. The answer must be unique: +// importFromBuildList returns an error if multiple modules attempt to provide +// the same package. // -// If the package cannot be found in the current build list, -// Import returns an ImportMissingError as the error. -// If Import can identify a module that could be added to supply the package, -// the ImportMissingError records that module. -func Import(path string) (m module.Version, dir string, err error) { +// importFromBuildList can return a module with an empty m.Path, for packages in +// the standard library. +// +// importFromBuildList can return an empty directory string, for fake packages +// like "C" and "unsafe". +// +// If the package cannot be found in buildList, +// importFromBuildList returns an *ImportMissingError. +func importFromBuildList(ctx context.Context, path string, buildList []module.Version) (m module.Version, dir string, err error) { if strings.Contains(path, "@") { return module.Version{}, "", fmt.Errorf("import path should not have @version") } @@ -121,6 +201,10 @@ func Import(path string) (m module.Version, dir string, err error) { // There's no directory for import "C" or import "unsafe". return module.Version{}, "", nil } + // Before any further lookup, check that the path is valid. + if err := module.CheckImportPath(path); err != nil { + return module.Version{}, "", &invalidImportError{importPath: path, err: err} + } // Is the package in the standard library? pathIsStd := search.IsStandardImportPath(path) @@ -160,13 +244,23 @@ func Import(path string) (m module.Version, dir string, err error) { // Check each module on the build list. var dirs []string var mods []module.Version + haveSumErr := false for _, m := range buildList { if !maybeInModule(path, m.Path) { // Avoid possibly downloading irrelevant modules. continue } - root, isLocal, err := fetch(m) + needSum := true + root, isLocal, err := fetch(ctx, m, needSum) if err != nil { + if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { + // We are missing a sum needed to fetch a module in the build list. + // We can't verify that the package is unique, and we may not find + // the package at all. Keep checking other modules to decide which + // error to report. + haveSumErr = true + continue + } // Report fetch error. // Note that we don't know for sure this module is necessary, // but it certainly _could_ provide the package, and even if we @@ -182,88 +276,76 @@ func Import(path string) (m module.Version, dir string, err error) { dirs = append(dirs, dir) } } - if len(mods) == 1 { - return mods[0], dirs[0], nil - } - if len(mods) > 0 { + if len(mods) > 1 { return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} } - - // Look up module containing the package, for addition to the build list. - // Goal is to determine the module, download it to dir, and return m, dir, ErrMissing. - if cfg.BuildMod == "readonly" { - var queryErr error - if !pathIsStd { - if cfg.BuildModReason == "" { - queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) - } else { - queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) - } - } - return module.Version{}, "", &ImportMissingError{Path: path, QueryErr: queryErr} + if haveSumErr { + return module.Version{}, "", &ImportMissingSumError{importPath: path, found: len(mods) > 0} } - if modRoot == "" && !allowMissingModuleImports { - return module.Version{}, "", &ImportMissingError{ - Path: path, - QueryErr: errors.New("working directory is not part of a module"), - } + if len(mods) == 1 { + return mods[0], dirs[0], nil } - // Not on build list. - // To avoid spurious remote fetches, next try the latest replacement for each module. - // (golang.org/issue/26241) - if modFile != nil { - latest := map[string]string{} // path -> version - for _, r := range modFile.Replace { - if maybeInModule(path, r.Old.Path) { - // Don't use semver.Max here; need to preserve +incompatible suffix. - v := latest[r.Old.Path] - if semver.Compare(r.Old.Version, v) > 0 { - v = r.Old.Version - } - latest[r.Old.Path] = v - } - } + return module.Version{}, "", &ImportMissingError{Path: path, isStd: pathIsStd} +} - mods = make([]module.Version, 0, len(latest)) - for p, v := range latest { - // If the replacement didn't specify a version, synthesize a - // pseudo-version with an appropriate major version and a timestamp below - // any real timestamp. That way, if the main module is used from within - // some other module, the user will be able to upgrade the requirement to - // any real version they choose. - if v == "" { - if _, pathMajor, ok := module.SplitPathVersion(p); ok && len(pathMajor) > 0 { - v = modfetch.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") +// queryImport attempts to locate a module that can be added to the current +// build list to provide the package with the given import path. +// +// Unlike QueryPattern, queryImport prefers to add a replaced version of a +// module *before* checking the proxies for a version to add. +func queryImport(ctx context.Context, path string) (module.Version, error) { + // To avoid spurious remote fetches, try the latest replacement for each + // module (golang.org/issue/26241). + if index != nil { + var mods []module.Version + for mp, mv := range index.highestReplaced { + if !maybeInModule(path, mp) { + continue + } + if mv == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(mp); ok && len(pathMajor) > 0 { + mv = modfetch.ZeroPseudoVersion(pathMajor[1:]) } else { - v = modfetch.PseudoVersion("v0", "", time.Time{}, "000000000000") + mv = modfetch.ZeroPseudoVersion("v0") } } - mods = append(mods, module.Version{Path: p, Version: v}) + mods = append(mods, module.Version{Path: mp, Version: mv}) } // Every module path in mods is a prefix of the import path. - // As in QueryPackage, prefer the longest prefix that satisfies the import. + // As in QueryPattern, prefer the longest prefix that satisfies the import. sort.Slice(mods, func(i, j int) bool { return len(mods[i].Path) > len(mods[j].Path) }) for _, m := range mods { - root, isLocal, err := fetch(m) + needSum := true + root, isLocal, err := fetch(ctx, m, needSum) if err != nil { - // Report fetch error as above. - return module.Version{}, "", err + if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { + return module.Version{}, &ImportMissingSumError{importPath: path} + } + return module.Version{}, err } if _, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { - return m, "", err + return m, err } else if ok { - return m, "", &ImportMissingError{Path: path, Module: m} + if cfg.BuildMod == "readonly" { + return module.Version{}, &ImportMissingError{Path: path, replaced: m} + } + return m, nil } } if len(mods) > 0 && module.CheckPath(path) != nil { // The package path is not valid to fetch remotely, - // so it can only exist if in a replaced module, + // so it can only exist in a replaced module, // and we know from the above loop that it is not. - return module.Version{}, "", &PackageNotInModuleError{ + return module.Version{}, &PackageNotInModuleError{ Mod: mods[0], Query: "latest", Pattern: path, @@ -272,36 +354,53 @@ func Import(path string) (m module.Version, dir string, err error) { } } - if pathIsStd { + if search.IsStandardImportPath(path) { // This package isn't in the standard library, isn't in any module already // in the build list, and isn't in any other module that the user has // shimmed in via a "replace" directive. // Moreover, the import path is reserved for the standard library, so - // QueryPackage cannot possibly find a module containing this package. + // QueryPattern cannot possibly find a module containing this package. // - // Instead of trying QueryPackage, report an ImportMissingError immediately. - return module.Version{}, "", &ImportMissingError{Path: path} + // Instead of trying QueryPattern, report an ImportMissingError immediately. + return module.Version{}, &ImportMissingError{Path: path, isStd: true} + } + + if cfg.BuildMod == "readonly" { + // In readonly mode, we can't write go.mod, so we shouldn't try to look up + // the module. If readonly mode was enabled explicitly, include that in + // the error message. + var queryErr error + if cfg.BuildModExplicit { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) + } else if cfg.BuildModReason != "" { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) + } + return module.Version{}, &ImportMissingError{Path: path, QueryErr: queryErr} } + // Look up module containing the package, for addition to the build list. + // Goal is to determine the module, download it to dir, + // and return m, dir, ImpportMissingError. fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) - candidates, err := QueryPackage(path, "latest", Allowed) + candidates, err := QueryPackages(ctx, path, "latest", Selected, CheckAllowed) if err != nil { - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // Return "cannot find module providing package […]" instead of whatever - // low-level error QueryPackage produced. - return module.Version{}, "", &ImportMissingError{Path: path, QueryErr: err} + // low-level error QueryPattern produced. + return module.Version{}, &ImportMissingError{Path: path, QueryErr: err} } else { - return module.Version{}, "", err + return module.Version{}, err } } - m = candidates[0].Mod - newMissingVersion := "" - for _, c := range candidates { + + candidate0MissingVersion := "" + for i, c := range candidates { cm := c.Mod + canAdd := true for _, bm := range buildList { if bm.Path == cm.Path && semver.Compare(bm.Version, cm.Version) > 0 { - // QueryPackage proposed that we add module cm to provide the package, + // QueryPattern proposed that we add module cm to provide the package, // but we already depend on a newer version of that module (and we don't // have the package). // @@ -309,13 +408,22 @@ func Import(path string) (m module.Version, dir string, err error) { // version (e.g., v1.0.0) of a module, but we have a newer version // of the same module in the build list (e.g., v1.0.1-beta), and // the package is not present there. - m = cm - newMissingVersion = bm.Version + canAdd = false + if i == 0 { + candidate0MissingVersion = bm.Version + } break } } + if canAdd { + return cm, nil + } + } + return module.Version{}, &ImportMissingError{ + Path: path, + Module: candidates[0].Mod, + newMissingVersion: candidate0MissingVersion, } - return m, "", &ImportMissingError{Path: path, Module: m, newMissingVersion: newMissingVersion} } // maybeInModule reports whether, syntactically, @@ -369,7 +477,7 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile if isLocal { for d := dir; d != mdir && len(d) > len(mdir); { haveGoMod := haveGoModCache.Do(d, func() interface{} { - fi, err := os.Stat(filepath.Join(d, "go.mod")) + fi, err := fsys.Stat(filepath.Join(d, "go.mod")) return err == nil && !fi.IsDir() }).(bool) @@ -392,57 +500,65 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile // We don't care about build tags, not even "+build ignore". // We're just looking for a plausible directory. res := haveGoFilesCache.Do(dir, func() interface{} { - ok, err := isDirWithGoFiles(dir) + ok, err := fsys.IsDirWithGoFiles(dir) return goFilesEntry{haveGoFiles: ok, err: err} }).(goFilesEntry) return dir, res.haveGoFiles, res.err } -func isDirWithGoFiles(dir string) (bool, error) { - f, err := os.Open(dir) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - defer f.Close() - - names, firstErr := f.Readdirnames(-1) - if firstErr != nil { - if fi, err := f.Stat(); err == nil && !fi.IsDir() { - return false, nil - } - - // Rewrite the error from ReadDirNames to include the path if not present. - // See https://golang.org/issue/38923. - var pe *os.PathError - if !errors.As(firstErr, &pe) { - firstErr = &os.PathError{Op: "readdir", Path: dir, Err: firstErr} - } +// fetch downloads the given module (or its replacement) +// and returns its location. +// +// needSum indicates whether the module may be downloaded in readonly mode +// without a go.sum entry. It should only be false for modules fetched +// speculatively (for example, for incompatible version filtering). The sum +// will still be verified normally. +// +// The isLocal return value reports whether the replacement, +// if any, is local to the filesystem. +func fetch(ctx context.Context, mod module.Version, needSum bool) (dir string, isLocal bool, err error) { + if mod == Target { + return ModRoot(), true, nil } - - for _, name := range names { - if strings.HasSuffix(name, ".go") { - info, err := os.Stat(filepath.Join(dir, name)) - if err == nil && info.Mode().IsRegular() { - // If any .go source file exists, the package exists regardless of - // errors for other source files. Leave further error reporting for - // later. - return true, nil + if r := Replacement(mod); r.Path != "" { + if r.Version == "" { + dir = r.Path + if !filepath.IsAbs(dir) { + dir = filepath.Join(ModRoot(), dir) } - if firstErr == nil { + // Ensure that the replacement directory actually exists: + // dirInModule does not report errors for missing modules, + // so if we don't report the error now, later failures will be + // very mysterious. + if _, err := fsys.Stat(dir); err != nil { if os.IsNotExist(err) { - // If the file was concurrently deleted, or was a broken symlink, - // convert the error to an opaque error instead of one matching - // os.IsNotExist. - err = errors.New(err.Error()) + // Semantically the module version itself “exists” — we just don't + // have its source code. Remove the equivalence to os.ErrNotExist, + // and make the message more concise while we're at it. + err = fmt.Errorf("replacement directory %s does not exist", r.Path) + } else { + err = fmt.Errorf("replacement directory %s: %w", r.Path, err) } - firstErr = err + return dir, true, module.VersionError(mod, err) } + return dir, true, nil } + mod = r + } + + if cfg.BuildMod == "readonly" && needSum && !modfetch.HaveSum(mod) { + return "", false, module.VersionError(mod, &sumMissingError{}) } - return false, firstErr + dir, err = modfetch.Download(ctx, mod) + return dir, false, err +} + +type sumMissingError struct { + suggestion string +} + +func (e *sumMissingError) Error() string { + return "missing go.sum entry" + e.suggestion } diff --git a/libgo/go/cmd/go/internal/modload/import_test.go b/libgo/go/cmd/go/internal/modload/import_test.go index accc60e..22d5b82 100644 --- a/libgo/go/cmd/go/internal/modload/import_test.go +++ b/libgo/go/cmd/go/internal/modload/import_test.go @@ -5,19 +5,25 @@ package modload import ( + "context" "internal/testenv" "regexp" "strings" "testing" + + "golang.org/x/mod/module" ) var importTests = []struct { path string + m module.Version err string }{ { path: "golang.org/x/net/context", - err: "missing module for import: golang.org/x/net@.* provides golang.org/x/net/context", + m: module.Version{ + Path: "golang.org/x/net", + }, }, { path: "golang.org/x/net", @@ -25,15 +31,23 @@ var importTests = []struct { }, { path: "golang.org/x/text", - err: "missing module for import: golang.org/x/text@.* provides golang.org/x/text", + m: module.Version{ + Path: "golang.org/x/text", + }, }, { path: "github.com/rsc/quote/buggy", - err: "missing module for import: github.com/rsc/quote@v1.5.2 provides github.com/rsc/quote/buggy", + m: module.Version{ + Path: "github.com/rsc/quote", + Version: "v1.5.2", + }, }, { path: "github.com/rsc/quote", - err: "missing module for import: github.com/rsc/quote@v1.5.2 provides github.com/rsc/quote", + m: module.Version{ + Path: "github.com/rsc/quote", + Version: "v1.5.2", + }, }, { path: "golang.org/x/foo/bar", @@ -41,7 +55,7 @@ var importTests = []struct { }, } -func TestImport(t *testing.T) { +func TestQueryImport(t *testing.T) { testenv.MustHaveExternalNetwork(t) testenv.MustHaveExecPath(t, "git") defer func(old bool) { @@ -49,15 +63,28 @@ func TestImport(t *testing.T) { }(allowMissingModuleImports) AllowMissingModuleImports() + ctx := context.Background() + for _, tt := range importTests { t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) { // Note that there is no build list, so Import should always fail. - m, dir, err := Import(tt.path) - if err == nil { - t.Fatalf("Import(%q) = %v, %v, nil; expected error", tt.path, m, dir) + m, err := queryImport(ctx, tt.path) + + if tt.err == "" { + if err != nil { + t.Fatalf("queryImport(_, %q): %v", tt.path, err) + } + } else { + if err == nil { + t.Fatalf("queryImport(_, %q) = %v, nil; expected error", tt.path, m) + } + if !regexp.MustCompile(tt.err).MatchString(err.Error()) { + t.Fatalf("queryImport(_, %q): error %q, want error matching %#q", tt.path, err, tt.err) + } } - if !regexp.MustCompile(tt.err).MatchString(err.Error()) { - t.Fatalf("Import(%q): error %q, want error matching %#q", tt.path, err, tt.err) + + if m.Path != tt.m.Path || (tt.m.Version != "" && m.Version != tt.m.Version) { + t.Errorf("queryImport(_, %q) = %v, _; want %v", tt.path, m, tt.m) } }) } diff --git a/libgo/go/cmd/go/internal/modload/init.go b/libgo/go/cmd/go/internal/modload/init.go index 664a2a1..3f70d04 100644 --- a/libgo/go/cmd/go/internal/modload/init.go +++ b/libgo/go/cmd/go/internal/modload/init.go @@ -6,23 +6,22 @@ package modload import ( "bytes" + "context" "encoding/json" "errors" "fmt" "go/build" "internal/lazyregexp" - "io/ioutil" "os" "path" "path/filepath" - "runtime/debug" "strconv" "strings" + "sync" "cmd/go/internal/base" - "cmd/go/internal/cache" "cmd/go/internal/cfg" - "cmd/go/internal/load" + "cmd/go/internal/fsys" "cmd/go/internal/lockedfile" "cmd/go/internal/modconv" "cmd/go/internal/modfetch" @@ -35,8 +34,7 @@ import ( ) var ( - mustUseModules = false - initialized bool + initialized bool modRoot string Target module.Version @@ -52,20 +50,42 @@ var ( gopath string - CmdModInit bool // running 'go mod init' - CmdModModule string // module argument for 'go mod init' + // RootMode determines whether a module root is needed. + RootMode Root + + // ForceUseModules may be set to force modules to be enabled when + // GO111MODULE=auto or to report an error when GO111MODULE=off. + ForceUseModules bool allowMissingModuleImports bool ) +type Root int + +const ( + // AutoRoot is the default for most commands. modload.Init will look for + // a go.mod file in the current directory or any parent. If none is found, + // modules may be disabled (GO111MODULE=on) or commands may run in a + // limited module mode. + AutoRoot Root = iota + + // NoRoot is used for commands that run in module mode and ignore any go.mod + // file the current directory or in parent directories. + NoRoot + + // NeedRoot is used for commands that must run in module mode and don't + // make sense without a main module. + NeedRoot +) + // ModFile returns the parsed go.mod file. // -// Note that after calling ImportPaths or LoadBuildList, +// Note that after calling LoadPackages or LoadAllModules, // the require statements in the modfile.File are no longer // the source of truth and will be ignored: edits made directly // will be lost at the next call to WriteGoMod. // To make permanent changes to the require statements -// in go.mod, edit it before calling ImportPaths or LoadBuildList. +// in go.mod, edit it before loading. func ModFile() *modfile.File { Init() if modFile == nil { @@ -92,19 +112,27 @@ func Init() { // Keep in sync with WillBeEnabled. We perform extra validation here, and // there are lots of diagnostics and side effects, so we can't use // WillBeEnabled directly. + var mustUseModules bool env := cfg.Getenv("GO111MODULE") switch env { default: base.Fatalf("go: unknown environment setting GO111MODULE=%s", env) - case "auto", "": - mustUseModules = false - case "on": + case "auto": + mustUseModules = ForceUseModules + case "on", "": mustUseModules = true case "off": + if ForceUseModules { + base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") + } mustUseModules = false return } + if err := fsys.Init(base.Cwd); err != nil { + base.Fatalf("go: %v", err) + } + // Disable any prompting for passwords by Git. // Only has an effect for 2.3.0 or later, but avoiding // the prompt in earlier versions is just too hard. @@ -132,15 +160,23 @@ func Init() { os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no") } - if CmdModInit { - // Running 'go mod init': go.mod will be created in current directory. - modRoot = base.Cwd + if modRoot != "" { + // modRoot set before Init was called ("go mod init" does this). + // No need to search for go.mod. + } else if RootMode == NoRoot { + if cfg.ModFile != "" && !base.InGOFLAGS("-modfile") { + base.Fatalf("go: -modfile cannot be used with commands that ignore the current module") + } + modRoot = "" } else { modRoot = findModuleRoot(base.Cwd) if modRoot == "" { if cfg.ModFile != "" { base.Fatalf("go: cannot find main module, but -modfile was set.\n\t-modfile cannot be used to set the module root directory.") } + if RootMode == NeedRoot { + base.Fatalf("go: cannot find main module; see 'go help modules'") + } if !mustUseModules { // GO111MODULE is 'auto', and we can't find a module root. // Stay in GOPATH mode. @@ -154,38 +190,26 @@ func Init() { // when it happens. See golang.org/issue/26708. modRoot = "" fmt.Fprintf(os.Stderr, "go: warning: ignoring go.mod in system temp root %v\n", os.TempDir()) + if !mustUseModules { + return + } } } if cfg.ModFile != "" && !strings.HasSuffix(cfg.ModFile, ".mod") { base.Fatalf("go: -modfile=%s: file does not have .mod extension", cfg.ModFile) } - // We're in module mode. Install the hooks to make it work. - - if c := cache.Default(); c == nil { - // With modules, there are no install locations for packages - // other than the build cache. - base.Fatalf("go: cannot use modules with build cache disabled") - } - + // We're in module mode. Set any global variables that need to be set. list := filepath.SplitList(cfg.BuildContext.GOPATH) if len(list) == 0 || list[0] == "" { base.Fatalf("missing $GOPATH") } gopath = list[0] - if _, err := os.Stat(filepath.Join(gopath, "go.mod")); err == nil { + if _, err := fsys.Stat(filepath.Join(gopath, "go.mod")); err == nil { base.Fatalf("$GOPATH/go.mod exists but should not") } cfg.ModulesEnabled = true - load.ModBinDir = BinDir - load.ModLookup = Lookup - load.ModPackageModuleInfo = PackageModuleInfo - load.ModImportPaths = ImportPaths - load.ModPackageBuildInfo = PackageBuildInfo - load.ModInfoProg = ModInfoProg - load.ModImportFromFiles = ImportFromFiles - load.ModDirImportPath = DirImportPath if modRoot == "" { // We're in module mode, but not inside a module. @@ -211,10 +235,6 @@ func Init() { } } -func init() { - load.ModInit = Init -} - // WillBeEnabled checks whether modules should be enabled but does not // initialize modules by installing hooks. If Init has already been called, // WillBeEnabled returns the same result as Enabled. @@ -225,10 +245,12 @@ func init() { // be called until the command is installed and flags are parsed. Instead of // calling Init and Enabled, the main package can call this function. func WillBeEnabled() bool { - if modRoot != "" || mustUseModules { + if modRoot != "" || cfg.ModulesEnabled { + // Already enabled. return true } if initialized { + // Initialized, not enabled. return false } @@ -236,18 +258,14 @@ func WillBeEnabled() bool { // exits, so it can't call this function directly. env := cfg.Getenv("GO111MODULE") switch env { - case "on": + case "on", "": return true - case "auto", "": + case "auto": break default: return false } - if CmdModInit { - // Running 'go mod init': go.mod will be created in current directory. - return true - } if modRoot := findModuleRoot(base.Cwd); modRoot == "" { // GO111MODULE is 'auto', and we can't find a module root. // Stay in GOPATH mode. @@ -269,7 +287,7 @@ func WillBeEnabled() bool { // (usually through MustModRoot). func Enabled() bool { Init() - return modRoot != "" || mustUseModules + return modRoot != "" || cfg.ModulesEnabled } // ModRoot returns the root of the main module. @@ -303,16 +321,7 @@ func ModFilePath() string { return filepath.Join(modRoot, "go.mod") } -// printStackInDie causes die to print a stack trace. -// -// It is enabled by the testgo tag, and helps to diagnose paths that -// unexpectedly require a main module. -var printStackInDie = false - func die() { - if printStackInDie { - debug.PrintStack() - } if cfg.Getenv("GO111MODULE") == "off" { base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") } @@ -330,12 +339,16 @@ func die() { base.Fatalf("go: cannot find main module; see 'go help modules'") } -// InitMod sets Target and, if there is a main module, parses the initial build -// list from its go.mod file, creating and populating that file if needed. +// LoadModFile sets Target and, if there is a main module, parses the initial +// build list from its go.mod file. // -// As a side-effect, InitMod sets a default for cfg.BuildMod if it does not +// LoadModFile may make changes in memory, like adding a go directive and +// ensuring requirements are consistent. WriteGoMod should be called later to +// write changes out to disk or report errors in readonly mode. +// +// As a side-effect, LoadModFile sets a default for cfg.BuildMod if it does not // already have an explicit value. -func InitMod() { +func LoadModFile(ctx context.Context) { if len(buildList) > 0 { return } @@ -348,14 +361,6 @@ func InitMod() { return } - if CmdModInit { - // Running go mod init: do legacy module conversion - legacyModInit() - modFileToBuildList() - WriteGoMod() - return - } - gomod := ModFilePath() data, err := lockedfile.Read(gomod) if err != nil { @@ -363,7 +368,7 @@ func InitMod() { } var fixed bool - f, err := modfile.Parse(gomod, data, fixVersion(&fixed)) + f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed)) if err != nil { // Errors returned by modfile.Parse begin with file:line. base.Fatalf("go: errors parsing go.mod:\n%s\n", err) @@ -371,32 +376,133 @@ func InitMod() { modFile = f index = indexModFile(data, f, fixed) - if len(f.Syntax.Stmt) == 0 || f.Module == nil { - // Empty mod file. Must add module path. - path, err := findModulePath(modRoot) - if err != nil { - base.Fatalf("go: %v", err) - } - f.AddModuleStmt(path) + if f.Module == nil { + // No module declaration. Must add module path. + base.Fatalf("go: no module declaration in go.mod.\n\tRun 'go mod edit -module=example.com/mod' to specify the module path.") } - if len(f.Syntax.Stmt) == 1 && f.Module != nil { - // Entire file is just a module statement. - // Populate require if possible. - legacyModInit() + if err := checkModulePathLax(f.Module.Mod.Path); err != nil { + base.Fatalf("go: %v", err) } - modFileToBuildList() setDefaultBuildMod() + modFileToBuildList() if cfg.BuildMod == "vendor" { readVendorList() checkVendorConsistency() - } else { - // TODO(golang.org/issue/33326): if cfg.BuildMod != "readonly"? - WriteGoMod() } } +// CreateModFile initializes a new module by creating a go.mod file. +// +// If modPath is empty, CreateModFile will attempt to infer the path from the +// directory location within GOPATH. +// +// If a vendoring configuration file is present, CreateModFile will attempt to +// translate it to go.mod directives. The resulting build list may not be +// exactly the same as in the legacy configuration (for example, we can't get +// packages at multiple versions from the same module). +func CreateModFile(ctx context.Context, modPath string) { + modRoot = base.Cwd + Init() + modFilePath := ModFilePath() + if _, err := fsys.Stat(modFilePath); err == nil { + base.Fatalf("go: %s already exists", modFilePath) + } + + if modPath == "" { + var err error + modPath, err = findModulePath(modRoot) + if err != nil { + base.Fatalf("go: %v", err) + } + } else if err := checkModulePathLax(modPath); err != nil { + base.Fatalf("go: %v", err) + } + + fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", modPath) + modFile = new(modfile.File) + modFile.AddModuleStmt(modPath) + addGoStmt() // Add the go directive before converted module requirements. + + convertedFrom, err := convertLegacyConfig(modPath) + if convertedFrom != "" { + fmt.Fprintf(os.Stderr, "go: copying requirements from %s\n", base.ShortPath(convertedFrom)) + } + if err != nil { + base.Fatalf("go: %v", err) + } + + modFileToBuildList() + WriteGoMod() + + // Suggest running 'go mod tidy' unless the project is empty. Even if we + // imported all the correct requirements above, we're probably missing + // some sums, so the next build command in -mod=readonly will likely fail. + // + // We look for non-hidden .go files or subdirectories to determine whether + // this is an existing project. Walking the tree for packages would be more + // accurate, but could take much longer. + empty := true + files, _ := os.ReadDir(modRoot) + for _, f := range files { + name := f.Name() + if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") { + continue + } + if strings.HasSuffix(name, ".go") || f.IsDir() { + empty = false + break + } + } + if !empty { + fmt.Fprintf(os.Stderr, "go: run 'go mod tidy' to add module requirements and sums\n") + } +} + +// checkModulePathLax checks that the path meets some minimum requirements +// to avoid confusing users or the module cache. The requirements are weaker +// than those of module.CheckPath to allow room for weakening module path +// requirements in the future, but strong enough to help users avoid significant +// problems. +func checkModulePathLax(p string) error { + // TODO(matloob): Replace calls of this function in this CL with calls + // to module.CheckImportPath once it's been laxened, if it becomes laxened. + // See golang.org/issue/29101 for a discussion about whether to make CheckImportPath + // more lax or more strict. + + errorf := func(format string, args ...interface{}) error { + return fmt.Errorf("invalid module path %q: %s", p, fmt.Sprintf(format, args...)) + } + + // Disallow shell characters " ' * < > ? ` | to avoid triggering bugs + // with file systems and subcommands. Disallow file path separators : and \ + // because path separators other than / will confuse the module cache. + // See fileNameOK in golang.org/x/mod/module/module.go. + shellChars := "`" + `\"'*<>?|` + fsChars := `\:` + if i := strings.IndexAny(p, shellChars); i >= 0 { + return errorf("contains disallowed shell character %q", p[i]) + } + if i := strings.IndexAny(p, fsChars); i >= 0 { + return errorf("contains disallowed path separator character %q", p[i]) + } + + // Ensure path.IsAbs and build.IsLocalImport are false, and that the path is + // invariant under path.Clean, also to avoid confusing the module cache. + if path.IsAbs(p) { + return errorf("is an absolute path") + } + if build.IsLocalImport(p) { + return errorf("is a local import path") + } + if path.Clean(p) != p { + return errorf("is not clean") + } + + return nil +} + // fixVersion returns a modfile.VersionFixer implemented using the Query function. // // It resolves commit hashes and branch names to versions, @@ -404,7 +510,7 @@ func InitMod() { // and does nothing for versions that already appear to be canonical. // // The VersionFixer sets 'fixed' if it ever returns a non-canonical version. -func fixVersion(fixed *bool) modfile.VersionFixer { +func fixVersion(ctx context.Context, fixed *bool) modfile.VersionFixer { return func(path, vers string) (resolved string, err error) { defer func() { if err == nil && resolved != vers { @@ -436,7 +542,7 @@ func fixVersion(fixed *bool) modfile.VersionFixer { } } - info, err := Query(path, vers, "", nil) + info, err := Query(ctx, path, vers, "", nil) if err != nil { return "", err } @@ -465,7 +571,15 @@ func modFileToBuildList() { list := []module.Version{Target} for _, r := range modFile.Require { - list = append(list, r.Mod) + if index != nil && index.exclude[r.Mod] { + if cfg.BuildMod == "mod" { + fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } else { + fmt.Fprintf(os.Stderr, "go: ignoring requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } + } else { + list = append(list, r.Mod) + } } buildList = list } @@ -473,80 +587,62 @@ func modFileToBuildList() { // setDefaultBuildMod sets a default value for cfg.BuildMod // if it is currently empty. func setDefaultBuildMod() { - if cfg.BuildMod != "" { + if cfg.BuildModExplicit { // Don't override an explicit '-mod=' argument. return } - cfg.BuildMod = "mod" + if cfg.CmdName == "get" || strings.HasPrefix(cfg.CmdName, "mod ") { - // Don't set -mod implicitly for commands whose purpose is to - // manipulate the build list. + // 'get' and 'go mod' commands may update go.mod automatically. + // TODO(jayconrod): should this narrower? Should 'go mod download' or + // 'go mod graph' update go.mod by default? + cfg.BuildMod = "mod" return } if modRoot == "" { + cfg.BuildMod = "readonly" return } - if fi, err := os.Stat(filepath.Join(modRoot, "vendor")); err == nil && fi.IsDir() { + if fi, err := fsys.Stat(filepath.Join(modRoot, "vendor")); err == nil && fi.IsDir() { modGo := "unspecified" - if index.goVersion != "" { - if semver.Compare("v"+index.goVersion, "v1.14") >= 0 { + if index.goVersionV != "" { + if semver.Compare(index.goVersionV, "v1.14") >= 0 { // The Go version is at least 1.14, and a vendor directory exists. // Set -mod=vendor by default. cfg.BuildMod = "vendor" cfg.BuildModReason = "Go version in go.mod is at least 1.14 and vendor directory exists." return } else { - modGo = index.goVersion + modGo = index.goVersionV[1:] } } - // Since a vendor directory exists, we have a non-trivial reason for - // choosing -mod=mod, although it probably won't be used for anything. - // Record the reason anyway for consistency. - // It may be overridden if we switch to mod=readonly below. - cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s.", modGo) + // Since a vendor directory exists, we should record why we didn't use it. + // This message won't normally be shown, but it may appear with import errors. + cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s, so vendor directory was not used.", modGo) } - p := ModFilePath() - if fi, err := os.Stat(p); err == nil && !hasWritePerm(p, fi) { - cfg.BuildMod = "readonly" - cfg.BuildModReason = "go.mod file is read-only." - } + cfg.BuildMod = "readonly" } -func legacyModInit() { - if modFile == nil { - path, err := findModulePath(modRoot) - if err != nil { - base.Fatalf("go: %v", err) - } - fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", path) - modFile = new(modfile.File) - modFile.AddModuleStmt(path) - addGoStmt() // Add the go directive before converted module requirements. - } - +// convertLegacyConfig imports module requirements from a legacy vendoring +// configuration file, if one is present. +func convertLegacyConfig(modPath string) (from string, err error) { for _, name := range altConfigs { cfg := filepath.Join(modRoot, name) - data, err := ioutil.ReadFile(cfg) + data, err := os.ReadFile(cfg) if err == nil { convert := modconv.Converters[name] if convert == nil { - return + return "", nil } - fmt.Fprintf(os.Stderr, "go: copying requirements from %s\n", base.ShortPath(cfg)) cfg = filepath.ToSlash(cfg) - if err := modconv.ConvertLegacyConfig(modFile, cfg, data); err != nil { - base.Fatalf("go: %v", err) - } - if len(modFile.Syntax.Stmt) == 1 { - // Add comment to avoid re-converting every time it runs. - modFile.AddComment("// go: no requirements found in " + name) - } - return + err := modconv.ConvertLegacyConfig(modFile, cfg, data) + return name, err } } + return "", nil } // addGoStmt adds a go directive to the go.mod file if it does not already include one. @@ -588,7 +684,7 @@ func findModuleRoot(dir string) (root string) { // Look for enclosing go.mod. for { - if fi, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil && !fi.IsDir() { + if fi, err := fsys.Stat(filepath.Join(dir, "go.mod")); err == nil && !fi.IsDir() { return dir } d := filepath.Dir(dir) @@ -612,7 +708,7 @@ func findAltConfig(dir string) (root, name string) { } for { for _, name := range altConfigs { - if fi, err := os.Stat(filepath.Join(dir, name)); err == nil && !fi.IsDir() { + if fi, err := fsys.Stat(filepath.Join(dir, name)); err == nil && !fi.IsDir() { return dir, name } } @@ -626,14 +722,6 @@ func findAltConfig(dir string) (root, name string) { } func findModulePath(dir string) (string, error) { - if CmdModModule != "" { - // Running go mod init x/y/z; return x/y/z. - if err := module.CheckImportPath(CmdModModule); err != nil { - return "", err - } - return CmdModModule, nil - } - // TODO(bcmills): once we have located a plausible module path, we should // query version control (if available) to verify that it matches the major // version of the most recent tag. @@ -642,9 +730,9 @@ func findModulePath(dir string) (string, error) { // Cast about for import comments, // first in top-level directory, then in subdirectories. - list, _ := ioutil.ReadDir(dir) + list, _ := os.ReadDir(dir) for _, info := range list { - if info.Mode().IsRegular() && strings.HasSuffix(info.Name(), ".go") { + if info.Type().IsRegular() && strings.HasSuffix(info.Name(), ".go") { if com := findImportComment(filepath.Join(dir, info.Name())); com != "" { return com, nil } @@ -652,9 +740,9 @@ func findModulePath(dir string) (string, error) { } for _, info1 := range list { if info1.IsDir() { - files, _ := ioutil.ReadDir(filepath.Join(dir, info1.Name())) + files, _ := os.ReadDir(filepath.Join(dir, info1.Name())) for _, info2 := range files { - if info2.Mode().IsRegular() && strings.HasSuffix(info2.Name(), ".go") { + if info2.Type().IsRegular() && strings.HasSuffix(info2.Name(), ".go") { if com := findImportComment(filepath.Join(dir, info1.Name(), info2.Name())); com != "" { return path.Dir(com), nil } @@ -664,7 +752,7 @@ func findModulePath(dir string) (string, error) { } // Look for Godeps.json declaring import path. - data, _ := ioutil.ReadFile(filepath.Join(dir, "Godeps/Godeps.json")) + data, _ := os.ReadFile(filepath.Join(dir, "Godeps/Godeps.json")) var cfg1 struct{ ImportPath string } json.Unmarshal(data, &cfg1) if cfg1.ImportPath != "" { @@ -672,7 +760,7 @@ func findModulePath(dir string) (string, error) { } // Look for vendor.json declaring import path. - data, _ = ioutil.ReadFile(filepath.Join(dir, "vendor/vendor.json")) + data, _ = os.ReadFile(filepath.Join(dir, "vendor/vendor.json")) var cfg2 struct{ RootPath string } json.Unmarshal(data, &cfg2) if cfg2.RootPath != "" { @@ -680,16 +768,35 @@ func findModulePath(dir string) (string, error) { } // Look for path in GOPATH. + var badPathErr error for _, gpdir := range filepath.SplitList(cfg.BuildContext.GOPATH) { if gpdir == "" { continue } if rel := search.InDir(dir, filepath.Join(gpdir, "src")); rel != "" && rel != "." { - return filepath.ToSlash(rel), nil + path := filepath.ToSlash(rel) + // TODO(matloob): replace this with module.CheckImportPath + // once it's been laxened. + // Only checkModulePathLax here. There are some unpublishable + // module names that are compatible with checkModulePathLax + // but they already work in GOPATH so don't break users + // trying to do a build with modules. gorelease will alert users + // publishing their modules to fix their paths. + if err := checkModulePathLax(path); err != nil { + badPathErr = err + break + } + return path, nil } } - msg := `cannot determine module path for source directory %s (outside GOPATH, module path must be specified) + reason := "outside GOPATH, module path must be specified" + if badPathErr != nil { + // return a different error message if the module was in GOPATH, but + // the module path determined above would be an invalid path. + reason = fmt.Sprintf("bad module path inferred from directory in GOPATH: %v", badPathErr) + } + msg := `cannot determine module path for source directory %s (%s) Example usage: 'go mod init example.com/m' to initialize a v0 or v1 module @@ -697,7 +804,7 @@ Example usage: Run 'go help mod init' for more information. ` - return "", fmt.Errorf(msg, dir) + return "", fmt.Errorf(msg, dir, reason) } var ( @@ -705,7 +812,7 @@ var ( ) func findImportComment(file string) string { - data, err := ioutil.ReadFile(file) + data, err := os.ReadFile(file) if err != nil { return "" } @@ -745,7 +852,7 @@ func MinReqs() mvs.Reqs { retain = append(retain, m.Path) } } - min, err := mvs.Req(Target, retain, Reqs()) + min, err := mvs.Req(Target, retain, &mvsReqs{buildList: buildList}) if err != nil { base.Fatalf("go: %v", err) } @@ -791,20 +898,23 @@ func WriteGoMod() { if dirty && cfg.BuildMod == "readonly" { // If we're about to fail due to -mod=readonly, // prefer to report a dirty go.mod over a dirty go.sum - if cfg.BuildModReason != "" { + if cfg.BuildModExplicit { + base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly") + } else if cfg.BuildModReason != "" { base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly\n\t(%s)", cfg.BuildModReason) } else { - base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly") + base.Fatalf("go: updates to go.mod needed; try 'go mod tidy' first") } } - // Always update go.sum, even if we didn't change go.mod: we may have - // downloaded modules that we didn't have before. - modfetch.WriteGoSum() if !dirty && cfg.CmdName != "mod tidy" { // The go.mod file has the same semantic content that it had before // (but not necessarily the same exact bytes). - // Ignore any intervening edits. + // Don't write go.mod, but write go.sum in case we added or trimmed sums. + // 'go mod init' shouldn't write go.sum, since it will be incomplete. + if cfg.CmdName != "mod init" { + modfetch.WriteGoSum(keepSums(true)) + } return } @@ -815,6 +925,12 @@ func WriteGoMod() { defer func() { // At this point we have determined to make the go.mod file on disk equal to new. index = indexModFile(new, modFile, false) + + // Update go.sum after releasing the side lock and refreshing the index. + // 'go mod init' shouldn't write go.sum, since it will be incomplete. + if cfg.CmdName != "mod init" { + modfetch.WriteGoSum(keepSums(true)) + } }() // Make a best-effort attempt to acquire the side lock, only to exclude @@ -849,3 +965,102 @@ func WriteGoMod() { base.Fatalf("go: updating go.mod: %v", err) } } + +// keepSums returns a set of module sums to preserve in go.sum. The set +// includes entries for all modules used to load packages (according to +// the last load function such as LoadPackages or ImportFromFiles). +// It also contains entries for go.mod files needed for MVS (the version +// of these entries ends with "/go.mod"). +// +// If addDirect is true, the set also includes sums for modules directly +// required by go.mod, as represented by the index, with replacements applied. +func keepSums(addDirect bool) map[module.Version]bool { + // Re-derive the build list using the current list of direct requirements. + // Keep the sum for the go.mod of each visited module version (or its + // replacement). + modkey := func(m module.Version) module.Version { + return module.Version{Path: m.Path, Version: m.Version + "/go.mod"} + } + keep := make(map[module.Version]bool) + var mu sync.Mutex + reqs := &keepSumReqs{ + Reqs: &mvsReqs{buildList: buildList}, + visit: func(m module.Version) { + // If we build using a replacement module, keep the sum for the replacement, + // since that's the code we'll actually use during a build. + mu.Lock() + r := Replacement(m) + if r.Path == "" { + keep[modkey(m)] = true + } else { + keep[modkey(r)] = true + } + mu.Unlock() + }, + } + buildList, err := mvs.BuildList(Target, reqs) + if err != nil { + panic(fmt.Sprintf("unexpected error reloading build list: %v", err)) + } + + // Add entries for modules in the build list with paths that are prefixes of + // paths of loaded packages. We need to retain sums for modules needed to + // report ambiguous import errors. We use our re-derived build list, + // since the global build list may have been tidied. + if loaded != nil { + actualMods := make(map[string]module.Version) + for _, m := range buildList[1:] { + if r := Replacement(m); r.Path != "" { + actualMods[m.Path] = r + } else { + actualMods[m.Path] = m + } + } + for _, pkg := range loaded.pkgs { + if pkg.testOf != nil || pkg.inStd || module.CheckImportPath(pkg.path) != nil { + continue + } + for prefix := pkg.path; prefix != "."; prefix = path.Dir(prefix) { + if m, ok := actualMods[prefix]; ok { + keep[m] = true + } + } + } + } + + // Add entries for modules directly required by go.mod. + if addDirect { + for m := range index.require { + var kept module.Version + if r := Replacement(m); r.Path != "" { + kept = r + } else { + kept = m + } + keep[kept] = true + keep[module.Version{Path: kept.Path, Version: kept.Version + "/go.mod"}] = true + } + } + + return keep +} + +// keepSumReqs embeds another Reqs implementation. The Required method +// calls visit for each version in the module graph. +type keepSumReqs struct { + mvs.Reqs + visit func(module.Version) +} + +func (r *keepSumReqs) Required(m module.Version) ([]module.Version, error) { + r.visit(m) + return r.Reqs.Required(m) +} + +func TrimGoSum() { + // Don't retain sums for direct requirements in go.mod. When TrimGoSum is + // called, go.mod has not been updated, and it may contain requirements on + // modules deleted from the build list. + addDirect := false + modfetch.TrimGoSum(keepSums(addDirect)) +} diff --git a/libgo/go/cmd/go/internal/modload/list.go b/libgo/go/cmd/go/internal/modload/list.go index 9400793..3491f94 100644 --- a/libgo/go/cmd/go/internal/modload/list.go +++ b/libgo/go/cmd/go/internal/modload/list.go @@ -5,47 +5,62 @@ package modload import ( + "context" "errors" "fmt" "os" + "runtime" "strings" "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modinfo" - "cmd/go/internal/par" "cmd/go/internal/search" "golang.org/x/mod/module" ) -func ListModules(args []string, listU, listVersions bool) []*modinfo.ModulePublic { - mods := listModules(args, listVersions) - if listU || listVersions { - var work par.Work +func ListModules(ctx context.Context, args []string, listU, listVersions, listRetracted bool) []*modinfo.ModulePublic { + mods := listModules(ctx, args, listVersions, listRetracted) + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + if listU || listVersions || listRetracted { for _, m := range mods { - work.Add(m) + add := func(m *modinfo.ModulePublic) { + sem <- token{} + go func() { + if listU { + addUpdate(ctx, m) + } + if listVersions { + addVersions(ctx, m, listRetracted) + } + if listRetracted || listU { + addRetraction(ctx, m) + } + <-sem + }() + } + + add(m) if m.Replace != nil { - work.Add(m.Replace) + add(m.Replace) } } - work.Do(10, func(item interface{}) { - m := item.(*modinfo.ModulePublic) - if listU { - addUpdate(m) - } - if listVersions { - addVersions(m) - } - }) } + // Fill semaphore channel to wait for all tasks to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } + return mods } -func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { - LoadBuildList() +func listModules(ctx context.Context, args []string, listVersions, listRetracted bool) []*modinfo.ModulePublic { + LoadAllModules(ctx) if len(args) == 0 { - return []*modinfo.ModulePublic{moduleInfo(buildList[0], true)} + return []*modinfo.ModulePublic{moduleInfo(ctx, buildList[0], true, listRetracted)} } var mods []*modinfo.ModulePublic @@ -71,7 +86,13 @@ func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { } } - info, err := Query(path, vers, current, nil) + allowed := CheckAllowed + if IsRevisionQuery(vers) || listRetracted { + // Allow excluded and retracted versions if the user asked for a + // specific revision or used 'go list -retracted'. + allowed = nil + } + info, err := Query(ctx, path, vers, current, allowed) if err != nil { mods = append(mods, &modinfo.ModulePublic{ Path: path, @@ -80,7 +101,8 @@ func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { }) continue } - mods = append(mods, moduleInfo(module.Version{Path: path, Version: info.Version}, false)) + mod := moduleInfo(ctx, module.Version{Path: path, Version: info.Version}, false, listRetracted) + mods = append(mods, mod) continue } @@ -105,7 +127,7 @@ func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { matched = true if !matchedBuildList[i] { matchedBuildList[i] = true - mods = append(mods, moduleInfo(m, true)) + mods = append(mods, moduleInfo(ctx, m, true, listRetracted)) } } } @@ -115,9 +137,10 @@ func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { // Don't make the user provide an explicit '@latest' when they're // explicitly asking what the available versions are. // Instead, resolve the module, even if it isn't an existing dependency. - info, err := Query(arg, "latest", "", nil) + info, err := Query(ctx, arg, "latest", "", nil) if err == nil { - mods = append(mods, moduleInfo(module.Version{Path: arg, Version: info.Version}, false)) + mod := moduleInfo(ctx, module.Version{Path: arg, Version: info.Version}, false, listRetracted) + mods = append(mods, mod) } else { mods = append(mods, &modinfo.ModulePublic{ Path: arg, diff --git a/libgo/go/cmd/go/internal/modload/load.go b/libgo/go/cmd/go/internal/modload/load.go index f5f74ba..90e990d 100644 --- a/libgo/go/cmd/go/internal/modload/load.go +++ b/libgo/go/cmd/go/internal/modload/load.go @@ -4,63 +4,196 @@ package modload +// This file contains the module-mode package loader, as well as some accessory +// functions pertaining to the package import graph. +// +// There are two exported entry points into package loading — LoadPackages and +// ImportFromFiles — both implemented in terms of loadFromRoots, which itself +// manipulates an instance of the loader struct. +// +// Although most of the loading state is maintained in the loader struct, +// one key piece - the build list - is a global, so that it can be modified +// separate from the loading operation, such as during "go get" +// upgrades/downgrades or in "go mod" operations. +// TODO(#40775): It might be nice to make the loader take and return +// a buildList rather than hard-coding use of the global. +// +// Loading is an iterative process. On each iteration, we try to load the +// requested packages and their transitive imports, then try to resolve modules +// for any imported packages that are still missing. +// +// The first step of each iteration identifies a set of “root” packages. +// Normally the root packages are exactly those matching the named pattern +// arguments. However, for the "all" meta-pattern, the final set of packages is +// computed from the package import graph, and therefore cannot be an initial +// input to loading that graph. Instead, the root packages for the "all" pattern +// are those contained in the main module, and allPatternIsRoot parameter to the +// loader instructs it to dynamically expand those roots to the full "all" +// pattern as loading progresses. +// +// The pkgInAll flag on each loadPkg instance tracks whether that +// package is known to match the "all" meta-pattern. +// A package matches the "all" pattern if: +// - it is in the main module, or +// - it is imported by any test in the main module, or +// - it is imported by another package in "all", or +// - the main module specifies a go version ≤ 1.15, and the package is imported +// by a *test of* another package in "all". +// +// When we implement lazy loading, we will record the modules providing packages +// in "all" even when we are only loading individual packages, so we set the +// pkgInAll flag regardless of the whether the "all" pattern is a root. +// (This is necessary to maintain the “import invariant” described in +// https://golang.org/design/36460-lazy-module-loading.) +// +// Because "go mod vendor" prunes out the tests of vendored packages, the +// behavior of the "all" pattern with -mod=vendor in Go 1.11–1.15 is the same +// as the "all" pattern (regardless of the -mod flag) in 1.16+. +// The allClosesOverTests parameter to the loader indicates whether the "all" +// pattern should close over tests (as in Go 1.11–1.15) or stop at only those +// packages transitively imported by the packages and tests in the main module +// ("all" in Go 1.16+ and "go mod vendor" in Go 1.11+). +// +// Note that it is possible for a loaded package NOT to be in "all" even when we +// are loading the "all" pattern. For example, packages that are transitive +// dependencies of other roots named on the command line must be loaded, but are +// not in "all". (The mod_notall test illustrates this behavior.) +// Similarly, if the LoadTests flag is set but the "all" pattern does not close +// over test dependencies, then when we load the test of a package that is in +// "all" but outside the main module, the dependencies of that test will not +// necessarily themselves be in "all". (That configuration does not arise in Go +// 1.11–1.15, but it will be possible in Go 1.16+.) +// +// Loading proceeds from the roots, using a parallel work-queue with a limit on +// the amount of active work (to avoid saturating disks, CPU cores, and/or +// network connections). Each package is added to the queue the first time it is +// imported by another package. When we have finished identifying the imports of +// a package, we add the test for that package if it is needed. A test may be +// needed if: +// - the package matches a root pattern and tests of the roots were requested, or +// - the package is in the main module and the "all" pattern is requested +// (because the "all" pattern includes the dependencies of tests in the main +// module), or +// - the package is in "all" and the definition of "all" we are using includes +// dependencies of tests (as is the case in Go ≤1.15). +// +// After all available packages have been loaded, we examine the results to +// identify any requested or imported packages that are still missing, and if +// so, which modules we could add to the module graph in order to make the +// missing packages available. We add those to the module graph and iterate, +// until either all packages resolve successfully or we cannot identify any +// module that would resolve any remaining missing package. +// +// If the main module is “tidy” (that is, if "go mod tidy" is a no-op for it) +// and all requested packages are in "all", then loading completes in a single +// iteration. +// TODO(bcmills): We should also be able to load in a single iteration if the +// requested packages all come from modules that are themselves tidy, regardless +// of whether those packages are in "all". Today, that requires two iterations +// if those packages are not found in existing dependencies of the main module. + import ( "bytes" - "cmd/go/internal/base" - "cmd/go/internal/cfg" - "cmd/go/internal/imports" - "cmd/go/internal/modfetch" - "cmd/go/internal/mvs" - "cmd/go/internal/par" - "cmd/go/internal/search" - "cmd/go/internal/str" + "context" "errors" "fmt" "go/build" + "io/fs" "os" "path" pathpkg "path" "path/filepath" + "reflect" + "runtime" "sort" "strings" + "sync" + "sync/atomic" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/fsys" + "cmd/go/internal/imports" + "cmd/go/internal/modfetch" + "cmd/go/internal/mvs" + "cmd/go/internal/par" + "cmd/go/internal/search" + "cmd/go/internal/str" "golang.org/x/mod/module" ) -// buildList is the list of modules to use for building packages. -// It is initialized by calling ImportPaths, ImportFromFiles, -// LoadALL, or LoadBuildList, each of which uses loaded.load. -// -// Ideally, exactly ONE of those functions would be called, -// and exactly once. Most of the time, that's true. -// During "go get" it may not be. TODO(rsc): Figure out if -// that restriction can be established, or else document why not. -// -var buildList []module.Version - // loaded is the most recently-used package loader. // It holds details about individual packages. -// -// Note that loaded.buildList is only valid during a load operation; -// afterward, it is copied back into the global buildList, -// which should be used instead. var loaded *loader -// ImportPaths returns the set of packages matching the args (patterns), -// on the target platform. Modules may be added to the build list -// to satisfy new imports. -func ImportPaths(patterns []string) []*search.Match { - matches := ImportPathsQuiet(patterns, imports.Tags()) - search.WarnUnmatched(matches) - return matches +// PackageOpts control the behavior of the LoadPackages function. +type PackageOpts struct { + // Tags are the build tags in effect (as interpreted by the + // cmd/go/internal/imports package). + // If nil, treated as equivalent to imports.Tags(). + Tags map[string]bool + + // ResolveMissingImports indicates that we should attempt to add module + // dependencies as needed to resolve imports of packages that are not found. + // + // For commands that support the -mod flag, resolving imports may still fail + // if the flag is set to "readonly" (the default) or "vendor". + ResolveMissingImports bool + + // AllowPackage, if non-nil, is called after identifying the module providing + // each package. If AllowPackage returns a non-nil error, that error is set + // for the package, and the imports and test of that package will not be + // loaded. + // + // AllowPackage may be invoked concurrently by multiple goroutines, + // and may be invoked multiple times for a given package path. + AllowPackage func(ctx context.Context, path string, mod module.Version) error + + // LoadTests loads the test dependencies of each package matching a requested + // pattern. If ResolveMissingImports is also true, test dependencies will be + // resolved if missing. + LoadTests bool + + // UseVendorAll causes the "all" package pattern to be interpreted as if + // running "go mod vendor" (or building with "-mod=vendor"). + // + // This is a no-op for modules that declare 'go 1.16' or higher, for which this + // is the default (and only) interpretation of the "all" pattern in module mode. + UseVendorAll bool + + // AllowErrors indicates that LoadPackages should not terminate the process if + // an error occurs. + AllowErrors bool + + // SilenceErrors indicates that LoadPackages should not print errors + // that occur while loading packages. SilenceErrors implies AllowErrors. + SilenceErrors bool + + // SilenceUnmatchedWarnings suppresses the warnings normally emitted for + // patterns that did not match any packages. + SilenceUnmatchedWarnings bool } -// ImportPathsQuiet is like ImportPaths but does not warn about patterns with -// no matches. It also lets the caller specify a set of build tags to match -// packages. The build tags should typically be imports.Tags() or -// imports.AnyTags(); a nil map has no special meaning. -func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { - updateMatches := func(matches []*search.Match, iterating bool) { +// LoadPackages identifies the set of packages matching the given patterns and +// loads the packages in the import graph rooted at that set. +func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (matches []*search.Match, loadedPackages []string) { + LoadModFile(ctx) + if opts.Tags == nil { + opts.Tags = imports.Tags() + } + + patterns = search.CleanPatterns(patterns) + matches = make([]*search.Match, 0, len(patterns)) + allPatternIsRoot := false + for _, pattern := range patterns { + matches = append(matches, search.NewMatch(pattern)) + if pattern == "all" { + allPatternIsRoot = true + } + } + + updateMatches := func(ld *loader) { for _, m := range matches { switch { case m.IsLocal(): @@ -87,7 +220,7 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { // indicates that. ModRoot() - if !iterating { + if ld != nil { m.AddError(err) } continue @@ -100,19 +233,18 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { case strings.Contains(m.Pattern(), "..."): m.Errs = m.Errs[:0] - matchPackages(m, loaded.tags, includeStd, buildList) + matchPackages(ctx, m, opts.Tags, includeStd, buildList) case m.Pattern() == "all": - loaded.testAll = true - if iterating { - // Enumerate the packages in the main module. - // We'll load the dependencies as we find them. + if ld == nil { + // The initial roots are the packages in the main module. + // loadFromRoots will expand that to "all". m.Errs = m.Errs[:0] - matchPackages(m, loaded.tags, omitStd, []module.Version{Target}) + matchPackages(ctx, m, opts.Tags, omitStd, []module.Version{Target}) } else { // Starting with the packages in the main module, // enumerate the full list of "all". - m.Pkgs = loaded.computePatternAll(m.Pkgs) + m.Pkgs = ld.computePatternAll() } case m.Pattern() == "std" || m.Pattern() == "cmd": @@ -126,49 +258,71 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { } } - InitMod() + loaded = loadFromRoots(loaderParams{ + PackageOpts: opts, - var matches []*search.Match - for _, pattern := range search.CleanPatterns(patterns) { - matches = append(matches, search.NewMatch(pattern)) - } + allClosesOverTests: index.allPatternClosesOverTests() && !opts.UseVendorAll, + allPatternIsRoot: allPatternIsRoot, - loaded = newLoader(tags) - loaded.load(func() []string { - var roots []string - updateMatches(matches, true) - for _, m := range matches { - roots = append(roots, m.Pkgs...) - } - return roots + listRoots: func() (roots []string) { + updateMatches(nil) + for _, m := range matches { + roots = append(roots, m.Pkgs...) + } + return roots + }, }) // One last pass to finalize wildcards. - updateMatches(matches, false) - checkMultiplePaths() - WriteGoMod() + updateMatches(loaded) - return matches -} + // Report errors, if any. + checkMultiplePaths() + for _, pkg := range loaded.pkgs { + if pkg.err != nil { + if pkg.flags.has(pkgInAll) { + if imErr := (*ImportMissingError)(nil); errors.As(pkg.err, &imErr) { + imErr.inAll = true + } else if sumErr := (*ImportMissingSumError)(nil); errors.As(pkg.err, &sumErr) { + sumErr.inAll = true + } + } -// checkMultiplePaths verifies that a given module path is used as itself -// or as a replacement for another module, but not both at the same time. -// -// (See https://golang.org/issue/26607 and https://golang.org/issue/34650.) -func checkMultiplePaths() { - firstPath := make(map[module.Version]string, len(buildList)) - for _, mod := range buildList { - src := mod - if rep := Replacement(mod); rep.Path != "" { - src = rep + if !opts.SilenceErrors { + if opts.AllowErrors { + fmt.Fprintf(os.Stderr, "%s: %v\n", pkg.stackText(), pkg.err) + } else { + base.Errorf("%s: %v", pkg.stackText(), pkg.err) + } + } } - if prev, ok := firstPath[src]; !ok { - firstPath[src] = mod.Path - } else if prev != mod.Path { - base.Errorf("go: %s@%s used for two different module paths (%s and %s)", src.Path, src.Version, prev, mod.Path) + if !pkg.isTest() { + loadedPackages = append(loadedPackages, pkg.path) + } + } + if !opts.SilenceErrors { + // Also list errors in matching patterns (such as directory permission + // errors for wildcard patterns). + for _, match := range matches { + for _, err := range match.Errs { + if opts.AllowErrors { + fmt.Fprintf(os.Stderr, "%v\n", err) + } else { + base.Errorf("%v", err) + } + } } } base.ExitIfErrors() + + if !opts.SilenceUnmatchedWarnings { + search.WarnUnmatched(matches) + } + + // Success! Update go.mod (if needed) and return the results. + WriteGoMod() + sort.Strings(loadedPackages) + return matches, loadedPackages } // matchLocalDirs is like m.MatchDirs, but tries to avoid scanning directories @@ -217,11 +371,11 @@ func resolveLocalPackage(dir string) (string, error) { // If the named directory does not exist or contains no Go files, // the package does not exist. // Other errors may affect package loading, but not resolution. - if _, err := os.Stat(absDir); err != nil { + if _, err := fsys.Stat(absDir); err != nil { if os.IsNotExist(err) { // Canonicalize OS-specific errors to errDirectoryNotFound so that error // messages will be easier for users to search for. - return "", &os.PathError{Op: "stat", Path: absDir, Err: errDirectoryNotFound} + return "", &fs.PathError{Op: "stat", Path: absDir, Err: errDirectoryNotFound} } return "", err } @@ -307,7 +461,7 @@ var ( // pathInModuleCache returns the import path of the directory dir, // if dir is in the module cache copy of a module in our build list. func pathInModuleCache(dir string) string { - for _, m := range buildList[1:] { + tryMod := func(m module.Version) (string, bool) { var root string var err error if repl := Replacement(m); repl.Path != "" && repl.Version == "" { @@ -321,13 +475,26 @@ func pathInModuleCache(dir string) string { root, err = modfetch.DownloadDir(m) } if err != nil { - continue + return "", false } - if sub := search.InDir(dir, root); sub != "" { - sub = filepath.ToSlash(sub) - if !strings.Contains(sub, "/vendor/") && !strings.HasPrefix(sub, "vendor/") && !strings.Contains(sub, "@") { - return path.Join(m.Path, filepath.ToSlash(sub)) - } + + sub := search.InDir(dir, root) + if sub == "" { + return "", false + } + sub = filepath.ToSlash(sub) + if strings.Contains(sub, "/vendor/") || strings.HasPrefix(sub, "vendor/") || strings.Contains(sub, "@") { + return "", false + } + + return path.Join(m.Path, filepath.ToSlash(sub)), true + } + + for _, m := range buildList[1:] { + if importPath, ok := tryMod(m); ok { + // checkMultiplePaths ensures that a module can be used for at most one + // requirement, so this must be it. + return importPath } } return "" @@ -335,8 +502,8 @@ func pathInModuleCache(dir string) string { // ImportFromFiles adds modules to the build list as needed // to satisfy the imports in the named Go source files. -func ImportFromFiles(gofiles []string) { - InitMod() +func ImportFromFiles(ctx context.Context, gofiles []string) { + LoadModFile(ctx) tags := imports.Tags() imports, testImports, err := imports.ScanFiles(gofiles, tags) @@ -344,12 +511,17 @@ func ImportFromFiles(gofiles []string) { base.Fatalf("go: %v", err) } - loaded = newLoader(tags) - loaded.load(func() []string { - var roots []string - roots = append(roots, imports...) - roots = append(roots, testImports...) - return roots + loaded = loadFromRoots(loaderParams{ + PackageOpts: PackageOpts{ + Tags: tags, + ResolveMissingImports: true, + }, + allClosesOverTests: index.allPatternClosesOverTests(), + listRoots: func() (roots []string) { + roots = append(roots, imports...) + roots = append(roots, testImports...) + return roots + }, }) WriteGoMod() } @@ -357,9 +529,10 @@ func ImportFromFiles(gofiles []string) { // DirImportPath returns the effective import path for dir, // provided it is within the main module, or else returns ".". func DirImportPath(dir string) string { - if modRoot == "" { + if !HasModRoot() { return "." } + LoadModFile(context.TODO()) if !filepath.IsAbs(dir) { dir = filepath.Join(base.Cwd, dir) @@ -380,130 +553,20 @@ func DirImportPath(dir string) string { return "." } -// LoadBuildList loads and returns the build list from go.mod. -// The loading of the build list happens automatically in ImportPaths: -// LoadBuildList need only be called if ImportPaths is not -// (typically in commands that care about the module but -// no particular package). -func LoadBuildList() []module.Version { - InitMod() - ReloadBuildList() - WriteGoMod() - return buildList -} - -func ReloadBuildList() []module.Version { - loaded = newLoader(imports.Tags()) - loaded.load(func() []string { return nil }) - return buildList -} - -// LoadALL returns the set of all packages in the current module -// and their dependencies in any other modules, without filtering -// due to build tags, except "+build ignore". -// It adds modules to the build list as needed to satisfy new imports. -// This set is useful for deciding whether a particular import is needed -// anywhere in a module. -func LoadALL() []string { - return loadAll(true) -} - -// LoadVendor is like LoadALL but only follows test dependencies -// for tests in the main module. Tests in dependency modules are -// ignored completely. -// This set is useful for identifying the which packages to include in a vendor directory. -func LoadVendor() []string { - return loadAll(false) -} - -func loadAll(testAll bool) []string { - InitMod() - - loaded = newLoader(imports.AnyTags()) - loaded.isALL = true - loaded.testAll = testAll - if !testAll { - loaded.testRoots = true - } - all := TargetPackages("...") - loaded.load(func() []string { return all.Pkgs }) - checkMultiplePaths() - WriteGoMod() - - var paths []string - for _, pkg := range loaded.pkgs { - if pkg.err != nil { - base.Errorf("%s: %v", pkg.stackText(), pkg.err) - continue - } - paths = append(paths, pkg.path) - } - for _, err := range all.Errs { - base.Errorf("%v", err) - } - base.ExitIfErrors() - return paths -} - // TargetPackages returns the list of packages in the target (top-level) module // matching pattern, which may be relative to the working directory, under all // build tag settings. -func TargetPackages(pattern string) *search.Match { +func TargetPackages(ctx context.Context, pattern string) *search.Match { // TargetPackages is relative to the main module, so ensure that the main // module is a thing that can contain packages. + LoadModFile(ctx) ModRoot() m := search.NewMatch(pattern) - matchPackages(m, imports.AnyTags(), omitStd, []module.Version{Target}) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{Target}) return m } -// BuildList returns the module build list, -// typically constructed by a previous call to -// LoadBuildList or ImportPaths. -// The caller must not modify the returned list. -func BuildList() []module.Version { - return buildList -} - -// SetBuildList sets the module build list. -// The caller is responsible for ensuring that the list is valid. -// SetBuildList does not retain a reference to the original list. -func SetBuildList(list []module.Version) { - buildList = append([]module.Version{}, list...) -} - -// TidyBuildList trims the build list to the minimal requirements needed to -// retain the same versions of all packages from the preceding Load* or -// ImportPaths* call. -func TidyBuildList() { - used := map[module.Version]bool{Target: true} - for _, pkg := range loaded.pkgs { - used[pkg.mod] = true - } - - keep := []module.Version{Target} - var direct []string - for _, m := range buildList[1:] { - if used[m] { - keep = append(keep, m) - if loaded.direct[m.Path] { - direct = append(direct, m.Path) - } - } else if cfg.BuildV { - if _, ok := index.require[m]; ok { - fmt.Fprintf(os.Stderr, "unused %s\n", m.Path) - } - } - } - - min, err := mvs.Req(Target, direct, &mvsReqs{buildList: keep}) - if err != nil { - base.Fatalf("go: %v", err) - } - buildList = append([]module.Version{Target}, min...) -} - // ImportMap returns the actual package import path // for an import path found in source code. // If the given import path does not appear in the source code @@ -558,12 +621,6 @@ func PackageImports(path string) (imports, testImports []string) { return imports, testImports } -// ModuleUsedDirectly reports whether the main module directly imports -// some package in the module with the given path. -func ModuleUsedDirectly(path string) bool { - return loaded.direct[path] -} - // Lookup returns the source directory, import path, and any loading error for // the package at path as imported from the package in parentDir. // Lookup requires that one of the Load functions in this package has already @@ -599,129 +656,193 @@ func Lookup(parentPath string, parentIsStd bool, path string) (dir, realPath str // the required packages for a particular build, // checking that the packages are available in the module set, // and updating the module set if needed. -// Loading is an iterative process: try to load all the needed packages, -// but if imports are missing, try to resolve those imports, and repeat. -// -// Although most of the loading state is maintained in the loader struct, -// one key piece - the build list - is a global, so that it can be modified -// separate from the loading operation, such as during "go get" -// upgrades/downgrades or in "go mod" operations. -// TODO(rsc): It might be nice to make the loader take and return -// a buildList rather than hard-coding use of the global. type loader struct { - tags map[string]bool // tags for scanDir - testRoots bool // include tests for roots - isALL bool // created with LoadALL - testAll bool // include tests for all packages - forceStdVendor bool // if true, load standard-library dependencies from the vendor subtree + loaderParams + + work *par.Queue // reset on each iteration roots []*loadPkg - pkgs []*loadPkg - work *par.Work // current work queue - pkgCache *par.Cache // map from string to *loadPkg + pkgCache *par.Cache // package path (string) → *loadPkg + pkgs []*loadPkg // transitive closure of loaded packages and tests; populated in buildStacks // computed at end of iterations - direct map[string]bool // imported directly by main module - goVersion map[string]string // go version recorded in each module + direct map[string]bool // imported directly by main module } -// LoadTests controls whether the loaders load tests of the root packages. -var LoadTests bool - -func newLoader(tags map[string]bool) *loader { - ld := new(loader) - ld.tags = tags - ld.testRoots = LoadTests +// loaderParams configure the packages loaded by, and the properties reported +// by, a loader instance. +type loaderParams struct { + PackageOpts - // Inside the "std" and "cmd" modules, we prefer to use the vendor directory - // unless the command explicitly changes the module graph. - if !targetInGorootSrc || (cfg.CmdName != "get" && !strings.HasPrefix(cfg.CmdName, "mod ")) { - ld.forceStdVendor = true - } + allClosesOverTests bool // Does the "all" pattern include the transitive closure of tests of packages in "all"? + allPatternIsRoot bool // Is the "all" pattern an additional root? - return ld + listRoots func() []string } func (ld *loader) reset() { + select { + case <-ld.work.Idle(): + default: + panic("loader.reset when not idle") + } + ld.roots = nil - ld.pkgs = nil - ld.work = new(par.Work) ld.pkgCache = new(par.Cache) + ld.pkgs = nil } // A loadPkg records information about a single loaded package. type loadPkg struct { - path string // import path + // Populated at construction time: + path string // import path + testOf *loadPkg + + // Populated at construction time and updated by (*loader).applyPkgFlags: + flags atomicLoadPkgFlags + + // Populated by (*loader).load: mod module.Version // module providing package dir string // directory containing source code - imports []*loadPkg // packages imported by this one err error // error loading package - stack *loadPkg // package importing this one in minimal import stack for this pkg - test *loadPkg // package with test imports, if we need test - testOf *loadPkg - testImports []string // test-only imports, saved for use by pkg.test. + imports []*loadPkg // packages imported by this one + testImports []string // test-only imports, saved for use by pkg.test. + inStd bool + + // Populated by (*loader).pkgTest: + testOnce sync.Once + test *loadPkg + + // Populated by postprocessing in (*loader).buildStacks: + stack *loadPkg // package importing this one in minimal import stack for this pkg +} + +// loadPkgFlags is a set of flags tracking metadata about a package. +type loadPkgFlags int8 + +const ( + // pkgInAll indicates that the package is in the "all" package pattern, + // regardless of whether we are loading the "all" package pattern. + // + // When the pkgInAll flag and pkgImportsLoaded flags are both set, the caller + // who set the last of those flags must propagate the pkgInAll marking to all + // of the imports of the marked package. + // + // A test is marked with pkgInAll if that test would promote the packages it + // imports to be in "all" (such as when the test is itself within the main + // module, or when ld.allClosesOverTests is true). + pkgInAll loadPkgFlags = 1 << iota + + // pkgIsRoot indicates that the package matches one of the root package + // patterns requested by the caller. + // + // If LoadTests is set, then when pkgIsRoot and pkgImportsLoaded are both set, + // the caller who set the last of those flags must populate a test for the + // package (in the pkg.test field). + // + // If the "all" pattern is included as a root, then non-test packages in "all" + // are also roots (and must be marked pkgIsRoot). + pkgIsRoot + + // pkgImportsLoaded indicates that the imports and testImports fields of a + // loadPkg have been populated. + pkgImportsLoaded +) + +// has reports whether all of the flags in cond are set in f. +func (f loadPkgFlags) has(cond loadPkgFlags) bool { + return f&cond == cond +} + +// An atomicLoadPkgFlags stores a loadPkgFlags for which individual flags can be +// added atomically. +type atomicLoadPkgFlags struct { + bits int32 +} + +// update sets the given flags in af (in addition to any flags already set). +// +// update returns the previous flag state so that the caller may determine which +// flags were newly-set. +func (af *atomicLoadPkgFlags) update(flags loadPkgFlags) (old loadPkgFlags) { + for { + old := atomic.LoadInt32(&af.bits) + new := old | int32(flags) + if new == old || atomic.CompareAndSwapInt32(&af.bits, old, new) { + return loadPkgFlags(old) + } + } +} + +// has reports whether all of the flags in cond are set in af. +func (af *atomicLoadPkgFlags) has(cond loadPkgFlags) bool { + return loadPkgFlags(atomic.LoadInt32(&af.bits))&cond == cond +} + +// isTest reports whether pkg is a test of another package. +func (pkg *loadPkg) isTest() bool { + return pkg.testOf != nil } var errMissing = errors.New("cannot find package") -// load attempts to load the build graph needed to process a set of root packages. -// The set of root packages is defined by the addRoots function, -// which must call add(path) with the import path of each root package. -func (ld *loader) load(roots func() []string) { +// loadFromRoots attempts to load the build graph needed to process a set of +// root packages and their dependencies. +// +// The set of root packages is returned by the params.listRoots function, and +// expanded to the full set of packages by tracing imports (and possibly tests) +// as needed. +func loadFromRoots(params loaderParams) *loader { + ld := &loader{ + loaderParams: params, + work: par.NewQueue(runtime.GOMAXPROCS(0)), + } + var err error - reqs := Reqs() + reqs := &mvsReqs{buildList: buildList} buildList, err = mvs.BuildList(Target, reqs) if err != nil { base.Fatalf("go: %v", err) } - added := make(map[string]bool) + addedModuleFor := make(map[string]bool) for { ld.reset() - if roots != nil { - // Note: the returned roots can change on each iteration, - // since the expansion of package patterns depends on the - // build list we're using. - for _, path := range roots() { - ld.work.Add(ld.pkg(path, true)) + + // Load the root packages and their imports. + // Note: the returned roots can change on each iteration, + // since the expansion of package patterns depends on the + // build list we're using. + inRoots := map[*loadPkg]bool{} + for _, path := range ld.listRoots() { + root := ld.pkg(path, pkgIsRoot) + if !inRoots[root] { + ld.roots = append(ld.roots, root) + inRoots[root] = true } } - ld.work.Do(10, ld.doPkg) + + // ld.pkg adds imported packages to the work queue and calls applyPkgFlags, + // which adds tests (and test dependencies) as needed. + // + // When all of the work in the queue has completed, we'll know that the + // transitive closure of dependencies has been loaded. + <-ld.work.Idle() + ld.buildStacks() - numAdded := 0 - haveMod := make(map[module.Version]bool) - for _, m := range buildList { - haveMod[m] = true - } - modAddedBy := make(map[module.Version]*loadPkg) - for _, pkg := range ld.pkgs { - if err, ok := pkg.err.(*ImportMissingError); ok && err.Module.Path != "" { - if err.newMissingVersion != "" { - base.Fatalf("go: %s: package provided by %s at latest version %s but not at required version %s", pkg.stackText(), err.Module.Path, err.Module.Version, err.newMissingVersion) - } - fmt.Fprintf(os.Stderr, "go: found %s in %s %s\n", pkg.path, err.Module.Path, err.Module.Version) - if added[pkg.path] { - base.Fatalf("go: %s: looping trying to add package", pkg.stackText()) - } - added[pkg.path] = true - numAdded++ - if !haveMod[err.Module] { - haveMod[err.Module] = true - modAddedBy[err.Module] = pkg - buildList = append(buildList, err.Module) - } - continue - } - // Leave other errors for Import or load.Packages to report. + + if !ld.ResolveMissingImports || (!HasModRoot() && !allowMissingModuleImports) { + // We've loaded as much as we can without resolving missing imports. + break } - base.ExitIfErrors() - if numAdded == 0 { + modAddedBy := ld.resolveMissingImports(addedModuleFor) + if len(modAddedBy) == 0 { break } // Recompute buildList with all our additions. - reqs = Reqs() + reqs = &mvsReqs{buildList: buildList} buildList, err = mvs.BuildList(Target, reqs) if err != nil { // If an error was found in a newly added module, report the package @@ -749,102 +870,287 @@ func (ld *loader) load(roots func() []string) { } } - // Add Go versions, computed during walk. - ld.goVersion = make(map[string]string) - for _, m := range buildList { - v, _ := reqs.(*mvsReqs).versions.Load(m) - ld.goVersion[m.Path], _ = v.(string) - } - - // Mix in direct markings (really, lack of indirect markings) - // from go.mod, unless we scanned the whole module - // and can therefore be sure we know better than go.mod. - if !ld.isALL && modFile != nil { + // If we didn't scan all of the imports from the main module, or didn't use + // imports.AnyTags, then we didn't necessarily load every package that + // contributes “direct” imports — so we can't safely mark existing + // dependencies as indirect-only. + // Conservatively mark those dependencies as direct. + if modFile != nil && (!ld.allPatternIsRoot || !reflect.DeepEqual(ld.Tags, imports.AnyTags())) { for _, r := range modFile.Require { if !r.Indirect { ld.direct[r.Mod.Path] = true } } } + + return ld } -// pkg returns the *loadPkg for path, creating and queuing it if needed. -// If the package should be tested, its test is created but not queued -// (the test is queued after processing pkg). -// If isRoot is true, the pkg is being queued as one of the roots of the work graph. -func (ld *loader) pkg(path string, isRoot bool) *loadPkg { - return ld.pkgCache.Do(path, func() interface{} { - pkg := &loadPkg{ - path: path, +// resolveMissingImports adds module dependencies to the global build list +// in order to resolve missing packages from pkgs. +// +// The newly-resolved packages are added to the addedModuleFor map, and +// resolveMissingImports returns a map from each newly-added module version to +// the first package for which that module was added. +func (ld *loader) resolveMissingImports(addedModuleFor map[string]bool) (modAddedBy map[module.Version]*loadPkg) { + var needPkgs []*loadPkg + for _, pkg := range ld.pkgs { + if pkg.err == nil { + continue } - if ld.testRoots && isRoot || ld.testAll { - test := &loadPkg{ - path: path, - testOf: pkg, - } - pkg.test = test + if pkg.isTest() { + // If we are missing a test, we are also missing its non-test version, and + // we should only add the missing import once. + continue + } + if !errors.As(pkg.err, new(*ImportMissingError)) { + // Leave other errors for Import or load.Packages to report. + continue + } + + needPkgs = append(needPkgs, pkg) + + pkg := pkg + ld.work.Add(func() { + pkg.mod, pkg.err = queryImport(context.TODO(), pkg.path) + }) + } + <-ld.work.Idle() + + modAddedBy = map[module.Version]*loadPkg{} + for _, pkg := range needPkgs { + if pkg.err != nil { + continue + } + + fmt.Fprintf(os.Stderr, "go: found %s in %s %s\n", pkg.path, pkg.mod.Path, pkg.mod.Version) + if addedModuleFor[pkg.path] { + // TODO(bcmills): This should only be an error if pkg.mod is the same + // version we already tried to add previously. + base.Fatalf("go: %s: looping trying to add package", pkg.stackText()) } - if isRoot { - ld.roots = append(ld.roots, pkg) + if modAddedBy[pkg.mod] == nil { + modAddedBy[pkg.mod] = pkg + buildList = append(buildList, pkg.mod) } - ld.work.Add(pkg) + } + + return modAddedBy +} + +// pkg locates the *loadPkg for path, creating and queuing it for loading if +// needed, and updates its state to reflect the given flags. +// +// The imports of the returned *loadPkg will be loaded asynchronously in the +// ld.work queue, and its test (if requested) will also be populated once +// imports have been resolved. When ld.work goes idle, all transitive imports of +// the requested package (and its test, if requested) will have been loaded. +func (ld *loader) pkg(path string, flags loadPkgFlags) *loadPkg { + if flags.has(pkgImportsLoaded) { + panic("internal error: (*loader).pkg called with pkgImportsLoaded flag set") + } + + pkg := ld.pkgCache.Do(path, func() interface{} { + pkg := &loadPkg{ + path: path, + } + ld.applyPkgFlags(pkg, flags) + + ld.work.Add(func() { ld.load(pkg) }) return pkg }).(*loadPkg) + + ld.applyPkgFlags(pkg, flags) + return pkg } -// doPkg processes a package on the work queue. -func (ld *loader) doPkg(item interface{}) { - // TODO: what about replacements? - pkg := item.(*loadPkg) - var imports []string - if pkg.testOf != nil { - pkg.dir = pkg.testOf.dir - pkg.mod = pkg.testOf.mod - imports = pkg.testOf.testImports - } else { - if strings.Contains(pkg.path, "@") { - // Leave for error during load. - return +// applyPkgFlags updates pkg.flags to set the given flags and propagate the +// (transitive) effects of those flags, possibly loading or enqueueing further +// packages as a result. +func (ld *loader) applyPkgFlags(pkg *loadPkg, flags loadPkgFlags) { + if flags == 0 { + return + } + + if flags.has(pkgInAll) && ld.allPatternIsRoot && !pkg.isTest() { + // This package matches a root pattern by virtue of being in "all". + flags |= pkgIsRoot + } + + old := pkg.flags.update(flags) + new := old | flags + if new == old || !new.has(pkgImportsLoaded) { + // We either didn't change the state of pkg, or we don't know anything about + // its dependencies yet. Either way, we can't usefully load its test or + // update its dependencies. + return + } + + if !pkg.isTest() { + // Check whether we should add (or update the flags for) a test for pkg. + // ld.pkgTest is idempotent and extra invocations are inexpensive, + // so it's ok if we call it more than is strictly necessary. + wantTest := false + switch { + case ld.allPatternIsRoot && pkg.mod == Target: + // We are loading the "all" pattern, which includes packages imported by + // tests in the main module. This package is in the main module, so we + // need to identify the imports of its test even if LoadTests is not set. + // + // (We will filter out the extra tests explicitly in computePatternAll.) + wantTest = true + + case ld.allPatternIsRoot && ld.allClosesOverTests && new.has(pkgInAll): + // This variant of the "all" pattern includes imports of tests of every + // package that is itself in "all", and pkg is in "all", so its test is + // also in "all" (as above). + wantTest = true + + case ld.LoadTests && new.has(pkgIsRoot): + // LoadTest explicitly requests tests of “the root packages”. + wantTest = true } - if build.IsLocalImport(pkg.path) || filepath.IsAbs(pkg.path) { - // Leave for error during load. - // (Module mode does not allow local imports.) - return + + if wantTest { + var testFlags loadPkgFlags + if pkg.mod == Target || (ld.allClosesOverTests && new.has(pkgInAll)) { + // Tests of packages in the main module are in "all", in the sense that + // they cause the packages they import to also be in "all". So are tests + // of packages in "all" if "all" closes over test dependencies. + testFlags |= pkgInAll + } + ld.pkgTest(pkg, testFlags) } + } - pkg.mod, pkg.dir, pkg.err = Import(pkg.path) - if pkg.dir == "" { - return + if new.has(pkgInAll) && !old.has(pkgInAll|pkgImportsLoaded) { + // We have just marked pkg with pkgInAll, or we have just loaded its + // imports, or both. Now is the time to propagate pkgInAll to the imports. + for _, dep := range pkg.imports { + ld.applyPkgFlags(dep, pkgInAll) } - if cfg.BuildContext.Compiler == "gccgo" && pkg.mod.Path == "" { - return + } +} + +// load loads an individual package. +func (ld *loader) load(pkg *loadPkg) { + if strings.Contains(pkg.path, "@") { + // Leave for error during load. + return + } + if build.IsLocalImport(pkg.path) || filepath.IsAbs(pkg.path) { + // Leave for error during load. + // (Module mode does not allow local imports.) + return + } + + if search.IsMetaPackage(pkg.path) { + pkg.err = &invalidImportError{ + importPath: pkg.path, + err: fmt.Errorf("%q is not an importable package; see 'go help packages'", pkg.path), + } + return + } + + pkg.mod, pkg.dir, pkg.err = importFromBuildList(context.TODO(), pkg.path, buildList) + if pkg.dir == "" { + return + } + if pkg.mod == Target { + // Go ahead and mark pkg as in "all". This provides the invariant that a + // package that is *only* imported by other packages in "all" is always + // marked as such before loading its imports. + // + // We don't actually rely on that invariant at the moment, but it may + // improve efficiency somewhat and makes the behavior a bit easier to reason + // about (by reducing churn on the flag bits of dependencies), and costs + // essentially nothing (these atomic flag ops are essentially free compared + // to scanning source code for imports). + ld.applyPkgFlags(pkg, pkgInAll) + } + if ld.AllowPackage != nil { + if err := ld.AllowPackage(context.TODO(), pkg.path, pkg.mod); err != nil { + pkg.err = err } + } - var testImports []string + pkg.inStd = (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "") + + var imports, testImports []string + + if cfg.BuildContext.Compiler == "gccgo" && pkg.inStd { + // We can't scan standard packages for gccgo. + } else { var err error - imports, testImports, err = scanDir(pkg.dir, ld.tags) + imports, testImports, err = scanDir(pkg.dir, ld.Tags) if err != nil { pkg.err = err return } - if pkg.test != nil { - pkg.testImports = testImports - } } - inStd := (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "") + pkg.imports = make([]*loadPkg, 0, len(imports)) + var importFlags loadPkgFlags + if pkg.flags.has(pkgInAll) { + importFlags = pkgInAll + } for _, path := range imports { - if inStd { + if pkg.inStd { + // Imports from packages in "std" and "cmd" should resolve using + // GOROOT/src/vendor even when "std" is not the main module. path = ld.stdVendor(pkg.path, path) } - pkg.imports = append(pkg.imports, ld.pkg(path, false)) + pkg.imports = append(pkg.imports, ld.pkg(path, importFlags)) } + pkg.testImports = testImports - // Now that pkg.dir, pkg.mod, pkg.testImports are set, we can queue pkg.test. - // TODO: All that's left is creating new imports. Why not just do it now? - if pkg.test != nil { - ld.work.Add(pkg.test) + ld.applyPkgFlags(pkg, pkgImportsLoaded) +} + +// pkgTest locates the test of pkg, creating it if needed, and updates its state +// to reflect the given flags. +// +// pkgTest requires that the imports of pkg have already been loaded (flagged +// with pkgImportsLoaded). +func (ld *loader) pkgTest(pkg *loadPkg, testFlags loadPkgFlags) *loadPkg { + if pkg.isTest() { + panic("pkgTest called on a test package") } + + createdTest := false + pkg.testOnce.Do(func() { + pkg.test = &loadPkg{ + path: pkg.path, + testOf: pkg, + mod: pkg.mod, + dir: pkg.dir, + err: pkg.err, + inStd: pkg.inStd, + } + ld.applyPkgFlags(pkg.test, testFlags) + createdTest = true + }) + + test := pkg.test + if createdTest { + test.imports = make([]*loadPkg, 0, len(pkg.testImports)) + var importFlags loadPkgFlags + if test.flags.has(pkgInAll) { + importFlags = pkgInAll + } + for _, path := range pkg.testImports { + if pkg.inStd { + path = ld.stdVendor(test.path, path) + } + test.imports = append(test.imports, ld.pkg(path, importFlags)) + } + pkg.testImports = nil + ld.applyPkgFlags(test, pkgImportsLoaded) + } else { + ld.applyPkgFlags(test, testFlags) + } + + return test } // stdVendor returns the canonical import path for the package with the given @@ -855,13 +1161,25 @@ func (ld *loader) stdVendor(parentPath, path string) string { } if str.HasPathPrefix(parentPath, "cmd") { - if ld.forceStdVendor || Target.Path != "cmd" { + if Target.Path != "cmd" { vendorPath := pathpkg.Join("cmd", "vendor", path) if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { return vendorPath } } - } else if ld.forceStdVendor || Target.Path != "std" { + } else if Target.Path != "std" || str.HasPathPrefix(parentPath, "vendor") { + // If we are outside of the 'std' module, resolve imports from within 'std' + // to the vendor directory. + // + // Do the same for importers beginning with the prefix 'vendor/' even if we + // are *inside* of the 'std' module: the 'vendor/' packages that resolve + // globally from GOROOT/src/vendor (and are listed as part of 'go list std') + // are distinct from the real module dependencies, and cannot import + // internal packages from the real module. + // + // (Note that although the 'vendor/' packages match the 'std' *package* + // pattern, they are not part of the std *module*, and do not affect + // 'go mod tidy' and similar module commands when working within std.) vendorPath := pathpkg.Join("vendor", path) if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { return vendorPath @@ -874,30 +1192,13 @@ func (ld *loader) stdVendor(parentPath, path string) string { // computePatternAll returns the list of packages matching pattern "all", // starting with a list of the import paths for the packages in the main module. -func (ld *loader) computePatternAll(paths []string) []string { - seen := make(map[*loadPkg]bool) - var all []string - var walk func(*loadPkg) - walk = func(pkg *loadPkg) { - if seen[pkg] { - return - } - seen[pkg] = true - if pkg.testOf == nil { +func (ld *loader) computePatternAll() (all []string) { + for _, pkg := range ld.pkgs { + if pkg.flags.has(pkgInAll) && !pkg.isTest() { all = append(all, pkg.path) } - for _, p := range pkg.imports { - walk(p) - } - if p := pkg.test; p != nil { - walk(p) - } - } - for _, path := range paths { - walk(ld.pkg(path, false)) } sort.Strings(all) - return all } @@ -1019,7 +1320,7 @@ func (pkg *loadPkg) why() string { // Why returns the "go mod why" output stanza for the given package, // without the leading # comment. -// The package graph must have been loaded already, usually by LoadALL. +// The package graph must have been loaded already, usually by LoadPackages. // If there is no reason for the package to be in the current build, // Why returns an empty string. func Why(path string) string { diff --git a/libgo/go/cmd/go/internal/modload/modfile.go b/libgo/go/cmd/go/internal/modload/modfile.go index 9f4ec5a..eb05e9f 100644 --- a/libgo/go/cmd/go/internal/modload/modfile.go +++ b/libgo/go/cmd/go/internal/modload/modfile.go @@ -5,25 +5,45 @@ package modload import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "unicode" + "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/lockedfile" + "cmd/go/internal/modfetch" + "cmd/go/internal/par" + "cmd/go/internal/trace" "golang.org/x/mod/modfile" "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) +// narrowAllVersionV is the Go version (plus leading "v") at which the +// module-module "all" pattern no longer closes over the dependencies of +// tests outside of the main module. +const narrowAllVersionV = "v1.16" +const go116EnableNarrowAll = true + var modFile *modfile.File // A modFileIndex is an index of data corresponding to a modFile // at a specific point in time. type modFileIndex struct { - data []byte - dataNeedsFix bool // true if fixVersion applied a change while parsing data - module module.Version - goVersion string - require map[module.Version]requireMeta - replace map[module.Version]module.Version - exclude map[module.Version]bool + data []byte + dataNeedsFix bool // true if fixVersion applied a change while parsing data + module module.Version + goVersionV string // GoVersion with "v" prefix + require map[module.Version]requireMeta + replace map[module.Version]module.Version + highestReplaced map[string]string // highest replaced version of each module path; empty string for wildcard-only replacements + exclude map[module.Version]bool } // index is the index of the go.mod file as of when it was last read or written. @@ -33,9 +53,179 @@ type requireMeta struct { indirect bool } -// Allowed reports whether module m is allowed (not excluded) by the main module's go.mod. -func Allowed(m module.Version) bool { - return index == nil || !index.exclude[m] +// CheckAllowed returns an error equivalent to ErrDisallowed if m is excluded by +// the main module's go.mod or retracted by its author. Most version queries use +// this to filter out versions that should not be used. +func CheckAllowed(ctx context.Context, m module.Version) error { + if err := CheckExclusions(ctx, m); err != nil { + return err + } + if err := CheckRetractions(ctx, m); err != nil { + return err + } + return nil +} + +// ErrDisallowed is returned by version predicates passed to Query and similar +// functions to indicate that a version should not be considered. +var ErrDisallowed = errors.New("disallowed module version") + +// CheckExclusions returns an error equivalent to ErrDisallowed if module m is +// excluded by the main module's go.mod file. +func CheckExclusions(ctx context.Context, m module.Version) error { + if index != nil && index.exclude[m] { + return module.VersionError(m, errExcluded) + } + return nil +} + +var errExcluded = &excludedError{} + +type excludedError struct{} + +func (e *excludedError) Error() string { return "excluded by go.mod" } +func (e *excludedError) Is(err error) bool { return err == ErrDisallowed } + +// CheckRetractions returns an error if module m has been retracted by +// its author. +func CheckRetractions(ctx context.Context, m module.Version) error { + if m.Version == "" { + // Main module, standard library, or file replacement module. + // Cannot be retracted. + return nil + } + + // Look up retraction information from the latest available version of + // the module. Cache retraction information so we don't parse the go.mod + // file repeatedly. + type entry struct { + retract []retraction + err error + } + path := m.Path + e := retractCache.Do(path, func() (v interface{}) { + ctx, span := trace.StartSpan(ctx, "checkRetractions "+path) + defer span.Done() + + if repl := Replacement(module.Version{Path: m.Path}); repl.Path != "" { + // All versions of the module were replaced with a local directory. + // Don't load retractions. + return &entry{nil, nil} + } + + // Find the latest version of the module. + // Ignore exclusions from the main module's go.mod. + const ignoreSelected = "" + var allowAll AllowedFunc + rev, err := Query(ctx, path, "latest", ignoreSelected, allowAll) + if err != nil { + return &entry{nil, err} + } + + // Load go.mod for that version. + // If the version is replaced, we'll load retractions from the replacement. + // + // If there's an error loading the go.mod, we'll return it here. + // These errors should generally be ignored by callers of checkRetractions, + // since they happen frequently when we're offline. These errors are not + // equivalent to ErrDisallowed, so they may be distinguished from + // retraction errors. + // + // We load the raw file here: the go.mod file may have a different module + // path that we expect if the module or its repository was renamed. + // We still want to apply retractions to other aliases of the module. + rm := module.Version{Path: path, Version: rev.Version} + if repl := Replacement(rm); repl.Path != "" { + rm = repl + } + summary, err := rawGoModSummary(rm) + if err != nil { + return &entry{nil, err} + } + return &entry{summary.retract, nil} + }).(*entry) + + if err := e.err; err != nil { + // Attribute the error to the version being checked, not the version from + // which the retractions were to be loaded. + var mErr *module.ModuleError + if errors.As(err, &mErr) { + err = mErr.Err + } + return &retractionLoadingError{m: m, err: err} + } + + var rationale []string + isRetracted := false + for _, r := range e.retract { + if semver.Compare(r.Low, m.Version) <= 0 && semver.Compare(m.Version, r.High) <= 0 { + isRetracted = true + if r.Rationale != "" { + rationale = append(rationale, r.Rationale) + } + } + } + if isRetracted { + return module.VersionError(m, &ModuleRetractedError{Rationale: rationale}) + } + return nil +} + +var retractCache par.Cache + +type ModuleRetractedError struct { + Rationale []string +} + +func (e *ModuleRetractedError) Error() string { + msg := "retracted by module author" + if len(e.Rationale) > 0 { + // This is meant to be a short error printed on a terminal, so just + // print the first rationale. + msg += ": " + ShortRetractionRationale(e.Rationale[0]) + } + return msg +} + +func (e *ModuleRetractedError) Is(err error) bool { + return err == ErrDisallowed +} + +type retractionLoadingError struct { + m module.Version + err error +} + +func (e *retractionLoadingError) Error() string { + return fmt.Sprintf("loading module retractions for %v: %v", e.m, e.err) +} + +func (e *retractionLoadingError) Unwrap() error { + return e.err +} + +// ShortRetractionRationale returns a retraction rationale string that is safe +// to print in a terminal. It returns hard-coded strings if the rationale +// is empty, too long, or contains non-printable characters. +func ShortRetractionRationale(rationale string) string { + const maxRationaleBytes = 500 + if i := strings.Index(rationale, "\n"); i >= 0 { + rationale = rationale[:i] + } + rationale = strings.TrimSpace(rationale) + if rationale == "" { + return "retracted by module author" + } + if len(rationale) > maxRationaleBytes { + return "(rationale omitted: too long)" + } + for _, r := range rationale { + if !unicode.IsGraphic(r) && !unicode.IsSpace(r) { + return "(rationale omitted: contains non-printable characters)" + } + } + // NOTE: the go.mod parser rejects invalid UTF-8, so we don't check that here. + return rationale } // Replacement returns the replacement for mod, if any, from go.mod. @@ -66,9 +256,11 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd i.module = modFile.Module.Mod } - i.goVersion = "" + i.goVersionV = "" if modFile.Go != nil { - i.goVersion = modFile.Go.Version + // We're going to use the semver package to compare Go versions, so go ahead + // and add the "v" prefix it expects once instead of every time. + i.goVersionV = "v" + modFile.Go.Version } i.require = make(map[module.Version]requireMeta, len(modFile.Require)) @@ -84,6 +276,14 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd i.replace[r.Old] = r.New } + i.highestReplaced = make(map[string]string) + for _, r := range modFile.Replace { + v, ok := i.highestReplaced[r.Old.Path] + if !ok || semver.Compare(r.Old.Version, v) > 0 { + i.highestReplaced[r.Old.Path] = r.Old.Version + } + } + i.exclude = make(map[module.Version]bool, len(modFile.Exclude)) for _, x := range modFile.Exclude { i.exclude[x.Mod] = true @@ -92,6 +292,23 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd return i } +// allPatternClosesOverTests reports whether the "all" pattern includes +// dependencies of tests outside the main module (as in Go 1.11–1.15). +// (Otherwise — as in Go 1.16+ — the "all" pattern includes only the packages +// transitively *imported by* the packages and tests in the main module.) +func (i *modFileIndex) allPatternClosesOverTests() bool { + if !go116EnableNarrowAll { + return true + } + if i != nil && semver.Compare(i.goVersionV, narrowAllVersionV) < 0 { + // The module explicitly predates the change in "all" for lazy loading, so + // continue to use the older interpretation. (If i == nil, we not in any + // module at all and should use the latest semantics.) + return true + } + return false +} + // modFileIsDirty reports whether the go.mod file differs meaningfully // from what was indexed. // If modFile has been changed (even cosmetically) since it was first read, @@ -114,11 +331,11 @@ func (i *modFileIndex) modFileIsDirty(modFile *modfile.File) bool { } if modFile.Go == nil { - if i.goVersion != "" { + if i.goVersionV != "" { return true } - } else if modFile.Go.Version != i.goVersion { - if i.goVersion == "" && cfg.BuildMod == "readonly" { + } else if "v"+modFile.Go.Version != i.goVersionV { + if i.goVersionV == "" && cfg.BuildMod == "readonly" { // go.mod files did not always require a 'go' version, so do not error out // if one is missing — we may be inside an older module in the module // cache, and should bias toward providing useful behavior. @@ -162,3 +379,213 @@ func (i *modFileIndex) modFileIsDirty(modFile *modfile.File) bool { return false } + +// rawGoVersion records the Go version parsed from each module's go.mod file. +// +// If a module is replaced, the version of the replacement is keyed by the +// replacement module.Version, not the version being replaced. +var rawGoVersion sync.Map // map[module.Version]string + +// A modFileSummary is a summary of a go.mod file for which we do not need to +// retain complete information — for example, the go.mod file of a dependency +// module. +type modFileSummary struct { + module module.Version + goVersionV string // GoVersion with "v" prefix + require []module.Version + retract []retraction +} + +// A retraction consists of a retracted version interval and rationale. +// retraction is like modfile.Retract, but it doesn't point to the syntax tree. +type retraction struct { + modfile.VersionInterval + Rationale string +} + +// goModSummary returns a summary of the go.mod file for module m, +// taking into account any replacements for m, exclusions of its dependencies, +// and/or vendoring. +// +// m must be a version in the module graph, reachable from the Target module. +// In readonly mode, the go.sum file must contain an entry for m's go.mod file +// (or its replacement). goModSummary must not be called for the Target module +// itself, as its requirements may change. Use rawGoModSummary for other +// module versions. +// +// The caller must not modify the returned summary. +func goModSummary(m module.Version) (*modFileSummary, error) { + if m == Target { + panic("internal error: goModSummary called on the Target module") + } + + if cfg.BuildMod == "vendor" { + summary := &modFileSummary{ + module: module.Version{Path: m.Path}, + } + if vendorVersion[m.Path] != m.Version { + // This module is not vendored, so packages cannot be loaded from it and + // it cannot be relevant to the build. + return summary, nil + } + + // For every module other than the target, + // return the full list of modules from modules.txt. + readVendorList() + + // TODO(#36876): Load the "go" version from vendor/modules.txt and store it + // in rawGoVersion with the appropriate key. + + // We don't know what versions the vendored module actually relies on, + // so assume that it requires everything. + summary.require = vendorList + return summary, nil + } + + actual := Replacement(m) + if actual.Path == "" { + actual = m + } + if cfg.BuildMod == "readonly" && actual.Version != "" { + key := module.Version{Path: actual.Path, Version: actual.Version + "/go.mod"} + if !modfetch.HaveSum(key) { + suggestion := fmt.Sprintf("; try 'go mod download %s' to add it", m.Path) + return nil, module.VersionError(actual, &sumMissingError{suggestion: suggestion}) + } + } + summary, err := rawGoModSummary(actual) + if err != nil { + return nil, err + } + + if actual.Version == "" { + // The actual module is a filesystem-local replacement, for which we have + // unfortunately not enforced any sort of invariants about module lines or + // matching module paths. Anything goes. + // + // TODO(bcmills): Remove this special-case, update tests, and add a + // release note. + } else { + if summary.module.Path == "" { + return nil, module.VersionError(actual, errors.New("parsing go.mod: missing module line")) + } + + // In theory we should only allow mpath to be unequal to m.Path here if the + // version that we fetched lacks an explicit go.mod file: if the go.mod file + // is explicit, then it should match exactly (to ensure that imports of other + // packages within the module are interpreted correctly). Unfortunately, we + // can't determine that information from the module proxy protocol: we'll have + // to leave that validation for when we load actual packages from within the + // module. + if mpath := summary.module.Path; mpath != m.Path && mpath != actual.Path { + return nil, module.VersionError(actual, fmt.Errorf(`parsing go.mod: + module declares its path as: %s + but was required as: %s`, mpath, m.Path)) + } + } + + if index != nil && len(index.exclude) > 0 { + // Drop any requirements on excluded versions. + // Don't modify the cached summary though, since we might need the raw + // summary separately. + haveExcludedReqs := false + for _, r := range summary.require { + if index.exclude[r] { + haveExcludedReqs = true + break + } + } + if haveExcludedReqs { + s := new(modFileSummary) + *s = *summary + s.require = make([]module.Version, 0, len(summary.require)) + for _, r := range summary.require { + if !index.exclude[r] { + s.require = append(s.require, r) + } + } + summary = s + } + } + return summary, nil +} + +// rawGoModSummary returns a new summary of the go.mod file for module m, +// ignoring all replacements that may apply to m and excludes that may apply to +// its dependencies. +// +// rawGoModSummary cannot be used on the Target module. +func rawGoModSummary(m module.Version) (*modFileSummary, error) { + if m == Target { + panic("internal error: rawGoModSummary called on the Target module") + } + + type cached struct { + summary *modFileSummary + err error + } + c := rawGoModSummaryCache.Do(m, func() interface{} { + summary := new(modFileSummary) + var f *modfile.File + if m.Version == "" { + // m is a replacement module with only a file path. + dir := m.Path + if !filepath.IsAbs(dir) { + dir = filepath.Join(ModRoot(), dir) + } + gomod := filepath.Join(dir, "go.mod") + + data, err := lockedfile.Read(gomod) + if err != nil { + return cached{nil, module.VersionError(m, fmt.Errorf("reading %s: %v", base.ShortPath(gomod), err))} + } + f, err = modfile.ParseLax(gomod, data, nil) + if err != nil { + return cached{nil, module.VersionError(m, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err))} + } + } else { + if !semver.IsValid(m.Version) { + // Disallow the broader queries supported by fetch.Lookup. + base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", m.Path, m.Version) + } + + data, err := modfetch.GoMod(m.Path, m.Version) + if err != nil { + return cached{nil, err} + } + f, err = modfile.ParseLax("go.mod", data, nil) + if err != nil { + return cached{nil, module.VersionError(m, fmt.Errorf("parsing go.mod: %v", err))} + } + } + + if f.Module != nil { + summary.module = f.Module.Mod + } + if f.Go != nil && f.Go.Version != "" { + rawGoVersion.LoadOrStore(m, f.Go.Version) + summary.goVersionV = "v" + f.Go.Version + } + if len(f.Require) > 0 { + summary.require = make([]module.Version, 0, len(f.Require)) + for _, req := range f.Require { + summary.require = append(summary.require, req.Mod) + } + } + if len(f.Retract) > 0 { + summary.retract = make([]retraction, 0, len(f.Retract)) + for _, ret := range f.Retract { + summary.retract = append(summary.retract, retraction{ + VersionInterval: ret.VersionInterval, + Rationale: ret.Rationale, + }) + } + } + + return cached{summary, nil} + }).(cached) + + return c.summary, c.err +} + +var rawGoModSummaryCache par.Cache // module.Version → rawGoModSummary result diff --git a/libgo/go/cmd/go/internal/modload/mvs.go b/libgo/go/cmd/go/internal/modload/mvs.go index 5dd009d..167d681 100644 --- a/libgo/go/cmd/go/internal/modload/mvs.go +++ b/libgo/go/cmd/go/internal/modload/mvs.go @@ -5,21 +5,13 @@ package modload import ( + "context" "errors" - "fmt" "os" - "path/filepath" "sort" - "sync" - "cmd/go/internal/base" - "cmd/go/internal/cfg" - "cmd/go/internal/lockedfile" "cmd/go/internal/modfetch" - "cmd/go/internal/mvs" - "cmd/go/internal/par" - "golang.org/x/mod/modfile" "golang.org/x/mod/module" "golang.org/x/mod/semver" ) @@ -28,133 +20,24 @@ import ( // with any exclusions or replacements applied internally. type mvsReqs struct { buildList []module.Version - cache par.Cache - versions sync.Map -} - -// Reqs returns the current module requirement graph. -// Future calls to SetBuildList do not affect the operation -// of the returned Reqs. -func Reqs() mvs.Reqs { - r := &mvsReqs{ - buildList: buildList, - } - return r } func (r *mvsReqs) Required(mod module.Version) ([]module.Version, error) { - type cached struct { - list []module.Version - err error - } - - c := r.cache.Do(mod, func() interface{} { - list, err := r.required(mod) - if err != nil { - return cached{nil, err} - } - for i, mv := range list { - if index != nil { - for index.exclude[mv] { - mv1, err := r.next(mv) - if err != nil { - return cached{nil, err} - } - if mv1.Version == "none" { - return cached{nil, fmt.Errorf("%s(%s) depends on excluded %s(%s) with no newer version available", mod.Path, mod.Version, mv.Path, mv.Version)} - } - mv = mv1 - } - } - list[i] = mv - } - - return cached{list, nil} - }).(cached) - - return c.list, c.err -} - -func (r *mvsReqs) modFileToList(f *modfile.File) []module.Version { - list := make([]module.Version, 0, len(f.Require)) - for _, r := range f.Require { - list = append(list, r.Mod) - } - return list -} - -// required returns a unique copy of the requirements of mod. -func (r *mvsReqs) required(mod module.Version) ([]module.Version, error) { if mod == Target { - if modFile != nil && modFile.Go != nil { - r.versions.LoadOrStore(mod, modFile.Go.Version) - } - return append([]module.Version(nil), r.buildList[1:]...), nil - } - - if cfg.BuildMod == "vendor" { - // For every module other than the target, - // return the full list of modules from modules.txt. - readVendorList() - return append([]module.Version(nil), vendorList...), nil - } - - origPath := mod.Path - if repl := Replacement(mod); repl.Path != "" { - if repl.Version == "" { - // TODO: need to slip the new version into the tags list etc. - dir := repl.Path - if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) - } - gomod := filepath.Join(dir, "go.mod") - data, err := lockedfile.Read(gomod) - if err != nil { - return nil, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err) - } - f, err := modfile.ParseLax(gomod, data, nil) - if err != nil { - return nil, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err) - } - if f.Go != nil { - r.versions.LoadOrStore(mod, f.Go.Version) - } - return r.modFileToList(f), nil - } - mod = repl + // Use the build list as it existed when r was constructed, not the current + // global build list. + return r.buildList[1:], nil } if mod.Version == "none" { return nil, nil } - if !semver.IsValid(mod.Version) { - // Disallow the broader queries supported by fetch.Lookup. - base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", mod.Path, mod.Version) - } - - data, err := modfetch.GoMod(mod.Path, mod.Version) + summary, err := goModSummary(mod) if err != nil { return nil, err } - f, err := modfile.ParseLax("go.mod", data, nil) - if err != nil { - return nil, module.VersionError(mod, fmt.Errorf("parsing go.mod: %v", err)) - } - - if f.Module == nil { - return nil, module.VersionError(mod, errors.New("parsing go.mod: missing module line")) - } - if mpath := f.Module.Mod.Path; mpath != origPath && mpath != mod.Path { - return nil, module.VersionError(mod, fmt.Errorf(`parsing go.mod: - module declares its path as: %s - but was required as: %s`, mpath, origPath)) - } - if f.Go != nil { - r.versions.LoadOrStore(mod, f.Go.Version) - } - - return r.modFileToList(f), nil + return summary.require, nil } // Max returns the maximum of v1 and v2 according to semver.Compare. @@ -164,7 +47,7 @@ func (r *mvsReqs) required(mod module.Version) ([]module.Version, error) { // be chosen over other versions of the same module in the module dependency // graph. func (*mvsReqs) Max(v1, v2 string) string { - if v1 != "" && semver.Compare(v1, v2) == -1 { + if v1 != "" && (v2 == "" || semver.Compare(v1, v2) == -1) { return v2 } return v1 @@ -176,25 +59,50 @@ func (*mvsReqs) Upgrade(m module.Version) (module.Version, error) { return m, nil } -func versions(path string) ([]string, error) { +func versions(ctx context.Context, path string, allowed AllowedFunc) ([]string, error) { // Note: modfetch.Lookup and repo.Versions are cached, // so there's no need for us to add extra caching here. var versions []string err := modfetch.TryProxies(func(proxy string) error { - repo, err := modfetch.Lookup(proxy, path) - if err == nil { - versions, err = repo.Versions("") + repo, err := lookupRepo(proxy, path) + if err != nil { + return err + } + allVersions, err := repo.Versions("") + if err != nil { + return err } - return err + allowedVersions := make([]string, 0, len(allVersions)) + for _, v := range allVersions { + if err := allowed(ctx, module.Version{Path: path, Version: v}); err == nil { + allowedVersions = append(allowedVersions, v) + } else if !errors.Is(err, ErrDisallowed) { + return err + } + } + versions = allowedVersions + return nil }) return versions, err } // Previous returns the tagged version of m.Path immediately prior to // m.Version, or version "none" if no prior version is tagged. +// +// Since the version of Target is not found in the version list, +// it has no previous version. func (*mvsReqs) Previous(m module.Version) (module.Version, error) { - list, err := versions(m.Path) + // TODO(golang.org/issue/38714): thread tracing context through MVS. + + if m == Target { + return module.Version{Path: m.Path, Version: "none"}, nil + } + + list, err := versions(context.TODO(), m.Path, CheckAllowed) if err != nil { + if errors.Is(err, os.ErrNotExist) { + return module.Version{Path: m.Path, Version: "none"}, nil + } return module.Version{}, err } i := sort.Search(len(list), func(i int) bool { return semver.Compare(list[i], m.Version) >= 0 }) @@ -208,7 +116,8 @@ func (*mvsReqs) Previous(m module.Version) (module.Version, error) { // It is only used by the exclusion processing in the Required method, // not called directly by MVS. func (*mvsReqs) next(m module.Version) (module.Version, error) { - list, err := versions(m.Path) + // TODO(golang.org/issue/38714): thread tracing context through MVS. + list, err := versions(context.TODO(), m.Path, CheckAllowed) if err != nil { return module.Version{}, err } @@ -218,42 +127,3 @@ func (*mvsReqs) next(m module.Version) (module.Version, error) { } return module.Version{Path: m.Path, Version: "none"}, nil } - -// fetch downloads the given module (or its replacement) -// and returns its location. -// -// The isLocal return value reports whether the replacement, -// if any, is local to the filesystem. -func fetch(mod module.Version) (dir string, isLocal bool, err error) { - if mod == Target { - return ModRoot(), true, nil - } - if r := Replacement(mod); r.Path != "" { - if r.Version == "" { - dir = r.Path - if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) - } - // Ensure that the replacement directory actually exists: - // dirInModule does not report errors for missing modules, - // so if we don't report the error now, later failures will be - // very mysterious. - if _, err := os.Stat(dir); err != nil { - if os.IsNotExist(err) { - // Semantically the module version itself “exists” — we just don't - // have its source code. Remove the equivalence to os.ErrNotExist, - // and make the message more concise while we're at it. - err = fmt.Errorf("replacement directory %s does not exist", r.Path) - } else { - err = fmt.Errorf("replacement directory %s: %w", r.Path, err) - } - return dir, true, module.VersionError(mod, err) - } - return dir, true, nil - } - mod = r - } - - dir, err = modfetch.Download(mod) - return dir, false, err -} diff --git a/libgo/go/cmd/go/internal/modload/mvs_test.go b/libgo/go/cmd/go/internal/modload/mvs_test.go new file mode 100644 index 0000000..50e93c3 --- /dev/null +++ b/libgo/go/cmd/go/internal/modload/mvs_test.go @@ -0,0 +1,31 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "testing" +) + +func TestReqsMax(t *testing.T) { + type testCase struct { + a, b, want string + } + reqs := new(mvsReqs) + for _, tc := range []testCase{ + {a: "v0.1.0", b: "v0.2.0", want: "v0.2.0"}, + {a: "v0.2.0", b: "v0.1.0", want: "v0.2.0"}, + {a: "", b: "v0.1.0", want: ""}, // "" is Target.Version + {a: "v0.1.0", b: "", want: ""}, + {a: "none", b: "v0.1.0", want: "v0.1.0"}, + {a: "v0.1.0", b: "none", want: "v0.1.0"}, + {a: "none", b: "", want: ""}, + {a: "", b: "none", want: ""}, + } { + max := reqs.Max(tc.a, tc.b) + if max != tc.want { + t.Errorf("(%T).Max(%q, %q) = %q; want %q", reqs, tc.a, tc.b, max, tc.want) + } + } +} diff --git a/libgo/go/cmd/go/internal/modload/query.go b/libgo/go/cmd/go/internal/modload/query.go index acc886b..e35e0fc 100644 --- a/libgo/go/cmd/go/internal/modload/query.go +++ b/libgo/go/cmd/go/internal/modload/query.go @@ -5,19 +5,23 @@ package modload import ( + "context" "errors" "fmt" + "io/fs" "os" pathpkg "path" "path/filepath" + "sort" "strings" "sync" + "time" "cmd/go/internal/cfg" "cmd/go/internal/imports" "cmd/go/internal/modfetch" "cmd/go/internal/search" - "cmd/go/internal/str" + "cmd/go/internal/trace" "golang.org/x/mod/module" "golang.org/x/mod/semver" @@ -43,27 +47,43 @@ import ( // with non-prereleases preferred over prereleases. // - a repository commit identifier or tag, denoting that commit. // -// current denotes the current version of the module; it may be "" if the -// current version is unknown or should not be considered. If query is +// current denotes the currently-selected version of the module; it may be +// "none" if no version is currently selected, or "" if the currently-selected +// version is unknown or should not be considered. If query is // "upgrade" or "patch", current will be returned if it is a newer // semantic version or a chronologically later pseudo-version than the // version that would otherwise be chosen. This prevents accidental downgrades // from newer pre-release or development versions. // -// If the allowed function is non-nil, Query excludes any versions for which -// allowed returns false. +// The allowed function (which may be nil) is used to filter out unsuitable +// versions (see AllowedFunc documentation for details). If the query refers to +// a specific revision (for example, "master"; see IsRevisionQuery), and the +// revision is disallowed by allowed, Query returns the error. If the query +// does not refer to a specific revision (for example, "latest"), Query +// acts as if versions disallowed by allowed do not exist. // // If path is the path of the main module and the query is "latest", // Query returns Target.Version as the version. -func Query(path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func Query(ctx context.Context, path, query, current string, allowed AllowedFunc) (*modfetch.RevInfo, error) { var info *modfetch.RevInfo err := modfetch.TryProxies(func(proxy string) (err error) { - info, err = queryProxy(proxy, path, query, current, allowed) + info, err = queryProxy(ctx, proxy, path, query, current, allowed) return err }) return info, err } +// AllowedFunc is used by Query and other functions to filter out unsuitable +// versions, for example, those listed in exclude directives in the main +// module's go.mod file. +// +// An AllowedFunc returns an error equivalent to ErrDisallowed for an unsuitable +// version. Any other error indicates the function was unable to determine +// whether the version should be allowed, for example, the function was unable +// to fetch or parse a go.mod file containing retractions. Typically, errors +// other than ErrDisallowd may be ignored. +type AllowedFunc func(context.Context, module.Version) error + var errQueryDisabled error = queryDisabledError{} type queryDisabledError struct{} @@ -75,138 +95,56 @@ func (queryDisabledError) Error() string { return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) } -func queryProxy(proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { - if current != "" && !semver.IsValid(current) { +func queryProxy(ctx context.Context, proxy, path, query, current string, allowed AllowedFunc) (*modfetch.RevInfo, error) { + ctx, span := trace.StartSpan(ctx, "modload.queryProxy "+path+" "+query) + defer span.Done() + + if current != "" && current != "none" && !semver.IsValid(current) { return nil, fmt.Errorf("invalid previous version %q", current) } if cfg.BuildMod == "vendor" { return nil, errQueryDisabled } if allowed == nil { - allowed = func(module.Version) bool { return true } + allowed = func(context.Context, module.Version) error { return nil } } - // Parse query to detect parse errors (and possibly handle query) - // before any network I/O. - badVersion := func(v string) (*modfetch.RevInfo, error) { - return nil, fmt.Errorf("invalid semantic version %q in range %q", v, query) - } - matchesMajor := func(v string) bool { - _, pathMajor, ok := module.SplitPathVersion(path) - if !ok { - return false + if path == Target.Path && (query == "upgrade" || query == "patch") { + if err := allowed(ctx, Target); err != nil { + return nil, fmt.Errorf("internal error: main module version is not allowed: %w", err) } - return module.CheckPathMajor(v, pathMajor) == nil + return &modfetch.RevInfo{Version: Target.Version}, nil } - var ( - ok func(module.Version) bool - prefix string - preferOlder bool - mayUseLatest bool - preferIncompatible bool = strings.HasSuffix(current, "+incompatible") - ) - switch { - case query == "latest": - ok = allowed - mayUseLatest = true - case query == "upgrade": - ok = allowed - mayUseLatest = true - - case query == "patch": - if current == "" { - ok = allowed - mayUseLatest = true - } else { - prefix = semver.MajorMinor(current) - ok = func(m module.Version) bool { - return matchSemverPrefix(prefix, m.Version) && allowed(m) - } - } - - case strings.HasPrefix(query, "<="): - v := query[len("<="):] - if !semver.IsValid(v) { - return badVersion(v) - } - if isSemverPrefix(v) { - // Refuse to say whether <=v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). - return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) - } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) <= 0 && allowed(m) - } - if !matchesMajor(v) { - preferIncompatible = true - } - - case strings.HasPrefix(query, "<"): - v := query[len("<"):] - if !semver.IsValid(v) { - return badVersion(v) - } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) < 0 && allowed(m) - } - if !matchesMajor(v) { - preferIncompatible = true - } - - case strings.HasPrefix(query, ">="): - v := query[len(">="):] - if !semver.IsValid(v) { - return badVersion(v) - } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) >= 0 && allowed(m) - } - preferOlder = true - if !matchesMajor(v) { - preferIncompatible = true - } + if path == "std" || path == "cmd" { + return nil, fmt.Errorf("can't query specific version (%q) of standard-library module %q", query, path) + } - case strings.HasPrefix(query, ">"): - v := query[len(">"):] - if !semver.IsValid(v) { - return badVersion(v) - } - if isSemverPrefix(v) { - // Refuse to say whether >v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). - return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) - } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) > 0 && allowed(m) - } - preferOlder = true - if !matchesMajor(v) { - preferIncompatible = true - } + repo, err := lookupRepo(proxy, path) + if err != nil { + return nil, err + } - case semver.IsValid(query) && isSemverPrefix(query): - ok = func(m module.Version) bool { - return matchSemverPrefix(query, m.Version) && allowed(m) - } - prefix = query + "." - if !matchesMajor(query) { - preferIncompatible = true - } + // Parse query to detect parse errors (and possibly handle query) + // before any network I/O. + qm, err := newQueryMatcher(path, query, current, allowed) + if (err == nil && qm.canStat) || err == errRevQuery { + // Direct lookup of a commit identifier or complete (non-prefix) semantic + // version. - default: - // Direct lookup of semantic version or commit identifier. - // // If the identifier is not a canonical semver tag — including if it's a // semver tag with a +metadata suffix — then modfetch.Stat will populate // info.Version with a suitable pseudo-version. - info, err := modfetch.Stat(proxy, path, query) + info, err := repo.Stat(query) if err != nil { queryErr := err // The full query doesn't correspond to a tag. If it is a semantic version // with a +metadata suffix, see if there is a tag without that suffix: // semantic versioning defines them to be equivalent. - if vers := module.CanonicalVersion(query); vers != "" && vers != query { - info, err = modfetch.Stat(proxy, path, vers) - if !errors.Is(err, os.ErrNotExist) { + canonicalQuery := module.CanonicalVersion(query) + if canonicalQuery != "" && query != canonicalQuery { + info, err = repo.Stat(canonicalQuery) + if err != nil && !errors.Is(err, fs.ErrNotExist) { return info, err } } @@ -214,36 +152,20 @@ func queryProxy(proxy, path, query, current string, allowed func(module.Version) return nil, queryErr } } - if !allowed(module.Version{Path: path, Version: info.Version}) { - return nil, fmt.Errorf("%s@%s excluded", path, info.Version) + if err := allowed(ctx, module.Version{Path: path, Version: info.Version}); errors.Is(err, ErrDisallowed) { + return nil, err } return info, nil - } - - if path == Target.Path { - if query != "latest" { - return nil, fmt.Errorf("can't query specific version (%q) for the main module (%s)", query, path) - } - if !allowed(Target) { - return nil, fmt.Errorf("internal error: main module version is not allowed") - } - return &modfetch.RevInfo{Version: Target.Version}, nil - } - - if str.HasPathPrefix(path, "std") || str.HasPathPrefix(path, "cmd") { - return nil, fmt.Errorf("explicit requirement on standard-library module %s not allowed", path) + } else if err != nil { + return nil, err } // Load versions and execute query. - repo, err := modfetch.Lookup(proxy, path) - if err != nil { - return nil, err - } - versions, err := repo.Versions(prefix) + versions, err := repo.Versions(qm.prefix) if err != nil { return nil, err } - releases, prereleases, err := filterVersions(path, versions, ok, preferIncompatible) + releases, prereleases, err := qm.filterVersions(ctx, versions) if err != nil { return nil, err } @@ -254,11 +176,30 @@ func queryProxy(proxy, path, query, current string, allowed func(module.Version) return nil, err } - // For "upgrade" and "patch", make sure we don't accidentally downgrade - // from a newer prerelease or from a chronologically newer pseudoversion. - if current != "" && (query == "upgrade" || query == "patch") { + if (query == "upgrade" || query == "patch") && modfetch.IsPseudoVersion(current) && !rev.Time.IsZero() { + // Don't allow "upgrade" or "patch" to move from a pseudo-version + // to a chronologically older version or pseudo-version. + // + // If the current version is a pseudo-version from an untagged branch, it + // may be semantically lower than the "latest" release or the latest + // pseudo-version on the main branch. A user on such a version is unlikely + // to intend to “upgrade” to a version that already existed at that point + // in time. + // + // We do this only if the current version is a pseudo-version: if the + // version is tagged, the author of the dependency module has given us + // explicit information about their intended precedence of this version + // relative to other versions, and we shouldn't contradict that + // information. (For example, v1.0.1 might be a backport of a fix already + // incorporated into v1.1.0, in which case v1.0.1 would be chronologically + // newer but v1.1.0 is still an “upgrade”; or v1.0.2 might be a revert of + // an unsuccessful fix in v1.0.1, in which case the v1.0.2 commit may be + // older than the v1.0.1 commit despite the tag itself being newer.) currentTime, err := modfetch.PseudoVersionTime(current) - if semver.Compare(rev.Version, current) < 0 || (err == nil && rev.Time.Before(currentTime)) { + if err == nil && rev.Time.Before(currentTime) { + if err := allowed(ctx, module.Version{Path: path, Version: current}); errors.Is(err, ErrDisallowed) { + return nil, err + } return repo.Stat(current) } } @@ -266,7 +207,7 @@ func queryProxy(proxy, path, query, current string, allowed func(module.Version) return rev, nil } - if preferOlder { + if qm.preferLower { if len(releases) > 0 { return lookup(releases[0]) } @@ -282,22 +223,44 @@ func queryProxy(proxy, path, query, current string, allowed func(module.Version) } } - if mayUseLatest { - // Special case for "latest": if no tags match, use latest commit in repo, - // provided it is not excluded. + if qm.mayUseLatest { latest, err := repo.Latest() if err == nil { - if allowed(module.Version{Path: path, Version: latest.Version}) { + if qm.allowsVersion(ctx, latest.Version) { return lookup(latest.Version) } - } else if !errors.Is(err, os.ErrNotExist) { + } else if !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + } + + if (query == "upgrade" || query == "patch") && current != "" && current != "none" { + // "upgrade" and "patch" may stay on the current version if allowed. + if err := allowed(ctx, module.Version{Path: path, Version: current}); errors.Is(err, ErrDisallowed) { return nil, err } + return lookup(current) } return nil, &NoMatchingVersionError{query: query, current: current} } +// IsRevisionQuery returns true if vers is a version query that may refer to +// a particular version or revision in a repository like "v1.0.0", "master", +// or "0123abcd". IsRevisionQuery returns false if vers is a query that +// chooses from among available versions like "latest" or ">v1.0.0". +func IsRevisionQuery(vers string) bool { + if vers == "latest" || + vers == "upgrade" || + vers == "patch" || + strings.HasPrefix(vers, "<") || + strings.HasPrefix(vers, ">") || + (semver.IsValid(vers) && isSemverPrefix(vers)) { + return false + } + return true +} + // isSemverPrefix reports whether v is a semantic version prefix: v1 or v1.2 (not v1.2.3). // The caller is assumed to have checked that semver.IsValid(v) is true. func isSemverPrefix(v string) bool { @@ -316,25 +279,190 @@ func isSemverPrefix(v string) bool { return true } -// matchSemverPrefix reports whether the shortened semantic version p -// matches the full-width (non-shortened) semantic version v. -func matchSemverPrefix(p, v string) bool { - return len(v) > len(p) && v[len(p)] == '.' && v[:len(p)] == p && semver.Prerelease(v) == "" +type queryMatcher struct { + path string + prefix string + filter func(version string) bool + allowed AllowedFunc + canStat bool // if true, the query can be resolved by repo.Stat + preferLower bool // if true, choose the lowest matching version + mayUseLatest bool + preferIncompatible bool +} + +var errRevQuery = errors.New("query refers to a non-semver revision") + +// newQueryMatcher returns a new queryMatcher that matches the versions +// specified by the given query on the module with the given path. +// +// If the query can only be resolved by statting a non-SemVer revision, +// newQueryMatcher returns errRevQuery. +func newQueryMatcher(path string, query, current string, allowed AllowedFunc) (*queryMatcher, error) { + badVersion := func(v string) (*queryMatcher, error) { + return nil, fmt.Errorf("invalid semantic version %q in range %q", v, query) + } + + matchesMajor := func(v string) bool { + _, pathMajor, ok := module.SplitPathVersion(path) + if !ok { + return false + } + return module.CheckPathMajor(v, pathMajor) == nil + } + + qm := &queryMatcher{ + path: path, + allowed: allowed, + preferIncompatible: strings.HasSuffix(current, "+incompatible"), + } + + switch { + case query == "latest": + qm.mayUseLatest = true + + case query == "upgrade": + if current == "" || current == "none" { + qm.mayUseLatest = true + } else { + qm.mayUseLatest = modfetch.IsPseudoVersion(current) + qm.filter = func(mv string) bool { return semver.Compare(mv, current) >= 0 } + } + + case query == "patch": + if current == "none" { + return nil, &NoPatchBaseError{path} + } + if current == "" { + qm.mayUseLatest = true + } else { + qm.mayUseLatest = modfetch.IsPseudoVersion(current) + qm.prefix = semver.MajorMinor(current) + "." + qm.filter = func(mv string) bool { return semver.Compare(mv, current) >= 0 } + } + + case strings.HasPrefix(query, "<="): + v := query[len("<="):] + if !semver.IsValid(v) { + return badVersion(v) + } + if isSemverPrefix(v) { + // Refuse to say whether <=v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). + return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) <= 0 } + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, "<"): + v := query[len("<"):] + if !semver.IsValid(v) { + return badVersion(v) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) < 0 } + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, ">="): + v := query[len(">="):] + if !semver.IsValid(v) { + return badVersion(v) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) >= 0 } + qm.preferLower = true + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, ">"): + v := query[len(">"):] + if !semver.IsValid(v) { + return badVersion(v) + } + if isSemverPrefix(v) { + // Refuse to say whether >v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). + return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) > 0 } + qm.preferLower = true + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case semver.IsValid(query): + if isSemverPrefix(query) { + qm.prefix = query + "." + // Do not allow the query "v1.2" to match versions lower than "v1.2.0", + // such as prereleases for that version. (https://golang.org/issue/31972) + qm.filter = func(mv string) bool { return semver.Compare(mv, query) >= 0 } + } else { + qm.canStat = true + qm.filter = func(mv string) bool { return semver.Compare(mv, query) == 0 } + qm.prefix = semver.Canonical(query) + } + if !matchesMajor(query) { + qm.preferIncompatible = true + } + + default: + return nil, errRevQuery + } + + return qm, nil +} + +// allowsVersion reports whether version v is allowed by the prefix, filter, and +// AllowedFunc of qm. +func (qm *queryMatcher) allowsVersion(ctx context.Context, v string) bool { + if qm.prefix != "" && !strings.HasPrefix(v, qm.prefix) { + return false + } + if qm.filter != nil && !qm.filter(v) { + return false + } + if qm.allowed != nil { + if err := qm.allowed(ctx, module.Version{Path: qm.path, Version: v}); errors.Is(err, ErrDisallowed) { + return false + } + } + return true } // filterVersions classifies versions into releases and pre-releases, filtering // out: -// 1. versions that do not satisfy the 'ok' predicate, and +// 1. versions that do not satisfy the 'allowed' predicate, and // 2. "+incompatible" versions, if a compatible one satisfies the predicate // and the incompatible version is not preferred. -func filterVersions(path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { +// +// If the allowed predicate returns an error not equivalent to ErrDisallowed, +// filterVersions returns that error. +func (qm *queryMatcher) filterVersions(ctx context.Context, versions []string) (releases, prereleases []string, err error) { + needIncompatible := qm.preferIncompatible + var lastCompatible string for _, v := range versions { - if !ok(module.Version{Path: path, Version: v}) { + if !qm.allowsVersion(ctx, v) { continue } - if !preferIncompatible { + if !needIncompatible { + // We're not yet sure whether we need to include +incomptaible versions. + // Keep track of the last compatible version we've seen, and use the + // presence (or absence) of a go.mod file in that version to decide: a + // go.mod file implies that the module author is supporting modules at a + // compatible version (and we should ignore +incompatible versions unless + // requested explicitly), while a lack of go.mod file implies the + // potential for legacy (pre-modules) versioning without semantic import + // paths (and thus *with* +incompatible versions). + // + // This isn't strictly accurate if the latest compatible version has been + // replaced by a local file path, because we do not allow file-path + // replacements without a go.mod file: the user would have needed to add + // one. However, replacing the last compatible version while + // simultaneously expecting to upgrade implicitly to a +incompatible + // version seems like an extreme enough corner case to ignore for now. + if !strings.HasSuffix(v, "+incompatible") { lastCompatible = v } else if lastCompatible != "" { @@ -342,19 +470,22 @@ func filterVersions(path string, versions []string, ok func(module.Version) bool // ignore any version with a higher (+incompatible) major version. (See // https://golang.org/issue/34165.) Note that we even prefer a // compatible pre-release over an incompatible release. - - ok, err := versionHasGoMod(module.Version{Path: path, Version: lastCompatible}) + ok, err := versionHasGoMod(ctx, module.Version{Path: qm.path, Version: lastCompatible}) if err != nil { return nil, nil, err } if ok { + // The last compatible version has a go.mod file, so that's the + // highest version we're willing to consider. Don't bother even + // looking at higher versions, because they're all +incompatible from + // here onward. break } // No acceptable compatible release has a go.mod file, so the versioning // for the module might not be module-aware, and we should respect // legacy major-version tags. - preferIncompatible = true + needIncompatible = true } } @@ -374,34 +505,42 @@ type QueryResult struct { Packages []string } -// QueryPackage looks up the module(s) containing path at a revision matching -// query. The results are sorted by module path length in descending order. -// -// If the package is in the main module, QueryPackage considers only the main -// module and only the version "latest", without checking for other possible -// modules. -func QueryPackage(path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { - m := search.NewMatch(path) - if m.IsLocal() || !m.IsLiteral() { - return nil, fmt.Errorf("pattern %s is not an importable package", path) +// QueryPackages is like QueryPattern, but requires that the pattern match at +// least one package and omits the non-package result (if any). +func QueryPackages(ctx context.Context, pattern, query string, current func(string) string, allowed AllowedFunc) ([]QueryResult, error) { + pkgMods, modOnly, err := QueryPattern(ctx, pattern, query, current, allowed) + + if len(pkgMods) == 0 && err == nil { + return nil, &PackageNotInModuleError{ + Mod: modOnly.Mod, + Replacement: Replacement(modOnly.Mod), + Query: query, + Pattern: pattern, + } } - return QueryPattern(path, query, allowed) + + return pkgMods, err } // QueryPattern looks up the module(s) containing at least one package matching // the given pattern at the given version. The results are sorted by module path -// length in descending order. +// length in descending order. If any proxy provides a non-empty set of candidate +// modules, no further proxies are tried. // -// QueryPattern queries modules with package paths up to the first "..." -// in the pattern. For the pattern "example.com/a/b.../c", QueryPattern would -// consider prefixes of "example.com/a". If multiple modules have versions -// that match the query and packages that match the pattern, QueryPattern -// picks the one with the longest module path. +// For wildcard patterns, QueryPattern looks in modules with package paths up to +// the first "..." in the pattern. For the pattern "example.com/a/b.../c", +// QueryPattern would consider prefixes of "example.com/a". // // If any matching package is in the main module, QueryPattern considers only // the main module and only the version "latest", without checking for other // possible modules. -func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +// +// QueryPattern always returns at least one QueryResult (which may be only +// modOnly) or a non-nil error. +func QueryPattern(ctx context.Context, pattern, query string, current func(string) string, allowed AllowedFunc) (pkgMods []QueryResult, modOnly *QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query) + defer span.Done() + base := pattern firstError := func(m *search.Match) error { @@ -412,12 +551,16 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q } var match func(mod module.Version, root string, isLocal bool) *search.Match + matchPattern := search.MatchPattern(pattern) if i := strings.Index(pattern, "..."); i >= 0 { base = pathpkg.Dir(pattern[:i+3]) + if base == "." { + return nil, nil, &WildcardInFirstElementError{Pattern: pattern, Query: query} + } match = func(mod module.Version, root string, isLocal bool) *search.Match { m := search.NewMatch(pattern) - matchPackages(m, imports.AnyTags(), omitStd, []module.Version{mod}) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{mod}) return m } } else { @@ -436,23 +579,41 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q } } + var queryMatchesMainModule bool if HasModRoot() { m := match(Target, modRoot, true) if len(m.Pkgs) > 0 { - if query != "latest" { - return nil, fmt.Errorf("can't query specific version for package %s in the main module (%s)", pattern, Target.Path) + if query != "upgrade" && query != "patch" { + return nil, nil, &QueryMatchesPackagesInMainModuleError{ + Pattern: pattern, + Query: query, + Packages: m.Pkgs, + } } - if !allowed(Target) { - return nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed", pattern, Target.Path) + if err := allowed(ctx, Target); err != nil { + return nil, nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed: %w", pattern, Target.Path, err) } return []QueryResult{{ Mod: Target, Rev: &modfetch.RevInfo{Version: Target.Version}, Packages: m.Pkgs, - }}, nil + }}, nil, nil } if err := firstError(m); err != nil { - return nil, err + return nil, nil, err + } + + if matchPattern(Target.Path) { + queryMatchesMainModule = true + } + + if (query == "upgrade" || query == "patch") && queryMatchesMainModule { + if err := allowed(ctx, Target); err == nil { + modOnly = &QueryResult{ + Mod: Target, + Rev: &modfetch.RevInfo{Version: Target.Version}, + } + } } } @@ -461,29 +622,42 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q candidateModules = modulePrefixesExcludingTarget(base) ) if len(candidateModules) == 0 { - return nil, &PackageNotInModuleError{ - Mod: Target, - Query: query, - Pattern: pattern, + if modOnly != nil { + return nil, modOnly, nil + } else if queryMatchesMainModule { + return nil, nil, &QueryMatchesMainModuleError{ + Pattern: pattern, + Query: query, + } + } else { + return nil, nil, &PackageNotInModuleError{ + Mod: Target, + Query: query, + Pattern: pattern, + } } } - err := modfetch.TryProxies(func(proxy string) error { - queryModule := func(path string) (r QueryResult, err error) { - current := findCurrentVersion(path) + err = modfetch.TryProxies(func(proxy string) error { + queryModule := func(ctx context.Context, path string) (r QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern.queryModule ["+proxy+"] "+path) + defer span.Done() + + pathCurrent := current(path) r.Mod.Path = path - r.Rev, err = queryProxy(proxy, path, query, current, allowed) + r.Rev, err = queryProxy(ctx, proxy, path, query, pathCurrent, allowed) if err != nil { return r, err } r.Mod.Version = r.Rev.Version - root, isLocal, err := fetch(r.Mod) + needSum := true + root, isLocal, err := fetch(ctx, r.Mod, needSum) if err != nil { return r, err } m := match(r.Mod, root, isLocal) r.Packages = m.Pkgs - if len(r.Packages) == 0 { + if len(r.Packages) == 0 && !matchPattern(path) { if err := firstError(m); err != nil { return r, err } @@ -497,12 +671,25 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q return r, nil } - var err error - results, err = queryPrefixModules(candidateModules, queryModule) + allResults, err := queryPrefixModules(ctx, candidateModules, queryModule) + results = allResults[:0] + for _, r := range allResults { + if len(r.Packages) == 0 { + modOnly = &r + } else { + results = append(results, r) + } + } return err }) - return results, err + if queryMatchesMainModule && len(results) == 0 && modOnly == nil && errors.Is(err, fs.ErrNotExist) { + return nil, nil, &QueryMatchesMainModuleError{ + Pattern: pattern, + Query: query, + } + } + return results[:len(results):len(results)], modOnly, err } // modulePrefixesExcludingTarget returns all prefixes of path that may plausibly @@ -528,21 +715,10 @@ func modulePrefixesExcludingTarget(path string) []string { return prefixes } -func findCurrentVersion(path string) string { - for _, m := range buildList { - if m.Path == path { - return m.Version - } - } - return "" -} - -type prefixResult struct { - QueryResult - err error -} +func queryPrefixModules(ctx context.Context, candidateModules []string, queryModule func(ctx context.Context, path string) (QueryResult, error)) (found []QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.queryPrefixModules") + defer span.Done() -func queryPrefixModules(candidateModules []string, queryModule func(path string) (QueryResult, error)) (found []QueryResult, err error) { // If the path we're attempting is not in the module cache and we don't have a // fetch result cached either, we'll end up making a (potentially slow) // request to the proxy or (often even slower) the origin server. @@ -555,8 +731,9 @@ func queryPrefixModules(candidateModules []string, queryModule func(path string) var wg sync.WaitGroup wg.Add(len(candidateModules)) for i, p := range candidateModules { + ctx := trace.StartGoroutine(ctx) go func(p string, r *result) { - r.QueryResult, r.err = queryModule(p) + r.QueryResult, r.err = queryModule(ctx, p) wg.Done() }(p, &results[i]) } @@ -568,6 +745,7 @@ func queryPrefixModules(candidateModules []string, queryModule func(path string) var ( noPackage *PackageNotInModuleError noVersion *NoMatchingVersionError + noPatchBase *NoPatchBaseError notExistErr error ) for _, r := range results { @@ -584,8 +762,12 @@ func queryPrefixModules(candidateModules []string, queryModule func(path string) if noVersion == nil { noVersion = rErr } + case *NoPatchBaseError: + if noPatchBase == nil { + noPatchBase = rErr + } default: - if errors.Is(rErr, os.ErrNotExist) { + if errors.Is(rErr, fs.ErrNotExist) { if notExistErr == nil { notExistErr = rErr } @@ -615,6 +797,8 @@ func queryPrefixModules(candidateModules []string, queryModule func(path string) err = noPackage case noVersion != nil: err = noVersion + case noPatchBase != nil: + err = noPatchBase case notExistErr != nil: err = notExistErr default: @@ -628,7 +812,7 @@ func queryPrefixModules(candidateModules []string, queryModule func(path string) // A NoMatchingVersionError indicates that Query found a module at the requested // path, but not at any versions satisfying the query string and allow-function. // -// NOTE: NoMatchingVersionError MUST NOT implement Is(os.ErrNotExist). +// NOTE: NoMatchingVersionError MUST NOT implement Is(fs.ErrNotExist). // // If the module came from a proxy, that proxy had to return a successful status // code for the versions it knows about, and thus did not have the opportunity @@ -639,17 +823,39 @@ type NoMatchingVersionError struct { func (e *NoMatchingVersionError) Error() string { currentSuffix := "" - if (e.query == "upgrade" || e.query == "patch") && e.current != "" { + if (e.query == "upgrade" || e.query == "patch") && e.current != "" && e.current != "none" { currentSuffix = fmt.Sprintf(" (current version is %s)", e.current) } return fmt.Sprintf("no matching versions for query %q", e.query) + currentSuffix } +// A NoPatchBaseError indicates that Query was called with the query "patch" +// but with a current version of "" or "none". +type NoPatchBaseError struct { + path string +} + +func (e *NoPatchBaseError) Error() string { + return fmt.Sprintf(`can't query version "patch" of module %s: no existing version is required`, e.path) +} + +// A WildcardInFirstElementError indicates that a pattern passed to QueryPattern +// had a wildcard in its first path element, and therefore had no pattern-prefix +// modules to search in. +type WildcardInFirstElementError struct { + Pattern string + Query string +} + +func (e *WildcardInFirstElementError) Error() string { + return fmt.Sprintf("no modules to query for %s@%s because first path element contains a wildcard", e.Pattern, e.Query) +} + // A PackageNotInModuleError indicates that QueryPattern found a candidate // module at the requested version, but that module did not contain any packages // matching the requested pattern. // -// NOTE: PackageNotInModuleError MUST NOT implement Is(os.ErrNotExist). +// NOTE: PackageNotInModuleError MUST NOT implement Is(fs.ErrNotExist). // // If the module came from a proxy, that proxy had to return a successful status // code for the versions it knows about, and thus did not have the opportunity @@ -698,8 +904,9 @@ func (e *PackageNotInModuleError) ImportPath() string { } // ModuleHasRootPackage returns whether module m contains a package m.Path. -func ModuleHasRootPackage(m module.Version) (bool, error) { - root, isLocal, err := fetch(m) +func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { + needSum := false + root, isLocal, err := fetch(ctx, m, needSum) if err != nil { return false, err } @@ -707,11 +914,202 @@ func ModuleHasRootPackage(m module.Version) (bool, error) { return ok, err } -func versionHasGoMod(m module.Version) (bool, error) { - root, _, err := fetch(m) +func versionHasGoMod(ctx context.Context, m module.Version) (bool, error) { + needSum := false + root, _, err := fetch(ctx, m, needSum) if err != nil { return false, err } fi, err := os.Stat(filepath.Join(root, "go.mod")) return err == nil && !fi.IsDir(), nil } + +// A versionRepo is a subset of modfetch.Repo that can report information about +// available versions, but cannot fetch specific source files. +type versionRepo interface { + ModulePath() string + Versions(prefix string) ([]string, error) + Stat(rev string) (*modfetch.RevInfo, error) + Latest() (*modfetch.RevInfo, error) +} + +var _ versionRepo = modfetch.Repo(nil) + +func lookupRepo(proxy, path string) (repo versionRepo, err error) { + err = module.CheckPath(path) + if err == nil { + repo = modfetch.Lookup(proxy, path) + } else { + repo = emptyRepo{path: path, err: err} + } + + if index == nil { + return repo, err + } + if _, ok := index.highestReplaced[path]; !ok { + return repo, err + } + + return &replacementRepo{repo: repo}, nil +} + +// An emptyRepo is a versionRepo that contains no versions. +type emptyRepo struct { + path string + err error +} + +var _ versionRepo = emptyRepo{} + +func (er emptyRepo) ModulePath() string { return er.path } +func (er emptyRepo) Versions(prefix string) ([]string, error) { return nil, nil } +func (er emptyRepo) Stat(rev string) (*modfetch.RevInfo, error) { return nil, er.err } +func (er emptyRepo) Latest() (*modfetch.RevInfo, error) { return nil, er.err } + +// A replacementRepo augments a versionRepo to include the replacement versions +// (if any) found in the main module's go.mod file. +// +// A replacementRepo suppresses "not found" errors for otherwise-nonexistent +// modules, so a replacementRepo should only be constructed for a module that +// actually has one or more valid replacements. +type replacementRepo struct { + repo versionRepo +} + +var _ versionRepo = (*replacementRepo)(nil) + +func (rr *replacementRepo) ModulePath() string { return rr.repo.ModulePath() } + +// Versions returns the versions from rr.repo augmented with any matching +// replacement versions. +func (rr *replacementRepo) Versions(prefix string) ([]string, error) { + repoVersions, err := rr.repo.Versions(prefix) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + versions := repoVersions + if index != nil && len(index.replace) > 0 { + path := rr.ModulePath() + for m, _ := range index.replace { + if m.Path == path && strings.HasPrefix(m.Version, prefix) && m.Version != "" && !modfetch.IsPseudoVersion(m.Version) { + versions = append(versions, m.Version) + } + } + } + + if len(versions) == len(repoVersions) { // No replacement versions added. + return versions, nil + } + + sort.Slice(versions, func(i, j int) bool { + return semver.Compare(versions[i], versions[j]) < 0 + }) + uniq := versions[:1] + for _, v := range versions { + if v != uniq[len(uniq)-1] { + uniq = append(uniq, v) + } + } + return uniq, nil +} + +func (rr *replacementRepo) Stat(rev string) (*modfetch.RevInfo, error) { + info, err := rr.repo.Stat(rev) + if err == nil || index == nil || len(index.replace) == 0 { + return info, err + } + + v := module.CanonicalVersion(rev) + if v != rev { + // The replacements in the go.mod file list only canonical semantic versions, + // so a non-canonical version can't possibly have a replacement. + return info, err + } + + path := rr.ModulePath() + _, pathMajor, ok := module.SplitPathVersion(path) + if ok && pathMajor == "" { + if err := module.CheckPathMajor(v, pathMajor); err != nil && semver.Build(v) == "" { + v += "+incompatible" + } + } + + if r := Replacement(module.Version{Path: path, Version: v}); r.Path == "" { + return info, err + } + return rr.replacementStat(v) +} + +func (rr *replacementRepo) Latest() (*modfetch.RevInfo, error) { + info, err := rr.repo.Latest() + + if index != nil { + path := rr.ModulePath() + if v, ok := index.highestReplaced[path]; ok { + if v == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(path); ok && len(pathMajor) > 0 { + v = modfetch.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") + } else { + v = modfetch.PseudoVersion("v0", "", time.Time{}, "000000000000") + } + } + + if err != nil || semver.Compare(v, info.Version) > 0 { + return rr.replacementStat(v) + } + } + } + + return info, err +} + +func (rr *replacementRepo) replacementStat(v string) (*modfetch.RevInfo, error) { + rev := &modfetch.RevInfo{Version: v} + if modfetch.IsPseudoVersion(v) { + rev.Time, _ = modfetch.PseudoVersionTime(v) + rev.Short, _ = modfetch.PseudoVersionRev(v) + } + return rev, nil +} + +// A QueryMatchesMainModuleError indicates that a query requests +// a version of the main module that cannot be satisfied. +// (The main module's version cannot be changed.) +type QueryMatchesMainModuleError struct { + Pattern string + Query string +} + +func (e *QueryMatchesMainModuleError) Error() string { + if e.Pattern == Target.Path { + return fmt.Sprintf("can't request version %q of the main module (%s)", e.Query, e.Pattern) + } + + return fmt.Sprintf("can't request version %q of pattern %q that includes the main module (%s)", e.Query, e.Pattern, Target.Path) +} + +// A QueryMatchesPackagesInMainModuleError indicates that a query cannot be +// satisfied because it matches one or more packages found in the main module. +type QueryMatchesPackagesInMainModuleError struct { + Pattern string + Query string + Packages []string +} + +func (e *QueryMatchesPackagesInMainModuleError) Error() string { + if len(e.Packages) > 1 { + return fmt.Sprintf("pattern %s matches %d packages in the main module, so can't request version %s", e.Pattern, len(e.Packages), e.Query) + } + + if search.IsMetaPackage(e.Pattern) || strings.Contains(e.Pattern, "...") { + return fmt.Sprintf("pattern %s matches package %s in the main module, so can't request version %s", e.Pattern, e.Packages[0], e.Query) + } + + return fmt.Sprintf("package %s is in the main module, so can't request version %s", e.Packages[0], e.Query) +} diff --git a/libgo/go/cmd/go/internal/modload/query_test.go b/libgo/go/cmd/go/internal/modload/query_test.go index 247e4c4..e225a0e 100644 --- a/libgo/go/cmd/go/internal/modload/query_test.go +++ b/libgo/go/cmd/go/internal/modload/query_test.go @@ -5,8 +5,8 @@ package modload import ( + "context" "internal/testenv" - "io/ioutil" "log" "os" "path" @@ -26,7 +26,7 @@ func TestMain(m *testing.M) { func testMain(m *testing.M) int { cfg.GOPROXY = "direct" - dir, err := ioutil.TempDir("", "modload-test-") + dir, err := os.MkdirTemp("", "modload-test-") if err != nil { log.Fatal(err) } @@ -44,7 +44,7 @@ var ( queryRepoV3 = queryRepo + "/v3" // Empty version list (no semver tags), not actually empty. - emptyRepo = "vcs-test.golang.org/git/emptytest.git" + emptyRepoPath = "vcs-test.golang.org/git/emptytest.git" ) var queryTests = []struct { @@ -120,14 +120,14 @@ var queryTests = []struct { {path: queryRepo, query: "upgrade", current: "v1.9.10-pre2+metadata", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, {path: queryRepo, query: "upgrade", current: "v0.0.0-20190513201126-42abcb6df8ee", vers: "v0.0.0-20190513201126-42abcb6df8ee"}, {path: queryRepo, query: "upgrade", allow: "NOMATCH", err: `no matching versions for query "upgrade"`}, - {path: queryRepo, query: "upgrade", current: "v1.9.9", allow: "NOMATCH", err: `no matching versions for query "upgrade" (current version is v1.9.9)`}, + {path: queryRepo, query: "upgrade", current: "v1.9.9", allow: "NOMATCH", err: `vcs-test.golang.org/git/querytest.git@v1.9.9: disallowed module version`}, {path: queryRepo, query: "upgrade", current: "v1.99.99", err: `vcs-test.golang.org/git/querytest.git@v1.99.99: invalid version: unknown revision v1.99.99`}, {path: queryRepo, query: "patch", current: "", vers: "v1.9.9"}, {path: queryRepo, query: "patch", current: "v0.1.0", vers: "v0.1.2"}, {path: queryRepo, query: "patch", current: "v1.9.0", vers: "v1.9.9"}, {path: queryRepo, query: "patch", current: "v1.9.10-pre1", vers: "v1.9.10-pre1"}, {path: queryRepo, query: "patch", current: "v1.9.10-pre2+metadata", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, - {path: queryRepo, query: "patch", current: "v1.99.99", err: `no matching versions for query "patch" (current version is v1.99.99)`}, + {path: queryRepo, query: "patch", current: "v1.99.99", err: `vcs-test.golang.org/git/querytest.git@v1.99.99: invalid version: unknown revision v1.99.99`}, {path: queryRepo, query: ">v1.9.9", vers: "v1.9.10-pre1"}, {path: queryRepo, query: ">v1.10.0", err: `no matching versions for query ">v1.10.0"`}, {path: queryRepo, query: ">=v1.10.0", err: `no matching versions for query ">=v1.10.0"`}, @@ -170,42 +170,46 @@ var queryTests = []struct { // That should prevent us from resolving any version for the /v3 path. {path: queryRepoV3, query: "latest", err: `no matching versions for query "latest"`}, - {path: emptyRepo, query: "latest", vers: "v0.0.0-20180704023549-7bb914627242"}, - {path: emptyRepo, query: ">v0.0.0", err: `no matching versions for query ">v0.0.0"`}, - {path: emptyRepo, query: "<v10.0.0", err: `no matching versions for query "<v10.0.0"`}, + {path: emptyRepoPath, query: "latest", vers: "v0.0.0-20180704023549-7bb914627242"}, + {path: emptyRepoPath, query: ">v0.0.0", err: `no matching versions for query ">v0.0.0"`}, + {path: emptyRepoPath, query: "<v10.0.0", err: `no matching versions for query "<v10.0.0"`}, } func TestQuery(t *testing.T) { testenv.MustHaveExternalNetwork(t) testenv.MustHaveExecPath(t, "git") + ctx := context.Background() + for _, tt := range queryTests { allow := tt.allow if allow == "" { allow = "*" } - allowed := func(m module.Version) bool { - ok, _ := path.Match(allow, m.Version) - return ok + allowed := func(ctx context.Context, m module.Version) error { + if ok, _ := path.Match(allow, m.Version); !ok { + return module.VersionError(m, ErrDisallowed) + } + return nil } tt := tt t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.query+"/"+tt.current+"/"+allow, func(t *testing.T) { t.Parallel() - info, err := Query(tt.path, tt.query, tt.current, allowed) + info, err := Query(ctx, tt.path, tt.query, tt.current, allowed) if tt.err != "" { if err == nil { - t.Errorf("Query(%q, %q, %v) = %v, want error %q", tt.path, tt.query, allow, info.Version, tt.err) + t.Errorf("Query(_, %q, %q, %q, %v) = %v, want error %q", tt.path, tt.query, tt.current, allow, info.Version, tt.err) } else if err.Error() != tt.err { - t.Errorf("Query(%q, %q, %v): %v, want error %q", tt.path, tt.query, allow, err, tt.err) + t.Errorf("Query(_, %q, %q, %q, %v): %v\nwant error %q", tt.path, tt.query, tt.current, allow, err, tt.err) } return } if err != nil { - t.Fatalf("Query(%q, %q, %v): %v", tt.path, tt.query, allow, err) + t.Fatalf("Query(_, %q, %q, %q, %v): %v\nwant %v", tt.path, tt.query, tt.current, allow, err, tt.vers) } if info.Version != tt.vers { - t.Errorf("Query(%q, %q, %v) = %v, want %v", tt.path, tt.query, allow, info.Version, tt.vers) + t.Errorf("Query(_, %q, %q, %q, %v) = %v, want %v", tt.path, tt.query, tt.current, allow, info.Version, tt.vers) } }) } diff --git a/libgo/go/cmd/go/internal/modload/search.go b/libgo/go/cmd/go/internal/modload/search.go index 146c94a..4c73c78 100644 --- a/libgo/go/cmd/go/internal/modload/search.go +++ b/libgo/go/cmd/go/internal/modload/search.go @@ -5,13 +5,16 @@ package modload import ( + "context" "fmt" + "io/fs" "os" "path/filepath" "runtime" "strings" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/imports" "cmd/go/internal/search" @@ -28,7 +31,7 @@ const ( // matchPackages is like m.MatchPackages, but uses a local variable (rather than // a global) for tags, can include or exclude packages in the standard library, // and is restricted to the given list of modules. -func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { +func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { m.Pkgs = []string{} isMatch := func(string) bool { return true } @@ -53,7 +56,7 @@ func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modu walkPkgs := func(root, importPathRoot string, prune pruning) { root = filepath.Clean(root) - err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error { if err != nil { m.AddError(err) return nil @@ -84,8 +87,8 @@ func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modu } if !fi.IsDir() { - if fi.Mode()&os.ModeSymlink != 0 && want { - if target, err := os.Stat(path); err == nil && target.IsDir() { + if fi.Mode()&fs.ModeSymlink != 0 && want { + if target, err := fsys.Stat(path); err == nil && target.IsDir() { fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) } } @@ -154,7 +157,8 @@ func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modu isLocal = true } else { var err error - root, isLocal, err = fetch(mod) + const needSum = true + root, isLocal, err = fetch(ctx, mod, needSum) if err != nil { m.AddError(err) continue @@ -171,3 +175,46 @@ func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modu return } + +// MatchInModule identifies the packages matching the given pattern within the +// given module version, which does not need to be in the build list or module +// requirement graph. +// +// If m is the zero module.Version, MatchInModule matches the pattern +// against the standard library (std and cmd) in GOROOT/src. +func MatchInModule(ctx context.Context, pattern string, m module.Version, tags map[string]bool) *search.Match { + match := search.NewMatch(pattern) + if m == (module.Version{}) { + matchPackages(ctx, match, tags, includeStd, nil) + } + + LoadModFile(ctx) + + if !match.IsLiteral() { + matchPackages(ctx, match, tags, omitStd, []module.Version{m}) + return match + } + + const needSum = true + root, isLocal, err := fetch(ctx, m, needSum) + if err != nil { + match.Errs = []error{err} + return match + } + + dir, haveGoFiles, err := dirInModule(pattern, m.Path, root, isLocal) + if err != nil { + match.Errs = []error{err} + return match + } + if haveGoFiles { + if _, _, err := scanDir(dir, tags); err != imports.ErrNoGo { + // ErrNoGo indicates that the directory is not actually a Go package, + // perhaps due to the tags in use. Any other non-nil error indicates a + // problem with one or more of the Go source files, but such an error does + // not stop the package from existing, so it has no impact on matching. + match.Pkgs = []string{pattern} + } + } + return match +} diff --git a/libgo/go/cmd/go/internal/modload/stat_openfile.go b/libgo/go/cmd/go/internal/modload/stat_openfile.go index 931aaf1..5842b858 100644 --- a/libgo/go/cmd/go/internal/modload/stat_openfile.go +++ b/libgo/go/cmd/go/internal/modload/stat_openfile.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix js,wasm plan9 +// +build js,wasm plan9 // On plan9, per http://9p.io/magic/man2html/2/access: “Since file permissions // are checked by the server and group information is not known to the client, @@ -13,12 +13,13 @@ package modload import ( + "io/fs" "os" ) // hasWritePerm reports whether the current user has permission to write to the // file with the given info. -func hasWritePerm(path string, _ os.FileInfo) bool { +func hasWritePerm(path string, _ fs.FileInfo) bool { if f, err := os.OpenFile(path, os.O_WRONLY, 0); err == nil { f.Close() return true diff --git a/libgo/go/cmd/go/internal/modload/stat_unix.go b/libgo/go/cmd/go/internal/modload/stat_unix.go index a53d17e..032fff8 100644 --- a/libgo/go/cmd/go/internal/modload/stat_unix.go +++ b/libgo/go/cmd/go/internal/modload/stat_unix.go @@ -2,11 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd hurd linux netbsd openbsd solaris +// +build aix darwin dragonfly freebsd hurd linux netbsd openbsd solaris package modload import ( + "io/fs" "os" "syscall" ) @@ -17,7 +18,7 @@ import ( // Although the root user on most Unix systems can write to files even without // permission, hasWritePerm reports false if no appropriate permission bit is // set even if the current user is root. -func hasWritePerm(path string, fi os.FileInfo) bool { +func hasWritePerm(path string, fi fs.FileInfo) bool { if os.Getuid() == 0 { // The root user can access any file, but we still want to default to // read-only mode if the go.mod file is marked as globally non-writable. diff --git a/libgo/go/cmd/go/internal/modload/stat_windows.go b/libgo/go/cmd/go/internal/modload/stat_windows.go index d7826cf..0ac2391 100644 --- a/libgo/go/cmd/go/internal/modload/stat_windows.go +++ b/libgo/go/cmd/go/internal/modload/stat_windows.go @@ -6,13 +6,11 @@ package modload -import ( - "os" -) +import "io/fs" // hasWritePerm reports whether the current user has permission to write to the // file with the given info. -func hasWritePerm(_ string, fi os.FileInfo) bool { +func hasWritePerm(_ string, fi fs.FileInfo) bool { // Windows has a read-only attribute independent of ACLs, so use that to // determine whether the file is intended to be overwritten. // diff --git a/libgo/go/cmd/go/internal/modload/testgo.go b/libgo/go/cmd/go/internal/modload/testgo.go deleted file mode 100644 index 6b34f5b..0000000 --- a/libgo/go/cmd/go/internal/modload/testgo.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build testgo - -package modload - -func init() { - printStackInDie = true -} diff --git a/libgo/go/cmd/go/internal/modload/vendor.go b/libgo/go/cmd/go/internal/modload/vendor.go index 71f68ef..80d4905 100644 --- a/libgo/go/cmd/go/internal/modload/vendor.go +++ b/libgo/go/cmd/go/internal/modload/vendor.go @@ -7,7 +7,7 @@ package modload import ( "errors" "fmt" - "io/ioutil" + "io/fs" "os" "path/filepath" "strings" @@ -40,9 +40,9 @@ func readVendorList() { vendorPkgModule = make(map[string]module.Version) vendorVersion = make(map[string]string) vendorMeta = make(map[module.Version]vendorMetadata) - data, err := ioutil.ReadFile(filepath.Join(ModRoot(), "vendor/modules.txt")) + data, err := os.ReadFile(filepath.Join(ModRoot(), "vendor/modules.txt")) if err != nil { - if !errors.Is(err, os.ErrNotExist) { + if !errors.Is(err, fs.ErrNotExist) { base.Fatalf("go: %s", err) } return @@ -133,7 +133,7 @@ func checkVendorConsistency() { readVendorList() pre114 := false - if modFile.Go == nil || semver.Compare("v"+modFile.Go.Version, "v1.14") < 0 { + if semver.Compare(index.goVersionV, "v1.14") < 0 { // Go versions before 1.14 did not include enough information in // vendor/modules.txt to check for consistency. // If we know that we're on an earlier version, relax the consistency check. @@ -150,6 +150,8 @@ func checkVendorConsistency() { } } + // Iterate over the Require directives in their original (not indexed) order + // so that the errors match the original file. for _, r := range modFile.Require { if !vendorMeta[r.Mod].Explicit { if pre114 { diff --git a/libgo/go/cmd/go/internal/mvs/errors.go b/libgo/go/cmd/go/internal/mvs/errors.go new file mode 100644 index 0000000..5564965 --- /dev/null +++ b/libgo/go/cmd/go/internal/mvs/errors.go @@ -0,0 +1,101 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mvs + +import ( + "fmt" + "strings" + + "golang.org/x/mod/module" +) + +// BuildListError decorates an error that occurred gathering requirements +// while constructing a build list. BuildListError prints the chain +// of requirements to the module where the error occurred. +type BuildListError struct { + Err error + stack []buildListErrorElem +} + +type buildListErrorElem struct { + m module.Version + + // nextReason is the reason this module depends on the next module in the + // stack. Typically either "requires", or "updating to". + nextReason string +} + +// NewBuildListError returns a new BuildListError wrapping an error that +// occurred at a module found along the given path of requirements and/or +// upgrades, which must be non-empty. +// +// The isUpgrade function reports whether a path step is due to an upgrade. +// A nil isUpgrade function indicates that none of the path steps are due to upgrades. +func NewBuildListError(err error, path []module.Version, isUpgrade func(from, to module.Version) bool) *BuildListError { + stack := make([]buildListErrorElem, 0, len(path)) + for len(path) > 1 { + reason := "requires" + if isUpgrade != nil && isUpgrade(path[0], path[1]) { + reason = "updating to" + } + stack = append(stack, buildListErrorElem{ + m: path[0], + nextReason: reason, + }) + path = path[1:] + } + stack = append(stack, buildListErrorElem{m: path[0]}) + + return &BuildListError{ + Err: err, + stack: stack, + } +} + +// Module returns the module where the error occurred. If the module stack +// is empty, this returns a zero value. +func (e *BuildListError) Module() module.Version { + if len(e.stack) == 0 { + return module.Version{} + } + return e.stack[len(e.stack)-1].m +} + +func (e *BuildListError) Error() string { + b := &strings.Builder{} + stack := e.stack + + // Don't print modules at the beginning of the chain without a + // version. These always seem to be the main module or a + // synthetic module ("target@"). + for len(stack) > 0 && stack[0].m.Version == "" { + stack = stack[1:] + } + + if len(stack) == 0 { + b.WriteString(e.Err.Error()) + } else { + for _, elem := range stack[:len(stack)-1] { + fmt.Fprintf(b, "%s %s\n\t", elem.m, elem.nextReason) + } + // Ensure that the final module path and version are included as part of the + // error message. + m := stack[len(stack)-1].m + if mErr, ok := e.Err.(*module.ModuleError); ok { + actual := module.Version{Path: mErr.Path, Version: mErr.Version} + if v, ok := mErr.Err.(*module.InvalidVersionError); ok { + actual.Version = v.Version + } + if actual == m { + fmt.Fprintf(b, "%v", e.Err) + } else { + fmt.Fprintf(b, "%s (replaced by %s): %v", m, actual, mErr.Err) + } + } else { + fmt.Fprintf(b, "%v", module.VersionError(m, e.Err)) + } + } + return b.String() +} diff --git a/libgo/go/cmd/go/internal/mvs/mvs.go b/libgo/go/cmd/go/internal/mvs/mvs.go index 1f8eaa1..b630b61 100644 --- a/libgo/go/cmd/go/internal/mvs/mvs.go +++ b/libgo/go/cmd/go/internal/mvs/mvs.go @@ -9,7 +9,6 @@ package mvs import ( "fmt" "sort" - "strings" "sync" "sync/atomic" @@ -61,59 +60,6 @@ type Reqs interface { Previous(m module.Version) (module.Version, error) } -// BuildListError decorates an error that occurred gathering requirements -// while constructing a build list. BuildListError prints the chain -// of requirements to the module where the error occurred. -type BuildListError struct { - Err error - stack []buildListErrorElem -} - -type buildListErrorElem struct { - m module.Version - - // nextReason is the reason this module depends on the next module in the - // stack. Typically either "requires", or "upgraded to". - nextReason string -} - -// Module returns the module where the error occurred. If the module stack -// is empty, this returns a zero value. -func (e *BuildListError) Module() module.Version { - if len(e.stack) == 0 { - return module.Version{} - } - return e.stack[0].m -} - -func (e *BuildListError) Error() string { - b := &strings.Builder{} - stack := e.stack - - // Don't print modules at the beginning of the chain without a - // version. These always seem to be the main module or a - // synthetic module ("target@"). - for len(stack) > 0 && stack[len(stack)-1].m.Version == "" { - stack = stack[:len(stack)-1] - } - - for i := len(stack) - 1; i >= 1; i-- { - fmt.Fprintf(b, "%s@%s %s\n\t", stack[i].m.Path, stack[i].m.Version, stack[i].nextReason) - } - if len(stack) == 0 { - b.WriteString(e.Err.Error()) - } else { - // Ensure that the final module path and version are included as part of the - // error message. - if _, ok := e.Err.(*module.ModuleError); ok { - fmt.Fprintf(b, "%v", e.Err) - } else { - fmt.Fprintf(b, "%v", module.VersionError(stack[0].m, e.Err)) - } - } - return b.String() -} - // BuildList returns the build list for the target module. // // target is the root vertex of a module requirement graph. For cmd/go, this is @@ -162,19 +108,23 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m node := &modGraphNode{m: m} mu.Lock() modGraph[m] = node - if v, ok := min[m.Path]; !ok || reqs.Max(v, m.Version) != v { - min[m.Path] = m.Version + if m.Version != "none" { + if v, ok := min[m.Path]; !ok || reqs.Max(v, m.Version) != v { + min[m.Path] = m.Version + } } mu.Unlock() - required, err := reqs.Required(m) - if err != nil { - setErr(node, err) - return - } - node.required = required - for _, r := range node.required { - work.Add(r) + if m.Version != "none" { + required, err := reqs.Required(m) + if err != nil { + setErr(node, err) + return + } + node.required = required + for _, r := range node.required { + work.Add(r) + } } if upgrade != nil { @@ -202,18 +152,30 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m q = q[1:] if node.err != nil { - err := &BuildListError{ - Err: node.err, - stack: []buildListErrorElem{{m: node.m}}, - } + pathUpgrade := map[module.Version]module.Version{} + + // Construct the error path reversed (from the error to the main module), + // then reverse it to obtain the usual order (from the main module to + // the error). + errPath := []module.Version{node.m} for n, prev := neededBy[node], node; n != nil; n, prev = neededBy[n], n { - reason := "requires" if n.upgrade == prev.m { - reason = "updating to" + pathUpgrade[n.m] = prev.m } - err.stack = append(err.stack, buildListErrorElem{m: n.m, nextReason: reason}) + errPath = append(errPath, n.m) } - return nil, err + i, j := 0, len(errPath)-1 + for i < j { + errPath[i], errPath[j] = errPath[j], errPath[i] + i++ + j-- + } + + isUpgrade := func(from, to module.Version) bool { + return pathUpgrade[from] == to + } + + return nil, NewBuildListError(node.err, errPath, isUpgrade) } neighbors := node.required @@ -250,6 +212,9 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m n := modGraph[module.Version{Path: path, Version: vers}] required := n.required for _, r := range required { + if r.Version == "none" { + continue + } v := min[r.Path] if r.Path != target.Path && reqs.Max(v, r.Version) != v { panic(fmt.Sprintf("mistake: version %q does not satisfy requirement %+v", v, r)) // TODO: Don't panic. @@ -370,16 +335,36 @@ func Upgrade(target module.Version, reqs Reqs, upgrade ...module.Version) ([]mod if err != nil { return nil, err } - // TODO: Maybe if an error is given, - // rerun with BuildList(upgrade[0], reqs) etc - // to find which ones are the buggy ones. + + pathInList := make(map[string]bool, len(list)) + for _, m := range list { + pathInList[m.Path] = true + } list = append([]module.Version(nil), list...) - list = append(list, upgrade...) - return BuildList(target, &override{target, list, reqs}) + + upgradeTo := make(map[string]string, len(upgrade)) + for _, u := range upgrade { + if !pathInList[u.Path] { + list = append(list, module.Version{Path: u.Path, Version: "none"}) + } + if prev, dup := upgradeTo[u.Path]; dup { + upgradeTo[u.Path] = reqs.Max(prev, u.Version) + } else { + upgradeTo[u.Path] = u.Version + } + } + + return buildList(target, &override{target, list, reqs}, func(m module.Version) (module.Version, error) { + if v, ok := upgradeTo[m.Path]; ok { + return module.Version{Path: m.Path, Version: v}, nil + } + return m, nil + }) } // Downgrade returns a build list for the target module -// in which the given additional modules are downgraded. +// in which the given additional modules are downgraded, +// potentially overriding the requirements of the target. // // The versions to be downgraded may be unreachable from reqs.Latest and // reqs.Previous, but the methods of reqs must otherwise handle such versions diff --git a/libgo/go/cmd/go/internal/mvs/mvs_test.go b/libgo/go/cmd/go/internal/mvs/mvs_test.go index 9a30a8c..721cd96 100644 --- a/libgo/go/cmd/go/internal/mvs/mvs_test.go +++ b/libgo/go/cmd/go/internal/mvs/mvs_test.go @@ -54,7 +54,7 @@ build A: A B C D2 E2 name: cross1V A: B2 C D2 E1 -B1: +B1: B2: D1 C: D2 D1: E2 @@ -63,7 +63,7 @@ build A: A B2 C D2 E2 name: cross1U A: B1 C -B1: +B1: B2: D1 C: D2 D1: E2 @@ -72,7 +72,7 @@ build A: A B1 C D2 E1 upgrade A B2: A B2 C D2 E2 name: cross1R -A: B C +A: B C B: D2 C: D1 D1: E2 @@ -165,7 +165,7 @@ M: A1 B1 A1: X1 B1: X2 X1: I1 -X2: +X2: build M: M A1 B1 I1 X2 # Upgrade from B1 to B2 should not drop the transitive dep on D. @@ -229,28 +229,31 @@ E1: F1: downgrade A F1: A B1 E1 -name: down3 -A: +name: downcycle +A: A B2 +B2: A +B1: +downgrade A B1: A B1 # golang.org/issue/25542. name: noprev1 A: B4 C2 -B2.hidden: -C2: +B2.hidden: +C2: downgrade A B2.hidden: A B2.hidden C2 name: noprev2 A: B4 C2 -B2.hidden: -B1: -C2: +B2.hidden: +B1: +C2: downgrade A B2.hidden: A B2.hidden C2 name: noprev3 A: B4 C2 -B3: -B2.hidden: -C2: +B3: +B2.hidden: +C2: downgrade A B2.hidden: A B2.hidden C2 # Cycles involving the target. @@ -315,8 +318,15 @@ M: A1 B1 A1: X1 B1: X2 X1: I1 -X2: +X2: req M: A1 B1 + +name: reqnone +M: Anone B1 D1 E1 +B1: Cnone D1 +E1: Fnone +build M: M B1 D1 E1 +req M: B1 E1 ` func Test(t *testing.T) { @@ -331,6 +341,9 @@ func Test(t *testing.T) { for _, fn := range fns { fn(t) } + if len(fns) == 0 { + t.Errorf("no functions tested") + } }) } } @@ -478,9 +491,9 @@ func (r reqsMap) Max(v1, v2 string) string { } func (r reqsMap) Upgrade(m module.Version) (module.Version, error) { - var u module.Version + u := module.Version{Version: "none"} for k := range r { - if k.Path == m.Path && u.Version < k.Version && !strings.HasSuffix(k.Version, ".hidden") { + if k.Path == m.Path && r.Max(u.Version, k.Version) == k.Version && !strings.HasSuffix(k.Version, ".hidden") { u = k } } diff --git a/libgo/go/cmd/go/internal/par/queue.go b/libgo/go/cmd/go/internal/par/queue.go new file mode 100644 index 0000000..180bc75 --- /dev/null +++ b/libgo/go/cmd/go/internal/par/queue.go @@ -0,0 +1,88 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package par + +import "fmt" + +// Queue manages a set of work items to be executed in parallel. The number of +// active work items is limited, and excess items are queued sequentially. +type Queue struct { + maxActive int + st chan queueState +} + +type queueState struct { + active int // number of goroutines processing work; always nonzero when len(backlog) > 0 + backlog []func() + idle chan struct{} // if non-nil, closed when active becomes 0 +} + +// NewQueue returns a Queue that executes up to maxActive items in parallel. +// +// maxActive must be positive. +func NewQueue(maxActive int) *Queue { + if maxActive < 1 { + panic(fmt.Sprintf("par.NewQueue called with nonpositive limit (%d)", maxActive)) + } + + q := &Queue{ + maxActive: maxActive, + st: make(chan queueState, 1), + } + q.st <- queueState{} + return q +} + +// Add adds f as a work item in the queue. +// +// Add returns immediately, but the queue will be marked as non-idle until after +// f (and any subsequently-added work) has completed. +func (q *Queue) Add(f func()) { + st := <-q.st + if st.active == q.maxActive { + st.backlog = append(st.backlog, f) + q.st <- st + return + } + if st.active == 0 { + // Mark q as non-idle. + st.idle = nil + } + st.active++ + q.st <- st + + go func() { + for { + f() + + st := <-q.st + if len(st.backlog) == 0 { + if st.active--; st.active == 0 && st.idle != nil { + close(st.idle) + } + q.st <- st + return + } + f, st.backlog = st.backlog[0], st.backlog[1:] + q.st <- st + } + }() +} + +// Idle returns a channel that will be closed when q has no (active or enqueued) +// work outstanding. +func (q *Queue) Idle() <-chan struct{} { + st := <-q.st + defer func() { q.st <- st }() + + if st.idle == nil { + st.idle = make(chan struct{}) + if st.active == 0 { + close(st.idle) + } + } + + return st.idle +} diff --git a/libgo/go/cmd/go/internal/par/queue_test.go b/libgo/go/cmd/go/internal/par/queue_test.go new file mode 100644 index 0000000..1331e65 --- /dev/null +++ b/libgo/go/cmd/go/internal/par/queue_test.go @@ -0,0 +1,79 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package par + +import ( + "sync" + "testing" +) + +func TestQueueIdle(t *testing.T) { + q := NewQueue(1) + select { + case <-q.Idle(): + default: + t.Errorf("NewQueue(1) is not initially idle.") + } + + started := make(chan struct{}) + unblock := make(chan struct{}) + q.Add(func() { + close(started) + <-unblock + }) + + <-started + idle := q.Idle() + select { + case <-idle: + t.Errorf("NewQueue(1) is marked idle while processing work.") + default: + } + + close(unblock) + <-idle // Should be closed as soon as the Add callback returns. +} + +func TestQueueBacklog(t *testing.T) { + const ( + maxActive = 2 + totalWork = 3 * maxActive + ) + + q := NewQueue(maxActive) + t.Logf("q = NewQueue(%d)", maxActive) + + var wg sync.WaitGroup + wg.Add(totalWork) + started := make([]chan struct{}, totalWork) + unblock := make(chan struct{}) + for i := range started { + started[i] = make(chan struct{}) + i := i + q.Add(func() { + close(started[i]) + <-unblock + wg.Done() + }) + } + + for i, c := range started { + if i < maxActive { + <-c // Work item i should be started immediately. + } else { + select { + case <-c: + t.Errorf("Work item %d started before previous items finished.", i) + default: + } + } + } + + close(unblock) + for _, c := range started[maxActive:] { + <-c + } + wg.Wait() +} diff --git a/libgo/go/cmd/go/internal/renameio/renameio.go b/libgo/go/cmd/go/internal/renameio/renameio.go index d573cc6..9788171 100644 --- a/libgo/go/cmd/go/internal/renameio/renameio.go +++ b/libgo/go/cmd/go/internal/renameio/renameio.go @@ -8,6 +8,7 @@ package renameio import ( "bytes" "io" + "io/fs" "math/rand" "os" "path/filepath" @@ -24,18 +25,18 @@ func Pattern(filename string) string { return filepath.Join(filepath.Dir(filename), filepath.Base(filename)+patternSuffix) } -// WriteFile is like ioutil.WriteFile, but first writes data to an arbitrary +// WriteFile is like os.WriteFile, but first writes data to an arbitrary // file in the same directory as filename, then renames it atomically to the // final name. // // That ensures that the final location, if it exists, is always a complete file. -func WriteFile(filename string, data []byte, perm os.FileMode) (err error) { +func WriteFile(filename string, data []byte, perm fs.FileMode) (err error) { return WriteToFile(filename, bytes.NewReader(data), perm) } // WriteToFile is a variant of WriteFile that accepts the data as an io.Reader // instead of a slice. -func WriteToFile(filename string, data io.Reader, perm os.FileMode) (err error) { +func WriteToFile(filename string, data io.Reader, perm fs.FileMode) (err error) { f, err := tempFile(filepath.Dir(filename), filepath.Base(filename), perm) if err != nil { return err @@ -66,7 +67,7 @@ func WriteToFile(filename string, data io.Reader, perm os.FileMode) (err error) return robustio.Rename(f.Name(), filename) } -// ReadFile is like ioutil.ReadFile, but on Windows retries spurious errors that +// ReadFile is like os.ReadFile, but on Windows retries spurious errors that // may occur if the file is concurrently replaced. // // Errors are classified heuristically and retries are bounded, so even this @@ -80,7 +81,7 @@ func ReadFile(filename string) ([]byte, error) { } // tempFile creates a new temporary file with given permission bits. -func tempFile(dir, prefix string, perm os.FileMode) (f *os.File, err error) { +func tempFile(dir, prefix string, perm fs.FileMode) (f *os.File, err error) { for i := 0; i < 10000; i++ { name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+patternSuffix) f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm) diff --git a/libgo/go/cmd/go/internal/renameio/renameio_test.go b/libgo/go/cmd/go/internal/renameio/renameio_test.go index df8ddab..5b2ed83 100644 --- a/libgo/go/cmd/go/internal/renameio/renameio_test.go +++ b/libgo/go/cmd/go/internal/renameio/renameio_test.go @@ -10,7 +10,6 @@ import ( "encoding/binary" "errors" "internal/testenv" - "io/ioutil" "math/rand" "os" "path/filepath" @@ -30,7 +29,7 @@ func TestConcurrentReadsAndWrites(t *testing.T) { testenv.SkipFlaky(t, 33041) } - dir, err := ioutil.TempDir("", "renameio") + dir, err := os.MkdirTemp("", "renameio") if err != nil { t.Fatal(err) } @@ -144,10 +143,12 @@ func TestConcurrentReadsAndWrites(t *testing.T) { // As long as those are the only errors and *some* of the reads succeed, we're happy. minReadSuccesses = attempts / 4 - case "darwin": - // The filesystem on macOS 10.14 occasionally fails with "no such file or - // directory" errors. See https://golang.org/issue/33041. The flake rate is - // fairly low, so ensure that at least 75% of attempts succeed. + case "darwin", "ios": + // The filesystem on certain versions of macOS (10.14) and iOS (affected + // versions TBD) occasionally fail with "no such file or directory" errors. + // See https://golang.org/issue/33041 and https://golang.org/issue/42066. + // The flake rate is fairly low, so ensure that at least 75% of attempts + // succeed. minReadSuccesses = attempts - (attempts / 4) } diff --git a/libgo/go/cmd/go/internal/renameio/umask_test.go b/libgo/go/cmd/go/internal/renameio/umask_test.go index d75d67c..65e4fa5 100644 --- a/libgo/go/cmd/go/internal/renameio/umask_test.go +++ b/libgo/go/cmd/go/internal/renameio/umask_test.go @@ -7,7 +7,7 @@ package renameio import ( - "io/ioutil" + "io/fs" "os" "path/filepath" "syscall" @@ -15,7 +15,7 @@ import ( ) func TestWriteFileModeAppliesUmask(t *testing.T) { - dir, err := ioutil.TempDir("", "renameio") + dir, err := os.MkdirTemp("", "renameio") if err != nil { t.Fatalf("Failed to create temporary directory: %v", err) } @@ -36,7 +36,7 @@ func TestWriteFileModeAppliesUmask(t *testing.T) { t.Fatalf("Stat %q (looking for mode %#o): %s", file, mode, err) } - if fi.Mode()&os.ModePerm != 0640 { - t.Errorf("Stat %q: mode %#o want %#o", file, fi.Mode()&os.ModePerm, 0640) + if fi.Mode()&fs.ModePerm != 0640 { + t.Errorf("Stat %q: mode %#o want %#o", file, fi.Mode()&fs.ModePerm, 0640) } } diff --git a/libgo/go/cmd/go/internal/robustio/robustio.go b/libgo/go/cmd/go/internal/robustio/robustio.go index 76e47ad..ce3dbbd 100644 --- a/libgo/go/cmd/go/internal/robustio/robustio.go +++ b/libgo/go/cmd/go/internal/robustio/robustio.go @@ -22,7 +22,7 @@ func Rename(oldpath, newpath string) error { return rename(oldpath, newpath) } -// ReadFile is like ioutil.ReadFile, but on Windows retries errors that may +// ReadFile is like os.ReadFile, but on Windows retries errors that may // occur if the file is concurrently replaced. // // (See golang.org/issue/31247 and golang.org/issue/32188.) diff --git a/libgo/go/cmd/go/internal/robustio/robustio_flaky.go b/libgo/go/cmd/go/internal/robustio/robustio_flaky.go index d4cb7e6..5bd44bd 100644 --- a/libgo/go/cmd/go/internal/robustio/robustio_flaky.go +++ b/libgo/go/cmd/go/internal/robustio/robustio_flaky.go @@ -8,7 +8,6 @@ package robustio import ( "errors" - "io/ioutil" "math/rand" "os" "syscall" @@ -70,11 +69,11 @@ func rename(oldpath, newpath string) (err error) { }) } -// readFile is like ioutil.ReadFile, but retries ephemeral errors. +// readFile is like os.ReadFile, but retries ephemeral errors. func readFile(filename string) ([]byte, error) { var b []byte err := retry(func() (err error, mayRetry bool) { - b, err = ioutil.ReadFile(filename) + b, err = os.ReadFile(filename) // Unlike in rename, we do not retry errFileNotFound here: it can occur // as a spurious error, but the file may also genuinely not exist, so the diff --git a/libgo/go/cmd/go/internal/robustio/robustio_other.go b/libgo/go/cmd/go/internal/robustio/robustio_other.go index 907b556..6fe7b7e 100644 --- a/libgo/go/cmd/go/internal/robustio/robustio_other.go +++ b/libgo/go/cmd/go/internal/robustio/robustio_other.go @@ -7,7 +7,6 @@ package robustio import ( - "io/ioutil" "os" ) @@ -16,7 +15,7 @@ func rename(oldpath, newpath string) error { } func readFile(filename string) ([]byte, error) { - return ioutil.ReadFile(filename) + return os.ReadFile(filename) } func removeAll(path string) error { diff --git a/libgo/go/cmd/go/internal/run/run.go b/libgo/go/cmd/go/internal/run/run.go index 2edae38..99578b2 100644 --- a/libgo/go/cmd/go/internal/run/run.go +++ b/libgo/go/cmd/go/internal/run/run.go @@ -6,6 +6,7 @@ package run import ( + "context" "fmt" "os" "path" @@ -57,7 +58,7 @@ func printStderr(args ...interface{}) (int, error) { return fmt.Fprint(os.Stderr, args...) } -func runRun(cmd *base.Command, args []string) { +func runRun(ctx context.Context, cmd *base.Command, args []string) { work.BuildInit() var b work.Builder b.Init() @@ -76,9 +77,9 @@ func runRun(cmd *base.Command, args []string) { base.Fatalf("go run: cannot run *_test.go files (%s)", file) } } - p = load.GoFilesPackage(files) + p = load.GoFilesPackage(ctx, files) } else if len(args) > 0 && !strings.HasPrefix(args[0], "-") { - pkgs := load.PackagesAndErrors(args[:1]) + pkgs := load.PackagesAndErrors(ctx, args[:1]) if len(pkgs) == 0 { base.Fatalf("go run: no packages loaded from %s", args[0]) } @@ -140,12 +141,12 @@ func runRun(cmd *base.Command, args []string) { } a1 := b.LinkAction(work.ModeBuild, work.ModeBuild, p) a := &work.Action{Mode: "go run", Func: buildRunProgram, Args: cmdArgs, Deps: []*work.Action{a1}} - b.Do(a) + b.Do(ctx, a) } // buildRunProgram is the action for running a binary that has already // been compiled. We ignore exit status. -func buildRunProgram(b *work.Builder, a *work.Action) error { +func buildRunProgram(b *work.Builder, ctx context.Context, a *work.Action) error { cmdline := str.StringList(work.FindExecCmd(), a.Deps[0].Target, a.Args) if cfg.BuildN || cfg.BuildX { b.Showcmd("", "%s", strings.Join(cmdline, " ")) diff --git a/libgo/go/cmd/go/internal/search/search.go b/libgo/go/cmd/go/internal/search/search.go index 4efef24..18738cf 100644 --- a/libgo/go/cmd/go/internal/search/search.go +++ b/libgo/go/cmd/go/internal/search/search.go @@ -7,8 +7,10 @@ package search import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "fmt" "go/build" + "io/fs" "os" "path" "path/filepath" @@ -127,7 +129,7 @@ func (m *Match) MatchPackages() { if m.pattern == "cmd" { root += "cmd" + string(filepath.Separator) } - err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error { if err != nil { return err // Likely a permission error, which could interfere with matching. } @@ -153,8 +155,8 @@ func (m *Match) MatchPackages() { } if !fi.IsDir() { - if fi.Mode()&os.ModeSymlink != 0 && want { - if target, err := os.Stat(path); err == nil && target.IsDir() { + if fi.Mode()&fs.ModeSymlink != 0 && want { + if target, err := fsys.Stat(path); err == nil && target.IsDir() { fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) } } @@ -263,7 +265,7 @@ func (m *Match) MatchDirs() { } } - err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { + err := fsys.Walk(dir, func(path string, fi fs.FileInfo, err error) error { if err != nil { return err // Likely a permission error, which could interfere with matching. } @@ -272,7 +274,7 @@ func (m *Match) MatchDirs() { } top := false if path == dir { - // filepath.Walk starts at dir and recurses. For the recursive case, + // Walk starts at dir and recurses. For the recursive case, // the path is the result of filepath.Join, which calls filepath.Clean. // The initial case is not Cleaned, though, so we do this explicitly. // @@ -293,7 +295,7 @@ func (m *Match) MatchDirs() { if !top && cfg.ModulesEnabled { // Ignore other modules found in subdirectories. - if fi, err := os.Stat(filepath.Join(path, "go.mod")); err == nil && !fi.IsDir() { + if fi, err := fsys.Stat(filepath.Join(path, "go.mod")); err == nil && !fi.IsDir() { return filepath.SkipDir } } diff --git a/libgo/go/cmd/go/internal/str/path.go b/libgo/go/cmd/go/internal/str/path.go index 95d91a3..51ab2af 100644 --- a/libgo/go/cmd/go/internal/str/path.go +++ b/libgo/go/cmd/go/internal/str/path.go @@ -5,7 +5,6 @@ package str import ( - "path" "path/filepath" "strings" ) @@ -50,47 +49,3 @@ func HasFilePathPrefix(s, prefix string) bool { return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix } } - -// GlobsMatchPath reports whether any path prefix of target -// matches one of the glob patterns (as defined by path.Match) -// in the comma-separated globs list. -// It ignores any empty or malformed patterns in the list. -func GlobsMatchPath(globs, target string) bool { - for globs != "" { - // Extract next non-empty glob in comma-separated list. - var glob string - if i := strings.Index(globs, ","); i >= 0 { - glob, globs = globs[:i], globs[i+1:] - } else { - glob, globs = globs, "" - } - if glob == "" { - continue - } - - // A glob with N+1 path elements (N slashes) needs to be matched - // against the first N+1 path elements of target, - // which end just before the N+1'th slash. - n := strings.Count(glob, "/") - prefix := target - // Walk target, counting slashes, truncating at the N+1'th slash. - for i := 0; i < len(target); i++ { - if target[i] == '/' { - if n == 0 { - prefix = target[:i] - break - } - n-- - } - } - if n > 0 { - // Not enough prefix elements. - continue - } - matched, _ := path.Match(glob, prefix) - if matched { - return true - } - } - return false -} diff --git a/libgo/go/cmd/go/internal/str/str_test.go b/libgo/go/cmd/go/internal/str/str_test.go new file mode 100644 index 0000000..147ce1a --- /dev/null +++ b/libgo/go/cmd/go/internal/str/str_test.go @@ -0,0 +1,27 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package str + +import "testing" + +var foldDupTests = []struct { + list []string + f1, f2 string +}{ + {StringList("math/rand", "math/big"), "", ""}, + {StringList("math", "strings"), "", ""}, + {StringList("strings"), "", ""}, + {StringList("strings", "strings"), "strings", "strings"}, + {StringList("Rand", "rand", "math", "math/rand", "math/Rand"), "Rand", "rand"}, +} + +func TestFoldDup(t *testing.T) { + for _, tt := range foldDupTests { + f1, f2 := FoldDup(tt.list) + if f1 != tt.f1 || f2 != tt.f2 { + t.Errorf("foldDup(%q) = %q, %q, want %q, %q", tt.list, f1, f2, tt.f1, tt.f2) + } + } +} diff --git a/libgo/go/cmd/go/internal/test/flagdefs_test.go b/libgo/go/cmd/go/internal/test/flagdefs_test.go index 7562415..ab5440b 100644 --- a/libgo/go/cmd/go/internal/test/flagdefs_test.go +++ b/libgo/go/cmd/go/internal/test/flagdefs_test.go @@ -16,9 +16,14 @@ func TestPassFlagToTestIncludesAllTestFlags(t *testing.T) { return } name := strings.TrimPrefix(f.Name, "test.") - if name != "testlogfile" && !passFlagToTest[name] { - t.Errorf("passFlagToTest missing entry for %q (flag test.%s)", name, name) - t.Logf("(Run 'go generate cmd/go/internal/test' if it should be added.)") + switch name { + case "testlogfile", "paniconexit0": + // These are internal flags. + default: + if !passFlagToTest[name] { + t.Errorf("passFlagToTest missing entry for %q (flag test.%s)", name, name) + t.Logf("(Run 'go generate cmd/go/internal/test' if it should be added.)") + } } }) diff --git a/libgo/go/cmd/go/internal/test/genflags.go b/libgo/go/cmd/go/internal/test/genflags.go index 512fa16..5e83d53 100644 --- a/libgo/go/cmd/go/internal/test/genflags.go +++ b/libgo/go/cmd/go/internal/test/genflags.go @@ -62,9 +62,10 @@ func testFlags() []string { } name := strings.TrimPrefix(f.Name, "test.") - if name == "testlogfile" { - // test.testlogfile is “for use only by cmd/go” - } else { + switch name { + case "testlogfile", "paniconexit0": + // These flags are only for use by cmd/go. + default: names = append(names, name) } }) diff --git a/libgo/go/cmd/go/internal/test/test.go b/libgo/go/cmd/go/internal/test/test.go index 77bfc11..50fe2db 100644 --- a/libgo/go/cmd/go/internal/test/test.go +++ b/libgo/go/cmd/go/internal/test/test.go @@ -6,12 +6,13 @@ package test import ( "bytes" + "context" "crypto/sha256" "errors" "fmt" "go/build" "io" - "io/ioutil" + "io/fs" "os" "os/exec" "path" @@ -28,8 +29,8 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/lockedfile" - "cmd/go/internal/modload" "cmd/go/internal/str" + "cmd/go/internal/trace" "cmd/go/internal/work" "cmd/internal/test2json" ) @@ -148,6 +149,7 @@ In addition to the build flags, the flags handled by 'go test' itself are: -i Install packages that are dependencies of the test. Do not run the test. + The -i flag is deprecated. Compiled packages are cached automatically. -json Convert test output to JSON suitable for automated processing. @@ -565,18 +567,36 @@ var defaultVetFlags = []string{ // "-unusedresult", } -func runTest(cmd *base.Command, args []string) { - modload.LoadTests = true +func runTest(ctx context.Context, cmd *base.Command, args []string) { + load.ModResolveTests = true pkgArgs, testArgs = testFlags(args) + if cfg.DebugTrace != "" { + var close func() error + var err error + ctx, close, err = trace.Start(ctx, cfg.DebugTrace) + if err != nil { + base.Fatalf("failed to start trace: %v", err) + } + defer func() { + if err := close(); err != nil { + base.Fatalf("failed to stop trace: %v", err) + } + }() + } + + ctx, span := trace.StartSpan(ctx, fmt.Sprint("Running ", cmd.Name(), " command")) + defer span.Done() + work.FindExecCmd() // initialize cached result work.BuildInit() work.VetFlags = testVet.flags work.VetExplicit = testVet.explicit - pkgs = load.PackagesForBuild(pkgArgs) + pkgs = load.PackagesAndErrors(ctx, pkgArgs) + load.CheckPackageErrors(pkgs) if len(pkgs) == 0 { base.Fatalf("no packages to test") } @@ -621,6 +641,7 @@ func runTest(cmd *base.Command, args []string) { b.Init() if cfg.BuildI { + fmt.Fprint(os.Stderr, "go test: -i flag is deprecated\n") cfg.BuildV = testV deps := make(map[string]bool) @@ -658,7 +679,9 @@ func runTest(cmd *base.Command, args []string) { sort.Strings(all) a := &work.Action{Mode: "go test -i"} - for _, p := range load.PackagesForBuild(all) { + pkgs := load.PackagesAndErrors(ctx, all) + load.CheckPackageErrors(pkgs) + for _, p := range pkgs { if cfg.BuildToolchainName == "gccgo" && p.Standard { // gccgo's standard library packages // can not be reinstalled. @@ -666,7 +689,7 @@ func runTest(cmd *base.Command, args []string) { } a.Deps = append(a.Deps, b.CompileAction(work.ModeInstall, work.ModeInstall, p)) } - b.Do(a) + b.Do(ctx, a) if !testC || a.Failed { return } @@ -683,7 +706,7 @@ func runTest(cmd *base.Command, args []string) { } // Select for coverage all dependencies matching the testCoverPaths patterns. - for _, p := range load.TestPackageList(pkgs) { + for _, p := range load.TestPackageList(ctx, pkgs) { haveMatch := false for i := range testCoverPaths { if match[i](p) { @@ -745,7 +768,7 @@ func runTest(cmd *base.Command, args []string) { ensureImport(p, "sync/atomic") } - buildTest, runTest, printTest, err := builderTest(&b, p) + buildTest, runTest, printTest, err := builderTest(&b, ctx, p) if err != nil { str := err.Error() str = strings.TrimPrefix(str, "\n") @@ -786,7 +809,7 @@ func runTest(cmd *base.Command, args []string) { } } - b.Do(root) + b.Do(ctx, root) } // ensures that package p imports the named package @@ -812,7 +835,7 @@ var windowsBadWords = []string{ "update", } -func builderTest(b *work.Builder, p *load.Package) (buildAction, runAction, printAction *work.Action, err error) { +func builderTest(b *work.Builder, ctx context.Context, p *load.Package) (buildAction, runAction, printAction *work.Action, err error) { if len(p.TestGoFiles)+len(p.XTestGoFiles) == 0 { build := b.CompileAction(work.ModeBuild, work.ModeBuild, p) run := &work.Action{Mode: "test run", Package: p, Deps: []*work.Action{build}} @@ -835,7 +858,7 @@ func builderTest(b *work.Builder, p *load.Package) (buildAction, runAction, prin DeclVars: declareCoverVars, } } - pmain, ptest, pxtest, err := load.TestPackagesFor(p, cover) + pmain, ptest, pxtest, err := load.TestPackagesFor(ctx, p, cover) if err != nil { return nil, nil, nil, err } @@ -863,7 +886,7 @@ func builderTest(b *work.Builder, p *load.Package) (buildAction, runAction, prin if !cfg.BuildN { // writeTestmain writes _testmain.go, // using the test description gathered in t. - if err := ioutil.WriteFile(testDir+"_testmain.go", *pmain.Internal.TestmainGo, 0666); err != nil { + if err := os.WriteFile(testDir+"_testmain.go", *pmain.Internal.TestmainGo, 0666); err != nil { return nil, nil, nil, err } } @@ -1068,7 +1091,7 @@ func (lockedStdout) Write(b []byte) (int, error) { } // builderRunTest is the action for running a test binary. -func (c *runCache) builderRunTest(b *work.Builder, a *work.Action) error { +func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work.Action) error { if a.Failed { // We were unable to build the binary. a.Failed = false @@ -1145,7 +1168,8 @@ func (c *runCache) builderRunTest(b *work.Builder, a *work.Action) error { if !c.disableCache && len(execCmd) == 0 { testlogArg = []string{"-test.testlogfile=" + a.Objdir + "testlog.txt"} } - args := str.StringList(execCmd, a.Deps[0].BuiltTarget(), testlogArg, testArgs) + panicArg := "-test.paniconexit0" + args := str.StringList(execCmd, a.Deps[0].BuiltTarget(), testlogArg, panicArg, testArgs) if testCoverProfile != "" { // Write coverage to temporary profile, for merging later. @@ -1539,13 +1563,18 @@ func hashOpen(name string) (cache.ActionID, error) { } hashWriteStat(h, info) if info.IsDir() { - names, err := ioutil.ReadDir(name) + files, err := os.ReadDir(name) if err != nil { fmt.Fprintf(h, "err %v\n", err) } - for _, f := range names { + for _, f := range files { fmt.Fprintf(h, "file %s ", f.Name()) - hashWriteStat(h, f) + finfo, err := f.Info() + if err != nil { + fmt.Fprintf(h, "err %v\n", err) + } else { + hashWriteStat(h, finfo) + } } } else if info.Mode().IsRegular() { // Because files might be very large, do not attempt @@ -1579,7 +1608,7 @@ func hashStat(name string) cache.ActionID { return h.Sum() } -func hashWriteStat(h io.Writer, info os.FileInfo) { +func hashWriteStat(h io.Writer, info fs.FileInfo) { fmt.Fprintf(h, "stat %d %x %v %v\n", info.Size(), uint64(info.Mode()), info.ModTime(), info.IsDir()) } @@ -1594,7 +1623,7 @@ func (c *runCache) saveOutput(a *work.Action) { } // See comment about two-level lookup in tryCacheWithID above. - testlog, err := ioutil.ReadFile(a.Objdir + "testlog.txt") + testlog, err := os.ReadFile(a.Objdir + "testlog.txt") if err != nil || !bytes.HasPrefix(testlog, testlogMagic) || testlog[len(testlog)-1] != '\n' { if cache.DebugTest { if err != nil { @@ -1645,7 +1674,7 @@ func coveragePercentage(out []byte) string { } // builderCleanTest is the action for cleaning up after a test. -func builderCleanTest(b *work.Builder, a *work.Action) error { +func builderCleanTest(b *work.Builder, ctx context.Context, a *work.Action) error { if cfg.BuildWork { return nil } @@ -1657,7 +1686,7 @@ func builderCleanTest(b *work.Builder, a *work.Action) error { } // builderPrintTest is the action for printing a test result. -func builderPrintTest(b *work.Builder, a *work.Action) error { +func builderPrintTest(b *work.Builder, ctx context.Context, a *work.Action) error { clean := a.Deps[0] run := clean.Deps[0] if run.TestOutput != nil { @@ -1668,7 +1697,7 @@ func builderPrintTest(b *work.Builder, a *work.Action) error { } // builderNoTest is the action for testing a package with no test files. -func builderNoTest(b *work.Builder, a *work.Action) error { +func builderNoTest(b *work.Builder, ctx context.Context, a *work.Action) error { var stdout io.Writer = os.Stdout if testJSON { json := test2json.NewConverter(lockedStdout{}, a.Package.ImportPath, test2json.Timestamp) @@ -1680,7 +1709,7 @@ func builderNoTest(b *work.Builder, a *work.Action) error { } // printExitStatus is the action for printing the exit status -func printExitStatus(b *work.Builder, a *work.Action) error { +func printExitStatus(b *work.Builder, ctx context.Context, a *work.Action) error { if !testJSON && len(pkgArgs) != 0 { if base.GetExitStatus() != 0 { fmt.Println("FAIL") diff --git a/libgo/go/cmd/go/internal/test/testflag.go b/libgo/go/cmd/go/internal/test/testflag.go index 4f0a892..d2671ff5 100644 --- a/libgo/go/cmd/go/internal/test/testflag.go +++ b/libgo/go/cmd/go/internal/test/testflag.go @@ -212,6 +212,10 @@ func testFlags(args []string) (packageNames, passToTest []string) { } }) + // firstUnknownFlag helps us report an error when flags not known to 'go + // test' are used along with -i or -c. + firstUnknownFlag := "" + explicitArgs := make([]string, 0, len(args)) inPkgList := false afterFlagWithoutValue := false @@ -288,6 +292,10 @@ func testFlags(args []string) (packageNames, passToTest []string) { break } + if firstUnknownFlag == "" { + firstUnknownFlag = nd.RawArg + } + explicitArgs = append(explicitArgs, nd.RawArg) args = remainingArgs if !nd.HasValue { @@ -312,6 +320,14 @@ func testFlags(args []string) (packageNames, passToTest []string) { args = remainingArgs } + if firstUnknownFlag != "" && (testC || cfg.BuildI) { + buildFlag := "-c" + if !testC { + buildFlag = "-i" + } + fmt.Fprintf(os.Stderr, "flag %s is not a 'go test' flag (unknown flags cannot be used with %s)\n", firstUnknownFlag, buildFlag) + exitWithUsage() + } var injectedFlags []string if testJSON { diff --git a/libgo/go/cmd/go/internal/tool/tool.go b/libgo/go/cmd/go/internal/tool/tool.go index 930eecb..7f4dc86 100644 --- a/libgo/go/cmd/go/internal/tool/tool.go +++ b/libgo/go/cmd/go/internal/tool/tool.go @@ -6,6 +6,7 @@ package tool import ( + "context" "fmt" "os" "os/exec" @@ -48,7 +49,7 @@ func init() { CmdTool.Flag.BoolVar(&toolN, "n", false, "") } -func runTool(cmd *base.Command, args []string) { +func runTool(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { listTools() return diff --git a/libgo/go/cmd/go/internal/trace/trace.go b/libgo/go/cmd/go/internal/trace/trace.go new file mode 100644 index 0000000..f108a2b --- /dev/null +++ b/libgo/go/cmd/go/internal/trace/trace.go @@ -0,0 +1,206 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package trace + +import ( + "cmd/internal/traceviewer" + "context" + "encoding/json" + "errors" + "os" + "strings" + "sync/atomic" + "time" +) + +// Constants used in event fields. +// See https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU +// for more details. +const ( + phaseDurationBegin = "B" + phaseDurationEnd = "E" + phaseFlowStart = "s" + phaseFlowEnd = "f" + + bindEnclosingSlice = "e" +) + +var traceStarted int32 + +func getTraceContext(ctx context.Context) (traceContext, bool) { + if atomic.LoadInt32(&traceStarted) == 0 { + return traceContext{}, false + } + v := ctx.Value(traceKey{}) + if v == nil { + return traceContext{}, false + } + return v.(traceContext), true +} + +// StartSpan starts a trace event with the given name. The Span ends when its Done method is called. +func StartSpan(ctx context.Context, name string) (context.Context, *Span) { + tc, ok := getTraceContext(ctx) + if !ok { + return ctx, nil + } + childSpan := &Span{t: tc.t, name: name, tid: tc.tid, start: time.Now()} + tc.t.writeEvent(&traceviewer.Event{ + Name: childSpan.name, + Time: float64(childSpan.start.UnixNano()) / float64(time.Microsecond), + TID: childSpan.tid, + Phase: phaseDurationBegin, + }) + ctx = context.WithValue(ctx, traceKey{}, traceContext{tc.t, tc.tid}) + return ctx, childSpan +} + +// StartGoroutine associates the context with a new Thread ID. The Chrome trace viewer associates each +// trace event with a thread, and doesn't expect events with the same thread id to happen at the +// same time. +func StartGoroutine(ctx context.Context) context.Context { + tc, ok := getTraceContext(ctx) + if !ok { + return ctx + } + return context.WithValue(ctx, traceKey{}, traceContext{tc.t, tc.t.getNextTID()}) +} + +// Flow marks a flow indicating that the 'to' span depends on the 'from' span. +// Flow should be called while the 'to' span is in progress. +func Flow(ctx context.Context, from *Span, to *Span) { + tc, ok := getTraceContext(ctx) + if !ok || from == nil || to == nil { + return + } + + id := tc.t.getNextFlowID() + tc.t.writeEvent(&traceviewer.Event{ + Name: from.name + " -> " + to.name, + Category: "flow", + ID: id, + Time: float64(from.end.UnixNano()) / float64(time.Microsecond), + Phase: phaseFlowStart, + TID: from.tid, + }) + tc.t.writeEvent(&traceviewer.Event{ + Name: from.name + " -> " + to.name, + Category: "flow", // TODO(matloob): Add Category to Flow? + ID: id, + Time: float64(to.start.UnixNano()) / float64(time.Microsecond), + Phase: phaseFlowEnd, + TID: to.tid, + BindPoint: bindEnclosingSlice, + }) +} + +type Span struct { + t *tracer + + name string + tid uint64 + start time.Time + end time.Time +} + +func (s *Span) Done() { + if s == nil { + return + } + s.end = time.Now() + s.t.writeEvent(&traceviewer.Event{ + Name: s.name, + Time: float64(s.end.UnixNano()) / float64(time.Microsecond), + TID: s.tid, + Phase: phaseDurationEnd, + }) +} + +type tracer struct { + file chan traceFile // 1-buffered + + nextTID uint64 + nextFlowID uint64 +} + +func (t *tracer) writeEvent(ev *traceviewer.Event) error { + f := <-t.file + defer func() { t.file <- f }() + var err error + if f.entries == 0 { + _, err = f.sb.WriteString("[\n") + } else { + _, err = f.sb.WriteString(",") + } + f.entries++ + if err != nil { + return nil + } + + if err := f.enc.Encode(ev); err != nil { + return err + } + + // Write event string to output file. + _, err = f.f.WriteString(f.sb.String()) + f.sb.Reset() + return err +} + +func (t *tracer) Close() error { + f := <-t.file + defer func() { t.file <- f }() + + _, firstErr := f.f.WriteString("]") + if err := f.f.Close(); firstErr == nil { + firstErr = err + } + return firstErr +} + +func (t *tracer) getNextTID() uint64 { + return atomic.AddUint64(&t.nextTID, 1) +} + +func (t *tracer) getNextFlowID() uint64 { + return atomic.AddUint64(&t.nextFlowID, 1) +} + +// traceKey is the context key for tracing information. It is unexported to prevent collisions with context keys defined in +// other packages. +type traceKey struct{} + +type traceContext struct { + t *tracer + tid uint64 +} + +// Start starts a trace which writes to the given file. +func Start(ctx context.Context, file string) (context.Context, func() error, error) { + atomic.StoreInt32(&traceStarted, 1) + if file == "" { + return nil, nil, errors.New("no trace file supplied") + } + f, err := os.Create(file) + if err != nil { + return nil, nil, err + } + t := &tracer{file: make(chan traceFile, 1)} + sb := new(strings.Builder) + t.file <- traceFile{ + f: f, + sb: sb, + enc: json.NewEncoder(sb), + } + ctx = context.WithValue(ctx, traceKey{}, traceContext{t: t}) + return ctx, t.Close, nil +} + +type traceFile struct { + f *os.File + sb *strings.Builder + enc *json.Encoder + entries int64 +} diff --git a/libgo/go/cmd/go/internal/txtar/archive.go b/libgo/go/cmd/go/internal/txtar/archive.go index c384f33..1796684 100644 --- a/libgo/go/cmd/go/internal/txtar/archive.go +++ b/libgo/go/cmd/go/internal/txtar/archive.go @@ -34,7 +34,7 @@ package txtar import ( "bytes" "fmt" - "io/ioutil" + "os" "strings" ) @@ -66,7 +66,7 @@ func Format(a *Archive) []byte { // ParseFile parses the named file as an archive. func ParseFile(file string) (*Archive, error) { - data, err := ioutil.ReadFile(file) + data, err := os.ReadFile(file) if err != nil { return nil, err } diff --git a/libgo/go/cmd/go/internal/get/discovery.go b/libgo/go/cmd/go/internal/vcs/discovery.go index afa6ef4..327b44c 100644 --- a/libgo/go/cmd/go/internal/get/discovery.go +++ b/libgo/go/cmd/go/internal/vcs/discovery.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( "encoding/xml" diff --git a/libgo/go/cmd/go/internal/get/pkg_test.go b/libgo/go/cmd/go/internal/vcs/discovery_test.go index fc6a179..eb99fdf6 100644 --- a/libgo/go/cmd/go/internal/get/pkg_test.go +++ b/libgo/go/cmd/go/internal/vcs/discovery_test.go @@ -2,35 +2,14 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( - "cmd/go/internal/str" "reflect" "strings" "testing" ) -var foldDupTests = []struct { - list []string - f1, f2 string -}{ - {str.StringList("math/rand", "math/big"), "", ""}, - {str.StringList("math", "strings"), "", ""}, - {str.StringList("strings"), "", ""}, - {str.StringList("strings", "strings"), "strings", "strings"}, - {str.StringList("Rand", "rand", "math", "math/rand", "math/Rand"), "Rand", "rand"}, -} - -func TestFoldDup(t *testing.T) { - for _, tt := range foldDupTests { - f1, f2 := str.FoldDup(tt.list) - if f1 != tt.f1 || f2 != tt.f2 { - t.Errorf("foldDup(%q) = %q, %q, want %q, %q", tt.list, f1, f2, tt.f1, tt.f2) - } - } -} - var parseMetaGoImportsTests = []struct { in string mod ModuleMode diff --git a/libgo/go/cmd/go/internal/get/vcs.go b/libgo/go/cmd/go/internal/vcs/vcs.go index fd37fcb..4894ecd 100644 --- a/libgo/go/cmd/go/internal/get/vcs.go +++ b/libgo/go/cmd/go/internal/vcs/vcs.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( "encoding/json" @@ -10,6 +10,7 @@ import ( "fmt" "internal/lazyregexp" "internal/singleflight" + "io/fs" "log" urlpkg "net/url" "os" @@ -21,29 +22,32 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/load" + "cmd/go/internal/search" + "cmd/go/internal/str" "cmd/go/internal/web" + + "golang.org/x/mod/module" ) // A vcsCmd describes how to use a version control system // like Mercurial, Git, or Subversion. -type vcsCmd struct { - name string - cmd string // name of binary to invoke command +type Cmd struct { + Name string + Cmd string // name of binary to invoke command - createCmd []string // commands to download a fresh copy of a repository - downloadCmd []string // commands to download updates into an existing repository + CreateCmd []string // commands to download a fresh copy of a repository + DownloadCmd []string // commands to download updates into an existing repository - tagCmd []tagCmd // commands to list tags - tagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd - tagSyncCmd []string // commands to sync to specific tag - tagSyncDefault []string // commands to sync to default tag + TagCmd []tagCmd // commands to list tags + TagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd + TagSyncCmd []string // commands to sync to specific tag + TagSyncDefault []string // commands to sync to default tag - scheme []string - pingCmd string + Scheme []string + PingCmd string - remoteRepo func(v *vcsCmd, rootDir string) (remoteRepo string, err error) - resolveRepo func(v *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) + RemoteRepo func(v *Cmd, rootDir string) (remoteRepo string, err error) + ResolveRepo func(v *Cmd, rootDir, remoteRepo string) (realRepo string, err error) } var defaultSecureScheme = map[string]bool{ @@ -54,7 +58,7 @@ var defaultSecureScheme = map[string]bool{ "ssh": true, } -func (v *vcsCmd) isSecure(repo string) bool { +func (v *Cmd) IsSecure(repo string) bool { u, err := urlpkg.Parse(repo) if err != nil { // If repo is not a URL, it's not secure. @@ -63,8 +67,8 @@ func (v *vcsCmd) isSecure(repo string) bool { return v.isSecureScheme(u.Scheme) } -func (v *vcsCmd) isSecureScheme(scheme string) bool { - switch v.cmd { +func (v *Cmd) isSecureScheme(scheme string) bool { + switch v.Cmd { case "git": // GIT_ALLOW_PROTOCOL is an environment variable defined by Git. It is a // colon-separated list of schemes that are allowed to be used with git @@ -89,7 +93,7 @@ type tagCmd struct { } // vcsList lists the known version control systems -var vcsList = []*vcsCmd{ +var vcsList = []*Cmd{ vcsHg, vcsGit, vcsSvn, @@ -97,11 +101,15 @@ var vcsList = []*vcsCmd{ vcsFossil, } +// vcsMod is a stub for the "mod" scheme. It's returned by +// repoRootForImportPathDynamic, but is otherwise not treated as a VCS command. +var vcsMod = &Cmd{Name: "mod"} + // vcsByCmd returns the version control system for the given // command name (hg, git, svn, bzr). -func vcsByCmd(cmd string) *vcsCmd { +func vcsByCmd(cmd string) *Cmd { for _, vcs := range vcsList { - if vcs.cmd == cmd { + if vcs.Cmd == cmd { return vcs } } @@ -109,31 +117,31 @@ func vcsByCmd(cmd string) *vcsCmd { } // vcsHg describes how to use Mercurial. -var vcsHg = &vcsCmd{ - name: "Mercurial", - cmd: "hg", +var vcsHg = &Cmd{ + Name: "Mercurial", + Cmd: "hg", - createCmd: []string{"clone -U -- {repo} {dir}"}, - downloadCmd: []string{"pull"}, + CreateCmd: []string{"clone -U -- {repo} {dir}"}, + DownloadCmd: []string{"pull"}, // We allow both tag and branch names as 'tags' // for selecting a version. This lets people have // a go.release.r60 branch and a go1 branch // and make changes in both, without constantly // editing .hgtags. - tagCmd: []tagCmd{ + TagCmd: []tagCmd{ {"tags", `^(\S+)`}, {"branches", `^(\S+)`}, }, - tagSyncCmd: []string{"update -r {tag}"}, - tagSyncDefault: []string{"update default"}, + TagSyncCmd: []string{"update -r {tag}"}, + TagSyncDefault: []string{"update default"}, - scheme: []string{"https", "http", "ssh"}, - pingCmd: "identify -- {scheme}://{repo}", - remoteRepo: hgRemoteRepo, + Scheme: []string{"https", "http", "ssh"}, + PingCmd: "identify -- {scheme}://{repo}", + RemoteRepo: hgRemoteRepo, } -func hgRemoteRepo(vcsHg *vcsCmd, rootDir string) (remoteRepo string, err error) { +func hgRemoteRepo(vcsHg *Cmd, rootDir string) (remoteRepo string, err error) { out, err := vcsHg.runOutput(rootDir, "paths default") if err != nil { return "", err @@ -142,45 +150,45 @@ func hgRemoteRepo(vcsHg *vcsCmd, rootDir string) (remoteRepo string, err error) } // vcsGit describes how to use Git. -var vcsGit = &vcsCmd{ - name: "Git", - cmd: "git", +var vcsGit = &Cmd{ + Name: "Git", + Cmd: "git", - createCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, - downloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, + CreateCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, + DownloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, - tagCmd: []tagCmd{ + TagCmd: []tagCmd{ // tags/xxx matches a git tag named xxx // origin/xxx matches a git branch named xxx on the default remote repository {"show-ref", `(?:tags|origin)/(\S+)$`}, }, - tagLookupCmd: []tagCmd{ + TagLookupCmd: []tagCmd{ {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`}, }, - tagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, + TagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, // both createCmd and downloadCmd update the working dir. // No need to do more here. We used to 'checkout master' // but that doesn't work if the default branch is not named master. // DO NOT add 'checkout master' here. // See golang.org/issue/9032. - tagSyncDefault: []string{"submodule update --init --recursive"}, + TagSyncDefault: []string{"submodule update --init --recursive"}, - scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, + Scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, // Leave out the '--' separator in the ls-remote command: git 2.7.4 does not // support such a separator for that command, and this use should be safe // without it because the {scheme} value comes from the predefined list above. // See golang.org/issue/33836. - pingCmd: "ls-remote {scheme}://{repo}", + PingCmd: "ls-remote {scheme}://{repo}", - remoteRepo: gitRemoteRepo, + RemoteRepo: gitRemoteRepo, } // scpSyntaxRe matches the SCP-like addresses used by Git to access // repositories by SSH. var scpSyntaxRe = lazyregexp.New(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) -func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error) { +func gitRemoteRepo(vcsGit *Cmd, rootDir string) (remoteRepo string, err error) { cmd := "config remote.origin.url" errParse := errors.New("unable to parse output of git " + cmd) errRemoteOriginNotFound := errors.New("remote origin not found") @@ -216,7 +224,7 @@ func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error // Iterate over insecure schemes too, because this function simply // reports the state of the repo. If we can't see insecure schemes then // we can't report the actual repo URL. - for _, s := range vcsGit.scheme { + for _, s := range vcsGit.Scheme { if repoURL.Scheme == s { return repoURL.String(), nil } @@ -225,27 +233,27 @@ func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error } // vcsBzr describes how to use Bazaar. -var vcsBzr = &vcsCmd{ - name: "Bazaar", - cmd: "bzr", +var vcsBzr = &Cmd{ + Name: "Bazaar", + Cmd: "bzr", - createCmd: []string{"branch -- {repo} {dir}"}, + CreateCmd: []string{"branch -- {repo} {dir}"}, // Without --overwrite bzr will not pull tags that changed. // Replace by --overwrite-tags after http://pad.lv/681792 goes in. - downloadCmd: []string{"pull --overwrite"}, + DownloadCmd: []string{"pull --overwrite"}, - tagCmd: []tagCmd{{"tags", `^(\S+)`}}, - tagSyncCmd: []string{"update -r {tag}"}, - tagSyncDefault: []string{"update -r revno:-1"}, + TagCmd: []tagCmd{{"tags", `^(\S+)`}}, + TagSyncCmd: []string{"update -r {tag}"}, + TagSyncDefault: []string{"update -r revno:-1"}, - scheme: []string{"https", "http", "bzr", "bzr+ssh"}, - pingCmd: "info -- {scheme}://{repo}", - remoteRepo: bzrRemoteRepo, - resolveRepo: bzrResolveRepo, + Scheme: []string{"https", "http", "bzr", "bzr+ssh"}, + PingCmd: "info -- {scheme}://{repo}", + RemoteRepo: bzrRemoteRepo, + ResolveRepo: bzrResolveRepo, } -func bzrRemoteRepo(vcsBzr *vcsCmd, rootDir string) (remoteRepo string, err error) { +func bzrRemoteRepo(vcsBzr *Cmd, rootDir string) (remoteRepo string, err error) { outb, err := vcsBzr.runOutput(rootDir, "config parent_location") if err != nil { return "", err @@ -253,7 +261,7 @@ func bzrRemoteRepo(vcsBzr *vcsCmd, rootDir string) (remoteRepo string, err error return strings.TrimSpace(string(outb)), nil } -func bzrResolveRepo(vcsBzr *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) { +func bzrResolveRepo(vcsBzr *Cmd, rootDir, remoteRepo string) (realRepo string, err error) { outb, err := vcsBzr.runOutput(rootDir, "info "+remoteRepo) if err != nil { return "", err @@ -287,22 +295,22 @@ func bzrResolveRepo(vcsBzr *vcsCmd, rootDir, remoteRepo string) (realRepo string } // vcsSvn describes how to use Subversion. -var vcsSvn = &vcsCmd{ - name: "Subversion", - cmd: "svn", +var vcsSvn = &Cmd{ + Name: "Subversion", + Cmd: "svn", - createCmd: []string{"checkout -- {repo} {dir}"}, - downloadCmd: []string{"update"}, + CreateCmd: []string{"checkout -- {repo} {dir}"}, + DownloadCmd: []string{"update"}, // There is no tag command in subversion. // The branch information is all in the path names. - scheme: []string{"https", "http", "svn", "svn+ssh"}, - pingCmd: "info -- {scheme}://{repo}", - remoteRepo: svnRemoteRepo, + Scheme: []string{"https", "http", "svn", "svn+ssh"}, + PingCmd: "info -- {scheme}://{repo}", + RemoteRepo: svnRemoteRepo, } -func svnRemoteRepo(vcsSvn *vcsCmd, rootDir string) (remoteRepo string, err error) { +func svnRemoteRepo(vcsSvn *Cmd, rootDir string) (remoteRepo string, err error) { outb, err := vcsSvn.runOutput(rootDir, "info") if err != nil { return "", err @@ -337,22 +345,22 @@ func svnRemoteRepo(vcsSvn *vcsCmd, rootDir string) (remoteRepo string, err error const fossilRepoName = ".fossil" // vcsFossil describes how to use Fossil (fossil-scm.org) -var vcsFossil = &vcsCmd{ - name: "Fossil", - cmd: "fossil", +var vcsFossil = &Cmd{ + Name: "Fossil", + Cmd: "fossil", - createCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"}, - downloadCmd: []string{"up"}, + CreateCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"}, + DownloadCmd: []string{"up"}, - tagCmd: []tagCmd{{"tag ls", `(.*)`}}, - tagSyncCmd: []string{"up tag:{tag}"}, - tagSyncDefault: []string{"up trunk"}, + TagCmd: []tagCmd{{"tag ls", `(.*)`}}, + TagSyncCmd: []string{"up tag:{tag}"}, + TagSyncDefault: []string{"up trunk"}, - scheme: []string{"https", "http"}, - remoteRepo: fossilRemoteRepo, + Scheme: []string{"https", "http"}, + RemoteRepo: fossilRemoteRepo, } -func fossilRemoteRepo(vcsFossil *vcsCmd, rootDir string) (remoteRepo string, err error) { +func fossilRemoteRepo(vcsFossil *Cmd, rootDir string) (remoteRepo string, err error) { out, err := vcsFossil.runOutput(rootDir, "remote-url") if err != nil { return "", err @@ -360,8 +368,8 @@ func fossilRemoteRepo(vcsFossil *vcsCmd, rootDir string) (remoteRepo string, err return strings.TrimSpace(string(out)), nil } -func (v *vcsCmd) String() string { - return v.name +func (v *Cmd) String() string { + return v.Name } // run runs the command line cmd in the given directory. @@ -371,24 +379,24 @@ func (v *vcsCmd) String() string { // If an error occurs, run prints the command line and the // command's combined stdout+stderr to standard error. // Otherwise run discards the command's output. -func (v *vcsCmd) run(dir string, cmd string, keyval ...string) error { +func (v *Cmd) run(dir string, cmd string, keyval ...string) error { _, err := v.run1(dir, cmd, keyval, true) return err } // runVerboseOnly is like run but only generates error output to standard error in verbose mode. -func (v *vcsCmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { +func (v *Cmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { _, err := v.run1(dir, cmd, keyval, false) return err } // runOutput is like run but returns the output of the command. -func (v *vcsCmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { +func (v *Cmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { return v.run1(dir, cmd, keyval, true) } // run1 is the generalized implementation of run and runOutput. -func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { +func (v *Cmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { m := make(map[string]string) for i := 0; i < len(keyval); i += 2 { m[keyval[i]] = keyval[i+1] @@ -401,9 +409,9 @@ func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) if len(args) >= 2 && args[0] == "-go-internal-mkdir" { var err error if filepath.IsAbs(args[1]) { - err = os.Mkdir(args[1], os.ModePerm) + err = os.Mkdir(args[1], fs.ModePerm) } else { - err = os.Mkdir(filepath.Join(dir, args[1]), os.ModePerm) + err = os.Mkdir(filepath.Join(dir, args[1]), fs.ModePerm) } if err != nil { return nil, err @@ -420,25 +428,25 @@ func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) args = args[2:] } - _, err := exec.LookPath(v.cmd) + _, err := exec.LookPath(v.Cmd) if err != nil { fmt.Fprintf(os.Stderr, "go: missing %s command. See https://golang.org/s/gogetcmd\n", - v.name) + v.Name) return nil, err } - cmd := exec.Command(v.cmd, args...) + cmd := exec.Command(v.Cmd, args...) cmd.Dir = dir cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) if cfg.BuildX { fmt.Fprintf(os.Stderr, "cd %s\n", dir) - fmt.Fprintf(os.Stderr, "%s %s\n", v.cmd, strings.Join(args, " ")) + fmt.Fprintf(os.Stderr, "%s %s\n", v.Cmd, strings.Join(args, " ")) } out, err := cmd.Output() if err != nil { if verbose || cfg.BuildV { - fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.cmd, strings.Join(args, " ")) + fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.Cmd, strings.Join(args, " ")) if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 { os.Stderr.Write(ee.Stderr) } else { @@ -449,15 +457,15 @@ func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) return out, err } -// ping pings to determine scheme to use. -func (v *vcsCmd) ping(scheme, repo string) error { - return v.runVerboseOnly(".", v.pingCmd, "scheme", scheme, "repo", repo) +// Ping pings to determine scheme to use. +func (v *Cmd) Ping(scheme, repo string) error { + return v.runVerboseOnly(".", v.PingCmd, "scheme", scheme, "repo", repo) } -// create creates a new copy of repo in dir. +// Create creates a new copy of repo in dir. // The parent of dir must exist; dir must not. -func (v *vcsCmd) create(dir, repo string) error { - for _, cmd := range v.createCmd { +func (v *Cmd) Create(dir, repo string) error { + for _, cmd := range v.CreateCmd { if err := v.run(".", cmd, "dir", dir, "repo", repo); err != nil { return err } @@ -465,9 +473,9 @@ func (v *vcsCmd) create(dir, repo string) error { return nil } -// download downloads any new changes for the repo in dir. -func (v *vcsCmd) download(dir string) error { - for _, cmd := range v.downloadCmd { +// Download downloads any new changes for the repo in dir. +func (v *Cmd) Download(dir string) error { + for _, cmd := range v.DownloadCmd { if err := v.run(dir, cmd); err != nil { return err } @@ -475,10 +483,10 @@ func (v *vcsCmd) download(dir string) error { return nil } -// tags returns the list of available tags for the repo in dir. -func (v *vcsCmd) tags(dir string) ([]string, error) { +// Tags returns the list of available tags for the repo in dir. +func (v *Cmd) Tags(dir string) ([]string, error) { var tags []string - for _, tc := range v.tagCmd { + for _, tc := range v.TagCmd { out, err := v.runOutput(dir, tc.cmd) if err != nil { return nil, err @@ -493,12 +501,12 @@ func (v *vcsCmd) tags(dir string) ([]string, error) { // tagSync syncs the repo in dir to the named tag, // which either is a tag returned by tags or is v.tagDefault. -func (v *vcsCmd) tagSync(dir, tag string) error { - if v.tagSyncCmd == nil { +func (v *Cmd) TagSync(dir, tag string) error { + if v.TagSyncCmd == nil { return nil } if tag != "" { - for _, tc := range v.tagLookupCmd { + for _, tc := range v.TagLookupCmd { out, err := v.runOutput(dir, tc.cmd, "tag", tag) if err != nil { return err @@ -512,8 +520,8 @@ func (v *vcsCmd) tagSync(dir, tag string) error { } } - if tag == "" && v.tagSyncDefault != nil { - for _, cmd := range v.tagSyncDefault { + if tag == "" && v.TagSyncDefault != nil { + for _, cmd := range v.TagSyncDefault { if err := v.run(dir, cmd); err != nil { return err } @@ -521,7 +529,7 @@ func (v *vcsCmd) tagSync(dir, tag string) error { return nil } - for _, cmd := range v.tagSyncCmd { + for _, cmd := range v.TagSyncCmd { if err := v.run(dir, cmd, "tag", tag); err != nil { return err } @@ -532,7 +540,7 @@ func (v *vcsCmd) tagSync(dir, tag string) error { // A vcsPath describes how to convert an import path into a // version control system and repository name. type vcsPath struct { - prefix string // prefix this description applies to + pathPrefix string // prefix this description applies to regexp *lazyregexp.Regexp // compiled pattern for import path repo string // repository to use (expand with match of re) vcs string // version control system to use (expand with match of re) @@ -540,11 +548,11 @@ type vcsPath struct { schemelessRepo bool // if true, the repo pattern lacks a scheme } -// vcsFromDir inspects dir and its parents to determine the +// FromDir inspects dir and its parents to determine the // version control system and code repository to use. // On return, root is the import path // corresponding to the root of the repository. -func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { +func FromDir(dir, srcRoot string) (vcs *Cmd, root string, err error) { // Clean and double-check that dir is in (a subdirectory of) srcRoot. dir = filepath.Clean(dir) srcRoot = filepath.Clean(srcRoot) @@ -552,13 +560,13 @@ func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) } - var vcsRet *vcsCmd + var vcsRet *Cmd var rootRet string origDir := dir for len(dir) > len(srcRoot) { for _, vcs := range vcsList { - if _, err := os.Stat(filepath.Join(dir, "."+vcs.cmd)); err == nil { + if _, err := os.Stat(filepath.Join(dir, "."+vcs.Cmd)); err == nil { root := filepath.ToSlash(dir[len(srcRoot)+1:]) // Record first VCS we find, but keep looking, // to detect mistakes like one kind of VCS inside another. @@ -568,12 +576,12 @@ func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { continue } // Allow .git inside .git, which can arise due to submodules. - if vcsRet == vcs && vcs.cmd == "git" { + if vcsRet == vcs && vcs.Cmd == "git" { continue } // Otherwise, we have one VCS inside a different VCS. return nil, "", fmt.Errorf("directory %q uses %s, but parent %q uses %s", - filepath.Join(srcRoot, rootRet), vcsRet.cmd, filepath.Join(srcRoot, root), vcs.cmd) + filepath.Join(srcRoot, rootRet), vcsRet.Cmd, filepath.Join(srcRoot, root), vcs.Cmd) } } @@ -587,15 +595,149 @@ func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { } if vcsRet != nil { + if err := checkGOVCS(vcsRet, rootRet); err != nil { + return nil, "", err + } return vcsRet, rootRet, nil } return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir) } -// checkNestedVCS checks for an incorrectly-nested VCS-inside-VCS +// A govcsRule is a single GOVCS rule like private:hg|svn. +type govcsRule struct { + pattern string + allowed []string +} + +// A govcsConfig is a full GOVCS configuration. +type govcsConfig []govcsRule + +func parseGOVCS(s string) (govcsConfig, error) { + s = strings.TrimSpace(s) + if s == "" { + return nil, nil + } + var cfg govcsConfig + have := make(map[string]string) + for _, item := range strings.Split(s, ",") { + item = strings.TrimSpace(item) + if item == "" { + return nil, fmt.Errorf("empty entry in GOVCS") + } + i := strings.Index(item, ":") + if i < 0 { + return nil, fmt.Errorf("malformed entry in GOVCS (missing colon): %q", item) + } + pattern, list := strings.TrimSpace(item[:i]), strings.TrimSpace(item[i+1:]) + if pattern == "" { + return nil, fmt.Errorf("empty pattern in GOVCS: %q", item) + } + if list == "" { + return nil, fmt.Errorf("empty VCS list in GOVCS: %q", item) + } + if search.IsRelativePath(pattern) { + return nil, fmt.Errorf("relative pattern not allowed in GOVCS: %q", pattern) + } + if old := have[pattern]; old != "" { + return nil, fmt.Errorf("unreachable pattern in GOVCS: %q after %q", item, old) + } + have[pattern] = item + allowed := strings.Split(list, "|") + for i, a := range allowed { + a = strings.TrimSpace(a) + if a == "" { + return nil, fmt.Errorf("empty VCS name in GOVCS: %q", item) + } + allowed[i] = a + } + cfg = append(cfg, govcsRule{pattern, allowed}) + } + return cfg, nil +} + +func (c *govcsConfig) allow(path string, private bool, vcs string) bool { + for _, rule := range *c { + match := false + switch rule.pattern { + case "private": + match = private + case "public": + match = !private + default: + // Note: rule.pattern is known to be comma-free, + // so MatchPrefixPatterns is only matching a single pattern for us. + match = module.MatchPrefixPatterns(rule.pattern, path) + } + if !match { + continue + } + for _, allow := range rule.allowed { + if allow == vcs || allow == "all" { + return true + } + } + return false + } + + // By default, nothing is allowed. + return false +} + +var ( + govcs govcsConfig + govcsErr error + govcsOnce sync.Once +) + +// defaultGOVCS is the default setting for GOVCS. +// Setting GOVCS adds entries ahead of these but does not remove them. +// (They are appended to the parsed GOVCS setting.) +// +// The rationale behind allowing only Git and Mercurial is that +// these two systems have had the most attention to issues +// of being run as clients of untrusted servers. In contrast, +// Bazaar, Fossil, and Subversion have primarily been used +// in trusted, authenticated environments and are not as well +// scrutinized as attack surfaces. +// +// See golang.org/issue/41730 for details. +var defaultGOVCS = govcsConfig{ + {"private", []string{"all"}}, + {"public", []string{"git", "hg"}}, +} + +func checkGOVCS(vcs *Cmd, root string) error { + if vcs == vcsMod { + // Direct module (proxy protocol) fetches don't + // involve an external version control system + // and are always allowed. + return nil + } + + govcsOnce.Do(func() { + govcs, govcsErr = parseGOVCS(os.Getenv("GOVCS")) + govcs = append(govcs, defaultGOVCS...) + }) + if govcsErr != nil { + return govcsErr + } + + private := module.MatchPrefixPatterns(cfg.GOPRIVATE, root) + if !govcs.allow(root, private, vcs.Cmd) { + what := "public" + if private { + what = "private" + } + return fmt.Errorf("GOVCS disallows using %s for %s %s", vcs.Cmd, what, root) + } + + return nil +} + +// CheckNested checks for an incorrectly-nested VCS-inside-VCS // situation for dir, checking parents up until srcRoot. -func checkNestedVCS(vcs *vcsCmd, dir, srcRoot string) error { +func CheckNested(vcs *Cmd, dir, srcRoot string) error { if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { return fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) } @@ -603,17 +745,17 @@ func checkNestedVCS(vcs *vcsCmd, dir, srcRoot string) error { otherDir := dir for len(otherDir) > len(srcRoot) { for _, otherVCS := range vcsList { - if _, err := os.Stat(filepath.Join(otherDir, "."+otherVCS.cmd)); err == nil { + if _, err := os.Stat(filepath.Join(otherDir, "."+otherVCS.Cmd)); err == nil { // Allow expected vcs in original dir. if otherDir == dir && otherVCS == vcs { continue } // Allow .git inside .git, which can arise due to submodules. - if otherVCS == vcs && vcs.cmd == "git" { + if otherVCS == vcs && vcs.Cmd == "git" { continue } // Otherwise, we have one VCS inside a different VCS. - return fmt.Errorf("directory %q uses %s, but parent %q uses %s", dir, vcs.cmd, otherDir, otherVCS.cmd) + return fmt.Errorf("directory %q uses %s, but parent %q uses %s", dir, vcs.Cmd, otherDir, otherVCS.Cmd) } } // Move to parent. @@ -633,9 +775,7 @@ type RepoRoot struct { Repo string // repository URL, including scheme Root string // import path corresponding to root of repo IsCustom bool // defined by served <meta> tags (as opposed to hard-coded pattern) - VCS string // vcs type ("mod", "git", ...) - - vcs *vcsCmd // internal: vcs command access + VCS *Cmd } func httpPrefix(s string) string { @@ -662,7 +802,7 @@ func RepoRootForImportPath(importPath string, mod ModuleMode, security web.Secur if err == errUnknownSite { rr, err = repoRootForImportDynamic(importPath, mod, security) if err != nil { - err = load.ImportErrorf(importPath, "unrecognized import path %q: %v", importPath, err) + err = importErrorf(importPath, "unrecognized import path %q: %v", importPath, err) } } if err != nil { @@ -677,7 +817,7 @@ func RepoRootForImportPath(importPath string, mod ModuleMode, security web.Secur if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.Root, "...") { // Do not allow wildcards in the repo root. rr = nil - err = load.ImportErrorf(importPath, "cannot expand ... in %q", importPath) + err = importErrorf(importPath, "cannot expand ... in %q", importPath) } return rr, err } @@ -687,6 +827,20 @@ var errUnknownSite = errors.New("dynamic lookup required to find mapping") // repoRootFromVCSPaths attempts to map importPath to a repoRoot // using the mappings defined in vcsPaths. func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths []*vcsPath) (*RepoRoot, error) { + if str.HasPathPrefix(importPath, "example.net") { + // TODO(rsc): This should not be necessary, but it's required to keep + // tests like ../../testdata/script/mod_get_extra.txt from using the network. + // That script has everything it needs in the replacement set, but it is still + // doing network calls. + return nil, fmt.Errorf("no modules on example.net") + } + if importPath == "rsc.io" { + // This special case allows tests like ../../testdata/script/govcs.txt + // to avoid making any network calls. The module lookup for a path + // like rsc.io/nonexist.svn/foo needs to not make a network call for + // a lookup on rsc.io. + return nil, fmt.Errorf("rsc.io is not a module") + } // A common error is to use https://packagepath because that's what // hg and git require. Diagnose this helpfully. if prefix := httpPrefix(importPath); prefix != "" { @@ -695,20 +849,20 @@ func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths return nil, fmt.Errorf("%q not allowed in import path", prefix+"//") } for _, srv := range vcsPaths { - if !strings.HasPrefix(importPath, srv.prefix) { + if !str.HasPathPrefix(importPath, srv.pathPrefix) { continue } m := srv.regexp.FindStringSubmatch(importPath) if m == nil { - if srv.prefix != "" { - return nil, load.ImportErrorf(importPath, "invalid %s import path %q", srv.prefix, importPath) + if srv.pathPrefix != "" { + return nil, importErrorf(importPath, "invalid %s import path %q", srv.pathPrefix, importPath) } continue } // Build map of named subexpression matches for expand. match := map[string]string{ - "prefix": srv.prefix, + "prefix": srv.pathPrefix + "/", "import": importPath, } for i, name := range srv.regexp.SubexpNames() { @@ -731,19 +885,22 @@ func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths if vcs == nil { return nil, fmt.Errorf("unknown version control system %q", match["vcs"]) } + if err := checkGOVCS(vcs, match["root"]); err != nil { + return nil, err + } var repoURL string if !srv.schemelessRepo { repoURL = match["repo"] } else { - scheme := vcs.scheme[0] // default to first scheme + scheme := vcs.Scheme[0] // default to first scheme repo := match["repo"] - if vcs.pingCmd != "" { + if vcs.PingCmd != "" { // If we know how to test schemes, scan to find one. - for _, s := range vcs.scheme { + for _, s := range vcs.Scheme { if security == web.SecureOnly && !vcs.isSecureScheme(s) { continue } - if vcs.ping(s, repo) == nil { + if vcs.Ping(s, repo) == nil { scheme = s break } @@ -754,8 +911,7 @@ func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths rr := &RepoRoot{ Repo: repoURL, Root: match["root"], - VCS: vcs.cmd, - vcs: vcs, + VCS: vcs, } return rr, nil } @@ -846,17 +1002,25 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se if err := validateRepoRoot(mmi.RepoRoot); err != nil { return nil, fmt.Errorf("%s: invalid repo root %q: %v", resp.URL, mmi.RepoRoot, err) } - vcs := vcsByCmd(mmi.VCS) - if vcs == nil && mmi.VCS != "mod" { - return nil, fmt.Errorf("%s: unknown vcs %q", resp.URL, mmi.VCS) + var vcs *Cmd + if mmi.VCS == "mod" { + vcs = vcsMod + } else { + vcs = vcsByCmd(mmi.VCS) + if vcs == nil { + return nil, fmt.Errorf("%s: unknown vcs %q", resp.URL, mmi.VCS) + } + } + + if err := checkGOVCS(vcs, mmi.Prefix); err != nil { + return nil, err } rr := &RepoRoot{ Repo: mmi.RepoRoot, Root: mmi.Prefix, IsCustom: true, - VCS: mmi.VCS, - vcs: vcs, + VCS: vcs, } return rr, nil } @@ -949,18 +1113,6 @@ type metaImport struct { Prefix, VCS, RepoRoot string } -// pathPrefix reports whether sub is a prefix of s, -// only considering entire path components. -func pathPrefix(s, sub string) bool { - // strings.HasPrefix is necessary but not sufficient. - if !strings.HasPrefix(s, sub) { - return false - } - // The remainder after the prefix must either be empty or start with a slash. - rem := s[len(sub):] - return rem == "" || rem[0] == '/' -} - // A ImportMismatchError is returned where metaImport/s are present // but none match our import path. type ImportMismatchError struct { @@ -984,7 +1136,7 @@ func matchGoImport(imports []metaImport, importPath string) (metaImport, error) errImportMismatch := ImportMismatchError{importPath: importPath} for i, im := range imports { - if !pathPrefix(importPath, im.Prefix) { + if !str.HasPathPrefix(importPath, im.Prefix) { errImportMismatch.mismatches = append(errImportMismatch.mismatches, im.Prefix) continue } @@ -1026,52 +1178,52 @@ func expand(match map[string]string, s string) string { var vcsPaths = []*vcsPath{ // Github { - prefix: "github.com/", - regexp: lazyregexp.New(`^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[\p{L}0-9_.\-]+)*$`), - vcs: "git", - repo: "https://{root}", - check: noVCSSuffix, + pathPrefix: "github.com", + regexp: lazyregexp.New(`^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`), + vcs: "git", + repo: "https://{root}", + check: noVCSSuffix, }, // Bitbucket { - prefix: "bitbucket.org/", - regexp: lazyregexp.New(`^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`), - repo: "https://{root}", - check: bitbucketVCS, + pathPrefix: "bitbucket.org", + regexp: lazyregexp.New(`^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`), + repo: "https://{root}", + check: bitbucketVCS, }, // IBM DevOps Services (JazzHub) { - prefix: "hub.jazz.net/git/", - regexp: lazyregexp.New(`^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`), - vcs: "git", - repo: "https://{root}", - check: noVCSSuffix, + pathPrefix: "hub.jazz.net/git", + regexp: lazyregexp.New(`^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`), + vcs: "git", + repo: "https://{root}", + check: noVCSSuffix, }, // Git at Apache { - prefix: "git.apache.org/", - regexp: lazyregexp.New(`^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/[A-Za-z0-9_.\-]+)*$`), - vcs: "git", - repo: "https://{root}", + pathPrefix: "git.apache.org", + regexp: lazyregexp.New(`^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/[A-Za-z0-9_.\-]+)*$`), + vcs: "git", + repo: "https://{root}", }, // Git at OpenStack { - prefix: "git.openstack.org/", - regexp: lazyregexp.New(`^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/[A-Za-z0-9_.\-]+)*$`), - vcs: "git", - repo: "https://{root}", + pathPrefix: "git.openstack.org", + regexp: lazyregexp.New(`^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/[A-Za-z0-9_.\-]+)*$`), + vcs: "git", + repo: "https://{root}", }, // chiselapp.com for fossil { - prefix: "chiselapp.com/", - regexp: lazyregexp.New(`^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$`), - vcs: "fossil", - repo: "https://{root}", + pathPrefix: "chiselapp.com", + regexp: lazyregexp.New(`^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$`), + vcs: "fossil", + repo: "https://{root}", }, // General syntax for any server. @@ -1089,11 +1241,11 @@ var vcsPaths = []*vcsPath{ var vcsPathsAfterDynamic = []*vcsPath{ // Launchpad. See golang.org/issue/11436. { - prefix: "launchpad.net/", - regexp: lazyregexp.New(`^(?P<root>launchpad\.net/((?P<project>[A-Za-z0-9_.\-]+)(?P<series>/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`), - vcs: "bzr", - repo: "https://{root}", - check: launchpadVCS, + pathPrefix: "launchpad.net", + regexp: lazyregexp.New(`^(?P<root>launchpad\.net/((?P<project>[A-Za-z0-9_.\-]+)(?P<series>/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`), + vcs: "bzr", + repo: "https://{root}", + check: launchpadVCS, }, } @@ -1103,7 +1255,7 @@ var vcsPathsAfterDynamic = []*vcsPath{ func noVCSSuffix(match map[string]string) error { repo := match["repo"] for _, vcs := range vcsList { - if strings.HasSuffix(repo, "."+vcs.cmd) { + if strings.HasSuffix(repo, "."+vcs.Cmd) { return fmt.Errorf("invalid version control suffix in %s path", match["prefix"]) } } @@ -1133,7 +1285,7 @@ func bitbucketVCS(match map[string]string) error { // VCS it uses. See issue 5375. root := match["root"] for _, vcs := range []string{"git", "hg"} { - if vcsByCmd(vcs).ping("https", root) == nil { + if vcsByCmd(vcs).Ping("https", root) == nil { resp.SCM = vcs break } @@ -1180,3 +1332,32 @@ func launchpadVCS(match map[string]string) error { } return nil } + +// importError is a copy of load.importError, made to avoid a dependency cycle +// on cmd/go/internal/load. It just needs to satisfy load.ImportPathError. +type importError struct { + importPath string + err error +} + +func importErrorf(path, format string, args ...interface{}) error { + err := &importError{importPath: path, err: fmt.Errorf(format, args...)} + if errStr := err.Error(); !strings.Contains(errStr, path) { + panic(fmt.Sprintf("path %q not in error %q", path, errStr)) + } + return err +} + +func (e *importError) Error() string { + return e.err.Error() +} + +func (e *importError) Unwrap() error { + // Don't return e.err directly, since we're only wrapping an error if %w + // was passed to ImportErrorf. + return errors.Unwrap(e.err) +} + +func (e *importError) ImportPath() string { + return e.importPath +} diff --git a/libgo/go/cmd/go/internal/get/vcs_test.go b/libgo/go/cmd/go/internal/vcs/vcs_test.go index 91800ba..c5c7a32 100644 --- a/libgo/go/cmd/go/internal/get/vcs_test.go +++ b/libgo/go/cmd/go/internal/vcs/vcs_test.go @@ -2,20 +2,28 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( "errors" "internal/testenv" - "io/ioutil" "os" "path" "path/filepath" + "strings" "testing" "cmd/go/internal/web" ) +func init() { + // GOVCS defaults to public:git|hg,private:all, + // which breaks many tests here - they can't use non-git, non-hg VCS at all! + // Change to fully permissive. + // The tests of the GOVCS setting itself are in ../../testdata/script/govcs.txt. + os.Setenv("GOVCS", "*:all") +} + // Test that RepoRootForImportPath determines the correct RepoRoot for a given importPath. // TODO(cmang): Add tests for SVN and BZR. func TestRepoRootForImportPath(t *testing.T) { @@ -28,30 +36,27 @@ func TestRepoRootForImportPath(t *testing.T) { { "github.com/golang/groupcache", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://github.com/golang/groupcache", }, }, - // Unicode letters in directories (issue 18660). + // Unicode letters in directories are not valid. { "github.com/user/unicode/испытание", - &RepoRoot{ - vcs: vcsGit, - Repo: "https://github.com/user/unicode", - }, + nil, }, // IBM DevOps Services tests { "hub.jazz.net/git/user1/pkgname", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://hub.jazz.net/git/user1/pkgname", }, }, { "hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://hub.jazz.net/git/user1/pkgname", }, }, @@ -92,7 +97,7 @@ func TestRepoRootForImportPath(t *testing.T) { { "hub.jazz.net/git/user/pkg.name", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://hub.jazz.net/git/user/pkg.name", }, }, @@ -105,7 +110,7 @@ func TestRepoRootForImportPath(t *testing.T) { { "git.openstack.org/openstack/swift", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.openstack.org/openstack/swift", }, }, @@ -115,14 +120,14 @@ func TestRepoRootForImportPath(t *testing.T) { { "git.openstack.org/openstack/swift.git", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.openstack.org/openstack/swift.git", }, }, { "git.openstack.org/openstack/swift/go/hummingbird", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.openstack.org/openstack/swift", }, }, @@ -151,21 +156,21 @@ func TestRepoRootForImportPath(t *testing.T) { { "git.apache.org/package-name.git", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.apache.org/package-name.git", }, }, { "git.apache.org/package-name_2.x.git/path/to/lib", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.apache.org/package-name_2.x.git", }, }, { "chiselapp.com/user/kyle/repository/fossilgg", &RepoRoot{ - vcs: vcsFossil, + VCS: vcsFossil, Repo: "https://chiselapp.com/user/kyle/repository/fossilgg", }, }, @@ -194,22 +199,22 @@ func TestRepoRootForImportPath(t *testing.T) { t.Errorf("RepoRootForImportPath(%q): %v", test.path, err) continue } - if got.vcs.name != want.vcs.name || got.Repo != want.Repo { - t.Errorf("RepoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.vcs, got.Repo, want.vcs, want.Repo) + if got.VCS.Name != want.VCS.Name || got.Repo != want.Repo { + t.Errorf("RepoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.VCS, got.Repo, want.VCS, want.Repo) } } } // Test that vcsFromDir correctly inspects a given directory and returns the right VCS and root. func TestFromDir(t *testing.T) { - tempDir, err := ioutil.TempDir("", "vcstest") + tempDir, err := os.MkdirTemp("", "vcstest") if err != nil { t.Fatal(err) } defer os.RemoveAll(tempDir) for j, vcs := range vcsList { - dir := filepath.Join(tempDir, "example.com", vcs.name, "."+vcs.cmd) + dir := filepath.Join(tempDir, "example.com", vcs.Name, "."+vcs.Cmd) if j&1 == 0 { err := os.MkdirAll(dir, 0755) if err != nil { @@ -228,24 +233,24 @@ func TestFromDir(t *testing.T) { } want := RepoRoot{ - vcs: vcs, - Root: path.Join("example.com", vcs.name), + VCS: vcs, + Root: path.Join("example.com", vcs.Name), } var got RepoRoot - got.vcs, got.Root, err = vcsFromDir(dir, tempDir) + got.VCS, got.Root, err = FromDir(dir, tempDir) if err != nil { t.Errorf("FromDir(%q, %q): %v", dir, tempDir, err) continue } - if got.vcs.name != want.vcs.name || got.Root != want.Root { - t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.vcs, got.Root, want.vcs, want.Root) + if got.VCS.Name != want.VCS.Name || got.Root != want.Root { + t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.VCS, got.Root, want.VCS, want.Root) } } } func TestIsSecure(t *testing.T) { tests := []struct { - vcs *vcsCmd + vcs *Cmd url string secure bool }{ @@ -270,7 +275,7 @@ func TestIsSecure(t *testing.T) { } for _, test := range tests { - secure := test.vcs.isSecure(test.url) + secure := test.vcs.IsSecure(test.url) if secure != test.secure { t.Errorf("%s isSecure(%q) = %t; want %t", test.vcs, test.url, secure, test.secure) } @@ -279,7 +284,7 @@ func TestIsSecure(t *testing.T) { func TestIsSecureGitAllowProtocol(t *testing.T) { tests := []struct { - vcs *vcsCmd + vcs *Cmd url string secure bool }{ @@ -310,7 +315,7 @@ func TestIsSecureGitAllowProtocol(t *testing.T) { defer os.Unsetenv("GIT_ALLOW_PROTOCOL") os.Setenv("GIT_ALLOW_PROTOCOL", "https:foo") for _, test := range tests { - secure := test.vcs.isSecure(test.url) + secure := test.vcs.IsSecure(test.url) if secure != test.secure { t.Errorf("%s isSecure(%q) = %t; want %t", test.vcs, test.url, secure, test.secure) } @@ -476,3 +481,98 @@ func TestValidateRepoRoot(t *testing.T) { } } } + +var govcsTests = []struct { + govcs string + path string + vcs string + ok bool +}{ + {"private:all", "is-public.com/foo", "zzz", false}, + {"private:all", "is-private.com/foo", "zzz", true}, + {"public:all", "is-public.com/foo", "zzz", true}, + {"public:all", "is-private.com/foo", "zzz", false}, + {"public:all,private:none", "is-public.com/foo", "zzz", true}, + {"public:all,private:none", "is-private.com/foo", "zzz", false}, + {"*:all", "is-public.com/foo", "zzz", true}, + {"golang.org:git", "golang.org/x/text", "zzz", false}, + {"golang.org:git", "golang.org/x/text", "git", true}, + {"golang.org:zzz", "golang.org/x/text", "zzz", true}, + {"golang.org:zzz", "golang.org/x/text", "git", false}, + {"golang.org:zzz", "golang.org/x/text", "zzz", true}, + {"golang.org:zzz", "golang.org/x/text", "git", false}, + {"golang.org:git|hg", "golang.org/x/text", "hg", true}, + {"golang.org:git|hg", "golang.org/x/text", "git", true}, + {"golang.org:git|hg", "golang.org/x/text", "zzz", false}, + {"golang.org:all", "golang.org/x/text", "hg", true}, + {"golang.org:all", "golang.org/x/text", "git", true}, + {"golang.org:all", "golang.org/x/text", "zzz", true}, + {"other.xyz/p:none,golang.org/x:git", "other.xyz/p/x", "git", false}, + {"other.xyz/p:none,golang.org/x:git", "unexpected.com", "git", false}, + {"other.xyz/p:none,golang.org/x:git", "golang.org/x/text", "zzz", false}, + {"other.xyz/p:none,golang.org/x:git", "golang.org/x/text", "git", true}, + {"other.xyz/p:none,golang.org/x:zzz", "golang.org/x/text", "zzz", true}, + {"other.xyz/p:none,golang.org/x:zzz", "golang.org/x/text", "git", false}, + {"other.xyz/p:none,golang.org/x:git|hg", "golang.org/x/text", "hg", true}, + {"other.xyz/p:none,golang.org/x:git|hg", "golang.org/x/text", "git", true}, + {"other.xyz/p:none,golang.org/x:git|hg", "golang.org/x/text", "zzz", false}, + {"other.xyz/p:none,golang.org/x:all", "golang.org/x/text", "hg", true}, + {"other.xyz/p:none,golang.org/x:all", "golang.org/x/text", "git", true}, + {"other.xyz/p:none,golang.org/x:all", "golang.org/x/text", "zzz", true}, + {"other.xyz/p:none,golang.org/x:git", "golang.org/y/text", "zzz", false}, + {"other.xyz/p:none,golang.org/x:git", "golang.org/y/text", "git", false}, + {"other.xyz/p:none,golang.org/x:zzz", "golang.org/y/text", "zzz", false}, + {"other.xyz/p:none,golang.org/x:zzz", "golang.org/y/text", "git", false}, + {"other.xyz/p:none,golang.org/x:git|hg", "golang.org/y/text", "hg", false}, + {"other.xyz/p:none,golang.org/x:git|hg", "golang.org/y/text", "git", false}, + {"other.xyz/p:none,golang.org/x:git|hg", "golang.org/y/text", "zzz", false}, + {"other.xyz/p:none,golang.org/x:all", "golang.org/y/text", "hg", false}, + {"other.xyz/p:none,golang.org/x:all", "golang.org/y/text", "git", false}, + {"other.xyz/p:none,golang.org/x:all", "golang.org/y/text", "zzz", false}, +} + +func TestGOVCS(t *testing.T) { + for _, tt := range govcsTests { + cfg, err := parseGOVCS(tt.govcs) + if err != nil { + t.Errorf("parseGOVCS(%q): %v", tt.govcs, err) + continue + } + private := strings.HasPrefix(tt.path, "is-private") + ok := cfg.allow(tt.path, private, tt.vcs) + if ok != tt.ok { + t.Errorf("parseGOVCS(%q).allow(%q, %v, %q) = %v, want %v", + tt.govcs, tt.path, private, tt.vcs, ok, tt.ok) + } + } +} + +var govcsErrors = []struct { + s string + err string +}{ + {`,`, `empty entry in GOVCS`}, + {`,x`, `empty entry in GOVCS`}, + {`x,`, `malformed entry in GOVCS (missing colon): "x"`}, + {`x:y,`, `empty entry in GOVCS`}, + {`x`, `malformed entry in GOVCS (missing colon): "x"`}, + {`x:`, `empty VCS list in GOVCS: "x:"`}, + {`x:|`, `empty VCS name in GOVCS: "x:|"`}, + {`x:y|`, `empty VCS name in GOVCS: "x:y|"`}, + {`x:|y`, `empty VCS name in GOVCS: "x:|y"`}, + {`x:y,z:`, `empty VCS list in GOVCS: "z:"`}, + {`x:y,z:|`, `empty VCS name in GOVCS: "z:|"`}, + {`x:y,z:|w`, `empty VCS name in GOVCS: "z:|w"`}, + {`x:y,z:w|`, `empty VCS name in GOVCS: "z:w|"`}, + {`x:y,z:w||v`, `empty VCS name in GOVCS: "z:w||v"`}, + {`x:y,x:z`, `unreachable pattern in GOVCS: "x:z" after "x:y"`}, +} + +func TestGOVCSErrors(t *testing.T) { + for _, tt := range govcsErrors { + _, err := parseGOVCS(tt.s) + if err == nil || !strings.Contains(err.Error(), tt.err) { + t.Errorf("parseGOVCS(%s): err=%v, want %v", tt.s, err, tt.err) + } + } +} diff --git a/libgo/go/cmd/go/internal/version/version.go b/libgo/go/cmd/go/internal/version/version.go index a5b1365..58cbd32 100644 --- a/libgo/go/cmd/go/internal/version/version.go +++ b/libgo/go/cmd/go/internal/version/version.go @@ -7,8 +7,10 @@ package version import ( "bytes" + "context" "encoding/binary" "fmt" + "io/fs" "os" "path/filepath" "runtime" @@ -51,7 +53,7 @@ var ( versionV = CmdVersion.Flag.Bool("v", false, "") ) -func runVersion(cmd *base.Command, args []string) { +func runVersion(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { // If any of this command's flags were passed explicitly, error // out, because they only make sense with arguments. @@ -86,8 +88,15 @@ func runVersion(cmd *base.Command, args []string) { // scanDir scans a directory for executables to run scanFile on. func scanDir(dir string) { - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if d.Type().IsRegular() || d.Type()&fs.ModeSymlink != 0 { + info, err := d.Info() + if err != nil { + if *versionV { + fmt.Fprintf(os.Stderr, "%s: %v\n", path, err) + } + return nil + } scanFile(path, info, *versionV) } return nil @@ -95,7 +104,7 @@ func scanDir(dir string) { } // isExe reports whether the file should be considered executable. -func isExe(file string, info os.FileInfo) bool { +func isExe(file string, info fs.FileInfo) bool { if runtime.GOOS == "windows" { return strings.HasSuffix(strings.ToLower(file), ".exe") } @@ -106,8 +115,8 @@ func isExe(file string, info os.FileInfo) bool { // If mustPrint is true, scanFile will report any error reading file. // Otherwise (mustPrint is false, because scanFile is being called // by scanDir) scanFile prints nothing for non-Go executables. -func scanFile(file string, info os.FileInfo, mustPrint bool) { - if info.Mode()&os.ModeSymlink != 0 { +func scanFile(file string, info fs.FileInfo, mustPrint bool) { + if info.Mode()&fs.ModeSymlink != 0 { // Accept file symlinks only. i, err := os.Stat(file) if err != nil || !i.Mode().IsRegular() { @@ -118,6 +127,7 @@ func scanFile(file string, info os.FileInfo, mustPrint bool) { } info = i } + if !isExe(file, info) { if mustPrint { fmt.Fprintf(os.Stderr, "%s: not executable file\n", file) @@ -144,7 +154,7 @@ func scanFile(file string, info os.FileInfo, mustPrint bool) { fmt.Printf("%s: %s\n", file, vers) if *versionM && mod != "" { - fmt.Printf("\t%s\n", strings.Replace(mod[:len(mod)-1], "\n", "\n\t", -1)) + fmt.Printf("\t%s\n", strings.ReplaceAll(mod[:len(mod)-1], "\n", "\n\t")) } } diff --git a/libgo/go/cmd/go/internal/vet/vet.go b/libgo/go/cmd/go/internal/vet/vet.go index 4ec58de..4257c90c 100644 --- a/libgo/go/cmd/go/internal/vet/vet.go +++ b/libgo/go/cmd/go/internal/vet/vet.go @@ -6,11 +6,15 @@ package vet import ( + "context" + "fmt" + "path/filepath" + "cmd/go/internal/base" + "cmd/go/internal/cfg" "cmd/go/internal/load" - "cmd/go/internal/modload" + "cmd/go/internal/trace" "cmd/go/internal/work" - "path/filepath" ) // Break init loop. @@ -48,11 +52,28 @@ See also: go fmt, go fix. `, } -func runVet(cmd *base.Command, args []string) { - modload.LoadTests = true +func runVet(ctx context.Context, cmd *base.Command, args []string) { + load.ModResolveTests = true vetFlags, pkgArgs := vetFlags(args) + if cfg.DebugTrace != "" { + var close func() error + var err error + ctx, close, err = trace.Start(ctx, cfg.DebugTrace) + if err != nil { + base.Fatalf("failed to start trace: %v", err) + } + defer func() { + if err := close(); err != nil { + base.Fatalf("failed to stop trace: %v", err) + } + }() + } + + ctx, span := trace.StartSpan(ctx, fmt.Sprint("Running ", cmd.Name(), " command")) + defer span.Done() + work.BuildInit() work.VetFlags = vetFlags if len(vetFlags) > 0 { @@ -66,7 +87,8 @@ func runVet(cmd *base.Command, args []string) { } } - pkgs := load.PackagesForBuild(pkgArgs) + pkgs := load.PackagesAndErrors(ctx, pkgArgs) + load.CheckPackageErrors(pkgs) if len(pkgs) == 0 { base.Fatalf("no packages to vet") } @@ -76,7 +98,7 @@ func runVet(cmd *base.Command, args []string) { root := &work.Action{Mode: "go vet"} for _, p := range pkgs { - _, ptest, pxtest, err := load.TestPackagesFor(p, nil) + _, ptest, pxtest, err := load.TestPackagesFor(ctx, p, nil) if err != nil { base.Errorf("%v", err) continue @@ -92,5 +114,5 @@ func runVet(cmd *base.Command, args []string) { root.Deps = append(root.Deps, b.VetAction(work.ModeBuild, work.ModeBuild, pxtest)) } } - b.Do(root) + b.Do(ctx, root) } diff --git a/libgo/go/cmd/go/internal/web/api.go b/libgo/go/cmd/go/internal/web/api.go index 5708188..9053b16 100644 --- a/libgo/go/cmd/go/internal/web/api.go +++ b/libgo/go/cmd/go/internal/web/api.go @@ -13,9 +13,8 @@ import ( "bytes" "fmt" "io" - "io/ioutil" + "io/fs" "net/url" - "os" "strings" "unicode" "unicode/utf8" @@ -56,7 +55,7 @@ func (e *HTTPError) Error() string { } if err := e.Err; err != nil { - if pErr, ok := e.Err.(*os.PathError); ok && strings.HasSuffix(e.URL, pErr.Path) { + if pErr, ok := e.Err.(*fs.PathError); ok && strings.HasSuffix(e.URL, pErr.Path) { // Remove the redundant copy of the path. err = pErr.Err } @@ -67,7 +66,7 @@ func (e *HTTPError) Error() string { } func (e *HTTPError) Is(target error) bool { - return target == os.ErrNotExist && (e.StatusCode == 404 || e.StatusCode == 410) + return target == fs.ErrNotExist && (e.StatusCode == 404 || e.StatusCode == 410) } func (e *HTTPError) Unwrap() error { @@ -87,7 +86,7 @@ func GetBytes(u *url.URL) ([]byte, error) { if err := resp.Err(); err != nil { return nil, err } - b, err := ioutil.ReadAll(resp.Body) + b, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("reading %s: %v", u.Redacted(), err) } @@ -130,7 +129,7 @@ func (r *Response) formatErrorDetail() string { } // Ensure that r.errorDetail has been populated. - _, _ = io.Copy(ioutil.Discard, r.Body) + _, _ = io.Copy(io.Discard, r.Body) s := r.errorDetail.buf.String() if !utf8.ValidString(s) { diff --git a/libgo/go/cmd/go/internal/web/file_test.go b/libgo/go/cmd/go/internal/web/file_test.go index 6339469..3734df5 100644 --- a/libgo/go/cmd/go/internal/web/file_test.go +++ b/libgo/go/cmd/go/internal/web/file_test.go @@ -6,7 +6,7 @@ package web import ( "errors" - "io/ioutil" + "io/fs" "os" "path/filepath" "testing" @@ -15,7 +15,7 @@ import ( func TestGetFileURL(t *testing.T) { const content = "Hello, file!\n" - f, err := ioutil.TempFile("", "web-TestGetFileURL") + f, err := os.CreateTemp("", "web-TestGetFileURL") if err != nil { t.Fatal(err) } @@ -54,7 +54,7 @@ func TestGetNonexistentFile(t *testing.T) { } b, err := GetBytes(u) - if !errors.Is(err, os.ErrNotExist) { - t.Fatalf("GetBytes(%v) = %q, %v; want _, os.ErrNotExist", u, b, err) + if !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("GetBytes(%v) = %q, %v; want _, fs.ErrNotExist", u, b, err) } } diff --git a/libgo/go/cmd/go/internal/web/http.go b/libgo/go/cmd/go/internal/web/http.go index e050980..72fa2b2 100644 --- a/libgo/go/cmd/go/internal/web/http.go +++ b/libgo/go/cmd/go/internal/web/http.go @@ -80,6 +80,13 @@ func get(security SecurityMode, url *urlpkg.URL) (*Response, error) { return res, nil } + if url.Host == "localhost.localdev" { + return nil, fmt.Errorf("no such host localhost.localdev") + } + if os.Getenv("TESTGONETWORK") == "panic" && !strings.HasPrefix(url.Host, "127.0.0.1") && !strings.HasPrefix(url.Host, "0.0.0.0") { + panic("use of network: " + url.String()) + } + fetch := func(url *urlpkg.URL) (*urlpkg.URL, *http.Response, error) { // Note: The -v build flag does not mean "print logging information", // despite its historical misuse for this in GOPATH-based go get. diff --git a/libgo/go/cmd/go/internal/work/action.go b/libgo/go/cmd/go/internal/work/action.go index b872928..5320121 100644 --- a/libgo/go/cmd/go/internal/work/action.go +++ b/libgo/go/cmd/go/internal/work/action.go @@ -10,11 +10,11 @@ import ( "bufio" "bytes" "container/heap" + "context" "debug/elf" "encoding/json" "fmt" "internal/xcoff" - "io/ioutil" "os" "path/filepath" "runtime" @@ -26,6 +26,7 @@ import ( "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/load" + "cmd/go/internal/trace" "cmd/internal/buildid" ) @@ -64,13 +65,13 @@ type Builder struct { // An Action represents a single action in the action graph. type Action struct { - Mode string // description of action operation - Package *load.Package // the package this action works on - Deps []*Action // actions that must happen before this one - Func func(*Builder, *Action) error // the action itself (nil = no-op) - IgnoreFail bool // whether to run f even if dependencies fail - TestOutput *bytes.Buffer // test output buffer - Args []string // additional args for runProgram + Mode string // description of action operation + Package *load.Package // the package this action works on + Deps []*Action // actions that must happen before this one + Func func(*Builder, context.Context, *Action) error // the action itself (nil = no-op) + IgnoreFail bool // whether to run f even if dependencies fail + TestOutput *bytes.Buffer // test output buffer + Args []string // additional args for runProgram triggers []*Action // inverse of deps @@ -92,10 +93,12 @@ type Action struct { output []byte // output redirect buffer (nil means use b.Print) // Execution state. - pending int // number of deps yet to complete - priority int // relative execution priority - Failed bool // whether the action failed - json *actionJSON // action graph information + pending int // number of deps yet to complete + priority int // relative execution priority + Failed bool // whether the action failed + json *actionJSON // action graph information + nonGoOverlay map[string]string // map from non-.go source files to copied files in objdir. Nil if no overlay is used. + traceSpan *trace.Span } // BuildActionID returns the action ID section of a's build ID. @@ -250,7 +253,7 @@ func (b *Builder) Init() { if cfg.BuildN { b.WorkDir = "$WORK" } else { - tmp, err := ioutil.TempDir(cfg.Getenv("GOTMPDIR"), "go-build") + tmp, err := os.MkdirTemp(cfg.Getenv("GOTMPDIR"), "go-build") if err != nil { base.Fatalf("go: creating work dir: %v", err) } diff --git a/libgo/go/cmd/go/internal/work/build.go b/libgo/go/cmd/go/internal/work/build.go index 7146c9c..7f2617c 100644 --- a/libgo/go/cmd/go/internal/work/build.go +++ b/libgo/go/cmd/go/internal/work/build.go @@ -5,23 +5,33 @@ package work import ( + "context" "errors" "fmt" "go/build" + "internal/goroot" "os" "os/exec" + "path" "path/filepath" "runtime" "strings" "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/load" + "cmd/go/internal/modfetch" + "cmd/go/internal/modload" "cmd/go/internal/search" + "cmd/go/internal/trace" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" ) var CmdBuild = &base.Command{ - UsageLine: "go build [-o output] [-i] [build flags] [packages]", + UsageLine: "go build [-o output] [build flags] [packages]", Short: "compile packages and dependencies", Long: ` Build compiles the packages named by the import paths, @@ -44,10 +54,12 @@ serving only as a check that the packages can be built. The -o flag forces build to write the resulting executable or object to the named output file or directory, instead of the default behavior described -in the last two paragraphs. If the named output is a directory that exists, -then any resulting executables will be written to that directory. +in the last two paragraphs. If the named output is an existing directory or +ends with a slash or backslash, then any resulting executables +will be written to that directory. The -i flag installs the packages that are dependencies of the target. +The -i flag is deprecated. Compiled packages are cached automatically. The build flags are shared by the build, clean, get, install, list, run, and test commands: @@ -112,6 +124,17 @@ and test commands: directory, but it is not accessed. When -modfile is specified, an alternate go.sum file is also used: its path is derived from the -modfile flag by trimming the ".mod" extension and appending ".sum". + -overlay file + read a JSON config file that provides an overlay for build operations. + The file is a JSON struct with a single field, named 'Replace', that + maps each disk file path (a string) to its backing file path, so that + a build will run as if the disk file path exists with the contents + given by the backing file paths, or as if the disk file path does not + exist if its backing file path is empty. Support for the -overlay flag + has some limitations:importantly, cgo files included from outside the + include path must be in the same directory as the Go package they are + included from, and overlays will not appear when binaries and tests are + run through go run and go test respectively. -pkgdir dir install and load all packages from dir instead of the usual locations. For example, when building with a non-standard configuration, @@ -238,13 +261,12 @@ const ( // AddBuildFlags adds the flags common to the build, clean, get, // install, list, run, and test commands. func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { + base.AddBuildFlagsNX(&cmd.Flag) cmd.Flag.BoolVar(&cfg.BuildA, "a", false, "") - cmd.Flag.BoolVar(&cfg.BuildN, "n", false, "") cmd.Flag.IntVar(&cfg.BuildP, "p", cfg.BuildP, "") if mask&OmitVFlag == 0 { cmd.Flag.BoolVar(&cfg.BuildV, "v", false, "") } - cmd.Flag.BoolVar(&cfg.BuildX, "x", false, "") cmd.Flag.Var(&load.BuildAsmflags, "asmflags", "") cmd.Flag.Var(buildCompiler{}, "compiler", "") @@ -252,10 +274,15 @@ func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { cmd.Flag.Var(&load.BuildGcflags, "gcflags", "") cmd.Flag.Var(&load.BuildGccgoflags, "gccgoflags", "") if mask&OmitModFlag == 0 { - cmd.Flag.StringVar(&cfg.BuildMod, "mod", "", "") + base.AddModFlag(&cmd.Flag) } if mask&OmitModCommonFlags == 0 { - AddModCommonFlags(cmd) + base.AddModCommonFlags(&cmd.Flag) + } else { + // Add the overlay flag even when we don't add the rest of the mod common flags. + // This only affects 'go get' in GOPATH mode, but add the flag anyway for + // consistency. + cmd.Flag.StringVar(&fsys.OverlayFile, "overlay", "", "") } cmd.Flag.StringVar(&cfg.BuildContext.InstallSuffix, "installsuffix", "", "") cmd.Flag.Var(&load.BuildLdflags, "ldflags", "") @@ -270,13 +297,7 @@ func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { // Undocumented, unstable debugging flags. cmd.Flag.StringVar(&cfg.DebugActiongraph, "debug-actiongraph", "", "") -} - -// AddModCommonFlags adds the module-related flags common to build commands -// and 'go mod' subcommands. -func AddModCommonFlags(cmd *base.Command) { - cmd.Flag.BoolVar(&cfg.ModCacheRW, "modcacherw", false, "") - cmd.Flag.StringVar(&cfg.ModFile, "modfile", "", "") + cmd.Flag.StringVar(&cfg.DebugTrace, "debug-trace", "", "") } // tagsFlag is the implementation of the -tags flag. @@ -343,12 +364,13 @@ var pkgsFilter = func(pkgs []*load.Package) []*load.Package { return pkgs } var runtimeVersion = runtime.Version() -func runBuild(cmd *base.Command, args []string) { +func runBuild(ctx context.Context, cmd *base.Command, args []string) { BuildInit() var b Builder b.Init() - pkgs := load.PackagesForBuild(args) + pkgs := load.PackagesAndErrors(ctx, args) + load.CheckPackageErrors(pkgs) explicitO := len(cfg.BuildO) > 0 @@ -375,9 +397,10 @@ func runBuild(cmd *base.Command, args []string) { depMode := ModeBuild if cfg.BuildI { depMode = ModeInstall + fmt.Fprint(os.Stderr, "go build: -i flag is deprecated\n") } - pkgs = omitTestOnly(pkgsFilter(load.Packages(args))) + pkgs = omitTestOnly(pkgsFilter(pkgs)) // Special case -o /dev/null by not writing at all. if cfg.BuildO == os.DevNull { @@ -385,10 +408,13 @@ func runBuild(cmd *base.Command, args []string) { } if cfg.BuildO != "" { - // If the -o name exists and is a directory, then + // If the -o name exists and is a directory or + // ends with a slash or backslash, then // write all main packages to that directory. // Otherwise require only a single package be built. - if fi, err := os.Stat(cfg.BuildO); err == nil && fi.IsDir() { + if fi, err := os.Stat(cfg.BuildO); (err == nil && fi.IsDir()) || + strings.HasSuffix(cfg.BuildO, "/") || + strings.HasSuffix(cfg.BuildO, string(os.PathSeparator)) { if !explicitO { base.Fatalf("go build: build output %q already exists and is a directory", cfg.BuildO) } @@ -407,7 +433,7 @@ func runBuild(cmd *base.Command, args []string) { if len(a.Deps) == 0 { base.Fatalf("go build: no main packages to build") } - b.Do(a) + b.Do(ctx, a) return } if len(pkgs) > 1 { @@ -420,7 +446,7 @@ func runBuild(cmd *base.Command, args []string) { p.Stale = true // must build - not up to date p.StaleReason = "build -o flag in use" a := b.AutoAction(ModeInstall, depMode, p) - b.Do(a) + b.Do(ctx, a) return } @@ -431,11 +457,11 @@ func runBuild(cmd *base.Command, args []string) { if cfg.BuildBuildmode == "shared" { a = b.buildmodeShared(ModeBuild, depMode, args, pkgs, a) } - b.Do(a) + b.Do(ctx, a) } var CmdInstall = &base.Command{ - UsageLine: "go install [-i] [build flags] [packages]", + UsageLine: "go install [build flags] [packages]", Short: "compile and install packages and dependencies", Long: ` Install compiles and installs the packages named by the import paths. @@ -445,11 +471,39 @@ variable, which defaults to $GOPATH/bin or $HOME/go/bin if the GOPATH environment variable is not set. Executables in $GOROOT are installed in $GOROOT/bin or $GOTOOLDIR instead of $GOBIN. +If the arguments have version suffixes (like @latest or @v1.0.0), "go install" +builds packages in module-aware mode, ignoring the go.mod file in the current +directory or any parent directory, if there is one. This is useful for +installing executables without affecting the dependencies of the main module. +To eliminate ambiguity about which module versions are used in the build, the +arguments must satisfy the following constraints: + +- Arguments must be package paths or package patterns (with "..." wildcards). + They must not be standard packages (like fmt), meta-patterns (std, cmd, + all), or relative or absolute file paths. +- All arguments must have the same version suffix. Different queries are not + allowed, even if they refer to the same version. +- All arguments must refer to packages in the same module at the same version. +- No module is considered the "main" module. If the module containing + packages named on the command line has a go.mod file, it must not contain + directives (replace and exclude) that would cause it to be interpreted + differently than if it were the main module. The module must not require + a higher version of itself. +- Package path arguments must refer to main packages. Pattern arguments + will only match main packages. + +If the arguments don't have version suffixes, "go install" may run in +module-aware mode or GOPATH mode, depending on the GO111MODULE environment +variable and the presence of a go.mod file. See 'go help modules' for details. +If module-aware mode is enabled, "go install" runs in the context of the main +module. + When module-aware mode is disabled, other packages are installed in the directory $GOPATH/pkg/$GOOS_$GOARCH. When module-aware mode is enabled, other packages are built and cached but not installed. The -i flag installs the dependencies of the named packages as well. +The -i flag is deprecated. Compiled packages are cached automatically. For more about the build flags, see 'go help build'. For more about specifying packages, see 'go help packages'. @@ -514,9 +568,61 @@ func libname(args []string, pkgs []*load.Package) (string, error) { return "lib" + libname + ".so", nil } -func runInstall(cmd *base.Command, args []string) { +func runInstall(ctx context.Context, cmd *base.Command, args []string) { + // TODO(golang.org/issue/41696): print a deprecation message for the -i flag + // whenever it's set (or just remove it). For now, we don't print a message + // if all named packages are in GOROOT. cmd/dist (run by make.bash) uses + // 'go install -i' when bootstrapping, and we don't want to show deprecation + // messages in that case. + for _, arg := range args { + if strings.Contains(arg, "@") && !build.IsLocalImport(arg) && !filepath.IsAbs(arg) { + if cfg.BuildI { + fmt.Fprint(os.Stderr, "go install: -i flag is deprecated\n") + } + installOutsideModule(ctx, args) + return + } + } + BuildInit() - InstallPackages(args, load.PackagesForBuild(args)) + pkgs := load.PackagesAndErrors(ctx, args) + if cfg.ModulesEnabled && !modload.HasModRoot() { + haveErrors := false + allMissingErrors := true + for _, pkg := range pkgs { + if pkg.Error == nil { + continue + } + haveErrors = true + if missingErr := (*modload.ImportMissingError)(nil); !errors.As(pkg.Error, &missingErr) { + allMissingErrors = false + break + } + } + if haveErrors && allMissingErrors { + latestArgs := make([]string, len(args)) + for i := range args { + latestArgs[i] = args[i] + "@latest" + } + hint := strings.Join(latestArgs, " ") + base.Fatalf("go install: version is required when current directory is not in a module\n\tTry 'go install %s' to install the latest version", hint) + } + } + load.CheckPackageErrors(pkgs) + if cfg.BuildI { + allGoroot := true + for _, pkg := range pkgs { + if !pkg.Goroot { + allGoroot = false + break + } + } + if !allGoroot { + fmt.Fprint(os.Stderr, "go install: -i flag is deprecated\n") + } + } + + InstallPackages(ctx, args, pkgs) } // omitTestOnly returns pkgs with test-only packages removed. @@ -536,7 +642,10 @@ func omitTestOnly(pkgs []*load.Package) []*load.Package { return list } -func InstallPackages(patterns []string, pkgs []*load.Package) { +func InstallPackages(ctx context.Context, patterns []string, pkgs []*load.Package) { + ctx, span := trace.StartSpan(ctx, "InstallPackages "+strings.Join(patterns, " ")) + defer span.Done() + if cfg.GOBIN != "" && !filepath.IsAbs(cfg.GOBIN) { base.Fatalf("cannot install, GOBIN must be an absolute path") } @@ -605,7 +714,7 @@ func InstallPackages(patterns []string, pkgs []*load.Package) { a = b.buildmodeShared(ModeInstall, ModeInstall, patterns, pkgs, a) } - b.Do(a) + b.Do(ctx, a) base.ExitIfErrors() // Success. If this command is 'go install' with no arguments @@ -636,6 +745,157 @@ func InstallPackages(patterns []string, pkgs []*load.Package) { } } +// installOutsideModule implements 'go install pkg@version'. It builds and +// installs one or more main packages in module mode while ignoring any go.mod +// in the current directory or parent directories. +// +// See golang.org/issue/40276 for details and rationale. +func installOutsideModule(ctx context.Context, args []string) { + modload.ForceUseModules = true + modload.RootMode = modload.NoRoot + modload.AllowMissingModuleImports() + modload.Init() + + // Check that the arguments satisfy syntactic constraints. + var version string + for _, arg := range args { + if i := strings.Index(arg, "@"); i >= 0 { + version = arg[i+1:] + if version == "" { + base.Fatalf("go install %s: version must not be empty", arg) + } + break + } + } + patterns := make([]string, len(args)) + for i, arg := range args { + if !strings.HasSuffix(arg, "@"+version) { + base.Errorf("go install %s: all arguments must have the same version (@%s)", arg, version) + continue + } + p := arg[:len(arg)-len(version)-1] + switch { + case build.IsLocalImport(p): + base.Errorf("go install %s: argument must be a package path, not a relative path", arg) + case filepath.IsAbs(p): + base.Errorf("go install %s: argument must be a package path, not an absolute path", arg) + case search.IsMetaPackage(p): + base.Errorf("go install %s: argument must be a package path, not a meta-package", arg) + case path.Clean(p) != p: + base.Errorf("go install %s: argument must be a clean package path", arg) + case !strings.Contains(p, "...") && search.IsStandardImportPath(p) && goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, p): + base.Errorf("go install %s: argument must not be a package in the standard library", arg) + default: + patterns[i] = p + } + } + base.ExitIfErrors() + BuildInit() + + // Query the module providing the first argument, load its go.mod file, and + // check that it doesn't contain directives that would cause it to be + // interpreted differently if it were the main module. + // + // If multiple modules match the first argument, accept the longest match + // (first result). It's possible this module won't provide packages named by + // later arguments, and other modules would. Let's not try to be too + // magical though. + allowed := modload.CheckAllowed + if modload.IsRevisionQuery(version) { + // Don't check for retractions if a specific revision is requested. + allowed = nil + } + noneSelected := func(path string) (version string) { return "none" } + qrs, err := modload.QueryPackages(ctx, patterns[0], version, noneSelected, allowed) + if err != nil { + base.Fatalf("go install %s: %v", args[0], err) + } + installMod := qrs[0].Mod + data, err := modfetch.GoMod(installMod.Path, installMod.Version) + if err != nil { + base.Fatalf("go install %s: %v", args[0], err) + } + f, err := modfile.Parse("go.mod", data, nil) + if err != nil { + base.Fatalf("go install %s: %s: %v", args[0], installMod, err) + } + directiveFmt := "go install %s: %s\n" + + "\tThe go.mod file for the module providing named packages contains one or\n" + + "\tmore %s directives. It must not contain directives that would cause\n" + + "\tit to be interpreted differently than if it were the main module." + if len(f.Replace) > 0 { + base.Fatalf(directiveFmt, args[0], installMod, "replace") + } + if len(f.Exclude) > 0 { + base.Fatalf(directiveFmt, args[0], installMod, "exclude") + } + + // Since we are in NoRoot mode, the build list initially contains only + // the dummy command-line-arguments module. Add a requirement on the + // module that provides the packages named on the command line. + if err := modload.EditBuildList(ctx, nil, []module.Version{installMod}); err != nil { + base.Fatalf("go install %s: %v", args[0], err) + } + + // Load packages for all arguments. Ignore non-main packages. + // Print a warning if an argument contains "..." and matches no main packages. + // PackagesAndErrors already prints warnings for patterns that don't match any + // packages, so be careful not to double print. + matchers := make([]func(string) bool, len(patterns)) + for i, p := range patterns { + if strings.Contains(p, "...") { + matchers[i] = search.MatchPattern(p) + } + } + + // TODO(golang.org/issue/40276): don't report errors loading non-main packages + // matched by a pattern. + pkgs := load.PackagesAndErrors(ctx, patterns) + load.CheckPackageErrors(pkgs) + mainPkgs := make([]*load.Package, 0, len(pkgs)) + mainCount := make([]int, len(patterns)) + nonMainCount := make([]int, len(patterns)) + for _, pkg := range pkgs { + if pkg.Name == "main" { + mainPkgs = append(mainPkgs, pkg) + for i := range patterns { + if matchers[i] != nil && matchers[i](pkg.ImportPath) { + mainCount[i]++ + } + } + } else { + for i := range patterns { + if matchers[i] == nil && patterns[i] == pkg.ImportPath { + base.Errorf("go install: package %s is not a main package", pkg.ImportPath) + } else if matchers[i] != nil && matchers[i](pkg.ImportPath) { + nonMainCount[i]++ + } + } + } + } + base.ExitIfErrors() + for i, p := range patterns { + if matchers[i] != nil && mainCount[i] == 0 && nonMainCount[i] > 0 { + fmt.Fprintf(os.Stderr, "go: warning: %q matched no main packages\n", p) + } + } + + // Check that named packages are all provided by the same module. + for _, pkg := range mainPkgs { + if pkg.Module == nil { + // Packages in std, cmd, and their vendored dependencies + // don't have this field set. + base.Errorf("go install: package %s not provided by module %s", pkg.ImportPath, installMod) + } else if pkg.Module.Path != installMod.Path || pkg.Module.Version != installMod.Version { + base.Errorf("go install: package %s provided by module %s@%s\n\tAll packages must be provided by the same module (%s).", pkg.ImportPath, pkg.Module.Path, pkg.Module.Version, installMod) + } + } + base.ExitIfErrors() + + // Build and install the packages. + InstallPackages(ctx, patterns, mainPkgs) +} + // ExecCmd is the command to use to run user binaries. // Normally it is empty, meaning run the binaries directly. // If cross-compiling and running on a remote system or diff --git a/libgo/go/cmd/go/internal/work/build_test.go b/libgo/go/cmd/go/internal/work/build_test.go index c33de26..eaf2639 100644 --- a/libgo/go/cmd/go/internal/work/build_test.go +++ b/libgo/go/cmd/go/internal/work/build_test.go @@ -7,7 +7,7 @@ package work import ( "bytes" "fmt" - "io/ioutil" + "io/fs" "os" "path/filepath" "reflect" @@ -169,7 +169,7 @@ func TestSharedLibName(t *testing.T) { for _, data := range testData { func() { if data.rootedAt != "" { - tmpGopath, err := ioutil.TempDir("", "gopath") + tmpGopath, err := os.MkdirTemp("", "gopath") if err != nil { t.Fatal(err) } @@ -221,10 +221,8 @@ func pkgImportPath(pkgpath string) *load.Package { // See https://golang.org/issue/18878. func TestRespectSetgidDir(t *testing.T) { switch runtime.GOOS { - case "darwin": - if runtime.GOARCH == "arm64" { - t.Skip("can't set SetGID bit with chmod on iOS") - } + case "ios": + t.Skip("can't set SetGID bit with chmod on iOS") case "windows", "plan9": t.Skip("chown/chmod setgid are not supported on Windows or Plan 9") } @@ -239,7 +237,7 @@ func TestRespectSetgidDir(t *testing.T) { return cmdBuf.WriteString(fmt.Sprint(a...)) } - setgiddir, err := ioutil.TempDir("", "SetGroupID") + setgiddir, err := os.MkdirTemp("", "SetGroupID") if err != nil { t.Fatal(err) } @@ -255,13 +253,13 @@ func TestRespectSetgidDir(t *testing.T) { } // Change setgiddir's permissions to include the SetGID bit. - if err := os.Chmod(setgiddir, 0755|os.ModeSetgid); err != nil { + if err := os.Chmod(setgiddir, 0755|fs.ModeSetgid); err != nil { t.Fatal(err) } - pkgfile, err := ioutil.TempFile("", "pkgfile") + pkgfile, err := os.CreateTemp("", "pkgfile") if err != nil { - t.Fatalf("ioutil.TempFile(\"\", \"pkgfile\"): %v", err) + t.Fatalf("os.CreateTemp(\"\", \"pkgfile\"): %v", err) } defer os.Remove(pkgfile.Name()) defer pkgfile.Close() diff --git a/libgo/go/cmd/go/internal/work/buildid.go b/libgo/go/cmd/go/internal/work/buildid.go index 6613b6f..d769881 100644 --- a/libgo/go/cmd/go/internal/work/buildid.go +++ b/libgo/go/cmd/go/internal/work/buildid.go @@ -7,7 +7,6 @@ package work import ( "bytes" "fmt" - "io/ioutil" "os" "os/exec" "strings" @@ -15,6 +14,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cache" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/str" "cmd/internal/buildid" ) @@ -31,7 +31,7 @@ import ( // // actionID/[.../]contentID // -// where the actionID and contentID are prepared by hashToString below. +// where the actionID and contentID are prepared by buildid.HashToString below. // and are found by looking for the first or last slash. // Usually the buildID is simply actionID/contentID, but see below for an // exception. @@ -108,31 +108,6 @@ func contentID(buildID string) string { return buildID[strings.LastIndex(buildID, buildIDSeparator)+1:] } -// hashToString converts the hash h to a string to be recorded -// in package archives and binaries as part of the build ID. -// We use the first 96 bits of the hash and encode it in base64, -// resulting in a 16-byte string. Because this is only used for -// detecting the need to rebuild installed files (not for lookups -// in the object file cache), 96 bits are sufficient to drive the -// probability of a false "do not need to rebuild" decision to effectively zero. -// We embed two different hashes in archives and four in binaries, -// so cutting to 16 bytes is a significant savings when build IDs are displayed. -// (16*4+3 = 67 bytes compared to 64*4+3 = 259 bytes for the -// more straightforward option of printing the entire h in hex). -func hashToString(h [cache.HashSize]byte) string { - const b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" - const chunks = 5 - var dst [chunks * 4]byte - for i := 0; i < chunks; i++ { - v := uint32(h[3*i])<<16 | uint32(h[3*i+1])<<8 | uint32(h[3*i+2]) - dst[4*i+0] = b64[(v>>18)&0x3F] - dst[4*i+1] = b64[(v>>12)&0x3F] - dst[4*i+2] = b64[(v>>6)&0x3F] - dst[4*i+3] = b64[v&0x3F] - } - return string(dst[:]) -} - // toolID returns the unique ID to use for the current copy of the // named tool (asm, compile, cover, link). // @@ -368,7 +343,7 @@ func (b *Builder) gccgoBuildIDFile(a *Action) (string, error) { } } - if err := ioutil.WriteFile(sfile, buf.Bytes(), 0666); err != nil { + if err := os.WriteFile(sfile, buf.Bytes(), 0666); err != nil { return "", err } @@ -400,11 +375,12 @@ func (b *Builder) buildID(file string) string { // fileHash returns the content hash of the named file. func (b *Builder) fileHash(file string) string { + file, _ = fsys.OverlayPath(file) sum, err := cache.FileHash(file) if err != nil { return "" } - return hashToString(sum) + return buildid.HashToString(sum) } // useCache tries to satisfy the action a, which has action ID actionHash, @@ -425,9 +401,9 @@ func (b *Builder) useCache(a *Action, actionHash cache.ActionID, target string) // It's important that the overall buildID be unlikely verging on impossible // to appear in the output by chance, but that should be taken care of by // the actionID half; if it also appeared in the input that would be like an - // engineered 96-bit partial SHA256 collision. + // engineered 120-bit partial SHA256 collision. a.actionID = actionHash - actionID := hashToString(actionHash) + actionID := buildid.HashToString(actionHash) if a.json != nil { a.json.ActionID = actionID } @@ -480,7 +456,7 @@ func (b *Builder) useCache(a *Action, actionHash cache.ActionID, target string) // build IDs of completed actions. oldBuildID := a.buildID a.buildID = id[1] + buildIDSeparator + id[2] - linkID := hashToString(b.linkActionID(a.triggers[0])) + linkID := buildid.HashToString(b.linkActionID(a.triggers[0])) if id[0] == linkID { // Best effort attempt to display output from the compile and link steps. // If it doesn't work, it doesn't work: reusing the cached binary is more @@ -654,7 +630,7 @@ func (b *Builder) updateBuildID(a *Action, target string, rewrite bool) error { if err != nil { return err } - newID := a.buildID[:strings.LastIndex(a.buildID, buildIDSeparator)] + buildIDSeparator + hashToString(hash) + newID := a.buildID[:strings.LastIndex(a.buildID, buildIDSeparator)] + buildIDSeparator + buildid.HashToString(hash) if len(newID) != len(a.buildID) { return fmt.Errorf("internal error: build ID length mismatch %q vs %q", a.buildID, newID) } @@ -670,7 +646,7 @@ func (b *Builder) updateBuildID(a *Action, target string, rewrite bool) error { } if rewrite { - w, err := os.OpenFile(target, os.O_WRONLY, 0) + w, err := os.OpenFile(target, os.O_RDWR, 0) if err != nil { return err } @@ -713,6 +689,7 @@ func (b *Builder) updateBuildID(a *Action, target string, rewrite bool) error { return err } a.Package.Export = c.OutputFile(outputID) + a.Package.BuildID = a.buildID } } } diff --git a/libgo/go/cmd/go/internal/work/exec.go b/libgo/go/cmd/go/internal/work/exec.go index 3898b20..c0b69f9 100644 --- a/libgo/go/cmd/go/internal/work/exec.go +++ b/libgo/go/cmd/go/internal/work/exec.go @@ -8,12 +8,14 @@ package work import ( "bytes" + "cmd/go/internal/fsys" + "context" "encoding/json" "errors" "fmt" "internal/lazyregexp" "io" - "io/ioutil" + "io/fs" "log" "math/rand" "os" @@ -30,7 +32,9 @@ import ( "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/load" + "cmd/go/internal/modload" "cmd/go/internal/str" + "cmd/go/internal/trace" ) // actionList returns the list of actions in the dag rooted at root @@ -54,7 +58,10 @@ func actionList(root *Action) []*Action { } // do runs the action graph rooted at root. -func (b *Builder) Do(root *Action) { +func (b *Builder) Do(ctx context.Context, root *Action) { + ctx, span := trace.StartSpan(ctx, "exec.Builder.Do ("+root.Mode+" "+root.Target+")") + defer span.Done() + if !b.IsCmdList { // If we're doing real work, take time at the end to trim the cache. c := cache.Default() @@ -86,7 +93,7 @@ func (b *Builder) Do(root *Action) { base.Fatalf("go: refusing to write action graph to %v\n", file) } js := actionGraphJSON(root) - if err := ioutil.WriteFile(file, []byte(js), 0666); err != nil { + if err := os.WriteFile(file, []byte(js), 0666); err != nil { fmt.Fprintf(os.Stderr, "go: writing action graph: %v\n", err) base.SetExitStatus(1) } @@ -110,13 +117,24 @@ func (b *Builder) Do(root *Action) { // Handle runs a single action and takes care of triggering // any actions that are runnable as a result. - handle := func(a *Action) { + handle := func(ctx context.Context, a *Action) { if a.json != nil { a.json.TimeStart = time.Now() } var err error if a.Func != nil && (!a.Failed || a.IgnoreFail) { - err = a.Func(b, a) + // TODO(matloob): Better action descriptions + desc := "Executing action " + if a.Package != nil { + desc += "(" + a.Mode + " " + a.Package.Desc() + ")" + } + ctx, span := trace.StartSpan(ctx, desc) + a.traceSpan = span + for _, d := range a.Deps { + trace.Flow(ctx, d.traceSpan, a.traceSpan) + } + err = a.Func(b, ctx, a) + span.Done() } if a.json != nil { a.json.TimeDone = time.Now() @@ -164,6 +182,7 @@ func (b *Builder) Do(root *Action) { for i := 0; i < par; i++ { wg.Add(1) go func() { + ctx := trace.StartGoroutine(ctx) defer wg.Done() for { select { @@ -176,7 +195,7 @@ func (b *Builder) Do(root *Action) { b.exec.Lock() a := b.ready.pop() b.exec.Unlock() - handle(a) + handle(ctx, a) case <-base.Interrupted: base.SetExitStatus(1) return @@ -253,7 +272,7 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { fmt.Fprintf(h, "asm %q %q %q\n", b.toolID("asm"), forcedAsmflags, p.Internal.Asmflags) } - // GO386, GOARM, GOMIPS, etc. + // GOARM, GOMIPS, etc. key, val := cfg.GetArchEnv() fmt.Fprintf(h, "%s=%s\n", key, val) @@ -320,6 +339,7 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { p.SysoFiles, p.SwigFiles, p.SwigCXXFiles, + p.EmbedFiles, ) for _, file := range inputFiles { fmt.Fprintf(h, "file %s %s\n", file, b.fileHash(filepath.Join(p.Dir, file))) @@ -386,7 +406,7 @@ const ( // build is the action for building a single package. // Note that any new influence on this logic must be reported in b.buildActionID above as well. -func (b *Builder) build(a *Action) (err error) { +func (b *Builder) build(ctx context.Context, a *Action) (err error) { p := a.Package bit := func(x uint32, b bool) uint32 { @@ -414,6 +434,7 @@ func (b *Builder) build(a *Action) (err error) { need &^= needBuild if b.NeedExport { p.Export = a.built + p.BuildID = a.buildID } if need&needCompiledGoFiles != 0 { if err := b.loadCachedSrcFiles(a); err == nil { @@ -516,6 +537,34 @@ func (b *Builder) build(a *Action) (err error) { } } + // Compute overlays for .c/.cc/.h/etc. and if there are any overlays + // put correct contents of all those files in the objdir, to ensure + // the correct headers are included. nonGoOverlay is the overlay that + // points from nongo files to the copied files in objdir. + nonGoFileLists := [][]string{a.Package.CFiles, a.Package.SFiles, a.Package.CXXFiles, a.Package.HFiles, a.Package.FFiles} +OverlayLoop: + for _, fs := range nonGoFileLists { + for _, f := range fs { + if _, ok := fsys.OverlayPath(mkAbs(p.Dir, f)); ok { + a.nonGoOverlay = make(map[string]string) + break OverlayLoop + } + } + } + if a.nonGoOverlay != nil { + for _, fs := range nonGoFileLists { + for i := range fs { + from := mkAbs(p.Dir, fs[i]) + opath, _ := fsys.OverlayPath(from) + dst := objdir + filepath.Base(fs[i]) + if err := b.copyFile(dst, opath, 0666, false); err != nil { + return err + } + a.nonGoOverlay[from] = dst + } + } + } + // Run SWIG on each .swig and .swigcxx file. // Each run will generate two files, a .go file and a .c or .cxx file. // The .go file will use import "C" and is to be processed by cgo. @@ -586,7 +635,7 @@ func (b *Builder) build(a *Action) (err error) { sfiles, gccfiles = filter(sfiles, sfiles[:0], gccfiles) } else { for _, sfile := range sfiles { - data, err := ioutil.ReadFile(filepath.Join(a.Package.Dir, sfile)) + data, err := os.ReadFile(filepath.Join(a.Package.Dir, sfile)) if err == nil { if bytes.HasPrefix(data, []byte("TEXT")) || bytes.Contains(data, []byte("\nTEXT")) || bytes.HasPrefix(data, []byte("DATA")) || bytes.Contains(data, []byte("\nDATA")) || @@ -676,8 +725,28 @@ func (b *Builder) build(a *Action) (err error) { } } + // Prepare Go embed config if needed. + // Unlike the import config, it's okay for the embed config to be empty. + var embedcfg []byte + if len(p.Internal.Embed) > 0 { + var embed struct { + Patterns map[string][]string + Files map[string]string + } + embed.Patterns = p.Internal.Embed + embed.Files = make(map[string]string) + for _, file := range p.EmbedFiles { + embed.Files[file] = filepath.Join(p.Dir, file) + } + js, err := json.MarshalIndent(&embed, "", "\t") + if err != nil { + return fmt.Errorf("marshal embedcfg: %v", err) + } + embedcfg = js + } + if p.Internal.BuildInfo != "" && cfg.ModulesEnabled { - if err := b.writeFile(objdir+"_gomod_.go", load.ModInfoProg(p.Internal.BuildInfo, cfg.BuildToolchainName == "gccgo")); err != nil { + if err := b.writeFile(objdir+"_gomod_.go", modload.ModInfoProg(p.Internal.BuildInfo, cfg.BuildToolchainName == "gccgo")); err != nil { return err } gofiles = append(gofiles, objdir+"_gomod_.go") @@ -685,7 +754,7 @@ func (b *Builder) build(a *Action) (err error) { // Compile Go. objpkg := objdir + "_pkg_.a" - ofile, out, err := BuildToolchain.gc(b, a, objpkg, icfg.Bytes(), symabis, len(sfiles) > 0, gofiles) + ofile, out, err := BuildToolchain.gc(b, a, objpkg, icfg.Bytes(), embedcfg, symabis, len(sfiles) > 0, gofiles) if len(out) > 0 { output := b.processOutput(out) if p.Module != nil && !allowedVersion(p.Module.GoVersion) { @@ -698,7 +767,7 @@ func (b *Builder) build(a *Action) (err error) { } if err != nil { if p.Module != nil && !allowedVersion(p.Module.GoVersion) { - b.showOutput(a, a.Package.Dir, a.Package.Desc(), "note: module requires Go "+p.Module.GoVersion) + b.showOutput(a, a.Package.Dir, a.Package.Desc(), "note: module requires Go "+p.Module.GoVersion+"\n") } return err } @@ -906,12 +975,13 @@ func (b *Builder) loadCachedSrcFiles(a *Action) error { // vetConfig is the configuration passed to vet describing a single package. type vetConfig struct { - ID string // package ID (example: "fmt [fmt.test]") - Compiler string // compiler name (gc, gccgo) - Dir string // directory containing package - ImportPath string // canonical import path ("package path") - GoFiles []string // absolute paths to package source files - NonGoFiles []string // absolute paths to package non-Go files + ID string // package ID (example: "fmt [fmt.test]") + Compiler string // compiler name (gc, gccgo) + Dir string // directory containing package + ImportPath string // canonical import path ("package path") + GoFiles []string // absolute paths to package source files + NonGoFiles []string // absolute paths to package non-Go files + IgnoredFiles []string // absolute paths to ignored source files ImportMap map[string]string // map import path in source code to package path PackageFile map[string]string // map package path to .a file with export data @@ -935,20 +1005,23 @@ func buildVetConfig(a *Action, srcfiles []string) { } } + ignored := str.StringList(a.Package.IgnoredGoFiles, a.Package.IgnoredOtherFiles) + // Pass list of absolute paths to vet, // so that vet's error messages will use absolute paths, // so that we can reformat them relative to the directory // in which the go command is invoked. vcfg := &vetConfig{ - ID: a.Package.ImportPath, - Compiler: cfg.BuildToolchainName, - Dir: a.Package.Dir, - GoFiles: mkAbsFiles(a.Package.Dir, gofiles), - NonGoFiles: mkAbsFiles(a.Package.Dir, nongofiles), - ImportPath: a.Package.ImportPath, - ImportMap: make(map[string]string), - PackageFile: make(map[string]string), - Standard: make(map[string]bool), + ID: a.Package.ImportPath, + Compiler: cfg.BuildToolchainName, + Dir: a.Package.Dir, + GoFiles: mkAbsFiles(a.Package.Dir, gofiles), + NonGoFiles: mkAbsFiles(a.Package.Dir, nongofiles), + IgnoredFiles: mkAbsFiles(a.Package.Dir, ignored), + ImportPath: a.Package.ImportPath, + ImportMap: make(map[string]string), + PackageFile: make(map[string]string), + Standard: make(map[string]bool), } a.vetCfg = vcfg for i, raw := range a.Package.Internal.RawImports { @@ -993,7 +1066,7 @@ var VetFlags []string // VetExplicit records whether the vet flags were set explicitly on the command line. var VetExplicit bool -func (b *Builder) vet(a *Action) error { +func (b *Builder) vet(ctx context.Context, a *Action) error { // a.Deps[0] is the build of the package being vetted. // a.Deps[1] is the build of the "fmt" package. @@ -1036,17 +1109,28 @@ func (b *Builder) vet(a *Action) error { // This is OK as long as the packages that are farther down the // dependency tree turn on *more* analysis, as here. // (The unsafeptr check does not write any facts for use by - // later vet runs.) + // later vet runs, nor does unreachable.) if a.Package.Goroot && !VetExplicit && VetTool == "" { + // Turn off -unsafeptr checks. + // There's too much unsafe.Pointer code + // that vet doesn't like in low-level packages + // like runtime, sync, and reflect. // Note that $GOROOT/src/buildall.bash // does the same for the misc-compile trybots // and should be updated if these flags are // changed here. - // - // There's too much unsafe.Pointer code - // that vet doesn't like in low-level packages - // like runtime, sync, and reflect. vetFlags = []string{"-unsafeptr=false"} + + // Also turn off -unreachable checks during go test. + // During testing it is very common to make changes + // like hard-coded forced returns or panics that make + // code unreachable. It's unreasonable to insist on files + // not having any unreachable code during "go test". + // (buildall.bash still runs with -unreachable enabled + // for the overall whole-tree scan.) + if cfg.CmdName == "test" { + vetFlags = append(vetFlags, "-unreachable=false") + } } // Note: We could decide that vet should compute export data for @@ -1159,12 +1243,17 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) { fmt.Fprintf(h, "linkflags %q\n", p.Internal.Ldflags) } - // GO386, GOARM, GOMIPS, etc. + // GOARM, GOMIPS, etc. key, val := cfg.GetArchEnv() fmt.Fprintf(h, "%s=%s\n", key, val) - // The linker writes source file paths that say GOROOT_FINAL. - fmt.Fprintf(h, "GOROOT=%s\n", cfg.GOROOT_FINAL) + // The linker writes source file paths that say GOROOT_FINAL, but + // only if -trimpath is not specified (see ld() in gc.go). + gorootFinal := cfg.GOROOT_FINAL + if cfg.BuildTrimpath { + gorootFinal = trimPathGoRootFinal + } + fmt.Fprintf(h, "GOROOT=%s\n", gorootFinal) // GO_EXTLINK_ENABLED controls whether the external linker is used. fmt.Fprintf(h, "GO_EXTLINK_ENABLED=%s\n", cfg.Getenv("GO_EXTLINK_ENABLED")) @@ -1184,7 +1273,7 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) { // link is the action for linking a single command. // Note that any new influence on this logic must be reported in b.linkActionID above as well. -func (b *Builder) link(a *Action) (err error) { +func (b *Builder) link(ctx context.Context, a *Action) (err error) { if b.useCache(a, b.linkActionID(a), a.Package.Target) || b.IsCmdList { return nil } @@ -1376,14 +1465,14 @@ func (b *Builder) getPkgConfigFlags(p *load.Package) (cflags, ldflags []string, return } -func (b *Builder) installShlibname(a *Action) error { +func (b *Builder) installShlibname(ctx context.Context, a *Action) error { if err := allowInstall(a); err != nil { return err } // TODO: BuildN a1 := a.Deps[0] - err := ioutil.WriteFile(a.Target, []byte(filepath.Base(a1.Target)+"\n"), 0666) + err := os.WriteFile(a.Target, []byte(filepath.Base(a1.Target)+"\n"), 0666) if err != nil { return err } @@ -1425,7 +1514,7 @@ func (b *Builder) linkSharedActionID(a *Action) cache.ActionID { return h.Sum() } -func (b *Builder) linkShared(a *Action) (err error) { +func (b *Builder) linkShared(ctx context.Context, a *Action) (err error) { if b.useCache(a, b.linkSharedActionID(a), a.Target) || b.IsCmdList { return nil } @@ -1451,7 +1540,7 @@ func (b *Builder) linkShared(a *Action) (err error) { } // BuildInstallFunc is the action for installing a single package or executable. -func BuildInstallFunc(b *Builder, a *Action) (err error) { +func BuildInstallFunc(b *Builder, ctx context.Context, a *Action) (err error) { defer func() { if err != nil && err != errPrintedOutput { // a.Package == nil is possible for the go install -buildmode=shared @@ -1524,7 +1613,7 @@ func BuildInstallFunc(b *Builder, a *Action) (err error) { return err } - perm := os.FileMode(0666) + perm := fs.FileMode(0666) if a1.Mode == "link" { switch cfg.BuildBuildmode { case "c-archive", "c-shared", "plugin": @@ -1573,7 +1662,7 @@ func (b *Builder) cleanup(a *Action) { } // moveOrCopyFile is like 'mv src dst' or 'cp src dst'. -func (b *Builder) moveOrCopyFile(dst, src string, perm os.FileMode, force bool) error { +func (b *Builder) moveOrCopyFile(dst, src string, perm fs.FileMode, force bool) error { if cfg.BuildN { b.Showcmd("", "mv %s %s", src, dst) return nil @@ -1599,7 +1688,7 @@ func (b *Builder) moveOrCopyFile(dst, src string, perm os.FileMode, force bool) // we have to copy the file to retain the correct permissions. // https://golang.org/issue/18878 if fi, err := os.Stat(filepath.Dir(dst)); err == nil { - if fi.IsDir() && (fi.Mode()&os.ModeSetgid) != 0 { + if fi.IsDir() && (fi.Mode()&fs.ModeSetgid) != 0 { return b.copyFile(dst, src, perm, force) } } @@ -1634,7 +1723,7 @@ func (b *Builder) moveOrCopyFile(dst, src string, perm os.FileMode, force bool) } // copyFile is like 'cp src dst'. -func (b *Builder) copyFile(dst, src string, perm os.FileMode, force bool) error { +func (b *Builder) copyFile(dst, src string, perm fs.FileMode, force bool) error { if cfg.BuildN || cfg.BuildX { b.Showcmd("", "cp %s %s", src, dst) if cfg.BuildN { @@ -1700,11 +1789,11 @@ func (b *Builder) writeFile(file string, text []byte) error { if cfg.BuildN { return nil } - return ioutil.WriteFile(file, text, 0666) + return os.WriteFile(file, text, 0666) } // Install the cgo export header file, if there is one. -func (b *Builder) installHeader(a *Action) error { +func (b *Builder) installHeader(ctx context.Context, a *Action) error { src := a.Objdir + "_cgo_install.h" if _, err := os.Stat(src); os.IsNotExist(err) { // If the file does not exist, there are no exported @@ -1963,6 +2052,13 @@ func (b *Builder) runOut(a *Action, dir string, env []string, cmdargs ...interfa defer cleanup() cmd.Dir = dir cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) + + // Add the TOOLEXEC_IMPORTPATH environment variable for -toolexec tools. + // It doesn't really matter if -toolexec isn't being used. + if a != nil && a.Package != nil { + cmd.Env = append(cmd.Env, "TOOLEXEC_IMPORTPATH="+a.Package.ImportPath) + } + cmd.Env = append(cmd.Env, env...) start := time.Now() err := cmd.Run() @@ -2080,9 +2176,7 @@ func mkAbs(dir, f string) string { type toolchain interface { // gc runs the compiler in a specific directory on a set of files // and returns the name of the generated output file. - // - // TODO: This argument list is long. Consider putting it in a struct. - gc(b *Builder, a *Action, archive string, importcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, out []byte, err error) + gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, out []byte, err error) // cc runs the toolchain's C compiler in a directory on a C file // to produce an output file. cc(b *Builder, a *Action, ofile, cfile string) error @@ -2122,7 +2216,7 @@ func (noToolchain) linker() string { return "" } -func (noToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, out []byte, err error) { +func (noToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, out []byte, err error) { return "", nil, noCompiler() } @@ -2204,7 +2298,11 @@ func (b *Builder) ccompile(a *Action, p *load.Package, outfile string, flags []s } } - output, err := b.runOut(a, filepath.Dir(file), b.cCompilerEnv(), compiler, flags, "-o", outfile, "-c", filepath.Base(file)) + overlayPath := file + if p, ok := a.nonGoOverlay[overlayPath]; ok { + overlayPath = p + } + output, err := b.runOut(a, filepath.Dir(overlayPath), b.cCompilerEnv(), compiler, flags, "-o", outfile, "-c", filepath.Base(overlayPath)) if len(output) > 0 { // On FreeBSD 11, when we pass -g to clang 3.8 it // invokes its internal assembler with -dwarf-version=2. @@ -2247,7 +2345,8 @@ func (b *Builder) gccld(a *Action, p *load.Package, objdir, outfile string, flag cmdargs := []interface{}{cmd, "-o", outfile, objs, flags} dir := p.Dir - out, err := b.runOut(a, dir, b.cCompilerEnv(), cmdargs...) + out, err := b.runOut(a, base.Cwd, b.cCompilerEnv(), cmdargs...) + if len(out) > 0 { // Filter out useless linker warnings caused by bugs outside Go. // See also cmd/link/internal/ld's hostlink method. @@ -2403,7 +2502,7 @@ func (b *Builder) compilerCmd(compiler []string, incdir, workdir string) []strin // On OS X, some of the compilers behave as if -fno-common // is always set, and the Mach-O linker in 6l/8l assumes this. // See https://golang.org/issue/3253. - if cfg.Goos == "darwin" { + if cfg.Goos == "darwin" || cfg.Goos == "ios" { a = append(a, "-fno-common") } @@ -2445,7 +2544,7 @@ func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { tmp := os.DevNull if runtime.GOOS == "windows" { - f, err := ioutil.TempFile(b.WorkDir, "") + f, err := os.CreateTemp(b.WorkDir, "") if err != nil { return false } @@ -2585,7 +2684,8 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo cgoLDFLAGS = append([]string{"-fsanitize=memory"}, cgoLDFLAGS...) } - // Allows including _cgo_export.h from .[ch] files in the package. + // Allows including _cgo_export.h, as well as the user's .h files, + // from .[ch] files in the package. cgoCPPFLAGS = append(cgoCPPFLAGS, "-I", objdir) // cgo @@ -2642,7 +2742,23 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo cgoflags = append(cgoflags, "-exportheader="+objdir+"_cgo_install.h") } - if err := b.run(a, p.Dir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil { + execdir := p.Dir + + // Rewrite overlaid paths in cgo files. + // cgo adds //line and #line pragmas in generated files with these paths. + var trimpath []string + for i := range cgofiles { + path := mkAbs(p.Dir, cgofiles[i]) + if opath, ok := fsys.OverlayPath(path); ok { + cgofiles[i] = opath + trimpath = append(trimpath, opath+"=>"+path) + } + } + if len(trimpath) > 0 { + cgoflags = append(cgoflags, "-trimpath", strings.Join(trimpath, ";")) + } + + if err := b.run(a, execdir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil { return nil, nil, err } outGo = append(outGo, gofiles...) @@ -2735,7 +2851,7 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo continue } - src, err := ioutil.ReadFile(f) + src, err := os.ReadFile(f) if err != nil { return nil, nil, err } @@ -2811,7 +2927,7 @@ func (b *Builder) dynimport(a *Action, p *load.Package, objdir, importGo, cgoExe return err } - linkobj := str.StringList(ofile, outObj, p.SysoFiles) + linkobj := str.StringList(ofile, outObj, mkAbsFiles(p.Dir, p.SysoFiles)) dynobj := objdir + "_cgo_.o" // we need to use -pie for Linux/ARM to get accurate imported sym @@ -2836,7 +2952,7 @@ func (b *Builder) dynimport(a *Action, p *load.Package, objdir, importGo, cgoExe if p.Standard && p.ImportPath == "runtime/cgo" { cgoflags = []string{"-dynlinker"} // record path to dynamic linker } - return b.run(a, p.Dir, p.ImportPath, b.cCompilerEnv(), cfg.BuildToolexec, cgoExe, "-dynpackage", p.Name, "-dynimport", dynobj, "-dynout", importGo, cgoflags) + return b.run(a, base.Cwd, p.ImportPath, b.cCompilerEnv(), cfg.BuildToolexec, cgoExe, "-dynpackage", p.Name, "-dynimport", dynobj, "-dynout", importGo, cgoflags) } // Run SWIG on all SWIG input files. @@ -2965,14 +3081,14 @@ func (b *Builder) swigDoIntSize(objdir string) (intsize string, err error) { return "$INTBITS", nil } src := filepath.Join(b.WorkDir, "swig_intsize.go") - if err = ioutil.WriteFile(src, []byte(swigIntSizeCode), 0666); err != nil { + if err = os.WriteFile(src, []byte(swigIntSizeCode), 0666); err != nil { return } srcs := []string{src} - p := load.GoFilesPackage(srcs) + p := load.GoFilesPackage(context.TODO(), srcs) - if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, "", false, srcs); e != nil { + if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, nil, "", false, srcs); e != nil { return "32", nil } return "64", nil @@ -3125,14 +3241,14 @@ func passLongArgsInResponseFiles(cmd *exec.Cmd) (cleanup func()) { return } - tf, err := ioutil.TempFile("", "args") + tf, err := os.CreateTemp("", "args") if err != nil { log.Fatalf("error writing long arguments to response file: %v", err) } cleanup = func() { os.Remove(tf.Name()) } var buf bytes.Buffer for _, arg := range cmd.Args[1:] { - fmt.Fprintf(&buf, "%s\n", arg) + fmt.Fprintf(&buf, "%s\n", encodeArg(arg)) } if _, err := tf.Write(buf.Bytes()); err != nil { tf.Close() @@ -3147,6 +3263,12 @@ func passLongArgsInResponseFiles(cmd *exec.Cmd) (cleanup func()) { return cleanup } +// Windows has a limit of 32 KB arguments. To be conservative and not worry +// about whether that includes spaces or not, just use 30 KB. Darwin's limit is +// less clear. The OS claims 256KB, but we've seen failures with arglen as +// small as 50KB. +const ArgLengthForResponseFile = (30 << 10) + func useResponseFile(path string, argLen int) bool { // Unless the program uses objabi.Flagparse, which understands // response files, don't use response files. @@ -3158,11 +3280,7 @@ func useResponseFile(path string, argLen int) bool { return false } - // Windows has a limit of 32 KB arguments. To be conservative and not - // worry about whether that includes spaces or not, just use 30 KB. - // Darwin's limit is less clear. The OS claims 256KB, but we've seen - // failures with arglen as small as 50KB. - if argLen > (30 << 10) { + if argLen > ArgLengthForResponseFile { return true } @@ -3175,3 +3293,25 @@ func useResponseFile(path string, argLen int) bool { return false } + +// encodeArg encodes an argument for response file writing. +func encodeArg(arg string) string { + // If there aren't any characters we need to reencode, fastpath out. + if !strings.ContainsAny(arg, "\\\n") { + return arg + } + var b strings.Builder + for _, r := range arg { + switch r { + case '\\': + b.WriteByte('\\') + b.WriteByte('\\') + case '\n': + b.WriteByte('\\') + b.WriteByte('n') + default: + b.WriteRune(r) + } + } + return b.String() +} diff --git a/libgo/go/cmd/go/internal/work/exec_test.go b/libgo/go/cmd/go/internal/work/exec_test.go new file mode 100644 index 0000000..4eb762cb --- /dev/null +++ b/libgo/go/cmd/go/internal/work/exec_test.go @@ -0,0 +1,86 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "bytes" + "cmd/internal/objabi" + "fmt" + "math/rand" + "testing" + "time" + "unicode/utf8" +) + +func TestEncodeArgs(t *testing.T) { + t.Parallel() + tests := []struct { + arg, want string + }{ + {"", ""}, + {"hello", "hello"}, + {"hello\n", "hello\\n"}, + {"hello\\", "hello\\\\"}, + {"hello\nthere", "hello\\nthere"}, + {"\\\n", "\\\\\\n"}, + } + for _, test := range tests { + if got := encodeArg(test.arg); got != test.want { + t.Errorf("encodeArg(%q) = %q, want %q", test.arg, got, test.want) + } + } +} + +func TestEncodeDecode(t *testing.T) { + t.Parallel() + tests := []string{ + "", + "hello", + "hello\\there", + "hello\nthere", + "hello 中国", + "hello \n中\\国", + } + for _, arg := range tests { + if got := objabi.DecodeArg(encodeArg(arg)); got != arg { + t.Errorf("objabi.DecodeArg(encodeArg(%q)) = %q", arg, got) + } + } +} + +func TestEncodeDecodeFuzz(t *testing.T) { + if testing.Short() { + t.Skip("fuzz test is slow") + } + t.Parallel() + + nRunes := ArgLengthForResponseFile + 100 + rBuffer := make([]rune, nRunes) + buf := bytes.NewBuffer([]byte(string(rBuffer))) + + seed := time.Now().UnixNano() + t.Logf("rand seed: %v", seed) + rng := rand.New(rand.NewSource(seed)) + + for i := 0; i < 50; i++ { + // Generate a random string of runes. + buf.Reset() + for buf.Len() < ArgLengthForResponseFile+1 { + var r rune + for { + r = rune(rng.Intn(utf8.MaxRune + 1)) + if utf8.ValidRune(r) { + break + } + } + fmt.Fprintf(buf, "%c", r) + } + arg := buf.String() + + if got := objabi.DecodeArg(encodeArg(arg)); got != arg { + t.Errorf("[%d] objabi.DecodeArg(encodeArg(%q)) = %q [seed: %v]", i, arg, got, seed) + } + } +} diff --git a/libgo/go/cmd/go/internal/work/gc.go b/libgo/go/cmd/go/internal/work/gc.go index f1d08e0..cc4e2b2b 100644 --- a/libgo/go/cmd/go/internal/work/gc.go +++ b/libgo/go/cmd/go/internal/work/gc.go @@ -9,7 +9,6 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "log" "os" "path/filepath" @@ -18,6 +17,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/load" "cmd/go/internal/str" "cmd/internal/objabi" @@ -25,6 +25,9 @@ import ( "crypto/sha1" ) +// The 'path' used for GOROOT_FINAL when -trimpath is specified +const trimPathGoRootFinal = "go" + // The Go toolchain. type gcToolchain struct{} @@ -48,7 +51,7 @@ func pkgPath(a *Action) string { return ppath } -func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { +func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { p := a.Package objdir := a.Objdir if archive != "" { @@ -85,7 +88,11 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s extFiles := len(p.CgoFiles) + len(p.CFiles) + len(p.CXXFiles) + len(p.MFiles) + len(p.FFiles) + len(p.SFiles) + len(p.SysoFiles) + len(p.SwigFiles) + len(p.SwigCXXFiles) if p.Standard { switch p.ImportPath { - case "bytes", "internal/poll", "net", "os", "runtime/pprof", "runtime/trace", "sync", "syscall", "time": + case "bytes", "internal/poll", "net", "os": + fallthrough + case "runtime/metrics", "runtime/pprof", "runtime/trace": + fallthrough + case "sync", "syscall", "time": extFiles++ } } @@ -129,6 +136,12 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s } args = append(args, "-importcfg", objdir+"importcfg") } + if embedcfg != nil { + if err := b.writeFile(objdir+"embedcfg", embedcfg); err != nil { + return "", nil, err + } + args = append(args, "-embedcfg", objdir+"embedcfg") + } if ofile == archive { args = append(args, "-pack") } @@ -142,10 +155,26 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s } for _, f := range gofiles { - args = append(args, mkAbs(p.Dir, f)) - } - - output, err = b.runOut(a, p.Dir, nil, args...) + f := mkAbs(p.Dir, f) + + // Handle overlays. Convert path names using OverlayPath + // so these paths can be handed directly to tools. + // Deleted files won't show up in when scanning directories earlier, + // so OverlayPath will never return "" (meaning a deleted file) here. + // TODO(#39958): Handle cases where the package directory + // doesn't exist on disk (this can happen when all the package's + // files are in an overlay): the code expects the package directory + // to exist and runs some tools in that directory. + // TODO(#39958): Process the overlays when the + // gofiles, cgofiles, cfiles, sfiles, and cxxfiles variables are + // created in (*Builder).build. Doing that requires rewriting the + // code that uses those values to expect absolute paths. + f, _ = fsys.OverlayPath(f) + + args = append(args, f) + } + + output, err = b.runOut(a, base.Cwd, nil, args...) return ofile, output, err } @@ -232,17 +261,63 @@ func (a *Action) trimpath() string { if len(objdir) > 1 && objdir[len(objdir)-1] == filepath.Separator { objdir = objdir[:len(objdir)-1] } - rewrite := objdir + "=>" + rewrite := "" - // For "go build -trimpath", rewrite package source directory - // to a file system-independent path (just the import path). + rewriteDir := a.Package.Dir if cfg.BuildTrimpath { if m := a.Package.Module; m != nil && m.Version != "" { - rewrite += ";" + a.Package.Dir + "=>" + m.Path + "@" + m.Version + strings.TrimPrefix(a.Package.ImportPath, m.Path) + rewriteDir = m.Path + "@" + m.Version + strings.TrimPrefix(a.Package.ImportPath, m.Path) } else { - rewrite += ";" + a.Package.Dir + "=>" + a.Package.ImportPath + rewriteDir = a.Package.ImportPath + } + rewrite += a.Package.Dir + "=>" + rewriteDir + ";" + } + + // Add rewrites for overlays. The 'from' and 'to' paths in overlays don't need to have + // same basename, so go from the overlay contents file path (passed to the compiler) + // to the path the disk path would be rewritten to. + + cgoFiles := make(map[string]bool) + for _, f := range a.Package.CgoFiles { + cgoFiles[f] = true + } + + // TODO(matloob): Higher up in the stack, when the logic for deciding when to make copies + // of c/c++/m/f/hfiles is consolidated, use the same logic that Build uses to determine + // whether to create the copies in objdir to decide whether to rewrite objdir to the + // package directory here. + var overlayNonGoRewrites string // rewrites for non-go files + hasCgoOverlay := false + if fsys.OverlayFile != "" { + for _, filename := range a.Package.AllFiles() { + path := filename + if !filepath.IsAbs(path) { + path = filepath.Join(a.Package.Dir, path) + } + base := filepath.Base(path) + isGo := strings.HasSuffix(filename, ".go") || strings.HasSuffix(filename, ".s") + isCgo := cgoFiles[filename] || !isGo + overlayPath, isOverlay := fsys.OverlayPath(path) + if isCgo && isOverlay { + hasCgoOverlay = true + } + if !isCgo && isOverlay { + rewrite += overlayPath + "=>" + filepath.Join(rewriteDir, base) + ";" + } else if isCgo { + // Generate rewrites for non-Go files copied to files in objdir. + if filepath.Dir(path) == a.Package.Dir { + // This is a file copied to objdir. + overlayNonGoRewrites += filepath.Join(objdir, base) + "=>" + filepath.Join(rewriteDir, base) + ";" + } + } else { + // Non-overlay Go files are covered by the a.Package.Dir rewrite rule above. + } } } + if hasCgoOverlay { + rewrite += overlayNonGoRewrites + } + rewrite += objdir + "=>" return rewrite } @@ -259,6 +334,21 @@ func asmArgs(a *Action, p *load.Package) []interface{} { } } } + if objabi.IsRuntimePackagePath(pkgpath) { + args = append(args, "-compiling-runtime") + if objabi.Regabi_enabled != 0 { + // In order to make it easier to port runtime assembly + // to the register ABI, we introduce a macro + // indicating the experiment is enabled. + // + // Note: a similar change also appears in + // cmd/dist/build.go. + // + // TODO(austin): Remove this once we commit to the + // register ABI (#40724). + args = append(args, "-D=GOEXPERIMENT_REGABI=1") + } + } if cfg.Goarch == "mips" || cfg.Goarch == "mipsle" { // Define GOMIPS_value from cfg.GOMIPS. @@ -279,9 +369,10 @@ func (gcToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) var ofiles []string for _, sfile := range sfiles { + overlayPath, _ := fsys.OverlayPath(mkAbs(p.Dir, sfile)) ofile := a.Objdir + sfile[:len(sfile)-len(".s")] + ".o" ofiles = append(ofiles, ofile) - args1 := append(args, "-o", ofile, mkAbs(p.Dir, sfile)) + args1 := append(args, "-o", ofile, overlayPath) if err := b.run(a, p.Dir, p.ImportPath, nil, args1...); err != nil { return nil, err } @@ -297,7 +388,8 @@ func (gcToolchain) symabis(b *Builder, a *Action, sfiles []string) (string, erro if p.ImportPath == "runtime/cgo" && strings.HasPrefix(sfile, "gcc_") { continue } - args = append(args, mkAbs(p.Dir, sfile)) + op, _ := fsys.OverlayPath(mkAbs(p.Dir, sfile)) + args = append(args, op) } // Supply an empty go_asm.h as if the compiler had been run. @@ -333,11 +425,11 @@ func toolVerify(a *Action, b *Builder, p *load.Package, newTool string, ofile st if err := b.run(a, p.Dir, p.ImportPath, nil, newArgs...); err != nil { return err } - data1, err := ioutil.ReadFile(ofile) + data1, err := os.ReadFile(ofile) if err != nil { return err } - data2, err := ioutil.ReadFile(ofile + ".new") + data2, err := os.ReadFile(ofile + ".new") if err != nil { return err } @@ -487,7 +579,7 @@ func pluginPath(a *Action) string { } fmt.Fprintf(h, "build ID: %s\n", buildID) for _, file := range str.StringList(p.GoFiles, p.CgoFiles, p.SFiles) { - data, err := ioutil.ReadFile(filepath.Join(p.Dir, file)) + data, err := os.ReadFile(filepath.Join(p.Dir, file)) if err != nil { base.Fatalf("go: %s", err) } @@ -560,7 +652,7 @@ func (gcToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) env := []string{} if cfg.BuildTrimpath { - env = append(env, "GOROOT_FINAL=go") + env = append(env, "GOROOT_FINAL="+trimPathGoRootFinal) } return b.run(root, dir, root.Package.ImportPath, env, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags, mainpkg) } diff --git a/libgo/go/cmd/go/internal/work/gccgo.go b/libgo/go/cmd/go/internal/work/gccgo.go index dbaff70..793a33366 100644 --- a/libgo/go/cmd/go/internal/work/gccgo.go +++ b/libgo/go/cmd/go/internal/work/gccgo.go @@ -6,7 +6,6 @@ package work import ( "fmt" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -15,6 +14,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/load" "cmd/go/internal/str" "cmd/internal/pkgpath" @@ -62,7 +62,7 @@ func checkGccgoBin() { base.Exit() } -func (tools gccgoToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { +func (tools gccgoToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { p := a.Package objdir := a.Objdir out := "_go_.o" @@ -93,13 +93,37 @@ func (tools gccgoToolchain) gc(b *Builder, a *Action, archive string, importcfg args = append(args, "-I", root) } } - if cfg.BuildTrimpath && b.gccSupportsFlag(args[:1], "-ffile-prefix-map=a=b") { - args = append(args, "-ffile-prefix-map="+base.Cwd+"=.") - args = append(args, "-ffile-prefix-map="+b.WorkDir+"=/tmp/go-build") + + if b.gccSupportsFlag(args[:1], "-ffile-prefix-map=a=b") { + if cfg.BuildTrimpath { + args = append(args, "-ffile-prefix-map="+base.Cwd+"=.") + args = append(args, "-ffile-prefix-map="+b.WorkDir+"=/tmp/go-build") + } + if fsys.OverlayFile != "" { + for _, name := range gofiles { + absPath := mkAbs(p.Dir, name) + overlayPath, ok := fsys.OverlayPath(absPath) + if !ok { + continue + } + toPath := absPath + // gccgo only applies the last matching rule, so also handle the case where + // BuildTrimpath is true and the path is relative to base.Cwd. + if cfg.BuildTrimpath && str.HasFilePathPrefix(toPath, base.Cwd) { + toPath = "." + toPath[len(base.Cwd):] + } + args = append(args, "-ffile-prefix-map="+overlayPath+"="+toPath) + } + } } + args = append(args, a.Package.Internal.Gccgoflags...) for _, f := range gofiles { - args = append(args, mkAbs(p.Dir, f)) + f := mkAbs(p.Dir, f) + // Overlay files if necessary. + // See comment on gctoolchain.gc about overlay TODOs + f, _ = fsys.OverlayPath(f) + args = append(args, f) } output, err = b.runOut(a, p.Dir, nil, args) @@ -174,7 +198,7 @@ func (tools gccgoToolchain) asm(b *Builder, a *Action, sfiles []string) ([]strin base := filepath.Base(sfile) ofile := a.Objdir + base[:len(base)-len(".s")] + ".o" ofiles = append(ofiles, ofile) - sfile = mkAbs(p.Dir, sfile) + sfile, _ = fsys.OverlayPath(mkAbs(p.Dir, sfile)) defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} if pkgpath := tools.gccgoCleanPkgpath(b, p); pkgpath != "" { defs = append(defs, `-D`, `GOPKGPATH=`+pkgpath) @@ -246,7 +270,7 @@ func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string } readCgoFlags := func(flagsFile string) error { - flags, err := ioutil.ReadFile(flagsFile) + flags, err := os.ReadFile(flagsFile) if err != nil { return err } diff --git a/libgo/go/cmd/go/internal/work/init.go b/libgo/go/cmd/go/internal/work/init.go index c168364..ba7c7c2 100644 --- a/libgo/go/cmd/go/internal/work/init.go +++ b/libgo/go/cmd/go/internal/work/init.go @@ -9,7 +9,8 @@ package work import ( "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/load" + "cmd/go/internal/fsys" + "cmd/go/internal/modload" "cmd/internal/objabi" "cmd/internal/sys" "flag" @@ -21,9 +22,12 @@ import ( ) func BuildInit() { - load.ModInit() + modload.Init() instrumentInit() buildModeInit() + if err := fsys.Init(base.Cwd); err != nil { + base.Fatalf("go: %v", err) + } // Make sure -pkgdir is absolute, because we run commands // in different directories. @@ -37,6 +41,13 @@ func BuildInit() { cfg.BuildPkgdir = p } + // Make sure CC and CXX are absolute paths + for _, key := range []string{"CC", "CXX"} { + if path := cfg.Getenv(key); !filepath.IsAbs(path) && path != "" && path != filepath.Base(path) { + base.Fatalf("go %s: %s environment variable is relative; must be absolute path: %s\n", flag.Args()[0], key, path) + } + } + // For each experiment that has been enabled in the toolchain, define a // build tag with the same name but prefixed by "goexperiment." which can be // used for compiling alternative files for the experiment. This allows @@ -68,7 +79,7 @@ func instrumentInit() { } if cfg.BuildRace { if !sys.RaceDetectorSupported(cfg.Goos, cfg.Goarch) { - fmt.Fprintf(os.Stderr, "go %s: -race is only supported on linux/amd64, linux/ppc64le, linux/arm64, freebsd/amd64, netbsd/amd64, darwin/amd64 and windows/amd64\n", flag.Args()[0]) + fmt.Fprintf(os.Stderr, "go %s: -race is only supported on linux/amd64, linux/ppc64le, linux/arm64, freebsd/amd64, netbsd/amd64, darwin/amd64, darwin/arm64, and windows/amd64\n", flag.Args()[0]) base.SetExitStatus(2) base.Exit() } @@ -121,7 +132,7 @@ func buildModeInit() { codegenArg = "-fPIC" } else { switch cfg.Goos { - case "darwin": + case "darwin", "ios": switch cfg.Goarch { case "arm64": codegenArg = "-shared" @@ -157,6 +168,9 @@ func buildModeInit() { ldBuildmode = "pie" case "windows": ldBuildmode = "pie" + case "ios": + codegenArg = "-shared" + ldBuildmode = "pie" case "darwin": switch cfg.Goarch { case "arm64": @@ -227,7 +241,8 @@ func buildModeInit() { if gccgo { codegenArg = "-fPIC" } else { - forcedAsmflags = append(forcedAsmflags, "-D=GOBUILDMODE_shared=1") + forcedAsmflags = append(forcedAsmflags, "-D=GOBUILDMODE_shared=1", + "-linkshared") codegenArg = "-dynlink" forcedGcflags = append(forcedGcflags, "-linkshared") // TODO(mwhudson): remove -w when that gets fixed in linker. @@ -252,7 +267,7 @@ func buildModeInit() { switch cfg.BuildMod { case "": - // ok + // Behavior will be determined automatically, as if no flag were passed. case "readonly", "vendor", "mod": if !cfg.ModulesEnabled && !base.InGOFLAGS("-mod") { base.Fatalf("build flag -mod=%s only valid when using modules", cfg.BuildMod) diff --git a/libgo/go/cmd/go/internal/work/security.go b/libgo/go/cmd/go/internal/work/security.go index 0d96282..36bbab3 100644 --- a/libgo/go/cmd/go/internal/work/security.go +++ b/libgo/go/cmd/go/internal/work/security.go @@ -132,6 +132,7 @@ var validCompilerFlagsWithNextArg = []string{ "-U", "-I", "-framework", + "-include", "-isysroot", "-isystem", "--sysroot", @@ -177,7 +178,8 @@ var validLinkerFlags = []*lazyregexp.Regexp{ re(`-Wl,-Bdynamic`), re(`-Wl,-berok`), re(`-Wl,-Bstatic`), - re(`-WL,-O([^@,\-][^,]*)?`), + re(`-Wl,-Bsymbolic-functions`), + re(`-Wl,-O([^@,\-][^,]*)?`), re(`-Wl,-d[ny]`), re(`-Wl,--disable-new-dtags`), re(`-Wl,-e[=,][a-zA-Z0-9]*`), diff --git a/libgo/go/cmd/go/internal/work/security_test.go b/libgo/go/cmd/go/internal/work/security_test.go index aec9789..4f2e0eb 100644 --- a/libgo/go/cmd/go/internal/work/security_test.go +++ b/libgo/go/cmd/go/internal/work/security_test.go @@ -65,6 +65,8 @@ var goodCompilerFlags = [][]string{ {"-I", "=/usr/include/libxml2"}, {"-I", "dir"}, {"-I", "$SYSROOT/dir"}, + {"-isystem", "/usr/include/mozjs-68"}, + {"-include", "/usr/include/mozjs-68/RequiredDefines.h"}, {"-framework", "Chocolate"}, {"-x", "c"}, {"-v"}, @@ -96,6 +98,7 @@ var badCompilerFlags = [][]string{ {"-I", "@foo"}, {"-I", "-foo"}, {"-I", "=@obj"}, + {"-include", "@foo"}, {"-framework", "-Caffeine"}, {"-framework", "@Home"}, {"-x", "--c"}, diff --git a/libgo/go/cmd/go/main.go b/libgo/go/cmd/go/main.go index 11913d6..c4af195 100644 --- a/libgo/go/cmd/go/main.go +++ b/libgo/go/cmd/go/main.go @@ -7,6 +7,7 @@ package main import ( + "context" "flag" "fmt" "log" @@ -34,6 +35,7 @@ import ( "cmd/go/internal/run" "cmd/go/internal/test" "cmd/go/internal/tool" + "cmd/go/internal/trace" "cmd/go/internal/version" "cmd/go/internal/vet" "cmd/go/internal/work" @@ -73,10 +75,11 @@ func init() { modload.HelpModules, modget.HelpModuleGet, modfetch.HelpModuleAuth, - modfetch.HelpModulePrivate, help.HelpPackages, + modfetch.HelpPrivate, test.HelpTestflag, test.HelpTestfunc, + modget.HelpVCS, } } @@ -190,7 +193,10 @@ BigCmdLoop: cmd.Flag.Parse(args[1:]) args = cmd.Flag.Args() } - cmd.Run(cmd, args) + ctx := maybeStartTrace(context.Background()) + ctx, span := trace.StartSpan(ctx, fmt.Sprint("Running ", cmd.Name(), " command")) + cmd.Run(ctx, cmd, args) + span.Done() base.Exit() return } @@ -212,3 +218,21 @@ func mainUsage() { help.PrintUsage(os.Stderr, base.Go) os.Exit(2) } + +func maybeStartTrace(pctx context.Context) context.Context { + if cfg.DebugTrace == "" { + return pctx + } + + ctx, close, err := trace.Start(pctx, cfg.DebugTrace) + if err != nil { + base.Fatalf("failed to start trace: %v", err) + } + base.AtExit(func() { + if err := close(); err != nil { + base.Fatalf("failed to stop trace: %v", err) + } + }) + + return ctx +} diff --git a/libgo/go/cmd/go/proxy_test.go b/libgo/go/cmd/go/proxy_test.go index 2a4d293..e390c73 100644 --- a/libgo/go/cmd/go/proxy_test.go +++ b/libgo/go/cmd/go/proxy_test.go @@ -12,7 +12,7 @@ import ( "flag" "fmt" "io" - "io/ioutil" + "io/fs" "log" "net" "net/http" @@ -74,12 +74,12 @@ func StartProxy() { var modList []module.Version func readModList() { - infos, err := ioutil.ReadDir("testdata/mod") + files, err := os.ReadDir("testdata/mod") if err != nil { log.Fatal(err) } - for _, info := range infos { - name := info.Name() + for _, f := range files { + name := f.Name() if !strings.HasSuffix(name, ".txt") { continue } @@ -91,7 +91,7 @@ func readModList() { encPath := strings.ReplaceAll(name[:i], "_", "/") path, err := module.UnescapePath(encPath) if err != nil { - if encPath != "example.com/invalidpath/v1" { + if testing.Verbose() && encPath != "example.com/invalidpath/v1" { fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err) } continue @@ -131,11 +131,10 @@ func proxyHandler(w http.ResponseWriter, r *http.Request) { } path := r.URL.Path[len("/mod/"):] - // /mod/quiet/ does not print errors. - quiet := false - if strings.HasPrefix(path, "quiet/") { - path = path[len("quiet/"):] - quiet = true + // /mod/invalid returns faulty responses. + if strings.HasPrefix(path, "invalid/") { + w.Write([]byte("invalid")) + return } // Next element may opt into special behavior. @@ -214,7 +213,7 @@ func proxyHandler(w http.ResponseWriter, r *http.Request) { enc := path[:i] modPath, err := module.UnescapePath(enc) if err != nil { - if !quiet { + if testing.Verbose() { fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err) } http.NotFound(w, r) @@ -267,7 +266,7 @@ func proxyHandler(w http.ResponseWriter, r *http.Request) { enc, file := path[:i], path[i+len("/@v/"):] path, err := module.UnescapePath(enc) if err != nil { - if !quiet { + if testing.Verbose() { fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err) } http.NotFound(w, r) @@ -333,10 +332,10 @@ func proxyHandler(w http.ResponseWriter, r *http.Request) { a, err := readArchive(path, vers) if err != nil { - if !quiet { + if testing.Verbose() { fmt.Fprintf(os.Stderr, "go proxy: no archive %s %s: %v\n", path, vers, err) } - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { http.NotFound(w, r) } else { http.Error(w, "cannot load archive", 500) @@ -387,7 +386,7 @@ func proxyHandler(w http.ResponseWriter, r *http.Request) { }).(cached) if c.err != nil { - if !quiet { + if testing.Verbose() { fmt.Fprintf(os.Stderr, "go proxy: %v\n", c.err) } http.Error(w, c.err.Error(), 500) @@ -444,7 +443,7 @@ func readArchive(path, vers string) (*txtar.Archive, error) { return a }).(*txtar.Archive) if a == nil { - return nil, os.ErrNotExist + return nil, fs.ErrNotExist } return a, nil } @@ -471,13 +470,13 @@ func proxyGoSum(path, vers string) ([]byte, error) { } h1, err := dirhash.Hash1(names, func(name string) (io.ReadCloser, error) { data := files[name] - return ioutil.NopCloser(bytes.NewReader(data)), nil + return io.NopCloser(bytes.NewReader(data)), nil }) if err != nil { return nil, err } h1mod, err := dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) { - return ioutil.NopCloser(bytes.NewReader(gomod)), nil + return io.NopCloser(bytes.NewReader(gomod)), nil }) if err != nil { return nil, err diff --git a/libgo/go/cmd/go/script_test.go b/libgo/go/cmd/go/script_test.go index 2e8f18a..dfaa405 100644 --- a/libgo/go/cmd/go/script_test.go +++ b/libgo/go/cmd/go/script_test.go @@ -14,7 +14,7 @@ import ( "fmt" "go/build" "internal/testenv" - "io/ioutil" + "io/fs" "os" "os/exec" "path/filepath" @@ -22,6 +22,7 @@ import ( "runtime" "strconv" "strings" + "sync" "testing" "time" @@ -38,9 +39,7 @@ import ( // TestScript runs the tests in testdata/script/*.txt. func TestScript(t *testing.T) { testenv.MustHaveGoBuild(t) - if skipExternal { - t.Skipf("skipping external tests on %s/%s", runtime.GOOS, runtime.GOARCH) - } + testenv.SkipIfShortAndSlow(t) files, err := filepath.Glob("testdata/script/*.txt") if err != nil { @@ -135,12 +134,16 @@ func (ts *testScript) setup() { "GOSUMDB=" + testSumDBVerifierKey, "GONOPROXY=", "GONOSUMDB=", + "GOVCS=*:all", "PWD=" + ts.cd, tempEnvName() + "=" + filepath.Join(ts.workdir, "tmp"), "devnull=" + os.DevNull, "goversion=" + goVersion(ts), ":=" + string(os.PathListSeparator), } + if !testenv.HasExternalNetwork() { + ts.env = append(ts.env, "TESTGONETWORK=panic", "TESTGOVCS=panic") + } if runtime.GOOS == "plan9" { ts.env = append(ts.env, "path="+testBin+string(filepath.ListSeparator)+os.Getenv("path")) @@ -216,7 +219,7 @@ func (ts *testScript) run() { for _, f := range a.Files { name := ts.mkabs(ts.expand(f.Name, false)) ts.check(os.MkdirAll(filepath.Dir(name), 0777)) - ts.check(ioutil.WriteFile(name, f.Data, 0666)) + ts.check(os.WriteFile(name, f.Data, 0666)) } // With -v or -testwork, start log with full environment. @@ -296,6 +299,8 @@ Script: ok = os.Geteuid() == 0 case "symlink": ok = testenv.HasSymlink() + case "case-sensitive": + ok = isCaseSensitive(ts.t) default: if strings.HasPrefix(cond.tag, "exec:") { prog := cond.tag[len("exec:"):] @@ -364,6 +369,41 @@ Script: } } +var ( + onceCaseSensitive sync.Once + caseSensitive bool +) + +func isCaseSensitive(t *testing.T) bool { + onceCaseSensitive.Do(func() { + tmpdir, err := os.MkdirTemp("", "case-sensitive") + if err != nil { + t.Fatal("failed to create directory to determine case-sensitivity:", err) + } + defer os.RemoveAll(tmpdir) + + fcap := filepath.Join(tmpdir, "FILE") + if err := os.WriteFile(fcap, []byte{}, 0644); err != nil { + t.Fatal("error writing file to determine case-sensitivity:", err) + } + + flow := filepath.Join(tmpdir, "file") + _, err = os.ReadFile(flow) + switch { + case err == nil: + caseSensitive = false + return + case os.IsNotExist(err): + caseSensitive = true + return + default: + t.Fatal("unexpected error reading file when determining case-sensitivity:", err) + } + }) + + return caseSensitive +} + // scriptCmds are the script command implementations. // Keep list and the implementations below sorted by name. // @@ -409,9 +449,9 @@ func (ts *testScript) cmdAddcrlf(want simpleStatus, args []string) { for _, file := range args { file = ts.mkabs(file) - data, err := ioutil.ReadFile(file) + data, err := os.ReadFile(file) ts.check(err) - ts.check(ioutil.WriteFile(file, bytes.ReplaceAll(data, []byte("\n"), []byte("\r\n")), 0666)) + ts.check(os.WriteFile(file, bytes.ReplaceAll(data, []byte("\n"), []byte("\r\n")), 0666)) } } @@ -462,7 +502,7 @@ func (ts *testScript) cmdChmod(want simpleStatus, args []string) { ts.fatalf("usage: chmod perm paths...") } perm, err := strconv.ParseUint(args[0], 0, 32) - if err != nil || perm&uint64(os.ModePerm) != perm { + if err != nil || perm&uint64(fs.ModePerm) != perm { ts.fatalf("invalid mode: %s", args[0]) } for _, arg := range args[1:] { @@ -470,7 +510,7 @@ func (ts *testScript) cmdChmod(want simpleStatus, args []string) { if !filepath.IsAbs(path) { path = filepath.Join(ts.cd, arg) } - err := os.Chmod(path, os.FileMode(perm)) + err := os.Chmod(path, fs.FileMode(perm)) ts.check(err) } } @@ -516,12 +556,12 @@ func (ts *testScript) doCmdCmp(args []string, env, quiet bool) { } else if name1 == "stderr" { text1 = ts.stderr } else { - data, err := ioutil.ReadFile(ts.mkabs(name1)) + data, err := os.ReadFile(ts.mkabs(name1)) ts.check(err) text1 = string(data) } - data, err := ioutil.ReadFile(ts.mkabs(name2)) + data, err := os.ReadFile(ts.mkabs(name2)) ts.check(err) text2 = string(data) @@ -557,7 +597,7 @@ func (ts *testScript) cmdCp(want simpleStatus, args []string) { var ( src string data []byte - mode os.FileMode + mode fs.FileMode ) switch arg { case "stdout": @@ -573,14 +613,14 @@ func (ts *testScript) cmdCp(want simpleStatus, args []string) { info, err := os.Stat(src) ts.check(err) mode = info.Mode() & 0777 - data, err = ioutil.ReadFile(src) + data, err = os.ReadFile(src) ts.check(err) } targ := dst if dstDir { targ = filepath.Join(dst, filepath.Base(src)) } - err := ioutil.WriteFile(targ, data, mode) + err := os.WriteFile(targ, data, mode) switch want { case failure: if err == nil { @@ -856,7 +896,7 @@ func scriptMatch(ts *testScript, want simpleStatus, args []string, text, name st isGrep := name == "grep" if isGrep { name = args[1] // for error messages - data, err := ioutil.ReadFile(ts.mkabs(args[1])) + data, err := os.ReadFile(ts.mkabs(args[1])) ts.check(err) text = string(data) } @@ -1218,7 +1258,12 @@ func (ts *testScript) parse(line string) command { if cmd.name != "" { cmd.args = append(cmd.args, arg) - isRegexp = false // Commands take only one regexp argument, so no subsequent args are regexps. + // Commands take only one regexp argument (after the optional flags), + // so no subsequent args are regexps. Liberally assume an argument that + // starts with a '-' is a flag. + if len(arg) == 0 || arg[0] != '-' { + isRegexp = false + } return } diff --git a/libgo/go/cmd/go/testdata/addmod.go b/libgo/go/cmd/go/testdata/addmod.go index d9c3aab..58376b7 100644 --- a/libgo/go/cmd/go/testdata/addmod.go +++ b/libgo/go/cmd/go/testdata/addmod.go @@ -22,7 +22,7 @@ import ( "bytes" "flag" "fmt" - "io/ioutil" + "io/fs" "log" "os" "os/exec" @@ -57,7 +57,7 @@ func main() { log.SetFlags(0) var err error - tmpdir, err = ioutil.TempDir("", "addmod-") + tmpdir, err = os.MkdirTemp("", "addmod-") if err != nil { log.Fatal(err) } @@ -81,7 +81,7 @@ func main() { exitCode := 0 for _, arg := range flag.Args() { - if err := ioutil.WriteFile(filepath.Join(tmpdir, "go.mod"), []byte("module m\n"), 0666); err != nil { + if err := os.WriteFile(filepath.Join(tmpdir, "go.mod"), []byte("module m\n"), 0666); err != nil { fatalf("%v", err) } run(goCmd, "get", "-d", arg) @@ -97,13 +97,13 @@ func main() { continue } path, vers, dir := f[0], f[1], f[2] - mod, err := ioutil.ReadFile(filepath.Join(gopath, "pkg/mod/cache/download", path, "@v", vers+".mod")) + mod, err := os.ReadFile(filepath.Join(gopath, "pkg/mod/cache/download", path, "@v", vers+".mod")) if err != nil { log.Printf("%s: %v", arg, err) exitCode = 1 continue } - info, err := ioutil.ReadFile(filepath.Join(gopath, "pkg/mod/cache/download", path, "@v", vers+".info")) + info, err := os.ReadFile(filepath.Join(gopath, "pkg/mod/cache/download", path, "@v", vers+".info")) if err != nil { log.Printf("%s: %v", arg, err) exitCode = 1 @@ -121,13 +121,13 @@ func main() { {Name: ".info", Data: info}, } dir = filepath.Clean(dir) - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if !info.Mode().IsRegular() { + err = filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error { + if !info.Type().IsRegular() { return nil } name := info.Name() if name == "go.mod" || strings.HasSuffix(name, ".go") { - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { return err } @@ -143,7 +143,7 @@ func main() { data := txtar.Format(a) target := filepath.Join("mod", strings.ReplaceAll(path, "/", "_")+"_"+vers+".txt") - if err := ioutil.WriteFile(target, data, 0666); err != nil { + if err := os.WriteFile(target, data, 0666); err != nil { log.Printf("%s: %v", arg, err) exitCode = 1 continue diff --git a/libgo/go/cmd/go/testdata/savedir.go b/libgo/go/cmd/go/testdata/savedir.go index 48a6318..d469c31 100644 --- a/libgo/go/cmd/go/testdata/savedir.go +++ b/libgo/go/cmd/go/testdata/savedir.go @@ -17,7 +17,7 @@ package main import ( "flag" "fmt" - "io/ioutil" + "io/fs" "log" "os" "path/filepath" @@ -48,7 +48,7 @@ func main() { a := new(txtar.Archive) dir = filepath.Clean(dir) - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error { if path == dir { return nil } @@ -59,10 +59,10 @@ func main() { } return nil } - if !info.Mode().IsRegular() { + if !info.Type().IsRegular() { return nil } - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { log.Fatal(err) } diff --git a/libgo/go/cmd/go/testdata/script/README b/libgo/go/cmd/go/testdata/script/README index 76d6651..d658ceb 100644 --- a/libgo/go/cmd/go/testdata/script/README +++ b/libgo/go/cmd/go/testdata/script/README @@ -85,6 +85,7 @@ should only run when the condition is satisfied. The available conditions are: - [link] for testenv.HasLink() - [root] for os.Geteuid() == 0 - [symlink] for testenv.HasSymlink() + - [case-sensitive] for whether the file system is case-sensitive - [exec:prog] for whether prog is available for execution (found by exec.LookPath) - [GODEBUG:value] for whether value is one of the comma-separated entries in the GODEBUG variable - [buildmode:value] for whether -buildmode=value is supported diff --git a/libgo/go/cmd/go/testdata/script/build_GOTMPDIR.txt b/libgo/go/cmd/go/testdata/script/build_GOTMPDIR.txt index c93ca93..1073517 100644 --- a/libgo/go/cmd/go/testdata/script/build_GOTMPDIR.txt +++ b/libgo/go/cmd/go/testdata/script/build_GOTMPDIR.txt @@ -1,15 +1,50 @@ -env GO111MODULE=off -[short] skip - # Set GOCACHE to a clean directory to ensure that 'go build' has work to report. -env GOCACHE=$WORK/gocache +[!windows] env GOCACHE=$WORK/gocache +[windows] env GOCACHE=$WORK\gocache -# Build should use GOTMPDIR if set. -env GOTMPDIR=$WORK/my-favorite-tmpdir +# 'go build' should use GOTMPDIR if set. +[!windows] env GOTMPDIR=$WORK/my-favorite-tmpdir +[windows] env GOTMPDIR=$WORK\my-favorite-tmpdir mkdir $GOTMPDIR -go build -work hello.go +go build -x hello.go stderr ^WORK=.*my-favorite-tmpdir +# Make GOTMPDIR a regular file. This prevents the creation of work directories, +# so we can check that certain commands don't create them. +# This simulates running on a full disk or a read-only volume. +rm $GOTMPDIR +cp hello.go $GOTMPDIR # any file will do + +# 'go build' should fail if GOTMPDIR is read-only. +! go build -x . +stderr '^go: creating work dir: \w+ '$GOTMPDIR + +# 'go list' should only fail if it needs to build something. +go list -x . +! stderr 'creating work dir' +stdout m +go list -m all +stdout m +! go list -x -export . +stderr '^go: creating work dir: \w+ '$GOTMPDIR + +# 'go clean -cache' and 'go clean -modcache' should not fail. +go clean -x -cache +! stderr 'creating work dir' +go clean -x -modcache +! stderr 'creating work dir' + +# 'go env' should not fail for specific variables. +# Without arguments, it needs to initialize a builder to load cgo flags, and +# that uses a temporary directory. +! go env +stderr '^go: creating work dir: \w+ '$GOTMPDIR +go env GOROOT + +-- go.mod -- +module m + +go 1.15 -- hello.go -- package main func main() { println("hello") } diff --git a/libgo/go/cmd/go/testdata/script/build_trimpath.txt b/libgo/go/cmd/go/testdata/script/build_trimpath.txt index ad78bcf..e1ea0a4 100644 --- a/libgo/go/cmd/go/testdata/script/build_trimpath.txt +++ b/libgo/go/cmd/go/testdata/script/build_trimpath.txt @@ -9,6 +9,8 @@ env GO111MODULE=on mkdir $WORK/a/src/paths $WORK/b/src/paths cp paths.go $WORK/a/src/paths cp paths.go $WORK/b/src/paths +cp overlay.json $WORK/a/src/paths +cp overlay.json $WORK/b/src/paths cp go.mod $WORK/a/src/paths/ cp go.mod $WORK/b/src/paths/ @@ -43,6 +45,29 @@ go build -trimpath -o $WORK/paths-b.exe cmp -q $WORK/paths-a.exe $WORK/paths-b.exe +# Same sequence of tests but with overlays. +# A binary built without -trimpath should contain the module root dir +# and GOROOT for debugging and stack traces. +cd $WORK/a/src/paths +go build -overlay overlay.json -o $WORK/paths-dbg.exe ./overlaydir +exec $WORK/paths-dbg.exe $WORK/paths-dbg.exe +stdout 'binary contains module root: true' +stdout 'binary contains GOROOT: true' + +# A binary built with -trimpath should not contain the current workspace +# or GOROOT. +go build -overlay overlay.json -trimpath -o $WORK/paths-a.exe ./overlaydir +exec $WORK/paths-a.exe $WORK/paths-a.exe +stdout 'binary contains module root: false' +stdout 'binary contains GOROOT: false' + +# Two binaries built from identical packages in different directories +# should be identical. +cd $WORK/b/src/paths +go build -overlay overlay.json -trimpath -o $WORK/paths-b.exe ./overlaydir +cmp -q $WORK/paths-a.exe $WORK/paths-b.exe + + # Same sequence of tests but in GOPATH mode. # A binary built without -trimpath should contain GOPATH and GOROOT. env GO111MODULE=off @@ -96,7 +121,6 @@ package main import ( "bytes" "fmt" - "io/ioutil" "log" "os" "os/exec" @@ -106,7 +130,7 @@ import ( func main() { exe := os.Args[1] - data, err := ioutil.ReadFile(exe) + data, err := os.ReadFile(exe) if err != nil { log.Fatal(err) } @@ -129,7 +153,8 @@ func check(data []byte, desc, dir string) { containsSlashDir := bytes.Contains(data, []byte(filepath.ToSlash(dir))) fmt.Printf("binary contains %s: %v\n", desc, containsDir || containsSlashDir) } - +-- overlay.json -- +{ "Replace": { "overlaydir/paths.go": "paths.go" } } -- go.mod -- module paths diff --git a/libgo/go/cmd/go/testdata/script/env_write.txt b/libgo/go/cmd/go/testdata/script/env_write.txt index 1cc0846..dc69652 100644 --- a/libgo/go/cmd/go/testdata/script/env_write.txt +++ b/libgo/go/cmd/go/testdata/script/env_write.txt @@ -24,6 +24,16 @@ stdout GOARCH= stdout GOOS= stdout GOROOT= +# go env ignores invalid flag in GOFLAGS environment variable +env GOFLAGS='=true' +go env + +# checking errors +! go env -w +stderr 'go env -w: no KEY=VALUE arguments given' +! go env -u +stderr 'go env -u: no arguments given' + # go env -w changes default setting env root= [windows] env root=c: @@ -59,6 +69,8 @@ go env -u GOPATH stderr 'unknown go command variable GODEBUG' ! go env -w GOEXE=.bat stderr 'GOEXE cannot be modified' +! go env -w GOVERSION=customversion +stderr 'GOVERSION cannot be modified' ! go env -w GOENV=/env stderr 'GOENV can only be set using the OS environment' @@ -97,6 +109,50 @@ stderr 'GOPATH entry cannot start with shell metacharacter' ! go env -w GOPATH=./go stderr 'GOPATH entry is relative; must be absolute path' +# go env -w rejects invalid GOTMPDIR values +! go env -w GOTMPDIR=x +stderr 'go env -w: GOTMPDIR must be an absolute path' + +# go env -w should accept absolute GOTMPDIR value +# and should not create it +[windows] go env -w GOTMPDIR=$WORK\x\y\z +[!windows] go env -w GOTMPDIR=$WORK/x/y/z +! exists $WORK/x/y/z +# we should be able to clear an env +go env -u GOTMPDIR +go env GOTMPDIR +stdout ^$ + +[windows] go env -w GOTMPDIR=$WORK\x\y\z +[!windows] go env -w GOTMPDIR=$WORK/x/y/z +go env -w GOTMPDIR= +go env GOTMPDIR +stdout ^$ + +# go env -w rejects relative CC values +[!windows] go env -w CC=/usr/bin/clang +go env -w CC=clang +[!windows] ! go env -w CC=./clang +[!windows] ! go env -w CC=bin/clang +[!windows] stderr 'go env -w: CC entry is relative; must be absolute path' + +[windows] go env -w CC=$WORK\bin\clang +[windows] ! go env -w CC=.\clang +[windows] ! go env -w CC=bin\clang +[windows] stderr 'go env -w: CC entry is relative; must be absolute path' + +# go env -w rejects relative CXX values +[!windows] go env -w CC=/usr/bin/cpp +go env -w CXX=cpp +[!windows] ! go env -w CXX=./cpp +[!windows] ! go env -w CXX=bin/cpp +[!windows] stderr 'go env -w: CXX entry is relative; must be absolute path' + +[windows] go env -w CXX=$WORK\bin\cpp +[windows] ! go env -w CXX=.\cpp +[windows] ! go env -w CXX=bin\cpp +[windows] stderr 'go env -w: CXX entry is relative; must be absolute path' + # go env -w/-u checks validity of GOOS/ARCH combinations [gccgo] skip env GOOS= diff --git a/libgo/go/cmd/go/testdata/script/gcflags_patterns.txt b/libgo/go/cmd/go/testdata/script/gcflags_patterns.txt index 5374493..f23cece 100644 --- a/libgo/go/cmd/go/testdata/script/gcflags_patterns.txt +++ b/libgo/go/cmd/go/testdata/script/gcflags_patterns.txt @@ -63,6 +63,10 @@ stderr 'link.* -X=math.pi=3' go build -n -ldflags=-X=math.pi=3 stderr 'link.* -X=math.pi=3' +# cgo.a should not be a dependency of internally-linked go package +go build -ldflags='-linkmode=external -linkmode=internal' -n prog.go +! stderr 'packagefile .*runtime/cgo.a' + -- z1/z.go -- package z1 import _ "y" diff --git a/libgo/go/cmd/go/testdata/script/get_unicode.txt b/libgo/go/cmd/go/testdata/script/get_unicode.txt deleted file mode 100644 index d3b82bd..0000000 --- a/libgo/go/cmd/go/testdata/script/get_unicode.txt +++ /dev/null @@ -1,40 +0,0 @@ -env GO111MODULE=off - -[!exec:git] skip -[short] skip - -# Construct a repository that imports a non-ASCII path. -cd $WORK/_origin/example.com/unicode -exec git init -exec git config user.name 'Nameless Gopher' -exec git config user.email 'nobody@golang.org' -exec git add unicode.go -exec git commit -m 'add unicode.go' - -# Clone the repo into GOPATH so that 'go get -u' can find it. -mkdir $GOPATH/src/example.com/unicode -cd $GOPATH/src/example.com/unicode -exec git clone $WORK/_origin/example.com/unicode . - -# Construct the imported repository. -cd $WORK/_origin/example.com/испытание -exec git init -exec git config user.name 'Nameless Gopher' -exec git config user.email 'nobody@golang.org' -exec git add испытание.go -exec git commit -m 'add испытание.go' - -# Clone that repo into GOPATH too. -mkdir $GOPATH/src/example.com/испытание -cd $GOPATH/src/example.com/испытание -exec git clone $WORK/_origin/example.com/испытание . - -# Upgrading the importer should pull from the non-ASCII repo. -cd $GOPATH -go get -u example.com/unicode - --- $WORK/_origin/example.com/unicode/unicode.go -- -package unicode -import _ "example.com/испытание" --- $WORK/_origin/example.com/испытание/испытание.go -- -package испытание diff --git a/libgo/go/cmd/go/testdata/script/gopath_moved_repo.txt b/libgo/go/cmd/go/testdata/script/gopath_moved_repo.txt index 869980d..99d80bf 100644 --- a/libgo/go/cmd/go/testdata/script/gopath_moved_repo.txt +++ b/libgo/go/cmd/go/testdata/script/gopath_moved_repo.txt @@ -45,7 +45,6 @@ package main import ( "bytes" - "io/ioutil" "log" "os" ) @@ -57,11 +56,11 @@ func main() { base := []byte(os.Args[1]) path := os.Args[2] - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { log.Fatal(err) } - err = ioutil.WriteFile(path, bytes.ReplaceAll(data, base, append(base, "XXX"...)), 0644) + err = os.WriteFile(path, bytes.ReplaceAll(data, base, append(base, "XXX"...)), 0644) if err != nil { log.Fatal(err) } diff --git a/libgo/go/cmd/go/testdata/script/link_syso_issue33139.txt b/libgo/go/cmd/go/testdata/script/link_syso_issue33139.txt index 3030ee9..26034c9 100644 --- a/libgo/go/cmd/go/testdata/script/link_syso_issue33139.txt +++ b/libgo/go/cmd/go/testdata/script/link_syso_issue33139.txt @@ -8,14 +8,13 @@ # See: https://github.com/golang/go/issues/8912 [linux] [ppc64] skip -# External linking is not supported on linux/riscv, linux/riscv64. -# See: https://github.com/golang/go/issues/36739 -[linux] [riscv] skip -[linux] [riscv64] skip - cc -c -o syso/objTestImpl.syso syso/src/objTestImpl.c go build -ldflags='-linkmode=external' ./cmd/main.go +-- go.mod -- +module m + +go 1.16 -- syso/objTest.s -- #include "textflag.h" @@ -37,7 +36,7 @@ void objTestImpl() { /* Empty */ } -- cmd/main.go -- package main -import "syso" +import "m/syso" func main() { syso.ObjTest() diff --git a/libgo/go/cmd/go/testdata/script/list_bad_import.txt b/libgo/go/cmd/go/testdata/script/list_bad_import.txt index b8f9d58..dbec350 100644 --- a/libgo/go/cmd/go/testdata/script/list_bad_import.txt +++ b/libgo/go/cmd/go/testdata/script/list_bad_import.txt @@ -15,10 +15,11 @@ stdout 'incomplete' stdout 'bad dep: .*example.com[/\\]notfound' # Listing with -deps should also fail. -# BUG: Today, it does not. -# ! go list -deps example.com/direct -# stderr example.com[/\\]notfound -go list -deps example.com/direct +! go list -deps example.com/direct +stderr example.com[/\\]notfound + +# But -e -deps should succeed. +go list -e -deps example.com/direct stdout example.com/notfound @@ -31,10 +32,11 @@ stdout incomplete stdout 'bad dep: .*example.com[/\\]notfound' # Again, -deps should fail. -# BUG: Again, it does not. -# ! go list -deps example.com/indirect -# stderr example.com[/\\]notfound -go list -deps example.com/indirect +! go list -deps example.com/indirect +stderr example.com[/\\]notfound + +# But -deps -e should succeed. +go list -e -deps example.com/indirect stdout example.com/notfound diff --git a/libgo/go/cmd/go/testdata/script/list_test_err.txt b/libgo/go/cmd/go/testdata/script/list_test_err.txt index a174b5e..c6f1ecf4 100644 --- a/libgo/go/cmd/go/testdata/script/list_test_err.txt +++ b/libgo/go/cmd/go/testdata/script/list_test_err.txt @@ -22,6 +22,9 @@ go list -e -test -deps -f '{{.ImportPath}} {{.Error | printf "%q"}}' syntaxerr stdout 'pkgdep <nil>' stdout 'testdep_a <nil>' stdout 'testdep_b <nil>' +stdout 'syntaxerr <nil>' +stdout 'syntaxerr \[syntaxerr.test\] <nil>' +stdout 'syntaxerr_test \[syntaxerr.test\] <nil>' stdout 'syntaxerr\.test "[^"]*expected declaration' ! stderr 'expected declaration' diff --git a/libgo/go/cmd/go/testdata/script/mod_auth.txt b/libgo/go/cmd/go/testdata/script/mod_auth.txt index 5bcbcd1..d8ea586 100644 --- a/libgo/go/cmd/go/testdata/script/mod_auth.txt +++ b/libgo/go/cmd/go/testdata/script/mod_auth.txt @@ -3,11 +3,12 @@ env GO111MODULE=on env GOPROXY=direct env GOSUMDB=off +env GOVCS='*:off' # Without credentials, downloading a module from a path that requires HTTPS # basic auth should fail. env NETRC=$WORK/empty -! go list all +! go mod tidy stderr '^\tserver response: ACCESS DENIED, buddy$' stderr '^\tserver response: File\? What file\?$' diff --git a/libgo/go/cmd/go/testdata/script/mod_bad_domain.txt b/libgo/go/cmd/go/testdata/script/mod_bad_domain.txt index ec0d474..20199c1 100644 --- a/libgo/go/cmd/go/testdata/script/mod_bad_domain.txt +++ b/libgo/go/cmd/go/testdata/script/mod_bad_domain.txt @@ -2,22 +2,34 @@ env GO111MODULE=on # explicit get should report errors about bad names ! go get appengine -stderr '^go get appengine: package appengine is not in GOROOT \(.*\)$' +stderr '^go get: malformed module path "appengine": missing dot in first path element$' ! go get x/y.z stderr 'malformed module path "x/y.z": missing dot in first path element' + # 'go list -m' should report errors about module names, never GOROOT. ! go list -m -versions appengine stderr 'malformed module path "appengine": missing dot in first path element' ! go list -m -versions x/y.z stderr 'malformed module path "x/y.z": missing dot in first path element' + # build should report all unsatisfied imports, # but should be more definitive about non-module import paths ! go build ./useappengine -stderr 'cannot find package' +stderr '^useappengine[/\\]x.go:2:8: cannot find package$' ! go build ./usenonexistent -stderr 'cannot find module providing package nonexistent.rsc.io' +stderr '^usenonexistent[/\\]x.go:2:8: no required module provides package nonexistent.rsc.io; try ''go mod tidy'' to add it$' + + +# 'get -d' should be similarly definitive + +go get -d ./useappengine # TODO(#41315): This should fail. + # stderr '^useappengine[/\\]x.go:2:8: cannot find package$' + +! go get -d ./usenonexistent +stderr '^x/usenonexistent imports\n\tnonexistent.rsc.io: cannot find module providing package nonexistent.rsc.io$' + # go mod vendor and go mod tidy should ignore appengine imports. rm usenonexistent/x.go diff --git a/libgo/go/cmd/go/testdata/script/mod_bad_filenames.txt b/libgo/go/cmd/go/testdata/script/mod_bad_filenames.txt index 6e0c8bd..eb556f4 100644 --- a/libgo/go/cmd/go/testdata/script/mod_bad_filenames.txt +++ b/libgo/go/cmd/go/testdata/script/mod_bad_filenames.txt @@ -3,9 +3,9 @@ env GO111MODULE=on ! go get rsc.io/badfile1 rsc.io/badfile2 rsc.io/badfile3 rsc.io/badfile4 rsc.io/badfile5 ! stderr 'unzip.*badfile1' stderr 'unzip.*badfile2[\\/]@v[\\/]v1.0.0.zip:.*malformed file path "☺.go": invalid char ''☺''' -stderr 'unzip.*badfile3[\\/]@v[\\/]v1.0.0.zip: malformed file path "x\?y.go": invalid char ''\?''' -stderr 'unzip.*badfile4[\\/]@v[\\/]v1.0.0.zip: case-insensitive file name collision: "x/Y.go" and "x/y.go"' -stderr 'unzip.*badfile5[\\/]@v[\\/]v1.0.0.zip: case-insensitive file name collision: "x/y" and "x/Y"' +stderr 'unzip.*badfile3[\\/]@v[\\/]v1.0.0.zip: rsc.io[\\/]badfile3@v1.0.0[\\/]x\?y.go: malformed file path "x\?y.go": invalid char ''\?''' +stderr 'unzip.*badfile4[\\/]@v[\\/]v1.0.0.zip: rsc.io[\\/]badfile4@v1.0.0[\\/]x[\\/]y.go: case-insensitive file name collision: "x/Y.go" and "x/y.go"' +stderr 'unzip.*badfile5[\\/]@v[\\/]v1.0.0.zip: rsc.io[\\/]badfile5@v1.0.0[\\/]x[\\/]Y[\\/]zz[\\/]ww.go: case-insensitive file name collision: "x/y" and "x/Y"' -- go.mod -- module x diff --git a/libgo/go/cmd/go/testdata/script/mod_build_info_err.txt b/libgo/go/cmd/go/testdata/script/mod_build_info_err.txt index 87a099b..cee055e 100644 --- a/libgo/go/cmd/go/testdata/script/mod_build_info_err.txt +++ b/libgo/go/cmd/go/testdata/script/mod_build_info_err.txt @@ -1,8 +1,19 @@ # This test verifies that line numbers are included in module import errors. # Verifies golang.org/issue/34393. -go list -e -deps -f '{{with .Error}}{{.Pos}}: {{.Err}}{{end}}' ./main -stdout 'bad[/\\]bad.go:3:8: malformed module path "🐧.example.com/string": invalid char ''🐧''' +go list -e -mod=mod -deps -f '{{with .Error}}{{.Pos}}: {{.Err}}{{end}}' ./main +stdout '^bad[/\\]bad.go:3:8: malformed import path "🐧.example.com/string": invalid char ''🐧''$' + +# TODO(#26909): This should include an import stack. +# (Today it includes only a file and line.) +! go build ./main +stderr '^bad[/\\]bad.go:3:8: malformed import path "🐧.example.com/string": invalid char ''🐧''$' + +# TODO(#41688): This should include a file and line, and report the reason for the error.. +# (Today it includes only an import stack.) +! go get -d ./main +stderr '^m/main imports\n\tm/bad imports\n\t🐧.example.com/string: malformed import path "🐧.example.com/string": invalid char ''🐧''$' + -- go.mod -- module m diff --git a/libgo/go/cmd/go/testdata/script/mod_case.txt b/libgo/go/cmd/go/testdata/script/mod_case.txt index ee818c2..4a46986 100644 --- a/libgo/go/cmd/go/testdata/script/mod_case.txt +++ b/libgo/go/cmd/go/testdata/script/mod_case.txt @@ -1,6 +1,6 @@ env GO111MODULE=on -go get rsc.io/QUOTE +go get -d go list -m all stdout '^rsc.io/quote v1.5.2' stdout '^rsc.io/QUOTE v1.5.2' @@ -9,7 +9,7 @@ go list -f 'DIR {{.Dir}} DEPS {{.Deps}}' rsc.io/QUOTE/QUOTE stdout 'DEPS.*rsc.io/quote' stdout 'DIR.*!q!u!o!t!e' -go get rsc.io/QUOTE@v1.5.3-PRE +go get -d rsc.io/QUOTE@v1.5.3-PRE go list -m all stdout '^rsc.io/QUOTE v1.5.3-PRE' @@ -18,3 +18,8 @@ stdout '!q!u!o!t!e@v1.5.3-!p!r!e' -- go.mod -- module x + +-- use.go -- +package use + +import _ "rsc.io/QUOTE/QUOTE" diff --git a/libgo/go/cmd/go/testdata/script/mod_case_cgo.txt b/libgo/go/cmd/go/testdata/script/mod_case_cgo.txt index 917bce9..f3d6aaa 100644 --- a/libgo/go/cmd/go/testdata/script/mod_case_cgo.txt +++ b/libgo/go/cmd/go/testdata/script/mod_case_cgo.txt @@ -2,7 +2,9 @@ env GO111MODULE=on -go get rsc.io/CGO +go get -d rsc.io/CGO +[short] stop + go build rsc.io/CGO -- go.mod -- diff --git a/libgo/go/cmd/go/testdata/script/mod_concurrent.txt b/libgo/go/cmd/go/testdata/script/mod_concurrent.txt index e03e5e5..8c21525 100644 --- a/libgo/go/cmd/go/testdata/script/mod_concurrent.txt +++ b/libgo/go/cmd/go/testdata/script/mod_concurrent.txt @@ -1,6 +1,7 @@ env GO111MODULE=on # Concurrent builds should succeed, even if they need to download modules. +go get -d ./x ./y go build ./x & go build ./y wait diff --git a/libgo/go/cmd/go/testdata/script/mod_doc.txt b/libgo/go/cmd/go/testdata/script/mod_doc.txt index aac3db0..595ad67 100644 --- a/libgo/go/cmd/go/testdata/script/mod_doc.txt +++ b/libgo/go/cmd/go/testdata/script/mod_doc.txt @@ -1,6 +1,7 @@ # go doc should find module documentation env GO111MODULE=on +env GOFLAGS=-mod=mod [short] skip # Check when module x is inside GOPATH/src. @@ -48,6 +49,7 @@ stderr '^doc: cannot find module providing package example.com/hello: module loo # path used in source code, not to the absolute path relative to GOROOT. cd $GOROOT/src +env GOFLAGS= go doc cryptobyte stdout '// import "golang.org/x/crypto/cryptobyte"' diff --git a/libgo/go/cmd/go/testdata/script/mod_domain_root.txt b/libgo/go/cmd/go/testdata/script/mod_domain_root.txt index e34cc29..14745b5 100644 --- a/libgo/go/cmd/go/testdata/script/mod_domain_root.txt +++ b/libgo/go/cmd/go/testdata/script/mod_domain_root.txt @@ -2,7 +2,7 @@ # (example.com not example.com/something) env GO111MODULE=on -go build +go get -d -- go.mod -- module x diff --git a/libgo/go/cmd/go/testdata/script/mod_dot.txt b/libgo/go/cmd/go/testdata/script/mod_dot.txt index c220e99..e581320 100644 --- a/libgo/go/cmd/go/testdata/script/mod_dot.txt +++ b/libgo/go/cmd/go/testdata/script/mod_dot.txt @@ -4,8 +4,12 @@ env GO111MODULE=on # in an empty directory should refer to the path '.' and should not attempt # to resolve an external module. cd dir +! go get +stderr '^go get: no package in current directory$' ! go get . -stderr 'go get \.: path .* is not a package in module rooted at .*[/\\]dir$' +stderr '^go get \.: no package in current directory$' +! go get ./subdir +stderr '^go get: \.[/\\]subdir \('$WORK'[/\\]gopath[/\\]src[/\\]dir[/\\]subdir\) is not a package in module rooted at '$WORK'[/\\]gopath[/\\]src[/\\]dir$' ! go list ! stderr 'cannot find module providing package' stderr '^no Go files in '$WORK'[/\\]gopath[/\\]src[/\\]dir$' diff --git a/libgo/go/cmd/go/testdata/script/mod_download.txt b/libgo/go/cmd/go/testdata/script/mod_download.txt index 3573928..8a9faff 100644 --- a/libgo/go/cmd/go/testdata/script/mod_download.txt +++ b/libgo/go/cmd/go/testdata/script/mod_download.txt @@ -1,14 +1,15 @@ env GO111MODULE=on -env GOPROXY=$GOPROXY/quiet -# download with version should print nothing +# download with version should print nothing. +# It should not load retractions from the .mod file from the latest version. go mod download rsc.io/quote@v1.5.0 ! stdout . ! stderr . - exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.0.info exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.0.mod exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.0.zip +! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info +! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod # download of an invalid path should report the error [short] skip @@ -32,53 +33,59 @@ stdout '^\t"GoModSum": "h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe\+TKr0="' go list -m all ! stdout rsc.io -# add to go.mod so we can test non-query downloads -go mod edit -require rsc.io/quote@v1.5.2 -! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info -! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod -! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip - -# module loading will page in the info and mod files -go list -m all +# download query should have downloaded go.mod for the highest release version +# in order to find retractions when resolving the query '@<=v1.5.0'. exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod ! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip +# add to go.mod so we can test non-query downloads +go mod edit -require rsc.io/quote@v1.5.3-pre1 +! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.info +! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.mod +! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.zip + +# module loading will page in the info and mod files +go list -m -mod=mod all +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.info +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.mod +! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.zip + # download will fetch and unpack the zip file go mod download -exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info -exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod -exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip -exists $GOPATH/pkg/mod/rsc.io/quote@v1.5.2 +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.info +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.mod +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.zip +exists $GOPATH/pkg/mod/rsc.io/quote@v1.5.3-pre1 # download repopulates deleted files and directories independently. -rm $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info +rm $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.info go mod download -exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info -rm $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.info +rm $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.mod go mod download -exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod -rm $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.mod +rm $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.zip go mod download -exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip -rm -r $GOPATH/pkg/mod/rsc.io/quote@v1.5.2 +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.zip +rm -r $GOPATH/pkg/mod/rsc.io/quote@v1.5.3-pre1 go mod download -exists $GOPATH/pkg/mod/rsc.io/quote@v1.5.2 +exists $GOPATH/pkg/mod/rsc.io/quote@v1.5.3-pre1 # download reports the locations of downloaded files go mod download -json stdout '^\t"Path": "rsc.io/quote"' -stdout '^\t"Version": "v1.5.2"' -stdout '^\t"Info": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.2.info"' -stdout '^\t"GoMod": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.2.mod"' -stdout '^\t"Zip": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.2.zip"' -stdout '^\t"Dir": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)rsc.io(\\\\|/)quote@v1.5.2"' +stdout '^\t"Version": "v1.5.3-pre1"' +stdout '^\t"Info": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.3-pre1.info"' +stdout '^\t"GoMod": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.3-pre1.mod"' +stdout '^\t"Zip": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.3-pre1.zip"' +stdout '^\t"Dir": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)rsc.io(\\\\|/)quote@v1.5.3-pre1"' # download will follow replacements -go mod edit -require rsc.io/quote@v1.5.1 -replace rsc.io/quote@v1.5.1=rsc.io/quote@v1.5.3-pre1 +go mod edit -require rsc.io/quote@v1.5.1 -replace rsc.io/quote@v1.5.1=rsc.io/quote@v1.5.2 go mod download ! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.1.zip -exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.zip +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip # download will not follow replacements for explicit module queries go mod download -json rsc.io/quote@v1.5.1 @@ -86,19 +93,27 @@ exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.1.zip # download reports errors encountered when locating modules ! go mod download bad/path -stderr '^module bad/path: not a known dependency$' +stderr '^go mod download: module bad/path: not a known dependency$' ! go mod download bad/path@latest -stderr '^bad/path@latest: malformed module path "bad/path": missing dot in first path element$' +stderr '^go mod download: bad/path@latest: malformed module path "bad/path": missing dot in first path element$' ! go mod download rsc.io/quote@v1.999.999 -stderr '^rsc.io/quote@v1.999.999: reading .*/v1.999.999.info: 404 Not Found$' +stderr '^go mod download: rsc.io/quote@v1.999.999: reading .*/v1.999.999.info: 404 Not Found$' ! go mod download -json bad/path stdout '^\t"Error": "module bad/path: not a known dependency"' -# download main module returns an error +# download main module produces a warning or error go mod download m stderr '^go mod download: skipping argument m that resolves to the main module\n' -go mod download m@latest -stderr '^go mod download: skipping argument m@latest that resolves to the main module\n' +! go mod download m@latest +stderr '^go mod download: m@latest: malformed module path "m": missing dot in first path element$' + +# download updates go.mod and populates go.sum +cd update +! exists go.sum +go mod download +grep '^rsc.io/sampler v1.3.0 ' go.sum +go list -m rsc.io/sampler +stdout '^rsc.io/sampler v1.3.0$' # allow go mod download without go.mod env GO111MODULE=auto @@ -115,3 +130,13 @@ stderr 'get '$GOPROXY -- go.mod -- module m + +-- update/go.mod -- +module m + +go 1.16 + +require ( + rsc.io/quote v1.5.2 + rsc.io/sampler v1.2.1 // older version than in build list +) diff --git a/libgo/go/cmd/go/testdata/script/mod_download_json.txt b/libgo/go/cmd/go/testdata/script/mod_download_json.txt index 01c35dd..9555adf 100644 --- a/libgo/go/cmd/go/testdata/script/mod_download_json.txt +++ b/libgo/go/cmd/go/testdata/script/mod_download_json.txt @@ -1,10 +1,9 @@ env GO111MODULE=on -env GOPROXY=$GOPROXY/quiet env GOSUMDB=$sumdb' '$proxy/sumdb-wrong # download -json with version should print JSON on sumdb failure ! go mod download -json 'rsc.io/quote@<=v1.5.0' -stdout '"Error": ".*verifying module' +stdout '"Error": ".*verifying (module|go.mod)' -- go.mod -- module m diff --git a/libgo/go/cmd/go/testdata/script/mod_edit.txt b/libgo/go/cmd/go/testdata/script/mod_edit.txt index 898d852..78485eb 100644 --- a/libgo/go/cmd/go/testdata/script/mod_edit.txt +++ b/libgo/go/cmd/go/testdata/script/mod_edit.txt @@ -16,15 +16,19 @@ cmpenv go.mod $WORK/go.mod.init cmpenv go.mod $WORK/go.mod.init # go mod edits -go mod edit -droprequire=x.1 -require=x.1@v1.0.0 -require=x.2@v1.1.0 -droprequire=x.2 -exclude='x.1 @ v1.2.0' -exclude=x.1@v1.2.1 -replace=x.1@v1.3.0=y.1@v1.4.0 -replace='x.1@v1.4.0 = ../z' +go mod edit -droprequire=x.1 -require=x.1@v1.0.0 -require=x.2@v1.1.0 -droprequire=x.2 -exclude='x.1 @ v1.2.0' -exclude=x.1@v1.2.1 -replace=x.1@v1.3.0=y.1@v1.4.0 -replace='x.1@v1.4.0 = ../z' -retract=v1.6.0 -retract=[v1.1.0,v1.2.0] -retract=[v1.3.0,v1.4.0] -retract=v1.0.0 cmpenv go.mod $WORK/go.mod.edit1 -go mod edit -droprequire=x.1 -dropexclude=x.1@v1.2.1 -dropreplace=x.1@v1.3.0 -require=x.3@v1.99.0 +go mod edit -droprequire=x.1 -dropexclude=x.1@v1.2.1 -dropreplace=x.1@v1.3.0 -require=x.3@v1.99.0 -dropretract=v1.0.0 -dropretract=[v1.1.0,v1.2.0] cmpenv go.mod $WORK/go.mod.edit2 # go mod edit -json go mod edit -json cmpenv stdout $WORK/go.mod.json +# go mod edit -json (retractions with rationales) +go mod edit -json $WORK/go.mod.retractrationale +cmp stdout $WORK/go.mod.retractrationale.json + # go mod edit -json (empty mod file) go mod edit -json $WORK/go.mod.empty cmp stdout $WORK/go.mod.empty.json @@ -40,11 +44,11 @@ cmpenv go.mod $WORK/go.mod.edit5 # go mod edit -fmt cp $WORK/go.mod.badfmt go.mod go mod edit -fmt -print # -print should avoid writing file -cmpenv stdout $WORK/go.mod.edit6 +cmpenv stdout $WORK/go.mod.goodfmt cmp go.mod $WORK/go.mod.badfmt go mod edit -fmt # without -print, should write file (and nothing to stdout) ! stdout . -cmpenv go.mod $WORK/go.mod.edit6 +cmpenv go.mod $WORK/go.mod.goodfmt # go mod edit -module cd $WORK/m @@ -84,6 +88,13 @@ replace ( x.1 v1.3.0 => y.1 v1.4.0 x.1 v1.4.0 => ../z ) + +retract ( + v1.6.0 + [v1.3.0, v1.4.0] + [v1.1.0, v1.2.0] + v1.0.0 +) -- $WORK/go.mod.edit2 -- module x.x/y/z @@ -93,6 +104,11 @@ exclude x.1 v1.2.0 replace x.1 v1.4.0 => ../z +retract ( + v1.6.0 + [v1.3.0, v1.4.0] +) + require x.3 v1.99.0 -- $WORK/go.mod.json -- { @@ -122,6 +138,16 @@ require x.3 v1.99.0 "Path": "../z" } } + ], + "Retract": [ + { + "Low": "v1.6.0", + "High": "v1.6.0" + }, + { + "Low": "v1.3.0", + "High": "v1.4.0" + } ] } -- $WORK/go.mod.edit3 -- @@ -136,6 +162,11 @@ replace ( x.1 v1.4.0 => y.1/v2 v2.3.5 ) +retract ( + v1.6.0 + [v1.3.0, v1.4.0] +) + require x.3 v1.99.0 -- $WORK/go.mod.edit4 -- module x.x/y/z @@ -146,6 +177,11 @@ exclude x.1 v1.2.0 replace x.1 => y.1/v2 v2.3.6 +retract ( + v1.6.0 + [v1.3.0, v1.4.0] +) + require x.3 v1.99.0 -- $WORK/go.mod.edit5 -- module x.x/y/z @@ -154,15 +190,10 @@ go $goversion exclude x.1 v1.2.0 -require x.3 v1.99.0 --- $WORK/go.mod.edit6 -- -module x.x/y/z - -go 1.10 - -exclude x.1 v1.2.0 - -replace x.1 => y.1/v2 v2.3.6 +retract ( + v1.6.0 + [v1.3.0, v1.4.0] +) require x.3 v1.99.0 -- $WORK/local/go.mod.edit -- @@ -183,10 +214,64 @@ exclude x.1 v1.2.0 replace x.1 => y.1/v2 v2.3.6 require x.3 v1.99.0 + +retract [ "v1.8.1" , "v1.8.2" ] +-- $WORK/go.mod.goodfmt -- +module x.x/y/z + +go 1.10 + +exclude x.1 v1.2.0 + +replace x.1 => y.1/v2 v2.3.6 + +require x.3 v1.99.0 + +retract [v1.8.1, v1.8.2] -- $WORK/m/go.mod.edit -- module x.x/y/z go $goversion +-- $WORK/go.mod.retractrationale -- +module x.x/y/z + +go 1.15 + +// a +retract v1.0.0 + +// b +retract ( + v1.0.1 + v1.0.2 // c +) +-- $WORK/go.mod.retractrationale.json -- +{ + "Module": { + "Path": "x.x/y/z" + }, + "Go": "1.15", + "Require": null, + "Exclude": null, + "Replace": null, + "Retract": [ + { + "Low": "v1.0.0", + "High": "v1.0.0", + "Rationale": "a" + }, + { + "Low": "v1.0.1", + "High": "v1.0.1", + "Rationale": "b" + }, + { + "Low": "v1.0.2", + "High": "v1.0.2", + "Rationale": "c" + } + ] +} -- $WORK/go.mod.empty -- -- $WORK/go.mod.empty.json -- { @@ -195,5 +280,6 @@ go $goversion }, "Require": null, "Exclude": null, - "Replace": null + "Replace": null, + "Retract": null } diff --git a/libgo/go/cmd/go/testdata/script/mod_enabled.txt b/libgo/go/cmd/go/testdata/script/mod_enabled.txt index 10fa103..39f1ece 100644 --- a/libgo/go/cmd/go/testdata/script/mod_enabled.txt +++ b/libgo/go/cmd/go/testdata/script/mod_enabled.txt @@ -25,7 +25,7 @@ cd $GOPATH/foo/bar/baz go env GOMOD stdout foo[/\\]go.mod -# GO111MODULE unset should be equivalent to auto. +# GO111MODULE unset should be equivalent to on. env GO111MODULE= cd $GOPATH/src/x/y/z @@ -34,7 +34,7 @@ stdout $GOPATH[/\\]src[/\\]x[/\\]y[/\\]z[/\\]go.mod cd $GOPATH/src/x/y go env GOMOD -! stdout . +stdout 'NUL|/dev/null' # GO111MODULE=on should trigger everywhere env GO111MODULE=on diff --git a/libgo/go/cmd/go/testdata/script/mod_find.txt b/libgo/go/cmd/go/testdata/script/mod_find.txt index 7fbe9fb..9468acf 100644 --- a/libgo/go/cmd/go/testdata/script/mod_find.txt +++ b/libgo/go/cmd/go/testdata/script/mod_find.txt @@ -19,6 +19,11 @@ go mod init stderr 'module example.com/x/y$' rm go.mod +# go mod init rejects a zero-length go.mod file +cp $devnull go.mod # can't use touch to create it because Windows +! go mod init +stderr 'go.mod already exists' + # Module path from Godeps/Godeps.json overrides GOPATH. cd $GOPATH/src/example.com/x/y/z go mod init diff --git a/libgo/go/cmd/go/testdata/script/mod_get_commit.txt b/libgo/go/cmd/go/testdata/script/mod_get_commit.txt index d108242..4649491 100644 --- a/libgo/go/cmd/go/testdata/script/mod_get_commit.txt +++ b/libgo/go/cmd/go/testdata/script/mod_get_commit.txt @@ -1,5 +1,4 @@ env GO111MODULE=on -env GOPROXY=$GOPROXY/quiet [short] skip # @commit should resolve @@ -33,11 +32,11 @@ go install -x golang.org/x/text/language ! go get -d -x golang.org/x/text/foo@14c0d48 # get pseudo-version should record that version -go get rsc.io/quote@v0.0.0-20180214005840-23179ee8a569 +go get -d rsc.io/quote@v0.0.0-20180214005840-23179ee8a569 grep 'rsc.io/quote v0.0.0-20180214005840-23179ee8a569' go.mod # but as commit should record as v1.5.1 -go get rsc.io/quote@23179ee8 +go get -d rsc.io/quote@23179ee8 grep 'rsc.io/quote v1.5.1' go.mod # go mod edit -require does not interpret commits diff --git a/libgo/go/cmd/go/testdata/script/mod_get_downgrade.txt b/libgo/go/cmd/go/testdata/script/mod_get_downgrade.txt index ee9ac96..a954c10 100644 --- a/libgo/go/cmd/go/testdata/script/mod_get_downgrade.txt +++ b/libgo/go/cmd/go/testdata/script/mod_get_downgrade.txt @@ -3,30 +3,33 @@ env GO111MODULE=on # downgrade sampler should downgrade quote cp go.mod.orig go.mod -go get rsc.io/sampler@v1.0.0 +go get -d rsc.io/sampler@v1.0.0 go list -m all stdout 'rsc.io/quote v1.4.0' stdout 'rsc.io/sampler v1.0.0' # downgrade sampler away should downgrade quote further -go get rsc.io/sampler@none +go get -d rsc.io/sampler@none go list -m all stdout 'rsc.io/quote v1.3.0' # downgrade should report inconsistencies and not change go.mod -go get rsc.io/quote@v1.5.1 +go get -d rsc.io/quote@v1.5.1 go list -m all stdout 'rsc.io/quote v1.5.1' stdout 'rsc.io/sampler v1.3.0' -! go get rsc.io/sampler@v1.0.0 rsc.io/quote@v1.5.2 golang.org/x/text@none -stderr 'go get: inconsistent versions:\n\trsc.io/quote@v1.5.2 requires golang.org/x/text@v0.0.0-20170915032832-14c0d48ead0c \(not golang.org/x/text@none\), rsc.io/sampler@v1.3.0 \(not rsc.io/sampler@v1.0.0\)' + +! go get -d rsc.io/sampler@v1.0.0 rsc.io/quote@v1.5.2 golang.org/x/text@none +stderr '^go get: rsc.io/quote@v1.5.2 requires rsc.io/sampler@v1.3.0, not rsc.io/sampler@v1.0.0$' +stderr '^go get: rsc.io/quote@v1.5.2 requires golang.org/x/text@v0.0.0-20170915032832-14c0d48ead0c, not golang.org/x/text@none$' + go list -m all stdout 'rsc.io/quote v1.5.1' stdout 'rsc.io/sampler v1.3.0' # go get -u args should limit upgrades cp go.mod.empty go.mod -go get -u rsc.io/quote@v1.4.0 rsc.io/sampler@v1.0.0 +go get -d -u rsc.io/quote@v1.4.0 rsc.io/sampler@v1.0.0 go list -m all stdout 'rsc.io/quote v1.4.0' stdout 'rsc.io/sampler v1.0.0' @@ -37,7 +40,7 @@ stdout 'rsc.io/sampler v1.0.0' cp go.mod.orig go.mod go list -m -versions example.com/latemigrate/v2 stdout v2.0.0 # proxy may serve incompatible versions -go get rsc.io/quote@none +go get -d rsc.io/quote@none go list -m all ! stdout 'example.com/latemigrate/v2' diff --git a/libgo/go/cmd/go/testdata/script/mod_get_incompatible.txt b/libgo/go/cmd/go/testdata/script/mod_get_incompatible.txt index b210715..8000ee6 100644 --- a/libgo/go/cmd/go/testdata/script/mod_get_incompatible.txt +++ b/libgo/go/cmd/go/testdata/script/mod_get_incompatible.txt @@ -1,15 +1,15 @@ env GO111MODULE=on -go list x +go get -d x go list -m all stdout 'rsc.io/breaker v2.0.0\+incompatible' cp go.mod2 go.mod -go get rsc.io/breaker@7307b30 +go get -d rsc.io/breaker@7307b30 go list -m all stdout 'rsc.io/breaker v2.0.0\+incompatible' -go get rsc.io/breaker@v2.0.0 +go get -d rsc.io/breaker@v2.0.0 go list -m all stdout 'rsc.io/breaker v2.0.0\+incompatible' diff --git a/libgo/go/cmd/go/testdata/script/mod_get_indirect.txt b/libgo/go/cmd/go/testdata/script/mod_get_indirect.txt index f25e170..e1cc1ab 100644 --- a/libgo/go/cmd/go/testdata/script/mod_get_indirect.txt +++ b/libgo/go/cmd/go/testdata/script/mod_get_indirect.txt @@ -27,7 +27,7 @@ grep 'golang.org/x/text v0.3.0 // indirect$' go.mod # indirect tag should be removed upon seeing direct import. cp $WORK/tmp/uselang.go x.go -go list +go get -d grep 'rsc.io/quote v1.5.2$' go.mod grep 'golang.org/x/text [v0-9a-f\.-]+$' go.mod diff --git a/libgo/go/cmd/go/testdata/script/mod_get_latest_pseudo.txt b/libgo/go/cmd/go/testdata/script/mod_get_latest_pseudo.txt index 825ee8c..241a0c2 100644 --- a/libgo/go/cmd/go/testdata/script/mod_get_latest_pseudo.txt +++ b/libgo/go/cmd/go/testdata/script/mod_get_latest_pseudo.txt @@ -5,6 +5,6 @@ env GO111MODULE=on go mod init m -go list example.com/notags +go get -d example.com/notags go list -m all stdout '^example.com/notags v0.0.0-20190507143103-cc8cbe209b64$' diff --git a/libgo/go/cmd/go/testdata/script/mod_get_main.txt b/libgo/go/cmd/go/testdata/script/mod_get_main.txt index 408a5b5..50b2fee 100644 --- a/libgo/go/cmd/go/testdata/script/mod_get_main.txt +++ b/libgo/go/cmd/go/testdata/script/mod_get_main.txt @@ -3,13 +3,13 @@ cp go.mod.orig go.mod # relative and absolute paths must be within the main module. ! go get -d .. -stderr '^go get \.\.: path '$WORK'[/\\]gopath is not a package in module rooted at '$WORK'[/\\]gopath[/\\]src$' +stderr '^go get: \.\. \('$WORK'[/\\]gopath\) is not within module rooted at '$WORK'[/\\]gopath[/\\]src$' ! go get -d $WORK -stderr '^go get '$WORK': path '$WORK' is not a package in module rooted at '$WORK'[/\\]gopath[/\\]src$' +stderr '^go get: '$WORK' is not within module rooted at '$WORK'[/\\]gopath[/\\]src$' ! go get -d ../... -stderr '^go get: pattern \.\./\.\.\.: directory prefix \.\. outside available modules$' +stderr '^go get: \.\./\.\.\. \('$WORK'[/\\]gopath([/\\]...)?\) is not within module rooted at '$WORK'[/\\]gopath[/\\]src$' ! go get -d $WORK/... -stderr '^go get: pattern '$WORK'[/\\]\.\.\.: directory prefix \.\.[/\\]\.\. outside available modules$' +stderr '^go get: '$WORK'[/\\]\.\.\. is not within module rooted at '$WORK'[/\\]gopath[/\\]src$' # @patch and @latest within the main module refer to the current version. # The main module won't be upgraded, but missing dependencies will be added. @@ -22,21 +22,25 @@ go get -d rsc.io/x@patch grep 'rsc.io/quote v1.5.2' go.mod cp go.mod.orig go.mod -# The main module cannot be updated to @latest, which is a specific version. -! go get -d rsc.io/x@latest -stderr '^go get rsc.io/x@latest: can.t request explicit version of path in main module$' - -# The main module cannot be updated to a specific version. -! go get -d rsc.io/x@v0.1.0 -stderr '^go get rsc.io/x@v0.1.0: can.t request explicit version of path in main module$' -! go get -d rsc.io/x@v0.1.0 -stderr '^go get rsc.io/x@v0.1.0: can.t request explicit version of path in main module$' # Upgrading a package pattern not contained in the main module should not # attempt to upgrade the main module. go get -d rsc.io/quote/...@v1.5.1 grep 'rsc.io/quote v1.5.1' go.mod + +# The main module cannot be updated to a specific version. +! go get -d rsc.io@v0.1.0 +stderr '^go get: can''t request version "v0.1.0" of the main module \(rsc.io\)$' + +# A package in the main module can't be upgraded either. +! go get -d rsc.io/x@v0.1.0 +stderr '^go get: package rsc.io/x is in the main module, so can''t request version v0.1.0$' + +# Nor can a pattern matching packages in the main module. +! go get -d rsc.io/x/...@latest +stderr '^go get: pattern rsc.io/x/... matches package rsc.io/x in the main module, so can''t request version latest$' + -- go.mod.orig -- module rsc.io diff --git a/libgo/go/cmd/go/testdata/script/mod_get_moved.txt b/libgo/go/cmd/go/testdata/script/mod_get_moved.txt index edc41cf..8430a73 100644 --- a/libgo/go/cmd/go/testdata/script/mod_get_moved.txt +++ b/libgo/go/cmd/go/testdata/script/mod_get_moved.txt @@ -1,5 +1,4 @@ env GO111MODULE=on -env GOPROXY=$GOPROXY/quiet [short] skip # A 'go get' that worked at a previous version should continue to work at that version, @@ -10,7 +9,7 @@ go list -m all stdout 'example.com/split v1.0.0' # A 'go get' that simultaneously upgrades away conflicting package defitions is not ambiguous. -go get example.com/split/subpkg@v1.1.0 +go get -d example.com/split/subpkg@v1.1.0 # A 'go get' without an upgrade should find the package. rm go.mod @@ -29,7 +28,9 @@ go list -m all stdout 'example.com/join/subpkg v1.0.0' # A 'go get' that simultaneously upgrades away conflicting package definitions is not ambiguous. -go get example.com/join/subpkg@v1.1.0 +# (A wildcard pattern applies to both packages and modules, +# because we define wildcard matching to apply after version resolution.) +go get -d example.com/join/subpkg/...@v1.1.0 # A 'go get' without an upgrade should find the package. rm go.mod diff --git a/libgo/go/cmd/go/testdata/script/mod_get_newcycle.txt b/libgo/go/cmd/go/testdata/script/mod_get_newcycle.txt index 5c197bb..f71620c 100644 --- a/libgo/go/cmd/go/testdata/script/mod_get_newcycle.txt +++ b/libgo/go/cmd/go/testdata/script/mod_get_newcycle.txt @@ -11,5 +11,4 @@ go mod init m cmp stderr stderr-expected -- stderr-expected -- -go get: inconsistent versions: - example.com/newcycle/a@v1.0.0 requires example.com/newcycle/a@v1.0.1 (not example.com/newcycle/a@v1.0.0) +go get: example.com/newcycle/a@v1.0.0 requires example.com/newcycle/a@v1.0.1, not example.com/newcycle/a@v1.0.0 diff --git a/libgo/go/cmd/go/testdata/script/mod_get_none.txt b/libgo/go/cmd/go/testdata/script/mod_get_none.txt index 5aec209..b358f05 100644 --- a/libgo/go/cmd/go/testdata/script/mod_get_none.txt +++ b/libgo/go/cmd/go/testdata/script/mod_get_none.txt @@ -3,10 +3,10 @@ env GO111MODULE=on go mod init example.com/foo # 'go get bar@none' should be a no-op if module bar is not active. -go get example.com/bar@none +go get -d example.com/bar@none go list -m all ! stdout example.com/bar -go get example.com/bar@none +go get -d example.com/bar@none go list -m all ! stdout example.com/bar diff --git a/libgo/go/cmd/go/testdata/script/mod_get_patterns.txt b/libgo/go/cmd/go/testdata/script/mod_get_patterns.txt index 8adc4b0..aee4374 100644 --- a/libgo/go/cmd/go/testdata/script/mod_get_patterns.txt +++ b/libgo/go/cmd/go/testdata/script/mod_get_patterns.txt @@ -10,21 +10,27 @@ grep 'require rsc.io/quote' go.mod cp go.mod.orig go.mod ! go get -d rsc.io/quote/x... -stderr 'go get rsc.io/quote/x...: module rsc.io/quote@upgrade found \(v1.5.2\), but does not contain packages matching rsc.io/quote/x...' +stderr 'go get: module rsc.io/quote@upgrade found \(v1.5.2\), but does not contain packages matching rsc.io/quote/x...' ! grep 'require rsc.io/quote' go.mod ! go get -d rsc.io/quote/x/... -stderr 'go get rsc.io/quote/x/...: module rsc.io/quote@upgrade found \(v1.5.2\), but does not contain packages matching rsc.io/quote/x/...' +stderr 'go get: module rsc.io/quote@upgrade found \(v1.5.2\), but does not contain packages matching rsc.io/quote/x/...' ! grep 'require rsc.io/quote' go.mod # If a pattern matches no packages within a module, the module should not -# be upgraded, even if the module path matches the pattern. +# be upgraded, even if the module path is a prefix of the pattern. cp go.mod.orig go.mod go mod edit -require example.com/nest@v1.0.0 go get -d example.com/nest/sub/y... grep 'example.com/nest/sub v1.0.0' go.mod grep 'example.com/nest v1.0.0' go.mod +# However, if the pattern matches the module path itself, the module +# should be upgraded even if it contains no matching packages. +go get -d example.com/n...t +grep 'example.com/nest v1.1.0' go.mod +grep 'example.com/nest/sub v1.0.0' go.mod + -- go.mod.orig -- module m diff --git a/libgo/go/cmd/go/testdata/script/mod_get_test.txt b/libgo/go/cmd/go/testdata/script/mod_get_test.txt index 3680ca2..23722bd 100644 --- a/libgo/go/cmd/go/testdata/script/mod_get_test.txt +++ b/libgo/go/cmd/go/testdata/script/mod_get_test.txt @@ -2,7 +2,7 @@ env GO111MODULE=on # By default, 'go get' should ignore tests cp go.mod.empty go.mod -go get m/a +go get -d m/a ! grep rsc.io/quote go.mod # 'go get -t' should consider test dependencies of the named package. diff --git a/libgo/go/cmd/go/testdata/script/mod_get_trailing_slash.txt b/libgo/go/cmd/go/testdata/script/mod_get_trailing_slash.txt index 7b5d90c..3b38d8b 100644 --- a/libgo/go/cmd/go/testdata/script/mod_get_trailing_slash.txt +++ b/libgo/go/cmd/go/testdata/script/mod_get_trailing_slash.txt @@ -1,3 +1,6 @@ +# Populate go.sum +go mod download + # go list should succeed to load a package ending with ".go" if the path does # not correspond to an existing local file. Listing a pattern ending with # ".go/" should try to list a package regardless of whether a file exists at the diff --git a/libgo/go/cmd/go/testdata/script/mod_get_upgrade.txt b/libgo/go/cmd/go/testdata/script/mod_get_upgrade.txt index 6a14dfd..eeb6d6f 100644 --- a/libgo/go/cmd/go/testdata/script/mod_get_upgrade.txt +++ b/libgo/go/cmd/go/testdata/script/mod_get_upgrade.txt @@ -1,6 +1,6 @@ env GO111MODULE=on -go get rsc.io/quote@v1.5.1 +go get -d rsc.io/quote@v1.5.1 go list -m all stdout 'rsc.io/quote v1.5.1' grep 'rsc.io/quote v1.5.1$' go.mod diff --git a/libgo/go/cmd/go/testdata/script/mod_go_version.txt b/libgo/go/cmd/go/testdata/script/mod_go_version.txt index 37f1735..97d9975 100644 --- a/libgo/go/cmd/go/testdata/script/mod_go_version.txt +++ b/libgo/go/cmd/go/testdata/script/mod_go_version.txt @@ -8,12 +8,19 @@ go build sub.1 go build subver.1 ! stderr 'module requires' ! go build badsub.1 -stderr 'module requires Go 1.11111' +stderr '^note: module requires Go 1.11111$' go build versioned.1 go mod edit -require versioned.1@v1.1.0 ! go build versioned.1 -stderr 'module requires Go 1.99999' +stderr '^note: module requires Go 1.99999$' + +[short] stop + +# The message should be printed even if the compiler emits no output. +go build -o $WORK/nooutput.exe nooutput.go +! go build -toolexec=$WORK/nooutput.exe versioned.1 +stderr '^# versioned.1\nnote: module requires Go 1.99999$' -- go.mod -- module m @@ -71,3 +78,33 @@ go 1.99999 -- versioned2/x.go -- package x invalid syntax + +-- nooutput.go -- +// +build ignore + +package main + +import ( + "bytes" + "os" + "os/exec" + "strings" +) + +func main() { + stderr := new(bytes.Buffer) + stdout := new(bytes.Buffer) + + cmd := exec.Command(os.Args[1], os.Args[2:]...) + cmd.Stderr = stderr + cmd.Stdout = stdout + + err := cmd.Run() + if strings.HasPrefix(os.Args[2], "-V") { + os.Stderr.Write(stderr.Bytes()) + os.Stdout.Write(stdout.Bytes()) + } + if err != nil { + os.Exit(1) + } +} diff --git a/libgo/go/cmd/go/testdata/script/mod_gobuild_import.txt b/libgo/go/cmd/go/testdata/script/mod_gobuild_import.txt index 9484962..3a13366 100644 --- a/libgo/go/cmd/go/testdata/script/mod_gobuild_import.txt +++ b/libgo/go/cmd/go/testdata/script/mod_gobuild_import.txt @@ -19,7 +19,7 @@ exec $WORK/testimport$GOEXE other/x/y/z/w . stdout w2.go ! exec $WORK/testimport$GOEXE gobuild.example.com/x/y/z/w . -stderr 'cannot find module providing package gobuild.example.com/x/y/z/w' +stderr 'no required module provides package gobuild.example.com/x/y/z/w; try ''go get -d gobuild.example.com/x/y/z/w'' to add it' cd z exec $WORK/testimport$GOEXE other/x/y/z/w . diff --git a/libgo/go/cmd/go/testdata/script/mod_gonoproxy.txt b/libgo/go/cmd/go/testdata/script/mod_gonoproxy.txt index a9e0ca4..2047869 100644 --- a/libgo/go/cmd/go/testdata/script/mod_gonoproxy.txt +++ b/libgo/go/cmd/go/testdata/script/mod_gonoproxy.txt @@ -7,26 +7,32 @@ env dbname=localhost.localdev/sumdb # disagree with sumdb fails cp go.mod.orig go.mod env GOSUMDB=$sumdb' '$proxy/sumdb-wrong -! go get rsc.io/quote +! go get -d rsc.io/quote stderr 'SECURITY ERROR' # GONOSUMDB bypasses sumdb, for rsc.io/quote, rsc.io/sampler, golang.org/x/text env GONOSUMDB='*/quote,*/*mple*,golang.org/x' -go get rsc.io/quote +go get -d rsc.io/quote rm go.sum env GOPRIVATE='*/quote,*/*mple*,golang.org/x' env GONOPROXY=none # that is, proxy all despite GOPRIVATE -go get rsc.io/quote +go get -d rsc.io/quote + +# Download .info files needed for 'go list -m all' later. +# TODO(#42723): either 'go list -m' should not read these files, +# or 'go get' and 'go mod tidy' should download them. +go list -m all +stdout '^golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c$' # When GOPROXY is not empty but contains no entries, an error should be reported. env GOPROXY=',' -! go get golang.org/x/text +! go get -d golang.org/x/text stderr '^go get golang.org/x/text: GOPROXY list is not the empty string, but contains no entries$' # When GOPROXY=off, fetching modules not matched by GONOPROXY fails. env GONOPROXY=*/fortune env GOPROXY=off -! go get golang.org/x/text +! go get -d golang.org/x/text stderr '^go get golang.org/x/text: module lookup disabled by GOPROXY=off$' # GONOPROXY bypasses proxy @@ -34,13 +40,13 @@ stderr '^go get golang.org/x/text: module lookup disabled by GOPROXY=off$' [!exec:git] skip env GOPRIVATE=none env GONOPROXY='*/fortune' -! go get rsc.io/fortune # does not exist in real world, only on test proxy +! go get -d rsc.io/fortune # does not exist in real world, only on test proxy stderr 'git ls-remote' env GOSUMDB= env GONOPROXY= env GOPRIVATE='*/x' -go get golang.org/x/text +go get -d golang.org/x/text go list -m all ! stdout 'text.*v0.0.0-2017' # should not have the version from the proxy diff --git a/libgo/go/cmd/go/testdata/script/mod_gopkg_unstable.txt b/libgo/go/cmd/go/testdata/script/mod_gopkg_unstable.txt index 9d288a6..5ad9106 100644 --- a/libgo/go/cmd/go/testdata/script/mod_gopkg_unstable.txt +++ b/libgo/go/cmd/go/testdata/script/mod_gopkg_unstable.txt @@ -12,7 +12,7 @@ go list env GOPROXY=direct env GOSUMDB=off -go get gopkg.in/macaroon-bakery.v2-unstable/bakery +go get -d gopkg.in/macaroon-bakery.v2-unstable/bakery go list -m all stdout 'gopkg.in/macaroon-bakery.v2-unstable v2.0.0-[0-9]+-[0-9a-f]+$' diff --git a/libgo/go/cmd/go/testdata/script/mod_import.txt b/libgo/go/cmd/go/testdata/script/mod_import.txt index 3985b43..28358b5 100644 --- a/libgo/go/cmd/go/testdata/script/mod_import.txt +++ b/libgo/go/cmd/go/testdata/script/mod_import.txt @@ -1,7 +1,7 @@ env GO111MODULE=on # latest rsc.io/quote should be v1.5.2 not v1.5.3-pre1 -go list +go get -d go list -m all stdout 'rsc.io/quote v1.5.2' diff --git a/libgo/go/cmd/go/testdata/script/mod_in_testdata_dir.txt b/libgo/go/cmd/go/testdata/script/mod_in_testdata_dir.txt index f582569..66f79fa 100644 --- a/libgo/go/cmd/go/testdata/script/mod_in_testdata_dir.txt +++ b/libgo/go/cmd/go/testdata/script/mod_in_testdata_dir.txt @@ -8,8 +8,8 @@ env GO111MODULE=on cd $WORK/testdata go mod init testdata.tld/foo -# Building a package within that module should resolve its dependencies. -go build +# Getting a package within that module should resolve its dependencies. +go get -d grep 'rsc.io/quote' go.mod # Tidying the module should preserve those dependencies. @@ -26,7 +26,7 @@ exists vendor/rsc.io/quote cd $WORK/_ignored go mod init testdata.tld/foo -go build +go get grep 'rsc.io/quote' go.mod go mod tidy diff --git a/libgo/go/cmd/go/testdata/script/mod_indirect.txt b/libgo/go/cmd/go/testdata/script/mod_indirect.txt index 87a3f0b..6ea1cae 100644 --- a/libgo/go/cmd/go/testdata/script/mod_indirect.txt +++ b/libgo/go/cmd/go/testdata/script/mod_indirect.txt @@ -1,6 +1,6 @@ env GO111MODULE=on -# golang.org/issue/31248: module requirements imposed by dependency versions +# golang.org/issue/31248: required modules imposed by dependency versions # older than the selected version must still be taken into account. env GOFLAGS=-mod=readonly diff --git a/libgo/go/cmd/go/testdata/script/mod_init_dep.txt b/libgo/go/cmd/go/testdata/script/mod_init_dep.txt index 755076e..f8cf1d5 100644 --- a/libgo/go/cmd/go/testdata/script/mod_init_dep.txt +++ b/libgo/go/cmd/go/testdata/script/mod_init_dep.txt @@ -1,24 +1,14 @@ env GO111MODULE=on +env GOFLAGS=-mod=mod # modconv uses git directly to examine what old 'go get' would [!net] skip [!exec:git] skip -# go build should populate go.mod from Gopkg.lock -cp go.mod1 go.mod -go build +# go mod init should populate go.mod from Gopkg.lock +go mod init x stderr 'copying requirements from Gopkg.lock' go list -m all -! stderr 'copying requirements from Gopkg.lock' -stdout 'rsc.io/sampler v1.0.0' - -# go list should populate go.mod from Gopkg.lock -cp go.mod1 go.mod -go list -stderr 'copying requirements from Gopkg.lock' -go list -! stderr 'copying requirements from Gopkg.lock' -go list -m all stdout 'rsc.io/sampler v1.0.0' # test dep replacement @@ -26,9 +16,6 @@ cd y go mod init cmpenv go.mod go.mod.replace --- go.mod1 -- -module x - -- x.go -- package x @@ -54,4 +41,4 @@ go $goversion replace z v1.0.0 => rsc.io/quote v1.0.0 -require rsc.io/quote v1.0.0
\ No newline at end of file +require rsc.io/quote v1.0.0 diff --git a/libgo/go/cmd/go/testdata/script/mod_init_path.txt b/libgo/go/cmd/go/testdata/script/mod_init_path.txt index 637c29f..ccdfc92 100644 --- a/libgo/go/cmd/go/testdata/script/mod_init_path.txt +++ b/libgo/go/cmd/go/testdata/script/mod_init_path.txt @@ -1,7 +1,7 @@ env GO111MODULE=on ! go mod init . -stderr 'malformed import path' +stderr '^go: invalid module path "\.": is a local import path$' cd x go mod init example.com/x diff --git a/libgo/go/cmd/go/testdata/script/mod_install_versioned.txt b/libgo/go/cmd/go/testdata/script/mod_install_versioned.txt index 03986d0..c6bce41 100644 --- a/libgo/go/cmd/go/testdata/script/mod_install_versioned.txt +++ b/libgo/go/cmd/go/testdata/script/mod_install_versioned.txt @@ -1,9 +1,11 @@ env GO111MODULE=on +go get -d rsc.io/fortune go list -f '{{.Target}}' rsc.io/fortune ! stdout fortune@v1 stdout 'fortune(\.exe)?$' +go get -d rsc.io/fortune/v2 go list -f '{{.Target}}' rsc.io/fortune/v2 ! stdout v2 stdout 'fortune(\.exe)?$' diff --git a/libgo/go/cmd/go/testdata/script/mod_internal.txt b/libgo/go/cmd/go/testdata/script/mod_internal.txt index 1193d52..687269d 100644 --- a/libgo/go/cmd/go/testdata/script/mod_internal.txt +++ b/libgo/go/cmd/go/testdata/script/mod_internal.txt @@ -3,30 +3,34 @@ env GO111MODULE=on # golang.org/x/internal should be importable from other golang.org/x modules. go mod edit -module=golang.org/x/anything -go build . +go get -d . # ...and their tests... go test stdout PASS # ...but that should not leak into other modules. +go get -d ./baddep ! go build ./baddep stderr golang.org[/\\]notx[/\\]useinternal stderr 'use of internal package golang.org/x/.* not allowed' # Internal packages in the standard library should not leak into modules. +go get -d ./fromstd ! go build ./fromstd stderr 'use of internal package internal/testenv not allowed' # Dependencies should be able to use their own internal modules... go mod edit -module=golang.org/notx -go build ./throughdep +go get -d ./throughdep # ... but other modules should not, even if they have transitive dependencies. +go get -d . ! go build . stderr 'use of internal package golang.org/x/.* not allowed' # And transitive dependencies still should not leak. +go get -d ./baddep ! go build ./baddep stderr golang.org[/\\]notx[/\\]useinternal stderr 'use of internal package golang.org/x/.* not allowed' @@ -34,15 +38,17 @@ stderr 'use of internal package golang.org/x/.* not allowed' # Replacing an internal module should keep it internal to the same paths. go mod edit -module=golang.org/notx go mod edit -replace golang.org/x/internal=./replace/golang.org/notx/internal -go build ./throughdep +go get -d ./throughdep +go get -d ./baddep ! go build ./baddep stderr golang.org[/\\]notx[/\\]useinternal stderr 'use of internal package golang.org/x/.* not allowed' go mod edit -replace golang.org/x/internal=./vendor/golang.org/x/internal -go build ./throughdep +go get -d ./throughdep +go get -d ./baddep ! go build ./baddep stderr golang.org[/\\]notx[/\\]useinternal stderr 'use of internal package golang.org/x/.* not allowed' diff --git a/libgo/go/cmd/go/testdata/script/mod_invalid_version.txt b/libgo/go/cmd/go/testdata/script/mod_invalid_version.txt index 7e1bc9e..43b9564 100644 --- a/libgo/go/cmd/go/testdata/script/mod_invalid_version.txt +++ b/libgo/go/cmd/go/testdata/script/mod_invalid_version.txt @@ -4,6 +4,7 @@ env GO111MODULE=on env GOPROXY=direct env GOSUMDB=off +env GOFLAGS=-mod=mod # Regression test for golang.org/issue/27173: if the user (or go.mod file) # requests a pseudo-version that does not match both the module path and commit @@ -18,7 +19,7 @@ cp go.mod.orig go.mod go mod edit -require golang.org/x/text@14c0d48ead0c cd outside ! go list -m golang.org/x/text -stderr 'go: example.com@v0.0.0: parsing ../go.mod: '$WORK'/gopath/src/go.mod:5: require golang.org/x/text: version "14c0d48ead0c" invalid: must be of the form v1.2.3' +stderr 'go: example.com@v0.0.0 \(replaced by \./\..\): parsing ../go.mod: '$WORK'/gopath/src/go.mod:5: require golang.org/x/text: version "14c0d48ead0c" invalid: must be of the form v1.2.3' cd .. go list -m golang.org/x/text stdout 'golang.org/x/text v0.1.1-0.20170915032832-14c0d48ead0c' @@ -46,7 +47,7 @@ cp go.mod.orig go.mod go mod edit -require golang.org/x/text@v2.1.1-0.20170915032832-14c0d48ead0c cd outside ! go list -m golang.org/x/text -stderr 'go: example.com@v0.0.0: parsing ../go.mod: '$WORK'/gopath/src/go.mod:5: require golang.org/x/text: version "v2.1.1-0.20170915032832-14c0d48ead0c" invalid: should be v0 or v1, not v2' +stderr 'go: example.com@v0.0.0 \(replaced by \./\.\.\): parsing ../go.mod: '$WORK'/gopath/src/go.mod:5: require golang.org/x/text: version "v2.1.1-0.20170915032832-14c0d48ead0c" invalid: should be v0 or v1, not v2' cd .. ! go list -m golang.org/x/text stderr $WORK'/gopath/src/go.mod:5: require golang.org/x/text: version "v2.1.1-0.20170915032832-14c0d48ead0c" invalid: should be v0 or v1, not v2' @@ -221,7 +222,7 @@ stdout 'github.com/pierrec/lz4 v2.0.5\+incompatible' # not resolve to a pseudo-version with a different major version. cp go.mod.orig go.mod ! go get -d github.com/pierrec/lz4@v2.0.8 -stderr 'go get github.com/pierrec/lz4@v2.0.8: github.com/pierrec/lz4@v2.0.8: invalid version: module contains a go.mod file, so major version must be compatible: should be v0 or v1, not v2' +stderr 'go get: github.com/pierrec/lz4@v2.0.8: invalid version: module contains a go.mod file, so major version must be compatible: should be v0 or v1, not v2' # An invalid +incompatible suffix for a canonical version should error out, # not resolve to a pseudo-version. diff --git a/libgo/go/cmd/go/testdata/script/mod_issue35317.txt b/libgo/go/cmd/go/testdata/script/mod_issue35317.txt index 92416a5..b1852ab 100644 --- a/libgo/go/cmd/go/testdata/script/mod_issue35317.txt +++ b/libgo/go/cmd/go/testdata/script/mod_issue35317.txt @@ -5,4 +5,4 @@ env GO111MODULE=on [short] skip go mod init example.com -go get golang.org/x/text@v0.3.0 golang.org/x/internal@v0.1.0 golang.org/x/exp@none +go get -d golang.org/x/text@v0.3.0 golang.org/x/internal@v0.1.0 golang.org/x/exp@none diff --git a/libgo/go/cmd/go/testdata/script/mod_list.txt b/libgo/go/cmd/go/testdata/script/mod_list.txt index 17b33fc..1ba6d7c 100644 --- a/libgo/go/cmd/go/testdata/script/mod_list.txt +++ b/libgo/go/cmd/go/testdata/script/mod_list.txt @@ -2,12 +2,12 @@ env GO111MODULE=on [short] skip # list {{.Dir}} shows main module and go.mod but not not-yet-downloaded dependency dir. -go list -m -f '{{.Path}} {{.Main}} {{.GoMod}} {{.Dir}}' all +go list -mod=mod -m -f '{{.Path}} {{.Main}} {{.GoMod}} {{.Dir}}' all stdout '^x true .*[\\/]src[\\/]go.mod .*[\\/]src$' stdout '^rsc.io/quote false .*[\\/]v1.5.2.mod $' # list {{.Dir}} shows dependency after download (and go list without -m downloads it) -go list -f '{{.Dir}}' rsc.io/quote +go list -mod=mod -f '{{.Dir}}' rsc.io/quote stdout '.*mod[\\/]rsc.io[\\/]quote@v1.5.2$' # downloaded dependencies are read-only @@ -20,7 +20,7 @@ go clean -modcache # list {{.Dir}} shows replaced directories cp go.mod2 go.mod -go list -f {{.Dir}} rsc.io/quote +go list -mod=mod -f {{.Dir}} rsc.io/quote go list -m -f '{{.Path}} {{.Version}} {{.Dir}}{{with .Replace}} {{.GoMod}} => {{.Version}} {{.Dir}} {{.GoMod}}{{end}}' all stdout 'mod[\\/]rsc.io[\\/]quote@v1.5.1' stdout 'v1.3.0.*mod[\\/]rsc.io[\\/]sampler@v1.3.1 .*[\\/]v1.3.1.mod => v1.3.1.*sampler@v1.3.1 .*[\\/]v1.3.1.mod' @@ -30,7 +30,7 @@ go list std stdout ^math/big # rsc.io/quote/buggy should be listable as a package -go list rsc.io/quote/buggy +go list -mod=mod rsc.io/quote/buggy # rsc.io/quote/buggy should not be listable as a module go list -m -e -f '{{.Error.Err}}' nonexist rsc.io/quote/buggy diff --git a/libgo/go/cmd/go/testdata/script/mod_list_bad_import.txt b/libgo/go/cmd/go/testdata/script/mod_list_bad_import.txt index 8a66e0b..3cd50b0 100644 --- a/libgo/go/cmd/go/testdata/script/mod_list_bad_import.txt +++ b/libgo/go/cmd/go/testdata/script/mod_list_bad_import.txt @@ -12,10 +12,11 @@ stdout 'incomplete' stdout 'bad dep: .*example.com/notfound' # Listing with -deps should also fail. -# BUG: Today, it does not. -# ! go list -deps example.com/direct -# stderr example.com/notfound -go list -deps example.com/direct +! go list -deps example.com/direct +stderr example.com/notfound + +# But -e -deps should succeed. +go list -e -deps example.com/direct stdout example.com/notfound @@ -28,16 +29,17 @@ stdout incomplete stdout 'bad dep: .*example.com/notfound' # Again, -deps should fail. -# BUG: Again, it does not. -# ! go list -deps example.com/indirect -# stderr example.com/notfound -go list -deps example.com/indirect +! go list -deps example.com/indirect +stderr example.com/notfound + +# But -e -deps should succeed. +go list -e -deps example.com/indirect stdout example.com/notfound # Listing the missing dependency directly should fail outright... ! go list -f '{{if .Error}}error{{end}} {{if .Incomplete}}incomplete{{end}}' example.com/notfound -stderr 'cannot find module providing package example.com/notfound' +stderr 'no required module provides package example.com/notfound; try ''go get -d example.com/notfound'' to add it' ! stdout error ! stdout incomplete diff --git a/libgo/go/cmd/go/testdata/script/mod_list_dir.txt b/libgo/go/cmd/go/testdata/script/mod_list_dir.txt index 6653435..1adab8f 100644 --- a/libgo/go/cmd/go/testdata/script/mod_list_dir.txt +++ b/libgo/go/cmd/go/testdata/script/mod_list_dir.txt @@ -2,6 +2,9 @@ # go list with path to directory should work +# populate go.sum +go get -d + env GO111MODULE=off go list -f '{{.ImportPath}}' $GOROOT/src/math stdout ^math$ @@ -29,3 +32,5 @@ require rsc.io/quote v1.5.2 -- x.go -- package x + +import _ "rsc.io/quote" diff --git a/libgo/go/cmd/go/testdata/script/mod_list_direct.txt b/libgo/go/cmd/go/testdata/script/mod_list_direct.txt index 8f85871..62a472f 100644 --- a/libgo/go/cmd/go/testdata/script/mod_list_direct.txt +++ b/libgo/go/cmd/go/testdata/script/mod_list_direct.txt @@ -10,7 +10,7 @@ env GOSUMDB=off # For a while, (*modfetch.codeRepo).Stat was not checking for a go.mod file, # which would produce a hard error at the subsequent call to GoMod. -go list all +go get -d -- go.mod -- module example.com diff --git a/libgo/go/cmd/go/testdata/script/mod_list_pseudo.txt b/libgo/go/cmd/go/testdata/script/mod_list_pseudo.txt index 3a10b3a..056c093 100644 --- a/libgo/go/cmd/go/testdata/script/mod_list_pseudo.txt +++ b/libgo/go/cmd/go/testdata/script/mod_list_pseudo.txt @@ -10,30 +10,25 @@ go mod download github.com/dmitshur-test/modtest5@v0.5.0-alpha go mod download github.com/dmitshur-test/modtest5@v0.5.0-alpha.0.20190619023908-3da23a9deb9e cmp $GOPATH/pkg/mod/cache/download/github.com/dmitshur-test/modtest5/@v/list $WORK/modtest5.list +env GOSUMDB=off # don't verify go.mod files when loading retractions env GOPROXY=file:///$GOPATH/pkg/mod/cache/download env GOPATH=$WORK/gopath2 mkdir $GOPATH -go list -m -json github.com/dmitshur-test/modtest5@latest -cmp stdout $WORK/modtest5.json +go list -m -f '{{.Path}} {{.Version}} {{.Time.Format "2006-01-02"}}' github.com/dmitshur-test/modtest5@latest +stdout '^github.com/dmitshur-test/modtest5 v0.5.0-alpha 2019-06-18$' # If the module proxy contains only pseudo-versions, 'latest' should stat # the version with the most recent timestamp — not the highest semantic # version — and return its metadata. env GOPROXY=file:///$WORK/tinyproxy -go list -m -json dmitri.shuralyov.com/test/modtest3@latest -cmp stdout $WORK/modtest3.json +go list -m -f '{{.Path}} {{.Version}} {{.Time.Format "2006-01-02"}}' dmitri.shuralyov.com/test/modtest3@latest +stdout '^dmitri.shuralyov.com/test/modtest3 v0.0.0-20181023043359-a85b471d5412 2018-10-22$' -- $WORK/modtest5.list -- v0.0.0-20190619020302-197a620e0c9a v0.5.0-alpha v0.5.0-alpha.0.20190619023908-3da23a9deb9e --- $WORK/modtest5.json -- -{ - "Path": "github.com/dmitshur-test/modtest5", - "Version": "v0.5.0-alpha", - "Time": "2019-06-18T19:04:46-07:00" -} -- $WORK/tinyproxy/dmitri.shuralyov.com/test/modtest3/@v/list -- v0.1.0-0.20161023043300-000000000000 v0.0.0-20181023043359-a85b471d5412 @@ -42,9 +37,3 @@ v0.0.0-20181023043359-a85b471d5412 "Version": "v0.0.0-20181023043359-a85b471d5412", "Time": "2018-10-22T21:33:59-07:00" } --- $WORK/modtest3.json -- -{ - "Path": "dmitri.shuralyov.com/test/modtest3", - "Version": "v0.0.0-20181023043359-a85b471d5412", - "Time": "2018-10-22T21:33:59-07:00" -} diff --git a/libgo/go/cmd/go/testdata/script/mod_list_replace_dir.txt b/libgo/go/cmd/go/testdata/script/mod_list_replace_dir.txt index cad7fe2..f2f2d2b 100644 --- a/libgo/go/cmd/go/testdata/script/mod_list_replace_dir.txt +++ b/libgo/go/cmd/go/testdata/script/mod_list_replace_dir.txt @@ -2,8 +2,11 @@ # module within the module cache. # Verifies golang.org/issue/29548 -env GO111MODULE=on -go mod download rsc.io/quote@v1.5.1 rsc.io/quote@v1.5.2 +# Populate go.sum and download dependencies. +go get -d + +# Ensure v1.5.2 is also in the cache so we can list it. +go mod download rsc.io/quote@v1.5.2 ! go list $GOPATH/pkg/mod/rsc.io/quote@v1.5.2 stderr '^directory ..[/\\]pkg[/\\]mod[/\\]rsc.io[/\\]quote@v1.5.2 outside available modules$' @@ -17,3 +20,8 @@ module example.com/quoter require rsc.io/quote v1.5.2 replace rsc.io/quote => rsc.io/quote v1.5.1 + +-- use.go -- +package use + +import _ "rsc.io/quote" diff --git a/libgo/go/cmd/go/testdata/script/mod_list_std.txt b/libgo/go/cmd/go/testdata/script/mod_list_std.txt index 76a3b00..baf7908 100644 --- a/libgo/go/cmd/go/testdata/script/mod_list_std.txt +++ b/libgo/go/cmd/go/testdata/script/mod_list_std.txt @@ -6,8 +6,13 @@ env GOPROXY=off # Outside of GOROOT, our vendored packages should be reported as part of the standard library. go list -f '{{if .Standard}}{{.ImportPath}}{{end}}' std cmd -stdout ^vendor/golang.org/x/net/http2/hpack +stdout ^vendor/golang\.org/x/net/http2/hpack stdout ^cmd/vendor/golang\.org/x/arch/x86/x86asm +! stdout ^golang\.org/x/ + +# The dependencies of those packages should also be vendored. +go list -deps vendor/golang.org/x/crypto/chacha20 +stdout ^vendor/golang\.org/x/crypto/internal/subtle # cmd/... should match the same packages it used to match in GOPATH mode. go list cmd/... @@ -23,40 +28,57 @@ stdout ^bytes$ ! stdout ^builtin$ ! stdout ^cmd/ ! stdout ^vendor/ +! stdout ^golang\.org/x/ + +# Vendored dependencies should appear with their 'vendor/' paths in std (they're +# in GOROOT/src, but not in the 'std' module following the usual module-boundary +# rules). -# Within the std module, listing ./... should omit the 'std' prefix: -# the package paths should be the same via ./... or the 'std' meta-pattern. -# TODO(golang.org/issue/30241): Make that work. -# Today, they are listed in 'std' but not './...'. cd $GOROOT/src -go list ./... -! stdout ^vendor/golang.org/x # TODO: should be included, or should be omitted from 'std'. -cp stdout $WORK/listdot.txt go list std -stdout ^vendor/golang.org/x # TODO: remove vendor/ prefix -# TODO: cmp stdout $WORK/listdot.txt +stdout ^vendor/golang.org/x/net/http2/hpack +! stdout ^golang\.org/x + +# The dependencies of packages with an explicit 'vendor/' prefix should +# still themselves resolve to vendored packages. +go list -deps vendor/golang.org/x/crypto/chacha20 +stdout ^vendor/golang.org/x/crypto/internal/subtle +! stdout ^golang\.org/x + +# Within the std module, the dependencies of the non-vendored packages within +# std should appear to come from modules, but they should be loaded from the +# vendor directory (just like ordinary vendored module dependencies). go list all -stdout ^vendor/golang.org/x # TODO: remove vendor/ prefix. +stdout ^golang.org/x/ ! stdout ^std/ +! stdout ^cmd/ +! stdout ^vendor/ +go list -deps -f '{{if not .Standard}}{{.ImportPath}}{{end}}' std +! stdout ^vendor/golang.org/x/net/http2/hpack +stdout ^golang.org/x/net/http2/hpack -# Within the std module, the vendored dependencies of std should appear -# to come from the actual modules. -# TODO(golang.org/issue/30241): Make that work. -# Today, they still have the vendor/ prefix. -go list std -stdout ^vendor/golang.org/x/net/http2/hpack # TODO -! stdout ^golang.org/x/net/http2/hpack # TODO +go list -f '{{.Dir}}' golang.org/x/net/http2/hpack +stdout $GOROOT[/\\]src[/\\]vendor -go list -deps -f '{{if not .Standard}}{{.ImportPath}}{{end}}' std -# ! stdout ^vendor/golang.org/x/net/http2/hpack # TODO -! stdout ^golang.org/x/net/http2/hpack # TODO +# Within the std module, the packages within the module should omit the 'std/' +# prefix (they retain their own identities), but should respect normal module +# boundaries (vendored packages are not included in the module, even though they +# are included in the 'std' pattern). + +go list ./... +stdout ^bytes$ +! stdout ^builtin$ +! stdout ^cmd/ +! stdout ^vendor/ +! stdout ^golang\.org/x/ # Within std, the vendored dependencies of cmd should still appear to be part of cmd. + go list -f '{{if .Standard}}{{.ImportPath}}{{end}}' cmd stdout ^cmd/vendor/golang\.org/x/arch/x86/x86asm diff --git a/libgo/go/cmd/go/testdata/script/mod_list_test.txt b/libgo/go/cmd/go/testdata/script/mod_list_test.txt index a99e4f3..f697af6 100644 --- a/libgo/go/cmd/go/testdata/script/mod_list_test.txt +++ b/libgo/go/cmd/go/testdata/script/mod_list_test.txt @@ -3,9 +3,19 @@ env GO111MODULE=on # go list -compiled -test must handle test-only packages # golang.org/issue/27097. go list -compiled -test +stdout -count=4 '^.' # 4 lines stdout '^m$' stdout '^m\.test$' stdout '^m \[m\.test\]$' +stdout '^m_test \[m\.test\]$' + +# https://golang.org/issue/39974: test packages should have the Module field populated. +go list -test -f '{{.ImportPath}}{{with .Module}}: {{.Path}}{{end}}' +stdout -count=4 '^.' # 4 lines +stdout '^m: m$' +stdout '^m\.test: m$' +stdout '^m \[m\.test\]: m$' +stdout '^m_test \[m\.test\]: m$' -- go.mod -- module m @@ -14,3 +24,7 @@ module m package x import "testing" func Test(t *testing.T) {} +-- x_x_test.go -- +package x_test +import "testing" +func Test(t *testing.T) {} diff --git a/libgo/go/cmd/go/testdata/script/mod_list_upgrade.txt b/libgo/go/cmd/go/testdata/script/mod_list_upgrade.txt index 474df0d..0cef04b 100644 --- a/libgo/go/cmd/go/testdata/script/mod_list_upgrade.txt +++ b/libgo/go/cmd/go/testdata/script/mod_list_upgrade.txt @@ -1,5 +1,9 @@ env GO111MODULE=on +# Populate go.sum +go list -m -mod=mod all + +# Check for upgrades. go list -m -u all stdout 'rsc.io/quote v1.2.0 \[v1\.5\.2\]' diff --git a/libgo/go/cmd/go/testdata/script/mod_load_badchain.txt b/libgo/go/cmd/go/testdata/script/mod_load_badchain.txt index 67d9a15..c0c382b 100644 --- a/libgo/go/cmd/go/testdata/script/mod_load_badchain.txt +++ b/libgo/go/cmd/go/testdata/script/mod_load_badchain.txt @@ -28,10 +28,10 @@ cmp stderr list-expected # Try listing a package that imports a package # in a module without a requirement. go mod edit -droprequire example.com/badchain/a -! go list m/use +! go list -mod=mod m/use cmp stderr list-missing-expected -! go list -test m/testuse +! go list -mod=mod -test m/testuse cmp stderr list-missing-test-expected -- go.mod.orig -- @@ -40,6 +40,19 @@ module m go 1.13 require example.com/badchain/a v1.0.0 +-- go.sum -- +example.com/badchain/a v1.0.0 h1:iJDLiHLmpQgr9Zrv+44UqywAE2IG6WkHnH4uG08vf+s= +example.com/badchain/a v1.0.0/go.mod h1:6/gnCYHdVrs6mUgatUYUSbuHxEY+/yWedmTggLz23EI= +example.com/badchain/a v1.1.0 h1:cPxQpsOjaIrn05yDfl4dFFgGSbjYmytLqtIIBfTsEqA= +example.com/badchain/a v1.1.0/go.mod h1:T15b2BEK+RY7h7Lr2dgS38p1pgH5/t7Kf5nQXBlcW/A= +example.com/badchain/b v1.0.0 h1:kjDVlBxpjQavYxHE7ECCyyXhfwsfhWIqvghfRgPktSA= +example.com/badchain/b v1.0.0/go.mod h1:sYsH934pMc3/A2vQZh019qrWmp4+k87l3O0VFUYqL+I= +example.com/badchain/b v1.1.0 h1:iEALV+DRN62FArnYylBR4YwCALn/hCdITvhdagHa0L4= +example.com/badchain/b v1.1.0/go.mod h1:mlCgKO7lRZ+ijwMFIBFRPCGt5r5oqCcHdhSSE0VL4uY= +example.com/badchain/c v1.0.0 h1:lOeUHQKR7SboSH7Bj6eIDWoNHaDQXI0T2GfaH2x9fNA= +example.com/badchain/c v1.0.0/go.mod h1:4U3gzno17SaQ2koSVNxITu9r60CeLSgye9y4/5LnfOE= +example.com/badchain/c v1.1.0 h1:VtTg1g7fOutWKHQf+ag04KLRpdMGSfQ9s9tagVtGW14= +example.com/badchain/c v1.1.0/go.mod h1:tyoJj5qh+qtb48sflwdVvk4R+OjPQEY2UJOoibsVLPk= -- use/use.go -- package use @@ -56,14 +69,13 @@ import ( func Test(t *testing.T) {} -- update-main-expected -- -go: example.com/badchain/c upgrade => v1.1.0 go get: example.com/badchain/c@v1.0.0 updating to example.com/badchain/c@v1.1.0: parsing go.mod: module declares its path as: badchain.example.com/c but was required as: example.com/badchain/c -- update-a-expected -- -go: example.com/badchain/a upgrade => v1.1.0 -go get: example.com/badchain/a@v1.1.0 requires +go get: example.com/badchain/a@v1.0.0 updating to + example.com/badchain/a@v1.1.0 requires example.com/badchain/b@v1.1.0 requires example.com/badchain/c@v1.1.0: parsing go.mod: module declares its path as: badchain.example.com/c diff --git a/libgo/go/cmd/go/testdata/script/mod_load_badmod.txt b/libgo/go/cmd/go/testdata/script/mod_load_badmod.txt index 68c8b37..fa22e18 100644 --- a/libgo/go/cmd/go/testdata/script/mod_load_badmod.txt +++ b/libgo/go/cmd/go/testdata/script/mod_load_badmod.txt @@ -1,14 +1,13 @@ # Unknown lines should be ignored in dependency go.mod files. -env GO111MODULE=on -go list -m all +go list -m -mod=mod all # ... and in replaced dependency go.mod files. cp go.mod go.mod.usesub -go list -m all +go list -m -mod=mod all # ... but not in the main module. cp go.mod.bad go.mod -! go list -m all +! go list -m -mod=mod all stderr 'unknown directive: hello' -- go.mod -- diff --git a/libgo/go/cmd/go/testdata/script/mod_load_badzip.txt b/libgo/go/cmd/go/testdata/script/mod_load_badzip.txt index c5ba18e..65374d2 100644 --- a/libgo/go/cmd/go/testdata/script/mod_load_badzip.txt +++ b/libgo/go/cmd/go/testdata/script/mod_load_badzip.txt @@ -5,10 +5,8 @@ env GO111MODULE=on stderr 'zip for rsc.io/badzip@v1.0.0 has unexpected file rsc.io/badzip@v1.0.0.txt' ! grep rsc.io/badzip go.mod -# TODO(golang.org/issue/31730): 'go build' should print the error below if the -# requirement is not present. go mod edit -require rsc.io/badzip@v1.0.0 -! go build rsc.io/badzip +! go build -mod=mod rsc.io/badzip stderr 'zip for rsc.io/badzip@v1.0.0 has unexpected file rsc.io/badzip@v1.0.0.txt' -- go.mod -- diff --git a/libgo/go/cmd/go/testdata/script/mod_missingpkg_prerelease.txt b/libgo/go/cmd/go/testdata/script/mod_missingpkg_prerelease.txt index 319ff85..9c250e7 100644 --- a/libgo/go/cmd/go/testdata/script/mod_missingpkg_prerelease.txt +++ b/libgo/go/cmd/go/testdata/script/mod_missingpkg_prerelease.txt @@ -1,7 +1,7 @@ env GO111MODULE=on -! go list use.go -stderr 'example.com/missingpkg/deprecated: package provided by example.com/missingpkg at latest version v1.0.0 but not at required version v1.0.1-beta' +! go list -mod=mod -deps use.go +stderr '^use.go:4:2: package example.com/missingpkg/deprecated provided by example.com/missingpkg at latest version v1.0.0 but not at required version v1.0.1-beta$' -- go.mod -- module m diff --git a/libgo/go/cmd/go/testdata/script/mod_modinfo.txt b/libgo/go/cmd/go/testdata/script/mod_modinfo.txt index 731f53c..c56ed91 100644 --- a/libgo/go/cmd/go/testdata/script/mod_modinfo.txt +++ b/libgo/go/cmd/go/testdata/script/mod_modinfo.txt @@ -9,6 +9,7 @@ env GO111MODULE=on cd x go mod edit -require=rsc.io/quote@v1.5.2 go mod edit -replace=rsc.io/quote@v1.5.2=rsc.io/quote@v1.0.0 +go mod tidy # populate go.sum # Build a binary and ensure that it can output its own debug info. # The debug info should be accessible before main starts (golang.org/issue/29628). @@ -71,7 +72,6 @@ package main import ( "bytes" "encoding/hex" - "io/ioutil" "log" "os" @@ -79,7 +79,7 @@ import ( ) func main() { - b, err := ioutil.ReadFile(os.Args[0]) + b, err := os.ReadFile(os.Args[0]) if err != nil { log.Fatal(err) } diff --git a/libgo/go/cmd/go/testdata/script/mod_multirepo.txt b/libgo/go/cmd/go/testdata/script/mod_multirepo.txt index 7f977e8..0f335a1 100644 --- a/libgo/go/cmd/go/testdata/script/mod_multirepo.txt +++ b/libgo/go/cmd/go/testdata/script/mod_multirepo.txt @@ -7,6 +7,7 @@ go list -deps -f {{.Dir}} # v2 import should use a downloaded module # both without an explicit go.mod entry ... cp tmp/use_v2.go x.go +go get -d . go list -deps -f {{.Dir}} stdout 'pkg[\\/]mod[\\/]rsc.io[\\/]quote[\\/]v2@v2.0.1$' diff --git a/libgo/go/cmd/go/testdata/script/mod_off.txt b/libgo/go/cmd/go/testdata/script/mod_off.txt index cada6de..a73a58d 100644 --- a/libgo/go/cmd/go/testdata/script/mod_off.txt +++ b/libgo/go/cmd/go/testdata/script/mod_off.txt @@ -4,7 +4,7 @@ env GO111MODULE=off # GO111MODULE=off when outside of GOPATH will fatal # with an error message, even with some source code in the directory and a go.mod. ! go mod init -stderr 'go mod init: modules disabled by GO111MODULE=off; see ''go help modules''' +stderr 'go: modules disabled by GO111MODULE=off; see ''go help modules''' ! go mod graph stderr 'go: modules disabled by GO111MODULE=off; see ''go help modules''' ! go mod verify @@ -16,7 +16,7 @@ stderr 'go: modules disabled by GO111MODULE=off; see ''go help modules''' mkdir z cd z ! go mod init -stderr 'go mod init: modules disabled by GO111MODULE=off; see ''go help modules''' +stderr 'go: modules disabled by GO111MODULE=off; see ''go help modules''' ! go mod graph stderr 'go: modules disabled by GO111MODULE=off; see ''go help modules''' ! go mod verify diff --git a/libgo/go/cmd/go/testdata/script/mod_off_init.txt b/libgo/go/cmd/go/testdata/script/mod_off_init.txt index 1339c8a..2aec0b3 100644 --- a/libgo/go/cmd/go/testdata/script/mod_off_init.txt +++ b/libgo/go/cmd/go/testdata/script/mod_off_init.txt @@ -2,4 +2,4 @@ # ignored anyway due to GO111MODULE=off. env GO111MODULE=off ! go mod init -stderr 'go mod init: modules disabled by GO111MODULE=off; see ''go help modules''' +stderr 'go: modules disabled by GO111MODULE=off; see ''go help modules''' diff --git a/libgo/go/cmd/go/testdata/script/mod_outside.txt b/libgo/go/cmd/go/testdata/script/mod_outside.txt index 03ef576..8f01b5d 100644 --- a/libgo/go/cmd/go/testdata/script/mod_outside.txt +++ b/libgo/go/cmd/go/testdata/script/mod_outside.txt @@ -39,6 +39,11 @@ stdout '^fmt$' go list ./needmod/needmod.go stdout 'command-line-arguments' +# 'go list' on a package from a module should fail. +! go list example.com/printversion +stderr '^no required module provides package example.com/printversion: working directory is not part of a module$' + + # 'go list -m' with an explicit version should resolve that version. go list -m example.com/version@latest stdout 'example.com/version v1.1.0' @@ -69,10 +74,9 @@ go clean -n ! stdout . ! stderr . -# 'go mod graph' should not display anything, since there are no active modules. -go mod graph -! stdout . -! stderr . +# 'go mod graph' should fail, since there's no module graph. +! go mod graph +stderr 'cannot find main module' # 'go mod why' should fail, since there is no main module to depend on anything. ! go mod why -m example.com/version @@ -126,12 +130,12 @@ stderr 'cannot find main module' # 'go get -u all' upgrades the transitive import graph of the main module, # which is empty. ! go get -u all -stderr 'go get all: cannot match "all": working directory is not part of a module' +stderr 'go get: cannot match "all": working directory is not part of a module' # 'go get' should check the proposed module graph for consistency, # even though we won't write it anywhere. ! go get -d example.com/printversion@v1.0.0 example.com/version@none -stderr 'inconsistent versions' +stderr '^go get: example.com/printversion@v1.0.0 requires example.com/version@v1.0.0, not example.com/version@none$' # 'go get -d' should download and extract the source code needed to build the requested version. rm -r $GOPATH/pkg/mod/example.com @@ -152,7 +156,7 @@ stderr 'cannot find main module' # 'go build' of source files should fail if they import anything outside std. ! go build -n ./needmod/needmod.go -stderr 'needmod[/\\]needmod.go:10:2: cannot find module providing package example.com/version: working directory is not part of a module' +stderr '^needmod[/\\]needmod.go:10:2: no required module provides package example.com/version: working directory is not part of a module$' # 'go build' of source files should succeed if they do not import anything outside std. go build -n -o ignore ./stdonly/stdonly.go @@ -175,21 +179,26 @@ go doc fmt # 'go doc' should fail for a package path outside a module. ! go doc example.com/version -stderr 'doc: cannot find module providing package example.com/version: working directory is not part of a module' +stderr 'doc: no required module provides package example.com/version: working directory is not part of a module' -# 'go install' with a version should fail due to syntax. -! go install example.com/printversion@v1.0.0 -stderr 'can only use path@version syntax with' +# 'go install' with a version should succeed if all constraints are met. +# See mod_install_pkg_version. +rm $GOPATH/bin +go install example.com/printversion@v0.1.0 +exists $GOPATH/bin/printversion$GOEXE # 'go install' should fail if a package argument must be resolved to a module. ! go install example.com/printversion -stderr 'cannot find module providing package example.com/printversion: working directory is not part of a module' +stderr '^go install: version is required when current directory is not in a module\n\tTry ''go install example.com/printversion@latest'' to install the latest version$' # 'go install' should fail if a source file imports a package that must be # resolved to a module. ! go install ./needmod/needmod.go -stderr 'needmod[/\\]needmod.go:10:2: cannot find module providing package example.com/version: working directory is not part of a module' +stderr 'needmod[/\\]needmod.go:10:2: no required module provides package example.com/version: working directory is not part of a module' +# 'go install' should succeed with a package in GOROOT. +go install cmd/addr2line +! stderr . # 'go run' with a verison should fail due to syntax. ! go run example.com/printversion@v1.0.0 @@ -197,12 +206,12 @@ stderr 'can only use path@version syntax with' # 'go run' should fail if a package argument must be resolved to a module. ! go run example.com/printversion -stderr 'cannot find module providing package example.com/printversion: working directory is not part of a module' +stderr '^no required module provides package example.com/printversion: working directory is not part of a module$' # 'go run' should fail if a source file imports a package that must be # resolved to a module. ! go run ./needmod/needmod.go -stderr 'needmod[/\\]needmod.go:10:2: cannot find module providing package example.com/version: working directory is not part of a module' +stderr '^needmod[/\\]needmod.go:10:2: no required module provides package example.com/version: working directory is not part of a module$' # 'go fmt' should be able to format files outside of a module. @@ -220,7 +229,7 @@ stdout 'main is example.com/printversion v0.1.0' stdout 'using example.com/version v1.1.0' # 'go get' of a versioned binary should build and install the latest version -# using its minimal module requirements, ignoring replacements and exclusions. +# using its minimal required modules, ignoring replacements and exclusions. go get example.com/printversion exec ../bin/printversion stdout 'path is example.com/printversion' diff --git a/libgo/go/cmd/go/testdata/script/mod_permissions.txt b/libgo/go/cmd/go/testdata/script/mod_permissions.txt index 11fb475..2d32dcd 100644 --- a/libgo/go/cmd/go/testdata/script/mod_permissions.txt +++ b/libgo/go/cmd/go/testdata/script/mod_permissions.txt @@ -12,7 +12,7 @@ chmod 0640 go.mod chmod 0604 go.sum go mod edit -module=golang.org/issue/34634 -go build . +go get -d cmp go.mod go.mod.want cmp go.sum go.sum.want diff --git a/libgo/go/cmd/go/testdata/script/mod_proxy_list.txt b/libgo/go/cmd/go/testdata/script/mod_proxy_list.txt index 849cf2c..89129f4 100644 --- a/libgo/go/cmd/go/testdata/script/mod_proxy_list.txt +++ b/libgo/go/cmd/go/testdata/script/mod_proxy_list.txt @@ -3,34 +3,34 @@ env proxy=$GOPROXY # Proxy that can't serve should fail. env GOPROXY=$proxy/404 -! go get rsc.io/quote@v1.0.0 +! go get -d rsc.io/quote@v1.0.0 stderr '404 Not Found' # get should walk down the proxy list past 404 and 410 responses. env GOPROXY=$proxy/404,$proxy/410,$proxy -go get rsc.io/quote@v1.1.0 +go get -d rsc.io/quote@v1.1.0 # get should not walk past other 4xx errors if proxies are separated with ','. env GOPROXY=$proxy/403,$proxy -! go get rsc.io/quote@v1.2.0 +! go get -d rsc.io/quote@v1.2.0 stderr 'reading.*/403/rsc.io/.*: 403 Forbidden' # get should not walk past non-4xx errors if proxies are separated with ','. env GOPROXY=$proxy/500,$proxy -! go get rsc.io/quote@v1.3.0 +! go get -d rsc.io/quote@v1.3.0 stderr 'reading.*/500/rsc.io/.*: 500 Internal Server Error' # get should walk past other 4xx errors if proxies are separated with '|'. env GOPROXY=$proxy/403|https://0.0.0.0|$proxy -go get rsc.io/quote@v1.2.0 +go get -d rsc.io/quote@v1.2.0 # get should walk past non-4xx errors if proxies are separated with '|'. env GOPROXY=$proxy/500|https://0.0.0.0|$proxy -go get rsc.io/quote@v1.3.0 +go get -d rsc.io/quote@v1.3.0 # get should return the final error if that's all we have. env GOPROXY=$proxy/404,$proxy/410 -! go get rsc.io/quote@v1.4.0 +! go get -d rsc.io/quote@v1.4.0 stderr 'reading.*/410/rsc.io/.*: 410 Gone' -- go.mod -- diff --git a/libgo/go/cmd/go/testdata/script/mod_query.txt b/libgo/go/cmd/go/testdata/script/mod_query.txt index e87ca30..e101857 100644 --- a/libgo/go/cmd/go/testdata/script/mod_query.txt +++ b/libgo/go/cmd/go/testdata/script/mod_query.txt @@ -1,5 +1,10 @@ env GO111MODULE=on +# Populate go.sum. +# TODO(golang.org/issue/41297): we shouldn't need go.sum. None of the commands +# below depend on the build list. +go mod download + go list -m -versions rsc.io/quote stdout '^rsc.io/quote v1.0.0 v1.1.0 v1.2.0 v1.2.1 v1.3.0 v1.4.0 v1.5.0 v1.5.1 v1.5.2 v1.5.3-pre1$' @@ -30,3 +35,8 @@ stdout 'no matching versions for query ">v1.5.3"' -- go.mod -- module x require rsc.io/quote v1.0.0 + +-- use.go -- +package use + +import _ "rsc.io/quote" diff --git a/libgo/go/cmd/go/testdata/script/mod_query_empty.txt b/libgo/go/cmd/go/testdata/script/mod_query_empty.txt index 4d8259b..f8b6e3e 100644 --- a/libgo/go/cmd/go/testdata/script/mod_query_empty.txt +++ b/libgo/go/cmd/go/testdata/script/mod_query_empty.txt @@ -8,7 +8,7 @@ go mod download example.com/join@v1.1.0 env GOPROXY=file:///$WORK/badproxy cp go.mod.orig go.mod ! go get -d example.com/join/subpkg -stderr 'go get example.com/join/subpkg: example.com/join/subpkg@v0.0.0-20190624000000-123456abcdef: .*' +stderr 'go get: example.com/join/subpkg@v0.0.0-20190624000000-123456abcdef: .*' # If @v/list is empty, the 'go' command should still try to resolve # other module paths. @@ -40,7 +40,7 @@ env GOPROXY=file:///$WORK/gatekeeper chmod 0000 $WORK/gatekeeper/example.com/join/subpkg/@latest cp go.mod.orig go.mod ! go get -d example.com/join/subpkg -stderr 'go get example.com/join/subpkg: module example.com/join/subpkg: (invalid character .+|reading file://.*/gatekeeper/example.com/join/subpkg/@latest: .+)' +stderr 'go get: module example.com/join/subpkg: (invalid response from proxy ".+": invalid character .+|reading file://.*/gatekeeper/example.com/join/subpkg/@latest: .+)' -- go.mod.orig -- module example.com/othermodule diff --git a/libgo/go/cmd/go/testdata/script/mod_query_exclude.txt b/libgo/go/cmd/go/testdata/script/mod_query_exclude.txt index 1ae0d17..b001969 100644 --- a/libgo/go/cmd/go/testdata/script/mod_query_exclude.txt +++ b/libgo/go/cmd/go/testdata/script/mod_query_exclude.txt @@ -1,24 +1,43 @@ env GO111MODULE=on -env GOPROXY=$GOPROXY/quiet + +# list excluded version +go list -modfile=go.exclude.mod -m rsc.io/quote@v1.5.0 +stdout '^rsc.io/quote v1.5.0$' + +# list versions should not print excluded versions +go list -m -versions rsc.io/quote +stdout '\bv1.5.0\b' +go list -modfile=go.exclude.mod -m -versions rsc.io/quote +! stdout '\bv1.5.0\b' + +# list query with excluded version +go list -m rsc.io/quote@>=v1.5 +stdout '^rsc.io/quote v1.5.0$' +go list -modfile=go.exclude.mod -m rsc.io/quote@>=v1.5 +stdout '^rsc.io/quote v1.5.1$' # get excluded version -cp go.mod1 go.mod -! go get rsc.io/quote@v1.5.0 -stderr 'rsc.io/quote@v1.5.0 excluded' +cp go.exclude.mod go.exclude.mod.orig +! go get -modfile=go.exclude.mod -d rsc.io/quote@v1.5.0 +stderr '^go get: rsc.io/quote@v1.5.0: excluded by go.mod$' # get non-excluded version -cp go.mod1 go.mod -go get rsc.io/quote@v1.5.1 +cp go.exclude.mod.orig go.exclude.mod +go get -modfile=go.exclude.mod -d rsc.io/quote@v1.5.1 stderr 'rsc.io/quote v1.5.1' -# get range with excluded version -cp go.mod1 go.mod -go get rsc.io/quote@>=v1.5 -go list -m ...quote +# get query with excluded version +cp go.exclude.mod.orig go.exclude.mod +go get -modfile=go.exclude.mod -d rsc.io/quote@>=v1.5 +go list -modfile=go.exclude.mod -m ...quote stdout 'rsc.io/quote v1.5.[1-9]' --- go.mod1 -- +-- go.mod -- module x + +-- go.exclude.mod -- +module x + exclude rsc.io/quote v1.5.0 -- x.go -- diff --git a/libgo/go/cmd/go/testdata/script/mod_readonly.txt b/libgo/go/cmd/go/testdata/script/mod_readonly.txt index ac58126..ca8cd6e 100644 --- a/libgo/go/cmd/go/testdata/script/mod_readonly.txt +++ b/libgo/go/cmd/go/testdata/script/mod_readonly.txt @@ -10,17 +10,16 @@ stderr '^x.go:2:8: cannot find module providing package rsc\.io/quote: import lo ! stderr '\(\)' # If we don't have a reason for -mod=readonly, don't log an empty one. cmp go.mod go.mod.empty -# -mod=readonly should be set implicitly if the go.mod file is read-only -chmod 0400 go.mod +# -mod=readonly should be set by default. env GOFLAGS= ! go list all -stderr '^x.go:2:8: cannot find module providing package rsc\.io/quote: import lookup disabled by -mod=readonly\n\t\(go.mod file is read-only\.\)$' +stderr '^x.go:2:8: no required module provides package rsc\.io/quote; try ''go mod tidy'' to add it$' +cmp go.mod go.mod.empty -chmod 0600 go.mod env GOFLAGS=-mod=readonly # update go.mod - go get allowed -go get rsc.io/quote +go get -d rsc.io/quote grep rsc.io/quote go.mod # update go.mod - go mod tidy allowed @@ -42,24 +41,50 @@ go list -m all # -mod=readonly should reject inconsistent go.mod files # (ones that would be rewritten). -go mod edit -require rsc.io/sampler@v1.2.0 +go get -d rsc.io/sampler@v1.2.0 +go mod edit -require rsc.io/quote@v1.5.2 cp go.mod go.mod.inconsistent ! go list stderr 'go: updates to go.mod needed, disabled by -mod=readonly' cmp go.mod go.mod.inconsistent +# We get a different message when -mod=readonly is used by default. +env GOFLAGS= +! go list +stderr '^go: updates to go.mod needed; try ''go mod tidy'' first$' + # However, it should not reject files missing a 'go' directive, # since that was not always required. cp go.mod.nogo go.mod go list all +cmp go.mod go.mod.nogo # Nor should it reject files with redundant (not incorrect) # requirements. cp go.mod.redundant go.mod go list all +cmp go.mod go.mod.redundant cp go.mod.indirect go.mod go list all +cmp go.mod go.mod.indirect + + +# If we identify a missing package as a dependency of some other package in the +# main module, we should suggest 'go mod tidy' instead of resolving it. + +cp go.mod.untidy go.mod +! go list all +stderr '^x.go:2:8: no required module provides package rsc.io/quote; try ''go mod tidy'' to add it$' + +! go list -deps . +stderr '^x.go:2:8: no required module provides package rsc.io/quote; try ''go mod tidy'' to add it$' + +# However, if we didn't see an import from the main module, we should suggest +# 'go get -d' instead, because we don't know whether 'go mod tidy' would add it. +! go list rsc.io/quote +stderr '^no required module provides package rsc.io/quote; try ''go get -d rsc.io/quote'' to add it$' + -- go.mod -- module m @@ -96,3 +121,11 @@ require ( rsc.io/sampler v1.3.0 // indirect rsc.io/testonly v1.0.0 // indirect ) +-- go.mod.untidy -- +module m + +go 1.20 + +require ( + rsc.io/sampler v1.3.0 // indirect +) diff --git a/libgo/go/cmd/go/testdata/script/mod_replace.txt b/libgo/go/cmd/go/testdata/script/mod_replace.txt index c21f172..dc9667f 100644 --- a/libgo/go/cmd/go/testdata/script/mod_replace.txt +++ b/libgo/go/cmd/go/testdata/script/mod_replace.txt @@ -4,7 +4,7 @@ env GO111MODULE=on cp go.mod go.mod.orig # Make sure the test builds without replacement. -go build -o a1.exe . +go build -mod=mod -o a1.exe . exec ./a1.exe stdout 'Don''t communicate by sharing memory' @@ -32,7 +32,7 @@ stderr 'rsc.io/quote/v3@v3.0.0 used for two different module paths \(not-rsc.io/ # Modules that do not (yet) exist upstream can be replaced too. cp go.mod.orig go.mod go mod edit -replace=not-rsc.io/quote/v3@v3.1.0=./local/rsc.io/quote/v3 -go build -o a5.exe ./usenewmodule +go build -mod=mod -o a5.exe ./usenewmodule ! stderr 'finding not-rsc.io/quote/v3' grep 'not-rsc.io/quote/v3 v3.1.0' go.mod exec ./a5.exe diff --git a/libgo/go/cmd/go/testdata/script/mod_replace_gopkgin.txt b/libgo/go/cmd/go/testdata/script/mod_replace_gopkgin.txt index 28c1196..df752d9 100644 --- a/libgo/go/cmd/go/testdata/script/mod_replace_gopkgin.txt +++ b/libgo/go/cmd/go/testdata/script/mod_replace_gopkgin.txt @@ -11,6 +11,7 @@ env GO111MODULE=on env GOPROXY=direct env GOSUMDB=off +env GOFLAGS=-mod=mod # Replacing gopkg.in/[…].vN with a repository with a root go.mod file # specifying […].vN and a compatible version should succeed, even if @@ -34,7 +35,7 @@ go list -m gopkg.in/src-d/go-git.v4 # A mismatched gopkg.in path should not be able to replace a different major version. cd ../3-to-gomod-4 ! go list -m gopkg.in/src-d/go-git.v3 -stderr '^go: gopkg\.in/src-d/go-git\.v3@v3.0.0-20190801152248-0d1a009cbb60: invalid version: go\.mod has non-\.\.\.\.v3 module path "gopkg\.in/src-d/go-git\.v4" at revision 0d1a009cbb60$' +stderr '^go: gopkg\.in/src-d/go-git\.v3@v3\.2\.0 \(replaced by gopkg\.in/src-d/go-git\.v3@v3\.0\.0-20190801152248-0d1a009cbb60\): version "v3\.0\.0-20190801152248-0d1a009cbb60" invalid: go\.mod has non-\.\.\.\.v3 module path "gopkg\.in/src-d/go-git\.v4" at revision 0d1a009cbb60$' -- 4-to-4/go.mod -- module golang.org/issue/34254 diff --git a/libgo/go/cmd/go/testdata/script/mod_replace_import.txt b/libgo/go/cmd/go/testdata/script/mod_replace_import.txt index 54b1a12..2add31f 100644 --- a/libgo/go/cmd/go/testdata/script/mod_replace_import.txt +++ b/libgo/go/cmd/go/testdata/script/mod_replace_import.txt @@ -1,12 +1,12 @@ env GO111MODULE=on -# 'go list -mod=readonly' should not add requirements even if they can be -# resolved locally. +# 'go list' should not add requirements even if they can be resolved locally. cp go.mod go.mod.orig -! go list -mod=readonly all +! go list all cmp go.mod go.mod.orig # 'go list' should resolve imports using replacements. +go get -d go list all stdout 'example.com/a/b$' stdout 'example.com/x/v3$' @@ -25,10 +25,11 @@ stdout 'example.com/v v1.12.0 => ./v12' # The go command should print an informative error when the matched # module does not contain a package. +# TODO(#26909): Ideally these errors should include line numbers for the imports within the main module. cd fail -! go list all -stderr '^m.go:4:2: module w@latest found \(v0.0.0-00010101000000-000000000000, replaced by ../w\), but does not contain package w$' -stderr '^m.go:5:2: nonexist@v0.1.0: replacement directory ../nonexist does not exist$' +! go mod tidy +stderr '^localhost.fail imports\n\tw: module w@latest found \(v0.0.0-00010101000000-000000000000, replaced by ../w\), but does not contain package w$' +stderr '^localhost.fail imports\n\tnonexist: nonexist@v0.1.0: replacement directory ../nonexist does not exist$' -- go.mod -- module example.com/m diff --git a/libgo/go/cmd/go/testdata/script/mod_require_exclude.txt b/libgo/go/cmd/go/testdata/script/mod_require_exclude.txt index 60f7e3f..9156d4c 100644 --- a/libgo/go/cmd/go/testdata/script/mod_require_exclude.txt +++ b/libgo/go/cmd/go/testdata/script/mod_require_exclude.txt @@ -1,16 +1,51 @@ # build with no newer version to satisfy exclude env GO111MODULE=on -! go list -m all -stderr 'no newer version available' +cp go.mod go.mod.orig + +# With the selected version excluded, commands that query that version without +# updating go.mod should fail. + +! go list -mod=readonly -m all +stderr '^go: ignoring requirement on excluded version rsc.io/sampler v1\.99\.99$' +stderr '^go: updates to go.mod needed, disabled by -mod=readonly$' +! stdout '^rsc.io/sampler v1.99.99' +cmp go.mod go.mod.orig + +! go list -mod=vendor -m rsc.io/sampler +stderr '^go: ignoring requirement on excluded version rsc.io/sampler v1\.99\.99$' +stderr '^go list -m: module rsc.io/sampler: can''t resolve module using the vendor directory\n\t\(Use -mod=mod or -mod=readonly to bypass\.\)$' +! stdout '^rsc.io/sampler v1.99.99' +cmp go.mod go.mod.orig + +# With the selected version excluded, commands that load only modules should +# drop the excluded module. + +go list -m -mod=mod all +stderr '^go: dropping requirement on excluded version rsc.io/sampler v1\.99\.99$' +stdout '^x$' +! stdout '^rsc.io/sampler' +cmp go.mod go.moddrop + +# With the latest version excluded, 'go list' should resolve needed packages +# from the next-highest version. + +cp go.mod.orig go.mod +go list -mod=mod -f '{{with .Module}}{{.Path}} {{.Version}}{{end}}' all +stderr '^go: dropping requirement on excluded version rsc.io/sampler v1\.99\.99$' +stdout '^x $' +! stdout '^rsc.io/sampler v1.99.99' +stdout '^rsc.io/sampler v1.3.0' # build with newer version available cp go.mod2 go.mod -go list -m all +go list -mod=mod -f '{{with .Module}}{{.Path}} {{.Version}}{{end}}' all +stderr '^go: dropping requirement on excluded version rsc.io/quote v1\.5\.1$' stdout 'rsc.io/quote v1.5.2' # build with excluded newer version cp go.mod3 go.mod -go list -m all +go list -mod=mod -f '{{with .Module}}{{.Path}} {{.Version}}{{end}}' all +! stderr '^go: dropping requirement' stdout 'rsc.io/quote v1.5.1' -- x.go -- @@ -19,15 +54,28 @@ import _ "rsc.io/quote" -- go.mod -- module x -exclude rsc.io/sampler latest -require rsc.io/sampler latest +go 1.13 + +exclude rsc.io/sampler v1.99.99 +require rsc.io/sampler v1.99.99 +-- go.moddrop -- +module x + +go 1.13 + +exclude rsc.io/sampler v1.99.99 -- go.mod2 -- module x + +go 1.13 + exclude rsc.io/quote v1.5.1 require rsc.io/quote v1.5.1 - -- go.mod3 -- module x + +go 1.13 + exclude rsc.io/quote v1.5.2 require rsc.io/quote v1.5.1 diff --git a/libgo/go/cmd/go/testdata/script/mod_retention.txt b/libgo/go/cmd/go/testdata/script/mod_retention.txt index 1d83e6c..a4441c4 100644 --- a/libgo/go/cmd/go/testdata/script/mod_retention.txt +++ b/libgo/go/cmd/go/testdata/script/mod_retention.txt @@ -7,7 +7,7 @@ env GO111MODULE=on # Control case: verify that go.mod.tidy is actually tidy. cp go.mod.tidy go.mod -go list all +go list -mod=mod all cmp go.mod go.mod.tidy @@ -35,7 +35,7 @@ cmp go.mod go.mod.tidy # "// indirect" comments should be removed if direct dependencies are seen. # changes. cp go.mod.indirect go.mod -go list all +go list -mod=mod all cmp go.mod go.mod.tidy # "// indirect" comments should be added if appropriate. @@ -63,7 +63,7 @@ cmp go.mod go.mod.tidy # A missing "go" version directive should be added. # However, that should not remove other redundant requirements. cp go.mod.nogo go.mod -go list all +go list -mod=mod all cmpenv go.mod go.mod.currentgo diff --git a/libgo/go/cmd/go/testdata/script/mod_std_vendor.txt b/libgo/go/cmd/go/testdata/script/mod_std_vendor.txt index 5986cff..fb954d7 100644 --- a/libgo/go/cmd/go/testdata/script/mod_std_vendor.txt +++ b/libgo/go/cmd/go/testdata/script/mod_std_vendor.txt @@ -37,12 +37,10 @@ stderr 'use of vendored package' # When run within the 'std' module, 'go list -test' should report vendored # transitive dependencies at their original module paths. -# TODO(golang.org/issue/30241): Make that work. -# Today, they're standard packages as long as they exist. cd $GOROOT/src go list -test -f '{{range .Deps}}{{.}}{{"\n"}}{{end}}' net/http -stdout ^vendor/golang.org/x/net/http2/hpack # TODO: remove vendor/ prefix -! stdout ^golang.org/x/net/http2/hpack +stdout ^golang.org/x/net/http2/hpack +! stdout ^vendor/golang.org/x/net/http2/hpack -- go.mod -- module m diff --git a/libgo/go/cmd/go/testdata/script/mod_sumdb.txt b/libgo/go/cmd/go/testdata/script/mod_sumdb.txt index caf97e9..9a688e1 100644 --- a/libgo/go/cmd/go/testdata/script/mod_sumdb.txt +++ b/libgo/go/cmd/go/testdata/script/mod_sumdb.txt @@ -9,12 +9,18 @@ env dbname=localhost.localdev/sumdb cp go.mod.orig go.mod env GOSUMDB=$sumdb' '$proxy/sumdb-wrong ! go get -d rsc.io/quote -stderr 'go get rsc.io/quote: rsc.io/quote@v1.5.2: verifying module: checksum mismatch' +stderr 'go get: rsc.io/quote@v1.5.2: verifying module: checksum mismatch' stderr 'downloaded: h1:3fEy' stderr 'localhost.localdev/sumdb: h1:wrong' stderr 'SECURITY ERROR\nThis download does NOT match the one reported by the checksum server.' ! go get -d rsc.io/sampler ! go get -d golang.org/x/text + +go mod edit -require rsc.io/quote@v1.5.2 +! go mod tidy +stderr 'go: rsc.io/quote@v1.5.2: verifying go.mod: checksum mismatch' +stderr 'SECURITY ERROR\n' + rm go.sum # switching to truthful sumdb detects timeline inconsistency diff --git a/libgo/go/cmd/go/testdata/script/mod_sumdb_file_path.txt b/libgo/go/cmd/go/testdata/script/mod_sumdb_file_path.txt index 6108c0a..22fcbf3 100644 --- a/libgo/go/cmd/go/testdata/script/mod_sumdb_file_path.txt +++ b/libgo/go/cmd/go/testdata/script/mod_sumdb_file_path.txt @@ -13,7 +13,7 @@ env GOPATH=$WORK/gopath1 [windows] env GOPROXY=file:///$WORK/sumproxy,https://proxy.golang.org [!windows] env GOPROXY=file://$WORK/sumproxy,https://proxy.golang.org ! go get -d golang.org/x/text@v0.3.2 -stderr '^go get golang.org/x/text@v0.3.2: golang.org/x/text@v0.3.2: verifying module: golang.org/x/text@v0.3.2: reading file://.*/sumdb/sum.golang.org/lookup/golang.org/x/text@v0.3.2: (no such file or directory|.*cannot find the path specified.*)' +stderr '^go get: golang.org/x/text@v0.3.2: verifying module: golang.org/x/text@v0.3.2: reading file://.*/sumdb/sum.golang.org/lookup/golang.org/x/text@v0.3.2: (no such file or directory|.*cannot find the path specified.*)' # If the proxy does not claim to support the database, # checksum verification should fall through to the next proxy, diff --git a/libgo/go/cmd/go/testdata/script/mod_sumdb_golang.txt b/libgo/go/cmd/go/testdata/script/mod_sumdb_golang.txt index 40a07fc..cc0b0da 100644 --- a/libgo/go/cmd/go/testdata/script/mod_sumdb_golang.txt +++ b/libgo/go/cmd/go/testdata/script/mod_sumdb_golang.txt @@ -9,7 +9,7 @@ env GOPROXY=https://proxy.golang.org go env GOSUMDB stdout '^sum.golang.org$' -# download direct from github +# Download direct from github. [!net] skip [!exec:git] skip env GOSUMDB=sum.golang.org @@ -17,11 +17,13 @@ env GOPROXY=direct go get -d rsc.io/quote@v1.5.2 cp go.sum saved.sum -# download from proxy.golang.org with go.sum entry already +# Download from proxy.golang.org with go.sum entry already. +# Use 'go list' instead of 'go get' since the latter may download extra go.mod +# files not listed in go.sum. go clean -modcache env GOSUMDB= env GOPROXY= -go get -x -d rsc.io/quote@v1.5.2 +go list -x -deps rsc.io/quote ! stderr github stderr proxy.golang.org/rsc.io/quote ! stderr sum.golang.org/tile @@ -32,7 +34,7 @@ cmp go.sum saved.sum # Should use the checksum database to validate new go.sum lines, # but not need to fetch any new data from the proxy. rm go.sum -go get -x -d rsc.io/quote@v1.5.2 +go list -mod=mod -x rsc.io/quote ! stderr github ! stderr proxy.golang.org/rsc.io/quote stderr sum.golang.org/tile @@ -43,7 +45,7 @@ cmp go.sum saved.sum env TESTGOPROXY404=1 go clean -modcache rm go.sum -go get -x -d rsc.io/quote@v1.5.2 +go list -mod=mod -x rsc.io/quote stderr 'proxy.golang.org.*404 testing' stderr github.com/rsc cmp go.sum saved.sum diff --git a/libgo/go/cmd/go/testdata/script/mod_sumdb_proxy.txt b/libgo/go/cmd/go/testdata/script/mod_sumdb_proxy.txt index 7bbc3f9..70b8e3f 100644 --- a/libgo/go/cmd/go/testdata/script/mod_sumdb_proxy.txt +++ b/libgo/go/cmd/go/testdata/script/mod_sumdb_proxy.txt @@ -17,14 +17,12 @@ rm $GOPATH/pkg/mod/cache/download/sumdb rm go.sum # direct access fails (because localhost.localdev does not exist) -# The text of the error message is hard to predict because some DNS servers -# will resolve unknown domains like localhost.localdev to a real IP -# to serve ads. +# web.get is providing the error message - there's no actual network access. cp go.mod.orig go.mod env GOSUMDB=$sumdb env GOPROXY=direct ! go get -d rsc.io/fortune@v1.0.0 -stderr 'verifying.*localhost.localdev' +stderr 'verifying module: rsc.io/fortune@v1.0.0: .*: no such host localhost.localdev' rm $GOPATH/pkg/mod/cache/download/sumdb rm go.sum diff --git a/libgo/go/cmd/go/testdata/script/mod_symlink.txt b/libgo/go/cmd/go/testdata/script/mod_symlink.txt index 49bece2..dbc23fb 100644 --- a/libgo/go/cmd/go/testdata/script/mod_symlink.txt +++ b/libgo/go/cmd/go/testdata/script/mod_symlink.txt @@ -1,16 +1,19 @@ env GO111MODULE=on [!symlink] skip -# 'go list' should resolve modules of imported packages. +# 'go get -d' should resolve modules of imported packages. +go get -d go list -deps -f '{{.Module}}' . stdout golang.org/x/text +go get -d ./subpkg go list -deps -f '{{.Module}}' ./subpkg stdout golang.org/x/text # Create a copy of the module using symlinks in src/links. mkdir links symlink links/go.mod -> $GOPATH/src/go.mod +symlink links/go.sum -> $GOPATH/src/go.sum symlink links/issue.go -> $GOPATH/src/issue.go mkdir links/subpkg symlink links/subpkg/issue.go -> $GOPATH/src/subpkg/issue.go diff --git a/libgo/go/cmd/go/testdata/script/mod_test.txt b/libgo/go/cmd/go/testdata/script/mod_test.txt index 8f2da2f..50f0035 100644 --- a/libgo/go/cmd/go/testdata/script/mod_test.txt +++ b/libgo/go/cmd/go/testdata/script/mod_test.txt @@ -1,4 +1,5 @@ env GO111MODULE=on +env GOFLAGS=-mod=mod [short] skip # TODO(bcmills): Convert the 'go test' calls below to 'go list -test' once 'go diff --git a/libgo/go/cmd/go/testdata/script/mod_test_cached.txt b/libgo/go/cmd/go/testdata/script/mod_test_cached.txt index ffd573c..3da4358 100644 --- a/libgo/go/cmd/go/testdata/script/mod_test_cached.txt +++ b/libgo/go/cmd/go/testdata/script/mod_test_cached.txt @@ -51,26 +51,25 @@ bar package foo_test import ( - "io/ioutil" "os" "path/filepath" "testing" ) func TestWriteTmp(t *testing.T) { - dir, err := ioutil.TempDir("", "") + dir, err := os.MkdirTemp("", "") if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) - err = ioutil.WriteFile(filepath.Join(dir, "x"), nil, 0666) + err = os.WriteFile(filepath.Join(dir, "x"), nil, 0666) if err != nil { t.Fatal(err) } } func TestReadTestdata(t *testing.T) { - _, err := ioutil.ReadFile("testdata/foo.txt") + _, err := os.ReadFile("testdata/foo.txt") if err != nil { t.Fatal(err) } diff --git a/libgo/go/cmd/go/testdata/script/mod_tidy_replace.txt b/libgo/go/cmd/go/testdata/script/mod_tidy_replace.txt index c3158f8..dd99438 100644 --- a/libgo/go/cmd/go/testdata/script/mod_tidy_replace.txt +++ b/libgo/go/cmd/go/testdata/script/mod_tidy_replace.txt @@ -1,4 +1,5 @@ env GO111MODULE=on +env GOFLAGS=-mod=mod [short] skip # golang.org/issue/30166: 'go mod tidy' should not crash if a replaced module is @@ -34,7 +35,7 @@ grep 'golang.org/x/text' go.mod # 'go get' and 'go mod tidy' should follow the requirements of the replacements, # not the originals, even if that results in a set of versions that are # misleading or redundant without those replacements. -go get rsc.io/sampler@v1.2.0 +go get -d rsc.io/sampler@v1.2.0 go mod tidy go list -m all stdout 'rsc.io/quote/v3 v3.0.0' diff --git a/libgo/go/cmd/go/testdata/script/mod_upgrade_patch.txt b/libgo/go/cmd/go/testdata/script/mod_upgrade_patch.txt index 3939e54..8b34f8b 100644 --- a/libgo/go/cmd/go/testdata/script/mod_upgrade_patch.txt +++ b/libgo/go/cmd/go/testdata/script/mod_upgrade_patch.txt @@ -2,12 +2,18 @@ env GO111MODULE=on [short] skip # Initially, we are at v1.0.0 for all dependencies. +go get -d cp go.mod go.mod.orig go list -m all stdout '^patch.example.com/direct v1.0.0' stdout '^patch.example.com/indirect v1.0.0' ! stdout '^patch.example.com/depofdirectpatch' +# @patch should be rejected for modules not already in the build list. +! go get -d patch.example.com/depofdirectpatch@patch +stderr '^go get: can''t query version "patch" of module patch.example.com/depofdirectpatch: no existing version is required$' +cmp go.mod.orig go.mod + # get -u=patch, with no arguments, should patch-update all dependencies # of the package in the current directory, pulling in transitive dependencies # and also patching those. @@ -18,7 +24,7 @@ stdout '^patch.example.com/direct v1.0.1' stdout '^patch.example.com/indirect v1.0.1' stdout '^patch.example.com/depofdirectpatch v1.0.0' -# 'get all@patch' should be equivalent to 'get -u=patch all' +# 'get all@patch' should patch the modules that provide packages in 'all'. cp go.mod.orig go.mod go get -d all@patch go list -m all @@ -26,6 +32,15 @@ stdout '^patch.example.com/direct v1.0.1' stdout '^patch.example.com/indirect v1.0.1' stdout '^patch.example.com/depofdirectpatch v1.0.0' +# ...but 'all@patch' should fail if any of the affected modules do not already +# have a selected version. +cp go.mod.orig go.mod +go mod edit -droprequire=patch.example.com/direct +cp go.mod go.mod.dropped +! go get -d all@patch +stderr '^go get all@patch: can''t query version "patch" of module patch.example.com/direct: no existing version is required$' +cmp go.mod.dropped go.mod + # Requesting the direct dependency with -u=patch but without an explicit version # should patch-update it and its dependencies. cp go.mod.orig go.mod @@ -68,10 +83,10 @@ stdout '^patch.example.com/direct v1.1.0' stdout '^patch.example.com/indirect v1.0.1' ! stdout '^patch.example.com/depofdirectpatch' -# Standard-library packages cannot be upgraded explicitly. +# Standard library packages cannot be upgraded explicitly. cp go.mod.orig go.mod ! go get cmd/vet@patch -stderr 'cannot use pattern .* with explicit version' +stderr 'go get: can''t request explicit version "patch" of standard library package cmd/vet$' # However, standard-library packages without explicit versions are fine. go get -d -u=patch -d cmd/go @@ -84,6 +99,7 @@ go get -d example.com/noroot@patch go list -m all stdout '^example.com/noroot v1.0.1$' + -- go.mod -- module x diff --git a/libgo/go/cmd/go/testdata/script/mod_vcs_missing.txt b/libgo/go/cmd/go/testdata/script/mod_vcs_missing.txt index a755935..f8be43c 100644 --- a/libgo/go/cmd/go/testdata/script/mod_vcs_missing.txt +++ b/libgo/go/cmd/go/testdata/script/mod_vcs_missing.txt @@ -5,14 +5,14 @@ env GO111MODULE=on env GOPROXY=direct cd empty -! go list launchpad.net/gocheck +! go get -d launchpad.net/gocheck stderr '"bzr": executable file not found' cd .. # 1.11 used to give the cryptic error "cannot find module for path" here, but # only for a main package. cd main -! go build +! go build -mod=mod stderr '"bzr": executable file not found' cd .. diff --git a/libgo/go/cmd/go/testdata/script/mod_vendor_auto.txt b/libgo/go/cmd/go/testdata/script/mod_vendor_auto.txt index 53120dc..1b362ed 100644 --- a/libgo/go/cmd/go/testdata/script/mod_vendor_auto.txt +++ b/libgo/go/cmd/go/testdata/script/mod_vendor_auto.txt @@ -177,7 +177,7 @@ stdout '^'$WORK'[/\\]auto[/\\]vendor[/\\]example.com[/\\]version$' # 'go get' should update from the network or module cache, # even if a vendor directory is present. -go get -u example.com/printversion +go get -d example.com/version@v1.1.0 ! go list -f {{.Dir}} -tags tools all stderr '^go: inconsistent vendoring' diff --git a/libgo/go/cmd/go/testdata/script/mod_vendor_build.txt b/libgo/go/cmd/go/testdata/script/mod_vendor_build.txt index 0c359ce..3b8eec0 100644 --- a/libgo/go/cmd/go/testdata/script/mod_vendor_build.txt +++ b/libgo/go/cmd/go/testdata/script/mod_vendor_build.txt @@ -1,13 +1,16 @@ env GO111MODULE=on [short] skip +# Populate go.mod and go.sum. +go mod tidy + # initial conditions: using sampler v1.3.0, not listed in go.mod. go list -deps stdout rsc.io/sampler ! grep 'rsc.io/sampler v1.3.0' go.mod # update to v1.3.1, now indirect in go.mod. -go get rsc.io/sampler@v1.3.1 +go get -d rsc.io/sampler@v1.3.1 grep 'rsc.io/sampler v1.3.1 // indirect' go.mod cp go.mod go.mod.good diff --git a/libgo/go/cmd/go/testdata/script/mod_verify.txt b/libgo/go/cmd/go/testdata/script/mod_verify.txt index 646bc62..43812d0 100644 --- a/libgo/go/cmd/go/testdata/script/mod_verify.txt +++ b/libgo/go/cmd/go/testdata/script/mod_verify.txt @@ -12,20 +12,18 @@ go mod verify ! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.1.0.zip # With bad go.sum, sync (which must download) fails. -# Even if the bad sum is in the old legacy go.modverify file. rm go.sum -cp go.sum.bad go.modverify +cp go.sum.bad go.sum ! go mod tidy stderr 'checksum mismatch' ! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.1.0.zip -# With good go.sum, sync works (and moves go.modverify to go.sum). +# With good go.sum, sync works. rm go.sum -cp go.sum.good go.modverify +cp go.sum.good go.sum go mod tidy exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.1.0.zip exists $GOPATH/pkg/mod/rsc.io/quote@v1.1.0/quote.go -! exists go.modverify # go.sum should have the new checksum for go.mod grep '^rsc.io/quote v1.1.0/go.mod ' go.sum @@ -58,7 +56,7 @@ go mod tidy # Packages below module root should not be mentioned in go.sum. rm go.sum go mod edit -droprequire rsc.io/quote -go list rsc.io/quote/buggy # re-resolves import path and updates go.mod +go get -d rsc.io/quote/buggy grep '^rsc.io/quote v1.5.2/go.mod ' go.sum ! grep buggy go.sum diff --git a/libgo/go/cmd/go/testdata/script/mod_why.txt b/libgo/go/cmd/go/testdata/script/mod_why.txt index 10a4f9f..b3036fa 100644 --- a/libgo/go/cmd/go/testdata/script/mod_why.txt +++ b/libgo/go/cmd/go/testdata/script/mod_why.txt @@ -1,6 +1,10 @@ env GO111MODULE=on [short] skip +# Populate go.sum. +go mod tidy +cp go.mod go.mod.orig + go list -test all stdout rsc.io/quote stdout golang.org/x/text/language @@ -17,7 +21,7 @@ cmp stdout why-text-module.txt go mod why rsc.io/testonly cmp stdout why-testonly.txt -# why a module used only in tests? +# why a module used only in a test of a dependency? go mod why -m rsc.io/testonly cmp stdout why-testonly.txt @@ -41,6 +45,14 @@ cmp stdout why-both.txt go mod why -m rsc.io/quote rsc.io/sampler cmp stdout why-both-module.txt +# package in a module that isn't even in the module graph +# (https://golang.org/issue/26977) +go mod why rsc.io/fortune +cmp stdout why-missing.txt + +# None of these command should have changed the go.mod file. +cmp go.mod go.mod.orig + -- go.mod -- module mymodule require rsc.io/quote v1.5.2 @@ -113,3 +125,6 @@ mymodule/y mymodule/y.test rsc.io/quote rsc.io/sampler +-- why-missing.txt -- +# rsc.io/fortune +(main module does not need package rsc.io/fortune) diff --git a/libgo/go/cmd/go/testdata/script/modfile_flag.txt b/libgo/go/cmd/go/testdata/script/modfile_flag.txt index f05bf03..0ad0880 100644 --- a/libgo/go/cmd/go/testdata/script/modfile_flag.txt +++ b/libgo/go/cmd/go/testdata/script/modfile_flag.txt @@ -37,10 +37,10 @@ go mod why rsc.io/quote # 'go list' and other commands with build flags should work. # They should update the alternate go.mod when a dependency is missing. go mod edit -droprequire rsc.io/quote -go list . +go list -mod=mod . grep rsc.io/quote go.alt.mod -go build -n . -go test -n . +go build -n -mod=mod . +go test -n -mod=mod . go get -d rsc.io/quote diff --git a/libgo/go/cmd/go/testdata/script/sum_readonly.txt b/libgo/go/cmd/go/testdata/script/sum_readonly.txt deleted file mode 100644 index 8aa6116..0000000 --- a/libgo/go/cmd/go/testdata/script/sum_readonly.txt +++ /dev/null @@ -1,29 +0,0 @@ -# Test that go.sum does not get updated when -mod=readonly flag is set -env GO111MODULE=on - -go get -d rsc.io/quote -go mod tidy - -# go.sum != dirty; -mod=readonly -go list -mod=readonly - -# dirty up go.sum by removing it. -rm go.sum - -# go.sum == dirty; -mod=readonly -! go list -mod=readonly - -stderr 'go: updates to go.sum needed, disabled by -mod=readonly' - --- go.mod -- -module m - --- main.go -- - -package main - -import "rsc.io/quote" - -func main() { - println(quote.Hello()) -}
\ No newline at end of file diff --git a/libgo/go/cmd/go/testdata/script/test_cache_inputs.txt b/libgo/go/cmd/go/testdata/script/test_cache_inputs.txt index 57602e9..50486e1 100644 --- a/libgo/go/cmd/go/testdata/script/test_cache_inputs.txt +++ b/libgo/go/cmd/go/testdata/script/test_cache_inputs.txt @@ -137,7 +137,7 @@ exit 0 package testcache import ( - "io/ioutil" + "io" "os" "testing" ) @@ -159,7 +159,7 @@ func TestOddFileContent(t *testing.T) { if err != nil { t.Fatal(err) } - data, err := ioutil.ReadAll(f) + data, err := io.ReadAll(f) f.Close() if err != nil { t.Fatal(err) diff --git a/libgo/go/cmd/go/testdata/script/test_compile_tempfile.txt b/libgo/go/cmd/go/testdata/script/test_compile_tempfile.txt index 9124108..05f721a 100644 --- a/libgo/go/cmd/go/testdata/script/test_compile_tempfile.txt +++ b/libgo/go/cmd/go/testdata/script/test_compile_tempfile.txt @@ -1,7 +1,7 @@ [short] skip # Ensure that the target of 'go build -o' can be an existing, empty file so that -# its name can be reserved using ioutil.TempFile or the 'mktemp` command. +# its name can be reserved using os.CreateTemp or the 'mktemp` command. go build -o empty-file$GOEXE main.go diff --git a/libgo/go/cmd/go/testdata/script/test_generated_main.txt b/libgo/go/cmd/go/testdata/script/test_generated_main.txt index 75ffa9c..2e991a5 100644 --- a/libgo/go/cmd/go/testdata/script/test_generated_main.txt +++ b/libgo/go/cmd/go/testdata/script/test_generated_main.txt @@ -12,7 +12,6 @@ package x import ( "os" "path/filepath" - "io/ioutil" "regexp" "testing" ) @@ -23,7 +22,7 @@ func Test(t *testing.T) { t.Fatal(err) } testmainPath := filepath.Join(filepath.Dir(exePath), "_testmain.go") - source, err := ioutil.ReadFile(testmainPath) + source, err := os.ReadFile(testmainPath) if err != nil { t.Fatal(err) } diff --git a/libgo/go/cmd/go/testdata/script/version.txt b/libgo/go/cmd/go/testdata/script/version.txt index 87cb6be..8615a4a 100644 --- a/libgo/go/cmd/go/testdata/script/version.txt +++ b/libgo/go/cmd/go/testdata/script/version.txt @@ -20,6 +20,7 @@ env GO111MODULE=on [short] skip # Check that 'go version' and 'go version -m' work on a binary built in module mode. +go get -d rsc.io/fortune go build -o fortune.exe rsc.io/fortune go version fortune.exe stdout '^fortune.exe: .+' diff --git a/libgo/go/cmd/go/testdata/script/vet_flags.txt b/libgo/go/cmd/go/testdata/script/vet_flags.txt index b55dada..8002367 100644 --- a/libgo/go/cmd/go/testdata/script/vet_flags.txt +++ b/libgo/go/cmd/go/testdata/script/vet_flags.txt @@ -5,21 +5,25 @@ env GO111MODULE=on # Issue 35837: "go vet -<analyzer> <std package>" should use the requested # analyzers, not the default analyzers for 'go test'. -go vet -n -unreachable=false encoding/binary -stderr '-unreachable=false' +go vet -n -buildtags=false runtime +stderr '-buildtags=false' ! stderr '-unsafeptr=false' # Issue 37030: "go vet <std package>" without other flags should disable the # unsafeptr check by default. -go vet -n encoding/binary +go vet -n runtime stderr '-unsafeptr=false' ! stderr '-unreachable=false' # However, it should be enabled if requested explicitly. -go vet -n -unsafeptr encoding/binary +go vet -n -unsafeptr runtime stderr '-unsafeptr' ! stderr '-unsafeptr=false' +# -unreachable is disabled during test but on during plain vet. +go test -n runtime +stderr '-unreachable=false' + # A flag terminator should be allowed before the package list. go vet -n -- . @@ -63,10 +67,10 @@ stderr '[/\\]vet'$GOEXE'["]? .* -errorsas .* ["]?\$WORK[/\\][^ ]*[/\\]vet\.cfg' # "go test" on a standard package should by default disable an explicit list. go test -x -run=none encoding/binary -stderr '[/\\]vet'$GOEXE'["]? -unsafeptr=false ["]?\$WORK[/\\][^ ]*[/\\]vet\.cfg' +stderr '[/\\]vet'$GOEXE'["]? -unsafeptr=false -unreachable=false ["]?\$WORK[/\\][^ ]*[/\\]vet\.cfg' go test -x -vet= -run=none encoding/binary -stderr '[/\\]vet'$GOEXE'["]? -unsafeptr=false ["]?\$WORK[/\\][^ ]*[/\\]vet\.cfg' +stderr '[/\\]vet'$GOEXE'["]? -unsafeptr=false -unreachable=false ["]?\$WORK[/\\][^ ]*[/\\]vet\.cfg' # Both should allow users to override via the -vet flag. go test -x -vet=unreachable -run=none . diff --git a/libgo/go/cmd/gofmt/gofmt.go b/libgo/go/cmd/gofmt/gofmt.go index 8c56af7..2793c2c 100644 --- a/libgo/go/cmd/gofmt/gofmt.go +++ b/libgo/go/cmd/gofmt/gofmt.go @@ -14,7 +14,7 @@ import ( "go/scanner" "go/token" "io" - "io/ioutil" + "io/fs" "os" "path/filepath" "runtime" @@ -73,7 +73,7 @@ func initParserMode() { } } -func isGoFile(f os.FileInfo) bool { +func isGoFile(f fs.DirEntry) bool { // ignore non-Go files name := f.Name() return !f.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go") @@ -81,7 +81,7 @@ func isGoFile(f os.FileInfo) bool { // If in == nil, the source is the contents of the file with the given filename. func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error { - var perm os.FileMode = 0644 + var perm fs.FileMode = 0644 if in == nil { f, err := os.Open(filename) if err != nil { @@ -96,7 +96,7 @@ func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error perm = fi.Mode().Perm() } - src, err := ioutil.ReadAll(in) + src, err := io.ReadAll(in) if err != nil { return err } @@ -136,7 +136,7 @@ func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error if err != nil { return err } - err = ioutil.WriteFile(filename, res, perm) + err = os.WriteFile(filename, res, perm) if err != nil { os.Rename(bakname, filename) return err @@ -163,7 +163,7 @@ func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error return err } -func visitFile(path string, f os.FileInfo, err error) error { +func visitFile(path string, f fs.DirEntry, err error) error { if err == nil && isGoFile(f) { err = processFile(path, nil, os.Stdout, false) } @@ -176,7 +176,7 @@ func visitFile(path string, f os.FileInfo, err error) error { } func walkDir(path string) { - filepath.Walk(path, visitFile) + filepath.WalkDir(path, visitFile) } func main() { @@ -275,9 +275,9 @@ const chmodSupported = runtime.GOOS != "windows" // backupFile writes data to a new file named filename<number> with permissions perm, // with <number randomly chosen such that the file name is unique. backupFile returns // the chosen file name. -func backupFile(filename string, data []byte, perm os.FileMode) (string, error) { +func backupFile(filename string, data []byte, perm fs.FileMode) (string, error) { // create backup file - f, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename)) + f, err := os.CreateTemp(filepath.Dir(filename), filepath.Base(filename)) if err != nil { return "", err } diff --git a/libgo/go/cmd/gofmt/gofmt_test.go b/libgo/go/cmd/gofmt/gofmt_test.go index 98d3eb7..bf2adfe 100644 --- a/libgo/go/cmd/gofmt/gofmt_test.go +++ b/libgo/go/cmd/gofmt/gofmt_test.go @@ -7,7 +7,6 @@ package main import ( "bytes" "flag" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -93,7 +92,7 @@ func runTest(t *testing.T, in, out string) { return } - expected, err := ioutil.ReadFile(out) + expected, err := os.ReadFile(out) if err != nil { t.Error(err) return @@ -102,7 +101,7 @@ func runTest(t *testing.T, in, out string) { if got := buf.Bytes(); !bytes.Equal(got, expected) { if *update { if in != out { - if err := ioutil.WriteFile(out, got, 0666); err != nil { + if err := os.WriteFile(out, got, 0666); err != nil { t.Error(err) } return @@ -116,7 +115,7 @@ func runTest(t *testing.T, in, out string) { if err == nil { t.Errorf("%s", d) } - if err := ioutil.WriteFile(in+".gofmt", got, 0666); err != nil { + if err := os.WriteFile(in+".gofmt", got, 0666); err != nil { t.Error(err) } } @@ -157,7 +156,7 @@ func TestCRLF(t *testing.T) { const input = "testdata/crlf.input" // must contain CR/LF's const golden = "testdata/crlf.golden" // must not contain any CR's - data, err := ioutil.ReadFile(input) + data, err := os.ReadFile(input) if err != nil { t.Error(err) } @@ -165,7 +164,7 @@ func TestCRLF(t *testing.T) { t.Errorf("%s contains no CR/LF's", input) } - data, err = ioutil.ReadFile(golden) + data, err = os.ReadFile(golden) if err != nil { t.Error(err) } @@ -175,7 +174,7 @@ func TestCRLF(t *testing.T) { } func TestBackupFile(t *testing.T) { - dir, err := ioutil.TempDir("", "gofmt_test") + dir, err := os.MkdirTemp("", "gofmt_test") if err != nil { t.Fatal(err) } diff --git a/libgo/go/cmd/gofmt/long_test.go b/libgo/go/cmd/gofmt/long_test.go index e2a6208..4a82170 100644 --- a/libgo/go/cmd/gofmt/long_test.go +++ b/libgo/go/cmd/gofmt/long_test.go @@ -16,6 +16,7 @@ import ( "go/printer" "go/token" "io" + "io/fs" "os" "path/filepath" "runtime" @@ -107,12 +108,12 @@ func testFiles(t *testing.T, filenames <-chan string, done chan<- int) { func genFilenames(t *testing.T, filenames chan<- string) { defer close(filenames) - handleFile := func(filename string, fi os.FileInfo, err error) error { + handleFile := func(filename string, d fs.DirEntry, err error) error { if err != nil { t.Error(err) return nil } - if isGoFile(fi) { + if isGoFile(d) { filenames <- filename nfiles++ } @@ -123,13 +124,13 @@ func genFilenames(t *testing.T, filenames chan<- string) { if *files != "" { for _, filename := range strings.Split(*files, ",") { fi, err := os.Stat(filename) - handleFile(filename, fi, err) + handleFile(filename, &statDirEntry{fi}, err) } return // ignore files under -root } // otherwise, test all Go files under *root - filepath.Walk(*root, handleFile) + filepath.WalkDir(*root, handleFile) } func TestAll(t *testing.T) { @@ -163,3 +164,12 @@ func TestAll(t *testing.T) { fmt.Printf("processed %d files\n", nfiles) } } + +type statDirEntry struct { + info fs.FileInfo +} + +func (d *statDirEntry) Name() string { return d.info.Name() } +func (d *statDirEntry) IsDir() bool { return d.info.IsDir() } +func (d *statDirEntry) Type() fs.FileMode { return d.info.Mode().Type() } +func (d *statDirEntry) Info() (fs.FileInfo, error) { return d.info, nil } diff --git a/libgo/go/cmd/internal/buildid/buildid.go b/libgo/go/cmd/internal/buildid/buildid.go index ac238d7..1e8855d 100644 --- a/libgo/go/cmd/internal/buildid/buildid.go +++ b/libgo/go/cmd/internal/buildid/buildid.go @@ -10,18 +10,15 @@ import ( "fmt" "internal/xcoff" "io" + "io/fs" "os" "strconv" "strings" ) var ( - errBuildIDToolchain = fmt.Errorf("build ID only supported in gc toolchain") errBuildIDMalformed = fmt.Errorf("malformed object file") - errBuildIDUnknown = fmt.Errorf("lost build ID") -) -var ( bangArch = []byte("!<arch>") pkgdef = []byte("__.PKGDEF") goobject = []byte("go object ") @@ -109,7 +106,7 @@ func ReadFile(name string) (id string, err error) { // in cmd/go/internal/work/exec.go. func readGccgoArchive(name string, f *os.File) (string, error) { bad := func() (string, error) { - return "", &os.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed} + return "", &fs.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed} } off := int64(8) @@ -167,7 +164,7 @@ func readGccgoArchive(name string, f *os.File) (string, error) { // in cmd/go/internal/work/exec.go. func readGccgoBigArchive(name string, f *os.File) (string, error) { bad := func() (string, error) { - return "", &os.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed} + return "", &fs.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed} } // Read fixed-length header. @@ -309,13 +306,38 @@ func readRaw(name string, data []byte) (id string, err error) { j := bytes.Index(data[i+len(goBuildPrefix):], goBuildEnd) if j < 0 { - return "", &os.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed} + return "", &fs.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed} } quoted := data[i+len(goBuildPrefix)-1 : i+len(goBuildPrefix)+j+1] id, err = strconv.Unquote(string(quoted)) if err != nil { - return "", &os.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed} + return "", &fs.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed} } return id, nil } + +// HashToString converts the hash h to a string to be recorded +// in package archives and binaries as part of the build ID. +// We use the first 120 bits of the hash (5 chunks of 24 bits each) and encode +// it in base64, resulting in a 20-byte string. Because this is only used for +// detecting the need to rebuild installed files (not for lookups +// in the object file cache), 120 bits are sufficient to drive the +// probability of a false "do not need to rebuild" decision to effectively zero. +// We embed two different hashes in archives and four in binaries, +// so cutting to 20 bytes is a significant savings when build IDs are displayed. +// (20*4+3 = 83 bytes compared to 64*4+3 = 259 bytes for the +// more straightforward option of printing the entire h in base64). +func HashToString(h [32]byte) string { + const b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" + const chunks = 5 + var dst [chunks * 4]byte + for i := 0; i < chunks; i++ { + v := uint32(h[3*i])<<16 | uint32(h[3*i+1])<<8 | uint32(h[3*i+2]) + dst[4*i+0] = b64[(v>>18)&0x3F] + dst[4*i+1] = b64[(v>>12)&0x3F] + dst[4*i+2] = b64[(v>>6)&0x3F] + dst[4*i+3] = b64[v&0x3F] + } + return string(dst[:]) +} diff --git a/libgo/go/cmd/internal/buildid/buildid_test.go b/libgo/go/cmd/internal/buildid/buildid_test.go index 904c2c6..e832f99 100644 --- a/libgo/go/cmd/internal/buildid/buildid_test.go +++ b/libgo/go/cmd/internal/buildid/buildid_test.go @@ -11,6 +11,7 @@ import ( "io/ioutil" "os" "reflect" + "strings" "testing" ) @@ -146,3 +147,33 @@ func TestFindAndHash(t *testing.T) { } } } + +func TestExcludedReader(t *testing.T) { + const s = "0123456789abcdefghijklmn" + tests := []struct { + start, end int64 // excluded range + results []string // expected results of reads + }{ + {12, 15, []string{"0123456789", "ab\x00\x00\x00fghij", "klmn"}}, // within one read + {8, 21, []string{"01234567\x00\x00", "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", "\x00lmn"}}, // across multiple reads + {10, 20, []string{"0123456789", "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", "klmn"}}, // a whole read + {0, 5, []string{"\x00\x00\x00\x00\x0056789", "abcdefghij", "klmn"}}, // start + {12, 24, []string{"0123456789", "ab\x00\x00\x00\x00\x00\x00\x00\x00", "\x00\x00\x00\x00"}}, // end + } + p := make([]byte, 10) + for _, test := range tests { + r := &excludedReader{strings.NewReader(s), 0, test.start, test.end} + for _, res := range test.results { + n, err := r.Read(p) + if err != nil { + t.Errorf("read failed: %v", err) + } + if n != len(res) { + t.Errorf("unexpected number of bytes read: want %d, got %d", len(res), n) + } + if string(p[:n]) != res { + t.Errorf("unexpected bytes: want %q, got %q", res, p[:n]) + } + } + } +} diff --git a/libgo/go/cmd/internal/buildid/note.go b/libgo/go/cmd/internal/buildid/note.go index 2d26ea9..f5b6fc5 100644 --- a/libgo/go/cmd/internal/buildid/note.go +++ b/libgo/go/cmd/internal/buildid/note.go @@ -11,6 +11,7 @@ import ( "encoding/binary" "fmt" "io" + "io/fs" "os" ) @@ -96,7 +97,7 @@ func readELF(name string, f *os.File, data []byte) (buildid string, err error) { ef, err := elf.NewFile(bytes.NewReader(data)) if err != nil { - return "", &os.PathError{Path: name, Op: "parse", Err: err} + return "", &fs.PathError{Path: name, Op: "parse", Err: err} } var gnu string for _, p := range ef.Progs { @@ -181,13 +182,13 @@ func readMacho(name string, f *os.File, data []byte) (buildid string, err error) mf, err := macho.NewFile(f) if err != nil { - return "", &os.PathError{Path: name, Op: "parse", Err: err} + return "", &fs.PathError{Path: name, Op: "parse", Err: err} } sect := mf.Section("__text") if sect == nil { // Every binary has a __text section. Something is wrong. - return "", &os.PathError{Path: name, Op: "parse", Err: fmt.Errorf("cannot find __text section")} + return "", &fs.PathError{Path: name, Op: "parse", Err: fmt.Errorf("cannot find __text section")} } // It should be in the first few bytes, but read a lot just in case, diff --git a/libgo/go/cmd/internal/buildid/rewrite.go b/libgo/go/cmd/internal/buildid/rewrite.go index 5be5455..a792895 100644 --- a/libgo/go/cmd/internal/buildid/rewrite.go +++ b/libgo/go/cmd/internal/buildid/rewrite.go @@ -6,7 +6,9 @@ package buildid import ( "bytes" + "cmd/internal/codesign" "crypto/sha256" + "debug/macho" "fmt" "io" ) @@ -26,6 +28,11 @@ func FindAndHash(r io.Reader, id string, bufSize int) (matches []int64, hash [32 zeros := make([]byte, len(id)) idBytes := []byte(id) + // For Mach-O files, we want to exclude the code signature. + // The code signature contains hashes of the whole file (except the signature + // itself), including the buildid. So the buildid cannot contain the signature. + r = excludeMachoCodeSignature(r) + // The strategy is to read the file through buf, looking for id, // but we need to worry about what happens if id is broken up // and returned in parts by two different reads. @@ -87,5 +94,69 @@ func Rewrite(w io.WriterAt, pos []int64, id string) error { return err } } + + // Update Mach-O code signature, if any. + if f, cmd, ok := findMachoCodeSignature(w); ok { + if codesign.Size(int64(cmd.Dataoff), "a.out") == int64(cmd.Datasize) { + // Update the signature if the size matches, so we don't need to + // fix up headers. Binaries generated by the Go linker should have + // the expected size. Otherwise skip. + text := f.Segment("__TEXT") + cs := make([]byte, cmd.Datasize) + codesign.Sign(cs, w.(io.Reader), "a.out", int64(cmd.Dataoff), int64(text.Offset), int64(text.Filesz), f.Type == macho.TypeExec) + if _, err := w.WriteAt(cs, int64(cmd.Dataoff)); err != nil { + return err + } + } + } + return nil } + +func excludeMachoCodeSignature(r io.Reader) io.Reader { + _, cmd, ok := findMachoCodeSignature(r) + if !ok { + return r + } + return &excludedReader{r, 0, int64(cmd.Dataoff), int64(cmd.Dataoff + cmd.Datasize)} +} + +// excludedReader wraps an io.Reader. Reading from it returns the bytes from +// the underlying reader, except that when the byte offset is within the +// range between start and end, it returns zero bytes. +type excludedReader struct { + r io.Reader + off int64 // current offset + start, end int64 // the range to be excluded (read as zero) +} + +func (r *excludedReader) Read(p []byte) (int, error) { + n, err := r.r.Read(p) + if n > 0 && r.off+int64(n) > r.start && r.off < r.end { + cstart := r.start - r.off + if cstart < 0 { + cstart = 0 + } + cend := r.end - r.off + if cend > int64(n) { + cend = int64(n) + } + zeros := make([]byte, cend-cstart) + copy(p[cstart:cend], zeros) + } + r.off += int64(n) + return n, err +} + +func findMachoCodeSignature(r interface{}) (*macho.File, codesign.CodeSigCmd, bool) { + ra, ok := r.(io.ReaderAt) + if !ok { + return nil, codesign.CodeSigCmd{}, false + } + f, err := macho.NewFile(ra) + if err != nil { + return nil, codesign.CodeSigCmd{}, false + } + cmd, ok := codesign.FindCodeSigCmd(f) + return f, cmd, ok +} diff --git a/libgo/go/cmd/internal/codesign/codesign.go b/libgo/go/cmd/internal/codesign/codesign.go new file mode 100644 index 0000000..0517a10 --- /dev/null +++ b/libgo/go/cmd/internal/codesign/codesign.go @@ -0,0 +1,268 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package codesign provides basic functionalities for +// ad-hoc code signing of Mach-O files. +// +// This is not a general tool for code-signing. It is made +// specifically for the Go toolchain. It uses the same +// ad-hoc signing algorithm as the Darwin linker. +package codesign + +import ( + "crypto/sha256" + "debug/macho" + "encoding/binary" + "io" +) + +// Code signature layout. +// +// The code signature is a block of bytes that contains +// a SuperBlob, which contains one or more Blobs. For ad-hoc +// signing, a single CodeDirectory Blob suffices. +// +// A SuperBlob starts with its header (the binary representation +// of the SuperBlob struct), followed by a list of (in our case, +// one) Blobs (offset and size). A CodeDirectory Blob starts +// with its head (the binary representation of CodeDirectory struct), +// followed by the identifier (as a C string) and the hashes, at +// the corresponding offsets. +// +// The signature data must be included in the __LINKEDIT segment. +// In the Mach-O file header, an LC_CODE_SIGNATURE load command +// points to the data. + +const ( + pageSizeBits = 12 + pageSize = 1 << pageSizeBits +) + +const LC_CODE_SIGNATURE = 0x1d + +// Constants and struct layouts are from +// https://opensource.apple.com/source/xnu/xnu-4903.270.47/osfmk/kern/cs_blobs.h + +const ( + CSMAGIC_REQUIREMENT = 0xfade0c00 // single Requirement blob + CSMAGIC_REQUIREMENTS = 0xfade0c01 // Requirements vector (internal requirements) + CSMAGIC_CODEDIRECTORY = 0xfade0c02 // CodeDirectory blob + CSMAGIC_EMBEDDED_SIGNATURE = 0xfade0cc0 // embedded form of signature data + CSMAGIC_DETACHED_SIGNATURE = 0xfade0cc1 // multi-arch collection of embedded signatures + + CSSLOT_CODEDIRECTORY = 0 // slot index for CodeDirectory +) + +const ( + CS_HASHTYPE_SHA1 = 1 + CS_HASHTYPE_SHA256 = 2 + CS_HASHTYPE_SHA256_TRUNCATED = 3 + CS_HASHTYPE_SHA384 = 4 +) + +const ( + CS_EXECSEG_MAIN_BINARY = 0x1 // executable segment denotes main binary + CS_EXECSEG_ALLOW_UNSIGNED = 0x10 // allow unsigned pages (for debugging) + CS_EXECSEG_DEBUGGER = 0x20 // main binary is debugger + CS_EXECSEG_JIT = 0x40 // JIT enabled + CS_EXECSEG_SKIP_LV = 0x80 // skip library validation + CS_EXECSEG_CAN_LOAD_CDHASH = 0x100 // can bless cdhash for execution + CS_EXECSEG_CAN_EXEC_CDHASH = 0x200 // can execute blessed cdhash +) + +type Blob struct { + typ uint32 // type of entry + offset uint32 // offset of entry + // data follows +} + +func (b *Blob) put(out []byte) []byte { + out = put32be(out, b.typ) + out = put32be(out, b.offset) + return out +} + +const blobSize = 2 * 4 + +type SuperBlob struct { + magic uint32 // magic number + length uint32 // total length of SuperBlob + count uint32 // number of index entries following + // blobs []Blob +} + +func (s *SuperBlob) put(out []byte) []byte { + out = put32be(out, s.magic) + out = put32be(out, s.length) + out = put32be(out, s.count) + return out +} + +const superBlobSize = 3 * 4 + +type CodeDirectory struct { + magic uint32 // magic number (CSMAGIC_CODEDIRECTORY) + length uint32 // total length of CodeDirectory blob + version uint32 // compatibility version + flags uint32 // setup and mode flags + hashOffset uint32 // offset of hash slot element at index zero + identOffset uint32 // offset of identifier string + nSpecialSlots uint32 // number of special hash slots + nCodeSlots uint32 // number of ordinary (code) hash slots + codeLimit uint32 // limit to main image signature range + hashSize uint8 // size of each hash in bytes + hashType uint8 // type of hash (cdHashType* constants) + _pad1 uint8 // unused (must be zero) + pageSize uint8 // log2(page size in bytes); 0 => infinite + _pad2 uint32 // unused (must be zero) + scatterOffset uint32 + teamOffset uint32 + _pad3 uint32 + codeLimit64 uint64 + execSegBase uint64 + execSegLimit uint64 + execSegFlags uint64 + // data follows +} + +func (c *CodeDirectory) put(out []byte) []byte { + out = put32be(out, c.magic) + out = put32be(out, c.length) + out = put32be(out, c.version) + out = put32be(out, c.flags) + out = put32be(out, c.hashOffset) + out = put32be(out, c.identOffset) + out = put32be(out, c.nSpecialSlots) + out = put32be(out, c.nCodeSlots) + out = put32be(out, c.codeLimit) + out = put8(out, c.hashSize) + out = put8(out, c.hashType) + out = put8(out, c._pad1) + out = put8(out, c.pageSize) + out = put32be(out, c._pad2) + out = put32be(out, c.scatterOffset) + out = put32be(out, c.teamOffset) + out = put32be(out, c._pad3) + out = put64be(out, c.codeLimit64) + out = put64be(out, c.execSegBase) + out = put64be(out, c.execSegLimit) + out = put64be(out, c.execSegFlags) + return out +} + +const codeDirectorySize = 13*4 + 4 + 4*8 + +// CodeSigCmd is Mach-O LC_CODE_SIGNATURE load command. +type CodeSigCmd struct { + Cmd uint32 // LC_CODE_SIGNATURE + Cmdsize uint32 // sizeof this command (16) + Dataoff uint32 // file offset of data in __LINKEDIT segment + Datasize uint32 // file size of data in __LINKEDIT segment +} + +func FindCodeSigCmd(f *macho.File) (CodeSigCmd, bool) { + get32 := f.ByteOrder.Uint32 + for _, l := range f.Loads { + data := l.Raw() + cmd := get32(data) + if cmd == LC_CODE_SIGNATURE { + return CodeSigCmd{ + cmd, + get32(data[4:]), + get32(data[8:]), + get32(data[12:]), + }, true + } + } + return CodeSigCmd{}, false +} + +func put32be(b []byte, x uint32) []byte { binary.BigEndian.PutUint32(b, x); return b[4:] } +func put64be(b []byte, x uint64) []byte { binary.BigEndian.PutUint64(b, x); return b[8:] } +func put8(b []byte, x uint8) []byte { b[0] = x; return b[1:] } +func puts(b, s []byte) []byte { n := copy(b, s); return b[n:] } + +// Size computes the size of the code signature. +// id is the identifier used for signing (a field in CodeDirectory blob, which +// has no significance in ad-hoc signing). +func Size(codeSize int64, id string) int64 { + nhashes := (codeSize + pageSize - 1) / pageSize + idOff := int64(codeDirectorySize) + hashOff := idOff + int64(len(id)+1) + cdirSz := hashOff + nhashes*sha256.Size + return int64(superBlobSize+blobSize) + cdirSz +} + +// Sign generates an ad-hoc code signature and writes it to out. +// out must have length at least Size(codeSize, id). +// data is the file content without the signature, of size codeSize. +// textOff and textSize is the file offset and size of the text segment. +// isMain is true if this is a main executable. +// id is the identifier used for signing (a field in CodeDirectory blob, which +// has no significance in ad-hoc signing). +func Sign(out []byte, data io.Reader, id string, codeSize, textOff, textSize int64, isMain bool) { + nhashes := (codeSize + pageSize - 1) / pageSize + idOff := int64(codeDirectorySize) + hashOff := idOff + int64(len(id)+1) + sz := Size(codeSize, id) + + // emit blob headers + sb := SuperBlob{ + magic: CSMAGIC_EMBEDDED_SIGNATURE, + length: uint32(sz), + count: 1, + } + blob := Blob{ + typ: CSSLOT_CODEDIRECTORY, + offset: superBlobSize + blobSize, + } + cdir := CodeDirectory{ + magic: CSMAGIC_CODEDIRECTORY, + length: uint32(sz) - (superBlobSize + blobSize), + version: 0x20400, + flags: 0x20002, // adhoc | linkerSigned + hashOffset: uint32(hashOff), + identOffset: uint32(idOff), + nCodeSlots: uint32(nhashes), + codeLimit: uint32(codeSize), + hashSize: sha256.Size, + hashType: CS_HASHTYPE_SHA256, + pageSize: uint8(pageSizeBits), + execSegBase: uint64(textOff), + execSegLimit: uint64(textSize), + } + if isMain { + cdir.execSegFlags = CS_EXECSEG_MAIN_BINARY + } + + outp := out + outp = sb.put(outp) + outp = blob.put(outp) + outp = cdir.put(outp) + + // emit the identifier + outp = puts(outp, []byte(id+"\000")) + + // emit hashes + var buf [pageSize]byte + h := sha256.New() + p := 0 + for p < int(codeSize) { + n, err := io.ReadFull(data, buf[:]) + if err == io.EOF { + break + } + if err != nil && err != io.ErrUnexpectedEOF { + panic(err) + } + if p+n > int(codeSize) { + n = int(codeSize) - p + } + p += n + h.Reset() + h.Write(buf[:n]) + b := h.Sum(nil) + outp = puts(outp, b[:]) + } +} diff --git a/libgo/go/cmd/internal/objabi/doc.go b/libgo/go/cmd/internal/objabi/doc.go deleted file mode 100644 index 08e922b..0000000 --- a/libgo/go/cmd/internal/objabi/doc.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// NOTE: There are *three* independent implementations of this object -// file format in the Go source tree: -// -// - cmd/internal/goobj/read.go (used by cmd/addr2line, cmd/nm, cmd/objdump, cmd/pprof) -// - cmd/internal/obj/objfile.go (used by cmd/asm and cmd/compile) -// - cmd/link/internal/objfile.go (used by cmd/link) -// -// When changing the object file format, remember to change all three. - -// Originally, Go object files were Plan 9 object files, but no longer. -// Now they are more like standard object files, in that each symbol is defined -// by an associated memory image (bytes) and a list of relocations to apply -// during linking. We do not (yet?) use a standard file format, however. -// For now, the format is chosen to be as simple as possible to read and write. -// It may change for reasons of efficiency, or we may even switch to a -// standard file format if there are compelling benefits to doing so. -// See golang.org/s/go13linker for more background. -// -// The file format is: -// -// - magic header: "\x00go114ld" -// - byte 1 - version number -// - sequence of strings giving dependencies (imported packages) -// - empty string (marks end of sequence) -// - number of entries in the following sequence -// - sequence of filename strings to generate debug information -// - sequence of symbol references used by the defined symbols -// - byte 0xff (marks end of sequence) -// - sequence of integer lengths: -// - total data length -// - total number of relocations -// - total number of pcdata -// - total number of automatics -// - total number of funcdata -// - total number of files -// - data, the content of the defined symbols -// - sequence of defined symbols -// - byte 0xff (marks end of sequence) -// - magic footer: "\xffgo114ld" -// -// All integers are stored in a zigzag varint format. -// See golang.org/s/go12symtab for a definition. -// -// Data blocks and strings are both stored as an integer -// followed by that many bytes. -// -// A symbol reference is a string name followed by an ABI or -1 for static. -// -// A symbol points to other symbols using an index into the symbol -// reference sequence. Index 0 corresponds to a nil symbol pointer. -// In the symbol layout described below "symref index" stands for this -// index. -// -// Each symbol is laid out as the following fields: -// -// - byte 0xfe (sanity check for synchronization) -// - type [byte] -// - name & ABI [symref index] -// - flags [int] -// 1<<0 dupok -// 1<<1 local -// 1<<2 add to typelink table -// - size [int] -// - gotype [symref index] -// - p [data block] -// - nr [int] -// - r [nr relocations, sorted by off] -// -// If type == STEXT, there are a few more fields: -// -// - args [int] -// - locals [int] -// - nosplit [int] -// - flags [int] -// 1<<0 leaf -// 1<<1 C function -// 1<<2 function may call reflect.Type.Method -// 1<<3 function compiled with -shared -// - nlocal [int] -// - local [nlocal automatics] -// - pcln [pcln table] -// -// Each relocation has the encoding: -// -// - off [int] -// - siz [int] -// - type [int] -// - add [int] -// - sym [symref index] -// -// Each local has the encoding: -// -// - asym [symref index] -// - offset [int] -// - type [int] -// - gotype [symref index] -// -// The pcln table has the encoding: -// -// - pcsp [data block] -// - pcfile [data block] -// - pcline [data block] -// - pcinline [data block] -// - npcdata [int] -// - pcdata [npcdata data blocks] -// - nfuncdata [int] -// - funcdata [nfuncdata symref index] -// - funcdatasym [nfuncdata ints] -// - nfile [int] -// - file [nfile symref index] -// - ninlinedcall [int] -// - inlinedcall [ninlinedcall int symref int symref] -// -// The file layout and meaning of type integers are architecture-independent. -// -// TODO(rsc): The file format is good for a first pass but needs work. -// - There are SymID in the object file that should really just be strings. -package objabi diff --git a/libgo/go/cmd/internal/objabi/flag.go b/libgo/go/cmd/internal/objabi/flag.go index 79ad2cc..3fd73f3 100644 --- a/libgo/go/cmd/internal/objabi/flag.go +++ b/libgo/go/cmd/internal/objabi/flag.go @@ -5,6 +5,7 @@ package objabi import ( + "bytes" "flag" "fmt" "io" @@ -59,6 +60,9 @@ func expandArgs(in []string) (out []string) { log.Fatal(err) } args := strings.Split(strings.TrimSpace(strings.Replace(string(slurp), "\r", "", -1)), "\n") + for i, arg := range args { + args[i] = DecodeArg(arg) + } out = append(out, expandArgs(args)...) } else if out != nil { out = append(out, s) @@ -160,3 +164,38 @@ func (f fn1) Set(s string) error { } func (f fn1) String() string { return "" } + +// DecodeArg decodes an argument. +// +// This function is public for testing with the parallel encoder. +func DecodeArg(arg string) string { + // If no encoding, fastpath out. + if !strings.ContainsAny(arg, "\\\n") { + return arg + } + + // We can't use strings.Builder as this must work at bootstrap. + var b bytes.Buffer + var wasBS bool + for _, r := range arg { + if wasBS { + switch r { + case '\\': + b.WriteByte('\\') + case 'n': + b.WriteByte('\n') + default: + // This shouldn't happen. The only backslashes that reach here + // should encode '\n' and '\\' exclusively. + panic("badly formatted input") + } + } else if r == '\\' { + wasBS = true + continue + } else { + b.WriteRune(r) + } + wasBS = false + } + return b.String() +} diff --git a/libgo/go/cmd/internal/objabi/flag_test.go b/libgo/go/cmd/internal/objabi/flag_test.go new file mode 100644 index 0000000..935b9c2 --- /dev/null +++ b/libgo/go/cmd/internal/objabi/flag_test.go @@ -0,0 +1,26 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package objabi + +import "testing" + +func TestDecodeArg(t *testing.T) { + t.Parallel() + tests := []struct { + arg, want string + }{ + {"", ""}, + {"hello", "hello"}, + {"hello\\n", "hello\n"}, + {"hello\\nthere", "hello\nthere"}, + {"hello\\\\there", "hello\\there"}, + {"\\\\\\n", "\\\n"}, + } + for _, test := range tests { + if got := DecodeArg(test.arg); got != test.want { + t.Errorf("decodoeArg(%q) = %q, want %q", test.arg, got, test.want) + } + } +} diff --git a/libgo/go/cmd/internal/objabi/funcdata.go b/libgo/go/cmd/internal/objabi/funcdata.go index d5bacb5..faa2863 100644 --- a/libgo/go/cmd/internal/objabi/funcdata.go +++ b/libgo/go/cmd/internal/objabi/funcdata.go @@ -11,17 +11,15 @@ package objabi // ../../../runtime/symtab.go. const ( - PCDATA_RegMapIndex = 0 // if !go115ReduceLiveness - PCDATA_UnsafePoint = 0 // if go115ReduceLiveness + PCDATA_UnsafePoint = 0 PCDATA_StackMapIndex = 1 PCDATA_InlTreeIndex = 2 FUNCDATA_ArgsPointerMaps = 0 FUNCDATA_LocalsPointerMaps = 1 - FUNCDATA_RegPointerMaps = 2 // if !go115ReduceLiveness - FUNCDATA_StackObjects = 3 - FUNCDATA_InlTree = 4 - FUNCDATA_OpenCodedDeferInfo = 5 + FUNCDATA_StackObjects = 2 + FUNCDATA_InlTree = 3 + FUNCDATA_OpenCodedDeferInfo = 4 // ArgsSizeUnknown is set in Func.argsize to mark all functions // whose argument size is unknown (C vararg functions, and @@ -32,11 +30,6 @@ const ( // Special PCDATA values. const ( - // PCDATA_RegMapIndex values. - // - // Only if !go115ReduceLiveness. - PCDATA_RegMapUnsafe = -2 // Unsafe for async preemption - // PCDATA_UnsafePoint values. PCDATA_UnsafePointSafe = -1 // Safe for async preemption PCDATA_UnsafePointUnsafe = -2 // Unsafe for async preemption diff --git a/libgo/go/cmd/internal/objabi/funcid.go b/libgo/go/cmd/internal/objabi/funcid.go index 0fda1db..1d098ee 100644 --- a/libgo/go/cmd/internal/objabi/funcid.go +++ b/libgo/go/cmd/internal/objabi/funcid.go @@ -4,11 +4,6 @@ package objabi -import ( - "strconv" - "strings" -) - // A FuncID identifies particular functions that need to be treated // specially by the runtime. // Note that in some situations involving plugins, there may be multiple @@ -31,7 +26,7 @@ const ( FuncID_gcBgMarkWorker FuncID_systemstack_switch FuncID_systemstack - FuncID_cgocallback_gofunc + FuncID_cgocallback FuncID_gogo FuncID_externalthreadhandler FuncID_debugCallV1 @@ -44,7 +39,10 @@ const ( // Get the function ID for the named function in the named file. // The function should be package-qualified. -func GetFuncID(name, file string) FuncID { +func GetFuncID(name string, isWrapper bool) FuncID { + if isWrapper { + return FuncID_wrapper + } switch name { case "runtime.main": return FuncID_runtime_main @@ -72,8 +70,8 @@ func GetFuncID(name, file string) FuncID { return FuncID_systemstack_switch case "runtime.systemstack": return FuncID_systemstack - case "runtime.cgocallback_gofunc": - return FuncID_cgocallback_gofunc + case "runtime.cgocallback": + return FuncID_cgocallback case "runtime.gogo": return FuncID_gogo case "runtime.externalthreadhandler": @@ -98,17 +96,5 @@ func GetFuncID(name, file string) FuncID { // Don't show in the call stack (used when invoking defer functions) return FuncID_wrapper } - if file == "<autogenerated>" { - return FuncID_wrapper - } - if strings.HasPrefix(name, "runtime.call") { - if _, err := strconv.Atoi(name[12:]); err == nil { - // runtime.callXX reflect call wrappers. - return FuncID_wrapper - } - } - if strings.HasSuffix(name, "-fm") { - return FuncID_wrapper - } return FuncID_normal } diff --git a/libgo/go/cmd/internal/objabi/head.go b/libgo/go/cmd/internal/objabi/head.go index 95b8db3..48ff292 100644 --- a/libgo/go/cmd/internal/objabi/head.go +++ b/libgo/go/cmd/internal/objabi/head.go @@ -54,7 +54,7 @@ func (h *HeadType) Set(s string) error { switch s { case "aix": *h = Haix - case "darwin": + case "darwin", "ios": *h = Hdarwin case "dragonfly": *h = Hdragonfly diff --git a/libgo/go/cmd/internal/objabi/line.go b/libgo/go/cmd/internal/objabi/line.go index 178c836..0733b65 100644 --- a/libgo/go/cmd/internal/objabi/line.go +++ b/libgo/go/cmd/internal/objabi/line.go @@ -37,25 +37,36 @@ func AbsFile(dir, file, rewrites string) string { abs = filepath.Join(dir, file) } + abs, rewritten := ApplyRewrites(abs, rewrites) + if !rewritten && hasPathPrefix(abs, GOROOT) { + abs = "$GOROOT" + abs[len(GOROOT):] + } + + if abs == "" { + abs = "??" + } + return abs +} + +// ApplyRewrites returns the filename for file in the given directory, +// as rewritten by the rewrites argument. +// +// The rewrites argument is a ;-separated list of rewrites. +// Each rewrite is of the form "prefix" or "prefix=>replace", +// where prefix must match a leading sequence of path elements +// and is either removed entirely or replaced by the replacement. +func ApplyRewrites(file, rewrites string) (string, bool) { start := 0 for i := 0; i <= len(rewrites); i++ { if i == len(rewrites) || rewrites[i] == ';' { - if new, ok := applyRewrite(abs, rewrites[start:i]); ok { - abs = new - goto Rewritten + if new, ok := applyRewrite(file, rewrites[start:i]); ok { + return new, true } start = i + 1 } } - if hasPathPrefix(abs, GOROOT) { - abs = "$GOROOT" + abs[len(GOROOT):] - } -Rewritten: - if abs == "" { - abs = "??" - } - return abs + return file, false } // applyRewrite applies the rewrite to the path, diff --git a/libgo/go/cmd/internal/objabi/path.go b/libgo/go/cmd/internal/objabi/path.go index 2a42179..fd1c998 100644 --- a/libgo/go/cmd/internal/objabi/path.go +++ b/libgo/go/cmd/internal/objabi/path.go @@ -39,3 +39,25 @@ func PathToPrefix(s string) string { return string(p) } + +// IsRuntimePackagePath examines 'pkgpath' and returns TRUE if it +// belongs to the collection of "runtime-related" packages, including +// "runtime" itself, "reflect", "syscall", and the +// "runtime/internal/*" packages. The compiler and/or assembler in +// some cases need to be aware of when they are building such a +// package, for example to enable features such as ABI selectors in +// assembly sources. +func IsRuntimePackagePath(pkgpath string) bool { + rval := false + switch pkgpath { + case "runtime": + rval = true + case "reflect": + rval = true + case "syscall": + rval = true + default: + rval = strings.HasPrefix(pkgpath, "runtime/internal") + } + return rval +} diff --git a/libgo/go/cmd/internal/objabi/reloctype.go b/libgo/go/cmd/internal/objabi/reloctype.go index f029a3c..649f690 100644 --- a/libgo/go/cmd/internal/objabi/reloctype.go +++ b/libgo/go/cmd/internal/objabi/reloctype.go @@ -89,6 +89,17 @@ const ( // should be linked into the final binary, even if there are no other // direct references. (This is used for types reachable by reflection.) R_USETYPE + // R_USEIFACE marks a type is converted to an interface in the function this + // relocation is applied to. The target is a type descriptor. + // This is a marker relocation (0-sized), for the linker's reachabililty + // analysis. + R_USEIFACE + // R_USEIFACEMETHOD marks an interface method that is used in the function + // this relocation is applied to. The target is an interface type descriptor. + // The addend is the offset of the method in the type descriptor. + // This is a marker relocation (0-sized), for the linker's reachabililty + // analysis. + R_USEIFACEMETHOD // R_METHODOFF resolves to a 32-bit offset from the beginning of the section // holding the data being relocated to the referenced symbol. // It is a variant of R_ADDROFF used when linking from the uncommonType of a @@ -145,6 +156,9 @@ const ( // R_ARM64_LDST8 sets a LD/ST immediate value to bits [11:0] of a local address. R_ARM64_LDST8 + // R_ARM64_LDST16 sets a LD/ST immediate value to bits [11:1] of a local address. + R_ARM64_LDST16 + // R_ARM64_LDST32 sets a LD/ST immediate value to bits [11:2] of a local address. R_ARM64_LDST32 @@ -212,6 +226,14 @@ const ( // AUIPC + S-type instruction pair. R_RISCV_PCREL_STYPE + // R_RISCV_TLS_IE_ITYPE resolves a 32-bit TLS initial-exec TOC offset + // address using an AUIPC + I-type instruction pair. + R_RISCV_TLS_IE_ITYPE + + // R_RISCV_TLS_IE_STYPE resolves a 32-bit TLS initial-exec TOC offset + // address using an AUIPC + S-type instruction pair. + R_RISCV_TLS_IE_STYPE + // R_PCRELDBL relocates s390x 2-byte aligned PC-relative addresses. // TODO(mundaym): remove once variants can be serialized - see issue 14218. R_PCRELDBL diff --git a/libgo/go/cmd/internal/objabi/reloctype_string.go b/libgo/go/cmd/internal/objabi/reloctype_string.go index 83dfe71..658a44f 100644 --- a/libgo/go/cmd/internal/objabi/reloctype_string.go +++ b/libgo/go/cmd/internal/objabi/reloctype_string.go @@ -4,9 +4,75 @@ package objabi import "strconv" -const _RelocType_name = "R_ADDRR_ADDRPOWERR_ADDRARM64R_ADDRMIPSR_ADDROFFR_WEAKADDROFFR_SIZER_CALLR_CALLARMR_CALLARM64R_CALLINDR_CALLPOWERR_CALLMIPSR_CALLRISCVR_CONSTR_PCRELR_TLS_LER_TLS_IER_GOTOFFR_PLT0R_PLT1R_PLT2R_USEFIELDR_USETYPER_METHODOFFR_POWER_TOCR_GOTPCRELR_JMPMIPSR_DWARFSECREFR_DWARFFILEREFR_ARM64_TLS_LER_ARM64_TLS_IER_ARM64_GOTPCRELR_ARM64_GOTR_ARM64_PCRELR_ARM64_LDST8R_ARM64_LDST32R_ARM64_LDST64R_ARM64_LDST128R_POWER_TLS_LER_POWER_TLS_IER_POWER_TLSR_ADDRPOWER_DSR_ADDRPOWER_GOTR_ADDRPOWER_PCRELR_ADDRPOWER_TOCRELR_ADDRPOWER_TOCREL_DSR_RISCV_PCREL_ITYPER_RISCV_PCREL_STYPER_PCRELDBLR_ADDRMIPSUR_ADDRMIPSTLSR_ADDRCUOFFR_WASMIMPORTR_XCOFFREF" +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[R_ADDR-1] + _ = x[R_ADDRPOWER-2] + _ = x[R_ADDRARM64-3] + _ = x[R_ADDRMIPS-4] + _ = x[R_ADDROFF-5] + _ = x[R_WEAKADDROFF-6] + _ = x[R_SIZE-7] + _ = x[R_CALL-8] + _ = x[R_CALLARM-9] + _ = x[R_CALLARM64-10] + _ = x[R_CALLIND-11] + _ = x[R_CALLPOWER-12] + _ = x[R_CALLMIPS-13] + _ = x[R_CALLRISCV-14] + _ = x[R_CONST-15] + _ = x[R_PCREL-16] + _ = x[R_TLS_LE-17] + _ = x[R_TLS_IE-18] + _ = x[R_GOTOFF-19] + _ = x[R_PLT0-20] + _ = x[R_PLT1-21] + _ = x[R_PLT2-22] + _ = x[R_USEFIELD-23] + _ = x[R_USETYPE-24] + _ = x[R_USEIFACE-25] + _ = x[R_USEIFACEMETHOD-26] + _ = x[R_METHODOFF-27] + _ = x[R_POWER_TOC-28] + _ = x[R_GOTPCREL-29] + _ = x[R_JMPMIPS-30] + _ = x[R_DWARFSECREF-31] + _ = x[R_DWARFFILEREF-32] + _ = x[R_ARM64_TLS_LE-33] + _ = x[R_ARM64_TLS_IE-34] + _ = x[R_ARM64_GOTPCREL-35] + _ = x[R_ARM64_GOT-36] + _ = x[R_ARM64_PCREL-37] + _ = x[R_ARM64_LDST8-38] + _ = x[R_ARM64_LDST16-39] + _ = x[R_ARM64_LDST32-40] + _ = x[R_ARM64_LDST64-41] + _ = x[R_ARM64_LDST128-42] + _ = x[R_POWER_TLS_LE-43] + _ = x[R_POWER_TLS_IE-44] + _ = x[R_POWER_TLS-45] + _ = x[R_ADDRPOWER_DS-46] + _ = x[R_ADDRPOWER_GOT-47] + _ = x[R_ADDRPOWER_PCREL-48] + _ = x[R_ADDRPOWER_TOCREL-49] + _ = x[R_ADDRPOWER_TOCREL_DS-50] + _ = x[R_RISCV_PCREL_ITYPE-51] + _ = x[R_RISCV_PCREL_STYPE-52] + _ = x[R_RISCV_TLS_IE_ITYPE-53] + _ = x[R_RISCV_TLS_IE_STYPE-54] + _ = x[R_PCRELDBL-55] + _ = x[R_ADDRMIPSU-56] + _ = x[R_ADDRMIPSTLS-57] + _ = x[R_ADDRCUOFF-58] + _ = x[R_WASMIMPORT-59] + _ = x[R_XCOFFREF-60] +} + +const _RelocType_name = "R_ADDRR_ADDRPOWERR_ADDRARM64R_ADDRMIPSR_ADDROFFR_WEAKADDROFFR_SIZER_CALLR_CALLARMR_CALLARM64R_CALLINDR_CALLPOWERR_CALLMIPSR_CALLRISCVR_CONSTR_PCRELR_TLS_LER_TLS_IER_GOTOFFR_PLT0R_PLT1R_PLT2R_USEFIELDR_USETYPER_USEIFACER_USEIFACEMETHODR_METHODOFFR_POWER_TOCR_GOTPCRELR_JMPMIPSR_DWARFSECREFR_DWARFFILEREFR_ARM64_TLS_LER_ARM64_TLS_IER_ARM64_GOTPCRELR_ARM64_GOTR_ARM64_PCRELR_ARM64_LDST8R_ARM64_LDST16R_ARM64_LDST32R_ARM64_LDST64R_ARM64_LDST128R_POWER_TLS_LER_POWER_TLS_IER_POWER_TLSR_ADDRPOWER_DSR_ADDRPOWER_GOTR_ADDRPOWER_PCRELR_ADDRPOWER_TOCRELR_ADDRPOWER_TOCREL_DSR_RISCV_PCREL_ITYPER_RISCV_PCREL_STYPER_RISCV_TLS_IE_ITYPER_RISCV_TLS_IE_STYPER_PCRELDBLR_ADDRMIPSUR_ADDRMIPSTLSR_ADDRCUOFFR_WASMIMPORTR_XCOFFREF" -var _RelocType_index = [...]uint16{0, 6, 17, 28, 38, 47, 60, 66, 72, 81, 92, 101, 112, 122, 133, 140, 147, 155, 163, 171, 177, 183, 189, 199, 208, 219, 230, 240, 249, 262, 276, 290, 304, 320, 331, 344, 357, 371, 385, 400, 414, 428, 439, 453, 468, 485, 503, 524, 543, 562, 572, 583, 596, 607, 619, 629} +var _RelocType_index = [...]uint16{0, 6, 17, 28, 38, 47, 60, 66, 72, 81, 92, 101, 112, 122, 133, 140, 147, 155, 163, 171, 177, 183, 189, 199, 208, 218, 234, 245, 256, 266, 275, 288, 302, 316, 330, 346, 357, 370, 383, 397, 411, 425, 440, 454, 468, 479, 493, 508, 525, 543, 564, 583, 602, 622, 642, 652, 663, 676, 687, 699, 709} func (i RelocType) String() string { i -= 1 diff --git a/libgo/go/cmd/internal/objabi/symkind.go b/libgo/go/cmd/internal/objabi/symkind.go index 374aaa6..6c99112 100644 --- a/libgo/go/cmd/internal/objabi/symkind.go +++ b/libgo/go/cmd/internal/objabi/symkind.go @@ -56,7 +56,12 @@ const ( // Thread-local data that is initially all 0s STLSBSS // Debugging data - SDWARFINFO + SDWARFCUINFO + SDWARFCONST + SDWARFFCN + SDWARFABSFCN + SDWARFTYPE + SDWARFVAR SDWARFRANGE SDWARFLOC SDWARFLINES diff --git a/libgo/go/cmd/internal/objabi/symkind_string.go b/libgo/go/cmd/internal/objabi/symkind_string.go index 919a666..1b1c394 100644 --- a/libgo/go/cmd/internal/objabi/symkind_string.go +++ b/libgo/go/cmd/internal/objabi/symkind_string.go @@ -16,17 +16,22 @@ func _() { _ = x[SBSS-5] _ = x[SNOPTRBSS-6] _ = x[STLSBSS-7] - _ = x[SDWARFINFO-8] - _ = x[SDWARFRANGE-9] - _ = x[SDWARFLOC-10] - _ = x[SDWARFLINES-11] - _ = x[SABIALIAS-12] - _ = x[SLIBFUZZER_EXTRA_COUNTER-13] + _ = x[SDWARFCUINFO-8] + _ = x[SDWARFCONST-9] + _ = x[SDWARFFCN-10] + _ = x[SDWARFABSFCN-11] + _ = x[SDWARFTYPE-12] + _ = x[SDWARFVAR-13] + _ = x[SDWARFRANGE-14] + _ = x[SDWARFLOC-15] + _ = x[SDWARFLINES-16] + _ = x[SABIALIAS-17] + _ = x[SLIBFUZZER_EXTRA_COUNTER-18] } -const _SymKind_name = "SxxxSTEXTSRODATASNOPTRDATASDATASBSSSNOPTRBSSSTLSBSSSDWARFINFOSDWARFRANGESDWARFLOCSDWARFLINESSABIALIASSLIBFUZZER_EXTRA_COUNTER" +const _SymKind_name = "SxxxSTEXTSRODATASNOPTRDATASDATASBSSSNOPTRBSSSTLSBSSSDWARFCUINFOSDWARFCONSTSDWARFFCNSDWARFABSFCNSDWARFTYPESDWARFVARSDWARFRANGESDWARFLOCSDWARFLINESSABIALIASSLIBFUZZER_EXTRA_COUNTER" -var _SymKind_index = [...]uint8{0, 4, 9, 16, 26, 31, 35, 44, 51, 61, 72, 81, 92, 101, 125} +var _SymKind_index = [...]uint8{0, 4, 9, 16, 26, 31, 35, 44, 51, 63, 74, 83, 95, 105, 114, 125, 134, 145, 154, 178} func (i SymKind) String() string { if i >= SymKind(len(_SymKind_index)-1) { diff --git a/libgo/go/cmd/internal/objabi/util.go b/libgo/go/cmd/internal/objabi/util.go index 30d9ac5..d9cf495 100644 --- a/libgo/go/cmd/internal/objabi/util.go +++ b/libgo/go/cmd/internal/objabi/util.go @@ -25,7 +25,6 @@ var ( GOARCH = envOr("GOARCH", defaultGOARCH) GOOS = envOr("GOOS", defaultGOOS) GO386 = envOr("GO386", defaultGO386) - GOAMD64 = goamd64() GOARM = goarm() GOMIPS = gomips() GOMIPS64 = gomips64() @@ -37,17 +36,16 @@ var ( const ( ElfRelocOffset = 256 - MachoRelocOffset = 2048 // reserve enough space for ELF relocations - Go115AMD64 = "alignedjumps" // Should be "alignedjumps" or "normaljumps"; this replaces environment variable introduced in CL 219357. + MachoRelocOffset = 2048 // reserve enough space for ELF relocations ) -// TODO(1.16): assuming no issues in 1.15 release, remove this and related constant. -func goamd64() string { - return Go115AMD64 -} - func goarm() int { - switch v := envOr("GOARM", defaultGOARM); v { + def := defaultGOARM + if GOOS == "android" && GOARCH == "arm" { + // Android arm devices always support GOARM=7. + def = "7" + } + switch v := envOr("GOARM", def); v { case "5": return 5 case "6": @@ -131,12 +129,16 @@ func init() { addexp(f) } } -} -func Framepointer_enabled(goos, goarch string) bool { - return framepointer_enabled != 0 && (goarch == "amd64" || goarch == "arm64" && (goos == "linux" || goos == "darwin")) + // regabi is only supported on amd64. + if GOARCH != "amd64" { + Regabi_enabled = 0 + } } +// Note: must agree with runtime.framepointer_enabled. +var Framepointer_enabled = GOARCH == "amd64" || GOARCH == "arm64" && (GOOS == "linux" || GOOS == "darwin" || GOOS == "ios") + func addexp(s string) { // Could do general integer parsing here, but the runtime copy doesn't yet. v := 1 @@ -159,10 +161,10 @@ func addexp(s string) { } var ( - framepointer_enabled int = 1 Fieldtrack_enabled int Preemptibleloops_enabled int Staticlockranking_enabled int + Regabi_enabled int ) // Toolchain experiments. @@ -174,9 +176,9 @@ var exper = []struct { val *int }{ {"fieldtrack", &Fieldtrack_enabled}, - {"framepointer", &framepointer_enabled}, {"preemptibleloops", &Preemptibleloops_enabled}, {"staticlockranking", &Staticlockranking_enabled}, + {"regabi", &Regabi_enabled}, } var defaultExpstring = Expstring() diff --git a/libgo/go/cmd/internal/sys/supported.go b/libgo/go/cmd/internal/sys/supported.go index c27b3b9..ef7c017 100644 --- a/libgo/go/cmd/internal/sys/supported.go +++ b/libgo/go/cmd/internal/sys/supported.go @@ -13,7 +13,9 @@ func RaceDetectorSupported(goos, goarch string) bool { switch goos { case "linux": return goarch == "amd64" || goarch == "ppc64le" || goarch == "arm64" - case "darwin", "freebsd", "netbsd", "windows": + case "darwin": + return goarch == "amd64" || goarch == "arm64" + case "freebsd", "netbsd", "windows": return goarch == "amd64" default: return false @@ -32,13 +34,14 @@ func MSanSupported(goos, goarch string) bool { } // MustLinkExternal reports whether goos/goarch requires external linking. +// (This is the opposite of internal/testenv.CanInternalLink. Keep them in sync.) func MustLinkExternal(goos, goarch string) bool { switch goos { case "android": if goarch != "arm64" { return true } - case "darwin": + case "ios": if goarch == "arm64" { return true } @@ -69,7 +72,7 @@ func BuildModeSupported(compiler, buildmode, goos, goarch string) bool { case "linux/amd64", "linux/arm", "linux/arm64", "linux/386", "linux/ppc64le", "linux/s390x", "android/amd64", "android/arm", "android/arm64", "android/386", "freebsd/amd64", - "darwin/amd64", + "darwin/amd64", "darwin/arm64", "windows/amd64", "windows/386": return true } @@ -83,10 +86,11 @@ func BuildModeSupported(compiler, buildmode, goos, goarch string) bool { case "pie": switch platform { - case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/s390x", + case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/riscv64", "linux/s390x", "android/amd64", "android/arm", "android/arm64", "android/386", "freebsd/amd64", - "darwin/amd64", + "darwin/amd64", "darwin/arm64", + "ios/amd64", "ios/arm64", "aix/ppc64", "windows/386", "windows/amd64", "windows/arm": return true @@ -104,7 +108,7 @@ func BuildModeSupported(compiler, buildmode, goos, goarch string) bool { switch platform { case "linux/amd64", "linux/arm", "linux/arm64", "linux/386", "linux/s390x", "linux/ppc64le", "android/amd64", "android/arm", "android/arm64", "android/386", - "darwin/amd64", + "darwin/amd64", "darwin/arm64", "freebsd/amd64": return true } @@ -114,3 +118,14 @@ func BuildModeSupported(compiler, buildmode, goos, goarch string) bool { return false } } + +func InternalLinkPIESupported(goos, goarch string) bool { + switch goos + "/" + goarch { + case "darwin/amd64", "darwin/arm64", + "linux/amd64", "linux/arm64", + "android/arm64", + "windows-amd64", "windows-386", "windows-arm": + return true + } + return false +} diff --git a/libgo/go/cmd/internal/sys/supported_test.go b/libgo/go/cmd/internal/sys/supported_test.go new file mode 100644 index 0000000..1217814 --- /dev/null +++ b/libgo/go/cmd/internal/sys/supported_test.go @@ -0,0 +1,18 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sys + +import ( + "internal/testenv" + "runtime" + "testing" +) + +func TestMustLinkExternalMatchesTestenv(t *testing.T) { + // MustLinkExternal and testenv.CanInternalLink are the exact opposite. + if b := MustLinkExternal(runtime.GOOS, runtime.GOARCH); b != !testenv.CanInternalLink() { + t.Fatalf("MustLinkExternal() == %v, testenv.CanInternalLink() == %v, don't match", b, testenv.CanInternalLink()) + } +} diff --git a/libgo/go/cmd/internal/traceviewer/format.go b/libgo/go/cmd/internal/traceviewer/format.go new file mode 100644 index 0000000..8714774 --- /dev/null +++ b/libgo/go/cmd/internal/traceviewer/format.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package traceviewer provides definitions of the JSON data structures +// used by the Chrome trace viewer. +// +// The official description of the format is in this file: +// https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview +package traceviewer + +type Data struct { + Events []*Event `json:"traceEvents"` + Frames map[string]Frame `json:"stackFrames"` + TimeUnit string `json:"displayTimeUnit"` +} + +type Event struct { + Name string `json:"name,omitempty"` + Phase string `json:"ph"` + Scope string `json:"s,omitempty"` + Time float64 `json:"ts"` + Dur float64 `json:"dur,omitempty"` + PID uint64 `json:"pid"` + TID uint64 `json:"tid"` + ID uint64 `json:"id,omitempty"` + BindPoint string `json:"bp,omitempty"` + Stack int `json:"sf,omitempty"` + EndStack int `json:"esf,omitempty"` + Arg interface{} `json:"args,omitempty"` + Cname string `json:"cname,omitempty"` + Category string `json:"cat,omitempty"` +} + +type Frame struct { + Name string `json:"name"` + Parent int `json:"parent,omitempty"` +} diff --git a/libgo/go/cmd/vendor/modules.txt b/libgo/go/cmd/vendor/modules.txt new file mode 100644 index 0000000..4e47f41 --- /dev/null +++ b/libgo/go/cmd/vendor/modules.txt @@ -0,0 +1,91 @@ +# github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2 +## explicit +github.com/google/pprof/driver +github.com/google/pprof/internal/binutils +github.com/google/pprof/internal/driver +github.com/google/pprof/internal/elfexec +github.com/google/pprof/internal/graph +github.com/google/pprof/internal/measurement +github.com/google/pprof/internal/plugin +github.com/google/pprof/internal/report +github.com/google/pprof/internal/symbolizer +github.com/google/pprof/internal/symbolz +github.com/google/pprof/internal/transport +github.com/google/pprof/profile +github.com/google/pprof/third_party/d3 +github.com/google/pprof/third_party/d3flamegraph +github.com/google/pprof/third_party/svgpan +# github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 +github.com/ianlancetaylor/demangle +# golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff +## explicit +golang.org/x/arch/arm/armasm +golang.org/x/arch/arm64/arm64asm +golang.org/x/arch/ppc64/ppc64asm +golang.org/x/arch/x86/x86asm +# golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 +## explicit +golang.org/x/crypto/ed25519 +golang.org/x/crypto/ed25519/internal/edwards25519 +golang.org/x/crypto/ssh/terminal +# golang.org/x/mod v0.4.0 +## explicit +golang.org/x/mod/internal/lazyregexp +golang.org/x/mod/modfile +golang.org/x/mod/module +golang.org/x/mod/semver +golang.org/x/mod/sumdb +golang.org/x/mod/sumdb/dirhash +golang.org/x/mod/sumdb/note +golang.org/x/mod/sumdb/tlog +golang.org/x/mod/zip +# golang.org/x/sys v0.0.0-20201204225414-ed752295db88 +## explicit +golang.org/x/sys/internal/unsafeheader +golang.org/x/sys/unix +golang.org/x/sys/windows +# golang.org/x/tools v0.0.0-20201211025543-abf6a1d87e11 +## explicit +golang.org/x/tools/go/analysis +golang.org/x/tools/go/analysis/internal/analysisflags +golang.org/x/tools/go/analysis/internal/facts +golang.org/x/tools/go/analysis/passes/asmdecl +golang.org/x/tools/go/analysis/passes/assign +golang.org/x/tools/go/analysis/passes/atomic +golang.org/x/tools/go/analysis/passes/bools +golang.org/x/tools/go/analysis/passes/buildtag +golang.org/x/tools/go/analysis/passes/cgocall +golang.org/x/tools/go/analysis/passes/composite +golang.org/x/tools/go/analysis/passes/copylock +golang.org/x/tools/go/analysis/passes/ctrlflow +golang.org/x/tools/go/analysis/passes/errorsas +golang.org/x/tools/go/analysis/passes/framepointer +golang.org/x/tools/go/analysis/passes/httpresponse +golang.org/x/tools/go/analysis/passes/ifaceassert +golang.org/x/tools/go/analysis/passes/inspect +golang.org/x/tools/go/analysis/passes/internal/analysisutil +golang.org/x/tools/go/analysis/passes/loopclosure +golang.org/x/tools/go/analysis/passes/lostcancel +golang.org/x/tools/go/analysis/passes/nilfunc +golang.org/x/tools/go/analysis/passes/printf +golang.org/x/tools/go/analysis/passes/shift +golang.org/x/tools/go/analysis/passes/stdmethods +golang.org/x/tools/go/analysis/passes/stringintconv +golang.org/x/tools/go/analysis/passes/structtag +golang.org/x/tools/go/analysis/passes/testinggoroutine +golang.org/x/tools/go/analysis/passes/tests +golang.org/x/tools/go/analysis/passes/unmarshal +golang.org/x/tools/go/analysis/passes/unreachable +golang.org/x/tools/go/analysis/passes/unsafeptr +golang.org/x/tools/go/analysis/passes/unusedresult +golang.org/x/tools/go/analysis/unitchecker +golang.org/x/tools/go/ast/astutil +golang.org/x/tools/go/ast/inspector +golang.org/x/tools/go/cfg +golang.org/x/tools/go/types/objectpath +golang.org/x/tools/go/types/typeutil +golang.org/x/tools/internal/analysisinternal +golang.org/x/tools/internal/lsp/fuzzy +# golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 +golang.org/x/xerrors +golang.org/x/xerrors/internal diff --git a/libgo/go/cmd/vet/main.go b/libgo/go/cmd/vet/main.go index 6381de8..d50c45d 100644 --- a/libgo/go/cmd/vet/main.go +++ b/libgo/go/cmd/vet/main.go @@ -14,6 +14,7 @@ import ( "golang.org/x/tools/go/analysis/passes/composite" "golang.org/x/tools/go/analysis/passes/copylock" "golang.org/x/tools/go/analysis/passes/errorsas" + "golang.org/x/tools/go/analysis/passes/framepointer" "golang.org/x/tools/go/analysis/passes/httpresponse" "golang.org/x/tools/go/analysis/passes/ifaceassert" "golang.org/x/tools/go/analysis/passes/loopclosure" @@ -24,6 +25,7 @@ import ( "golang.org/x/tools/go/analysis/passes/stdmethods" "golang.org/x/tools/go/analysis/passes/stringintconv" "golang.org/x/tools/go/analysis/passes/structtag" + "golang.org/x/tools/go/analysis/passes/testinggoroutine" "golang.org/x/tools/go/analysis/passes/tests" "golang.org/x/tools/go/analysis/passes/unmarshal" "golang.org/x/tools/go/analysis/passes/unreachable" @@ -44,6 +46,7 @@ func main() { composite.Analyzer, copylock.Analyzer, errorsas.Analyzer, + framepointer.Analyzer, httpresponse.Analyzer, ifaceassert.Analyzer, loopclosure.Analyzer, @@ -55,6 +58,7 @@ func main() { stringintconv.Analyzer, structtag.Analyzer, tests.Analyzer, + testinggoroutine.Analyzer, unmarshal.Analyzer, unreachable.Analyzer, unsafeptr.Analyzer, diff --git a/libgo/go/cmd/vet/vet_test.go b/libgo/go/cmd/vet/vet_test.go index 66732ad..4c8dd3c 100644 --- a/libgo/go/cmd/vet/vet_test.go +++ b/libgo/go/cmd/vet/vet_test.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "internal/testenv" - "io/ioutil" "log" "os" "os/exec" @@ -33,7 +32,7 @@ func TestMain(m *testing.M) { } func testMain(m *testing.M) int { - dir, err := ioutil.TempDir("", "vet_test") + dir, err := os.MkdirTemp("", "vet_test") if err != nil { fmt.Fprintln(os.Stderr, err) return 1 @@ -349,7 +348,7 @@ var ( func wantedErrors(file, short string) (errs []wantedError) { cache := make(map[string]*regexp.Regexp) - src, err := ioutil.ReadFile(file) + src, err := os.ReadFile(file) if err != nil { log.Fatal(err) } |